@zintrust/workers 0.1.31 → 0.1.52

package/src/createQueueWorker.ts

@@ -1,5 +1,116 @@
 import type { BullMQPayload, QueueMessage } from '@zintrust/core';
-import { Logger, Queue } from '@zintrust/core';
+import * as Core from '@zintrust/core';
+import { Env, Logger, Queue } from '@zintrust/core';
+
+const RETRY_BASE_DELAY_MS = 1000;
+const RETRY_MAX_DELAY_MS = 30000;
+
+const getJobStateTracker = (): unknown => {
+  try {
+    return (Core as Record<string, unknown>)['JobStateTracker'];
+  } catch {
+    return undefined;
+  }
+};
+
+const getJobHeartbeatStore = (): unknown => {
+  try {
+    return (Core as Record<string, unknown>)['JobHeartbeatStore'];
+  } catch {
+    return undefined;
+  }
+};
+
+const getTimeoutManager = (): unknown => {
+  try {
+    return (Core as Record<string, unknown>)['TimeoutManager'];
+  } catch {
+    return undefined;
+  }
+};
+
+const getEnvInt = (key: string, fallback: number): number => {
+  const getter = (Env as { getInt?: (name: string, defaultValue: number) => number }).getInt;
+  if (typeof getter === 'function') {
+    return getter(key, fallback);
+  }
+
+  const raw = (Env as Record<string, unknown>)[key];
+  if (typeof raw === 'number' && Number.isFinite(raw)) return Math.floor(raw);
+  if (typeof raw === 'string' && raw.trim() !== '') {
+    const parsed = Number(raw);
+    if (Number.isFinite(parsed)) return Math.floor(parsed);
+  }
+  return fallback;
+};
+
+const resolveQueueJobTimeoutMs = (): number => {
+  const timeoutManager = getTimeoutManager();
+  const tm = (timeoutManager ?? {}) as { getQueueJobTimeoutMs?: () => number };
+  if (typeof tm.getQueueJobTimeoutMs === 'function') {
+    return tm.getQueueJobTimeoutMs();
+  }
+  return Math.max(1000, getEnvInt('QUEUE_JOB_TIMEOUT', 60) * 1000);
+};
+
+const runWithTimeout = async <T>(
+  operation: () => Promise<T>,
+  timeoutMs: number,
+  operationName: string
+): Promise<T> => {
+  const timeoutManager = getTimeoutManager();
+  const tm = (timeoutManager ?? {}) as {
+    withTimeout?: <R>(
+      op: () => Promise<R>,
+      t: number,
+      name: string,
+      timeoutHandler?: () => Promise<R>
+    ) => Promise<R>;
+  };
+  if (typeof tm.withTimeout === 'function') {
+    return tm.withTimeout(operation, timeoutMs, operationName);
+  }
+  return operation();
+};
+
+const isTimeoutError = (error: unknown): boolean => {
+  const timeoutManager = getTimeoutManager();
+  const tm = (timeoutManager ?? {}) as { isTimeoutError?: (value: unknown) => boolean };
+  if (typeof tm.isTimeoutError === 'function') {
+    return tm.isTimeoutError(error);
+  }
+  if (error instanceof Error) {
+    return error.message.toLowerCase().includes('timed out');
+  }
+  return false;
+};
+
+const normalizeAttempts = (value: unknown): number => {
+  if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
+    return Math.floor(value);
+  }
+  return 0;
+};
+
+const getAttemptsFromMessage = <TPayload>(message: QueueMessage<TPayload>): number => {
+  const payloadAttempts =
+    typeof message.payload === 'object' && message.payload !== null
+      ? normalizeAttempts((message.payload as Record<string, unknown>)['attempts'])
+      : 0;
+  const messageAttempts = normalizeAttempts(
+    (message as QueueMessage<TPayload> & { attempts?: number }).attempts
+  );
+  return Math.max(payloadAttempts, messageAttempts);
+};
+
+const getRetryDelayMs = (nextAttempts: number): number => {
+  const exponentialDelay = Math.min(
+    RETRY_BASE_DELAY_MS * 2 ** Math.max(0, nextAttempts - 1),
+    RETRY_MAX_DELAY_MS
+  );
+  const jitterMs = Math.floor(Math.random() * 250); // NOSONAR
+  return exponentialDelay + jitterMs;
+};
 
 type QueueWorker = {
   processOne: (queueName?: string, driverName?: string) => Promise<boolean>;
@@ -36,58 +147,332 @@ const buildBaseLogFields = <TPayload>(
   };
 };
 
-const createProcessOne = <TPayload>(
-  options: CreateQueueWorkerOptions<TPayload>
-): ((queueName?: string, driverName?: string) => Promise<boolean>) => {
-  return async (queueName = options.defaultQueueName, driverName?: string): Promise<boolean> => {
-    const message = await Queue.dequeue<TPayload>(queueName, driverName);
-    if (!message) return false;
+type TrackerApi = {
+  started?: (input: {
+    queueName: string;
+    jobId: string;
+    attempts: number;
+    timeoutMs?: number;
+    workerName?: string;
+    workerInstanceId?: string;
+  }) => Promise<void>;
+  heartbeat?: (input: {
+    queueName: string;
+    jobId: string;
+    workerInstanceId?: string;
+  }) => Promise<void>;
+  completed?: (input: {
+    queueName: string;
+    jobId: string;
+    processingTimeMs?: number;
+  }) => Promise<void>;
+  timedOut?: (input: {
+    queueName: string;
+    jobId: string;
+    reason?: string;
+    error?: unknown;
+  }) => Promise<void>;
+  failed?: (input: {
+    queueName: string;
+    jobId: string;
+    attempts?: number;
+    isFinal?: boolean;
+    retryAt?: string;
+    error?: unknown;
+  }) => Promise<void>;
+};
+
+type HeartbeatStoreApi = {
+  heartbeat?: (input: {
+    queueName: string;
+    jobId: string;
+    workerInstanceId?: string;
+    intervalMs?: number;
+  }) => Promise<void>;
+  remove?: (queueName: string, jobId: string) => Promise<void>;
+};
 
-    const baseLogFields = buildBaseLogFields(message, options.getLogFields);
+const getWorkerInstanceId = (): string | undefined => {
+  return typeof (Env as Record<string, unknown>)['WORKER_INSTANCE_ID'] === 'string'
+    ? String((Env as Record<string, unknown>)['WORKER_INSTANCE_ID'])
+    : undefined;
+};
+
+const getTrackerApi = (): TrackerApi => {
+  return ((getJobStateTracker() ?? {}) as TrackerApi) ?? {};
+};
+
+const getHeartbeatStoreApi = (): HeartbeatStoreApi => {
+  return ((getJobHeartbeatStore() ?? {}) as HeartbeatStoreApi) ?? {};
+};
 
-    // Check for delayed execution
-    const payload = message.payload as Record<string, unknown> & { timestamp?: number };
-    const rawTimestamp = 'timestamp' in payload ? payload['timestamp'] : 0;
-    const timestamp = typeof rawTimestamp === 'number' ? rawTimestamp : 0;
+const removeHeartbeatIfSupported = async (queueName: string, jobId: string): Promise<void> => {
+  const heartbeatStore = getHeartbeatStoreApi();
+  if (typeof heartbeatStore.remove === 'function') {
+    await heartbeatStore.remove(queueName, jobId);
+  }
+};
 
-    if (timestamp > Date.now()) {
-      Logger.info(`${options.kindLabel} not due yet, re-queueing`, {
-        ...baseLogFields,
-        dueAt: new Date(timestamp).toISOString(),
+const scheduleHeartbeatLoop = (
+  trackerApi: TrackerApi,
+  queueName: string,
+  jobId: string,
+  workerInstanceId: string | undefined,
+  heartbeatIntervalMs: number
+): ReturnType<typeof setInterval> => {
+  return setInterval(() => {
+    if (typeof trackerApi.heartbeat === 'function') {
+      void trackerApi.heartbeat({
+        queueName,
+        jobId,
+        workerInstanceId,
       });
-      // Re-queue original payload
-      await Queue.enqueue(queueName, message.payload as BullMQPayload, driverName);
-      await Queue.ack(queueName, message.id, driverName);
-      return false;
     }
 
-    try {
-      Logger.info(`Processing queued ${options.kindLabel}`, baseLogFields);
-      await options.handle(message.payload);
-      await Queue.ack(queueName, message.id, driverName);
-      Logger.info(`${options.kindLabel} processed successfully`, baseLogFields);
-      return true;
-    } catch (error) {
-      const attempts = (message as QueueMessage<TPayload> & { attempts?: number }).attempts ?? 0;
-
-      Logger.error(`Failed to process ${options.kindLabel}`, {
-        ...baseLogFields,
-        error,
-        attempts,
+    const heartbeatStore = getHeartbeatStoreApi();
+    if (typeof heartbeatStore.heartbeat === 'function') {
+      void heartbeatStore.heartbeat({
+        queueName,
+        jobId,
+        workerInstanceId,
+        intervalMs: heartbeatIntervalMs,
       });
+    }
+  }, heartbeatIntervalMs);
+};
 
-      if (attempts < options.maxAttempts) {
-        await Queue.enqueue(queueName, message.payload as BullMQPayload, driverName);
-        Logger.info(`${options.kindLabel} re-queued for retry`, {
-          ...baseLogFields,
-          attempts: attempts + 1,
-        });
-      }
+const checkAndRequeueIfNotDue = async <TPayload>(
+  options: CreateQueueWorkerOptions<TPayload>,
+  queueName: string,
+  driverName: string | undefined,
+  message: QueueMessage<TPayload>,
+  baseLogFields: Record<string, unknown>
+): Promise<boolean> => {
+  const payload = message.payload as Record<string, unknown> & { timestamp?: number };
+  const rawTimestamp = 'timestamp' in payload ? payload['timestamp'] : 0;
+  const timestamp = typeof rawTimestamp === 'number' ? rawTimestamp : 0;
 
-      await Queue.ack(queueName, message.id, driverName);
-      // We processed the message (even if it failed), so return true to continue processing
-      return true;
+  if (timestamp <= Date.now()) return false;
+
+  Logger.info(`${options.kindLabel} not due yet, re-queueing`, {
+    ...baseLogFields,
+    dueAt: new Date(timestamp).toISOString(),
+  });
+  await Queue.enqueue(queueName, message.payload as BullMQPayload, driverName);
+  await Queue.ack(queueName, message.id, driverName);
+  return true;
+};
+
+const onProcessSuccess = async <TPayload>(input: {
+  options: CreateQueueWorkerOptions<TPayload>;
+  trackerApi: TrackerApi;
+  queueName: string;
+  driverName?: string;
+  message: QueueMessage<TPayload>;
+  startedAtMs: number;
+  baseLogFields: Record<string, unknown>;
+}): Promise<boolean> => {
+  await Queue.ack(input.queueName, input.message.id, input.driverName);
+
+  if (typeof input.trackerApi.completed === 'function') {
+    await input.trackerApi.completed({
+      queueName: input.queueName,
+      jobId: input.message.id,
+      processingTimeMs: Date.now() - input.startedAtMs,
+    });
+  }
+
+  await removeHeartbeatIfSupported(input.queueName, input.message.id);
+  Logger.info(`${input.options.kindLabel} processed successfully`, input.baseLogFields);
+  return true;
+};
+
+const onProcessFailure = async <TPayload>(input: {
+  options: CreateQueueWorkerOptions<TPayload>;
+  trackerApi: TrackerApi;
+  queueName: string;
+  driverName?: string;
+  message: QueueMessage<TPayload>;
+  baseLogFields: Record<string, unknown>;
+  error: unknown;
+}): Promise<boolean> => {
+  const attempts = getAttemptsFromMessage(input.message);
+  const nextAttempts = attempts + 1;
+  const isFinal = nextAttempts >= input.options.maxAttempts;
+  let retryAt: string | undefined;
+
+  Logger.error(`Failed to process ${input.options.kindLabel}`, {
+    ...input.baseLogFields,
+    error: input.error,
+    attempts: nextAttempts,
+  });
+
+  if (isTimeoutError(input.error) && typeof input.trackerApi.timedOut === 'function') {
+    await input.trackerApi.timedOut({
+      queueName: input.queueName,
+      jobId: input.message.id,
+      reason: `Worker processing exceeded timeout for ${input.options.kindLabel}`,
+      error: input.error,
+    });
+  }
+
+  if (nextAttempts < input.options.maxAttempts) {
+    const retryDelayMs = getRetryDelayMs(nextAttempts);
+    retryAt = new Date(Date.now() + retryDelayMs).toISOString();
+    const currentPayload =
+      typeof input.message.payload === 'object' && input.message.payload !== null
+        ? (input.message.payload as Record<string, unknown>)
+        : ({ payload: input.message.payload } as Record<string, unknown>);
+
+    const payloadForRetry: BullMQPayload = {
+      ...currentPayload,
+      attempts: nextAttempts,
+      timestamp: Date.now() + retryDelayMs,
+    };
+
+    await Queue.enqueue(input.queueName, payloadForRetry, input.driverName);
+    Logger.info(`${input.options.kindLabel} re-queued for retry`, {
+      ...input.baseLogFields,
+      attempts: nextAttempts,
+      retryDelayMs,
+    });
+  }
+
+  await Queue.ack(input.queueName, input.message.id, input.driverName);
+  await removeHeartbeatIfSupported(input.queueName, input.message.id);
+
+  if (typeof input.trackerApi.failed === 'function') {
+    await input.trackerApi.failed({
+      queueName: input.queueName,
+      jobId: input.message.id,
+      attempts: nextAttempts,
+      isFinal,
+      retryAt,
+      error: input.error,
+    });
+  }
+
+  return true;
+};
+
+const startTrackingAndHeartbeat = async <TPayload>(input: {
+  options: CreateQueueWorkerOptions<TPayload>;
+  trackerApi: TrackerApi;
+  queueName: string;
+  message: QueueMessage<TPayload>;
+}): Promise<{ startedAtMs: number; heartbeatTimer?: ReturnType<typeof setInterval> }> => {
+  const startedAtMs = Date.now();
+  const timeoutMs = resolveQueueJobTimeoutMs();
+  const heartbeatIntervalMs = Math.max(1000, getEnvInt('JOB_HEARTBEAT_INTERVAL_MS', 10000));
+  const attempts = getAttemptsFromMessage(input.message);
+  const workerInstanceId = getWorkerInstanceId();
+
+  if (typeof input.trackerApi.started === 'function') {
+    await input.trackerApi.started({
+      queueName: input.queueName,
+      jobId: input.message.id,
+      attempts: attempts + 1,
+      timeoutMs,
+      workerName: input.options.kindLabel,
+      workerInstanceId,
+    });
+  }
+
+  const heartbeatStore = getHeartbeatStoreApi();
+  if (typeof heartbeatStore.heartbeat === 'function') {
+    await heartbeatStore.heartbeat({
+      queueName: input.queueName,
+      jobId: input.message.id,
+      workerInstanceId,
+      intervalMs: heartbeatIntervalMs,
+    });
+  }
+
+  const heartbeatTimer = scheduleHeartbeatLoop(
+    input.trackerApi,
+    input.queueName,
+    input.message.id,
+    workerInstanceId,
+    heartbeatIntervalMs
+  );
+
+  return { startedAtMs, heartbeatTimer };
+};
+
+const processQueueMessage = async <TPayload>(
+  options: CreateQueueWorkerOptions<TPayload>,
+  queueName: string,
+  driverName?: string
+): Promise<boolean> => {
+  const message = await Queue.dequeue<TPayload>(queueName, driverName);
+  if (!message) return false;
+
+  const baseLogFields = buildBaseLogFields(message, options.getLogFields);
+
+  const isRequeued = await checkAndRequeueIfNotDue(
+    options,
+    queueName,
+    driverName,
+    message,
+    baseLogFields
+  );
+  if (isRequeued) return false;
+
+  let heartbeatTimer: ReturnType<typeof setInterval> | undefined;
+  const trackerApi = getTrackerApi();
+  const timeoutMs = resolveQueueJobTimeoutMs();
+  let startedAtMs: number;
+
+  try {
+    const tracking = await startTrackingAndHeartbeat({
+      options,
+      trackerApi,
+      queueName,
+      message,
+    });
+    startedAtMs = tracking.startedAtMs;
+    heartbeatTimer = tracking.heartbeatTimer;
+
+    Logger.info(`Processing queued ${options.kindLabel}`, baseLogFields);
+    await runWithTimeout(
+      async () => {
+        await options.handle(message.payload);
+      },
+      timeoutMs,
+      `${options.kindLabel}:${queueName}:${message.id}`
+    );
+
+    return onProcessSuccess({
+      options,
+      trackerApi,
+      queueName,
+      driverName,
+      message,
+      startedAtMs,
+      baseLogFields,
+    });
+  } catch (error) {
+    return onProcessFailure({
+      options,
+      trackerApi,
+      queueName,
+      driverName,
+      message,
+      baseLogFields,
+      error,
+    });
+  } finally {
+    if (heartbeatTimer !== undefined) {
+      clearInterval(heartbeatTimer);
     }
+  }
+};
+
+const createProcessOne = <TPayload>(
+  options: CreateQueueWorkerOptions<TPayload>
+): ((queueName?: string, driverName?: string) => Promise<boolean>) => {
+  return async (queueName = options.defaultQueueName, driverName?: string): Promise<boolean> => {
+    return processQueueMessage(options, queueName, driverName);
   };
 };
 

package/src/dashboard/workers-api.ts

@@ -552,7 +552,14 @@ async function getRedisQueueData(): Promise<QueueData> {
       throw ErrorFactory.createConfigError('Redis driver not configured');
     }
 
-    const monitor = QueueMonitor.create({ redis: redisConfig });
+    const monitor = QueueMonitor.create({
+      redis: {
+        host: redisConfig.host || 'localhost',
+        port: redisConfig.port || 6379,
+        db: redisConfig.database || 1,
+        password: redisConfig.password,
+      },
+    });
     const snapshot = await monitor.getSnapshot();
 
     let totalJobs = 0;

package/src/http/WorkerController.ts

@@ -4,7 +4,14 @@
  * HTTP handlers for worker management API
  */
 
-import { Env, Logger, getValidatedBody, type IRequest, type IResponse } from '@zintrust/core';
+import {
+  Cloudflare,
+  Env,
+  Logger,
+  getValidatedBody,
+  type IRequest,
+  type IResponse,
+} from '@zintrust/core';
 import type { Job } from 'bullmq';
 import { CanaryController } from '../CanaryController';
 import { HealthMonitor } from '../HealthMonitor';
@@ -34,42 +41,52 @@ const getBody = (req: IRequest): Record<string, unknown> => {
 
 // ==================== Core Worker Operations ====================
 
-/**
- * Create a new worker instance
- * @param req.body.name - Worker name (required)
- * @param req.body.queueName - Queue name (required)
- * @param req.body.processor - Job processor function (required; internal only)
- * @param req.body.version - Worker version (optional)
- * @param req.body.options - BullMQ worker options (optional)
- * @param req.body.infrastructure - Infrastructure config (optional)
- * @param req.body.features - Feature flags (optional)
- * @param req.body.datacenter - Datacenter placement config (optional)
- * @returns Success response with worker name
- */
-async function create(req: IRequest, res: IResponse): Promise<void> {
-  Logger.info('WorkerController.create called');
-  try {
-    const body = req.data() as unknown as WorkerFactoryConfig;
+const isCloudflareEnv = (): boolean => Cloudflare.getWorkersEnv() !== null;
 
-    // Validate required fields
-    if (!body.name || !body.queueName || !body.processor || !body.version) {
-      return res.setStatus(400).json({
-        error: 'Missing required fields',
-        message: 'name, queueName, processor, and version are required',
-        code: 'MISSING_REQUIRED_FIELDS',
-      });
-    }
+const validateCreatePayload = (body: WorkerFactoryConfig, res: IResponse): boolean => {
+  if (!body.name || !body.queueName || !body.processor || !body.version) {
+    res.setStatus(400).json({
+      error: 'Missing required fields',
+      message: 'name, queueName, processor, and version are required',
+      code: 'MISSING_REQUIRED_FIELDS',
+    });
+    return false;
+  }
+  return true;
+};
+
+const respondIfWorkerExists = async (
+  name: string,
+  persistenceOverride: ReturnType<typeof resolvePersistenceOverride>,
+  res: IResponse
+): Promise<boolean> => {
+  const existing = await WorkerFactory.getPersisted(name, persistenceOverride);
+  if (!existing) return false;
+
+  res.status(409).json({
+    ok: false,
+    error: `Worker ${name} already exists`,
+    code: 'WORKER_EXISTS',
+    worker: existing,
+  });
+  return true;
+};
 
-    const rawProcessor = body.processor;
-    let processor: (job: Job) => Promise<unknown>;
-    let processorSpec: string | undefined;
+const resolveCreateProcessor = async (
+  body: WorkerFactoryConfig,
+  res: IResponse
+): Promise<{ processor: (job: Job) => Promise<unknown>; processorSpec?: string } | null> => {
+  const rawProcessor = body.processor;
+  let processor = rawProcessor as (job: Job) => Promise<unknown>;
+  let processorSpec: string | undefined;
 
+  if (!isCloudflareEnv()) {
     if (typeof rawProcessor === 'string') {
       processorSpec = rawProcessor;
       const resolved = await WorkerFactory.resolveProcessorSpec(rawProcessor);
       if (!resolved) {
         res.setStatus(400).json({ error: 'Processor spec could not be resolved' });
-        return;
+        return null;
       }
       processor = resolved;
     } else {
@@ -78,23 +95,81 @@ async function create(req: IRequest, res: IResponse): Promise<void> {
 
     if (typeof processor !== 'function') {
       res.setStatus(400).json({ error: 'Processor must be a function or resolvable path' });
-      return;
+      return null;
     }
 
-    const config = {
-      ...(body as WorkerFactoryConfig),
-      processor,
-      processorSpec,
-    };
+    return { processor, processorSpec };
+  }
 
-    await WorkerFactory.create(config);
+  // Cloudflare environment: treat string as spec, otherwise accept as-is
+  if (typeof rawProcessor === 'string') {
+    processorSpec = rawProcessor;
+    processor = async () => {};
+  }
+
+  return { processor, processorSpec };
+};
 
+const finalizeWorkerCreate = async (config: WorkerFactoryConfig, res: IResponse): Promise<void> => {
+  const globalAutoStart = Env.getBool('WORKER_AUTO_START', false);
+  const workerAutoStart = config.autoStart ?? globalAutoStart;
+  const isCloudflare = isCloudflareEnv();
+
+  if (!isCloudflare && globalAutoStart && workerAutoStart) {
+    await WorkerFactory.create(config);
     res.json({
       ok: true,
       workerName: config.name,
       status: 'creating',
       message: 'Worker creation started. Check status endpoint for progress.',
     });
+    return;
+  }
+
+  await WorkerFactory.register(config);
+  res.json({
+    ok: true,
+    workerName: config.name,
+    status: 'registered',
+    message: isCloudflare
+      ? 'Worker registered. Cloudflare environment detected; worker will be picked up by external processor.'
+      : 'Worker registered successfully.',
+  });
+};
+
+/**
+ * Create a new worker instance
+ * @param req.body.name - Worker name (required)
+ * @param req.body.queueName - Queue name (required)
+ * @param req.body.processor - Job processor function (required; internal only)
+ * @param req.body.version - Worker version (optional)
+ * @param req.body.options - BullMQ worker options (optional)
+ * @param req.body.infrastructure - Infrastructure config (optional)
+ * @param req.body.features - Feature flags (optional)
+ * @param req.body.datacenter - Datacenter placement config (optional)
+ * @returns Success response with worker name
+ */
+async function create(req: IRequest, res: IResponse): Promise<void> {
+  Logger.info('WorkerController.create called');
+  try {
+    const body = req.data() as unknown as WorkerFactoryConfig;
+    if (!validateCreatePayload(body, res)) return;
+
+    const name = body.name;
+    const persistenceOverride = resolvePersistenceOverride(req);
+    const exists = await respondIfWorkerExists(name, persistenceOverride, res);
+    if (exists) return;
+
+    const resolvedProcessor = await resolveCreateProcessor(body, res);
+    if (!resolvedProcessor) return;
+
+    const config = {
+      ...(body as WorkerFactoryConfig),
+      processor: resolvedProcessor.processor,
+      processorSpec: resolvedProcessor.processorSpec,
+    };
+
+    await finalizeWorkerCreate(config, res);
   } catch (error) {
     Logger.error('WorkerController.create failed', error);
     res.setStatus(500).json({ error: (error as Error).message });