@zintrust/workers 0.4.4 → 0.4.34

package/src/WorkerInit.ts

@@ -148,7 +148,7 @@ type PersistenceOverride = WorkerPersistenceConfig;
 
 type AutoStartTask = AutoStartCandidate & {
   persistenceOverride: PersistenceOverride;
-  source: 'database' | 'redis' | 'memory';
+  source: 'database' | 'redis' | 'memory' | 'file';
 };
 
 const resolveAutoStartCandidates = (records: AutoStartCandidate[]): AutoStartCandidate[] => {
@@ -229,6 +229,79 @@ const collectAutoStartTasks = async (): Promise<AutoStartTask[]> => {
   return tasks;
 };
 
+export const buildFileBackedAutoStartTasks = (
+  records: AutoStartCandidate[],
+  warn: (message: string) => void = Logger.warn
+): AutoStartTask[] => {
+  const tasks: AutoStartTask[] = [];
+  const seenWorkerNames = new Set<string>();
+  const candidates = resolveAutoStartCandidates(records);
+
+  for (const record of candidates) {
+    if (seenWorkerNames.has(record.name)) {
+      warn(
+        `Worker ${record.name} appears multiple times in file-backed discovery; keeping the first definition and skipping duplicates.`
+      );
+      continue;
+    }
+
+    seenWorkerNames.add(record.name);
+    tasks.push({
+      ...record,
+      persistenceOverride: { driver: 'memory' },
+      source: 'file',
+    });
+  }
+
+  return tasks;
+};
+
+export const selectAutoStartTasks = (
+  persistedTasks: AutoStartTask[],
+  fileRecords: AutoStartCandidate[],
+  warn: (message: string) => void = Logger.warn
+): AutoStartTask[] => {
+  if (persistedTasks.length > 0) {
+    return persistedTasks;
+  }
+
+  return buildFileBackedAutoStartTasks(fileRecords, warn);
+};
+
+export const selectAutoStartNames = (
+  persistedRecords: AutoStartCandidate[],
+  fileRecords: AutoStartCandidate[],
+  warn: (message: string) => void = Logger.warn
+): { names: string[]; source: 'persisted' | 'file' | 'none' } => {
+  const persistedNames = resolveAutoStartCandidates(persistedRecords).map((record) => record.name);
+
+  if (persistedNames.length > 0) {
+    return { names: persistedNames, source: 'persisted' };
+  }
+
+  const fileNames = buildFileBackedAutoStartTasks(fileRecords, warn).map((record) => record.name);
+  return { names: fileNames, source: fileNames.length > 0 ? 'file' : 'none' };
+};
+
+const collectFileBackedAutoStartTasks = async (): Promise<AutoStartTask[]> => {
+  try {
+    const records = await WorkerFactory.listFileBackedRecords();
+    const tasks = buildFileBackedAutoStartTasks(records);
+
+    Logger.debug('File-backed auto-start discovery', {
+      totalRecords: records.length,
+      candidateCount: tasks.length,
+    });
+
+    return tasks;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    Logger.warn(`File-backed auto-start discovery failed: ${message}`);
+  }
+
+  return [];
+};
+
 const isWorkerTrulyRunning = async (name: string): Promise<boolean> => {
   const existing = WorkerFactory.get(name);
   if (!existing) return false;
@@ -357,16 +430,21 @@ async function autoStartPersistedWorkers(): Promise<void> {
   }
 
   try {
-    const candidates = await collectAutoStartTasks();
+    let candidates = await collectAutoStartTasks();
+
+    if (candidates.length === 0) {
+      candidates = await collectFileBackedAutoStartTasks();
+    }
 
     const results = await Promise.all(candidates.map(async (record) => autoStartOneWorker(record)));
 
     const startedCount = results.filter((item) => item.started).length;
     const skippedCount = results.filter((item) => item.skipped).length;
-    Logger.info('Auto-started persisted workers', {
+    Logger.info('Auto-started workers', {
       total: candidates.length,
       started: startedCount,
       skipped: skippedCount,
+      source: candidates[0]?.source ?? 'none',
     });
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);

package/src/createQueueWorker.ts

@@ -2,6 +2,17 @@ import type { BullMQPayload, QueueMessage } from '@zintrust/core';
 import * as Core from '@zintrust/core';
 import { Env, Logger, Queue } from '@zintrust/core';
 
+type QueueApi = Readonly<{
+  enqueue: (queue: string, payload: BullMQPayload, driverName?: string) => Promise<string>;
+  dequeue: <TPayload>(
+    queue: string,
+    driverName?: string
+  ) => Promise<QueueMessage<TPayload> | undefined>;
+  ack: (queue: string, id: string, driverName?: string) => Promise<void>;
+}>;
+
+const TypedQueue = Queue as QueueApi;
+
 const RETRY_BASE_DELAY_MS = 1000;
 const RETRY_MAX_DELAY_MS = 30000;
 
@@ -97,9 +108,7 @@ const getAttemptsFromMessage = <TPayload>(message: QueueMessage<TPayload>): numb
     typeof message.payload === 'object' && message.payload !== null
       ? normalizeAttempts((message.payload as Record<string, unknown>)['attempts'])
       : 0;
-  const messageAttempts = normalizeAttempts(
-    (message as QueueMessage<TPayload> & { attempts?: number }).attempts
-  );
+  const messageAttempts = normalizeAttempts(message.attempts);
   return Math.max(payloadAttempts, messageAttempts);
 };
 
@@ -127,6 +136,11 @@ type QueueWorker = {
     signal?: AbortSignal;
     maxDurationMs?: number;
   }) => Promise<number>;
+  __zintrustQueueWorkerMeta?: Readonly<{
+    kindLabel: string;
+    defaultQueueName: string;
+    maxAttempts: number;
+  }>;
 };
 
 export type CreateQueueWorkerOptions<TPayload> = {
@@ -147,6 +161,14 @@ const buildBaseLogFields = <TPayload>(
   };
 };
 
+const toBullMQPayload = <TPayload>(payload: TPayload): BullMQPayload => {
+  if (typeof payload === 'object' && payload !== null) {
+    return { ...(payload as Record<string, unknown>) };
+  }
+
+  return { payload };
+};
+
 type TrackerApi = {
   started?: (input: {
     queueName: string;
@@ -258,8 +280,8 @@ const checkAndRequeueIfNotDue = async <TPayload>(
     ...baseLogFields,
     dueAt: new Date(timestamp).toISOString(),
   });
-  await Queue.enqueue(queueName, message.payload as BullMQPayload, driverName);
-  await Queue.ack(queueName, message.id, driverName);
+  await TypedQueue.enqueue(queueName, toBullMQPayload(message.payload), driverName);
+  await TypedQueue.ack(queueName, message.id, driverName);
   return true;
 };
 
@@ -272,7 +294,7 @@ const onProcessSuccess = async <TPayload>(input: {
   startedAtMs: number;
   baseLogFields: Record<string, unknown>;
 }): Promise<boolean> => {
-  await Queue.ack(input.queueName, input.message.id, input.driverName);
+  await TypedQueue.ack(input.queueName, input.message.id, input.driverName);
 
   if (typeof input.trackerApi.completed === 'function') {
     await input.trackerApi.completed({
@@ -319,10 +341,7 @@ const onProcessFailure = async <TPayload>(input: {
   if (nextAttempts < input.options.maxAttempts) {
     const retryDelayMs = getRetryDelayMs(nextAttempts);
     retryAt = new Date(Date.now() + retryDelayMs).toISOString();
-    const currentPayload =
-      typeof input.message.payload === 'object' && input.message.payload !== null
-        ? (input.message.payload as Record<string, unknown>)
-        : ({ payload: input.message.payload } as Record<string, unknown>);
+    const currentPayload = toBullMQPayload(input.message.payload);
 
     const payloadForRetry: BullMQPayload = {
       ...currentPayload,
@@ -330,7 +349,7 @@ const onProcessFailure = async <TPayload>(input: {
       timestamp: Date.now() + retryDelayMs,
     };
 
-    await Queue.enqueue(input.queueName, payloadForRetry, input.driverName);
+    await TypedQueue.enqueue(input.queueName, payloadForRetry, input.driverName);
     Logger.info(`${input.options.kindLabel} re-queued for retry`, {
       ...input.baseLogFields,
       attempts: nextAttempts,
@@ -338,7 +357,7 @@ const onProcessFailure = async <TPayload>(input: {
     });
   }
 
-  await Queue.ack(input.queueName, input.message.id, input.driverName);
+  await TypedQueue.ack(input.queueName, input.message.id, input.driverName);
   await removeHeartbeatIfSupported(input.queueName, input.message.id);
 
   if (typeof input.trackerApi.failed === 'function') {
@@ -404,7 +423,7 @@ const processQueueMessage = async <TPayload>(
   queueName: string,
   driverName?: string
 ): Promise<boolean> => {
-  const message = await Queue.dequeue<TPayload>(queueName, driverName);
+  const message = await TypedQueue.dequeue<TPayload>(queueName, driverName);
   if (!message) return false;
 
   const baseLogFields = buildBaseLogFields(message, options.getLogFields);
@@ -597,6 +616,17 @@ export function createQueueWorker<TPayload>(
   const processAll = createProcessAll(options.defaultQueueName, processOne);
   const runOnce = createRunOnce(options.defaultQueueName, processOne);
   const startWorker = createStartWorker(options.kindLabel, options.defaultQueueName, processOne);
+  const queueWorkerMeta = Object.freeze({
+    kindLabel: options.kindLabel,
+    defaultQueueName: options.defaultQueueName,
+    maxAttempts: options.maxAttempts,
+  });
 
-  return Object.freeze({ processOne, processAll, runOnce, startWorker });
+  return Object.freeze({
+    processOne,
+    processAll,
+    runOnce,
+    startWorker,
+    __zintrustQueueWorkerMeta: queueWorkerMeta,
+  });
 }

package/src/dashboard/workers-api.ts

@@ -278,6 +278,10 @@ async function getWorkersFromMixedPersistence(
       ...transformToWorkerData(redisRecords, 'redis'),
     ];
 
+    if (workers.length === 0) {
+      return getWorkersFromFileFallback(limit, query.includeInactive === true);
+    }
+
     return {
       workers,
       total:
@@ -312,6 +316,11 @@ async function getWorkersFromSinglePersistence(
       { driver: normalizedDriver },
       { offset, limit, includeInactive: query.includeInactive }
     );
+
+    if (driverRecords.length === 0) {
+      return getWorkersFromFileFallback(limit, query.includeInactive === true);
+    }
+
     const workers = transformToWorkerData(driverRecords, normalizedDriver);
 
     return {
@@ -333,6 +342,35 @@ async function getWorkersFromSinglePersistence(
   }
 }
 
+async function getWorkersFromFileFallback(
+  limit: number,
+  includeInactive: boolean
+): Promise<PersistenceResult> {
+  try {
+    const discovered = await WorkerFactory.listFileBackedRecords();
+    const filtered = includeInactive
+      ? discovered
+      : discovered.filter((record) => record.activeStatus !== false);
+
+    return {
+      workers: transformToWorkerData(filtered, 'memory'),
+      total: filtered.length,
+      drivers: getAvailableDriversFromDrivers(['memory']),
+      effectiveLimit: limit,
+      prePaginated: false,
+    };
+  } catch (error) {
+    Logger.debug('File-backed worker fallback failed', error);
+    return {
+      workers: [],
+      total: 0,
+      drivers: getAvailableDriversFromDrivers(['memory']),
+      effectiveLimit: limit,
+      prePaginated: false,
+    };
+  }
+}
+
 const normalizeDriver = (driver: string): WorkerDriver => {
   if (driver === 'db' || driver === 'database') return 'database';
   if (driver === 'redis') return 'redis';
@@ -527,17 +565,17 @@ async function getQueueData(): Promise<QueueData> {
     // Get queue statistics based on QUEUE_DRIVER
     switch (queueDriver) {
       case 'redis':
-        return getRedisQueueData();
+        return await getRedisQueueData();
       case 'database':
-        return getDatabaseQueueData();
+        return await getDatabaseQueueData();
       case 'db':
-        return getDatabaseQueueData();
+        return await getDatabaseQueueData();
       default:
-        return getMemoryQueueData();
+        return await getMemoryQueueData();
     }
   } catch (error) {
     Logger.error('Error fetching queue data:', error);
-    return getMemoryQueueData();
+    return await getMemoryQueueData();
   }
 }
 
@@ -603,7 +641,12 @@ async function getDatabaseQueueData(): Promise<QueueData> {
     const db = await useEnsureDbConnected();
 
     // Get queue statistics from actual database tables using proper query builder
-    const queueStats = (await db
+    const queueStats: {
+      totalQueues: number;
+      totalJobs: number;
+      processingJobs: number;
+      failedJobs: number;
+    } | null = await db
       .table('queue_jobs')
       .select('COUNT(DISTINCT queue) as totalQueues')
       .selectAs('COUNT(*)', 'totalJobs')
@@ -612,12 +655,7 @@ async function getDatabaseQueueData(): Promise<QueueData> {
         'processingJobs'
       )
       .selectAs('SUM(CASE WHEN failed_at IS NOT NULL THEN 1 ELSE 0 END)', 'failedJobs')
-      .first()) as {
-      totalQueues: number;
-      totalJobs: number;
-      processingJobs: number;
-      failedJobs: number;
-    } | null;
+      .first();
 
     const stats = queueStats || {
       totalQueues: 0,
@@ -733,7 +771,9 @@ async function enrichWithDetails(workers: WorkerData[]): Promise<WorkerData[]> {
 async function buildWorkerDetails(worker: WorkerData): Promise<WorkerData> {
   try {
     const persistenceOverride = resolvePersistenceOverride(worker.driver);
-    const persisted = await WorkerFactory.getPersisted(worker.name, persistenceOverride);
+    const persisted =
+      (await WorkerFactory.getPersisted(worker.name, persistenceOverride)) ??
+      (await WorkerFactory.getFileBackedRecord(worker.name));
     const health = await getWorkerHealthSnapshot(worker.name, worker.health);
     const metrics = await getWorkerMetricsSnapshot(worker.name, worker);
     const configuration = buildWorkerConfiguration(worker, persisted);
@@ -896,6 +936,13 @@ export async function getWorkerDetails(name: string, driver?: string): Promise<W
     }
   }
 
+  if (!worker) {
+    const fileBacked = await WorkerFactory.getFileBackedRecord(name);
+    if (fileBacked) {
+      worker = buildWorkerFromRecord(fileBacked, 'memory');
+    }
+  }
+
   if (!worker) {
     throw ErrorFactory.createWorkerError(`Worker ${name} not found`);
   }

package/src/index.ts

@@ -44,7 +44,12 @@ export type {
   WorkerFactoryConfig,
   WorkerPersistenceConfig,
 } from './WorkerFactory';
-export { WorkerInit } from './WorkerInit';
+export {
+  buildFileBackedAutoStartTasks,
+  selectAutoStartNames,
+  selectAutoStartTasks,
+  WorkerInit,
+} from './WorkerInit';
 export { WorkerShutdown } from './WorkerShutdown';
 
 // HTTP Controllers & Routes
@@ -66,10 +71,10 @@ export type {
   WorkerConfig,
   WorkerCostConfig,
   WorkerObservabilityConfig,
-  WorkerStatus,
-  WorkerVersioningConfig,
   WorkersConfigOverrides,
   WorkersGlobalConfig,
+  WorkerStatus,
+  WorkerVersioningConfig,
 } from '@zintrust/core';
 
 // Re-export bullmq types for type compatibility
@@ -92,6 +97,7 @@ export type {
 } from './ChaosEngineering';
 export type { ISLAConfig, ISLAReport, ISLAStatus, ISLAViolation, ITimeRange } from './SLAMonitor';
 
+export type * from './config/workerConfig';
 export type * from './type';
 
 /**

package/src/register.ts

@@ -1,12 +1,12 @@
+type Registry = {
+  register: (id: string, provider: CliCommandProvider) => void;
+};
+
 type CliCommandProvider = {
   getCommand: () => unknown;
   name?: string;
 };
 
-type Registry = {
-  register: (id: string, provider: CliCommandProvider) => void;
-};
-
 type WorkerCommandsModule = {
   WorkerCommands: {
     createWorkerListCommand: () => CliCommandProvider;
@@ -20,7 +20,12 @@ type WorkerCommandsModule = {
 };
 
 const commandModule = (await (async (): Promise<WorkerCommandsModule> => {
-  return (await import('@zintrust/core/cli')) as unknown as WorkerCommandsModule;
+  const workerCommandsSpecifier = '@zintrust/core/worker-commands';
+  try {
+    return (await import(workerCommandsSpecifier)) as unknown as WorkerCommandsModule;
+  } catch {
+    return (await import('@zintrust/core/cli')) as unknown as WorkerCommandsModule;
+  }
 })()) satisfies WorkerCommandsModule;
 
 const getWorkerProviders = (): Array<[string, CliCommandProvider]> => {
@@ -59,12 +64,12 @@ registerWorkerCliCommands({
 });
 
 try {
-  const core = (await import('@zintrust/core/cli')) as unknown as {
+  const coreCli = (await import('@zintrust/core/cli')) as unknown as {
     OptionalCliCommandRegistry?: Registry;
   };
 
-  if (core.OptionalCliCommandRegistry !== undefined) {
-    registerWorkerCliCommands(core.OptionalCliCommandRegistry);
+  if (coreCli.OptionalCliCommandRegistry !== undefined) {
+    registerWorkerCliCommands(coreCli.OptionalCliCommandRegistry);
   }
 } catch {
   // no-op