@zintrust/core 0.4.17 → 0.4.19

package/src/tools/queue/QueueReliabilityOrchestrator.ts

@@ -1,97 +0,0 @@
-import { Env } from '@config/env';
-import { Logger } from '@config/logger';
-import { JobReconciliationRunner } from '@queue/JobReconciliationRunner';
-import { JobRecoveryDaemon } from '@queue/JobRecoveryDaemon';
-import { StalledJobMonitor } from '@queue/StalledJobMonitor';
-
-type TimerRef = ReturnType<typeof setInterval>;
-
-type UnrefableTimer = { unref: () => void };
-
-const isUnrefableTimer = (timer: unknown): timer is UnrefableTimer => {
-  return (
-    typeof timer === 'object' &&
-    timer !== null &&
-    'unref' in timer &&
-    typeof (timer as UnrefableTimer).unref === 'function'
-  );
-};
-
-type OrchestratorState = {
-  started: boolean;
-  reconciliationTimer?: TimerRef;
-  recoveryTimer?: TimerRef;
-  stalledTimer?: TimerRef;
-};
-
-const state: OrchestratorState = {
-  started: false,
-};
-
-const clearTimer = (timer: TimerRef | undefined): void => {
-  if (timer === undefined) return;
-  clearInterval(timer);
-};
-
-const startInterval = (handler: () => Promise<void>, intervalMs: number): TimerRef => {
-  const timer = setInterval(() => {
-    handler().catch((error: unknown) => {
-      Logger.warn('Queue reliability interval failed', {
-        error: error instanceof Error ? error : String(error),
-      });
-    });
-  }, intervalMs);
-
-  if (isUnrefableTimer(timer)) {
-    timer.unref();
-  }
-
-  return timer;
-};
-
-export const QueueReliabilityOrchestrator = Object.freeze({
-  isEnabled(): boolean {
-    return Env.getBool('JOB_RELIABILITY_ENABLED', true);
-  },
-
-  start(): void {
-    if (!this.isEnabled()) return;
-    if (state.started) return;
-
-    const reconciliationMs = Math.max(5000, Env.getInt('JOB_RECONCILIATION_INTERVAL_MS', 60000));
-    const recoveryMs = Math.max(5000, Env.getInt('JOB_RECOVERY_INTERVAL_MS', 30000));
-    const stalledMs = Math.max(5000, Env.getInt('STALLED_JOB_CHECK_INTERVAL_MS', 30000));
-
-    state.reconciliationTimer = startInterval(async () => {
-      await JobReconciliationRunner.runOnce();
-    }, reconciliationMs);
-
-    state.recoveryTimer = startInterval(async () => {
-      await JobRecoveryDaemon.runOnce();
-    }, recoveryMs);
-
-    state.stalledTimer = startInterval(async () => {
-      await StalledJobMonitor.scanOnce();
-    }, stalledMs);
-
-    state.started = true;
-    Logger.info('Queue reliability orchestrator started', {
-      reconciliationMs,
-      recoveryMs,
-      stalledMs,
-    });
-  },
-
-  stop(): void {
-    clearTimer(state.reconciliationTimer);
-    clearTimer(state.recoveryTimer);
-    clearTimer(state.stalledTimer);
-
-    state.reconciliationTimer = undefined;
-    state.recoveryTimer = undefined;
-    state.stalledTimer = undefined;
-    state.started = false;
-  },
-});
-
-export default QueueReliabilityOrchestrator;

package/src/tools/queue/QueueRuntimeRegistration.ts

@@ -1,128 +0,0 @@
-import { Logger } from '@config/logger';
-import { Env } from '@config/env';
-import type { QueueConfig } from '@config/queue';
-import { ErrorFactory } from '@exceptions/ZintrustError';
-import { ZintrustLang } from '@lang/lang';
-import { existsSync } from '@node-singletons/fs';
-import * as path from '@node-singletons/path';
-import { pathToFileURL } from '@node-singletons/url';
-import { detectRuntime } from '@runtime/detectRuntime';
-import { DatabaseQueue } from '@tools/queue/drivers/Database';
-import { autoRegisterJobStateTrackerPersistenceFromEnv } from '@tools/queue/JobStateTrackerDbPersistence';
-import { QueueReliabilityOrchestrator } from '@tools/queue/QueueReliabilityOrchestrator';
-
-import { InMemoryQueue } from '@tools/queue/drivers/InMemory';
-import { Queue } from '@tools/queue/Queue';
-
-/**
- * Register queue drivers from runtime config.
- *
- * This follows the framework's config-driven availability pattern:
- * - Built-in drivers are registered so `QUEUE_DRIVER=sync|inmemory|redis` works out of the box.
- * - If the configured default is registered, it is ALSO registered as 'default'.
- * - Unknown/unregistered driver names still throw when selected.
- */
-const registerRedisDriverIfAvailable = async (): Promise<boolean> => {
-  try {
-    const mod = (await import('@zintrust/queue-redis')) as unknown as {
-      RedisQueue?: typeof Queue;
-      BullMQRedisQueue?: typeof Queue;
-    };
-
-    if (mod.RedisQueue !== undefined) {
-      Queue.register('redis', mod.RedisQueue as unknown as Parameters<typeof Queue.register>[1]);
-      return true;
-    }
-
-    if (mod.BullMQRedisQueue !== undefined) {
-      Queue.register(
-        'redis',
-        mod.BullMQRedisQueue as unknown as Parameters<typeof Queue.register>[1]
-      );
-      return true;
-    }
-  } catch {
-    // Fall back to local dist build output when running inside the core repo Docker image.
-    // In that environment, `@zintrust/queue-redis` is not installed in node_modules,
-    // but the compiled package is available at `dist/packages/queue-redis`.
-    try {
-      const cwd =
-        typeof process !== 'undefined' && typeof process.cwd === 'function' ? process.cwd() : '';
-      if (cwd.trim() === '') return false;
-
-      const localEntry = path.join(cwd, 'dist', 'packages', 'queue-redis', 'src', 'index.js');
-      if (!existsSync(localEntry)) return false;
-
-      const url = pathToFileURL(localEntry).href;
-      const localMod = (await import(url)) as unknown as {
-        RedisQueue?: typeof Queue;
-        BullMQRedisQueue?: typeof Queue;
-      };
-
-      if (localMod.RedisQueue !== undefined) {
-        Queue.register(
-          'redis',
-          localMod.RedisQueue as unknown as Parameters<typeof Queue.register>[1]
-        );
-        return true;
-      }
-
-      if (localMod.BullMQRedisQueue !== undefined) {
-        Queue.register(
-          'redis',
-          localMod.BullMQRedisQueue as unknown as Parameters<typeof Queue.register>[1]
-        );
-        return true;
-      }
-    } catch {
-      // ignore
-    }
-
-    return false;
-  }
-
-  return false;
-};
-
-export async function registerQueuesFromRuntimeConfig(config: QueueConfig): Promise<void> {
-  autoRegisterJobStateTrackerPersistenceFromEnv();
-
-  // Built-in drivers (core)
-  Queue.register('inmemory', InMemoryQueue);
-  Queue.register(ZintrustLang.DATABASE, DatabaseQueue);
-  // Project templates use QUEUE_DRIVER=sync; treat this as an alias of in-memory.
-  Queue.register('sync', InMemoryQueue);
-  const defaultName = (config.default ?? '').toString().trim().toLowerCase();
-  if (defaultName.length === 0) {
-    throw ErrorFactory.createConfigError('Queue default driver is not configured');
-  }
-
-  if (defaultName === 'redis') {
-    const registered = await registerRedisDriverIfAvailable();
-    if (!registered) {
-      throw ErrorFactory.createConfigError(
-        'Redis queue driver is not registered. Install queue:redis via zin plugin install.'
-      );
-    }
-  }
-
-  try {
-    const drv = Queue.get(defaultName);
-    Queue.register('default', drv);
-
-    if (Env.getBool('JOB_RELIABILITY_AUTOSTART', false)) {
-      QueueReliabilityOrchestrator.start();
-    }
-  } catch (error) {
-    const { isCloudflare } = detectRuntime();
-    if (isCloudflare) {
-      Logger.warn(
-        `[queue] Default driver '${defaultName}' is unavailable in Cloudflare runtime; falling back to 'sync'.`
-      );
-      Queue.register('default', Queue.get('sync'));
-      return;
-    }
-
-    throw ErrorFactory.createConfigError('Queue default driver is not available', error);
-  }
-}

package/src/tools/queue/QueueTracing.ts

@@ -1,211 +0,0 @@
-import { OpenTelemetry } from '@/observability/OpenTelemetry';
-import { generateUuid } from '@common/utility';
-import { Env } from '@config/env';
-import { Logger } from '@config/logger';
-
-export type QueueTraceOperation = 'enqueue' | 'dequeue' | 'ack' | 'length' | 'drain';
-
-export type QueueTraceStatus = 'ok' | 'error';
-
-export type QueueTraceEvent = {
-  traceId: string;
-  spanId: string;
-  queueName: string;
-  operation: QueueTraceOperation;
-  status: QueueTraceStatus;
-  startedAt: string;
-  endedAt: string;
-  durationMs: number;
-  error?: string;
-  attributes?: Record<string, unknown>;
-};
-
-export type QueueTraceExporter = (events: ReadonlyArray<QueueTraceEvent>) => void | Promise<void>;
-
-const storedEvents: QueueTraceEvent[] = [];
-const pendingEvents: QueueTraceEvent[] = [];
-const exporters = new Set<QueueTraceExporter>();
-
-const normalizeSampleRate = (): number => {
-  const value = Env.getFloat('QUEUE_TRACING_SAMPLE_RATE', 1);
-  if (Number.isFinite(value) === false) return 1;
-  return Math.max(0, Math.min(1, value));
-};
-
-const isEnabled = (): boolean => Env.getBool('QUEUE_TRACING_ENABLED', false);
-
-const shouldSample = (): boolean => {
-  const rate = normalizeSampleRate();
-  if (rate >= 1) return true;
-  if (rate <= 0) return false;
-  return Math.random() <= rate; // NOSONAR - Math.random() is sufficient for sampling decisions in this context.
-};
-
-const maxEvents = (): number => Math.max(1, Env.getInt('QUEUE_TRACING_MAX_EVENTS', 5000));
-
-const retentionMs = (): number =>
-  Math.max(1000, Env.getInt('QUEUE_TRACING_RETENTION_MS', 86400000));
-
-const exportBatchSize = (): number =>
-  Math.max(1, Env.getInt('QUEUE_TRACING_EXPORT_BATCH_SIZE', 20));
-
-const shouldExportToOtel = (): boolean => Env.getBool('QUEUE_TRACING_EXPORT_OTEL', true);
-
-const pruneStored = (): void => {
-  const cutoff = Date.now() - retentionMs();
-  const filtered = storedEvents.filter((event) => {
-    const ended = new Date(event.endedAt).getTime();
-    if (Number.isNaN(ended)) return false;
-    return ended >= cutoff;
-  });
-
-  storedEvents.splice(0, storedEvents.length, ...filtered);
-
-  const keep = maxEvents();
-  if (storedEvents.length > keep) {
-    storedEvents.splice(0, storedEvents.length - keep);
-  }
-};
-
-const finalize = (event: QueueTraceEvent): void => {
-  storedEvents.push(event);
-  pendingEvents.push(event);
-  pruneStored();
-};
-
-const normalizeError = (error: unknown): string => {
-  if (error instanceof Error) return error.message;
-  return String(error);
-};
-
-const tryExportToOpenTelemetry = (batch: ReadonlyArray<QueueTraceEvent>): void => {
-  if (shouldExportToOtel() === false) return;
-
-  const recordQueueSpan = OpenTelemetry.recordQueueOperationSpan as
-    | ((input: {
-        queueName: string;
-        operation: 'enqueue' | 'dequeue' | 'ack' | 'length' | 'drain';
-        durationMs: number;
-        status: 'ok' | 'error';
-      }) => void)
-    | undefined;
-  if (recordQueueSpan === undefined) return;
-
-  batch.forEach((event) => {
-    recordQueueSpan({
-      queueName: event.queueName,
-      operation: event.operation,
-      durationMs: event.durationMs,
-      status: event.status,
-    });
-  });
-};
-
-export const QueueTracing = Object.freeze({
-  isEnabled,
-
-  registerExporter(exporter: QueueTraceExporter): void {
-    exporters.add(exporter);
-  },
-
-  unregisterExporter(exporter: QueueTraceExporter): void {
-    exporters.delete(exporter);
-  },
-
-  async flush(): Promise<number> {
-    if (pendingEvents.length === 0) return 0;
-
-    const batch = pendingEvents.splice(0, pendingEvents.length);
-    tryExportToOpenTelemetry(batch);
-
-    if (exporters.size > 0) {
-      const settled = await Promise.allSettled(
-        Array.from(exporters).map(async (exporter) => exporter(batch))
-      );
-
-      settled.forEach((result) => {
-        if (result.status === 'rejected') {
-          Logger.warn('Queue trace exporter failed', {
-            error: String(result.reason),
-          });
-        }
-      });
-    }
-
-    return batch.length;
-  },
-
-  snapshot(limit = 100): QueueTraceEvent[] {
-    const safeLimit = Math.max(1, Math.floor(limit));
-    pruneStored();
-    if (storedEvents.length <= safeLimit) return [...storedEvents];
-    return storedEvents.slice(storedEvents.length - safeLimit);
-  },
-
-  prune(): void {
-    pruneStored();
-  },
-
-  reset(): void {
-    storedEvents.splice(0, storedEvents.length);
-    pendingEvents.splice(0, pendingEvents.length);
-    exporters.clear();
-  },
-
-  async traceOperation<T>(input: {
-    queueName: string;
-    operation: QueueTraceOperation;
-    attributes?: Record<string, unknown>;
-    execute: () => Promise<T>;
-  }): Promise<T> {
-    if (isEnabled() === false || shouldSample() === false) {
-      return input.execute();
-    }
-
-    const startedAtMs = Date.now();
-    const traceId = generateUuid();
-    const spanId = generateUuid();
-
-    try {
-      const result = await input.execute();
-      finalize({
-        traceId,
-        spanId,
-        queueName: input.queueName,
-        operation: input.operation,
-        status: 'ok',
-        startedAt: new Date(startedAtMs).toISOString(),
-        endedAt: new Date().toISOString(),
-        durationMs: Math.max(0, Date.now() - startedAtMs),
-        attributes: input.attributes,
-      });
-
-      if (pendingEvents.length >= exportBatchSize()) {
-        void this.flush();
-      }
-
-      return result;
-    } catch (error) {
-      finalize({
-        traceId,
-        spanId,
-        queueName: input.queueName,
-        operation: input.operation,
-        status: 'error',
-        startedAt: new Date(startedAtMs).toISOString(),
-        endedAt: new Date().toISOString(),
-        durationMs: Math.max(0, Date.now() - startedAtMs),
-        error: normalizeError(error),
-        attributes: input.attributes,
-      });
-
-      if (pendingEvents.length >= exportBatchSize()) {
-        void this.flush();
-      }
-
-      throw error;
-    }
-  },
-});
-
-export default QueueTracing;

package/src/tools/queue/StalledJobMonitor.ts

@@ -1,28 +0,0 @@
-import { Logger } from '@config/logger';
-import { JobHeartbeatStore } from '@queue/JobHeartbeatStore';
-import { JobStateTracker } from '@queue/JobStateTracker';
-
-export const StalledJobMonitor = Object.freeze({
-  async scanOnce(): Promise<number> {
-    const expired = await JobHeartbeatStore.listExpired();
-
-    await Promise.all(
-      expired.map(async (row) => {
-        await JobStateTracker.stalled({
-          queueName: row.queueName,
-          jobId: row.jobId,
-          reason: 'Heartbeat expired',
-        });
-        await JobHeartbeatStore.remove(row.queueName, row.jobId);
-      })
-    );
-
-    if (expired.length > 0) {
-      Logger.warn('Stalled jobs detected from heartbeat store', { count: expired.length });
-    }
-
-    return expired.length;
-  },
-});
-
-export default StalledJobMonitor;

package/src/tools/queue/TimeoutManager.ts

@@ -1,102 +0,0 @@
-import { Env } from '@config/env';
-import { ErrorFactory } from '@exceptions/ZintrustError';
-
-export type TimeoutRetryConfig = {
-  timeoutMs: number;
-  maxRetries: number;
-  retryDelayMs: number;
-  operationName?: string;
-};
-
-const sleep = async (ms: number): Promise<void> => {
-  if (ms <= 0) return;
-  await new Promise<void>((resolve) => {
-    const timeoutRef = globalThis.setTimeout(() => {
-      clearTimeout(timeoutRef);
-      resolve();
-    }, ms);
-  });
-};
-
-const createTimeoutError = (operationName: string, timeoutMs: number): Error =>
-  ErrorFactory.createTryCatchError(`Operation timed out: ${operationName}`, {
-    code: 'QUEUE_TIMEOUT',
-    operationName,
-    timeoutMs,
-  });
-
-const isTimeoutError = (error: unknown): boolean => {
-  if (error === null || error === undefined || typeof error !== 'object') return false;
-  const code = (error as { code?: unknown }).code;
-  if (code === 'QUEUE_TIMEOUT') return true;
-  const details = (error as { details?: unknown }).details;
-  if (details !== null && details !== undefined && typeof details === 'object') {
-    return (details as { code?: unknown }).code === 'QUEUE_TIMEOUT';
-  }
-  return false;
-};
-
-export const TimeoutManager = Object.freeze({
-  getQueueJobTimeoutMs(): number {
-    const timeoutSeconds = Env.getInt('QUEUE_JOB_TIMEOUT', 60);
-    return Math.max(1000, timeoutSeconds * 1000);
-  },
-
-  async withTimeout<T>(
-    operation: () => Promise<T>,
-    timeoutMs: number,
-    operationName: string,
-    timeoutHandler?: () => Promise<T>
-  ): Promise<T> {
-    let timeoutId: ReturnType<typeof setTimeout> | undefined;
-
-    const timeoutPromise = new Promise<never>((_, reject) => {
-      timeoutId = globalThis.setTimeout(
-        () => {
-          reject(createTimeoutError(operationName, timeoutMs));
-        },
-        Math.max(1, timeoutMs)
-      );
-    });
-
-    try {
-      return await Promise.race([operation(), timeoutPromise]);
-    } catch (error) {
-      if (timeoutHandler !== undefined && isTimeoutError(error)) {
-        return await timeoutHandler();
-      }
-      throw error;
-    } finally {
-      if (timeoutId !== undefined) {
-        clearTimeout(timeoutId);
-      }
-    }
-  },
-
-  async withTimeoutRetry<T>(operation: () => Promise<T>, config: TimeoutRetryConfig): Promise<T> {
-    const retries = Math.max(0, Math.floor(config.maxRetries));
-    const attemptRun = async (attempt: number): Promise<T> => {
-      try {
-        return await this.withTimeout(
-          operation,
-          config.timeoutMs,
-          config.operationName ?? 'queue-operation'
-        );
-      } catch (error) {
-        const shouldRetry = isTimeoutError(error) && attempt < retries;
-        if (!shouldRetry) {
-          throw error;
-        }
-        const delay = Math.max(0, Math.floor(config.retryDelayMs * 2 ** attempt));
-        await sleep(delay);
-        return attemptRun(attempt + 1);
-      }
-    };
-
-    return attemptRun(0);
-  },
-
-  isTimeoutError,
-});
-
-export default TimeoutManager;