@zintrust/core 0.4.16 → 0.4.17

package/src/tools/queue/JobStateTrackerDbPersistence.ts

@@ -0,0 +1,299 @@
+import { Env } from '@config/env';
+import { Logger } from '@config/logger';
+import { ZintrustLang } from '@lang/lang';
+import type { IDatabase } from '@orm/Database';
+import { useDatabase } from '@orm/Database';
+import type {
+  JobStateTrackerPersistenceAdapter,
+  JobTrackingRecord,
+  JobTrackingTransition,
+} from '@queue/JobStateTracker';
+import { JobStateTracker } from '@queue/JobStateTracker';
+
+type JobStateTrackerDbOptions = {
+  connectionName?: string;
+  jobsTable?: string;
+  transitionsTable?: string;
+  persistPayload?: boolean;
+  persistResult?: boolean;
+};
+
+const toJson = (value: unknown): string | null => {
+  if (value === undefined) return null;
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return null;
+  }
+};
+
+const toNonNullJson = (value: unknown): string | null => {
+  const json = toJson(value);
+  return json ?? null;
+};
+
+const toSqlDateTime = (isoLike: string | undefined): string | null => {
+  if (typeof isoLike !== 'string' || isoLike.trim().length === 0) return null;
+  const parsed = new Date(isoLike);
+  if (Number.isNaN(parsed.getTime())) return null;
+  return parsed.toISOString().slice(0, 19).replace('T', ' ');
+};
+
+const getConnectionName = (options: JobStateTrackerDbOptions): string => {
+  if (options.connectionName !== undefined && options.connectionName.trim().length > 0) {
+    return options.connectionName.trim();
+  }
+  return Env.get('JOB_TRACKING_DB_CONNECTION', 'default');
+};
+
+const getJobsTable = (options: JobStateTrackerDbOptions): string => {
+  if (options.jobsTable !== undefined && options.jobsTable.trim().length > 0) {
+    return options.jobsTable.trim();
+  }
+  return Env.get('JOB_TRACKING_DB_TABLE', 'zintrust_jobs');
+};
+
+const getTransitionsTable = (options: JobStateTrackerDbOptions): string => {
+  if (options.transitionsTable !== undefined && options.transitionsTable.trim().length > 0) {
+    return options.transitionsTable.trim();
+  }
+  return Env.get('JOB_TRACKING_DB_TRANSITIONS_TABLE', 'zintrust_job_transitions');
+};
+
+const getDatabase = (connectionName: string): IDatabase | null => {
+  try {
+    return useDatabase(undefined, connectionName);
+  } catch (error) {
+    Logger.warn('Job tracking database connection is unavailable', {
+      connectionName,
+      error,
+    });
+    return null;
+  }
+};
+
+const serializeJobRecord = (
+  record: JobTrackingRecord,
+  options: JobStateTrackerDbOptions
+): Record<string, unknown> => {
+  return {
+    job_id: record.jobId,
+    queue_name: record.queueName,
+    status: record.status,
+    attempts: record.attempts,
+    max_attempts: record.maxAttempts ?? null,
+    payload_json: toNonNullJson(record.payload),
+    result_json: options.persistResult === false ? null : toJson(record.result),
+    last_error: record.lastError ?? null,
+    last_error_code: record.lastErrorCode ?? null,
+    retry_at: toSqlDateTime(record.retryAt),
+    timeout_at: toSqlDateTime(record.timeoutAt),
+    heartbeat_at: toSqlDateTime(record.heartbeatAt),
+    expected_completion_at: toSqlDateTime(record.expectedCompletionAt),
+    worker_name: record.workerName ?? null,
+    worker_instance_id: record.workerInstanceId ?? null,
+    worker_region: record.workerRegion ?? null,
+    worker_version: record.workerVersion ?? null,
+    recovered_at: toSqlDateTime(record.recoveredAt),
+    idempotency_key: record.idempotencyKey ?? null,
+    created_at: toSqlDateTime(record.createdAt),
+    started_at: toSqlDateTime(record.startedAt),
+    completed_at: toSqlDateTime(record.completedAt),
+    updated_at: toSqlDateTime(record.updatedAt),
+  };
+};
+
+const serializeJobRecordForInsert = (
+  record: JobTrackingRecord,
+  options: JobStateTrackerDbOptions
+): Record<string, unknown> => {
+  const payload = serializeJobRecord(record, options);
+
+  // Ensure payload_json is always non-null for inserts (some schemas require NOT NULL).
+  if (payload['payload_json'] === null) {
+    payload['payload_json'] = '{}';
+  }
+
+  return payload;
+};
+
+type UpdateEntry = {
+  key: string;
+  enabled: boolean;
+  value: unknown;
+};
+
+const applyUpdateEntries = (update: Record<string, unknown>, entries: UpdateEntry[]): void => {
+  entries.forEach((entry) => {
+    if (entry.enabled) update[entry.key] = entry.value;
+  });
+};
+
+const resolveResultEntry = (
+  record: JobTrackingRecord,
+  options: JobStateTrackerDbOptions,
+  base: Record<string, unknown>
+): UpdateEntry => {
+  if (options.persistResult === false) {
+    return { key: 'result_json', enabled: true, value: null };
+  }
+
+  const enabled = record.result !== undefined && base['result_json'] !== null;
+  return { key: 'result_json', enabled, value: base['result_json'] };
+};
+
+const serializeJobRecordForUpdate = (
+  record: JobTrackingRecord,
+  options: JobStateTrackerDbOptions
+): Record<string, unknown> => {
+  const base = serializeJobRecord(record, options);
+  const update: Record<string, unknown> = {
+    status: base['status'],
+    attempts: base['attempts'],
+    updated_at: base['updated_at'],
+  };
+
+  applyUpdateEntries(update, [
+    { key: 'max_attempts', enabled: record.maxAttempts !== undefined, value: base['max_attempts'] },
+    resolveResultEntry(record, options, base),
+    { key: 'last_error', enabled: record.lastError !== undefined, value: base['last_error'] },
+    {
+      key: 'last_error_code',
+      enabled: record.lastErrorCode !== undefined,
+      value: base['last_error_code'],
+    },
+    { key: 'retry_at', enabled: record.retryAt !== undefined, value: base['retry_at'] },
+    { key: 'timeout_at', enabled: record.timeoutAt !== undefined, value: base['timeout_at'] },
+    { key: 'heartbeat_at', enabled: record.heartbeatAt !== undefined, value: base['heartbeat_at'] },
+    {
+      key: 'expected_completion_at',
+      enabled: record.expectedCompletionAt !== undefined,
+      value: base['expected_completion_at'],
+    },
+    { key: 'worker_name', enabled: record.workerName !== undefined, value: base['worker_name'] },
+    {
+      key: 'worker_instance_id',
+      enabled: record.workerInstanceId !== undefined,
+      value: base['worker_instance_id'],
+    },
+    {
+      key: 'worker_region',
+      enabled: record.workerRegion !== undefined,
+      value: base['worker_region'],
+    },
+    {
+      key: 'worker_version',
+      enabled: record.workerVersion !== undefined,
+      value: base['worker_version'],
+    },
+    { key: 'recovered_at', enabled: record.recoveredAt !== undefined, value: base['recovered_at'] },
+    {
+      key: 'idempotency_key',
+      enabled: record.idempotencyKey !== undefined,
+      value: base['idempotency_key'],
+    },
+    { key: 'started_at', enabled: record.startedAt !== undefined, value: base['started_at'] },
+    { key: 'completed_at', enabled: record.completedAt !== undefined, value: base['completed_at'] },
+  ]);
+
+  return update;
+};
+
+const serializeTransition = (transition: JobTrackingTransition): Record<string, unknown> => {
+  return {
+    job_id: transition.jobId,
+    queue_name: transition.queueName,
+    from_status: transition.fromStatus,
+    to_status: transition.toStatus,
+    reason: transition.reason,
+    attempts: transition.attempts ?? null,
+    error: transition.error ?? null,
+    transitioned_at: toSqlDateTime(transition.timestamp),
+  };
+};
+
+export const createJobStateTrackerDbPersistence = (
+  options: JobStateTrackerDbOptions = {}
+): JobStateTrackerPersistenceAdapter => {
+  const connectionName = getConnectionName(options);
+  const jobsTable = getJobsTable(options);
+  const transitionsTable = getTransitionsTable(options);
+
+  const persistTransitions = (): boolean =>
+    Env.getBool('JOB_TRACKING_PERSIST_TRANSITIONS_ENABLED', false);
+
+  const shouldInsertNewRow = (record: JobTrackingRecord): boolean => {
+    // zintrust_jobs is an enqueue-fallback buffer: only jobs that failed to enqueue
+    // should be inserted into persistence. Once the job is in QUEUE_DRIVER, we only
+    // update the existing row (e.g., status=enqueued) but never create new rows.
+    return record.status === 'pending_recovery';
+  };
+
+  const upsertJob = async (record: JobTrackingRecord): Promise<void> => {
+    const db = getDatabase(connectionName);
+    if (db === null) return;
+
+    const existing = await db
+      .table(jobsTable)
+      .where('job_id', '=', record.jobId)
+      .where('queue_name', '=', record.queueName)
+      .first<{ status?: unknown }>();
+
+    if (existing) {
+      const existingStatus =
+        typeof existing.status === 'string' ? existing.status.trim().toLowerCase() : '';
+      if (existingStatus === 'enqueued') return;
+
+      const payload = serializeJobRecordForUpdate(record, options);
+      await db
+        .table(jobsTable)
+        .where('job_id', '=', record.jobId)
+        .where('queue_name', '=', record.queueName)
+        .update(payload);
+      return;
+    }
+
+    if (!shouldInsertNewRow(record)) return;
+    const payload = serializeJobRecordForInsert(record, options);
+    await db.table(jobsTable).insert(payload);
+  };
+
+  const insertTransition = async (transition: JobTrackingTransition): Promise<void> => {
+    if (persistTransitions() === false) return;
+    const db = getDatabase(connectionName);
+    if (db === null) return;
+
+    await db.table(transitionsTable).insert(serializeTransition(transition));
+  };
+
+  return Object.freeze({
+    upsertJob,
+    insertTransition,
+  });
+};
+
+export const autoRegisterJobStateTrackerPersistenceFromEnv = (): boolean => {
+  const trackingEnabled = Env.getBool('JOB_TRACKING_ENABLED', true);
+  const persistenceEnabled = Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false);
+
+  if (trackingEnabled === false || persistenceEnabled === false) {
+    JobStateTracker.clearPersistenceAdapter();
+    return false;
+  }
+
+  const driver = Env.get('JOB_TRACKING_PERSISTENCE_DRIVER', ZintrustLang.DATABASE)
+    .trim()
+    .toLowerCase();
+  if (driver !== ZintrustLang.DATABASE) {
+    Logger.warn('Unsupported job tracking persistence driver, skipping auto-registration', {
+      driver,
+    });
+    JobStateTracker.clearPersistenceAdapter();
+    return false;
+  }
+
+  JobStateTracker.registerPersistenceAdapter(createJobStateTrackerDbPersistence());
+  return true;
+};
+
+export default createJobStateTrackerDbPersistence;

package/src/tools/queue/LockProvider.ts

@@ -0,0 +1,350 @@
+/* eslint-disable @typescript-eslint/require-await */
+/**
+ * Lock Provider Implementation
+ * Provides distributed lock management for job deduplication
+ */
+
+import { ErrorFactory } from '@/exceptions/ZintrustError';
+import type {
+  Lock,
+  LockOptions,
+  LockProvider,
+  LockProviderConfig,
+  LockStatus,
+} from '@/types/Queue';
+import { Logger } from '@config/logger';
+import { createBaseDrivers } from '@config/queue';
+import { createRedisConnection } from '@config/workers';
+import { ZintrustLang } from '@lang/lang';
+import type { Redis } from 'ioredis';
+
+// Singleton Redis client for locks to avoid connection spam
+let redisClient: Redis | null = null;
+
+function getRedisClient(): Redis {
+  if (!redisClient) {
+    const redisConfig = createBaseDrivers().redis;
+    // Adapt queue config to worker config format if needed
+    redisClient = createRedisConnection({
+      host: redisConfig.host,
+      port: redisConfig.port,
+      password: redisConfig.password,
+      db: redisConfig.database,
+    });
+  }
+  return redisClient;
+}
+
+/**
+ * Close Redis connection and cleanup resources
+ */
+export async function closeLockProvider(): Promise<void> {
+  if (redisClient) {
+    try {
+      await redisClient.quit();
+    } catch (error) {
+      Logger.warn('Error closing Redis lock provider connection', error);
+    } finally {
+      redisClient = null;
+    }
+  }
+}
+
+const METRICS_SUFFIX = {
+  attempts: 'metrics:attempts',
+  acquired: 'metrics:acquired',
+  collisions: 'metrics:collisions',
+} as const;
+
+const recordLockAttempt = async (prefix: string, acquired: boolean): Promise<void> => {
+  const client = getRedisClient();
+  try {
+    const attemptsKey = `${prefix}${METRICS_SUFFIX.attempts}`;
+    const acquiredKey = `${prefix}${METRICS_SUFFIX.acquired}`;
+    const collisionsKey = `${prefix}${METRICS_SUFFIX.collisions}`;
+
+    const operations = [client.incr(attemptsKey)];
+    operations.push(acquired ? client.incr(acquiredKey) : client.incr(collisionsKey));
+    await Promise.allSettled(operations);
+  } catch (error) {
+    Logger.debug('Lock metrics update failed', { error });
+  }
+};
+
+function createAcquireMethod(prefix: string, defaultTtl: number) {
+  return async function acquire(key: string, options: LockOptions = {}): Promise<Lock> {
+    const lockKey = `${prefix}${key}`;
+    const ttl = options.ttl ?? defaultTtl;
+    const client = getRedisClient();
+
+    try {
+      // Use SET NX PX to acquire lock
+      const result = await client.set(lockKey, 'locked', 'PX', ttl, 'NX');
+      const acquired = result === 'OK';
+      const expires = new Date(Date.now() + ttl);
+
+      Logger.debug(`Lock acquisition attempt`, { key: lockKey, ttl, acquired });
+      void recordLockAttempt(prefix, acquired);
+
+      return {
+        key: lockKey,
+        ttl,
+        acquired,
+        expires,
+      };
+    } catch (error) {
+      Logger.error(`Failed to acquire lock`, { key: lockKey, error });
+      throw error;
+    }
+  };
+}
+
+function normalizeLockKey(prefix: string, key: string): string {
+  return key.startsWith(prefix) ? key : `${prefix}${key}`;
+}
+
+function createReleaseMethod(prefix: string) {
+  return async function release(lock: Lock): Promise<void> {
+    try {
+      const client = getRedisClient();
+      const lockKey = normalizeLockKey(prefix, lock.key);
+      await client.del(lockKey);
+      Logger.debug(`Lock release`, { key: lockKey });
+    } catch (error) {
+      Logger.error(`Failed to release lock`, { key: lock.key, error });
+      throw error;
+    }
+  };
+}
+
+function createExtendMethod(prefix: string) {
+  return async function extend(lock: Lock, ttl: number): Promise<boolean> {
+    try {
+      const client = getRedisClient();
+      const lockKey = normalizeLockKey(prefix, lock.key);
+      // Use PEXPIRE to extend
+      const result = await client.pexpire(lockKey, ttl);
+      const success = result === 1;
+
+      if (success) {
+        const newExpires = new Date(Date.now() + ttl);
+        Logger.debug(`Lock extension`, { key: lockKey, ttl, newExpires });
+        lock.ttl = ttl;
+        lock.expires = newExpires;
+      }
+
+      return success;
+    } catch (error) {
+      Logger.error(`Failed to extend lock`, { key: lock.key, error });
+      return false;
+    }
+  };
+}
+
+function createStatusMethod(prefix: string) {
+  return async function status(key: string): Promise<LockStatus> {
+    const lockKey = `${prefix}${key}`;
+
+    try {
+      const client = getRedisClient();
+      const ttl = await client.pttl(lockKey);
+      const exists = ttl > 0;
+
+      Logger.debug(`Lock status check`, { key: lockKey, exists, ttl });
+
+      return {
+        exists,
+        ttl: exists ? ttl : undefined,
+        expires: exists ? new Date(Date.now() + ttl) : undefined,
+      };
+    } catch (error) {
+      Logger.error(`Failed to check lock status`, { key: lockKey, error });
+      return { exists: false };
+    }
+  };
+}
+
+function createListMethod(prefix: string) {
+  return async function list(pattern: string = '*'): Promise<string[]> {
+    try {
+      const client = getRedisClient();
+      const searchPattern = `${prefix}${pattern}`;
+      const keys: string[] = [];
+      let cursor = '0';
+
+      do {
+        // SCAN to avoid blocking Redis in production environments
+        // eslint-disable-next-line no-await-in-loop
+        const [nextCursor, batch] = await client.scan(
+          cursor,
+          'MATCH',
+          searchPattern,
+          'COUNT',
+          '200'
+        );
+        cursor = nextCursor;
+        keys.push(...batch);
+      } while (cursor !== '0');
+
+      return keys.map((k) => k.replace(prefix, ''));
+    } catch (error) {
+      Logger.error(`Failed to list locks`, { pattern, error });
+      return [];
+    }
+  };
+}
+
+/**
+ * Redis-based Lock Provider Implementation
+ */
+export function createRedisLockProvider(config: LockProviderConfig): LockProvider {
+  const prefix = config.prefix ?? ZintrustLang.ZINTRUST_LOCKS_PREFIX;
+  const defaultTtl = config.defaultTtl ?? ZintrustLang.ZINTRUST_LOCKS_TTL;
+
+  return {
+    acquire: createAcquireMethod(prefix, defaultTtl),
+    release: createReleaseMethod(prefix),
+    extend: createExtendMethod(prefix),
+    status: createStatusMethod(prefix),
+    list: createListMethod(prefix),
+  };
+}
+
+/**
+ * Memory-based Lock Provider (for testing/sync driver)
+ */
+function globToRegExp(pattern: string): RegExp {
+  // Escape all regex metacharacters, then turn '*' into '.*' as a wildcard
+  const escaped = pattern.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`);
+  const regexSource = escaped.replaceAll(String.raw`\*`, '.*');
+  return new RegExp(`^${regexSource}$`);
+}
+
+export function createMemoryLockProvider(config: LockProviderConfig): LockProvider {
+  const locks = new Map<string, Lock>();
+  const prefix = config.prefix ?? ZintrustLang.MEMORY_LOCKS;
+  const defaultTtl = config.defaultTtl ?? ZintrustLang.ZINTRUST_LOCKS_TTL;
+
+  return {
+    async acquire(key: string, options: LockOptions = {}): Promise<Lock> {
+      const lockKey = `${prefix}${key}`;
+      const ttl = options.ttl ?? defaultTtl;
+      const expires = new Date(Date.now() + ttl);
+
+      if (locks.has(lockKey)) {
+        const existingLock = locks.get(lockKey);
+        if (existingLock !== undefined && existingLock.expires > new Date()) {
+          return {
+            key: lockKey,
+            ttl,
+            acquired: false,
+            expires,
+          };
+        }
+        // Lock expired, remove it
+        locks.delete(lockKey);
+      }
+
+      const lock: Lock = {
+        key: lockKey,
+        ttl,
+        acquired: true,
+        expires,
+      };
+
+      locks.set(lockKey, lock);
+      Logger.debug(`Memory lock acquired`, { key: lockKey, ttl });
+
+      return lock;
+    },
+
+    async release(lock: Lock): Promise<void> {
+      const lockKey = normalizeLockKey(prefix, lock.key);
+      locks.delete(lockKey);
+      Logger.debug(`Memory lock released`, { key: lockKey });
+    },
+
+    async extend(lock: Lock, ttl: number): Promise<boolean> {
+      const lockKey = normalizeLockKey(prefix, lock.key);
+      const existingLock = locks.get(lockKey);
+      if (!existingLock) {
+        return false;
+      }
+
+      existingLock.ttl = ttl;
+      existingLock.expires = new Date(Date.now() + ttl);
+      Logger.debug(`Memory lock extended`, { key: lockKey, ttl });
+
+      return true;
+    },
+
+    async status(key: string): Promise<LockStatus> {
+      const lockKey = `${prefix}${key}`;
+      const lock = locks.get(lockKey);
+
+      if (!lock) {
+        return { exists: false };
+      }
+
+      if (lock.expires <= new Date()) {
+        locks.delete(lockKey);
+        return { exists: false };
+      }
+
+      return {
+        exists: true,
+        ttl: lock.ttl,
+        expires: lock.expires,
+      };
+    },
+
+    async list(pattern: string = '*'): Promise<string[]> {
+      // Simple glob-style match for memory provider ('*' as wildcard)
+      const regex = globToRegExp(pattern);
+      const keys: string[] = [];
+
+      for (const key of locks.keys()) {
+        const strippedKey = key.replace(prefix, '');
+        if (regex.test(strippedKey)) {
+          keys.push(strippedKey);
+        }
+      }
+      return keys;
+    },
+  };
+}
+
+/**
+ * Lock Provider Registry
+ */
+const lockProviders = new Map<string, LockProvider>();
+
+export function registerLockProvider(name: string, provider: LockProvider): void {
+  lockProviders.set(name, provider);
+  Logger.info(`Lock provider registered`, { name });
+}
+export function getLockProvider(name: string): LockProvider | undefined {
+  return lockProviders.get(name);
+}
+
+/**
+ * Clear all registered lock providers (for testing purposes)
+ * @internal
+ */
+export function clearLockProviders(): void {
+  lockProviders.clear();
+}
+
+/**
+ * Create lock provider based on configuration
+ */
+export function createLockProvider(config: LockProviderConfig): LockProvider {
+  switch (config.type) {
+    case ZintrustLang.REDIS:
+      return createRedisLockProvider(config);
+    case ZintrustLang.MEMORY:
+      return createMemoryLockProvider(config);
+    default:
+      throw ErrorFactory.createConfigError(`Unsupported lock provider type: ${config.type}`);
+  }
+}