@zintrust/core 2.4.1 → 2.4.2

package/src/tools/queue/DeduplicationBuilder.js

@@ -0,0 +1,77 @@
+/**
+ * Deduplication Builder - Plain function implementation for BullMQ job deduplication
+ * Follows ZinTrust's preference for functions over classes
+ */
+import { createValidationError } from '../../exceptions/ZintrustError.js';
+/**
+ * Creates a deduplication builder for configuring job deduplication options
+ * @returns {DeduplicationBuilder} Builder instance with fluent interface
+ */
+export function createDeduplicationBuilder() {
+    const state = {};
+    return {
+        /**
+         * Set the unique identifier for deduplication
+         * @param id - Unique identifier string
+         */
+        id(id) {
+            state.id = id;
+            return this;
+        },
+        /**
+         * Set time-to-live for deduplication lock in milliseconds
+         * @param ms - TTL in milliseconds
+         */
+        expireAfter(ms) {
+            state.ttl = ms;
+            return this;
+        },
+        /**
+         * Prevent automatic lock release - requires manual release
+         */
+        dontRelease() {
+            state.dontRelease = true;
+            return this;
+        },
+        /**
+         * Replace existing job with same ID instead of ignoring
+         */
+        replace() {
+            state.replace = true;
+            return this;
+        },
+        /**
+         * Set release strategy for the deduplication lock
+         * @param strategy - Release strategy (delay, 'success', or condition object)
+         */
+        releaseAfter(strategy) {
+            state.releaseAfter = strategy;
+            return this;
+        },
+        /**
+         * Build the final deduplication options
+         * @returns {DeduplicationOptions} Configured deduplication options
+         */
+        build() {
+            if (state.id === null || state.id === undefined) {
+                throw createValidationError('Deduplication ID is required. Call .id() before .build()');
+            }
+            const options = {
+                id: state.id,
+            };
+            if (state.ttl !== undefined) {
+                options.ttl = state.ttl;
+            }
+            if (state.dontRelease === true) {
+                options.dontRelease = true;
+            }
+            if (state.replace === true) {
+                options.replace = true;
+            }
+            if (state.releaseAfter !== undefined) {
+                options.releaseAfter = state.releaseAfter;
+            }
+            return options;
+        },
+    };
+}

package/src/tools/queue/DeduplicationKey.d.ts

@@ -0,0 +1,2 @@
+export declare const resolveDeduplicationLockKey: (queueName: string, deduplicationId: string) => string;
+//# sourceMappingURL=DeduplicationKey.d.ts.map

package/src/tools/queue/DeduplicationKey.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeduplicationKey.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/DeduplicationKey.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,2BAA2B,GAAI,WAAW,MAAM,EAAE,iBAAiB,MAAM,KAAG,MASxF,CAAC"}

package/src/tools/queue/DeduplicationKey.js

@@ -0,0 +1,8 @@
+export const resolveDeduplicationLockKey = (queueName, deduplicationId) => {
+    const normalizedQueueName = String(queueName ?? '').trim();
+    const normalizedDeduplicationId = String(deduplicationId ?? '').trim();
+    if (normalizedQueueName.length === 0) {
+        return normalizedDeduplicationId;
+    }
+    return `queue:${normalizedQueueName}:${normalizedDeduplicationId}`;
+};

package/src/tools/queue/IdempotencyManager.d.ts

@@ -0,0 +1,6 @@
+import { type BullMQPayload } from './Queue';
+export declare const IdempotencyManager: Readonly<{
+    enqueueIdempotent(queueName: string, payload: BullMQPayload, idempotencyKey: string, ttlMs?: number): Promise<string>;
+}>;
+export default IdempotencyManager;
+//# sourceMappingURL=IdempotencyManager.d.ts.map

package/src/tools/queue/IdempotencyManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IdempotencyManager.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/IdempotencyManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAS,KAAK,aAAa,EAAE,MAAM,cAAc,CAAC;AAOzD,eAAO,MAAM,kBAAkB;iCAEhB,MAAM,WACR,aAAa,kBACN,MAAM,mBAErB,OAAO,CAAC,MAAM,CAAC;EA8BlB,CAAC;AAEH,eAAe,kBAAkB,CAAC"}

package/src/tools/queue/IdempotencyManager.js

@@ -0,0 +1,36 @@
+import { Env } from '../../config/env.js';
+import { useDatabase } from '../../orm/Database.js';
+import { Queue } from './Queue.js';
+const getDb = () => useDatabase(undefined, Env.get('JOB_TRACKING_DB_CONNECTION', 'default'));
+const toSqlDateTime = (value) => value.toISOString().slice(0, 19).replace('T', ' ');
+export const IdempotencyManager = Object.freeze({
+    async enqueueIdempotent(queueName, payload, idempotencyKey, ttlMs = Env.getInt('IDEMPOTENCY_DEFAULT_TTL_MS', 86400000)) {
+        const key = idempotencyKey.trim();
+        if (key === '') {
+            return Queue.enqueue(queueName, payload);
+        }
+        if (Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false)) {
+            const db = getDb();
+            const cutoff = new Date(Date.now() - Math.max(1000, ttlMs));
+            const row = await db
+                .table(Env.get('JOB_TRACKING_DB_TABLE', 'zintrust_jobs'))
+                .select('job_id', 'status')
+                .where('idempotency_key', '=', key)
+                .where('created_at', '>=', toSqlDateTime(cutoff))
+                .whereIn('status', ['pending', 'active', 'completed'])
+                .first();
+            if (row?.job_id !== undefined)
+                return row.job_id;
+        }
+        return Queue.enqueue(queueName, {
+            ...payload,
+            uniqueId: key,
+            deduplication: {
+                id: key,
+                ttl: ttlMs,
+            },
+            __zintrustIdempotencyKey: key,
+        });
+    },
+});
+export default IdempotencyManager;

package/src/tools/queue/JobHeartbeatStore.d.ts

@@ -0,0 +1,16 @@
+export declare const JobHeartbeatStore: Readonly<{
+    getIntervalMs(): number;
+    heartbeat(input: {
+        queueName: string;
+        jobId: string;
+        workerInstanceId?: string;
+        intervalMs?: number;
+    }): Promise<void>;
+    remove(queueName: string, jobId: string): Promise<void>;
+    listExpired(limit?: number): Promise<Array<{
+        queueName: string;
+        jobId: string;
+    }>>;
+}>;
+export default JobHeartbeatStore;
+//# sourceMappingURL=JobHeartbeatStore.d.ts.map

package/src/tools/queue/JobHeartbeatStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobHeartbeatStore.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobHeartbeatStore.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,iBAAiB;qBACX,MAAM;qBAIA;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,IAAI,CAAC;sBAmCO,MAAM,SAAS,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;iCAU7B,OAAO,CAAC,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;EAgBpF,CAAC;AAEH,eAAe,iBAAiB,CAAC"}

package/src/tools/queue/JobHeartbeatStore.js

@@ -0,0 +1,67 @@
+import { Env } from '../../config/env.js';
+import { useDatabase } from '../../orm/Database.js';
+const HEARTBEAT_TABLE = 'zintrust_job_heartbeats';
+const getConnection = () => useDatabase(undefined, Env.get('JOB_TRACKING_DB_CONNECTION', 'default'));
+const now = () => new Date();
+const toSqlDateTime = (value) => value.toISOString().slice(0, 19).replace('T', ' ');
+export const JobHeartbeatStore = Object.freeze({
+    getIntervalMs() {
+        return Math.max(1000, Env.getInt('JOB_HEARTBEAT_INTERVAL_MS', 10000));
+    },
+    async heartbeat(input) {
+        if (!Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false))
+            return;
+        const db = getConnection();
+        const intervalMs = input.intervalMs ?? this.getIntervalMs();
+        const current = now();
+        const expected = new Date(current.getTime() + intervalMs * 2);
+        const payload = {
+            queue_name: input.queueName,
+            job_id: input.jobId,
+            worker_instance_id: input.workerInstanceId ?? null,
+            heartbeat_interval_ms: intervalMs,
+            last_heartbeat_at: toSqlDateTime(current),
+            expected_next_heartbeat_at: toSqlDateTime(expected),
+            updated_at: toSqlDateTime(current),
+        };
+        const existing = await db
+            .table(HEARTBEAT_TABLE)
+            .where('queue_name', '=', input.queueName)
+            .where('job_id', '=', input.jobId)
+            .first();
+        if (existing !== null && existing !== undefined) {
+            await db.table(HEARTBEAT_TABLE).where('id', '=', existing.id).update(payload);
+            return;
+        }
+        await db.table(HEARTBEAT_TABLE).insert({
+            ...payload,
+            created_at: toSqlDateTime(current),
+        });
+    },
+    async remove(queueName, jobId) {
+        if (!Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false))
+            return;
+        const db = getConnection();
+        await db
+            .table(HEARTBEAT_TABLE)
+            .where('queue_name', '=', queueName)
+            .where('job_id', '=', jobId)
+            .delete();
+    },
+    async listExpired(limit = 500) {
+        if (!Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false))
+            return [];
+        const db = getConnection();
+        const rows = await db
+            .table(HEARTBEAT_TABLE)
+            .select('queue_name', 'job_id')
+            .where('expected_next_heartbeat_at', '<=', toSqlDateTime(now()))
+            .limit(Math.max(1, limit))
+            .get();
+        return (rows ?? []).map((row) => ({
+            queueName: row.queue_name,
+            jobId: row.job_id,
+        }));
+    },
+});
+export default JobHeartbeatStore;

package/src/tools/queue/JobReconciliationRunner.d.ts

@@ -0,0 +1,16 @@
+export declare const JobReconciliationRunner: Readonly<{
+    reconcileInMemory(): Promise<{
+        stalled: number;
+        timeout: number;
+    }>;
+    reconcileFromPersistence(limit?: number): Promise<number>;
+    runOnce(): Promise<{
+        inMemory: {
+            stalled: number;
+            timeout: number;
+        };
+        persisted: number;
+    }>;
+}>;
+export default JobReconciliationRunner;
+//# sourceMappingURL=JobReconciliationRunner.d.ts.map

package/src/tools/queue/JobReconciliationRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobReconciliationRunner.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobReconciliationRunner.ts"],"names":[],"mappings":"AAiBA,eAAO,MAAM,uBAAuB;yBACP,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;8CA6B3B,OAAO,CAAC,MAAM,CAAC;eAqB3C,OAAO,CAAC;QAAE,QAAQ,EAAE;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;EAc/F,CAAC;AAEH,eAAe,uBAAuB,CAAC"}

package/src/tools/queue/JobReconciliationRunner.js

@@ -0,0 +1,56 @@
+import { Env } from '../../config/env.js';
+import { Logger } from '../../config/logger.js';
+import { useDatabase } from '../../orm/Database.js';
+import { JobStateTracker } from './JobStateTracker.js';
+const getDb = () => useDatabase(undefined, Env.get('JOB_TRACKING_DB_CONNECTION', 'default'));
+const toSqlDateTime = (value) => value.toISOString().slice(0, 19).replace('T', ' ');
+export const JobReconciliationRunner = Object.freeze({
+    async reconcileInMemory() {
+        const timeoutMs = Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000);
+        const heartbeatGraceMs = Math.max(1000, Env.getInt('JOB_HEARTBEAT_GRACE_MS', 20000));
+        const staleHeartbeats = JobStateTracker.listHeartbeatExpired(heartbeatGraceMs);
+        await Promise.all(staleHeartbeats.map(async (row) => JobStateTracker.stalled({
+            queueName: row.queueName,
+            jobId: row.jobId,
+            reason: 'Missing heartbeat during reconciliation',
+        })));
+        const timedOut = JobStateTracker.listActiveOlderThan(timeoutMs);
+        await Promise.all(timedOut.map(async (row) => JobStateTracker.timedOut({
+            queueName: row.queueName,
+            jobId: row.jobId,
+            reason: 'Exceeded queue timeout during reconciliation',
+        })));
+        return { stalled: staleHeartbeats.length, timeout: timedOut.length };
+    },
+    async reconcileFromPersistence(limit = 500) {
+        if (!Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false))
+            return 0;
+        const db = getDb();
+        const staleAfterMs = Math.max(1000, Env.getInt('JOB_RECONCILIATION_STALE_MS', 120000));
+        const cutoff = new Date(Date.now() - staleAfterMs);
+        // Persistence is an enqueue-fallback buffer (jobs that failed to enqueue).
+        // Do not reconcile stale 'pending'/'active' from DB: QUEUE_DRIVER is the source of truth.
+        // pending_recovery rows are handled by JobRecoveryDaemon, which performs the enqueue attempt.
+        const candidates = await db
+            .table(Env.get('JOB_TRACKING_DB_TABLE', 'zintrust_jobs'))
+            .select('queue_name', 'job_id')
+            .whereIn('status', ['pending_recovery'])
+            .where('updated_at', '<=', toSqlDateTime(cutoff))
+            .limit(Math.max(1, limit))
+            .get();
+        return candidates.length;
+    },
+    async runOnce() {
+        const inMemory = await this.reconcileInMemory();
+        const persisted = await this.reconcileFromPersistence();
+        if (inMemory.stalled > 0 || inMemory.timeout > 0 || persisted > 0) {
+            Logger.warn('Queue reconciliation updated jobs', {
+                stalled: inMemory.stalled,
+                timeout: inMemory.timeout,
+                persisted,
+            });
+        }
+        return { inMemory, persisted };
+    },
+});
+export default JobReconciliationRunner;

package/src/tools/queue/JobRecoveryDaemon.d.ts

@@ -0,0 +1,27 @@
+import { type JobTrackingRecord } from './JobStateTracker';
+export type DlqReplayReasonCode = 'bug_fixed' | 'transient_dependency' | 'operator_override';
+type DlqReplayRequest = {
+    reasonCode: DlqReplayReasonCode;
+    replayedBy: string;
+    queueName?: string;
+    limit?: number;
+    maxPerSecond?: number;
+    minAgeMs?: number;
+};
+type DlqReplayResult = {
+    scanned: number;
+    replayed: number;
+    skipped: number;
+};
+export declare const JobRecoveryDaemon: Readonly<{
+    recoverOne(record: JobTrackingRecord): Promise<"requeued" | "dead_letter" | "manual_review">;
+    runOnce(): Promise<{
+        scanned: number;
+        requeued: number;
+        deadLetter: number;
+        manualReview: number;
+    }>;
+    replayDeadLetter(input: DlqReplayRequest): Promise<DlqReplayResult>;
+}>;
+export default JobRecoveryDaemon;
+//# sourceMappingURL=JobRecoveryDaemon.d.ts.map

package/src/tools/queue/JobRecoveryDaemon.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobRecoveryDaemon.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobRecoveryDaemon.ts"],"names":[],"mappings":"AAKA,OAAO,EAAmB,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAejF,MAAM,MAAM,mBAAmB,GAAG,WAAW,GAAG,sBAAsB,GAAG,mBAAmB,CAAC;AAE7F,KAAK,gBAAgB,GAAG;IACtB,UAAU,EAAE,mBAAmB,CAAC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAuFF,eAAO,MAAM,iBAAiB;uBAElB,iBAAiB,GACxB,OAAO,CAAC,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;eA2EvC,OAAO,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;4BAkD4B,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;EA2DzE,CAAC;AAiFH,eAAe,iBAAiB,CAAC"}

package/src/tools/queue/JobRecoveryDaemon.js

@@ -0,0 +1,303 @@
+import { Env } from '../../config/env.js';
+import { Logger } from '../../config/logger.js';
+import { ErrorFactory } from '../../exceptions/ZintrustError.js';
+import { useDatabase } from '../../orm/Database.js';
+import { JobStateTracker } from './JobStateTracker.js';
+import { Queue } from './Queue.js';
+const parsePayload = (payload) => {
+    if (payload !== null && payload !== undefined && typeof payload === 'object') {
+        return payload;
+    }
+    return { payload };
+};
+const shouldManualReview = (record) => {
+    const error = String(record.lastError ?? '').toLowerCase();
+    return error.includes('fatal') || error.includes('corrupt');
+};
+const replayReasonCodes = new Set([
+    'bug_fixed',
+    'transient_dependency',
+    'operator_override',
+]);
+const resolveLimit = (requested) => {
+    const maxBatch = Math.max(1, Env.getInt('DLQ_REPLAY_MAX_BATCH_SIZE', 25));
+    if (typeof requested !== 'number' || Number.isFinite(requested) === false || requested <= 0) {
+        return maxBatch;
+    }
+    return Math.min(maxBatch, Math.floor(requested));
+};
+const resolveRate = (requested) => {
+    const maxQps = Math.max(1, Env.getInt('DLQ_REPLAY_MAX_QPS', 5));
+    if (typeof requested !== 'number' || Number.isFinite(requested) === false || requested <= 0) {
+        return maxQps;
+    }
+    return Math.min(maxQps, Math.floor(requested));
+};
+const resolveMinAgeMs = (requested) => {
+    const minAllowed = Math.max(0, Env.getInt('DLQ_REPLAY_MIN_AGE_MS', 60000));
+    if (typeof requested !== 'number' || Number.isFinite(requested) === false || requested < 0) {
+        return minAllowed;
+    }
+    return Math.max(minAllowed, Math.floor(requested));
+};
+const validateReplayReasonCode = (reasonCode) => {
+    if (replayReasonCodes.has(reasonCode))
+        return;
+    throw ErrorFactory.createConfigError(`Unsupported DLQ replay reason code: ${reasonCode}`);
+};
+const validateReplayActor = (replayedBy) => {
+    if (replayedBy.trim().length === 0) {
+        throw ErrorFactory.createConfigError('DLQ replay actor is required');
+    }
+    const allowedActorsRaw = Env.get('DLQ_REPLAY_ALLOWED_ACTORS', '').trim();
+    if (allowedActorsRaw.length === 0)
+        return;
+    const allowedActors = allowedActorsRaw
+        .split(',')
+        .map((value) => value.trim())
+        .filter((value) => value.length > 0);
+    if (allowedActors.length === 0)
+        return;
+    if (allowedActors.includes(replayedBy.trim()))
+        return;
+    throw ErrorFactory.createConfigError(`DLQ replay actor is not allowed: ${replayedBy}`);
+};
+const isReplayEligible = (record, minAgeMs) => {
+    if (record.status !== 'dead_letter')
+        return false;
+    const updated = new Date(record.updatedAt).getTime();
+    if (Number.isNaN(updated))
+        return false;
+    return updated <= Date.now() - minAgeMs;
+};
+const toReplayPayload = (record, reasonCode, replayedBy) => {
+    const parsed = parsePayload(record.payload);
+    const lineage = {
+        originalJobId: record.jobId,
+        replayReasonCode: reasonCode,
+        replayedBy,
+        replayedAt: new Date().toISOString(),
+    };
+    const existingLineage = parsed['__dlqReplayLineage'];
+    const lineageArray = Array.isArray(existingLineage)
+        ? [...existingLineage.map((entry) => ({ ...entry })), lineage]
+        : [lineage];
+    return {
+        ...parsed,
+        __dlqReplayLineage: lineageArray,
+    };
+};
+export const JobRecoveryDaemon = Object.freeze({
+    async recoverOne(record) {
+        const maxAttempts = typeof record.maxAttempts === 'number' ? record.maxAttempts : 3;
+        if (record.attempts >= maxAttempts) {
+            await JobStateTracker.setTerminalStatus({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                status: 'dead_letter',
+                reason: 'Max retry attempts reached during recovery',
+            });
+            return 'dead_letter';
+        }
+        if (shouldManualReview(record)) {
+            await JobStateTracker.setTerminalStatus({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                status: 'manual_review',
+                reason: 'Fatal recovery signature detected',
+            });
+            return 'manual_review';
+        }
+        const payload = parsePayload(record.payload);
+        // 30s, 1m, 3m backoff strategy
+        const getBackoffMs = (attempt) => {
+            if (attempt === 0)
+                return 30000;
+            if (attempt === 1)
+                return 60000;
+            return 180000;
+        };
+        const backoffMs = getBackoffMs(record.attempts);
+        try {
+            await Queue.enqueue(record.queueName, {
+                ...payload,
+                jobId: record.jobId,
+                attempts: maxAttempts,
+                _currentAttempts: record.attempts + 1,
+                timestamp: Date.now() + backoffMs,
+            });
+            // Once the job is in QUEUE_DRIVER (or already exists), we never re-enqueue from DB/recovery again.
+            await JobStateTracker.handedOffToQueue({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                reason: 'Enqueue-fallback job handed off to QUEUE_DRIVER',
+            });
+            return 'requeued';
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            const message = errorMessage.toLowerCase();
+            if (message.includes('jobid') && message.includes('already exists')) {
+                await JobStateTracker.handedOffToQueue({
+                    queueName: record.queueName,
+                    jobId: record.jobId,
+                    reason: 'Job already exists in queue driver',
+                });
+                return 'requeued';
+            }
+            await JobStateTracker.pendingRecovery({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                reason: 'Enqueue-fallback retry failed during recovery daemon run',
+                attempts: record.attempts + 1,
+                maxAttempts,
+                retryAt: new Date(Date.now() + backoffMs).toISOString(),
+                error,
+            });
+            throw error;
+        }
+    },
+    async runOnce() {
+        const minAgeMs = Math.max(0, Env.getInt('JOB_RECOVERY_MIN_AGE_MS', 5000));
+        const candidates = JobStateTracker.listRecoverable(minAgeMs);
+        const persisted = await listRecoverableFromPersistence(minAgeMs);
+        // De-dupe jobs that exist in both in-memory tracker and persistence.
+        const seen = new Set();
+        const allCandidates = [...candidates, ...persisted].filter((row) => {
+            const key = `${row.queueName}:${row.jobId}`;
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+        let requeued = 0;
+        let deadLetter = 0;
+        let manualReview = 0;
+        const concurrency = 10;
+        const batches = [];
+        for (let offset = 0; offset < allCandidates.length; offset += concurrency) {
+            const slice = allCandidates.slice(offset, offset + concurrency);
+            batches.push(Promise.all(slice.map(async (candidate) => this.recoverOne(candidate))));
+        }
+        const batchResults = await Promise.all(batches);
+        batchResults.flat().forEach((result) => {
+            if (result === 'requeued')
+                requeued += 1;
+            if (result === 'dead_letter')
+                deadLetter += 1;
+            if (result === 'manual_review')
+                manualReview += 1;
+        });
+        if (allCandidates.length > 0) {
+            Logger.info('Queue recovery daemon completed scan', {
+                scanned: allCandidates.length,
+                requeued,
+                deadLetter,
+                manualReview,
+            });
+        }
+        return {
+            scanned: allCandidates.length,
+            requeued,
+            deadLetter,
+            manualReview,
+        };
+    },
+    async replayDeadLetter(input) {
+        validateReplayReasonCode(input.reasonCode);
+        validateReplayActor(input.replayedBy);
+        const limit = resolveLimit(input.limit);
+        const maxPerSecond = resolveRate(input.maxPerSecond);
+        const minAgeMs = resolveMinAgeMs(input.minAgeMs);
+        const replayBudget = Math.max(1, maxPerSecond);
+        const candidates = JobStateTracker.list({
+            queueName: input.queueName,
+            status: 'dead_letter',
+            limit,
+        })
+            .filter((record) => isReplayEligible(record, minAgeMs))
+            .slice(0, replayBudget);
+        let replayed = 0;
+        let skipped = 0;
+        await candidates.reduce(async (chain, candidate) => {
+            await chain;
+            if (candidate.queueName.trim().length === 0) {
+                skipped += 1;
+                return;
+            }
+            const replayPayload = toReplayPayload(candidate, input.reasonCode, input.replayedBy.trim());
+            const replayJobId = await Queue.enqueue(candidate.queueName, replayPayload);
+            await JobStateTracker.setTerminalStatus({
+                queueName: candidate.queueName,
+                jobId: candidate.jobId,
+                status: 'dead_letter',
+                reason: `DLQ replayed (${input.reasonCode}) by ${input.replayedBy.trim()} as ${replayJobId}`,
+            });
+            replayed += 1;
+        }, Promise.resolve());
+        Logger.info('DLQ replay governance batch completed', {
+            scanned: candidates.length,
+            replayed,
+            skipped,
+            reasonCode: input.reasonCode,
+            replayedBy: input.replayedBy.trim(),
+            queueName: input.queueName,
+            limit,
+            maxPerSecond,
+            minAgeMs,
+        });
+        return {
+            scanned: candidates.length,
+            replayed,
+            skipped,
+        };
+    },
+});
+const toSqlDateTime = (value) => value.toISOString().slice(0, 19).replace('T', ' ');
+const getPersistenceDb = () => useDatabase(undefined, Env.get('JOB_TRACKING_DB_CONNECTION', 'default'));
+const listRecoverableFromPersistence = async (minAgeMs) => {
+    if (!Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false))
+        return [];
+    const db = getPersistenceDb();
+    const cutoff = new Date(Date.now() - Math.max(0, Math.floor(minAgeMs)));
+    const rows = await db
+        .table(Env.get('JOB_TRACKING_DB_TABLE', 'zintrust_jobs'))
+        .select('queue_name', 'job_id', 'attempts', 'max_attempts', 'payload_json', 'retry_at', 'updated_at')
+        .where('status', '=', 'pending_recovery')
+        .where('updated_at', '<=', toSqlDateTime(cutoff))
+        .limit(Math.max(1, Env.getInt('JOB_RECOVERY_DB_SCAN_LIMIT', 100)))
+        .get();
+    return (rows ?? []).map((row) => {
+        let payload = {};
+        try {
+            payload = JSON.parse(String(row.payload_json ?? '{}'));
+        }
+        catch {
+            payload = {};
+        }
+        const attempts = typeof row.attempts === 'number' && Number.isFinite(row.attempts)
+            ? Math.max(0, Math.floor(row.attempts))
+            : 0;
+        const maxAttempts = typeof row.max_attempts === 'number' && Number.isFinite(row.max_attempts)
+            ? Math.max(1, Math.floor(row.max_attempts))
+            : undefined;
+        const updatedAtRaw = typeof row.updated_at === 'string' ? row.updated_at : undefined;
+        const updatedAt = updatedAtRaw !== undefined && updatedAtRaw.trim().length > 0
+            ? updatedAtRaw
+            : new Date(0).toISOString();
+        const retryAtRaw = typeof row.retry_at === 'string' ? row.retry_at : undefined;
+        const retryAt = retryAtRaw !== undefined && retryAtRaw.trim().length > 0 ? retryAtRaw : undefined;
+        return {
+            queueName: String(row.queue_name),
+            jobId: String(row.job_id),
+            status: 'pending_recovery',
+            attempts,
+            maxAttempts,
+            createdAt: new Date().toISOString(),
+            updatedAt,
+            payload,
+            retryAt,
+        };
+    });
+};
+export default JobRecoveryDaemon;