@zintrust/core 0.1.41 → 0.1.42

package/src/tools/queue/JobRecoveryDaemon.js

@@ -1,6 +1,7 @@
 import { Env } from '../../config/env.js';
 import { Logger } from '../../config/logger.js';
 import { ErrorFactory } from '../../exceptions/ZintrustError.js';
+import { useDatabase } from '../../orm/Database.js';
 import { JobStateTracker } from './JobStateTracker.js';
 import { Queue } from './Queue.js';
 const parsePayload = (payload) => {
@@ -108,28 +109,78 @@ export const JobRecoveryDaemon = Object.freeze({
             return 'manual_review';
         }
         const payload = parsePayload(record.payload);
-        const backoffMs = Math.min(300000, Math.max(1000, 1000 * 2 ** Math.max(0, record.attempts)));
-        await Queue.enqueue(record.queueName, {
-            ...payload,
-            attempts: Math.max(0, record.attempts),
-            timestamp: Date.now() + backoffMs,
-        });
-        await JobStateTracker.markedRecovered({
-            queueName: record.queueName,
-            jobId: record.jobId,
-            reason: 'Recovered and re-queued by recovery daemon',
-            retryAt: new Date(Date.now() + backoffMs).toISOString(),
-        });
-        return 'requeued';
+        // 30s, 1m, 3m backoff strategy
+        const getBackoffMs = (attempt) => {
+            if (attempt === 0)
+                return 30000;
+            if (attempt === 1)
+                return 60000;
+            return 180000;
+        };
+        const backoffMs = getBackoffMs(record.attempts);
+        try {
+            await Queue.enqueue(record.queueName, {
+                ...payload,
+                uniqueId: record.jobId, // Preserves job ID (prevents duplication)
+                attempts: maxAttempts,
+                _currentAttempts: record.attempts + 1,
+                timestamp: Date.now() + backoffMs,
+            });
+            // Once the job is in QUEUE_DRIVER (or already exists), we never re-enqueue from DB/recovery again.
+            await JobStateTracker.handedOffToQueue({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                reason: 'Enqueue-fallback job handed off to QUEUE_DRIVER',
+            });
+            return 'requeued';
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            const message = errorMessage.toLowerCase();
+            if (message.includes('jobid') && message.includes('already exists')) {
+                await JobStateTracker.handedOffToQueue({
+                    queueName: record.queueName,
+                    jobId: record.jobId,
+                    reason: 'Job already exists in queue driver',
+                });
+                return 'requeued';
+            }
+            await JobStateTracker.pendingRecovery({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                reason: 'Enqueue-fallback retry failed during recovery daemon run',
+                attempts: record.attempts + 1,
+                maxAttempts,
+                retryAt: new Date(Date.now() + backoffMs).toISOString(),
+                error,
+            });
+            throw error;
+        }
     },
     async runOnce() {
         const minAgeMs = Math.max(0, Env.getInt('JOB_RECOVERY_MIN_AGE_MS', 5000));
         const candidates = JobStateTracker.listRecoverable(minAgeMs);
+        const persisted = await listRecoverableFromPersistence(minAgeMs);
+        // De-dupe jobs that exist in both in-memory tracker and persistence.
+        const seen = new Set();
+        const allCandidates = [...candidates, ...persisted].filter((row) => {
+            const key = `${row.queueName}:${row.jobId}`;
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
         let requeued = 0;
         let deadLetter = 0;
         let manualReview = 0;
-        const results = await Promise.all(candidates.map(async (candidate) => this.recoverOne(candidate)));
-        results.forEach((result) => {
+        const concurrency = 10;
+        const batches = [];
+        for (let offset = 0; offset < allCandidates.length; offset += concurrency) {
+            const slice = allCandidates.slice(offset, offset + concurrency);
+            batches.push(Promise.all(slice.map(async (candidate) => this.recoverOne(candidate))));
+        }
+        const batchResults = await Promise.all(batches);
+        batchResults.flat().forEach((result) => {
             if (result === 'requeued')
                 requeued += 1;
             if (result === 'dead_letter')
@@ -137,16 +188,16 @@ export const JobRecoveryDaemon = Object.freeze({
             if (result === 'manual_review')
                 manualReview += 1;
         });
-        if (candidates.length > 0) {
+        if (allCandidates.length > 0) {
             Logger.info('Queue recovery daemon completed scan', {
-                scanned: candidates.length,
+                scanned: allCandidates.length,
                 requeued,
                 deadLetter,
                 manualReview,
             });
         }
         return {
-            scanned: candidates.length,
+            scanned: allCandidates.length,
             requeued,
             deadLetter,
             manualReview,
@@ -202,4 +253,51 @@ export const JobRecoveryDaemon = Object.freeze({
         };
     },
 });
+const toSqlDateTime = (value) => value.toISOString().slice(0, 19).replace('T', ' ');
+const getPersistenceDb = () => useDatabase(undefined, Env.get('JOB_TRACKING_DB_CONNECTION', 'default'));
+const listRecoverableFromPersistence = async (minAgeMs) => {
+    if (!Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false))
+        return [];
+    const db = getPersistenceDb();
+    const cutoff = new Date(Date.now() - Math.max(0, Math.floor(minAgeMs)));
+    const rows = await db
+        .table(Env.get('JOB_TRACKING_DB_TABLE', 'zintrust_jobs'))
+        .select('queue_name', 'job_id', 'attempts', 'max_attempts', 'payload_json', 'retry_at', 'updated_at')
+        .where('status', '=', 'pending_recovery')
+        .where('updated_at', '<=', toSqlDateTime(cutoff))
+        .limit(Math.max(1, Env.getInt('JOB_RECOVERY_DB_SCAN_LIMIT', 100)))
+        .get();
+    return (rows ?? []).map((row) => {
+        let payload = {};
+        try {
+            payload = JSON.parse(String(row.payload_json ?? '{}'));
+        }
+        catch {
+            payload = {};
+        }
+        const attempts = typeof row.attempts === 'number' && Number.isFinite(row.attempts)
+            ? Math.max(0, Math.floor(row.attempts))
+            : 0;
+        const maxAttempts = typeof row.max_attempts === 'number' && Number.isFinite(row.max_attempts)
+            ? Math.max(1, Math.floor(row.max_attempts))
+            : undefined;
+        const updatedAtRaw = typeof row.updated_at === 'string' ? row.updated_at : undefined;
+        const updatedAt = updatedAtRaw !== undefined && updatedAtRaw.trim().length > 0
+            ? updatedAtRaw
+            : new Date(0).toISOString();
+        const retryAtRaw = typeof row.retry_at === 'string' ? row.retry_at : undefined;
+        const retryAt = retryAtRaw !== undefined && retryAtRaw.trim().length > 0 ? retryAtRaw : undefined;
+        return {
+            queueName: String(row.queue_name),
+            jobId: String(row.job_id),
+            status: 'pending_recovery',
+            attempts,
+            maxAttempts,
+            createdAt: new Date().toISOString(),
+            updatedAt,
+            payload,
+            retryAt,
+        };
+    });
+};
 export default JobRecoveryDaemon;

package/src/tools/queue/JobStateTracker.d.ts

@@ -1,4 +1,4 @@
-export type JobTrackingStatus = 'pending' | 'active' | 'completed' | 'failed' | 'stalled' | 'timeout' | 'pending_recovery' | 'dead_letter' | 'manual_review' | 'delayed';
+export type JobTrackingStatus = 'pending' | 'active' | 'enqueued' | 'completed' | 'failed' | 'stalled' | 'timeout' | 'pending_recovery' | 'dead_letter' | 'manual_review' | 'delayed';
 export type JobTrackingRecord = {
     jobId: string;
     queueName: string;
@@ -44,6 +44,7 @@ export declare const JobStateTracker: Readonly<{
         queueName: string;
         jobId: string;
         payload?: unknown;
+        attempts?: number;
         maxAttempts?: number;
         expectedCompletionAt?: string;
         idempotencyKey?: string;
@@ -91,6 +92,9 @@ export declare const JobStateTracker: Readonly<{
     pendingRecovery(input: {
         queueName: string;
         jobId: string;
+        attempts?: number;
+        maxAttempts?: number;
+        retryAt?: string;
         reason?: string;
         error?: unknown;
     }): Promise<void>;
@@ -100,6 +104,11 @@ export declare const JobStateTracker: Readonly<{
         reason?: string;
         retryAt?: string;
     }): Promise<void>;
+    handedOffToQueue(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+    }): Promise<void>;
     setTerminalStatus(input: {
         queueName: string;
         jobId: string;

package/src/tools/queue/JobStateTracker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"JobStateTracker.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTracker.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,SAAS,GACT,SAAS,GACT,kBAAkB,GAClB,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEd,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,iCAAiC;IAChD,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,CAAC,UAAU,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAwRD,eAAO,MAAM,eAAe;iBACb,OAAO;oBAIE;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAYI;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,GAAG,OAAO,CAAC,IAAI,CAAC;qBA0BM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;kBAgBG;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;qBAkBM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,GAAG,OAAO,CAAC,IAAI,CAAC;oBASK;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAgBI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;2BAc7D;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;2BAeY;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;6BAec;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,GAAG,WAAW,CAAC;QACjE,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;wCAUmB,iCAAiC,GAAG,IAAI;+BAIjD,IAAI;mBAIhB,MAAM,SAAS,MAAM,GAAG,iBAAiB,GAAG,SAAS;mBAIrD;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,iBAAiB,EAAE;6BAgBE;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,qBAAqB,EAAE;kCAgBG,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;mCAW/C,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;8BAWrD,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;uCAYvC,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;2BAW5D,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;aAW7C,IAAI;EAKb,CAAC;AAEH,eAAe,eAAe,CAAC"}
+{"version":3,"file":"JobStateTracker.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTracker.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,QAAQ,GACR,UAAU,GACV,WAAW,GACX,QAAQ,GACR,SAAS,GACT,SAAS,GACT,kBAAkB,GAClB,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEd,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,iCAAiC;IAChD,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,CAAC,UAAU,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAwRD,eAAO,MAAM,eAAe;iBACb,OAAO;oBAIE;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAYI;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,GAAG,OAAO,CAAC,IAAI,CAAC;qBA0BM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;kBAgBG;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;qBAkBM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,GAAG,OAAO,CAAC,IAAI,CAAC;oBASK;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAgBI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;2BAc7D;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;2BAqBY;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;4BAea;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;6BAgBc;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,GAAG,WAAW,CAAC;QACjE,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;wCAUmB,iCAAiC,GAAG,IAAI;+BAIjD,IAAI;mBAIhB,MAAM,SAAS,MAAM,GAAG,iBAAiB,GAAG,SAAS;mBAIrD;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,iBAAiB,EAAE;6BAgBE;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,qBAAqB,EAAE;kCAgBG,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;mCAW/C,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;8BAWrD,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;uCAqBvC,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;2BAW5D,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;aAW7C,IAAI;EAKb,CAAC;AAEH,eAAe,eAAe,CAAC"}

package/src/tools/queue/JobStateTracker.js

@@ -175,7 +175,7 @@ export const JobStateTracker = Object.freeze({
         if (isEnabled() === false)
             return;
         const transition = updateStatus(input.queueName, input.jobId, 'pending', 'Job enqueued', {
-            attempts: 0,
+            attempts: typeof input.attempts === 'number' ? input.attempts : 0,
             maxAttempts: toFinitePositiveInt(input.maxAttempts),
             payload: QueueDataRedactor.sanitizePayload(input.payload),
             expectedCompletionAt: input.expectedCompletionAt,
@@ -259,6 +259,11 @@ export const JobStateTracker = Object.freeze({
         if (isEnabled() === false)
             return;
         const transition = updateStatus(input.queueName, input.jobId, 'pending_recovery', input.reason ?? 'Job pending recovery', {
+            attempts: typeof input.attempts === 'number' && Number.isFinite(input.attempts)
+                ? Math.max(0, Math.floor(input.attempts))
+                : undefined,
+            maxAttempts: toFinitePositiveInt(input.maxAttempts),
+            retryAt: input.retryAt,
             error: normalizeError(input.error) ?? input.reason,
             lastErrorCode: normalizeErrorCode(input.error),
         });
@@ -273,6 +278,16 @@ export const JobStateTracker = Object.freeze({
         });
         await persistLatest(input.queueName, input.jobId, transition);
     },
+    async handedOffToQueue(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'enqueued', input.reason ?? 'Job handed off to queue driver', {
+            retryAt: undefined,
+            timeoutAt: undefined,
+            error: undefined,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
     async setTerminalStatus(input) {
         if (isEnabled() === false)
             return;
@@ -345,12 +360,19 @@ export const JobStateTracker = Object.freeze({
     },
     listRecoverable(maxAgeMs, queueName) {
         const threshold = Date.now() - Math.max(0, Math.floor(maxAgeMs));
-        const recoverable = new Set(['pending_recovery', 'timeout', 'stalled']);
+        const recoverable = new Set(['pending_recovery']);
         return Array.from(trackedJobs.values()).filter((row) => {
             if (!recoverable.has(row.status))
                 return false;
             if (queueName !== undefined && row.queueName !== queueName)
                 return false;
+            // Respect scheduled retry window to avoid hot-loop recovery churn.
+            if (row.retryAt !== undefined && row.retryAt.trim().length > 0) {
+                const retryAtMs = new Date(row.retryAt).getTime();
+                if (!Number.isNaN(retryAtMs) && retryAtMs > Date.now()) {
+                    return false;
+                }
+            }
             const reference = toEpochMs(row.updatedAt);
             if (reference === null)
                 return false;

package/src/tools/queue/JobStateTrackerDbPersistence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"JobStateTrackerDbPersistence.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTrackerDbPersistence.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,iCAAiC,EAGlC,MAAM,wBAAwB,CAAC;AAEhC,KAAK,wBAAwB,GAAG;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AA+FF,eAAO,MAAM,kCAAkC,GAC7C,UAAS,wBAA6B,KACrC,iCAuCF,CAAC;AAEF,eAAO,MAAM,6CAA6C,QAAO,OAsBhE,CAAC;AAEF,eAAe,kCAAkC,CAAC"}
+{"version":3,"file":"JobStateTrackerDbPersistence.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTrackerDbPersistence.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,iCAAiC,EAGlC,MAAM,wBAAwB,CAAC;AAGhC,KAAK,wBAAwB,GAAG;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAoMF,eAAO,MAAM,kCAAkC,GAC7C,UAAS,wBAA6B,KACrC,iCAwDF,CAAC;AAEF,eAAO,MAAM,6CAA6C,QAAO,OAsBhE,CAAC;AAEF,eAAe,kCAAkC,CAAC"}

package/src/tools/queue/JobStateTrackerDbPersistence.js

@@ -13,6 +13,10 @@ const toJson = (value) => {
         return null;
     }
 };
+const toNonNullJson = (value) => {
+    const json = toJson(value);
+    return json ?? null;
+};
 const toSqlDateTime = (isoLike) => {
     if (typeof isoLike !== 'string' || isoLike.trim().length === 0)
         return null;
@@ -58,7 +62,7 @@ const serializeJobRecord = (record, options) => {
         status: record.status,
         attempts: record.attempts,
         max_attempts: record.maxAttempts ?? null,
-        payload_json: options.persistPayload === false ? null : toJson(record.payload),
+        payload_json: toNonNullJson(record.payload),
         result_json: options.persistResult === false ? null : toJson(record.result),
         last_error: record.lastError ?? null,
         last_error_code: record.lastErrorCode ?? null,
@@ -78,6 +82,78 @@ const serializeJobRecord = (record, options) => {
         updated_at: toSqlDateTime(record.updatedAt),
     };
 };
+const serializeJobRecordForInsert = (record, options) => {
+    const payload = serializeJobRecord(record, options);
+    // Ensure payload_json is always non-null for inserts (some schemas require NOT NULL).
+    if (payload['payload_json'] === null) {
+        payload['payload_json'] = '{}';
+    }
+    return payload;
+};
+const applyUpdateEntries = (update, entries) => {
+    entries.forEach((entry) => {
+        if (entry.enabled)
+            update[entry.key] = entry.value;
+    });
+};
+const resolveResultEntry = (record, options, base) => {
+    if (options.persistResult === false) {
+        return { key: 'result_json', enabled: true, value: null };
+    }
+    const enabled = record.result !== undefined && base['result_json'] !== null;
+    return { key: 'result_json', enabled, value: base['result_json'] };
+};
+const serializeJobRecordForUpdate = (record, options) => {
+    const base = serializeJobRecord(record, options);
+    const update = {
+        status: base['status'],
+        attempts: base['attempts'],
+        updated_at: base['updated_at'],
+    };
+    applyUpdateEntries(update, [
+        { key: 'max_attempts', enabled: record.maxAttempts !== undefined, value: base['max_attempts'] },
+        resolveResultEntry(record, options, base),
+        { key: 'last_error', enabled: record.lastError !== undefined, value: base['last_error'] },
+        {
+            key: 'last_error_code',
+            enabled: record.lastErrorCode !== undefined,
+            value: base['last_error_code'],
+        },
+        { key: 'retry_at', enabled: record.retryAt !== undefined, value: base['retry_at'] },
+        { key: 'timeout_at', enabled: record.timeoutAt !== undefined, value: base['timeout_at'] },
+        { key: 'heartbeat_at', enabled: record.heartbeatAt !== undefined, value: base['heartbeat_at'] },
+        {
+            key: 'expected_completion_at',
+            enabled: record.expectedCompletionAt !== undefined,
+            value: base['expected_completion_at'],
+        },
+        { key: 'worker_name', enabled: record.workerName !== undefined, value: base['worker_name'] },
+        {
+            key: 'worker_instance_id',
+            enabled: record.workerInstanceId !== undefined,
+            value: base['worker_instance_id'],
+        },
+        {
+            key: 'worker_region',
+            enabled: record.workerRegion !== undefined,
+            value: base['worker_region'],
+        },
+        {
+            key: 'worker_version',
+            enabled: record.workerVersion !== undefined,
+            value: base['worker_version'],
+        },
+        { key: 'recovered_at', enabled: record.recoveredAt !== undefined, value: base['recovered_at'] },
+        {
+            key: 'idempotency_key',
+            enabled: record.idempotencyKey !== undefined,
+            value: base['idempotency_key'],
+        },
+        { key: 'started_at', enabled: record.startedAt !== undefined, value: base['started_at'] },
+        { key: 'completed_at', enabled: record.completedAt !== undefined, value: base['completed_at'] },
+    ]);
+    return update;
+};
 const serializeTransition = (transition) => {
     return {
         job_id: transition.jobId,
@@ -94,17 +170,27 @@ export const createJobStateTrackerDbPersistence = (options = {}) => {
     const connectionName = getConnectionName(options);
     const jobsTable = getJobsTable(options);
     const transitionsTable = getTransitionsTable(options);
+    const persistTransitions = () => Env.getBool('JOB_TRACKING_PERSIST_TRANSITIONS_ENABLED', false);
+    const shouldInsertNewRow = (record) => {
+        // zintrust_jobs is an enqueue-fallback buffer: only jobs that failed to enqueue
+        // should be inserted into persistence. Once the job is in QUEUE_DRIVER, we only
+        // update the existing row (e.g., status=enqueued) but never create new rows.
+        return record.status === 'pending_recovery';
+    };
     const upsertJob = async (record) => {
         const db = getDatabase(connectionName);
         if (db === null)
             return;
-        const payload = serializeJobRecord(record, options);
         const existing = await db
             .table(jobsTable)
             .where('job_id', '=', record.jobId)
             .where('queue_name', '=', record.queueName)
             .first();
         if (existing) {
+            const existingStatus = typeof existing.status === 'string' ? existing.status.trim().toLowerCase() : '';
+            if (existingStatus === 'enqueued')
+                return;
+            const payload = serializeJobRecordForUpdate(record, options);
             await db
                 .table(jobsTable)
                 .where('job_id', '=', record.jobId)
@@ -112,9 +198,14 @@ export const createJobStateTrackerDbPersistence = (options = {}) => {
                 .update(payload);
             return;
         }
+        if (!shouldInsertNewRow(record))
+            return;
+        const payload = serializeJobRecordForInsert(record, options);
         await db.table(jobsTable).insert(payload);
     };
     const insertTransition = async (transition) => {
+        if (persistTransitions() === false)
+            return;
         const db = getDatabase(connectionName);
         if (db === null)
             return;