@zintrust/core 2.4.1 → 2.4.2

package/src/tools/queue/JobStateTracker.d.ts

@@ -0,0 +1,140 @@
+export type JobTrackingStatus = 'pending' | 'active' | 'enqueued' | 'completed' | 'failed' | 'stalled' | 'timeout' | 'pending_recovery' | 'dead_letter' | 'manual_review' | 'delayed';
+export type JobTrackingRecord = {
+    jobId: string;
+    queueName: string;
+    status: JobTrackingStatus;
+    attempts: number;
+    maxAttempts?: number;
+    payload?: unknown;
+    result?: unknown;
+    lastError?: string;
+    lastErrorCode?: string;
+    retryAt?: string;
+    timeoutAt?: string;
+    heartbeatAt?: string;
+    expectedCompletionAt?: string;
+    workerName?: string;
+    workerInstanceId?: string;
+    workerRegion?: string;
+    workerVersion?: string;
+    recoveredAt?: string;
+    idempotencyKey?: string;
+    createdAt: string;
+    startedAt?: string;
+    completedAt?: string;
+    updatedAt: string;
+};
+export type JobTrackingTransition = {
+    jobId: string;
+    queueName: string;
+    fromStatus: JobTrackingStatus | null;
+    toStatus: JobTrackingStatus;
+    reason: string;
+    timestamp: string;
+    attempts?: number;
+    error?: string;
+};
+export interface JobStateTrackerPersistenceAdapter {
+    upsertJob(record: JobTrackingRecord): Promise<void>;
+    insertTransition(transition: JobTrackingTransition): Promise<void>;
+}
+export declare const JobStateTracker: Readonly<{
+    isEnabled(): boolean;
+    enqueued(input: {
+        queueName: string;
+        jobId: string;
+        payload?: unknown;
+        attempts?: number;
+        maxAttempts?: number;
+        expectedCompletionAt?: string;
+        idempotencyKey?: string;
+    }): Promise<void>;
+    started(input: {
+        queueName: string;
+        jobId: string;
+        attempts?: number;
+        timeoutMs?: number;
+        workerName?: string;
+        workerInstanceId?: string;
+        workerRegion?: string;
+        workerVersion?: string;
+    }): Promise<void>;
+    completed(input: {
+        queueName: string;
+        jobId: string;
+        processingTimeMs?: number;
+        result?: unknown;
+    }): Promise<void>;
+    failed(input: {
+        queueName: string;
+        jobId: string;
+        attempts?: number;
+        isFinal: boolean;
+        retryAt?: string;
+        error?: unknown;
+    }): Promise<void>;
+    heartbeat(input: {
+        queueName: string;
+        jobId: string;
+        workerInstanceId?: string;
+    }): Promise<void>;
+    timedOut(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+        error?: unknown;
+    }): Promise<void>;
+    stalled(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+    }): Promise<void>;
+    pendingRecovery(input: {
+        queueName: string;
+        jobId: string;
+        attempts?: number;
+        maxAttempts?: number;
+        retryAt?: string;
+        reason?: string;
+        error?: unknown;
+    }): Promise<void>;
+    markedRecovered(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+        retryAt?: string;
+    }): Promise<void>;
+    handedOffToQueue(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+    }): Promise<void>;
+    setTerminalStatus(input: {
+        queueName: string;
+        jobId: string;
+        status: "dead_letter" | "manual_review" | "failed" | "completed";
+        reason: string;
+        error?: unknown;
+    }): Promise<void>;
+    registerPersistenceAdapter(adapter: JobStateTrackerPersistenceAdapter): void;
+    clearPersistenceAdapter(): void;
+    get(queueName: string, jobId: string): JobTrackingRecord | undefined;
+    list(options?: {
+        queueName?: string;
+        status?: JobTrackingStatus;
+        limit?: number;
+    }): JobTrackingRecord[];
+    getTransitions(options?: {
+        queueName?: string;
+        jobId?: string;
+        limit?: number;
+    }): JobTrackingTransition[];
+    listActiveOlderThan(maxAgeMs: number, queueName?: string): JobTrackingRecord[];
+    listPendingOlderThan(maxAgeMs: number, queueName?: string): JobTrackingRecord[];
+    listRecoverable(maxAgeMs: number, queueName?: string): JobTrackingRecord[];
+    listHeartbeatExpired(maxSilenceMs: number, queueName?: string): JobTrackingRecord[];
+    getSummary(queueName?: string): Record<string, number>;
+    reset(): void;
+}>;
+export default JobStateTracker;
+//# sourceMappingURL=JobStateTracker.d.ts.map

package/src/tools/queue/JobStateTracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobStateTracker.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTracker.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,QAAQ,GACR,UAAU,GACV,WAAW,GACX,QAAQ,GACR,SAAS,GACT,SAAS,GACT,kBAAkB,GAClB,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEd,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,iCAAiC;IAChD,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,CAAC,UAAU,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAwRD,eAAO,MAAM,eAAe;iBACb,OAAO;oBAIE;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAYI;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,GAAG,OAAO,CAAC,IAAI,CAAC;qBA0BM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;kBAgBG;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;qBAkBM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,GAAG,OAAO,CAAC,IAAI,CAAC;oBASK;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAgBI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;2BAc7D;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;2BAqBY;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;4BAea;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;6BAgBc;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,GAAG,WAAW,CAAC;QACjE,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;wCAUmB,iCAAiC,GAAG,IAAI;+BAIjD,IAAI;mBAIhB,MAAM,SAAS,MAAM,GAAG,iBAAiB,GAAG,SAAS;mBAIrD;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,iBAAiB,EAAE;6BAgBE;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,qBAAqB,EAAE;kCAgBG,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;mCAW/C,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;8BAWrD,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;uCAqBvC,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;2BAW5D,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;aAW7C,IAAI;EAKb,CAAC;AAEH,eAAe,eAAe,CAAC"}

package/src/tools/queue/JobStateTracker.js

@@ -0,0 +1,409 @@
+import { Env } from '../../config/env.js';
+import { Logger } from '../../config/logger.js';
+import { QueueDataRedactor } from './QueueDataRedactor.js';
+const trackedJobs = new Map();
+const transitions = [];
+let persistenceAdapter = null;
+const getKey = (queueName, jobId) => `${queueName}:${jobId}`;
+const nowIso = () => new Date().toISOString();
+const isEnabled = () => Env.getBool('JOB_TRACKING_ENABLED', true);
+const isPersistenceEnabled = () => Env.getBool('JOB_TRACKING_PERSISTENCE_ENABLED', false);
+const toFinitePositiveInt = (value) => {
+    if (typeof value !== 'number' || Number.isFinite(value) === false)
+        return undefined;
+    if (value <= 0)
+        return undefined;
+    return Math.floor(value);
+};
+const normalizeError = (error) => {
+    if (error instanceof Error)
+        return QueueDataRedactor.redactText(error.message);
+    if (typeof error === 'string' && error.trim().length > 0) {
+        return QueueDataRedactor.redactText(error);
+    }
+    return undefined;
+};
+const normalizeErrorCode = (error) => {
+    if (error === null || error === undefined || typeof error !== 'object')
+        return undefined;
+    const raw = error.code;
+    if (typeof raw !== 'string')
+        return undefined;
+    const trimmed = raw.trim();
+    return trimmed === '' ? undefined : trimmed;
+};
+const pruneJobs = () => {
+    const maxJobs = Env.getInt('JOB_TRACKING_MAX_JOBS', 20000);
+    while (trackedJobs.size > maxJobs) {
+        const oldestKey = trackedJobs.keys().next().value;
+        if (oldestKey === undefined)
+            break;
+        trackedJobs.delete(oldestKey);
+    }
+};
+const appendTransition = (transition) => {
+    transitions.push(transition);
+    const maxTransitions = Env.getInt('JOB_TRACKING_MAX_TRANSITIONS', 50000);
+    if (transitions.length > maxTransitions) {
+        transitions.splice(0, transitions.length - maxTransitions);
+    }
+};
+const choose = (first, second, third) => {
+    if (first !== undefined)
+        return first;
+    if (second !== undefined)
+        return second;
+    return third;
+};
+const toEpochMs = (value) => {
+    if (value === undefined || value.trim().length === 0)
+        return null;
+    const parsed = new Date(value).getTime();
+    if (Number.isNaN(parsed))
+        return null;
+    return parsed;
+};
+const resolveAttempts = (existing, options) => choose(options.attempts, existing?.attempts, 0);
+const resolveMaxAttempts = (existing, options) => choose(options.maxAttempts, existing?.maxAttempts);
+const resolvePayload = (existing, options) => choose(options.payload, existing?.payload);
+const resolveResult = (existing, options) => choose(options.result, existing?.result);
+const resolveError = (existing, options) => choose(options.error, existing?.lastError);
+const resolveRetryAt = (existing, options) => choose(options.retryAt, existing?.retryAt);
+const resolveTimeoutAt = (existing, options) => choose(options.timeoutAt, existing?.timeoutAt);
+const resolveExpectedCompletionAt = (existing, options) => choose(options.expectedCompletionAt, existing?.expectedCompletionAt);
+const resolveWorkerName = (existing, options) => choose(options.workerName, existing?.workerName);
+const resolveWorkerInstanceId = (existing, options) => choose(options.workerInstanceId, existing?.workerInstanceId);
+const resolveWorkerRegion = (existing, options) => choose(options.workerRegion, existing?.workerRegion);
+const resolveWorkerVersion = (existing, options) => choose(options.workerVersion, existing?.workerVersion);
+const resolveHeartbeatAt = (existing, options) => choose(options.heartbeatAt, existing?.heartbeatAt);
+const resolveLastErrorCode = (existing, options) => choose(options.lastErrorCode, existing?.lastErrorCode);
+const resolveRecoveredAt = (existing, options) => choose(options.recoveredAt, existing?.recoveredAt);
+const resolveIdempotencyKey = (existing, options) => choose(options.idempotencyKey, existing?.idempotencyKey);
+const resolveCreatedAt = (existing, timestamp) => choose(existing?.createdAt, timestamp, timestamp);
+const resolveStartedAt = (existing, options) => choose(options.startedAt, existing?.startedAt);
+const resolveCompletedAt = (existing, options) => choose(options.completedAt, existing?.completedAt);
+const buildRecord = (existing, queueName, jobId, toStatus, timestamp, options) => {
+    const resolvedOptions = options ?? {};
+    const attempts = resolveAttempts(existing, resolvedOptions);
+    const maxAttempts = resolveMaxAttempts(existing, resolvedOptions);
+    const payload = resolvePayload(existing, resolvedOptions);
+    const result = resolveResult(existing, resolvedOptions);
+    const lastError = resolveError(existing, resolvedOptions);
+    const retryAt = resolveRetryAt(existing, resolvedOptions);
+    const timeoutAt = resolveTimeoutAt(existing, resolvedOptions);
+    const expectedCompletionAt = resolveExpectedCompletionAt(existing, resolvedOptions);
+    const workerName = resolveWorkerName(existing, resolvedOptions);
+    const workerInstanceId = resolveWorkerInstanceId(existing, resolvedOptions);
+    const workerRegion = resolveWorkerRegion(existing, resolvedOptions);
+    const workerVersion = resolveWorkerVersion(existing, resolvedOptions);
+    const heartbeatAt = resolveHeartbeatAt(existing, resolvedOptions);
+    const lastErrorCode = resolveLastErrorCode(existing, resolvedOptions);
+    const recoveredAt = resolveRecoveredAt(existing, resolvedOptions);
+    const idempotencyKey = resolveIdempotencyKey(existing, resolvedOptions);
+    const createdAt = resolveCreatedAt(existing, timestamp);
+    const startedAt = resolveStartedAt(existing, resolvedOptions);
+    const completedAt = resolveCompletedAt(existing, resolvedOptions);
+    return {
+        jobId,
+        queueName,
+        status: toStatus,
+        attempts,
+        maxAttempts,
+        payload,
+        result,
+        lastError,
+        lastErrorCode,
+        retryAt,
+        timeoutAt,
+        expectedCompletionAt,
+        workerName,
+        workerInstanceId,
+        workerRegion,
+        workerVersion,
+        heartbeatAt,
+        recoveredAt,
+        idempotencyKey,
+        createdAt,
+        startedAt,
+        completedAt,
+        updatedAt: timestamp,
+    };
+};
+const updateStatus = (queueName, jobId, toStatus, reason, options) => {
+    const key = getKey(queueName, jobId);
+    const existing = trackedJobs.get(key);
+    const timestamp = nowIso();
+    const next = buildRecord(existing, queueName, jobId, toStatus, timestamp, options);
+    trackedJobs.set(key, next);
+    const transition = {
+        jobId,
+        queueName,
+        fromStatus: existing?.status ?? null,
+        toStatus,
+        reason,
+        timestamp,
+        attempts: next.attempts,
+        error: options?.error,
+    };
+    appendTransition(transition);
+    pruneJobs();
+    return transition;
+};
+const persistLatest = async (queueName, jobId, transition) => {
+    if (isPersistenceEnabled() === false)
+        return;
+    if (persistenceAdapter === null)
+        return;
+    const record = trackedJobs.get(getKey(queueName, jobId));
+    if (record === undefined)
+        return;
+    try {
+        await persistenceAdapter.upsertJob(record);
+        if (transition !== undefined) {
+            await persistenceAdapter.insertTransition(transition);
+        }
+    }
+    catch (error) {
+        Logger.warn('Job tracking persistence failed', { queueName, jobId, error });
+    }
+};
+export const JobStateTracker = Object.freeze({
+    isEnabled() {
+        return isEnabled();
+    },
+    async enqueued(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'pending', 'Job enqueued', {
+            attempts: typeof input.attempts === 'number' ? input.attempts : 0,
+            maxAttempts: toFinitePositiveInt(input.maxAttempts),
+            payload: QueueDataRedactor.sanitizePayload(input.payload),
+            expectedCompletionAt: input.expectedCompletionAt,
+            idempotencyKey: input.idempotencyKey,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async started(input) {
+        if (isEnabled() === false)
+            return;
+        const timeoutMs = typeof input.timeoutMs === 'number' && Number.isFinite(input.timeoutMs) && input.timeoutMs > 0
+            ? Math.floor(input.timeoutMs)
+            : undefined;
+        const transition = updateStatus(input.queueName, input.jobId, 'active', 'Job processing started', {
+            attempts: toFinitePositiveInt(input.attempts) ?? 1,
+            startedAt: nowIso(),
+            expectedCompletionAt: timeoutMs === undefined ? undefined : new Date(Date.now() + timeoutMs).toISOString(),
+            workerName: input.workerName,
+            workerInstanceId: input.workerInstanceId,
+            workerRegion: input.workerRegion,
+            workerVersion: input.workerVersion,
+            heartbeatAt: nowIso(),
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async completed(input) {
+        if (isEnabled() === false)
+            return;
+        const reason = typeof input.processingTimeMs === 'number' && Number.isFinite(input.processingTimeMs)
+            ? `Job completed in ${Math.max(0, Math.floor(input.processingTimeMs))}ms`
+            : 'Job completed';
+        const transition = updateStatus(input.queueName, input.jobId, 'completed', reason, {
+            completedAt: nowIso(),
+            result: QueueDataRedactor.sanitizePayload(input.result),
+            retryAt: undefined,
+            error: undefined,
+            timeoutAt: undefined,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async failed(input) {
+        if (isEnabled() === false)
+            return;
+        const errorMessage = normalizeError(input.error);
+        const transition = updateStatus(input.queueName, input.jobId, input.isFinal ? 'failed' : 'pending', input.isFinal ? 'Job failed permanently' : 'Job failed and scheduled for retry', {
+            attempts: toFinitePositiveInt(input.attempts),
+            error: errorMessage,
+            lastErrorCode: normalizeErrorCode(input.error),
+            retryAt: input.isFinal ? undefined : input.retryAt,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async heartbeat(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'active', 'Heartbeat updated', {
+            heartbeatAt: nowIso(),
+            workerInstanceId: input.workerInstanceId,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async timedOut(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'timeout', input.reason ?? 'Job timed out', {
+            timeoutAt: nowIso(),
+            error: normalizeError(input.error) ?? input.reason ?? 'Job timed out',
+            lastErrorCode: normalizeErrorCode(input.error),
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async stalled(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'stalled', input.reason ?? 'Job stalled', {
+            error: input.reason,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async pendingRecovery(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'pending_recovery', input.reason ?? 'Job pending recovery', {
+            attempts: typeof input.attempts === 'number' && Number.isFinite(input.attempts)
+                ? Math.max(0, Math.floor(input.attempts))
+                : undefined,
+            maxAttempts: toFinitePositiveInt(input.maxAttempts),
+            retryAt: input.retryAt,
+            error: normalizeError(input.error) ?? input.reason,
+            lastErrorCode: normalizeErrorCode(input.error),
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async markedRecovered(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'pending', input.reason ?? 'Job re-queued for recovery', {
+            recoveredAt: nowIso(),
+            retryAt: input.retryAt,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async handedOffToQueue(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, 'enqueued', input.reason ?? 'Job handed off to queue driver', {
+            retryAt: undefined,
+            timeoutAt: undefined,
+            error: undefined,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    async setTerminalStatus(input) {
+        if (isEnabled() === false)
+            return;
+        const transition = updateStatus(input.queueName, input.jobId, input.status, input.reason, {
+            error: normalizeError(input.error),
+            lastErrorCode: normalizeErrorCode(input.error),
+            completedAt: input.status === 'completed' ? nowIso() : undefined,
+        });
+        await persistLatest(input.queueName, input.jobId, transition);
+    },
+    registerPersistenceAdapter(adapter) {
+        persistenceAdapter = adapter;
+    },
+    clearPersistenceAdapter() {
+        persistenceAdapter = null;
+    },
+    get(queueName, jobId) {
+        return trackedJobs.get(getKey(queueName, jobId));
+    },
+    list(options) {
+        const filterQueueName = options?.queueName;
+        const filterStatus = options?.status;
+        const limit = typeof options?.limit === 'number' && Number.isFinite(options.limit) && options.limit > 0
+            ? Math.floor(options.limit)
+            : 100;
+        const rows = Array.from(trackedJobs.values())
+            .filter((row) => (filterQueueName === undefined ? true : row.queueName === filterQueueName))
+            .filter((row) => (filterStatus === undefined ? true : row.status === filterStatus))
+            .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
+        return rows.slice(0, limit);
+    },
+    getTransitions(options) {
+        const filterQueueName = options?.queueName;
+        const filterJobId = options?.jobId;
+        const limit = typeof options?.limit === 'number' && Number.isFinite(options.limit) && options.limit > 0
+            ? Math.floor(options.limit)
+            : 200;
+        const rows = transitions
+            .filter((row) => (filterQueueName === undefined ? true : row.queueName === filterQueueName))
+            .filter((row) => (filterJobId === undefined ? true : row.jobId === filterJobId));
+        if (rows.length <= limit)
+            return [...rows];
+        return rows.slice(rows.length - limit);
+    },
+    listActiveOlderThan(maxAgeMs, queueName) {
+        const threshold = Date.now() - Math.max(0, Math.floor(maxAgeMs));
+        return Array.from(trackedJobs.values()).filter((row) => {
+            if (row.status !== 'active')
+                return false;
+            if (queueName !== undefined && row.queueName !== queueName)
+                return false;
+            const reference = toEpochMs(row.startedAt) ?? toEpochMs(row.updatedAt);
+            if (reference === null)
+                return false;
+            return reference <= threshold;
+        });
+    },
+    listPendingOlderThan(maxAgeMs, queueName) {
+        const threshold = Date.now() - Math.max(0, Math.floor(maxAgeMs));
+        return Array.from(trackedJobs.values()).filter((row) => {
+            if (row.status !== 'pending')
+                return false;
+            if (queueName !== undefined && row.queueName !== queueName)
+                return false;
+            const reference = toEpochMs(row.updatedAt) ?? toEpochMs(row.createdAt);
+            if (reference === null)
+                return false;
+            return reference <= threshold;
+        });
+    },
+    listRecoverable(maxAgeMs, queueName) {
+        const threshold = Date.now() - Math.max(0, Math.floor(maxAgeMs));
+        const recoverable = new Set(['pending_recovery']);
+        return Array.from(trackedJobs.values()).filter((row) => {
+            if (!recoverable.has(row.status))
+                return false;
+            if (queueName !== undefined && row.queueName !== queueName)
+                return false;
+            // Respect scheduled retry window to avoid hot-loop recovery churn.
+            if (row.retryAt !== undefined && row.retryAt.trim().length > 0) {
+                const retryAtMs = new Date(row.retryAt).getTime();
+                if (!Number.isNaN(retryAtMs) && retryAtMs > Date.now()) {
+                    return false;
+                }
+            }
+            const reference = toEpochMs(row.updatedAt);
+            if (reference === null)
+                return false;
+            return reference <= threshold;
+        });
+    },
+    listHeartbeatExpired(maxSilenceMs, queueName) {
+        const threshold = Date.now() - Math.max(0, Math.floor(maxSilenceMs));
+        return Array.from(trackedJobs.values()).filter((row) => {
+            if (row.status !== 'active')
+                return false;
+            if (queueName !== undefined && row.queueName !== queueName)
+                return false;
+            const reference = toEpochMs(row.heartbeatAt) ?? toEpochMs(row.updatedAt);
+            if (reference === null)
+                return false;
+            return reference <= threshold;
+        });
+    },
+    getSummary(queueName) {
+        const rows = Array.from(trackedJobs.values()).filter((row) => queueName === undefined ? true : row.queueName === queueName);
+        return rows.reduce((acc, row) => {
+            const key = row.status;
+            acc[key] = (acc[key] ?? 0) + 1;
+            return acc;
+        }, {});
+    },
+    reset() {
+        trackedJobs.clear();
+        transitions.splice(0, transitions.length);
+        persistenceAdapter = null;
+    },
+});
+export default JobStateTracker;

package/src/tools/queue/JobStateTrackerDbPersistence.d.ts

@@ -0,0 +1,12 @@
+import type { JobStateTrackerPersistenceAdapter } from './JobStateTracker';
+type JobStateTrackerDbOptions = {
+    connectionName?: string;
+    jobsTable?: string;
+    transitionsTable?: string;
+    persistPayload?: boolean;
+    persistResult?: boolean;
+};
+export declare const createJobStateTrackerDbPersistence: (options?: JobStateTrackerDbOptions) => JobStateTrackerPersistenceAdapter;
+export declare const autoRegisterJobStateTrackerPersistenceFromEnv: () => boolean;
+export default createJobStateTrackerDbPersistence;
+//# sourceMappingURL=JobStateTrackerDbPersistence.d.ts.map

package/src/tools/queue/JobStateTrackerDbPersistence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobStateTrackerDbPersistence.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTrackerDbPersistence.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,iCAAiC,EAGlC,MAAM,wBAAwB,CAAC;AAGhC,KAAK,wBAAwB,GAAG;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AAoMF,eAAO,MAAM,kCAAkC,GAC7C,UAAS,wBAA6B,KACrC,iCAwDF,CAAC;AAEF,eAAO,MAAM,6CAA6C,QAAO,OAsBhE,CAAC;AAEF,eAAe,kCAAkC,CAAC"}