@zintrust/core 0.1.40 → 0.1.41

package/src/tools/queue/JobRecoveryDaemon.js

@@ -0,0 +1,205 @@
+import { Env } from '../../config/env.js';
+import { Logger } from '../../config/logger.js';
+import { ErrorFactory } from '../../exceptions/ZintrustError.js';
+import { JobStateTracker } from './JobStateTracker.js';
+import { Queue } from './Queue.js';
+const parsePayload = (payload) => {
+    if (payload !== null && payload !== undefined && typeof payload === 'object') {
+        return payload;
+    }
+    return { payload };
+};
+const shouldManualReview = (record) => {
+    const error = String(record.lastError ?? '').toLowerCase();
+    return error.includes('fatal') || error.includes('corrupt');
+};
+const replayReasonCodes = new Set([
+    'bug_fixed',
+    'transient_dependency',
+    'operator_override',
+]);
+const resolveLimit = (requested) => {
+    const maxBatch = Math.max(1, Env.getInt('DLQ_REPLAY_MAX_BATCH_SIZE', 25));
+    if (typeof requested !== 'number' || Number.isFinite(requested) === false || requested <= 0) {
+        return maxBatch;
+    }
+    return Math.min(maxBatch, Math.floor(requested));
+};
+const resolveRate = (requested) => {
+    const maxQps = Math.max(1, Env.getInt('DLQ_REPLAY_MAX_QPS', 5));
+    if (typeof requested !== 'number' || Number.isFinite(requested) === false || requested <= 0) {
+        return maxQps;
+    }
+    return Math.min(maxQps, Math.floor(requested));
+};
+const resolveMinAgeMs = (requested) => {
+    const minAllowed = Math.max(0, Env.getInt('DLQ_REPLAY_MIN_AGE_MS', 60000));
+    if (typeof requested !== 'number' || Number.isFinite(requested) === false || requested < 0) {
+        return minAllowed;
+    }
+    return Math.max(minAllowed, Math.floor(requested));
+};
+const validateReplayReasonCode = (reasonCode) => {
+    if (replayReasonCodes.has(reasonCode))
+        return;
+    throw ErrorFactory.createConfigError(`Unsupported DLQ replay reason code: ${reasonCode}`);
+};
+const validateReplayActor = (replayedBy) => {
+    if (replayedBy.trim().length === 0) {
+        throw ErrorFactory.createConfigError('DLQ replay actor is required');
+    }
+    const allowedActorsRaw = Env.get('DLQ_REPLAY_ALLOWED_ACTORS', '').trim();
+    if (allowedActorsRaw.length === 0)
+        return;
+    const allowedActors = allowedActorsRaw
+        .split(',')
+        .map((value) => value.trim())
+        .filter((value) => value.length > 0);
+    if (allowedActors.length === 0)
+        return;
+    if (allowedActors.includes(replayedBy.trim()))
+        return;
+    throw ErrorFactory.createConfigError(`DLQ replay actor is not allowed: ${replayedBy}`);
+};
+const isReplayEligible = (record, minAgeMs) => {
+    if (record.status !== 'dead_letter')
+        return false;
+    const updated = new Date(record.updatedAt).getTime();
+    if (Number.isNaN(updated))
+        return false;
+    return updated <= Date.now() - minAgeMs;
+};
+const toReplayPayload = (record, reasonCode, replayedBy) => {
+    const parsed = parsePayload(record.payload);
+    const lineage = {
+        originalJobId: record.jobId,
+        replayReasonCode: reasonCode,
+        replayedBy,
+        replayedAt: new Date().toISOString(),
+    };
+    const existingLineage = parsed['__dlqReplayLineage'];
+    const lineageArray = Array.isArray(existingLineage)
+        ? [...existingLineage.map((entry) => ({ ...entry })), lineage]
+        : [lineage];
+    return {
+        ...parsed,
+        __dlqReplayLineage: lineageArray,
+    };
+};
+export const JobRecoveryDaemon = Object.freeze({
+    async recoverOne(record) {
+        const maxAttempts = typeof record.maxAttempts === 'number' ? record.maxAttempts : 3;
+        if (record.attempts >= maxAttempts) {
+            await JobStateTracker.setTerminalStatus({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                status: 'dead_letter',
+                reason: 'Max retry attempts reached during recovery',
+            });
+            return 'dead_letter';
+        }
+        if (shouldManualReview(record)) {
+            await JobStateTracker.setTerminalStatus({
+                queueName: record.queueName,
+                jobId: record.jobId,
+                status: 'manual_review',
+                reason: 'Fatal recovery signature detected',
+            });
+            return 'manual_review';
+        }
+        const payload = parsePayload(record.payload);
+        const backoffMs = Math.min(300000, Math.max(1000, 1000 * 2 ** Math.max(0, record.attempts)));
+        await Queue.enqueue(record.queueName, {
+            ...payload,
+            attempts: Math.max(0, record.attempts),
+            timestamp: Date.now() + backoffMs,
+        });
+        await JobStateTracker.markedRecovered({
+            queueName: record.queueName,
+            jobId: record.jobId,
+            reason: 'Recovered and re-queued by recovery daemon',
+            retryAt: new Date(Date.now() + backoffMs).toISOString(),
+        });
+        return 'requeued';
+    },
+    async runOnce() {
+        const minAgeMs = Math.max(0, Env.getInt('JOB_RECOVERY_MIN_AGE_MS', 5000));
+        const candidates = JobStateTracker.listRecoverable(minAgeMs);
+        let requeued = 0;
+        let deadLetter = 0;
+        let manualReview = 0;
+        const results = await Promise.all(candidates.map(async (candidate) => this.recoverOne(candidate)));
+        results.forEach((result) => {
+            if (result === 'requeued')
+                requeued += 1;
+            if (result === 'dead_letter')
+                deadLetter += 1;
+            if (result === 'manual_review')
+                manualReview += 1;
+        });
+        if (candidates.length > 0) {
+            Logger.info('Queue recovery daemon completed scan', {
+                scanned: candidates.length,
+                requeued,
+                deadLetter,
+                manualReview,
+            });
+        }
+        return {
+            scanned: candidates.length,
+            requeued,
+            deadLetter,
+            manualReview,
+        };
+    },
+    async replayDeadLetter(input) {
+        validateReplayReasonCode(input.reasonCode);
+        validateReplayActor(input.replayedBy);
+        const limit = resolveLimit(input.limit);
+        const maxPerSecond = resolveRate(input.maxPerSecond);
+        const minAgeMs = resolveMinAgeMs(input.minAgeMs);
+        const replayBudget = Math.max(1, maxPerSecond);
+        const candidates = JobStateTracker.list({
+            queueName: input.queueName,
+            status: 'dead_letter',
+            limit,
+        })
+            .filter((record) => isReplayEligible(record, minAgeMs))
+            .slice(0, replayBudget);
+        let replayed = 0;
+        let skipped = 0;
+        await candidates.reduce(async (chain, candidate) => {
+            await chain;
+            if (candidate.queueName.trim().length === 0) {
+                skipped += 1;
+                return;
+            }
+            const replayPayload = toReplayPayload(candidate, input.reasonCode, input.replayedBy.trim());
+            const replayJobId = await Queue.enqueue(candidate.queueName, replayPayload);
+            await JobStateTracker.setTerminalStatus({
+                queueName: candidate.queueName,
+                jobId: candidate.jobId,
+                status: 'dead_letter',
+                reason: `DLQ replayed (${input.reasonCode}) by ${input.replayedBy.trim()} as ${replayJobId}`,
+            });
+            replayed += 1;
+        }, Promise.resolve());
+        Logger.info('DLQ replay governance batch completed', {
+            scanned: candidates.length,
+            replayed,
+            skipped,
+            reasonCode: input.reasonCode,
+            replayedBy: input.replayedBy.trim(),
+            queueName: input.queueName,
+            limit,
+            maxPerSecond,
+            minAgeMs,
+        });
+        return {
+            scanned: candidates.length,
+            replayed,
+            skipped,
+        };
+    },
+});
+export default JobRecoveryDaemon;

package/src/tools/queue/JobStateTracker.d.ts

@@ -0,0 +1,131 @@
+export type JobTrackingStatus = 'pending' | 'active' | 'completed' | 'failed' | 'stalled' | 'timeout' | 'pending_recovery' | 'dead_letter' | 'manual_review' | 'delayed';
+export type JobTrackingRecord = {
+    jobId: string;
+    queueName: string;
+    status: JobTrackingStatus;
+    attempts: number;
+    maxAttempts?: number;
+    payload?: unknown;
+    result?: unknown;
+    lastError?: string;
+    lastErrorCode?: string;
+    retryAt?: string;
+    timeoutAt?: string;
+    heartbeatAt?: string;
+    expectedCompletionAt?: string;
+    workerName?: string;
+    workerInstanceId?: string;
+    workerRegion?: string;
+    workerVersion?: string;
+    recoveredAt?: string;
+    idempotencyKey?: string;
+    createdAt: string;
+    startedAt?: string;
+    completedAt?: string;
+    updatedAt: string;
+};
+export type JobTrackingTransition = {
+    jobId: string;
+    queueName: string;
+    fromStatus: JobTrackingStatus | null;
+    toStatus: JobTrackingStatus;
+    reason: string;
+    timestamp: string;
+    attempts?: number;
+    error?: string;
+};
+export interface JobStateTrackerPersistenceAdapter {
+    upsertJob(record: JobTrackingRecord): Promise<void>;
+    insertTransition(transition: JobTrackingTransition): Promise<void>;
+}
+export declare const JobStateTracker: Readonly<{
+    isEnabled(): boolean;
+    enqueued(input: {
+        queueName: string;
+        jobId: string;
+        payload?: unknown;
+        maxAttempts?: number;
+        expectedCompletionAt?: string;
+        idempotencyKey?: string;
+    }): Promise<void>;
+    started(input: {
+        queueName: string;
+        jobId: string;
+        attempts?: number;
+        timeoutMs?: number;
+        workerName?: string;
+        workerInstanceId?: string;
+        workerRegion?: string;
+        workerVersion?: string;
+    }): Promise<void>;
+    completed(input: {
+        queueName: string;
+        jobId: string;
+        processingTimeMs?: number;
+        result?: unknown;
+    }): Promise<void>;
+    failed(input: {
+        queueName: string;
+        jobId: string;
+        attempts?: number;
+        isFinal: boolean;
+        retryAt?: string;
+        error?: unknown;
+    }): Promise<void>;
+    heartbeat(input: {
+        queueName: string;
+        jobId: string;
+        workerInstanceId?: string;
+    }): Promise<void>;
+    timedOut(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+        error?: unknown;
+    }): Promise<void>;
+    stalled(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+    }): Promise<void>;
+    pendingRecovery(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+        error?: unknown;
+    }): Promise<void>;
+    markedRecovered(input: {
+        queueName: string;
+        jobId: string;
+        reason?: string;
+        retryAt?: string;
+    }): Promise<void>;
+    setTerminalStatus(input: {
+        queueName: string;
+        jobId: string;
+        status: "dead_letter" | "manual_review" | "failed" | "completed";
+        reason: string;
+        error?: unknown;
+    }): Promise<void>;
+    registerPersistenceAdapter(adapter: JobStateTrackerPersistenceAdapter): void;
+    clearPersistenceAdapter(): void;
+    get(queueName: string, jobId: string): JobTrackingRecord | undefined;
+    list(options?: {
+        queueName?: string;
+        status?: JobTrackingStatus;
+        limit?: number;
+    }): JobTrackingRecord[];
+    getTransitions(options?: {
+        queueName?: string;
+        jobId?: string;
+        limit?: number;
+    }): JobTrackingTransition[];
+    listActiveOlderThan(maxAgeMs: number, queueName?: string): JobTrackingRecord[];
+    listPendingOlderThan(maxAgeMs: number, queueName?: string): JobTrackingRecord[];
+    listRecoverable(maxAgeMs: number, queueName?: string): JobTrackingRecord[];
+    listHeartbeatExpired(maxSilenceMs: number, queueName?: string): JobTrackingRecord[];
+    getSummary(queueName?: string): Record<string, number>;
+    reset(): void;
+}>;
+export default JobStateTracker;
+//# sourceMappingURL=JobStateTracker.d.ts.map

package/src/tools/queue/JobStateTracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobStateTracker.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/JobStateTracker.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,SAAS,GACT,SAAS,GACT,kBAAkB,GAClB,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEd,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,iCAAiC;IAChD,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,gBAAgB,CAAC,UAAU,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAwRD,eAAO,MAAM,eAAe;iBACb,OAAO;oBAIE;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAYI;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,GAAG,OAAO,CAAC,IAAI,CAAC;qBA0BM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;kBAgBG;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;qBAkBM;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,GAAG,OAAO,CAAC,IAAI,CAAC;oBASK;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;mBAgBI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;2BAc7D;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;2BAeY;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;6BAec;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,GAAG,WAAW,CAAC;QACjE,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;wCAUmB,iCAAiC,GAAG,IAAI;+BAIjD,IAAI;mBAIhB,MAAM,SAAS,MAAM,GAAG,iBAAiB,GAAG,SAAS;mBAIrD;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,iBAAiB,CAAC;QAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,iBAAiB,EAAE;6BAgBE;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,qBAAqB,EAAE;kCAgBG,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;mCAW/C,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;8BAWrD,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;uCAYvC,MAAM,cAAc,MAAM,GAAG,iBAAiB,EAAE;2BAW5D,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;aAW7C,IAAI;EAKb,CAAC;AAEH,eAAe,eAAe,CAAC"}