@zintrust/core 0.1.40 → 0.1.42

package/src/tools/queue/Queue.js

@@ -1,6 +1,10 @@
+import { generateUuid } from '../../common/utility.js';
 import { ZintrustLang } from '../../lang/lang.js';
 import { Env } from '../../config/env.js';
+import { Logger } from '../../config/logger.js';
 import { ErrorFactory } from '../../exceptions/ZintrustError.js';
+import { JobStateTracker } from './JobStateTracker.js';
+import { QueueTracing } from './QueueTracing.js';
 import { RedisKeys } from '../redis/RedisKeyManager.js';
 let redis_key_prefix;
 /**
@@ -15,6 +19,61 @@ export const resolveLockPrefix = () => {
     return redis_key_prefix;
 };
 const drivers = new Map();
+const resolveDriverName = (name) => {
+    const resolved = (name ?? Env.QUEUE_CONNECTION) || Env.QUEUE_DRIVER || ZintrustLang.INMEMORY;
+    return (resolved !== null && resolved !== undefined ? String(resolved) : ZintrustLang.INMEMORY)
+        .trim()
+        .toLowerCase();
+};
+const shouldPreserveExistingStatus = (queueName, jobId) => {
+    const existing = JobStateTracker.get(queueName, jobId);
+    return existing?.status === 'pending_recovery';
+};
+const resolveRequestedUniqueId = (payload) => {
+    if (typeof payload?.uniqueId !== 'string')
+        return undefined;
+    const normalized = payload.uniqueId.trim();
+    return normalized.length > 0 ? normalized : undefined;
+};
+const resolveMaxAttempts = (payload) => {
+    if (typeof payload?.attempts !== 'number' || !Number.isFinite(payload.attempts))
+        return undefined;
+    return payload.attempts > 0 ? Math.floor(payload.attempts) : undefined;
+};
+const resolveExpectedCompletionAt = () => {
+    return new Date(Date.now() + Math.max(1000, Env.getInt('QUEUE_JOB_TIMEOUT', 60) * 1000)).toISOString();
+};
+const markEnqueued = async (input) => {
+    await JobStateTracker.enqueued({
+        queueName: input.queueName,
+        jobId: input.jobId,
+        payload: input.payload,
+        maxAttempts: resolveMaxAttempts(input.payload),
+        expectedCompletionAt: resolveExpectedCompletionAt(),
+        idempotencyKey: input.requestedUniqueId,
+    });
+};
+const createFallbackJobId = (requestedUniqueId) => {
+    if (requestedUniqueId !== undefined)
+        return requestedUniqueId;
+    return `fallback-${generateUuid()}`;
+};
+const markFailedEnqueue = async (input) => {
+    const fallbackJobId = createFallbackJobId(input.requestedUniqueId);
+    await markEnqueued({
+        queueName: input.queueName,
+        jobId: fallbackJobId,
+        payload: input.payload,
+        requestedUniqueId: input.requestedUniqueId,
+    });
+    await JobStateTracker.pendingRecovery({
+        queueName: input.queueName,
+        jobId: fallbackJobId,
+        reason: 'Queue enqueue failed; marked pending recovery by core queue layer',
+        error: input.error,
+    });
+    return fallbackJobId;
+};
 export const Queue = Object.freeze({
     register(name, driver) {
         drivers.set(name.toLowerCase(), driver);
@@ -23,35 +82,120 @@ export const Queue = Object.freeze({
         drivers.clear();
     },
     get(name) {
-        const resolved = (name ?? Env.QUEUE_CONNECTION) || Env.QUEUE_DRIVER || ZintrustLang.INMEMORY;
-        const driverName = (resolved !== null && resolved !== undefined ? String(resolved) : ZintrustLang.INMEMORY)
-            .trim()
-            .toLowerCase();
+        const driverName = resolveDriverName(name);
         const driver = drivers.get(driverName);
-        if (!driver)
+        if (!driver) {
             throw ErrorFactory.createConfigError(`Queue driver not registered: ${driverName}`);
+        }
         return driver;
     },
     async enqueue(queue, payload, driverName) {
-        const driver = Queue.get(driverName);
-        const jobId = await driver.enqueue(queue, payload);
-        return jobId;
+        const resolvedDriver = resolveDriverName(driverName);
+        const requestedUniqueId = resolveRequestedUniqueId(payload);
+        try {
+            const jobId = await QueueTracing.traceOperation({
+                queueName: queue,
+                operation: 'enqueue',
+                attributes: {
+                    driverName: resolvedDriver,
+                    hasUniqueId: requestedUniqueId !== undefined,
+                },
+                execute: async () => {
+                    const driver = Queue.get(driverName);
+                    return driver.enqueue(queue, payload);
+                },
+            });
+            Logger.info('Queue enqueue succeeded', {
+                queue,
+                driver: resolvedDriver,
+                jobId,
+                requestedUniqueId,
+            });
+            if (shouldPreserveExistingStatus(queue, jobId)) {
+                Logger.warn('Queue enqueue returned job already marked pending recovery; preserving status', {
+                    queue,
+                    driver: resolvedDriver,
+                    jobId,
+                    requestedUniqueId,
+                });
+                return jobId;
+            }
+            await markEnqueued({
+                queueName: queue,
+                jobId,
+                payload,
+                requestedUniqueId,
+            });
+            return jobId;
+        }
+        catch (error) {
+            const fallbackJobId = await markFailedEnqueue({
+                queueName: queue,
+                payload,
+                requestedUniqueId,
+                error,
+            });
+            Logger.warn('Queue enqueue failed', {
+                queue,
+                driver: resolvedDriver,
+                fallbackJobId,
+                requestedUniqueId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+        }
     },
     async dequeue(queue, driverName) {
-        const driver = Queue.get(driverName);
-        return driver.dequeue(queue);
+        return QueueTracing.traceOperation({
+            queueName: queue,
+            operation: 'dequeue',
+            attributes: {
+                driverName: driverName ?? null,
+            },
+            execute: async () => {
+                const driver = Queue.get(driverName);
+                return driver.dequeue(queue);
+            },
+        });
     },
     async ack(queue, id, driverName) {
-        const driver = Queue.get(driverName);
-        return driver.ack(queue, id);
+        return QueueTracing.traceOperation({
+            queueName: queue,
+            operation: 'ack',
+            attributes: {
+                driverName: driverName ?? null,
+            },
+            execute: async () => {
+                const driver = Queue.get(driverName);
+                await driver.ack(queue, id);
+            },
+        });
     },
     async length(queue, driverName) {
-        const driver = Queue.get(driverName);
-        return driver.length(queue);
+        return QueueTracing.traceOperation({
+            queueName: queue,
+            operation: 'length',
+            attributes: {
+                driverName: driverName ?? null,
+            },
+            execute: async () => {
+                const driver = Queue.get(driverName);
+                return driver.length(queue);
+            },
+        });
     },
     async drain(queue, driverName) {
-        const driver = Queue.get(driverName);
-        return driver.drain(queue);
+        return QueueTracing.traceOperation({
+            queueName: queue,
+            operation: 'drain',
+            attributes: {
+                driverName: driverName ?? null,
+            },
+            execute: async () => {
+                const driver = Queue.get(driverName);
+                await driver.drain(queue);
+            },
+        });
     },
 });
 export default Queue;

package/src/tools/queue/QueueDataRedactor.d.ts

@@ -0,0 +1,6 @@
+export declare const QueueDataRedactor: Readonly<{
+    sanitizePayload<T>(payload: T): T;
+    redactText(value: string): string;
+}>;
+export default QueueDataRedactor;
+//# sourceMappingURL=QueueDataRedactor.d.ts.map

package/src/tools/queue/QueueDataRedactor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"QueueDataRedactor.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueDataRedactor.ts"],"names":[],"mappings":"AA+CA,eAAO,MAAM,iBAAiB;oBACZ,CAAC,WAAW,CAAC,GAAG,CAAC;sBAIf,MAAM,GAAG,MAAM;EAGjC,CAAC;AAEH,eAAe,iBAAiB,CAAC"}

package/src/tools/queue/QueueDataRedactor.js

@@ -0,0 +1,45 @@
+const SENSITIVE_KEY_PATTERN = /pass(word)?|token|secret|api[_-]?key|auth|authorization|cookie|session|credential|private[_-]?key/i;
+const SECRET_VALUE_PATTERN = /(bearer\s+[a-z0-9._-]+|sk_[a-z0-9]{8,}|pk_[a-z0-9]{8,}|[a-f0-9]{32,})/gi;
+const redactString = (value) => {
+    if (value.trim() === '')
+        return value;
+    return value.replace(SECRET_VALUE_PATTERN, '[REDACTED]');
+};
+const sanitizeValue = (value) => {
+    if (value === null || value === undefined)
+        return value;
+    if (typeof value === 'string') {
+        return redactString(value);
+    }
+    if (typeof value === 'number' || typeof value === 'boolean') {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => sanitizeValue(item));
+    }
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    if (typeof value === 'object') {
+        const source = value;
+        const result = {};
+        for (const [key, val] of Object.entries(source)) {
+            if (SENSITIVE_KEY_PATTERN.test(key)) {
+                result[key] = '[REDACTED]';
+                continue;
+            }
+            result[key] = sanitizeValue(val);
+        }
+        return result;
+    }
+    return String(value);
+};
+export const QueueDataRedactor = Object.freeze({
+    sanitizePayload(payload) {
+        return sanitizeValue(payload);
+    },
+    redactText(value) {
+        return redactString(value);
+    },
+});
+export default QueueDataRedactor;

package/src/tools/queue/QueueExtensions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"QueueExtensions.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueExtensions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAsB,WAAW,EAAE,MAAM,eAAe,CAAC;AAKzF,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAKlD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAWrD;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,aAAa,EACtB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,MAAM,CAAC,CAOjB;AAED;;GAEG;AACH,wBAAgB,8BAA8B,IAAI,IAAI,CAuBrD;AAED;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAEvF;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE;IACvB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAkC3C,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,gBAAgB;IAC3B;;OAEG;uCAEgB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,aAC7B,MAAM,GAChB,kBAAkB;IAOrB;;OAEG;uCAEgB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,mBACvB,MAAM,QACjB,MAAM,GACX,kBAAkB;CAStB,CAAC"}
+{"version":3,"file":"QueueExtensions.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueExtensions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAsB,WAAW,EAAE,MAAM,eAAe,CAAC;AAKzF,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAKlD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAWrD;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,aAAa,EACtB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,MAAM,CAAC,CAQjB;AAED;;GAEG;AACH,wBAAgB,8BAA8B,IAAI,IAAI,CAuBrD;AAED;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAEvF;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE;IACvB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAkC3C,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,gBAAgB;IAC3B;;OAEG;uCAEgB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,aAC7B,MAAM,GAChB,kBAAkB;IAOrB;;OAEG;uCAEgB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,mBACvB,MAAM,QACjB,MAAM,GACX,kBAAkB;CAStB,CAAC"}

package/src/tools/queue/QueueExtensions.js

@@ -32,7 +32,8 @@ export function extendQueue(config) {
 export async function enqueueAdvanced(name, payload, options = {}) {
     if (advancedQueueRef === null) {
         Logger.warn(`Advanced queue not initialized, falling back to standard enqueue`);
-        return Queue.enqueue(name, payload);
+        const getQueue = await Queue?.enqueue(name, payload);
+        return getQueue ?? '';
     }
     return advancedQueueRef.enqueue(name, payload, options);
 }

package/src/tools/queue/QueueReliabilityMetrics.d.ts

@@ -0,0 +1,38 @@
+export type QueueReliabilitySnapshot = {
+    pending: number;
+    active: number;
+    completed: number;
+    failed: number;
+    stalled: number;
+    timeout: number;
+    pendingRecovery: number;
+    deadLetter: number;
+    manualReview: number;
+};
+export type QueueReliabilityAlertId = 'HighJobFailureRate' | 'StalledJobsAccumulating' | 'QueueDepthGrowing' | 'ManualReviewBacklog';
+export type QueueReliabilityAlert = {
+    id: QueueReliabilityAlertId;
+    severity: 'warning' | 'critical';
+    message: string;
+    value: number;
+    threshold: number;
+    runbook: string;
+};
+export type QueueReliabilityDashboardSnapshot = {
+    queueName: string;
+    generatedAt: string;
+    metrics: QueueReliabilitySnapshot & {
+        totalTracked: number;
+        queueDepth: number;
+        failureRate: number;
+    };
+    alerts: QueueReliabilityAlert[];
+    runbooks: Record<QueueReliabilityAlertId, string>;
+};
+export declare const QueueReliabilityMetrics: Readonly<{
+    snapshot(queueName?: string): QueueReliabilitySnapshot;
+    runbookMap(): Record<QueueReliabilityAlertId, string>;
+    dashboardSnapshot(queueName?: string): Promise<QueueReliabilityDashboardSnapshot>;
+}>;
+export default QueueReliabilityMetrics;
+//# sourceMappingURL=QueueReliabilityMetrics.d.ts.map

package/src/tools/queue/QueueReliabilityMetrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"QueueReliabilityMetrics.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueReliabilityMetrics.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,wBAAwB,GAAG;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAC/B,oBAAoB,GACpB,yBAAyB,GACzB,mBAAmB,GACnB,qBAAqB,CAAC;AAE1B,MAAM,MAAM,qBAAqB,GAAG;IAClC,EAAE,EAAE,uBAAuB,CAAC;IAC5B,QAAQ,EAAE,SAAS,GAAG,UAAU,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,iCAAiC,GAAG;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,wBAAwB,GAAG;QAClC,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,EAAE,qBAAqB,EAAE,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;CACnD,CAAC;AAwFF,eAAO,MAAM,uBAAuB;yBACb,MAAM,GAAG,wBAAwB;kBAexC,MAAM,CAAC,uBAAuB,EAAE,MAAM,CAAC;kCAIjB,MAAM,GAAG,OAAO,CAAC,iCAAiC,CAAC;EA0CvF,CAAC;AAEH,eAAe,uBAAuB,CAAC"}

package/src/tools/queue/QueueReliabilityMetrics.js

@@ -0,0 +1,131 @@
+import { Env } from '../../config/env.js';
+import { JobStateTracker } from './JobStateTracker.js';
+import { Queue } from './Queue.js';
+const toNumber = (input, key) => input[key] ?? 0;
+const toRate = (failed, completed) => {
+    const denominator = failed + completed;
+    if (denominator <= 0)
+        return 0;
+    return failed / denominator;
+};
+const resolveRunbookBase = () => {
+    const base = Env.get('JOB_RUNBOOK_BASE_URL', '/docs/runbooks').trim();
+    if (base === '')
+        return '/docs/runbooks';
+    return base.endsWith('/') ? base.slice(0, -1) : base;
+};
+const getRunbookMap = () => {
+    const base = resolveRunbookBase();
+    return {
+        HighJobFailureRate: `${base}/high-job-failure-rate`,
+        StalledJobsAccumulating: `${base}/stalled-jobs-accumulating`,
+        QueueDepthGrowing: `${base}/queue-depth-growing`,
+        ManualReviewBacklog: `${base}/manual-review-backlog`,
+    };
+};
+const evaluateAlerts = (queueName, metrics) => {
+    const runbooks = getRunbookMap();
+    const alerts = [];
+    const failureThreshold = Math.max(0, Env.getFloat('JOB_ALERT_FAILURE_RATE_THRESHOLD', 0.1));
+    const stalledThreshold = Math.max(1, Env.getInt('JOB_ALERT_STALLED_THRESHOLD', 50));
+    const queueDepthThreshold = Math.max(1, Env.getInt('JOB_ALERT_QUEUE_DEPTH_THRESHOLD', 1000));
+    const manualReviewThreshold = Math.max(1, Env.getInt('JOB_ALERT_MANUAL_REVIEW_THRESHOLD', 10));
+    if (metrics.failureRate > failureThreshold) {
+        alerts.push({
+            id: 'HighJobFailureRate',
+            severity: 'critical',
+            message: `Queue ${queueName} failure rate is above threshold`,
+            value: Number(metrics.failureRate.toFixed(6)),
+            threshold: failureThreshold,
+            runbook: runbooks.HighJobFailureRate,
+        });
+    }
+    if (metrics.stalled > stalledThreshold) {
+        alerts.push({
+            id: 'StalledJobsAccumulating',
+            severity: 'critical',
+            message: `Queue ${queueName} stalled jobs exceed threshold`,
+            value: metrics.stalled,
+            threshold: stalledThreshold,
+            runbook: runbooks.StalledJobsAccumulating,
+        });
+    }
+    if (metrics.queueDepth > queueDepthThreshold) {
+        alerts.push({
+            id: 'QueueDepthGrowing',
+            severity: 'warning',
+            message: `Queue ${queueName} depth is above threshold`,
+            value: metrics.queueDepth,
+            threshold: queueDepthThreshold,
+            runbook: runbooks.QueueDepthGrowing,
+        });
+    }
+    if (metrics.manualReview > manualReviewThreshold) {
+        alerts.push({
+            id: 'ManualReviewBacklog',
+            severity: 'warning',
+            message: `Queue ${queueName} manual-review backlog is above threshold`,
+            value: metrics.manualReview,
+            threshold: manualReviewThreshold,
+            runbook: runbooks.ManualReviewBacklog,
+        });
+    }
+    return alerts;
+};
+export const QueueReliabilityMetrics = Object.freeze({
+    snapshot(queueName) {
+        const summary = JobStateTracker.getSummary(queueName);
+        return {
+            pending: toNumber(summary, 'pending'),
+            active: toNumber(summary, 'active'),
+            completed: toNumber(summary, 'completed'),
+            failed: toNumber(summary, 'failed'),
+            stalled: toNumber(summary, 'stalled'),
+            timeout: toNumber(summary, 'timeout'),
+            pendingRecovery: toNumber(summary, 'pending_recovery'),
+            deadLetter: toNumber(summary, 'dead_letter'),
+            manualReview: toNumber(summary, 'manual_review'),
+        };
+    },
+    runbookMap() {
+        return getRunbookMap();
+    },
+    async dashboardSnapshot(queueName) {
+        const normalizedQueueName = typeof queueName === 'string' ? queueName.trim() : '';
+        const resolvedQueueName = normalizedQueueName === ''
+            ? Env.get('JOB_DASHBOARD_DEFAULT_QUEUE', 'default')
+            : normalizedQueueName;
+        const metrics = this.snapshot(resolvedQueueName);
+        let queueDepth = 0;
+        try {
+            queueDepth = await Queue.length(resolvedQueueName);
+        }
+        catch {
+            queueDepth = 0;
+        }
+        const totalTracked = metrics.pending +
+            metrics.active +
+            metrics.completed +
+            metrics.failed +
+            metrics.stalled +
+            metrics.timeout +
+            metrics.pendingRecovery +
+            metrics.deadLetter +
+            metrics.manualReview;
+        const failureRate = toRate(metrics.failed + metrics.deadLetter, metrics.completed);
+        const derivedMetrics = {
+            ...metrics,
+            totalTracked,
+            queueDepth,
+            failureRate,
+        };
+        return {
+            queueName: resolvedQueueName,
+            generatedAt: new Date().toISOString(),
+            metrics: derivedMetrics,
+            alerts: evaluateAlerts(resolvedQueueName, derivedMetrics),
+            runbooks: getRunbookMap(),
+        };
+    },
+});
+export default QueueReliabilityMetrics;

package/src/tools/queue/QueueReliabilityOrchestrator.d.ts

@@ -0,0 +1,7 @@
+export declare const QueueReliabilityOrchestrator: Readonly<{
+    isEnabled(): boolean;
+    start(): void;
+    stop(): void;
+}>;
+export default QueueReliabilityOrchestrator;
+//# sourceMappingURL=QueueReliabilityOrchestrator.d.ts.map

package/src/tools/queue/QueueReliabilityOrchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"QueueReliabilityOrchestrator.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueReliabilityOrchestrator.ts"],"names":[],"mappings":"AAiCA,eAAO,MAAM,4BAA4B;iBAC1B,OAAO;aAIX,IAAI;YA4BL,IAAI;EAUZ,CAAC;AAEH,eAAe,4BAA4B,CAAC"}

package/src/tools/queue/QueueReliabilityOrchestrator.js

@@ -0,0 +1,59 @@
+import { Env } from '../../config/env.js';
+import { Logger } from '../../config/logger.js';
+import { JobReconciliationRunner } from './JobReconciliationRunner.js';
+import { JobRecoveryDaemon } from './JobRecoveryDaemon.js';
+import { StalledJobMonitor } from './StalledJobMonitor.js';
+const state = {
+    started: false,
+};
+const clearTimer = (timer) => {
+    if (timer === undefined)
+        return;
+    clearInterval(timer);
+};
+const startInterval = (handler, intervalMs) => setInterval(() => {
+    handler().catch((error) => {
+        Logger.warn('Queue reliability interval failed', {
+            error: error instanceof Error ? error : String(error),
+        });
+    });
+}, intervalMs);
+export const QueueReliabilityOrchestrator = Object.freeze({
+    isEnabled() {
+        return Env.getBool('JOB_RELIABILITY_ENABLED', true);
+    },
+    start() {
+        if (!this.isEnabled())
+            return;
+        if (state.started)
+            return;
+        const reconciliationMs = Math.max(5000, Env.getInt('JOB_RECONCILIATION_INTERVAL_MS', 60000));
+        const recoveryMs = Math.max(5000, Env.getInt('JOB_RECOVERY_INTERVAL_MS', 30000));
+        const stalledMs = Math.max(5000, Env.getInt('STALLED_JOB_CHECK_INTERVAL_MS', 30000));
+        state.reconciliationTimer = startInterval(async () => {
+            await JobReconciliationRunner.runOnce();
+        }, reconciliationMs);
+        state.recoveryTimer = startInterval(async () => {
+            await JobRecoveryDaemon.runOnce();
+        }, recoveryMs);
+        state.stalledTimer = startInterval(async () => {
+            await StalledJobMonitor.scanOnce();
+        }, stalledMs);
+        state.started = true;
+        Logger.info('Queue reliability orchestrator started', {
+            reconciliationMs,
+            recoveryMs,
+            stalledMs,
+        });
+    },
+    stop() {
+        clearTimer(state.reconciliationTimer);
+        clearTimer(state.recoveryTimer);
+        clearTimer(state.stalledTimer);
+        state.reconciliationTimer = undefined;
+        state.recoveryTimer = undefined;
+        state.stalledTimer = undefined;
+        state.started = false;
+    },
+});
+export default QueueReliabilityOrchestrator;

package/src/tools/queue/QueueRuntimeRegistration.d.ts

@@ -1,11 +1,3 @@
 import type { QueueConfig } from '../../config/queue';
-/**
- * Register queue drivers from runtime config.
- *
- * This follows the framework's config-driven availability pattern:
- * - Built-in drivers are registered so `QUEUE_DRIVER=sync|inmemory|redis` works out of the box.
- * - If the configured default is registered, it is ALSO registered as 'default'.
- * - Unknown/unregistered driver names still throw when selected.
- */
-export declare function registerQueuesFromRuntimeConfig(config: QueueConfig): void;
+export declare function registerQueuesFromRuntimeConfig(config: QueueConfig): Promise<void>;
 //# sourceMappingURL=QueueRuntimeRegistration.d.ts.map

package/src/tools/queue/QueueRuntimeRegistration.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"QueueRuntimeRegistration.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueRuntimeRegistration.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAOjD;;;;;;;GAOG;AACH,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAazE"}
+{"version":3,"file":"QueueRuntimeRegistration.d.ts","sourceRoot":"","sources":["../../../../src/tools/queue/QueueRuntimeRegistration.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAoFjD,wBAAsB,+BAA+B,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAyCxF"}

package/src/tools/queue/QueueRuntimeRegistration.js

@@ -1,5 +1,14 @@
+import { Logger } from '../../config/logger.js';
+import { Env } from '../../config/env.js';
 import { ErrorFactory } from '../../exceptions/ZintrustError.js';
+import { ZintrustLang } from '../../lang/lang.js';
+import { existsSync } from '../../node-singletons/fs.js';
+import * as path from '../../node-singletons/path.js';
+import { pathToFileURL } from '../../node-singletons/url.js';
+import { detectRuntime } from '../../runtime/detectRuntime.js';
 import { DatabaseQueue } from '../queue/drivers/Database.js';
+import { autoRegisterJobStateTrackerPersistenceFromEnv } from '../queue/JobStateTrackerDbPersistence.js';
+import { QueueReliabilityOrchestrator } from '../queue/QueueReliabilityOrchestrator.js';
 import { InMemoryQueue } from '../queue/drivers/InMemory.js';
 import { Queue } from '../queue/Queue.js';
 /**
@@ -10,16 +19,78 @@ import { Queue } from '../queue/Queue.js';
  * - If the configured default is registered, it is ALSO registered as 'default'.
  * - Unknown/unregistered driver names still throw when selected.
  */
-export function registerQueuesFromRuntimeConfig(config) {
+const registerRedisDriverIfAvailable = async () => {
+    try {
+        const mod = (await import('@zintrust/queue-redis'));
+        if (mod.RedisQueue !== undefined) {
+            Queue.register('redis', mod.RedisQueue);
+            return true;
+        }
+        if (mod.BullMQRedisQueue !== undefined) {
+            Queue.register('redis', mod.BullMQRedisQueue);
+            return true;
+        }
+    }
+    catch {
+        // Fall back to local dist build output when running inside the core repo Docker image.
+        // In that environment, `@zintrust/queue-redis` is not installed in node_modules,
+        // but the compiled package is available at `dist/packages/queue-redis`.
+        try {
+            const cwd = typeof process !== 'undefined' && typeof process.cwd === 'function' ? process.cwd() : '';
+            if (cwd.trim() === '')
+                return false;
+            const localEntry = path.join(cwd, 'dist', 'packages', 'queue-redis', 'src', 'index.js');
+            if (!existsSync(localEntry))
+                return false;
+            const url = pathToFileURL(localEntry).href;
+            const localMod = (await import(url));
+            if (localMod.RedisQueue !== undefined) {
+                Queue.register('redis', localMod.RedisQueue);
+                return true;
+            }
+            if (localMod.BullMQRedisQueue !== undefined) {
+                Queue.register('redis', localMod.BullMQRedisQueue);
+                return true;
+            }
+        }
+        catch {
+            // ignore
+        }
+        return false;
+    }
+    return false;
+};
+export async function registerQueuesFromRuntimeConfig(config) {
+    autoRegisterJobStateTrackerPersistenceFromEnv();
     // Built-in drivers (core)
     Queue.register('inmemory', InMemoryQueue);
-    Queue.register('db', DatabaseQueue);
+    Queue.register(ZintrustLang.DATABASE, DatabaseQueue);
     // Project templates use QUEUE_DRIVER=sync; treat this as an alias of in-memory.
     Queue.register('sync', InMemoryQueue);
     const defaultName = (config.default ?? '').toString().trim().toLowerCase();
     if (defaultName.length === 0) {
         throw ErrorFactory.createConfigError('Queue default driver is not configured');
     }
-    const drv = Queue.get(defaultName);
-    Queue.register('default', drv);
+    if (defaultName === 'redis') {
+        const registered = await registerRedisDriverIfAvailable();
+        if (!registered) {
+            throw ErrorFactory.createConfigError('Redis queue driver is not registered. Install queue:redis via zin plugin install.');
+        }
+    }
+    try {
+        const drv = Queue.get(defaultName);
+        Queue.register('default', drv);
+        if (Env.getBool('JOB_RELIABILITY_AUTOSTART', false)) {
+            QueueReliabilityOrchestrator.start();
+        }
+    }
+    catch (error) {
+        const { isCloudflare } = detectRuntime();
+        if (isCloudflare) {
+            Logger.warn(`[queue] Default driver '${defaultName}' is unavailable in Cloudflare runtime; falling back to 'sync'.`);
+            Queue.register('default', Queue.get('sync'));
+            return;
+        }
+        throw ErrorFactory.createConfigError('Queue default driver is not available', error);
+    }
 }