@vorionsys/platform-core 0.1.0

package/dist/intent/queues.js

@@ -0,0 +1,1393 @@
+import { Queue, Worker, QueueEvents } from 'bullmq';
+import { randomUUID } from 'node:crypto';
+import { SpanStatusCode } from '@opentelemetry/api';
+import { secureRandomInt } from '../common/random.js';
+import { TrustEngine } from '../trust-engine/index.js';
+import { RuleEvaluator } from '../basis/evaluator.js';
+import { createEnforcementService } from '../enforce/index.js';
+import { createLogger, createTracedLogger } from '../common/logger.js';
+import { getRedis } from '../common/redis.js';
+import { getConfig } from '../common/config.js';
+import { getTraceContext, runWithTraceContext, createTraceContext, addTraceToJobData, extractTraceFromJobData, } from '../common/trace.js';
+import { createSystemTenantContext, } from '../common/tenant-context.js';
+import { recordJobResult, updateQueueGauges, dlqSize, recordPolicyEvaluation, recordPolicyOverride, recordCircuitBreakerStateChange, recordCircuitBreakerExecution, } from './metrics.js';
+import { getCircuitBreaker, } from '../common/circuit-breaker.js';
+import { getPolicyLoader, createPolicyEvaluator, } from '../policy/index.js';
+import { createAuditService, createAuditHelper, } from '../audit/index.js';
+import { createWebhookService } from './webhooks.js';
+import { getCachedTrustScore, cacheTrustScore, } from '../common/trust-cache.js';
+import { tracePolicyEvaluate, recordPolicyEvaluationResult, getTracer, } from './tracing.js';
+import { createProofService } from '../proof/index.js';
+import { createGateway, } from '../cognigate/index.js';
+import { getSemanticGovernanceIntegration, } from '../semantic-governance/index.js';
+const logger = createLogger({ component: 'intent-queue' });
+// Singleton Cognigate gateway instance
+let cognigateGateway = null;
+/**
+ * Get or create the Cognigate gateway instance
+ */
+function getCognigateGateway() {
+    if (!cognigateGateway) {
+        const config = getConfig();
+        cognigateGateway = createGateway({
+            timeoutMs: config.cognigate.timeout,
+            maxMemoryMb: config.cognigate.maxMemoryMb,
+            maxCpuPercent: config.cognigate.maxCpuPercent,
+        });
+        logger.info('Cognigate gateway initialized');
+    }
+    return cognigateGateway;
+}
+const queueNames = {
+    intake: 'intent:intake',
+    evaluate: 'intent:evaluate',
+    decision: 'intent:decision',
+    execute: 'intent:execute',
+    deadLetter: 'intent:dead-letter',
+};
+/**
+ * Get job options from config with jitter for backoff
+ */
+function getJobOptions() {
+    const config = getConfig();
+    const baseDelay = config.intent.retryBackoffMs;
+    // Add 0-25% jitter to prevent thundering herd
+    const jitter = secureRandomInt(0, Math.floor(baseDelay * 0.25) + 1);
+    return {
+        attempts: config.intent.maxRetries,
+        backoff: {
+            type: 'exponential',
+            delay: baseDelay + jitter,
+        },
+        removeOnComplete: {
+            age: 3600, // Keep completed jobs for 1 hour
+            count: 1000,
+        },
+        removeOnFail: false, // Keep failed jobs for inspection
+    };
+}
+// Redis connection pool for queue infrastructure
+// Instead of duplicating connections per queue/worker, we reuse a pool of connections
+const connectionPool = new Map();
+const MAX_POOL_CONNECTIONS = 10;
+let connectionIndex = 0;
+/**
+ * Get a pooled Redis connection for queue infrastructure.
+ * Reuses connections to prevent excessive connection proliferation.
+ * BullMQ queues and workers can share connections safely.
+ */
+function connection() {
+    const poolKey = `queue-conn-${connectionIndex % MAX_POOL_CONNECTIONS}`;
+    if (!connectionPool.has(poolKey)) {
+        const conn = getRedis().duplicate();
+        connectionPool.set(poolKey, conn);
+        logger.debug({ poolKey, poolSize: connectionPool.size }, 'Created new pooled Redis connection');
+    }
+    connectionIndex++;
+    return connectionPool.get(poolKey);
+}
+/**
+ * Close all pooled connections during shutdown
+ */
+export async function closePooledConnections() {
+    const closePromises = [];
+    const entries = Array.from(connectionPool.entries());
+    for (let i = 0; i < entries.length; i++) {
+        const [key, conn] = entries[i];
+        closePromises.push(conn.quit()
+            .then(() => {
+            logger.debug({ key }, 'Closed pooled connection');
+        })
+            .catch((error) => {
+            logger.warn({ key, error }, 'Error closing pooled connection');
+        }));
+    }
+    await Promise.all(closePromises);
+    connectionPool.clear();
+    connectionIndex = 0;
+    logger.info('All pooled Redis connections closed');
+}
+const trustEngine = new TrustEngine();
+const policyLoader = getPolicyLoader();
+const policyEvaluator = createPolicyEvaluator();
+const ruleEvaluator = new RuleEvaluator();
+const enforcementService = createEnforcementService({});
+const auditService = createAuditService();
+const auditHelper = createAuditHelper(auditService);
+const webhookService = createWebhookService();
+// Policy evaluation circuit breaker - prevents cascading failures when policy engine is unhealthy
+// Uses per-service circuit breaker configuration from the registry
+const policyCircuitBreaker = getCircuitBreaker('policyEngine', (from, to) => {
+    recordCircuitBreakerStateChange('policy-engine', from, to);
+    logger.info({ circuitBreaker: 'policy-engine', from, to }, 'Policy circuit breaker state changed');
+});
+/**
+ * Action priority map - lower number = more restrictive
+ */
+const ACTION_PRIORITY = {
+    deny: 1,
+    escalate: 2,
+    limit: 3,
+    constrain: 4,
+    monitor: 5,
+    terminate: 6,
+    allow: 7,
+};
+/**
+ * Convert policy action interface to control action string
+ */
+function policyActionToControlAction(action) {
+    return action.action;
+}
+/**
+ * Determine the most restrictive action between rule and policy evaluations
+ * Returns the more restrictive action (deny > escalate > limit > monitor > terminate > allow)
+ */
+function getMostRestrictiveAction(ruleAction, policyAction) {
+    if (!policyAction) {
+        return ruleAction;
+    }
+    const rulePriority = ACTION_PRIORITY[ruleAction];
+    const policyPriority = ACTION_PRIORITY[policyAction];
+    // Return the action with lower priority number (more restrictive)
+    return rulePriority <= policyPriority
+        ? ruleAction
+        : policyAction;
+}
+// Queue instances
+export const intentIntakeQueue = new Queue(queueNames.intake, {
+    connection: connection(),
+    defaultJobOptions: getJobOptions(),
+});
+export const intentEvaluateQueue = new Queue(queueNames.evaluate, {
+    connection: connection(),
+    defaultJobOptions: getJobOptions(),
+});
+export const intentDecisionQueue = new Queue(queueNames.decision, {
+    connection: connection(),
+    defaultJobOptions: getJobOptions(),
+});
+export const intentExecuteQueue = new Queue(queueNames.execute, {
+    connection: connection(),
+    defaultJobOptions: getJobOptions(),
+});
+export const deadLetterQueue = new Queue(queueNames.deadLetter, {
+    connection: connection(),
+    defaultJobOptions: {
+        removeOnComplete: false,
+        removeOnFail: false,
+    },
+});
+// Queue event listeners for monitoring
+const queueEvents = [];
+[queueNames.intake, queueNames.evaluate, queueNames.decision, queueNames.execute].forEach((name) => {
+    const events = new QueueEvents(name, { connection: connection() });
+    queueEvents.push(events);
+    events.on('failed', ({ jobId, failedReason }) => {
+        logger.error({ queue: name, jobId, failedReason }, 'Job failed');
+    });
+    events.on('stalled', ({ jobId }) => {
+        logger.warn({ queue: name, jobId }, 'Job stalled');
+    });
+});
+// Worker instances for graceful shutdown
+const workers = [];
+let workersStarted = false;
+let intentService = null;
+let isShuttingDown = false;
+/**
+ * Move a failed job to the dead letter queue after max retries
+ */
+async function moveToDeadLetterQueue(job, error, stage) {
+    try {
+        await deadLetterQueue.add('failed-intent', {
+            originalQueue: stage,
+            jobId: job.id,
+            jobData: job.data,
+            error: {
+                message: error.message,
+                stack: error.stack,
+            },
+            attemptsMade: job.attemptsMade,
+            failedAt: new Date().toISOString(),
+        });
+        logger.info({ jobId: job.id, stage }, 'Moved failed job to dead letter queue');
+    }
+    catch (dlqError) {
+        logger.error({ jobId: job.id, stage, error: dlqError }, 'Failed to move job to dead letter queue');
+    }
+}
+/**
+ * Update intent status to failed when job exhausts retries
+ */
+async function markIntentFailed(intentId, tenantId, error) {
+    if (!intentService)
+        return;
+    // Create system context for background job processing
+    const ctx = createSystemTenantContext(tenantId, { userId: 'system:intent-queue-worker' });
+    try {
+        await intentService.updateStatus(ctx, intentId, 'failed');
+        await intentService.recordEvaluation(ctx, intentId, {
+            stage: 'error',
+            error: {
+                message: error.message,
+                timestamp: new Date().toISOString(),
+            },
+        });
+        // Record failed intent audit event
+        await auditHelper.recordIntentEvent(tenantId, 'intent.failed', intentId, { type: 'service', id: 'intent-queue' }, {
+            outcome: 'failure',
+            reason: error.message,
+            stateChange: {
+                after: { status: 'failed' },
+            },
+            metadata: {
+                errorStack: error.stack,
+            },
+        });
+    }
+    catch (updateError) {
+        logger.error({ intentId, error: updateError }, 'Failed to mark intent as failed');
+    }
+}
+export async function enqueueIntentSubmission(intent, options) {
+    // Get current trace context or create new one
+    const traceContext = options?.traceContext ?? getTraceContext() ?? createTraceContext();
+    const jobData = addTraceToJobData({
+        intentId: intent.id,
+        tenantId: intent.tenantId,
+        namespace: options?.namespace,
+    }, traceContext);
+    await intentIntakeQueue.add('intent.submitted', jobData);
+    logger.debug({ intentId: intent.id, traceId: traceContext.traceId }, 'Intent enqueued for processing');
+}
+export function registerIntentWorkers(service) {
+    if (workersStarted)
+        return;
+    workersStarted = true;
+    intentService = service;
+    const config = getConfig();
+    const concurrency = config.intent.queueConcurrency;
+    const lockDuration = config.intent.jobTimeoutMs;
+    // Intake Worker
+    const intakeWorker = new Worker(queueNames.intake, async (job) => {
+        const startTime = Date.now();
+        if (!intentService || isShuttingDown)
+            return;
+        // Extract trace context from job data or create new one
+        const traceContext = extractTraceFromJobData(job.data) ?? createTraceContext();
+        // Run the job within the trace context
+        return runWithTraceContext(traceContext, async () => {
+            const { intentId, tenantId, namespace } = job.data;
+            // Create system context for background job processing
+            const ctx = createSystemTenantContext(tenantId, { userId: 'system:intake-worker' });
+            const jobLogger = createTracedLogger({ component: 'intake-worker', jobId: job.id }, traceContext.traceId, traceContext.spanId);
+            try {
+                const intent = await intentService.get(ctx, intentId);
+                if (!intent) {
+                    jobLogger.warn({ intentId }, 'Intent no longer exists');
+                    return;
+                }
+                // Check cache first, then fall back to trust engine
+                let trustRecord = await getCachedTrustScore(intent.entityId, tenantId);
+                if (!trustRecord) {
+                    // Cache miss - fetch from trust engine
+                    trustRecord = await trustEngine.getScore(intent.entityId, ctx) ?? null;
+                    // Cache the result for future lookups
+                    if (trustRecord) {
+                        await cacheTrustScore(intent.entityId, tenantId, trustRecord);
+                    }
+                }
+                const trustSnapshot = trustRecord
+                    ? {
+                        score: trustRecord.score,
+                        level: trustRecord.level,
+                        components: trustRecord.components,
+                    }
+                    : null;
+                await intentService.updateTrustMetadata(ctx, intent.id, trustSnapshot, trustRecord?.level, trustRecord?.score);
+                await intentService.updateStatus(ctx, intent.id, 'evaluating', 'pending');
+                await intentService.recordEvaluation(ctx, intent.id, {
+                    stage: 'trust-snapshot',
+                    result: trustSnapshot,
+                });
+                // Record audit event for intent submission
+                await auditHelper.recordIntentEvent(tenantId, 'intent.submitted', intentId, { type: 'agent', id: intent.entityId, name: intent.metadata['agentName'] }, {
+                    outcome: 'success',
+                    metadata: {
+                        intentType: intent.intentType,
+                        goal: intent.goal,
+                        trustScore: trustRecord?.score,
+                        trustLevel: trustRecord?.level,
+                        namespace,
+                    },
+                }).catch((auditError) => {
+                    jobLogger.warn({ error: auditError }, 'Failed to record intake audit event');
+                });
+                // Propagate trace context to next queue
+                const nextJobData = addTraceToJobData({
+                    intentId: intent.id,
+                    tenantId: intent.tenantId,
+                    namespace,
+                }, traceContext);
+                await intentEvaluateQueue.add('intent.evaluate', nextJobData);
+                recordJobResult('intake', 'success', (Date.now() - startTime) / 1000);
+            }
+            catch (error) {
+                recordJobResult('intake', 'failure', (Date.now() - startTime) / 1000);
+                throw error;
+            }
+        });
+    }, {
+        connection: connection(),
+        concurrency,
+        lockDuration,
+        stalledInterval: lockDuration + 5000,
+    });
+    intakeWorker.on('failed', async (job, error) => {
+        if (!job)
+            return;
+        const { intentId, tenantId } = job.data;
+        logger.error({ jobId: job.id, intentId, error: error.message }, 'Intent intake job failed');
+        // Move to DLQ after final retry
+        if (job.attemptsMade >= config.intent.maxRetries) {
+            await moveToDeadLetterQueue(job, error, 'intake');
+            await markIntentFailed(intentId, tenantId, error);
+        }
+    });
+    workers.push(intakeWorker);
+    // Evaluate Worker
+    const evaluateWorker = new Worker(queueNames.evaluate, async (job) => {
+        const startTime = Date.now();
+        if (!intentService || isShuttingDown)
+            return;
+        const { intentId, tenantId, namespace } = job.data;
+        // Create system context for background job processing
+        const ctx = createSystemTenantContext(tenantId, { userId: 'system:evaluate-worker' });
+        try {
+            const intent = await intentService.get(ctx, intentId);
+            if (!intent) {
+                logger.warn({ intentId }, 'Intent no longer exists');
+                return;
+            }
+            // Record evaluation started audit event
+            auditHelper.recordIntentEvent(tenantId, 'intent.evaluation.started', intentId, { type: 'service', id: 'intent-queue' }, {
+                outcome: 'success',
+                metadata: {
+                    intentType: intent.intentType,
+                    namespace,
+                },
+            }).catch((auditError) => {
+                logger.warn({ error: auditError }, 'Failed to record evaluation started audit event');
+            });
+            // Build evaluation context (shared by rule and policy evaluators)
+            const evaluationContext = {
+                intent: {
+                    id: intent.id,
+                    type: intent.intentType ?? 'generic',
+                    goal: intent.goal,
+                    context: intent.context,
+                },
+                entity: {
+                    id: intent.entityId,
+                    type: intent.metadata['entityType'] ?? 'agent',
+                    trustScore: intent.trustScore ?? 0,
+                    trustLevel: (intent.trustLevel ?? 0),
+                    attributes: intent.metadata,
+                },
+                environment: {
+                    timestamp: new Date().toISOString(),
+                    timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+                    requestId: randomUUID(),
+                },
+                custom: {
+                    namespace: namespace ?? config.intent.defaultNamespace,
+                },
+            };
+            // PARALLEL EVALUATION: Run rule and policy evaluation concurrently
+            // This roughly halves evaluation latency by running both evaluations at the same time
+            const policyNamespace = namespace ?? config.intent.defaultNamespace;
+            const ruleEvalStartTime = Date.now();
+            const policyEvalStartTime = Date.now();
+            // Build policy evaluation context (slightly different structure than rule context)
+            const policyContext = {
+                intent: {
+                    ...intent,
+                    intentType: intent.intentType ?? 'generic',
+                },
+                entity: {
+                    id: intent.entityId,
+                    type: intent.metadata['entityType'] ?? 'agent',
+                    trustScore: intent.trustScore ?? 0,
+                    trustLevel: (intent.trustLevel ?? 0),
+                    attributes: intent.metadata,
+                },
+                environment: {
+                    timestamp: new Date().toISOString(),
+                    timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+                    requestId: randomUUID(),
+                },
+            };
+            // Run rule evaluation, policy evaluation, and semantic governance in parallel
+            const semanticGovernance = getSemanticGovernanceIntegration();
+            const [evaluation, policyResult, semanticResult] = await Promise.all([
+                // Rule-based evaluation (existing system)
+                ruleEvaluator.evaluate(evaluationContext).then((result) => {
+                    const ruleEvalDuration = Date.now() - ruleEvalStartTime;
+                    logger.debug({
+                        intentId: intent.id,
+                        rulesEvaluated: result.rulesEvaluated.length,
+                        passed: result.passed,
+                        finalAction: result.finalAction,
+                        durationMs: ruleEvalDuration,
+                    }, 'Rule evaluation completed');
+                    return result;
+                }),
+                // Policy-based evaluation (new policy engine) with circuit breaker and tracing
+                // Circuit breaker prevents cascading failures when policy engine is unhealthy
+                (async () => {
+                    // Check if circuit is open - if so, skip policy evaluation immediately
+                    const isCircuitOpen = await policyCircuitBreaker.isOpen();
+                    if (isCircuitOpen) {
+                        recordCircuitBreakerExecution('policy-engine', 'rejected');
+                        logger.warn({ intentId: intent.id, circuitState: 'OPEN' }, 'Policy circuit breaker is OPEN, skipping policy evaluation and using rules only');
+                        return { evaluation: null, circuitOpen: true };
+                    }
+                    return tracePolicyEvaluate(intent.id, intent.tenantId, policyNamespace, async (span) => {
+                        // Execute policy evaluation through circuit breaker
+                        const circuitResult = await policyCircuitBreaker.execute(async () => {
+                            const policies = await policyLoader.getPolicies(intent.tenantId, policyNamespace);
+                            if (policies.length === 0) {
+                                recordPolicyEvaluationResult(span, 0, 0, 'none');
+                                return null;
+                            }
+                            const policyEvaluation = await policyEvaluator.evaluateMultiple(policies, policyContext);
+                            // Record policy evaluation metrics
+                            const policyEvalDuration = (Date.now() - policyEvalStartTime) / 1000;
+                            const matchedCount = policyEvaluation.policiesEvaluated.filter((p) => p.matchedRules.length > 0).length;
+                            const policyResultType = policyEvaluation.finalAction === 'allow'
+                                ? 'allow'
+                                : policyEvaluation.finalAction === 'deny'
+                                    ? 'deny'
+                                    : 'escalate';
+                            // Record span attributes for policy evaluation
+                            recordPolicyEvaluationResult(span, policyEvaluation.policiesEvaluated.length, matchedCount, policyResultType);
+                            recordPolicyEvaluation(intent.tenantId, policyNamespace, policyResultType, policyEvalDuration, matchedCount);
+                            logger.debug({
+                                intentId: intent.id,
+                                policiesEvaluated: policyEvaluation.policiesEvaluated.length,
+                                finalAction: policyEvaluation.finalAction,
+                                durationMs: Date.now() - policyEvalStartTime,
+                            }, 'Policy evaluation completed');
+                            return policyEvaluation;
+                        });
+                        // Handle circuit breaker result
+                        if (circuitResult.success) {
+                            recordCircuitBreakerExecution('policy-engine', 'success');
+                            return { evaluation: circuitResult.result ?? null };
+                        }
+                        else if (circuitResult.circuitOpen) {
+                            // Circuit opened during execution (shouldn't happen as we check above)
+                            recordCircuitBreakerExecution('policy-engine', 'rejected');
+                            logger.warn({ intentId: intent.id }, 'Policy circuit breaker opened during evaluation, continuing with rules only');
+                            return { evaluation: null, circuitOpen: true };
+                        }
+                        else {
+                            // Execution failed - circuit breaker already recorded the failure
+                            recordCircuitBreakerExecution('policy-engine', 'failure');
+                            logger.warn({ intentId: intent.id, error: circuitResult.error }, 'Policy evaluation failed (circuit breaker tracked), continuing with rules only');
+                            return { evaluation: null, error: circuitResult.error };
+                        }
+                    });
+                })(),
+                // Semantic governance validation (Layer 5 - confused deputy protection)
+                // Runs in parallel with rule and policy evaluation
+                (async () => {
+                    if (!semanticGovernance.isEnabled()) {
+                        return { valid: true, signals: [], durationMs: 0 };
+                    }
+                    const semanticStartTime = Date.now();
+                    try {
+                        const result = await semanticGovernance.validateIntent({
+                            intentId: intent.id,
+                            tenantId: intent.tenantId,
+                            entityId: intent.entityId,
+                            intentType: intent.intentType ?? undefined,
+                            goal: intent.goal,
+                            context: intent.context ?? undefined,
+                            trustScore: intent.trustScore ?? undefined,
+                            trustLevel: intent.trustLevel ?? undefined,
+                            messageSource: 'intent-submission',
+                            authenticated: true,
+                        });
+                        logger.debug({
+                            intentId: intent.id,
+                            valid: result.valid,
+                            signalsCount: result.signals.length,
+                            durationMs: result.durationMs,
+                        }, 'Semantic governance validation completed');
+                        return result;
+                    }
+                    catch (error) {
+                        logger.warn({ intentId: intent.id, error }, 'Semantic governance validation error (non-blocking)');
+                        return {
+                            valid: true,
+                            signals: [{
+                                    type: 'semantic.validation.error',
+                                    value: -0.05,
+                                    weight: 0.5,
+                                    severity: 'medium',
+                                    source: 'semantic-governance',
+                                    metadata: { error: error instanceof Error ? error.message : String(error) },
+                                }],
+                            durationMs: Date.now() - semanticStartTime,
+                            warnings: [`Semantic validation error: ${error instanceof Error ? error.message : 'Unknown error'}`],
+                        };
+                    }
+                })(),
+            ]);
+            // Extract policy evaluation result (may be null if no policies or error)
+            const policyEvaluation = policyResult.evaluation;
+            // Log parallel evaluation timing metrics
+            const totalEvalDuration = Date.now() - ruleEvalStartTime;
+            logger.debug({
+                intentId: intent.id,
+                totalDurationMs: totalEvalDuration,
+                parallelExecution: true,
+                semanticValid: semanticResult.valid,
+                semanticSignals: semanticResult.signals.length,
+            }, 'Parallel evaluation completed');
+            // Record semantic signals for trust engine using batch processing
+            if (semanticResult.signals.length > 0) {
+                const timestamp = new Date().toISOString();
+                const signalPromises = semanticResult.signals.map((signal) => trustEngine.recordSignal({
+                    id: randomUUID(),
+                    entityId: intent.entityId,
+                    type: signal.type,
+                    value: signal.value,
+                    weight: signal.weight,
+                    source: signal.source,
+                    timestamp,
+                    metadata: signal.metadata,
+                }, ctx).catch((error) => {
+                    // Log individual signal failures without blocking other signals
+                    logger.warn({ error, signalType: signal.type, entityId: intent.entityId }, 'Failed to record semantic signal');
+                    return null; // Return null to indicate failure without throwing
+                }));
+                await Promise.all(signalPromises);
+            }
+            await intentService.recordEvaluation(ctx, intent.id, {
+                stage: 'basis',
+                evaluation,
+                namespace: namespace ?? config.intent.defaultNamespace,
+            });
+            // Record semantic governance evaluation
+            await intentService.recordEvaluation(ctx, intent.id, {
+                stage: 'semantic-governance',
+                result: {
+                    valid: semanticResult.valid,
+                    reason: semanticResult.reason,
+                    signalsEmitted: semanticResult.signals.length,
+                    durationMs: semanticResult.durationMs,
+                    warnings: semanticResult.warnings,
+                },
+            });
+            // Record evaluation completed audit event
+            auditHelper.recordIntentEvent(tenantId, 'intent.evaluation.completed', intentId, { type: 'service', id: 'intent-queue' }, {
+                outcome: 'success',
+                metadata: {
+                    rulesPassed: evaluation.passed,
+                    rulesAction: evaluation.finalAction,
+                    rulesEvaluated: evaluation.rulesEvaluated.length,
+                    policyAction: policyEvaluation?.finalAction,
+                    policiesEvaluated: policyEvaluation?.policiesEvaluated.length ?? 0,
+                    semanticValid: semanticResult.valid,
+                    semanticSignals: semanticResult.signals.length,
+                    namespace,
+                },
+            }).catch((auditError) => {
+                logger.warn({ error: auditError }, 'Failed to record evaluation completed audit event');
+            });
+            await intentDecisionQueue.add('intent.decision', {
+                intentId: intent.id,
+                tenantId: intent.tenantId,
+                evaluation,
+                policyEvaluation,
+                semanticValidation: {
+                    valid: semanticResult.valid,
+                    reason: semanticResult.reason,
+                    signals: semanticResult.signals,
+                    warnings: semanticResult.warnings,
+                },
+            });
+            recordJobResult('evaluate', 'success', (Date.now() - startTime) / 1000);
+        }
+        catch (error) {
+            recordJobResult('evaluate', 'failure', (Date.now() - startTime) / 1000);
+            throw error;
+        }
+    }, {
+        connection: connection(),
+        concurrency,
+        lockDuration,
+        stalledInterval: lockDuration + 5000,
+    });
+    evaluateWorker.on('failed', async (job, error) => {
+        if (!job)
+            return;
+        const { intentId, tenantId } = job.data;
+        logger.error({ jobId: job.id, intentId, error: error.message }, 'Intent evaluation job failed');
+        if (job.attemptsMade >= config.intent.maxRetries) {
+            await moveToDeadLetterQueue(job, error, 'evaluate');
+            await markIntentFailed(intentId, tenantId, error);
+        }
+    });
+    workers.push(evaluateWorker);
+    // Decision Worker
+    const decisionWorker = new Worker(queueNames.decision, async (job) => {
+        const startTime = Date.now();
+        if (!intentService || isShuttingDown)
+            return;
+        const { intentId, tenantId, evaluation, policyEvaluation, semanticValidation } = job.data;
+        // Create system context for background job processing
+        const ctx = createSystemTenantContext(tenantId, { userId: 'system:decision-worker' });
+        try {
+            const intent = await intentService.get(ctx, intentId);
+            if (!intent) {
+                logger.warn({ intentId }, 'Intent no longer exists');
+                return;
+            }
+            // Re-validate trust at decision stage if configured
+            if (config.intent.revalidateTrustAtDecision) {
+                const currentTrust = await trustEngine.getScore(intent.entityId, ctx);
+                const currentLevel = currentTrust?.level ?? 0;
+                const requiredLevel = intentService.getRequiredTrustLevel(intent.intentType);
+                // Record the trust gate check
+                await intentService.recordEvaluation(ctx, intent.id, {
+                    stage: 'trust-gate',
+                    passed: currentLevel >= requiredLevel,
+                    requiredLevel,
+                    actualLevel: currentLevel,
+                });
+                if (currentLevel < requiredLevel) {
+                    logger.warn({ intentId, requiredLevel, currentLevel }, 'Trust level degraded below requirement at decision stage');
+                    await intentService.updateStatus(ctx, intent.id, 'denied', 'evaluating');
+                    recordJobResult('decision', 'success', (Date.now() - startTime) / 1000);
+                    return;
+                }
+                // Update trust metadata if changed
+                if (currentTrust && currentTrust.level !== intent.trustLevel) {
+                    await intentService.updateTrustMetadata(ctx, intent.id, {
+                        score: currentTrust.score,
+                        level: currentTrust.level,
+                        components: currentTrust.components,
+                        revalidatedAt: new Date().toISOString(),
+                    }, currentTrust.level, currentTrust.score);
+                }
+            }
+            // Get rule-based enforcement decision
+            const ruleDecision = await enforcementService.decide({
+                intent,
+                evaluation,
+                tenantId: intent.tenantId,
+                trustScore: intent.trustScore ?? 0,
+                trustLevel: (intent.trustLevel ?? 0),
+            });
+            // Determine final action considering policy evaluation and semantic governance
+            let finalAction = ruleDecision.action;
+            let policyOverride = false;
+            let semanticOverride = false;
+            // First check semantic governance - if it failed, deny
+            if (semanticValidation && !semanticValidation.valid) {
+                finalAction = 'deny';
+                semanticOverride = true;
+                logger.warn({
+                    intentId: intent.id,
+                    ruleAction: ruleDecision.action,
+                    semanticReason: semanticValidation.reason,
+                    finalAction,
+                }, 'Semantic governance validation failed - denying intent');
+            }
+            else if (policyEvaluation && policyEvaluation.finalAction) {
+                const combinedAction = getMostRestrictiveAction(ruleDecision.action, policyEvaluation.finalAction);
+                if (combinedAction !== ruleDecision.action) {
+                    policyOverride = true;
+                    finalAction = combinedAction;
+                    // Record policy override metric
+                    recordPolicyOverride(intent.tenantId, ruleDecision.action, policyEvaluation.finalAction);
+                    logger.info({
+                        intentId: intent.id,
+                        ruleAction: ruleDecision.action,
+                        policyAction: policyEvaluation.finalAction,
+                        finalAction,
+                    }, 'Policy evaluation overrode rule decision');
+                }
+            }
+            // Record rule, policy, and semantic evaluations
+            await intentService.recordEvaluation(ctx, intent.id, {
+                stage: 'decision',
+                decision: {
+                    ruleDecision: {
+                        action: ruleDecision.action,
+                        constraintsEvaluated: ruleDecision.constraintsEvaluated,
+                    },
+                    policyDecision: policyEvaluation
+                        ? {
+                            action: policyEvaluation.finalAction,
+                            reason: policyEvaluation.reason,
+                            policiesEvaluated: policyEvaluation.policiesEvaluated.length,
+                            matchedPolicies: policyEvaluation.policiesEvaluated
+                                .filter((p) => p.matchedRules.length > 0)
+                                .map((p) => ({ policyId: p.policyId, action: p.action, reason: p.reason })),
+                        }
+                        : null,
+                    semanticDecision: semanticValidation
+                        ? {
+                            valid: semanticValidation.valid,
+                            reason: semanticValidation.reason,
+                            signalsCount: semanticValidation.signals.length,
+                            warnings: semanticValidation.warnings,
+                        }
+                        : null,
+                    finalAction,
+                    policyOverride,
+                    semanticOverride,
+                },
+            });
+            const nextStatus = finalAction === 'allow'
+                ? 'approved'
+                : finalAction === 'deny'
+                    ? 'denied'
+                    : finalAction === 'escalate'
+                        ? 'escalated'
+                        : 'pending';
+            await intentService.updateStatus(ctx, intent.id, nextStatus, 'evaluating');
+            // Trigger webhook notifications for approved/denied statuses (fire and forget)
+            if (nextStatus === 'approved') {
+                webhookService.notifyIntent('intent.approved', intentId, tenantId, { finalAction, policyOverride, semanticOverride }).catch((error) => {
+                    logger.warn({ error, intentId }, 'Failed to send webhook notification');
+                });
+                // Enqueue for execution
+                await intentExecuteQueue.add('intent.execute', {
+                    intentId: intent.id,
+                    tenantId: intent.tenantId,
+                    decision: ruleDecision,
+                });
+            }
+            else if (nextStatus === 'denied') {
+                // Determine reason - semantic override takes precedence
+                const reason = semanticOverride
+                    ? semanticValidation?.reason ?? 'Denied by semantic governance'
+                    : policyEvaluation?.reason ?? 'Denied by policy';
+                webhookService.notifyIntent('intent.denied', intentId, tenantId, { finalAction, reason, semanticOverride }).catch((error) => {
+                    logger.warn({ error, intentId }, 'Failed to send webhook notification');
+                });
+            }
+            // Note: 'escalated' status webhook is handled by escalation creation elsewhere
+            // Determine the audit event type based on final action
+            const auditEventType = finalAction === 'allow'
+                ? 'intent.approved'
+                : finalAction === 'deny'
+                    ? 'intent.denied'
+                    : 'intent.escalated';
+            // Record decision audit event
+            auditHelper.recordIntentEvent(tenantId, auditEventType, intentId, { type: 'service', id: 'intent-queue' }, {
+                outcome: 'success',
+                stateChange: {
+                    before: { status: 'evaluating' },
+                    after: { status: nextStatus },
+                },
+                metadata: {
+                    ruleAction: ruleDecision.action,
+                    policyAction: policyEvaluation?.finalAction,
+                    semanticValid: semanticValidation?.valid,
+                    semanticReason: semanticValidation?.reason,
+                    finalAction,
+                    policyOverride,
+                    semanticOverride,
+                    trustScore: intent.trustScore,
+                    trustLevel: intent.trustLevel,
+                },
+            }).catch((auditError) => {
+                logger.warn({ error: auditError }, 'Failed to record decision audit event');
+            });
+            recordJobResult('decision', 'success', (Date.now() - startTime) / 1000);
+        }
+        catch (error) {
+            recordJobResult('decision', 'failure', (Date.now() - startTime) / 1000);
+            throw error;
+        }
+    }, {
+        connection: connection(),
+        concurrency,
+        lockDuration,
+        stalledInterval: lockDuration + 5000,
+    });
+    decisionWorker.on('failed', async (job, error) => {
+        if (!job)
+            return;
+        const { intentId, tenantId } = job.data;
+        logger.error({ jobId: job.id, intentId, error: error.message }, 'Intent decision job failed');
+        if (job.attemptsMade >= config.intent.maxRetries) {
+            await moveToDeadLetterQueue(job, error, 'decision');
+            await markIntentFailed(intentId, tenantId, error);
+        }
+    });
+    workers.push(decisionWorker);
+    // Execution worker - processes approved intents
+    const tracer = getTracer();
+    const executionWorker = new Worker(queueNames.execute, async (job) => {
+        const startTime = Date.now();
+        if (!intentService || isShuttingDown)
+            return;
+        const { intentId, tenantId, decision } = job.data;
+        // Create system context for background job processing
+        const ctx = createSystemTenantContext(tenantId, { userId: 'system:execution-worker' });
+        const span = tracer.startSpan('intent.execution', {
+            attributes: { intentId, tenantId },
+        });
+        let intent = null;
+        try {
+            // Get intent
+            intent = await intentService.get(ctx, intentId);
+            if (!intent) {
+                logger.warn({ intentId }, 'Intent not found for execution');
+                return;
+            }
+            // Transition to executing
+            await intentService.updateStatus(ctx, intentId, 'executing', 'approved');
+            // Execute via Cognigate
+            const gateway = getCognigateGateway();
+            const executionContext = {
+                intent,
+                decision,
+                resourceLimits: {
+                    maxMemoryMb: 512,
+                    maxCpuPercent: 50,
+                    timeoutMs: 300000,
+                },
+            };
+            const executionResult = await gateway.execute(executionContext);
+            // Handle execution failure
+            if (!executionResult.success) {
+                logger.warn({ intentId, error: executionResult.error }, 'Cognigate execution failed');
+                // Record failure signal
+                await trustEngine.recordSignal({
+                    id: randomUUID(),
+                    entityId: intent.entityId,
+                    type: 'behavioral.execution_failure',
+                    value: -0.15,
+                    weight: 1.0,
+                    source: 'cognigate-execution',
+                    timestamp: new Date().toISOString(),
+                }, ctx);
+                // Update to failed
+                await intentService.updateStatus(ctx, intentId, 'failed', 'executing');
+                // Audit
+                await auditHelper.recordIntentEvent(tenantId, 'intent.failed', intentId, { type: 'service', id: 'cognigate-gateway' }, { outcome: 'failure', metadata: { error: executionResult.error, resourceUsage: executionResult.resourceUsage } });
+                span.setStatus({ code: SpanStatusCode.ERROR, message: executionResult.error });
+                recordJobResult('execution', 'failure', (Date.now() - startTime) / 1000);
+                return;
+            }
+            // Record trust signal for successful execution
+            await trustEngine.recordSignal({
+                id: randomUUID(),
+                entityId: intent.entityId,
+                type: 'behavioral.execution_success',
+                value: 0.1,
+                weight: 1.0,
+                source: 'cognigate-execution',
+                timestamp: new Date().toISOString(),
+            }, ctx);
+            // Create proof record
+            const proofService = createProofService();
+            await proofService.create({
+                intent,
+                decision,
+                inputs: intent.context ?? {},
+                outputs: executionResult.outputs,
+                tenantId, // SECURITY: Required for per-tenant lock isolation
+            });
+            // Update to completed
+            await intentService.updateStatus(ctx, intentId, 'completed', 'executing');
+            // Audit
+            await auditHelper.recordIntentEvent(tenantId, 'intent.completed', intentId, { type: 'service', id: 'cognigate-gateway' }, { outcome: 'success', metadata: { executionResult: { success: executionResult.success, outputs: executionResult.outputs, resourceUsage: executionResult.resourceUsage } } });
+            // Webhook
+            const webhookPayload = {
+                success: executionResult.success,
+                outputs: executionResult.outputs,
+                resourceUsage: executionResult.resourceUsage,
+                completedAt: executionResult.completedAt,
+            };
+            webhookService.notifyIntent('intent.completed', intentId, tenantId, webhookPayload).catch((err) => {
+                logger.warn({ err, intentId }, 'Failed to send completion webhook');
+            });
+            span.setStatus({ code: SpanStatusCode.OK });
+            recordJobResult('execution', 'success', (Date.now() - startTime) / 1000);
+        }
+        catch (error) {
+            span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+            logger.error({ error, intentId }, 'Execution failed');
+            // Record failure signal
+            await trustEngine.recordSignal({
+                id: randomUUID(),
+                entityId: intent?.entityId ?? 'unknown',
+                type: 'behavioral.execution_failure',
+                value: -0.2,
+                weight: 1.0,
+                source: 'intent-execution-worker',
+                timestamp: new Date().toISOString(),
+            }, ctx);
+            // Update to failed
+            await intentService.updateStatus(ctx, intentId, 'failed', 'executing');
+            recordJobResult('execution', 'failure', (Date.now() - startTime) / 1000);
+            throw error;
+        }
+        finally {
+            span.end();
+        }
+    }, {
+        connection: connection(),
+        concurrency,
+        lockDuration,
+        stalledInterval: lockDuration + 5000,
+    });
+    executionWorker.on('failed', async (job, error) => {
+        if (!job)
+            return;
+        const { intentId, tenantId } = job.data;
+        logger.error({ jobId: job.id, intentId, error: error.message }, 'Execution job failed');
+        if (job.attemptsMade >= config.intent.maxRetries) {
+            await moveToDeadLetterQueue(job, error, 'execution');
+        }
+    });
+    workers.push(executionWorker);
+    logger.info({ concurrency, lockDuration, maxRetries: config.intent.maxRetries }, 'Intent workers registered');
+}
+/**
+ * Gracefully shutdown all workers
+ * Waits for in-flight jobs to complete before closing
+ */
+export async function shutdownWorkers(timeoutMs = 30000) {
+    if (!workersStarted)
+        return;
+    isShuttingDown = true;
+    logger.info('Initiating graceful shutdown of intent workers');
+    const shutdownPromises = workers.map(async (worker) => {
+        try {
+            await worker.close();
+        }
+        catch (error) {
+            logger.error({ error }, 'Error closing worker');
+        }
+    });
+    // Close queue event listeners
+    const eventClosePromises = queueEvents.map(async (events) => {
+        try {
+            await events.close();
+        }
+        catch (error) {
+            logger.error({ error }, 'Error closing queue events');
+        }
+    });
+    // Wait for all with timeout
+    const allPromises = [...shutdownPromises, ...eventClosePromises];
+    await Promise.race([
+        Promise.all(allPromises),
+        new Promise((resolve) => {
+            setTimeout(() => {
+                logger.warn('Shutdown timeout reached, forcing close');
+                resolve();
+            }, timeoutMs);
+        }),
+    ]);
+    // Close pooled Redis connections
+    await closePooledConnections();
+    workersStarted = false;
+    isShuttingDown = false;
+    workers.length = 0;
+    queueEvents.length = 0;
+    logger.info('Intent workers shutdown complete');
+}
+/**
+ * Get queue health status and update Prometheus gauges
+ */
+export async function getQueueHealth() {
+    const [intakeCounts, evaluateCounts, decisionCounts, dlqCount] = await Promise.all([
+        intentIntakeQueue.getJobCounts('waiting', 'active', 'failed'),
+        intentEvaluateQueue.getJobCounts('waiting', 'active', 'failed'),
+        intentDecisionQueue.getJobCounts('waiting', 'active', 'failed'),
+        deadLetterQueue.getJobCounts('waiting'),
+    ]);
+    // Update Prometheus gauges
+    updateQueueGauges('intake', intakeCounts.waiting ?? 0, intakeCounts.active ?? 0);
+    updateQueueGauges('evaluate', evaluateCounts.waiting ?? 0, evaluateCounts.active ?? 0);
+    updateQueueGauges('decision', decisionCounts.waiting ?? 0, decisionCounts.active ?? 0);
+    dlqSize.set(dlqCount.waiting ?? 0);
+    return {
+        intake: {
+            waiting: intakeCounts.waiting ?? 0,
+            active: intakeCounts.active ?? 0,
+            failed: intakeCounts.failed ?? 0,
+        },
+        evaluate: {
+            waiting: evaluateCounts.waiting ?? 0,
+            active: evaluateCounts.active ?? 0,
+            failed: evaluateCounts.failed ?? 0,
+        },
+        decision: {
+            waiting: decisionCounts.waiting ?? 0,
+            active: decisionCounts.active ?? 0,
+            failed: decisionCounts.failed ?? 0,
+        },
+        deadLetter: {
+            count: dlqCount.waiting ?? 0,
+        },
+    };
+}
+/**
+ * Retry a job from the dead letter queue
+ */
+export async function retryDeadLetterJob(jobId) {
+    const job = await deadLetterQueue.getJob(jobId);
+    if (!job) {
+        logger.warn({ jobId }, 'Dead letter job not found');
+        return false;
+    }
+    const { originalQueue, jobData } = job.data;
+    // Re-enqueue to original queue
+    const targetQueue = originalQueue === 'intake'
+        ? intentIntakeQueue
+        : originalQueue === 'evaluate'
+            ? intentEvaluateQueue
+            : intentDecisionQueue;
+    await targetQueue.add('retry', jobData);
+    await job.remove();
+    logger.info({ jobId, originalQueue }, 'Dead letter job retried');
+    return true;
+}
+/**
+ * Get the policy evaluation circuit breaker status
+ * Useful for health checks and monitoring
+ */
+export async function getPolicyCircuitBreakerStatus() {
+    return policyCircuitBreaker.getStatus();
+}
+/**
+ * Force the policy circuit breaker to open state
+ * Useful for manual intervention during incidents
+ */
+export async function forcePolicyCircuitBreakerOpen() {
+    await policyCircuitBreaker.forceOpen();
+    logger.warn({}, 'Policy circuit breaker forcibly opened');
+}
+/**
+ * Force the policy circuit breaker to closed state
+ * Useful for recovery after manual intervention
+ */
+export async function forcePolicyCircuitBreakerClose() {
+    await policyCircuitBreaker.forceClose();
+    logger.info({}, 'Policy circuit breaker forcibly closed');
+}
+/**
+ * Reset the policy circuit breaker state
+ * Clears all failure counts and state
+ */
+export async function resetPolicyCircuitBreaker() {
+    await policyCircuitBreaker.reset();
+    logger.info({}, 'Policy circuit breaker reset');
+}
+// Canary queue for health checks
+let canaryQueue = null;
+let canaryWorker = null;
+const canaryResults = new Map();
+/**
+ * Initialize canary queue for health checks
+ */
+function initCanaryQueue() {
+    if (!canaryQueue) {
+        canaryQueue = new Queue('intent:canary', {
+            connection: connection(),
+            defaultJobOptions: {
+                removeOnComplete: { age: 60, count: 10 }, // Keep for 1 minute max
+                removeOnFail: { age: 60, count: 10 },
+                attempts: 1,
+            },
+        });
+        // Create a worker to process canary jobs
+        canaryWorker = new Worker('intent:canary', async (job) => {
+            const data = job.data;
+            const latencyMs = Date.now() - data.timestamp;
+            canaryResults.set(data.healthCheckId, {
+                success: true,
+                latencyMs,
+            });
+            return { processed: true, latencyMs };
+        }, {
+            connection: connection(),
+            concurrency: 1,
+        });
+        canaryWorker.on('failed', (job, error) => {
+            if (job) {
+                const data = job.data;
+                canaryResults.set(data.healthCheckId, {
+                    success: false,
+                    latencyMs: Date.now() - data.timestamp,
+                    error: error.message,
+                });
+            }
+        });
+        logger.debug('Canary queue initialized for health checks');
+    }
+    return canaryQueue;
+}
+/**
+ * Shutdown the canary queue and worker
+ */
+export async function shutdownCanaryQueue() {
+    if (canaryWorker) {
+        await canaryWorker.close();
+        canaryWorker = null;
+    }
+    if (canaryQueue) {
+        await canaryQueue.close();
+        canaryQueue = null;
+    }
+    canaryResults.clear();
+}
+/**
+ * Run a canary job to test actual job processing
+ * Returns the processing latency or null if the test times out
+ */
+async function runCanaryJob(timeoutMs = 5000) {
+    const queue = initCanaryQueue();
+    const healthCheckId = randomUUID();
+    const startTime = Date.now();
+    try {
+        // Add canary job
+        await queue.add('canary', {
+            type: 'canary',
+            timestamp: startTime,
+            healthCheckId,
+        });
+        // Wait for result with timeout
+        const pollInterval = 100;
+        let elapsed = 0;
+        while (elapsed < timeoutMs) {
+            const result = canaryResults.get(healthCheckId);
+            if (result) {
+                canaryResults.delete(healthCheckId);
+                return result;
+            }
+            await new Promise((resolve) => setTimeout(resolve, pollInterval));
+            elapsed += pollInterval;
+        }
+        // Timeout
+        return {
+            success: false,
+            latencyMs: timeoutMs,
+            error: 'Canary job timed out - workers may be unavailable or overloaded',
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            latencyMs: Date.now() - startTime,
+            error: error instanceof Error ? error.message : 'Unknown error',
+        };
+    }
+}
+/**
+ * Enhanced queue health check
+ *
+ * Performs comprehensive health check including:
+ * - Queue depth and job counts
+ * - Worker availability check via canary job
+ * - Processing latency measurement
+ * - Dead letter queue monitoring
+ *
+ * @param options.runCanary - Whether to run a canary job (default: true)
+ * @param options.canaryTimeoutMs - Timeout for canary job (default: 5000ms)
+ * @param options.maxQueueDepth - Maximum allowed queue depth before unhealthy (default: 10000)
+ */
+export async function checkQueueHealth(options) {
+    const config = getConfig();
+    const runCanary = options?.runCanary ?? true;
+    const canaryTimeoutMs = options?.canaryTimeoutMs ?? 5000;
+    const maxQueueDepth = options?.maxQueueDepth ?? config.intent.queueDepthThreshold ?? 10000;
+    // Get job counts from all queues in parallel
+    const [intakeCounts, evaluateCounts, decisionCounts, executeCounts, dlqCount,] = await Promise.all([
+        intentIntakeQueue.getJobCounts('waiting', 'active', 'failed', 'completed'),
+        intentEvaluateQueue.getJobCounts('waiting', 'active', 'failed', 'completed'),
+        intentDecisionQueue.getJobCounts('waiting', 'active', 'failed', 'completed'),
+        intentExecuteQueue.getJobCounts('waiting', 'active', 'failed', 'completed'),
+        deadLetterQueue.getJobCounts('waiting'),
+    ]);
+    // Calculate totals
+    const totalWaiting = (intakeCounts.waiting ?? 0) +
+        (evaluateCounts.waiting ?? 0) +
+        (decisionCounts.waiting ?? 0) +
+        (executeCounts.waiting ?? 0);
+    const totalActive = (intakeCounts.active ?? 0) +
+        (evaluateCounts.active ?? 0) +
+        (decisionCounts.active ?? 0) +
+        (executeCounts.active ?? 0);
+    const totalFailed = (intakeCounts.failed ?? 0) +
+        (evaluateCounts.failed ?? 0) +
+        (decisionCounts.failed ?? 0) +
+        (executeCounts.failed ?? 0);
+    const totalCompleted = (intakeCounts.completed ?? 0) +
+        (evaluateCounts.completed ?? 0) +
+        (decisionCounts.completed ?? 0) +
+        (executeCounts.completed ?? 0);
+    const queueDepth = totalWaiting + totalActive;
+    const deadLetterCount = dlqCount.waiting ?? 0;
+    // Update Prometheus gauges
+    updateQueueGauges('intake', intakeCounts.waiting ?? 0, intakeCounts.active ?? 0);
+    updateQueueGauges('evaluate', evaluateCounts.waiting ?? 0, evaluateCounts.active ?? 0);
+    updateQueueGauges('decision', decisionCounts.waiting ?? 0, decisionCounts.active ?? 0);
+    updateQueueGauges('execute', executeCounts.waiting ?? 0, executeCounts.active ?? 0);
+    dlqSize.set(deadLetterCount);
+    // Run canary job to test actual processing
+    let canaryJobResult;
+    let workersAvailable = workersStarted && !isShuttingDown;
+    let processingLatency = 0;
+    if (runCanary && workersAvailable) {
+        canaryJobResult = await runCanaryJob(canaryTimeoutMs);
+        workersAvailable = canaryJobResult.success;
+        processingLatency = canaryJobResult.latencyMs;
+    }
+    // Determine health status
+    const isHealthy = workersAvailable &&
+        queueDepth <= maxQueueDepth &&
+        !isShuttingDown;
+    const result = {
+        healthy: isHealthy,
+        activeJobs: totalActive,
+        waitingJobs: totalWaiting,
+        completedJobs: totalCompleted,
+        failedJobs: totalFailed,
+        processingLatency,
+        workersAvailable,
+        queueDepth,
+        deadLetterCount,
+        details: {
+            intake: {
+                waiting: intakeCounts.waiting ?? 0,
+                active: intakeCounts.active ?? 0,
+                failed: intakeCounts.failed ?? 0,
+                completed: intakeCounts.completed ?? 0,
+            },
+            evaluate: {
+                waiting: evaluateCounts.waiting ?? 0,
+                active: evaluateCounts.active ?? 0,
+                failed: evaluateCounts.failed ?? 0,
+                completed: evaluateCounts.completed ?? 0,
+            },
+            decision: {
+                waiting: decisionCounts.waiting ?? 0,
+                active: decisionCounts.active ?? 0,
+                failed: decisionCounts.failed ?? 0,
+                completed: decisionCounts.completed ?? 0,
+            },
+            execute: {
+                waiting: executeCounts.waiting ?? 0,
+                active: executeCounts.active ?? 0,
+                failed: executeCounts.failed ?? 0,
+                completed: executeCounts.completed ?? 0,
+            },
+            deadLetter: {
+                count: deadLetterCount,
+            },
+        },
+        ...(canaryJobResult && { canaryJobResult }),
+    };
+    if (!isHealthy) {
+        logger.warn({
+            healthy: isHealthy,
+            workersAvailable,
+            queueDepth,
+            maxQueueDepth,
+            isShuttingDown,
+            canaryResult: canaryJobResult,
+        }, 'Queue health check failed');
+    }
+    return result;
+}
+/**
+ * Check if queue workers are currently running
+ */
+export function areWorkersRunning() {
+    return workersStarted && !isShuttingDown;
+}
+/**
+ * Get the current queue depth (waiting + active jobs)
+ */
+export async function getQueueDepth() {
+    const health = await getQueueHealth();
+    return (health.intake.waiting +
+        health.intake.active +
+        health.evaluate.waiting +
+        health.evaluate.active +
+        health.decision.waiting +
+        health.decision.active);
+}
+/**
+ * Pause all queue workers (stop processing new jobs)
+ * Existing jobs in progress will continue to completion
+ */
+export async function pauseWorkers() {
+    logger.info('Pausing queue workers');
+    const pausePromises = workers.map(async (worker) => {
+        try {
+            await worker.pause();
+        }
+        catch (error) {
+            logger.warn({ error }, 'Error pausing worker');
+        }
+    });
+    await Promise.all(pausePromises);
+    logger.info('All queue workers paused');
+}
+/**
+ * Resume all queue workers
+ */
+export async function resumeWorkers() {
+    logger.info('Resuming queue workers');
+    const resumePromises = workers.map(async (worker) => {
+        try {
+            worker.resume();
+        }
+        catch (error) {
+            logger.warn({ error }, 'Error resuming worker');
+        }
+    });
+    await Promise.all(resumePromises);
+    logger.info('All queue workers resumed');
+}
+/**
+ * Wait for all active jobs to complete with timeout
+ * Returns true if all jobs completed, false if timeout was reached
+ */
+export async function waitForActiveJobs(timeoutMs) {
+    const startTime = Date.now();
+    const pollInterval = 500;
+    while (Date.now() - startTime < timeoutMs) {
+        const health = await getQueueHealth();
+        const activeJobs = health.intake.active +
+            health.evaluate.active +
+            health.decision.active;
+        if (activeJobs === 0) {
+            return { completed: true, remainingJobs: 0 };
+        }
+        logger.debug({ activeJobs, elapsedMs: Date.now() - startTime, timeoutMs }, 'Waiting for active jobs to complete');
+        await new Promise((resolve) => setTimeout(resolve, pollInterval));
+    }
+    // Timeout reached - get final count
+    const finalHealth = await getQueueHealth();
+    const remainingJobs = finalHealth.intake.active +
+        finalHealth.evaluate.active +
+        finalHealth.decision.active;
+    return { completed: false, remainingJobs };
+}
+//# sourceMappingURL=queues.js.map