npm - @wildo-ai/saas-backend-lib - Versions diffs - 1.1.8 → 1.1.9 - Mend

@wildo-ai/saas-backend-lib 1.1.8 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (174) hide show

package/dist/esm/queue/queue.backend.service.js CHANGED Viewed

@@ -91,15 +91,19 @@ let QueueBackendService = class QueueBackendService {
         // Assert the operations dead-letter queue FIRST — the main queues below name
         // it as their dead-letter target, and asserting the target ahead of the
         // source keeps the topology self-consistent. This queue is TERMINAL: it has
-        // no dead-letter args of its own (a parked message stays here until a human /
-        // tool inspects it — the queue has no consumer).
+        // no dead-letter args of its own and no consumer. A parked message stays until a human / tool
+        // inspects it OR the platform's bound removes it: the per-application broker policy
+        // `wildo-deadletter-bound.<appId>` (14-day `message-ttl`, `max-length` 10 000, `overflow:
+        // drop-head`), declared by the RabbitMQ provider module when the application's credentials are
+        // minted (#2214). A policy rather than an argument, because RabbitMQ refuses to change the
+        // arguments of a queue that already exists.
         //
         // ## Dead-letter inspection is a BROKER-level concern — decided 2026-09-08 (#469)
         //
-        // Having no consumer is deliberate, not unfinished. Wildo ships no listing of parked jobs, no
-        // replay verb and no queue-depth health signal, and it is not going to: an operator inspects and
-        // replays a parked message with the broker's own tooling (the RabbitMQ management UI or CLI
-        // against this queue by name).
+        // Having no consumer is deliberate, not unfinished. Wildo ships no listing of parked jobs and no
+        // replay verb, and it is not going to: an operator inspects and replays a parked message with the
+        // broker's own tooling (the RabbitMQ management UI or CLI against this queue by name). A DEPTH
+        // signal is the one exception, added by #2214 (below), because it exposes no payload.
         //
         // Do not read the absence as an oversight and wire a surface here. A job-execution read surface
         // exposes job PAYLOADS, which may carry personal data, and would need its own authorization and
@@ -109,6 +113,13 @@ let QueueBackendService = class QueueBackendService {
         //
         // If this ever bites, the option to revisit FIRST is queue DEPTH as a health signal — it catches
         // the fleet-level problem without exposing a single payload.
+        //
+        // It bit on 2026-10-01 (#2214): `scheduled.deadletter` reached 359 messages for one application
+        // in a day of restarts and nothing said so. Two things changed, and the no-consumer decision did
+        // not: the queues are now BOUNDED by the broker policy above, and the worker runtime that owns
+        // them warns at boot for each non-empty one (`reportParkedDeadLetters` — a depth, never a
+        // payload). This API-pod path declares only the operations dead-letter queue and does not own
+        // the scheduled/callbacks ones, so the boot report lives in `initializeForWorker`.
         const deadLetterQueue = buildQueueName(this.appId, 'operations', QueueVariant.DEADLETTER);
         await this.adapter.assertQueue(deadLetterQueue, {
             durable: true,
@@ -222,8 +233,11 @@ let QueueBackendService = class QueueBackendService {
         }
         // Assert the dead-letter queues FIRST — each is the dead-letter target of the
         // matching main queue below, so it must be declared ahead of the source that
-        // names it. Each DLQ is TERMINAL (no dead-letter args of its own): a parked
-        // message stays until inspected; these queues have no consumer.
+        // names it. Each DLQ is TERMINAL (no dead-letter args of its own, no consumer): a parked
+        // message stays until inspected or until the platform's per-application broker POLICY removes it
+        // (14-day TTL, at most 10 000 messages, oldest dropped first — see the operations-DLQ comment in
+        // `initializeForApiPod` and `RABBITMQ_DEAD_LETTER_QUEUE_BOUND`, #2214). Their depth is reported
+        // at boot by `reportParkedDeadLetters`, below.
         //
         // TOPOLOGY-MIGRATION NOTE: the main queues below are asserted WITH
         // `x-dead-letter-*` arguments. RabbitMQ will NOT mutate the arguments of an
@@ -235,7 +249,9 @@ let QueueBackendService = class QueueBackendService {
         // are auto-recreated here with the DLX args. This project is greenfield (no
         // production data), so a one-time manual redeclare is the chosen migration —
         // see `.claude/plans/queue-permanent-dlx.md` for the rejected alternatives
-        // (versioning the queue name; broker-side policies).
+        // (versioning the queue name; broker-side policies). That rejection is about the DLX ROUTING
+        // arguments, which stay queue arguments. The dead-letter queues' BOUND is a different concern
+        // and IS a broker policy (#2214), because it must reach queues that already exist.
         const operationsDeadLetterQueue = buildQueueName(this.appId, 'operations', QueueVariant.DEADLETTER);
         await this.adapter.assertQueue(operationsDeadLetterQueue, { durable: true });
         const scheduledDeadLetterQueue = buildQueueName(this.appId, 'scheduled', QueueVariant.DEADLETTER);
@@ -259,9 +275,16 @@ let QueueBackendService = class QueueBackendService {
             deadLetterExchange: '',
             deadLetterRoutingKey: operationsDeadLetterQueue,
         });
-        // Assert scheduled queue → parks permanently-failed scheduled executions in
-        // `scheduled.deadletter` for inspection (they still re-run on the next
-        // schedule; the parked token is a stale audit copy, not a re-execution).
+        // Assert scheduled queue → parks the scheduled deliveries this application could not accept in
+        // `scheduled.deadletter` for inspection: deliveries that are not authentic (a trust failure),
+        // addressed to another application, or authentic-but-late ones whose missed occurrence could
+        // not be RECORDED. A late delivery that was recorded is acknowledged instead (#2212) — its
+        // durable record is the evidence, so the broker no longer keeps a copy of every restart's
+        // backlog. The schedule continues with its next occurrence either way.
+        //
+        // The dead-letter queues carry no arguments of their own; their bound (TTL + max length) is a
+        // broker POLICY applied when the platform provisions the application (#2214), because RabbitMQ
+        // refuses to change the arguments of a queue that already exists.
         const scheduledQueue = buildQueueName(this.appId, 'scheduled');
         await this.adapter.assertQueue(scheduledQueue, {
             durable: true,
@@ -275,12 +298,45 @@ let QueueBackendService = class QueueBackendService {
             deadLetterExchange: '',
             deadLetterRoutingKey: callbacksDeadLetterQueue,
         });
+        await this.reportParkedDeadLetters([operationsDeadLetterQueue, scheduledDeadLetterQueue, callbacksDeadLetterQueue]);
         this.initialized = true;
         this.logDebug('Queue service initialized for worker', {
             appId: this.appId,
             replyQueue: this.replyQueueName,
         });
     }
+    /**
+     * Say, once at boot, which dead-letter queues hold parked messages (#2214).
+     *
+     * Dead-letter queues have no consumer by decision (#469): an operator inspects and replays a parked
+     * message with the broker's own tooling, because a job payload may carry personal data. What that
+     * decision left out is any way to learn that something IS parked — `scheduled.deadletter` reached
+     * 359 messages in one day of local restarts and nothing said so. #469 named queue DEPTH as the
+     * first thing to revisit, and this is that: a count and a queue name, never a payload.
+     *
+     * At boot, because a restart is exactly when a backlog lands, and in the runtime that just declared
+     * these queues (so they exist: a passive check of a missing queue closes the channel). Best-effort:
+     * a depth that cannot be read is skipped, never a reason to fail initialization.
+     */
+    async reportParkedDeadLetters(deadLetterQueues) {
+        for (const queueName of deadLetterQueues) {
+            let parked = null;
+            try {
+                parked = await this.adapter.getQueueDepth(queueName);
+            }
+            catch {
+                parked = null;
+            }
+            if (parked !== null && parked > 0) {
+                this.logger.warn('Dead-letter queue holds parked messages', {
+                    queueName, parked,
+                    remedy: 'These are deliveries this application could not accept. Inspect or replay them with the broker\'s '
+                        + 'own tooling (the RabbitMQ management UI or CLI, against this queue by name). The platform bounds the '
+                        + 'queue by age and length, so old messages expire on their own.',
+                });
+            }
+        }
+    }
     /**
      * Enqueue operation (fire and forget).
      * Returns immediately after publishing.

package/dist/esm/queue/queue.backend.service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"queue.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/queue.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,4BAA4B,EAAE,MAAM,oFAAoF,CAAC;AAClI,OAAO,EAAE,6BAA6B,EAAE,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AAE3H,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAExF,OAAO,EAAE,uBAAuB,EAAE,MAAM,+DAA+D,CAAC;AACxG,OAAO,EAAE,4BAA4B,EAAE,MAAM,gEAAgE,CAAC;AAC9G,OAAO,EAEL,gCAAgC,EAChC,sCAAsC,EAGtC,OAAO,EACP,YAAY,EACZ,WAAW,EAEX,SAAS,EACT,YAAY,EAEZ,0BAA0B,EAC1B,kCAAkC,EAClC,qBAAqB,EACrB,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAE/B,4FAA4F;AAC5F,qFAAqF;AACrF,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5F,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,MAAM,CAAC;IACvD,SAAS,EAAE,SAAS;IACpB,aAAa,EAAE,SAAS,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC;AAsBH,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,MAAkC;IAClE,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAAC,MAAkC;IACjE,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9E,CAAC;AAGM,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAaX;IAEA;IAEA;IAEA;IAEA;IAEA;IAtBT,QAAQ,CAA+D;IACzE,MAAM,GAAkB,IAAI,CAAC;IAC7B,cAAc,GAAkB,IAAI,CAAC;IAC5B,cAAc,GAI1B,IAAI,GAAG,EAAE,CAAC;IACP,WAAW,GAAG,KAAK,CAAC;IAE5B,YAEmB,gBAAyC,EAEzC,qBAAmD,EAEnD,MAAqB,EAErB,OAAqB,EAErB,iBAAkD,EAElD,cAA+C;QAV/C,qBAAgB,GAAhB,gBAAgB,CAAyB;QAEzC,0BAAqB,GAArB,qBAAqB,CAA8B;QAEnD,WAAM,GAAN,MAAM,CAAe;QAErB,YAAO,GAAP,OAAO,CAAc;QAErB,sBAAiB,GAAjB,iBAAiB,CAAiC;QAElD,mBAAc,GAAd,cAAc,CAAiC;QAEhE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC9D,kEAAkE;IACpE,CAAC;IAED,6EAA6E;IAC7E,IAAY,KAAK;QACf,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,mBAAmB;QAC9B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CACtD,IAAI,CAAC,KAAK,EACV,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC,CACnE,CAAC;QAEF,6EAA6E;QAC7E,wEAAwE;QACxE,4EAA4E;QAC5E,8EAA8E;QAC9E,iDAAiD;QACjD,EAAE;QACF,kFAAkF;QAClF,EAAE;QACF,8FAA8F;QAC9F,iGAAiG;QACjG,4FAA4F;QAC5F,+BAA+B;QAC/B,EAAE;QACF,gGAAgG;QAChG,gGAAgG;QAChG,+FAA+F;QAC/F,6FAA6F;QAC7F,6EAA6E;QAC7E,EAAE;QACF,iGAAiG;QACjG,6DAA6D;QAC7D,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QAC1F,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE;YAC9C,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,+EAA+E;QAC/E,wEAAwE;QACxE,2EAA2E;QAC3E,6EAA6E;QAC7E,8EAA8E;QAC9E,+DAA+D;QAC/D,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE;YAC9C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,eAAe;SACtC,CAAC,CAAC;QAEH,6EAA6E;QAC7E,8DAA8D;QAC9D,iEAAiE;QACjE,+EAA+E;QAC/E,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE;YAC5C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,eAAe;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,uCAAuC,EAAE;YACrD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,cAAc;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACI,KAAK,CAAC,mBAAmB,CAAC,UAA+C,EAAE;QAChF,MAAM,EAAE,qBAAqB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACjD,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,6CAA6C;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;QAEjD;;;;;;;WAOG;QACH,IAAI,SAAS,CAAC,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,OAAO,YAAY,6BAA6B,EAAE,CAAC;YAC1F,+DAA+D;YAC/D,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ;mBACnD,OAAO,CAAC,GAAG,CAAC,iBAAiB;mBAC7B,OAAO,CAAC;YACb,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ;mBACnD,OAAO,CAAC,GAAG,CAAC,iBAAiB;mBAC7B,OAAO,CAAC;YAEb,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;gBACrB,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI;gBACtC,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI;gBACtC,QAAQ;gBACR,QAAQ;gBACR,gEAAgE;gBAChE,gEAAgE;gBAChE,8DAA8D;gBAC9D,iEAAiE;gBACjE,+DAA+D;gBAC/D,4DAA4D;gBAC5D,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK;aACzC,CAAC,CAAC;QACL,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,CAAC;QAED,sDAAsD;QACtD,8CAA8C;QAC9C,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CACtD,IAAI,CAAC,KAAK,EACV,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC,CACnE,CAAC;QAEF,wFAAwF;QACxF,mFAAmF;QACnF,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,mFAAmF,EAAE;gBACjG,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,IAAI,CAAC,cAAc;aAChC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,8EAA8E;QAC9E,6EAA6E;QAC7E,4EAA4E;QAC5E,gEAAgE;QAChE,EAAE;QACF,mEAAmE;QACnE,4EAA4E;QAC5E,4EAA4E;QAC5E,6DAA6D;QAC7D,4EAA4E;QAC5E,qEAAqE;QACrE,6EAA6E;QAC7E,4EAA4E;QAC5E,6EAA6E;QAC7E,2EAA2E;QAC3E,qDAAqD;QACrD,MAAM,yBAAyB,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QACpG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,yBAAyB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAE7E,MAAM,wBAAwB,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5E,MAAM,wBAAwB,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5E,2EAA2E;QAC3E,8EAA8E;QAC9E,wEAAwE;QACxE,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE;YAC9C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,yBAAyB;SAChD,CAAC,CAAC;QAEH,gFAAgF;QAChF,iEAAiE;QACjE,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE;YAC5C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,yBAAyB;SAChD,CAAC,CAAC;QAEH,4EAA4E;QAC5E,uEAAuE;QACvE,yEAAyE;QACzE,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,EAAE;YAC7C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,wBAAwB;SAC/C,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,EAAE;YAC7C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,wBAAwB;SAC/C,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,sCAAsC,EAAE;YACpD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,cAAc;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,gBAAgB,CAC3B,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,eAA4C,EAC5C,qBAAwD;QAExD,MAAM,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;QAE7C,iDAAiD;QACjD,MAAM,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErG,oBAAoB;QACpB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;QAErH,oCAAoC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5D,+CAA+C;QAC/C,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,mBAAmB;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE3C,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE;YAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS;YACT,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;YACtC,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,qBAAqB;SAChD,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACzC,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,cAAc,CACzB,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,SAAkB,EAClB,eAA4C,EAC5C,qBAAwD;QAExD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,SAAS,IAAI,WAAW,CAAC,SAAS,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAEtF,sBAAsB;QACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErG,6BAA6B;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;QACrH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,qBAAqB,GAAG,aAAa,CAAC;QAC1C,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;QAElC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5D,kFAAkF;QAClF,kFAAkF;QAClF,oFAAoF;QACpF,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,2BAA2B;QAC3B,MAAM,aAAa,GAAG,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;gBACpC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAC1C,MAAM,CAAC,IAAI,KAAK,CAAC,0CAA0C,OAAO,kCAAkC,CAAC,CAAC,CAAC;YACzG,CAAC,EAAE,OAAO,CAAC,CAAC;YAEZ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;QAEH,gFAAgF;QAChF,gFAAgF;QAChF,iFAAiF;QACjF,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE;YACrE,aAAa;YACb,OAAO,EAAE,cAAc;SACxB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,QAAQ,CAAC,uBAAuB,EAAE;gBACrC,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,mFAAmF;gBACnF,qBAAqB,EAAE,aAAa;gBACpC,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YACf,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACvD,IAAI,OAAO,EAAE,CAAC;gBACZ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,qBAAqB,EAAE,aAAa,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7K,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,mBAAmB,CAC9B,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,eAA4C,EAC5C,qBAAwD;QAExD,sBAAsB;QACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErG,0FAA0F;QAC1F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;QAErH,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5D,+CAA+C;QAC/C,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE3C,IAAI,CAAC,QAAQ,CAAC,4BAA4B,EAAE;YAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS;YACT,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,qBAAqB;SAChD,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CAC3B,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,eAA4C,EAC5C,qBAAwD;QAExD,MAAM,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,qBAAqB,CAAC;QAEnE,0EAA0E;QAC1E,6EAA6E;QAC7E,4EAA4E;QAC5E,0EAA0E;QAC1E,8DAA8D;QAC9D,IAAI,aAAa,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpH,IAAI,qBAAyC,CAAC;QAE9C,uEAAuE;QACvE,IAAI,aAAa,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;YACpE,qBAAqB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACnE,aAAa,GAAG,SAAS,CAAC,CAAC,yBAAyB;YACpD,IAAI,CAAC,QAAQ,CAAC,iCAAiC,EAAE,EAAE,qBAAqB,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,2CAA2C;QAC3C,MAAM,aAAa,GAAG,kCAAkC,CAAC,oBAAoB,CAAC,SAAgB,CAAC,CAAC;QAEhG,MAAM,GAAG,GAAe;YACtB,KAAK,EAAE,MAAM,EAAE;YACf,OAAO,EAAE,OAAO,CAAC,kBAAkB;YACnC,uBAAuB,EAAE,eAAe,CAAC,CAAC,CAAC,gCAAgC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9G,6BAA6B,EAAE,qBAAqB,CAAC,CAAC,CAAC,sCAAsC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS;YACtI,aAAa,EAAE,IAAI,CAAC,KAAK;YAEzB,gBAAgB,EAAE,kCAAkC,CAAC,sBAAsB,CAAC,aAAa,CAAC;YAC1F,iBAAiB,EAAE,0BAA0B,CAAC,sBAAsB,CAAC,aAAa,CAAC;YAEnF,aAAa;YACb,qBAAqB;YACrB,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAE3F,oBAAoB,EAAE,0BAA0B,CAAC,yBAAyB,CAAC;gBACzE,aAAa,EAAE,gBAAgB,CAAC,aAAa;gBAC7C,YAAY,EAAE,gBAAgB,CAAC,YAAY;gBAC3C,cAAc,EAAE,CAAC,GAAG,gBAAgB,CAAC,cAAc,CAAC;gBACpD,qBAAqB,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,qBAAqB,EAAE;gBACrE,oBAAoB,EAAE,CAAC,GAAG,gBAAgB,CAAC,oBAAoB,CAAC;gBAChE,8FAA8F;gBAC9F,gGAAgG;gBAChG,2EAA2E;gBAC3E,GAAG,CAAC,gBAAgB,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnH,GAAG,gBAAgB,CAAC,mBAAmB;aACxC,CAAC;YAEF,8FAA8F;YAC9F,gGAAgG;YAChG,kGAAkG;YAClG,yEAAyE;YACzE,mBAAmB,EAAE,gBAAgB,CAAC,aAAa;YACnD,oGAAoG;YACpG,+FAA+F;YAC/F,GAAG,CAAC,GAAG,EAAE,GAAG,MAAM,WAAW,GAAG,4BAA4B,EAAE,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;YAE/G,WAAW,EAAE,YAAY,CAAC,GAAG;YAC7B,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,WAAW,CAAC,MAAM;YACpD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,WAAW,CAAC,UAAU,IAAI,qBAAqB,CAAC,UAAU;YACtE,wEAAwE;YACxE,oEAAoE;YACpE,sEAAsE;YACtE,0EAA0E;YAC1E,WAAW,EAAE;gBACX,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBAChG,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBAC7F,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;aACjG;YACD,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,qBAAqB,CAAC,SAAS;YAEnE,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,QAAgB;QAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACtD,OAAO,CAAC,QAAQ,KAAK,WAAW,CAAC,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,QAAQ,CAAC;YACzE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC;YACjE,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,aAAqB,EAAE,MAAiB;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;QAED,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAE1C,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC9E,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;QACxG,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,KAAK,CAAC,kBAAkB,CAC7B,SAAiB,EACjB,OAAgB,EAChB,OAiBC;QAED,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC;QACzD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,qBAAqB,CAAC,UAAU,CAAC;QAC3E,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAExE,uEAAuE;QACvE,kEAAkE;QAClE,IAAI,aAAa,GAAG,0BAA0B,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC5E,IAAI,qBAAyC,CAAC;QAE9C,kCAAkC;QAClC,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;YACnD,qBAAqB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjE,aAAa,GAAG,SAAgB,CAAC;QACnC,CAAC;QAED,MAAM,GAAG,GAAe;YACtB,KAAK,EAAE,MAAM,EAAE;YACf,OAAO,EAAE,OAAO,CAAC,YAAY;YAC7B,aAAa,EAAE,IAAI,CAAC,KAAK;YAEzB,0FAA0F;YAC1F,uFAAuF;YACvF,6EAA6E;YAC7E,QAAQ,EAAE,SAAS;YAEnB,gBAAgB,EAAE,SAAS,EAAE,2CAA2C;YACxE,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC;YAEhD,aAAa;YACb,qBAAqB;YAErB,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnC,aAAa,EAAE,OAAO;gBACtB,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;aACpC,CAAC;YAEF,8FAA8F;YAC9F,kFAAkF;YAClF,kGAAkG;YAClG,2FAA2F;YAC3F,kGAAkG;YAClG,mBAAmB,EAAE,uBAAuB,EAAE;YAC9C,yEAAyE;YACzE,GAAG,CAAC,GAAG,EAAE,GAAG,MAAM,WAAW,GAAG,4BAA4B,EAAE,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;YAE/G,WAAW,EAAE,YAAY,CAAC,QAAQ;YAClC,QAAQ;YACR,UAAU,EAAE,CAAC;YACb,UAAU;YACV,WAAW,EAAE;gBACX,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBAClF,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBAC/E,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;aACnF;YACD,SAAS;YAET,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,oCAAoC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAExD,6BAA6B;QAC7B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,+FAA+F;QAC/F,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QACtC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,2BAA2B,EAAE;YACzC,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS;YACT,SAAS;YACT,OAAO;YACP,kBAAkB,EAAE,CAAC,CAAC,qBAAqB;SAC5C,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACzC,CAAC;IAED;;OAEG;IACI,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,wBAAwB;QACxB,KAAK,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3D,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,gBAAgB;QAChB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAE3B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC3C,CAAC;CACF,CAAA;AA5qBY,mBAAmB;IAD/B,UAAU,EAAE;IAaR,WAAA,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3C,WAAA,MAAM,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,CAAA;IAEhD,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;IAEnD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;qCATjB,uBAAuB;QAElB,4BAA4B;QAE3C,aAAa,UAIF,+BAA+B;QAElC,+BAA+B;GAvBvD,mBAAmB,CA4qB/B","sourcesContent":["/*\n Queue Backend Service\n \n Main entry point for queue operations.\n * Handles enqueuing jobs with authorization checks.\n /\n\nimport { injectable, inject } from 'inversify';\nimport { v4 as uuidv4 } from 'uuid';\nimport { z } from 'zod';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';\nimport { AuthorizationsBackendService } from '~wildo-backend-lib/security/authorizations/services/authorizations.backend.service';\nimport { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';\nimport { projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport type { QueueAdapter } from './adapters/queue-adapter.backend.interface';\nimport { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';\nimport { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';\nimport { ExecutionContext } from '~wildo-backend-lib/runtime/models/execution-context.backend.schemas';\nimport { getCurrentCorrelationId } from '~wildo-backend-lib/runtime/models/request-context.als.backend';\nimport { snapshotWorkAttributionFrame } from '~wildo-backend-lib/runtime/models/work-attribution.als.backend';\nimport {\n JobPayload,\n JobResourceResponseOptionsSchema,\n JobResourceMutationPreconditionsSchema,\n type JobResourceMutationPreconditions,\n type JobResourceResponseOptions,\n JobType,\n JobInitiator,\n JobPriority,\n JobResult,\n JobStatus,\n QueueVariant,\n QueueRetryBackoffConfig,\n ResourceSerializationUtils,\n ResourcesConfig_OperationPathUtils,\n QUEUE_CONFIG_DEFAULTS,\n QUEUE_RETRY_DEFAULTS,\n buildQueueName,\n WildoError_BackendSchema,\n} from '@wildo-ai/saas-models';\n\n// RabbitMQ JSON transport converts dates to strings. Restore only the declared date fields;\n// validate the whole canonical error before letting it reach the HTTP error handler.\nconst replyDate = z.union([z.date(), z.iso.datetime().transform(value => new Date(value))]);\nconst replyErrorSchema = WildoError_BackendSchema.extend({\n timestamp: replyDate,\n lastRetryTime: replyDate.optional(),\n});\n\n\n/\n Result of an enqueue operation.\n /\nexport interface EnqueueResult {\n /* Unique job identifier /\n jobId: string;\n /* Queue the job was published to /\n queueName: string;\n}\n\n/\n What a queued resource operation is aimed at: ONE row, or the rows a bulk operation resolved.\n \n A bulk target cannot travel in `inputId` — it is a single id — and the selector is stripped from\n * the payload once the service has read it, so before this a bulk operation reached the worker with\n * no target at all (#1461).\n /\nexport type QueueJobTarget = string \| readonly string[];\n\n/* The single-row target, or `undefined` when the job is aimed at a set of rows. /\nexport function singleJobTargetId(target: QueueJobTarget \| undefined): string \| undefined {\n return typeof target === 'string' ? target : undefined;\n}\n\n/* The bulk target, or `undefined` when the job is aimed at one row (or at none). /\nexport function bulkJobTargetIds(target: QueueJobTarget \| undefined): string[] \| undefined {\n return Array.isArray(target) && target.length > 0 ? [...target] : undefined;\n}\n\n@injectable()\nexport class QueueBackendService {\n protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n private _appId: string \| null = null;\n private replyQueueName: string \| null = null;\n private readonly pendingReplies: Map<string, {\n resolve: (result: any) => void;\n reject: (error: unknown) => void;\n timeout: NodeJS.Timeout;\n }> = new Map();\n private initialized = false;\n\n constructor(\n @inject(SAAS_SERVICE_TYPES.AppConfigService)\n private readonly appConfigService: AppConfigurationService,\n @inject(SAAS_SERVICE_TYPES.AuthorizationsService)\n private readonly authorizationsService: AuthorizationsBackendService,\n @inject(SAAS_SERVICE_TYPES.Logger)\n private readonly logger: LoggerService,\n @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n private readonly adapter: QueueAdapter,\n @inject(SAAS_SERVICE_TYPES.JobStatusTrackingService)\n private readonly jobStatusTracking: JobStatusTrackingBackendService,\n @inject(SAAS_SERVICE_TYPES.JobPayloadStorageService)\n private readonly payloadStorage: JobPayloadStorageBackendService,\n ) {\n this.logDebug = this.logger.debugByType('QueueBackendService');\n this.logger.setDebugTypeLogging('QueueBackendService', false);\n // Lazy-load appId to avoid accessing config before initialization\n }\n\n /* Lazy getter for applicationId - avoids accessing config in constructor /\n private get appId(): string {\n if (!this._appId) {\n this._appId = this.appConfigService.applicationId;\n }\n return this._appId;\n }\n\n /\n Initialize queue service for API pod.\n * Sets up reply queue for RPC pattern.\n /\n public async initializeForApiPod(): Promise<void> {\n if (this.initialized) return;\n\n // Ensure adapter is connected\n if (!this.adapter.isConnected()) {\n await this.adapter.connect();\n }\n\n // Set up reply queue for RPC\n this.replyQueueName = await this.adapter.setupReplyQueue(\n this.appId,\n (correlationId, result) => this.handleReply(correlationId, result)\n );\n\n // Assert the operations dead-letter queue FIRST — the main queues below name\n // it as their dead-letter target, and asserting the target ahead of the\n // source keeps the topology self-consistent. This queue is TERMINAL: it has\n // no dead-letter args of its own (a parked message stays here until a human /\n // tool inspects it — the queue has no consumer).\n //\n // ## Dead-letter inspection is a BROKER-level concern — decided 2026-09-08 (#469)\n //\n // Having no consumer is deliberate, not unfinished. Wildo ships no listing of parked jobs, no\n // replay verb and no queue-depth health signal, and it is not going to: an operator inspects and\n // replays a parked message with the broker's own tooling (the RabbitMQ management UI or CLI\n // against this queue by name).\n //\n // Do not read the absence as an oversight and wire a surface here. A job-execution read surface\n // exposes job PAYLOADS, which may carry personal data, and would need its own authorization and\n // privacy classification — which is why the decision went the other way. The display behaviour\n // that used to be authored for `jobExecutions` / `jobPayloadStorage` was deleted in the same\n // change, because it rendered nowhere and made this absence look accidental.\n //\n // If this ever bites, the option to revisit FIRST is queue DEPTH as a health signal — it catches\n // the fleet-level problem without exposing a single payload.\n const deadLetterQueue = buildQueueName(this.appId, 'operations', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(deadLetterQueue, {\n durable: true,\n });\n\n // Assert operations queue, dead-lettering permanent failures to the DLQ above.\n // A consumer that exhausts retries (or hits a non-retryable / malformed\n // message) calls `channel.nack(msg, false, false)`; with the DLX wired the\n // broker parks that message in `operations.deadletter` instead of discarding\n // it. Default exchange ('') + routing key = the DLQ name routes by queue name\n // — the same plugin-free mechanism the retry delay queues use.\n const operationsQueue = buildQueueName(this.appId, 'operations');\n await this.adapter.assertQueue(operationsQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: deadLetterQueue,\n });\n\n // Assert priority queue. Priority is a dispatch VARIANT of operations, not a\n // distinct category, so its permanent failures share the SAME\n // `operations.deadletter` (there is intentionally no per-variant\n // `operations.priority.deadletter` — see queue-monitoring.backend.service.ts).\n const priorityQueue = buildQueueName(this.appId, 'operations', QueueVariant.PRIORITY);\n await this.adapter.assertQueue(priorityQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: deadLetterQueue,\n });\n\n this.initialized = true;\n this.logDebug('Queue service initialized for API pod', {\n appId: this.appId,\n replyQueue: this.replyQueueName,\n });\n }\n\n /\n Initialize queue service for worker pod.\n \n This method initializes all queue infrastructure needed for a pod that:\n * - Consumes and processes jobs (worker functionality)\n * - Enqueues jobs and receives RPC replies (API functionality)\n \n Every backend API pod serves both roles, hence this comprehensive initialization.\n \n `ownsSharedJobTopology` (default `true`) decides whether this runtime DECLARES the shared\n * application job queues — `operations`, `operations.priority`, `scheduled`, `callbacks` and their\n * `.deadletter` queues. The runtime that consumes those queues owns them. A runtime that connects\n * only for its own dedicated topology — a minion, which consumes just its `minion-tick.<name>`\n * queue — passes `false` and gets the broker connection and its own reply queue, nothing else.\n \n WHY A NON-OWNER MUST NOT DECLARE THEM ANYWAY. A minion's broker credential is least-privilege:\n * its `read` permission admits only its tick queue and its own reply queue (`buildMinionConsumeRegex`\n * in the rabbitmq-self infra provider module). Measured 2026-09-14 on Wonder Todos: asserting\n * `operations` WITH its `x-dead-letter-exchange` arguments was refused with\n * `403 ACCESS_REFUSED - read access to queue '…operations'`, while the three `.deadletter` queues —\n * declared with no DLX arguments — had just been accepted. The refusal aborted the initialization\n * and left this service uninitialized. Beyond permissions, a second declarer of a queue can only\n * agree with the owner's arguments or break them (RabbitMQ refuses to mutate a live queue's\n * arguments), so the owner is the one place those arguments are stated.\n /\n public async initializeForWorker(options: { ownsSharedJobTopology?: boolean } = {}): Promise<void> {\n const { ownsSharedJobTopology = true } = options;\n if (this.initialized) return;\n\n // Configure RabbitMQ adapter from app config\n const appConfig = this.appConfigService.config;\n const appSecrets = this.appConfigService.secrets;\n\n /\n * Broker connection settings apply to the BROKER adapter, and to nothing else (#502).\n \n `configure()` takes a `RabbitMQConfig` and is deliberately absent from `QueueAdapter` — an\n * in-memory queue has no host, port or vhost to be told about. Narrowing here rather than\n * widening the port keeps that honest: the alternative was a `configure(config: unknown)` every\n * adapter must accept and most must ignore, which is a hole in the port wearing a signature.\n /\n if (appConfig.database?.rabbitmq && this.adapter instanceof RabbitMQAdapterBackendService) {\n // Credentials from secrets first, then env vars, then defaults\n const username = appSecrets.database?.rabbitmq?.username\n \|\| process.env.RABBITMQ_USERNAME\n \|\| 'guest';\n const password = appSecrets.database?.rabbitmq?.password\n \|\| process.env.RABBITMQ_PASSWORD\n \|\| 'guest';\n\n this.adapter.configure({\n host: appConfig.database.rabbitmq.host,\n port: appConfig.database.rabbitmq.port,\n username,\n password,\n // Slice 2.5c-3 of `infra-provider-planner.md`: optional per-app\n // virtual host. Authored at `backingServices.rabbitmq.vhost` in\n // `wildo.infra.<envName>.config.ts`, threaded by apps-manager\n // into the persisted `Wildo_Application_Config`, read here. When\n // omitted the adapter falls back to the default vhost `/` (the\n // adapter's existing `this.config.vhost \|\| '/'` semantics).\n vhost: appConfig.database.rabbitmq.vhost,\n });\n }\n\n // Ensure adapter is connected\n if (!this.adapter.isConnected()) {\n await this.adapter.connect();\n }\n\n // Set up reply queue for RPC pattern (enqueueAndWait)\n // Each pod gets its own exclusive reply queue\n this.replyQueueName = await this.adapter.setupReplyQueue(\n this.appId,\n (correlationId, result) => this.handleReply(correlationId, result)\n );\n\n // A runtime that does not consume the shared job queues does not own them, and does not\n // declare them (see the method JSDoc: a minion's credential cannot, and must not).\n if (!ownsSharedJobTopology) {\n this.initialized = true;\n this.logDebug('Queue service initialized for a runtime that does not own the shared job topology', {\n appId: this.appId,\n replyQueue: this.replyQueueName,\n });\n return;\n }\n\n // Assert the dead-letter queues FIRST — each is the dead-letter target of the\n // matching main queue below, so it must be declared ahead of the source that\n // names it. Each DLQ is TERMINAL (no dead-letter args of its own): a parked\n // message stays until inspected; these queues have no consumer.\n //\n // TOPOLOGY-MIGRATION NOTE: the main queues below are asserted WITH\n // `x-dead-letter-` arguments. RabbitMQ will NOT mutate the arguments of an\n // already-existing durable queue — asserting `operations` with new DLX args\n // when it already exists as plain `{ durable: true }` throws\n // 406 PRECONDITION_FAILED and closes the channel. On deployments that ran a\n // pre-DLX build, the existing `operations` / `operations.priority` /\n // `scheduled` / `callbacks` / `domain.` queues must be DELETED ONCE so they\n // are auto-recreated here with the DLX args. This project is greenfield (no\n // production data), so a one-time manual redeclare is the chosen migration —\n // see `.claude/plans/queue-permanent-dlx.md` for the rejected alternatives\n // (versioning the queue name; broker-side policies).\n const operationsDeadLetterQueue = buildQueueName(this.appId, 'operations', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(operationsDeadLetterQueue, { durable: true });\n\n const scheduledDeadLetterQueue = buildQueueName(this.appId, 'scheduled', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(scheduledDeadLetterQueue, { durable: true });\n\n const callbacksDeadLetterQueue = buildQueueName(this.appId, 'callbacks', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(callbacksDeadLetterQueue, { durable: true });\n\n // Assert operations queue, dead-lettering permanent failures (nack without\n // requeue) to `operations.deadletter` via the default exchange ('') routed by\n // queue name. Without this a permanently-failed job would be discarded.\n const operationsQueue = buildQueueName(this.appId, 'operations');\n await this.adapter.assertQueue(operationsQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: operationsDeadLetterQueue,\n });\n\n // Priority is a dispatch variant of operations → shares `operations.deadletter`\n // (no per-variant DLQ; see queue-monitoring.backend.service.ts).\n const priorityQueue = buildQueueName(this.appId, 'operations', QueueVariant.PRIORITY);\n await this.adapter.assertQueue(priorityQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: operationsDeadLetterQueue,\n });\n\n // Assert scheduled queue → parks permanently-failed scheduled executions in\n // `scheduled.deadletter` for inspection (they still re-run on the next\n // schedule; the parked token is a stale audit copy, not a re-execution).\n const scheduledQueue = buildQueueName(this.appId, 'scheduled');\n await this.adapter.assertQueue(scheduledQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: scheduledDeadLetterQueue,\n });\n\n // Assert callbacks queue → dead-letters to `callbacks.deadletter`.\n const callbacksQueue = buildQueueName(this.appId, 'callbacks');\n await this.adapter.assertQueue(callbacksQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: callbacksDeadLetterQueue,\n });\n\n this.initialized = true;\n this.logDebug('Queue service initialized for worker', {\n appId: this.appId,\n replyQueue: this.replyQueueName,\n });\n }\n\n /\n Enqueue operation (fire and forget).\n * Returns immediately after publishing.\n /\n public async enqueueOperation(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<EnqueueResult> {\n const operation = executionContext.operation;\n\n // Authorization check (same as controller layer)\n await this.authorizationsService.authorizeInController(singleJobTargetId(inputId), executionContext);\n\n // Build job payload\n const job = await this.buildJobPayload(executionContext, inputData, inputId, responseOptions, mutationPreconditions);\n\n // Determine queue based on priority\n const queueName = this.getOperationsQueueName(job.priority);\n\n // Create job tracking record before publishing\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n // Publish to queue\n await this.adapter.publish(queueName, job);\n\n this.logDebug('Job enqueued', {\n jobId: job.jobId,\n queueName,\n operationPathKey: job.operationPathKey,\n hasExternalPayload: !!job.inputPayloadStorageId,\n });\n\n return { jobId: job.jobId, queueName };\n }\n\n /\n Enqueue operation and wait for result (RPC pattern).\n * Used for QUEUED runtime mode. timeoutMs bounds reply waiting from immediately\n * before publication; it includes broker waiting and execution, not authorization\n * or payload/execution-record preparation. Expiry forgets the reply correlation only: it neither\n * cancels the worker nor determines whether its business effects happened.\n * A late reply is ignored; callers must not infer that retrying is safe.\n /\n public async enqueueAndWait<T>(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n timeoutMs?: number,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<T> {\n if (!this.replyQueueName) {\n throw new Error('Reply queue not initialized. Call initializeForApiPod() first.');\n }\n\n const operation = executionContext.operation;\n const queueConfig = operation.queueConfig ?? QUEUE_CONFIG_DEFAULTS;\n const timeout = timeoutMs ?? queueConfig.timeoutMs ?? QUEUE_CONFIG_DEFAULTS.timeoutMs;\n\n // Authorization check\n await this.authorizationsService.authorizeInController(singleJobTargetId(inputId), executionContext);\n\n // Build job with RPC headers\n const job = await this.buildJobPayload(executionContext, inputData, inputId, responseOptions, mutationPreconditions);\n const correlationId = uuidv4();\n job.rpcReplyCorrelationId = correlationId;\n job.replyTo = this.replyQueueName;\n\n const replyQueueName = this.replyQueueName;\n const queueName = this.getOperationsQueueName(job.priority);\n\n // The consumer only updates this record; RPC has the same producer-owned creation\n // lifecycle as the other operation modes. Finish persistence before registering a\n // reply deadline so a failed create cannot publish untracked work or leak a waiter.\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n // Create promise for reply\n const resultPromise = new Promise<T>((resolve, reject) => {\n const timeoutHandle = setTimeout(() => {\n this.pendingReplies.delete(correlationId);\n reject(new Error(`Timed out waiting for job result after ${timeout}ms; execution may still complete`));\n }, timeout);\n\n this.pendingReplies.set(correlationId, { resolve, reject, timeout: timeoutHandle });\n });\n\n // Adopt the reply promise immediately. Waiting for publish first would hide the\n // deadline from the caller and could leave its rejection temporarily unhandled.\n // Publication can finish after expiry: timeout still does not mean cancellation.\n void Promise.resolve().then(() => this.adapter.publish(queueName, job, {\n correlationId,\n replyTo: replyQueueName,\n })).then(() => {\n this.logDebug('Job enqueued with RPC', {\n jobId: job.jobId,\n // RPC reply matching is separate from the originating request's trace correlation.\n rpcReplyCorrelationId: correlationId,\n queueName,\n });\n }).catch(error => {\n const pending = this.pendingReplies.get(correlationId);\n if (pending) {\n clearTimeout(pending.timeout);\n this.pendingReplies.delete(correlationId);\n pending.reject(error);\n } else {\n this.logger.warn('RPC publication failed after the caller was settled', { jobId: job.jobId, rpcReplyCorrelationId: correlationId, ...projectCaughtErrorLogFields(error) });\n }\n });\n\n return resultPromise;\n }\n\n /\n Enqueue with callback (for BACKGROUND_JOB).\n * Returns job ID immediately, result delivered via callback mechanism.\n /\n public async enqueueWithCallback(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<EnqueueResult> {\n // Authorization check\n await this.authorizationsService.authorizeInController(singleJobTargetId(inputId), executionContext);\n\n // Build job (completion handlers run in the consuming process through its local registry)\n const job = await this.buildJobPayload(executionContext, inputData, inputId, responseOptions, mutationPreconditions);\n\n // Publish to operations queue\n const queueName = this.getOperationsQueueName(job.priority);\n\n // Create job tracking record before publishing\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n await this.adapter.publish(queueName, job);\n\n this.logDebug('Job enqueued with callback', {\n jobId: job.jobId,\n queueName,\n hasExternalPayload: !!job.inputPayloadStorageId,\n });\n\n return { jobId: job.jobId, queueName };\n }\n\n /\n Build job payload from execution context and input data.\n * Stores large payloads (>100KB) externally and stores only the storage ID.\n /\n private async buildJobPayload(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<JobPayload> {\n const operation = executionContext.operation;\n const queueConfig = operation.queueConfig ?? QUEUE_CONFIG_DEFAULTS;\n\n // Serialize input data. Use the bigint-safe serializer: a resource `data`\n // payload may carry `z_money` (MoneyValue) amounts stored as `bigint`, which\n // a bare `JSON.stringify` cannot serialize (\"Do not know how to serialize a\n // BigInt\"). See ResourceSerializationUtils.bigIntSafeJsonReplacer for the\n // string-encoding rationale and the worker round-trip caveat.\n let inputDataJson = inputData !== undefined ? ResourceSerializationUtils.stringifyWithBigInt(inputData) : undefined;\n let inputPayloadStorageId: string \| undefined;\n\n // Store large payloads externally to avoid RabbitMQ performance issues\n if (inputDataJson && this.payloadStorage.shouldStore(inputDataJson)) {\n inputPayloadStorageId = await this.payloadStorage.store(inputData);\n inputDataJson = undefined; // Don't embed in message\n this.logDebug('Large payload stored externally', { inputPayloadStorageId });\n }\n\n // Extract operation path for serialization\n const operationPath = ResourcesConfig_OperationPathUtils.extractOperationPath(operation as any);\n\n const job: JobPayload = {\n jobId: uuidv4(),\n jobType: JobType.RESOURCE_OPERATION,\n resourceResponseOptions: responseOptions ? JobResourceResponseOptionsSchema.parse(responseOptions) : undefined,\n resourceMutationPreconditions: mutationPreconditions ? JobResourceMutationPreconditionsSchema.parse(mutationPreconditions) : undefined,\n applicationId: this.appId,\n\n operationPathKey: ResourcesConfig_OperationPathUtils.createOperationPathKey(operationPath),\n operationPathJson: ResourceSerializationUtils.serializeOperationPath(operationPath),\n\n inputDataJson,\n inputPayloadStorageId,\n ...(singleJobTargetId(inputId) !== undefined ? { inputId: singleJobTargetId(inputId) } : {}),\n ...(bulkJobTargetIds(inputId) !== undefined ? { inputIds: bulkJobTargetIds(inputId) } : {}),\n\n executionContextJson: ResourceSerializationUtils.serializeExecutionContext({\n executionType: executionContext.executionType,\n initiatorIds: executionContext.initiatorIds,\n initiatorRoles: [...executionContext.initiatorRoles],\n getContextResourceIds: () => executionContext.getContextResourceIds(),\n relationshipContexts: [...executionContext.relationshipContexts],\n // Attribution evidence, so the worker's audit rows name the same on-behalf-of user, agent and\n // delegating client as the request that enqueued the job. The worker rebuilds from these alone:\n // no ambient frame reaches a delivery (see the in-memory adapter's drain).\n ...(executionContext.auditActorUserId !== undefined ? { auditActorUserId: executionContext.auditActorUserId } : {}),\n ...executionContext.attributionEvidence,\n }),\n\n // Phase 6 — carry the enqueuing request's correlationId onto the job so the worker that later\n // executes it adopts the same id (request → job trace join). The EC already holds the request's\n // correlationId (it adopted the ambient request frame's id at construction, R13/Model A), so this\n // is the single authoritative source — no separate ALS read needed here.\n originCorrelationId: executionContext.correlationId,\n // #1598 F14: the work frame the job was enqueued inside travels with it, so a model call the worker\n // makes is attributed to the same tenant and case. Engine-written from the ambient frame only.\n ...(() => { const attribution = snapshotWorkAttributionFrame(); return attribution ? { attribution } : {}; })(),\n\n initiatedBy: JobInitiator.API,\n priority: queueConfig.priority ?? JobPriority.NORMAL,\n retryCount: 0,\n maxRetries: queueConfig.maxRetries ?? QUEUE_CONFIG_DEFAULTS.maxRetries,\n // Resolve backoff from the operation's authored queueConfig.retryConfig\n // (read off `operation.queueConfig`, whose type carries the field —\n // QUEUE_CONFIG_DEFAULTS does not), applying defaults per-field so the\n // transported payload carries fully-resolved values the worker can honor.\n retryConfig: {\n baseDelayMs: operation.queueConfig?.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n maxDelayMs: operation.queueConfig?.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n retryPolicy: operation.queueConfig?.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n },\n timeoutMs: queueConfig.timeoutMs ?? QUEUE_CONFIG_DEFAULTS.timeoutMs,\n\n createdAt: new Date(),\n };\n\n return job;\n }\n\n /\n Get operations queue name based on priority.\n * Accepts string because job.priority from schema is a string union.\n /\n private getOperationsQueueName(priority: string): string {\n const base = buildQueueName(this.appId, 'operations');\n return (priority === JobPriority.HIGH \|\| priority === JobPriority.CRITICAL)\n ? buildQueueName(this.appId, 'operations', QueueVariant.PRIORITY)\n : base;\n }\n\n /\n Handle RPC reply from worker.\n /\n private handleReply(correlationId: string, result: JobResult): void {\n const pending = this.pendingReplies.get(correlationId);\n if (!pending) {\n this.logger.warn('Received reply for unknown correlationId', { correlationId });\n return;\n }\n\n clearTimeout(pending.timeout);\n this.pendingReplies.delete(correlationId);\n\n if (result.status === JobStatus.COMPLETED) {\n const parsedResult = result.resultJson ? JSON.parse(result.resultJson) : null;\n pending.resolve(parsedResult);\n } else {\n const error = replyErrorSchema.safeParse(result.error);\n pending.reject(error.success ? error.data : new Error('Job failed without a valid structured error'));\n }\n }\n\n /\n Enqueue a custom batch job.\n \n This method is used for fire-and-forget batch operations that don't\n * need to go through the standard resource operation flow.\n \n @param batchType - Identifier for the batch type (e.g., 'email.batch-send')\n * @param payload - The batch job payload (must be serializable)\n * @param options - Optional configuration for the job\n * @returns The job ID and queue name\n /\n public async enqueueCustomBatch(\n batchType: string,\n payload: unknown,\n options?: {\n priority?: JobPriority;\n maxRetries?: number;\n /* Carried as job metadata only; custom batch execution is not cancelled after this interval. /\n timeoutMs?: number;\n /* Optional backoff timing override (base delay / cap / curve). /\n retryConfig?: QueueRetryBackoffConfig;\n /\n Hold the job on the broker for this long before it becomes consumable (#1066).\n \n Published through the adapter's `scheduleDelayedRetry`, the same durable mechanism the consumer\n * uses for its own retries: a per-delay TTL queue that dead-letters back on RabbitMQ, a timer in\n * memory. It exists so a handler that wants to try again later can hand the wait to the broker\n * instead of sleeping inside the job, which holds the queue's only consumer (`prefetch(1)`) for\n * the whole wait and stalls every job behind it.\n /\n delayMs?: number;\n }\n ): Promise<EnqueueResult> {\n const priority = options?.priority ?? JobPriority.NORMAL;\n const maxRetries = options?.maxRetries ?? QUEUE_CONFIG_DEFAULTS.maxRetries;\n const timeoutMs = options?.timeoutMs ?? QUEUE_CONFIG_DEFAULTS.timeoutMs;\n\n // Serialize payload with the bigint-safe serializer (payload may carry\n // z_money/MoneyValue bigint amounts; see bigIntSafeJsonReplacer).\n let inputDataJson = ResourceSerializationUtils.stringifyWithBigInt(payload);\n let inputPayloadStorageId: string \| undefined;\n\n // Store large payloads externally\n if (this.payloadStorage.shouldStore(inputDataJson)) {\n inputPayloadStorageId = await this.payloadStorage.store(payload);\n inputDataJson = undefined as any;\n }\n\n const job: JobPayload = {\n jobId: uuidv4(),\n jobType: JobType.CUSTOM_BATCH,\n applicationId: this.appId,\n\n // batchRef is REQUIRED for custom batches: `JobExecutorBackendService.executeCustomBatch`\n // throws JOB_BATCH_REF_NOT_FOUND if it's missing. Without this line every custom batch\n // job (email batches, M2M webhook deliveries, …) would fail at the executor.\n batchRef: batchType,\n\n operationPathKey: batchType, // Use batch type as the operation path key\n operationPathJson: JSON.stringify({ batchType }),\n\n inputDataJson,\n inputPayloadStorageId,\n\n executionContextJson: JSON.stringify({\n executionType: 'BATCH',\n initiatorIds: { systemBatch: true },\n }),\n\n // Phase 6 (F1) — a custom batch has no enqueuing EC, but its DISPATCHER (SIEM / M2M-webhook /\n // notification delivery) usually runs inside the TRIGGERING request's frame. Seed\n // originCorrelationId from the ambient frame (SD10 pattern) so the batch job's worker-side logs +\n // audit join that request's trace. `undefined` when dispatched outside any request (e.g. a\n // background flush), in which case the consumer mints a fresh id — still independently traceable.\n originCorrelationId: getCurrentCorrelationId(),\n // #1598 F14: the dispatcher's work frame, like its correlation id above.\n ...(() => { const attribution = snapshotWorkAttributionFrame(); return attribution ? { attribution } : {}; })(),\n\n initiatedBy: JobInitiator.INTERNAL,\n priority,\n retryCount: 0,\n maxRetries,\n retryConfig: {\n baseDelayMs: options?.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n maxDelayMs: options?.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n retryPolicy: options?.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n },\n timeoutMs,\n\n createdAt: new Date(),\n };\n\n // Determine queue based on priority\n const queueName = this.getOperationsQueueName(priority);\n\n // Create job tracking record\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n // Publish to queue — or hand it to the broker's delay mechanism when the caller asked to wait.\n const delayMs = options?.delayMs ?? 0;\n if (delayMs > 0) {\n await this.adapter.scheduleDelayedRetry(queueName, job, delayMs);\n } else {\n await this.adapter.publish(queueName, job);\n }\n\n this.logDebug('Custom batch job enqueued', {\n jobId: job.jobId,\n batchType,\n queueName,\n delayMs,\n hasExternalPayload: !!inputPayloadStorageId,\n });\n\n return { jobId: job.jobId, queueName };\n }\n\n /\n Get RabbitMQ adapter for direct operations.\n /\n public getAdapter(): QueueAdapter {\n return this.adapter;\n }\n\n /\n Close queue service gracefully.\n */\n public async close(): Promise<void> {\n // Clear pending replies\n for (const [correlationId, pending] of this.pendingReplies) {\n clearTimeout(pending.timeout);\n pending.reject(new Error('Queue service closing'));\n }\n this.pendingReplies.clear();\n\n // Close adapter\n await this.adapter.close();\n\n this.initialized = false;\n this.logger.info('Queue service closed');\n }\n}\n"]}
1	+ {"version":3,"file":"queue.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/queue.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;;;;;;;;;;;;;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AACpF,OAAO,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAC7F,OAAO,EAAE,uBAAuB,EAAE,MAAM,sEAAsE,CAAC;AAC/G,OAAO,EAAE,4BAA4B,EAAE,MAAM,oFAAoF,CAAC;AAClI,OAAO,EAAE,6BAA6B,EAAE,MAAM,6CAA6C,CAAC;AAC5F,OAAO,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AAE3H,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AACxF,OAAO,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAExF,OAAO,EAAE,uBAAuB,EAAE,MAAM,+DAA+D,CAAC;AACxG,OAAO,EAAE,4BAA4B,EAAE,MAAM,gEAAgE,CAAC;AAC9G,OAAO,EAEL,gCAAgC,EAChC,sCAAsC,EAGtC,OAAO,EACP,YAAY,EACZ,WAAW,EAEX,SAAS,EACT,YAAY,EAEZ,0BAA0B,EAC1B,kCAAkC,EAClC,qBAAqB,EACrB,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAE/B,4FAA4F;AAC5F,qFAAqF;AACrF,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5F,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,MAAM,CAAC;IACvD,SAAS,EAAE,SAAS;IACpB,aAAa,EAAE,SAAS,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC;AAsBH,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,MAAkC;IAClE,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAAC,MAAkC;IACjE,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9E,CAAC;AAGM,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAaX;IAEA;IAEA;IAEA;IAEA;IAEA;IAtBT,QAAQ,CAA+D;IACzE,MAAM,GAAkB,IAAI,CAAC;IAC7B,cAAc,GAAkB,IAAI,CAAC;IAC5B,cAAc,GAI1B,IAAI,GAAG,EAAE,CAAC;IACP,WAAW,GAAG,KAAK,CAAC;IAE5B,YAEmB,gBAAyC,EAEzC,qBAAmD,EAEnD,MAAqB,EAErB,OAAqB,EAErB,iBAAkD,EAElD,cAA+C;QAV/C,qBAAgB,GAAhB,gBAAgB,CAAyB;QAEzC,0BAAqB,GAArB,qBAAqB,CAA8B;QAEnD,WAAM,GAAN,MAAM,CAAe;QAErB,YAAO,GAAP,OAAO,CAAc;QAErB,sBAAiB,GAAjB,iBAAiB,CAAiC;QAElD,mBAAc,GAAd,cAAc,CAAiC;QAEhE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC9D,kEAAkE;IACpE,CAAC;IAED,6EAA6E;IAC7E,IAAY,KAAK;QACf,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,mBAAmB;QAC9B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CACtD,IAAI,CAAC,KAAK,EACV,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC,CACnE,CAAC;QAEF,6EAA6E;QAC7E,wEAAwE;QACxE,4EAA4E;QAC5E,8FAA8F;QAC9F,oFAAoF;QACpF,0FAA0F;QAC1F,+FAA+F;QAC/F,2FAA2F;QAC3F,4CAA4C;QAC5C,EAAE;QACF,kFAAkF;QAClF,EAAE;QACF,iGAAiG;QACjG,kGAAkG;QAClG,+FAA+F;QAC/F,sFAAsF;QACtF,EAAE;QACF,gGAAgG;QAChG,gGAAgG;QAChG,+FAA+F;QAC/F,6FAA6F;QAC7F,6EAA6E;QAC7E,EAAE;QACF,iGAAiG;QACjG,6DAA6D;QAC7D,EAAE;QACF,gGAAgG;QAChG,iGAAiG;QACjG,+FAA+F;QAC/F,0FAA0F;QAC1F,8FAA8F;QAC9F,mFAAmF;QACnF,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QAC1F,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE;YAC9C,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,+EAA+E;QAC/E,wEAAwE;QACxE,2EAA2E;QAC3E,6EAA6E;QAC7E,8EAA8E;QAC9E,+DAA+D;QAC/D,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE;YAC9C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,eAAe;SACtC,CAAC,CAAC;QAEH,6EAA6E;QAC7E,8DAA8D;QAC9D,iEAAiE;QACjE,+EAA+E;QAC/E,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE;YAC5C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,eAAe;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,uCAAuC,EAAE;YACrD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,cAAc;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACI,KAAK,CAAC,mBAAmB,CAAC,UAA+C,EAAE;QAChF,MAAM,EAAE,qBAAqB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACjD,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,6CAA6C;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;QAEjD;;;;;;;WAOG;QACH,IAAI,SAAS,CAAC,QAAQ,EAAE,QAAQ,IAAI,IAAI,CAAC,OAAO,YAAY,6BAA6B,EAAE,CAAC;YAC1F,+DAA+D;YAC/D,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ;mBACnD,OAAO,CAAC,GAAG,CAAC,iBAAiB;mBAC7B,OAAO,CAAC;YACb,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ;mBACnD,OAAO,CAAC,GAAG,CAAC,iBAAiB;mBAC7B,OAAO,CAAC;YAEb,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;gBACrB,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI;gBACtC,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI;gBACtC,QAAQ;gBACR,QAAQ;gBACR,gEAAgE;gBAChE,gEAAgE;gBAChE,8DAA8D;gBAC9D,iEAAiE;gBACjE,+DAA+D;gBAC/D,4DAA4D;gBAC5D,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK;aACzC,CAAC,CAAC;QACL,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,CAAC;QAED,sDAAsD;QACtD,8CAA8C;QAC9C,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CACtD,IAAI,CAAC,KAAK,EACV,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC,CACnE,CAAC;QAEF,wFAAwF;QACxF,mFAAmF;QACnF,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,mFAAmF,EAAE;gBACjG,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,IAAI,CAAC,cAAc;aAChC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,8EAA8E;QAC9E,6EAA6E;QAC7E,yFAAyF;QACzF,iGAAiG;QACjG,iGAAiG;QACjG,gGAAgG;QAChG,+CAA+C;QAC/C,EAAE;QACF,mEAAmE;QACnE,4EAA4E;QAC5E,4EAA4E;QAC5E,6DAA6D;QAC7D,4EAA4E;QAC5E,qEAAqE;QACrE,6EAA6E;QAC7E,4EAA4E;QAC5E,6EAA6E;QAC7E,2EAA2E;QAC3E,6FAA6F;QAC7F,8FAA8F;QAC9F,mFAAmF;QACnF,MAAM,yBAAyB,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QACpG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,yBAAyB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAE7E,MAAM,wBAAwB,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5E,MAAM,wBAAwB,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5E,2EAA2E;QAC3E,8EAA8E;QAC9E,wEAAwE;QACxE,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE;YAC9C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,yBAAyB;SAChD,CAAC,CAAC;QAEH,gFAAgF;QAChF,iEAAiE;QACjE,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE;YAC5C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,yBAAyB;SAChD,CAAC,CAAC;QAEH,+FAA+F;QAC/F,8FAA8F;QAC9F,6FAA6F;QAC7F,2FAA2F;QAC3F,0FAA0F;QAC1F,uEAAuE;QACvE,EAAE;QACF,8FAA8F;QAC9F,+FAA+F;QAC/F,kEAAkE;QAClE,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,EAAE;YAC7C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,wBAAwB;SAC/C,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,cAAc,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,EAAE;YAC7C,OAAO,EAAE,IAAI;YACb,kBAAkB,EAAE,EAAE;YACtB,oBAAoB,EAAE,wBAAwB;SAC/C,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,uBAAuB,CAAC,CAAC,yBAAyB,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC,CAAC;QAEpH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,sCAAsC,EAAE;YACpD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI,CAAC,cAAc;SAChC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACK,KAAK,CAAC,uBAAuB,CAAC,gBAAmC;QACvE,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YACzC,IAAI,MAAM,GAAkB,IAAI,CAAC;YACjC,IAAI,CAAC;gBAAC,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,MAAM,GAAG,IAAI,CAAC;YAAC,CAAC;YACtF,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE;oBAC1D,SAAS,EAAE,MAAM;oBACjB,MAAM,EAAE,oGAAoG;0BACxG,uGAAuG;0BACvG,+DAA+D;iBACpE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,gBAAgB,CAC3B,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,eAA4C,EAC5C,qBAAwD;QAExD,MAAM,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;QAE7C,iDAAiD;QACjD,MAAM,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErG,oBAAoB;QACpB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;QAErH,oCAAoC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5D,+CAA+C;QAC/C,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,mBAAmB;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE3C,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE;YAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS;YACT,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;YACtC,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,qBAAqB;SAChD,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACzC,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,cAAc,CACzB,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,SAAkB,EAClB,eAA4C,EAC5C,qBAAwD;QAExD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,SAAS,IAAI,WAAW,CAAC,SAAS,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAEtF,sBAAsB;QACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErG,6BAA6B;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;QACrH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,qBAAqB,GAAG,aAAa,CAAC;QAC1C,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;QAElC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5D,kFAAkF;QAClF,kFAAkF;QAClF,oFAAoF;QACpF,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,2BAA2B;QAC3B,MAAM,aAAa,GAAG,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACvD,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;gBACpC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAC1C,MAAM,CAAC,IAAI,KAAK,CAAC,0CAA0C,OAAO,kCAAkC,CAAC,CAAC,CAAC;YACzG,CAAC,EAAE,OAAO,CAAC,CAAC;YAEZ,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;QAEH,gFAAgF;QAChF,gFAAgF;QAChF,iFAAiF;QACjF,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE;YACrE,aAAa;YACb,OAAO,EAAE,cAAc;SACxB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,QAAQ,CAAC,uBAAuB,EAAE;gBACrC,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,mFAAmF;gBACnF,qBAAqB,EAAE,aAAa;gBACpC,SAAS;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YACf,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACvD,IAAI,OAAO,EAAE,CAAC;gBACZ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,qBAAqB,EAAE,aAAa,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7K,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,mBAAmB,CAC9B,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,eAA4C,EAC5C,qBAAwD;QAExD,sBAAsB;QACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAErG,0FAA0F;QAC1F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;QAErH,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5D,+CAA+C;QAC/C,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE3C,IAAI,CAAC,QAAQ,CAAC,4BAA4B,EAAE;YAC1C,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS;YACT,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,qBAAqB;SAChD,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CAC3B,gBAAuC,EACvC,SAAkB,EAClB,OAAwB,EACxB,eAA4C,EAC5C,qBAAwD;QAExD,MAAM,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,qBAAqB,CAAC;QAEnE,0EAA0E;QAC1E,6EAA6E;QAC7E,4EAA4E;QAC5E,0EAA0E;QAC1E,8DAA8D;QAC9D,IAAI,aAAa,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpH,IAAI,qBAAyC,CAAC;QAE9C,uEAAuE;QACvE,IAAI,aAAa,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;YACpE,qBAAqB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACnE,aAAa,GAAG,SAAS,CAAC,CAAC,yBAAyB;YACpD,IAAI,CAAC,QAAQ,CAAC,iCAAiC,EAAE,EAAE,qBAAqB,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,2CAA2C;QAC3C,MAAM,aAAa,GAAG,kCAAkC,CAAC,oBAAoB,CAAC,SAAgB,CAAC,CAAC;QAEhG,MAAM,GAAG,GAAe;YACtB,KAAK,EAAE,MAAM,EAAE;YACf,OAAO,EAAE,OAAO,CAAC,kBAAkB;YACnC,uBAAuB,EAAE,eAAe,CAAC,CAAC,CAAC,gCAAgC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9G,6BAA6B,EAAE,qBAAqB,CAAC,CAAC,CAAC,sCAAsC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS;YACtI,aAAa,EAAE,IAAI,CAAC,KAAK;YAEzB,gBAAgB,EAAE,kCAAkC,CAAC,sBAAsB,CAAC,aAAa,CAAC;YAC1F,iBAAiB,EAAE,0BAA0B,CAAC,sBAAsB,CAAC,aAAa,CAAC;YAEnF,aAAa;YACb,qBAAqB;YACrB,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAE3F,oBAAoB,EAAE,0BAA0B,CAAC,yBAAyB,CAAC;gBACzE,aAAa,EAAE,gBAAgB,CAAC,aAAa;gBAC7C,YAAY,EAAE,gBAAgB,CAAC,YAAY;gBAC3C,cAAc,EAAE,CAAC,GAAG,gBAAgB,CAAC,cAAc,CAAC;gBACpD,qBAAqB,EAAE,GAAG,EAAE,CAAC,gBAAgB,CAAC,qBAAqB,EAAE;gBACrE,oBAAoB,EAAE,CAAC,GAAG,gBAAgB,CAAC,oBAAoB,CAAC;gBAChE,8FAA8F;gBAC9F,gGAAgG;gBAChG,2EAA2E;gBAC3E,GAAG,CAAC,gBAAgB,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnH,GAAG,gBAAgB,CAAC,mBAAmB;aACxC,CAAC;YAEF,8FAA8F;YAC9F,gGAAgG;YAChG,kGAAkG;YAClG,yEAAyE;YACzE,mBAAmB,EAAE,gBAAgB,CAAC,aAAa;YACnD,oGAAoG;YACpG,+FAA+F;YAC/F,GAAG,CAAC,GAAG,EAAE,GAAG,MAAM,WAAW,GAAG,4BAA4B,EAAE,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;YAE/G,WAAW,EAAE,YAAY,CAAC,GAAG;YAC7B,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,WAAW,CAAC,MAAM;YACpD,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,WAAW,CAAC,UAAU,IAAI,qBAAqB,CAAC,UAAU;YACtE,wEAAwE;YACxE,oEAAoE;YACpE,sEAAsE;YACtE,0EAA0E;YAC1E,WAAW,EAAE;gBACX,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBAChG,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBAC7F,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;aACjG;YACD,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,qBAAqB,CAAC,SAAS;YAEnE,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,QAAgB;QAC7C,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACtD,OAAO,CAAC,QAAQ,KAAK,WAAW,CAAC,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,QAAQ,CAAC;YACzE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC;YACjE,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,aAAqB,EAAE,MAAiB;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;QAED,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAE1C,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC9E,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC,CAAC;QACxG,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,KAAK,CAAC,kBAAkB,CAC7B,SAAiB,EACjB,OAAgB,EAChB,OAiBC;QAED,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC;QACzD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,qBAAqB,CAAC,UAAU,CAAC;QAC3E,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,qBAAqB,CAAC,SAAS,CAAC;QAExE,uEAAuE;QACvE,kEAAkE;QAClE,IAAI,aAAa,GAAG,0BAA0B,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC5E,IAAI,qBAAyC,CAAC;QAE9C,kCAAkC;QAClC,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;YACnD,qBAAqB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjE,aAAa,GAAG,SAAgB,CAAC;QACnC,CAAC;QAED,MAAM,GAAG,GAAe;YACtB,KAAK,EAAE,MAAM,EAAE;YACf,OAAO,EAAE,OAAO,CAAC,YAAY;YAC7B,aAAa,EAAE,IAAI,CAAC,KAAK;YAEzB,0FAA0F;YAC1F,uFAAuF;YACvF,6EAA6E;YAC7E,QAAQ,EAAE,SAAS;YAEnB,gBAAgB,EAAE,SAAS,EAAE,2CAA2C;YACxE,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC;YAEhD,aAAa;YACb,qBAAqB;YAErB,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnC,aAAa,EAAE,OAAO;gBACtB,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;aACpC,CAAC;YAEF,8FAA8F;YAC9F,kFAAkF;YAClF,kGAAkG;YAClG,2FAA2F;YAC3F,kGAAkG;YAClG,mBAAmB,EAAE,uBAAuB,EAAE;YAC9C,yEAAyE;YACzE,GAAG,CAAC,GAAG,EAAE,GAAG,MAAM,WAAW,GAAG,4BAA4B,EAAE,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;YAE/G,WAAW,EAAE,YAAY,CAAC,QAAQ;YAClC,QAAQ;YACR,UAAU,EAAE,CAAC;YACb,UAAU;YACV,WAAW,EAAE;gBACX,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;gBAClF,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,IAAI,oBAAoB,CAAC,UAAU;gBAC/E,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,IAAI,oBAAoB,CAAC,WAAW;aACnF;YACD,SAAS;YAET,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,oCAAoC;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAExD,6BAA6B;QAC7B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAE7D,+FAA+F;QAC/F,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QACtC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,2BAA2B,EAAE;YACzC,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS;YACT,SAAS;YACT,OAAO;YACP,kBAAkB,EAAE,CAAC,CAAC,qBAAqB;SAC5C,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACzC,CAAC;IAED;;OAEG;IACI,UAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,wBAAwB;QACxB,KAAK,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3D,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,gBAAgB;QAChB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAE3B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC3C,CAAC;CACF,CAAA;AAjuBY,mBAAmB;IAD/B,UAAU,EAAE;IAaR,WAAA,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3C,WAAA,MAAM,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,CAAA;IAEhD,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IAEjD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;IAEnD,WAAA,MAAM,CAAC,kBAAkB,CAAC,wBAAwB,CAAC,CAAA;qCATjB,uBAAuB;QAElB,4BAA4B;QAE3C,aAAa,UAIF,+BAA+B;QAElC,+BAA+B;GAvBvD,mBAAmB,CAiuB/B","sourcesContent":["/*\n Queue Backend Service\n \n Main entry point for queue operations.\n * Handles enqueuing jobs with authorization checks.\n /\n\nimport { injectable, inject } from 'inversify';\nimport { v4 as uuidv4 } from 'uuid';\nimport { z } from 'zod';\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';\nimport { AuthorizationsBackendService } from '~wildo-backend-lib/security/authorizations/services/authorizations.backend.service';\nimport { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';\nimport { projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport type { QueueAdapter } from './adapters/queue-adapter.backend.interface';\nimport { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';\nimport { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';\nimport { ExecutionContext } from '~wildo-backend-lib/runtime/models/execution-context.backend.schemas';\nimport { getCurrentCorrelationId } from '~wildo-backend-lib/runtime/models/request-context.als.backend';\nimport { snapshotWorkAttributionFrame } from '~wildo-backend-lib/runtime/models/work-attribution.als.backend';\nimport {\n JobPayload,\n JobResourceResponseOptionsSchema,\n JobResourceMutationPreconditionsSchema,\n type JobResourceMutationPreconditions,\n type JobResourceResponseOptions,\n JobType,\n JobInitiator,\n JobPriority,\n JobResult,\n JobStatus,\n QueueVariant,\n QueueRetryBackoffConfig,\n ResourceSerializationUtils,\n ResourcesConfig_OperationPathUtils,\n QUEUE_CONFIG_DEFAULTS,\n QUEUE_RETRY_DEFAULTS,\n buildQueueName,\n WildoError_BackendSchema,\n} from '@wildo-ai/saas-models';\n\n// RabbitMQ JSON transport converts dates to strings. Restore only the declared date fields;\n// validate the whole canonical error before letting it reach the HTTP error handler.\nconst replyDate = z.union([z.date(), z.iso.datetime().transform(value => new Date(value))]);\nconst replyErrorSchema = WildoError_BackendSchema.extend({\n timestamp: replyDate,\n lastRetryTime: replyDate.optional(),\n});\n\n\n/\n Result of an enqueue operation.\n /\nexport interface EnqueueResult {\n /* Unique job identifier /\n jobId: string;\n /* Queue the job was published to /\n queueName: string;\n}\n\n/\n What a queued resource operation is aimed at: ONE row, or the rows a bulk operation resolved.\n \n A bulk target cannot travel in `inputId` — it is a single id — and the selector is stripped from\n * the payload once the service has read it, so before this a bulk operation reached the worker with\n * no target at all (#1461).\n /\nexport type QueueJobTarget = string \| readonly string[];\n\n/* The single-row target, or `undefined` when the job is aimed at a set of rows. /\nexport function singleJobTargetId(target: QueueJobTarget \| undefined): string \| undefined {\n return typeof target === 'string' ? target : undefined;\n}\n\n/* The bulk target, or `undefined` when the job is aimed at one row (or at none). /\nexport function bulkJobTargetIds(target: QueueJobTarget \| undefined): string[] \| undefined {\n return Array.isArray(target) && target.length > 0 ? [...target] : undefined;\n}\n\n@injectable()\nexport class QueueBackendService {\n protected logDebug: (message: string, context?: Record<string, unknown>) => void;\n private _appId: string \| null = null;\n private replyQueueName: string \| null = null;\n private readonly pendingReplies: Map<string, {\n resolve: (result: any) => void;\n reject: (error: unknown) => void;\n timeout: NodeJS.Timeout;\n }> = new Map();\n private initialized = false;\n\n constructor(\n @inject(SAAS_SERVICE_TYPES.AppConfigService)\n private readonly appConfigService: AppConfigurationService,\n @inject(SAAS_SERVICE_TYPES.AuthorizationsService)\n private readonly authorizationsService: AuthorizationsBackendService,\n @inject(SAAS_SERVICE_TYPES.Logger)\n private readonly logger: LoggerService,\n @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)\n private readonly adapter: QueueAdapter,\n @inject(SAAS_SERVICE_TYPES.JobStatusTrackingService)\n private readonly jobStatusTracking: JobStatusTrackingBackendService,\n @inject(SAAS_SERVICE_TYPES.JobPayloadStorageService)\n private readonly payloadStorage: JobPayloadStorageBackendService,\n ) {\n this.logDebug = this.logger.debugByType('QueueBackendService');\n this.logger.setDebugTypeLogging('QueueBackendService', false);\n // Lazy-load appId to avoid accessing config before initialization\n }\n\n /* Lazy getter for applicationId - avoids accessing config in constructor /\n private get appId(): string {\n if (!this._appId) {\n this._appId = this.appConfigService.applicationId;\n }\n return this._appId;\n }\n\n /\n Initialize queue service for API pod.\n * Sets up reply queue for RPC pattern.\n /\n public async initializeForApiPod(): Promise<void> {\n if (this.initialized) return;\n\n // Ensure adapter is connected\n if (!this.adapter.isConnected()) {\n await this.adapter.connect();\n }\n\n // Set up reply queue for RPC\n this.replyQueueName = await this.adapter.setupReplyQueue(\n this.appId,\n (correlationId, result) => this.handleReply(correlationId, result)\n );\n\n // Assert the operations dead-letter queue FIRST — the main queues below name\n // it as their dead-letter target, and asserting the target ahead of the\n // source keeps the topology self-consistent. This queue is TERMINAL: it has\n // no dead-letter args of its own and no consumer. A parked message stays until a human / tool\n // inspects it OR the platform's bound removes it: the per-application broker policy\n // `wildo-deadletter-bound.<appId>` (14-day `message-ttl`, `max-length` 10 000, `overflow:\n // drop-head`), declared by the RabbitMQ provider module when the application's credentials are\n // minted (#2214). A policy rather than an argument, because RabbitMQ refuses to change the\n // arguments of a queue that already exists.\n //\n // ## Dead-letter inspection is a BROKER-level concern — decided 2026-09-08 (#469)\n //\n // Having no consumer is deliberate, not unfinished. Wildo ships no listing of parked jobs and no\n // replay verb, and it is not going to: an operator inspects and replays a parked message with the\n // broker's own tooling (the RabbitMQ management UI or CLI against this queue by name). A DEPTH\n // signal is the one exception, added by #2214 (below), because it exposes no payload.\n //\n // Do not read the absence as an oversight and wire a surface here. A job-execution read surface\n // exposes job PAYLOADS, which may carry personal data, and would need its own authorization and\n // privacy classification — which is why the decision went the other way. The display behaviour\n // that used to be authored for `jobExecutions` / `jobPayloadStorage` was deleted in the same\n // change, because it rendered nowhere and made this absence look accidental.\n //\n // If this ever bites, the option to revisit FIRST is queue DEPTH as a health signal — it catches\n // the fleet-level problem without exposing a single payload.\n //\n // It bit on 2026-10-01 (#2214): `scheduled.deadletter` reached 359 messages for one application\n // in a day of restarts and nothing said so. Two things changed, and the no-consumer decision did\n // not: the queues are now BOUNDED by the broker policy above, and the worker runtime that owns\n // them warns at boot for each non-empty one (`reportParkedDeadLetters` — a depth, never a\n // payload). This API-pod path declares only the operations dead-letter queue and does not own\n // the scheduled/callbacks ones, so the boot report lives in `initializeForWorker`.\n const deadLetterQueue = buildQueueName(this.appId, 'operations', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(deadLetterQueue, {\n durable: true,\n });\n\n // Assert operations queue, dead-lettering permanent failures to the DLQ above.\n // A consumer that exhausts retries (or hits a non-retryable / malformed\n // message) calls `channel.nack(msg, false, false)`; with the DLX wired the\n // broker parks that message in `operations.deadletter` instead of discarding\n // it. Default exchange ('') + routing key = the DLQ name routes by queue name\n // — the same plugin-free mechanism the retry delay queues use.\n const operationsQueue = buildQueueName(this.appId, 'operations');\n await this.adapter.assertQueue(operationsQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: deadLetterQueue,\n });\n\n // Assert priority queue. Priority is a dispatch VARIANT of operations, not a\n // distinct category, so its permanent failures share the SAME\n // `operations.deadletter` (there is intentionally no per-variant\n // `operations.priority.deadletter` — see queue-monitoring.backend.service.ts).\n const priorityQueue = buildQueueName(this.appId, 'operations', QueueVariant.PRIORITY);\n await this.adapter.assertQueue(priorityQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: deadLetterQueue,\n });\n\n this.initialized = true;\n this.logDebug('Queue service initialized for API pod', {\n appId: this.appId,\n replyQueue: this.replyQueueName,\n });\n }\n\n /\n Initialize queue service for worker pod.\n \n This method initializes all queue infrastructure needed for a pod that:\n * - Consumes and processes jobs (worker functionality)\n * - Enqueues jobs and receives RPC replies (API functionality)\n \n Every backend API pod serves both roles, hence this comprehensive initialization.\n \n `ownsSharedJobTopology` (default `true`) decides whether this runtime DECLARES the shared\n * application job queues — `operations`, `operations.priority`, `scheduled`, `callbacks` and their\n * `.deadletter` queues. The runtime that consumes those queues owns them. A runtime that connects\n * only for its own dedicated topology — a minion, which consumes just its `minion-tick.<name>`\n * queue — passes `false` and gets the broker connection and its own reply queue, nothing else.\n \n WHY A NON-OWNER MUST NOT DECLARE THEM ANYWAY. A minion's broker credential is least-privilege:\n * its `read` permission admits only its tick queue and its own reply queue (`buildMinionConsumeRegex`\n * in the rabbitmq-self infra provider module). Measured 2026-09-14 on Wonder Todos: asserting\n * `operations` WITH its `x-dead-letter-exchange` arguments was refused with\n * `403 ACCESS_REFUSED - read access to queue '…operations'`, while the three `.deadletter` queues —\n * declared with no DLX arguments — had just been accepted. The refusal aborted the initialization\n * and left this service uninitialized. Beyond permissions, a second declarer of a queue can only\n * agree with the owner's arguments or break them (RabbitMQ refuses to mutate a live queue's\n * arguments), so the owner is the one place those arguments are stated.\n /\n public async initializeForWorker(options: { ownsSharedJobTopology?: boolean } = {}): Promise<void> {\n const { ownsSharedJobTopology = true } = options;\n if (this.initialized) return;\n\n // Configure RabbitMQ adapter from app config\n const appConfig = this.appConfigService.config;\n const appSecrets = this.appConfigService.secrets;\n\n /\n * Broker connection settings apply to the BROKER adapter, and to nothing else (#502).\n \n `configure()` takes a `RabbitMQConfig` and is deliberately absent from `QueueAdapter` — an\n * in-memory queue has no host, port or vhost to be told about. Narrowing here rather than\n * widening the port keeps that honest: the alternative was a `configure(config: unknown)` every\n * adapter must accept and most must ignore, which is a hole in the port wearing a signature.\n /\n if (appConfig.database?.rabbitmq && this.adapter instanceof RabbitMQAdapterBackendService) {\n // Credentials from secrets first, then env vars, then defaults\n const username = appSecrets.database?.rabbitmq?.username\n \|\| process.env.RABBITMQ_USERNAME\n \|\| 'guest';\n const password = appSecrets.database?.rabbitmq?.password\n \|\| process.env.RABBITMQ_PASSWORD\n \|\| 'guest';\n\n this.adapter.configure({\n host: appConfig.database.rabbitmq.host,\n port: appConfig.database.rabbitmq.port,\n username,\n password,\n // Slice 2.5c-3 of `infra-provider-planner.md`: optional per-app\n // virtual host. Authored at `backingServices.rabbitmq.vhost` in\n // `wildo.infra.<envName>.config.ts`, threaded by apps-manager\n // into the persisted `Wildo_Application_Config`, read here. When\n // omitted the adapter falls back to the default vhost `/` (the\n // adapter's existing `this.config.vhost \|\| '/'` semantics).\n vhost: appConfig.database.rabbitmq.vhost,\n });\n }\n\n // Ensure adapter is connected\n if (!this.adapter.isConnected()) {\n await this.adapter.connect();\n }\n\n // Set up reply queue for RPC pattern (enqueueAndWait)\n // Each pod gets its own exclusive reply queue\n this.replyQueueName = await this.adapter.setupReplyQueue(\n this.appId,\n (correlationId, result) => this.handleReply(correlationId, result)\n );\n\n // A runtime that does not consume the shared job queues does not own them, and does not\n // declare them (see the method JSDoc: a minion's credential cannot, and must not).\n if (!ownsSharedJobTopology) {\n this.initialized = true;\n this.logDebug('Queue service initialized for a runtime that does not own the shared job topology', {\n appId: this.appId,\n replyQueue: this.replyQueueName,\n });\n return;\n }\n\n // Assert the dead-letter queues FIRST — each is the dead-letter target of the\n // matching main queue below, so it must be declared ahead of the source that\n // names it. Each DLQ is TERMINAL (no dead-letter args of its own, no consumer): a parked\n // message stays until inspected or until the platform's per-application broker POLICY removes it\n // (14-day TTL, at most 10 000 messages, oldest dropped first — see the operations-DLQ comment in\n // `initializeForApiPod` and `RABBITMQ_DEAD_LETTER_QUEUE_BOUND`, #2214). Their depth is reported\n // at boot by `reportParkedDeadLetters`, below.\n //\n // TOPOLOGY-MIGRATION NOTE: the main queues below are asserted WITH\n // `x-dead-letter-` arguments. RabbitMQ will NOT mutate the arguments of an\n // already-existing durable queue — asserting `operations` with new DLX args\n // when it already exists as plain `{ durable: true }` throws\n // 406 PRECONDITION_FAILED and closes the channel. On deployments that ran a\n // pre-DLX build, the existing `operations` / `operations.priority` /\n // `scheduled` / `callbacks` / `domain.` queues must be DELETED ONCE so they\n // are auto-recreated here with the DLX args. This project is greenfield (no\n // production data), so a one-time manual redeclare is the chosen migration —\n // see `.claude/plans/queue-permanent-dlx.md` for the rejected alternatives\n // (versioning the queue name; broker-side policies). That rejection is about the DLX ROUTING\n // arguments, which stay queue arguments. The dead-letter queues' BOUND is a different concern\n // and IS a broker policy (#2214), because it must reach queues that already exist.\n const operationsDeadLetterQueue = buildQueueName(this.appId, 'operations', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(operationsDeadLetterQueue, { durable: true });\n\n const scheduledDeadLetterQueue = buildQueueName(this.appId, 'scheduled', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(scheduledDeadLetterQueue, { durable: true });\n\n const callbacksDeadLetterQueue = buildQueueName(this.appId, 'callbacks', QueueVariant.DEADLETTER);\n await this.adapter.assertQueue(callbacksDeadLetterQueue, { durable: true });\n\n // Assert operations queue, dead-lettering permanent failures (nack without\n // requeue) to `operations.deadletter` via the default exchange ('') routed by\n // queue name. Without this a permanently-failed job would be discarded.\n const operationsQueue = buildQueueName(this.appId, 'operations');\n await this.adapter.assertQueue(operationsQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: operationsDeadLetterQueue,\n });\n\n // Priority is a dispatch variant of operations → shares `operations.deadletter`\n // (no per-variant DLQ; see queue-monitoring.backend.service.ts).\n const priorityQueue = buildQueueName(this.appId, 'operations', QueueVariant.PRIORITY);\n await this.adapter.assertQueue(priorityQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: operationsDeadLetterQueue,\n });\n\n // Assert scheduled queue → parks the scheduled deliveries this application could not accept in\n // `scheduled.deadletter` for inspection: deliveries that are not authentic (a trust failure),\n // addressed to another application, or authentic-but-late ones whose missed occurrence could\n // not be RECORDED. A late delivery that was recorded is acknowledged instead (#2212) — its\n // durable record is the evidence, so the broker no longer keeps a copy of every restart's\n // backlog. The schedule continues with its next occurrence either way.\n //\n // The dead-letter queues carry no arguments of their own; their bound (TTL + max length) is a\n // broker POLICY applied when the platform provisions the application (#2214), because RabbitMQ\n // refuses to change the arguments of a queue that already exists.\n const scheduledQueue = buildQueueName(this.appId, 'scheduled');\n await this.adapter.assertQueue(scheduledQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: scheduledDeadLetterQueue,\n });\n\n // Assert callbacks queue → dead-letters to `callbacks.deadletter`.\n const callbacksQueue = buildQueueName(this.appId, 'callbacks');\n await this.adapter.assertQueue(callbacksQueue, {\n durable: true,\n deadLetterExchange: '',\n deadLetterRoutingKey: callbacksDeadLetterQueue,\n });\n\n await this.reportParkedDeadLetters([operationsDeadLetterQueue, scheduledDeadLetterQueue, callbacksDeadLetterQueue]);\n\n this.initialized = true;\n this.logDebug('Queue service initialized for worker', {\n appId: this.appId,\n replyQueue: this.replyQueueName,\n });\n }\n\n /\n Say, once at boot, which dead-letter queues hold parked messages (#2214).\n \n Dead-letter queues have no consumer by decision (#469): an operator inspects and replays a parked\n * message with the broker's own tooling, because a job payload may carry personal data. What that\n * decision left out is any way to learn that something IS parked — `scheduled.deadletter` reached\n * 359 messages in one day of local restarts and nothing said so. #469 named queue DEPTH as the\n * first thing to revisit, and this is that: a count and a queue name, never a payload.\n \n At boot, because a restart is exactly when a backlog lands, and in the runtime that just declared\n * these queues (so they exist: a passive check of a missing queue closes the channel). Best-effort:\n * a depth that cannot be read is skipped, never a reason to fail initialization.\n /\n private async reportParkedDeadLetters(deadLetterQueues: readonly string[]): Promise<void> {\n for (const queueName of deadLetterQueues) {\n let parked: number \| null = null;\n try { parked = await this.adapter.getQueueDepth(queueName); } catch { parked = null; }\n if (parked !== null && parked > 0) {\n this.logger.warn('Dead-letter queue holds parked messages', {\n queueName, parked,\n remedy: 'These are deliveries this application could not accept. Inspect or replay them with the broker\\'s '\n + 'own tooling (the RabbitMQ management UI or CLI, against this queue by name). The platform bounds the '\n + 'queue by age and length, so old messages expire on their own.',\n });\n }\n }\n }\n\n /\n Enqueue operation (fire and forget).\n * Returns immediately after publishing.\n /\n public async enqueueOperation(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<EnqueueResult> {\n const operation = executionContext.operation;\n\n // Authorization check (same as controller layer)\n await this.authorizationsService.authorizeInController(singleJobTargetId(inputId), executionContext);\n\n // Build job payload\n const job = await this.buildJobPayload(executionContext, inputData, inputId, responseOptions, mutationPreconditions);\n\n // Determine queue based on priority\n const queueName = this.getOperationsQueueName(job.priority);\n\n // Create job tracking record before publishing\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n // Publish to queue\n await this.adapter.publish(queueName, job);\n\n this.logDebug('Job enqueued', {\n jobId: job.jobId,\n queueName,\n operationPathKey: job.operationPathKey,\n hasExternalPayload: !!job.inputPayloadStorageId,\n });\n\n return { jobId: job.jobId, queueName };\n }\n\n /\n Enqueue operation and wait for result (RPC pattern).\n * Used for QUEUED runtime mode. timeoutMs bounds reply waiting from immediately\n * before publication; it includes broker waiting and execution, not authorization\n * or payload/execution-record preparation. Expiry forgets the reply correlation only: it neither\n * cancels the worker nor determines whether its business effects happened.\n * A late reply is ignored; callers must not infer that retrying is safe.\n /\n public async enqueueAndWait<T>(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n timeoutMs?: number,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<T> {\n if (!this.replyQueueName) {\n throw new Error('Reply queue not initialized. Call initializeForApiPod() first.');\n }\n\n const operation = executionContext.operation;\n const queueConfig = operation.queueConfig ?? QUEUE_CONFIG_DEFAULTS;\n const timeout = timeoutMs ?? queueConfig.timeoutMs ?? QUEUE_CONFIG_DEFAULTS.timeoutMs;\n\n // Authorization check\n await this.authorizationsService.authorizeInController(singleJobTargetId(inputId), executionContext);\n\n // Build job with RPC headers\n const job = await this.buildJobPayload(executionContext, inputData, inputId, responseOptions, mutationPreconditions);\n const correlationId = uuidv4();\n job.rpcReplyCorrelationId = correlationId;\n job.replyTo = this.replyQueueName;\n\n const replyQueueName = this.replyQueueName;\n const queueName = this.getOperationsQueueName(job.priority);\n\n // The consumer only updates this record; RPC has the same producer-owned creation\n // lifecycle as the other operation modes. Finish persistence before registering a\n // reply deadline so a failed create cannot publish untracked work or leak a waiter.\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n // Create promise for reply\n const resultPromise = new Promise<T>((resolve, reject) => {\n const timeoutHandle = setTimeout(() => {\n this.pendingReplies.delete(correlationId);\n reject(new Error(`Timed out waiting for job result after ${timeout}ms; execution may still complete`));\n }, timeout);\n\n this.pendingReplies.set(correlationId, { resolve, reject, timeout: timeoutHandle });\n });\n\n // Adopt the reply promise immediately. Waiting for publish first would hide the\n // deadline from the caller and could leave its rejection temporarily unhandled.\n // Publication can finish after expiry: timeout still does not mean cancellation.\n void Promise.resolve().then(() => this.adapter.publish(queueName, job, {\n correlationId,\n replyTo: replyQueueName,\n })).then(() => {\n this.logDebug('Job enqueued with RPC', {\n jobId: job.jobId,\n // RPC reply matching is separate from the originating request's trace correlation.\n rpcReplyCorrelationId: correlationId,\n queueName,\n });\n }).catch(error => {\n const pending = this.pendingReplies.get(correlationId);\n if (pending) {\n clearTimeout(pending.timeout);\n this.pendingReplies.delete(correlationId);\n pending.reject(error);\n } else {\n this.logger.warn('RPC publication failed after the caller was settled', { jobId: job.jobId, rpcReplyCorrelationId: correlationId, ...projectCaughtErrorLogFields(error) });\n }\n });\n\n return resultPromise;\n }\n\n /\n Enqueue with callback (for BACKGROUND_JOB).\n * Returns job ID immediately, result delivered via callback mechanism.\n /\n public async enqueueWithCallback(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<EnqueueResult> {\n // Authorization check\n await this.authorizationsService.authorizeInController(singleJobTargetId(inputId), executionContext);\n\n // Build job (completion handlers run in the consuming process through its local registry)\n const job = await this.buildJobPayload(executionContext, inputData, inputId, responseOptions, mutationPreconditions);\n\n // Publish to operations queue\n const queueName = this.getOperationsQueueName(job.priority);\n\n // Create job tracking record before publishing\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n await this.adapter.publish(queueName, job);\n\n this.logDebug('Job enqueued with callback', {\n jobId: job.jobId,\n queueName,\n hasExternalPayload: !!job.inputPayloadStorageId,\n });\n\n return { jobId: job.jobId, queueName };\n }\n\n /\n Build job payload from execution context and input data.\n * Stores large payloads (>100KB) externally and stores only the storage ID.\n /\n private async buildJobPayload(\n executionContext: ExecutionContext<any>,\n inputData: unknown,\n inputId?: QueueJobTarget,\n responseOptions?: JobResourceResponseOptions,\n mutationPreconditions?: JobResourceMutationPreconditions\n ): Promise<JobPayload> {\n const operation = executionContext.operation;\n const queueConfig = operation.queueConfig ?? QUEUE_CONFIG_DEFAULTS;\n\n // Serialize input data. Use the bigint-safe serializer: a resource `data`\n // payload may carry `z_money` (MoneyValue) amounts stored as `bigint`, which\n // a bare `JSON.stringify` cannot serialize (\"Do not know how to serialize a\n // BigInt\"). See ResourceSerializationUtils.bigIntSafeJsonReplacer for the\n // string-encoding rationale and the worker round-trip caveat.\n let inputDataJson = inputData !== undefined ? ResourceSerializationUtils.stringifyWithBigInt(inputData) : undefined;\n let inputPayloadStorageId: string \| undefined;\n\n // Store large payloads externally to avoid RabbitMQ performance issues\n if (inputDataJson && this.payloadStorage.shouldStore(inputDataJson)) {\n inputPayloadStorageId = await this.payloadStorage.store(inputData);\n inputDataJson = undefined; // Don't embed in message\n this.logDebug('Large payload stored externally', { inputPayloadStorageId });\n }\n\n // Extract operation path for serialization\n const operationPath = ResourcesConfig_OperationPathUtils.extractOperationPath(operation as any);\n\n const job: JobPayload = {\n jobId: uuidv4(),\n jobType: JobType.RESOURCE_OPERATION,\n resourceResponseOptions: responseOptions ? JobResourceResponseOptionsSchema.parse(responseOptions) : undefined,\n resourceMutationPreconditions: mutationPreconditions ? JobResourceMutationPreconditionsSchema.parse(mutationPreconditions) : undefined,\n applicationId: this.appId,\n\n operationPathKey: ResourcesConfig_OperationPathUtils.createOperationPathKey(operationPath),\n operationPathJson: ResourceSerializationUtils.serializeOperationPath(operationPath),\n\n inputDataJson,\n inputPayloadStorageId,\n ...(singleJobTargetId(inputId) !== undefined ? { inputId: singleJobTargetId(inputId) } : {}),\n ...(bulkJobTargetIds(inputId) !== undefined ? { inputIds: bulkJobTargetIds(inputId) } : {}),\n\n executionContextJson: ResourceSerializationUtils.serializeExecutionContext({\n executionType: executionContext.executionType,\n initiatorIds: executionContext.initiatorIds,\n initiatorRoles: [...executionContext.initiatorRoles],\n getContextResourceIds: () => executionContext.getContextResourceIds(),\n relationshipContexts: [...executionContext.relationshipContexts],\n // Attribution evidence, so the worker's audit rows name the same on-behalf-of user, agent and\n // delegating client as the request that enqueued the job. The worker rebuilds from these alone:\n // no ambient frame reaches a delivery (see the in-memory adapter's drain).\n ...(executionContext.auditActorUserId !== undefined ? { auditActorUserId: executionContext.auditActorUserId } : {}),\n ...executionContext.attributionEvidence,\n }),\n\n // Phase 6 — carry the enqueuing request's correlationId onto the job so the worker that later\n // executes it adopts the same id (request → job trace join). The EC already holds the request's\n // correlationId (it adopted the ambient request frame's id at construction, R13/Model A), so this\n // is the single authoritative source — no separate ALS read needed here.\n originCorrelationId: executionContext.correlationId,\n // #1598 F14: the work frame the job was enqueued inside travels with it, so a model call the worker\n // makes is attributed to the same tenant and case. Engine-written from the ambient frame only.\n ...(() => { const attribution = snapshotWorkAttributionFrame(); return attribution ? { attribution } : {}; })(),\n\n initiatedBy: JobInitiator.API,\n priority: queueConfig.priority ?? JobPriority.NORMAL,\n retryCount: 0,\n maxRetries: queueConfig.maxRetries ?? QUEUE_CONFIG_DEFAULTS.maxRetries,\n // Resolve backoff from the operation's authored queueConfig.retryConfig\n // (read off `operation.queueConfig`, whose type carries the field —\n // QUEUE_CONFIG_DEFAULTS does not), applying defaults per-field so the\n // transported payload carries fully-resolved values the worker can honor.\n retryConfig: {\n baseDelayMs: operation.queueConfig?.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n maxDelayMs: operation.queueConfig?.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n retryPolicy: operation.queueConfig?.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n },\n timeoutMs: queueConfig.timeoutMs ?? QUEUE_CONFIG_DEFAULTS.timeoutMs,\n\n createdAt: new Date(),\n };\n\n return job;\n }\n\n /\n Get operations queue name based on priority.\n * Accepts string because job.priority from schema is a string union.\n /\n private getOperationsQueueName(priority: string): string {\n const base = buildQueueName(this.appId, 'operations');\n return (priority === JobPriority.HIGH \|\| priority === JobPriority.CRITICAL)\n ? buildQueueName(this.appId, 'operations', QueueVariant.PRIORITY)\n : base;\n }\n\n /\n Handle RPC reply from worker.\n /\n private handleReply(correlationId: string, result: JobResult): void {\n const pending = this.pendingReplies.get(correlationId);\n if (!pending) {\n this.logger.warn('Received reply for unknown correlationId', { correlationId });\n return;\n }\n\n clearTimeout(pending.timeout);\n this.pendingReplies.delete(correlationId);\n\n if (result.status === JobStatus.COMPLETED) {\n const parsedResult = result.resultJson ? JSON.parse(result.resultJson) : null;\n pending.resolve(parsedResult);\n } else {\n const error = replyErrorSchema.safeParse(result.error);\n pending.reject(error.success ? error.data : new Error('Job failed without a valid structured error'));\n }\n }\n\n /\n Enqueue a custom batch job.\n \n This method is used for fire-and-forget batch operations that don't\n * need to go through the standard resource operation flow.\n \n @param batchType - Identifier for the batch type (e.g., 'email.batch-send')\n * @param payload - The batch job payload (must be serializable)\n * @param options - Optional configuration for the job\n * @returns The job ID and queue name\n /\n public async enqueueCustomBatch(\n batchType: string,\n payload: unknown,\n options?: {\n priority?: JobPriority;\n maxRetries?: number;\n /* Carried as job metadata only; custom batch execution is not cancelled after this interval. /\n timeoutMs?: number;\n /* Optional backoff timing override (base delay / cap / curve). /\n retryConfig?: QueueRetryBackoffConfig;\n /\n Hold the job on the broker for this long before it becomes consumable (#1066).\n \n Published through the adapter's `scheduleDelayedRetry`, the same durable mechanism the consumer\n * uses for its own retries: a per-delay TTL queue that dead-letters back on RabbitMQ, a timer in\n * memory. It exists so a handler that wants to try again later can hand the wait to the broker\n * instead of sleeping inside the job, which holds the queue's only consumer (`prefetch(1)`) for\n * the whole wait and stalls every job behind it.\n /\n delayMs?: number;\n }\n ): Promise<EnqueueResult> {\n const priority = options?.priority ?? JobPriority.NORMAL;\n const maxRetries = options?.maxRetries ?? QUEUE_CONFIG_DEFAULTS.maxRetries;\n const timeoutMs = options?.timeoutMs ?? QUEUE_CONFIG_DEFAULTS.timeoutMs;\n\n // Serialize payload with the bigint-safe serializer (payload may carry\n // z_money/MoneyValue bigint amounts; see bigIntSafeJsonReplacer).\n let inputDataJson = ResourceSerializationUtils.stringifyWithBigInt(payload);\n let inputPayloadStorageId: string \| undefined;\n\n // Store large payloads externally\n if (this.payloadStorage.shouldStore(inputDataJson)) {\n inputPayloadStorageId = await this.payloadStorage.store(payload);\n inputDataJson = undefined as any;\n }\n\n const job: JobPayload = {\n jobId: uuidv4(),\n jobType: JobType.CUSTOM_BATCH,\n applicationId: this.appId,\n\n // batchRef is REQUIRED for custom batches: `JobExecutorBackendService.executeCustomBatch`\n // throws JOB_BATCH_REF_NOT_FOUND if it's missing. Without this line every custom batch\n // job (email batches, M2M webhook deliveries, …) would fail at the executor.\n batchRef: batchType,\n\n operationPathKey: batchType, // Use batch type as the operation path key\n operationPathJson: JSON.stringify({ batchType }),\n\n inputDataJson,\n inputPayloadStorageId,\n\n executionContextJson: JSON.stringify({\n executionType: 'BATCH',\n initiatorIds: { systemBatch: true },\n }),\n\n // Phase 6 (F1) — a custom batch has no enqueuing EC, but its DISPATCHER (SIEM / M2M-webhook /\n // notification delivery) usually runs inside the TRIGGERING request's frame. Seed\n // originCorrelationId from the ambient frame (SD10 pattern) so the batch job's worker-side logs +\n // audit join that request's trace. `undefined` when dispatched outside any request (e.g. a\n // background flush), in which case the consumer mints a fresh id — still independently traceable.\n originCorrelationId: getCurrentCorrelationId(),\n // #1598 F14: the dispatcher's work frame, like its correlation id above.\n ...(() => { const attribution = snapshotWorkAttributionFrame(); return attribution ? { attribution } : {}; })(),\n\n initiatedBy: JobInitiator.INTERNAL,\n priority,\n retryCount: 0,\n maxRetries,\n retryConfig: {\n baseDelayMs: options?.retryConfig?.baseDelayMs ?? QUEUE_RETRY_DEFAULTS.baseDelayMs,\n maxDelayMs: options?.retryConfig?.maxDelayMs ?? QUEUE_RETRY_DEFAULTS.maxDelayMs,\n retryPolicy: options?.retryConfig?.retryPolicy ?? QUEUE_RETRY_DEFAULTS.retryPolicy,\n },\n timeoutMs,\n\n createdAt: new Date(),\n };\n\n // Determine queue based on priority\n const queueName = this.getOperationsQueueName(priority);\n\n // Create job tracking record\n await this.jobStatusTracking.createJobRecord(job, queueName);\n\n // Publish to queue — or hand it to the broker's delay mechanism when the caller asked to wait.\n const delayMs = options?.delayMs ?? 0;\n if (delayMs > 0) {\n await this.adapter.scheduleDelayedRetry(queueName, job, delayMs);\n } else {\n await this.adapter.publish(queueName, job);\n }\n\n this.logDebug('Custom batch job enqueued', {\n jobId: job.jobId,\n batchType,\n queueName,\n delayMs,\n hasExternalPayload: !!inputPayloadStorageId,\n });\n\n return { jobId: job.jobId, queueName };\n }\n\n /\n Get RabbitMQ adapter for direct operations.\n /\n public getAdapter(): QueueAdapter {\n return this.adapter;\n }\n\n /\n Close queue service gracefully.\n */\n public async close(): Promise<void> {\n // Clear pending replies\n for (const [correlationId, pending] of this.pendingReplies) {\n clearTimeout(pending.timeout);\n pending.reject(new Error('Queue service closing'));\n }\n this.pendingReplies.clear();\n\n // Close adapter\n await this.adapter.close();\n\n this.initialized = false;\n this.logger.info('Queue service closed');\n }\n}\n"]}

package/dist/esm/queue/scheduled-evidence-coverage.backend.d.ts CHANGED Viewed

@@ -89,5 +89,13 @@ export declare function assessScheduledEvidenceCoverage(input: {
     unavailableResults: ScheduledEvidenceObservation[];
     deferred: ScheduledEvidenceObservation[];
     unexpected: ScheduledEvidenceObservation[];
+    /**
+     * Occurrences that were never run because their delivery reached the application after its
+     * admission deadline (#2212) — usually because the application was not consuming; see
+     * `ScheduledDeliveryLateness` for the case the scheduler shares. Their slots stay in `missing`
+     * (they did not run) and they are also in `unsuccessful`; listed apart because their cause is
+     * RECORDED, which is what distinguishes a restart's gap from an unexplained one.
+     */
+    lateDelivered: ScheduledEvidenceObservation[];
 }>;
 //# sourceMappingURL=scheduled-evidence-coverage.backend.d.ts.map

package/dist/esm/queue/scheduled-evidence-coverage.backend.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"scheduled-evidence-coverage.backend.d.ts","sourceRoot":"","sources":["../../../../src/queue/scheduled-evidence-coverage.backend.ts"],"names":[],"mappings":"AACA,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,uBAAuB,EAEvB,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAE7F,oBAAY,yBAAyB;IACnC,QAAQ,aAAa;IACrB,UAAU,eAAe;IACzB,OAAO,YAAY;CACpB;AACD,gHAAgH;AAChH,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,IAAI,CAAC;IACX,EAAE,EAAE,IAAI,CAAC;IACT,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB;AACD,MAAM,WAAW,4BAA4B;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,gBAAgB,EAAE,kBAAkB,CAAC;IACrC,MAAM,CAAC,EAAE,uBAAuB,CAAC;IACjC,YAAY,CAAC,EAAE,6BAA6B,CAAC;IAC7C,OAAO,CAAC,EAAE,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC1C,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sFAAsF;IACtF,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED;;;;;;;GAOG;AACH,wBAAsB,+BAA+B,CACnD,KAAK,EAAE;IACL;;;;;;OAMG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjG,IAAI,EAAE,IAAI,CAAC;IACX,EAAE,EAAE,IAAI,CAAC;IACT,gBAAgB,EAAE,gCAAgC,EAAE,CAAC;IACrD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,EACD,YAAY,EAAE;IACZ,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,uBAAuB,CAAC,CAAC;IACzE,OAAO,EAAE,IAAI,CAAC,+BAA+B,EAAE,iCAAiC,CAAC,CAAC;CACnF;;;mBAVsB,MAAM;mBAAa,MAAM;2BAAqB,MAAM;qBAAe,MAAM;;;;;;;;;;;;;;;;;;;;;;;~~GA+KjG~~"}
1	+ {"version":3,"file":"scheduled-evidence-coverage.backend.d.ts","sourceRoot":"","sources":["../../../../src/queue/scheduled-evidence-coverage.backend.ts"],"names":[],"mappings":"AACA,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,uBAAuB,EAEvB,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAE7F,oBAAY,yBAAyB;IACnC,QAAQ,aAAa;IACrB,UAAU,eAAe;IACzB,OAAO,YAAY;CACpB;AACD,gHAAgH;AAChH,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,IAAI,CAAC;IACX,EAAE,EAAE,IAAI,CAAC;IACT,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB;AACD,MAAM,WAAW,4BAA4B;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,WAAW,CAAC,EAAE,IAAI,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,gBAAgB,EAAE,kBAAkB,CAAC;IACrC,MAAM,CAAC,EAAE,uBAAuB,CAAC;IACjC,YAAY,CAAC,EAAE,6BAA6B,CAAC;IAC7C,OAAO,CAAC,EAAE,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC1C,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sFAAsF;IACtF,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED;;;;;;;GAOG;AACH,wBAAsB,+BAA+B,CACnD,KAAK,EAAE;IACL;;;;;;OAMG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjG,IAAI,EAAE,IAAI,CAAC;IACX,EAAE,EAAE,IAAI,CAAC;IACT,gBAAgB,EAAE,gCAAgC,EAAE,CAAC;IACrD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,EACD,YAAY,EAAE;IACZ,QAAQ,EAAE,IAAI,CAAC,+BAA+B,EAAE,uBAAuB,CAAC,CAAC;IACzE,OAAO,EAAE,IAAI,CAAC,+BAA+B,EAAE,iCAAiC,CAAC,CAAC;CACnF;;;mBAVsB,MAAM;mBAAa,MAAM;2BAAqB,MAAM;qBAAe,MAAM;;;;;;;;;;;;;;;;;;;;;;;IAwL9F;;;;;;OAMG;;GAGN"}

package/dist/esm/queue/scheduled-evidence-coverage.backend.js CHANGED Viewed

@@ -133,6 +133,9 @@ export async function assessScheduledEvidenceCoverage(input, dependencies) {
             completedAt: row.completedAt,
             invocationStatus: ScheduledRunEvidenceSchema.shape.invocationStatus.parse(row.status),
             retained: false,
+            // A live row carries no archived source, but a late-delivery row says what it is structurally
+            // (#2212) — so a restart's gap is explained even for a target that retains no evidence.
+            ...(row.scheduledState?.lateness !== undefined ? { source: ScheduledEvidenceSource.LATE_DELIVERY } : {}),
         });
     }
     const values = [...observations.values()];
@@ -151,7 +154,12 @@ export async function assessScheduledEvidenceCoverage(input, dependencies) {
         return {
             scheduledAt: new Date(instant),
             runs,
-            retained: runs.some((value) => value.retained),
+            // Covered only by evidence of an occurrence that was DEALT WITH. A late delivery is retained
+            // evidence that the occurrence did NOT run (#2212), so it explains the slot — `lateDelivered`
+            // names it — but never covers it: counting it would let one long restart report a window
+            // COMPLETE, and would give the same event two verdicts depending on whether the target keeps
+            // an evidence policy (archived → covered, live row → missing).
+            retained: runs.some((value) => value.retained && value.source !== ScheduledEvidenceSource.LATE_DELIVERY),
             awaitingStartWindow: instant + input.allowedStartDelayMs > now,
         };
     });
@@ -179,9 +187,19 @@ export async function assessScheduledEvidenceCoverage(input, dependencies) {
         slots,
         unknownTime: values.filter((value) => !value.scheduledAt),
         unsuccessful: values.filter((value) => value.invocationStatus !== JobExecutionStatus.COMPLETED),
-        unavailableResults: values.filter((value) => !value.retained || value.resultStatus !== ScheduledEvidenceResultStatus.CAPTURED),
+        // A late delivery is not a run, so it has no result to be unavailable: it is reported once, in
+        // `lateDelivered`, never also as a "retained run with no readable result" (#2212).
+        unavailableResults: values.filter((value) => value.source !== ScheduledEvidenceSource.LATE_DELIVERY && (!value.retained || value.resultStatus !== ScheduledEvidenceResultStatus.CAPTURED)),
         deferred: values.filter((value) => value.deferred === true),
         unexpected: values.filter((value) => value.scheduledAt && !expectedInstants.has(value.scheduledAt.getTime())),
+        /**
+         * Occurrences that were never run because their delivery reached the application after its
+         * admission deadline (#2212) — usually because the application was not consuming; see
+         * `ScheduledDeliveryLateness` for the case the scheduler shares. Their slots stay in `missing`
+         * (they did not run) and they are also in `unsuccessful`; listed apart because their cause is
+         * RECORDED, which is what distinguishes a restart's gap from an unexplained one.
+         */
+        lateDelivered: values.filter((value) => value.source === ScheduledEvidenceSource.LATE_DELIVERY),
     };
 }
 //# sourceMappingURL=scheduled-evidence-coverage.backend.js.map

package/dist/esm/queue/scheduled-evidence-coverage.backend.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"scheduled-evidence-coverage.backend.js","sourceRoot":"","sources":["../../../../src/queue/scheduled-evidence-coverage.backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,uBAAuB,EACvB,0BAA0B,GAE3B,MAAM,uBAAuB,CAAC;AAI/B,MAAM,CAAN,IAAY,yBAIX;AAJD,WAAY,yBAAyB;IACnC,kDAAqB,CAAA;IACrB,sDAAyB,CAAA;IACzB,gDAAmB,CAAA;AACrB,CAAC,EAJW,yBAAyB,KAAzB,yBAAyB,QAIpC;AAyBD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,KAeC,EACD,YAGC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC;IACnC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAC/B,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,EACvB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACnB,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QACtB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpB,IAAI,IAAI,EAAE;QACV,CAAC,KAAK,CAAC,aAAa;QACpB,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS;QACvB,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;QACxB,KAAK,GAAG,CAAC;QACT,KAAK,GAAG,KAAK;QACb,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC;QAC3C,KAAK,CAAC,mBAAmB,GAAG,CAAC;QAC7B,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACrJ,CAAC,EACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACjG,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAClC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;QAC7B,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvB,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YACrB,KAAK,IAAI,GAAG;YACZ,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE;YAC5B,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE;YACxB,OAAO,OAAO,CAAC,OAAO,KAAK,SAAS,EACpC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,GAAG,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE;YAAE,SAAS;QACzC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,EAC9B,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,GAAG,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACzE,IAAI,EAAE,GAAG,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC/F,MAAM,GAAG,EAAE,CAAC;QACZ,yGAAyG;QACzG,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,OAAO,IAAI,EAAE;oBAAE,MAAM;gBACzB,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;oBAC9B,iBAAiB,GAAG,IAAI,CAAC;oBACzB,MAAM;gBACR,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,MAAM,GAAG,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC/F,IAAI,EAAE,GAAG,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IACxE,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxC,YAAY,CAAC,QAAQ,CAAC,qBAAqB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC;QACtF,YAAY,CAAC,OAAO,CAAC,+BAA+B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC;KAChG,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAwC,CAAC;IACrE,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC9H,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;YAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,QAAQ,EAAE,IAAI;YACd,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;YAC3C,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,sBAAsB,EACpB,QAAQ,CAAC,MAAM,KAAK,uBAAuB,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,WAAW;gBACjG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,mBAAmB;gBAC5F,CAAC,CAAC,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1C,IAAI,WAA6B,CAAC;QAClC,yFAAyF;QACzF,mFAAmF;QACnF,IAAI,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpE,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAC3D,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;oBAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;wBAAE,WAAW,GAAG,WAAW,CAAC;gBACxE,CAAC;gBACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;oBACtC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;oBAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;wBAAE,WAAW,GAAG,SAAS,CAAC;gBACpE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0EAA0E;YAC5E,CAAC;QACH,CAAC;QACD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YAAE,SAAS;QAC3F,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YAAE,SAAS;QACvJ,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE;YAC1B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,WAAW;YACX,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,gBAAgB,EAAE,0BAA0B,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YACrF,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0C,CAAC;IACpE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,WAAW;YAAE,SAAS;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACL,WAAW,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC;YAC9B,IAAI;YACJ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC9C,mBAAmB,EAAE,OAAO,GAAG,KAAK,CAAC,mBAAmB,GAAG,GAAG;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC;IAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvF,MAAM,iBAAiB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvG,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrF,OAAO;QACL,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,gBAAgB,EAAE,QAAQ;QAC1B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,QAAQ,EAAE,OAAO;YACf,CAAC,CAAC,yBAAyB,CAAC,OAAO;YACnC,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC,MAAM;gBACvD,CAAC,CAAC,yBAAyB,CAAC,UAAU;gBACtC,CAAC,CAAC,yBAAyB,CAAC,QAAQ;QACxC,OAAO;QACP,SAAS;QACT,QAAQ,EAAE,QAAQ,CAAC,MAAM;QACzB,OAAO;QACP,iBAAiB;QACjB,KAAK;QACL,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;QACzD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,KAAK,kBAAkB,CAAC,SAAS,CAAC;QAC/F,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,YAAY,KAAK,6BAA6B,CAAC,QAAQ,CAAC;QAC9H,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC;QAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;KAC9G,CAAC;AACJ,CAAC","sourcesContent":["import { CronExpressionParser } from \"cron-parser\";\nimport {\n JobExecutionStatus,\n ScheduledEvidenceResultStatus,\n ScheduledEvidenceSource,\n ScheduledRunEvidenceSchema,\n type ScheduledRunEvidence,\n} from \"@wildo-ai/saas-models\";\nimport type { JobPayloadStorageBackendService } from \"./job-payload-storage.backend.service\";\nimport type { JobStatusTrackingBackendService } from \"./job-status-tracking.backend.service\";\n\nexport enum ScheduledEvidenceCoverage {\n COMPLETE = \"complete\",\n INCOMPLETE = \"incomplete\",\n UNKNOWN = \"unknown\",\n}\n/** Accepted historical configuration supplied by the trusted report caller, not inferred from observed runs. /\nexport interface ScheduledEvidenceScheduleSegment {\n from: Date;\n to: Date;\n cron: string;\n timezone: string;\n enabled: boolean;\n authorityRef: string;\n}\nexport interface ScheduledEvidenceObservation {\n jobId: string;\n scheduledAt?: Date;\n startedAt?: Date;\n completedAt?: Date;\n retained: boolean;\n invocationStatus: JobExecutionStatus;\n source?: ScheduledEvidenceSource;\n resultStatus?: ScheduledEvidenceResultStatus;\n results?: ScheduledRunEvidence[\"results\"];\n deferred?: boolean;\n /* Undefined for reconciled/operator-confirmed outcomes or missing start evidence. /\n startedWithinTolerance?: boolean;\n}\n\n/\n Assess archived occurrence coverage against an explicitly supplied historical schedule basis.\n * This is a trusted backend measurement seam, not an authorization endpoint or a claim that the\n * supplied history has been independently authenticated. Callers must retain its authorityRefs\n * with the report. Current schedule defaults cannot stand in for absent historical segments.\n * Intervals are [from,to); coverage is separate from successful invocation, result completeness,\n * timeliness and deferred business work. Pending operational rows are visible but not archived proof.\n /\nexport async function assessScheduledEvidenceCoverage(\n input: {\n /\n The REPORT's scope — the application RECORD id (`applications._id`), which is what every\n * `jobExecutions` row and every archived evidence row is anchored to. The tracker and the\n * payload store scope their own scans to that id; this value is what the returned rows are\n * checked AGAINST, so a caller that passes the platform id gets a refusal rather than a\n * clean, plausible, empty report (the 288-missing-occurrences finding the controller records).\n /\n applicationId: string;\n target: { queueName: string; batchRef?: string; operationPathKey?: string; minionName?: string };\n from: Date;\n to: Date;\n scheduleSegments: ScheduledEvidenceScheduleSegment[];\n allowedStartDelayMs: number;\n limit?: number;\n },\n dependencies: {\n payloads: Pick<JobPayloadStorageBackendService, \"listScheduledEvidence\">;\n tracker: Pick<JobStatusTrackingBackendService, \"listScheduledEvidenceCandidates\">;\n },\n) {\n const limit = input.limit ?? 10000;\n const from = input.from.getTime(),\n to = input.to.getTime(),\n now = Date.now();\n if (\n !Number.isFinite(from) \|\|\n !Number.isFinite(to) \|\|\n from >= to \|\|\n !input.applicationId \|\|\n !input.target.queueName \|\|\n !Number.isInteger(limit) \|\|\n limit < 1 \|\|\n limit > 10000 \|\|\n !Number.isFinite(input.allowedStartDelayMs) \|\|\n input.allowedStartDelayMs < 0 \|\|\n [input.target.batchRef, input.target.operationPathKey, input.target.minionName].filter((value) => typeof value === \"string\" && value.length > 0).length !==\n 1\n ) {\n throw new Error(\"Invalid scheduled evidence coverage request\");\n }\n const reasons: string[] = [];\n const expected: number[] = [];\n let cursor = from;\n let expectedTruncated = false;\n const segments = [...input.scheduleSegments].sort((a, b) => a.from.getTime() - b.from.getTime());\n for (const segment of segments) {\n const start = segment.from.getTime(),\n end = segment.to.getTime();\n if (\n !Number.isFinite(start) \|\|\n !Number.isFinite(end) \|\|\n start >= end \|\|\n !segment.authorityRef.trim() \|\|\n !segment.timezone.trim() \|\|\n typeof segment.enabled !== \"boolean\"\n ) {\n throw new Error(\"Invalid historical schedule segment\");\n }\n if (end <= from \|\| start >= to) continue;\n const lo = Math.max(start, from),\n hi = Math.min(end, to);\n if (lo < cursor) throw new Error(\"Historical schedule segments overlap\");\n if (lo > cursor) reasons.push(\"Historical schedule authority does not cover the whole period\");\n cursor = hi;\n // Parse even disabled segments: malformed declarations must not look like an intentional empty schedule.\n const cron = CronExpressionParser.parse(segment.cron, { currentDate: new Date(lo - 1), endDate: new Date(hi), tz: segment.timezone });\n if (segment.enabled) {\n while (cron.hasNext()) {\n const instant = cron.next().getTime();\n if (instant >= hi) break;\n if (expected.length === limit) {\n expectedTruncated = true;\n break;\n }\n expected.push(instant);\n }\n }\n }\n if (cursor < to) reasons.push(\"Historical schedule authority does not cover the whole period\");\n if (to > now) reasons.push(\"The requested period includes future time\");\n const [archive, live] = await Promise.all([\n dependencies.payloads.listScheduledEvidence(input.target, input.from, input.to, limit),\n dependencies.tracker.listScheduledEvidenceCandidates(input.target, input.from, input.to, limit),\n ]);\n const observations = new Map<string, ScheduledEvidenceObservation>();\n for (const evidence of archive.records) {\n if (evidence.applicationId !== input.applicationId) throw new Error(\"Evidence store application does not match report scope\");\n observations.set(evidence.jobId, {\n jobId: evidence.jobId,\n scheduledAt: evidence.scheduledAt,\n startedAt: evidence.startedAt,\n completedAt: evidence.completedAt,\n retained: true,\n invocationStatus: evidence.invocationStatus,\n source: evidence.source,\n resultStatus: evidence.resultStatus,\n results: evidence.results,\n deferred: evidence.deferred,\n startedWithinTolerance:\n evidence.source === ScheduledEvidenceSource.EXECUTION && evidence.startedAt && evidence.scheduledAt\n ? evidence.startedAt.getTime() <= evidence.scheduledAt.getTime() + input.allowedStartDelayMs\n : undefined,\n });\n }\n for (const row of live.records) {\n if (observations.has(row.jobId)) continue;\n let scheduledAt: Date \| undefined;\n // Legacy rows use authenticated receipt time solely to select unknown-time observations.\n // Publication/receipt time never substitutes for an expected scheduled occurrence.\n let publishedAt = row.queuedAt ? new Date(row.queuedAt) : undefined;\n if (row.scheduledState) {\n try {\n const payload = JSON.parse(row.scheduledState.payloadJson);\n if (payload.createdAt !== undefined) {\n const publication = new Date(payload.createdAt);\n if (Number.isFinite(publication.getTime())) publishedAt = publication;\n }\n if (payload.scheduledAt !== undefined) {\n const candidate = new Date(payload.scheduledAt);\n if (Number.isFinite(candidate.getTime())) scheduledAt = candidate;\n }\n } catch {\n / Historical unknown stays unknown; publication is never a substitute. */\n }\n }\n if (scheduledAt && (scheduledAt.getTime() < from \|\| scheduledAt.getTime() >= to)) continue;\n if (!scheduledAt && (!publishedAt \|\| !Number.isFinite(publishedAt.getTime()) \|\| publishedAt.getTime() < from \|\| publishedAt.getTime() >= to)) continue;\n observations.set(row.jobId, {\n jobId: row.jobId,\n scheduledAt,\n startedAt: row.startedAt,\n completedAt: row.completedAt,\n invocationStatus: ScheduledRunEvidenceSchema.shape.invocationStatus.parse(row.status),\n retained: false,\n });\n }\n const values = [...observations.values()];\n const byInstant = new Map<number, ScheduledEvidenceObservation[]>();\n for (const value of values) {\n if (!value.scheduledAt) continue;\n const instant = value.scheduledAt.getTime();\n const runs = byInstant.get(instant) ?? [];\n runs.push(value);\n byInstant.set(instant, runs);\n }\n const expectedInstants = new Set(expected);\n const slots = expected.map((instant) => {\n const runs = byInstant.get(instant) ?? [];\n return {\n scheduledAt: new Date(instant),\n runs,\n retained: runs.some((value) => value.retained),\n awaitingStartWindow: instant + input.allowedStartDelayMs > now,\n };\n });\n const truncated = archive.truncated \|\| live.truncated \|\| expectedTruncated;\n const missing = slots.filter((slot) => !slot.retained).map((slot) => slot.scheduledAt);\n const duplicateInstants = slots.filter((slot) => slot.runs.length > 1).map((slot) => slot.scheduledAt);\n const unknown = reasons.length > 0 \|\| slots.some((slot) => slot.awaitingStartWindow);\n return {\n applicationId: input.applicationId,\n target: input.target,\n from: input.from,\n to: input.to,\n scheduleSegments: segments,\n allowedStartDelayMs: input.allowedStartDelayMs,\n coverage: unknown\n ? ScheduledEvidenceCoverage.UNKNOWN\n : truncated \|\| missing.length \|\| duplicateInstants.length\n ? ScheduledEvidenceCoverage.INCOMPLETE\n : ScheduledEvidenceCoverage.COMPLETE,\n reasons,\n truncated,\n expected: expected.length,\n missing,\n duplicateInstants,\n slots,\n unknownTime: values.filter((value) => !value.scheduledAt),\n unsuccessful: values.filter((value) => value.invocationStatus !== JobExecutionStatus.COMPLETED),\n unavailableResults: values.filter((value) => !value.retained \|\| value.resultStatus !== ScheduledEvidenceResultStatus.CAPTURED),\n deferred: values.filter((value) => value.deferred === true),\n unexpected: values.filter((value) => value.scheduledAt && !expectedInstants.has(value.scheduledAt.getTime())),\n };\n}\n"]}
1	+ {"version":3,"file":"scheduled-evidence-coverage.backend.js","sourceRoot":"","sources":["../../../../src/queue/scheduled-evidence-coverage.backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACL,kBAAkB,EAClB,6BAA6B,EAC7B,uBAAuB,EACvB,0BAA0B,GAE3B,MAAM,uBAAuB,CAAC;AAI/B,MAAM,CAAN,IAAY,yBAIX;AAJD,WAAY,yBAAyB;IACnC,kDAAqB,CAAA;IACrB,sDAAyB,CAAA;IACzB,gDAAmB,CAAA;AACrB,CAAC,EAJW,yBAAyB,KAAzB,yBAAyB,QAIpC;AAyBD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,KAeC,EACD,YAGC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC;IACnC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAC/B,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,EACvB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACnB,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QACtB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpB,IAAI,IAAI,EAAE;QACV,CAAC,KAAK,CAAC,aAAa;QACpB,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS;QACvB,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;QACxB,KAAK,GAAG,CAAC;QACT,KAAK,GAAG,KAAK;QACb,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC;QAC3C,KAAK,CAAC,mBAAmB,GAAG,CAAC;QAC7B,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM;YACrJ,CAAC,EACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACjG,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAClC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;QAC7B,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvB,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;YACrB,KAAK,IAAI,GAAG;YACZ,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE;YAC5B,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE;YACxB,OAAO,OAAO,CAAC,OAAO,KAAK,SAAS,EACpC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,GAAG,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE;YAAE,SAAS;QACzC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,EAC9B,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,GAAG,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACzE,IAAI,EAAE,GAAG,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC/F,MAAM,GAAG,EAAE,CAAC;QACZ,yGAAyG;QACzG,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,OAAO,IAAI,EAAE;oBAAE,MAAM;gBACzB,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;oBAC9B,iBAAiB,GAAG,IAAI,CAAC;oBACzB,MAAM;gBACR,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,MAAM,GAAG,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC/F,IAAI,EAAE,GAAG,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IACxE,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxC,YAAY,CAAC,QAAQ,CAAC,qBAAqB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC;QACtF,YAAY,CAAC,OAAO,CAAC,+BAA+B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC;KAChG,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAwC,CAAC;IACrE,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC9H,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;YAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,QAAQ,EAAE,IAAI;YACd,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;YAC3C,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,sBAAsB,EACpB,QAAQ,CAAC,MAAM,KAAK,uBAAuB,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,WAAW;gBACjG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,mBAAmB;gBAC5F,CAAC,CAAC,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1C,IAAI,WAA6B,CAAC;QAClC,yFAAyF;QACzF,mFAAmF;QACnF,IAAI,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpE,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAC3D,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;oBAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;wBAAE,WAAW,GAAG,WAAW,CAAC;gBACxE,CAAC;gBACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;oBACtC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;oBAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;wBAAE,WAAW,GAAG,SAAS,CAAC;gBACpE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0EAA0E;YAC5E,CAAC;QACH,CAAC;QACD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YAAE,SAAS;QAC3F,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YAAE,SAAS;QACvJ,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE;YAC1B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,WAAW;YACX,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,gBAAgB,EAAE,0BAA0B,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YACrF,QAAQ,EAAE,KAAK;YACf,8FAA8F;YAC9F,wFAAwF;YACxF,GAAG,CAAC,GAAG,CAAC,cAAc,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,uBAAuB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzG,CAAC,CAAC;IACL,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0C,CAAC;IACpE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,WAAW;YAAE,SAAS;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACL,WAAW,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC;YAC9B,IAAI;YACJ,6FAA6F;YAC7F,8FAA8F;YAC9F,yFAAyF;YACzF,6FAA6F;YAC7F,+DAA+D;YAC/D,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,uBAAuB,CAAC,aAAa,CAAC;YACxG,mBAAmB,EAAE,OAAO,GAAG,KAAK,CAAC,mBAAmB,GAAG,GAAG;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC;IAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvF,MAAM,iBAAiB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvG,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrF,OAAO;QACL,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,gBAAgB,EAAE,QAAQ;QAC1B,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;QAC9C,QAAQ,EAAE,OAAO;YACf,CAAC,CAAC,yBAAyB,CAAC,OAAO;YACnC,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC,MAAM;gBACvD,CAAC,CAAC,yBAAyB,CAAC,UAAU;gBACtC,CAAC,CAAC,yBAAyB,CAAC,QAAQ;QACxC,OAAO;QACP,SAAS;QACT,QAAQ,EAAE,QAAQ,CAAC,MAAM;QACzB,OAAO;QACP,iBAAiB;QACjB,KAAK;QACL,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;QACzD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,KAAK,kBAAkB,CAAC,SAAS,CAAC;QAC/F,+FAA+F;QAC/F,mFAAmF;QACnF,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,uBAAuB,CAAC,aAAa,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,YAAY,KAAK,6BAA6B,CAAC,QAAQ,CAAC,CAAC;QAC1L,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC;QAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7G;;;;;;WAMG;QACH,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,uBAAuB,CAAC,aAAa,CAAC;KAChG,CAAC;AACJ,CAAC","sourcesContent":["import { CronExpressionParser } from \"cron-parser\";\nimport {\n JobExecutionStatus,\n ScheduledEvidenceResultStatus,\n ScheduledEvidenceSource,\n ScheduledRunEvidenceSchema,\n type ScheduledRunEvidence,\n} from \"@wildo-ai/saas-models\";\nimport type { JobPayloadStorageBackendService } from \"./job-payload-storage.backend.service\";\nimport type { JobStatusTrackingBackendService } from \"./job-status-tracking.backend.service\";\n\nexport enum ScheduledEvidenceCoverage {\n COMPLETE = \"complete\",\n INCOMPLETE = \"incomplete\",\n UNKNOWN = \"unknown\",\n}\n/** Accepted historical configuration supplied by the trusted report caller, not inferred from observed runs. /\nexport interface ScheduledEvidenceScheduleSegment {\n from: Date;\n to: Date;\n cron: string;\n timezone: string;\n enabled: boolean;\n authorityRef: string;\n}\nexport interface ScheduledEvidenceObservation {\n jobId: string;\n scheduledAt?: Date;\n startedAt?: Date;\n completedAt?: Date;\n retained: boolean;\n invocationStatus: JobExecutionStatus;\n source?: ScheduledEvidenceSource;\n resultStatus?: ScheduledEvidenceResultStatus;\n results?: ScheduledRunEvidence[\"results\"];\n deferred?: boolean;\n /* Undefined for reconciled/operator-confirmed outcomes or missing start evidence. /\n startedWithinTolerance?: boolean;\n}\n\n/\n Assess archived occurrence coverage against an explicitly supplied historical schedule basis.\n * This is a trusted backend measurement seam, not an authorization endpoint or a claim that the\n * supplied history has been independently authenticated. Callers must retain its authorityRefs\n * with the report. Current schedule defaults cannot stand in for absent historical segments.\n * Intervals are [from,to); coverage is separate from successful invocation, result completeness,\n * timeliness and deferred business work. Pending operational rows are visible but not archived proof.\n /\nexport async function assessScheduledEvidenceCoverage(\n input: {\n /\n The REPORT's scope — the application RECORD id (`applications._id`), which is what every\n * `jobExecutions` row and every archived evidence row is anchored to. The tracker and the\n * payload store scope their own scans to that id; this value is what the returned rows are\n * checked AGAINST, so a caller that passes the platform id gets a refusal rather than a\n * clean, plausible, empty report (the 288-missing-occurrences finding the controller records).\n /\n applicationId: string;\n target: { queueName: string; batchRef?: string; operationPathKey?: string; minionName?: string };\n from: Date;\n to: Date;\n scheduleSegments: ScheduledEvidenceScheduleSegment[];\n allowedStartDelayMs: number;\n limit?: number;\n },\n dependencies: {\n payloads: Pick<JobPayloadStorageBackendService, \"listScheduledEvidence\">;\n tracker: Pick<JobStatusTrackingBackendService, \"listScheduledEvidenceCandidates\">;\n },\n) {\n const limit = input.limit ?? 10000;\n const from = input.from.getTime(),\n to = input.to.getTime(),\n now = Date.now();\n if (\n !Number.isFinite(from) \|\|\n !Number.isFinite(to) \|\|\n from >= to \|\|\n !input.applicationId \|\|\n !input.target.queueName \|\|\n !Number.isInteger(limit) \|\|\n limit < 1 \|\|\n limit > 10000 \|\|\n !Number.isFinite(input.allowedStartDelayMs) \|\|\n input.allowedStartDelayMs < 0 \|\|\n [input.target.batchRef, input.target.operationPathKey, input.target.minionName].filter((value) => typeof value === \"string\" && value.length > 0).length !==\n 1\n ) {\n throw new Error(\"Invalid scheduled evidence coverage request\");\n }\n const reasons: string[] = [];\n const expected: number[] = [];\n let cursor = from;\n let expectedTruncated = false;\n const segments = [...input.scheduleSegments].sort((a, b) => a.from.getTime() - b.from.getTime());\n for (const segment of segments) {\n const start = segment.from.getTime(),\n end = segment.to.getTime();\n if (\n !Number.isFinite(start) \|\|\n !Number.isFinite(end) \|\|\n start >= end \|\|\n !segment.authorityRef.trim() \|\|\n !segment.timezone.trim() \|\|\n typeof segment.enabled !== \"boolean\"\n ) {\n throw new Error(\"Invalid historical schedule segment\");\n }\n if (end <= from \|\| start >= to) continue;\n const lo = Math.max(start, from),\n hi = Math.min(end, to);\n if (lo < cursor) throw new Error(\"Historical schedule segments overlap\");\n if (lo > cursor) reasons.push(\"Historical schedule authority does not cover the whole period\");\n cursor = hi;\n // Parse even disabled segments: malformed declarations must not look like an intentional empty schedule.\n const cron = CronExpressionParser.parse(segment.cron, { currentDate: new Date(lo - 1), endDate: new Date(hi), tz: segment.timezone });\n if (segment.enabled) {\n while (cron.hasNext()) {\n const instant = cron.next().getTime();\n if (instant >= hi) break;\n if (expected.length === limit) {\n expectedTruncated = true;\n break;\n }\n expected.push(instant);\n }\n }\n }\n if (cursor < to) reasons.push(\"Historical schedule authority does not cover the whole period\");\n if (to > now) reasons.push(\"The requested period includes future time\");\n const [archive, live] = await Promise.all([\n dependencies.payloads.listScheduledEvidence(input.target, input.from, input.to, limit),\n dependencies.tracker.listScheduledEvidenceCandidates(input.target, input.from, input.to, limit),\n ]);\n const observations = new Map<string, ScheduledEvidenceObservation>();\n for (const evidence of archive.records) {\n if (evidence.applicationId !== input.applicationId) throw new Error(\"Evidence store application does not match report scope\");\n observations.set(evidence.jobId, {\n jobId: evidence.jobId,\n scheduledAt: evidence.scheduledAt,\n startedAt: evidence.startedAt,\n completedAt: evidence.completedAt,\n retained: true,\n invocationStatus: evidence.invocationStatus,\n source: evidence.source,\n resultStatus: evidence.resultStatus,\n results: evidence.results,\n deferred: evidence.deferred,\n startedWithinTolerance:\n evidence.source === ScheduledEvidenceSource.EXECUTION && evidence.startedAt && evidence.scheduledAt\n ? evidence.startedAt.getTime() <= evidence.scheduledAt.getTime() + input.allowedStartDelayMs\n : undefined,\n });\n }\n for (const row of live.records) {\n if (observations.has(row.jobId)) continue;\n let scheduledAt: Date \| undefined;\n // Legacy rows use authenticated receipt time solely to select unknown-time observations.\n // Publication/receipt time never substitutes for an expected scheduled occurrence.\n let publishedAt = row.queuedAt ? new Date(row.queuedAt) : undefined;\n if (row.scheduledState) {\n try {\n const payload = JSON.parse(row.scheduledState.payloadJson);\n if (payload.createdAt !== undefined) {\n const publication = new Date(payload.createdAt);\n if (Number.isFinite(publication.getTime())) publishedAt = publication;\n }\n if (payload.scheduledAt !== undefined) {\n const candidate = new Date(payload.scheduledAt);\n if (Number.isFinite(candidate.getTime())) scheduledAt = candidate;\n }\n } catch {\n / Historical unknown stays unknown; publication is never a substitute. /\n }\n }\n if (scheduledAt && (scheduledAt.getTime() < from \|\| scheduledAt.getTime() >= to)) continue;\n if (!scheduledAt && (!publishedAt \|\| !Number.isFinite(publishedAt.getTime()) \|\| publishedAt.getTime() < from \|\| publishedAt.getTime() >= to)) continue;\n observations.set(row.jobId, {\n jobId: row.jobId,\n scheduledAt,\n startedAt: row.startedAt,\n completedAt: row.completedAt,\n invocationStatus: ScheduledRunEvidenceSchema.shape.invocationStatus.parse(row.status),\n retained: false,\n // A live row carries no archived source, but a late-delivery row says what it is structurally\n // (#2212) — so a restart's gap is explained even for a target that retains no evidence.\n ...(row.scheduledState?.lateness !== undefined ? { source: ScheduledEvidenceSource.LATE_DELIVERY } : {}),\n });\n }\n const values = [...observations.values()];\n const byInstant = new Map<number, ScheduledEvidenceObservation[]>();\n for (const value of values) {\n if (!value.scheduledAt) continue;\n const instant = value.scheduledAt.getTime();\n const runs = byInstant.get(instant) ?? [];\n runs.push(value);\n byInstant.set(instant, runs);\n }\n const expectedInstants = new Set(expected);\n const slots = expected.map((instant) => {\n const runs = byInstant.get(instant) ?? [];\n return {\n scheduledAt: new Date(instant),\n runs,\n // Covered only by evidence of an occurrence that was DEALT WITH. A late delivery is retained\n // evidence that the occurrence did NOT run (#2212), so it explains the slot — `lateDelivered`\n // names it — but never covers it: counting it would let one long restart report a window\n // COMPLETE, and would give the same event two verdicts depending on whether the target keeps\n // an evidence policy (archived → covered, live row → missing).\n retained: runs.some((value) => value.retained && value.source !== ScheduledEvidenceSource.LATE_DELIVERY),\n awaitingStartWindow: instant + input.allowedStartDelayMs > now,\n };\n });\n const truncated = archive.truncated \|\| live.truncated \|\| expectedTruncated;\n const missing = slots.filter((slot) => !slot.retained).map((slot) => slot.scheduledAt);\n const duplicateInstants = slots.filter((slot) => slot.runs.length > 1).map((slot) => slot.scheduledAt);\n const unknown = reasons.length > 0 \|\| slots.some((slot) => slot.awaitingStartWindow);\n return {\n applicationId: input.applicationId,\n target: input.target,\n from: input.from,\n to: input.to,\n scheduleSegments: segments,\n allowedStartDelayMs: input.allowedStartDelayMs,\n coverage: unknown\n ? ScheduledEvidenceCoverage.UNKNOWN\n : truncated \|\| missing.length \|\| duplicateInstants.length\n ? ScheduledEvidenceCoverage.INCOMPLETE\n : ScheduledEvidenceCoverage.COMPLETE,\n reasons,\n truncated,\n expected: expected.length,\n missing,\n duplicateInstants,\n slots,\n unknownTime: values.filter((value) => !value.scheduledAt),\n unsuccessful: values.filter((value) => value.invocationStatus !== JobExecutionStatus.COMPLETED),\n // A late delivery is not a run, so it has no result to be unavailable: it is reported once, in\n // `lateDelivered`, never also as a \"retained run with no readable result\" (#2212).\n unavailableResults: values.filter((value) => value.source !== ScheduledEvidenceSource.LATE_DELIVERY && (!value.retained \|\| value.resultStatus !== ScheduledEvidenceResultStatus.CAPTURED)),\n deferred: values.filter((value) => value.deferred === true),\n unexpected: values.filter((value) => value.scheduledAt && !expectedInstants.has(value.scheduledAt.getTime())),\n /\n Occurrences that were never run because their delivery reached the application after its\n * admission deadline (#2212) — usually because the application was not consuming; see\n * `ScheduledDeliveryLateness` for the case the scheduler shares. Their slots stay in `missing`\n * (they did not run) and they are also in `unsuccessful`; listed apart because their cause is\n * RECORDED, which is what distinguishes a restart's gap from an unexplained one.\n */\n lateDelivered: values.filter((value) => value.source === ScheduledEvidenceSource.LATE_DELIVERY),\n };\n}\n"]}