npm - @wildo-ai/saas-backend-lib - Versions diffs - 1.1.8 → 1.1.9 - Mend

@wildo-ai/saas-backend-lib 1.1.8 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (174) hide show

package/dist/esm/queue/scheduled-execution.backend.service.d.ts CHANGED Viewed

@@ -1,4 +1,4 @@
-import { ScheduledReviewOutcome, type JobExecution, type JobPayload, type JobResult, type ScheduledAttemptContext, type ScheduledReconciliationResult, type ScheduledRecoveryPolicy, type ScheduledEvidencePolicy } from '@wildo-ai/saas-models';
+import { ScheduledReviewOutcome, type JobExecution, type JobPayload, type JobResult, type ScheduledAttemptContext, type ScheduledReconciliationResult, type ScheduledRecoveryPolicy, type ScheduledEvidencePolicy, ScheduledDeliveryLateness } from '@wildo-ai/saas-models';
 import type { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';
 import type { LoggerService } from '../monitoring/logger/services/logger.service';
 import type { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';
@@ -42,6 +42,8 @@ export declare class ScheduledExecutionRunner {
      * itself; if it waits past its recovery deadline it goes to review, where it is visible.
      */
     private readonly activeTargets;
+    /** Late deliveries gathered per (queue, target) until their drain goes quiet — see {@link recordLateDelivery}. */
+    private readonly lateDigests;
     constructor(tracker: JobStatusTrackingBackendService, logger: Pick<LoggerService, 'info' | 'warn' | 'error'>, payloadStorage?: Pick<JobPayloadStorageBackendService, 'store'> & Partial<Pick<JobPayloadStorageBackendService, 'storeScheduledEvidence'>>, activity?: {
         started(): void;
         completed(): void;
@@ -68,6 +70,45 @@ export declare class ScheduledExecutionRunner {
      * review) or a legacy row whose effects are unknown.
      */
     admit(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver): Promise<JobExecution | undefined>;
+    /**
+     * Record an AUTHENTIC delivery that arrived after its admission deadline as a MISSED occurrence
+     * (#2212), and report the drain it belongs to once.
+     *
+     * The delivery is never admitted and never run — a missed occurrence is skipped, and the schedule
+     * continues (`ScheduledDeliveryLateness`). What changes is what it leaves behind. It used to be
+     * failed as an `AUTHENTICATION` error, logged twice, dead-lettered and otherwise forgotten, so a
+     * restart left a gap in scheduled-evidence coverage that nothing could tell apart from an
+     * unexplained one. It now becomes a terminal row (`FINISHED`, `FAILED`, `lateness` set) and, when
+     * the target retains evidence, an archived `LATE_DELIVERY` record — an explained gap.
+     *
+     * Never overwrites an occurrence an earlier receipt owns: a delivery admitted on time whose
+     * acknowledgement was lost is redelivered late, and that occurrence may well have run. The row is
+     * first created in `REVIEW` (never scanned for execution) and only a row this path created moves
+     * to `FINISHED`; anything else is left alone. A crash between the two writes leaves a `REVIEW` row
+     * with no attempts, which an operator can abandon or retry like any other.
+     *
+     * Never throws. Returns `true` when the occurrence is durable (recorded now, or already held), so
+     * the caller can acknowledge the delivery; `false` when it could not be recorded, so the caller
+     * dead-letters it and the copy survives.
+     */
+    recordLateDelivery(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver, lateness: ScheduledDeliveryLateness): Promise<boolean>;
+    /**
+     * What the reader should conclude, by WHICH gate fired: only an expired token proves the queue
+     * went unconsumed for the whole window; an occurrence past the window on a current token shares
+     * its delay with the scheduler (`ScheduledDeliveryLateness`).
+     */
+    private lateDigestRemedy;
+    /** Flush every pending late-delivery summary now (shutdown, and tests). */
+    flushLateDeliveryDigests(): void;
+    private lateDigestFor;
+    private scheduleLateDigest;
+    /**
+     * One line per drain per target. WARN, not ERROR: a missed occurrence is the expected consequence
+     * of the application not running, and the line says so — but it is still a gap in what the
+     * schedule promised, so it must not be silent either. Escalates to ERROR only when some deliveries
+     * could not be recorded, because then the gap is no longer explained durably.
+     */
+    private reportLateDigest;
     /**
      * Start an admitted occurrence WITHOUT waiting for it, under the same bound as recovery.
      *
@@ -109,6 +150,8 @@ export declare class ScheduledExecutionRunner {
     private process;
     /** The batch, operation or minion an occurrence runs, read from its durable payload. */
     private targetOf;
+    /** Prefixed by kind: a batch and a minion that happen to share a name are different targets. */
+    private targetKey;
     private failureDetails;
     private ended;
     private review;

package/dist/esm/queue/scheduled-execution.backend.service.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"scheduled-execution.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/scheduled-execution.backend.service.ts"],"names":[],"mappings":"AACA,OAAO,EAG2E,sBAAsB,EACtG,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,uBAAuB,EAClD,KAAK,6BAA6B,EAAE,KAAK,uBAAuB,EAC9F,KAAK,uBAAuB,~~EAE7B~~,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,8CAA8C,CAAC;AAClF,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;~~AAU7F~~,iGAAiG;AACjG,MAAM,WAAW,0BAA0B;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,uBAAuB,CAAC;IACzC,OAAO,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9D,SAAS,CAAC,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;CACtF;AACD,MAAM,MAAM,0BAA0B,GAAG,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC,CAAC;AAE9G;;;;;;;GAOG;AACH,qBAAa,wBAAwB;~~IAsBjC~~,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;~~IAxB5B~~,OAAO,CAAC,KAAK,CAAC,CAAiC;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,4FAA4F;IAC5F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;IACrD;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;~~gBAGhC~~,OAAO,EAAE,+BAA+B,EACxC,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,EACtD,cAAc,CAAC,EAAE,IAAI,CAAC,+BAA+B,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,+BAA+B,EAAE,wBAAwB,CAAC,CAAC,EAC1I,QAAQ,CAAC,EAAE;QAAE,OAAO,IAAI,IAAI,CAAC;QAAC,SAAS,IAAI,IAAI,CAAA;KAAE;IAGpE;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;IAyBvH;;;;;;OAMG;IACH,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,0BAA0B,GAAG,IAAI;IAUpE,iFAAiF;IAC3E,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;OAMG;IACG,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;IAWrG;;;;;;;;;OASG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,IAAI;IAWnE,mGAAmG;IACnG,IAAI,IAAI,IAAI;~~IAMN~~,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBjF,2FAA2F;YAC7E,YAAY;IAQ1B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,gBAAgB;IAMxB,wFAAwF;IACxF,OAAO,CAAC,UAAU;YAMJ,OAAO;IA8GrB,wFAAwF;IACxF,OAAO,CAAC,QAAQ;~~IAahB~~,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,KAAK;YAIC,MAAM;YASN,IAAI;YAaJ,QAAQ;IAgBtB,6GAA6G;YAC/F,eAAe;IAsB7B;;;;OAIG;IACG,aAAa,CAAC,KAAK,EAAE;QACzB,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAC3D,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,sBAAsB,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;KACrF,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;CA4DvD"}
1	+ {"version":3,"file":"scheduled-execution.backend.service.d.ts","sourceRoot":"","sources":["../../../../src/queue/scheduled-execution.backend.service.ts"],"names":[],"mappings":"AACA,OAAO,EAG2E,sBAAsB,EACtG,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,uBAAuB,EAClD,KAAK,6BAA6B,EAAE,KAAK,uBAAuB,EAC9F,KAAK,uBAAuB,EAE5B,yBAAyB,EAC1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,8CAA8C,CAAC;AAClF,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AA8C7F,iGAAiG;AACjG,MAAM,WAAW,0BAA0B;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,uBAAuB,CAAC;IACzC,OAAO,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9D,SAAS,CAAC,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;CACtF;AACD,MAAM,MAAM,0BAA0B,GAAG,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,0BAA0B,GAAG,SAAS,CAAC,CAAC;AAE9G;;;;;;;GAOG;AACH,qBAAa,wBAAwB;IAwBjC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IA1B5B,OAAO,CAAC,KAAK,CAAC,CAAiC;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,4FAA4F;IAC5F,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;IACrD;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,kHAAkH;IAClH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyC;gBAGlD,OAAO,EAAE,+BAA+B,EACxC,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,EACtD,cAAc,CAAC,EAAE,IAAI,CAAC,+BAA+B,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,+BAA+B,EAAE,wBAAwB,CAAC,CAAC,EAC1I,QAAQ,CAAC,EAAE;QAAE,OAAO,IAAI,IAAI,CAAC;QAAC,SAAS,IAAI,IAAI,CAAA;KAAE;IAGpE;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;IAyBvH;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,kBAAkB,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,EAC9F,QAAQ,EAAE,yBAAyB,GAAG,OAAO,CAAC,OAAO,CAAC;IA2CxD;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAgBxB,2EAA2E;IAC3E,wBAAwB,IAAI,IAAI;IAIhC,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,kBAAkB;IAS1B;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAgBxB;;;;;;OAMG;IACH,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,0BAA0B,GAAG,IAAI;IAUpE,iFAAiF;IAC3E,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;OAMG;IACG,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;IAWrG;;;;;;;;;OASG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,IAAI;IAWnE,mGAAmG;IACnG,IAAI,IAAI,IAAI;IAON,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBjF,2FAA2F;YAC7E,YAAY;IAQ1B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,gBAAgB;IAMxB,wFAAwF;IACxF,OAAO,CAAC,UAAU;YAMJ,OAAO;IA8GrB,wFAAwF;IACxF,OAAO,CAAC,QAAQ;IAQhB,gGAAgG;IAChG,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,KAAK;YAIC,MAAM;YASN,IAAI;YAaJ,QAAQ;IAgBtB,6GAA6G;YAC/F,eAAe;IAsB7B;;;;OAIG;IACG,aAAa,CAAC,KAAK,EAAE;QACzB,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAC3D,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,sBAAsB,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;KACrF,EAAE,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;CA4DvD"}

package/dist/esm/queue/scheduled-execution.backend.service.js CHANGED Viewed

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { ErrorType, ErrorSeverity, ErrorCustomMessageReference, ErrorHandling_Strategy, isWildoBackendError, JOB_EXECUTION_TTL_SECONDS, JobExecutionStatus, JobPayloadSchema, JobRetryUtils, JobStatus, normalizeScheduledRecoveryPolicy, QUEUE_RETRY_DEFAULTS, ScheduledExecutionPhase, ScheduledReconciliationOutcome, ScheduledReplaySafety, ScheduledReviewOutcome, ScheduledEvidencePolicySchema, ScheduledEvidenceSource, } from '@wildo-ai/saas-models';
+import { ErrorType, ErrorSeverity, ErrorCustomMessageReference, ErrorHandling_Strategy, isWildoBackendError, JOB_EXECUTION_TTL_SECONDS, JobExecutionStatus, JobPayloadSchema, JobRetryUtils, JobStatus, normalizeScheduledRecoveryPolicy, QUEUE_RETRY_DEFAULTS, ScheduledExecutionPhase, ScheduledReconciliationOutcome, ScheduledReplaySafety, ScheduledReviewOutcome, ScheduledEvidencePolicySchema, ScheduledEvidenceSource, scheduledEvidenceInvocationStatus, ScheduledDeliveryLateness, } from '@wildo-ai/saas-models';
 import { createScheduledRunEvidence } from './scheduled-evidence.backend.js';
 import { describeCaughtError } from '../errors/utils/error-log-level-classification.backend.utils.js';
 /**
@@ -7,6 +7,21 @@ import { describeCaughtError } from '../errors/utils/error-log-level-classificat
  * since both go through `process`. Other replicas arbitrate the same rows through compare-and-swap.
  */
 const SCHEDULED_EXECUTION_CONCURRENCY = 8;
+/**
+ * How long a target's late deliveries are gathered before ONE summary is logged.
+ *
+ * A backlog drains in about a second — measured at 139 deliveries in 0.9 s after a restart (#2212) —
+ * so a short quiet period closes the drain without holding the summary noticeably.
+ */
+const LATE_DELIVERY_DIGEST_QUIET_MS = 2000;
+/**
+ * The longest a late-delivery summary is held, however steadily deliveries keep arriving: a consumer
+ * that stays permanently behind produces gaps shorter than the quiet period forever, and must still
+ * be reported rather than silenced by its own persistence.
+ */
+const LATE_DELIVERY_DIGEST_MAX_HOLD_MS = 30_000;
+/** Reported on an occurrence recorded from a late delivery; the structured cause is `ScheduledExecutionState.lateness`. */
+const LATE_DELIVERY_REASON = 'Delivery reached this application after its admission deadline; the occurrence was not run';
 /**
  * Durable scheduled execution owner shared by application consumers and minions.
  *
@@ -39,6 +54,8 @@ export class ScheduledExecutionRunner {
      * itself; if it waits past its recovery deadline it goes to review, where it is visible.
      */
     activeTargets = new Set();
+    /** Late deliveries gathered per (queue, target) until their drain goes quiet — see {@link recordLateDelivery}. */
+    lateDigests = new Map();
     constructor(tracker, logger, payloadStorage, activity) {
         this.tracker = tracker;
         this.logger = logger;
@@ -94,6 +111,156 @@ export class ScheduledExecutionRunner {
         }
         return row;
     }
+    /**
+     * Record an AUTHENTIC delivery that arrived after its admission deadline as a MISSED occurrence
+     * (#2212), and report the drain it belongs to once.
+     *
+     * The delivery is never admitted and never run — a missed occurrence is skipped, and the schedule
+     * continues (`ScheduledDeliveryLateness`). What changes is what it leaves behind. It used to be
+     * failed as an `AUTHENTICATION` error, logged twice, dead-lettered and otherwise forgotten, so a
+     * restart left a gap in scheduled-evidence coverage that nothing could tell apart from an
+     * unexplained one. It now becomes a terminal row (`FINISHED`, `FAILED`, `lateness` set) and, when
+     * the target retains evidence, an archived `LATE_DELIVERY` record — an explained gap.
+     *
+     * Never overwrites an occurrence an earlier receipt owns: a delivery admitted on time whose
+     * acknowledgement was lost is redelivered late, and that occurrence may well have run. The row is
+     * first created in `REVIEW` (never scanned for execution) and only a row this path created moves
+     * to `FINISHED`; anything else is left alone. A crash between the two writes leaves a `REVIEW` row
+     * with no attempts, which an operator can abandon or retry like any other.
+     *
+     * Never throws. Returns `true` when the occurrence is durable (recorded now, or already held), so
+     * the caller can acknowledge the delivery; `false` when it could not be recorded, so the caller
+     * dead-letters it and the copy survives.
+     */
+    async recordLateDelivery(job, queueName, resolve, lateness) {
+        const digest = this.lateDigestFor(queueName, job);
+        digest.lateness.add(lateness);
+        const instant = job.scheduledAt ?? job.createdAt;
+        if (!digest.earliest || instant < digest.earliest)
+            digest.earliest = instant;
+        if (!digest.latest || instant > digest.latest)
+            digest.latest = instant;
+        try {
+            // Resolution only selects the evidence policy; a missing handler still gets its miss recorded.
+            let contract;
+            try {
+                contract = await resolve(job);
+            }
+            catch {
+                contract = undefined;
+            }
+            const now = new Date();
+            const evidencePolicy = contract?.evidencePolicy ? ScheduledEvidencePolicySchema.parse(contract.evidencePolicy) : undefined;
+            const row = await this.tracker.ensureScheduledJobRecord(job, queueName, {
+                phase: ScheduledExecutionPhase.REVIEW, revision: 0, payloadJson: JSON.stringify(job),
+                policy: normalizeScheduledRecoveryPolicy(contract?.policy), evidencePolicy,
+                reason: LATE_DELIVERY_REASON, lateness, deadline: now, nextAttemptAt: now,
+                attemptCount: 0, attempts: [], resolutions: [], reconciledSafe: false,
+            });
+            const state = row.scheduledState;
+            if (!state || state.lateness === undefined || state.revision !== 0 || state.attemptCount !== 0
+                || state.phase !== ScheduledExecutionPhase.REVIEW) {
+                digest.alreadyHeld += 1;
+                return true;
+            }
+            const source = ScheduledEvidenceSource.LATE_DELIVERY;
+            const evidence = createScheduledRunEvidence(row, now, source, scheduledEvidenceInvocationStatus(source));
+            const terminal = await this.tracker.compareScheduledState(row.jobId, state.revision, {
+                ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.FINISHED,
+                evidence, evidenceArchiveAfter: evidence ? now : undefined,
+            }, {
+                status: scheduledEvidenceInvocationStatus(source), completedAt: now,
+                expiresAt: new Date(now.getTime() + JOB_EXECUTION_TTL_SECONDS * 1000),
+            });
+            // A lost compare-and-swap means a concurrent late receipt of the same occurrence recorded it.
+            if (terminal) {
+                digest.recorded += 1;
+                this.launchArchive(terminal);
+            }
+            else
+                digest.alreadyHeld += 1;
+            return true;
+        }
+        catch (error) {
+            digest.unrecorded += 1;
+            digest.firstError ??= describeCaughtError(error);
+            return false;
+        }
+        finally {
+            this.scheduleLateDigest(digest);
+        }
+    }
+    /**
+     * What the reader should conclude, by WHICH gate fired: only an expired token proves the queue
+     * went unconsumed for the whole window; an occurrence past the window on a current token shares
+     * its delay with the scheduler (`ScheduledDeliveryLateness`).
+     */
+    lateDigestRemedy(digest) {
+        if (digest.unrecorded > 0) {
+            return 'Some missed occurrences could not be recorded and were dead-lettered; see firstError. Nothing was run.';
+        }
+        const causes = [];
+        if (digest.lateness.has(ScheduledDeliveryLateness.TOKEN_EXPIRED)) {
+            causes.push('nothing consumed the scheduled queue for longer than the 5-minute admission window (the backend or minion was stopped or restarting)');
+        }
+        if (digest.lateness.has(ScheduledDeliveryLateness.OUTSIDE_ADMISSION_WINDOW)) {
+            causes.push('the occurrence was older than the admission window when consumed, a delay shared between the scheduler publishing it and this application consuming it');
+        }
+        return `Nothing to do: these occurrences were missed because ${causes.join('; or ')}. `
+            + 'They are recorded as late deliveries, were not run, and the schedule continues. '
+            + 'Recurring bursts mean the application is often unavailable for longer than that window.';
+    }
+    /** Flush every pending late-delivery summary now (shutdown, and tests). */
+    flushLateDeliveryDigests() {
+        for (const digest of [...this.lateDigests.values()])
+            this.reportLateDigest(digest);
+    }
+    lateDigestFor(queueName, job) {
+        const target = this.targetKey(job) ?? `job:${job.jobType}`;
+        const key = `${queueName}\u0000${target}`;
+        let digest = this.lateDigests.get(key);
+        if (!digest) {
+            digest = { queueName, target, recorded: 0, alreadyHeld: 0, unrecorded: 0, lateness: new Set(), openedAt: Date.now() };
+            this.lateDigests.set(key, digest);
+        }
+        return digest;
+    }
+    scheduleLateDigest(digest) {
+        if (digest.timer)
+            clearTimeout(digest.timer);
+        const held = Date.now() - digest.openedAt;
+        if (held >= LATE_DELIVERY_DIGEST_MAX_HOLD_MS) {
+            this.reportLateDigest(digest);
+            return;
+        }
+        digest.timer = setTimeout(() => this.reportLateDigest(digest), Math.min(LATE_DELIVERY_DIGEST_QUIET_MS, LATE_DELIVERY_DIGEST_MAX_HOLD_MS - held));
+        digest.timer.unref();
+    }
+    /**
+     * One line per drain per target. WARN, not ERROR: a missed occurrence is the expected consequence
+     * of the application not running, and the line says so — but it is still a gap in what the
+     * schedule promised, so it must not be silent either. Escalates to ERROR only when some deliveries
+     * could not be recorded, because then the gap is no longer explained durably.
+     */
+    reportLateDigest(digest) {
+        if (digest.timer)
+            clearTimeout(digest.timer);
+        this.lateDigests.delete(`${digest.queueName}\u0000${digest.target}`);
+        const total = digest.recorded + digest.alreadyHeld + digest.unrecorded;
+        if (total === 0)
+            return;
+        const fields = {
+            queueName: digest.queueName, target: digest.target, missed: total,
+            recorded: digest.recorded, alreadyHeld: digest.alreadyHeld, unrecorded: digest.unrecorded,
+            lateness: [...digest.lateness].sort(), earliest: digest.earliest, latest: digest.latest,
+            remedy: this.lateDigestRemedy(digest),
+            ...(digest.firstError ? { firstError: digest.firstError } : {}),
+        };
+        if (digest.unrecorded > 0)
+            this.logger.error('Scheduled occurrences delivered after their admission window', fields);
+        else
+            this.logger.warn('Scheduled occurrences delivered after their admission window', fields);
+    }
     /**
      * Start an admitted occurrence WITHOUT waiting for it, under the same bound as recovery.
      *
@@ -165,6 +332,7 @@ export class ScheduledExecutionRunner {
         if (this.timer)
             clearInterval(this.timer);
         this.timer = undefined;
+        this.flushLateDeliveryDigests();
     }
     async scan(queueName, resolve) {
         if (this.scanning)
@@ -384,20 +552,22 @@ export class ScheduledExecutionRunner {
         if (!payloadJson)
             return undefined;
         try {
-            const payload = JSON.parse(payloadJson);
-            // Prefixed by kind: a batch and a minion that happen to share a name are different targets.
-            if (typeof payload.batchRef === 'string' && payload.batchRef !== '')
-                return `batch:${payload.batchRef}`;
-            if (typeof payload.operationPathKey === 'string' && payload.operationPathKey !== '')
-                return `operation:${payload.operationPathKey}`;
-            if (typeof payload.minionName === 'string' && payload.minionName !== '')
-                return `minion:${payload.minionName}`;
-            return undefined;
+            return this.targetKey(JSON.parse(payloadJson));
         }
         catch {
             return undefined;
         }
     }
+    /** Prefixed by kind: a batch and a minion that happen to share a name are different targets. */
+    targetKey(payload) {
+        if (typeof payload.batchRef === 'string' && payload.batchRef !== '')
+            return `batch:${payload.batchRef}`;
+        if (typeof payload.operationPathKey === 'string' && payload.operationPathKey !== '')
+            return `operation:${payload.operationPathKey}`;
+        if (typeof payload.minionName === 'string' && payload.minionName !== '')
+            return `minion:${payload.minionName}`;
+        return undefined;
+    }
     failureDetails(error) {
         if (isWildoBackendError(error))
             return error;

package/dist/esm/queue/scheduled-execution.backend.service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"scheduled-execution.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/scheduled-execution.backend.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,SAAS,EAAE,aAAa,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,kBAAkB,EACjJ,gBAAgB,EAAE,aAAa,EAAE,SAAS,EAAE,gCAAgC,EAAE,oBAAoB,EAClG,uBAAuB,EAAE,8BAA8B,EAAE,qBAAqB,EAAE,sBAAsB,EAItG,6BAA6B,EAAE,uBAAuB,GACvD,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAE1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,8EAA8E,CAAC;AACnH;;;GAGG;AACH,MAAM,+BAA+B,GAAG,CAAC,CAAC;AAW1C;;;;;;;GAOG;AACH,MAAM,OAAO,wBAAwB;IAsBhB;IACA;IACA;IACA;IAxBX,KAAK,CAAkC;IACvC,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,KAAK,CAAC;IACT,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,UAAU,GAAG,CAAC,CAAC;IACN,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,4FAA4F;IAC3E,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IACrD;;;;;;;;;OASG;IACc,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnD,YACmB,OAAwC,EACxC,MAAsD,EACtD,cAA0I,EAC1I,QAAiD;QAHjD,YAAO,GAAP,OAAO,CAAiC;QACxC,WAAM,GAAN,MAAM,CAAgD;QACtD,mBAAc,GAAd,cAAc,CAA4H;QAC1I,aAAQ,GAAR,QAAQ,CAAyC;IACjE,CAAC;IAEJ;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,KAAK,CAAC,GAAe,EAAE,SAAiB,EAAE,OAAmC;QACjF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,gCAAgC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClE,IAAI,QAAQ;YAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,GAAG,EAAE,SAAS,EAAE;YACtE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM;YAChF,cAAc,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,6BAA6B,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;YACnH,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,8CAA8C,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM;YACpI,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,aAAa,EAAE,GAAG;YAC5E,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK;SACtE,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,EAAE,CAAC;YAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2DAA2D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;QACzI,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;YACxB,gGAAgG;YAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1G,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,KAAK,KAAK,uBAAuB,CAAC,KAAK,EAAE,CAAC;YAChI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK;gBACzF,KAAK,EAAE,GAAG,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,GAAiB,EAAE,OAAmC;QAC3D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,+BAA+B;YAAE,OAAO;QAChE,MAAM,GAAG,GAAkB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAClE,6FAA6F;YAC7F,sFAAsF;YACtF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;YAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,GAAe,EAAE,SAAiB,EAAE,OAAmC;QACnF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC;YAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAAC,CAAC;QACzC,OAAO,KAAK,EAAE,CAAC;YACb,6FAA6F;YAC7F,sFAAsF;YACtF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,SAAiB,EAAE,OAAmC;QAC1D,IAAI,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC9E,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,MAAM,IAAI,GAAG,GAAG,EAAE;YAAG,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACpE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3F,CAAC,CAAC,CAAC;QAAC,CAAC,CAAC;QACN,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,EAAE,CAAC;IACT,CAAC;IAED,mGAAmG;IACnG,IAAI;QACF,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,OAAmC;QAC/D,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;YACxE,mFAAmF;YACnF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,+BAA+B;oBAAE,MAAM;gBAC/D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;oBAAE,SAAS;gBACzC,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBACxD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7F,CAAC,CAAC,CAAC;YACL,CAAC;YACD,kGAAkG;YAClG,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC;gBAC/F,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,KAAK,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBAC1D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC3F,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;gBAAS,CAAC;YAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IAED,2FAA2F;IACnF,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,UAAkB;QAC9D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;QACnF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;gBAAE,MAAM;YAC3E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,GAAiB;QACrC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QAChF,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAEO,gBAAgB,CAAC,MAA+B,EAAE,QAAoC;QAC5F,IAAI,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACnF,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,wFAAwF;IAChF,UAAU,CAAC,KAA8B;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACjD,OAAO,gBAAgB,CAAC,KAAK,CAAC,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACtF,WAAW,EAAE,UAAU,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,GAAiB,EAAE,OAAmC,EAAE,UAAmB;QAC/F,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QACnE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC;YACjC,IAAI,CAAC,KAAK,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,uBAAuB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;gBAAE,OAAO;YAC/G,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE;gBAAE,OAAO;YACjE,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACnC,IAAI,QAAgD,CAAC;YACrD,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,gCAAgC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,gCAAgC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;oBACtJ,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,kDAAkD,CAAC,CAAC;oBAAC,OAAO;gBACrF,CAAC;gBACD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,+CAA+C,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC5F,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,uDAAuD,CAAC,CAAC;gBAAC,OAAO;YAC1F,CAAC;YACD,IAAI,KAAK,CAAC,KAAK,KAAK,uBAAuB,CAAC,WAAW,EAAE,CAAC;gBACxD,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,0DAA0D,CAAC,CAAC;gBAAC,OAAO;YAC7F,CAAC;YACD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU;gBAAE,OAAO;YACvE,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC;YAC1C,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBAC7E,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,8CAA8C,CAAC,CAAC;gBAAC,OAAO;YACjF,CAAC;YACD,MAAM,SAAS,GAAG,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC;YACvH,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC;YAChG,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;YAC3B,MAAM,YAAY,GAA4B;gBAC5C,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,CAAC;gBAC1F,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;gBACpF,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;aAC3E,CAAC;YACF,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,YAAY,EAAE;gBAC9F,MAAM,EAAE,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK;aACvG,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC7G,MAAM,OAAO,GAA4B,EAAE,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,CAAC,YAAY,EAAE,CAAC;YACjI,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,IAAI,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;gBACjC,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;oBACtC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,cAAc;wBAAE,OAAO;oBAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE;4BAClF,GAAG,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;yBAC5G,CAAC,CAAC;wBACH,IAAI,IAAI;4BAAE,OAAO,GAAG,IAAI,CAAC;;4BAAM,IAAI,GAAG,IAAI,CAAC;oBAC7C,CAAC;oBAAC,MAAM,CAAC;wBAAC,IAAI,GAAG,IAAI,CAAC;oBAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;YACzC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,MAA6B,CAAC;YAClC,IAAI,cAAyD,CAAC;YAC9D,IAAI,OAAgB,CAAC;YACrB,IAAI,sBAA0C,CAAC;YAC/C,IAAI,UAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,IAAI,SAAS;oBAAE,cAAc,GAAG,MAAM,QAAQ,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC;;oBAC9D,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,UAAU,GAAG,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,SAAS,IAAI,cAAc,EAAE,OAAO,KAAK,8BAA8B,CAAC,SAAS,CAAC;gBAClI,UAAU,GAAG,cAAc,EAAE,UAAU,IAAI,MAAM,EAAE,UAAU,CAAC;gBAC9D,IAAI,UAAU,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;oBAC3C,IAAI,CAAC;wBACH,IAAI,CAAC,IAAI,CAAC,cAAc;4BAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;wBACrF,sBAAsB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBACvE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC9H,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,OAAO,GAAG,KAAK,CAAC;YAAC,CAAC;oBAC5B,CAAC;gBAAC,aAAa,CAAC,SAAS,CAAC,CAAC;gBAAC,MAAM,UAAU,CAAC;YAAC,CAAC;YACvD,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qEAAqE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAAC,OAAO;YAC1I,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,cAAc,EAAE,OAAO,KAAK,8BAA8B,CAAC,SAAS,EAAE,CAAC;oBACzE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,uBAAuB,CAAC,cAAc,CAAC,CAAC;oBAAC,OAAO;gBACnH,CAAC;gBACD,IAAI,cAAc,EAAE,OAAO,KAAK,8BAA8B,CAAC,aAAa,EAAE,CAAC;oBAC7E,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;oBAAC,OAAO;gBACzC,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;gBAAC,OAAO;YAC3F,CAAC;YACD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,sBAAsB,EAAE,UAAU,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACzH,OAAO,GAAG,MAAM,EAAE,KAAK,IAAI,OAAO,CAAC;YACnC,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;YAC7C,MAAM,SAAS,GAAG,CAAC,mBAAmB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,qBAAqB,KAAK,SAAS;mBACzF,OAAO,CAAC,qBAAqB,KAAK,sBAAsB,CAAC,KAAK,CAAC;YACpE,IAAI,MAAM,CAAC,qBAAqB,IAAI,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBACtG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,4FAA4F;gBAC5F,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,8DAA8D,EAAE,OAAO,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,SAAS;gBAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5D,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,wFAAwF;IAChF,QAAQ,CAAC,GAAiB;QAChC,MAAM,WAAW,GAAG,GAAG,CAAC,cAAc,EAAE,WAAW,CAAC;QACpD,IAAI,CAAC,WAAW;YAAE,OAAO,SAAS,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAA6E,CAAC;YACpH,4FAA4F;YAC5F,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,EAAE;gBAAE,OAAO,SAAS,OAAO,CAAC,QAAQ,EAAE,CAAC;YACxG,IAAI,OAAO,OAAO,CAAC,gBAAgB,KAAK,QAAQ,IAAI,OAAO,CAAC,gBAAgB,KAAK,EAAE;gBAAE,OAAO,aAAa,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACpI,IAAI,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,KAAK,EAAE;gBAAE,OAAO,UAAU,OAAO,CAAC,UAAU,EAAE,CAAC;YAC/G,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;IAC/B,CAAC;IAEO,cAAc,CAAC,KAAc;QACnC,IAAI,mBAAmB,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7C,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE;YACxG,sBAAsB,EAAE,2BAA2B,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG;YAC7E,OAAO,EAAE,EAAE,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,KAA8B,EAAE,OAAe;QAC3D,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACnI,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAiB,EAAE,MAAc,EAAE,OAAiB;QACvE,MAAM,KAAK,GAAG,GAAG,CAAC,cAAe,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YAClF,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,MAAM,EAAE,MAAM;YACrF,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;SACpC,EAAE,EAAE,MAAM,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACrH,IAAI,OAAO;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACtG,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,GAAiB,EAAE,cAAuB,EAAE,OAAiB;QAC9E,MAAM,KAAK,GAAG,GAAG,CAAC,cAAe,CAAC;QAClC,MAAM,IAAI,GAAG,aAAa,CAAC,cAAc,CAAC,EAAE,GAAG,oBAAoB,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;QACrI,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChF,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/G,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,uDAAuD,EAAE,OAAO,CAAC,CAAC;YAAC,OAAO;QACnG,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YAClE,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,OAAO,EAAE,cAAc;YAC9F,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC;SAClI,EAAE,EAAE,MAAM,EAAE,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACzH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAiB,EAAE,sBAA+B,EAAE,UAAmB,EAC5F,MAAM,GAAG,uBAAuB,CAAC,SAAS;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,cAAe,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACxG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YACnF,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,QAAQ;YAC/E,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;YAC1D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC;SACzC,EAAE;YACD,MAAM,EAAE,kBAAkB,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,sBAAsB;YAC9E,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,yBAAyB,GAAG,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;SACrI,CAAC,CAAC;QACH,IAAI,QAAQ;YAAE,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED,6GAA6G;IACrG,KAAK,CAAC,eAAe,CAAC,GAAiB;QAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC;QACjC,IAAI,KAAK,EAAE,KAAK,KAAK,uBAAuB,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,gBAAgB;YAAE,OAAO;QAC3G,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAC/D,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;YACxI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0EAA0E,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YACnH,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,sBAAsB;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/G,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,4EAA4E;YACjG,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YAC/C,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QACzI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gEAAgE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChI,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC;gBAC1G,oBAAoB,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAGnB,EAAE,OAAmC;QACpC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACjH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,GAAG,EAAE,cAAc,CAAC;QAClC,0FAA0F;QAC1F,4FAA4F;QAC5F,+FAA+F;QAC/F,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK;eACvH,KAAK,CAAC,KAAK,KAAK,uBAAuB,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,gBAAgB;YAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvK,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACrG,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC1B,IAAI,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;QAC1C,IAAI,QAAQ,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,KAAK,sBAAsB,CAAC,KAAK,CAAC;QAC7D,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,KAAK,sBAAsB,CAAC,iBAAiB,CAAC;QAC7E,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,sBAAsB,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnI,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;YACvD,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACxE,MAAM,aAAa,GAAG,gCAAgC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACxE,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC/C,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;gBAC7B,+FAA+F;gBAC/F,MAAM,GAAG,aAAa,CAAC;gBACvB,IAAI,KAAK,CAAC,MAAM,KAAK,8CAA8C,EAAE,CAAC;oBACpE,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,6BAA6B,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtH,CAAC;gBACD,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;YAC9G,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,gCAAgC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBAC5G,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;YACxF,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,IAAI,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,MAAM,CAAC;mBAC/E,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QACD,IAAI,sBAA0C,CAAC;QAC/C,IAAI,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACxF,sBAAsB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAC,GAAG,EAAE,GAAG,EACtE,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,qBAAqB,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,EAC/F,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAC1F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YAC/E,GAAG,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC;YACxE,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;YAC1D,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC,QAAQ;YACjF,6EAA6E;YAC7E,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG;YACzC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnH,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACnD,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,sBAAsB;YACtH,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,yBAAyB,GAAG,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC;SAC7G,CAAC,CAAC;QACH,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAC1E,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACxI,CAAC;CACF","sourcesContent":["import { randomUUID } from 'node:crypto';\nimport {\n ErrorType, ErrorSeverity, ErrorCustomMessageReference, ErrorHandling_Strategy, isWildoBackendError, JOB_EXECUTION_TTL_SECONDS, JobExecutionStatus,\n JobPayloadSchema, JobRetryUtils, JobStatus, normalizeScheduledRecoveryPolicy, QUEUE_RETRY_DEFAULTS,\n ScheduledExecutionPhase, ScheduledReconciliationOutcome, ScheduledReplaySafety, ScheduledReviewOutcome,\n type JobExecution, type JobPayload, type JobResult, type ScheduledAttemptContext,\n type ScheduledExecutionState, type ScheduledReconciliationResult, type ScheduledRecoveryPolicy,\n type ScheduledEvidencePolicy,\n ScheduledEvidencePolicySchema, ScheduledEvidenceSource,\n} from '@wildo-ai/saas-models';\nimport type { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';\nimport type { LoggerService } from '../monitoring/logger/services/logger.service';\nimport type { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';\nimport { createScheduledRunEvidence } from './scheduled-evidence.backend';\n\nimport { describeCaughtError } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\n/*\n How many scheduled occurrences one runner executes at once — first deliveries and recoveries alike,\n * since both go through `process`. Other replicas arbitrate the same rows through compare-and-swap.\n /\nconst SCHEDULED_EXECUTION_CONCURRENCY = 8;\n\n/* Resolved from local handler declarations, never from an untrusted message's policy fields. /\nexport interface ScheduledExecutionContract {\n policy?: Partial<ScheduledRecoveryPolicy>;\n evidencePolicy?: ScheduledEvidencePolicy;\n execute(context: ScheduledAttemptContext): Promise<JobResult>;\n reconcile?(context: ScheduledAttemptContext): Promise<ScheduledReconciliationResult>;\n}\nexport type ScheduledExecutionResolver = (job: JobPayload) => Promise<ScheduledExecutionContract \| undefined>;\n\n/\n Durable scheduled execution owner shared by application consumers and minions.\n \n Broker receipt authenticates and inserts an occurrence. A database scanner subsequently owns\n * recovery, so a five-minute JWT never becomes a long-lived replay credential. Every admission and\n * mutation uses the persisted revision. Leases detect lost contact; they do not cancel external work.\n * Applications opting into recovery must provide effect-level idempotency or reconciliation.\n /\nexport class ScheduledExecutionRunner {\n private timer?: ReturnType<typeof setInterval>;\n private scanning = false;\n private archiving = false;\n private readonly archivesActive = new Set<string>();\n private generation = 0;\n private readonly active = new Set<string>();\n /* Runs started by {@link launch} and not yet settled — what {@link whenIdle} waits for. /\n private readonly launched = new Set<Promise<void>>();\n /\n The targets (batch, operation or minion) with an occurrence running in THIS runner.\n \n A consumer used to hold one delivery at a time, so two occurrences of one target never ran\n * together here — by accident of the queue rather than by design, and at the price of dropping\n * whatever waited too long. Now that receipt no longer waits on execution, the non-overlap is kept\n * deliberately: a second occurrence of a running target stays due and the scanner starts it once the\n * first settles. A sweep that takes longer than its schedule therefore queues instead of racing\n * itself; if it waits past its recovery deadline it goes to review, where it is visible.\n /\n private readonly activeTargets = new Set<string>();\n\n constructor(\n private readonly tracker: JobStatusTrackingBackendService,\n private readonly logger: Pick<LoggerService, 'info' \| 'warn' \| 'error'>,\n private readonly payloadStorage?: Pick<JobPayloadStorageBackendService, 'store'> & Partial<Pick<JobPayloadStorageBackendService, 'storeScheduledEvidence'>>,\n private readonly activity?: { started(): void; completed(): void },\n ) {}\n\n /\n Make one broker delivery DURABLE, and hand back the occurrence to run — without running it.\n \n This is the whole of what a broker receipt has to do. The signed token proves the delivery is\n * genuine for five minutes, so its only job is to get the occurrence into the database inside that\n * window; from there the database owns it (see the class comment).\n \n ## Why receipt and execution are separate calls\n \n They were one: {@link receive} inserted the occurrence and then awaited the whole run, so the\n * consumer could only acknowledge the broker message once the job FINISHED. A queue consumer holds\n * one unacknowledged message at a time, so every later delivery sat unreceived behind a long run,\n * and when that run outlasted five minutes each of them failed verification (\"jwt expired\") and was\n * dead-lettered — the occurrence was lost without ever being recorded. Measured on Wonder Todos,\n * 2026-09-30: an `immudb.audit-reconcile` run that never returned held every scheduled job in the\n * application (erasure sweep, retention sweep, organization purge) until the next backend restart.\n * The consumers now {@link admit}, acknowledge, then {@link launch}.\n \n Returns `undefined` when there is nothing to run: no local handler (the receipt is retained for\n * review) or a legacy row whose effects are unknown.\n /\n async admit(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver): Promise<JobExecution \| undefined> {\n const contract = await resolve(job);\n const policy = normalizeScheduledRecoveryPolicy(contract?.policy);\n if (contract) this.assertReconciler(policy, contract);\n const now = new Date();\n const row = await this.tracker.ensureScheduledJobRecord(job, queueName, {\n phase: contract ? ScheduledExecutionPhase.READY : ScheduledExecutionPhase.REVIEW,\n evidencePolicy: contract?.evidencePolicy ? ScheduledEvidencePolicySchema.parse(contract.evidencePolicy) : undefined,\n reason: contract ? undefined : 'Handler unavailable at authenticated receipt', revision: 0, payloadJson: JSON.stringify(job), policy,\n deadline: new Date(now.getTime() + policy.maxRecoveryMs), nextAttemptAt: now,\n attemptCount: 0, attempts: [], resolutions: [], reconciledSafe: false,\n });\n if (!contract) { this.logger.warn('Scheduled target unavailable; receipt retained for review', { jobId: job.jobId }); return undefined; }\n if (!row.scheduledState) {\n // Legacy active receipts have unknown effects. Never turn an old PROCESSING row into a new run.\n this.logger.warn('Legacy scheduled occurrence requires review', { jobId: job.jobId, status: row.status });\n return undefined;\n }\n if (this.active.has(job.jobId) \|\| row.scheduledState.revision > 0 \|\| row.scheduledState.phase !== ScheduledExecutionPhase.READY) {\n this.logger.info('Repeated scheduled receipt reused durable occurrence', { jobId: job.jobId,\n phase: row.scheduledState.phase, attemptCount: row.scheduledState.attemptCount });\n }\n return row;\n }\n\n /\n Start an admitted occurrence WITHOUT waiting for it, under the same bound as recovery.\n \n When every execution slot is taken the occurrence is simply left due: it is already durable, and\n * the recovery scanner starts it within a second of a slot freeing. Nothing is dropped and nothing\n * is waited on, so the caller can acknowledge its broker message at once.\n /\n launch(row: JobExecution, resolve: ScheduledExecutionResolver): void {\n if (this.active.size >= SCHEDULED_EXECUTION_CONCURRENCY) return;\n const run: Promise<void> = this.process(row, resolve).catch(error => {\n // Intake is already durable. Neither tracking nor handler failures hand retry ownership back\n // to the broker: the scanner recovers only under the captured effect-safety contract.\n this.logger.error('Scheduled attempt could not settle durably', { jobId: row.jobId, error: String(error) });\n }).finally(() => { this.launched.delete(run); });\n this.launched.add(run);\n }\n\n /* Resolves once every run {@link launch} started has settled. Never rejects. /\n async whenIdle(): Promise<void> {\n while (this.launched.size > 0) await Promise.allSettled([...this.launched]);\n }\n\n /\n Admit and RUN one occurrence, resolving when the attempt has settled.\n \n For a caller that owns no broker message (tests, and any in-process driver). A broker consumer must\n * not use it: holding the delivery for the length of the run is what dropped every job queued behind\n * a long one — see {@link admit}.\n /\n async receive(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver): Promise<void> {\n const row = await this.admit(job, queueName, resolve);\n if (!row) return;\n try { await this.process(row, resolve); }\n catch (error) {\n // Intake is already durable. Neither tracking nor handler failures hand retry ownership back\n // to the broker: the scanner recovers only under the captured effect-safety contract.\n this.logger.error('Scheduled attempt could not settle durably', { jobId: row.jobId, error: String(error) });\n }\n }\n\n /\n Start the recovery scanner for ONE queue.\n \n Callers name the queue and nothing else. The runner used to take an `applicationId` here as\n * well, and every caller — the application consumer and the minion tick consumer — passed the\n * PLATFORM id, because that is what their queue names are built from. The tracker then scanned\n * rows under it, which only worked because the same platform id had been stamped into the rows.\n * The row identity is the tracker's to resolve (`JobStatusTrackingBackendService.canonicalApplicationId`);\n * removing the parameter is what makes it impossible for a caller to scan under the wrong one.\n /\n start(queueName: string, resolve: ScheduledExecutionResolver): void {\n if (this.timer) throw new Error('Scheduled recovery scanner already started');\n this.generation += 1;\n const tick = () => { void this.scan(queueName, resolve).catch(error => {\n this.logger.error('Scheduled recovery scan failed', { queueName, error: String(error) });\n }); };\n this.timer = setInterval(tick, 1000);\n this.timer.unref();\n tick();\n }\n\n /* Stop admitting recovery; already admitted work retains heartbeat ownership until settlement. /\n stop(): void {\n this.generation += 1;\n if (this.timer) clearInterval(this.timer);\n this.timer = undefined;\n }\n\n async scan(queueName: string, resolve: ScheduledExecutionResolver): Promise<void> {\n if (this.scanning) return;\n this.scanning = true;\n const generation = this.generation;\n try {\n const rows = await this.tracker.listScheduledDue(queueName, new Date());\n // Bounded process concurrency; other replicas arbitrate the same rows through CAS.\n for (const row of rows) {\n if (generation !== this.generation) break;\n if (this.active.size >= SCHEDULED_EXECUTION_CONCURRENCY) break;\n if (this.active.has(row.jobId)) continue;\n void this.process(row, resolve, generation).catch(error => {\n this.logger.error('Scheduled recovery failed', { jobId: row.jobId, error: String(error) });\n });\n }\n // Archival has its own bounded pass. A storage outage must not fill the recovery admission queue.\n if (generation === this.generation && !this.archiving && this.tracker.listScheduledArchivesDue) {\n this.archiving = true;\n void this.scanArchives(queueName, generation).catch(error => {\n this.logger.error('Scheduled evidence scan failed', { queueName, error: String(error) });\n }).finally(() => { this.archiving = false; });\n }\n } finally { this.scanning = false; }\n }\n\n /* Archive I/O cannot hold the recovery scanner or consume its handler admission slots. /\n private async scanArchives(queueName: string, generation: number): Promise<void> {\n const pending = await this.tracker.listScheduledArchivesDue(queueName, new Date());\n for (const row of pending) {\n if (generation !== this.generation \|\| this.archivesActive.size >= 4) break;\n this.launchArchive(row);\n }\n }\n\n private launchArchive(row: JobExecution): void {\n if (this.archivesActive.size >= 4 \|\| this.archivesActive.has(row.jobId)) return;\n this.archivesActive.add(row.jobId);\n void this.archiveEvidence(row).catch(error => {\n this.logger.error('Scheduled evidence retry failed', { jobId: row.jobId, error: String(error) });\n }).finally(() => { this.archivesActive.delete(row.jobId); });\n }\n\n private assertReconciler(policy: ScheduledRecoveryPolicy, contract: ScheduledExecutionContract): void {\n if (policy.replaySafety === ScheduledReplaySafety.RECONCILE && !contract.reconcile) {\n throw new Error('Reconciliation policy requires a locally registered reconciler');\n }\n }\n\n /* Restore transport dates for both scanner execution and attributed operator retry. /\n private restoreJob(state: ScheduledExecutionState): JobPayload {\n const serialized = JSON.parse(state.payloadJson);\n return JobPayloadSchema.parse({ ...serialized, createdAt: new Date(serialized.createdAt),\n scheduledAt: serialized.scheduledAt === undefined ? undefined : new Date(serialized.scheduledAt) });\n }\n\n private async process(row: JobExecution, resolve: ScheduledExecutionResolver, generation?: number): Promise<void> {\n if (this.active.has(row.jobId)) return;\n const target = this.targetOf(row);\n if (target !== undefined && this.activeTargets.has(target)) return;\n this.active.add(row.jobId);\n if (target !== undefined) this.activeTargets.add(target);\n this.activity?.started();\n try {\n const state = row.scheduledState;\n if (!state \|\| [ScheduledExecutionPhase.REVIEW, ScheduledExecutionPhase.FINISHED].includes(state.phase)) return;\n if (new Date(state.nextAttemptAt).getTime() > Date.now()) return;\n const job = this.restoreJob(state);\n let contract: ScheduledExecutionContract \| undefined;\n try {\n contract = await resolve(job);\n if (!contract \|\| JSON.stringify(normalizeScheduledRecoveryPolicy(contract.policy)) !== JSON.stringify(normalizeScheduledRecoveryPolicy(state.policy))) {\n await this.review(row, 'Handler unavailable or recovery contract changed'); return;\n }\n this.assertReconciler(state.policy, contract);\n } catch { await this.review(row, 'Local recovery contract could not be resolved'); return; }\n if (Date.now() >= new Date(state.deadline).getTime() \|\| state.attemptCount >= state.policy.maxAttempts) {\n await this.review(row, 'Recovery deadline or total admission budget exhausted'); return;\n }\n if (state.phase === ScheduledExecutionPhase.RECONCILING) {\n await this.review(row, 'Reconciliation worker contact lost; outcome is uncertain'); return;\n }\n if (generation !== undefined && generation !== this.generation) return;\n const recovering = state.attemptCount > 0;\n if (recovering && state.policy.replaySafety === ScheduledReplaySafety.MANUAL) {\n await this.review(row, 'Interrupted execution requires manual review'); return;\n }\n const reconcile = recovering && state.policy.replaySafety === ScheduledReplaySafety.RECONCILE && !state.reconciledSafe;\n const phase = reconcile ? ScheduledExecutionPhase.RECONCILING : ScheduledExecutionPhase.RUNNING;\n const now = new Date();\n const owner = randomUUID();\n const claimedState: ScheduledExecutionState = {\n ...state, phase, owner, revision: state.revision + 1, attemptCount: state.attemptCount + 1,\n reconciledSafe: false, nextAttemptAt: new Date(now.getTime() + state.policy.leaseMs),\n attempts: [...state.attempts, { attemptId: owner, phase, startedAt: now }],\n };\n let current = await this.tracker.compareScheduledState(row.jobId, state.revision, claimedState, {\n status: JobExecutionStatus.PROCESSING, startedAt: now, retryCount: state.attemptCount, workerId: owner,\n });\n if (!current) { this.logger.info('Duplicate scheduled admission suppressed', { jobId: row.jobId }); return; }\n const context: ScheduledAttemptContext = { occurrenceId: row.jobId, attemptId: owner, attemptNumber: claimedState.attemptCount };\n let lost = false;\n let heartbeats = Promise.resolve();\n const heartbeat = setInterval(() => {\n heartbeats = heartbeats.then(async () => {\n if (lost \|\| !current?.scheduledState) return;\n const previous = current.scheduledState;\n try {\n const next = await this.tracker.compareScheduledState(row.jobId, previous.revision, {\n ...previous, revision: previous.revision + 1, nextAttemptAt: new Date(Date.now() + previous.policy.leaseMs),\n });\n if (next) current = next; else lost = true;\n } catch { lost = true; }\n });\n }, Math.floor(state.policy.leaseMs / 3));\n heartbeat.unref();\n let result: JobResult \| undefined;\n let reconciliation: ScheduledReconciliationResult \| undefined;\n let failure: unknown;\n let outputPayloadStorageId: string \| undefined;\n let resultJson: string \| undefined;\n try {\n if (reconcile) reconciliation = await contract.reconcile!(context);\n else result = await contract.execute(context);\n const successful = result?.status === JobStatus.COMPLETED \|\| reconciliation?.outcome === ScheduledReconciliationOutcome.COMPLETED;\n resultJson = reconciliation?.resultJson ?? result?.resultJson;\n if (successful && resultJson !== undefined) {\n try {\n if (!this.payloadStorage) throw new Error('Scheduled result storage is unavailable');\n outputPayloadStorageId = await this.payloadStorage.store(resultJson);\n } catch (error) {\n this.logger.error('Scheduled output storage failed after successful execution', { jobId: row.jobId, error: String(error) });\n }\n }\n } catch (error) { failure = error; }\n finally { clearInterval(heartbeat); await heartbeats; }\n if (lost \|\| !current?.scheduledState) {\n this.logger.warn('Scheduled attempt lost ownership; outcome cannot overwrite recovery', { jobId: row.jobId, attemptId: owner }); return;\n }\n if (reconcile) {\n if (reconciliation?.outcome === ScheduledReconciliationOutcome.COMPLETED) {\n await this.complete(current, outputPayloadStorageId, resultJson, ScheduledEvidenceSource.RECONCILIATION); return;\n }\n if (reconciliation?.outcome === ScheduledReconciliationOutcome.SAFE_TO_RETRY) {\n await this.wait(current, true); return;\n }\n await this.review(current, 'Reconciliation failed or was inconclusive', failure); return;\n }\n if (result?.status === JobStatus.COMPLETED) { await this.complete(current, outputPayloadStorageId, resultJson); return; }\n failure = result?.error ?? failure;\n const policy = current.scheduledState.policy;\n const retryable = !isWildoBackendError(failure) \|\| failure.errorHandlingStrategy === undefined\n \|\| failure.errorHandlingStrategy === ErrorHandling_Strategy.RETRY;\n if (policy.retryReportedFailures && retryable && policy.replaySafety !== ScheduledReplaySafety.MANUAL) {\n await this.wait(current, false, failure);\n } else {\n // A reported failure can follow effects. Preserve it for an explicit decision, without TTL.\n await this.review(current, 'Execution reported failure; automatic retry is not permitted', failure);\n }\n } finally {\n this.active.delete(row.jobId);\n if (target !== undefined) this.activeTargets.delete(target);\n this.activity?.completed();\n }\n }\n\n /* The batch, operation or minion an occurrence runs, read from its durable payload. /\n private targetOf(row: JobExecution): string \| undefined {\n const payloadJson = row.scheduledState?.payloadJson;\n if (!payloadJson) return undefined;\n try {\n const payload = JSON.parse(payloadJson) as { batchRef?: unknown; operationPathKey?: unknown; minionName?: unknown };\n // Prefixed by kind: a batch and a minion that happen to share a name are different targets.\n if (typeof payload.batchRef === 'string' && payload.batchRef !== '') return `batch:${payload.batchRef}`;\n if (typeof payload.operationPathKey === 'string' && payload.operationPathKey !== '') return `operation:${payload.operationPathKey}`;\n if (typeof payload.minionName === 'string' && payload.minionName !== '') return `minion:${payload.minionName}`;\n return undefined;\n } catch { return undefined; }\n }\n\n private failureDetails(error: unknown): NonNullable<JobExecution['error']> {\n if (isWildoBackendError(error)) return error;\n return { type: ErrorType.INTERNAL, severity: ErrorSeverity.HIGH, timestamp: new Date(), backendActions: [],\n customMessageReference: ErrorCustomMessageReference.INTERNAL, statusCode: 500,\n context: { message: describeCaughtError(error) } };\n }\n\n private ended(state: ScheduledExecutionState, outcome: string): ScheduledExecutionState['attempts'] {\n return state.attempts.map(attempt => attempt.attemptId === state.owner ? { ...attempt, endedAt: new Date(), outcome } : attempt);\n }\n\n private async review(row: JobExecution, reason: string, failure?: unknown): Promise<void> {\n const state = row.scheduledState!;\n const updated = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.REVIEW, reason,\n attempts: this.ended(state, reason),\n }, { status: JobExecutionStatus.FAILED, ...(failure !== undefined ? { error: this.failureDetails(failure) } : {}) });\n if (updated) this.logger.warn('Scheduled occurrence requires review', { jobId: row.jobId, reason });\n }\n\n private async wait(row: JobExecution, reconciledSafe: boolean, failure?: unknown): Promise<void> {\n const state = row.scheduledState!;\n const base = JobRetryUtils.calculateDelay({ ...QUEUE_RETRY_DEFAULTS, ...state.policy.backoff }, Math.max(0, state.attemptCount - 1));\n const delay = Math.round(base (1 - state.policy.jitterRatio * Math.random()));\n if (state.attemptCount >= state.policy.maxAttempts \|\| Date.now() + delay >= new Date(state.deadline).getTime()) {\n await this.review(row, 'Recovery deadline or total admission budget exhausted', failure); return;\n }\n await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.WAITING, reconciledSafe,\n nextAttemptAt: new Date(Date.now() + delay), attempts: this.ended(state, reconciledSafe ? 'Reconciled safe' : 'Reported failure'),\n }, { status: JobExecutionStatus.RETRYING, ...(failure !== undefined ? { error: this.failureDetails(failure) } : {}) });\n }\n\n private async complete(row: JobExecution, outputPayloadStorageId?: string, resultJson?: string,\n source = ScheduledEvidenceSource.EXECUTION): Promise<void> {\n const state = row.scheduledState!;\n const now = new Date();\n const evidence = createScheduledRunEvidence(row, now, source, JobExecutionStatus.COMPLETED, resultJson);\n const terminal = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.FINISHED,\n evidence, evidenceArchiveAfter: evidence ? now : undefined,\n attempts: this.ended(state, 'Completed'),\n }, {\n status: JobExecutionStatus.COMPLETED, completedAt: now, outputPayloadStorageId,\n expiresAt: new Date(Math.max(now.getTime() + JOB_EXECUTION_TTL_SECONDS * 1000, new Date(state.deadline).getTime() + 10 * 60 * 1000)),\n });\n if (terminal) this.launchArchive(terminal);\n }\n\n /** Publication follows terminal CAS. Failure retries only this projection, never the business invocation. /\n private async archiveEvidence(row: JobExecution): Promise<void> {\n const state = row.scheduledState;\n if (state?.phase !== ScheduledExecutionPhase.FINISHED \|\| !state.evidence \|\| state.evidenceArchived) return;\n if (new Date(state.evidence.expiresAt).getTime() <= Date.now()) {\n const { evidence: _expired, ...rest } = state;\n await this.tracker.compareScheduledState(row.jobId, state.revision, { ...rest, revision: state.revision + 1, evidenceArchived: false });\n this.logger.warn('Scheduled evidence expired before archival; coverage remains unavailable', { jobId: row.jobId });\n return;\n }\n try {\n if (!this.payloadStorage?.storeScheduledEvidence) throw new Error('Scheduled evidence archive is unavailable');\n const stored = await this.payloadStorage.storeScheduledEvidence(state.evidence);\n if (!stored) return; // Expired while dispatching: the next pass disposes the pending projection.\n const { evidence: _archived, ...rest } = state;\n await this.tracker.compareScheduledState(row.jobId, state.revision, { ...rest, revision: state.revision + 1, evidenceArchived: true });\n } catch (error) {\n this.logger.error('Scheduled evidence archival failed; retained for bounded retry', { jobId: row.jobId, error: String(error) });\n await this.tracker.compareScheduledState(row.jobId, state.revision, { ...state, revision: state.revision + 1,\n evidenceArchiveAfter: new Date(Date.now() + 60000) });\n }\n }\n\n /\n Trusted backend operator seam: callers must authorize their operator before invoking this method.\n * It cannot create a fresh occurrence or turn a manual contract into a replayable handler. The\n * decision and attribution are persisted in the same CAS as its effect; stale decisions fail.\n /\n async resolveReview(input: {\n queueName: string; jobId: string; expectedRevision: number;\n actor: string; reason: string; outcome: ScheduledReviewOutcome; resultJson?: string;\n }, resolve: ScheduledExecutionResolver): Promise<void> {\n if (!input.actor.trim() \|\| !input.reason.trim()) throw new Error('Operator attribution and reason are required');\n const row = await this.tracker.getByJobId(input.jobId);\n const state = row?.scheduledState;\n // Ownership is checked against the id the tracker STAMPS, not against anything the caller\n // supplies: a caller cannot name the application, so it cannot review another application's\n // row by naming it, and a row still anchored to the retired platform id is refused as foreign.\n if (!row \|\| row.applicationId !== await this.tracker.canonicalApplicationId() \|\| row.queueName !== input.queueName \|\| !state\n \|\| state.phase !== ScheduledExecutionPhase.REVIEW \|\| state.revision !== input.expectedRevision) throw new Error('Scheduled review target changed or is unavailable');\n if (state.resolutions.length >= 100) throw new Error('Scheduled operator decision budget exhausted');\n let policy = state.policy;\n let evidencePolicy = state.evidencePolicy;\n let deadline = new Date(state.deadline);\n const retry = input.outcome === ScheduledReviewOutcome.RETRY;\n const completed = input.outcome === ScheduledReviewOutcome.CONFIRM_COMPLETED;\n if (!retry && !completed && input.outcome !== ScheduledReviewOutcome.ABANDON) throw new Error('Unknown scheduled review decision');\n if (retry) {\n const contract = await resolve(this.restoreJob(state));\n if (!contract) throw new Error('Scheduled target is still unavailable');\n const currentPolicy = normalizeScheduledRecoveryPolicy(contract.policy);\n this.assertReconciler(currentPolicy, contract);\n if (state.attemptCount === 0) {\n // No effects ever began. Bind the now-present handler, without extending the original horizon.\n policy = currentPolicy;\n if (state.reason === 'Handler unavailable at authenticated receipt') {\n evidencePolicy = contract.evidencePolicy ? ScheduledEvidencePolicySchema.parse(contract.evidencePolicy) : undefined;\n }\n deadline = new Date(Math.min(deadline.getTime(), new Date(row.createdAt).getTime() + policy.maxRecoveryMs));\n } else if (JSON.stringify(currentPolicy) !== JSON.stringify(normalizeScheduledRecoveryPolicy(state.policy))) {\n throw new Error('Recovery contract changed; operator cannot override handler safety');\n }\n if ((state.attemptCount > 0 && policy.replaySafety === ScheduledReplaySafety.MANUAL)\n \|\| state.attemptCount >= policy.maxAttempts \|\| Date.now() >= deadline.getTime()) {\n throw new Error('Retry exceeds the declared safety or recovery budget');\n }\n }\n let outputPayloadStorageId: string \| undefined;\n if (completed && input.resultJson !== undefined) {\n if (!this.payloadStorage) throw new Error('Resolution evidence storage is unavailable');\n outputPayloadStorageId = await this.payloadStorage.store(input.resultJson);\n }\n const now = new Date();\n const evidence = retry ? undefined : createScheduledRunEvidence(row, now,\n completed ? ScheduledEvidenceSource.OPERATOR_CONFIRMATION : ScheduledEvidenceSource.ABANDONMENT,\n completed ? JobExecutionStatus.COMPLETED : JobExecutionStatus.FAILED, input.resultJson);\n const next = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, policy, evidencePolicy, deadline, revision: state.revision + 1,\n evidence, evidenceArchiveAfter: evidence ? now : undefined,\n phase: retry ? ScheduledExecutionPhase.WAITING : ScheduledExecutionPhase.FINISHED,\n // A retry decision cannot impersonate the handler's reconciliation evidence.\n reconciledSafe: false, nextAttemptAt: now,\n resolutions: [...state.resolutions, { actor: input.actor, reason: input.reason, at: now, outcome: input.outcome }],\n }, retry ? { status: JobExecutionStatus.RETRYING } : {\n status: completed ? JobExecutionStatus.COMPLETED : JobExecutionStatus.FAILED, completedAt: now, outputPayloadStorageId,\n expiresAt: new Date(Math.max(now.getTime() + JOB_EXECUTION_TTL_SECONDS 1000, deadline.getTime() + 600000)),\n });\n if (!next) throw new Error('Scheduled review decision lost its revision');\n if (!retry) this.launchArchive(next);\n this.logger.info('Scheduled review resolved', { jobId: row.jobId, actor: input.actor, reason: input.reason, outcome: input.outcome });\n }\n}\n"]}
1	+ {"version":3,"file":"scheduled-execution.backend.service.js","sourceRoot":"","sources":["../../../../src/queue/scheduled-execution.backend.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,SAAS,EAAE,aAAa,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,kBAAkB,EACjJ,gBAAgB,EAAE,aAAa,EAAE,SAAS,EAAE,gCAAgC,EAAE,oBAAoB,EAClG,uBAAuB,EAAE,8BAA8B,EAAE,qBAAqB,EAAE,sBAAsB,EAItG,6BAA6B,EAAE,uBAAuB,EAAE,iCAAiC,EACzF,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAE1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,8EAA8E,CAAC;AACnH;;;GAGG;AACH,MAAM,+BAA+B,GAAG,CAAC,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,6BAA6B,GAAG,IAAI,CAAC;AAE3C;;;;GAIG;AACH,MAAM,gCAAgC,GAAG,MAAM,CAAC;AAEhD,2HAA2H;AAC3H,MAAM,oBAAoB,GAAG,4FAA4F,CAAC;AA6B1H;;;;;;;GAOG;AACH,MAAM,OAAO,wBAAwB;IAwBhB;IACA;IACA;IACA;IA1BX,KAAK,CAAkC;IACvC,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,KAAK,CAAC;IACT,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,UAAU,GAAG,CAAC,CAAC;IACN,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,4FAA4F;IAC3E,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IACrD;;;;;;;;;OASG;IACc,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACnD,kHAAkH;IACjG,WAAW,GAAG,IAAI,GAAG,EAA8B,CAAC;IAErE,YACmB,OAAwC,EACxC,MAAsD,EACtD,cAA0I,EAC1I,QAAiD;QAHjD,YAAO,GAAP,OAAO,CAAiC;QACxC,WAAM,GAAN,MAAM,CAAgD;QACtD,mBAAc,GAAd,cAAc,CAA4H;QAC1I,aAAQ,GAAR,QAAQ,CAAyC;IACjE,CAAC;IAEJ;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,KAAK,CAAC,GAAe,EAAE,SAAiB,EAAE,OAAmC;QACjF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,gCAAgC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClE,IAAI,QAAQ;YAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,GAAG,EAAE,SAAS,EAAE;YACtE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM;YAChF,cAAc,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,6BAA6B,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;YACnH,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,8CAA8C,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM;YACpI,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,aAAa,EAAE,GAAG;YAC5E,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK;SACtE,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,EAAE,CAAC;YAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2DAA2D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;QACzI,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;YACxB,gGAAgG;YAChG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1G,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,KAAK,KAAK,uBAAuB,CAAC,KAAK,EAAE,CAAC;YAChI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK;gBACzF,KAAK,EAAE,GAAG,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAe,EAAE,SAAiB,EAAE,OAAmC,EAC9F,QAAmC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAClD,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,SAAS,CAAC;QACjD,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,OAAO,GAAG,MAAM,CAAC,QAAQ;YAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;QAC7E,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM;YAAE,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;QACvE,IAAI,CAAC;YACH,+FAA+F;YAC/F,IAAI,QAAgD,CAAC;YACrD,IAAI,CAAC;gBAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,QAAQ,GAAG,SAAS,CAAC;YAAC,CAAC;YACtE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,6BAA6B,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC3H,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,GAAG,EAAE,SAAS,EAAE;gBACtE,KAAK,EAAE,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;gBACpF,MAAM,EAAE,gCAAgC,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,cAAc;gBAC1E,MAAM,EAAE,oBAAoB,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG;gBACzE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK;aACtE,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC;YACjC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC;mBACzF,KAAK,CAAC,KAAK,KAAK,uBAAuB,CAAC,MAAM,EAAE,CAAC;gBACpD,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;gBACxB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,aAAa,CAAC;YACrD,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC,CAAC;YACzG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;gBACnF,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,QAAQ;gBAC/E,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;aAC3D,EAAE;gBACD,MAAM,EAAE,iCAAiC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,GAAG;gBACnE,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,yBAAyB,GAAG,IAAI,CAAC;aACtE,CAAC,CAAC;YACH,8FAA8F;YAC9F,IAAI,QAAQ,EAAE,CAAC;gBAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;gBAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;;gBAAM,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;YACnG,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;YACvB,MAAM,CAAC,UAAU,KAAK,mBAAmB,CAAC,KAAK,CAAC,CAAC;YACjD,OAAO,KAAK,CAAC;QACf,CAAC;gBAAS,CAAC;YAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACK,gBAAgB,CAAC,MAA0B;QACjD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,wGAAwG,CAAC;QAClH,CAAC;QACD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,yBAAyB,CAAC,aAAa,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,sIAAsI,CAAC,CAAC;QACtJ,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,yBAAyB,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,wJAAwJ,CAAC,CAAC;QACxK,CAAC;QACD,OAAO,wDAAwD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI;cACnF,kFAAkF;cAClF,yFAAyF,CAAC;IAChG,CAAC;IAED,2EAA2E;IAC3E,wBAAwB;QACtB,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACrF,CAAC;IAEO,aAAa,CAAC,SAAiB,EAAE,GAAe;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,OAAO,EAAE,CAAC;QAC3D,MAAM,GAAG,GAAG,GAAG,SAAS,SAAS,MAAM,EAAE,CAAC;QAC1C,IAAI,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACtH,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,MAA0B;QACnD,IAAI,MAAM,CAAC,KAAK;YAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC1C,IAAI,IAAI,IAAI,gCAAgC,EAAE,CAAC;YAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACxF,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,6BAA6B,EAAE,gCAAgC,GAAG,IAAI,CAAC,CAAC,CAAC;QACpF,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACK,gBAAgB,CAAC,MAA0B;QACjD,IAAI,MAAM,CAAC,KAAK;YAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACvE,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO;QACxB,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK;YACjE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU;YACzF,QAAQ,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM;YACvF,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;YACrC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChE,CAAC;QACF,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8DAA8D,EAAE,MAAM,CAAC,CAAC;;YAChH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8DAA8D,EAAE,MAAM,CAAC,CAAC;IAChG,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,GAAiB,EAAE,OAAmC;QAC3D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,+BAA+B;YAAE,OAAO;QAChE,MAAM,GAAG,GAAkB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAClE,6FAA6F;YAC7F,sFAAsF;YACtF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;YAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,GAAe,EAAE,SAAiB,EAAE,OAAmC;QACnF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC;YAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAAC,CAAC;QACzC,OAAO,KAAK,EAAE,CAAC;YACb,6FAA6F;YAC7F,sFAAsF;YACtF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,SAAiB,EAAE,OAAmC;QAC1D,IAAI,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC9E,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,MAAM,IAAI,GAAG,GAAG,EAAE;YAAG,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACpE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3F,CAAC,CAAC,CAAC;QAAC,CAAC,CAAC;QACN,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,EAAE,CAAC;IACT,CAAC;IAED,mGAAmG;IACnG,IAAI;QACF,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,OAAmC;QAC/D,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;YACxE,mFAAmF;YACnF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,+BAA+B;oBAAE,MAAM;gBAC/D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;oBAAE,SAAS;gBACzC,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBACxD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7F,CAAC,CAAC,CAAC;YACL,CAAC;YACD,kGAAkG;YAClG,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC;gBAC/F,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,KAAK,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;oBAC1D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC3F,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;gBAAS,CAAC;YAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAAC,CAAC;IACtC,CAAC;IAED,2FAA2F;IACnF,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,UAAkB;QAC9D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;QACnF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;gBAAE,MAAM;YAC3E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,GAAiB;QACrC,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QAChF,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAEO,gBAAgB,CAAC,MAA+B,EAAE,QAAoC;QAC5F,IAAI,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACnF,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,wFAAwF;IAChF,UAAU,CAAC,KAA8B;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACjD,OAAO,gBAAgB,CAAC,KAAK,CAAC,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YACtF,WAAW,EAAE,UAAU,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,GAAiB,EAAE,OAAmC,EAAE,UAAmB;QAC/F,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QACnE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC;YACjC,IAAI,CAAC,KAAK,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,uBAAuB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;gBAAE,OAAO;YAC/G,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE;gBAAE,OAAO;YACjE,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACnC,IAAI,QAAgD,CAAC;YACrD,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,gCAAgC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,gCAAgC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;oBACtJ,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,kDAAkD,CAAC,CAAC;oBAAC,OAAO;gBACrF,CAAC;gBACD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,+CAA+C,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC5F,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,uDAAuD,CAAC,CAAC;gBAAC,OAAO;YAC1F,CAAC;YACD,IAAI,KAAK,CAAC,KAAK,KAAK,uBAAuB,CAAC,WAAW,EAAE,CAAC;gBACxD,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,0DAA0D,CAAC,CAAC;gBAAC,OAAO;YAC7F,CAAC;YACD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU;gBAAE,OAAO;YACvE,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC;YAC1C,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBAC7E,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,8CAA8C,CAAC,CAAC;gBAAC,OAAO;YACjF,CAAC;YACD,MAAM,SAAS,GAAG,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC;YACvH,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC;YAChG,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;YAC3B,MAAM,YAAY,GAA4B;gBAC5C,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,CAAC;gBAC1F,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;gBACpF,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;aAC3E,CAAC;YACF,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,YAAY,EAAE;gBAC9F,MAAM,EAAE,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,KAAK;aACvG,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC7G,MAAM,OAAO,GAA4B,EAAE,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,CAAC,YAAY,EAAE,CAAC;YACjI,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,IAAI,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;gBACjC,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;oBACtC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,cAAc;wBAAE,OAAO;oBAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,CAAC;oBACxC,IAAI,CAAC;wBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE;4BAClF,GAAG,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;yBAC5G,CAAC,CAAC;wBACH,IAAI,IAAI;4BAAE,OAAO,GAAG,IAAI,CAAC;;4BAAM,IAAI,GAAG,IAAI,CAAC;oBAC7C,CAAC;oBAAC,MAAM,CAAC;wBAAC,IAAI,GAAG,IAAI,CAAC;oBAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;YACzC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,MAA6B,CAAC;YAClC,IAAI,cAAyD,CAAC;YAC9D,IAAI,OAAgB,CAAC;YACrB,IAAI,sBAA0C,CAAC;YAC/C,IAAI,UAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,IAAI,SAAS;oBAAE,cAAc,GAAG,MAAM,QAAQ,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC;;oBAC9D,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,UAAU,GAAG,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,SAAS,IAAI,cAAc,EAAE,OAAO,KAAK,8BAA8B,CAAC,SAAS,CAAC;gBAClI,UAAU,GAAG,cAAc,EAAE,UAAU,IAAI,MAAM,EAAE,UAAU,CAAC;gBAC9D,IAAI,UAAU,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;oBAC3C,IAAI,CAAC;wBACH,IAAI,CAAC,IAAI,CAAC,cAAc;4BAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;wBACrF,sBAAsB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBACvE,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC9H,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,OAAO,GAAG,KAAK,CAAC;YAAC,CAAC;oBAC5B,CAAC;gBAAC,aAAa,CAAC,SAAS,CAAC,CAAC;gBAAC,MAAM,UAAU,CAAC;YAAC,CAAC;YACvD,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qEAAqE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAAC,OAAO;YAC1I,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,cAAc,EAAE,OAAO,KAAK,8BAA8B,CAAC,SAAS,EAAE,CAAC;oBACzE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,uBAAuB,CAAC,cAAc,CAAC,CAAC;oBAAC,OAAO;gBACnH,CAAC;gBACD,IAAI,cAAc,EAAE,OAAO,KAAK,8BAA8B,CAAC,aAAa,EAAE,CAAC;oBAC7E,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;oBAAC,OAAO;gBACzC,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;gBAAC,OAAO;YAC3F,CAAC;YACD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,sBAAsB,EAAE,UAAU,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACzH,OAAO,GAAG,MAAM,EAAE,KAAK,IAAI,OAAO,CAAC;YACnC,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC;YAC7C,MAAM,SAAS,GAAG,CAAC,mBAAmB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,qBAAqB,KAAK,SAAS;mBACzF,OAAO,CAAC,qBAAqB,KAAK,sBAAsB,CAAC,KAAK,CAAC;YACpE,IAAI,MAAM,CAAC,qBAAqB,IAAI,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBACtG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,4FAA4F;gBAC5F,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,8DAA8D,EAAE,OAAO,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,SAAS;gBAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5D,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,wFAAwF;IAChF,QAAQ,CAAC,GAAiB;QAChC,MAAM,WAAW,GAAG,GAAG,CAAC,cAAc,EAAE,WAAW,CAAC;QACpD,IAAI,CAAC,WAAW;YAAE,OAAO,SAAS,CAAC;QACnC,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAA6E,CAAC,CAAC;QAC7H,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;IAC/B,CAAC;IAED,gGAAgG;IACxF,SAAS,CAAC,OAAiF;QACjG,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,EAAE;YAAE,OAAO,SAAS,OAAO,CAAC,QAAQ,EAAE,CAAC;QACxG,IAAI,OAAO,OAAO,CAAC,gBAAgB,KAAK,QAAQ,IAAI,OAAO,CAAC,gBAAgB,KAAK,EAAE;YAAE,OAAO,aAAa,OAAO,CAAC,gBAAgB,EAAE,CAAC;QACpI,IAAI,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,KAAK,EAAE;YAAE,OAAO,UAAU,OAAO,CAAC,UAAU,EAAE,CAAC;QAC/G,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,cAAc,CAAC,KAAc;QACnC,IAAI,mBAAmB,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7C,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE;YACxG,sBAAsB,EAAE,2BAA2B,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG;YAC7E,OAAO,EAAE,EAAE,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,KAA8B,EAAE,OAAe;QAC3D,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACnI,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAiB,EAAE,MAAc,EAAE,OAAiB;QACvE,MAAM,KAAK,GAAG,GAAG,CAAC,cAAe,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YAClF,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,MAAM,EAAE,MAAM;YACrF,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;SACpC,EAAE,EAAE,MAAM,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACrH,IAAI,OAAO;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACtG,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,GAAiB,EAAE,cAAuB,EAAE,OAAiB;QAC9E,MAAM,KAAK,GAAG,GAAG,CAAC,cAAe,CAAC;QAClC,MAAM,IAAI,GAAG,aAAa,CAAC,cAAc,CAAC,EAAE,GAAG,oBAAoB,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;QACrI,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChF,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/G,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,uDAAuD,EAAE,OAAO,CAAC,CAAC;YAAC,OAAO;QACnG,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YAClE,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,OAAO,EAAE,cAAc;YAC9F,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC;SAClI,EAAE,EAAE,MAAM,EAAE,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACzH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAAiB,EAAE,sBAA+B,EAAE,UAAmB,EAC5F,MAAM,GAAG,uBAAuB,CAAC,SAAS;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,cAAe,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,0BAA0B,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACxG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YACnF,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,EAAE,uBAAuB,CAAC,QAAQ;YAC/E,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;YAC1D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC;SACzC,EAAE;YACD,MAAM,EAAE,kBAAkB,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,sBAAsB;YAC9E,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,yBAAyB,GAAG,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;SACrI,CAAC,CAAC;QACH,IAAI,QAAQ;YAAE,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED,6GAA6G;IACrG,KAAK,CAAC,eAAe,CAAC,GAAiB;QAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC;QACjC,IAAI,KAAK,EAAE,KAAK,KAAK,uBAAuB,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,gBAAgB;YAAE,OAAO;QAC3G,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAC/D,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;YACxI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0EAA0E,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YACnH,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,sBAAsB;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/G,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,4EAA4E;YACjG,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YAC/C,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QACzI,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gEAAgE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAChI,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC;gBAC1G,oBAAoB,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAAC,KAGnB,EAAE,OAAmC;QACpC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACjH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,GAAG,EAAE,cAAc,CAAC;QAClC,0FAA0F;QAC1F,4FAA4F;QAC5F,+FAA+F;QAC/F,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK;eACvH,KAAK,CAAC,KAAK,KAAK,uBAAuB,CAAC,MAAM,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,gBAAgB;YAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvK,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACrG,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC1B,IAAI,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC;QAC1C,IAAI,QAAQ,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,KAAK,sBAAsB,CAAC,KAAK,CAAC;QAC7D,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,KAAK,sBAAsB,CAAC,iBAAiB,CAAC;QAC7E,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,sBAAsB,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnI,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;YACvD,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACxE,MAAM,aAAa,GAAG,gCAAgC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACxE,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC/C,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;gBAC7B,+FAA+F;gBAC/F,MAAM,GAAG,aAAa,CAAC;gBACvB,IAAI,KAAK,CAAC,MAAM,KAAK,8CAA8C,EAAE,CAAC;oBACpE,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,6BAA6B,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtH,CAAC;gBACD,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;YAC9G,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,gCAAgC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBAC5G,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;YACxF,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,IAAI,MAAM,CAAC,YAAY,KAAK,qBAAqB,CAAC,MAAM,CAAC;mBAC/E,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QACD,IAAI,sBAA0C,CAAC;QAC/C,IAAI,SAAS,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACxF,sBAAsB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAC,GAAG,EAAE,GAAG,EACtE,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,qBAAqB,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,EAC/F,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAC1F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;YAC/E,GAAG,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,CAAC;YACxE,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;YAC1D,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC,QAAQ;YACjF,6EAA6E;YAC7E,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG;YACzC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnH,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACnD,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,sBAAsB;YACtH,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,yBAAyB,GAAG,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC;SAC7G,CAAC,CAAC;QACH,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAC1E,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACxI,CAAC;CACF","sourcesContent":["import { randomUUID } from 'node:crypto';\nimport {\n ErrorType, ErrorSeverity, ErrorCustomMessageReference, ErrorHandling_Strategy, isWildoBackendError, JOB_EXECUTION_TTL_SECONDS, JobExecutionStatus,\n JobPayloadSchema, JobRetryUtils, JobStatus, normalizeScheduledRecoveryPolicy, QUEUE_RETRY_DEFAULTS,\n ScheduledExecutionPhase, ScheduledReconciliationOutcome, ScheduledReplaySafety, ScheduledReviewOutcome,\n type JobExecution, type JobPayload, type JobResult, type ScheduledAttemptContext,\n type ScheduledExecutionState, type ScheduledReconciliationResult, type ScheduledRecoveryPolicy,\n type ScheduledEvidencePolicy,\n ScheduledEvidencePolicySchema, ScheduledEvidenceSource, scheduledEvidenceInvocationStatus,\n ScheduledDeliveryLateness,\n} from '@wildo-ai/saas-models';\nimport type { JobStatusTrackingBackendService } from './job-status-tracking.backend.service';\nimport type { LoggerService } from '../monitoring/logger/services/logger.service';\nimport type { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';\nimport { createScheduledRunEvidence } from './scheduled-evidence.backend';\n\nimport { describeCaughtError } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\n/*\n How many scheduled occurrences one runner executes at once — first deliveries and recoveries alike,\n * since both go through `process`. Other replicas arbitrate the same rows through compare-and-swap.\n /\nconst SCHEDULED_EXECUTION_CONCURRENCY = 8;\n\n/\n How long a target's late deliveries are gathered before ONE summary is logged.\n \n A backlog drains in about a second — measured at 139 deliveries in 0.9 s after a restart (#2212) —\n * so a short quiet period closes the drain without holding the summary noticeably.\n /\nconst LATE_DELIVERY_DIGEST_QUIET_MS = 2000;\n\n/\n The longest a late-delivery summary is held, however steadily deliveries keep arriving: a consumer\n * that stays permanently behind produces gaps shorter than the quiet period forever, and must still\n * be reported rather than silenced by its own persistence.\n /\nconst LATE_DELIVERY_DIGEST_MAX_HOLD_MS = 30_000;\n\n/* Reported on an occurrence recorded from a late delivery; the structured cause is `ScheduledExecutionState.lateness`. /\nconst LATE_DELIVERY_REASON = 'Delivery reached this application after its admission deadline; the occurrence was not run';\n\n/* One drain's late deliveries for one target — what the single summary line reports. /\ninterface LateDeliveryDigest {\n queueName: string;\n target: string;\n recorded: number;\n /* Already owned by an earlier receipt of the same occurrence (a redelivery): nothing new to record. /\n alreadyHeld: number;\n /* Could not be made durable; the delivery was dead-lettered so the copy survives. /\n unrecorded: number;\n firstError?: string;\n lateness: Set<ScheduledDeliveryLateness>;\n /* When this digest received its first delivery — bounds how long it may be held. /\n openedAt: number;\n earliest?: Date;\n latest?: Date;\n timer?: ReturnType<typeof setTimeout>;\n}\n\n/* Resolved from local handler declarations, never from an untrusted message's policy fields. /\nexport interface ScheduledExecutionContract {\n policy?: Partial<ScheduledRecoveryPolicy>;\n evidencePolicy?: ScheduledEvidencePolicy;\n execute(context: ScheduledAttemptContext): Promise<JobResult>;\n reconcile?(context: ScheduledAttemptContext): Promise<ScheduledReconciliationResult>;\n}\nexport type ScheduledExecutionResolver = (job: JobPayload) => Promise<ScheduledExecutionContract \| undefined>;\n\n/\n Durable scheduled execution owner shared by application consumers and minions.\n \n Broker receipt authenticates and inserts an occurrence. A database scanner subsequently owns\n * recovery, so a five-minute JWT never becomes a long-lived replay credential. Every admission and\n * mutation uses the persisted revision. Leases detect lost contact; they do not cancel external work.\n * Applications opting into recovery must provide effect-level idempotency or reconciliation.\n /\nexport class ScheduledExecutionRunner {\n private timer?: ReturnType<typeof setInterval>;\n private scanning = false;\n private archiving = false;\n private readonly archivesActive = new Set<string>();\n private generation = 0;\n private readonly active = new Set<string>();\n /* Runs started by {@link launch} and not yet settled — what {@link whenIdle} waits for. /\n private readonly launched = new Set<Promise<void>>();\n /\n The targets (batch, operation or minion) with an occurrence running in THIS runner.\n \n A consumer used to hold one delivery at a time, so two occurrences of one target never ran\n * together here — by accident of the queue rather than by design, and at the price of dropping\n * whatever waited too long. Now that receipt no longer waits on execution, the non-overlap is kept\n * deliberately: a second occurrence of a running target stays due and the scanner starts it once the\n * first settles. A sweep that takes longer than its schedule therefore queues instead of racing\n * itself; if it waits past its recovery deadline it goes to review, where it is visible.\n /\n private readonly activeTargets = new Set<string>();\n /* Late deliveries gathered per (queue, target) until their drain goes quiet — see {@link recordLateDelivery}. /\n private readonly lateDigests = new Map<string, LateDeliveryDigest>();\n\n constructor(\n private readonly tracker: JobStatusTrackingBackendService,\n private readonly logger: Pick<LoggerService, 'info' \| 'warn' \| 'error'>,\n private readonly payloadStorage?: Pick<JobPayloadStorageBackendService, 'store'> & Partial<Pick<JobPayloadStorageBackendService, 'storeScheduledEvidence'>>,\n private readonly activity?: { started(): void; completed(): void },\n ) {}\n\n /\n Make one broker delivery DURABLE, and hand back the occurrence to run — without running it.\n \n This is the whole of what a broker receipt has to do. The signed token proves the delivery is\n * genuine for five minutes, so its only job is to get the occurrence into the database inside that\n * window; from there the database owns it (see the class comment).\n \n ## Why receipt and execution are separate calls\n \n They were one: {@link receive} inserted the occurrence and then awaited the whole run, so the\n * consumer could only acknowledge the broker message once the job FINISHED. A queue consumer holds\n * one unacknowledged message at a time, so every later delivery sat unreceived behind a long run,\n * and when that run outlasted five minutes each of them failed verification (\"jwt expired\") and was\n * dead-lettered — the occurrence was lost without ever being recorded. Measured on Wonder Todos,\n * 2026-09-30: an `immudb.audit-reconcile` run that never returned held every scheduled job in the\n * application (erasure sweep, retention sweep, organization purge) until the next backend restart.\n * The consumers now {@link admit}, acknowledge, then {@link launch}.\n \n Returns `undefined` when there is nothing to run: no local handler (the receipt is retained for\n * review) or a legacy row whose effects are unknown.\n /\n async admit(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver): Promise<JobExecution \| undefined> {\n const contract = await resolve(job);\n const policy = normalizeScheduledRecoveryPolicy(contract?.policy);\n if (contract) this.assertReconciler(policy, contract);\n const now = new Date();\n const row = await this.tracker.ensureScheduledJobRecord(job, queueName, {\n phase: contract ? ScheduledExecutionPhase.READY : ScheduledExecutionPhase.REVIEW,\n evidencePolicy: contract?.evidencePolicy ? ScheduledEvidencePolicySchema.parse(contract.evidencePolicy) : undefined,\n reason: contract ? undefined : 'Handler unavailable at authenticated receipt', revision: 0, payloadJson: JSON.stringify(job), policy,\n deadline: new Date(now.getTime() + policy.maxRecoveryMs), nextAttemptAt: now,\n attemptCount: 0, attempts: [], resolutions: [], reconciledSafe: false,\n });\n if (!contract) { this.logger.warn('Scheduled target unavailable; receipt retained for review', { jobId: job.jobId }); return undefined; }\n if (!row.scheduledState) {\n // Legacy active receipts have unknown effects. Never turn an old PROCESSING row into a new run.\n this.logger.warn('Legacy scheduled occurrence requires review', { jobId: job.jobId, status: row.status });\n return undefined;\n }\n if (this.active.has(job.jobId) \|\| row.scheduledState.revision > 0 \|\| row.scheduledState.phase !== ScheduledExecutionPhase.READY) {\n this.logger.info('Repeated scheduled receipt reused durable occurrence', { jobId: job.jobId,\n phase: row.scheduledState.phase, attemptCount: row.scheduledState.attemptCount });\n }\n return row;\n }\n\n /\n Record an AUTHENTIC delivery that arrived after its admission deadline as a MISSED occurrence\n * (#2212), and report the drain it belongs to once.\n \n The delivery is never admitted and never run — a missed occurrence is skipped, and the schedule\n * continues (`ScheduledDeliveryLateness`). What changes is what it leaves behind. It used to be\n * failed as an `AUTHENTICATION` error, logged twice, dead-lettered and otherwise forgotten, so a\n * restart left a gap in scheduled-evidence coverage that nothing could tell apart from an\n * unexplained one. It now becomes a terminal row (`FINISHED`, `FAILED`, `lateness` set) and, when\n * the target retains evidence, an archived `LATE_DELIVERY` record — an explained gap.\n \n Never overwrites an occurrence an earlier receipt owns: a delivery admitted on time whose\n * acknowledgement was lost is redelivered late, and that occurrence may well have run. The row is\n * first created in `REVIEW` (never scanned for execution) and only a row this path created moves\n * to `FINISHED`; anything else is left alone. A crash between the two writes leaves a `REVIEW` row\n * with no attempts, which an operator can abandon or retry like any other.\n \n Never throws. Returns `true` when the occurrence is durable (recorded now, or already held), so\n * the caller can acknowledge the delivery; `false` when it could not be recorded, so the caller\n * dead-letters it and the copy survives.\n /\n async recordLateDelivery(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver,\n lateness: ScheduledDeliveryLateness): Promise<boolean> {\n const digest = this.lateDigestFor(queueName, job);\n digest.lateness.add(lateness);\n const instant = job.scheduledAt ?? job.createdAt;\n if (!digest.earliest \|\| instant < digest.earliest) digest.earliest = instant;\n if (!digest.latest \|\| instant > digest.latest) digest.latest = instant;\n try {\n // Resolution only selects the evidence policy; a missing handler still gets its miss recorded.\n let contract: ScheduledExecutionContract \| undefined;\n try { contract = await resolve(job); } catch { contract = undefined; }\n const now = new Date();\n const evidencePolicy = contract?.evidencePolicy ? ScheduledEvidencePolicySchema.parse(contract.evidencePolicy) : undefined;\n const row = await this.tracker.ensureScheduledJobRecord(job, queueName, {\n phase: ScheduledExecutionPhase.REVIEW, revision: 0, payloadJson: JSON.stringify(job),\n policy: normalizeScheduledRecoveryPolicy(contract?.policy), evidencePolicy,\n reason: LATE_DELIVERY_REASON, lateness, deadline: now, nextAttemptAt: now,\n attemptCount: 0, attempts: [], resolutions: [], reconciledSafe: false,\n });\n const state = row.scheduledState;\n if (!state \|\| state.lateness === undefined \|\| state.revision !== 0 \|\| state.attemptCount !== 0\n \|\| state.phase !== ScheduledExecutionPhase.REVIEW) {\n digest.alreadyHeld += 1;\n return true;\n }\n const source = ScheduledEvidenceSource.LATE_DELIVERY;\n const evidence = createScheduledRunEvidence(row, now, source, scheduledEvidenceInvocationStatus(source));\n const terminal = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.FINISHED,\n evidence, evidenceArchiveAfter: evidence ? now : undefined,\n }, {\n status: scheduledEvidenceInvocationStatus(source), completedAt: now,\n expiresAt: new Date(now.getTime() + JOB_EXECUTION_TTL_SECONDS 1000),\n });\n // A lost compare-and-swap means a concurrent late receipt of the same occurrence recorded it.\n if (terminal) { digest.recorded += 1; this.launchArchive(terminal); } else digest.alreadyHeld += 1;\n return true;\n } catch (error) {\n digest.unrecorded += 1;\n digest.firstError ??= describeCaughtError(error);\n return false;\n } finally { this.scheduleLateDigest(digest); }\n }\n\n /*\n What the reader should conclude, by WHICH gate fired: only an expired token proves the queue\n * went unconsumed for the whole window; an occurrence past the window on a current token shares\n * its delay with the scheduler (`ScheduledDeliveryLateness`).\n /\n private lateDigestRemedy(digest: LateDeliveryDigest): string {\n if (digest.unrecorded > 0) {\n return 'Some missed occurrences could not be recorded and were dead-lettered; see firstError. Nothing was run.';\n }\n const causes: string[] = [];\n if (digest.lateness.has(ScheduledDeliveryLateness.TOKEN_EXPIRED)) {\n causes.push('nothing consumed the scheduled queue for longer than the 5-minute admission window (the backend or minion was stopped or restarting)');\n }\n if (digest.lateness.has(ScheduledDeliveryLateness.OUTSIDE_ADMISSION_WINDOW)) {\n causes.push('the occurrence was older than the admission window when consumed, a delay shared between the scheduler publishing it and this application consuming it');\n }\n return `Nothing to do: these occurrences were missed because ${causes.join('; or ')}. `\n + 'They are recorded as late deliveries, were not run, and the schedule continues. '\n + 'Recurring bursts mean the application is often unavailable for longer than that window.';\n }\n\n /* Flush every pending late-delivery summary now (shutdown, and tests). /\n flushLateDeliveryDigests(): void {\n for (const digest of [...this.lateDigests.values()]) this.reportLateDigest(digest);\n }\n\n private lateDigestFor(queueName: string, job: JobPayload): LateDeliveryDigest {\n const target = this.targetKey(job) ?? `job:${job.jobType}`;\n const key = `${queueName}\\u0000${target}`;\n let digest = this.lateDigests.get(key);\n if (!digest) {\n digest = { queueName, target, recorded: 0, alreadyHeld: 0, unrecorded: 0, lateness: new Set(), openedAt: Date.now() };\n this.lateDigests.set(key, digest);\n }\n return digest;\n }\n\n private scheduleLateDigest(digest: LateDeliveryDigest): void {\n if (digest.timer) clearTimeout(digest.timer);\n const held = Date.now() - digest.openedAt;\n if (held >= LATE_DELIVERY_DIGEST_MAX_HOLD_MS) { this.reportLateDigest(digest); return; }\n digest.timer = setTimeout(() => this.reportLateDigest(digest),\n Math.min(LATE_DELIVERY_DIGEST_QUIET_MS, LATE_DELIVERY_DIGEST_MAX_HOLD_MS - held));\n digest.timer.unref();\n }\n\n /\n One line per drain per target. WARN, not ERROR: a missed occurrence is the expected consequence\n * of the application not running, and the line says so — but it is still a gap in what the\n * schedule promised, so it must not be silent either. Escalates to ERROR only when some deliveries\n * could not be recorded, because then the gap is no longer explained durably.\n /\n private reportLateDigest(digest: LateDeliveryDigest): void {\n if (digest.timer) clearTimeout(digest.timer);\n this.lateDigests.delete(`${digest.queueName}\\u0000${digest.target}`);\n const total = digest.recorded + digest.alreadyHeld + digest.unrecorded;\n if (total === 0) return;\n const fields = {\n queueName: digest.queueName, target: digest.target, missed: total,\n recorded: digest.recorded, alreadyHeld: digest.alreadyHeld, unrecorded: digest.unrecorded,\n lateness: [...digest.lateness].sort(), earliest: digest.earliest, latest: digest.latest,\n remedy: this.lateDigestRemedy(digest),\n ...(digest.firstError ? { firstError: digest.firstError } : {}),\n };\n if (digest.unrecorded > 0) this.logger.error('Scheduled occurrences delivered after their admission window', fields);\n else this.logger.warn('Scheduled occurrences delivered after their admission window', fields);\n }\n\n /\n Start an admitted occurrence WITHOUT waiting for it, under the same bound as recovery.\n \n When every execution slot is taken the occurrence is simply left due: it is already durable, and\n * the recovery scanner starts it within a second of a slot freeing. Nothing is dropped and nothing\n * is waited on, so the caller can acknowledge its broker message at once.\n /\n launch(row: JobExecution, resolve: ScheduledExecutionResolver): void {\n if (this.active.size >= SCHEDULED_EXECUTION_CONCURRENCY) return;\n const run: Promise<void> = this.process(row, resolve).catch(error => {\n // Intake is already durable. Neither tracking nor handler failures hand retry ownership back\n // to the broker: the scanner recovers only under the captured effect-safety contract.\n this.logger.error('Scheduled attempt could not settle durably', { jobId: row.jobId, error: String(error) });\n }).finally(() => { this.launched.delete(run); });\n this.launched.add(run);\n }\n\n /* Resolves once every run {@link launch} started has settled. Never rejects. /\n async whenIdle(): Promise<void> {\n while (this.launched.size > 0) await Promise.allSettled([...this.launched]);\n }\n\n /\n Admit and RUN one occurrence, resolving when the attempt has settled.\n \n For a caller that owns no broker message (tests, and any in-process driver). A broker consumer must\n * not use it: holding the delivery for the length of the run is what dropped every job queued behind\n * a long one — see {@link admit}.\n /\n async receive(job: JobPayload, queueName: string, resolve: ScheduledExecutionResolver): Promise<void> {\n const row = await this.admit(job, queueName, resolve);\n if (!row) return;\n try { await this.process(row, resolve); }\n catch (error) {\n // Intake is already durable. Neither tracking nor handler failures hand retry ownership back\n // to the broker: the scanner recovers only under the captured effect-safety contract.\n this.logger.error('Scheduled attempt could not settle durably', { jobId: row.jobId, error: String(error) });\n }\n }\n\n /\n Start the recovery scanner for ONE queue.\n \n Callers name the queue and nothing else. The runner used to take an `applicationId` here as\n * well, and every caller — the application consumer and the minion tick consumer — passed the\n * PLATFORM id, because that is what their queue names are built from. The tracker then scanned\n * rows under it, which only worked because the same platform id had been stamped into the rows.\n * The row identity is the tracker's to resolve (`JobStatusTrackingBackendService.canonicalApplicationId`);\n * removing the parameter is what makes it impossible for a caller to scan under the wrong one.\n /\n start(queueName: string, resolve: ScheduledExecutionResolver): void {\n if (this.timer) throw new Error('Scheduled recovery scanner already started');\n this.generation += 1;\n const tick = () => { void this.scan(queueName, resolve).catch(error => {\n this.logger.error('Scheduled recovery scan failed', { queueName, error: String(error) });\n }); };\n this.timer = setInterval(tick, 1000);\n this.timer.unref();\n tick();\n }\n\n /* Stop admitting recovery; already admitted work retains heartbeat ownership until settlement. /\n stop(): void {\n this.generation += 1;\n if (this.timer) clearInterval(this.timer);\n this.timer = undefined;\n this.flushLateDeliveryDigests();\n }\n\n async scan(queueName: string, resolve: ScheduledExecutionResolver): Promise<void> {\n if (this.scanning) return;\n this.scanning = true;\n const generation = this.generation;\n try {\n const rows = await this.tracker.listScheduledDue(queueName, new Date());\n // Bounded process concurrency; other replicas arbitrate the same rows through CAS.\n for (const row of rows) {\n if (generation !== this.generation) break;\n if (this.active.size >= SCHEDULED_EXECUTION_CONCURRENCY) break;\n if (this.active.has(row.jobId)) continue;\n void this.process(row, resolve, generation).catch(error => {\n this.logger.error('Scheduled recovery failed', { jobId: row.jobId, error: String(error) });\n });\n }\n // Archival has its own bounded pass. A storage outage must not fill the recovery admission queue.\n if (generation === this.generation && !this.archiving && this.tracker.listScheduledArchivesDue) {\n this.archiving = true;\n void this.scanArchives(queueName, generation).catch(error => {\n this.logger.error('Scheduled evidence scan failed', { queueName, error: String(error) });\n }).finally(() => { this.archiving = false; });\n }\n } finally { this.scanning = false; }\n }\n\n /* Archive I/O cannot hold the recovery scanner or consume its handler admission slots. /\n private async scanArchives(queueName: string, generation: number): Promise<void> {\n const pending = await this.tracker.listScheduledArchivesDue(queueName, new Date());\n for (const row of pending) {\n if (generation !== this.generation \|\| this.archivesActive.size >= 4) break;\n this.launchArchive(row);\n }\n }\n\n private launchArchive(row: JobExecution): void {\n if (this.archivesActive.size >= 4 \|\| this.archivesActive.has(row.jobId)) return;\n this.archivesActive.add(row.jobId);\n void this.archiveEvidence(row).catch(error => {\n this.logger.error('Scheduled evidence retry failed', { jobId: row.jobId, error: String(error) });\n }).finally(() => { this.archivesActive.delete(row.jobId); });\n }\n\n private assertReconciler(policy: ScheduledRecoveryPolicy, contract: ScheduledExecutionContract): void {\n if (policy.replaySafety === ScheduledReplaySafety.RECONCILE && !contract.reconcile) {\n throw new Error('Reconciliation policy requires a locally registered reconciler');\n }\n }\n\n /* Restore transport dates for both scanner execution and attributed operator retry. /\n private restoreJob(state: ScheduledExecutionState): JobPayload {\n const serialized = JSON.parse(state.payloadJson);\n return JobPayloadSchema.parse({ ...serialized, createdAt: new Date(serialized.createdAt),\n scheduledAt: serialized.scheduledAt === undefined ? undefined : new Date(serialized.scheduledAt) });\n }\n\n private async process(row: JobExecution, resolve: ScheduledExecutionResolver, generation?: number): Promise<void> {\n if (this.active.has(row.jobId)) return;\n const target = this.targetOf(row);\n if (target !== undefined && this.activeTargets.has(target)) return;\n this.active.add(row.jobId);\n if (target !== undefined) this.activeTargets.add(target);\n this.activity?.started();\n try {\n const state = row.scheduledState;\n if (!state \|\| [ScheduledExecutionPhase.REVIEW, ScheduledExecutionPhase.FINISHED].includes(state.phase)) return;\n if (new Date(state.nextAttemptAt).getTime() > Date.now()) return;\n const job = this.restoreJob(state);\n let contract: ScheduledExecutionContract \| undefined;\n try {\n contract = await resolve(job);\n if (!contract \|\| JSON.stringify(normalizeScheduledRecoveryPolicy(contract.policy)) !== JSON.stringify(normalizeScheduledRecoveryPolicy(state.policy))) {\n await this.review(row, 'Handler unavailable or recovery contract changed'); return;\n }\n this.assertReconciler(state.policy, contract);\n } catch { await this.review(row, 'Local recovery contract could not be resolved'); return; }\n if (Date.now() >= new Date(state.deadline).getTime() \|\| state.attemptCount >= state.policy.maxAttempts) {\n await this.review(row, 'Recovery deadline or total admission budget exhausted'); return;\n }\n if (state.phase === ScheduledExecutionPhase.RECONCILING) {\n await this.review(row, 'Reconciliation worker contact lost; outcome is uncertain'); return;\n }\n if (generation !== undefined && generation !== this.generation) return;\n const recovering = state.attemptCount > 0;\n if (recovering && state.policy.replaySafety === ScheduledReplaySafety.MANUAL) {\n await this.review(row, 'Interrupted execution requires manual review'); return;\n }\n const reconcile = recovering && state.policy.replaySafety === ScheduledReplaySafety.RECONCILE && !state.reconciledSafe;\n const phase = reconcile ? ScheduledExecutionPhase.RECONCILING : ScheduledExecutionPhase.RUNNING;\n const now = new Date();\n const owner = randomUUID();\n const claimedState: ScheduledExecutionState = {\n ...state, phase, owner, revision: state.revision + 1, attemptCount: state.attemptCount + 1,\n reconciledSafe: false, nextAttemptAt: new Date(now.getTime() + state.policy.leaseMs),\n attempts: [...state.attempts, { attemptId: owner, phase, startedAt: now }],\n };\n let current = await this.tracker.compareScheduledState(row.jobId, state.revision, claimedState, {\n status: JobExecutionStatus.PROCESSING, startedAt: now, retryCount: state.attemptCount, workerId: owner,\n });\n if (!current) { this.logger.info('Duplicate scheduled admission suppressed', { jobId: row.jobId }); return; }\n const context: ScheduledAttemptContext = { occurrenceId: row.jobId, attemptId: owner, attemptNumber: claimedState.attemptCount };\n let lost = false;\n let heartbeats = Promise.resolve();\n const heartbeat = setInterval(() => {\n heartbeats = heartbeats.then(async () => {\n if (lost \|\| !current?.scheduledState) return;\n const previous = current.scheduledState;\n try {\n const next = await this.tracker.compareScheduledState(row.jobId, previous.revision, {\n ...previous, revision: previous.revision + 1, nextAttemptAt: new Date(Date.now() + previous.policy.leaseMs),\n });\n if (next) current = next; else lost = true;\n } catch { lost = true; }\n });\n }, Math.floor(state.policy.leaseMs / 3));\n heartbeat.unref();\n let result: JobResult \| undefined;\n let reconciliation: ScheduledReconciliationResult \| undefined;\n let failure: unknown;\n let outputPayloadStorageId: string \| undefined;\n let resultJson: string \| undefined;\n try {\n if (reconcile) reconciliation = await contract.reconcile!(context);\n else result = await contract.execute(context);\n const successful = result?.status === JobStatus.COMPLETED \|\| reconciliation?.outcome === ScheduledReconciliationOutcome.COMPLETED;\n resultJson = reconciliation?.resultJson ?? result?.resultJson;\n if (successful && resultJson !== undefined) {\n try {\n if (!this.payloadStorage) throw new Error('Scheduled result storage is unavailable');\n outputPayloadStorageId = await this.payloadStorage.store(resultJson);\n } catch (error) {\n this.logger.error('Scheduled output storage failed after successful execution', { jobId: row.jobId, error: String(error) });\n }\n }\n } catch (error) { failure = error; }\n finally { clearInterval(heartbeat); await heartbeats; }\n if (lost \|\| !current?.scheduledState) {\n this.logger.warn('Scheduled attempt lost ownership; outcome cannot overwrite recovery', { jobId: row.jobId, attemptId: owner }); return;\n }\n if (reconcile) {\n if (reconciliation?.outcome === ScheduledReconciliationOutcome.COMPLETED) {\n await this.complete(current, outputPayloadStorageId, resultJson, ScheduledEvidenceSource.RECONCILIATION); return;\n }\n if (reconciliation?.outcome === ScheduledReconciliationOutcome.SAFE_TO_RETRY) {\n await this.wait(current, true); return;\n }\n await this.review(current, 'Reconciliation failed or was inconclusive', failure); return;\n }\n if (result?.status === JobStatus.COMPLETED) { await this.complete(current, outputPayloadStorageId, resultJson); return; }\n failure = result?.error ?? failure;\n const policy = current.scheduledState.policy;\n const retryable = !isWildoBackendError(failure) \|\| failure.errorHandlingStrategy === undefined\n \|\| failure.errorHandlingStrategy === ErrorHandling_Strategy.RETRY;\n if (policy.retryReportedFailures && retryable && policy.replaySafety !== ScheduledReplaySafety.MANUAL) {\n await this.wait(current, false, failure);\n } else {\n // A reported failure can follow effects. Preserve it for an explicit decision, without TTL.\n await this.review(current, 'Execution reported failure; automatic retry is not permitted', failure);\n }\n } finally {\n this.active.delete(row.jobId);\n if (target !== undefined) this.activeTargets.delete(target);\n this.activity?.completed();\n }\n }\n\n /* The batch, operation or minion an occurrence runs, read from its durable payload. /\n private targetOf(row: JobExecution): string \| undefined {\n const payloadJson = row.scheduledState?.payloadJson;\n if (!payloadJson) return undefined;\n try {\n return this.targetKey(JSON.parse(payloadJson) as { batchRef?: unknown; operationPathKey?: unknown; minionName?: unknown });\n } catch { return undefined; }\n }\n\n /* Prefixed by kind: a batch and a minion that happen to share a name are different targets. /\n private targetKey(payload: { batchRef?: unknown; operationPathKey?: unknown; minionName?: unknown }): string \| undefined {\n if (typeof payload.batchRef === 'string' && payload.batchRef !== '') return `batch:${payload.batchRef}`;\n if (typeof payload.operationPathKey === 'string' && payload.operationPathKey !== '') return `operation:${payload.operationPathKey}`;\n if (typeof payload.minionName === 'string' && payload.minionName !== '') return `minion:${payload.minionName}`;\n return undefined;\n }\n\n private failureDetails(error: unknown): NonNullable<JobExecution['error']> {\n if (isWildoBackendError(error)) return error;\n return { type: ErrorType.INTERNAL, severity: ErrorSeverity.HIGH, timestamp: new Date(), backendActions: [],\n customMessageReference: ErrorCustomMessageReference.INTERNAL, statusCode: 500,\n context: { message: describeCaughtError(error) } };\n }\n\n private ended(state: ScheduledExecutionState, outcome: string): ScheduledExecutionState['attempts'] {\n return state.attempts.map(attempt => attempt.attemptId === state.owner ? { ...attempt, endedAt: new Date(), outcome } : attempt);\n }\n\n private async review(row: JobExecution, reason: string, failure?: unknown): Promise<void> {\n const state = row.scheduledState!;\n const updated = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.REVIEW, reason,\n attempts: this.ended(state, reason),\n }, { status: JobExecutionStatus.FAILED, ...(failure !== undefined ? { error: this.failureDetails(failure) } : {}) });\n if (updated) this.logger.warn('Scheduled occurrence requires review', { jobId: row.jobId, reason });\n }\n\n private async wait(row: JobExecution, reconciledSafe: boolean, failure?: unknown): Promise<void> {\n const state = row.scheduledState!;\n const base = JobRetryUtils.calculateDelay({ ...QUEUE_RETRY_DEFAULTS, ...state.policy.backoff }, Math.max(0, state.attemptCount - 1));\n const delay = Math.round(base (1 - state.policy.jitterRatio * Math.random()));\n if (state.attemptCount >= state.policy.maxAttempts \|\| Date.now() + delay >= new Date(state.deadline).getTime()) {\n await this.review(row, 'Recovery deadline or total admission budget exhausted', failure); return;\n }\n await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.WAITING, reconciledSafe,\n nextAttemptAt: new Date(Date.now() + delay), attempts: this.ended(state, reconciledSafe ? 'Reconciled safe' : 'Reported failure'),\n }, { status: JobExecutionStatus.RETRYING, ...(failure !== undefined ? { error: this.failureDetails(failure) } : {}) });\n }\n\n private async complete(row: JobExecution, outputPayloadStorageId?: string, resultJson?: string,\n source = ScheduledEvidenceSource.EXECUTION): Promise<void> {\n const state = row.scheduledState!;\n const now = new Date();\n const evidence = createScheduledRunEvidence(row, now, source, JobExecutionStatus.COMPLETED, resultJson);\n const terminal = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, revision: state.revision + 1, phase: ScheduledExecutionPhase.FINISHED,\n evidence, evidenceArchiveAfter: evidence ? now : undefined,\n attempts: this.ended(state, 'Completed'),\n }, {\n status: JobExecutionStatus.COMPLETED, completedAt: now, outputPayloadStorageId,\n expiresAt: new Date(Math.max(now.getTime() + JOB_EXECUTION_TTL_SECONDS * 1000, new Date(state.deadline).getTime() + 10 * 60 * 1000)),\n });\n if (terminal) this.launchArchive(terminal);\n }\n\n /** Publication follows terminal CAS. Failure retries only this projection, never the business invocation. /\n private async archiveEvidence(row: JobExecution): Promise<void> {\n const state = row.scheduledState;\n if (state?.phase !== ScheduledExecutionPhase.FINISHED \|\| !state.evidence \|\| state.evidenceArchived) return;\n if (new Date(state.evidence.expiresAt).getTime() <= Date.now()) {\n const { evidence: _expired, ...rest } = state;\n await this.tracker.compareScheduledState(row.jobId, state.revision, { ...rest, revision: state.revision + 1, evidenceArchived: false });\n this.logger.warn('Scheduled evidence expired before archival; coverage remains unavailable', { jobId: row.jobId });\n return;\n }\n try {\n if (!this.payloadStorage?.storeScheduledEvidence) throw new Error('Scheduled evidence archive is unavailable');\n const stored = await this.payloadStorage.storeScheduledEvidence(state.evidence);\n if (!stored) return; // Expired while dispatching: the next pass disposes the pending projection.\n const { evidence: _archived, ...rest } = state;\n await this.tracker.compareScheduledState(row.jobId, state.revision, { ...rest, revision: state.revision + 1, evidenceArchived: true });\n } catch (error) {\n this.logger.error('Scheduled evidence archival failed; retained for bounded retry', { jobId: row.jobId, error: String(error) });\n await this.tracker.compareScheduledState(row.jobId, state.revision, { ...state, revision: state.revision + 1,\n evidenceArchiveAfter: new Date(Date.now() + 60000) });\n }\n }\n\n /\n Trusted backend operator seam: callers must authorize their operator before invoking this method.\n * It cannot create a fresh occurrence or turn a manual contract into a replayable handler. The\n * decision and attribution are persisted in the same CAS as its effect; stale decisions fail.\n /\n async resolveReview(input: {\n queueName: string; jobId: string; expectedRevision: number;\n actor: string; reason: string; outcome: ScheduledReviewOutcome; resultJson?: string;\n }, resolve: ScheduledExecutionResolver): Promise<void> {\n if (!input.actor.trim() \|\| !input.reason.trim()) throw new Error('Operator attribution and reason are required');\n const row = await this.tracker.getByJobId(input.jobId);\n const state = row?.scheduledState;\n // Ownership is checked against the id the tracker STAMPS, not against anything the caller\n // supplies: a caller cannot name the application, so it cannot review another application's\n // row by naming it, and a row still anchored to the retired platform id is refused as foreign.\n if (!row \|\| row.applicationId !== await this.tracker.canonicalApplicationId() \|\| row.queueName !== input.queueName \|\| !state\n \|\| state.phase !== ScheduledExecutionPhase.REVIEW \|\| state.revision !== input.expectedRevision) throw new Error('Scheduled review target changed or is unavailable');\n if (state.resolutions.length >= 100) throw new Error('Scheduled operator decision budget exhausted');\n let policy = state.policy;\n let evidencePolicy = state.evidencePolicy;\n let deadline = new Date(state.deadline);\n const retry = input.outcome === ScheduledReviewOutcome.RETRY;\n const completed = input.outcome === ScheduledReviewOutcome.CONFIRM_COMPLETED;\n if (!retry && !completed && input.outcome !== ScheduledReviewOutcome.ABANDON) throw new Error('Unknown scheduled review decision');\n if (retry) {\n const contract = await resolve(this.restoreJob(state));\n if (!contract) throw new Error('Scheduled target is still unavailable');\n const currentPolicy = normalizeScheduledRecoveryPolicy(contract.policy);\n this.assertReconciler(currentPolicy, contract);\n if (state.attemptCount === 0) {\n // No effects ever began. Bind the now-present handler, without extending the original horizon.\n policy = currentPolicy;\n if (state.reason === 'Handler unavailable at authenticated receipt') {\n evidencePolicy = contract.evidencePolicy ? ScheduledEvidencePolicySchema.parse(contract.evidencePolicy) : undefined;\n }\n deadline = new Date(Math.min(deadline.getTime(), new Date(row.createdAt).getTime() + policy.maxRecoveryMs));\n } else if (JSON.stringify(currentPolicy) !== JSON.stringify(normalizeScheduledRecoveryPolicy(state.policy))) {\n throw new Error('Recovery contract changed; operator cannot override handler safety');\n }\n if ((state.attemptCount > 0 && policy.replaySafety === ScheduledReplaySafety.MANUAL)\n \|\| state.attemptCount >= policy.maxAttempts \|\| Date.now() >= deadline.getTime()) {\n throw new Error('Retry exceeds the declared safety or recovery budget');\n }\n }\n let outputPayloadStorageId: string \| undefined;\n if (completed && input.resultJson !== undefined) {\n if (!this.payloadStorage) throw new Error('Resolution evidence storage is unavailable');\n outputPayloadStorageId = await this.payloadStorage.store(input.resultJson);\n }\n const now = new Date();\n const evidence = retry ? undefined : createScheduledRunEvidence(row, now,\n completed ? ScheduledEvidenceSource.OPERATOR_CONFIRMATION : ScheduledEvidenceSource.ABANDONMENT,\n completed ? JobExecutionStatus.COMPLETED : JobExecutionStatus.FAILED, input.resultJson);\n const next = await this.tracker.compareScheduledState(row.jobId, state.revision, {\n ...state, policy, evidencePolicy, deadline, revision: state.revision + 1,\n evidence, evidenceArchiveAfter: evidence ? now : undefined,\n phase: retry ? ScheduledExecutionPhase.WAITING : ScheduledExecutionPhase.FINISHED,\n // A retry decision cannot impersonate the handler's reconciliation evidence.\n reconciledSafe: false, nextAttemptAt: now,\n resolutions: [...state.resolutions, { actor: input.actor, reason: input.reason, at: now, outcome: input.outcome }],\n }, retry ? { status: JobExecutionStatus.RETRYING } : {\n status: completed ? JobExecutionStatus.COMPLETED : JobExecutionStatus.FAILED, completedAt: now, outputPayloadStorageId,\n expiresAt: new Date(Math.max(now.getTime() + JOB_EXECUTION_TTL_SECONDS 1000, deadline.getTime() + 600000)),\n });\n if (!next) throw new Error('Scheduled review decision lost its revision');\n if (!retry) this.launchArchive(next);\n this.logger.info('Scheduled review resolved', { jobId: row.jobId, actor: input.actor, reason: input.reason, outcome: input.outcome });\n }\n}\n"]}