@usine/engine 0.3.0 → 0.5.0

package/dist/engine/run-driver.d.ts

@@ -0,0 +1,68 @@
+import type { AttemptId } from "@usine/core/attempt";
+import type { RunId } from "@usine/core/run";
+import { Context, Crypto, Effect, Layer } from "effect";
+import { ItemProjectionWriter } from "../item-projection/item-projection.ts";
+import { StepExecutor } from "../step-executor.ts";
+import { RunStore } from "../store/store.ts";
+import { TriggerCondition } from "../trigger-condition.ts";
+import { AttemptAdmission } from "./admission.ts";
+import type { RunStoreError } from "./error.ts";
+import { UnconfirmedStop } from "./unconfirmed-stops.ts";
+/** An Attempt whose parent task the engine stops, with its Run's Loop and Item and the reason. */
+export type StopTarget = Pick<UnconfirmedStop, "attemptId" | "runId" | "loop" | "itemKey" | "reason">;
+declare const RunDriver_base: Context.ServiceClass<RunDriver, "@usine/engine/internal/RunDriver", {
+    /** Starts the fiber of an open Run, unless one already runs here. */
+    readonly launch: (runId: RunId) => Effect.Effect<void>;
+    /** Starts the fiber of a Run that an approval or operator retry reopened, after its previous fiber ends. */
+    readonly resume: (runId: RunId) => Effect.Effect<void>;
+    /** Interrupts the fiber of a closed Run and waits for it to end. */
+    readonly stop: (runId: RunId) => Effect.Effect<void>;
+    /**
+     * Gives the newest queued Event of the closed Run's Item its turn, after re-evaluating the
+     * trigger against the Item's current state. If the Item is gone, marks its projection gone.
+     */
+    readonly continueWithNewest: (closedRunId: RunId) => Effect.Effect<void, RunStoreError>;
+    /**
+     * Opens the queued Events whose continuation didn't run and starts a fiber for every open Run
+     * without one, except Runs waiting for approval. Also retries the parent task stops that
+     * `releaseKept`, `recoverWorkspaces`, or the end of an Attempt couldn't confirm, then runs
+     * `StepExecutor.sweep`.
+     */
+    readonly reconcile: () => Effect.Effect<void, RunStoreError>;
+    /** Returns the closed Attempts whose parent task stop `reconcile` retries. */
+    readonly unconfirmedStops: Effect.Effect<ReadonlyArray<UnconfirmedStop>>;
+    /**
+     * Reconciles at startup the checkout of every persisted Attempt on a configured Worker, before
+     * any Run admits its next Attempt. Keeps the checkout and occupies a Worker slot (#177) for
+     * an open Attempt whose parent task is live or unreadable, and for a closed Attempt whose
+     * parent task isn't confirmed stopped. See `checkoutRecoveryFor`.
+     */
+    readonly recoverWorkspaces: () => Effect.Effect<void, RunStoreError>;
+    /**
+     * Moves the slot that the fiber of Run `runId` holds to its Attempt `attemptId`, whose parent
+     * task may still run, so the slot stays taken after the fiber frees it, until `releaseKept`
+     * confirms that the parent task stopped (#475). Does nothing if the fiber holds no slot.
+     */
+    readonly holdSlot: (runId: RunId, attemptId: AttemptId) => Effect.Effect<void>;
+    /**
+     * Releases the slot kept for the Attempt of `stop`: the slot that `holdSlot` moved, or what
+     * `recoverWorkspaces` kept for reattaching it. Once the parent task is confirmed stopped for
+     * `stop.reason`, reconciles a kept checkout and releases the slot. Until then, both stay kept
+     * (#177, #475), no Run of the Loop on the Item starts an Attempt (#523), and each `reconcile`
+     * retries.
+     */
+    readonly releaseKept: (stop: StopTarget) => Effect.Effect<void>;
+}>;
+/**
+ * The service that drives open Runs in this process, one fiber per Run. A fiber parks a gated
+ * Run for approval, then reserves a Worker, starts or reattaches the Attempt, records its
+ * outcome, and repeats until the Run closes. The fiber that closes a Run gives the Item's newest
+ * queued Event its turn; cancel, supersede, and revoke do that themselves. A fiber that ends
+ * without closing its Run leaves it for `resume` or `reconcile`, as one does instead of starting
+ * an Attempt while a parent task on its Loop's Item isn't confirmed stopped (#523).
+ */
+export declare class RunDriver extends RunDriver_base {
+    static readonly layer: Layer.Layer<RunDriver, never, AttemptAdmission | Crypto.Crypto | ItemProjectionWriter | RunStore | StepExecutor | TriggerCondition>;
+}
+export {};
+//# sourceMappingURL=run-driver.d.ts.map

package/dist/engine/run-driver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-driver.d.ts","sourceRoot":"","sources":["../../src/engine/run-driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAG9D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAG7C,OAAO,EAEL,OAAO,EACP,MAAM,EAEN,MAAM,EAGN,KAAK,EAKN,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,OAAO,KAAK,EAKV,aAAa,EACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AA6DzD,kGAAkG;AAClG,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,eAAe,EACf,WAAW,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CACtD,CAAC;;IAmBE,qEAAqE;qBACpD,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACtD,4GAA4G;qBAC3F,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACtD,oEAAoE;mBACrD,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACpD;;;OAGG;iCAC0B,CAAC,WAAW,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC;IACvF;;;;;OAKG;wBACiB,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC;IAC5D,8EAA8E;+BACnD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;IACxE;;;;;OAKG;gCACyB,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC;IACpE;;;;OAIG;uBACgB,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAC9E;;;;;;OAMG;0BACmB,CAAC,IAAI,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;;AAnDnE;;;;;;;GAOG;AACH,qBAAa,SAAU,SAAQ,cA6CQ;IACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,sIAilBnB;CACH"}

package/dist/engine/run-driver.js

@@ -0,0 +1,448 @@
+import * as AttemptChannel from "@usine/core/attempt/channel";
+import { DefectFailure, InfraFailure } from "@usine/core/attempt/failure";
+import { Cause, Context, Crypto, DateTime, Effect, Fiber, FiberMap, Layer, Option, Ref, Result, Schedule, } from "effect";
+import { ItemProjectionWriter } from "../item-projection/item-projection.js";
+import { StepExecutor } from "../step-executor.js";
+import { RunStore } from "../store/store.js";
+import { TranscriptAllQuery } from "../transcript.js";
+import { TriggerCondition } from "../trigger-condition.js";
+import { AttemptAdmission } from "./admission.js";
+import { checkoutRecoveryFor, without } from "./checkout-recovery.js";
+import { runIdFor } from "./submission.js";
+import { UnconfirmedStop } from "./unconfirmed-stops.js";
+// Executor sinks fail with `AttemptFailure`: a persistence error becomes an `InfraFailure`, and
+// other store errors are defects.
+const asAttemptFailure = (effect) => Effect.catchTags(effect, {
+    EnginePersistenceError: (error) => new InfraFailure({
+        message: `Runtime store error: could not ${error.operation}`,
+        cause: error,
+    }),
+    RunNotFoundError: (error) => Effect.die(error),
+    RunOperationRejectedError: (error) => Effect.die(error),
+    RunRetryBlockedError: (error) => Effect.die(error),
+});
+/** The number of retries for a store operation that failed to persist. */
+const STORE_RETRIES = 4;
+/** The delay before the first retry; each later delay doubles. */
+const STORE_RETRY_DELAY = "100 millis";
+/**
+ * Retries a store operation that fails with `EnginePersistenceError`, with exponential backoff.
+ * Returns other errors and the last persistence error.
+ */
+const retryingStore = (runId, operation) => (effect) => effect.pipe(Effect.tapError((error) => error._tag === "EnginePersistenceError"
+    ? Effect.logWarning("A Run store operation failed to persist", {
+        runId,
+        operation,
+        cause: error.cause,
+    })
+    : Effect.void), Effect.retry({
+    schedule: Schedule.exponential(STORE_RETRY_DELAY),
+    times: STORE_RETRIES,
+    while: (error) => error._tag === "EnginePersistenceError",
+}));
+/** Returns the open Attempt if its parent task is live. */
+const reattaching = (open) => Option.flatMap(open, (found) => found._tag === "Live" ? Option.some(found.attempt) : Option.none());
+/**
+ * The service that drives open Runs in this process, one fiber per Run. A fiber parks a gated
+ * Run for approval, then reserves a Worker, starts or reattaches the Attempt, records its
+ * outcome, and repeats until the Run closes. The fiber that closes a Run gives the Item's newest
+ * queued Event its turn; cancel, supersede, and revoke do that themselves. A fiber that ends
+ * without closing its Run leaves it for `resume` or `reconcile`, as one does instead of starting
+ * an Attempt while a parent task on its Loop's Item isn't confirmed stopped (#523).
+ */
+export class RunDriver extends Context.Service()("@usine/engine/internal/RunDriver") {
+    static layer = Layer.effect(RunDriver, Effect.gen(function* () {
+        const store = yield* RunStore;
+        const executor = yield* StepExecutor;
+        const admission = yield* AttemptAdmission;
+        const condition = yield* TriggerCondition;
+        const items = yield* ItemProjectionWriter;
+        const crypto = yield* Crypto.Crypto;
+        const fibers = yield* FiberMap.make();
+        const checkouts = yield* checkoutRecoveryFor(store, executor);
+        // Closed Attempts whose parent task isn't confirmed stopped; each reconcile retries the stop.
+        const unconfirmed = yield* Ref.make(new Map());
+        // The slot that each Run's fiber holds, unless `holdSlot` moved it to the Run's Attempt.
+        const slots = yield* Ref.make(new Map());
+        /** Releases `lease`, the slot of the Run's fiber, unless `holdSlot` moved it. */
+        const releaseSlot = (runId, lease) => Ref.modify(slots, (held) => held.get(runId) === lease ? [true, without(held, runId)] : [false, held]).pipe(Effect.flatMap((owned) => owned
+            ? lease.release
+            : Effect.logDebug("The Run's slot stays with its Attempt until the parent task is confirmed stopped", { runId, worker: lease.worker })));
+        // Annotated because a fiber's continuation calls launch again, which TypeScript cannot infer.
+        const launch = Effect.fn("RunDriver.launch")(function* (runId) {
+            yield* Effect.annotateCurrentSpan({ runId });
+            if (yield* FiberMap.has(fibers, runId)) {
+                yield* Effect.logDebug("The Run's fiber already runs in this process", { runId });
+                return;
+            }
+            yield* Effect.logDebug("Starting the Run's fiber", { runId });
+            const launchedBy = yield* Effect.option(Effect.currentSpan);
+            yield* FiberMap.run(fibers, runId, drive(runId).pipe(
+            // Logged inside the Run's span: the launch span has ended by the time drive fails.
+            Effect.tapCause((cause) => Cause.hasInterruptsOnly(cause)
+                ? Effect.void
+                : Effect.logError("The Run's fiber stopped on an engine failure; the next reconcile or a restart resumes it", { runId, cause })), Effect.withSpan("RunDriver.drive", {
+                root: true,
+                links: Option.toArray(Option.map(launchedBy, (span) => ({ span, attributes: {} }))),
+            }), Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.void)), { onlyIfMissing: true });
+        });
+        const resume = Effect.fn("RunDriver.resume")(function* (runId) {
+            yield* Effect.annotateCurrentSpan({ runId });
+            const previous = yield* FiberMap.get(fibers, runId);
+            if (Option.isSome(previous)) {
+                // The previous fiber ends right after its last write; wait until it leaves the map.
+                yield* Effect.logDebug("Waiting for the Run's previous fiber to end", { runId });
+                yield* Fiber.await(previous.value);
+            }
+            yield* launch(runId);
+        });
+        const stop = Effect.fn("RunDriver.stop")(function* (runId) {
+            yield* Effect.annotateCurrentSpan({ runId });
+            if (!(yield* FiberMap.has(fibers, runId))) {
+                yield* Effect.logDebug("The closed Run has no fiber in this process", { runId });
+                return;
+            }
+            yield* FiberMap.remove(fibers, runId);
+            yield* Effect.logInfo("The closed Run's fiber stopped", { runId });
+        });
+        const markItemGone = Effect.fn("RunDriver.markItemGone")(function* (closedRunId, submission) {
+            yield* Effect.annotateCurrentSpan({ runId: closedRunId, itemKey: submission.itemKey });
+            return yield* items.markGone(submission.itemKey, yield* DateTime.now).pipe(retryingStore(closedRunId, "mark the item's projection gone"), Effect.catch((error) => Effect.logWarning("Could not mark the gone Item's projection; its Board card stays in place", {
+                completedRunId: closedRunId,
+                itemKey: submission.itemKey,
+                errorTag: error._tag,
+            }).pipe(Effect.as(false))));
+        });
+        const continueWithNewest = Effect.fn("RunDriver.continueWithNewest")(function* (closedRunId) {
+            yield* Effect.annotateCurrentSpan({ runId: closedRunId });
+            const queued = yield* store
+                .nextQueued(closedRunId)
+                .pipe(retryingStore(closedRunId, "read the queued events"));
+            if (Option.isNone(queued))
+                return;
+            const submission = queued.value.submission;
+            const fields = {
+                completedRunId: closedRunId,
+                eventId: submission.event.id,
+                loop: submission.loop,
+                itemKey: submission.itemKey,
+            };
+            const recheck = yield* condition.recheck(submission);
+            if (recheck !== "matches") {
+                // Revoke only the queue read before the remote check.
+                const revoked = yield* store
+                    .revokeQueued(queued.value)
+                    .pipe(retryingStore(closedRunId, "revoke the queued events"));
+                if (recheck === "no-longer-matches") {
+                    yield* Effect.logInfo("Queued Event no longer matches its trigger; revoked", {
+                        ...fields,
+                        revokedEvents: revoked,
+                    });
+                    return;
+                }
+                const projectionMarkedGone = yield* markItemGone(closedRunId, submission);
+                yield* Effect.logInfo("Queued Event's Item is gone from its source; revoked its queued Events", { ...fields, revokedEvents: revoked, projectionMarkedGone });
+                return;
+            }
+            const runId = yield* runIdFor(submission).pipe(Effect.provideService(Crypto.Crypto, crypto));
+            const claimed = yield* store
+                .claimQueued(submission, runId)
+                .pipe(retryingStore(closedRunId, "claim the queued event"));
+            if (Option.isNone(claimed)) {
+                // During the check, the Event was ignored, another Run opened for the Item, or a cycle
+                // reset superseded the Event.
+                yield* Effect.logInfo("Queued Event still matches its trigger but could no longer be claimed; no Run opened", fields);
+                return;
+            }
+            yield* Effect.logInfo("Queued Event still matches its trigger; opening its Run", {
+                ...fields,
+                runId: claimed.value,
+            });
+            yield* launch(claimed.value);
+        });
+        /**
+         * Reads the state of the Run's open Attempt, or `None` if there is none. If a read fails,
+         * logs a warning and returns `Interrupted`.
+         */
+        const openAttempt = Effect.fn("RunDriver.openAttempt")(function* (runId) {
+            yield* Effect.annotateCurrentSpan({ runId });
+            const current = yield* store.snapshot(runId).pipe(retryingStore(runId, "read the run"));
+            const open = current.attempts.at(-1);
+            if (current.run.state._tag !== "Running" ||
+                open === undefined ||
+                Option.isSome(open.outcome))
+                return Option.none();
+            const fields = { runId, attemptId: open.id, worker: open.worker };
+            yield* Effect.annotateCurrentSpan({ attemptId: open.id, worker: open.worker });
+            const live = yield* executor.isLive(open.id).pipe(Effect.result);
+            if (Result.isFailure(live)) {
+                yield* Effect.logWarning("Could not read whether the open Attempt is live; the Run does not reattach it", { ...fields, errorTag: live.failure._tag, message: live.failure.message });
+                return Option.some({ _tag: "Interrupted", attempt: open });
+            }
+            if (live.success) {
+                yield* Effect.logInfo("The open Attempt's parent task is live; reattaching it", fields);
+                return Option.some({ _tag: "Live", attempt: open });
+            }
+            const stored = yield* executor.outcome(open.id).pipe(Effect.result);
+            if (Result.isFailure(stored)) {
+                yield* Effect.logWarning("Could not read the open Attempt's stored outcome; the Run does not reattach it", { ...fields, errorTag: stored.failure._tag, message: stored.failure.message });
+                return Option.some({ _tag: "Interrupted", attempt: open });
+            }
+            if (Option.isNone(stored.success)) {
+                yield* Effect.logInfo("The open Attempt's parent task is gone; the Run does not reattach it", fields);
+                return Option.some({ _tag: "Interrupted", attempt: open });
+            }
+            const result = stored.success.value;
+            yield* Effect.logInfo("The open Attempt's parent task finished before this process recorded it; finishing the Attempt with its stored result", {
+                ...fields,
+                outcome: Result.match(result, {
+                    onSuccess: () => "Succeeded",
+                    onFailure: ({ _tag }) => _tag,
+                }),
+            });
+            return Option.some({ _tag: "Finished", attempt: open, result });
+        });
+        /**
+         * Returns the slot for the Run's next Attempt and the effect that reads the open Attempt once
+         * the slot is held. A reattaching Run uses the slot startup occupied or waits for one on the
+         * Attempt's Worker. Only the wait, through `restore`, is interruptible.
+         */
+        const slotFor = Effect.fn("RunDriver.slot")(function* (runId, submission, open, restore) {
+            yield* Effect.annotateCurrentSpan({ runId });
+            const found = Effect.succeed(open);
+            const reattached = reattaching(open);
+            if (Option.isSome(reattached)) {
+                const occupied = yield* checkouts.reattachSlot(reattached.value.id);
+                if (Option.isSome(occupied)) {
+                    yield* Effect.logInfo("The Run reattaches its open Attempt in the slot startup occupied on its Worker", { runId, attemptId: reattached.value.id, worker: occupied.value.worker });
+                    return { lease: occupied.value, open: found };
+                }
+            }
+            const lease = yield* restore(executor.reserve(runId, submission, Option.map(reattached, ({ worker }) => worker)));
+            // The parent task can stop during the wait, so read it again before reattaching.
+            return { lease, open: Option.isSome(reattached) ? openAttempt(runId) : found };
+        });
+        /**
+         * Runs one Attempt of the Run of `submission` on the reserved Worker and returns the
+         * resulting decision tag. `current` reads the open Attempt once the slot is held.
+         */
+        const executeAttempt = Effect.fn("RunDriver.attempt")(function* (runId, submission, worker, current) {
+            yield* Effect.annotateCurrentSpan({ runId, worker });
+            const open = yield* current;
+            if (Option.isSome(open) && open.value._tag === "Finished") {
+                const finished = yield* store
+                    .finishAttempt(open.value.attempt, open.value.result)
+                    .pipe(retryingStore(runId, "finish the attempt"));
+                if (finished._tag !== "RetryAttempt")
+                    return finished._tag;
+            }
+            const started = yield* store
+                .beginAttempt(runId, worker, Option.map(reattaching(open), ({ id }) => id))
+                .pipe(retryingStore(runId, "begin the attempt"));
+            if (started._tag === "AttemptRecoveryFailed")
+                return "RunFailed";
+            if (started._tag === "RunClosed")
+                return "Unchanged";
+            // An executor defect becomes a `DefectFailure`, which isn't retried automatically (ADR 0003).
+            const result = yield* executor
+                .execute(started.execution, {
+                append: (direction, envelope) => store.appendTranscript(direction, envelope).pipe(asAttemptFailure),
+                salvage: (salvage) => store.recordSalvage(started.execution.attempt.id, salvage).pipe(asAttemptFailure),
+                comments: (comments) => store.recordComments(started.execution.attempt.id, comments).pipe(asAttemptFailure),
+                outOfMemory: (kills) => store.recordOutOfMemory(started.execution.attempt.id, kills).pipe(asAttemptFailure),
+                windows: store
+                    .attemptWindows(started.execution.attempt.id)
+                    .pipe(retryingStore(runId, "read the attempt windows"), asAttemptFailure),
+                transcript: store
+                    .transcript(new TranscriptAllQuery({ attemptId: started.execution.attempt.id }))
+                    .pipe(retryingStore(runId, "read the attempt transcript"), asAttemptFailure, 
+                // Reattached Attempts recorded the current protocol version. A version bump must
+                // convert retired envelopes here for Attempts open across the upgrade.
+                Effect.map((entries) => entries.flatMap(({ envelope }) => envelope.version === AttemptChannel.PROTOCOL_VERSION ? [envelope] : []))),
+                // Until this point, the driver reconciles a checkout that startup kept, for example on
+                // a cancel (#180).
+                checkoutAcquired: checkouts.acquired(started.execution.attempt.id),
+            })
+                .pipe(Effect.catchDefect((defect) => Effect.logError("Step executor defect; operator intervention required", {
+                runId,
+                attemptId: started.execution.attempt.id,
+                defect: defect instanceof Error ? defect.message : String(defect),
+            }).pipe(Effect.andThen(new DefectFailure({
+                message: "The step executor crashed with an unexpected error",
+                cause: defect,
+            })))), Effect.result);
+            if (started.execution.reattach && Result.isFailure(result)) {
+                // Reconcile a reattach that failed before acquiring the kept checkout, such as on a
+                // credential error, so the next Attempt resumes from its salvage. `reconcile` does nothing
+                // once the checkout was acquired, and fails this fiber while the stop is unconfirmed.
+                yield* checkouts.reconcile(started.execution.attempt.id, "failed");
+            }
+            // Before the Attempt closes, so a cancel meanwhile still finds it open and keeps its slot.
+            if (Result.isFailure(result))
+                yield* confirmStopAfterFailure({
+                    attemptId: started.execution.attempt.id,
+                    runId,
+                    loop: submission.loop,
+                    itemKey: submission.itemKey,
+                    reason: "failed",
+                }, worker);
+            const finished = yield* store
+                .finishAttempt(started.execution.attempt, result)
+                .pipe(retryingStore(runId, "finish the attempt"));
+            return finished._tag;
+        });
+        // `launch` traces it as the root span of the Run's own trace.
+        const drive = Effect.fnUntraced(function* (runId) {
+            const submission = yield* store
+                .submission(runId)
+                .pipe(retryingStore(runId, "read the run's submission"));
+            yield* Effect.annotateCurrentSpan({
+                runId,
+                loop: submission.loop,
+                itemKey: submission.itemKey,
+            });
+            if (submission.approval === "required" &&
+                (yield* store
+                    .parkForApproval(runId)
+                    .pipe(retryingStore(runId, "park the run for approval")))._tag === "Waiting") {
+                yield* Effect.logInfo("Run waits for its approval", { runId });
+                return;
+            }
+            let outcome;
+            do {
+                // A reattached Attempt uses a slot on the Worker it started on (#177).
+                const open = yield* openAttempt(runId);
+                if (Option.isSome(open) && open.value._tag !== "Live") {
+                    // Reconcile before waiting for a slot, so the next Attempt resumes from the salvage and
+                    // the slot startup occupied is freed. If the stop can't be confirmed, this fiber fails
+                    // and keeps the slot; the next reconcile drives the Run again.
+                    yield* checkouts.reconcile(open.value.attempt.id, "startup");
+                }
+                // A reattachment starts no parent task, so only a new Attempt waits for the stop.
+                if (Option.isNone(reattaching(open))) {
+                    const blocking = yield* stopBlocking(submission);
+                    if (Option.isSome(blocking)) {
+                        // `reconcile` retries the stop, then starts a fiber for this Run again.
+                        yield* Effect.logInfo("The Run starts no Attempt while the parent task of an earlier Attempt on its Loop's Item is not confirmed stopped; each reconcile tries that stop again, then drives the Run", {
+                            runId,
+                            loop: submission.loop,
+                            itemKey: submission.itemKey,
+                            unstoppedAttemptId: blocking.value.attemptId,
+                            unstoppedRunId: blocking.value.runId,
+                            worker: blocking.value.worker,
+                            reason: blocking.value.reason,
+                        });
+                        return;
+                    }
+                }
+                // Only the slot wait is interruptible, so a cancel can stop a Run in `reserve`; the
+                // scheduler releases or reoffers the slot on interrupt. `releaseSlot` runs once the
+                // lease exists.
+                outcome = yield* Effect.uninterruptibleMask((restore) => slotFor(runId, submission, open, restore).pipe(Effect.tap(({ lease }) => Ref.update(slots, (held) => new Map(held).set(runId, lease))), Effect.flatMap(({ lease, open: current }) => restore(admission.withAttempt(runId, executeAttempt(runId, submission, lease.worker, current))).pipe(Effect.ensuring(releaseSlot(runId, lease))))));
+            } while (outcome === "RetryAttempt");
+            // Cancel, supersede, and revoke handle the Item's queued Events themselves and can
+            // interrupt this fiber at any point.
+            if (outcome === "Unchanged") {
+                yield* Effect.logInfo("Run closed by another operation; its fiber ends", { runId });
+                return;
+            }
+            yield* continueWithNewest(runId);
+        });
+        const holdSlot = Effect.fn("RunDriver.holdSlot")(function* (runId, attemptId) {
+            yield* Effect.annotateCurrentSpan({ runId, attemptId });
+            const lease = yield* Ref.modify(slots, (held) => [
+                Option.fromUndefinedOr(held.get(runId)),
+                without(held, runId),
+            ]);
+            if (Option.isNone(lease)) {
+                yield* Effect.logDebug("The Run's fiber holds no slot in this process; no slot moves to its Attempt", { runId, attemptId });
+                return;
+            }
+            yield* checkouts.hold(attemptId, lease.value);
+        }, Effect.uninterruptible);
+        /** Records `stop` for `reconcile` to retry, keeping the first record of its Attempt. */
+        const recordUnconfirmed = (stop) => Ref.update(unconfirmed, (stops) => stops.has(stop.attemptId) ? stops : new Map(stops).set(stop.attemptId, stop));
+        /**
+         * Confirms through `StepExecutor.interrupt` that the parent task of the Attempt of `stop`,
+         * which failed on `worker`, stopped. The executor stops the task when an Attempt fails but
+         * doesn't report whether the stop succeeded. If the stop isn't confirmed, the Attempt keeps
+         * the Run's slot, as on a cancel, no Run of the Loop on the Item starts an Attempt, and each
+         * `reconcile` retries the stop (#512, #523).
+         */
+        const confirmStopAfterFailure = Effect.fn("RunDriver.confirmStopAfterFailure")(function* (stop, worker) {
+            const { attemptId, runId } = stop;
+            const fields = { runId, attemptId, worker, reason: stop.reason };
+            yield* Effect.annotateCurrentSpan(fields);
+            const stopped = yield* executor.interrupt(attemptId, stop.reason).pipe(Effect.result);
+            if (Result.isSuccess(stopped)) {
+                yield* Effect.logDebug("The parent task of the failed Attempt is confirmed stopped; the Run's slot is freed", fields);
+                return;
+            }
+            yield* Effect.logWarning("Could not confirm that the parent task of the failed Attempt stopped; the Attempt keeps its Run's slot, no Attempt starts on its Loop's Item, and each reconcile tries the stop again", { ...fields, errorTag: stopped.failure._tag, message: stopped.failure.message });
+            yield* holdSlot(runId, attemptId).pipe(Effect.andThen(recordUnconfirmed(new UnconfirmedStop({ ...stop, worker }))), Effect.uninterruptible);
+        });
+        /** Returns the unconfirmed stop that keeps the Run of `submission` from starting an Attempt. */
+        const stopBlocking = (submission) => Ref.get(unconfirmed).pipe(Effect.map((stops) => Option.fromUndefinedOr([...stops.values()].find(({ loop, itemKey }) => loop === submission.loop && itemKey === submission.itemKey))));
+        const releaseKept = Effect.fn("RunDriver.releaseKept")(function* (stop) {
+            const { attemptId, runId, reason } = stop;
+            yield* Effect.annotateCurrentSpan({ runId, attemptId, reason });
+            const stopped = yield* checkouts.reconcile(attemptId, reason).pipe(Effect.result);
+            if (Result.isSuccess(stopped)) {
+                const recorded = yield* Ref.modify(unconfirmed, (stops) => [
+                    stops.has(attemptId),
+                    without(stops, attemptId),
+                ]);
+                if (recorded)
+                    yield* Effect.logInfo("The parent task of a closed Attempt is confirmed stopped; Runs on its Loop's Item start Attempts again", { runId, attemptId, loop: stop.loop, itemKey: stop.itemKey, reason });
+                return;
+            }
+            const { worker } = stopped.failure;
+            yield* Effect.annotateCurrentSpan({ worker });
+            yield* recordUnconfirmed(new UnconfirmedStop({ ...stop, worker }));
+            yield* Effect.logWarning("The parent task of a closed Attempt is not confirmed stopped; its Worker's slot, and any checkout startup kept, stay kept, no Attempt starts on its Loop's Item, and the next reconcile tries the stop again", { runId, attemptId, worker, loop: stop.loop, itemKey: stop.itemKey, reason });
+        });
+        const recoverWorkspaces = Effect.fn("RunDriver.recoverWorkspaces")(function* () {
+            // Closed Attempts whose parent task stop wasn't confirmed before the restart.
+            const unstopped = yield* checkouts.atStartup();
+            yield* Effect.forEach(unstopped, ({ attemptId, runId, loop, itemKey, worker }) => recordUnconfirmed(new UnconfirmedStop({ attemptId, runId, loop, itemKey, worker, reason: "startup" })), { discard: true });
+        });
+        const reconcile = Effect.fn("RunDriver.reconcile")(function* () {
+            // No Run fiber retries these stops, so reconcile does, before it starts the fibers they
+            // keep from starting an Attempt.
+            const unstopped = [...(yield* Ref.get(unconfirmed)).values()];
+            yield* Effect.forEach(unstopped, releaseKept, { discard: true });
+            const continuations = yield* store.pendingContinuations;
+            yield* Effect.forEach(continuations, (runId) => continueWithNewest(runId).pipe(Effect.tapError((error) => Effect.logError("Could not continue the queued Events behind a closed Run", {
+                runId,
+                errorTag: error._tag,
+            })), Effect.ignore), { discard: true });
+            const driven = yield* store.runIdsToDrive;
+            const withoutFiber = yield* Effect.filter(driven, (runId) => Effect.map(FiberMap.has(fibers, runId), (running) => !running));
+            yield* Effect.forEach(withoutFiber, launch, { discard: true });
+            // After the stops above, so a machine whose stop they confirmed is released before it.
+            yield* executor.sweep;
+            const fields = {
+                continuedAfterRunIds: continuations,
+                startedRunIds: withoutFiber,
+                drivenRunIds: driven.length,
+                retriedStopAttemptIds: unstopped.map(({ attemptId }) => attemptId),
+            };
+            if (continuations.length === 0 && withoutFiber.length === 0 && unstopped.length === 0)
+                yield* Effect.logDebug("Every Run to drive already has its fiber", fields);
+            else
+                yield* Effect.logInfo("Runs recovered from the run tables", fields);
+        });
+        return {
+            launch,
+            resume,
+            stop,
+            continueWithNewest,
+            reconcile,
+            recoverWorkspaces,
+            holdSlot,
+            releaseKept,
+            unconfirmedStops: Ref.get(unconfirmed).pipe(Effect.map((stops) => [...stops.values()])),
+        };
+    }));
+}
+//# sourceMappingURL=run-driver.js.map

package/dist/engine/run-driver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-driver.js","sourceRoot":"","sources":["../../src/engine/run-driver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,cAAc,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAuB,aAAa,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAI/F,OAAO,EACL,KAAK,EACL,OAAO,EACP,MAAM,EACN,QAAQ,EACR,MAAM,EACN,KAAK,EACL,QAAQ,EACR,KAAK,EACL,MAAM,EACN,GAAG,EACH,MAAM,EACN,QAAQ,GACT,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAStE,OAAO,EAAE,QAAQ,EAAmB,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,gGAAgG;AAChG,kCAAkC;AAClC,MAAM,gBAAgB,GAAG,CACvB,MAA0C,EACL,EAAE,CACvC,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE;IACvB,sBAAsB,EAAE,CAAC,KAA6B,EAAE,EAAE,CACxD,IAAI,YAAY,CAAC;QACf,OAAO,EAAE,kCAAkC,KAAK,CAAC,SAAS,EAAE;QAC5D,KAAK,EAAE,KAAK;KACb,CAAC;IACJ,gBAAgB,EAAE,CAAC,KAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;IAChE,yBAAyB,EAAE,CAAC,KAAgC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;IAClF,oBAAoB,EAAE,CAAC,KAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;CACzE,CAAC,CAAC;AAEL,0EAA0E;AAC1E,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,kEAAkE;AAClE,MAAM,iBAAiB,GAAG,YAAY,CAAC;AAEvC;;;GAGG;AACH,MAAM,aAAa,GACjB,CAAC,KAAY,EAAE,SAAiB,EAAE,EAAE,CACpC,CAAI,MAAuC,EAAmC,EAAE,CAC9E,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,KAAK,CAAC,IAAI,KAAK,wBAAwB;IACrC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,yCAAyC,EAAE;QAC3D,KAAK;QACL,SAAS;QACT,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC;IACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAChB,EACD,MAAM,CAAC,KAAK,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,iBAAiB,CAAC;IACjD,KAAK,EAAE,aAAa;IACpB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,wBAAwB;CAC1D,CAAC,CACH,CAAC;AAqBN,2DAA2D;AAC3D,MAAM,WAAW,GAAG,CAAC,IAAgC,EAA0B,EAAE,CAC/E,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAC7B,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CACnE,CAAC;AAEJ;;;;;;;GAOG;AACH,MAAM,OAAO,SAAU,SAAQ,OAAO,CAAC,OAAO,EA6C3C,CAAC,kCAAkC,CAAC;IACrC,MAAM,CAAU,KAAK,GAAG,KAAK,CAAC,MAAM,CAClC,SAAS,EACT,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC;QAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC;QACrC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,gBAAgB,CAAC;QAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,gBAAgB,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,oBAAoB,CAAC;QAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;QACpC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAS,CAAC;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC9D,8FAA8F;QAC9F,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAA0C,IAAI,GAAG,EAAE,CAAC,CAAC;QACxF,yFAAyF;QACzF,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAkC,IAAI,GAAG,EAAE,CAAC,CAAC;QAE1E,iFAAiF;QACjF,MAAM,WAAW,GAAG,CAAC,KAAY,EAAE,KAAkB,EAAE,EAAE,CACvD,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CACzB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CACzE,CAAC,IAAI,CACJ,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,KAAK;YACH,CAAC,CAAC,KAAK,CAAC,OAAO;YACf,CAAC,CAAC,MAAM,CAAC,QAAQ,CACb,kFAAkF,EAClF,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAChC,CACN,CACF,CAAC;QAEJ,8FAA8F;QAC9F,MAAM,MAAM,GAAmC,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,EACpF,KAAY;YAEZ,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,IAAI,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBAClF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9D,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5D,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CACjB,MAAM,EACN,KAAK,EACL,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI;YACf,mFAAmF;YACnF,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC;gBAC5B,CAAC,CAAC,MAAM,CAAC,IAAI;gBACb,CAAC,CAAC,MAAM,CAAC,QAAQ,CACb,0FAA0F,EAC1F,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CACN,EACD,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE;gBACjC,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;aACpF,CAAC,EACF,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CACvE,CACF,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,CACxB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAY;YAClE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACpD,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,oFAAoF;gBACpF,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,6CAA6C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBACjF,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC;YACD,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAY;YAC9D,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC1C,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,6CAA6C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBACjF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACtC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAC,QAAQ,CAAC,EAChE,WAAkB,EAClB,UAAsB;YAEtB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;YACvF,OAAO,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,CACxE,aAAa,CAAC,WAAW,EAAE,iCAAiC,CAAC,EAC7D,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACrB,MAAM,CAAC,UAAU,CACf,0EAA0E,EAC1E;gBACE,cAAc,EAAE,WAAW;gBAC3B,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,QAAQ,EAAE,KAAK,CAAC,IAAI;aACrB,CACF,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CACzB,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,8BAA8B,CAAC,CAAC,QAAQ,CAAC,EAC5E,WAAkB;YAElB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;YAC1D,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK;iBACxB,UAAU,CAAC,WAAW,CAAC;iBACvB,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC,CAAC;YAC9D,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;gBAAE,OAAO;YAClC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;YAC3C,MAAM,MAAM,GAAG;gBACb,cAAc,EAAE,WAAW;gBAC3B,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE;gBAC5B,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC;YACF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACrD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,sDAAsD;gBACtD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK;qBACzB,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;qBAC1B,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,0BAA0B,CAAC,CAAC,CAAC;gBAChE,IAAI,OAAO,KAAK,mBAAmB,EAAE,CAAC;oBACpC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,qDAAqD,EAAE;wBAC3E,GAAG,MAAM;wBACT,aAAa,EAAE,OAAO;qBACvB,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,oBAAoB,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;gBAC1E,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,wEAAwE,EACxE,EAAE,GAAG,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAC5D,CAAC;gBACF,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAC5C,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAC7C,CAAC;YACF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK;iBACzB,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC;iBAC9B,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC,CAAC;YAC9D,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,uFAAuF;gBACvF,8BAA8B;gBAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,sFAAsF,EACtF,MAAM,CACP,CAAC;gBACF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,yDAAyD,EAAE;gBAC/E,GAAG,MAAM;gBACT,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YACH,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH;;;WAGG;QACH,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAY;YAC5E,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;YACxF,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,IACE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS;gBACpC,IAAI,KAAK,SAAS;gBAClB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;gBAE3B,OAAO,MAAM,CAAC,IAAI,EAAe,CAAC;YACpC,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YAClE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/E,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,+EAA+E,EAC/E,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAC1E,CAAC;gBACF,OAAO,MAAM,CAAC,IAAI,CAAc,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,wDAAwD,EAAE,MAAM,CAAC,CAAC;gBACxF,OAAO,MAAM,CAAC,IAAI,CAAc,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACpE,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,gFAAgF,EAChF,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAC9E,CAAC;gBACF,OAAO,MAAM,CAAC,IAAI,CAAc,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,sEAAsE,EACtE,MAAM,CACP,CAAC;gBACF,OAAO,MAAM,CAAC,IAAI,CAAc,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;YACpC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,uHAAuH,EACvH;gBACE,GAAG,MAAM;gBACT,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE;oBAC5B,SAAS,EAAE,GAAG,EAAE,CAAC,WAAW;oBAC5B,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI;iBAC9B,CAAC;aACH,CACF,CAAC;YACF,OAAO,MAAM,CAAC,IAAI,CAAc,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH;;;;WAIG;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,EACnD,KAAY,EACZ,UAAsB,EACtB,IAAgC,EAChC,OAA4E;YAE5E,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACpE,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,gFAAgF,EAChF,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CACzE,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAC1B,QAAQ,CAAC,OAAO,CACd,KAAK,EACL,UAAU,EACV,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAC/C,CACF,CAAC;YACF,iFAAiF;YACjF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QACjF,CAAC,CAAC,CAAC;QAEH;;;WAGG;QACH,MAAM,cAAc,GAAG,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,EAC7D,KAAY,EACZ,UAAsB,EACtB,MAAkB,EAClB,OAAiE;YAEjE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC;YAC5B,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,KAAK;qBAC1B,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;qBACpD,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC;gBACpD,IAAI,QAAQ,CAAC,IAAI,KAAK,cAAc;oBAAE,OAAO,QAAQ,CAAC,IAAI,CAAC;YAC7D,CAAC;YACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK;iBACzB,YAAY,CACX,KAAK,EACL,MAAM,EACN,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAC9C;iBACA,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC;YACnD,IAAI,OAAO,CAAC,IAAI,KAAK,uBAAuB;gBAAE,OAAO,WAAW,CAAC;YACjE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW;gBAAE,OAAO,WAAW,CAAC;YAErD,8FAA8F;YAC9F,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ;iBAC3B,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE;gBAC1B,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,CAC9B,KAAK,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;gBACpE,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CACnB,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;gBACnF,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CACrB,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;gBACrF,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CACrB,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;gBACrF,OAAO,EAAE,KAAK;qBACX,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;qBAC5C,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,0BAA0B,CAAC,EAAE,gBAAgB,CAAC;gBAC3E,UAAU,EAAE,KAAK;qBACd,UAAU,CAAC,IAAI,kBAAkB,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;qBAC/E,IAAI,CACH,aAAa,CAAC,KAAK,EAAE,6BAA6B,CAAC,EACnD,gBAAgB;gBAChB,iFAAiF;gBACjF,uEAAuE;gBACvE,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACrB,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC/B,QAAQ,CAAC,OAAO,KAAK,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CACvE,CACF,CACF;gBACH,uFAAuF;gBACvF,mBAAmB;gBACnB,gBAAgB,EAAE,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;aACnE,CAAC;iBACD,IAAI,CACH,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,EAAE,CAC5B,MAAM,CAAC,QAAQ,CAAC,sDAAsD,EAAE;gBACtE,KAAK;gBACL,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;gBACvC,MAAM,EAAE,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;aAClE,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CACZ,IAAI,aAAa,CAAC;gBAChB,OAAO,EAAE,oDAAoD;gBAC7D,KAAK,EAAE,MAAM;aACd,CAAC,CACH,CACF,CACF,EACD,MAAM,CAAC,MAAM,CACd,CAAC;YACJ,IAAI,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3D,oFAAoF;gBACpF,2FAA2F;gBAC3F,sFAAsF;gBACtF,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACrE,CAAC;YACD,2FAA2F;YAC3F,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;gBAC1B,KAAK,CAAC,CAAC,uBAAuB,CAC5B;oBACE,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;oBACvC,KAAK;oBACL,IAAI,EAAE,UAAU,CAAC,IAAI;oBACrB,OAAO,EAAE,UAAU,CAAC,OAAO;oBAC3B,MAAM,EAAE,QAAQ;iBACjB,EACD,MAAM,CACP,CAAC;YACJ,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,KAAK;iBAC1B,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;iBAChD,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAC;YACpD,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,8DAA8D;QAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,KAAY;YACrD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,KAAK;iBAC5B,UAAU,CAAC,KAAK,CAAC;iBACjB,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC,CAAC;YAC3D,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC;gBAChC,KAAK;gBACL,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC,CAAC;YACH,IACE,UAAU,CAAC,QAAQ,KAAK,UAAU;gBAClC,CAAC,KAAK,CAAC,CAAC,KAAK;qBACV,eAAe,CAAC,KAAK,CAAC;qBACtB,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,EAC9E,CAAC;gBACD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/D,OAAO;YACT,CAAC;YAED,IAAI,OAA+B,CAAC;YACpC,GAAG,CAAC;gBACF,uEAAuE;gBACvE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACvC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBACtD,wFAAwF;oBACxF,uFAAuF;oBACvF,+DAA+D;oBAC/D,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;gBAC/D,CAAC;gBACD,kFAAkF;gBAClF,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;oBACjD,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5B,wEAAwE;wBACxE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,8KAA8K,EAC9K;4BACE,KAAK;4BACL,IAAI,EAAE,UAAU,CAAC,IAAI;4BACrB,OAAO,EAAE,UAAU,CAAC,OAAO;4BAC3B,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,SAAS;4BAC5C,cAAc,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;4BACpC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;4BAC7B,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;yBAC9B,CACF,CAAC;wBACF,OAAO;oBACT,CAAC;gBACH,CAAC;gBACD,oFAAoF;gBACpF,oFAAoF;gBACpF,gBAAgB;gBAChB,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,OAAO,EAAE,EAAE,CACtD,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAC5C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACvB,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAC7D,EACD,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAC1C,OAAO,CACL,SAAS,CAAC,WAAW,CACnB,KAAK,EACL,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CACzD,CACF,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CACnD,CACF,CACF,CAAC;YACJ,CAAC,QAAQ,OAAO,KAAK,cAAc,EAAE;YAErC,mFAAmF;YACnF,qCAAqC;YACrC,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;gBAC5B,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,EACxD,KAAY,EACZ,SAAoB;YAEpB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YACxD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/C,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;aACrB,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CACpB,6EAA6E,EAC7E,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;gBACF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;QAE3B,wFAAwF;QACxF,MAAM,iBAAiB,GAAG,CAAC,IAAqB,EAAE,EAAE,CAClD,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAChC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAC7E,CAAC;QAEJ;;;;;;WAMG;QACH,MAAM,uBAAuB,GAAG,MAAM,CAAC,EAAE,CAAC,mCAAmC,CAAC,CAAC,QAAQ,CAAC,EACtF,IAAgB,EAChB,MAAkB;YAElB,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;YAClC,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;YACjE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACtF,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CACpB,qFAAqF,EACrF,MAAM,CACP,CAAC;gBACF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,uLAAuL,EACvL,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAChF,CAAC;YACF,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,CACpC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,IAAI,eAAe,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAC3E,MAAM,CAAC,eAAe,CACvB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,gGAAgG;QAChG,MAAM,YAAY,GAAG,CAAC,UAAsB,EAAE,EAAE,CAC9C,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CACvB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,MAAM,CAAC,eAAe,CACpB,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CACtB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,IAAI,OAAO,KAAK,UAAU,CAAC,OAAO,CAClF,CACF,CACF,CACF,CAAC;QAEJ,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAgB;YAChF,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;YAC1C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAClF,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC;oBACzD,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC;oBACpB,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC;iBAC1B,CAAC,CAAC;gBACH,IAAI,QAAQ;oBACV,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,wGAAwG,EACxG,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CACrE,CAAC;gBACJ,OAAO;YACT,CAAC;YACD,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;YACnC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9C,KAAK,CAAC,CAAC,iBAAiB,CAAC,IAAI,eAAe,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YACnE,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,8MAA8M,EAC9M,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAC7E,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC,EAAE,CAAC,6BAA6B,CAAC,CAAC,QAAQ,CAAC;YAC1E,8EAA8E;YAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YAC/C,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,SAAS,EACT,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAC9C,iBAAiB,CACf,IAAI,eAAe,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CACpF,EACH,EAAE,OAAO,EAAE,IAAI,EAAE,CAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAAC;YAC1D,wFAAwF;YACxF,iCAAiC;YACjC,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9D,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACjE,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC;YACxD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,aAAa,EACb,CAAC,KAAK,EAAE,EAAE,CACR,kBAAkB,CAAC,KAAK,CAAC,CAAC,IAAI,CAC5B,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,QAAQ,CAAC,0DAA0D,EAAE;gBAC1E,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,IAAI;aACrB,CAAC,CACH,EACD,MAAM,CAAC,MAAM,CACd,EACH,EAAE,OAAO,EAAE,IAAI,EAAE,CAClB,CAAC;YACF,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;YAC1C,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAC1D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAC/D,CAAC;YACF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,uFAAuF;YACvF,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;YACtB,MAAM,MAAM,GAAG;gBACb,oBAAoB,EAAE,aAAa;gBACnC,aAAa,EAAE,YAAY;gBAC3B,YAAY,EAAE,MAAM,CAAC,MAAM;gBAC3B,qBAAqB,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC;aACnE,CAAC;YACF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;gBACnF,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,0CAA0C,EAAE,MAAM,CAAC,CAAC;;gBACxE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,oCAAoC,EAAE,MAAM,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,MAAM;YACN,MAAM;YACN,IAAI;YACJ,kBAAkB;YAClB,SAAS;YACT,iBAAiB;YACjB,QAAQ;YACR,WAAW;YACX,gBAAgB,EAAE,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;SACxF,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;CACH"}

package/dist/engine/shutdown.d.ts

@@ -1,10 +1,9 @@
 import { Context, Deferred } from "effect";
 declare const RunEngineShutdown_base: Context.ServiceClass<RunEngineShutdown, "@usine/engine/RunEngineShutdown", Deferred.Deferred<void, never>>;
 /**
- * Optional host-owned shutdown latch, provided before building an engine runtime.
- * Completing it closes Attempt admission even while startup recovery is running.
- * After startup, RunEngine.drain waits for admitted Attempts to persist their outcomes.
- * Allocate a fresh latch for each runtime; it cannot be reopened (ADR 0023).
+ * An optional shutdown latch owned by the host, provided before building an engine runtime.
+ * Completing it closes Attempt admission, even during startup recovery. It can't be reopened,
+ * so each runtime needs a new one (ADR 0023).
  */
 export declare class RunEngineShutdown extends RunEngineShutdown_base {
 }

package/dist/engine/shutdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shutdown.d.ts","sourceRoot":"","sources":["../../src/engine/shutdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;;AAE3C;;;;;GAKG;AACH,qBAAa,iBAAkB,SAAQ,sBAGD;CAAG"}
+{"version":3,"file":"shutdown.d.ts","sourceRoot":"","sources":["../../src/engine/shutdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;;AAE3C;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,sBAGD;CAAG"}

package/dist/engine/shutdown.js

@@ -1,9 +1,8 @@
 import { Context, Deferred } from "effect";
 /**
- * Optional host-owned shutdown latch, provided before building an engine runtime.
- * Completing it closes Attempt admission even while startup recovery is running.
- * After startup, RunEngine.drain waits for admitted Attempts to persist their outcomes.
- * Allocate a fresh latch for each runtime; it cannot be reopened (ADR 0023).
+ * An optional shutdown latch owned by the host, provided before building an engine runtime.
+ * Completing it closes Attempt admission, even during startup recovery. It can't be reopened,
+ * so each runtime needs a new one (ADR 0023).
  */
 export class RunEngineShutdown extends Context.Service()("@usine/engine/RunEngineShutdown") {
 }

package/dist/engine/shutdown.js.map

@@ -1 +1 @@
-{"version":3,"file":"shutdown.js","sourceRoot":"","sources":["../../src/engine/shutdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAE3C;;;;;GAKG;AACH,MAAM,OAAO,iBAAkB,SAAQ,OAAO,CAAC,OAAO,EAGnD,CAAC,iCAAiC,CAAC;CAAG"}
+{"version":3,"file":"shutdown.js","sourceRoot":"","sources":["../../src/engine/shutdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAE3C;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,OAAO,CAAC,OAAO,EAGnD,CAAC,iCAAiC,CAAC;CAAG"}