@usine/engine 0.3.0 → 0.5.0

package/dist/engine/checkout-recovery.js

@@ -0,0 +1,238 @@
+import { AttemptId } from "@usine/core/attempt";
+import { WorkerName } from "@usine/core/worker";
+import { Array as EffectArray, Effect, Option, Ref, Result, Schema } from "effect";
+/**
+ * An error for an Attempt whose parent task couldn't be confirmed stopped. Its slot of `worker`,
+ * and any checkout startup kept, stay kept, and the Run's next Attempt doesn't start.
+ */
+class ParentStopUnconfirmedError extends Schema.TaggedError("@usine/engine/internal/ParentStopUnconfirmedError")("ParentStopUnconfirmedError", {
+    attemptId: AttemptId,
+    worker: WorkerName,
+    cause: Schema.Defect(),
+}) {
+}
+/** Returns `map` without `key`. */
+export const without = (map, key) => {
+    const remaining = new Map(map);
+    remaining.delete(key);
+    return remaining;
+};
+/**
+ * Creates the checkout recovery steps that `RunDriver` uses at startup, on reattach, and on cancel.
+ *
+ * `atStartup` reconciles the checkouts left on Workers through `StepExecutor.recover`. It keeps
+ * the checkout, and occupies a Worker slot through `StepExecutor.occupy` (#177), for an open
+ * Attempt whose parent task is live or unreadable and for a closed Attempt whose parent task
+ * isn't confirmed stopped. `acquired` hands a kept checkout and slot to the reattaching
+ * execution. `hold` keeps the slot of an Attempt whose parent task may still run after a cancel
+ * or a failure (#475, #512). `reconcile` reconciles a kept checkout that the Run closes without
+ * that hand-off (#180), and releases the slot once the parent task is confirmed stopped.
+ */
+export const checkoutRecoveryFor = Effect.fn("CheckoutRecovery.for")(function* (store, executor) {
+    // The slots of Attempts whose parent task may still run, by Attempt.
+    const kept = yield* Ref.make(new Map());
+    /** Reconciles `attempt`'s checkout. On failure, logs an error and leaves the checkout on its Worker. */
+    const recover = (attempt) => executor
+        .recover(attempt, {
+        salvage: (salvage) => store.recordSalvage(attempt.attemptId, salvage),
+        comments: (comments) => store.recordComments(attempt.attemptId, comments),
+        windows: store.attemptWindows(attempt.attemptId),
+    })
+        .pipe(Effect.tapError((error) => Effect.logError("Attempt salvage reconciliation failed; work retained", {
+        attemptId: attempt.attemptId,
+        worker: attempt.worker,
+        message: String(error),
+    })), Effect.ignore);
+    /**
+     * Keeps `attempt`'s checkout and occupies a slot on its Worker, recording the lease in the same
+     * uninterruptible step.
+     */
+    const keep = Effect.fn("CheckoutRecovery.keep")(function* (attempt) {
+        yield* executor.occupy(attempt.worker, attempt.attemptId).pipe(Effect.flatMap((lease) => Effect.cached(lease.release).pipe(Effect.map((release) => ({
+            worker: lease.worker,
+            release,
+            checkoutKept: true,
+        })))), Effect.flatMap((occupied) => Ref.update(kept, (leases) => new Map(leases).set(attempt.attemptId, occupied))), Effect.uninterruptible);
+    });
+    /**
+     * Keeps the checkout and a slot for an open Attempt whose parent task is live or whose liveness
+     * can't be read. Otherwise reconciles the checkout.
+     */
+    const recoverOpen = Effect.fn("CheckoutRecovery.recoverOpen")(function* (attempt) {
+        const fields = { attemptId: attempt.attemptId, worker: attempt.worker };
+        yield* Effect.annotateCurrentSpan(fields);
+        const live = yield* executor.isLive(attempt.attemptId).pipe(Effect.result);
+        if (Result.isSuccess(live) && !live.success)
+            return yield* recover(attempt);
+        if (Result.isFailure(live))
+            yield* Effect.logWarning("Could not read whether an open Attempt's parent task is live; its checkout stays for its Run to decide", { ...fields, errorTag: live.failure._tag, message: live.failure.message });
+        else
+            yield* Effect.logInfo("Leaving the checkout of an open Attempt whose parent task is live for its Run to reattach", fields);
+        yield* keep(attempt);
+        yield* Effect.logInfo("Occupied a slot on the Worker of an open Attempt kept for its Run to reattach", fields);
+    });
+    /**
+     * Reconciles a closed Attempt's checkout if its parent task is confirmed stopped. Otherwise
+     * keeps the checkout and a slot, and returns the Attempt (#177).
+     */
+    const recoverClosed = Effect.fn("CheckoutRecovery.recoverClosed")(function* (attempt) {
+        const fields = { attemptId: attempt.attemptId, runId: attempt.runId, worker: attempt.worker };
+        yield* Effect.annotateCurrentSpan(fields);
+        // `recover` also stops the task, but its failure doesn't distinguish an unconfirmed stop from
+        // a failed salvage.
+        const stopped = yield* executor.interrupt(attempt.attemptId, "startup").pipe(Effect.result);
+        if (Result.isSuccess(stopped)) {
+            yield* recover(attempt);
+            return Option.none();
+        }
+        yield* Effect.logWarning("Could not confirm that the parent task of a closed Attempt stopped; its checkout and a slot of its Worker stay kept, and no Attempt starts on its Loop's Item, until a reconcile confirms the stop", { ...fields, errorTag: stopped.failure._tag, message: stopped.failure.message });
+        yield* keep(attempt);
+        return Option.some(attempt);
+    });
+    /**
+     * Reconciles every persisted Attempt on a configured Worker. Returns the closed Attempts whose
+     * parent task isn't confirmed stopped.
+     */
+    const atStartup = Effect.fn("CheckoutRecovery.atStartup")(function* () {
+        const configured = new Set(yield* executor.workers);
+        const candidates = yield* store.salvageCandidates;
+        const reconcilable = candidates.filter(({ worker }) => configured.has(worker));
+        const unconfigured = candidates.filter(({ worker }) => !configured.has(worker));
+        if (unconfigured.length > 0) {
+            yield* Effect.logWarning("Attempts recorded on Workers that are not configured were not reconciled; their checkouts stay until those Workers are configured again or their leftovers are discarded", {
+                attempts: unconfigured.length,
+                workers: [...new Set(unconfigured.map(({ worker }) => worker))].sort().join(","),
+            });
+        }
+        // Runs before Run recovery, so a retry starts only after the previous Attempt's work is
+        // recorded. A failed recovery leaves the checkout on disk.
+        const unstopped = yield* Effect.forEach(reconcilable, (attempt) => attempt.open
+            ? recoverOpen(attempt).pipe(Effect.as(Option.none()))
+            : recoverClosed(attempt));
+        return EffectArray.getSomes(unstopped);
+    });
+    /**
+     * Returns the slot that startup occupied with the checkout it kept for `attemptId`, or `None`.
+     * Until the execution acquires the checkout, the slot's `release` does nothing; `reconcile`
+     * frees it.
+     */
+    const reattachSlot = Effect.fn("CheckoutRecovery.reattachSlot")(function* (attemptId) {
+        const occupied = Option.filter(Option.fromUndefinedOr((yield* Ref.get(kept)).get(attemptId)), ({ checkoutKept }) => checkoutKept);
+        return Option.map(occupied, (entry) => ({
+            worker: entry.worker,
+            release: Ref.get(kept).pipe(Effect.flatMap((leases) => leases.get(attemptId) === entry
+                ? Effect.logInfo("The slot startup occupied stays kept with the checkout no execution acquired, until the Attempt's parent task is confirmed stopped", { attemptId, worker: entry.worker })
+                : entry.release)),
+        }));
+    });
+    /**
+     * Hands the kept checkout of `attemptId` to its execution; the Run then releases the slot when
+     * the Attempt ends. Does nothing for other Attempts.
+     */
+    const acquired = Effect.fn("CheckoutRecovery.acquired")(function* (attemptId) {
+        const handedOver = yield* Ref.modify(kept, (leases) => leases.get(attemptId)?.checkoutKept === true
+            ? [true, without(leases, attemptId)]
+            : [false, leases]);
+        if (handedOver)
+            yield* Effect.logInfo("The reattached Attempt's execution acquired the checkout startup kept; its executor releases the checkout from here, and its Run the slot once the Attempt ends", { attemptId });
+    });
+    /**
+     * Keeps `lease` as the slot of `attemptId`, an Attempt whose parent task may still run, until
+     * `reconcile` confirms that the task stopped. If the Attempt already has a slot here, releases
+     * `lease` instead.
+     */
+    const hold = Effect.fn("CheckoutRecovery.hold")(function* (attemptId, lease) {
+        const fields = { attemptId, worker: lease.worker };
+        yield* Effect.annotateCurrentSpan(fields);
+        const release = yield* Effect.cached(lease.release);
+        const held = yield* Ref.modify(kept, (leases) => leases.has(attemptId)
+            ? [false, leases]
+            : [
+                true,
+                new Map(leases).set(attemptId, {
+                    worker: lease.worker,
+                    release,
+                    checkoutKept: false,
+                }),
+            ]);
+        if (held) {
+            yield* Effect.logInfo("The Attempt keeps its Run's slot until its parent task is confirmed stopped", fields);
+            return;
+        }
+        yield* Effect.logDebug("The Attempt already has a slot kept until its parent task is confirmed stopped; releasing its Run's lease", fields);
+        yield* release;
+    }, Effect.uninterruptible);
+    /**
+     * Stops the parent task of `attemptId` for `reason`. Fails with `ParentStopUnconfirmedError` if
+     * the stop isn't confirmed.
+     */
+    const confirmStop = Effect.fn("CheckoutRecovery.confirmStop")(function* (attemptId, occupied, reason) {
+        const fields = {
+            attemptId,
+            worker: occupied.worker,
+            checkoutKept: occupied.checkoutKept,
+            reason,
+        };
+        yield* Effect.annotateCurrentSpan(fields);
+        const stopped = yield* executor.interrupt(attemptId, reason).pipe(Effect.result);
+        if (Result.isSuccess(stopped))
+            return;
+        const failure = { ...fields, errorTag: stopped.failure._tag, message: stopped.failure.message };
+        yield* occupied.checkoutKept
+            ? Effect.logWarning("Could not confirm that the parent task of an Attempt its Run does not reattach stopped; its checkout and its Worker's slot stay, and the Run starts no next Attempt until the stop is confirmed", failure)
+            : Effect.logWarning("Could not confirm that the parent task of a closed Attempt stopped; its Worker's slot stays taken, and no Attempt starts on its Loop's Item, until a reconcile confirms the stop", failure);
+        return yield* new ParentStopUnconfirmedError({
+            attemptId,
+            worker: occupied.worker,
+            cause: stopped.failure,
+        });
+    });
+    /** Reconciles the checkout startup kept for `attemptId`, whose parent task is confirmed stopped. */
+    const recoverKept = Effect.fn("CheckoutRecovery.recoverKept")(function* (attemptId) {
+        const found = yield* store.salvageCandidate(attemptId).pipe(Effect.result);
+        if (Result.isFailure(found)) {
+            yield* Effect.logError("Could not read the Attempt whose checkout startup kept; the checkout stays until the next startup", { attemptId, errorTag: found.failure._tag });
+            return;
+        }
+        if (Option.isNone(found.success)) {
+            yield* Effect.logWarning("The leftovers of the Attempt whose checkout startup kept were discarded; the checkout stays", { attemptId });
+            return;
+        }
+        const attempt = found.success.value;
+        const fields = { attemptId, worker: attempt.worker, open: attempt.open };
+        yield* Effect.annotateCurrentSpan(fields);
+        if (!(yield* executor.workers).includes(attempt.worker)) {
+            yield* Effect.logWarning("The Worker of the Attempt whose checkout startup kept is no longer configured; the checkout stays until it is configured again or its leftovers are discarded", fields);
+            return;
+        }
+        // The Run doesn't reattach the Attempt, so reconcile even if `isLive` still reports it, as for
+        // an unharvested parent report.
+        yield* Effect.logInfo("Reconciling the checkout startup kept for an Attempt that was not reattached", fields);
+        yield* recover(attempt);
+    });
+    /**
+     * Stops the parent task of `attemptId` for `reason` if a slot is kept for it, then reconciles a
+     * checkout startup kept and releases the slot. Fails with `ParentStopUnconfirmedError` and keeps
+     * both if the stop isn't confirmed.
+     */
+    const reconcile = Effect.fn("CheckoutRecovery.reconcile")(function* (attemptId, reason) {
+        yield* Effect.annotateCurrentSpan({ attemptId, reason });
+        const occupied = (yield* Ref.get(kept)).get(attemptId);
+        if (occupied === undefined) {
+            yield* Effect.logDebug("No slot kept for the Attempt awaits its parent task's stop", {
+                attemptId,
+            });
+            return;
+        }
+        // The parent task must stop before the reconcile, the next Attempt, or the slot's release.
+        yield* confirmStop(attemptId, occupied, reason);
+        if (occupied.checkoutKept)
+            yield* recoverKept(attemptId);
+        // Removed only after reconciling, so a cancel during the reconcile reconciles again and the
+        // slot stays taken while the parent task can still run (#177).
+        yield* Ref.update(kept, (leases) => without(leases, attemptId)).pipe(Effect.andThen(occupied.release), Effect.uninterruptible);
+        yield* Effect.logInfo("Released the slot kept for an Attempt whose parent task is confirmed stopped", { attemptId, worker: occupied.worker, checkoutKept: occupied.checkoutKept });
+    });
+    return { atStartup, reattachSlot, acquired, hold, reconcile };
+});
+//# sourceMappingURL=checkout-recovery.js.map

package/dist/engine/checkout-recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkout-recovery.js","sourceRoot":"","sources":["../../src/engine/checkout-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAmB,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAInF;;;GAGG;AACH,MAAM,0BAA2B,SAAQ,MAAM,CAAC,WAAW,CACzD,mDAAmD,CACpD,CAAC,4BAA4B,EAAE;IAC9B,SAAS,EAAE,SAAS;IACpB,MAAM,EAAE,UAAU;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;CACvB,CAAC;CAAG;AAEL,mCAAmC;AACnC,MAAM,CAAC,MAAM,OAAO,GAAG,CAAO,GAAsB,EAAE,GAAM,EAAqB,EAAE;IACjF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC/B,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAQF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC,EAAE,CAAC,sBAAsB,CAAC,CAAC,QAAQ,CAAC,EAC5E,KAA0B,EAC1B,QAAiC;IAEjC,qEAAqE;IACrE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAmC,IAAI,GAAG,EAAE,CAAC,CAAC;IAE1E,wGAAwG;IACxG,MAAM,OAAO,GAAG,CAAC,OAAwB,EAAE,EAAE,CAC3C,QAAQ;SACL,OAAO,CAAC,OAAO,EAAE;QAChB,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC;QACrE,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;QACzE,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC;KACjD,CAAC;SACD,IAAI,CACH,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,QAAQ,CAAC,sDAAsD,EAAE;QACtE,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;KACvB,CAAC,CACH,EACD,MAAM,CAAC,MAAM,CACd,CAAC;IAEN;;;OAGG;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAwB;QACjF,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAC5D,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAY,EAAE,CAAC,CAAC;YACjC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO;YACP,YAAY,EAAE,IAAI;SACnB,CAAC,CAAC,CACJ,CACF,EACD,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC1B,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC/E,EACD,MAAM,CAAC,eAAe,CACvB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,8BAA8B,CAAC,CAAC,QAAQ,CAAC,EACrE,OAAwB;QAExB,MAAM,MAAM,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACxE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3E,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC5E,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC;YACxB,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,wGAAwG,EACxG,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAC1E,CAAC;;YAEF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,2FAA2F,EAC3F,MAAM,CACP,CAAC;QACJ,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,+EAA+E,EAC/E,MAAM,CACP,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,aAAa,GAAG,MAAM,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC,QAAQ,CAAC,EACzE,OAAyB;QAEzB,MAAM,MAAM,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9F,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,8FAA8F;QAC9F,oBAAoB;QACpB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5F,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,MAAM,CAAC,IAAI,EAAoB,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,oMAAoM,EACpM,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAChF,CAAC;QACF,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC;QACjE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAClD,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/E,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAChF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,0KAA0K,EAC1K;gBACE,QAAQ,EAAE,YAAY,CAAC,MAAM;gBAC7B,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;aACjF,CACF,CAAC;QACJ,CAAC;QACD,wFAAwF;QACxF,2DAA2D;QAC3D,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAChE,OAAO,CAAC,IAAI;YACV,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAoB,CAAC,CAAC;YACvE,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAC3B,CAAC;QACF,OAAO,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH;;;;OAIG;IACH,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,+BAA+B,CAAC,CAAC,QAAQ,CAAC,EAAE,SAAoB;QAC7F,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAC5B,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAC7D,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,YAAY,CACnC,CAAC;QACF,OAAO,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAe,EAAE,CAAC,CAAC;YACnD,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CACzB,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CACxB,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,KAAK;gBAC7B,CAAC,CAAC,MAAM,CAAC,OAAO,CACZ,oIAAoI,EACpI,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CACpC;gBACH,CAAC,CAAC,KAAK,CAAC,OAAO,CAClB,CACF;SACF,CAAC,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EAAE,SAAoB;QACrF,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CACpD,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,YAAY,KAAK,IAAI;YAC1C,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CACpB,CAAC;QACF,IAAI,UAAU;YACZ,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,iKAAiK,EACjK,EAAE,SAAS,EAAE,CACd,CAAC;IACN,CAAC,CAAC,CAAC;IAEH;;;;OAIG;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EACvD,SAAoB,EACpB,KAAkB;QAElB,MAAM,MAAM,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAC9C,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC;YACjB,CAAC,CAAC;gBACE,IAAI;gBACJ,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE;oBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO;oBACP,YAAY,EAAE,KAAK;iBACpB,CAAC;aACH,CACN,CAAC;QACF,IAAI,IAAI,EAAE,CAAC;YACT,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,6EAA6E,EAC7E,MAAM,CACP,CAAC;YACF,OAAO;QACT,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CACpB,2GAA2G,EAC3G,MAAM,CACP,CAAC;QACF,KAAK,CAAC,CAAC,OAAO,CAAC;IACjB,CAAC,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;IAE3B;;;OAGG;IACH,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,8BAA8B,CAAC,CAAC,QAAQ,CAAC,EACrE,SAAoB,EACpB,QAAkB,EAClB,MAAkB;QAElB,MAAM,MAAM,GAAG;YACb,SAAS;YACT,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,MAAM;SACP,CAAC;QACF,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjF,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;YAAE,OAAO;QACtC,MAAM,OAAO,GAAG,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAChG,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY;YAC1B,CAAC,CAAC,MAAM,CAAC,UAAU,CACf,iMAAiM,EACjM,OAAO,CACR;YACH,CAAC,CAAC,MAAM,CAAC,UAAU,CACf,kLAAkL,EAClL,OAAO,CACR,CAAC;QACN,OAAO,KAAK,CAAC,CAAC,IAAI,0BAA0B,CAAC;YAC3C,SAAS;YACT,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK,EAAE,OAAO,CAAC,OAAO;SACvB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,oGAAoG;IACpG,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,8BAA8B,CAAC,CAAC,QAAQ,CAAC,EAAE,SAAoB;QAC3F,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3E,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CACpB,mGAAmG,EACnG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAC5C,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,6FAA6F,EAC7F,EAAE,SAAS,EAAE,CACd,CAAC;YACF,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACpC,MAAM,MAAM,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;QACzE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxD,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,+JAA+J,EAC/J,MAAM,CACP,CAAC;YACF,OAAO;QACT,CAAC;QACD,+FAA+F;QAC/F,gCAAgC;QAChC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,8EAA8E,EAC9E,MAAM,CACP,CAAC;QACF,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH;;;;OAIG;IACH,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC,EACjE,SAAoB,EACpB,MAAkB;QAElB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,4DAA4D,EAAE;gBACnF,SAAS;aACV,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,2FAA2F;QAC3F,KAAK,CAAC,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,YAAY;YAAE,KAAK,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACzD,4FAA4F;QAC5F,+DAA+D;QAC/D,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAClE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAChC,MAAM,CAAC,eAAe,CACvB,CAAC;QACF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,8EAA8E,EAC9E,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAC5E,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAW,CAAC;AACzE,CAAC,CAAC,CAAC"}

package/dist/engine/engine.d.ts

@@ -1,59 +1,100 @@
 import { Event } from "@usine/core/event";
-import { AttemptContext, AttemptId } from "@usine/core/attempt";
+import { AttemptId } from "@usine/core/attempt";
 import { Intervention } from "@usine/core/attempt/channel";
 import { ItemKey } from "@usine/core/item";
 import { LoopName } from "@usine/core/loop";
+import type { OperatorLogin } from "@usine/core/operator";
 import { RunId } from "@usine/core/run";
-import { Context, Crypto, Effect, Layer, Option } from "effect";
-import { SqlClient } from "effect/unstable/sql";
+import { ApprovalSource } from "@usine/core/run/approval";
+import type { WorkerName } from "@usine/core/worker";
+import { Context, Crypto, type Duration, Effect, Layer, Option } from "effect";
 import { RunDatabase } from "../database/database.ts";
-import { type RunEngineError } from "./error.ts";
+import { LeftoversDiscardRejectedError, type RunEngineError } from "./error.ts";
 import type { RunSnapshot } from "../run-reader/model.ts";
-import type { Submission, SubmissionResult } from "./submission.ts";
+import { type Submission, type SubmissionResult } from "./submission.ts";
+import type { RunAgainRequest, RunAgainResult } from "./run-again.ts";
+import { type RetryRequest } from "./retry.ts";
 import type { TranscriptEntry, TranscriptQuery } from "../transcript.ts";
 import { RunReader } from "../run-reader/run-reader.ts";
 import { ItemProjectionReader, ItemProjectionWriter } from "../item-projection/item-projection.ts";
 import { RepositoryLabelProjectionReader, RepositoryLabelProjectionWriter } from "../repository-label-projection/repository-label-projection.ts";
 import { StepExecutor } from "../step-executor.ts";
+import { TriggerCondition } from "../trigger-condition.ts";
 import { CursorStore } from "../cursor/cursor.ts";
 import { LoopControl } from "../loop-control/loop-control.ts";
 import { CycleBudgets } from "../cycle-budget/cycle-budget.ts";
+import { BoardMoves } from "../board-moves/board-moves.ts";
+import { UnconfirmedStops } from "./unconfirmed-stops.ts";
+/** The interval between periodic reconciles, which restart open Runs whose fiber stopped. */
+export declare const ReconcileInterval: Context.Reference<Duration.Input>;
 declare const RunEngine_base: Context.ServiceClass<RunEngine, "@usine/engine/RunEngine", {
-    /** Stops admitting Attempts, including retries, and waits for admitted work to settle. Interrupting this wait keeps admission closed until restart. */
+    /** Stops admitting Attempts, including retries, and waits for admitted Attempts to finish. Admission stays closed until restart. */
     readonly drain: () => Effect.Effect<void>;
-    /** Records the operator's approval Event and opens the Run's gate; execution still waits for a Worker slot. */
-    readonly approve: (runId: RunId, approval: Event) => Effect.Effect<void, RunEngineError>;
-    /** Cancels an open Run and interrupts its active Attempt. An already closed Run is unchanged. */
+    /** Records the approval Event and its source, and opens the Run's approval gate. A repeated approval keeps the first source. */
+    readonly approve: (runId: RunId, approval: Event, source: ApprovalSource) => Effect.Effect<void, RunEngineError>;
+    /**
+     * Cancels an open Run and interrupts its active Attempt; does nothing for a closed Run. If the
+     * Attempt's parent task can't be confirmed stopped, fails with `AttemptCancellationError`, and
+     * until a reconcile confirms the stop, the Attempt keeps its Worker slot (#475) and no Run of
+     * the Loop on the Item starts an Attempt (#523).
+     */
     readonly cancel: (runId: RunId) => Effect.Effect<void, RunEngineError>;
-    /** Reads the Run's current state, effective task and Agent, Step result, and Attempt history. */
+    /**
+     * Stops reconciling the Attempts recorded on `worker`, then removes with
+     * `StepExecutor.discard` what they left outside their checkouts, and returns the number of
+     * Attempts marked. Leaves their checkouts in place. Fails with
+     * `LeftoversDiscardRejectedError` if `worker` is in `StepExecutor.workers`.
+     */
+    readonly discardLeftovers: (worker: WorkerName) => Effect.Effect<number, RunEngineError | LeftoversDiscardRejectedError>;
+    /** Returns the Run's state, effective task and Agent, Step result, and Attempt history. */
     readonly inspect: (runId: RunId) => Effect.Effect<RunSnapshot, RunEngineError>;
-    /** Addresses one current Attempt explicitly so an old drawer cannot steer its replacement. */
+    /** Sends operator input to `attemptId`. Fails with `RunInterventionRejectedError` if it isn't the Run's running Attempt. */
     readonly intervene: (runId: RunId, attemptId: AttemptId, intervention: Intervention) => Effect.Effect<void, RunEngineError>;
-    /** Recovers unfinished Runs and outstanding actions from stored state; also runs when the engine starts. */
+    /**
+     * Starts a fiber for every open Run without one, except Runs waiting for approval, and opens the
+     * queued Events that no closed Run continued, and retries the parent task stops that a cancel,
+     * a failed Attempt, or startup couldn't confirm. A Run whose Loop and Item have an unconfirmed
+     * stop starts no Attempt until a reconcile confirms it. Runs at startup and every
+     * `ReconcileInterval`.
+     */
     readonly reconcile: () => Effect.Effect<void, RunEngineError>;
-    /** Requests another Attempt for a failed Run using its existing Event and effective task and Agent. */
-    readonly retry: (runId: RunId, amendment?: Option.Option<AttemptContext>) => Effect.Effect<void, RunEngineError>;
-    /** Cancels unstarted Runs and discards queued Events for a Loop's Item when it stops matching; running Attempts continue. */
+    /**
+     * Marks the current failed Attempt as resolved without changing its outcome. If the Attempt
+     * stopped with a question or blocker, the Loop evaluates the Item's queued Events again.
+     */
+    readonly resolveFailure: (runId: RunId, attemptId: AttemptId, operator: Option.Option<OperatorLogin>) => Effect.Effect<void, RunEngineError>;
+    /**
+     * Requests another Attempt for a failed Run with its existing Event, applying the request's
+     * task and Agent if set. The Attempt records the request as its start reason.
+     */
+    readonly retry: (runId: RunId, request?: RetryRequest) => Effect.Effect<void, RunEngineError>;
+    /**
+     * Opens a new Run for the Item state of a succeeded or canceled Run, with the request's task
+     * and Agent, as an operator Event. The Item's cycle budget and the Loop's approval gate apply.
+     */
+    readonly runAgain: (runId: RunId, request: RunAgainRequest) => Effect.Effect<RunAgainResult, RunEngineError>;
+    /** Cancels unstarted Runs and discards queued Events for a Loop's Item that stopped matching; running Attempts continue. */
     readonly revoke: (loop: LoopName, itemKey: ItemKey) => Effect.Effect<ReadonlyArray<RunId>, RunEngineError>;
     /**
-     * Records a matching Event and claims its Run, or queues it behind the Item's
-     * running Attempt. Repeated Events are deduplicated; execution continues separately.
+     * Records a matching Event and claims its Run, or queues it behind the Item's running Attempt.
+     * Deduplicates repeated Events.
      */
     readonly submit: (submission: Submission) => Effect.Effect<SubmissionResult, RunEngineError>;
-    /** Reads the requested portion of an Attempt's persisted transcript in append order. */
+    /** Returns the requested part of an Attempt's transcript, in append order. */
     readonly transcript: (query: TranscriptQuery) => Effect.Effect<ReadonlyArray<TranscriptEntry>, RunEngineError>;
 }>;
-/** Records Events and manages their Runs through approval, execution, retry, and cancellation. */
+/** The service that records Events and drives their Runs through approval, execution, retry, and cancellation. */
 export declare class RunEngine extends RunEngine_base {
 }
 /**
- * Provides Run execution and recovery on the supplied database.
- * `database` must supply Drizzle and Effect SQL clients for the same database,
- * so Run records and pending execution remain together across restarts.
- * The deployment supplies `Crypto` and the Worker implementation through `StepExecutor`.
+ * Returns the engine layers on `database`, which stores all durable Run state. Requires `Crypto`
+ * for Run IDs, `StepExecutor` for Workers, and `TriggerCondition` to re-check queued Events.
  */
-export declare const layerWithDatabase: <DatabaseError, DatabaseRequirements>(database: Layer.Layer<RunDatabase | SqlClient.SqlClient, DatabaseError, DatabaseRequirements>) => Layer.Layer<CursorStore | CycleBudgets | ItemProjectionReader | ItemProjectionWriter | LoopControl | RepositoryLabelProjectionReader | RepositoryLabelProjectionWriter | RunEngine | RunReader, RunEngineError | DatabaseError, Crypto.Crypto | StepExecutor | DatabaseRequirements>;
-export { Submission, Claimed, Queued, Ignored, CycleBlocked, IgnoreReason, SubmissionResult, } from "./submission.ts";
+export declare const layerWithDatabase: <DatabaseError, DatabaseRequirements>(database: Layer.Layer<RunDatabase, DatabaseError, DatabaseRequirements>) => Layer.Layer<BoardMoves | CursorStore | CycleBudgets | ItemProjectionReader | ItemProjectionWriter | LoopControl | RepositoryLabelProjectionReader | RepositoryLabelProjectionWriter | RunEngine | RunReader | UnconfirmedStops, RunEngineError | DatabaseError, Crypto.Crypto | StepExecutor | TriggerCondition | DatabaseRequirements>;
+export { Submission, Claimed, Queued, Suppressed, Ignored, CycleBlocked, IgnoreReason, SubmissionResult, } from "./submission.ts";
+export { RunAgainRequest, RunAgainResult } from "./run-again.ts";
+export { RetryRequest } from "./retry.ts";
 export { RunEngineShutdown } from "./shutdown.ts";
-export { RunNotFoundError, RunOperationRejectedError, EnginePersistenceError, AttemptCancellationError, RunInterventionRejectedError, RunStoreError, RunEngineError, } from "./error.ts";
+export { UnconfirmedStop, UnconfirmedStops } from "./unconfirmed-stops.ts";
+export { RunNotFoundError, RunOperationRejectedError, RunRetryBlockedError, EnginePersistenceError, AttemptCancellationError, RunInterventionRejectedError, LeftoversDiscardRejectedError, RunStoreError, RunEngineError, } from "./error.ts";
 //# sourceMappingURL=engine.d.ts.map

package/dist/engine/engine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine/engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AACnG,OAAO,EACL,+BAA+B,EAC/B,+BAA+B,EAChC,MAAM,+DAA+D,CAAC;AAQvE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;;IA4O3D,uJAAuJ;oBACvI,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACzC,+GAA+G;sBAC7F,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACxF,iGAAiG;qBAChF,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACtE,iGAAiG;sBAC/E,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC;IAC9E,8FAA8F;wBAC1E,CAClB,KAAK,EAAE,KAAK,EACZ,SAAS,EAAE,SAAS,EACpB,YAAY,EAAE,YAAY,KACvB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACxC,4GAA4G;wBACxF,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IAC7D,uGAAuG;oBACvF,CACd,KAAK,EAAE,KAAK,EACZ,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,KACtC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACxC,6HAA6H;qBAC5G,CACf,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,OAAO,KACb,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC;IACxD;;;OAGG;qBACc,CAAC,UAAU,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,cAAc,CAAC;IAC5F,wFAAwF;yBACnE,CACnB,KAAK,EAAE,eAAe,KACnB,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC;;AAtCtE,kGAAkG;AAClG,qBAAa,SAAU,SAAQ,cAuCD;CAAG;AAEjC;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,GAAI,aAAa,EAAE,oBAAoB,YACzD,KAAK,CAAC,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,SAAS,EAAE,aAAa,EAAE,oBAAoB,CAAC,KAC5F,KAAK,CAAC,KAAK,CACV,WAAW,GACX,YAAY,GACZ,oBAAoB,GACpB,oBAAoB,GACpB,WAAW,GACX,+BAA+B,GAC/B,+BAA+B,GAC/B,SAAS,GACT,SAAS,EACX,cAAc,GAAG,aAAa,EAC9B,MAAM,CAAC,MAAM,GAAG,YAAY,GAAG,oBAAoB,CA8BpD,CAAC;AAEF,OAAO,EACL,UAAU,EACV,OAAO,EACP,MAAM,EACN,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EACL,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,EACtB,wBAAwB,EACxB,4BAA4B,EAC5B,aAAa,EACb,cAAc,GACf,MAAM,YAAY,CAAC"}
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/engine/engine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAS,MAAM,EAAU,MAAM,QAAQ,CAAC;AAC9F,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAEL,6BAA6B,EAE7B,KAAK,cAAc,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAY,KAAK,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnF,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEzE,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AACnG,OAAO,EACL,+BAA+B,EAC/B,+BAA+B,EAChC,MAAM,+DAA+D,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAG3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,6FAA6F;AAC7F,eAAO,MAAM,iBAAiB,mCAG7B,CAAC;;IAwPE,oIAAoI;oBACpH,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACzC,gIAAgI;sBAC9G,CAChB,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,KAAK,EACf,MAAM,EAAE,cAAc,KACnB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACxC;;;;;OAKG;qBACc,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACtE;;;;;OAKG;+BACwB,CACzB,MAAM,EAAE,UAAU,KACf,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,6BAA6B,CAAC;IAC1E,2FAA2F;sBACzE,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC;IAC9E,4HAA4H;wBACxG,CAClB,KAAK,EAAE,KAAK,EACZ,SAAS,EAAE,SAAS,EACpB,YAAY,EAAE,YAAY,KACvB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACxC;;;;;;OAMG;wBACiB,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IAC7D;;;OAGG;6BACsB,CACvB,KAAK,EAAE,KAAK,EACZ,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,KACnC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IACxC;;;OAGG;oBACa,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,YAAY,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;IAC7F;;;OAGG;uBACgB,CACjB,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,eAAe,KACrB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,cAAc,CAAC;IAClD,4HAA4H;qBAC3G,CACf,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,OAAO,KACb,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC;IACxD;;;OAGG;qBACc,CAAC,UAAU,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,cAAc,CAAC;IAC5F,8EAA8E;yBACzD,CACnB,KAAK,EAAE,eAAe,KACnB,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC;;AA/EtE,kHAAkH;AAClH,qBAAa,SAAU,SAAQ,cAgFD;CAAG;AAEjC;;;GAGG;AACH,eAAO,MAAM,iBAAiB,GAAI,aAAa,EAAE,oBAAoB,YACzD,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,aAAa,EAAE,oBAAoB,CAAC,KACtE,KAAK,CAAC,KAAK,CACV,UAAU,GACV,WAAW,GACX,YAAY,GACZ,oBAAoB,GACpB,oBAAoB,GACpB,WAAW,GACX,+BAA+B,GAC/B,+BAA+B,GAC/B,SAAS,GACT,SAAS,GACT,gBAAgB,EAClB,cAAc,GAAG,aAAa,EAC9B,MAAM,CAAC,MAAM,GAAG,YAAY,GAAG,gBAAgB,GAAG,oBAAoB,CA+BvE,CAAC;AAEF,OAAO,EACL,UAAU,EACV,OAAO,EACP,MAAM,EACN,UAAU,EACV,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EACL,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,EAC7B,aAAa,EACb,cAAc,GACf,MAAM,YAAY,CAAC"}