@usine/executor 0.2.1 → 0.3.0

package/dist/remote/remote.js

@@ -0,0 +1,308 @@
+import { CHILD_KILL_SCRIPT } from "@usine/agents/child-host";
+import { ClaudeLoginLock } from "@usine/agents/runtime";
+import { InfraFailure } from "@usine/core/attempt/failure";
+import { RepositoryName, safeSegment } from "@usine/core/repository";
+import { harvestFacts, harvestPostedComments, } from "@usine/core/step/harvest";
+import { AttemptRuntime } from "@usine/durable/attempt-runtime";
+import { Deferred, Effect, Exit, Option, Schema, Scope } from "effect";
+import { durableSessionFor } from "../durable-session.js";
+import { CheckoutRequest, Executor, ExecutorError, ExecutorOperationError, WorkerCapabilityError, } from "../executor.js";
+import { verifyHarnessReadiness } from "../harness-preflight.js";
+import { keyedLockFor } from "../keyed-lock.js";
+import { withGitHooks } from "../pre-push-hook.js";
+import { cacheDirectoryName, provisionWorktree, reattachWorktree, recoverWorktree, } from "../provision.js";
+import { workerLeftoversFor } from "./leftovers.js";
+import { workerWatchesFor } from "./liveness.js";
+import { abandonedWorktreeSweepFor } from "./sweep.js";
+const isExecutorError = Schema.is(ExecutorError);
+const executorError = (message, cause) => new ExecutorOperationError({ operation: "prepare", message, cause });
+/** A script that writes `$2` to the file `$1`, creating its directory, and makes it executable. */
+const WRITE_EXECUTABLE_SCRIPT = String.raw `
+set -e
+mkdir -p "$(dirname "$1")"
+printf '%s' "$2" > "$1"
+chmod 755 "$1"
+`.trim();
+/**
+ * Returns the executor for Workers accessed over SSH. Each Attempt's Pi parent is a durable task
+ * in this process, and each child runs on the machine as one SSH command with a pid file next
+ * to the worktree. `worker` provides the machine, preflight, and repository access.
+ */
+export const remoteExecutorFor = (worker) => Effect.gen(function* () {
+    const { commands, label } = worker;
+    const span = (name) => `${worker.spanPrefix}.${name}`;
+    const fetches = yield* keyedLockFor();
+    const logins = yield* ClaudeLoginLock;
+    const sweepAbandonedWorktrees = yield* abandonedWorktreeSweepFor(commands);
+    const workerLeftovers = yield* workerLeftoversFor(commands, worker.machineFor, worker.reclaim);
+    const session = yield* durableSessionFor({ interruptGrace: worker.interruptGrace });
+    const runtime = yield* AttemptRuntime;
+    const workerWatches = yield* workerWatchesFor(commands);
+    const stopChildren = Effect.fn(span("stopChildren"))(function* (operation, machine, worktree) {
+        const host = machine.shell.destination;
+        yield* Effect.annotateCurrentSpan({ host, worktree });
+        const outcome = yield* commands.run(operation, machine.shell, "sh", [
+            "-c",
+            CHILD_KILL_SCRIPT,
+            "sh",
+            worktree,
+        ]);
+        yield* Effect.logInfo("Confirmed no child of the Attempt runs on the Worker's machine", {
+            host,
+            worktree,
+            children: outcome.trim().split("\n"),
+        });
+    });
+    /** Runs git on the machine with the Attempt's repository access, refreshing the credential first if it's about to expire. */
+    const accessGit = (operation, access) => (args) => access.refresh.pipe(Effect.andThen(commands.run(operation, access.shell, "git", args, {
+        environment: access.gitEnvironment,
+        scrub: access.scrub,
+    })));
+    const harvestCommand = (operation, worktree, access) => (command, args) => access.refresh
+        .pipe(Effect.andThen(commands.run(operation, access.shell, command, args, {
+        cwd: worktree,
+        environment: access.environment,
+        scrub: access.scrub,
+    })))
+        .pipe(Effect.map((stdout) => (stdout === "" ? [] : stdout.replace(/\n$/u, "").split("\n"))));
+    const recover = Effect.fn(span("recover"))(function* (attempt, record) {
+        yield* Effect.annotateCurrentSpan({ ...attempt });
+        yield* session.stopForRecovery(attempt);
+        const { machine, checkouts } = yield* workerLeftovers(attempt.worker);
+        const worktree = `${machine.workerDirectory}/attempts/${safeSegment(attempt.attemptId)}`;
+        const inspectCheckout = Effect.gen(function* () {
+            const inspected = yield* commands.inspect("cleanup", machine.shell, "test", [
+                "-d",
+                worktree,
+            ]);
+            if (inspected.exitCode === 1)
+                return false;
+            if (inspected.exitCode !== 0)
+                return yield* executorError("Can't inspect the recovery checkout", new Error(inspected.stderr));
+            return true;
+        });
+        // Checks the cached listing first to avoid an SSH round trip per persisted Attempt.
+        const present = checkouts.has(worktree) && (yield* inspectCheckout);
+        yield* session.recoverPostedComments(attempt, present
+            ? Option.some((spec, windows) => Effect.scoped(Effect.gen(function* () {
+                const grant = yield* worker.grantFor(attempt.repository);
+                const access = yield* grant.acquire(attempt.attemptId, attempt.worker, attempt.permissions, machine, "cleanup");
+                return yield* harvestPostedComments(spec, harvestCommand("cleanup", worktree, access), windows);
+            })))
+            : Option.none(), record);
+        if (!present)
+            return;
+        const reconcile = (git) => recoverWorktree({
+            attempt,
+            host: machine.shell.destination,
+            worktree,
+            cache: `${machine.workerDirectory}/repos/${cacheDirectoryName(attempt.repository)}`,
+            git,
+        }, record.salvage);
+        if (attempt.succeeded || attempt.permissions === "read") {
+            yield* stopChildren("cleanup", machine, worktree);
+            yield* reconcile((args) => commands.run("cleanup", machine.shell, "git", args));
+            return;
+        }
+        const grant = yield* worker.grantFor(attempt.repository);
+        yield* Effect.scoped(Effect.gen(function* () {
+            const access = yield* grant.acquire(attempt.attemptId, attempt.worker, attempt.permissions, machine, "cleanup");
+            yield* stopChildren("cleanup", machine, worktree);
+            yield* reconcile(accessGit("cleanup", access));
+        }));
+    });
+    const harvest = (workspace, access, windows) => harvestFacts(workspace.spec, harvestCommand("launch", workspace.worktree, access), windows).pipe(Effect.mapError((error) => new InfraFailure({
+        message: `Can't harvest the attempt's checkout: ${error.message}`,
+        cause: error,
+    })));
+    /**
+     * Checks that a reattached Attempt's parent task was created for this worktree and the
+     * worktree still exists, then stops children left from before the restart. Returns the
+     * parent's RunSpec.
+     */
+    const takeOver = Effect.fn(span("takeOver"))(function* (request, machine, worktree) {
+        const { cwd, spec } = yield* session.storedRequest(request.attemptId);
+        if (cwd !== worktree)
+            return yield* executorError(`The reattached attempt's checkout isn't on the machine of this ${label} worker`, new Error(cwd));
+        const inspected = yield* commands.inspect("prepare", machine.shell, "test", ["-d", worktree]);
+        if (inspected.exitCode === 1)
+            return yield* executorError("The reattached attempt's checkout no longer exists", new Error(worktree));
+        if (inspected.exitCode !== 0)
+            return yield* executorError("Can't inspect the reattached attempt's checkout", new Error(inspected.stderr.trim()));
+        yield* stopChildren("prepare", machine, worktree);
+        return spec;
+    });
+    /** Returns the `WorkspaceOps` for the machine; git runs with the Attempt's repository access. */
+    const workspaceOps = (request, machine, access, layout) => ({
+        ensureRoots: Effect.asVoid(commands.run("prepare", machine.shell, "mkdir", [
+            "-p",
+            `${machine.workerDirectory}/repos`,
+            `${machine.workerDirectory}/attempts`,
+        ])),
+        removePath: (operation, target) => Effect.asVoid(commands.run(operation, machine.shell, "rm", ["-rf", target])),
+        cacheExists: Effect.gen(function* () {
+            const inspected = yield* commands.inspect("prepare", machine.shell, "test", [
+                "-d",
+                layout.cache,
+            ]);
+            if (inspected.exitCode === 0)
+                return true;
+            if (inspected.exitCode === 1)
+                return false;
+            return yield* executorError(`Can't inspect the cached clone on worker "${request.worker}"`, new Error(inspected.stderr.trim()));
+        }),
+        promoteStaging: Effect.asVoid(commands.run("prepare", machine.shell, "mv", [layout.staging, layout.cache])),
+        git: (operation, args) => accessGit(operation, access)(args),
+        writeExecutable: (target, content) => Effect.asVoid(commands.run("prepare", machine.shell, "sh", [
+            "-c",
+            WRITE_EXECUTABLE_SCRIPT,
+            "sh",
+            target,
+            content,
+        ])),
+    });
+    const prepare = Effect.fn(span("prepare"))(function* (request) {
+        yield* Effect.annotateCurrentSpan({
+            attemptId: request.attemptId,
+            worker: request.worker,
+            repository: request.repository,
+            loop: request.loop,
+            permissions: request.permissions,
+            reattach: request.reattach,
+        });
+        if (request.permissions === "read" && Option.isSome(worker.readOnlyAccessGap)) {
+            const message = `${label} worker "${request.worker}" can't run loop "${request.loop}", which has permissions: read. ${worker.readOnlyAccessGap.value}.`;
+            yield* Effect.logWarning("Refused an Attempt that needs read-only repository access", {
+                attemptId: request.attemptId,
+                worker: request.worker,
+                loop: request.loop,
+                permissions: request.permissions,
+                message,
+            });
+            return yield* new WorkerCapabilityError({
+                worker: request.worker,
+                capability: "credentials",
+                message,
+                cause: new Error(worker.readOnlyAccessGap.value),
+            });
+        }
+        const grant = yield* worker.grantFor(request.repository);
+        const machine = yield* worker.machineFor(request.worker);
+        yield* verifyHarnessReadiness(request.worker, [request.agent.brain], worker.harnessProbe(machine), logins);
+        const host = machine.shell.destination;
+        const cache = `${machine.workerDirectory}/repos/${cacheDirectoryName(request.repository)}`;
+        const worktree = `${machine.workerDirectory}/attempts/${safeSegment(request.attemptId)}`;
+        const staging = `${cache}.cloning`;
+        yield* Effect.annotateCurrentSpan({ host });
+        yield* Effect.logInfo(request.reattach
+            ? `Reattaching the Attempt to its ${label} checkout`
+            : `Preparing ${label} Attempt workspace`, {
+            attemptId: request.attemptId,
+            worker: request.worker,
+            host,
+            repository: request.repository,
+            loop: request.loop,
+            permissions: request.permissions,
+            worktree,
+        });
+        yield* sweepAbandonedWorktrees(request.attemptId, request.worker, machine.shell, machine.workerDirectory);
+        // A reattachment reuses its checkout before any credential is granted.
+        const reattached = request.reattach
+            ? Option.some(yield* takeOver(request, machine, worktree))
+            : Option.none();
+        // A checkout kept after a failed remote stop never closes its scope, so its salvage result
+        // is completed here instead of by the checkout finalizer.
+        const salvaged = yield* Deferred.make();
+        const releaseWorkspace = Effect.fn(span("releaseWorkspace"))(function* (resource, exit) {
+            yield* Effect.annotateCurrentSpan({
+                attemptId: request.attemptId,
+                worker: request.worker,
+                host,
+            });
+            // Closes the checkout scope only if stopping the remote children succeeds.
+            yield* stopChildren("cleanup", machine, resource.provisioned.workspace.worktree);
+            yield* Scope.close(resource.scope, exit);
+            const salvage = yield* Effect.exit(resource.provisioned.salvaged);
+            yield* Deferred.done(salvaged, salvage);
+        }, (effect) => effect.pipe(Effect.tapError((error) => Effect.logError("Remote termination failed; Attempt checkout retained for recovery", {
+            attemptId: request.attemptId,
+            worker: request.worker,
+            host,
+            worktree,
+            message: error.message,
+        })), Effect.ignore, Effect.andThen(Deferred.succeed(salvaged, Option.none()))));
+        const acquire = Effect.gen(function* () {
+            // Credentials are scoped to the Attempt, not the checkout, so they are released exactly once
+            // even when checkout cleanup is skipped.
+            const access = yield* grant.acquire(request.attemptId, request.worker, request.permissions, machine, "prepare");
+            const ops = workspaceOps(request, machine, access, { cache, staging });
+            const checkout = Option.match(reattached, {
+                onNone: () => provisionWorktree({
+                    request,
+                    url: worker.repositoryUrl(machine, request.repository),
+                    host,
+                    layout: { cache, staging, worktree },
+                    ops,
+                }),
+                onSome: (spec) => reattachWorktree({ request, host, layout: { cache, worktree }, ops, spec }),
+            });
+            const { provisioned } = yield* Effect.acquireRelease(Effect.uninterruptibleMask((restore) => Effect.gen(function* () {
+                const scope = yield* Scope.make();
+                // No child can exist before preparation succeeds, so a partial checkout is always rolled
+                // back; only a completed workspace uses the guarded release above.
+                const provisioned = yield* restore(checkout).pipe(Scope.provide(scope), Effect.onError((cause) => Scope.close(scope, Exit.failCause(cause))));
+                return { provisioned, scope };
+            })), releaseWorkspace, { interruptible: true });
+            return { provisioned, access };
+        });
+        const { provisioned, access } = yield* (request.reattach ? acquire : fetches.withLock(request.repository)(acquire)).pipe(Effect.mapError((cause) => isExecutorError(cause)
+            ? cause
+            : executorError(`Can't prepare the ${label} attempt workspace`, cause)));
+        const prepared = provisioned.workspace;
+        yield* Effect.logInfo(`${label} Attempt workspace ready`, {
+            attemptId: request.attemptId,
+            worker: request.worker,
+            host,
+            repository: request.repository,
+            baseCommit: prepared.spec.baseCommit,
+            resume: Option.map(prepared.spec.resume, ({ ref }) => ref),
+            reattach: request.reattach,
+            worktree,
+        });
+        return {
+            workspace: prepared,
+            launch: (io) => Effect.scoped(Effect.gen(function* () {
+                // Uses the RunSpec's brain, including on reattachment. Each child reads the kind when it
+                // starts, so a preflight during the Attempt takes effect.
+                const gatewayCredential = worker.gatewayCredential(machine, prepared.spec.agent.brain.harness);
+                yield* workerWatches.watch({
+                    attemptId: request.attemptId,
+                    worker: request.worker,
+                    shell: machine.shell,
+                    report: (notice) => runtime.notify(request.attemptId, notice),
+                });
+                return yield* session.launch(prepared, io, {
+                    host: access.shell,
+                    environment: withGitHooks(access.environment, provisioned.gitHooks),
+                    gatewayCredential,
+                    harvest: (windows) => harvest(prepared, access, windows),
+                    harvestComments: (windows) => harvestPostedComments(prepared.spec, harvestCommand("launch", prepared.worktree, access), windows),
+                    keepForReattach: provisioned.keepForReattach,
+                });
+            })),
+            salvaged: Deferred.await(salvaged),
+            recordSalvage: provisioned.recordSalvage,
+        };
+    }, session.prepare);
+    return Executor.of({
+        kind: worker.kind,
+        recover,
+        cancel: session.cancel,
+        intervene: session.intervene,
+        isLive: session.isLive,
+        outcome: session.outcome,
+        preflight: worker.preflight,
+        prepare,
+    });
+});
+//# sourceMappingURL=remote.js.map

package/dist/remote/remote.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote.js","sourceRoot":"","sources":["../../src/remote/remote.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAuB,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAGhF,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAEL,YAAY,EACZ,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAA8B,MAAM,uBAAuB,CAAC;AACtF,OAAO,EAEL,eAAe,EACf,QAAQ,EACR,aAAa,EAEb,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EACL,kBAAkB,EAElB,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,GAEhB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjD,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,eAAe,GAAG,MAAM,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC;AAEjD,MAAM,aAAa,GAAG,CAAC,OAAe,EAAE,KAAc,EAA0B,EAAE,CAChF,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAEvE,mGAAmG;AACnG,MAAM,uBAAuB,GAAG,MAAM,CAAC,GAAG,CAAA;;;;;CAKzC,CAAC,IAAI,EAAE,CAAC;AAET;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAA0B,MAAuB,EAAE,EAAE,CACpF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACnC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;IAC9D,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,YAAY,EAAkB,CAAC;IACtD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC;IACtC,MAAM,uBAAuB,GAAG,KAAK,CAAC,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;IAC3E,MAAM,eAAe,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC;IACtC,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAExD,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC5D,SAA4B,EAC5B,OAAU,EACV,QAAgB;QAEhB,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;QACvC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE;YAClE,IAAI;YACJ,iBAAiB;YACjB,IAAI;YACJ,QAAQ;SACT,CAAC,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,gEAAgE,EAAE;YACtF,IAAI;YACJ,QAAQ;YACR,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SACrC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6HAA6H;IAC7H,MAAM,SAAS,GACb,CAAC,SAA4B,EAAE,MAAoB,EAAE,EAAE,CAAC,CAAC,IAA2B,EAAE,EAAE,CACtF,MAAM,CAAC,OAAO,CAAC,IAAI,CACjB,MAAM,CAAC,OAAO,CACZ,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE;QACjD,WAAW,EAAE,MAAM,CAAC,cAAc;QAClC,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC,CACH,CACF,CAAC;IAEN,MAAM,cAAc,GAClB,CAAC,SAA4B,EAAE,QAAgB,EAAE,MAAoB,EAAE,EAAE,CACzE,CAAC,OAAqB,EAAE,IAA2B,EAAE,EAAE,CACrD,MAAM,CAAC,OAAO;SACX,IAAI,CACH,MAAM,CAAC,OAAO,CACZ,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE;QACnD,GAAG,EAAE,QAAQ;QACb,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC,CACH,CACF;SACA,IAAI,CACH,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CACtF,CAAC;IAER,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAClD,OAAwB,EACxB,MAAuB;QAEvB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAClD,KAAK,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,eAAe,aAAa,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACzF,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE;gBAC1E,IAAI;gBACJ,QAAQ;aACT,CAAC,CAAC;YACH,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;gBAC1B,OAAO,KAAK,CAAC,CAAC,aAAa,CACzB,qCAAqC,EACrC,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAC5B,CAAC;YACJ,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,oFAAoF;QACpF,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC;QACpE,KAAK,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAClC,OAAO,EACP,OAAO;YACL,CAAC,CAAC,MAAM,CAAC,IAAI,CAAwB,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CACnD,MAAM,CAAC,MAAM,CACX,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACzD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CACjC,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,WAAW,EACnB,OAAO,EACP,SAAS,CACV,CAAC;gBACF,OAAO,KAAK,CAAC,CAAC,qBAAqB,CACjC,IAAI,EACJ,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,EAC3C,OAAO,CACR,CAAC;YACJ,CAAC,CAAC,CACH,CACF;YACH,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EACjB,MAAM,CACP,CAAC;QACF,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,MAAM,SAAS,GAAG,CAChB,GAA0E,EAC1E,EAAE,CACF,eAAe,CACb;YACE,OAAO;YACP,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,WAAW;YAC/B,QAAQ;YACR,KAAK,EAAE,GAAG,OAAO,CAAC,eAAe,UAAU,kBAAkB,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YACnF,GAAG;SACJ,EACD,MAAM,CAAC,OAAO,CACf,CAAC;QACJ,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YACxD,KAAK,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAClD,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzD,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAClB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CACjC,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,WAAW,EACnB,OAAO,EACP,SAAS,CACV,CAAC;YACF,KAAK,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAClD,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,CACd,SAA2B,EAC3B,MAAoB,EACpB,OAA2C,EAC3C,EAAE,CACF,YAAY,CACV,SAAS,CAAC,IAAI,EACd,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,EACpD,OAAO,CACR,CAAC,IAAI,CACJ,MAAM,CAAC,QAAQ,CACb,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,YAAY,CAAC;QACf,OAAO,EAAE,yCAAyC,KAAK,CAAC,OAAO,EAAE;QACjE,KAAK,EAAE,KAAK;KACb,CAAC,CACL,CACF,CAAC;IAEJ;;;;OAIG;IACH,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EACpD,OAAwB,EACxB,OAAU,EACV,QAAgB;QAEhB,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACtE,IAAI,GAAG,KAAK,QAAQ;YAClB,OAAO,KAAK,CAAC,CAAC,aAAa,CACzB,kEAAkE,KAAK,SAAS,EAChF,IAAI,KAAK,CAAC,GAAG,CAAC,CACf,CAAC;QACJ,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC9F,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,aAAa,CACzB,oDAAoD,EACpD,IAAI,KAAK,CAAC,QAAQ,CAAC,CACpB,CAAC;QACJ,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;YAC1B,OAAO,KAAK,CAAC,CAAC,aAAa,CACzB,iDAAiD,EACjD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CACnC,CAAC;QACJ,KAAK,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,iGAAiG;IACjG,MAAM,YAAY,GAAG,CACnB,OAAwB,EACxB,OAAU,EACV,MAAoB,EACpB,MAA4D,EAC/B,EAAE,CAAC,CAAC;QACjC,WAAW,EAAE,MAAM,CAAC,MAAM,CACxB,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE;YAC9C,IAAI;YACJ,GAAG,OAAO,CAAC,eAAe,QAAQ;YAClC,GAAG,OAAO,CAAC,eAAe,WAAW;SACtC,CAAC,CACH;QACD,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,CAChC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9E,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE;gBAC1E,IAAI;gBACJ,MAAM,CAAC,KAAK;aACb,CAAC,CAAC;YACH,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,OAAO,KAAK,CAAC,CAAC,aAAa,CACzB,6CAA6C,OAAO,CAAC,MAAM,GAAG,EAC9D,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CACnC,CAAC;QACJ,CAAC,CAAC;QACF,cAAc,EAAE,MAAM,CAAC,MAAM,CAC3B,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAC7E;QACD,GAAG,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;QAC5D,eAAe,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CACnC,MAAM,CAAC,MAAM,CACX,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE;YAC3C,IAAI;YACJ,uBAAuB;YACvB,IAAI;YACJ,MAAM;YACN,OAAO;SACR,CAAC,CACH;KACJ,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAwB;QAC5E,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC;YAChC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,WAAW,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC9E,MAAM,OAAO,GAAG,GAAG,KAAK,YAAY,OAAO,CAAC,MAAM,qBAAqB,OAAO,CAAC,IAAI,mCAAmC,MAAM,CAAC,iBAAiB,CAAC,KAAK,GAAG,CAAC;YACxJ,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,2DAA2D,EAAE;gBACpF,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO;aACR,CAAC,CAAC;YACH,OAAO,KAAK,CAAC,CAAC,IAAI,qBAAqB,CAAC;gBACtC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,UAAU,EAAE,aAAa;gBACzB,OAAO;gBACP,KAAK,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,CAAC;aACjD,CAAC,CAAC;QACL,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzD,KAAK,CAAC,CAAC,sBAAsB,CAC3B,OAAO,CAAC,MAAM,EACd,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EACrB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,EAC5B,MAAM,CACP,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;QACvC,MAAM,KAAK,GAAG,GAAG,OAAO,CAAC,eAAe,UAAU,kBAAkB,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3F,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,eAAe,aAAa,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACzF,MAAM,OAAO,GAAG,GAAG,KAAK,UAAU,CAAC;QACnC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,OAAO,CAAC,QAAQ;YACd,CAAC,CAAC,kCAAkC,KAAK,WAAW;YACpD,CAAC,CAAC,aAAa,KAAK,oBAAoB,EAC1C;YACE,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI;YACJ,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,QAAQ;SACT,CACF,CAAC;QACF,KAAK,CAAC,CAAC,uBAAuB,CAC5B,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,eAAe,CACxB,CAAC;QACF,uEAAuE;QACvE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ;YACjC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC1D,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAElB,2FAA2F;QAC3F,0DAA0D;QAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAiD,CAAC;QACvF,MAAM,gBAAgB,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAC1D,QAAQ,CAAC,EACP,QAAwF,EACxF,IAAiC;YAEjC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC;gBAChC,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,IAAI;aACL,CAAC,CAAC;YACH,2EAA2E;YAC3E,KAAK,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACjF,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAClE,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC,EACD,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,QAAQ,CAAC,mEAAmE,EAAE;YACnF,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI;YACJ,QAAQ;YACR,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC,CACH,EACD,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAC1D,CACJ,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClC,6FAA6F;YAC7F,yCAAyC;YACzC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CACjC,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,WAAW,EACnB,OAAO,EACP,SAAS,CACV,CAAC;YACF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE;gBACxC,MAAM,EAAE,GAAG,EAAE,CACX,iBAAiB,CAAC;oBAChB,OAAO;oBACP,GAAG,EAAE,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC;oBACtD,IAAI;oBACJ,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE;oBACpC,GAAG;iBACJ,CAAC;gBACJ,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CACf,gBAAgB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;aAC9E,CAAC,CAAC;YACH,MAAM,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,cAAc,CAClD,MAAM,CAAC,mBAAmB,CAAC,CAAC,OAAO,EAAE,EAAE,CACrC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAClC,yFAAyF;gBACzF,mEAAmE;gBACnE,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAC/C,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EACpB,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CACrE,CAAC;gBACF,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;YAChC,CAAC,CAAC,CACH,EACD,gBAAgB,EAChB,EAAE,aAAa,EAAE,IAAI,EAAE,CACxB,CAAC;YACF,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CACrC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAC3E,CAAC,IAAI,CACJ,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,eAAe,CAAC,KAAK,CAAC;YACpB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,aAAa,CAAC,qBAAqB,KAAK,oBAAoB,EAAE,KAAK,CAAC,CACzE,CACF,CAAC;QAEF,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC;QACvC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,0BAA0B,EAAE;YACxD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI;YACJ,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU;YACpC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC;YAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,QAAQ;SACT,CAAC,CAAC;QACH,OAAO;YACL,SAAS,EAAE,QAAQ;YACnB,MAAM,EAAE,CAAC,EAAuB,EAAE,EAAE,CAClC,MAAM,CAAC,MAAM,CACX,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClB,yFAAyF;gBACzF,0DAA0D;gBAC1D,MAAM,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAChD,OAAO,EACP,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAClC,CAAC;gBACF,KAAK,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;oBACzB,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;iBAC9D,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE;oBACzC,IAAI,EAAE,MAAM,CAAC,KAAK;oBAClB,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC;oBACnE,iBAAiB;oBACjB,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;oBACxD,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAC3B,qBAAqB,CACnB,QAAQ,CAAC,IAAI,EACb,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,EACnD,OAAO,CACR;oBACH,eAAe,EAAE,WAAW,CAAC,eAAe;iBAC7C,CAAC,CAAC;YACL,CAAC,CAAC,CACH;YACH,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;YAClC,aAAa,EAAE,WAAW,CAAC,aAAa;SACzC,CAAC;IACJ,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB,OAAO,QAAQ,CAAC,EAAE,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,OAAO;QACP,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO;KACR,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/remote/sweep.d.ts

@@ -0,0 +1,15 @@
+import type { RemoteShell } from "@usine/agents/child-host";
+import { Effect } from "effect";
+import { ExecutorOperationError } from "../executor.ts";
+import type { RemoteCommands } from "./commands.ts";
+/**
+ * Lists paths directly under a Worker's `attempts` directory, creating it first so a new Worker
+ * lists as empty. Verifies that each path is a direct child of `attemptsRoot`.
+ */
+export declare const attemptPathsOn: (commands: RemoteCommands, operation: "cancel" | "cleanup" | "launch" | "preflight" | "prepare", shell: RemoteShell, attemptsRoot: string) => Effect.Effect<readonly string[], ExecutorOperationError, never>;
+/**
+ * Returns a sweep that prunes stale cache registrations once per Worker and keeps crash
+ * leftovers. Only durable Attempt recovery removes an existing checkout.
+ */
+export declare const abandonedWorktreeSweepFor: (commands: RemoteCommands) => Effect.Effect<(attemptId: string & import("effect/Brand").Brand<"AttemptId">, worker: string & import("effect/Brand").Brand<"WorkerName">, shell: RemoteShell, workerDirectory: string) => Effect.Effect<void, never, never>, never, never>;
+//# sourceMappingURL=sweep.d.ts.map

package/dist/remote/sweep.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../../src/remote/sweep.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAG5D,OAAO,EAAE,MAAM,EAAe,MAAM,QAAQ,CAAC;AAC7C,OAAO,EAA0B,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAyCpD;;;GAGG;AACH,eAAO,MAAM,cAAc,+MAiBzB,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,yBAAyB,2QA0GrC,CAAC"}

package/dist/remote/sweep.js

@@ -0,0 +1,116 @@
+import { AttemptId } from "@usine/core/attempt";
+import { WorkerName } from "@usine/core/worker";
+import { Effect, Ref, Schema } from "effect";
+import { ExecutorOperationError } from "../executor.js";
+import { keyedLockFor } from "../keyed-lock.js";
+const SWEEP_CONCURRENCY = 4;
+const directChildSchema = (parent) => {
+    const prefix = `${parent}/`;
+    return Schema.String.check(Schema.makeFilter((value) => {
+        const segment = value.slice(prefix.length);
+        return (value.startsWith(prefix) &&
+            segment.length > 0 &&
+            segment !== "." &&
+            segment !== ".." &&
+            !segment.includes("/"));
+    }, { message: `Expected a direct child of ${parent}` }));
+};
+const directChildren = (operation, parent, output) => Schema.decodeUnknownEffect(Schema.Array(directChildSchema(parent)))(output.split("\0").filter((value) => value.length > 0)).pipe(Effect.mapError((cause) => new ExecutorOperationError({
+    operation,
+    message: `Worker returned an invalid path while listing ${parent}`,
+    cause,
+})));
+/**
+ * Lists paths directly under a Worker's `attempts` directory, creating it first so a new Worker
+ * lists as empty. Verifies that each path is a direct child of `attemptsRoot`.
+ */
+export const attemptPathsOn = Effect.fn("RemoteExecutor.attemptPathsOn")(function* (commands, operation, shell, attemptsRoot) {
+    yield* Effect.annotateCurrentSpan({ host: shell.destination, operation, attemptsRoot });
+    yield* commands.run(operation, shell, "mkdir", ["-p", attemptsRoot]);
+    const output = yield* commands.run(operation, shell, "find", [
+        attemptsRoot,
+        "-mindepth",
+        "1",
+        "-maxdepth",
+        "1",
+        "-print0",
+    ]);
+    return yield* directChildren(operation, attemptsRoot, output);
+});
+/**
+ * Returns a sweep that prunes stale cache registrations once per Worker and keeps crash
+ * leftovers. Only durable Attempt recovery removes an existing checkout.
+ */
+export const abandonedWorktreeSweepFor = Effect.fn("RemoteExecutor.abandonedWorktreeSweepFor")(function* (commands) {
+    const locks = yield* keyedLockFor();
+    const swept = yield* Ref.make(new Set());
+    const sweep = Effect.fn("RemoteExecutor.sweepAbandonedWorktrees")(function* (attemptId, worker, shell, workerDirectory) {
+        const host = shell.destination;
+        yield* Effect.annotateCurrentSpan({ attemptId, worker, host });
+        yield* locks.withLock(worker)(Effect.gen(function* () {
+            if ((yield* Ref.get(swept)).has(worker)) {
+                yield* Effect.logDebug("Remote abandoned-worktree sweep was already complete", {
+                    attemptId,
+                    worker,
+                    host,
+                });
+                return;
+            }
+            // Marked done before running so a failed sweep is not retried before every prepare; the
+            // next process start retries it.
+            yield* Ref.update(swept, (current) => new Set(current).add(worker));
+            const attemptsRoot = `${workerDirectory}/attempts`;
+            const reposRoot = `${workerDirectory}/repos`;
+            yield* Effect.logInfo("Sweeping abandoned Attempt worktrees on the remote Worker", {
+                attemptId,
+                worker,
+                host,
+            });
+            yield* commands.run("prepare", shell, "mkdir", ["-p", reposRoot]);
+            const attempts = yield* attemptPathsOn(commands, "prepare", shell, attemptsRoot);
+            // An unrecognized path can contain unsalvaged work from a crash; only recovery from the
+            // durable Attempt record removes it.
+            yield* Effect.forEach(attempts, (worktree) => Effect.logWarning("Preserving abandoned remote path for salvage reconciliation", {
+                attemptId,
+                worker,
+                host,
+                worktree,
+            }), { discard: true });
+            const caches = yield* commands
+                .run("prepare", shell, "find", [
+                reposRoot,
+                "-mindepth",
+                "1",
+                "-maxdepth",
+                "1",
+                "-type",
+                "d",
+                "-name",
+                "*.git",
+                "-print0",
+            ])
+                .pipe(Effect.flatMap((output) => directChildren("prepare", reposRoot, output)));
+            yield* Effect.forEach(caches, (cache) => commands.run("prepare", shell, "git", ["--git-dir", cache, "worktree", "prune"]).pipe(Effect.tapError((error) => Effect.logWarning("Could not prune a remote cache during the sweep", {
+                attemptId,
+                worker,
+                host,
+                cache,
+                message: error.message,
+            })), Effect.ignore), { concurrency: SWEEP_CONCURRENCY, discard: true });
+            yield* Effect.logInfo("Remote abandoned-worktree sweep completed", {
+                attemptId,
+                worker,
+                host,
+                retained: attempts.length,
+                caches: caches.length,
+            });
+        }).pipe(Effect.tapError((error) => Effect.logWarning("Remote abandoned-worktree sweep failed; continuing", {
+            attemptId,
+            worker,
+            host,
+            message: error.message,
+        })), Effect.ignore));
+    });
+    return sweep;
+});
+//# sourceMappingURL=sweep.js.map

package/dist/remote/sweep.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweep.js","sourceRoot":"","sources":["../../src/remote/sweep.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC7C,OAAO,EAA0B,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGhD,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,iBAAiB,GAAG,CAAC,MAAc,EAAwB,EAAE;IACjE,MAAM,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC;IAC5B,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CACxB,MAAM,CAAC,UAAU,CACf,CAAC,KAAK,EAAE,EAAE;QACR,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CACL,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;YACxB,OAAO,CAAC,MAAM,GAAG,CAAC;YAClB,OAAO,KAAK,GAAG;YACf,OAAO,KAAK,IAAI;YAChB,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CACvB,CAAC;IACJ,CAAC,EACD,EAAE,OAAO,EAAE,8BAA8B,MAAM,EAAE,EAAE,CACpD,CACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CACrB,SAA4B,EAC5B,MAAc,EACd,MAAc,EACgD,EAAE,CAChE,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CACjE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CACvD,CAAC,IAAI,CACJ,MAAM,CAAC,QAAQ,CACb,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,sBAAsB,CAAC;IACzB,SAAS;IACT,OAAO,EAAE,iDAAiD,MAAM,EAAE;IAClE,KAAK;CACN,CAAC,CACL,CACF,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC,EAAE,CAAC,+BAA+B,CAAC,CAAC,QAAQ,CAAC,EAChF,QAAwB,EACxB,SAA4B,EAC5B,KAAkB,EAClB,YAAoB;IAEpB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;IACxF,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;QAC3D,YAAY;QACZ,WAAW;QACX,GAAG;QACH,WAAW;QACX,GAAG;QACH,SAAS;KACV,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AAChE,CAAC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC,EAAE,CAAC,0CAA0C,CAAC,CAC5F,QAAQ,CAAC,EAAE,QAAwB;IACjC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,YAAY,EAAc,CAAC;IAChD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,EAAc,CAAC,CAAC;IAErD,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,wCAAwC,CAAC,CAAC,QAAQ,CAAC,EACzE,SAAoB,EACpB,MAAkB,EAClB,KAAkB,EAClB,eAAuB;QAEvB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC;QAC/B,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC3B,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,sDAAsD,EAAE;oBAC7E,SAAS;oBACT,MAAM;oBACN,IAAI;iBACL,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,wFAAwF;YACxF,iCAAiC;YACjC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,MAAM,YAAY,GAAG,GAAG,eAAe,WAAW,CAAC;YACnD,MAAM,SAAS,GAAG,GAAG,eAAe,QAAQ,CAAC;YAC7C,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,2DAA2D,EAAE;gBACjF,SAAS;gBACT,MAAM;gBACN,IAAI;aACL,CAAC,CAAC;YACH,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YAElE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;YACjF,wFAAwF;YACxF,qCAAqC;YACrC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,QAAQ,EACR,CAAC,QAAQ,EAAE,EAAE,CACX,MAAM,CAAC,UAAU,CAAC,6DAA6D,EAAE;gBAC/E,SAAS;gBACT,MAAM;gBACN,IAAI;gBACJ,QAAQ;aACT,CAAC,EACJ,EAAE,OAAO,EAAE,IAAI,EAAE,CAClB,CAAC;YAEF,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ;iBAC3B,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;gBAC7B,SAAS;gBACT,WAAW;gBACX,GAAG;gBACH,WAAW;gBACX,GAAG;gBACH,OAAO;gBACP,GAAG;gBACH,OAAO;gBACP,OAAO;gBACP,SAAS;aACV,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;YAClF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,MAAM,EACN,CAAC,KAAK,EAAE,EAAE,CACR,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CACnF,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,UAAU,CAAC,iDAAiD,EAAE;gBACnE,SAAS;gBACT,MAAM;gBACN,IAAI;gBACJ,KAAK;gBACL,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC,CACH,EACD,MAAM,CAAC,MAAM,CACd,EACH,EAAE,WAAW,EAAE,iBAAiB,EAAE,OAAO,EAAE,IAAI,EAAE,CAClD,CAAC;YAEF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,2CAA2C,EAAE;gBACjE,SAAS;gBACT,MAAM;gBACN,IAAI;gBACJ,QAAQ,EAAE,QAAQ,CAAC,MAAM;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;aACtB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,UAAU,CAAC,oDAAoD,EAAE;YACtE,SAAS;YACT,MAAM;YACN,IAAI;YACJ,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC,CACH,EACD,MAAM,CAAC,MAAM,CACd,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC,CACF,CAAC"}

package/dist/renewable-credential.d.ts

@@ -0,0 +1,17 @@
+import type { AttemptId } from "@usine/core/attempt";
+import type { LoopPermissions } from "@usine/core/loop";
+import type { RepositoryName } from "@usine/core/repository";
+import { Effect } from "effect";
+import type { CredentialUnavailableError, WorkerCredential } from "./executor.ts";
+/**
+ * Issues the Attempt's single-repository token with its Loop `permissions`, passes it to
+ * `publish`, and publishes a new one before each expiry until the caller's scope closes. Then
+ * wipes every issued token, so `Redacted.value` throws. Register cleanup for what `publish`
+ * writes before calling this, so the cleanup runs after renewal stops.
+ */
+export declare const renewableCredential: <A, E>(issue: (repository: RepositoryName, attemptId: AttemptId, permissions: LoopPermissions) => Effect.Effect<WorkerCredential, CredentialUnavailableError>, repository: string & import("effect/Brand").Brand<"RepositoryName">, attemptId: string & import("effect/Brand").Brand<"AttemptId">, permissions: "read" | "write", publish: (credential: WorkerCredential) => Effect.Effect<A, E>) => Effect.Effect<{
+    published: A;
+    refresh: () => Effect.Effect<void, E | CredentialUnavailableError, never>;
+    scrub: (text: string) => string;
+}, E | CredentialUnavailableError, import("effect/Scope").Scope>;
+//# sourceMappingURL=renewable-credential.d.ts.map

package/dist/renewable-credential.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"renewable-credential.d.ts","sourceRoot":"","sources":["../src/renewable-credential.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAmB,MAAM,EAA6B,MAAM,QAAQ,CAAC;AAC5E,OAAO,KAAK,EAEV,0BAA0B,EAC1B,gBAAgB,EACjB,MAAM,eAAe,CAAC;AAcvB;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,GAAwD,CAAC,EAAE,CAAC,mVAKpE,gBAAgB,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;;;kBAoD9C,MAAM,KAAG,MAAM;gEAK/B,CAAC"}

package/dist/renewable-credential.js

@@ -0,0 +1,58 @@
+import { Clock, DateTime, Effect, Redacted, SynchronizedRef } from "effect";
+/** The remaining lifetime, in milliseconds, below which a new token is issued. */
+const RENEWAL_WINDOW_MILLIS = 5 * 60_000;
+/**
+ * Issues the Attempt's single-repository token with its Loop `permissions`, passes it to
+ * `publish`, and publishes a new one before each expiry until the caller's scope closes. Then
+ * wipes every issued token, so `Redacted.value` throws. Register cleanup for what `publish`
+ * writes before calling this, so the cleanup runs after renewal stops.
+ */
+export const renewableCredential = Effect.fn("Executor.renewableCredential")(function* (issue, repository, attemptId, permissions, publish) {
+    yield* Effect.annotateCurrentSpan({ repository, attemptId, permissions });
+    const issued = new Set();
+    yield* Effect.addFinalizer(() => Effect.sync(() => {
+        for (const token of issued)
+            Redacted.wipeUnsafe(token);
+    }));
+    const publishFresh = Effect.fn("Executor.publishCredential")(function* () {
+        yield* Effect.annotateCurrentSpan({ repository, attemptId, permissions });
+        const credential = yield* issue(repository, attemptId, permissions);
+        issued.add(credential.value);
+        const published = yield* publish(credential);
+        yield* Effect.logInfo("Published renewed Attempt repository credential", {
+            repository,
+            attemptId,
+            permissions,
+            expiresAt: DateTime.formatIso(credential.expiresAt),
+        });
+        return { expiresAt: credential.expiresAt, published };
+    });
+    const first = yield* publishFresh();
+    const expiry = yield* SynchronizedRef.make(first.expiresAt);
+    const refresh = Effect.fn("Executor.refreshCredential")(function* () {
+        yield* Effect.annotateCurrentSpan({ repository, attemptId });
+        yield* SynchronizedRef.updateEffect(expiry, (expiresAt) => Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis;
+            if (DateTime.toEpochMillis(expiresAt) - now > RENEWAL_WINDOW_MILLIS)
+                return expiresAt;
+            return (yield* publishFresh()).expiresAt;
+        }));
+    });
+    // Forked in a scope opened before the checkout finalizers, so renewal continues through
+    // salvage and stops before the published credential is removed.
+    yield* Effect.forever(Effect.sleep("30 seconds").pipe(Effect.andThen(refresh()), Effect.tapError((error) => Effect.logWarning("Attempt credential renewal failed; will retry", {
+        repository,
+        attemptId,
+        message: String(error),
+    })), Effect.ignore)).pipe(Effect.forkScoped);
+    return {
+        published: first.published,
+        refresh,
+        scrub: (text) => {
+            for (const token of issued)
+                text = text.replaceAll(Redacted.value(token), "[REDACTED]");
+            return text;
+        },
+    };
+});
+//# sourceMappingURL=renewable-credential.js.map

package/dist/renewable-credential.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"renewable-credential.js","sourceRoot":"","sources":["../src/renewable-credential.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAO5E,kFAAkF;AAClF,MAAM,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC;AAWzC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC,EAAE,CAAC,8BAA8B,CAAC,CAAC,QAAQ,CAAC,EACpF,KAA6C,EAC7C,UAA0B,EAC1B,SAAoB,EACpB,WAA4B,EAC5B,OAA8D;IAE9D,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IACpD,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;QACf,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACzD,CAAC,CAAC,CACH,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC;QACpE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QACpE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC7C,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE;YACvE,UAAU;YACV,SAAS;YACT,WAAW;YACX,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC;SACpD,CAAC,CAAC;QACH,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC;IACxD,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,YAAY,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC;QAC/D,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;QAC7D,KAAK,CAAC,CAAC,eAAe,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,CACxD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC;YAC3C,IAAI,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,GAAG,GAAG,qBAAqB;gBAAE,OAAO,SAAS,CAAC;YACtF,OAAO,CAAC,KAAK,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,SAAS,CAAC;QAC3C,CAAC,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,wFAAwF;IACxF,gEAAgE;IAChE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAC7B,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EACzB,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,UAAU,CAAC,+CAA+C,EAAE;QACjE,UAAU;QACV,SAAS;QACT,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;KACvB,CAAC,CACH,EACD,MAAM,CAAC,MAAM,CACd,CACF,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC1B,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO;QACP,KAAK,EAAE,CAAC,IAAY,EAAU,EAAE;YAC9B,KAAK,MAAM,KAAK,IAAI,MAAM;gBAAE,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,CAAC;YACxF,OAAO,IAAI,CAAC;QACd,CAAC;KACkC,CAAC;AACxC,CAAC,CAAC,CAAC"}

package/dist/salvage.d.ts

@@ -0,0 +1,30 @@
+import type { AttemptId } from "@usine/core/attempt";
+import { type SalvageRef, SalvagePushed, SalvageRetained } from "@usine/core/attempt/salvage";
+import { CommitSha } from "@usine/core/repository";
+import type { WorkerName } from "@usine/core/worker";
+import { Effect, Option } from "effect";
+/** The git identity for commits usine creates in an Attempt's worktree. */
+export declare const SALVAGE_GIT_CONFIG: string[];
+export interface SalvageOptions<E extends Error> {
+    readonly attemptId: AttemptId;
+    readonly worker: WorkerName;
+    readonly host: string;
+    readonly worktree: string;
+    readonly baseCommit: Option.Option<CommitSha>;
+    readonly ref: SalvageRef;
+    /** Runs git with the Worker's repository credentials; `salvageWorktree` prepends `-C <worktree>`. */
+    readonly git: (args: ReadonlyArray<string>) => Effect.Effect<string, E>;
+}
+/**
+ * Returns the operator-facing line for the run drawer: the failure's message followed by its
+ * cause's message. For a failed command, the cause is the stderr, which local and SSH executors
+ * scrub of the Attempt's token.
+ */
+export declare const failureMessage: (failure: Error) => string;
+/**
+ * Commits everything in an unsuccessful Attempt's checkout and pushes HEAD to its salvage ref
+ * before the worktree is removed. Skips a clean checkout at its base commit. If the push fails,
+ * keeps the worktree and reports its path. Never fails; skipped and failed stages are logged.
+ */
+export declare const salvageWorktree: <E extends Error>(options: SalvageOptions<E>) => Effect.Effect<Option.Option<SalvagePushed | SalvageRetained>, never, never>;
+//# sourceMappingURL=salvage.d.ts.map

package/dist/salvage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"salvage.d.ts","sourceRoot":"","sources":["../src/salvage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAEL,KAAK,UAAU,EACf,aAAa,EACb,eAAe,EAChB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAkB,MAAM,QAAQ,CAAC;AAGxD,2EAA2E;AAC3E,eAAO,MAAM,kBAAkB,UAO9B,CAAC;AAEF,MAAM,WAAW,cAAc,CAAC,CAAC,SAAS,KAAK;IAC7C,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC9C,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IACzB,qGAAqG;IACrG,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;CACzE;AAID;;;;GAIG;AACH,eAAO,MAAM,cAAc,YAAa,KAAK,KAAG,MAG/C,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,GAAoD,CAAC,SAAS,KAAK,4GA8F7F,CAAC"}