@usine/executor 0.2.1 → 0.3.0

package/dist/remote/commands.js

@@ -0,0 +1,94 @@
+import { RemoteShell, sshArguments, } from "@usine/agents/child-host";
+import { Effect, Redacted, Stream } from "effect";
+import { ChildProcessSpawner } from "effect/unstable/process";
+import { allowlistedEnvironment, collectCommand, requireSuccess, } from "../child-command.js";
+/**
+ * The variables the local SSH client inherits. GitHub and factory credentials in the
+ * orchestrator environment aren't passed to the client or the machine.
+ */
+export const SSH_ENVIRONMENT_ALLOWLIST = [
+    "HOME",
+    "LANG",
+    "LC_ALL",
+    "PATH",
+    "SHELL",
+    "SSH_AGENT_PID",
+    "SSH_AUTH_SOCK",
+    "TMPDIR",
+    "USER",
+];
+/**
+ * The default SSH client for remote Workers. Its keepalive options make a dead TCP connection
+ * exit non-zero instead of hanging.
+ */
+export const DEFAULT_SSH = {
+    command: "ssh",
+    args: ["-o", "BatchMode=yes", "-o", "ServerAliveInterval=30", "-o", "ServerAliveCountMax=4"],
+};
+const encoder = new TextEncoder();
+/**
+ * Returns `shell` with `variables` sent through OpenSSH `SendEnv` instead of the command line,
+ * so the values aren't in arguments, logs, or process listings. The remote sshd drops names
+ * not in its `AcceptEnv`.
+ */
+export const sendingShell = (shell, variables) => {
+    const entries = Object.entries(variables);
+    if (entries.length === 0)
+        return shell;
+    return new RemoteShell({
+        ssh: {
+            command: shell.ssh.command,
+            args: [...shell.ssh.args, "-o", `SendEnv=${entries.map(([name]) => name).join(" ")}`],
+        },
+        sshEnvironment: {
+            ...shell.sshEnvironment,
+            ...Object.fromEntries(entries.map(([name, value]) => [name, Redacted.value(value)])),
+        },
+        destination: shell.destination,
+    });
+};
+/** The `RemoteCommands` that use this process's spawner and the allowlisted SSH environment. */
+export const remoteCommandsFor = Effect.gen(function* () {
+    const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+    const environment = yield* allowlistedEnvironment(SSH_ENVIRONMENT_ALLOWLIST);
+    const shellFor = (ssh, destination) => new RemoteShell({ ssh, sshEnvironment: environment, destination });
+    const executeRaw = Effect.fn("RemoteExecutor.executeRaw")(function* (operation, host, command, args, clientEnvironment) {
+        yield* Effect.annotateCurrentSpan({ host, operation });
+        return yield* collectCommand(spawner, operation, command, args, {
+            env: { ...clientEnvironment },
+            extendEnv: false,
+        });
+    });
+    const local = Effect.fn("RemoteExecutor.local")(function* (operation, host, command, args) {
+        const result = yield* executeRaw(operation, host, command, args, environment);
+        return yield* requireSuccess(operation, host, command, args, result);
+    });
+    const inspect = (operation, shell, command, args) => executeRaw(operation, shell.destination, shell.ssh.command, sshArguments(shell, { command, args }), shell.sshEnvironment);
+    const run = Effect.fn("RemoteExecutor.run")(function* (operation, shell, command, args, context = {}) {
+        const sshArgs = sshArguments(shell, {
+            cwd: context.cwd,
+            environment: context.environment,
+            command,
+            args,
+        });
+        const result = yield* executeRaw(operation, shell.destination, shell.ssh.command, sshArgs, shell.sshEnvironment);
+        const scrub = context.scrub ?? ((text) => text);
+        return yield* requireSuccess(operation, shell.destination, shell.ssh.command, sshArgs, {
+            ...result,
+            stdout: scrub(result.stdout),
+            stderr: scrub(result.stderr),
+        });
+    });
+    const runWithInput = Effect.fn("RemoteExecutor.runWithInput")(function* (operation, shell, command, args, input) {
+        yield* Effect.annotateCurrentSpan({ host: shell.destination, operation });
+        const sshArgs = sshArguments(shell, { command, args });
+        const result = yield* collectCommand(spawner, operation, shell.ssh.command, sshArgs, {
+            env: { ...shell.sshEnvironment },
+            extendEnv: false,
+            stdin: Stream.make(encoder.encode(Redacted.value(input))),
+        });
+        return yield* requireSuccess(operation, shell.destination, shell.ssh.command, sshArgs, result);
+    });
+    return { shellFor, local, inspect, run, runWithInput };
+});
+//# sourceMappingURL=commands.js.map

package/dist/remote/commands.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"commands.js","sourceRoot":"","sources":["../../src/remote/commands.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,WAAW,EAEX,YAAY,GACb,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EACL,sBAAsB,EACtB,cAAc,EAEd,cAAc,GACf,MAAM,qBAAqB,CAAC;AA+C7B;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,MAAM;IACN,MAAM;IACN,QAAQ;IACR,MAAM;IACN,OAAO;IACP,eAAe;IACf,eAAe;IACf,QAAQ;IACR,MAAM;CACE,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAe;IACrC,OAAO,EAAE,KAAK;IACd,IAAI,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,uBAAuB,CAAC;CAC7F,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,KAAkB,EAClB,SAA8D,EACjD,EAAE;IACf,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,IAAI,WAAW,CAAC;QACrB,GAAG,EAAE;YACH,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,OAAO;YAC1B,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;SACtF;QACD,cAAc,EAAE;YACd,GAAG,KAAK,CAAC,cAAc;YACvB,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACrF;QACD,WAAW,EAAE,KAAK,CAAC,WAAW;KAC/B,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,gGAAgG;AAChG,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IACnD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;IAC/D,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,sBAAsB,CAAC,yBAAyB,CAAC,CAAC;IAE7E,MAAM,QAAQ,GAAG,CAAC,GAAe,EAAE,WAA2B,EAAE,EAAE,CAChE,IAAI,WAAW,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;IAErE,MAAM,UAAU,GAAG,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EACjE,SAA4B,EAC5B,IAAY,EACZ,OAAe,EACf,IAA2B,EAC3B,iBAAmD;QAEnD,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QACvD,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE;YAC9D,GAAG,EAAE,EAAE,GAAG,iBAAiB,EAAE;YAC7B,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,sBAAsB,CAAC,CAAC,QAAQ,CAAC,EACvD,SAA4B,EAC5B,IAAY,EACZ,OAAe,EACf,IAA2B;QAE3B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAC9E,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,CACd,SAA4B,EAC5B,KAAkB,EAClB,OAAe,EACf,IAA2B,EAC2B,EAAE,CACxD,UAAU,CACR,SAAS,EACT,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,GAAG,CAAC,OAAO,EACjB,YAAY,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EACtC,KAAK,CAAC,cAAc,CACrB,CAAC;IAEJ,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,EACnD,SAA4B,EAC5B,KAAkB,EAClB,OAAe,EACf,IAA2B,EAC3B,OAAO,GAAkB,EAAE;QAE3B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE;YAClC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,OAAO;YACP,IAAI;SACL,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,UAAU,CAC9B,SAAS,EACT,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,GAAG,CAAC,OAAO,EACjB,OAAO,EACP,KAAK,CAAC,cAAc,CACrB,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACrF,GAAG,MAAM;YACT,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC5B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SAC7B,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,6BAA6B,CAAC,CAAC,QAAQ,CAAC,EACrE,SAA4B,EAC5B,KAAkB,EAClB,OAAe,EACf,IAA2B,EAC3B,KAAgC;QAEhC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACnF,GAAG,EAAE,EAAE,GAAG,KAAK,CAAC,cAAc,EAAE;YAChC,SAAS,EAAE,KAAK;YAChB,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;SAC1D,CAAC,CAAC;QACH,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,YAAY,EAA2B,CAAC;AAClF,CAAC,CAAC,CAAC"}

package/dist/remote/leftovers.d.ts

@@ -0,0 +1,14 @@
+import type { WorkerName } from "@usine/core/worker";
+import { Effect } from "effect";
+import type { RemoteCommands } from "./commands.ts";
+import type { RemoteMachine } from "./machine.ts";
+/**
+ * Resolves a Worker's machine, runs `reclaim` on it, and lists its leftover checkouts, once per
+ * Worker per executor. A Worker that can't be resolved or listed isn't reconciled until the
+ * next start.
+ */
+export declare const workerLeftoversFor: <M extends RemoteMachine, E>(commands: RemoteCommands, machineFor: (worker: WorkerName) => Effect.Effect<M, E>, reclaim: (worker: WorkerName) => Effect.Effect<void>) => Effect.Effect<(worker: WorkerName) => Effect.Effect<{
+    machine: M;
+    checkouts: Set<string>;
+}, E | import("../executor.ts").ExecutorOperationError, never>, never, never>;
+//# sourceMappingURL=leftovers.d.ts.map

package/dist/remote/leftovers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"leftovers.d.ts","sourceRoot":"","sources":["../../src/remote/leftovers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AASlD;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,GAAI,CAAC,SAAS,aAAa,EAAE,CAAC,YACjD,cAAc,cACZ,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,WAC9C,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;;;6EAuBnD,CAAC"}

package/dist/remote/leftovers.js

@@ -0,0 +1,22 @@
+import { Effect } from "effect";
+import { listedOncePerWorker } from "../leftovers.js";
+import { attemptPathsOn } from "./sweep.js";
+/**
+ * Resolves a Worker's machine, runs `reclaim` on it, and lists its leftover checkouts, once per
+ * Worker per executor. A Worker that can't be resolved or listed isn't reconciled until the
+ * next start.
+ */
+export const workerLeftoversFor = (commands, machineFor, reclaim) => listedOncePerWorker(Effect.fn("RemoteExecutor.listWorkerLeftovers")(function* (worker) {
+    const machine = yield* machineFor(worker);
+    const host = machine.shell.destination;
+    yield* Effect.annotateCurrentSpan({ worker, host });
+    yield* reclaim(worker);
+    const checkouts = new Set(yield* attemptPathsOn(commands, "cleanup", machine.shell, `${machine.workerDirectory}/attempts`));
+    yield* Effect.logInfo("Listed remote checkouts left for reconciliation", {
+        worker,
+        host,
+        checkouts: checkouts.size,
+    });
+    return { machine, checkouts };
+}));
+//# sourceMappingURL=leftovers.js.map

package/dist/remote/leftovers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"leftovers.js","sourceRoot":"","sources":["../../src/remote/leftovers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAQ5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,QAAwB,EACxB,UAAuD,EACvD,OAAoD,EACpD,EAAE,CACF,mBAAmB,CACjB,MAAM,CAAC,EAAE,CAAC,oCAAoC,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAkB;IAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;IACvC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,KAAK,CAAC,CAAC,cAAc,CACnB,QAAQ,EACR,SAAS,EACT,OAAO,CAAC,KAAK,EACb,GAAG,OAAO,CAAC,eAAe,WAAW,CACtC,CACF,CAAC;IACF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE;QACvE,MAAM;QACN,IAAI;QACJ,SAAS,EAAE,SAAS,CAAC,IAAI;KAC1B,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,SAAS,EAA+B,CAAC;AAC7D,CAAC,CAAC,CACH,CAAC"}

package/dist/remote/liveness.d.ts

@@ -0,0 +1,33 @@
+import { RemoteShell } from "@usine/agents/child-host";
+import type { AttemptId } from "@usine/core/attempt";
+import { type WorkerNotice } from "@usine/core/attempt/channel";
+import type { WorkerName } from "@usine/core/worker";
+import { Effect, type Scope } from "effect";
+import type { RemoteCommands } from "./commands.ts";
+interface WorkerWatcher {
+    readonly attemptId: AttemptId;
+    /** Records a notice in the Attempt's transcript. */
+    readonly report: (notice: WorkerNotice) => Effect.Effect<void>;
+}
+interface WorkerWatch extends WorkerWatcher {
+    readonly worker: WorkerName;
+    /** The machine's shell, without Attempt secrets. */
+    readonly shell: RemoteShell;
+}
+/** The liveness watches of one executor, one per Worker with a running Attempt. */
+export interface WorkerWatches {
+    /**
+     * Adds the Attempt to its Worker's watch until the caller's scope closes. Every minute, the
+     * watch runs `true` on the machine over a new SSH connection; when the machine stops or resumes
+     * responding, it calls `report` for each Attempt and logs a line. Never stops or fails an
+     * Attempt (ADR 0013).
+     */
+    readonly watch: (watch: WorkerWatch) => Effect.Effect<void, never, Scope.Scope>;
+}
+/**
+ * Returns the liveness watches that probe Workers' machines through `commands`. Each watch runs
+ * in the caller's scope and stops when that scope closes.
+ */
+export declare const workerWatchesFor: (commands: RemoteCommands) => Effect.Effect<WorkerWatches, never, Scope.Scope>;
+export {};
+//# sourceMappingURL=liveness.d.ts.map

package/dist/remote/liveness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"liveness.d.ts","sourceRoot":"","sources":["../../src/remote/liveness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EACL,KAAK,YAAY,EAGlB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAY,MAAM,EAA6B,KAAK,KAAK,EAAa,MAAM,QAAQ,CAAC;AAC5F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AA+GpD,UAAU,aAAa;IACrB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,UAAU,WAAY,SAAQ,aAAa;IACzC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,oDAAoD;IACpD,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;CAC7B;AAED,mFAAmF;AACnF,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;CACjF;AA+GD;;;GAGG;AACH,eAAO,MAAM,gBAAgB,aACjB,cAAc,KACvB,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CA+E9C,CAAC"}

package/dist/remote/liveness.js

@@ -0,0 +1,194 @@
+import { RemoteShell } from "@usine/agents/child-host";
+import { WorkerResponsive, WorkerUnresponsive, } from "@usine/core/attempt/channel";
+import { DateTime, Effect, Fiber, Match, Option, Ref, Semaphore } from "effect";
+const PROBE_INTERVAL = "1 minute";
+/** The SSH `ConnectTimeout` in seconds, covering the TCP connection, server banner, and key exchange. */
+const CONNECT_TIMEOUT_SECONDS = 20;
+/** The timeout for a whole probe, in seconds. A session that opens but never runs the command counts as unanswered. */
+const PROBE_TIMEOUT_SECONDS = 45;
+const reportTo = (notice, attemptIds) => attemptIds.length === 0 ? Option.none() : Option.some({ notice, attemptIds });
+/**
+ * Returns the next liveness after a probe at `now`, with the notice for the `watching` Attempts.
+ * On the first unanswered probe, every Attempt not yet notified gets a "stopped responding"
+ * notice. When the Worker responds again, only notified Attempts get a "responding again"
+ * notice.
+ */
+const observe = (worker, liveness, probe, now, watching) => {
+    if (probe._tag === "Answered")
+        return {
+            liveness: { _tag: "Answering", lastAnsweredAt: now },
+            report: liveness._tag === "Silent"
+                ? reportTo(new WorkerResponsive({ worker, lastAnsweredAt: liveness.lastAnsweredAt }), watching.filter((attemptId) => liveness.told.has(attemptId)))
+                : Option.none(),
+        };
+    const told = liveness._tag === "Silent" ? liveness.told : new Set();
+    return {
+        liveness: {
+            _tag: "Silent",
+            lastAnsweredAt: liveness.lastAnsweredAt,
+            unanswered: liveness._tag === "Silent" ? liveness.unanswered + 1 : 1,
+            reason: probe.reason,
+            told: new Set(watching),
+        },
+        report: reportTo(new WorkerUnresponsive({
+            worker,
+            lastAnsweredAt: liveness.lastAnsweredAt,
+            reason: probe.reason,
+        }), watching.filter((attemptId) => !told.has(attemptId))),
+    };
+};
+/**
+ * Returns the liveness after `attemptId` joins the watch. If the Worker is silent, an Attempt
+ * not yet notified gets a "stopped responding" notice with the last probe's failure.
+ */
+const joinSilent = (worker, liveness, attemptId) => liveness._tag === "Answering" || liveness.told.has(attemptId)
+    ? { liveness, report: Option.none() }
+    : {
+        liveness: { ...liveness, told: new Set([...liveness.told, attemptId]) },
+        report: reportTo(new WorkerUnresponsive({
+            worker,
+            lastAnsweredAt: liveness.lastAnsweredAt,
+            reason: liveness.reason,
+        }), [attemptId]),
+    };
+const lastLine = (text) => text
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line !== "")
+    .at(-1) ?? "";
+/**
+ * Probes one Worker every minute until interrupted, stores the result in `liveness`, and
+ * reports to the Attempts from `watchers`. Holds `lock` from reading `liveness` to the last
+ * report, so each Attempt gets its notices in order.
+ */
+const watchWorker = (commands, worker, machineShell, lock, liveness, watchers) => Effect.gen(function* () {
+    const host = machineShell.destination;
+    const shell = new RemoteShell({
+        ssh: {
+            command: machineShell.ssh.command,
+            args: [...machineShell.ssh.args, "-o", `ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}`],
+        },
+        sshEnvironment: machineShell.sshEnvironment,
+        destination: machineShell.destination,
+    });
+    const probe = commands.inspect("launch", shell, "true", []).pipe(Effect.timeoutOption(`${PROBE_TIMEOUT_SECONDS} seconds`), Effect.map(Option.match({
+        onNone: () => ({
+            _tag: "Unanswered",
+            reason: `No answer within ${PROBE_TIMEOUT_SECONDS} seconds`,
+        }),
+        onSome: (result) => result.exitCode === 0
+            ? { _tag: "Answered" }
+            : {
+                _tag: "Unanswered",
+                reason: [`ssh exited with code ${result.exitCode}`, lastLine(result.stderr)]
+                    .filter((part) => part !== "")
+                    .join(": "),
+            },
+    })), Effect.catch((error) => Effect.succeed({ _tag: "Unanswered", reason: error.message })));
+    const decide = (result) => Effect.gen(function* () {
+        const watching = watchers();
+        const attemptIds = watching.map(({ attemptId }) => attemptId);
+        const previous = yield* Ref.get(liveness);
+        const next = observe(worker, previous, result, yield* DateTime.now, attemptIds);
+        yield* Ref.set(liveness, next.liveness);
+        const fields = {
+            worker,
+            host,
+            attemptIds,
+            notifiedAttemptIds: Option.match(next.report, {
+                onNone: () => [],
+                onSome: ({ attemptIds }) => attemptIds,
+            }),
+            lastAnsweredAt: DateTime.formatIso(previous.lastAnsweredAt),
+        };
+        yield* Match.valueTags(result, {
+            Answered: () => previous._tag === "Silent"
+                ? Effect.logInfo("The Worker answers again", {
+                    ...fields,
+                    unansweredProbes: previous.unanswered,
+                })
+                : Effect.logDebug("The Worker answered its liveness probe", fields),
+            Unanswered: ({ reason }) => Effect.logWarning(previous._tag === "Silent"
+                ? "The Worker still does not answer"
+                : "The Worker stopped answering", {
+                ...fields,
+                unansweredProbes: previous._tag === "Silent" ? previous.unanswered + 1 : 1,
+                reason,
+            }),
+        });
+        if (Option.isSome(next.report)) {
+            const { notice, attemptIds: notified } = next.report.value;
+            yield* Effect.forEach(watching.filter(({ attemptId }) => notified.includes(attemptId)), ({ report }) => report(notice), { discard: true });
+        }
+    });
+    // Root span: the watch can run longer than the Attempt that started it.
+    const check = Effect.fn("RemoteExecutor.probeWorker", { root: true })(function* () {
+        yield* Effect.annotateCurrentSpan({ worker, host });
+        const result = yield* probe;
+        yield* Semaphore.withPermit(lock, decide(result));
+    });
+    return yield* Effect.sleep(PROBE_INTERVAL).pipe(Effect.andThen(check()), Effect.forever);
+});
+/**
+ * Returns the liveness watches that probe Workers' machines through `commands`. Each watch runs
+ * in the caller's scope and stops when that scope closes.
+ */
+export const workerWatchesFor = (commands) => Effect.gen(function* () {
+    const scope = yield* Effect.scope;
+    const lock = yield* Semaphore.make(1);
+    const watched = new Map();
+    const join = (watch) => Semaphore.withPermit(lock, Effect.gen(function* () {
+        const fields = { attemptId: watch.attemptId, worker: watch.worker };
+        const existing = watched.get(watch.worker);
+        if (existing !== undefined) {
+            existing.watchers.set(watch.attemptId, watch);
+            const current = yield* Ref.get(existing.liveness);
+            const joined = { ...fields, attempts: existing.watchers.size };
+            if (current._tag === "Answering")
+                return yield* Effect.logDebug("The Attempt joins its Worker's liveness watch", joined);
+            const next = joinSilent(watch.worker, current, watch.attemptId);
+            yield* Ref.set(existing.liveness, next.liveness);
+            yield* Effect.logInfo(Option.isSome(next.report)
+                ? "The Attempt joins its Worker's liveness watch while the Worker does not answer, and is told so"
+                : "The Attempt joins its Worker's liveness watch while the Worker does not answer; it was told so already", {
+                ...joined,
+                lastAnsweredAt: DateTime.formatIso(current.lastAnsweredAt),
+                unansweredProbes: current.unanswered,
+                reason: current.reason,
+            });
+            if (Option.isSome(next.report))
+                yield* watch.report(next.report.value.notice);
+            return;
+        }
+        const liveness = yield* Ref.make({
+            _tag: "Answering",
+            lastAnsweredAt: yield* DateTime.now,
+        });
+        const watchers = new Map([[watch.attemptId, watch]]);
+        const fiber = yield* watchWorker(commands, watch.worker, watch.shell, lock, liveness, () => [...watchers.values()]).pipe(Effect.forkIn(scope));
+        watched.set(watch.worker, { liveness, watchers, fiber });
+        yield* Effect.logDebug("Watching the Worker while Attempts run on it", {
+            ...fields,
+            host: watch.shell.destination,
+            interval: PROBE_INTERVAL,
+        });
+    }));
+    const leave = (watch) => Semaphore.withPermit(lock, Effect.gen(function* () {
+        const existing = watched.get(watch.worker);
+        if (existing === undefined)
+            return;
+        existing.watchers.delete(watch.attemptId);
+        if (existing.watchers.size > 0)
+            return;
+        watched.delete(watch.worker);
+        yield* Fiber.interrupt(existing.fiber);
+        yield* Effect.logDebug("Stopped watching the Worker; no Attempt runs on it", {
+            attemptId: watch.attemptId,
+            worker: watch.worker,
+        });
+    }));
+    return {
+        watch: (watch) => Effect.acquireRelease(join(watch), () => leave(watch)),
+    };
+});
+//# sourceMappingURL=liveness.js.map

package/dist/remote/liveness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"liveness.js","sourceRoot":"","sources":["../../src/remote/liveness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,OAAO,EAEL,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,6BAA6B,CAAC;AAErC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAc,SAAS,EAAE,MAAM,QAAQ,CAAC;AAG5F,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,yGAAyG;AACzG,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,uHAAuH;AACvH,MAAM,qBAAqB,GAAG,EAAE,CAAC;AA6BjC,MAAM,QAAQ,GAAG,CACf,MAAoB,EACpB,UAAoC,EACb,EAAE,CACzB,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,OAAO,GAAG,CACd,MAAkB,EAClB,QAAkB,EAClB,KAAY,EACZ,GAAiB,EACjB,QAAkC,EACrB,EAAE;IACf,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;QAC3B,OAAO;YACL,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,EAAE;YACpD,MAAM,EACJ,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBACxB,CAAC,CAAC,QAAQ,CACN,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,EACzE,QAAQ,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAC7D;gBACH,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;SACpB,CAAC;IACJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,EAAa,CAAC;IAC/E,OAAO;QACL,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,UAAU,EAAE,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpE,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC;SACxB;QACD,MAAM,EAAE,QAAQ,CACd,IAAI,kBAAkB,CAAC;YACrB,MAAM;YACN,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,EACF,QAAQ,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CACrD;KACF,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,GAAG,CAAC,MAAkB,EAAE,QAAkB,EAAE,SAAoB,EAAe,EAAE,CAC/F,QAAQ,CAAC,IAAI,KAAK,WAAW,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;IAC3D,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE;IACrC,CAAC,CAAC;QACE,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,EAAE;QACvE,MAAM,EAAE,QAAQ,CACd,IAAI,kBAAkB,CAAC;YACrB,MAAM;YACN,cAAc,EAAE,QAAQ,CAAC,cAAc;YACvC,MAAM,EAAE,QAAQ,CAAC,MAAM;SACxB,CAAC,EACF,CAAC,SAAS,CAAC,CACZ;KACF,CAAC;AAER,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE,CACxC,IAAI;KACD,KAAK,CAAC,IAAI,CAAC;KACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;KAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;KAC7B,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAyBlB;;;;GAIG;AACH,MAAM,WAAW,GAAG,CAClB,QAAwB,EACxB,MAAkB,EAClB,YAAyB,EACzB,IAAyB,EACzB,QAA2B,EAC3B,QAA4C,EACtB,EAAE,CACxB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,IAAI,GAAG,YAAY,CAAC,WAAW,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC;QAC5B,GAAG,EAAE;YACH,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,OAAO;YACjC,IAAI,EAAE,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAkB,uBAAuB,EAAE,CAAC;SACpF;QACD,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,WAAW,EAAE,YAAY,CAAC,WAAW;KACtC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAyB,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CACpF,MAAM,CAAC,aAAa,CAAC,GAAG,qBAAqB,UAAU,CAAC,EACxD,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,KAAK,CAAC;QACX,MAAM,EAAE,GAAU,EAAE,CAAC,CAAC;YACpB,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,oBAAoB,qBAAqB,UAAU;SAC5D,CAAC;QACF,MAAM,EAAE,CAAC,MAAM,EAAS,EAAE,CACxB,MAAM,CAAC,QAAQ,KAAK,CAAC;YACnB,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE;YACtB,CAAC,CAAC;gBACE,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,CAAC,wBAAwB,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;qBACzE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;qBAC7B,IAAI,CAAC,IAAI,CAAC;aACd;KACR,CAAC,CACH,EACD,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAC9F,CAAC;IAEF,MAAM,MAAM,GAAG,CAAC,MAAa,EAAE,EAAE,CAC/B,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,QAAQ,GAAG,QAAQ,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAChF,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG;YACb,MAAM;YACN,IAAI;YACJ,UAAU;YACV,kBAAkB,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE;gBAC5C,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;gBAChB,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,UAAU;aACvC,CAAC;YACF,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC;SAC5D,CAAC;QACF,KAAK,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE;YAC7B,QAAQ,EAAE,GAAG,EAAE,CACb,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBACxB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,0BAA0B,EAAE;oBACzC,GAAG,MAAM;oBACT,gBAAgB,EAAE,QAAQ,CAAC,UAAU;iBACtC,CAAC;gBACJ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,wCAAwC,EAAE,MAAM,CAAC;YACvE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,MAAM,CAAC,UAAU,CACf,QAAQ,CAAC,IAAI,KAAK,QAAQ;gBACxB,CAAC,CAAC,kCAAkC;gBACpC,CAAC,CAAC,8BAA8B,EAClC;gBACE,GAAG,MAAM;gBACT,gBAAgB,EAAE,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1E,MAAM;aACP,CACF;SACJ,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YAC3D,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAChE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAC9B,EAAE,OAAO,EAAE,IAAI,EAAE,CAClB,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,4BAA4B,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC;QAC7E,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC;QAC5B,KAAK,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;AAC3F,CAAC,CAAC,CAAC;AAQL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,QAAwB,EAC0B,EAAE,CACpD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE/C,MAAM,IAAI,GAAG,CAAC,KAAkB,EAAE,EAAE,CAClC,SAAS,CAAC,UAAU,CAClB,IAAI,EACJ,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,MAAM,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW;gBAC9B,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAC3B,+CAA+C,EAC/C,MAAM,CACP,CAAC;YACJ,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAChE,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;gBACxB,CAAC,CAAC,gGAAgG;gBAClG,CAAC,CAAC,wGAAwG,EAC5G;gBACE,GAAG,MAAM;gBACT,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC;gBAC1D,gBAAgB,EAAE,OAAO,CAAC,UAAU;gBACpC,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CACF,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAW;YACzC,IAAI,EAAE,WAAW;YACjB,cAAc,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG;SACpC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAA2B,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,WAAW,CAC9B,QAAQ,EACR,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,KAAK,EACX,IAAI,EACJ,QAAQ,EACR,GAAG,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAC7B,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACzD,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,EAAE;YACrE,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW;YAC7B,QAAQ,EAAE,cAAc;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CACH,CAAC;IAEJ,MAAM,KAAK,GAAG,CAAC,KAAkB,EAAE,EAAE,CACnC,SAAS,CAAC,UAAU,CAClB,IAAI,EACJ,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO;QACnC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;YAAE,OAAO;QACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7B,KAAK,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACvC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,oDAAoD,EAAE;YAC3E,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;IACL,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;QACL,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;KACjD,CAAC;AAC5B,CAAC,CAAC,CAAC"}

package/dist/remote/machine.d.ts

@@ -0,0 +1,64 @@
+import type { RemoteShell } from "@usine/agents/child-host";
+import type { BrainSpec, Harness } from "@usine/core/agent/brain";
+import type { AttemptId } from "@usine/core/attempt";
+import type { LoopPermissions } from "@usine/core/loop";
+import type { RepositoryName } from "@usine/core/repository";
+import type { GatewayCredentialKind, WorkerKind, WorkerName } from "@usine/core/worker";
+import type { Effect, Option, Scope } from "effect";
+import type { CredentialUnavailableError, ExecutorError, ExecutorOperation } from "../executor.ts";
+import type { HarnessProbe } from "../harness-preflight.ts";
+import type { RemoteCommands } from "./commands.ts";
+export interface RemoteMachine {
+    readonly shell: RemoteShell;
+    /** The Worker directory on the machine, with cached clones under `repos` and checkouts under `attempts`. */
+    readonly workerDirectory: string;
+}
+export interface RemoteAccess {
+    /**
+     * The shell for the Attempt's children, harvesting, and git commands: the machine's own, or one
+     * that sends the Attempt's secrets through the SSH connection (`sendingShell`).
+     */
+    readonly shell: RemoteShell;
+    /** The extra environment for the Attempt's children and harvesting commands. Values appear in the command line, so none can be secret. */
+    readonly environment: Readonly<Record<string, string>>;
+    /** The extra environment for the executor's git commands on the checkout. Values appear in the command line, so none can be secret. */
+    readonly gitEnvironment: Readonly<Record<string, string>>;
+    /** Renews the credential on the machine if it's about to expire. Run before each git and harvesting command. */
+    readonly refresh: Effect.Effect<void, ExecutorError>;
+    /** Removes every credential this process issued for the Attempt from a command's output. */
+    readonly scrub: (text: string) => string;
+}
+export interface RemoteGrant<M extends RemoteMachine> {
+    /**
+     * Grants the Attempt access to its repository on `machine` until the caller's scope closes,
+     * restricted to its Loop `permissions` if the Worker kind supports it.
+     */
+    readonly acquire: (attemptId: AttemptId, worker: WorkerName, permissions: LoopPermissions, machine: M, operation: ExecutorOperation) => Effect.Effect<RemoteAccess, ExecutorError, Scope.Scope>;
+}
+/** The Worker-kind implementation for the remote executor. */
+export interface RemoteWorker<M extends RemoteMachine> {
+    /** Reported as `Executor.kind`; the scheduler rejects Workers of other kinds. */
+    readonly kind: WorkerKind;
+    /** The span name prefix, for example `ExeDevExecutor`. */
+    readonly spanPrefix: string;
+    readonly label: string;
+    readonly commands: RemoteCommands;
+    /** Returns a Worker's machine. Fails with a typed error if the machine is missing or doesn't respond. */
+    readonly machineFor: (worker: WorkerName) => Effect.Effect<M, ExecutorError>;
+    /** See `Executor.preflight`. */
+    readonly preflight: (worker: WorkerName, brains: ReadonlyArray<BrainSpec>, maxConcurrentAttempts: number) => Effect.Effect<void, ExecutorError>;
+    readonly harnessProbe: (machine: M) => HarnessProbe;
+    /** Returns the URL the cached clone on `machine` fetches `repository` from. */
+    readonly repositoryUrl: (machine: M, repository: RepositoryName) => string;
+    /** Returns the credential kind that the Gateway on `machine` injects for `harness`. Returns `None` if the machine has no Gateway or the harness uses its own login. */
+    readonly gatewayCredential: (machine: M, harness: Harness) => Effect.Effect<Option.Option<GatewayCredentialKind>>;
+    /** Returns the access grant for `repository`. Fails if none is configured. */
+    readonly grantFor: (repository: RepositoryName) => Effect.Effect<RemoteGrant<M>, CredentialUnavailableError>;
+    /** The reason this Worker kind can't grant read-only repository access, or `None` if it can. */
+    readonly readOnlyAccessGap: Option.Option<string>;
+    /** Removes leftovers of earlier processes that no Attempt can use. Runs before the Worker's checkouts are listed. */
+    readonly reclaim: (worker: WorkerName) => Effect.Effect<void>;
+    /** The timeout for the parent task's cancellation, then separately for the Attempt to finish. */
+    readonly interruptGrace: Parameters<typeof Effect.timeoutOption>[1];
+}
+//# sourceMappingURL=machine.d.ts.map

package/dist/remote/machine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"machine.d.ts","sourceRoot":"","sources":["../../src/remote/machine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,qBAAqB,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACxF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AACpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,4GAA4G;IAC5G,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,0IAA0I;IAC1I,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACvD,uIAAuI;IACvI,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D,gHAAgH;IAChH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACrD,4FAA4F;IAC5F,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,WAAW,CAAC,CAAC,SAAS,aAAa;IAClD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,CAChB,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,UAAU,EAClB,WAAW,EAAE,eAAe,EAC5B,OAAO,EAAE,CAAC,EACV,SAAS,EAAE,iBAAiB,KACzB,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;CAC9D;AAED,8DAA8D;AAC9D,MAAM,WAAW,YAAY,CAAC,CAAC,SAAS,aAAa;IACnD,iFAAiF;IACjF,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,0DAA0D;IAC1D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,yGAAyG;IACzG,QAAQ,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAC7E,gCAAgC;IAChC,QAAQ,CAAC,SAAS,EAAE,CAClB,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAChC,qBAAqB,EAAE,MAAM,KAC1B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACxC,QAAQ,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,YAAY,CAAC;IACpD,+EAA+E;IAC/E,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,cAAc,KAAK,MAAM,CAAC;IAC3E,uKAAuK;IACvK,QAAQ,CAAC,iBAAiB,EAAE,CAC1B,OAAO,EAAE,CAAC,EACV,OAAO,EAAE,OAAO,KACb,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC;IACzD,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,CACjB,UAAU,EAAE,cAAc,KACvB,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,0BAA0B,CAAC,CAAC;IAC/D,gGAAgG;IAChG,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClD,qHAAqH;IACrH,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9D,iGAAiG;IACjG,QAAQ,CAAC,cAAc,EAAE,UAAU,CAAC,OAAO,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;CACrE"}

package/dist/remote/machine.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=machine.js.map

package/dist/remote/machine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"machine.js","sourceRoot":"","sources":["../../src/remote/machine.ts"],"names":[],"mappings":""}

package/dist/remote/remote.d.ts

@@ -0,0 +1,24 @@
+import { ClaudeLoginLock } from "@usine/agents/runtime";
+import { type AttemptFailure } from "@usine/core/attempt/failure";
+import type { AttemptRecovery, RecoverySink } from "@usine/core/attempt/salvage";
+import type * as AttemptChannel from "@usine/core/attempt/channel";
+import { AttemptRuntime } from "@usine/durable/attempt-runtime";
+import { Effect, Option, Scope } from "effect";
+import { CheckoutRequest, ExecutorError } from "../executor.ts";
+import type { RemoteMachine, RemoteWorker } from "./machine.ts";
+/**
+ * Returns the executor for Workers accessed over SSH. Each Attempt's Pi parent is a durable task
+ * in this process, and each child runs on the machine as one SSH command with a pid file next
+ * to the worktree. `worker` provides the machine, preflight, and repository access.
+ */
+export declare const remoteExecutorFor: <M extends RemoteMachine>(worker: RemoteWorker<M>) => Effect.Effect<{
+    readonly recover: <E>(attempt: AttemptRecovery, record: RecoverySink<E>) => Effect.Effect<void, E | ExecutorError>;
+    readonly kind: import("@usine/core/worker").WorkerKind;
+    readonly isLive: (attemptId: import("@usine/core/attempt").AttemptId) => Effect.Effect<boolean, AttemptFailure>;
+    readonly outcome: (attemptId: import("@usine/core/attempt").AttemptId) => Effect.Effect<Option.Option<import("effect/Result").Result<import("@usine/core/step/result").StepResult, AttemptFailure>>, AttemptFailure>;
+    readonly preflight: (worker: import("@usine/core/worker").WorkerName, brains: ReadonlyArray<import("@usine/core/agent/brain").BrainSpec>, maxConcurrentAttempts: number) => Effect.Effect<void, ExecutorError>;
+    readonly prepare: (request: CheckoutRequest) => Effect.Effect<import("../executor.ts").PreparedAttempt, ExecutorError, Scope.Scope>;
+    readonly cancel: (attemptId: import("@usine/core/attempt").AttemptId) => Effect.Effect<void, ExecutorError>;
+    readonly intervene: (attemptId: import("@usine/core/attempt").AttemptId, intervention: AttemptChannel.Intervention) => Effect.Effect<void, AttemptFailure>;
+}, never, AttemptRuntime | ClaudeLoginLock | Scope.Scope>;
+//# sourceMappingURL=remote.d.ts.map

package/dist/remote/remote.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote.d.ts","sourceRoot":"","sources":["../../src/remote/remote.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,6BAA6B,CAAC;AAChF,OAAO,KAAK,EAAE,eAAe,EAAkB,YAAY,EAAE,MAAM,6BAA6B,CAAC;AACjG,OAAO,KAAK,KAAK,cAAc,MAAM,6BAA6B,CAAC;AAOnE,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAY,MAAM,EAAQ,MAAM,EAAU,KAAK,EAAE,MAAM,QAAQ,CAAC;AAEvE,OAAO,EAEL,eAAe,EAEf,aAAa,EAId,MAAM,gBAAgB,CAAC;AAcxB,OAAO,KAAK,EAAgB,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAgB9E;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAAI,CAAC,SAAS,aAAa,UAAU,YAAY,CAAC,CAAC,CAAC;;;;;;;;;yDA6b9E,CAAC"}