@usine/executor 0.2.1 → 0.4.0

package/dist/ssh/model.d.ts

@@ -0,0 +1,153 @@
+import type { SshCommand } from "@usine/agents/child-host";
+import type { InstanceId } from "@usine/core/instance";
+import type { BoatSize, ContainerCpus, ContainerImage, ContainerMemoryMib, ContainerPids, GatewayAdminPort, GatewayCredentialKind, GatewayDomain, GatewayLlmProvider, HetznerImage, HetznerLabels, HetznerLocation, HetznerResourceName, HetznerServerType, MachineProvider, SshDestination, SshIdentityFile, SshPort, SshUser, WorkerName, WorkerRoot } from "@usine/core/worker";
+import type { Duration, Effect, Option, Redacted, Ref } from "effect";
+import type { HarnessProbe } from "../harness-preflight.ts";
+import type { RemoteMachine } from "../remote/machine.ts";
+import type { GatewayAdmin } from "./gateway/admin.ts";
+import type { GatewayForward } from "./gateway/forward.ts";
+/** The Gateway in front of an SSH Worker's machine on a private network (ADR 0027). */
+export interface SshWorkerGateway {
+    /** The Gateway's private DNS domain. GitHub is at `git.<domain>`, LLM providers at `llm.<domain>`, and the forward proxy at `<domain>:3128`. */
+    readonly domain: GatewayDomain;
+    /** The `ssh` destination for the Gateway: `user@host` or a Host alias. The admin API forward connects through it. */
+    readonly destination: SshDestination;
+    /** The admin API port on the Gateway's loopback interface. */
+    readonly adminPort: GatewayAdminPort;
+    /** The admin API bearer token. */
+    readonly adminToken: Redacted.Redacted<string>;
+    /** The LLM providers the Gateway has credentials for. A harness whose provider isn't listed uses its own login through the forward proxy. */
+    readonly llmProviders: ReadonlyArray<GatewayLlmProvider>;
+}
+/**
+ * The container of each Attempt on an SSH Worker whose machine has Docker. Each limit that is
+ * `None` is derived from the machine at preflight: its total less a reserve for the machine's own
+ * processes, divided by the Attempts that run on it at once.
+ */
+export interface SshWorkerContainer {
+    /** The image to run. If `None`, `ATTEMPT_IMAGE`. */
+    readonly image: Option.Option<ContainerImage>;
+    readonly cpus: Option.Option<ContainerCpus>;
+    readonly memoryMib: Option.Option<ContainerMemoryMib>;
+    /** The maximum number of processes and threads. */
+    readonly pids: Option.Option<ContainerPids>;
+}
+/** The LLM provider API keys that each Attempt on a provisioned Worker gets in its environment. */
+export interface ProviderApiKeys {
+    /** Claude Code's `ANTHROPIC_API_KEY`. */
+    readonly anthropic: Option.Option<Redacted.Redacted<string>>;
+    /** Codex's `OPENAI_API_KEY`. */
+    readonly openai: Option.Option<Redacted.Redacted<string>>;
+}
+/** Creates each Attempt's machine as a Hetzner Cloud server from a snapshot. */
+export interface HetznerProvisioner {
+    readonly provider: "hetzner";
+    /** The Hetzner Cloud API token, with read and write access to the project. */
+    readonly token: Redacted.Redacted<string>;
+    readonly serverType: HetznerServerType;
+    readonly location: HetznerLocation;
+    /** A snapshot's ID or description, or a system image's name. */
+    readonly image: HetznerImage;
+    /** The private network to attach each server to. Behind a Gateway, each server gets no public address. */
+    readonly network: Option.Option<HetznerResourceName>;
+    /** The project's firewall to apply to each server. */
+    readonly firewall: Option.Option<HetznerResourceName>;
+    /** The project's SSH key that each server authorizes. */
+    readonly sshKey: HetznerResourceName;
+    readonly sshUser: SshUser;
+    readonly labels: HetznerLabels;
+    readonly apiKeys: ProviderApiKeys;
+}
+/** Creates each Attempt's machine as a Boat sandbox. */
+export interface BoatProvisioner {
+    readonly provider: "boat";
+    /** The Boat API key. */
+    readonly token: Redacted.Redacted<string>;
+    readonly size: BoatSize;
+    readonly apiKeys: ProviderApiKeys;
+}
+export type SshProvisioner = HetznerProvisioner | BoatProvisioner;
+/** How the executor finds a Worker's machines: one long-lived machine, or one per Attempt. */
+export type SshWorkerMachine = {
+    readonly _tag: "Fixed";
+    /** The `ssh` destination: `user@host` or a Host alias. */
+    readonly destination: SshDestination;
+} | {
+    readonly _tag: "Provisioned";
+    readonly provisioner: SshProvisioner;
+};
+export interface SshWorkerAddress {
+    /**
+     * The Worker's name in `usine.yaml`; its state is in `workers/<name>` under `root`. A request
+     * for a Worker without an address fails with a `machine` capability error.
+     */
+    readonly name: WorkerName;
+    readonly machine: SshWorkerMachine;
+    /** The private key for authentication. If `None`, the SSH agent and configuration are used. */
+    readonly identityFile: Option.Option<SshIdentityFile>;
+    /** The SSH server port. If `None`, the SSH configuration's port, or a provisioned machine's, is used. */
+    readonly port: Option.Option<SshPort>;
+    /** The machine directory that contains `workers/<name>`. */
+    readonly root: WorkerRoot;
+    /** The Gateway in front of the machine. If `None`, each Attempt's token is written on the machine (ADR 0026). */
+    readonly gateway: Option.Option<SshWorkerGateway>;
+    /** Each Attempt's container. Used only if the machine has Docker. */
+    readonly container: SshWorkerContainer;
+}
+export interface SshExecutorOptions {
+    readonly workers: ReadonlyArray<SshWorkerAddress>;
+    /**
+     * The identity of this usine instance on the Workers' machines, which the instances that share
+     * a GitHub App share too. It labels each Attempt container, and tags each provisioned machine.
+     */
+    readonly instance: string;
+    /**
+     * Tells apart the instances that share a GitHub App. It labels each Attempt container too, so
+     * that a later process removes only this instance's containers.
+     */
+    readonly instanceId: InstanceId;
+    /** The local SSH command and the options placed before each Worker's port and identity file. Defaults to `ssh` in batch mode with keepalives. */
+    readonly ssh?: SshCommand;
+    /** The timeout for the parent task's cancellation, then separately for the Attempt to finish. Defaults to 10 seconds. */
+    readonly interruptGrace?: Parameters<typeof Effect.timeoutOption>[1];
+    /** Probes a brain's binary and login on a Worker's machine. Defaults to the `@usine/agents/preflight` checks over SSH, which send no prompt. Behind a Gateway, the Gateway checks also apply. */
+    readonly harnessReadiness?: (machine: SshMachine) => HarnessProbe;
+    /**
+     * The local directory for the host keys of provisioned machines, one file per machine, which
+     * the executor creates. Defaults to a temporary directory removed with the executor.
+     */
+    readonly knownHostsDirectory?: string;
+    /**
+     * The local directory where the executor records how to reach each long-lived Worker's machine
+     * when it adds the Worker, which `sshWorkerRecordsFor` reads. Defaults to a temporary
+     * directory removed with the executor.
+     */
+    readonly workerRecordsDirectory?: string;
+    /** The provider API endpoints, which tests replace. Defaults to the providers' own. */
+    readonly providerEndpoints?: Partial<Record<MachineProvider, string>>;
+    /** The longest wait for a created machine to answer over SSH. Defaults to 10 minutes. */
+    readonly machineReadyTimeout?: Duration.Input;
+    /** The pause between checks of a machine being created. Defaults to 3 seconds. */
+    readonly machinePollInterval?: Duration.Input;
+}
+export interface MachineGateway {
+    readonly settings: SshWorkerGateway;
+    readonly forward: GatewayForward;
+    readonly admin: GatewayAdmin;
+    /**
+     * The credential kind the Gateway uses for each provider, from the last preflight: Anthropic
+     * for Claude Code, OpenAI for Codex. `None` before the first preflight, for a provider that
+     * `llmProviders` doesn't list, or for Anthropic if the Gateway is too old to report it.
+     */
+    readonly credentials: Ref.Ref<GatewayCredentials>;
+    /** Whether a preflight already warned that the Gateway is too old to report its Anthropic credential kind. */
+    readonly warnedOfUnreportedCredential: Ref.Ref<boolean>;
+}
+/** The credential kind the Gateway holds for each provider; see `MachineGateway.credentials`. */
+export type GatewayCredentials = Readonly<Record<GatewayLlmProvider, Option.Option<GatewayCredentialKind>>>;
+export interface SshMachine extends RemoteMachine {
+    /** The Worker's address from `usine.yaml`; preflight checks its root. */
+    readonly address: SshWorkerAddress;
+    readonly gateway: Option.Option<MachineGateway>;
+}
+//# sourceMappingURL=model.d.ts.map

package/dist/ssh/model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../src/ssh/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,KAAK,EACV,QAAQ,EACR,aAAa,EACb,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,aAAa,EACb,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,eAAe,EACf,OAAO,EACP,OAAO,EACP,UAAU,EACV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AACtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,uFAAuF;AACvF,MAAM,WAAW,gBAAgB;IAC/B,gJAAgJ;IAChJ,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,qHAAqH;IACrH,QAAQ,CAAC,WAAW,EAAE,cAAc,CAAC;IACrC,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,gBAAgB,CAAC;IACrC,kCAAkC;IAClC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/C,6IAA6I;IAC7I,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAC;CAC1D;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,oDAAoD;IACpD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACtD,mDAAmD;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;CAC7C;AAED,mGAAmG;AACnG,MAAM,WAAW,eAAe;IAC9B,yCAAyC;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7D,gCAAgC;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;CAC3D;AAED,gFAAgF;AAChF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,iBAAiB,CAAC;IACvC,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,gEAAgE;IAChE,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,0GAA0G;IAC1G,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACrD,sDAAsD;IACtD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACtD,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;CACnC;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,wBAAwB;IACxB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;CACnC;AAED,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,eAAe,CAAC;AAElE,8FAA8F;AAC9F,MAAM,MAAM,gBAAgB,GACxB;IACE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,0DAA0D;IAC1D,QAAQ,CAAC,WAAW,EAAE,cAAc,CAAC;CACtC,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3E,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,+FAA+F;IAC/F,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACtD,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtC,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,iHAAiH;IACjH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClD,qEAAqE;IACrE,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;CACxC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAClD;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,iJAAiJ;IACjJ,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC;IAC1B,yHAAyH;IACzH,QAAQ,CAAC,cAAc,CAAC,EAAE,UAAU,CAAC,OAAO,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,iMAAiM;IACjM,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,YAAY,CAAC;IAClE;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,uFAAuF;IACvF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,yFAAyF;IACzF,QAAQ,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC;IAC9C,kFAAkF;IAClF,QAAQ,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAClD,8GAA8G;IAC9G,QAAQ,CAAC,4BAA4B,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;CACzD;AAED,iGAAiG;AACjG,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CACvC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CACjE,CAAC;AAEF,MAAM,WAAW,UAAW,SAAQ,aAAa;IAC/C,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;CACjD"}

package/dist/ssh/model.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=model.js.map

package/dist/ssh/model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/ssh/model.ts"],"names":[],"mappings":""}

package/dist/ssh/preflight.d.ts

@@ -0,0 +1,35 @@
+import type { RemoteShell } from "@usine/agents/child-host";
+import type { ClaudeLoginLock } from "@usine/agents/runtime";
+import type { BrainSpec } from "@usine/core/agent/brain";
+import type { WorkerName } from "@usine/core/worker";
+import { Effect, Option } from "effect";
+import { type ExecutorError, type PreflightReport, WorkerCapabilityError } from "../executor.ts";
+import { type HarnessProbe } from "../harness-preflight.ts";
+import type { RemoteCommands } from "../remote/commands.ts";
+import type { SshContainers } from "./container.ts";
+import type { SshMachine } from "./model.ts";
+import type { SshWorkerEntry } from "./workers.ts";
+/**
+ * Verifies an SSH Worker's machine: SSH connects, the root is a writable directory, Git runs,
+ * memory is enough for the concurrent Attempts, the Gateway (if any) responds and is reachable,
+ * Docker and the Attempt image work if the machine has Docker, and each brain is ready. Pulls
+ * the Attempt image and changes nothing else. With Docker, it derives the container limits that
+ * `usine.yaml` leaves unset for `maxConcurrentAttempts` containers, which the report gives, and
+ * brains are probed in a container from the image; without, as the SSH user, which the report
+ * gives as degraded. Claude brains are checked under the Worker's lock in `logins`.
+ */
+export declare const sshMachinePreflightFor: (commands: RemoteCommands, harnessProbe: (machine: SshMachine, shell: RemoteShell) => HarnessProbe, logins: ClaudeLoginLock["Service"], containers: SshContainers) => (machine: SshMachine, brains: readonly BrainSpec[], maxConcurrentAttempts: number) => Effect.Effect<{
+    degraded: Option.Option<string>;
+    deferred: Option.Option<string>;
+    containerLimits: Option.Option<import("@usine/core/worker").ContainerLimits>;
+}, import("../executor.ts").CredentialUnavailableError | import("../executor.ts").ExecutorOperationError | WorkerCapabilityError, never>;
+/**
+ * Verifies an SSH Worker: a long-lived Worker's machine with `verifyMachine`, or a provisioned
+ * Worker with `verifyProvisioned`, which has no machine before an Attempt.
+ */
+export declare const sshPreflightFor: (workerFor: (worker: WorkerName) => Effect.Effect<SshWorkerEntry, ExecutorError>, verifyMachine: ReturnType<typeof sshMachinePreflightFor>, verifyProvisioned: (entry: SshWorkerEntry, brains: ReadonlyArray<BrainSpec>, maxConcurrentAttempts: number) => Effect.Effect<PreflightReport, ExecutorError>) => (worker: string & import("effect/Brand").Brand<"WorkerName">, brains: readonly BrainSpec[], maxConcurrentAttempts: number) => Effect.Effect<PreflightReport | {
+    degraded: Option.Option<string>;
+    deferred: Option.Option<string>;
+    containerLimits: Option.Option<import("@usine/core/worker").ContainerLimits>;
+}, import("../executor.ts").CredentialUnavailableError | import("../executor.ts").ExecutorOperationError | WorkerCapabilityError, never>;
+//# sourceMappingURL=preflight.d.ts.map

package/dist/ssh/preflight.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preflight.d.ts","sourceRoot":"","sources":["../../src/ssh/preflight.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAgB,MAAM,EAAS,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC7D,OAAO,EACL,KAAK,aAAa,EAElB,KAAK,eAAe,EACpB,qBAAqB,EAEtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,YAAY,EAA0B,MAAM,yBAAyB,CAAC;AAEpF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAYnD;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,aACvB,cAAc,gBACV,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,KAAK,YAAY,UAC/D,eAAe,CAAC,SAAS,CAAC,cACtB,aAAa;;;;wIAqHvB,CAAC;AAEL;;;GAGG;AACH,eAAO,MAAM,eAAe,cACf,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,aAAa,CAAC,iBAChE,UAAU,CAAC,OAAO,sBAAsB,CAAC,qBACrC,CACjB,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAChC,qBAAqB,EAAE,MAAM,KAC1B,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC;;;;wIAa3B,CAAC"}

package/dist/ssh/preflight.js

@@ -0,0 +1,87 @@
+import { Array as Arr, Effect, Match, Option } from "effect";
+import { logPreflightFailure, WorkerCapabilityError, } from "../executor.js";
+import { verifyHarnessReadiness } from "../harness-preflight.js";
+import { verifyLinuxMemory } from "../memory-preflight.js";
+import { gatewayExportedEnvironment } from "./gateway/environment.js";
+import { verifyGateway } from "./gateway/preflight.js";
+import { limitFields } from "./limits.js";
+/** A script that succeeds if `$1` is a directory the SSH user can write to. */
+const WRITABLE_DIRECTORY_SCRIPT = 'test -d "$1" && test -w "$1"';
+const capabilityError = (worker, capability, message, cause) => new WorkerCapabilityError({ worker, capability, message, cause });
+/**
+ * Verifies an SSH Worker's machine: SSH connects, the root is a writable directory, Git runs,
+ * memory is enough for the concurrent Attempts, the Gateway (if any) responds and is reachable,
+ * Docker and the Attempt image work if the machine has Docker, and each brain is ready. Pulls
+ * the Attempt image and changes nothing else. With Docker, it derives the container limits that
+ * `usine.yaml` leaves unset for `maxConcurrentAttempts` containers, which the report gives, and
+ * brains are probed in a container from the image; without, as the SSH user, which the report
+ * gives as degraded. Claude brains are checked under the Worker's lock in `logins`.
+ */
+export const sshMachinePreflightFor = (commands, harnessProbe, logins, containers) => Effect.fn("SshExecutor.verifyMachine")(function* (machine, brains, maxConcurrentAttempts) {
+    const worker = machine.address.name;
+    const host = machine.shell.destination;
+    const root = machine.address.root;
+    yield* Effect.annotateCurrentSpan({ worker, host, maxConcurrentAttempts });
+    const connected = yield* commands
+        .inspect("preflight", machine.shell, "true", [])
+        .pipe(Effect.mapError((cause) => capabilityError(worker, "machine", `Can't connect to SSH worker "${worker}" at ${host}`, cause)));
+    if (connected.exitCode !== 0)
+        return yield* capabilityError(worker, "machine", `Can't connect to SSH worker "${worker}" at ${host}`, new Error(connected.stderr.trim() || `ssh exited with code ${String(connected.exitCode)}`));
+    const rootChecked = yield* commands
+        .inspect("preflight", machine.shell, "sh", ["-c", WRITABLE_DIRECTORY_SCRIPT, "sh", root])
+        .pipe(Effect.mapError((cause) => capabilityError(worker, "machine", `Can't inspect ${root} on ${host}`, cause)));
+    if (rootChecked.exitCode !== 0)
+        return yield* capabilityError(worker, "machine", `The root ${root} of SSH worker "${worker}" isn't a writable directory on ${host}`, new Error(rootChecked.stderr.trim() || `exit code ${String(rootChecked.exitCode)}`));
+    const gitVersion = (yield* commands
+        .run("preflight", machine.shell, "git", ["--version"])
+        .pipe(Effect.mapError((cause) => capabilityError(worker, "toolchain", `Worker "${worker}" can't run Git`, cause)))).trim();
+    if (!gitVersion.startsWith("git version "))
+        return yield* capabilityError(worker, "toolchain", `Worker "${worker}" returned an unrecognized Git version`, new Error(gitVersion));
+    yield* verifyLinuxMemory(worker, maxConcurrentAttempts, commands.run("preflight", machine.shell, "cat", ["/proc/meminfo"]));
+    if (Option.isSome(machine.gateway))
+        yield* verifyGateway(commands, worker, machine, machine.gateway.value);
+    const containerCheck = yield* containers.verify(machine, maxConcurrentAttempts);
+    const harnesses = Arr.dedupe(brains.map(({ harness }) => harness));
+    const probe = (shell) => verifyHarnessReadiness(worker, machine.provisioned, brains, harnessProbe(machine, shell), logins);
+    yield* containerCheck._tag === "Container" && harnesses.length > 0
+        ? Effect.scoped(containers
+            .preflightShell(machine, containerCheck.settings, harnesses, Option.match(machine.gateway, {
+            onNone: () => ({}),
+            onSome: ({ settings }) => gatewayExportedEnvironment(settings, `${machine.workerDirectory}/gh`),
+        }))
+            .pipe(Effect.flatMap(probe)))
+        : probe(machine.shell);
+    const fields = {
+        worker,
+        host,
+        root,
+        gitVersion,
+        isolation: containerCheck._tag,
+        harnesses: harnesses.join(","),
+    };
+    return yield* Match.valueTags(containerCheck, {
+        Container: ({ limits }) => Effect.as(Effect.logInfo("SSH Worker preflight verified", { ...fields, ...limitFields(limits) }), {
+            degraded: Option.none(),
+            deferred: Option.none(),
+            containerLimits: Option.some(limits.limits),
+        }),
+        SharedUser: ({ degraded }) => Effect.as(Effect.logInfo("SSH Worker preflight verified", fields), {
+            degraded: Option.some(degraded),
+            deferred: Option.none(),
+            containerLimits: Option.none(),
+        }),
+    });
+});
+/**
+ * Verifies an SSH Worker: a long-lived Worker's machine with `verifyMachine`, or a provisioned
+ * Worker with `verifyProvisioned`, which has no machine before an Attempt.
+ */
+export const sshPreflightFor = (workerFor, verifyMachine, verifyProvisioned) => Effect.fn("SshExecutor.preflight")(function* (worker, brains, maxConcurrentAttempts) {
+    yield* Effect.annotateCurrentSpan({ worker, maxConcurrentAttempts });
+    const entry = yield* workerFor(worker);
+    return yield* Option.match(entry.machine, {
+        onNone: () => verifyProvisioned(entry, brains, maxConcurrentAttempts),
+        onSome: (machine) => verifyMachine(machine, brains, maxConcurrentAttempts),
+    });
+}, logPreflightFailure);
+//# sourceMappingURL=preflight.js.map

package/dist/ssh/preflight.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"preflight.js","sourceRoot":"","sources":["../../src/ssh/preflight.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,IAAI,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC7D,OAAO,EAEL,mBAAmB,EAEnB,qBAAqB,GAEtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAqB,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACpF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAG3D,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI1C,+EAA+E;AAC/E,MAAM,yBAAyB,GAAG,8BAA8B,CAAC;AAEjE,MAAM,eAAe,GAAG,CACtB,MAAkB,EAClB,UAA4B,EAC5B,OAAe,EACf,KAAc,EACS,EAAE,CAAC,IAAI,qBAAqB,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAE9F;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,QAAwB,EACxB,YAAuE,EACvE,MAAkC,EAClC,UAAyB,EACzB,EAAE,CACF,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EAC9C,OAAmB,EACnB,MAAgC,EAChC,qBAA6B;IAE7B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;IAClC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,QAAQ;SAC9B,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;SAC/C,IAAI,CACH,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,eAAe,CACb,MAAM,EACN,SAAS,EACT,gCAAgC,MAAM,QAAQ,IAAI,EAAE,EACpD,KAAK,CACN,CACF,CACF,CAAC;IACJ,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;QAC1B,OAAO,KAAK,CAAC,CAAC,eAAe,CAC3B,MAAM,EACN,SAAS,EACT,gCAAgC,MAAM,QAAQ,IAAI,EAAE,EACpD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,wBAAwB,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAC3F,CAAC;IACJ,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,QAAQ;SAChC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,yBAAyB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;SACxF,IAAI,CACH,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,iBAAiB,IAAI,OAAO,IAAI,EAAE,EAAE,KAAK,CAAC,CAC9E,CACF,CAAC;IACJ,IAAI,WAAW,CAAC,QAAQ,KAAK,CAAC;QAC5B,OAAO,KAAK,CAAC,CAAC,eAAe,CAC3B,MAAM,EACN,SAAS,EACT,YAAY,IAAI,mBAAmB,MAAM,mCAAmC,IAAI,EAAE,EAClF,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,aAAa,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CACpF,CAAC;IACJ,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ;SAChC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC;SACrD,IAAI,CACH,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,eAAe,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,MAAM,iBAAiB,EAAE,KAAK,CAAC,CAChF,CACF,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,cAAc,CAAC;QACxC,OAAO,KAAK,CAAC,CAAC,eAAe,CAC3B,MAAM,EACN,WAAW,EACX,WAAW,MAAM,wCAAwC,EACzD,IAAI,KAAK,CAAC,UAAU,CAAC,CACtB,CAAC;IACJ,KAAK,CAAC,CAAC,iBAAiB,CACtB,MAAM,EACN,qBAAqB,EACrB,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,eAAe,CAAC,CAAC,CACnE,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;QAChC,KAAK,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzE,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,CAAC,KAAkB,EAAE,EAAE,CACnC,sBAAsB,CACpB,MAAM,EACN,OAAO,CAAC,WAAW,EACnB,MAAM,EACN,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,EAC5B,MAAM,CACP,CAAC;IACJ,KAAK,CAAC,CAAC,cAAc,CAAC,IAAI,KAAK,WAAW,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAChE,CAAC,CAAC,MAAM,CAAC,MAAM,CACX,UAAU;aACP,cAAc,CACb,OAAO,EACP,cAAc,CAAC,QAAQ,EACvB,SAAS,EACT,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE;YAC5B,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACvB,0BAA0B,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,eAAe,KAAK,CAAC;SACxE,CAAC,CACH;aACA,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAC/B;QACH,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG;QACb,MAAM;QACN,IAAI;QACJ,IAAI;QACJ,UAAU;QACV,SAAS,EAAE,cAAc,CAAC,IAAI;QAC9B,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC;KAC/B,CAAC;IACF,OAAO,KAAK,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE;QAC5C,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACxB,MAAM,CAAC,EAAE,CACP,MAAM,CAAC,OAAO,CAAC,+BAA+B,EAAE,EAAE,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,EACtF;YACE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE;YACvB,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE;YACvB,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SAClB,CAC5B;QACH,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC3B,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,EAAE,MAAM,CAAC,EAAE;YACjE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC/B,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE;YACvB,eAAe,EAAE,MAAM,CAAC,IAAI,EAAE;SACL,CAAC;KAC/B,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,SAA+E,EAC/E,aAAwD,EACxD,iBAIkD,EAClD,EAAE,CACF,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAC1C,MAAkB,EAClB,MAAgC,EAChC,qBAA6B;IAE7B,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE;QACxC,MAAM,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,qBAAqB,CAAC;QACrE,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,qBAAqB,CAAC;KAC3E,CAAC,CAAC;AACL,CAAC,EAAE,mBAAmB,CAAC,CAAC"}

package/dist/ssh/provisioner/boat.d.ts

@@ -0,0 +1,33 @@
+import { type WorkerName } from "@usine/core/worker";
+import { Effect, Option } from "effect";
+import { MachineUnavailableError, WorkerCapabilityError } from "../../executor.ts";
+import type { BoatProvisioner } from "../model.ts";
+import { type ListedMachine, type MachineTags, ProviderApiError } from "./model.ts";
+/** Boat's public API. */
+export declare const BOAT_API = "https://boat.dev/api/v1";
+/**
+ * Returns the Boat driver for `worker`'s sandboxes (#550). Each sandbox is created with none of
+ * the account's secrets (`noEnv`), no snapshots, which Boat can store outside the EU, no command
+ * monitoring, and no automatic stop, and the executor deletes it. Its name carries the instance,
+ * Worker, Run, creating process, and Attempt, because Boat has no labels. `publicKey` is the
+ * OpenSSH public key each sandbox authorizes.
+ */
+export declare const boatDriverFor: (worker: WorkerName, settings: BoatProvisioner, publicKey: string, base?: string) => Effect.Effect<{
+    provider: "boat";
+    verify: (_worker: string & import("effect/Brand").Brand<"WorkerName">, maxConcurrentAttempts: number) => Effect.Effect<{
+        warnings: string[];
+    }, WorkerCapabilityError, never>;
+    create: (_name: string, tags: MachineTags) => Effect.Effect<{
+        id: string & import("effect/Brand").Brand<"MachineId">;
+        hostKey: Option.Option<never>;
+    }, MachineUnavailableError, never>;
+    endpoint: (id: string & import("effect/Brand").Brand<"MachineId">, interval: Effect.Effect<void, never, never>) => Effect.Effect<{
+        destination: string & import("effect/Brand").Brand<"SshDestination">;
+        port: Option.Option<number & import("effect/Brand").Brand<"SshPort">>;
+        hostKey: Option.Option<string>;
+    }, MachineUnavailableError, never>;
+    tag: (id: string & import("effect/Brand").Brand<"MachineId">, tags: MachineTags) => Effect.Effect<void, ProviderApiError, never>;
+    remove: (id: string & import("effect/Brand").Brand<"MachineId">) => Effect.Effect<void, ProviderApiError, never>;
+    list: (instance: string, listed: string & import("effect/Brand").Brand<"WorkerName">) => Effect.Effect<ListedMachine[], ProviderApiError, never>;
+}, never, import("effect/http/HttpClient").HttpClient>;
+//# sourceMappingURL=boat.d.ts.map

package/dist/ssh/provisioner/boat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"boat.d.ts","sourceRoot":"","sources":["../../../src/ssh/provisioner/boat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAY,MAAM,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AAC1D,OAAO,EACL,uBAAuB,EAEvB,qBAAqB,EACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAEL,KAAK,aAAa,EAElB,KAAK,WAAW,EAChB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAEpB,yBAAyB;AACzB,eAAO,MAAM,QAAQ,4BAA4B,CAAC;AA+FlD;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,WAChB,UAAU,YACR,eAAe,aACd,MAAM,SACX,MAAM;;;;;;;;;;;;;;;;;sDAsPV,CAAC"}

package/dist/ssh/provisioner/boat.js

@@ -0,0 +1,247 @@
+import { MachineId, SshDestination, SshPort } from "@usine/core/worker";
+import { DateTime, Effect, Option, Schema } from "effect";
+import { MachineUnavailableError, WorkerCapabilityError, } from "../../executor.js";
+import { failureText, providerApiFor } from "./http.js";
+import { attemptTag, ProviderApiError, tagValue, } from "./model.js";
+/** Boat's public API. */
+export const BOAT_API = "https://boat.dev/api/v1";
+/** The prefix of the name of every sandbox that usine creates. */
+const NAME_PREFIX = "usine";
+const Sandbox = Schema.Struct({
+    id: Schema.String,
+    name: Schema.String,
+    state: Schema.String,
+    error: Schema.optionalKey(Schema.NullOr(Schema.String)),
+    createdAt: Schema.optionalKey(Schema.NullOr(Schema.String)),
+});
+const OneSandbox = Schema.Struct({ sandbox: Sandbox });
+const SandboxList = Schema.Struct({
+    sandboxes: Schema.Array(Sandbox),
+    pageInfo: Schema.optionalKey(Schema.Struct({ nextCursor: Schema.NullOr(Schema.String), hasMore: Schema.Boolean })),
+});
+const SshKey = Schema.Struct({
+    machineIp: Schema.optionalKey(Schema.NullOr(Schema.String)),
+    sshUser: Schema.optionalKey(Schema.String),
+    sshEndpoint: Schema.optionalKey(Schema.NullOr(Schema.String)),
+    hostKey: Schema.optionalKey(Schema.String),
+});
+const Limits = Schema.Struct({
+    canStart: Schema.optionalKey(Schema.Boolean),
+    activeSandboxes: Schema.optionalKey(Schema.Int),
+    maxActiveSandboxes: Schema.optionalKey(Schema.NullOr(Schema.Int)),
+    starts: Schema.optionalKey(Schema.Struct({
+        unlimited: Schema.optionalKey(Schema.Boolean),
+        day: Schema.optionalKey(Schema.Struct({ remaining: Schema.NullOr(Schema.Int) })),
+    })),
+    machineTypes: Schema.optionalKey(Schema.Array(Schema.Struct({
+        type: Schema.String,
+        available: Schema.Boolean,
+        reason: Schema.optionalKey(Schema.NullOr(Schema.String)),
+    }))),
+});
+/** The states in which a sandbox accepts SSH. */
+const RUNNING = new Set(["ready", "idle", "running"]);
+/** The states from which a new sandbox never runs. */
+const ENDED = new Set(["error", "cancelled", "archiving", "archived"]);
+/** Returns the name that carries a sandbox's tags; at most 120 characters, Boat's limit. */
+const nameOf = (tags) => [
+    NAME_PREFIX,
+    tagValue(tags.instance, 24),
+    tagValue(tags.worker, 24),
+    tagValue(tags.run, 32),
+    tagValue(tags.process, 12),
+    Option.match(tags.attempt, { onNone: () => "-", onSome: attemptTag }),
+].join(" ");
+/** Returns the reason a creation failure is retried, by Boat's documented statuses. */
+const creationReason = (failure) => {
+    if (failure._tag === "ProviderUnanswered")
+        return "unreachable";
+    if (failure.status === 503)
+        return "capacity";
+    if (failure.status === 402 || failure.status === 429)
+        return "quota";
+    if (failure.status === 401 || failure.status === 403 || failure.status === 400)
+        return "refused";
+    return "provider";
+};
+/** Returns where to connect: Boat's `sshEndpoint` forward if set, else the machine's address. */
+const destinationOf = (answer) => {
+    const forwarded = answer.sshEndpoint ?? undefined;
+    if (forwarded !== undefined) {
+        const separator = forwarded.lastIndexOf(":");
+        const port = Number(forwarded.slice(separator + 1));
+        if (separator > 0 && Number.isInteger(port) && port > 0 && port < 65_536)
+            return Option.some({
+                host: forwarded.slice(0, separator),
+                port: Option.some(SshPort.make(port)),
+            });
+    }
+    return Option.map(Option.fromNullishOr(answer.machineIp), (host) => ({
+        host,
+        port: Option.none(),
+    }));
+};
+/**
+ * Returns the Boat driver for `worker`'s sandboxes (#550). Each sandbox is created with none of
+ * the account's secrets (`noEnv`), no snapshots, which Boat can store outside the EU, no command
+ * monitoring, and no automatic stop, and the executor deletes it. Its name carries the instance,
+ * Worker, Run, creating process, and Attempt, because Boat has no labels. `publicKey` is the
+ * OpenSSH public key each sandbox authorizes.
+ */
+export const boatDriverFor = (worker, settings, publicKey, base = BOAT_API) => Effect.gen(function* () {
+    const api = yield* providerApiFor("boat", base, settings.token);
+    const apiError = (operation) => (failure) => new ProviderApiError({
+        provider: "boat",
+        operation,
+        status: failure._tag === "ProviderRefused" ? Option.some(failure.status) : Option.none(),
+        message: `Boat couldn't ${operation}: ${failureText(failure)}`,
+    });
+    const providerGap = (message, cause) => new WorkerCapabilityError({ worker, capability: "provider", message, cause });
+    const unavailable = (reason, message) => new MachineUnavailableError({ worker, provider: "boat", reason, message });
+    const verify = Effect.fn("BoatDriver.verify")(function* (_worker, maxConcurrentAttempts) {
+        yield* Effect.annotateCurrentSpan({ worker, maxConcurrentAttempts });
+        yield* api.send("GET", "/me");
+        const limits = yield* api
+            .send("GET", "/limits")
+            .pipe(Effect.flatMap(api.decode(Limits, "read the limits")));
+        const machineType = limits.machineTypes?.find(({ type }) => type === settings.size);
+        if (machineType !== undefined && !machineType.available)
+            return yield* providerGap(`Boat doesn't offer machine size ${settings.size} to the account of worker "${worker}"${machineType.reason === undefined || machineType.reason === null
+                ? ""
+                : `: ${machineType.reason}`}`, new Error(`size ${settings.size} unavailable`));
+        const maximum = limits.maxActiveSandboxes ?? undefined;
+        const warnings = [
+            ...(limits.canStart === false
+                ? [`Boat reports that the account of worker "${worker}" can't start sandboxes now.`]
+                : []),
+            ...(maximum !== undefined && maximum < maxConcurrentAttempts
+                ? [
+                    `Boat allows ${String(maximum)} active sandboxes, fewer than max_concurrent_attempts (${String(maxConcurrentAttempts)}).`,
+                ]
+                : []),
+        ];
+        yield* Effect.logInfo("Boat provisioner verified", {
+            worker,
+            size: settings.size,
+            canStart: limits.canStart ?? "not reported",
+            activeSandboxes: limits.activeSandboxes ?? "not reported",
+            maxActiveSandboxes: maximum ?? "not reported",
+            startsLeftToday: limits.starts?.unlimited === true
+                ? "unlimited"
+                : (limits.starts?.day?.remaining ?? "not reported"),
+        });
+        return { warnings };
+    }, (effect) => effect.pipe(Effect.catchTags({
+        ProviderUnanswered: (failure) => Effect.fail(providerGap(`Can't reach the Boat API for worker "${worker}": ${failure.detail}`, failure)),
+        ProviderRefused: (failure) => Effect.fail(providerGap(failure.status === 401
+            ? `Boat refused the API key of worker "${worker}". Create one with boat api-key create, and set it in provisioner.token_env.`
+            : `Boat refused a preflight request of worker "${worker}": ${failureText(failure)}`, failure)),
+    })));
+    const create = Effect.fn("BoatDriver.create")(function* (_name, tags) {
+        yield* Effect.annotateCurrentSpan({ worker, run: tags.run });
+        const response = yield* api.send("POST", "/sandboxes", {
+            body: {
+                type: settings.size,
+                name: nameOf(tags),
+                ttlSeconds: null,
+                noEnv: true,
+                snapshots: false,
+                monitoring: false,
+                failFast: true,
+            },
+        });
+        const created = yield* api.decode(OneSandbox, "create a sandbox")(response);
+        return { id: MachineId.make(created.sandbox.id), hostKey: Option.none() };
+    }, (effect) => effect.pipe(Effect.catchTags({
+        ProviderUnanswered: (failure) => Effect.fail(unavailable("unreachable", `Can't reach the Boat API to create a sandbox: ${failure.detail}`)),
+        ProviderRefused: (failure) => Effect.fail(unavailable(creationReason(failure), `Boat didn't create a ${settings.size} sandbox: ${failureText(failure)}`)),
+    })));
+    const endpoint = Effect.fn("BoatDriver.endpoint")(function* (id, interval) {
+        yield* Effect.annotateCurrentSpan({ worker, machine: id });
+        while (true) {
+            const { sandbox } = yield* api
+                .send("GET", `/sandboxes/${id}`)
+                .pipe(Effect.flatMap(api.decode(OneSandbox, "read the sandbox")));
+            if (RUNNING.has(sandbox.state))
+                break;
+            if (ENDED.has(sandbox.state))
+                return yield* unavailable("capacity", `The new Boat sandbox ended in state ${sandbox.state}${sandbox.error === undefined || sandbox.error === null ? "" : `: ${sandbox.error}`}`);
+            yield* Effect.logDebug("Waiting for the Boat sandbox to be ready", {
+                worker,
+                machine: id,
+                state: sandbox.state,
+            });
+            yield* interval;
+        }
+        const answer = yield* api
+            .send("POST", `/sandboxes/${id}/sshkey`, { body: { key: publicKey } })
+            .pipe(Effect.flatMap(api.decode(SshKey, "add the SSH key")));
+        const target = destinationOf(answer);
+        if (Option.isNone(target))
+            return yield* unavailable("provider", "Boat reported no SSH address for the new sandbox");
+        const user = answer.sshUser ?? "user";
+        return {
+            destination: SshDestination.make(`${user}@${target.value.host}`),
+            port: target.value.port,
+            hostKey: Option.fromUndefinedOr(answer.hostKey),
+        };
+    }, (effect) => effect.pipe(Effect.catchTags({
+        ProviderUnanswered: (failure) => Effect.fail(unavailable("unreachable", `Can't reach the Boat API to read a new sandbox: ${failure.detail}`)),
+        ProviderRefused: (failure) => Effect.fail(unavailable(failure.status === 404 ? "capacity" : creationReason(failure), failure.status === 404
+            ? "Boat removed the new sandbox before it was ready"
+            : `Boat couldn't ready the new sandbox: ${failureText(failure)}`)),
+    })));
+    const tag = Effect.fn("BoatDriver.tag")(function* (id, tags) {
+        yield* Effect.annotateCurrentSpan({ worker, machine: id });
+        yield* api
+            .send("PATCH", `/sandboxes/${id}`, { body: { name: nameOf(tags) } })
+            .pipe(Effect.mapError(apiError("rename the sandbox")));
+    });
+    const remove = Effect.fn("BoatDriver.remove")(function* (id) {
+        yield* Effect.annotateCurrentSpan({ worker, machine: id });
+        yield* api
+            .send("DELETE", `/sandboxes/${id}`, { headers: { "X-Ascii-Confirm-Delete": id } })
+            .pipe(Effect.asVoid, Effect.catchIf((failure) => failure._tag === "ProviderRefused" && failure.status === 404, () => Effect.void), Effect.mapError(apiError("delete the sandbox")));
+    });
+    const list = Effect.fn("BoatDriver.list")(function* (instance, listed) {
+        yield* Effect.annotateCurrentSpan({ worker: listed });
+        const prefix = `${NAME_PREFIX} ${tagValue(instance, 24)} ${tagValue(listed, 24)} `;
+        const machines = [];
+        let cursor = null;
+        do {
+            const query = new Map([["limit", "200"]]);
+            if (cursor !== null)
+                query.set("cursor", cursor);
+            const page = yield* api
+                .send("GET", "/sandboxes", { query: Object.fromEntries(query) })
+                .pipe(Effect.flatMap(api.decode(SandboxList, "list sandboxes")), Effect.mapError(apiError("list the sandboxes")));
+            for (const sandbox of page.sandboxes) {
+                if (!sandbox.name.startsWith(prefix))
+                    continue;
+                const [, , workerTag = "", run = "", process = "", attempt = "-"] = sandbox.name.split(" ");
+                machines.push({
+                    id: MachineId.make(sandbox.id),
+                    tags: {
+                        worker: workerTag,
+                        run,
+                        process,
+                        attempt: attempt === "-" ? Option.none() : Option.some(attempt),
+                    },
+                    createdAt: Option.getOrElse(Option.flatMap(Option.fromNullishOr(sandbox.createdAt), (at) => DateTime.make(at)), () => DateTime.makeUnsafe(0)),
+                });
+            }
+            cursor = page.pageInfo?.hasMore === true ? page.pageInfo.nextCursor : null;
+        } while (cursor !== null);
+        return machines;
+    });
+    return {
+        provider: "boat",
+        verify,
+        create,
+        endpoint,
+        tag,
+        remove,
+        list,
+    };
+});
+//# sourceMappingURL=boat.js.map

package/dist/ssh/provisioner/boat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"boat.js","sourceRoot":"","sources":["../../../src/ssh/provisioner/boat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAmB,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EACL,uBAAuB,EAEvB,qBAAqB,GACtB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,WAAW,EAAwB,cAAc,EAAE,MAAM,WAAW,CAAC;AAC9E,OAAO,EACL,UAAU,EAIV,gBAAgB,EAChB,QAAQ,GACT,MAAM,YAAY,CAAC;AAEpB,yBAAyB;AACzB,MAAM,CAAC,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAElD,kEAAkE;AAClE,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,EAAE,EAAE,MAAM,CAAC,MAAM;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,KAAK,EAAE,MAAM,CAAC,MAAM;IACpB,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;CAC5D,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AAEvD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAC1B,MAAM,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CACrF;CACF,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3D,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;CAC3C,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;IAC5C,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC;IAC/C,kBAAkB,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,MAAM,CAAC;QACZ,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7C,GAAG,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACjF,CAAC,CACH;IACD,YAAY,EAAE,MAAM,CAAC,WAAW,CAC9B,MAAM,CAAC,KAAK,CACV,MAAM,CAAC,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC,MAAM;QACnB,SAAS,EAAE,MAAM,CAAC,OAAO;QACzB,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;KACzD,CAAC,CACH,CACF;CACF,CAAC,CAAC;AAEH,iDAAiD;AACjD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;AACtD,sDAAsD;AACtD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEvE,4FAA4F;AAC5F,MAAM,MAAM,GAAG,CAAC,IAAiB,EAAU,EAAE,CAC3C;IACE,WAAW;IACX,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;IAC3B,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;IAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;CACtE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEd,uFAAuF;AACvF,MAAM,cAAc,GAAG,CAAC,OAAwB,EAA4B,EAAE;IAC5E,IAAI,OAAO,CAAC,IAAI,KAAK,oBAAoB;QAAE,OAAO,aAAa,CAAC;IAChE,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,UAAU,CAAC;IAC9C,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,OAAO,CAAC;IACrE,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACjG,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,aAAa,GAAG,CACpB,MAA0B,EACuD,EAAE;IACnF,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC;IAClD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,MAAM;YACtE,OAAO,MAAM,CAAC,IAAI,CAAC;gBACjB,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;gBACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACtC,CAAC,CAAC;IACP,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;KACpB,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,MAAkB,EAClB,QAAyB,EACzB,SAAiB,EACjB,IAAI,GAAW,QAAQ,EACvB,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEhE,MAAM,QAAQ,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,CAAC,OAAwB,EAAE,EAAE,CACnE,IAAI,gBAAgB,CAAC;QACnB,QAAQ,EAAE,MAAM;QAChB,SAAS;QACT,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;QACxF,OAAO,EAAE,iBAAiB,SAAS,KAAK,WAAW,CAAC,OAAO,CAAC,EAAE;KAC/D,CAAC,CAAC;IAEL,MAAM,WAAW,GAAG,CAAC,OAAe,EAAE,KAAc,EAAE,EAAE,CACtD,IAAI,qBAAqB,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAEhF,MAAM,WAAW,GAAG,CAAC,MAAgC,EAAE,OAAe,EAAE,EAAE,CACxE,IAAI,uBAAuB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAE7E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAC3C,QAAQ,CAAC,EAAE,OAAmB,EAAE,qBAA6B;QAC3D,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;QACrE,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG;aACtB,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC;aACtB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpF,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,SAAS;YACrD,OAAO,KAAK,CAAC,CAAC,WAAW,CACvB,mCAAmC,QAAQ,CAAC,IAAI,8BAA8B,MAAM,IAClF,WAAW,CAAC,MAAM,KAAK,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,IAAI;gBAC7D,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,KAAK,WAAW,CAAC,MAAM,EAC7B,EAAE,EACF,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,IAAI,cAAc,CAAC,CAC/C,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,CAAC,kBAAkB,IAAI,SAAS,CAAC;QACvD,MAAM,QAAQ,GAAG;YACf,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK;gBAC3B,CAAC,CAAC,CAAC,4CAA4C,MAAM,8BAA8B,CAAC;gBACpF,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,GAAG,qBAAqB;gBAC1D,CAAC,CAAC;oBACE,eAAe,MAAM,CAAC,OAAO,CAAC,0DAA0D,MAAM,CAAC,qBAAqB,CAAC,IAAI;iBAC1H;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;QACF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,2BAA2B,EAAE;YACjD,MAAM;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,cAAc;YAC3C,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,cAAc;YACzD,kBAAkB,EAAE,OAAO,IAAI,cAAc;YAC7C,eAAe,EACb,MAAM,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI;gBAC/B,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,IAAI,cAAc,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,EAAE,QAAQ,EAAE,CAAC;IACtB,CAAC,EACD,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,SAAS,CAAC;QACf,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE,CAC9B,MAAM,CAAC,IAAI,CACT,WAAW,CACT,wCAAwC,MAAM,MAAM,OAAO,CAAC,MAAM,EAAE,EACpE,OAAO,CACR,CACF;QACH,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAC3B,MAAM,CAAC,IAAI,CACT,WAAW,CACT,OAAO,CAAC,MAAM,KAAK,GAAG;YACpB,CAAC,CAAC,uCAAuC,MAAM,8EAA8E;YAC7H,CAAC,CAAC,+CAA+C,MAAM,MAAM,WAAW,CAAC,OAAO,CAAC,EAAE,EACrF,OAAO,CACR,CACF;KACJ,CAAC,CACH,CACJ,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAC3C,QAAQ,CAAC,EAAE,KAAa,EAAE,IAAiB;QACzC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE;YACrD,IAAI,EAAE;gBACJ,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC;gBAClB,UAAU,EAAE,IAAI;gBAChB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,KAAK;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACf;SACF,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5E,OAAO,EAAE,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IAC5E,CAAC,EACD,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,SAAS,CAAC;QACf,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE,CAC9B,MAAM,CAAC,IAAI,CACT,WAAW,CACT,aAAa,EACb,iDAAiD,OAAO,CAAC,MAAM,EAAE,CAClE,CACF;QACH,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAC3B,MAAM,CAAC,IAAI,CACT,WAAW,CACT,cAAc,CAAC,OAAO,CAAC,EACvB,wBAAwB,QAAQ,CAAC,IAAI,aAAa,WAAW,CAAC,OAAO,CAAC,EAAE,CACzE,CACF;KACJ,CAAC,CACH,CACJ,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC,qBAAqB,CAAC,CAC/C,QAAQ,CAAC,EAAE,EAAa,EAAE,QAA6B;QACrD,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3D,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG;iBAC3B,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC;iBAC/B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACpE,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,MAAM;YACtC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC1B,OAAO,KAAK,CAAC,CAAC,WAAW,CACvB,UAAU,EACV,uCAAuC,OAAO,CAAC,KAAK,GAClD,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,EACjF,EAAE,CACH,CAAC;YACJ,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,0CAA0C,EAAE;gBACjE,MAAM;gBACN,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YACH,KAAK,CAAC,CAAC,QAAQ,CAAC;QAClB,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG;aACtB,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC;aACrE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;YACvB,OAAO,KAAK,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAAC;QAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;QACtC,OAAO;YACL,WAAW,EAAE,cAAc,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAChE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI;YACvB,OAAO,EAAE,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;SAChD,CAAC;IACJ,CAAC,EACD,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,SAAS,CAAC;QACf,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE,CAC9B,MAAM,CAAC,IAAI,CACT,WAAW,CACT,aAAa,EACb,mDAAmD,OAAO,CAAC,MAAM,EAAE,CACpE,CACF;QACH,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE,CAC3B,MAAM,CAAC,IAAI,CACT,WAAW,CACT,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,EAC7D,OAAO,CAAC,MAAM,KAAK,GAAG;YACpB,CAAC,CAAC,kDAAkD;YACpD,CAAC,CAAC,wCAAwC,WAAW,CAAC,OAAO,CAAC,EAAE,CACnE,CACF;KACJ,CAAC,CACH,CACJ,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAa,EAAE,IAAiB;QACjF,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3D,KAAK,CAAC,CAAC,GAAG;aACP,IAAI,CAAC,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAa;QACpE,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3D,KAAK,CAAC,CAAC,GAAG;aACP,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,wBAAwB,EAAE,EAAE,EAAE,EAAE,CAAC;aACjF,IAAI,CACH,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,OAAO,CACZ,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,iBAAiB,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,EACzE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAClB,EACD,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAChD,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,EAAE,QAAgB,EAAE,MAAkB;QACvF,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,GAAG,WAAW,IAAI,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC;QACnF,MAAM,QAAQ,GAAyB,EAAE,CAAC;QAC1C,IAAI,MAAM,GAAkB,IAAI,CAAC;QACjC,GAAG,CAAC;YACF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,MAAM,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACjD,MAAM,IAAI,GAA4B,KAAK,CAAC,CAAC,GAAG;iBAC7C,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;iBAC/D,IAAI,CACH,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,EACzD,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAChD,CAAC;YACJ,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACrC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBAAE,SAAS;gBAC/C,MAAM,CAAC,EAAE,AAAD,EAAG,SAAS,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,GAAG,GAAG,CAAC,GAC/D,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC1B,QAAQ,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9B,IAAI,EAAE;wBACJ,MAAM,EAAE,SAAS;wBACjB,GAAG;wBACH,OAAO;wBACP,OAAO,EAAE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;qBAChE;oBACD,SAAS,EAAE,MAAM,CAAC,SAAS,CACzB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAClF,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAC7B;iBACF,CAAC,CAAC;YACL,CAAC;YACD,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7E,CAAC,QAAQ,MAAM,KAAK,IAAI,EAAE;QAC1B,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,MAAM;QACN,MAAM;QACN,QAAQ;QACR,GAAG;QACH,MAAM;QACN,IAAI;KACmB,CAAC;AAC5B,CAAC,CAAC,CAAC"}