@usine/executor 0.2.1 → 0.4.0

package/dist/ssh/gateway/preflight.js

@@ -0,0 +1,320 @@
+import { SSH_FAILURE } from "@usine/agents/child-host";
+import { unsupportedCodexVersion } from "@usine/agents/preflight";
+import { HarnessReady, HarnessUnavailableError } from "@usine/core/agent/capability";
+import { Effect, Option, Redacted, Ref } from "effect";
+import { ExecutorOperationError, WorkerCapabilityError } from "../../executor.js";
+import { sendingShell } from "../../remote/commands.js";
+import { forwardProxyUrl, GATEWAY_SENT_VARIABLES, gatewayEnvironment } from "./environment.js";
+/** A placeholder caller token that lets preflight check the sent variables without attaching anything. */
+const PROBE_TOKEN = Redacted.make("usine-preflight");
+/** A script that takes name and value pairs and prints each name whose variable doesn't have that value on the machine. */
+const SENT_VARIABLES_SCRIPT = `
+while [ "$#" -gt 1 ]; do
+  test "$(printenv "$1")" = "$2" || echo "$1"
+  shift 2
+done
+`.trim();
+/** Matches the Gateway's refusal header for a request without a caller token. */
+const REFUSED_WITHOUT_TOKEN = /^x-usine-gateway-refusal:\s*missing-caller-token\s*$/imu;
+const gatewayGap = (worker, message, cause) => new WorkerCapabilityError({ worker, capability: "gateway", message, cause });
+/** Returns the error for an SSH transport failure, which isn't a capability gap. */
+const unreachable = (host, command, result) => new ExecutorOperationError({
+    operation: "preflight",
+    message: `Can't reach ${host} to run ${command}`,
+    cause: new Error(result.stderr.trim() || `ssh exited with code ${SSH_FAILURE}`),
+});
+/**
+ * Logs a warning, once per Worker, that the Gateway is too old to report its Anthropic
+ * credential kind (#248). Later preflights log at debug level.
+ */
+const warnOfUnreportedCredential = (worker, gateway) => Ref.getAndSet(gateway.warnedOfUnreportedCredential, true).pipe(Effect.flatMap((warned) => warned
+    ? Effect.logDebug("SSH Worker's Gateway still does not report the kind of its Anthropic credential; already warned for this Worker", { worker, gateway: gateway.forward.gateway })
+    : Effect.logWarning("SSH Worker's Gateway does not report the kind of its Anthropic credential; its Claude Code children count in the Claude plan pool until the Gateway is upgraded", { worker, gateway: gateway.forward.gateway })));
+/** Each provider's display name and the Gateway variables that set its credential. */
+const PROVIDER_CREDENTIALS = {
+    anthropic: {
+        label: "Anthropic",
+        variables: "ANTHROPIC_API_KEY or USINE_GATEWAY_ANTHROPIC_SUBSCRIPTION_TOKEN",
+    },
+    openai: { label: "OpenAI", variables: "OPENAI_API_KEY or USINE_GATEWAY_CHATGPT_AUTH_FILE" },
+};
+const NO_CREDENTIALS = { anthropic: Option.none(), openai: Option.none() };
+/**
+ * Reads the Gateway's credential kind for each provider in the Worker's `llmProviders` and
+ * records them for the Worker (ADR 0013). Fails if the Gateway has no credential for a listed
+ * provider. If the Gateway is too old to report credentials, keeps the Claude plan pool for
+ * Anthropic with a warning, and fails for OpenAI, which such a Gateway can't serve for Codex.
+ */
+const recordProviderCredentials = Effect.fn("SshExecutor.recordProviderCredentials")(function* (worker, gateway) {
+    const listed = gateway.settings.llmProviders;
+    yield* Effect.annotateCurrentSpan({
+        worker,
+        gateway: gateway.forward.gateway,
+        llmProviders: listed.join(","),
+    });
+    if (listed.length === 0) {
+        yield* Ref.set(gateway.credentials, NO_CREDENTIALS);
+        return NO_CREDENTIALS;
+    }
+    const reported = yield* gateway.admin.providers.pipe(Effect.mapError((error) => gatewayGap(worker, error.message, error)));
+    if (Option.isNone(reported)) {
+        if (listed.includes("openai"))
+            return yield* gatewayGap(worker, `The gateway of SSH worker "${worker}" doesn't report its LLM credentials, so Codex can't use its OpenAI credential. Upgrade @usine/gateway on the gateway, or remove openai from llm_providers.`, new Error("the gateway answered 404 for GET /v1/providers"));
+        yield* warnOfUnreportedCredential(worker, gateway);
+        yield* Ref.set(gateway.credentials, NO_CREDENTIALS);
+        return NO_CREDENTIALS;
+    }
+    const { providers } = reported.value;
+    const kindOf = (provider) => Option.fromNullishOr(providers.find((held) => held.provider === provider)?.credential);
+    const missing = listed.find((provider) => Option.isNone(kindOf(provider)));
+    if (missing !== undefined)
+        return yield* gatewayGap(worker, `The gateway of SSH worker "${worker}" has no ${PROVIDER_CREDENTIALS[missing].label} credential, but the worker's llm_providers includes ${missing}. Set ${PROVIDER_CREDENTIALS[missing].variables} on the gateway, or remove ${missing} from llm_providers.`, new Error(`the gateway reports ${providers.map(({ provider }) => provider).join(", ") || "no provider"}`));
+    const credentials = {
+        anthropic: listed.includes("anthropic") ? kindOf("anthropic") : Option.none(),
+        openai: listed.includes("openai") ? kindOf("openai") : Option.none(),
+    };
+    yield* Ref.set(gateway.credentials, credentials);
+    return credentials;
+});
+/** Returns how a provider's harness authenticates on the Worker, for log lines. */
+const credentialMode = (kind, listed) => Option.match(kind, {
+    onNone: () => (listed ? "gateway-unreported" : "worker-login"),
+    onSome: (credential) => `gateway-${credential}`,
+});
+/**
+ * Returns the credential kind the Gateway injects for `harness`, as recorded by the last
+ * preflight: Anthropic's for Claude Code, OpenAI's for Codex. Pi uses its own login.
+ */
+export const gatewayCredentialFor = (gateway, harness) => {
+    switch (harness) {
+        case "claude":
+            return Effect.map(Ref.get(gateway.credentials), ({ anthropic }) => anthropic);
+        case "codex":
+            return Effect.map(Ref.get(gateway.credentials), ({ openai }) => openai);
+        case "pi":
+            return Effect.succeedNone;
+    }
+};
+/**
+ * Verifies the Gateway's admin API for an SSH Worker without attaching anything: the forward
+ * opens and the admin API accepts the admin token. Records the Gateway's credential kind for
+ * each provider in the Worker's `llmProviders`. Needs no Worker machine.
+ */
+export const verifyGatewayAdmin = Effect.fn("SshExecutor.verifyGatewayAdmin")(function* (worker, gateway) {
+    const { admin, forward, settings } = gateway;
+    yield* Effect.annotateCurrentSpan({ worker, domain: settings.domain, gateway: forward.gateway });
+    const open = yield* forward.current.pipe(Effect.catchTag("GatewayForwardClosed", (closed) => forward.reopen(closed.generation)), Effect.mapError((closed) => gatewayGap(worker, `Can't open the forward to the gateway admin API of SSH worker "${worker}" through ${closed.gateway}: ${closed.reason}. Run ${forward.command} to see the SSH error.`, closed)));
+    const { attachments } = yield* admin.list.pipe(Effect.mapError((error) => gatewayGap(worker, error.message, error)));
+    const credentials = yield* recordProviderCredentials(worker, gateway);
+    yield* Effect.logInfo("SSH Worker Gateway admin API verified", {
+        worker,
+        domain: settings.domain,
+        gateway: forward.gateway,
+        adminPort: settings.adminPort,
+        forwardPort: open.url.port,
+        llmProviders: settings.llmProviders.join(","),
+        anthropic: credentialMode(credentials.anthropic, settings.llmProviders.includes("anthropic")),
+        openai: credentialMode(credentials.openai, settings.llmProviders.includes("openai")),
+        attachments: attachments.filter((attachment) => attachment.worker === worker).length,
+    });
+});
+/**
+ * Verifies the Gateway from an SSH Worker's machine without attaching anything: the machine's
+ * sshd accepts the caller-token variables, and the machine can connect to `git.<domain>`, which
+ * rejects a request without a caller token.
+ */
+const verifyGatewayFromMachine = Effect.fn("SshExecutor.verifyGatewayFromMachine")(function* (commands, worker, machine, gateway) {
+    const { forward, settings } = gateway;
+    const gitHost = `git.${settings.domain}`;
+    yield* Effect.annotateCurrentSpan({ worker, domain: settings.domain, gateway: forward.gateway });
+    const { sent } = gatewayEnvironment({
+        domain: settings.domain,
+        llmProviders: settings.llmProviders,
+        ghConfigDirectory: machine.workerDirectory,
+        codexHome: machine.workerDirectory,
+        callerToken: PROBE_TOKEN,
+    });
+    const received = yield* commands.run("preflight", sendingShell(machine.shell, sent), "sh", [
+        "-c",
+        SENT_VARIABLES_SCRIPT,
+        "sh",
+        ...Object.entries(sent).flatMap(([name, value]) => [name, Redacted.value(value)]),
+    ]);
+    const dropped = received.split("\n").filter((name) => name !== "");
+    if (dropped.length > 0)
+        return yield* gatewayGap(worker, `The sshd on SSH worker "${worker}" doesn't accept ${dropped.join(", ")}. Add AcceptEnv ${GATEWAY_SENT_VARIABLES.join(" ")} to its sshd configuration.`, new Error(`variables not received: ${dropped.join(" ")}`));
+    const probed = yield* commands.inspect("preflight", machine.shell, "curl", [
+        "-sS",
+        "--noproxy",
+        "*",
+        "--max-time",
+        "15",
+        "-o",
+        "/dev/null",
+        "-D",
+        "-",
+        `http://${gitHost}/`,
+    ]);
+    if (probed.exitCode === SSH_FAILURE)
+        return yield* unreachable(machine.shell.destination, "curl", probed);
+    if (probed.exitCode !== 0 || !REFUSED_WITHOUT_TOKEN.test(probed.stdout))
+        return yield* gatewayGap(worker, `SSH worker "${worker}" can't reach its gateway at ${gitHost}`, new Error(probed.stderr.trim() ||
+            probed.stdout.split("\n")[0]?.trim() ||
+            `curl exited with code ${String(probed.exitCode)}`));
+    yield* Effect.logInfo("SSH Worker reaches its Gateway", {
+        worker,
+        host: machine.shell.destination,
+        domain: settings.domain,
+        gateway: forward.gateway,
+    });
+});
+/**
+ * Verifies the Gateway in front of an SSH Worker without attaching anything: the admin API
+ * checks of `verifyGatewayAdmin`, then the machine checks of `verifyGatewayFromMachine`.
+ */
+export const verifyGateway = (commands, worker, machine, gateway) => verifyGatewayAdmin(worker, gateway).pipe(Effect.andThen(verifyGatewayFromMachine(commands, worker, machine, gateway)));
+/**
+ * Checks only the binary of a harness whose credential the machine doesn't keep, the Gateway's
+ * or an API key in the Attempt's environment, named by `credential` in the log. Codex's version
+ * must also be supported.
+ */
+export const harnessBinaryProbe = Effect.fn("SshExecutor.probeHarnessBinary")(function* (commands, host, shell, harness, credential) {
+    const label = harness === "claude" ? "Claude Code" : "Codex";
+    yield* Effect.annotateCurrentSpan({ host, harness, credential });
+    const result = yield* commands.inspect("preflight", shell, harness, ["--version"]);
+    if (result.exitCode === SSH_FAILURE)
+        return yield* unreachable(host, harness, result);
+    if (result.exitCode !== 0)
+        return new HarnessUnavailableError({
+            harness,
+            capability: "binary",
+            message: `${harness} --version failed. Check the ${label} installation on the worker.`,
+        });
+    const version = result.stdout.trim();
+    const unsupported = harness === "codex" ? unsupportedCodexVersion(version) : Option.none();
+    if (Option.isSome(unsupported))
+        return unsupported.value;
+    yield* Effect.logInfo(`${label} uses ${credential}; checked its binary only`, {
+        host,
+        harness,
+        version,
+    });
+    return new HarnessReady({ harness });
+});
+/**
+ * The harnesses that can use the operator's login on the machine or the Gateway's credential
+ * for their provider, with the hosts each login calls through the forward proxy, which
+ * `infra/hetzner/gateway/egress-allowlist` lists, and the API host that an API key in the
+ * Attempt's environment calls.
+ */
+const LOGINS = {
+    claude: {
+        label: "Claude Code",
+        provider: "Anthropic",
+        gatewayProvider: "anthropic",
+        hosts: ["api.anthropic.com", "platform.claude.com"],
+        apiHost: "api.anthropic.com",
+    },
+    codex: {
+        label: "Codex",
+        provider: "OpenAI",
+        gatewayProvider: "openai",
+        hosts: ["chatgpt.com", "auth.openai.com"],
+        apiHost: "api.openai.com",
+    },
+};
+/**
+ * A script that takes the forward proxy URL and host names, and prints one line per host: the
+ * host, curl's exit code, the proxy's CONNECT status, and the host's HTTP status (`000` if the
+ * host didn't respond).
+ */
+const LOGIN_HOSTS_SCRIPT = `
+proxy=$1
+shift
+for host in "$@"; do
+  codes=$(curl -s -o /dev/null --max-time 10 --proxy "$proxy" -w '%{http_connect} %{http_code}' "https://$host/")
+  echo "$host $? $codes"
+done
+`.trim();
+const LOGIN_HOST_ANSWER = /^(\S+) (\d+) (\d{3}) (\d{3})$/u;
+/** The CONNECT status the forward proxy returns for a host that isn't in its allowlist. */
+const PROXY_REFUSED = "403";
+/**
+ * Probes a brain on a Worker behind a Gateway (ADR 0027). If the Gateway has the credential of
+ * the harness's provider, checks only the harness binary. Otherwise, Claude Code and Codex use
+ * the operator's login (ADR 0026), or an API key in the Attempt's environment if `apiKeyed`
+ * returns `true` for the harness: `probe` checks the login or the binary, then the machine must
+ * connect to the login's hosts, or the provider's API host, through the forward proxy. Pi goes
+ * to `probe`, which rejects it on any remote machine. The checks run through `shell`, on the
+ * machine or in a container on it.
+ */
+export const gatewayHarnessProbe = (commands, machine, gateway, probe, shell, apiKeyed) => {
+    const worker = machine.address.name;
+    const host = machine.shell.destination;
+    const { domain } = gateway.settings;
+    const proxy = forwardProxyUrl(domain);
+    /** Checks only the binary of a harness whose provider credential the Gateway holds. */
+    const binaryOnly = (harness) => harnessBinaryProbe(commands, host, shell, harness, `the Gateway's ${LOGINS[harness].provider} credential`);
+    const loginHostsReachable = Effect.fn("SshExecutor.probeLoginHostsBehindGateway")(function* (harness) {
+        const { label } = LOGINS[harness];
+        const keyed = apiKeyed(harness);
+        const hosts = keyed ? [LOGINS[harness].apiHost] : LOGINS[harness].hosts;
+        const uses = keyed ? "an API key" : "its login";
+        const alternative = keyed
+            ? `, or add ${LOGINS[harness].gatewayProvider} to the worker's llm_providers with the key on the gateway`
+            : "";
+        yield* Effect.annotateCurrentSpan({ worker, host, domain, harness });
+        const result = yield* commands.inspect("preflight", shell, "sh", [
+            "-c",
+            LOGIN_HOSTS_SCRIPT,
+            "sh",
+            proxy,
+            ...hosts,
+        ]);
+        if (result.exitCode === SSH_FAILURE)
+            return yield* unreachable(host, "curl", result);
+        const answers = new Map(result.stdout.split("\n").flatMap((line) => {
+            const match = LOGIN_HOST_ANSWER.exec(line.trim());
+            return match === null
+                ? []
+                : [[match[1], { exitCode: match[2], connect: match[3], status: match[4] }]];
+        }));
+        for (const loginHost of hosts) {
+            const answer = answers.get(loginHost);
+            if (answer !== undefined && answer.status !== "000")
+                continue;
+            const cause = new Error(answer === undefined
+                ? `no response from ${loginHost}: ${result.stderr.trim() || `sh exited with code ${String(result.exitCode)}`}`
+                : `curl exited with code ${answer.exitCode}; the proxy returned ${answer.connect} for CONNECT`);
+            return yield* gatewayGap(worker, answer?.connect === PROXY_REFUSED
+                ? `${label} uses ${uses} on SSH worker "${worker}", and the gateway's forward proxy blocks ${loginHost}. Add ${loginHost} to the gateway's egress allowlist${alternative}.`
+                : `${label} uses ${uses} on SSH worker "${worker}", which can't reach ${loginHost} through the gateway's forward proxy at ${proxy}`, cause);
+        }
+        yield* Effect.logInfo("SSH Worker reaches its harness login's hosts through the Gateway", {
+            worker,
+            harness,
+            domain,
+            hosts: hosts.join(","),
+        });
+    });
+    const loginOnMachine = Effect.fn("SshExecutor.probeLoginBehindGateway")(function* (brain, harness) {
+        yield* Effect.annotateCurrentSpan({ worker, host, domain, harness });
+        const readiness = yield* probe(brain);
+        if (readiness._tag === "HarnessUnavailableError")
+            return readiness;
+        yield* loginHostsReachable(harness);
+        return readiness;
+    });
+    return (brain) => {
+        switch (brain.harness) {
+            case "claude":
+            case "codex":
+                return gateway.settings.llmProviders.includes(LOGINS[brain.harness].gatewayProvider)
+                    ? binaryOnly(brain.harness)
+                    : loginOnMachine(brain, brain.harness);
+            case "pi":
+                return probe(brain);
+        }
+    };
+};
+//# sourceMappingURL=preflight.js.map

package/dist/ssh/gateway/preflight.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"preflight.js","sourceRoot":"","sources":["../../../src/ssh/gateway/preflight.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAElE,OAAO,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAErF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAElF,OAAO,EAAuB,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE7E,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE/F,0GAA0G;AAC1G,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAErD,2HAA2H;AAC3H,MAAM,qBAAqB,GAAG;;;;;CAK7B,CAAC,IAAI,EAAE,CAAC;AAET,iFAAiF;AACjF,MAAM,qBAAqB,GAAG,yDAAyD,CAAC;AAExF,MAAM,UAAU,GAAG,CAAC,MAAkB,EAAE,OAAe,EAAE,KAAc,EAAE,EAAE,CACzE,IAAI,qBAAqB,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;AAE/E,oFAAoF;AACpF,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,OAAe,EAAE,MAAqB,EAAE,EAAE,CAC3E,IAAI,sBAAsB,CAAC;IACzB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,eAAe,IAAI,WAAW,OAAO,EAAE;IAChD,KAAK,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,wBAAwB,WAAW,EAAE,CAAC;CAChF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,0BAA0B,GAAG,CAAC,MAAkB,EAAE,OAAuB,EAAE,EAAE,CACjF,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,4BAA4B,EAAE,IAAI,CAAC,CAAC,IAAI,CAC5D,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CACxB,MAAM;IACJ,CAAC,CAAC,MAAM,CAAC,QAAQ,CACb,iHAAiH,EACjH,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAC7C;IACH,CAAC,CAAC,MAAM,CAAC,UAAU,CACf,iKAAiK,EACjK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAC7C,CACN,CACF,CAAC;AAEJ,sFAAsF;AACtF,MAAM,oBAAoB,GAAG;IAC3B,SAAS,EAAE;QACT,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,iEAAiE;KAC7E;IACD,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,mDAAmD,EAAE;CAI5F,CAAC;AAEF,MAAM,cAAc,GAAuB,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;AAE/F;;;;;GAKG;AACH,MAAM,yBAAyB,GAAG,MAAM,CAAC,EAAE,CAAC,uCAAuC,CAAC,CAAC,QAAQ,CAAC,EAC5F,MAAkB,EAClB,OAAuB;IAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IAC7C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC;QAChC,MAAM;QACN,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO;QAChC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;KAC/B,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;QACpD,OAAO,cAAc,CAAC;IACxB,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAClD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CACrE,CAAC;IACF,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAC3B,OAAO,KAAK,CAAC,CAAC,UAAU,CACtB,MAAM,EACN,8BAA8B,MAAM,6JAA6J,EACjM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAC5D,CAAC;QACJ,KAAK,CAAC,CAAC,0BAA0B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnD,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;QACpD,OAAO,cAAc,CAAC;IACxB,CAAC;IACD,MAAM,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC;IACrC,MAAM,MAAM,GAAG,CAAC,QAA4B,EAAE,EAAE,CAC9C,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,UAAU,CAAC,CAAC;IACzF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3E,IAAI,OAAO,KAAK,SAAS;QACvB,OAAO,KAAK,CAAC,CAAC,UAAU,CACtB,MAAM,EACN,8BAA8B,MAAM,YAAY,oBAAoB,CAAC,OAAO,CAAC,CAAC,KAAK,wDAAwD,OAAO,SAAS,oBAAoB,CAAC,OAAO,CAAC,CAAC,SAAS,8BAA8B,OAAO,sBAAsB,EAC7P,IAAI,KAAK,CACP,uBAAuB,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,aAAa,EAAE,CAC/F,CACF,CAAC;IACJ,MAAM,WAAW,GAAuB;QACtC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;QAC7E,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;KACrE,CAAC;IACF,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IACjD,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC,CAAC;AAEH,mFAAmF;AACnF,MAAM,cAAc,GAAG,CAAC,IAA0C,EAAE,MAAe,EAAE,EAAE,CACrF,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE;IACjB,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC;IAC9D,MAAM,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,UAAU,EAAE;CAChD,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,OAAuB,EACvB,OAAgB,EACqC,EAAE;IACvD,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;QAChF,KAAK,OAAO;YACV,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC1E,KAAK,IAAI;YACP,OAAO,MAAM,CAAC,WAAW,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC,QAAQ,CAAC,EACrF,MAAkB,EAClB,OAAuB;IAEvB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACjG,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CACtC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EACtF,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CACzB,UAAU,CACR,MAAM,EACN,kEAAkE,MAAM,aAAa,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,MAAM,SAAS,OAAO,CAAC,OAAO,wBAAwB,EACrK,MAAM,CACP,CACF,CACF,CAAC;IACF,MAAM,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAC5C,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CACrE,CAAC;IACF,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,yBAAyB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,uCAAuC,EAAE;QAC7D,MAAM;QACN,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;QAC1B,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAC7C,SAAS,EAAE,cAAc,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC7F,MAAM,EAAE,cAAc,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpF,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;KACrF,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,wBAAwB,GAAG,MAAM,CAAC,EAAE,CAAC,sCAAsC,CAAC,CAAC,QAAQ,CAAC,EAC1F,QAAwB,EACxB,MAAkB,EAClB,OAAmB,EACnB,OAAuB;IAEvB,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IACtC,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,MAAM,EAAE,CAAC;IACzC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACjG,MAAM,EAAE,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAClC,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,iBAAiB,EAAE,OAAO,CAAC,eAAe;QAC1C,SAAS,EAAE,OAAO,CAAC,eAAe;QAClC,WAAW,EAAE,WAAW;KACzB,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE;QACzF,IAAI;QACJ,qBAAqB;QACrB,IAAI;QACJ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QACpB,OAAO,KAAK,CAAC,CAAC,UAAU,CACtB,MAAM,EACN,2BAA2B,MAAM,oBAAoB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC,6BAA6B,EACvJ,IAAI,KAAK,CAAC,2BAA2B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAC1D,CAAC;IAEJ,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE;QACzE,KAAK;QACL,WAAW;QACX,GAAG;QACH,YAAY;QACZ,IAAI;QACJ,IAAI;QACJ,WAAW;QACX,IAAI;QACJ,GAAG;QACH,UAAU,OAAO,GAAG;KACrB,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,QAAQ,KAAK,WAAW;QACjC,OAAO,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QACrE,OAAO,KAAK,CAAC,CAAC,UAAU,CACtB,MAAM,EACN,eAAe,MAAM,gCAAgC,OAAO,EAAE,EAC9D,IAAI,KAAK,CACP,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;YAClB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;YACpC,yBAAyB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CACrD,CACF,CAAC;IACJ,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAgC,EAAE;QACtD,MAAM;QACN,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,WAAW;QAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,QAAwB,EACxB,MAAkB,EAClB,OAAmB,EACnB,OAAuB,EACvB,EAAE,CACF,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CACtC,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAC7E,CAAC;AAEJ;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC,QAAQ,CAAC,EACrF,QAAwB,EACxB,IAAY,EACZ,KAAkB,EAClB,OAA2B,EAC3B,UAAkB;IAElB,MAAM,KAAK,GAAG,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC;IAC7D,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;IACnF,IAAI,MAAM,CAAC,QAAQ,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACtF,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;QACvB,OAAO,IAAI,uBAAuB,CAAC;YACjC,OAAO;YACP,UAAU,EAAE,QAAQ;YACpB,OAAO,EAAE,GAAG,OAAO,gCAAgC,KAAK,8BAA8B;SACvF,CAAC,CAAC;IACL,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3F,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;QAAE,OAAO,WAAW,CAAC,KAAK,CAAC;IACzD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,UAAU,2BAA2B,EAAE;QAC5E,IAAI;QACJ,OAAO;QACP,OAAO;KACR,CAAC,CAAC;IACH,OAAO,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,MAAM,GAAG;IACb,MAAM,EAAE;QACN,KAAK,EAAE,aAAa;QACpB,QAAQ,EAAE,WAAW;QACrB,eAAe,EAAE,WAAW;QAC5B,KAAK,EAAE,CAAC,mBAAmB,EAAE,qBAAqB,CAAC;QACnD,OAAO,EAAE,mBAAmB;KAC7B;IACD,KAAK,EAAE;QACL,KAAK,EAAE,OAAO;QACd,QAAQ,EAAE,QAAQ;QAClB,eAAe,EAAE,QAAQ;QACzB,KAAK,EAAE,CAAC,aAAa,EAAE,iBAAiB,CAAC;QACzC,OAAO,EAAE,gBAAgB;KAC1B;CAYF,CAAC;AAKF;;;;GAIG;AACH,MAAM,kBAAkB,GAAG;;;;;;;CAO1B,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,iBAAiB,GAAG,gCAAgC,CAAC;AAE3D,2FAA2F;AAC3F,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,QAAwB,EACxB,OAAmB,EACnB,OAAuB,EACvB,KAAmB,EACnB,KAAkB,EAClB,QAA4C,EAC9B,EAAE;IAChB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;IACvC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;IACpC,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAEtC,uFAAuF;IACvF,MAAM,UAAU,GAAG,CAAC,OAAqB,EAAE,EAAE,CAC3C,kBAAkB,CAChB,QAAQ,EACR,IAAI,EACJ,KAAK,EACL,OAAO,EACP,iBAAiB,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,aAAa,CACvD,CAAC;IAEJ,MAAM,mBAAmB,GAAG,MAAM,CAAC,EAAE,CAAC,0CAA0C,CAAC,CAAC,QAAQ,CAAC,EACzF,OAAqB;QAErB,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,KAAK,GAA0B,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC;QAC/F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;QAChD,MAAM,WAAW,GAAG,KAAK;YACvB,CAAC,CAAC,YAAY,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,4DAA4D;YACzG,CAAC,CAAC,EAAE,CAAC;QACP,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE;YAC/D,IAAI;YACJ,kBAAkB;YAClB,IAAI;YACJ,KAAK;YACL,GAAG,KAAK;SACT,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,QAAQ,KAAK,WAAW;YAAE,OAAO,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACrF,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACzC,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAClD,OAAO,KAAK,KAAK,IAAI;gBACnB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAU,CAAC,CAAC;QACzF,CAAC,CAAC,CACH,CAAC;QACF,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,SAAS;YAC9D,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,MAAM,KAAK,SAAS;gBAClB,CAAC,CAAC,oBAAoB,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,uBAAuB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC9G,CAAC,CAAC,yBAAyB,MAAM,CAAC,QAAQ,wBAAwB,MAAM,CAAC,OAAO,cAAc,CACjG,CAAC;YACF,OAAO,KAAK,CAAC,CAAC,UAAU,CACtB,MAAM,EACN,MAAM,EAAE,OAAO,KAAK,aAAa;gBAC/B,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,mBAAmB,MAAM,6CAA6C,SAAS,SAAS,SAAS,qCAAqC,WAAW,GAAG;gBAC3K,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,mBAAmB,MAAM,wBAAwB,SAAS,2CAA2C,KAAK,EAAE,EACrI,KAAK,CACN,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,kEAAkE,EAAE;YACxF,MAAM;YACN,OAAO;YACP,MAAM;YACN,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;SACvB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,MAAM,CAAC,EAAE,CAAC,qCAAqC,CAAC,CAAC,QAAQ,CAAC,EAC/E,KAAgB,EAChB,OAAqB;QAErB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,SAAS,CAAC,IAAI,KAAK,yBAAyB;YAAE,OAAO,SAAS,CAAC;QACnE,KAAK,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,KAAK,EAAE,EAAE;QACf,QAAQ,KAAK,CAAC,OAAO,EAAE,CAAC;YACtB,KAAK,QAAQ,CAAC;YACd,KAAK,OAAO;gBACV,OAAO,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC;oBAClF,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC;oBAC3B,CAAC,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3C,KAAK,IAAI;gBACP,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC,CAAC"}

package/dist/ssh/limits.d.ts

@@ -0,0 +1,60 @@
+import { type ContainerLimit, ContainerLimits } from "@usine/core/worker";
+import { Option, Schema } from "effect";
+import type { SshWorkerContainer } from "./model.ts";
+declare const MachineResources_base: Schema.Class<MachineResources, Schema.Struct<{
+    /** The CPUs available to the SSH user, as `nproc` counts them. */
+    readonly cpus: Schema.Int;
+    readonly memoryBytes: Schema.Int;
+    /** The most processes and threads the kernel runs at once: the lower of `threads-max` and `pid_max`. */
+    readonly tasks: Schema.Int;
+}>, {}>;
+/** What a Linux machine's kernel reports of its CPUs, memory, and tasks. */
+export declare class MachineResources extends MachineResources_base {
+}
+/**
+ * A script that prints the number of CPUs, the kernel's `threads-max` and `pid_max`, and
+ * `/proc/meminfo`.
+ */
+export declare const RESOURCES_SCRIPT = "nproc && cat /proc/sys/kernel/threads-max /proc/sys/kernel/pid_max /proc/meminfo";
+/** Parses the output of `RESOURCES_SCRIPT`. Returns `None` if a value is missing or not a positive integer. */
+export declare const resourcesFromOutput: (output: string) => Option.Option<MachineResources>;
+/** The limits of a Worker's containers and where they come from. */
+export type ResolvedLimits = 
+/** `usine.yaml` sets every limit. */
+{
+    readonly _tag: "Configured";
+    readonly limits: ContainerLimits;
+}
+/** `derived` come from `machine` for `slots` concurrent Attempts; `usine.yaml` sets the others. */
+ | {
+    readonly _tag: "Derived";
+    readonly limits: ContainerLimits;
+    readonly derived: ReadonlyArray<ContainerLimit>;
+    readonly machine: MachineResources;
+    readonly slots: number;
+};
+/** Returns the limits of `container` if `usine.yaml` sets every one, or `None`. */
+export declare const configuredLimits: (container: SshWorkerContainer) => Option.Option<ResolvedLimits>;
+/**
+ * Returns the limits of `container` when `slots` Attempts run on `machine` at once: the ones
+ * `usine.yaml` sets, and the others from `deriveLimits`.
+ */
+export declare const resolveLimits: (container: SshWorkerContainer, machine: MachineResources, slots: number) => ResolvedLimits;
+/** Returns the log fields of the limits of a Worker's containers. */
+export declare const limitFields: (resolved: ResolvedLimits) => {
+    cpus: number & import("effect/Brand").Brand<"ContainerCpus">;
+    memoryMib: number & import("effect/Brand").Brand<"ContainerMemoryMib">;
+    pids: number & import("effect/Brand").Brand<"ContainerPids">;
+    derivedLimits: string;
+} | {
+    cpus: number & import("effect/Brand").Brand<"ContainerCpus">;
+    memoryMib: number & import("effect/Brand").Brand<"ContainerMemoryMib">;
+    pids: number & import("effect/Brand").Brand<"ContainerPids">;
+    derivedLimits: string;
+    slots: number;
+    machineCpus: number;
+    machineMemoryMib: number;
+    machineTasks: number;
+};
+export {};
+//# sourceMappingURL=limits.d.ts.map

package/dist/ssh/limits.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"limits.d.ts","sourceRoot":"","sources":["../../src/ssh/limits.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EACnB,eAAe,EAGhB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAS,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;;IAQnD,kEAAkE;;;IAGlE,wGAAwG;;;AAP1G,4EAA4E;AAC5E,qBAAa,gBAAiB,SAAQ,qBAQpC;CAAG;AAEL;;;GAGG;AACH,eAAO,MAAM,gBAAgB,qFACuD,CAAC;AAIrF,+GAA+G;AAC/G,eAAO,MAAM,mBAAmB,WAAY,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CASlF,CAAC;AA6BF,oEAAoE;AACpE,MAAM,MAAM,cAAc;AACxB,qCAAqC;AACnC;IAAE,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;CAAE;AACnE,mGAAmG;GACjG;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IAChD,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAEN,mFAAmF;AACnF,eAAO,MAAM,gBAAgB,cAAe,kBAAkB,KAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAI1F,CAAC;AAEJ;;;GAGG;AACH,eAAO,MAAM,aAAa,cACb,kBAAkB,WACpB,gBAAgB,SAClB,MAAM,KACZ,cAkBF,CAAC;AAEF,qEAAqE;AACrE,eAAO,MAAM,WAAW,aAAc,cAAc;;;;;;;;;;;;;;CAclD,CAAC"}

package/dist/ssh/limits.js

@@ -0,0 +1,84 @@
+import { CONTAINER_LIMITS, ContainerCpus, ContainerLimits, ContainerMemoryMib, ContainerPids, } from "@usine/core/worker";
+import { Match, Option, Schema } from "effect";
+import { memoryFromMeminfo } from "../memory-preflight.js";
+const MIB = 1024 ** 2;
+/** What a Linux machine's kernel reports of its CPUs, memory, and tasks. */
+export class MachineResources extends Schema.Class("@usine/executor/MachineResources")({
+    /** The CPUs available to the SSH user, as `nproc` counts them. */
+    cpus: Schema.Int.check(Schema.isGreaterThan(0)),
+    memoryBytes: Schema.Int.check(Schema.isGreaterThan(0)),
+    /** The most processes and threads the kernel runs at once: the lower of `threads-max` and `pid_max`. */
+    tasks: Schema.Int.check(Schema.isGreaterThan(0)),
+}) {
+}
+/**
+ * A script that prints the number of CPUs, the kernel's `threads-max` and `pid_max`, and
+ * `/proc/meminfo`.
+ */
+export const RESOURCES_SCRIPT = "nproc && cat /proc/sys/kernel/threads-max /proc/sys/kernel/pid_max /proc/meminfo";
+const decodeResources = Schema.decodeUnknownOption(MachineResources);
+/** Parses the output of `RESOURCES_SCRIPT`. Returns `None` if a value is missing or not a positive integer. */
+export const resourcesFromOutput = (output) => {
+    const [cpus, threadsMax, pidMax, ...meminfo] = output.split("\n");
+    return Option.flatMap(memoryFromMeminfo(meminfo.join("\n")), ({ totalBytes }) => decodeResources({
+        cpus: Number(cpus),
+        memoryBytes: totalBytes,
+        tasks: Math.min(Number(threadsMax), Number(pidMax)),
+    }));
+};
+/** What the Attempt containers leave to the machine's own processes, such as sshd, Docker, and the cached clones' fetches. */
+const HOST_RESERVE = { cpus: 0.5, memoryMib: 1024, tasks: 1024 };
+/** The smallest limits that `usine.yaml` and Docker accept. */
+const SMALLEST = { cpus: 0.01, memoryMib: 256, pids: 64 };
+/**
+ * Returns the limits of each container when `slots` Attempts run on `machine` at once: the
+ * machine's total less a reserve for its own processes, divided by `slots`, rounded down to a
+ * hundredth of a CPU, a MiB, and a task, and no lower than the smallest limits Docker accepts.
+ */
+const deriveLimits = (machine, slots) => new ContainerLimits({
+    cpus: ContainerCpus.make(Math.max(SMALLEST.cpus, Math.floor(((machine.cpus - HOST_RESERVE.cpus) / slots) * 100) / 100)),
+    memoryMib: ContainerMemoryMib.make(Math.max(SMALLEST.memoryMib, Math.floor((machine.memoryBytes / MIB - HOST_RESERVE.memoryMib) / slots))),
+    pids: ContainerPids.make(Math.max(SMALLEST.pids, Math.floor((machine.tasks - HOST_RESERVE.tasks) / slots))),
+});
+/** Returns the limits of `container` if `usine.yaml` sets every one, or `None`. */
+export const configuredLimits = (container) => Option.map(Option.all({ cpus: container.cpus, memoryMib: container.memoryMib, pids: container.pids }), (limits) => ({ _tag: "Configured", limits: new ContainerLimits(limits) }));
+/**
+ * Returns the limits of `container` when `slots` Attempts run on `machine` at once: the ones
+ * `usine.yaml` sets, and the others from `deriveLimits`.
+ */
+export const resolveLimits = (container, machine, slots) => {
+    const derived = deriveLimits(machine, slots);
+    const unset = {
+        cpus: Option.isNone(container.cpus),
+        memoryMib: Option.isNone(container.memoryMib),
+        pids: Option.isNone(container.pids),
+    };
+    return {
+        _tag: "Derived",
+        limits: new ContainerLimits({
+            cpus: Option.getOrElse(container.cpus, () => derived.cpus),
+            memoryMib: Option.getOrElse(container.memoryMib, () => derived.memoryMib),
+            pids: Option.getOrElse(container.pids, () => derived.pids),
+        }),
+        derived: CONTAINER_LIMITS.filter((limit) => unset[limit]),
+        machine,
+        slots,
+    };
+};
+/** Returns the log fields of the limits of a Worker's containers. */
+export const limitFields = (resolved) => ({
+    cpus: resolved.limits.cpus,
+    memoryMib: resolved.limits.memoryMib,
+    pids: resolved.limits.pids,
+    ...Match.valueTags(resolved, {
+        Configured: () => ({ derivedLimits: "none" }),
+        Derived: ({ derived, machine, slots }) => ({
+            derivedLimits: derived.join(","),
+            slots,
+            machineCpus: machine.cpus,
+            machineMemoryMib: Math.floor(machine.memoryBytes / MIB),
+            machineTasks: machine.tasks,
+        }),
+    }),
+});
+//# sourceMappingURL=limits.js.map

package/dist/ssh/limits.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"limits.js","sourceRoot":"","sources":["../../src/ssh/limits.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,aAAa,EAEb,eAAe,EACf,kBAAkB,EAClB,aAAa,GACd,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAG3D,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC;AAEtB,4EAA4E;AAC5E,MAAM,OAAO,gBAAiB,SAAQ,MAAM,CAAC,KAAK,CAChD,kCAAkC,CACnC,CAAC;IACA,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/C,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IACtD,wGAAwG;IACxG,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;CACjD,CAAC;CAAG;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAC3B,kFAAkF,CAAC;AAErF,MAAM,eAAe,GAAG,MAAM,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;AAErE,+GAA+G;AAC/G,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAc,EAAmC,EAAE;IACrF,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClE,OAAO,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAC9E,eAAe,CAAC;QACd,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC;QAClB,WAAW,EAAE,UAAU;QACvB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;KACpD,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;AAEF,8HAA8H;AAC9H,MAAM,YAAY,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAW,CAAC;AAE1E,+DAA+D;AAC/D,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAW,CAAC;AAEnE;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,OAAyB,EAAE,KAAa,EAAmB,EAAE,CACjF,IAAI,eAAe,CAAC;IAClB,IAAI,EAAE,aAAa,CAAC,IAAI,CACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAC9F;IACD,SAAS,EAAE,kBAAkB,CAAC,IAAI,CAChC,IAAI,CAAC,GAAG,CACN,QAAQ,CAAC,SAAS,EAClB,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,WAAW,GAAG,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC,CACzE,CACF;IACD,IAAI,EAAE,aAAa,CAAC,IAAI,CACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAClF;CACF,CAAC,CAAC;AAeL,mFAAmF;AACnF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,SAA6B,EAAiC,EAAE,CAC/F,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,EAC1F,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAC1E,CAAC;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,SAA6B,EAC7B,OAAyB,EACzB,KAAa,EACG,EAAE;IAClB,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG;QACZ,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAC7C,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC;KACM,CAAC;IAC5C,OAAO;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,IAAI,eAAe,CAAC;YAC1B,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;YAC1D,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;YACzE,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SAC3D,CAAC;QACF,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzD,OAAO;QACP,KAAK;KACN,CAAC;AACJ,CAAC,CAAC;AAEF,qEAAqE;AACrE,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,QAAwB,EAAE,EAAE,CAAC,CAAC;IACxD,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI;IAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS;IACpC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI;IAC1B,GAAG,KAAK,CAAC,SAAS,CAAC,QAAQ,EAAE;QAC3B,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QAC7C,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;YACzC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAChC,KAAK;YACL,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,GAAG,GAAG,CAAC;YACvD,YAAY,EAAE,OAAO,CAAC,KAAK;SAC5B,CAAC;KACH,CAAC;CACH,CAAC,CAAC"}