@usine/executor 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -21
- package/README.md +212 -37
- package/dist/child-command.d.ts +5 -8
- package/dist/child-command.d.ts.map +1 -1
- package/dist/child-command.js +10 -14
- package/dist/child-command.js.map +1 -1
- package/dist/child-environment.d.ts +19 -0
- package/dist/child-environment.d.ts.map +1 -0
- package/dist/child-environment.js +18 -0
- package/dist/child-environment.js.map +1 -0
- package/dist/claude-login-lock.d.ts +9 -0
- package/dist/claude-login-lock.d.ts.map +1 -0
- package/dist/claude-login-lock.js +15 -0
- package/dist/claude-login-lock.js.map +1 -0
- package/dist/cloudflare/client.d.ts +50 -0
- package/dist/cloudflare/client.d.ts.map +1 -0
- package/dist/cloudflare/client.js +108 -0
- package/dist/cloudflare/client.js.map +1 -0
- package/dist/cloudflare/cloudflare.d.ts +17 -0
- package/dist/cloudflare/cloudflare.d.ts.map +1 -0
- package/dist/cloudflare/cloudflare.js +704 -0
- package/dist/cloudflare/cloudflare.js.map +1 -0
- package/dist/cloudflare/model.d.ts +42 -0
- package/dist/cloudflare/model.d.ts.map +1 -0
- package/dist/cloudflare/model.js +2 -0
- package/dist/cloudflare/model.js.map +1 -0
- package/dist/cloudflare/shell/environment.d.ts +12 -0
- package/dist/cloudflare/shell/environment.d.ts.map +1 -0
- package/dist/cloudflare/shell/environment.js +12 -0
- package/dist/cloudflare/shell/environment.js.map +1 -0
- package/dist/cloudflare/shell/shell.d.ts +2 -0
- package/dist/cloudflare/shell/shell.d.ts.map +1 -0
- package/dist/cloudflare/shell/shell.js +155 -0
- package/dist/cloudflare/shell/shell.js.map +1 -0
- package/dist/credential-environment.d.ts +20 -0
- package/dist/credential-environment.d.ts.map +1 -0
- package/dist/credential-environment.js +20 -0
- package/dist/credential-environment.js.map +1 -0
- package/dist/durable-session.d.ts +81 -0
- package/dist/durable-session.d.ts.map +1 -0
- package/dist/durable-session.js +550 -0
- package/dist/durable-session.js.map +1 -0
- package/dist/exe-dev/commands.d.ts +23 -13
- package/dist/exe-dev/commands.d.ts.map +1 -1
- package/dist/exe-dev/commands.js +11 -57
- package/dist/exe-dev/commands.js.map +1 -1
- package/dist/exe-dev/exe-dev.d.ts +9 -3
- package/dist/exe-dev/exe-dev.d.ts.map +1 -1
- package/dist/exe-dev/exe-dev.js +61 -128
- package/dist/exe-dev/exe-dev.js.map +1 -1
- package/dist/exe-dev/grants.d.ts.map +1 -1
- package/dist/exe-dev/grants.js +35 -15
- package/dist/exe-dev/grants.js.map +1 -1
- package/dist/exe-dev/model.d.ts +15 -19
- package/dist/exe-dev/model.d.ts.map +1 -1
- package/dist/exe-dev/model.js +9 -12
- package/dist/exe-dev/model.js.map +1 -1
- package/dist/exe-dev/preflight.d.ts +8 -4
- package/dist/exe-dev/preflight.d.ts.map +1 -1
- package/dist/exe-dev/preflight.js +26 -35
- package/dist/exe-dev/preflight.js.map +1 -1
- package/dist/executor.d.ts +292 -44
- package/dist/executor.d.ts.map +1 -1
- package/dist/executor.js +187 -19
- package/dist/executor.js.map +1 -1
- package/dist/guard.d.ts +57 -7
- package/dist/guard.d.ts.map +1 -1
- package/dist/guard.js +274 -59
- package/dist/guard.js.map +1 -1
- package/dist/harness-preflight.d.ts +27 -5
- package/dist/harness-preflight.d.ts.map +1 -1
- package/dist/harness-preflight.js +32 -31
- package/dist/harness-preflight.js.map +1 -1
- package/dist/keyed-lock.d.ts +2 -3
- package/dist/keyed-lock.d.ts.map +1 -1
- package/dist/keyed-lock.js +2 -3
- package/dist/keyed-lock.js.map +1 -1
- package/dist/leftovers.d.ts +9 -0
- package/dist/leftovers.d.ts.map +1 -0
- package/dist/leftovers.js +14 -0
- package/dist/leftovers.js.map +1 -0
- package/dist/local/credentials.d.ts +21 -0
- package/dist/local/credentials.d.ts.map +1 -0
- package/dist/local/credentials.js +39 -0
- package/dist/local/credentials.js.map +1 -0
- package/dist/local/leftovers.d.ts +8 -0
- package/dist/local/leftovers.d.ts.map +1 -0
- package/dist/local/leftovers.js +23 -0
- package/dist/local/leftovers.js.map +1 -0
- package/dist/local/local.d.ts +17 -11
- package/dist/local/local.d.ts.map +1 -1
- package/dist/local/local.js +205 -132
- package/dist/local/local.js.map +1 -1
- package/dist/memory-preflight.d.ts +27 -0
- package/dist/memory-preflight.d.ts.map +1 -0
- package/dist/memory-preflight.js +83 -0
- package/dist/memory-preflight.js.map +1 -0
- package/dist/pre-push-hook.d.ts +20 -0
- package/dist/pre-push-hook.d.ts.map +1 -0
- package/dist/pre-push-hook.js +33 -0
- package/dist/pre-push-hook.js.map +1 -0
- package/dist/prepared-attempt.d.ts +41 -3
- package/dist/prepared-attempt.d.ts.map +1 -1
- package/dist/prepared-attempt.js +109 -31
- package/dist/prepared-attempt.js.map +1 -1
- package/dist/provision.d.ts +129 -15
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +428 -69
- package/dist/provision.js.map +1 -1
- package/dist/remote/commands.d.ts +51 -0
- package/dist/remote/commands.d.ts.map +1 -0
- package/dist/remote/commands.js +94 -0
- package/dist/remote/commands.js.map +1 -0
- package/dist/remote/container.d.ts +90 -0
- package/dist/remote/container.d.ts.map +1 -0
- package/dist/remote/container.js +480 -0
- package/dist/remote/container.js.map +1 -0
- package/dist/remote/leftovers.d.ts +14 -0
- package/dist/remote/leftovers.d.ts.map +1 -0
- package/dist/remote/leftovers.js +22 -0
- package/dist/remote/leftovers.js.map +1 -0
- package/dist/remote/liveness.d.ts +33 -0
- package/dist/remote/liveness.d.ts.map +1 -0
- package/dist/remote/liveness.js +196 -0
- package/dist/remote/liveness.js.map +1 -0
- package/dist/remote/machine.d.ts +110 -0
- package/dist/remote/machine.d.ts.map +1 -0
- package/dist/remote/machine.js +2 -0
- package/dist/remote/machine.js.map +1 -0
- package/dist/remote/remote.d.ts +28 -0
- package/dist/remote/remote.d.ts.map +1 -0
- package/dist/remote/remote.js +556 -0
- package/dist/remote/remote.js.map +1 -0
- package/dist/remote/sweep.d.ts +17 -0
- package/dist/remote/sweep.d.ts.map +1 -0
- package/dist/remote/sweep.js +117 -0
- package/dist/remote/sweep.js.map +1 -0
- package/dist/renewable-credential.d.ts +17 -0
- package/dist/renewable-credential.d.ts.map +1 -0
- package/dist/renewable-credential.js +58 -0
- package/dist/renewable-credential.js.map +1 -0
- package/dist/salvage.d.ts +38 -0
- package/dist/salvage.d.ts.map +1 -0
- package/dist/salvage.js +198 -0
- package/dist/salvage.js.map +1 -0
- package/dist/ssh/api-keys.d.ts +36 -0
- package/dist/ssh/api-keys.d.ts.map +1 -0
- package/dist/ssh/api-keys.js +72 -0
- package/dist/ssh/api-keys.js.map +1 -0
- package/dist/ssh/codex-homes.d.ts +21 -0
- package/dist/ssh/codex-homes.d.ts.map +1 -0
- package/dist/ssh/codex-homes.js +78 -0
- package/dist/ssh/codex-homes.js.map +1 -0
- package/dist/ssh/container.d.ts +32 -0
- package/dist/ssh/container.d.ts.map +1 -0
- package/dist/ssh/container.js +282 -0
- package/dist/ssh/container.js.map +1 -0
- package/dist/ssh/credentials.d.ts +43 -0
- package/dist/ssh/credentials.d.ts.map +1 -0
- package/dist/ssh/credentials.js +86 -0
- package/dist/ssh/credentials.js.map +1 -0
- package/dist/ssh/gateway/admin.d.ts +45 -0
- package/dist/ssh/gateway/admin.d.ts.map +1 -0
- package/dist/ssh/gateway/admin.js +129 -0
- package/dist/ssh/gateway/admin.js.map +1 -0
- package/dist/ssh/gateway/attachments.d.ts +31 -0
- package/dist/ssh/gateway/attachments.d.ts.map +1 -0
- package/dist/ssh/gateway/attachments.js +168 -0
- package/dist/ssh/gateway/attachments.js.map +1 -0
- package/dist/ssh/gateway/environment.d.ts +71 -0
- package/dist/ssh/gateway/environment.d.ts.map +1 -0
- package/dist/ssh/gateway/environment.js +148 -0
- package/dist/ssh/gateway/environment.js.map +1 -0
- package/dist/ssh/gateway/forward.d.ts +74 -0
- package/dist/ssh/gateway/forward.d.ts.map +1 -0
- package/dist/ssh/gateway/forward.js +247 -0
- package/dist/ssh/gateway/forward.js.map +1 -0
- package/dist/ssh/gateway/preflight.d.ts +67 -0
- package/dist/ssh/gateway/preflight.d.ts.map +1 -0
- package/dist/ssh/gateway/preflight.js +320 -0
- package/dist/ssh/gateway/preflight.js.map +1 -0
- package/dist/ssh/limits.d.ts +60 -0
- package/dist/ssh/limits.d.ts.map +1 -0
- package/dist/ssh/limits.js +84 -0
- package/dist/ssh/limits.js.map +1 -0
- package/dist/ssh/model.d.ts +153 -0
- package/dist/ssh/model.d.ts.map +1 -0
- package/dist/ssh/model.js +2 -0
- package/dist/ssh/model.js.map +1 -0
- package/dist/ssh/preflight.d.ts +35 -0
- package/dist/ssh/preflight.d.ts.map +1 -0
- package/dist/ssh/preflight.js +87 -0
- package/dist/ssh/preflight.js.map +1 -0
- package/dist/ssh/provisioner/boat.d.ts +33 -0
- package/dist/ssh/provisioner/boat.d.ts.map +1 -0
- package/dist/ssh/provisioner/boat.js +247 -0
- package/dist/ssh/provisioner/boat.js.map +1 -0
- package/dist/ssh/provisioner/hetzner.d.ts +32 -0
- package/dist/ssh/provisioner/hetzner.d.ts.map +1 -0
- package/dist/ssh/provisioner/hetzner.js +337 -0
- package/dist/ssh/provisioner/hetzner.js.map +1 -0
- package/dist/ssh/provisioner/host-key.d.ts +11 -0
- package/dist/ssh/provisioner/host-key.d.ts.map +1 -0
- package/dist/ssh/provisioner/host-key.js +57 -0
- package/dist/ssh/provisioner/host-key.js.map +1 -0
- package/dist/ssh/provisioner/http.d.ts +45 -0
- package/dist/ssh/provisioner/http.d.ts.map +1 -0
- package/dist/ssh/provisioner/http.js +72 -0
- package/dist/ssh/provisioner/http.js.map +1 -0
- package/dist/ssh/provisioner/machines.d.ts +48 -0
- package/dist/ssh/provisioner/machines.d.ts.map +1 -0
- package/dist/ssh/provisioner/machines.js +686 -0
- package/dist/ssh/provisioner/machines.js.map +1 -0
- package/dist/ssh/provisioner/model.d.ts +92 -0
- package/dist/ssh/provisioner/model.d.ts.map +1 -0
- package/dist/ssh/provisioner/model.js +37 -0
- package/dist/ssh/provisioner/model.js.map +1 -0
- package/dist/ssh/ssh.d.ts +23 -0
- package/dist/ssh/ssh.d.ts.map +1 -0
- package/dist/ssh/ssh.js +145 -0
- package/dist/ssh/ssh.js.map +1 -0
- package/dist/ssh/worker-records.d.ts +18 -0
- package/dist/ssh/worker-records.d.ts.map +1 -0
- package/dist/ssh/worker-records.js +100 -0
- package/dist/ssh/worker-records.js.map +1 -0
- package/dist/ssh/workers.d.ts +80 -0
- package/dist/ssh/workers.d.ts.map +1 -0
- package/dist/ssh/workers.js +247 -0
- package/dist/ssh/workers.js.map +1 -0
- package/package.json +25 -9
- package/dist/exe-dev/channel.d.ts +0 -12
- package/dist/exe-dev/channel.d.ts.map +0 -1
- package/dist/exe-dev/channel.js +0 -42
- package/dist/exe-dev/channel.js.map +0 -1
- package/dist/exe-dev/shell.d.ts +0 -12
- package/dist/exe-dev/shell.d.ts.map +0 -1
- package/dist/exe-dev/shell.js +0 -31
- package/dist/exe-dev/shell.js.map +0 -1
- package/dist/exe-dev/sweep.d.ts +0 -10
- package/dist/exe-dev/sweep.d.ts.map +0 -1
- package/dist/exe-dev/sweep.js +0 -117
- package/dist/exe-dev/sweep.js.map +0 -1
- package/dist/invalid-frame.d.ts +0 -17
- package/dist/invalid-frame.d.ts.map +0 -1
- package/dist/invalid-frame.js +0 -111
- package/dist/invalid-frame.js.map +0 -1
- package/dist/run-host-session.d.ts +0 -27
- package/dist/run-host-session.d.ts.map +0 -1
- package/dist/run-host-session.js +0 -303
- package/dist/run-host-session.js.map +0 -1
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
import { CHILD_KILL_SCRIPT, RemoteShell } from "@usine/agents/child-host";
|
|
2
|
+
import { ContainerOutOfMemoryKills } from "@usine/core/attempt";
|
|
3
|
+
import { safeSegment } from "@usine/core/repository";
|
|
4
|
+
import { ContainerId, } from "@usine/core/worker";
|
|
5
|
+
import { Effect, Option, Redacted, Ref, Schema } from "effect";
|
|
6
|
+
import { ExecutorOperationError } from "../executor.js";
|
|
7
|
+
/** The container CLI on the machine. */
|
|
8
|
+
export const CONTAINER_RUNTIME = "docker";
|
|
9
|
+
export const PNPM_STORE_VOLUME = "usine-pnpm-store";
|
|
10
|
+
export const PNPM_STORE_PATH = "/home/usine/.local/share/pnpm/store";
|
|
11
|
+
export const PNPM_STORE_MOUNT = `type=volume,source=${PNPM_STORE_VOLUME},destination=${PNPM_STORE_PATH},volume-nocopy`;
|
|
12
|
+
/** Initializes volume `$3` using runtime `$1`, image `$2`, mount `$4`, and path `$5`. */
|
|
13
|
+
export const PREPARE_PNPM_STORE_SCRIPT = String.raw `
|
|
14
|
+
set -eu
|
|
15
|
+
"$1" volume create "$3" >/dev/null
|
|
16
|
+
exec "$1" run --rm --network none --user 0 --mount "$4" --entrypoint sh "$2" -ec '
|
|
17
|
+
owner=$(id -u usine)
|
|
18
|
+
current=$(stat -c %u "$1")
|
|
19
|
+
if [ "$current" = "$owner" ]; then exit 0; fi
|
|
20
|
+
if [ "$current" != 0 ] || [ -n "$(find "$1" -mindepth 1 -maxdepth 1 -print -quit)" ]; then
|
|
21
|
+
echo "The pnpm store belongs to UID $current, but the image usine user has UID $owner. Remove containers that reference it, then run: docker volume rm $2" >&2
|
|
22
|
+
exit 1
|
|
23
|
+
fi
|
|
24
|
+
chmod 0755 "$1"
|
|
25
|
+
chown -h "$owner" "$1"
|
|
26
|
+
' sh "$5" "$3"
|
|
27
|
+
`.trim();
|
|
28
|
+
/** The labels on each container, so a later process finds the ones it left. */
|
|
29
|
+
const INSTANCE_LABEL = "dev.usine.instance";
|
|
30
|
+
const INSTANCE_ID_LABEL = "dev.usine.instance-id";
|
|
31
|
+
const WORKER_LABEL = "dev.usine.worker";
|
|
32
|
+
const ATTEMPT_LABEL = "dev.usine.attempt";
|
|
33
|
+
/** The label of a preflight's container, which removes itself and never needs a sweep. */
|
|
34
|
+
const PREFLIGHT_LABEL = "dev.usine.preflight";
|
|
35
|
+
/** The lifetime of a preflight's container if no process removes it, in seconds. */
|
|
36
|
+
const PREFLIGHT_CONTAINER_SECONDS = 900;
|
|
37
|
+
/** The home directory of the image's `usine` user (`infra/worker-image/Dockerfile`). */
|
|
38
|
+
const CONTAINER_HOME = "/home/usine";
|
|
39
|
+
/** The image's entrypoint copies the files under this directory into `CONTAINER_HOME`. */
|
|
40
|
+
const SEED_DIRECTORY = "/usr/local/share/usine/seed";
|
|
41
|
+
export const sharedUser = { _tag: "SharedUser" };
|
|
42
|
+
/**
|
|
43
|
+
* The harness login files under the SSH user's home: `mounted` ones at the same path under the
|
|
44
|
+
* container's home, read-write because the harness rewrites them when it refreshes its OAuth
|
|
45
|
+
* token, and `seeded` ones copied there at start.
|
|
46
|
+
*/
|
|
47
|
+
const LOGIN_FILES = {
|
|
48
|
+
claude: { mounted: [".claude/.credentials.json"], seeded: [".claude.json"] },
|
|
49
|
+
codex: { mounted: [".codex/auth.json"], seeded: [".codex/config.toml"] },
|
|
50
|
+
pi: { mounted: [], seeded: [] },
|
|
51
|
+
};
|
|
52
|
+
/** Returns the name of `attemptId`'s container. */
|
|
53
|
+
export const attemptContainerName = (attemptId) => `usine-${safeSegment(attemptId)}`;
|
|
54
|
+
/**
|
|
55
|
+
* Returns the file whose presence records that `attemptId`'s checkout was mounted in a
|
|
56
|
+
* container. It's outside every container's mounts, so no Attempt can create or remove it.
|
|
57
|
+
*/
|
|
58
|
+
export const containerCheckoutMarker = (workerDirectory, attemptId) => `${workerDirectory}/container-checkouts/${safeSegment(attemptId)}`;
|
|
59
|
+
/** A script that creates the empty file `$1` and its directory. */
|
|
60
|
+
const MARK_SCRIPT = 'mkdir -p "$(dirname "$1")" && : > "$1"';
|
|
61
|
+
/**
|
|
62
|
+
* A script that removes a container named `$2` left from an earlier start, then prints the SSH
|
|
63
|
+
* user's home and each of the following arguments that exists: the first `$3` are absolute
|
|
64
|
+
* paths, and the rest are paths under the home.
|
|
65
|
+
*/
|
|
66
|
+
const PREPARE_SCRIPT = String.raw `
|
|
67
|
+
"$1" rm -f "$2" >/dev/null 2>&1 || true
|
|
68
|
+
printf '%s\n' "$HOME"
|
|
69
|
+
absolute=$3
|
|
70
|
+
shift 3
|
|
71
|
+
while [ "$#" -gt 0 ]; do
|
|
72
|
+
if [ "$absolute" -gt 0 ]; then path=$1; absolute=$((absolute - 1)); else path=$HOME/$1; fi
|
|
73
|
+
if test -e "$path"; then printf '%s\n' "$1"; fi
|
|
74
|
+
shift
|
|
75
|
+
done
|
|
76
|
+
`.trim();
|
|
77
|
+
/**
|
|
78
|
+
* A script that stops the children recorded in container `$2` for the checkout `$4` with the
|
|
79
|
+
* kill script `$3`. Prints `no-container` if the container doesn't exist and
|
|
80
|
+
* `container-stopped` if it doesn't run, since then no child can run in it.
|
|
81
|
+
*/
|
|
82
|
+
const STOP_CHILDREN_SCRIPT = String.raw `
|
|
83
|
+
if ! state=$("$1" container inspect --format '{{.State.Running}}' "$2" 2>&1); then
|
|
84
|
+
case "$state" in
|
|
85
|
+
*"No such"*) echo no-container; exit 0 ;;
|
|
86
|
+
esac
|
|
87
|
+
printf '%s\n' "$state" >&2
|
|
88
|
+
exit 1
|
|
89
|
+
fi
|
|
90
|
+
if [ "$state" != true ]; then echo container-stopped; exit 0; fi
|
|
91
|
+
exec "$1" exec "$2" sh -c "$3" sh "$4"
|
|
92
|
+
`.trim();
|
|
93
|
+
/** The `container inspect` format of a container's status, exit code, and out-of-memory kill. */
|
|
94
|
+
const STATE_FORMAT = "{{.State.Status}} {{.State.ExitCode}} {{.State.OOMKilled}}";
|
|
95
|
+
/**
|
|
96
|
+
* A script that prints `container <ID>` for container `$2`, then the `memory.events` of its
|
|
97
|
+
* cgroup if it runs, or `oom_kill 1` if it stopped because the kernel killed its init for memory.
|
|
98
|
+
*/
|
|
99
|
+
const MEMORY_EVENTS_SCRIPT = String.raw `
|
|
100
|
+
state=$("$1" container inspect --format '{{.Id}} {{.State.Running}} {{.State.OOMKilled}}' "$2") || exit 1
|
|
101
|
+
set -- "$1" "$2" $state
|
|
102
|
+
echo "container $3"
|
|
103
|
+
case "$4 $5" in
|
|
104
|
+
"true "*) exec "$1" exec "$2" cat /sys/fs/cgroup/memory.events ;;
|
|
105
|
+
*" true") echo "oom_kill 1" ;;
|
|
106
|
+
esac
|
|
107
|
+
`.trim();
|
|
108
|
+
/** The container's ID in the output of `MEMORY_EVENTS_SCRIPT`. */
|
|
109
|
+
const CONTAINER_LINE = /^container (\S*)$/mu;
|
|
110
|
+
/** The count of processes that the kernel killed for memory in a cgroup's `memory.events`. */
|
|
111
|
+
const OOM_KILL_LINE = /^oom_kill (\d+)$/mu;
|
|
112
|
+
const decodeContainerId = Schema.decodeUnknownEffect(ContainerId);
|
|
113
|
+
/**
|
|
114
|
+
* A script that lists the containers with both labels `$2` and `$3`, one `<name> <worker>` line
|
|
115
|
+
* each. Prints nothing if the machine has no container CLI.
|
|
116
|
+
*/
|
|
117
|
+
const LIST_SCRIPT = String.raw `
|
|
118
|
+
command -v "$1" >/dev/null 2>&1 || exit 0
|
|
119
|
+
exec "$1" ps --all --filter "label=$2" --filter "label=$3" --format '{{.Names}} {{.Label "dev.usine.worker"}}'
|
|
120
|
+
`.trim();
|
|
121
|
+
/**
|
|
122
|
+
* A script that runs `$@` with this script's standard input through a pipe. `docker run
|
|
123
|
+
* --env-file /dev/stdin` can't open a socket, which sshd and some process spawners give as
|
|
124
|
+
* standard input.
|
|
125
|
+
*/
|
|
126
|
+
const PIPED_STDIN_SCRIPT = 'cat | "$@"';
|
|
127
|
+
/** Returns a CSV field of a `--mount` option, quoted so a comma in a path is kept. */
|
|
128
|
+
const field = (key, value) => `"${key}=${value.replaceAll('"', '""')}"`;
|
|
129
|
+
const bindMount = (source, destination, access) => [
|
|
130
|
+
"type=bind",
|
|
131
|
+
field("source", source),
|
|
132
|
+
field("destination", destination),
|
|
133
|
+
...(access === "readonly" ? ["readonly"] : []),
|
|
134
|
+
].join(",");
|
|
135
|
+
/** Returns the `--env-file` content for `variables`. Fails on a value the file format can't hold. */
|
|
136
|
+
const environmentFile = (operation, variables) => {
|
|
137
|
+
const entries = Object.entries(variables);
|
|
138
|
+
const multiline = entries.find(([, value]) => value.includes("\n"));
|
|
139
|
+
if (multiline !== undefined)
|
|
140
|
+
return Effect.fail(new ExecutorOperationError({
|
|
141
|
+
operation,
|
|
142
|
+
message: `Can't set ${multiline[0]} in a container: its value has a line break`,
|
|
143
|
+
cause: new Error(`${multiline[0]} has a line break`),
|
|
144
|
+
}));
|
|
145
|
+
return Effect.succeed(Redacted.make(entries.map(([name, value]) => `${name}=${value}\n`).join("")));
|
|
146
|
+
};
|
|
147
|
+
/** Returns the log fields of the container of `spec` on the machine of `shell`. */
|
|
148
|
+
const containerFields = (shell, spec) => ({
|
|
149
|
+
worker: spec.worker,
|
|
150
|
+
host: shell.destination,
|
|
151
|
+
container: spec.name,
|
|
152
|
+
image: spec.settings.image,
|
|
153
|
+
...Option.match(spec.attemptId, {
|
|
154
|
+
onNone: () => ({}),
|
|
155
|
+
onSome: (attemptId) => ({ attemptId }),
|
|
156
|
+
}),
|
|
157
|
+
});
|
|
158
|
+
/** Returns the shell that runs commands in the container `name` on the machine of `shell`. */
|
|
159
|
+
const containerShell = (shell, name) => {
|
|
160
|
+
const container = { runtime: CONTAINER_RUNTIME, name };
|
|
161
|
+
return new RemoteShell({
|
|
162
|
+
ssh: shell.ssh,
|
|
163
|
+
sshEnvironment: shell.sshEnvironment,
|
|
164
|
+
destination: shell.destination,
|
|
165
|
+
container,
|
|
166
|
+
});
|
|
167
|
+
};
|
|
168
|
+
/**
|
|
169
|
+
* Returns the operations on the containers of a machine: start one for an Attempt or a
|
|
170
|
+
* preflight, stop an Attempt's children in it, and remove the containers that an earlier
|
|
171
|
+
* process left.
|
|
172
|
+
*/
|
|
173
|
+
export const containersFor = (commands) => {
|
|
174
|
+
const prepareStore = Effect.fn("RemoteExecutor.preparePnpmStore")(function* (operation, shell, image, worker) {
|
|
175
|
+
const fields = { worker, host: shell.destination, image, volume: PNPM_STORE_VOLUME };
|
|
176
|
+
yield* Effect.annotateCurrentSpan(fields);
|
|
177
|
+
const prepared = yield* commands.inspect(operation, shell, "sh", [
|
|
178
|
+
"-c",
|
|
179
|
+
PREPARE_PNPM_STORE_SCRIPT,
|
|
180
|
+
"sh",
|
|
181
|
+
CONTAINER_RUNTIME,
|
|
182
|
+
image,
|
|
183
|
+
PNPM_STORE_VOLUME,
|
|
184
|
+
PNPM_STORE_MOUNT,
|
|
185
|
+
PNPM_STORE_PATH,
|
|
186
|
+
]);
|
|
187
|
+
if (prepared.exitCode !== 0) {
|
|
188
|
+
const message = `Can't prepare pnpm store volume ${PNPM_STORE_VOLUME}: ${prepared.stderr.trim() || `exit code ${prepared.exitCode}`}`;
|
|
189
|
+
yield* Effect.logWarning(message, { ...fields, exitCode: prepared.exitCode });
|
|
190
|
+
return yield* new ExecutorOperationError({
|
|
191
|
+
operation,
|
|
192
|
+
message,
|
|
193
|
+
cause: new Error(prepared.stderr),
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
yield* Effect.logInfo("Prepared shared pnpm store volume", fields);
|
|
197
|
+
});
|
|
198
|
+
const remove = Effect.fn("RemoteExecutor.removeContainer")(function* (shell, name) {
|
|
199
|
+
yield* Effect.annotateCurrentSpan({ host: shell.destination, container: name });
|
|
200
|
+
yield* commands.run("cleanup", shell, CONTAINER_RUNTIME, ["rm", "--force", name]).pipe(Effect.tap(() => Effect.logInfo("Removed the container", { host: shell.destination, container: name })), Effect.tapError((error) => Effect.logWarning("Could not remove the container; the next process that uses the Worker removes it", { host: shell.destination, container: name, message: error.message })), Effect.ignore);
|
|
201
|
+
});
|
|
202
|
+
/**
|
|
203
|
+
* Starts the container of `spec` on the machine of `shell`, replacing one with the same name,
|
|
204
|
+
* and removes it when the caller's scope closes. Returns the shell that runs commands in it.
|
|
205
|
+
*/
|
|
206
|
+
const start = Effect.fn("RemoteExecutor.startContainer")(function* (operation, shell, spec) {
|
|
207
|
+
const fields = containerFields(shell, spec);
|
|
208
|
+
yield* Effect.annotateCurrentSpan(fields);
|
|
209
|
+
const logins = spec.harnesses.flatMap((harness) => [
|
|
210
|
+
...LOGIN_FILES[harness].mounted.map((file) => ({ file, mounted: true })),
|
|
211
|
+
...LOGIN_FILES[harness].seeded.map((file) => ({ file, mounted: false })),
|
|
212
|
+
]);
|
|
213
|
+
const [home = "", ...present] = (yield* commands.run(operation, shell, "sh", [
|
|
214
|
+
"-c",
|
|
215
|
+
PREPARE_SCRIPT,
|
|
216
|
+
"sh",
|
|
217
|
+
CONTAINER_RUNTIME,
|
|
218
|
+
spec.name,
|
|
219
|
+
String(spec.readOnly.length + spec.writable.length),
|
|
220
|
+
...spec.readOnly,
|
|
221
|
+
...spec.writable,
|
|
222
|
+
...logins.map(({ file }) => file),
|
|
223
|
+
])).split("\n");
|
|
224
|
+
if (!home.startsWith("/"))
|
|
225
|
+
return yield* new ExecutorOperationError({
|
|
226
|
+
operation,
|
|
227
|
+
message: "The SSH user's home on the machine isn't an absolute path",
|
|
228
|
+
cause: new Error(home),
|
|
229
|
+
});
|
|
230
|
+
const found = logins.filter(({ file }) => present.includes(file));
|
|
231
|
+
const readOnly = spec.readOnly.filter((path) => present.includes(path));
|
|
232
|
+
const writable = spec.writable.filter((path) => present.includes(path));
|
|
233
|
+
const missing = [...spec.readOnly, ...spec.writable].filter((path) => !present.includes(path));
|
|
234
|
+
if (missing.length > 0)
|
|
235
|
+
yield* Effect.logWarning("The container doesn't mount these missing machine paths", {
|
|
236
|
+
...fields,
|
|
237
|
+
missing: missing.join(","),
|
|
238
|
+
});
|
|
239
|
+
const envFile = yield* environmentFile(operation, {
|
|
240
|
+
...spec.environment,
|
|
241
|
+
...Object.fromEntries(Object.entries(spec.secrets).map(([name, value]) => [name, Redacted.value(value)])),
|
|
242
|
+
});
|
|
243
|
+
const lifetime = Option.match(spec.attemptId, {
|
|
244
|
+
onNone: () => ({
|
|
245
|
+
labels: [`${PREFLIGHT_LABEL}=${spec.worker}`],
|
|
246
|
+
options: ["--rm"],
|
|
247
|
+
command: ["sleep", String(PREFLIGHT_CONTAINER_SECONDS)],
|
|
248
|
+
}),
|
|
249
|
+
onSome: (attemptId) => ({
|
|
250
|
+
labels: [
|
|
251
|
+
`${INSTANCE_LABEL}=${spec.settings.instance}`,
|
|
252
|
+
`${INSTANCE_ID_LABEL}=${spec.settings.instanceId}`,
|
|
253
|
+
`${WORKER_LABEL}=${spec.worker}`,
|
|
254
|
+
`${ATTEMPT_LABEL}=${attemptId}`,
|
|
255
|
+
],
|
|
256
|
+
options: [],
|
|
257
|
+
command: [],
|
|
258
|
+
}),
|
|
259
|
+
});
|
|
260
|
+
const mounts = [
|
|
261
|
+
PNPM_STORE_MOUNT,
|
|
262
|
+
...Option.match(spec.checkout, {
|
|
263
|
+
onNone: () => [],
|
|
264
|
+
onSome: ({ path, parent }) => [
|
|
265
|
+
["type=tmpfs", field("destination", parent), "tmpfs-mode=1777"].join(","),
|
|
266
|
+
bindMount(path, path, "writable"),
|
|
267
|
+
],
|
|
268
|
+
}),
|
|
269
|
+
...readOnly.map((path) => bindMount(path, path, "readonly")),
|
|
270
|
+
...writable.map((path) => bindMount(path, path, "writable")),
|
|
271
|
+
...found.map(({ file, mounted }) => mounted
|
|
272
|
+
? bindMount(`${home}/${file}`, `${CONTAINER_HOME}/${file}`, "writable")
|
|
273
|
+
: bindMount(`${home}/${file}`, `${SEED_DIRECTORY}/${file}`, "readonly")),
|
|
274
|
+
];
|
|
275
|
+
// Host networking keeps the machine's nftables rules, which admit only the Gateway, in front
|
|
276
|
+
// of the container (ADR 0027, amended 2026-10-25).
|
|
277
|
+
const args = [
|
|
278
|
+
"run",
|
|
279
|
+
"--detach",
|
|
280
|
+
"--name",
|
|
281
|
+
spec.name,
|
|
282
|
+
...lifetime.options,
|
|
283
|
+
...lifetime.labels.flatMap((label) => ["--label", label]),
|
|
284
|
+
"--network",
|
|
285
|
+
"host",
|
|
286
|
+
"--cpus",
|
|
287
|
+
String(spec.settings.limits.cpus),
|
|
288
|
+
"--memory",
|
|
289
|
+
`${spec.settings.limits.memoryMib}m`,
|
|
290
|
+
"--pids-limit",
|
|
291
|
+
String(spec.settings.limits.pids),
|
|
292
|
+
"--env-file",
|
|
293
|
+
"/dev/stdin",
|
|
294
|
+
...mounts.flatMap((mount) => ["--mount", mount]),
|
|
295
|
+
spec.settings.image,
|
|
296
|
+
...lifetime.command,
|
|
297
|
+
];
|
|
298
|
+
yield* Effect.addFinalizer(() => remove(shell, spec.name));
|
|
299
|
+
yield* commands
|
|
300
|
+
.runWithInput(operation, shell, "sh", ["-c", PIPED_STDIN_SCRIPT, "sh", CONTAINER_RUNTIME, ...args], envFile)
|
|
301
|
+
.pipe(Effect.ensuring(Effect.sync(() => Redacted.wipeUnsafe(envFile))));
|
|
302
|
+
yield* Effect.logInfo("Started the container", {
|
|
303
|
+
...fields,
|
|
304
|
+
logins: found.map(({ file }) => file).join(","),
|
|
305
|
+
readOnly: readOnly.join(","),
|
|
306
|
+
});
|
|
307
|
+
return containerShell(shell, spec.name);
|
|
308
|
+
});
|
|
309
|
+
/**
|
|
310
|
+
* Stops the children recorded in the container `name` for the checkout `worktree`. Succeeds
|
|
311
|
+
* without stopping anything if the container doesn't exist or doesn't run.
|
|
312
|
+
*/
|
|
313
|
+
const stopChildren = Effect.fn("RemoteExecutor.stopContainerChildren")(function* (operation, shell, name, worktree) {
|
|
314
|
+
yield* Effect.annotateCurrentSpan({ host: shell.destination, container: name, worktree });
|
|
315
|
+
const outcome = yield* commands.run(operation, shell, "sh", [
|
|
316
|
+
"-c",
|
|
317
|
+
STOP_CHILDREN_SCRIPT,
|
|
318
|
+
"sh",
|
|
319
|
+
CONTAINER_RUNTIME,
|
|
320
|
+
name,
|
|
321
|
+
CHILD_KILL_SCRIPT,
|
|
322
|
+
worktree,
|
|
323
|
+
]);
|
|
324
|
+
yield* Effect.logInfo("Confirmed no child of the Attempt runs in its container", {
|
|
325
|
+
host: shell.destination,
|
|
326
|
+
container: name,
|
|
327
|
+
worktree,
|
|
328
|
+
children: outcome.trim().split("\n"),
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
/**
|
|
332
|
+
* Starts the container of `spec` again if it stopped, such as after an out-of-memory kill or a
|
|
333
|
+
* signal to its init, and returns whether the kernel killed its init for memory. The scope that
|
|
334
|
+
* started it still removes it. Fails if it doesn't exist.
|
|
335
|
+
*/
|
|
336
|
+
const ensureRunning = Effect.fn("RemoteExecutor.ensureContainerRunning")(function* (operation, shell, spec) {
|
|
337
|
+
const fields = containerFields(shell, spec);
|
|
338
|
+
yield* Effect.annotateCurrentSpan(fields);
|
|
339
|
+
const [status = "", exitCode = "", oomKilled = ""] = (yield* commands.run(operation, shell, CONTAINER_RUNTIME, ["container", "inspect", "--format", STATE_FORMAT, spec.name]))
|
|
340
|
+
.trim()
|
|
341
|
+
.split(" ");
|
|
342
|
+
if (status === "running") {
|
|
343
|
+
yield* Effect.logDebug("The container still runs", fields);
|
|
344
|
+
return false;
|
|
345
|
+
}
|
|
346
|
+
yield* Effect.logWarning("The container stopped; starting it again", {
|
|
347
|
+
...fields,
|
|
348
|
+
status,
|
|
349
|
+
exitCode,
|
|
350
|
+
oomKilled,
|
|
351
|
+
});
|
|
352
|
+
yield* commands.run(operation, shell, CONTAINER_RUNTIME, ["start", spec.name]);
|
|
353
|
+
return oomKilled === "true";
|
|
354
|
+
});
|
|
355
|
+
/**
|
|
356
|
+
* Returns the ID of the container of `spec` and the number of processes that the kernel killed
|
|
357
|
+
* in it because it reached its memory limit: the count of its cgroup while it runs, or 1 if it
|
|
358
|
+
* stopped because the kernel killed its init. A start creates a new cgroup, so the count
|
|
359
|
+
* restarts at 0.
|
|
360
|
+
*/
|
|
361
|
+
const outOfMemoryKills = Effect.fn("RemoteExecutor.containerOutOfMemoryKills")(function* (operation, shell, spec) {
|
|
362
|
+
const fields = containerFields(shell, spec);
|
|
363
|
+
yield* Effect.annotateCurrentSpan(fields);
|
|
364
|
+
const events = yield* commands.run(operation, shell, "sh", [
|
|
365
|
+
"-c",
|
|
366
|
+
MEMORY_EVENTS_SCRIPT,
|
|
367
|
+
"sh",
|
|
368
|
+
CONTAINER_RUNTIME,
|
|
369
|
+
spec.name,
|
|
370
|
+
]);
|
|
371
|
+
const container = yield* decodeContainerId(CONTAINER_LINE.exec(events)?.[1]).pipe(Effect.mapError((cause) => new ExecutorOperationError({
|
|
372
|
+
operation,
|
|
373
|
+
message: "The container runtime didn't print the container's ID",
|
|
374
|
+
cause,
|
|
375
|
+
})));
|
|
376
|
+
const kills = Number(OOM_KILL_LINE.exec(events)?.[1] ?? 0);
|
|
377
|
+
if (kills === 0)
|
|
378
|
+
yield* Effect.logDebug("The kernel killed no process in the container", {
|
|
379
|
+
...fields,
|
|
380
|
+
containerId: container,
|
|
381
|
+
});
|
|
382
|
+
else
|
|
383
|
+
yield* Effect.logWarning("The kernel killed processes in the container for memory", {
|
|
384
|
+
...fields,
|
|
385
|
+
containerId: container,
|
|
386
|
+
kills,
|
|
387
|
+
memoryMib: spec.settings.limits.memoryMib,
|
|
388
|
+
});
|
|
389
|
+
return { container, kills };
|
|
390
|
+
});
|
|
391
|
+
/** Tracks kills across container restarts and records them before removal. */
|
|
392
|
+
const trackMemory = Effect.fn("RemoteExecutor.trackContainerMemory")(function* (shell, spec) {
|
|
393
|
+
const fields = containerFields(shell, spec);
|
|
394
|
+
yield* Effect.annotateCurrentSpan(fields);
|
|
395
|
+
const recorder = yield* Ref.make(Option.none());
|
|
396
|
+
// A start gives the container a new cgroup, whose count doesn't include the killed init.
|
|
397
|
+
const killedInits = yield* Ref.make(0);
|
|
398
|
+
const recordKills = Effect.fn("RemoteExecutor.recordContainerOutOfMemoryKills")(function* () {
|
|
399
|
+
yield* Effect.annotateCurrentSpan(fields);
|
|
400
|
+
const record = yield* Ref.get(recorder);
|
|
401
|
+
if (Option.isNone(record))
|
|
402
|
+
return yield* Effect.logDebug("No launch ran in the container, so its out-of-memory kills aren't read", fields);
|
|
403
|
+
const { container, kills: counted } = yield* outOfMemoryKills("cleanup", shell, spec);
|
|
404
|
+
const kills = counted + (yield* Ref.get(killedInits));
|
|
405
|
+
if (kills === 0)
|
|
406
|
+
return;
|
|
407
|
+
yield* record.value(new ContainerOutOfMemoryKills({
|
|
408
|
+
container,
|
|
409
|
+
kills,
|
|
410
|
+
memoryMib: spec.settings.limits.memoryMib,
|
|
411
|
+
}));
|
|
412
|
+
}, Effect.tapError((error) => Effect.logWarning("Could not record the processes the kernel killed in the Attempt's container", { ...fields, errorTag: error._tag, message: String(error) })), Effect.ignore);
|
|
413
|
+
return {
|
|
414
|
+
recordKills,
|
|
415
|
+
recordTo: (record) => Ref.set(recorder, Option.some(record)),
|
|
416
|
+
ensureRunning: ensureRunning("cleanup", shell, spec).pipe(Effect.tap((oomKilled) => oomKilled ? Ref.update(killedInits, (n) => n + 1) : Effect.void)),
|
|
417
|
+
};
|
|
418
|
+
});
|
|
419
|
+
/**
|
|
420
|
+
* Removes the containers labeled with `instance` and `instanceId` on the machine of `shell`
|
|
421
|
+
* whose Worker label is `worker` or not in `kept`. Returns the names removed.
|
|
422
|
+
*/
|
|
423
|
+
const sweep = Effect.fn("RemoteExecutor.sweepContainers")(function* (shell, instance, instanceId, worker, kept) {
|
|
424
|
+
yield* Effect.annotateCurrentSpan({ host: shell.destination, worker, instance, instanceId });
|
|
425
|
+
const listed = (yield* commands.run("cleanup", shell, "sh", [
|
|
426
|
+
"-c",
|
|
427
|
+
LIST_SCRIPT,
|
|
428
|
+
"sh",
|
|
429
|
+
CONTAINER_RUNTIME,
|
|
430
|
+
`${INSTANCE_LABEL}=${instance}`,
|
|
431
|
+
`${INSTANCE_ID_LABEL}=${instanceId}`,
|
|
432
|
+
]))
|
|
433
|
+
.split("\n")
|
|
434
|
+
.flatMap((line) => {
|
|
435
|
+
const [name, owner = ""] = line.trim().split(" ");
|
|
436
|
+
return name === undefined || name === "" ? [] : [{ name, owner }];
|
|
437
|
+
});
|
|
438
|
+
const left = listed
|
|
439
|
+
.filter(({ owner }) => owner === worker || !kept.has(owner))
|
|
440
|
+
.map(({ name }) => name);
|
|
441
|
+
if (left.length === 0) {
|
|
442
|
+
yield* Effect.logDebug("No earlier process left a container on the Worker's machine", {
|
|
443
|
+
worker,
|
|
444
|
+
host: shell.destination,
|
|
445
|
+
instanceId,
|
|
446
|
+
running: listed.length,
|
|
447
|
+
});
|
|
448
|
+
return left;
|
|
449
|
+
}
|
|
450
|
+
yield* commands.run("cleanup", shell, CONTAINER_RUNTIME, ["rm", "--force", ...left]);
|
|
451
|
+
yield* Effect.logInfo("Removed the containers an earlier process left on the machine", {
|
|
452
|
+
worker,
|
|
453
|
+
host: shell.destination,
|
|
454
|
+
instanceId,
|
|
455
|
+
containers: left.join(","),
|
|
456
|
+
});
|
|
457
|
+
return left;
|
|
458
|
+
});
|
|
459
|
+
/** Records that the checkout of `marker` is mounted in a container, before any container starts. */
|
|
460
|
+
const mark = Effect.fn("RemoteExecutor.markContainerCheckout")(function* (operation, shell, marker) {
|
|
461
|
+
yield* Effect.annotateCurrentSpan({ host: shell.destination, marker });
|
|
462
|
+
yield* commands.run(operation, shell, "sh", ["-c", MARK_SCRIPT, "sh", marker]);
|
|
463
|
+
});
|
|
464
|
+
/** Returns whether a container has had the checkout of `marker` mounted. */
|
|
465
|
+
const marked = Effect.fn("RemoteExecutor.isContainerCheckout")(function* (operation, shell, marker) {
|
|
466
|
+
yield* Effect.annotateCurrentSpan({ host: shell.destination, marker });
|
|
467
|
+
const inspected = yield* commands.inspect(operation, shell, "test", ["-f", marker]);
|
|
468
|
+
if (inspected.exitCode === 0)
|
|
469
|
+
return true;
|
|
470
|
+
if (inspected.exitCode === 1)
|
|
471
|
+
return false;
|
|
472
|
+
return yield* new ExecutorOperationError({
|
|
473
|
+
operation,
|
|
474
|
+
message: "Can't inspect whether a container had the checkout",
|
|
475
|
+
cause: new Error(inspected.stderr.trim() || `test exited with code ${inspected.exitCode}`),
|
|
476
|
+
});
|
|
477
|
+
});
|
|
478
|
+
return { start, trackMemory, stopChildren, sweep, remove, mark, marked, prepareStore };
|
|
479
|
+
};
|
|
480
|
+
//# sourceMappingURL=container.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"container.js","sourceRoot":"","sources":["../../src/remote/container.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAuB,MAAM,0BAA0B,CAAC;AAE/F,OAAO,EAAkB,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAGhF,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,WAAW,GAIZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC/D,OAAO,EAA0B,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAGhF,wCAAwC;AACxC,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAQ,CAAC;AAE1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AACpD,MAAM,CAAC,MAAM,eAAe,GAAG,qCAAqC,CAAC;AACrE,MAAM,CAAC,MAAM,gBAAgB,GAAG,sBAAsB,iBAAiB,gBAAgB,eAAe,gBAAgB,CAAC;AAEvH,yFAAyF;AACzF,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC,GAAG,CAAA;;;;;;;;;;;;;;CAclD,CAAC,IAAI,EAAE,CAAC;AAET,+EAA+E;AAC/E,MAAM,cAAc,GAAG,oBAAoB,CAAC;AAC5C,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAClD,MAAM,YAAY,GAAG,kBAAkB,CAAC;AACxC,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAC1C,0FAA0F;AAC1F,MAAM,eAAe,GAAG,qBAAqB,CAAC;AAC9C,oFAAoF;AACpF,MAAM,2BAA2B,GAAG,GAAG,CAAC;AAExC,wFAAwF;AACxF,MAAM,cAAc,GAAG,aAAa,CAAC;AACrC,0FAA0F;AAC1F,MAAM,cAAc,GAAG,6BAA6B,CAAC;AAmBrD,MAAM,CAAC,MAAM,UAAU,GAAqB,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;AAEnE;;;;GAIG;AACH,MAAM,WAAW,GAAG;IAClB,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,2BAA2B,CAAC,EAAE,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE;IAC5E,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC,oBAAoB,CAAC,EAAE;IACxE,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;CAIhC,CAAC;AAEF,mDAAmD;AACnD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,SAAoB,EAAU,EAAE,CACnE,SAAS,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;AAEpC;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,eAAuB,EAAE,SAAoB,EAAU,EAAE,CAC/F,GAAG,eAAe,wBAAwB,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;AAErE,mEAAmE;AACnE,MAAM,WAAW,GAAG,wCAAwC,CAAC;AA0B7D;;;;GAIG;AACH,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAA;;;;;;;;;;CAUhC,CAAC,IAAI,EAAE,CAAC;AAET;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,MAAM,CAAC,GAAG,CAAA;;;;;;;;;;CAUtC,CAAC,IAAI,EAAE,CAAC;AAET,iGAAiG;AACjG,MAAM,YAAY,GAAG,4DAA4D,CAAC;AAElF;;;GAGG;AACH,MAAM,oBAAoB,GAAG,MAAM,CAAC,GAAG,CAAA;;;;;;;;CAQtC,CAAC,IAAI,EAAE,CAAC;AAET,kEAAkE;AAClE,MAAM,cAAc,GAAG,qBAAqB,CAAC;AAE7C,8FAA8F;AAC9F,MAAM,aAAa,GAAG,oBAAoB,CAAC;AAE3C,MAAM,iBAAiB,GAAG,MAAM,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;AAElE;;;GAGG;AACH,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAA;;;CAG7B,CAAC,IAAI,EAAE,CAAC;AAET;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,YAAY,CAAC;AAExC,sFAAsF;AACtF,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CAAC,IAAI,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;AAEhG,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,WAAmB,EAAE,MAA+B,EAAE,EAAE,CACzF;IACE,WAAW;IACX,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC;IACjC,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;CAC/C,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEd,qGAAqG;AACrG,MAAM,eAAe,GAAG,CACtB,SAA4B,EAC5B,SAA2C,EACuB,EAAE;IACpE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,IAAI,SAAS,KAAK,SAAS;QACzB,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,sBAAsB,CAAC;YACzB,SAAS;YACT,OAAO,EAAE,aAAa,SAAS,CAAC,CAAC,CAAC,6CAA6C;YAC/E,KAAK,EAAE,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC;SACrD,CAAC,CACH,CAAC;IACJ,OAAO,MAAM,CAAC,OAAO,CACnB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC7E,CAAC;AACJ,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,eAAe,GAAG,CAAC,KAAkB,EAAE,IAAmB,EAAE,EAAE,CAAC,CAAC;IACpE,MAAM,EAAE,IAAI,CAAC,MAAM;IACnB,IAAI,EAAE,KAAK,CAAC,WAAW;IACvB,SAAS,EAAE,IAAI,CAAC,IAAI;IACpB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;IAC1B,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;QAC9B,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;KACvC,CAAC;CACH,CAAC,CAAC;AAEH,8FAA8F;AAC9F,MAAM,cAAc,GAAG,CAAC,KAAkB,EAAE,IAAY,EAAe,EAAE;IACvE,MAAM,SAAS,GAAmB,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;IACvE,OAAO,IAAI,WAAW,CAAC;QACrB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS;KACV,CAAC,CAAC;AACL,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,QAAwB,EAAE,EAAE;IACxD,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,iCAAiC,CAAC,CAAC,QAAQ,CAAC,EACzE,SAA4B,EAC5B,KAAkB,EAClB,KAAqB,EACrB,MAAkB;QAElB,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QACrF,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;YAC/D,IAAI;YACJ,yBAAyB;YACzB,IAAI;YACJ,iBAAiB;YACjB,KAAK;YACL,iBAAiB;YACjB,gBAAgB;YAChB,eAAe;SAChB,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,mCAAmC,iBAAiB,KAAK,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,aAAa,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC;YACtI,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9E,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC;gBACvC,SAAS;gBACT,OAAO;gBACP,KAAK,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;aAClC,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,mCAAmC,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC,QAAQ,CAAC,EAClE,KAAkB,EAClB,IAAY;QAEZ,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChF,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CACpF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CACd,MAAM,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CACtF,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,UAAU,CACf,kFAAkF,EAClF,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CACrE,CACF,EACD,MAAM,CAAC,MAAM,CACd,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,+BAA+B,CAAC,CAAC,QAAQ,CAAC,EAChE,SAA4B,EAC5B,KAAkB,EAClB,IAAmB;QAEnB,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;SACzE,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;YAC3E,IAAI;YACJ,cAAc;YACd,IAAI;YACJ,iBAAiB;YACjB,IAAI,CAAC,IAAI;YACT,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACnD,GAAG,IAAI,CAAC,QAAQ;YAChB,GAAG,IAAI,CAAC,QAAQ;YAChB,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACvB,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC;gBACvC,SAAS;gBACT,OAAO,EAAE,2DAA2D;gBACpE,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC;aACvB,CAAC,CAAC;QACL,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YACpB,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,yDAAyD,EAAE;gBAClF,GAAG,MAAM;gBACT,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;aAC3B,CAAC,CAAC;QACL,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,EAAE;YAChD,GAAG,IAAI,CAAC,WAAW;YACnB,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CACnF;SACF,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;YAC5C,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;gBACb,MAAM,EAAE,CAAC,GAAG,eAAe,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC7C,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,2BAA2B,CAAC,CAAC;aACxD,CAAC;YACF,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gBACtB,MAAM,EAAE;oBACN,GAAG,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE;oBAC7C,GAAG,iBAAiB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE;oBAClD,GAAG,YAAY,IAAI,IAAI,CAAC,MAAM,EAAE;oBAChC,GAAG,aAAa,IAAI,SAAS,EAAE;iBAChC;gBACD,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE,EAAE;aACZ,CAAC;SACH,CAAC,CAAC;QACH,MAAM,MAAM,GAAG;YACb,gBAAgB;YAChB,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAC7B,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;gBAChB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;oBAC5B,CAAC,YAAY,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,CAAC,EAAE,iBAAiB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;oBACzE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC;iBAClC;aACF,CAAC;YACF,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAC5D,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAC5D,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CACjC,OAAO;gBACL,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,GAAG,cAAc,IAAI,IAAI,EAAE,EAAE,UAAU,CAAC;gBACvE,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,GAAG,cAAc,IAAI,IAAI,EAAE,EAAE,UAAU,CAAC,CAC1E;SACF,CAAC;QACF,6FAA6F;QAC7F,mDAAmD;QACnD,MAAM,IAAI,GAAG;YACX,KAAK;YACL,UAAU;YACV,QAAQ;YACR,IAAI,CAAC,IAAI;YACT,GAAG,QAAQ,CAAC,OAAO;YACnB,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACzD,WAAW;YACX,MAAM;YACN,QAAQ;YACR,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;YACjC,UAAU;YACV,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,GAAG;YACpC,cAAc;YACd,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;YACjC,YAAY;YACZ,YAAY;YACZ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAChD,IAAI,CAAC,QAAQ,CAAC,KAAK;YACnB,GAAG,QAAQ,CAAC,OAAO;SACpB,CAAC;QACF,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3D,KAAK,CAAC,CAAC,QAAQ;aACZ,YAAY,CACX,SAAS,EACT,KAAK,EACL,IAAI,EACJ,CAAC,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC,EAC5D,OAAO,CACR;aACA,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,uBAAuB,EAAE;YAC7C,GAAG,MAAM;YACT,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC/C,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;SAC7B,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,sCAAsC,CAAC,CAAC,QAAQ,CAAC,EAC9E,SAA4B,EAC5B,KAAkB,EAClB,IAAY,EACZ,QAAgB;QAEhB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC1F,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1D,IAAI;YACJ,oBAAoB;YACpB,IAAI;YACJ,iBAAiB;YACjB,IAAI;YACJ,iBAAiB;YACjB,QAAQ;SACT,CAAC,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,yDAAyD,EAAE;YAC/E,IAAI,EAAE,KAAK,CAAC,WAAW;YACvB,SAAS,EAAE,IAAI;YACf,QAAQ;YACR,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SACrC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH;;;;OAIG;IACH,MAAM,aAAa,GAAG,MAAM,CAAC,EAAE,CAAC,uCAAuC,CAAC,CAAC,QAAQ,CAAC,EAChF,SAA4B,EAC5B,KAAkB,EAClB,IAAmB;QAEnB,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,QAAQ,GAAG,EAAE,EAAE,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CACvE,SAAS,EACT,KAAK,EACL,iBAAiB,EACjB,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,CAC9D,CAAC;aACC,IAAI,EAAE;aACN,KAAK,CAAC,GAAG,CAAC,CAAC;QACd,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,EAAE,MAAM,CAAC,CAAC;YAC3D,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,0CAA0C,EAAE;YACnE,GAAG,MAAM;YACT,MAAM;YACN,QAAQ;YACR,SAAS;SACV,CAAC,CAAC;QACH,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/E,OAAO,SAAS,KAAK,MAAM,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH;;;;;OAKG;IACH,MAAM,gBAAgB,GAAG,MAAM,CAAC,EAAE,CAAC,0CAA0C,CAAC,CAAC,QAAQ,CAAC,EACtF,SAA4B,EAC5B,KAAkB,EAClB,IAAmB;QAEnB,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;YACzD,IAAI;YACJ,oBAAoB;YACpB,IAAI;YACJ,iBAAiB;YACjB,IAAI,CAAC,IAAI;SACV,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC/E,MAAM,CAAC,QAAQ,CACb,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,sBAAsB,CAAC;YACzB,SAAS;YACT,OAAO,EAAE,uDAAuD;YAChE,KAAK;SACN,CAAC,CACL,CACF,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3D,IAAI,KAAK,KAAK,CAAC;YACb,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,+CAA+C,EAAE;gBACtE,GAAG,MAAM;gBACT,WAAW,EAAE,SAAS;aACvB,CAAC,CAAC;;YAEH,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,yDAAyD,EAAE;gBAClF,GAAG,MAAM;gBACT,WAAW,EAAE,SAAS;gBACtB,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS;aAC1C,CAAC,CAAC;QACL,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,8EAA8E;IAC9E,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,qCAAqC,CAAC,CAAC,QAAQ,CAAC,EAC5E,KAAkB,EAClB,IAAmB;QAEnB,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC5C,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAuB,CAAC,CAAC;QACrE,yFAAyF;QACzF,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,gDAAgD,CAAC,CAC7E,QAAQ,CAAC;YACP,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;gBACvB,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAC3B,wEAAwE,EACxE,MAAM,CACP,CAAC;YACJ,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACtF,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO;YACxB,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CACjB,IAAI,yBAAyB,CAAC;gBAC5B,SAAS;gBACT,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS;aAC1C,CAAC,CACH,CAAC;QACJ,CAAC,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,MAAM,CAAC,UAAU,CACf,6EAA6E,EAC7E,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5D,CACF,EACD,MAAM,CAAC,MAAM,CACd,CAAC;QACF,OAAO;YACL,WAAW;YACX,QAAQ,EAAE,CAAC,MAA2B,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjF,aAAa,EAAE,aAAa,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,CACvD,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACvB,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAChE,CACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,gCAAgC,CAAC,CAAC,QAAQ,CAAC,EACjE,KAAkB,EAClB,QAAgB,EAChB,UAAsB,EACtB,MAAkB,EAClB,IAAyB;QAEzB,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7F,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;YAC1D,IAAI;YACJ,WAAW;YACX,IAAI;YACJ,iBAAiB;YACjB,GAAG,cAAc,IAAI,QAAQ,EAAE;YAC/B,GAAG,iBAAiB,IAAI,UAAU,EAAE;SACrC,CAAC,CAAC;aACA,KAAK,CAAC,IAAI,CAAC;aACX,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAChB,MAAM,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClD,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QACL,MAAM,IAAI,GAAG,MAAM;aAChB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aAC3D,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,6DAA6D,EAAE;gBACpF,MAAM;gBACN,IAAI,EAAE,KAAK,CAAC,WAAW;gBACvB,UAAU;gBACV,OAAO,EAAE,MAAM,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QACrF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+DAA+D,EAAE;YACrF,MAAM;YACN,IAAI,EAAE,KAAK,CAAC,WAAW;YACvB,UAAU;YACV,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;SAC3B,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,oGAAoG;IACpG,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,sCAAsC,CAAC,CAAC,QAAQ,CAAC,EACtE,SAA4B,EAC5B,KAAkB,EAClB,MAAc;QAEd,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACvE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,oCAAoC,CAAC,CAAC,QAAQ,CAAC,EACtE,SAA4B,EAC5B,KAAkB,EAClB,MAAc;QAEd,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACvE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACpF,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC3C,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC;YACvC,SAAS;YACT,OAAO,EAAE,oDAAoD;YAC7D,KAAK,EAAE,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,yBAAyB,SAAS,CAAC,QAAQ,EAAE,CAAC;SAC3F,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACzF,CAAC,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { WorkerName } from "@usine/core/worker";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
import type { RemoteCommands } from "./commands.ts";
|
|
4
|
+
import type { RemoteMachine } from "./machine.ts";
|
|
5
|
+
/**
|
|
6
|
+
* Resolves a Worker's machine, runs `reclaim` on it, and lists its leftover checkouts, once per
|
|
7
|
+
* Worker per executor. A Worker that can't be resolved or listed isn't reconciled until the
|
|
8
|
+
* next start.
|
|
9
|
+
*/
|
|
10
|
+
export declare const workerLeftoversFor: <M extends RemoteMachine, E>(commands: RemoteCommands, machineFor: (worker: WorkerName) => Effect.Effect<M, E>, reclaim: (worker: WorkerName) => Effect.Effect<void>) => Effect.Effect<(worker: WorkerName) => Effect.Effect<{
|
|
11
|
+
machine: M;
|
|
12
|
+
checkouts: Set<string>;
|
|
13
|
+
}, E | import("../executor.ts").ExecutorOperationError, never>, never, never>;
|
|
14
|
+
//# sourceMappingURL=leftovers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leftovers.d.ts","sourceRoot":"","sources":["../../src/remote/leftovers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AASlD;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,GAAI,CAAC,SAAS,aAAa,EAAE,CAAC,YACjD,cAAc,cACZ,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,WAC9C,CAAC,MAAM,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;;;6EAuBnD,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { listedOncePerWorker } from "../leftovers.js";
|
|
3
|
+
import { attemptPathsOn } from "./sweep.js";
|
|
4
|
+
/**
|
|
5
|
+
* Resolves a Worker's machine, runs `reclaim` on it, and lists its leftover checkouts, once per
|
|
6
|
+
* Worker per executor. A Worker that can't be resolved or listed isn't reconciled until the
|
|
7
|
+
* next start.
|
|
8
|
+
*/
|
|
9
|
+
export const workerLeftoversFor = (commands, machineFor, reclaim) => listedOncePerWorker(Effect.fn("RemoteExecutor.listWorkerLeftovers")(function* (worker) {
|
|
10
|
+
const machine = yield* machineFor(worker);
|
|
11
|
+
const host = machine.shell.destination;
|
|
12
|
+
yield* Effect.annotateCurrentSpan({ worker, host });
|
|
13
|
+
yield* reclaim(worker);
|
|
14
|
+
const checkouts = new Set(yield* attemptPathsOn(commands, "cleanup", machine.shell, `${machine.workerDirectory}/attempts`));
|
|
15
|
+
yield* Effect.logInfo("Listed remote checkouts left for reconciliation", {
|
|
16
|
+
worker,
|
|
17
|
+
host,
|
|
18
|
+
checkouts: checkouts.size,
|
|
19
|
+
});
|
|
20
|
+
return { machine, checkouts };
|
|
21
|
+
}));
|
|
22
|
+
//# sourceMappingURL=leftovers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leftovers.js","sourceRoot":"","sources":["../../src/remote/leftovers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAQ5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,QAAwB,EACxB,UAAuD,EACvD,OAAoD,EACpD,EAAE,CACF,mBAAmB,CACjB,MAAM,CAAC,EAAE,CAAC,oCAAoC,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAkB;IAC3E,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;IACvC,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,KAAK,CAAC,CAAC,cAAc,CACnB,QAAQ,EACR,SAAS,EACT,OAAO,CAAC,KAAK,EACb,GAAG,OAAO,CAAC,eAAe,WAAW,CACtC,CACF,CAAC;IACF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,iDAAiD,EAAE;QACvE,MAAM;QACN,IAAI;QACJ,SAAS,EAAE,SAAS,CAAC,IAAI;KAC1B,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,SAAS,EAA+B,CAAC;AAC7D,CAAC,CAAC,CACH,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { RemoteShell } from "@usine/agents/child-host";
|
|
2
|
+
import type { AttemptId } from "@usine/core/attempt";
|
|
3
|
+
import { type WorkerNotice } from "@usine/core/attempt/channel";
|
|
4
|
+
import type { WorkerName } from "@usine/core/worker";
|
|
5
|
+
import { Effect, type Scope } from "effect";
|
|
6
|
+
import type { RemoteCommands } from "./commands.ts";
|
|
7
|
+
interface WorkerWatcher {
|
|
8
|
+
readonly attemptId: AttemptId;
|
|
9
|
+
/** Records a notice in the Attempt's transcript. */
|
|
10
|
+
readonly report: (notice: WorkerNotice) => Effect.Effect<void>;
|
|
11
|
+
}
|
|
12
|
+
interface WorkerWatch extends WorkerWatcher {
|
|
13
|
+
readonly worker: WorkerName;
|
|
14
|
+
/** The machine's shell, without Attempt secrets. */
|
|
15
|
+
readonly shell: RemoteShell;
|
|
16
|
+
}
|
|
17
|
+
/** The liveness watches of one executor, one per machine with a running Attempt. */
|
|
18
|
+
export interface WorkerWatches {
|
|
19
|
+
/**
|
|
20
|
+
* Adds the Attempt to its Worker's watch until the caller's scope closes. Every minute, the
|
|
21
|
+
* watch runs `true` on the machine over a new SSH connection; when the machine stops or resumes
|
|
22
|
+
* responding, it calls `report` for each Attempt and logs a line. Never stops or fails an
|
|
23
|
+
* Attempt (ADR 0013).
|
|
24
|
+
*/
|
|
25
|
+
readonly watch: (watch: WorkerWatch) => Effect.Effect<void, never, Scope.Scope>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Returns the liveness watches that probe Workers' machines through `commands`. Each watch runs
|
|
29
|
+
* in the caller's scope and stops when that scope closes.
|
|
30
|
+
*/
|
|
31
|
+
export declare const workerWatchesFor: (commands: RemoteCommands) => Effect.Effect<WorkerWatches, never, Scope.Scope>;
|
|
32
|
+
export {};
|
|
33
|
+
//# sourceMappingURL=liveness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"liveness.d.ts","sourceRoot":"","sources":["../../src/remote/liveness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EACL,KAAK,YAAY,EAGlB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAY,MAAM,EAA6B,KAAK,KAAK,EAAa,MAAM,QAAQ,CAAC;AAC5F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AA+GpD,UAAU,aAAa;IACrB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,UAAU,WAAY,SAAQ,aAAa;IACzC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,oDAAoD;IACpD,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;CAC7B;AAED,oFAAoF;AACpF,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;CACjF;AA+GD;;;GAGG;AACH,eAAO,MAAM,gBAAgB,aACjB,cAAc,KACvB,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAiF9C,CAAC"}
|