@cotal-ai/cli 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/command.d.ts.map +1 -1
- package/dist/command.js +13 -0
- package/dist/command.js.map +1 -1
- package/dist/commands/agents.d.ts +5 -15
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +133 -72
- package/dist/commands/agents.js.map +1 -1
- package/dist/commands/clean.d.ts.map +1 -1
- package/dist/commands/clean.js +4 -3
- package/dist/commands/clean.js.map +1 -1
- package/dist/commands/describe.d.ts +72 -0
- package/dist/commands/describe.d.ts.map +1 -0
- package/dist/commands/describe.js +148 -0
- package/dist/commands/describe.js.map +1 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +27 -8
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/down-manifest.d.ts.map +1 -1
- package/dist/commands/down-manifest.js +38 -18
- package/dist/commands/down-manifest.js.map +1 -1
- package/dist/commands/down.d.ts +9 -0
- package/dist/commands/down.d.ts.map +1 -1
- package/dist/commands/down.js +38 -10
- package/dist/commands/down.js.map +1 -1
- package/dist/commands/ext.d.ts.map +1 -1
- package/dist/commands/ext.js +16 -10
- package/dist/commands/ext.js.map +1 -1
- package/dist/commands/join.d.ts.map +1 -1
- package/dist/commands/join.js +2 -4
- package/dist/commands/join.js.map +1 -1
- package/dist/commands/meshes-add.d.ts.map +1 -1
- package/dist/commands/meshes-add.js +23 -0
- package/dist/commands/meshes-add.js.map +1 -1
- package/dist/commands/models.d.ts.map +1 -1
- package/dist/commands/models.js +2 -2
- package/dist/commands/models.js.map +1 -1
- package/dist/commands/spawn-manifest.d.ts.map +1 -1
- package/dist/commands/spawn-manifest.js +22 -12
- package/dist/commands/spawn-manifest.js.map +1 -1
- package/dist/commands/spawn.d.ts +5 -0
- package/dist/commands/spawn.d.ts.map +1 -1
- package/dist/commands/spawn.js +4 -2
- package/dist/commands/spawn.js.map +1 -1
- package/dist/commands/status.js +25 -5
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/up.d.ts +17 -2
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +640 -49
- package/dist/commands/up.js.map +1 -1
- package/dist/console/app.d.ts +1 -1
- package/dist/console/app.d.ts.map +1 -1
- package/dist/console/app.js +16 -5
- package/dist/console/app.js.map +1 -1
- package/dist/console/commands.d.ts.map +1 -1
- package/dist/console/commands.js +5 -5
- package/dist/console/commands.js.map +1 -1
- package/dist/console/root.d.ts.map +1 -1
- package/dist/console/root.js +6 -1
- package/dist/console/root.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/attach-client.d.ts +37 -4
- package/dist/lib/attach-client.d.ts.map +1 -1
- package/dist/lib/attach-client.js +88 -18
- package/dist/lib/attach-client.js.map +1 -1
- package/dist/lib/auth-proc.d.ts +3 -1
- package/dist/lib/auth-proc.d.ts.map +1 -1
- package/dist/lib/auth-proc.js +15 -4
- package/dist/lib/auth-proc.js.map +1 -1
- package/dist/lib/connect.d.ts +1 -1
- package/dist/lib/connect.d.ts.map +1 -1
- package/dist/lib/connect.js +1 -1
- package/dist/lib/connect.js.map +1 -1
- package/dist/lib/control.d.ts +42 -11
- package/dist/lib/control.d.ts.map +1 -1
- package/dist/lib/control.js +251 -44
- package/dist/lib/control.js.map +1 -1
- package/dist/lib/delivery-proc.d.ts +19 -4
- package/dist/lib/delivery-proc.d.ts.map +1 -1
- package/dist/lib/delivery-proc.js +129 -43
- package/dist/lib/delivery-proc.js.map +1 -1
- package/dist/lib/manager-proc.d.ts +48 -5
- package/dist/lib/manager-proc.d.ts.map +1 -1
- package/dist/lib/manager-proc.js +131 -30
- package/dist/lib/manager-proc.js.map +1 -1
- package/dist/lib/manifest/ledger.d.ts +52 -8
- package/dist/lib/manifest/ledger.d.ts.map +1 -1
- package/dist/lib/manifest/ledger.js +77 -19
- package/dist/lib/manifest/ledger.js.map +1 -1
- package/dist/lib/manifest/live.d.ts +21 -16
- package/dist/lib/manifest/live.d.ts.map +1 -1
- package/dist/lib/manifest/live.js +33 -20
- package/dist/lib/manifest/live.js.map +1 -1
- package/dist/lib/restore.d.ts.map +1 -1
- package/dist/lib/restore.js +21 -3
- package/dist/lib/restore.js.map +1 -1
- package/dist/view/mesh-view.js +2 -2
- package/dist/view/mesh-view.js.map +1 -1
- package/package.json +4 -3
- package/dist/lib/pid.d.ts +0 -23
- package/dist/lib/pid.d.ts.map +0 -1
- package/dist/lib/pid.js +0 -39
- package/dist/lib/pid.js.map +0 -1
|
@@ -1,50 +1,77 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { existsSync, openSync, closeSync, writeFileSync, readFileSync, rmSync } from "node:fs";
|
|
3
3
|
import { DEFAULT_SERVER, mintCreds, newIdentity, waitForDeliveryLease, } from "@cotal-ai/core";
|
|
4
|
-
import { DELIVERY_CREDS_KEY, authDir, findCotalRoot, getSoleSpaceAuth, listSpaceAccounts, workspaceSecretStore } from "@cotal-ai/workspace";
|
|
4
|
+
import { DELIVERY_CREDS_KEY, authDir, findCotalRoot, getSoleSpaceAuth, listSpaceAccounts, parsePid, probeLiveness, workspaceSecretStore } from "@cotal-ai/workspace";
|
|
5
5
|
import { selfArgv } from "./self-exec.js";
|
|
6
6
|
import { resolveSpace } from "./status.js";
|
|
7
7
|
import { cotalPath } from "./paths.js";
|
|
8
|
-
import { ensureManager, managerHasDeliveryMarker,
|
|
8
|
+
import { MANAGER_PID_PATH, ensureManager, managerHasDeliveryMarker, managerLiveness, stopManager } from "./manager-proc.js";
|
|
9
9
|
const PID_PATH = () => cotalPath("delivery.pid");
|
|
10
10
|
// The daemon's cred goes through the secret-store seam; the shared key (== the filename, so the
|
|
11
11
|
// file stays `.cotal/delivery.creds`) comes from workspace — never a hand-copied literal.
|
|
12
12
|
const credsStore = () => workspaceSecretStore(findCotalRoot());
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
catch {
|
|
19
|
-
return false;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
/** True if the delivery daemon we started for this folder is still running (pid file + liveness). */
|
|
23
|
-
export function deliveryUp() {
|
|
13
|
+
/** The recorded daemon's liveness, THREE-VALUED plus absent. See {@link managerLiveness} for why the
|
|
14
|
+
* boolean collapse is the defect: `unknown` is reachable on a real kernel (a seccomp
|
|
15
|
+
* `SECCOMP_RET_ERRNO` filter or an LSM policy answers `kill(pid, 0)` with an arbitrary errno and
|
|
16
|
+
* libuv preserves it), and both ways of folding it into a boolean fail silently. */
|
|
17
|
+
export function deliveryLiveness(probe = probeLiveness) {
|
|
24
18
|
const p = PID_PATH();
|
|
25
19
|
if (!existsSync(p))
|
|
26
|
-
return
|
|
27
|
-
const
|
|
28
|
-
|
|
20
|
+
return "absent";
|
|
21
|
+
const raw = readFileSync(p, "utf8").trim();
|
|
22
|
+
if (raw === "")
|
|
23
|
+
return "absent";
|
|
24
|
+
const pid = parsePid(raw);
|
|
25
|
+
if (pid === undefined)
|
|
26
|
+
return "unattributable"; // see managerLiveness: never fold this into absent
|
|
27
|
+
return probe(pid);
|
|
28
|
+
}
|
|
29
|
+
/** True only if the daemon is PROVABLY running. Callers that ACT on the answer take
|
|
30
|
+
* {@link deliveryLiveness} instead; this cannot express "cannot tell". */
|
|
31
|
+
export function deliveryUp() {
|
|
32
|
+
return deliveryLiveness() === "alive";
|
|
29
33
|
}
|
|
30
34
|
/** True when this folder runs an authed mesh — the only mode with a delivery daemon (Plane-3 needs the
|
|
31
35
|
* trusted reader; open dev mode is live-only). */
|
|
32
36
|
function hasAuth() {
|
|
33
37
|
return listSpaceAccounts(authDir(findCotalRoot())).length > 0;
|
|
34
38
|
}
|
|
35
|
-
/**
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
/** The cutover preflight's verdict, THREE-VALUED, because the boolean it replaced was read before the
|
|
40
|
+
* refusal boundary and therefore defeated it.
|
|
41
|
+
*
|
|
42
|
+
* `managerUp()` folds `unknown` to false, so an unattributable manager with no marker (which IS the
|
|
43
|
+
* old Plane-3-hosting shape) read as "no old manager", the preflight skipped, and `ensureDelivery`
|
|
44
|
+
* went on to mint a credential, write a pidfile and start a second daemon. Only afterwards did
|
|
45
|
+
* `ensureManager` throw. Everything the refusal was supposed to prevent had already happened, and
|
|
46
|
+
* the daemon's own lease cannot help: it is a delivery-daemon-only mechanism and can neither prove
|
|
47
|
+
* nor exclude an old manager still hosting Plane 3.
|
|
48
|
+
*
|
|
49
|
+
* A guard that runs after the work is not a guard, so this one reads the tri-state directly and the
|
|
50
|
+
* caller refuses BEFORE anything is minted, written or started. */
|
|
51
|
+
function oldHostingManagerVerdict(probe = probeLiveness) {
|
|
52
|
+
const state = managerLiveness(probe);
|
|
53
|
+
if (state === "unknown" || state === "unattributable")
|
|
54
|
+
return "indeterminate";
|
|
55
|
+
if (state !== "alive")
|
|
56
|
+
return "proceed"; // dead or absent: nothing is hosting Plane 3
|
|
57
|
+
return managerHasDeliveryMarker() ? "proceed" : "stop-it"; // alive: the marker decides
|
|
40
58
|
}
|
|
41
59
|
/** Cutover preflight — the FIRST action, BEFORE the daemon can bind: stop any old Plane-3-hosting
|
|
42
60
|
* manager (live `manager.pid` without the delivery-aware marker) so it never double-binds the
|
|
43
61
|
* daemon's durables. A delivery-aware (this-build) manager is left running. No-op on a fresh install. */
|
|
44
|
-
export function stopOldHostingManagerIfPresent() {
|
|
45
|
-
|
|
62
|
+
export async function stopOldHostingManagerIfPresent(probe = probeLiveness, signal) {
|
|
63
|
+
const verdict = oldHostingManagerVerdict(probe);
|
|
64
|
+
// FIRST action, before any mint/write/start, so the refusal actually fences the daemon.
|
|
65
|
+
if (verdict === "indeterminate")
|
|
66
|
+
throw new Error(`the recorded manager pid (${readFileSync(MANAGER_PID_PATH(), "utf8").trim()}) cannot be attributed, so the delivery cutover preflight cannot run: the kernel answered neither "running" nor "no such process" (a seccomp filter or LSM policy does this inside some sandboxes).\n` +
|
|
67
|
+
`Refusing before the daemon starts. If that manager is an old Plane-3-hosting one it is still bound to fanout/reader, and starting the daemon anyway would double-bind them; the daemon's own lease cannot detect that.\n` +
|
|
68
|
+
`NEXT: verify the process yourself (\`ps -p <pid>\`). If it is gone, remove \`.cotal/manager.pid\` and re-run. If it is running, stop it with \`cotal down\` first.`);
|
|
69
|
+
if (verdict === "stop-it") {
|
|
46
70
|
console.error("• stopping an old Plane-3-hosting manager before starting the delivery daemon (cutover preflight)");
|
|
47
|
-
stopManager(
|
|
71
|
+
// stopManager THROWS rather than reporting a stop it did not achieve (EPERM, or a process that
|
|
72
|
+
// outlived SIGTERM), so reaching the next line is the proof the old manager is gone. Letting that
|
|
73
|
+
// throw propagate is the point: the daemon must not start beside a manager still bound to Plane 3.
|
|
74
|
+
await stopManager(probe, signal);
|
|
48
75
|
}
|
|
49
76
|
}
|
|
50
77
|
/** Start the delivery daemon detached (pid in `.cotal/delivery.pid`, output to `.cotal/delivery.log`),
|
|
@@ -60,6 +87,15 @@ export function startDeliveryDetached(o = {}) {
|
|
|
60
87
|
o.space ?? resolveSpace(process.cwd()),
|
|
61
88
|
"--server",
|
|
62
89
|
o.server ?? DEFAULT_SERVER,
|
|
90
|
+
// Propagate the broker's transport decision. The daemon cannot derive it: it learns everything
|
|
91
|
+
// from argv by design - it is a pre-minted scoped-cred client, injectable behind a SecretStore,
|
|
92
|
+
// and making it read the machine-local mesh registry to pick a transport would couple a
|
|
93
|
+
// hosted-composable daemon to a workstation artifact. So the launcher, which DOES know, tells it.
|
|
94
|
+
//
|
|
95
|
+
// Without this the daemon connects plaintext-capable to a TLS broker and nothing looks wrong,
|
|
96
|
+
// because it still upgrades on the server's INFO. It holds a STANDING credential and reconnects
|
|
97
|
+
// unattended, so that exposure would repeat on every reconnect with nobody watching.
|
|
98
|
+
...(o.tls ? ["--tls"] : []),
|
|
63
99
|
];
|
|
64
100
|
// Internal child re-exec (the `up` that reached here already seeded); the delivery daemon does not
|
|
65
101
|
// launch agents, so it skips the connector seed on boot (a direct `cotal deliver` still seeds).
|
|
@@ -74,10 +110,10 @@ export function startDeliveryDetached(o = {}) {
|
|
|
74
110
|
* never double-binds. Mints a SCOPED `delivery` cred from the local signer ONCE, writes it to
|
|
75
111
|
* `.cotal/delivery.creds` (0600), and launches the daemon WITHOUT signer access. Best-effort — callers
|
|
76
112
|
* treat it as non-fatal (a missing daemon degrades durable delivery, never live). */
|
|
77
|
-
export async function ensureDelivery(o = {}) {
|
|
113
|
+
export async function ensureDelivery(o = {}, probe = probeLiveness) {
|
|
78
114
|
if (!hasAuth())
|
|
79
115
|
return { running: false }; // open dev mode — no daemon, agents are live-only
|
|
80
|
-
if (
|
|
116
|
+
if (oldHostingManagerVerdict(probe) === "stop-it") {
|
|
81
117
|
console.error("✗ delivery: an old Plane-3-hosting manager is still live (no delivery-aware marker). Refusing to start the daemon - run `cotal down` first, then retry.");
|
|
82
118
|
return { running: false };
|
|
83
119
|
}
|
|
@@ -90,7 +126,15 @@ export async function ensureDelivery(o = {}) {
|
|
|
90
126
|
const creds = await mintCreds(auth, id, "delivery");
|
|
91
127
|
const space = o.space ?? resolveSpace(process.cwd());
|
|
92
128
|
const server = o.server ?? DEFAULT_SERVER;
|
|
93
|
-
|
|
129
|
+
const deliveryState = deliveryLiveness(probe);
|
|
130
|
+
// Same refusal as the manager: an unattributable pid must not be silently reused (a daemon
|
|
131
|
+
// reported running that is not there) nor silently replaced (two daemons on one fanout).
|
|
132
|
+
if (deliveryState === "unknown" || deliveryState === "unattributable")
|
|
133
|
+
throw new Error(`the recorded delivery daemon pid (${readFileSync(PID_PATH(), "utf8").trim()}) cannot be attributed: the kernel answered neither "running" nor "no such process".\n` +
|
|
134
|
+
`A seccomp filter or LSM policy that intercepts \`kill(pid, 0)\` does this, so it is expected inside some sandboxes and containers.\n` +
|
|
135
|
+
`Cotal will not guess: reusing it would report a daemon that is not there, and starting a second would put two daemons on one fanout.\n` +
|
|
136
|
+
`NEXT: verify the process yourself (\`ps -p <pid>\`). If it is gone, remove \`.cotal/delivery.pid\` and re-run. If it is running, use it or stop it.`);
|
|
137
|
+
if (deliveryState !== "alive") {
|
|
94
138
|
// The store's put hardens `.cotal/` first (the cred is born under a private ACL, no race) and
|
|
95
139
|
// lands it atomically — same path and bytes as before the seam.
|
|
96
140
|
await credsStore().put(DELIVERY_CREDS_KEY, creds);
|
|
@@ -109,25 +153,67 @@ export async function ensureDelivery(o = {}) {
|
|
|
109
153
|
* kill runs even if the creds delete fails (finally) — a delete error must never leave the daemon
|
|
110
154
|
* alive to outlive the teardown and reattach to a restarted broker; the error still propagates
|
|
111
155
|
* after the kill so the caller can surface it. */
|
|
112
|
-
export async function stopDelivery() {
|
|
113
|
-
|
|
156
|
+
export async function stopDelivery(probe = probeLiveness, signal) {
|
|
157
|
+
const send = signal ?? ((pid, sig) => process.kill(pid, sig));
|
|
158
|
+
const p = PID_PATH();
|
|
159
|
+
// ORDER MATTERS, AND IT USED TO BE BACKWARDS. The credential was deleted FIRST, in a try whose
|
|
160
|
+
// finally then signalled and removed the pidfile unconditionally. Under a refused signal that left
|
|
161
|
+
// a LIVE daemon with no pidfile and no standing credential: still connected, still serving, and its
|
|
162
|
+
// reconnect and renewal source deleted out from under it, while the function returned success.
|
|
163
|
+
// Nothing is removed now until the process is proven gone.
|
|
164
|
+
// Once death is PROVEN, both records are stale and the pidfile goes first: a creds-delete failure
|
|
165
|
+
// (a blocked path, a store error) must still propagate loudly, but it must not leave behind a
|
|
166
|
+
// pidfile for a process that is definitely gone. delivery-teardown.smoke.ts pins exactly that
|
|
167
|
+
// combination, and caught this when the first version of this fix ordered them the other way.
|
|
168
|
+
const removeRecords = async () => {
|
|
169
|
+
rmSync(p, { force: true });
|
|
114
170
|
await credsStore().delete(DELIVERY_CREDS_KEY);
|
|
171
|
+
};
|
|
172
|
+
if (!existsSync(p)) {
|
|
173
|
+
await credsStore().delete(DELIVERY_CREDS_KEY); // no pid recorded: no live daemon to strand
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
const raw = readFileSync(p, "utf8").trim();
|
|
177
|
+
const pid = parsePid(raw);
|
|
178
|
+
if (pid === undefined) {
|
|
179
|
+
if (raw === "") {
|
|
180
|
+
await removeRecords(); // pre-protocol husk, nothing behind it
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
// See the manager helper: unattributable content may front a live daemon, and clearing it here
|
|
184
|
+
// would also delete the standing credential of a process still using it.
|
|
185
|
+
throw new Error(`the delivery pidfile at ${p} is unattributable (${JSON.stringify(raw)}): it may still front a running daemon nobody can identify.\n` +
|
|
186
|
+
`Refusing to remove it or its standing credential, and refusing to report a clean stop.\n` +
|
|
187
|
+
`NEXT: find and stop that process, then remove the file by hand.`);
|
|
188
|
+
}
|
|
189
|
+
const before = probe(pid);
|
|
190
|
+
if (before === "dead") {
|
|
191
|
+
await removeRecords();
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
if (before === "unknown")
|
|
195
|
+
throw new Error(`refusing to stop the delivery daemon (pid ${pid}): its liveness cannot be determined (a seccomp filter or LSM policy answers kill(pid,0) with an arbitrary errno).\n` +
|
|
196
|
+
`The pidfile and standing credential are LEFT IN PLACE: removing them would strand a daemon that may still be connected, with its renewal source gone.\n` +
|
|
197
|
+
`NEXT: verify with \`ps -p ${pid}\`, then stop it yourself or remove \`.cotal/delivery.pid\` if it is gone.`);
|
|
198
|
+
try {
|
|
199
|
+
send(pid, "SIGTERM");
|
|
115
200
|
}
|
|
116
|
-
|
|
117
|
-
const
|
|
118
|
-
if (
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
try {
|
|
122
|
-
process.kill(pid, "SIGTERM");
|
|
123
|
-
}
|
|
124
|
-
catch {
|
|
125
|
-
/* already gone */
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
rmSync(p);
|
|
201
|
+
catch (e) {
|
|
202
|
+
const code = e.code;
|
|
203
|
+
if (code === "ESRCH") {
|
|
204
|
+
await removeRecords(); // raced to exit between probe and signal
|
|
205
|
+
return;
|
|
129
206
|
}
|
|
207
|
+
throw new Error(`refusing to stop the delivery daemon (pid ${pid}): the signal was rejected (${code ?? "unknown error"}).\n` +
|
|
208
|
+
`EPERM means it belongs to another user, so it is running and not ours to stop. The pidfile and standing credential are LEFT IN PLACE.\n` +
|
|
209
|
+
`NEXT: stop it as its owner, or remove \`.cotal/delivery.pid\` if you are certain it is gone.`);
|
|
130
210
|
}
|
|
211
|
+
const deadline = Date.now() + 15_000;
|
|
212
|
+
while (probe(pid) !== "dead" && Date.now() < deadline)
|
|
213
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
214
|
+
if (probe(pid) !== "dead")
|
|
215
|
+
throw new Error(`the delivery daemon (pid ${pid}) accepted SIGTERM but its death could not be confirmed; its pidfile and credential were preserved rather than recording a stop that did not happen.`);
|
|
216
|
+
await removeRecords();
|
|
131
217
|
}
|
|
132
218
|
/** Bring up the control plane in the correct cutover order: OLD-manager preflight → delivery daemon
|
|
133
219
|
* (auth only, fails closed on a live old manager) → manager (lifecycle, writes the delivery-aware
|
|
@@ -135,7 +221,7 @@ export async function stopDelivery() {
|
|
|
135
221
|
* first only to close the old-manager double-bind window and so freshly-spawned agents find the
|
|
136
222
|
* `ctl.delivery` responder for their boot self-join (a miss honest-degrades to live-only). */
|
|
137
223
|
export async function ensureControlPlane(o = {}) {
|
|
138
|
-
stopOldHostingManagerIfPresent();
|
|
224
|
+
await stopOldHostingManagerIfPresent();
|
|
139
225
|
await ensureDelivery(o);
|
|
140
226
|
return ensureManager(o);
|
|
141
227
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delivery-proc.js","sourceRoot":"","sources":["../../src/lib/delivery-proc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC/F,OAAO,EACL,cAAc,EACd,SAAS,EACT,WAAW,EACX,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"delivery-proc.js","sourceRoot":"","sources":["../../src/lib/delivery-proc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC/F,OAAO,EACL,cAAc,EACd,SAAS,EACT,WAAW,EACX,oBAAoB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,aAAa,EAAsB,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACzL,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,wBAAwB,EAAE,eAAe,EAAE,WAAW,EAAiB,MAAM,mBAAmB,CAAC;AAE3I,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;AACjD,gGAAgG;AAChG,0FAA0F;AAC1F,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,oBAAoB,CAAC,aAAa,EAAE,CAAC,CAAC;AAS/D;;;qFAGqF;AACrF,MAAM,UAAU,gBAAgB,CAAC,QAAuB,aAAa;IACnE,MAAM,CAAC,GAAG,QAAQ,EAAE,CAAC;IACrB,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IACpC,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,GAAG,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC;IAChC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAC,CAAC,mDAAmD;IACnG,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;AACpB,CAAC;AAED;2EAC2E;AAC3E,MAAM,UAAU,UAAU;IACxB,OAAO,gBAAgB,EAAE,KAAK,OAAO,CAAC;AACxC,CAAC;AAID;mDACmD;AACnD,SAAS,OAAO;IACd,OAAO,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;oEAWoE;AACpE,SAAS,wBAAwB,CAAC,QAAuB,aAAa;IACpE,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,gBAAgB;QAAE,OAAO,eAAe,CAAC;IAC9E,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC,CAAC,6CAA6C;IACtF,OAAO,wBAAwB,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,4BAA4B;AACzF,CAAC;AAED;;0GAE0G;AAC1G,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,QAAuB,aAAa,EAAE,MAAiB;IAC1G,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;IAChD,wFAAwF;IACxF,IAAI,OAAO,KAAK,eAAe;QAC7B,MAAM,IAAI,KAAK,CACb,6BAA6B,YAAY,CAAC,gBAAgB,EAAE,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,uMAAuM;YACjR,0NAA0N;YAC1N,oKAAoK,CACvK,CAAC;IACJ,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,mGAAmG,CAAC,CAAC;QACnH,+FAA+F;QAC/F,kGAAkG;QAClG,mGAAmG;QACnG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED;;8FAE8F;AAC9F,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE;IAChD,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG;QACX,GAAG,IAAI;QACP,SAAS;QACT,SAAS;QACT,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;QACtC,UAAU;QACV,CAAC,CAAC,MAAM,IAAI,cAAc;QAC1B,+FAA+F;QAC/F,gGAAgG;QAChG,wFAAwF;QACxF,kGAAkG;QAClG,EAAE;QACF,8FAA8F;QAC9F,gGAAgG;QAChG,qFAAqF;QACrF,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5B,CAAC;IACF,mGAAmG;IACnG,gGAAgG;IAChG,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,yBAAyB,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACxI,SAAS,CAAC,EAAE,CAAC,CAAC;IACd,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,aAAa,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;;sFAIsF;AACtF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAU,EAAE,EAAE,QAAuB,aAAa;IACrF,IAAI,CAAC,OAAO,EAAE;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,kDAAkD;IAC7F,IAAI,wBAAwB,CAAC,KAAK,CAAC,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,CACX,yJAAyJ,CAC1J,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IACD,uGAAuG;IACvG,wGAAwG;IACxG,sGAAsG;IACtG,kBAAkB;IAClB,MAAM,IAAI,GAAG,CAAC,MAAM,gBAAgB,CAAC,UAAU,EAAE,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC;IAC/E,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,cAAc,CAAC;IAC1C,MAAM,aAAa,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC9C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,KAAK,gBAAgB;QACnE,MAAM,IAAI,KAAK,CACb,qCAAqC,YAAY,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,wFAAwF;YAClK,sIAAsI;YACtI,wIAAwI;YACxI,qJAAqJ,CACxJ,CAAC;IACJ,IAAI,aAAa,KAAK,OAAO,EAAE,CAAC;QAC9B,8FAA8F;QAC9F,gEAAgE;QAChE,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;QAClD,qBAAqB,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,kGAAkG;IAClG,uGAAuG;IACvG,wGAAwG;IACxG,mEAAmE;IACnE,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACvF,IAAI,CAAC,KAAK;QACR,OAAO,CAAC,KAAK,CAAC,sGAAsG,CAAC,CAAC;IACxH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;;mDAGmD;AACnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAuB,aAAa,EAAE,MAAiB;IACxF,MAAM,IAAI,GAAa,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACxE,MAAM,CAAC,GAAG,QAAQ,EAAE,CAAC;IACrB,+FAA+F;IAC/F,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,2DAA2D;IAC3D,kGAAkG;IAClG,8FAA8F;IAC9F,8FAA8F;IAC9F,8FAA8F;IAC9F,MAAM,aAAa,GAAG,KAAK,IAAmB,EAAE;QAC9C,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3B,MAAM,UAAU,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAChD,CAAC,CAAC;IACF,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QACnB,MAAM,UAAU,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,4CAA4C;QAC3F,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACf,MAAM,aAAa,EAAE,CAAC,CAAC,uCAAuC;YAC9D,OAAO;QACT,CAAC;QACD,+FAA+F;QAC/F,yEAAyE;QACzE,MAAM,IAAI,KAAK,CACb,2BAA2B,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,+DAA+D;YACnI,0FAA0F;YAC1F,iEAAiE,CACpE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,aAAa,EAAE,CAAC;QACtB,OAAO;IACT,CAAC;IACD,IAAI,MAAM,KAAK,SAAS;QACtB,MAAM,IAAI,KAAK,CACb,6CAA6C,GAAG,sHAAsH;YACpK,yJAAyJ;YACzJ,6BAA6B,GAAG,4EAA4E,CAC/G,CAAC;IACJ,IAAI,CAAC;QACH,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,GAAI,CAA2B,CAAC,IAAI,CAAC;QAC/C,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,MAAM,aAAa,EAAE,CAAC,CAAC,yCAAyC;YAChE,OAAO;QACT,CAAC;QACD,MAAM,IAAI,KAAK,CACb,6CAA6C,GAAG,+BAA+B,IAAI,IAAI,eAAe,MAAM;YAC1G,yIAAyI;YACzI,8FAA8F,CACjG,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;IACrC,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ;QAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACpG,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,MAAM;QACvB,MAAM,IAAI,KAAK,CACb,4BAA4B,GAAG,sJAAsJ,CACtL,CAAC;IACJ,MAAM,aAAa,EAAE,CAAC;AACxB,CAAC;AAED;;;;+FAI+F;AAC/F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAU,EAAE;IACnD,MAAM,8BAA8B,EAAE,CAAC;IACvC,MAAM,cAAc,CAAC,CAAC,CAAC,CAAC;IACxB,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -1,4 +1,19 @@
|
|
|
1
|
-
|
|
1
|
+
import { type LivenessProbe } from "@cotal-ai/workspace";
|
|
2
|
+
/** Exported so the delivery cutover preflight can NAME the pid it refused on: an error that says
|
|
3
|
+
* "cannot be attributed" without saying which pid is not actionable. */
|
|
4
|
+
export declare const MANAGER_PID_PATH: () => string;
|
|
5
|
+
/** The recorded manager's liveness, THREE-VALUED plus absent, because collapsing it to a boolean is
|
|
6
|
+
* what made this dangerous. Both collapses are silent and both are wrong:
|
|
7
|
+
* `!== "dead"` -> an `unknown` reports UP forever; no retry clears it and nothing starts.
|
|
8
|
+
* `=== "alive"` -> an `unknown` reports DOWN and a second manager launches onto a possibly-live one.
|
|
9
|
+
* `unknown` is REACHABLE on a real kernel, not just under a test shim: a Linux seccomp filter
|
|
10
|
+
* (`SECCOMP_RET_ERRNO`) or an LSM policy can return an arbitrary errno for `kill(pid, 0)` without
|
|
11
|
+
* executing it at all, and libuv preserves it. Proven with a live seccomp BPF filter, not by
|
|
12
|
+
* interposition. So the caller has to SEE the third state and refuse. */
|
|
13
|
+
export declare function managerLiveness(probe?: LivenessProbe): "alive" | "dead" | "unknown" | "absent" | "unattributable";
|
|
14
|
+
/** True only if the manager is PROVABLY running. `unknown` is not up, and callers that would ACT on
|
|
15
|
+
* that answer must use {@link managerLiveness} instead: this boolean cannot express the difference
|
|
16
|
+
* between "not running" and "cannot tell", and acting on the difference is the whole point. */
|
|
2
17
|
export declare function managerUp(): boolean;
|
|
3
18
|
/** True if the live manager carries a delivery-aware marker BOUND to its current pid (i.e. it's THIS
|
|
4
19
|
* build, non-hosting). Fail-closed: the marker stores the pid it was written for, and this requires it
|
|
@@ -20,11 +35,20 @@ export declare function startManagerDetached(o?: {
|
|
|
20
35
|
attachHost?: string;
|
|
21
36
|
resumeAttempt?: string;
|
|
22
37
|
resumeCommitToken?: string;
|
|
38
|
+
wsPort?: number;
|
|
23
39
|
}): number;
|
|
24
40
|
/** Make the control plane available: reuse a manager already running for this folder, else start
|
|
25
41
|
* one detached. Best-effort — callers treat it as non-fatal. A caller that needs THE manager to
|
|
26
42
|
* carry a runtime/launch spec (`up -f`) must stop any leftover manager first — a reused one is
|
|
27
43
|
* taken as-is. */
|
|
44
|
+
/** Refuse to stand a manager up OVER a record we cannot read or attribute.
|
|
45
|
+
*
|
|
46
|
+
* Exported because `ensureManager` is not the only path that starts one: `cotal spawn -f` calls
|
|
47
|
+
* `startManagerDetached` directly after its own lease checks, and so skipped this entirely. A lease
|
|
48
|
+
* says nobody is ANSWERING; it does not say the recorded pid is dead. Overwriting an unknown or
|
|
49
|
+
* unattributable record is the same defect as deleting it, reached through a different verb, which
|
|
50
|
+
* is the third time that shape has appeared in this change. Any future starter calls this first. */
|
|
51
|
+
export declare function assertManagerRecordReplaceable(probe?: LivenessProbe): void;
|
|
28
52
|
export declare function ensureManager(o?: {
|
|
29
53
|
space?: string;
|
|
30
54
|
server?: string;
|
|
@@ -34,10 +58,29 @@ export declare function ensureManager(o?: {
|
|
|
34
58
|
attachHost?: string;
|
|
35
59
|
resumeAttempt?: string;
|
|
36
60
|
resumeCommitToken?: string;
|
|
37
|
-
|
|
61
|
+
wsPort?: number;
|
|
62
|
+
}, probe?: LivenessProbe): {
|
|
38
63
|
running: boolean;
|
|
39
64
|
};
|
|
40
|
-
/**
|
|
41
|
-
*
|
|
42
|
-
|
|
65
|
+
/** A signal, injectable for the same reason the probe is: `EPERM` from `kill` is producible only by
|
|
66
|
+
* another user's process or by kernel policy, so the branch that handles it is otherwise unreachable
|
|
67
|
+
* from a test. Production passes nothing. */
|
|
68
|
+
export type SignalFn = (pid: number, signal: NodeJS.Signals) => void;
|
|
69
|
+
/** What a stop actually achieved, because "void" let this function claim success it had not earned. */
|
|
70
|
+
export type StopVerdict = "stopped" | "already-gone";
|
|
71
|
+
/** Blocking sleep. DO NOT reintroduce this for death-waiting: it blocks the event loop, so a child
|
|
72
|
+
* of this process is never reaped, remains a signalable zombie, and reads `alive` forever. That
|
|
73
|
+
* turned `cotal up` + Ctrl-C into an exit-1 that left the broker running. Death-waits await. */
|
|
74
|
+
/** Stop the detached (pty) manager if we started one, and remove its records ONLY once it is gone.
|
|
75
|
+
*
|
|
76
|
+
* THIS USED TO CATCH EVERY SIGNAL FAILURE AS "already gone" AND DELETE THE RECORDS ANYWAY. That was
|
|
77
|
+
* survivable while `EPERM` was misread as dead, because the caller never got here: the cutover
|
|
78
|
+
* preflight skipped a manager it thought was not running. Resolving `EPERM` to `alive` (the fix this
|
|
79
|
+
* change exists for) makes the preflight recognise ANOTHER USER's live manager and call this, at
|
|
80
|
+
* which point the old code sent a signal it was not permitted to send, swallowed the refusal, and
|
|
81
|
+
* deleted the pidfile and marker of a process that was still running and still bound to Plane 3.
|
|
82
|
+
* A correct fix upstream reaching a latent destructive bug downstream is the worst shape available,
|
|
83
|
+
* so this refuses instead: records are removed only on proven death, never on a signal we could not
|
|
84
|
+
* send or a death we could not confirm. Found by review, with a kernel seccomp proof. */
|
|
85
|
+
export declare function stopManager(probe?: LivenessProbe, signal?: SignalFn | undefined): Promise<StopVerdict>;
|
|
43
86
|
//# sourceMappingURL=manager-proc.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager-proc.d.ts","sourceRoot":"","sources":["../../src/lib/manager-proc.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"manager-proc.d.ts","sourceRoot":"","sources":["../../src/lib/manager-proc.ts"],"names":[],"mappings":"AAMA,OAAO,EAA2B,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElF;yEACyE;AACzE,eAAO,MAAM,gBAAgB,QAAO,MAAkC,CAAC;AASvE;;;;;;;0EAO0E;AAC1E,wBAAgB,eAAe,CAAC,KAAK,GAAE,aAA6B,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,GAAG,gBAAgB,CAYhI;AAED;;gGAEgG;AAChG,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AAID;;;;2CAI2C;AAC3C,wBAAgB,wBAAwB,IAAI,OAAO,CAOlD;AAED;;;;0FAI0F;AAC1F,wBAAgB,oBAAoB,CAClC,CAAC,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GACzL,MAAM,CA0CR;AAED;;;mBAGmB;AACnB;;;;;;qGAMqG;AACrG,wBAAgB,8BAA8B,CAAC,KAAK,GAAE,aAA6B,GAAG,IAAI,CAczF;AAED,wBAAgB,aAAa,CAC3B,CAAC,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,EAC1L,KAAK,GAAE,aAA6B,GACnC;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAMtB;AAED;;8CAE8C;AAC9C,MAAM,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;AAErE,uGAAuG;AACvG,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,cAAc,CAAC;AAErD;;iGAEiG;AAGjG;;;;;;;;;;0FAU0F;AAC1F,wBAAsB,WAAW,CAAC,KAAK,GAAE,aAA6B,EAAE,MAAM,GAAE,QAAQ,GAAG,SAAqB,GAAG,OAAO,CAAC,WAAW,CAAC,CAsEtI"}
|
package/dist/lib/manager-proc.js
CHANGED
|
@@ -4,29 +4,46 @@ import { DEFAULT_SERVER } from "@cotal-ai/core";
|
|
|
4
4
|
import { selfArgv } from "./self-exec.js";
|
|
5
5
|
import { resolveSpace } from "./status.js";
|
|
6
6
|
import { cotalPath } from "./paths.js";
|
|
7
|
-
|
|
7
|
+
import { parsePid, probeLiveness } from "@cotal-ai/workspace";
|
|
8
|
+
/** Exported so the delivery cutover preflight can NAME the pid it refused on: an error that says
|
|
9
|
+
* "cannot be attributed" without saying which pid is not actionable. */
|
|
10
|
+
export const MANAGER_PID_PATH = () => cotalPath("manager.pid");
|
|
11
|
+
const PID_PATH = MANAGER_PID_PATH;
|
|
8
12
|
/** Sibling marker of `manager.pid`: written by THIS build's manager (which no longer hosts Plane-3 —
|
|
9
13
|
* the server-side delivery daemon does). Its presence beside a live `manager.pid` proves the manager is
|
|
10
14
|
* "delivery-aware" / non-hosting. A live `manager.pid` WITHOUT this marker is an OLD (pre-delivery-daemon)
|
|
11
15
|
* manager that still calls `startPlane3` — the delivery preflight stops it before the daemon binds, so an
|
|
12
16
|
* old hosting manager never double-binds `fanout`/`reader` against the new daemon. */
|
|
13
17
|
const DELIVERY_AWARE_MARKER = () => cotalPath("manager.delivery-aware");
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
/** True if the manager we started for this folder is still running (pid file + liveness). */
|
|
24
|
-
export function managerUp() {
|
|
18
|
+
/** The recorded manager's liveness, THREE-VALUED plus absent, because collapsing it to a boolean is
|
|
19
|
+
* what made this dangerous. Both collapses are silent and both are wrong:
|
|
20
|
+
* `!== "dead"` -> an `unknown` reports UP forever; no retry clears it and nothing starts.
|
|
21
|
+
* `=== "alive"` -> an `unknown` reports DOWN and a second manager launches onto a possibly-live one.
|
|
22
|
+
* `unknown` is REACHABLE on a real kernel, not just under a test shim: a Linux seccomp filter
|
|
23
|
+
* (`SECCOMP_RET_ERRNO`) or an LSM policy can return an arbitrary errno for `kill(pid, 0)` without
|
|
24
|
+
* executing it at all, and libuv preserves it. Proven with a live seccomp BPF filter, not by
|
|
25
|
+
* interposition. So the caller has to SEE the third state and refuse. */
|
|
26
|
+
export function managerLiveness(probe = probeLiveness) {
|
|
25
27
|
const p = PID_PATH();
|
|
26
28
|
if (!existsSync(p))
|
|
27
|
-
return
|
|
28
|
-
const
|
|
29
|
-
|
|
29
|
+
return "absent";
|
|
30
|
+
const raw = readFileSync(p, "utf8").trim();
|
|
31
|
+
if (raw === "")
|
|
32
|
+
return "absent"; // a pre-protocol husk: nothing is behind it
|
|
33
|
+
const pid = parsePid(raw);
|
|
34
|
+
// NOT `absent`. Folding non-empty corrupt content into "no manager recorded" is what let the
|
|
35
|
+
// ensure paths OVERWRITE it and launch a replacement, which is the same defect as deleting it:
|
|
36
|
+
// that record may front a live process nobody can identify. `absent` means no pidfile (or an
|
|
37
|
+
// empty husk); corrupt content is its own state and every action path must refuse on it.
|
|
38
|
+
if (pid === undefined)
|
|
39
|
+
return "unattributable";
|
|
40
|
+
return probe(pid);
|
|
41
|
+
}
|
|
42
|
+
/** True only if the manager is PROVABLY running. `unknown` is not up, and callers that would ACT on
|
|
43
|
+
* that answer must use {@link managerLiveness} instead: this boolean cannot express the difference
|
|
44
|
+
* between "not running" and "cannot tell", and acting on the difference is the whole point. */
|
|
45
|
+
export function managerUp() {
|
|
46
|
+
return managerLiveness() === "alive";
|
|
30
47
|
}
|
|
31
48
|
/** True if the live manager carries a delivery-aware marker BOUND to its current pid (i.e. it's THIS
|
|
32
49
|
* build, non-hosting). Fail-closed: the marker stores the pid it was written for, and this requires it
|
|
@@ -79,6 +96,8 @@ export function startManagerDetached(o = {}) {
|
|
|
79
96
|
...(o.launch ? ["--launch", o.launch] : []),
|
|
80
97
|
...(o.resumeAttempt ? ["--resume-attempt", o.resumeAttempt] : []),
|
|
81
98
|
...(o.resumeCommitToken ? ["--resume-commit-token", o.resumeCommitToken] : []),
|
|
99
|
+
// P2 item 6: the broker ws listener port (loopback) for the console session client's wsUrl.
|
|
100
|
+
...(o.wsPort !== undefined ? ["--ws-port", String(o.wsPort)] : []),
|
|
82
101
|
];
|
|
83
102
|
// This is an INTERNAL child re-exec: the `up`/`spawn` that reached here already ran the first-run
|
|
84
103
|
// connector seed, so the manager skips it on boot (a direct `cotal supervise` still seeds).
|
|
@@ -95,28 +114,110 @@ export function startManagerDetached(o = {}) {
|
|
|
95
114
|
* one detached. Best-effort — callers treat it as non-fatal. A caller that needs THE manager to
|
|
96
115
|
* carry a runtime/launch spec (`up -f`) must stop any leftover manager first — a reused one is
|
|
97
116
|
* taken as-is. */
|
|
98
|
-
|
|
99
|
-
|
|
117
|
+
/** Refuse to stand a manager up OVER a record we cannot read or attribute.
|
|
118
|
+
*
|
|
119
|
+
* Exported because `ensureManager` is not the only path that starts one: `cotal spawn -f` calls
|
|
120
|
+
* `startManagerDetached` directly after its own lease checks, and so skipped this entirely. A lease
|
|
121
|
+
* says nobody is ANSWERING; it does not say the recorded pid is dead. Overwriting an unknown or
|
|
122
|
+
* unattributable record is the same defect as deleting it, reached through a different verb, which
|
|
123
|
+
* is the third time that shape has appeared in this change. Any future starter calls this first. */
|
|
124
|
+
export function assertManagerRecordReplaceable(probe = probeLiveness) {
|
|
125
|
+
const state = managerLiveness(probe);
|
|
126
|
+
if (state === "unattributable")
|
|
127
|
+
throw new Error(`the manager pidfile at ${PID_PATH()} holds content that is not a pid (${JSON.stringify(readFileSync(PID_PATH(), "utf8").trim())}).\n` +
|
|
128
|
+
`Refusing to start a manager over it: that record may front a live process nobody can identify, and overwriting it would orphan the process while reporting a healthy control plane.\n` +
|
|
129
|
+
`NEXT: find and stop that process, then remove \`.cotal/manager.pid\` by hand.`);
|
|
130
|
+
if (state === "unknown")
|
|
131
|
+
throw new Error(`the recorded manager pid (${readFileSync(PID_PATH(), "utf8").trim()}) cannot be attributed: the kernel answered neither "running" nor "no such process" (a seccomp filter or LSM policy does this inside some sandboxes).\n` +
|
|
132
|
+
`Refusing to start a manager over it: it may still be running and bound to the control plane.\n` +
|
|
133
|
+
`NEXT: verify with \`ps -p <pid>\`. If it is gone, remove \`.cotal/manager.pid\` and re-run.`);
|
|
134
|
+
}
|
|
135
|
+
export function ensureManager(o = {}, probe = probeLiveness) {
|
|
136
|
+
const state = managerLiveness(probe);
|
|
137
|
+
if (state === "alive")
|
|
100
138
|
return { running: true };
|
|
139
|
+
assertManagerRecordReplaceable(probe); // refuses on unknown / unattributable
|
|
101
140
|
startManagerDetached(o);
|
|
102
141
|
return { running: true };
|
|
103
142
|
}
|
|
104
|
-
/**
|
|
105
|
-
*
|
|
106
|
-
|
|
143
|
+
/** Blocking sleep. DO NOT reintroduce this for death-waiting: it blocks the event loop, so a child
|
|
144
|
+
* of this process is never reaped, remains a signalable zombie, and reads `alive` forever. That
|
|
145
|
+
* turned `cotal up` + Ctrl-C into an exit-1 that left the broker running. Death-waits await. */
|
|
146
|
+
// (sleepSync removed: see above.)
|
|
147
|
+
/** Stop the detached (pty) manager if we started one, and remove its records ONLY once it is gone.
|
|
148
|
+
*
|
|
149
|
+
* THIS USED TO CATCH EVERY SIGNAL FAILURE AS "already gone" AND DELETE THE RECORDS ANYWAY. That was
|
|
150
|
+
* survivable while `EPERM` was misread as dead, because the caller never got here: the cutover
|
|
151
|
+
* preflight skipped a manager it thought was not running. Resolving `EPERM` to `alive` (the fix this
|
|
152
|
+
* change exists for) makes the preflight recognise ANOTHER USER's live manager and call this, at
|
|
153
|
+
* which point the old code sent a signal it was not permitted to send, swallowed the refusal, and
|
|
154
|
+
* deleted the pidfile and marker of a process that was still running and still bound to Plane 3.
|
|
155
|
+
* A correct fix upstream reaching a latent destructive bug downstream is the worst shape available,
|
|
156
|
+
* so this refuses instead: records are removed only on proven death, never on a signal we could not
|
|
157
|
+
* send or a death we could not confirm. Found by review, with a kernel seccomp proof. */
|
|
158
|
+
export async function stopManager(probe = probeLiveness, signal = undefined) {
|
|
159
|
+
const send = signal ?? ((pid, sig) => process.kill(pid, sig));
|
|
107
160
|
const p = PID_PATH();
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
161
|
+
const marker = DELIVERY_AWARE_MARKER();
|
|
162
|
+
const clear = () => {
|
|
163
|
+
rmSync(marker, { force: true });
|
|
164
|
+
rmSync(p, { force: true });
|
|
165
|
+
};
|
|
166
|
+
if (!existsSync(p)) {
|
|
167
|
+
rmSync(marker, { force: true }); // a marker with no pid records nothing
|
|
168
|
+
return "already-gone";
|
|
169
|
+
}
|
|
170
|
+
const raw = readFileSync(p, "utf8").trim();
|
|
171
|
+
const pid = parsePid(raw);
|
|
172
|
+
if (pid === undefined) {
|
|
173
|
+
// An EMPTY pidfile is a pre-protocol husk with nothing behind it, and clearing it is safe. ANY
|
|
174
|
+
// OTHER unattributable content (garbled, fractional, out of range) may still front a LIVE
|
|
175
|
+
// process we cannot identify or signal, so removing it would orphan that process while
|
|
176
|
+
// reporting a clean stop. The contract at the top of pid.ts says such content is never a pid to
|
|
177
|
+
// delete a record against; this is that rule at the destructive end, matching down.ts:298-311
|
|
178
|
+
// and stopAuthService, which already refuse. My first version cleared it.
|
|
179
|
+
if (raw === "") {
|
|
180
|
+
clear();
|
|
181
|
+
return "already-gone";
|
|
115
182
|
}
|
|
116
|
-
|
|
117
|
-
|
|
183
|
+
throw new Error(`the manager pidfile at ${p} is unattributable (${JSON.stringify(raw)}): it may still front a running process nobody can identify.\n` +
|
|
184
|
+
`Refusing to remove it or report a clean stop; the delivery-aware marker is preserved with it.\n` +
|
|
185
|
+
`NEXT: find and stop that process, then remove the file by hand.`);
|
|
186
|
+
}
|
|
187
|
+
const before = probe(pid);
|
|
188
|
+
if (before === "dead") {
|
|
189
|
+
clear();
|
|
190
|
+
return "already-gone";
|
|
191
|
+
}
|
|
192
|
+
if (before === "unknown")
|
|
193
|
+
throw new Error(`refusing to stop manager pid ${pid}: its liveness cannot be determined (the kernel answered neither "running" nor "no such process"; a seccomp filter or LSM policy does this).\n` +
|
|
194
|
+
`The pidfile and delivery-aware marker are LEFT IN PLACE: deleting them would orphan a process that may still be bound to the control plane.\n` +
|
|
195
|
+
`NEXT: verify with \`ps -p ${pid}\`, then stop it yourself or remove \`.cotal/manager.pid\` if it is gone.`);
|
|
196
|
+
try {
|
|
197
|
+
send(pid, "SIGTERM");
|
|
198
|
+
}
|
|
199
|
+
catch (e) {
|
|
200
|
+
const code = e.code;
|
|
201
|
+
if (code === "ESRCH") {
|
|
202
|
+
clear(); // died between the probe and the signal, which is an ordinary race and genuinely gone
|
|
203
|
+
return "already-gone";
|
|
118
204
|
}
|
|
205
|
+
throw new Error(`refusing to stop manager pid ${pid}: the signal was rejected (${code ?? "unknown error"}).\n` +
|
|
206
|
+
`EPERM here means the process belongs to another user, so it is running and NOT ours to stop. The pidfile and marker are LEFT IN PLACE.\n` +
|
|
207
|
+
`NEXT: stop it as its owner, or remove \`.cotal/manager.pid\` if you are certain it is gone.`);
|
|
119
208
|
}
|
|
120
|
-
|
|
209
|
+
// The signal was accepted, which is not the same as the process being gone. Prove it before
|
|
210
|
+
// removing the record, bounded, because a record deleted while its process lives is the defect.
|
|
211
|
+
// AWAIT, never a blocking sleep: this process spawned the manager, so the event loop must run
|
|
212
|
+
// for it to be reaped. A blocked loop leaves a zombie that still answers kill(pid,0).
|
|
213
|
+
for (let i = 0; i < 40 && probe(pid) === "alive"; i++)
|
|
214
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
215
|
+
const after = probe(pid);
|
|
216
|
+
if (after !== "dead")
|
|
217
|
+
throw new Error(`manager pid ${pid} accepted SIGTERM but was still ${after === "alive" ? "running" : "unattributable"} after 2s.\n` +
|
|
218
|
+
`The pidfile and marker are LEFT IN PLACE rather than recording a stop that did not happen.\n` +
|
|
219
|
+
`NEXT: check \`ps -p ${pid}\` and stop it directly if it is wedged.`);
|
|
220
|
+
clear();
|
|
221
|
+
return "stopped";
|
|
121
222
|
}
|
|
122
223
|
//# sourceMappingURL=manager-proc.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager-proc.js","sourceRoot":"","sources":["../../src/lib/manager-proc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC1G,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"manager-proc.js","sourceRoot":"","sources":["../../src/lib/manager-proc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC1G,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAsB,MAAM,qBAAqB,CAAC;AAElF;yEACyE;AACzE,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAW,EAAE,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;AACvE,MAAM,QAAQ,GAAG,gBAAgB,CAAC;AAClC;;;;uFAIuF;AACvF,MAAM,qBAAqB,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;AAExE;;;;;;;0EAO0E;AAC1E,MAAM,UAAU,eAAe,CAAC,QAAuB,aAAa;IAClE,MAAM,CAAC,GAAG,QAAQ,EAAE,CAAC;IACrB,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IACpC,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,GAAG,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC,CAAC,4CAA4C;IAC7E,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,6FAA6F;IAC7F,+FAA+F;IAC/F,6FAA6F;IAC7F,yFAAyF;IACzF,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAC;IAC/C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC;AACpB,CAAC;AAED;;gGAEgG;AAChG,MAAM,UAAU,SAAS;IACvB,OAAO,eAAe,EAAE,KAAK,OAAO,CAAC;AACvC,CAAC;AAID;;;;2CAI2C;AAC3C,MAAM,UAAU,wBAAwB;IACtC,MAAM,UAAU,GAAG,qBAAqB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;IAC3B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAClE,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,SAAS,KAAK,OAAO,CAAC;AACzF,CAAC;AAED;;;;0FAI0F;AAC1F,MAAM,UAAU,oBAAoB,CAClC,IAAwL,EAAE;IAE1L,4FAA4F;IAC5F,iGAAiG;IACjG,gGAAgG;IAChG,qEAAqE;IACrE,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IAC1D,iGAAiG;IACjG,+FAA+F;IAC/F,8FAA8F;IAC9F,yCAAyC;IACzC,IAAI,CAAC;QAAC,SAAS,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,gDAAgD,CAAC,CAAC;IAC9G,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG;QACX,GAAG,IAAI;QACP,WAAW;QACX,SAAS;QACT,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;QACtC,UAAU;QACV,CAAC,CAAC,MAAM,IAAI,cAAc;QAC1B,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,0FAA0F;QAC1F,gGAAgG;QAChG,mGAAmG;QACnG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,mGAAmG;QACnG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,uBAAuB,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,4FAA4F;QAC5F,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACnE,CAAC;IACF,kGAAkG;IAClG,4FAA4F;IAC5F,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,yBAAyB,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACxI,SAAS,CAAC,EAAE,CAAC,CAAC;IACd,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,aAAa,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,qGAAqG;IACrG,mGAAmG;IACnG,aAAa,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;mBAGmB;AACnB;;;;;;qGAMqG;AACrG,MAAM,UAAU,8BAA8B,CAAC,QAAuB,aAAa;IACjF,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,gBAAgB;QAC5B,MAAM,IAAI,KAAK,CACb,0BAA0B,QAAQ,EAAE,qCAAqC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM;YACpI,uLAAuL;YACvL,+EAA+E,CAClF,CAAC;IACJ,IAAI,KAAK,KAAK,SAAS;QACrB,MAAM,IAAI,KAAK,CACb,6BAA6B,YAAY,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,yJAAyJ;YAC3N,gGAAgG;YAChG,6FAA6F,CAChG,CAAC;AACN,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,IAAwL,EAAE,EAC1L,QAAuB,aAAa;IAEpC,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAChD,8BAA8B,CAAC,KAAK,CAAC,CAAC,CAAC,sCAAsC;IAC7E,oBAAoB,CAAC,CAAC,CAAC,CAAC;IACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAUD;;iGAEiG;AACjG,kCAAkC;AAElC;;;;;;;;;;0FAU0F;AAC1F,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAuB,aAAa,EAAE,SAA+B,SAAS;IAC9G,MAAM,IAAI,GAAa,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACxE,MAAM,CAAC,GAAG,QAAQ,EAAE,CAAC;IACrB,MAAM,MAAM,GAAG,qBAAqB,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC;IACF,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QACnB,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,uCAAuC;QACxE,OAAO,cAAc,CAAC;IACxB,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,+FAA+F;QAC/F,0FAA0F;QAC1F,uFAAuF;QACvF,gGAAgG;QAChG,8FAA8F;QAC9F,0EAA0E;QAC1E,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACf,KAAK,EAAE,CAAC;YACR,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,KAAK,CACb,0BAA0B,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,gEAAgE;YACnI,iGAAiG;YACjG,iEAAiE,CACpE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,KAAK,EAAE,CAAC;QACR,OAAO,cAAc,CAAC;IACxB,CAAC;IACD,IAAI,MAAM,KAAK,SAAS;QACtB,MAAM,IAAI,KAAK,CACb,gCAAgC,GAAG,gJAAgJ;YACjL,+IAA+I;YAC/I,6BAA6B,GAAG,2EAA2E,CAC9G,CAAC;IACJ,IAAI,CAAC;QACH,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,GAAI,CAA2B,CAAC,IAAI,CAAC;QAC/C,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,KAAK,EAAE,CAAC,CAAC,sFAAsF;YAC/F,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,KAAK,CACb,gCAAgC,GAAG,8BAA8B,IAAI,IAAI,eAAe,MAAM;YAC5F,0IAA0I;YAC1I,6FAA6F,CAChG,CAAC;IACJ,CAAC;IACD,4FAA4F;IAC5F,gGAAgG;IAChG,8FAA8F;IAC9F,sFAAsF;IACtF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,EAAE;QAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACnG,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,MAAM;QAClB,MAAM,IAAI,KAAK,CACb,eAAe,GAAG,mCAAmC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,cAAc;YACjH,8FAA8F;YAC9F,uBAAuB,GAAG,0CAA0C,CACvE,CAAC;IACJ,KAAK,EAAE,CAAC;IACR,OAAO,SAAS,CAAC;AACnB,CAAC"}
|