@jbkjr/relay 0.0.0-stage → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/LICENSES/Apache-2.0.txt +201 -0
- package/LICENSES/README.md +19 -0
- package/LICENSES/codex-NOTICE.txt +6 -0
- package/README.md +386 -2
- package/SECURITY.md +145 -0
- package/adapters/briefs/claude.md +43 -0
- package/adapters/briefs/codex.md +56 -0
- package/adapters/briefs/models.json +33 -0
- package/adapters/build.mjs +8 -0
- package/adapters/claude/.claude-plugin/marketplace.json +13 -0
- package/adapters/claude/relay/.claude-plugin/plugin.json +5 -0
- package/adapters/claude/relay/agents/codex.md +21 -0
- package/adapters/claude/relay/agents/operator.md +21 -0
- package/adapters/claude/relay/skills/ask/SKILL.md +20 -0
- package/adapters/claude/relay/skills/chat/SKILL.md +13 -0
- package/adapters/claude/relay/skills/delegate/SKILL.md +21 -0
- package/adapters/claude/relay/skills/discuss/SKILL.md +19 -0
- package/adapters/claude/relay/skills/review/SKILL.md +21 -0
- package/adapters/claude/relay/skills/tree/SKILL.md +10 -0
- package/adapters/claude/relay/skills/write-for-codex/SKILL.md +78 -0
- package/adapters/codex/.agents/plugins/marketplace.json +20 -0
- package/adapters/codex/AGENTS.snippet.md +5 -0
- package/adapters/codex/relay/.codex-plugin/plugin.json +6 -0
- package/adapters/codex/relay/skills/relay-ask/SKILL.md +20 -0
- package/adapters/codex/relay/skills/relay-chat/SKILL.md +13 -0
- package/adapters/codex/relay/skills/relay-delegate/SKILL.md +21 -0
- package/adapters/codex/relay/skills/relay-discuss/SKILL.md +19 -0
- package/adapters/codex/relay/skills/relay-review/SKILL.md +21 -0
- package/adapters/codex/relay/skills/relay-tree/SKILL.md +10 -0
- package/adapters/codex/relay/skills/relay-write-for-claude/SKILL.md +66 -0
- package/adapters/generate.mjs +124 -0
- package/bin/relay +59 -0
- package/codex-pin.json +12 -0
- package/dist/backends/acp.js +1424 -0
- package/dist/backends/claude.js +890 -0
- package/dist/backends/cli-version.js +112 -0
- package/dist/backends/codex-app.js +1525 -0
- package/dist/backends/codex-daybreak.js +75 -0
- package/dist/backends/codex-env.js +656 -0
- package/dist/backends/codex-network.js +304 -0
- package/dist/backends/codex-pin.js +320 -0
- package/dist/backends/codex-shell-policy.js +182 -0
- package/dist/backends/codex-stage2.js +998 -0
- package/dist/backends/codex-subagents.js +47 -0
- package/dist/backends/codex.js +637 -0
- package/dist/backends/copilot.js +246 -0
- package/dist/backends/env.js +147 -0
- package/dist/backends/gemini.js +607 -0
- package/dist/backends/grok.js +337 -0
- package/dist/backends/index.js +8 -0
- package/dist/backends/native-temp.js +187 -0
- package/dist/backends/never-admission.js +90 -0
- package/dist/backends/run.js +67 -0
- package/dist/backends/stream.js +230 -0
- package/dist/backends/tool-discovery.js +15 -0
- package/dist/backends/types.js +264 -0
- package/dist/build-info.json +5 -0
- package/dist/cli.js +344 -0
- package/dist/confine/bridge.js +113 -0
- package/dist/confine/broker-core.js +76 -0
- package/dist/confine/broker-http.js +650 -0
- package/dist/confine/broker-names.js +9 -0
- package/dist/confine/broker.js +258 -0
- package/dist/confine/canary.js +380 -0
- package/dist/confine/diagnostics.js +122 -0
- package/dist/confine/gemini-tools-main.js +8 -0
- package/dist/confine/gemini-tools.js +181 -0
- package/dist/confine/hardlinks.js +58 -0
- package/dist/confine/harness-config.js +16 -0
- package/dist/confine/index.js +117 -0
- package/dist/confine/leases.js +217 -0
- package/dist/confine/paths.js +6 -0
- package/dist/confine/peer.js +115 -0
- package/dist/confine/port-policy.js +8 -0
- package/dist/confine/ports.js +365 -0
- package/dist/confine/private-root.js +208 -0
- package/dist/confine/probe-stage2.js +501 -0
- package/dist/confine/probe.js +1109 -0
- package/dist/confine/proxy.js +783 -0
- package/dist/confine/sbcheck.js +208 -0
- package/dist/confine/seatbelt-never.js +41 -0
- package/dist/confine/seatbelt-stage2.js +139 -0
- package/dist/confine/seatbelt.js +1129 -0
- package/dist/confine/sockets.js +36 -0
- package/dist/confine/turn-probe.js +19 -0
- package/dist/confine/turn.js +86 -0
- package/dist/core/adoption.js +124 -0
- package/dist/core/api-keys.js +423 -0
- package/dist/core/approvals.js +740 -0
- package/dist/core/build-identity.js +248 -0
- package/dist/core/caller.js +749 -0
- package/dist/core/calls.js +102 -0
- package/dist/core/claude-settings.js +1045 -0
- package/dist/core/config.js +453 -0
- package/dist/core/context.js +186 -0
- package/dist/core/db.js +753 -0
- package/dist/core/dispatch.js +110 -0
- package/dist/core/env-broker.js +505 -0
- package/dist/core/env-policy.js +182 -0
- package/dist/core/escalation-authority.js +76 -0
- package/dist/core/escalation-contract.js +13 -0
- package/dist/core/escalation-person-path.js +131 -0
- package/dist/core/escalation-policy.js +79 -0
- package/dist/core/escalation-runtime.js +135 -0
- package/dist/core/escalation-state.js +42 -0
- package/dist/core/escalations.js +191 -0
- package/dist/core/events.js +235 -0
- package/dist/core/executables.js +147 -0
- package/dist/core/extensions.js +645 -0
- package/dist/core/families.js +204 -0
- package/dist/core/git.js +657 -0
- package/dist/core/identity.js +113 -0
- package/dist/core/liveness.js +68 -0
- package/dist/core/mediated-network-provider.js +81 -0
- package/dist/core/never-checkpoint.js +76 -0
- package/dist/core/never-coverage.js +97 -0
- package/dist/core/never-network.js +49 -0
- package/dist/core/never-policy.js +302 -0
- package/dist/core/never-proxy-policy.js +109 -0
- package/dist/core/never-schema.js +61 -0
- package/dist/core/never-tools.js +56 -0
- package/dist/core/notices.js +48 -0
- package/dist/core/ops.js +959 -0
- package/dist/core/outcome.js +224 -0
- package/dist/core/params.js +32 -0
- package/dist/core/presence.js +174 -0
- package/dist/core/procs.js +546 -0
- package/dist/core/prompts.js +363 -0
- package/dist/core/protected-config.js +238 -0
- package/dist/core/raw-network.js +192 -0
- package/dist/core/redact.js +152 -0
- package/dist/core/refusal.js +33 -0
- package/dist/core/relay-command.js +310 -0
- package/dist/core/reported.js +3 -0
- package/dist/core/restart-overlay.js +651 -0
- package/dist/core/retention.js +1135 -0
- package/dist/core/secrets.js +329 -0
- package/dist/core/session-token.js +175 -0
- package/dist/core/session.js +744 -0
- package/dist/core/shell.js +2 -0
- package/dist/core/shipped-pin.js +47 -0
- package/dist/core/shown.js +42 -0
- package/dist/core/slots.js +67 -0
- package/dist/core/stage2-executors.js +453 -0
- package/dist/core/stage2-switch.js +5 -0
- package/dist/core/steering.js +149 -0
- package/dist/core/storage.js +358 -0
- package/dist/core/toml.js +175 -0
- package/dist/core/views.js +324 -0
- package/dist/core/wait.js +114 -0
- package/dist/core/wake.js +207 -0
- package/dist/core/watch.js +247 -0
- package/dist/core/worktree.js +213 -0
- package/dist/hosts/claude-desktop.js +274 -0
- package/dist/hosts/claude.js +249 -0
- package/dist/hosts/codex.js +329 -0
- package/dist/hosts/desktop-shared.js +20 -0
- package/dist/hosts/index.js +11 -0
- package/dist/hosts/types.js +4 -0
- package/dist/install/basics.js +18 -0
- package/dist/install/claude-init.js +133 -0
- package/dist/install/common.js +655 -0
- package/dist/install/doctor.js +629 -0
- package/dist/install/qualify-codex.js +205 -0
- package/dist/install.js +524 -0
- package/dist/mcp/elicit.js +209 -0
- package/dist/mcp/gateway.js +218 -0
- package/dist/mcp/host.js +253 -0
- package/dist/mcp/server.js +1124 -0
- package/dist/mcp/stdio.js +37 -0
- package/dist/modes/discussion.js +393 -0
- package/dist/modes/review.js +722 -0
- package/dist/native/relay-sbcheck +0 -0
- package/dist/targets/briefs.js +86 -0
- package/dist/targets/claude.js +94 -0
- package/dist/targets/codex.js +66 -0
- package/dist/targets/copilot.js +93 -0
- package/dist/targets/gemini.js +119 -0
- package/dist/targets/grok.js +126 -0
- package/dist/targets/index.js +192 -0
- package/dist/targets/types.js +18 -0
- package/dist/targets/verified.js +35 -0
- package/dist/worker.js +113 -0
- package/docs/UPGRADING.md +1502 -0
- package/docs/pinned-codex.md +321 -0
- package/experiments/stage2-redteam/lib/oracles.mjs +214 -0
- package/experiments/stage2-redteam/lib/report.mjs +35 -0
- package/npm-shrinkwrap.json +1632 -0
- package/package.json +82 -4
- package/scripts/codex-qualify/codex-pin-store.mjs +353 -0
- package/scripts/codex-qualify/codex-qualifier-process.mjs +265 -0
- package/scripts/codex-qualify/codex-qualifier.example.json +5 -0
- package/scripts/codex-qualify/codex-qualifier.mjs +281 -0
- package/scripts/codex-qualify/codex-shipped-pin.mjs +158 -0
- package/scripts/codex-qualify/codex-stage2-pin.mjs +48 -0
- package/scripts/codex-qualify/codex-stage2.mjs +2381 -0
- package/scripts/codex-qualify/install-codex-qualifier.mjs +98 -0
- package/scripts/codex-qualify/main-module.mjs +14 -0
- package/scripts/codex-qualify/relay-codex-qualifier.plist +37 -0
- package/scripts/codex-qualify/scratch-env.mjs +31 -0
- package/scripts/codex-qualify/session.mjs +96 -0
- package/scripts/codex-qualify/stage2-escalation-cases.mjs +125 -0
- package/test/codex-permissions.test.ts +875 -0
- package/test/fixtures/boundary-probes.mjs +17 -0
- package/test/fixtures/denial-stream-host.mjs +7 -0
- package/test/fixtures/families.json +246 -0
- package/test/fixtures/install-snapshots/claude-desktop-foreign.txt +158 -0
- package/test/fixtures/install-snapshots/claude-desktop-fresh.txt +101 -0
- package/test/fixtures/install-snapshots/claude-desktop-installed.txt +161 -0
- package/test/fixtures/install-snapshots/claude-foreign.txt +133 -0
- package/test/fixtures/install-snapshots/claude-fresh.txt +77 -0
- package/test/fixtures/install-snapshots/claude-installed.txt +133 -0
- package/test/fixtures/install-snapshots/codex-foreign.txt +159 -0
- package/test/fixtures/install-snapshots/codex-fresh.txt +107 -0
- package/test/fixtures/install-snapshots/codex-installed.txt +161 -0
- package/test/fixtures/link-parent.mjs +6 -0
- package/test/fixtures/mediated-command.cjs +72 -0
- package/test/fixtures/native-skill-isolation.json +280 -0
- package/test/fixtures/pinned-codex-app-server.mjs +25 -0
- package/test/fixtures/qualified-ext-adapters.mjs +36 -0
- package/test/fixtures/refusals/README.md +52 -0
- package/test/fixtures/refusals/claude-source-result.json +7 -0
- package/test/fixtures/refusals/openai-app-capacity.json +24 -0
- package/test/fixtures/refusals/openai-app-cyberPolicy.json +24 -0
- package/test/fixtures/refusals/openai-gpt-6-astra-max.json +12 -0
- package/test/fixtures/refusals/openai-gpt-6.1-sol-xhigh.json +12 -0
- package/test/fixtures/refusals/openai-review-capacity.json +24 -0
- package/test/fixtures/relay-0.2.0.sql +57 -0
- package/test/fixtures/relay-0.9.1.gen.mjs +212 -0
- package/test/fixtures/relay-0.9.1.sql +340 -0
- package/test/fixtures/slow-boundary-probes.mjs +37 -0
- package/test/fixtures/stable-surfaces/config.json +1122 -0
- package/test/fixtures/stable-surfaces/escalation.json +128 -0
- package/test/fixtures/stable-surfaces/events.json +37 -0
- package/test/fixtures/stable-surfaces/gemini-native-resume.json +4 -0
- package/test/fixtures/stable-surfaces/inbox.jsonl +2 -0
- package/test/fixtures/stable-surfaces/notice.txt +1 -0
- package/test/fixtures/stable-surfaces/refusal-item.json +28 -0
- package/test/fixtures/stable-surfaces/refusal-review.json +23 -0
- package/test/fixtures/stable-surfaces/registry.json +49 -0
- package/test/fixtures/stable-surfaces/tools-desktop.json +575 -0
- package/test/fixtures/stable-surfaces/tools.json +716 -0
- package/test/fixtures/stable-surfaces/usage.txt +46 -0
- package/test/fixtures/stage2-acp-native.mjs +138 -0
- package/test/fixtures/stage2-app-server.mjs +23 -0
- package/test/fixtures/stage2-catalog.json +91 -0
- package/test/fixtures/stage2-hostile-web.mjs +35 -0
- package/test/fixtures/stage2-native-network.mjs +62 -0
- package/test/fixtures/stage2-worker.ts +87 -0
- package/test/fixtures/status-views/expected.json +7825 -0
- package/test/fixtures/status-views/registries.json +1 -0
- package/test/fixtures/steering.json +209 -0
- package/test/helpers.ts +740 -0
- package/test/host-env.ts +7 -0
- package/test/keychain-decoy.ts +197 -0
- package/test/live-scope.ts +44 -0
- package/test/mediated-network.test.ts +192 -0
- package/test/mediated-proxy.test.ts +264 -0
- package/test/native-temp-alias.ts +10 -0
- package/test/never-policy.test.ts +629 -0
- package/test/never-tools.test.ts +136 -0
- package/test/process-home.ts +18 -0
- package/test/proxy.test.ts +531 -0
- package/test/restart-overlay-live.test.ts +85 -0
- package/test/restart-overlay-profile.test.ts +66 -0
- package/test/restart-overlay.test.ts +416 -0
- package/test/stable-surfaces.test.ts +249 -0
- package/test/stage2-escalation-live.test.ts +544 -0
- package/test/stage2-escalation-qualification.test.ts +170 -0
- package/test/stage2-escalation.test.ts +747 -0
- package/test/stage2-executor.test.ts +849 -0
- package/test/stage2-grant.test.ts +603 -0
- package/test/stage2-native-live.test.ts +647 -0
- package/test/stage2-network-decisions.test.ts +70 -0
- package/test/stage2-probe.test.ts +845 -0
- package/test/stage2-standing-live.test.ts +203 -0
- package/test/stubs/broker-host.mjs +59 -0
- package/test/stubs/broker-peer.mjs +20 -0
- package/test/stubs/confined-harness.mjs +91 -0
- package/test/stubs/ext-broken-server.mjs +2 -0
- package/test/stubs/ext-odd-server.mjs +21 -0
- package/test/stubs/ext-server.mjs +31 -0
- package/test/stubs/fake-host-cli.mjs +218 -0
- package/test/stubs/procargs.mjs +30 -0
- package/test/stubs/stub-acp +4 -0
- package/test/stubs/stub-agent.mjs +322 -0
- package/test/stubs/stub-claude +4 -0
- package/test/stubs/stub-codex +4 -0
- package/test/stubs/targets/acp.mjs +307 -0
- package/test/stubs/targets/claude.mjs +149 -0
- package/test/stubs/targets/codex.mjs +479 -0
- package/test/stubs/targets/gemini.mjs +604 -0
- package/test/temp-root.ts +11 -0
- package/test/wp4-adapters.test.ts +353 -0
- package/test/wp4-interlock.test.ts +136 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// A release's shipped Codex pin (decision 18): `codex-pin.json` at relay's package root, written by
|
|
2
|
+
// the maintainer's qualification (scripts/codex-qualify/codex-shipped-pin.mjs writeShippedPin, from
|
|
3
|
+
// a passing local qualification only) and packed with the release. It is portable by construction:
|
|
4
|
+
// it holds the Codex release (version and npm's sha512 integrity, which fixes every byte of the
|
|
5
|
+
// binary) and relay's build identity (src/core/build-identity.ts), and nothing else. No path, user,
|
|
6
|
+
// host, time or machine detail is allowed; a manifest with any other field is refused, not trimmed.
|
|
7
|
+
// Relay accepts it only for exactly that relay build and runs only a Codex whose download matched
|
|
8
|
+
// that integrity (src/backends/codex-pin.ts). What it does not carry is the maintainer's machine:
|
|
9
|
+
// the trust is the maintainer's qualification on the maintainer's Mac (README, Security model).
|
|
10
|
+
import fs from "node:fs";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { BUILD_SCHEME, LEGACY_BUILD_SCHEMES, isBuildIdentity } from "./build-identity.js";
|
|
13
|
+
export const SHIPPED_PIN_FILE = "codex-pin.json";
|
|
14
|
+
export const SHIPPED_PIN_FORMAT = "relay-shipped-codex-pin-v1";
|
|
15
|
+
const exactKeys = (value, keys) => !!value && typeof value === "object" && !Array.isArray(value) && Object.keys(value).sort().join(",") === [...keys].sort().join(",");
|
|
16
|
+
/** The manifest, deeply frozen, or throws saying what is wrong. Any field beyond the format is refused. */
|
|
17
|
+
export function parseShippedPin(value) {
|
|
18
|
+
const v = value;
|
|
19
|
+
if (!exactKeys(v, ["format", "codex", "relay"]))
|
|
20
|
+
throw new Error("a shipped Codex pin has exactly format, codex and relay, and nothing else");
|
|
21
|
+
if (v.format !== SHIPPED_PIN_FORMAT)
|
|
22
|
+
throw new Error(`unknown shipped Codex pin format ${JSON.stringify(v.format)}`);
|
|
23
|
+
if (!exactKeys(v.codex, ["version", "npmIntegrity"]) || !/^\d+\.\d+\.\d+$/.test(v.codex.version) ||
|
|
24
|
+
!/^sha512-[A-Za-z0-9+/]{86}==$/.test(v.codex.npmIntegrity))
|
|
25
|
+
throw new Error("a shipped Codex pin's codex is exactly a stable version and npm's sha512 integrity");
|
|
26
|
+
if (exactKeys(v.relay, ["scheme", "version", "sha256"]) && LEGACY_BUILD_SCHEMES.includes(v.relay.scheme)) {
|
|
27
|
+
throw new Error(`a shipped Codex pin names relay's build under ${v.relay.scheme}, which does not cover relay's dependency lock; this relay matches only ${BUILD_SCHEME}, so the release must be qualified again`);
|
|
28
|
+
}
|
|
29
|
+
if (!exactKeys(v.relay, ["scheme", "version", "sha256"]) || !isBuildIdentity(v.relay))
|
|
30
|
+
throw new Error("a shipped Codex pin's relay is exactly relay's build identity (scheme, version, sha256)");
|
|
31
|
+
return Object.freeze({ format: v.format, codex: Object.freeze({ ...v.codex }), relay: Object.freeze({ ...v.relay }) });
|
|
32
|
+
}
|
|
33
|
+
/** The shipped pin in relay's package at `root`; null when this relay ships none (a git checkout never does). Throws on a present but invalid one. */
|
|
34
|
+
export function readShippedPin(root) {
|
|
35
|
+
let text;
|
|
36
|
+
try {
|
|
37
|
+
text = fs.readFileSync(path.join(root, SHIPPED_PIN_FILE), "utf8");
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
if (err.code === "ENOENT")
|
|
41
|
+
return null;
|
|
42
|
+
throw err;
|
|
43
|
+
}
|
|
44
|
+
return parseShippedPin(JSON.parse(text));
|
|
45
|
+
}
|
|
46
|
+
export const sameShippedPin = (a, b) => a.format === b.format && a.codex.version === b.codex.version && a.codex.npmIntegrity === b.codex.npmIntegrity &&
|
|
47
|
+
a.relay.scheme === b.relay.scheme && a.relay.version === b.relay.version && a.relay.sha256 === b.relay.sha256;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// How a launched agent's own text is shown to a person in one line: in relay's approval dialog,
|
|
2
|
+
// progress lines, the run tree and event texts. A leaf module, so that core and backends can use it.
|
|
3
|
+
/**
|
|
4
|
+
* Unicode characters that change how text displays without showing themselves: format characters
|
|
5
|
+
* (category Cf: bidi controls such as U+202E, zero-width characters, U+FEFF, soft hyphens, tags)
|
|
6
|
+
* and the line and paragraph separators U+2028 and U+2029, which some displays break lines at.
|
|
7
|
+
*/
|
|
8
|
+
const INVISIBLE = /[\p{Cf}\u2028\u2029]/u;
|
|
9
|
+
/**
|
|
10
|
+
* Text from a launched agent as one line of the approval dialog, losing nothing: a request whose
|
|
11
|
+
* text has line breaks or other control characters, or invisible format characters (INVISIBLE)
|
|
12
|
+
* that could make it display as something else (a right-to-left override reverses what follows
|
|
13
|
+
* it), is shown escaped (backslashes too, so that the escapes cannot be confused with the text),
|
|
14
|
+
* with how many lines it has, since Claude Code folds everything after the first lines and a
|
|
15
|
+
* second line could otherwise be a hidden second command. Format characters show as \u{XXXX}.
|
|
16
|
+
*/
|
|
17
|
+
export function shown(text) {
|
|
18
|
+
if (!/[\u0000-\u001f\u007f]/.test(text) && !INVISIBLE.test(text))
|
|
19
|
+
return text;
|
|
20
|
+
const lines = text.split(/\r\n|\r|\n/).length;
|
|
21
|
+
const escaped = text.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/\r/g, "\\r").replace(/\t/g, "\\t")
|
|
22
|
+
.replace(/[\u0000-\u001f\u007f]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`)
|
|
23
|
+
.replace(new RegExp(INVISIBLE.source, "gu"), (c) => `\\u{${c.codePointAt(0).toString(16).toUpperCase()}}`);
|
|
24
|
+
return `(${lines} line${lines === 1 ? "" : "s"}, shown escaped) ${escaped}`;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* A launched agent's text (a task, an activity, a value it chose) on one line: its whitespace,
|
|
28
|
+
* line breaks included, flattened to single spaces, then clipped to `max`. What is left that could
|
|
29
|
+
* change how it displays is shown escaped.
|
|
30
|
+
*/
|
|
31
|
+
export const oneLine = (text, max) => clipped(String(text ?? "").replace(/\s+/g, " ").trim(), max);
|
|
32
|
+
/**
|
|
33
|
+
* shown(text), with the text itself cut to `max` characters (the line count, when there is one, is
|
|
34
|
+
* kept in front): for progress lines, the run tree and event texts, where a long command is cut
|
|
35
|
+
* but its line breaks still show.
|
|
36
|
+
*/
|
|
37
|
+
export function clipped(text, max) {
|
|
38
|
+
const s = shown(text);
|
|
39
|
+
const head = /^\(\d+ lines?, shown escaped\) /.exec(s)?.[0] ?? "";
|
|
40
|
+
const body = s.slice(head.length);
|
|
41
|
+
return head + (body.length > max ? `${body.slice(0, max - 1)}…` : body);
|
|
42
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { db, now, reap, runConfig, tx } from "./db.js";
|
|
2
|
+
import { registryPoll } from "./wake.js";
|
|
3
|
+
/** The longest a slot request goes without looking. It looks sooner when another process commits to the registry, as one does that frees a slot. */
|
|
4
|
+
const LOOK_MS = 150;
|
|
5
|
+
/** How often a waiting request drops the slots of processes that have died. */
|
|
6
|
+
const REAP_MS = 3000;
|
|
7
|
+
/**
|
|
8
|
+
* Slots this request may not take: those held, and one for each request that came before it and
|
|
9
|
+
* is still waiting. One statement, so both counts are from the same moment.
|
|
10
|
+
*/
|
|
11
|
+
const taken = (runId, id) => db().prepare("SELECT (SELECT COUNT(*) FROM slots WHERE run_id = ? AND state = 'held') + (SELECT COUNT(*) FROM slots WHERE run_id = ? AND state = 'waiting' AND id < ?) AS c")
|
|
12
|
+
.get(runId, runId, id).c;
|
|
13
|
+
/**
|
|
14
|
+
* Concurrency limit: at most max_concurrent slots in state 'held' per run. Requests are granted in
|
|
15
|
+
* arrival order: a request takes a slot only while there is one for it and for every request that
|
|
16
|
+
* came before it, so the requests that get the free slots are the earliest ones, and each takes its
|
|
17
|
+
* own without waiting for those ahead of it to take theirs. A slot whose holder is blocked inside a
|
|
18
|
+
* relay call is 'lent' and does not count, so agents waiting on their own children cannot starve
|
|
19
|
+
* those children of capacity.
|
|
20
|
+
*/
|
|
21
|
+
export async function acquireSlot(runId, holder, shouldAbort) {
|
|
22
|
+
const max = runConfig(runId).max_concurrent;
|
|
23
|
+
const r = db().prepare("INSERT INTO slots (run_id, holder, pid, state, created_at) VALUES (?, ?, ?, 'waiting', ?)")
|
|
24
|
+
.run(runId, holder, process.pid, now());
|
|
25
|
+
const id = Number(r.lastInsertRowid);
|
|
26
|
+
const poll = registryPoll();
|
|
27
|
+
let reaped = -Infinity;
|
|
28
|
+
for (;;) {
|
|
29
|
+
if (shouldAbort()) {
|
|
30
|
+
releaseSlot(id);
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
if (now() - reaped >= REAP_MS) {
|
|
34
|
+
reap(runId);
|
|
35
|
+
reaped = now();
|
|
36
|
+
}
|
|
37
|
+
// Read first, and take the write lock only when there is room: requests waiting in many
|
|
38
|
+
// processes look each time the registry changes.
|
|
39
|
+
if (taken(runId, id) < max) {
|
|
40
|
+
const granted = tx(() => {
|
|
41
|
+
if (taken(runId, id) >= max)
|
|
42
|
+
return false;
|
|
43
|
+
db().prepare("UPDATE slots SET state = 'held' WHERE id = ?").run(id);
|
|
44
|
+
return true;
|
|
45
|
+
});
|
|
46
|
+
if (granted)
|
|
47
|
+
return id;
|
|
48
|
+
}
|
|
49
|
+
await poll.nap(LOOK_MS);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export function releaseSlot(id) {
|
|
53
|
+
db().prepare("DELETE FROM slots WHERE id = ?").run(id);
|
|
54
|
+
}
|
|
55
|
+
/** Lends the caller's slot for the duration of a blocking relay call. Returns an undo function. */
|
|
56
|
+
export function lendSlot(holder) {
|
|
57
|
+
if (!holder)
|
|
58
|
+
return () => { };
|
|
59
|
+
const r = db().prepare("UPDATE slots SET state = 'lent' WHERE holder = ? AND state = 'held'").run(holder);
|
|
60
|
+
if (!r.changes)
|
|
61
|
+
return () => { };
|
|
62
|
+
// Reclaimed immediately on return: the caller is already running, so making it wait for a free
|
|
63
|
+
// slot would only delay a tool result. New work still starts only while held < max.
|
|
64
|
+
return () => {
|
|
65
|
+
db().prepare("UPDATE slots SET state = 'held' WHERE holder = ? AND state = 'lent'").run(holder);
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
// Stage 2's executors on disk (docs/codex-confinement-design.md §2.2 steps 9, 14, 15): one record
|
|
2
|
+
// per running `codex exec-server`, so that relay can stop one whose own relay process died or was
|
|
3
|
+
// killed mid-turn (a crash, or a cancel that had to kill the worker). The worker that launched an
|
|
4
|
+
// executor tears it down itself and removes its record (backends/codex-env.ts); these records are
|
|
5
|
+
// for everyone else: reap (db.ts), cancel (ops.ts) and the retention sweep (retention.ts).
|
|
6
|
+
import { spawnSync } from "node:child_process";
|
|
7
|
+
import crypto from "node:crypto";
|
|
8
|
+
import fs from "node:fs";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { releaseLeasesOf } from "../confine/leases.js";
|
|
11
|
+
import { sandboxCheckUnavailable, sandboxReports } from "../confine/sbcheck.js";
|
|
12
|
+
import { relayHome } from "./config.js";
|
|
13
|
+
import { getNode, relayProcs, setParam } from "./db.js";
|
|
14
|
+
import { sessionParams } from "./params.js";
|
|
15
|
+
import { groupAlive, isGroupSignalRefusal, killGroupNow, killProcs, pidAlive, procStart, sameProcess } from "./procs.js";
|
|
16
|
+
import { finishLaunchExecutor, launchExecutorQuarantine } from "./restart-overlay.js";
|
|
17
|
+
/** `<relay home>/executors`, mode 0700. */
|
|
18
|
+
export const executorsDir = () => path.join(relayHome(), "executors");
|
|
19
|
+
const RECORD = /^ex-\d+-[0-9a-f]{8}\.json$/;
|
|
20
|
+
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
21
|
+
const norm = (start) => start.replace(/\s+/g, " ");
|
|
22
|
+
function ensureDir() {
|
|
23
|
+
const dir = executorsDir();
|
|
24
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
25
|
+
const st = fs.lstatSync(dir);
|
|
26
|
+
if (!st.isDirectory() || st.isSymbolicLink())
|
|
27
|
+
throw new Error(`${dir} is not a plain directory`);
|
|
28
|
+
if (st.mode & 0o077)
|
|
29
|
+
fs.chmodSync(dir, 0o700);
|
|
30
|
+
return dir;
|
|
31
|
+
}
|
|
32
|
+
/** Written whole or not at all: a temp file beside it, mode 0600, renamed into place. */
|
|
33
|
+
function writeAtomic(file, record) {
|
|
34
|
+
const tmp = `${file}.${crypto.randomBytes(4).toString("hex")}.tmp`;
|
|
35
|
+
try {
|
|
36
|
+
fs.writeFileSync(tmp, `${JSON.stringify(record)}\n`, { flag: "wx", mode: 0o600 });
|
|
37
|
+
fs.renameSync(tmp, file);
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
fs.rmSync(tmp, { force: true });
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/** Records a running executor; returns the record's file. The owner is this process. */
|
|
45
|
+
export function recordExecutor(r) {
|
|
46
|
+
const ownerStart = procStart(process.pid);
|
|
47
|
+
if (!ownerStart)
|
|
48
|
+
throw new Error("relay could not read its own process's start time, so it cannot record Codex's executor");
|
|
49
|
+
const record = {
|
|
50
|
+
v: 1, pid: r.pid, start: norm(r.start), pgid: r.pid, owner: { pid: process.pid, start: norm(ownerStart) },
|
|
51
|
+
runId: r.runId, nodeId: r.nodeId, turnId: r.turnId, ports: { exec: r.ports.exec, block: [...r.ports.block] }, tmpDir: r.tmpDir,
|
|
52
|
+
descendants: r.descendants ?? [], createdAt: Date.now(),
|
|
53
|
+
};
|
|
54
|
+
const file = path.join(ensureDir(), `ex-${r.pid}-${crypto.randomBytes(4).toString("hex")}.json`);
|
|
55
|
+
writeAtomic(file, record);
|
|
56
|
+
return file;
|
|
57
|
+
}
|
|
58
|
+
/** Adds descendants the watch found since; a record that is gone (torn down meanwhile) stays gone. */
|
|
59
|
+
export function noteDescendants(file, procs) {
|
|
60
|
+
const r = readRecord(file);
|
|
61
|
+
if (!r)
|
|
62
|
+
return;
|
|
63
|
+
const known = new Set(r.descendants.map((p) => `${p.pid} ${p.start}`));
|
|
64
|
+
const add = procs.filter((p) => !known.has(`${p.pid} ${norm(p.start)}`)).map((p) => ({ pid: p.pid, start: norm(p.start) }));
|
|
65
|
+
if (add.length)
|
|
66
|
+
writeAtomic(file, { ...r, descendants: [...r.descendants, ...add] });
|
|
67
|
+
}
|
|
68
|
+
export function forgetExecutor(file) {
|
|
69
|
+
fs.rmSync(file, { force: true });
|
|
70
|
+
}
|
|
71
|
+
function valid(x) {
|
|
72
|
+
const proc = (p) => p && Number.isInteger(p.pid) && p.pid > 1 && typeof p.start === "string" && p.start.length > 0;
|
|
73
|
+
return x && x.v === 1 && proc(x) && x.pgid === x.pid && proc(x.owner) && typeof x.nodeId === "string" && typeof x.runId === "string"
|
|
74
|
+
&& typeof x.tmpDir === "string" && path.isAbsolute(x.tmpDir) && Array.isArray(x.descendants) && x.descendants.every(proc)
|
|
75
|
+
&& x.ports && Number.isInteger(x.ports.exec) && Array.isArray(x.ports.block) && (x.tainted === undefined || typeof x.tainted === "string");
|
|
76
|
+
}
|
|
77
|
+
function readRecord(file) {
|
|
78
|
+
try {
|
|
79
|
+
const x = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
80
|
+
return valid(x) ? x : null;
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/** Every record in this relay home; `problems` names files that are not records relay wrote (left alone). */
|
|
87
|
+
export function executorRecords() {
|
|
88
|
+
const dir = executorsDir();
|
|
89
|
+
let names;
|
|
90
|
+
try {
|
|
91
|
+
const st = fs.lstatSync(dir, { throwIfNoEntry: false });
|
|
92
|
+
if (!st)
|
|
93
|
+
return { records: [], problems: [] };
|
|
94
|
+
if (!st.isDirectory() || st.isSymbolicLink())
|
|
95
|
+
return { records: [], problems: [`${dir} is not a plain directory, so relay did not read executor records from it`] };
|
|
96
|
+
names = fs.readdirSync(dir);
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
return { records: [], problems: [`${dir} could not be read (${err.message})`] };
|
|
100
|
+
}
|
|
101
|
+
const records = [];
|
|
102
|
+
const problems = [];
|
|
103
|
+
for (const name of names.sort()) {
|
|
104
|
+
if (!RECORD.test(name))
|
|
105
|
+
continue;
|
|
106
|
+
const file = path.join(dir, name);
|
|
107
|
+
const record = readRecord(file);
|
|
108
|
+
if (record)
|
|
109
|
+
records.push({ file, record });
|
|
110
|
+
else
|
|
111
|
+
problems.push(`${file} is not a readable executor record; left alone`);
|
|
112
|
+
}
|
|
113
|
+
return { records, problems };
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Whether the relay process that launched the executor may still be there to tear it down: the same
|
|
117
|
+
* process (pid and start time), or a live pid whose start time cannot be read (unknown is not gone).
|
|
118
|
+
*/
|
|
119
|
+
export function ownerMayLive(r) {
|
|
120
|
+
const { pid, start } = r.owner;
|
|
121
|
+
return sameProcess(pid, start, { fresh: true }) || (pidAlive(pid) && procStart(pid, { fresh: true }) === null);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Whether the executor's process group is still the one relay started: its leader is the process
|
|
125
|
+
* recorded (pid and start time), or the leader has exited and members remain (a group id is not
|
|
126
|
+
* given out again while it has members). A live leader with another start time is a reused number.
|
|
127
|
+
*/
|
|
128
|
+
export function executorGroupOurs(r) {
|
|
129
|
+
if (!groupAlive(r.pgid))
|
|
130
|
+
return false;
|
|
131
|
+
return !pidAlive(r.pid) || sameProcess(r.pid, r.start, { fresh: true });
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Every process of this user whose sandbox lets it write `tmpDir` and not the directory holding it:
|
|
135
|
+
* this turn's, wherever they detached to (sbcheck.ts). No other relay turn's profile allows another
|
|
136
|
+
* turn's temp dir; a broader sandbox (one allowing the home, say) also allows the parent and is left
|
|
137
|
+
* alone; unsandboxed processes are never counted, nor zombies. `tmpDir` must exist (the kernel denies
|
|
138
|
+
* every path that does not; sweepSurvivorsNow makes it again). A string, never an empty list, when
|
|
139
|
+
* the look could not be made: the helper unavailable or failing, ps failing, the directory missing,
|
|
140
|
+
* or the kernel unable to answer for a process that is still there.
|
|
141
|
+
*/
|
|
142
|
+
export function sweepTurnSurvivors(tmpDir) {
|
|
143
|
+
const unavailable = sandboxCheckUnavailable();
|
|
144
|
+
if (unavailable)
|
|
145
|
+
return unavailable;
|
|
146
|
+
if (!fs.existsSync(tmpDir))
|
|
147
|
+
return `the turn's temporary directory ${tmpDir} is not there, so the kernel cannot be asked about it`;
|
|
148
|
+
const uid = process.getuid?.() ?? -1;
|
|
149
|
+
const r = spawnSync("/bin/ps", ["-axo", "pid=,uid=,stat=,lstart="], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 20000, env: { PATH: "/usr/bin:/bin", LC_ALL: "C", LANG: "C" } });
|
|
150
|
+
if (r.error || r.status !== 0)
|
|
151
|
+
return `ps failed: ${r.error?.message ?? `exit ${r.status}`}`;
|
|
152
|
+
const starts = new Map();
|
|
153
|
+
for (const line of r.stdout.split("\n")) {
|
|
154
|
+
const m = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line);
|
|
155
|
+
// A zombie has exited: nothing of it runs, and the kernel has no sandbox to ask about.
|
|
156
|
+
if (m && Number(m[2]) === uid && Number(m[1]) !== process.pid && !m[3].startsWith("Z"))
|
|
157
|
+
starts.set(Number(m[1]), norm(m[4]));
|
|
158
|
+
}
|
|
159
|
+
if (!starts.size)
|
|
160
|
+
return "ps listed no processes of this user";
|
|
161
|
+
const reports = sandboxReports("file-write-data", [tmpDir, path.dirname(tmpDir)], [...starts.keys()]);
|
|
162
|
+
if (typeof reports === "string")
|
|
163
|
+
return reports;
|
|
164
|
+
const found = [];
|
|
165
|
+
for (const [pid, start] of starts) {
|
|
166
|
+
const rep = reports.get(pid);
|
|
167
|
+
const unanswered = !rep || typeof rep.sandboxed === "string" || (rep.sandboxed && rep.answers.some((a) => a.startsWith("error")));
|
|
168
|
+
if (unanswered) {
|
|
169
|
+
// Gone since ps listed it: nothing to ask about. Still there: the look is not complete.
|
|
170
|
+
if (sameProcess(pid, start, { fresh: true }))
|
|
171
|
+
return `the kernel could not say whether process ${pid} is in the turn's sandbox (${!rep ? "no answer" : typeof rep.sandboxed === "string" ? rep.sandboxed : rep.answers.find((a) => a.startsWith("error"))})`;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
if (rep.sandboxed === true && rep.answers[0] === "allow" && rep.answers[1] === "deny")
|
|
175
|
+
found.push({ pid, start });
|
|
176
|
+
}
|
|
177
|
+
return found;
|
|
178
|
+
}
|
|
179
|
+
/** How many looks a sweep takes at most before it gives up and counts what is left as survivors. */
|
|
180
|
+
export const SWEEP_CAP = 64;
|
|
181
|
+
const signalIfSame = (p, sig) => {
|
|
182
|
+
if (!sameProcess(p.pid, p.start, { fresh: true }))
|
|
183
|
+
return false;
|
|
184
|
+
try {
|
|
185
|
+
process.kill(p.pid, sig);
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
/**
|
|
193
|
+
* The survivor sweep (§2.2 step 15), until a look finds nothing new: each process a look finds is
|
|
194
|
+
* stopped (SIGSTOP), so that it cannot fork or detach before the next look; once a look finds
|
|
195
|
+
* nothing new, all are killed (SIGKILL) and a last look must find nothing at all. A look that fails
|
|
196
|
+
* stops the sweep with a problem (what was frozen is still killed), and so does reaching `cap`
|
|
197
|
+
* looks: a sweep with a problem has not shown that nothing is left. The turn's temp dir is made again
|
|
198
|
+
* if it is gone (sandbox rules name paths, so the kernel answers for it as before), and removed
|
|
199
|
+
* afterwards. Synchronous. `exclude`: processes already known and handled; `look` is for tests.
|
|
200
|
+
*/
|
|
201
|
+
export function sweepSurvivorsNow(tmpDir, o = {}) {
|
|
202
|
+
const look = o.look ?? sweepTurnSurvivors;
|
|
203
|
+
const cap = o.cap ?? SWEEP_CAP;
|
|
204
|
+
const exclude = new Set(o.exclude ?? []);
|
|
205
|
+
let made = false;
|
|
206
|
+
if (!o.look && !fs.existsSync(tmpDir)) {
|
|
207
|
+
try {
|
|
208
|
+
fs.mkdirSync(tmpDir, { mode: 0o700 });
|
|
209
|
+
made = true;
|
|
210
|
+
}
|
|
211
|
+
catch (err) {
|
|
212
|
+
return { swept: [], survivors: [], problem: `the turn's temporary directory ${tmpDir} is gone and could not be made again to ask the kernel about it (${err.message})` };
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const frozen = new Map();
|
|
216
|
+
let problem;
|
|
217
|
+
try {
|
|
218
|
+
let quiet = false;
|
|
219
|
+
for (let round = 0; round < cap && !quiet; round++) {
|
|
220
|
+
const found = look(tmpDir);
|
|
221
|
+
if (typeof found === "string") {
|
|
222
|
+
problem = found;
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
const fresh = found.filter((p) => !frozen.has(p.pid) && !exclude.has(p.pid));
|
|
226
|
+
if (!fresh.length)
|
|
227
|
+
quiet = true;
|
|
228
|
+
for (const p of fresh)
|
|
229
|
+
if (signalIfSame(p, "SIGSTOP"))
|
|
230
|
+
frozen.set(p.pid, p);
|
|
231
|
+
}
|
|
232
|
+
if (!quiet && !problem)
|
|
233
|
+
problem = `the survivor sweep still found new processes after ${cap} looks`;
|
|
234
|
+
for (const p of frozen.values())
|
|
235
|
+
signalIfSame(p, "SIGKILL");
|
|
236
|
+
const left = new Map(stillThere([...frozen.values()], 2000).map((p) => [p.pid, p]));
|
|
237
|
+
if (!problem) {
|
|
238
|
+
// The last look: after the kill, nothing of the turn may be left at all.
|
|
239
|
+
const last = look(tmpDir);
|
|
240
|
+
if (typeof last === "string")
|
|
241
|
+
problem = last;
|
|
242
|
+
else
|
|
243
|
+
for (const p of last)
|
|
244
|
+
if (!exclude.has(p.pid))
|
|
245
|
+
left.set(p.pid, p);
|
|
246
|
+
}
|
|
247
|
+
return { swept: [...frozen.values()], survivors: [...left.values()], ...(problem ? { problem } : {}) };
|
|
248
|
+
}
|
|
249
|
+
finally {
|
|
250
|
+
if (made)
|
|
251
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
/** Of the noted processes, those still running: same pid and start time, and not a zombie (one that has exited and waits for its parent). */
|
|
255
|
+
function running(procs) {
|
|
256
|
+
if (!procs.length)
|
|
257
|
+
return [];
|
|
258
|
+
const r = spawnSync("/bin/ps", ["-o", "pid=,stat=,lstart=", "-p", procs.map((p) => p.pid).join(",")], { encoding: "utf8", timeout: 20000, env: { PATH: "/usr/bin:/bin", LC_ALL: "C", LANG: "C" } });
|
|
259
|
+
// ps exits 1 when none of them is there; any other failure counts them all as running.
|
|
260
|
+
if (r.error || (r.status !== 0 && r.status !== 1))
|
|
261
|
+
return procs.filter((p) => sameProcess(p.pid, p.start, { fresh: true }));
|
|
262
|
+
const now = new Map();
|
|
263
|
+
for (const line of r.stdout.split("\n")) {
|
|
264
|
+
const m = /^\s*(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line);
|
|
265
|
+
if (m)
|
|
266
|
+
now.set(Number(m[1]), { zombie: m[2].startsWith("Z"), start: norm(m[3]) });
|
|
267
|
+
}
|
|
268
|
+
return procs.filter((p) => {
|
|
269
|
+
const n = now.get(p.pid);
|
|
270
|
+
return n !== undefined && !n.zombie && n.start === norm(p.start);
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
/** Waits, without yielding, up to `ms` for the noted processes to be gone; returns those still running. */
|
|
274
|
+
function stillThere(procs, ms) {
|
|
275
|
+
for (let waited = 0; waited < ms && running(procs).length; waited += 50)
|
|
276
|
+
pause(50);
|
|
277
|
+
return running(procs);
|
|
278
|
+
}
|
|
279
|
+
/** Keeps a record, tainted with `why`, with `left` as what is still to stop. */
|
|
280
|
+
export function taintExecutor(file, why, left) {
|
|
281
|
+
const r = readRecord(file);
|
|
282
|
+
if (!r)
|
|
283
|
+
return;
|
|
284
|
+
const known = new Set(left.map((p) => p.pid));
|
|
285
|
+
writeAtomic(file, { ...r, tainted: why, descendants: [...left.map((p) => ({ pid: p.pid, start: norm(p.start) })), ...r.descendants.filter((p) => !known.has(p.pid))] });
|
|
286
|
+
}
|
|
287
|
+
/** An executor-sandbox sweep cannot clear uncertainty about untraceable app-server processes. */
|
|
288
|
+
export function executorQuarantine(file, quarantine) {
|
|
289
|
+
if (quarantine !== undefined)
|
|
290
|
+
return quarantine;
|
|
291
|
+
const tainted = readRecord(file)?.tainted;
|
|
292
|
+
return tainted?.startsWith("untraceable app-server cleanup:") ? tainted : undefined;
|
|
293
|
+
}
|
|
294
|
+
/** Keep every identity and lease without signalling or sweeping after app-server cleanup was refused. */
|
|
295
|
+
export function quarantineRecordedExecutor(rec, why) {
|
|
296
|
+
const { file, record } = rec;
|
|
297
|
+
taintExecutor(file, why, record.descendants);
|
|
298
|
+
return { file, record, killed: false, swept: [], survivors: [record.pgid, ...record.descendants.map(p => p.pid)], released: [], tainted: why };
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Stops one recorded executor at once, as teardown would (§2.2 step 14), for an executor whose relay
|
|
302
|
+
* process is gone, or whose teardown left the record tainted: its process group (with everything
|
|
303
|
+
* procTree finds under it), the descendants recorded, then the survivor sweep by sandbox
|
|
304
|
+
* (sweepSurvivorsNow). Nothing is signalled that is not the process recorded (pid and start time).
|
|
305
|
+
* Only when all that comes out clean is the record removed and its lease given back; otherwise the
|
|
306
|
+
* record is kept, tainted, and the lease stays quarantined, for the next try.
|
|
307
|
+
* `sweep` is injectable for tests that simulate process identity and group-signal refusals.
|
|
308
|
+
*/
|
|
309
|
+
export function stopRecordedExecutorNow(rec, quarantine, sweep = sweepSurvivorsNow) {
|
|
310
|
+
const { file, record: r } = rec;
|
|
311
|
+
quarantine = executorQuarantine(file, quarantine) ?? launchExecutorQuarantine(r.runId, r.turnId);
|
|
312
|
+
let killed = false;
|
|
313
|
+
if (executorGroupOurs(r)) {
|
|
314
|
+
try {
|
|
315
|
+
killGroupNow(r.pgid, relayProcs);
|
|
316
|
+
}
|
|
317
|
+
catch (err) {
|
|
318
|
+
if (!isGroupSignalRefusal(err))
|
|
319
|
+
throw err;
|
|
320
|
+
// Keep the identities and lease. No descendant kill or sandbox sweep may follow refusal.
|
|
321
|
+
return quarantineRecordedExecutor(rec, quarantine ?? err.message);
|
|
322
|
+
}
|
|
323
|
+
killed = true;
|
|
324
|
+
}
|
|
325
|
+
killProcs(r.descendants);
|
|
326
|
+
const left = new Map(stillThere(r.descendants, 1000).map((p) => [p.pid, p]));
|
|
327
|
+
const result = sweep(r.tmpDir, { exclude: left.keys() });
|
|
328
|
+
for (const p of result.survivors)
|
|
329
|
+
left.set(p.pid, p);
|
|
330
|
+
const groupLeft = executorGroupOurs(r);
|
|
331
|
+
const survivors = [...left.keys(), ...(groupLeft ? [r.pgid] : [])];
|
|
332
|
+
const swept = result.swept.map((p) => p.pid);
|
|
333
|
+
// The kernel's executor-sandbox sweep cannot prove the absence of untraceable processes that
|
|
334
|
+
// escaped an app-server watch gap. Never turn that uncertainty into a clean lease release.
|
|
335
|
+
const tainted = quarantine ?? result.problem ?? (survivors.length ? `processes would not stop: ${survivors.join(", ")}` : undefined);
|
|
336
|
+
let released = [];
|
|
337
|
+
if (!tainted) {
|
|
338
|
+
finishLaunchExecutor(r.runId, r.turnId);
|
|
339
|
+
// The record first: while it is there, the lease naming it stays held.
|
|
340
|
+
forgetExecutor(file);
|
|
341
|
+
released = releaseLeasesOf({ pid: r.pid, start: r.start });
|
|
342
|
+
}
|
|
343
|
+
else {
|
|
344
|
+
taintExecutor(file, tainted, [...left.values()]);
|
|
345
|
+
}
|
|
346
|
+
return { file, record: r, killed, swept, survivors, released, ...(result.problem ? { sweep: result.problem } : {}), ...(tainted ? { tainted } : {}) };
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Stops every orphaned executor matching `which`: those whose owner may still live are its own to
|
|
350
|
+
* tear down, and are left alone, unless its teardown already ended with the record tainted.
|
|
351
|
+
* `quarantine` keeps each record and lease without signalling anything; `retain` still stops the
|
|
352
|
+
* executor but keeps its record, tainted with `retain`, and so its lease (stopRecordedExecutorNow).
|
|
353
|
+
*/
|
|
354
|
+
function stopOrphansNow(which, quarantine, retain) {
|
|
355
|
+
const out = [];
|
|
356
|
+
for (const rec of executorRecords().records) {
|
|
357
|
+
if (!which(rec.record) || (ownerMayLive(rec.record) && !rec.record.tainted))
|
|
358
|
+
continue;
|
|
359
|
+
out.push(quarantine !== undefined ? quarantineRecordedExecutor(rec, quarantine) : stopRecordedExecutorNow(rec, retain));
|
|
360
|
+
}
|
|
361
|
+
return out;
|
|
362
|
+
}
|
|
363
|
+
/** A note for the node's error, as stopOrphan gives: empty when nothing may still be running. */
|
|
364
|
+
export function stoppedNote(stopped) {
|
|
365
|
+
const left = stopped.flatMap((s) => s.survivors);
|
|
366
|
+
const unsure = stopped.filter((s) => s.tainted && !s.survivors.length).map((s) => s.tainted);
|
|
367
|
+
// An app-server marker survives every later sweep (executorQuarantine): only the person clears it.
|
|
368
|
+
const held = unsure.filter((t) => t.startsWith("untraceable app-server cleanup:"));
|
|
369
|
+
const retry = unsure.filter((t) => !t.startsWith("untraceable app-server cleanup:"));
|
|
370
|
+
return [
|
|
371
|
+
...(left.length ? [` Processes of a stage-2 executor (pids ${left.join(", ")}) may still be running, inside the turn's sandbox.`] : []),
|
|
372
|
+
...(retry.length ? [` relay could not confirm that every process of a stage-2 executor stopped (${retry.join("; ")}); its ports stay reserved until a later sweep does.`] : []),
|
|
373
|
+
...(held.length ? [` relay could not confirm that a stage-2 executor's app server stopped (${held.join("; ")}); the executor's record and ports stay quarantined, and no later sweep releases them: check that nothing of the turn is left, then remove the record yourself (docs/UPGRADING.md, Relay 1.1.3).`] : []),
|
|
374
|
+
].join("");
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Stops the stage-2 executors of a node whose relay process has gone (reap, after stopOrphan has
|
|
378
|
+
* killed the app server, so that it cannot reconnect to whatever later holds the executor's port;
|
|
379
|
+
* cancel, after stopSupervised). An executor whose owner may still be alive is left to it: a cancel
|
|
380
|
+
* whose worker exits cleanly tears its own down, and a turn a resume started meanwhile under a new
|
|
381
|
+
* worker is never touched. Returns a note for the node when something would not stop.
|
|
382
|
+
* If app-server cleanup was refused, `quarantine` keeps the record and leases without any derived
|
|
383
|
+
* executor cleanup: recorded descendants or the sandbox sweep could reach the refused group. If it
|
|
384
|
+
* is only unconfirmed (the app server may still run), `retain`, an `untraceable app-server cleanup:`
|
|
385
|
+
* reason, stops the executor but keeps its record and leases, so no later clean sweep releases them.
|
|
386
|
+
*/
|
|
387
|
+
export function stopNodeExecutorsNow(nodeId, quarantine, retain) {
|
|
388
|
+
return stoppedNote(stopOrphansNow((r) => r.nodeId === nodeId, quarantine, retain));
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* The retention sweep's part (retention.ts): every orphaned executor in this relay home, whatever its
|
|
392
|
+
* node (a crash that no reap has seen yet). Returns what it did, as lines, and what it could not.
|
|
393
|
+
*/
|
|
394
|
+
export function sweepOrphanedExecutors() {
|
|
395
|
+
const { problems } = executorRecords();
|
|
396
|
+
const stopped = [];
|
|
397
|
+
for (const s of stopOrphansNow(() => true)) {
|
|
398
|
+
const what = `the stage-2 executor ${s.record.pid} of ${s.record.nodeId}${s.record.turnId !== null ? ` turn ${s.record.turnId}` : ""}`;
|
|
399
|
+
if (s.survivors.length)
|
|
400
|
+
problems.push(`${what}: pids ${s.survivors.join(", ")} would not stop; its record and lease are kept`);
|
|
401
|
+
else if (s.tainted)
|
|
402
|
+
problems.push(`${what}: relay could not confirm that everything of the turn stopped (${s.tainted}); its record and lease are kept`);
|
|
403
|
+
else
|
|
404
|
+
stopped.push(`${what}${s.killed ? " (stopped)" : ""}${s.swept.length ? `, ${s.swept.length} detached process(es) stopped` : ""}`);
|
|
405
|
+
}
|
|
406
|
+
return { stopped, problems };
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Whether a record may still have something of its turn running: the executor, its process group, a
|
|
410
|
+
* process recorded as its descendant (by pid and start time; a reused pid does not count), or a
|
|
411
|
+
* taint (a teardown or sweep that could not be sure).
|
|
412
|
+
*/
|
|
413
|
+
const recordLive = (r) => Boolean(r.tainted) || sameProcess(r.pid, r.start) || executorGroupOurs(r) || r.descendants.some((p) => sameProcess(p.pid, p.start));
|
|
414
|
+
/**
|
|
415
|
+
* Every recorded executor of this relay home that may still have something running (recordLive):
|
|
416
|
+
* teardown keeps the record, tainted, with what would not stop or when it could not be sure. For the
|
|
417
|
+
* local-mode interlock (core/raw-network.ts), which refuses local mode while stage 2 has anything
|
|
418
|
+
* running. Throws when a file in the records directory cannot be read as a record, or the directory
|
|
419
|
+
* itself cannot: relay then cannot tell that nothing runs, and the interlock refuses. Read-only.
|
|
420
|
+
*/
|
|
421
|
+
export function liveExecutorRecords() {
|
|
422
|
+
const { records, problems } = executorRecords();
|
|
423
|
+
if (problems.length)
|
|
424
|
+
throw new Error(problems.join("; "));
|
|
425
|
+
return records.map((r) => r.record).filter(recordLive);
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Whether a record of this node may still have something running (recordLive, descendants and taint
|
|
429
|
+
* included): retention leaves the turn's temp dir alone while one does, since a later sweep asks
|
|
430
|
+
* the kernel about that path. True also when the records cannot all be read.
|
|
431
|
+
*/
|
|
432
|
+
export function nodeHasLiveExecutor(nodeId) {
|
|
433
|
+
const { records, problems } = executorRecords();
|
|
434
|
+
return problems.length > 0 || records.some(({ record: r }) => r.nodeId === nodeId && recordLive(r));
|
|
435
|
+
}
|
|
436
|
+
/** The node param naming each turn's skill-asset copy (SessionParams.ext_assets). */
|
|
437
|
+
const ASSETS = "ext_assets";
|
|
438
|
+
/** Records that `turnId` of `nodeId` copied its skills to `dir` (a `relay-ext-*` in relay's private root), for the sweep after a crash. */
|
|
439
|
+
export function noteSkillAssets(nodeId, turnId, dir) {
|
|
440
|
+
const n = getNode(nodeId);
|
|
441
|
+
if (!n)
|
|
442
|
+
return;
|
|
443
|
+
setParam(nodeId, ASSETS, { ...(sessionParams(n).ext_assets ?? {}), [String(turnId)]: dir });
|
|
444
|
+
}
|
|
445
|
+
/** Forgets a turn's skill-asset copy once it is removed. */
|
|
446
|
+
export function dropSkillAssets(nodeId, turnId) {
|
|
447
|
+
const n = getNode(nodeId);
|
|
448
|
+
const all = n ? sessionParams(n).ext_assets : null;
|
|
449
|
+
if (!all || !(String(turnId) in all))
|
|
450
|
+
return;
|
|
451
|
+
const { [String(turnId)]: _, ...rest } = all;
|
|
452
|
+
setParam(nodeId, ASSETS, Object.keys(rest).length ? rest : null);
|
|
453
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Leaf shared by overlay admission and standing grants; no registry/backend imports. */
|
|
2
|
+
export const STAGE2_FLAG_ENV = "RELAY_EXPERIMENTAL_CODEX_STAGE2";
|
|
3
|
+
export function stage2Enabled(env = process.env, cfg) {
|
|
4
|
+
return env[STAGE2_FLAG_ENV] === "1" || cfg?.targets?.codex?.stage2 === true;
|
|
5
|
+
}
|