@edgehero/pi-dispatch 4.3.1 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +84 -3
- package/README.md +21 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +118 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +136 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +190 -29
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +325 -0
- package/src/console-client.mjs +848 -0
- package/src/console-devices.mjs +372 -0
- package/src/console-install.mjs +239 -0
- package/src/console-open.mjs +99 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1657 -196
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +96 -6
- package/src/env-file.mjs +6 -4
- package/src/envelope.mjs +5 -3
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-host.mjs +42 -5
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +33 -3
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +734 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +4 -2
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +12 -2
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1550 -173
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +264 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +36 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +780 -38
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +402 -53
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
/** Host preparation for a human, outside every paid processor, budget and run-slot path (#771). */
|
|
2
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
3
|
+
import * as nodeFs from "node:fs";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { ensureJobsDir, ensureSandboxDir, loadConfig } from "./config.mjs";
|
|
6
|
+
import { parseTriggers } from "./triggers.mjs";
|
|
7
|
+
import { makeImagePreflight, resolveJobImage } from "./image-preflight.mjs";
|
|
8
|
+
import { resolveBackendName } from "./backend-registry.mjs";
|
|
9
|
+
import { resolveJobSize } from "./job-size.mjs";
|
|
10
|
+
import { loadProjects, projectOf } from "./projects.mjs";
|
|
11
|
+
import { loadScopedLimits } from "./scoped-limits.mjs";
|
|
12
|
+
import { SANDBOX_MANIFEST, pinSandbox, readManifest, sandboxDeadline, withManifestVersion } from "./sandbox-store.mjs";
|
|
13
|
+
import { cloneHostWorkspace, isCloneRefGone } from "./host-clone.mjs";
|
|
14
|
+
import { parseSandboxRef, resolveSandboxRef, sandboxRemoteUrl, sandboxRepoProblem } from "./sandbox-refs.mjs";
|
|
15
|
+
import { sandboxManifestGeneration } from "./sandbox-inventory.mjs";
|
|
16
|
+
import { WORKER_NAME_RE } from "./worker-name.mjs";
|
|
17
|
+
|
|
18
|
+
export const PREPARED_SANDBOX_ID_RE = /^sb-[a-f0-9]{32}$/;
|
|
19
|
+
export const newSandboxId = () => `sb-${randomBytes(16).toString("hex")}`;
|
|
20
|
+
const PREPARATION_LOCK_MAX_BYTES = 16 * 1024;
|
|
21
|
+
const hash = (value) => createHash("sha256").update(value).digest("hex");
|
|
22
|
+
const refused = (reason, message) => ({ refused: reason, message });
|
|
23
|
+
const configRefusal = () => refused("sandbox-config-invalid", "the host's sandbox configuration could not be read; run pi-dispatch doctor on that host");
|
|
24
|
+
|
|
25
|
+
export function sandboxTriggerDigest(text, index) {
|
|
26
|
+
const rows = JSON.parse(text)?.triggers;
|
|
27
|
+
if (!Number.isInteger(index) || index < 0 || !Array.isArray(rows) || !rows[index]) throw new Error("choose an existing trigger index");
|
|
28
|
+
return hash(JSON.stringify(rows[index]));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Console previews and worker decisions use the same pure resolution; the worker still re-reads its own files. */
|
|
32
|
+
export function planSandboxPreparation({ request, config, env = {}, triggersText, projects = [], limits = [] } = {}) {
|
|
33
|
+
if (!request || typeof request.host !== "string" || !WORKER_NAME_RE.test(request.host) || request.host !== config?.workerName) return refused("sandbox-host-mismatch", "the selected host does not own this sandbox preparation");
|
|
34
|
+
if (request.id !== undefined && !PREPARED_SANDBOX_ID_RE.test(request.id)) return refused("sandbox-id-invalid", "a prepared sandbox id must be sb- followed by 32 lowercase hexadecimal characters");
|
|
35
|
+
if (request.pin !== undefined && typeof request.pin !== "boolean") return refused("sandbox-pin-invalid", "pin must be true or false");
|
|
36
|
+
let triggers, raw, digest;
|
|
37
|
+
try {
|
|
38
|
+
triggers = parseTriggers(triggersText, config.triggersFile ?? "triggers.json");
|
|
39
|
+
digest = sandboxTriggerDigest(triggersText, request.triggerIndex);
|
|
40
|
+
raw = JSON.parse(triggersText).triggers[request.triggerIndex];
|
|
41
|
+
} catch { return refused("sandbox-trigger-unavailable", "the selected trigger is missing or its file does not load; refresh the console or run pi-dispatch doctor"); }
|
|
42
|
+
if (request.triggerDigest !== undefined && request.triggerDigest !== digest) return refused("sandbox-trigger-changed", "the selected trigger changed; preview the sandbox again");
|
|
43
|
+
// A disarmed trigger remains valid configuration. Preparing its image does not arm or run it.
|
|
44
|
+
const trigger = triggers[request.triggerIndex]?.on.type === "disarmed" ? raw : triggers[request.triggerIndex];
|
|
45
|
+
const kind = trigger?.run?.kind;
|
|
46
|
+
const repoProblem = sandboxRepoProblem(request.repo, kind);
|
|
47
|
+
if (repoProblem) return refused("sandbox-repo-invalid", repoProblem);
|
|
48
|
+
if (trigger.run.repository && request.repo.split("/").at(-1) !== trigger.run.repository) return refused("sandbox-repo-mismatch", "the repository does not match the selected trigger's repository");
|
|
49
|
+
const parsedRef = parseSandboxRef(request.ref ?? "HEAD", kind);
|
|
50
|
+
if (parsedRef.refused) return parsedRef;
|
|
51
|
+
if (!Number.isFinite(config.sandboxRetentionHours) || config.sandboxRetentionHours <= 0) return refused("sandbox-retention-off", "workspace retention is off on this host; set PI_SANDBOX_RETENTION_HOURS to a positive number before preparing");
|
|
52
|
+
const backend = resolveBackendName(trigger.run, config.defaultBackend);
|
|
53
|
+
if (!["local", "podman"].includes(backend)) return refused("sandbox-venue-unsupported", "the selected trigger's venue has no sandbox shell; choose a local or podman trigger");
|
|
54
|
+
if (!Array.isArray(config.backends) || !config.backends.includes(backend)) return refused("sandbox-venue-unblessed", "the selected host does not bless the trigger's venue in PI_BACKENDS");
|
|
55
|
+
if (typeof config.egress !== "boolean" || typeof config.egressProxy !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(config.egressProxy)) return configRefusal();
|
|
56
|
+
if (request.pin && (!Number.isInteger(config.sandboxPinDays) || config.sandboxPinDays < 1)) return configRefusal();
|
|
57
|
+
let image, size;
|
|
58
|
+
const project = projectOf({ kind, repo: request.repo }, projects);
|
|
59
|
+
try {
|
|
60
|
+
image = resolveJobImage(trigger.run, config.jobImage);
|
|
61
|
+
if (!image) return configRefusal();
|
|
62
|
+
size = resolveJobSize({ project, limits, env });
|
|
63
|
+
} catch { return configRefusal(); }
|
|
64
|
+
const plan = { host: request.host, repo: request.repo, ref: request.ref ?? "HEAD", triggerIndex: request.triggerIndex, triggerDigest: digest, kind, image, backend, size, egress: { armed: config.egress, proxy: config.egressProxy }, retentionHours: config.sandboxRetentionHours, pin: request.pin === true, pinDays: config.sandboxPinDays, project };
|
|
65
|
+
const planDigest = hash(JSON.stringify(plan));
|
|
66
|
+
if (request.planDigest !== undefined && request.planDigest !== planDigest) return refused("sandbox-plan-changed", "the host's image, size, venue, egress or retention changed; preview the sandbox again");
|
|
67
|
+
return { plan: { ...plan, planDigest } };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** The lazily created standalone CLI uses the same forge factories; a wired worker can supply its existing forgeFor. */
|
|
71
|
+
async function defaultForgeFor(kind, config, env) {
|
|
72
|
+
if (!config[kind]) return null;
|
|
73
|
+
if (kind === "github") {
|
|
74
|
+
const { makeGitHubAuth } = await import("./get-token.mjs");
|
|
75
|
+
return { auth: await makeGitHubAuth(config.github, { env }) };
|
|
76
|
+
}
|
|
77
|
+
if (kind === "gitlab") {
|
|
78
|
+
const { makeGitLabAuth } = await import("./gitlab-auth.mjs");
|
|
79
|
+
return { auth: await makeGitLabAuth(config.gitlab, { env }) };
|
|
80
|
+
}
|
|
81
|
+
if (kind === "forgejo") {
|
|
82
|
+
const { makeForgejoAuth } = await import("./forgejo-auth.mjs");
|
|
83
|
+
return { auth: await makeForgejoAuth(config.forgejo, { env }) };
|
|
84
|
+
}
|
|
85
|
+
if (kind === "azure") {
|
|
86
|
+
const { makeAzureAuth } = await import("./azure-auth.mjs");
|
|
87
|
+
return { auth: await makeAzureAuth(config.azure, { env }) };
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export async function observeSandboxPreparation({ backend, image, config, inspectImage = null, decideUser = null, readStore = null }) {
|
|
93
|
+
const { decideSandboxJobUser, readPodmanStore } = await import("./sandbox.mjs");
|
|
94
|
+
// Identity can use the image's own uid on some platforms without inspecting it. Presence is an independent gate:
|
|
95
|
+
// an absent image must be refused before minting or cloning, even on macOS, Windows, or the baked uid's account.
|
|
96
|
+
const inspect = inspectImage ?? makeImagePreflight({ image, bin: backend === "podman" ? "podman" : "docker" });
|
|
97
|
+
const inspected = await inspect({});
|
|
98
|
+
if (inspected?.unavailable) throw new Error("the sandbox image runtime did not answer");
|
|
99
|
+
if (inspected?.ok !== true) return refused("sandbox-image-unavailable", "the selected image is not available on this host; pull or build the trigger's image before preparing");
|
|
100
|
+
const user = await (decideUser ?? decideSandboxJobUser)({ manifest: { image, backend }, venue: backend, backendFloor: config.backendFloor });
|
|
101
|
+
// The interactive opener refuses unread observations because it has no queue. Preparation has a bounded retry,
|
|
102
|
+
// so the CLI's "try again" results must regain their infrastructure meaning here, without matching prose.
|
|
103
|
+
if (["job-user-unknown", "podman-conf-unread", "podman-unobserved"].includes(user?.refused)) throw new Error("the sandbox's user or isolation observations could not be read");
|
|
104
|
+
if (user?.refused === "job-user-image") {
|
|
105
|
+
const again = await inspect({});
|
|
106
|
+
if (again?.missing) return refused("sandbox-image-unavailable", "the selected image disappeared from this host; pull or build it before preparing");
|
|
107
|
+
throw new Error("the sandbox image's user capabilities could not be read");
|
|
108
|
+
}
|
|
109
|
+
if (user?.refused) return user;
|
|
110
|
+
const podmanStore = backend === "podman" ? await (readStore ?? readPodmanStore)() : null;
|
|
111
|
+
if (backend === "podman" && (typeof podmanStore !== "string" || !podmanStore)) throw new Error("the sandbox's rootless container store could not be read");
|
|
112
|
+
return { jobUser: { user: user?.user ?? null, home: user?.home ?? null }, ...(podmanStore ? { podmanStore } : {}) };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Each accepted request has one id and one final manifest. The exclusive per-id host lock protects direct CLI work
|
|
117
|
+
* against a queue retry, and a reconciled ready manifest is returned before another credential or clone is requested.
|
|
118
|
+
*/
|
|
119
|
+
export function createSandboxPreparer({ env = process.env, config = loadConfig(env), readTriggers = () => nodeFs.readFileSync(config.triggersFile ?? join(process.cwd(), "triggers.json"), "utf8"), projects = () => loadProjects(config), limits = () => loadScopedLimits(config), forgeFor = null, clone = cloneHostWorkspace, resolveRef = resolveSandboxRef, observe = observeSandboxPreparation, now = Date.now, fs = nodeFs, ensureDir = ensureSandboxDir, ensureJobs = ensureJobsDir, pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return e.code !== "ESRCH"; } } } = {}) {
|
|
120
|
+
return async function prepare(request) {
|
|
121
|
+
let planned;
|
|
122
|
+
try { planned = planSandboxPreparation({ request, config, env, triggersText: readTriggers(), projects: projects(), limits: limits() }); }
|
|
123
|
+
catch (error) { if (error?.piDispatchConfig || error?.code === "ENOENT") return configRefusal(); throw new Error("sandbox preparation configuration read failed", { cause: error }); }
|
|
124
|
+
if (planned.refused) return planned;
|
|
125
|
+
const { plan } = planned;
|
|
126
|
+
const id = request.id ?? newSandboxId();
|
|
127
|
+
const fingerprint = hash(JSON.stringify({ plan, by: request.by ?? null }));
|
|
128
|
+
let remote;
|
|
129
|
+
try {
|
|
130
|
+
remote = sandboxRemoteUrl(plan.kind, plan.repo, config);
|
|
131
|
+
ensureDir(config.sandboxDir, { env });
|
|
132
|
+
fs.mkdirSync(config.sandboxDir, { recursive: true, mode: 0o700 });
|
|
133
|
+
ensureDir(config.sandboxDir, { env });
|
|
134
|
+
ensureJobs(config.jobsDir, { env });
|
|
135
|
+
if (fs.statSync(config.jobsDir).dev !== fs.statSync(config.sandboxDir).dev) return refused("sandbox-config-invalid", "PI_JOBS_DIR and PI_SANDBOX_DIR must be on the same filesystem for atomic preparation; adjust this host's deployment before retrying");
|
|
136
|
+
} catch (error) { if (error?.piDispatchConfig) return configRefusal(); throw new Error("sandbox preparation directories could not be secured", { cause: error }); }
|
|
137
|
+
// The retention sweep removes non-directory entries in its root. This live lock belongs beside staging,
|
|
138
|
+
// outside that sweep, or a sweep during the awaited clone could let a second owner publish the same id.
|
|
139
|
+
const lock = join(config.jobsDir, `${id}.lock`);
|
|
140
|
+
let ownership;
|
|
141
|
+
try { ownership = takePreparationLock(lock, { fs, pidAlive, jobsDir: config.jobsDir }); }
|
|
142
|
+
catch (error) { throw new Error("sandbox preparation lock could not be acquired", { cause: error }); }
|
|
143
|
+
if (!ownership) return refused("sandbox-preparation-busy", "this sandbox is already being prepared, or its lock could not be safely recovered; inspect this host before retrying");
|
|
144
|
+
let staging = null;
|
|
145
|
+
try {
|
|
146
|
+
const dest = join(config.sandboxDir, id);
|
|
147
|
+
let exists = false;
|
|
148
|
+
try { fs.lstatSync(dest); exists = true; } catch (error) { if (error.code !== "ENOENT") throw error; }
|
|
149
|
+
if (exists) {
|
|
150
|
+
const manifest = readManifest({ sandboxDir: config.sandboxDir, jobId: id, fs });
|
|
151
|
+
if (manifest?.jobId === id && manifest.prepared?.requestDigest === fingerprint) return ready(manifest, plan.host, config.sandboxRetentionHours);
|
|
152
|
+
return refused("sandbox-id-collision", "the sandbox id already names another workspace; prepare again with a new id");
|
|
153
|
+
}
|
|
154
|
+
const observations = await observe({ backend: plan.backend, image: plan.image, config });
|
|
155
|
+
if (observations?.refused) return observations;
|
|
156
|
+
let pair;
|
|
157
|
+
try { pair = await forgeFor?.({ kind: plan.kind, repo: plan.repo }); if (!pair?.auth) pair = await defaultForgeFor(plan.kind, config, env); }
|
|
158
|
+
catch (error) { if (error?.piDispatchConfig) return refused("sandbox-credential-unavailable", "the host's forge credential is not configured; run pi-dispatch doctor on that host"); throw new Error("sandbox forge authentication temporarily failed", { cause: error }); }
|
|
159
|
+
if (typeof pair?.auth?.mintToken !== "function") return refused("sandbox-credential-unavailable", "the host has no credential for the selected forge");
|
|
160
|
+
// Narrow the installation token even when this host's ordinary jobs need write access.
|
|
161
|
+
let token;
|
|
162
|
+
try { token = await pair.auth.mintToken({ kind: plan.kind, repo: plan.repo, ...(plan.kind === "github" ? { permissions: { contents: "read" } } : {}) }); }
|
|
163
|
+
catch (error) { if (error?.piDispatchConfig) return refused("sandbox-credential-unavailable", "the host's forge credential cannot read this repository"); throw new Error("sandbox clone credential temporarily unavailable", { cause: error }); }
|
|
164
|
+
if (typeof token !== "string" || token.trim() === "") return refused("sandbox-credential-unavailable", "the host's forge credential is empty");
|
|
165
|
+
// The lock names this directory BEFORE it exists, closing the mkdir-to-record crash window. A restart
|
|
166
|
+
// reclaims exactly that dead owner's directory, including a clone too young for the boot orphan sweep.
|
|
167
|
+
const stagePath = join(config.jobsDir, ownership.stageName);
|
|
168
|
+
fs.mkdirSync(stagePath, { mode: 0o700 });
|
|
169
|
+
staging = stagePath;
|
|
170
|
+
const workspace = join(staging, "workspace");
|
|
171
|
+
let cloned;
|
|
172
|
+
try {
|
|
173
|
+
const resolved = await resolveRef({ kind: plan.kind, repo: plan.repo, ref: plan.ref, config, token });
|
|
174
|
+
if (resolved?.refused) return resolved;
|
|
175
|
+
cloned = await clone({ workspace, remote, ref: resolved.fetchRef, token, env });
|
|
176
|
+
} catch (error) {
|
|
177
|
+
if (isCloneRefGone(error)) return refused("sandbox-ref-unavailable", "the repository no longer serves the requested ref; choose another ref");
|
|
178
|
+
throw new Error("sandbox clone failed; check the host's forge connection", { cause: error });
|
|
179
|
+
} finally { token = null; }
|
|
180
|
+
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(cloned?.sha ?? "")) throw new Error("sandbox clone returned no commit id");
|
|
181
|
+
const at = now();
|
|
182
|
+
// Issue #761: a version only when a field needs one (`withManifestVersion`), so today's manifest is unchanged.
|
|
183
|
+
const manifest = withManifestVersion({ jobId: id, kind: plan.kind, origin: "prepared", image: plan.image, backend: plan.backend, jobUser: observations?.jobUser ?? null, ...(observations?.podmanStore ? { podmanStore: observations.podmanStore } : {}), size: plan.size, sandboxEgress: plan.egress, workspace: join(dest, "workspace"), baseSha: cloned.sha, createdAt: new Date(at).toISOString(), retainUntil: new Date(at + plan.retentionHours * 3_600_000).toISOString(), keepUntil: plan.pin ? new Date(at + plan.pinDays * 86_400_000).toISOString() : null, prepared: { requestId: id, requestDigest: fingerprint, triggerIndex: plan.triggerIndex, triggerDigest: plan.triggerDigest, repo: plan.repo, ref: plan.ref } });
|
|
184
|
+
fs.writeFileSync(join(staging, SANDBOX_MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx", mode: 0o600 });
|
|
185
|
+
// Nothing under the retention root is visible until the clone AND manifest are complete.
|
|
186
|
+
try { fs.lstatSync(dest); return refused("sandbox-id-collision", "another workspace appeared under this id; nothing was replaced"); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
|
187
|
+
fs.renameSync(staging, dest);
|
|
188
|
+
staging = null;
|
|
189
|
+
return ready(manifest, plan.host, config.sandboxRetentionHours);
|
|
190
|
+
} finally {
|
|
191
|
+
try { if (staging) fs.rmSync(staging, { recursive: true, force: true }); }
|
|
192
|
+
finally { releasePreparationLock(lock, ownership, fs); }
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Descriptor-backed lock identity, so a replaced path and an unreadable/FIFO lock are never judged stale. */
|
|
198
|
+
function readPreparationLock(path, fs) {
|
|
199
|
+
let fd;
|
|
200
|
+
try {
|
|
201
|
+
const before = fs.lstatSync(path);
|
|
202
|
+
if (!before.isFile() || before.size > PREPARATION_LOCK_MAX_BYTES) return null;
|
|
203
|
+
fd = fs.openSync(path, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0));
|
|
204
|
+
const stat = fs.fstatSync(fd);
|
|
205
|
+
if (!stat.isFile() || stat.size > PREPARATION_LOCK_MAX_BYTES || stat.dev !== before.dev || stat.ino !== before.ino) return null;
|
|
206
|
+
const body = fs.readFileSync(fd, "utf8");
|
|
207
|
+
return { body, dev: stat.dev, ino: stat.ino };
|
|
208
|
+
} catch { return null; }
|
|
209
|
+
finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function moveLockAside(path, why, fs) {
|
|
213
|
+
const moved = `${path}.${why}.${randomBytes(16).toString("hex")}`;
|
|
214
|
+
try { fs.renameSync(path, moved); return moved; } catch { return null; }
|
|
215
|
+
}
|
|
216
|
+
function sameLock(a, b) { return a && b && a.body === b.body && a.dev === b.dev && a.ino === b.ino; }
|
|
217
|
+
function restoreLock(moved, path, fs) {
|
|
218
|
+
try { fs.linkSync(moved, path); } catch {}
|
|
219
|
+
try { fs.unlinkSync(moved); } catch {}
|
|
220
|
+
}
|
|
221
|
+
function takePreparationLock(path, { fs, pidAlive, jobsDir }) {
|
|
222
|
+
const stageName = `job-sb-${randomBytes(16).toString("hex")}`;
|
|
223
|
+
const body = JSON.stringify({ pid: process.pid, tag: randomBytes(16).toString("hex"), jobsRoot: fs.realpathSync(jobsDir), stageName });
|
|
224
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
225
|
+
let fd;
|
|
226
|
+
try {
|
|
227
|
+
fd = fs.openSync(path, "wx", 0o600);
|
|
228
|
+
fs.writeFileSync(fd, body);
|
|
229
|
+
const stat = fs.fstatSync(fd);
|
|
230
|
+
return { body, dev: stat.dev, ino: stat.ino, stageName };
|
|
231
|
+
} catch (error) {
|
|
232
|
+
if (error.code !== "EEXIST") throw error;
|
|
233
|
+
const seen = readPreparationLock(path, fs);
|
|
234
|
+
let owner;
|
|
235
|
+
try { owner = JSON.parse(seen?.body); } catch { return null; }
|
|
236
|
+
if (!Number.isInteger(owner?.pid) || owner.pid < 1 || pidAlive(owner.pid)) return null;
|
|
237
|
+
// Like the checkpoint store: move, verify the exact observed generation, and only then discard it.
|
|
238
|
+
// An unlink of this path could remove a rival's new lock on evidence about the dead predecessor.
|
|
239
|
+
const moved = moveLockAside(path, "stale", fs);
|
|
240
|
+
if (!moved) continue;
|
|
241
|
+
if (!sameLock(readPreparationLock(moved, fs), seen)) { restoreLock(moved, path, fs); return null; }
|
|
242
|
+
try { reclaimPreparationStage(owner, jobsDir, fs); }
|
|
243
|
+
catch (error) { restoreLock(moved, path, fs); throw error; }
|
|
244
|
+
fs.unlinkSync(moved);
|
|
245
|
+
} finally { if (fd !== undefined) fs.closeSync(fd); }
|
|
246
|
+
}
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
function reclaimPreparationStage(owner, jobsDir, fs) {
|
|
250
|
+
if (owner.stageName === undefined) return;
|
|
251
|
+
if (!/^job-sb-[a-f0-9]{32}$/.test(owner.stageName) || owner.jobsRoot !== fs.realpathSync(jobsDir)) throw new Error("the old preparation's staging root changed; inspect it before retrying");
|
|
252
|
+
const stage = join(jobsDir, owner.stageName);
|
|
253
|
+
let stat;
|
|
254
|
+
try { stat = fs.lstatSync(stage); } catch (error) { if (error.code === "ENOENT") return; throw error; }
|
|
255
|
+
if (!stat.isDirectory() || (Number.isInteger(process.geteuid?.()) && stat.uid !== process.geteuid())) throw new Error("the old preparation's staging directory changed ownership or type");
|
|
256
|
+
fs.rmSync(stage, { recursive: true, force: true });
|
|
257
|
+
}
|
|
258
|
+
function releasePreparationLock(path, ownership, fs) {
|
|
259
|
+
const moved = moveLockAside(path, "release", fs);
|
|
260
|
+
if (!moved) return;
|
|
261
|
+
if (sameLock(readPreparationLock(moved, fs), ownership)) fs.unlinkSync(moved);
|
|
262
|
+
else restoreLock(moved, path, fs);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export const makeSandboxPreparer = createSandboxPreparer;
|
|
266
|
+
|
|
267
|
+
function ready(manifest, host, retentionHours) {
|
|
268
|
+
const deadline = sandboxDeadline(manifest, retentionHours);
|
|
269
|
+
return { id: manifest.jobId, host, until: deadline.until, pinned: deadline.source === "pin", backend: manifest.backend, prepared: true, generation: sandboxManifestGeneration(manifest) };
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export function createSandboxPinHandler({ config, now = Date.now, fs = nodeFs, pin = pinSandbox } = {}) {
|
|
273
|
+
return async (request) => {
|
|
274
|
+
if (request?.host !== config.workerName || !/^[A-Za-z0-9._:-]{1,200}$/.test(request?.runId ?? "")) return refused("sandbox-pin-invalid", "choose a kept workspace on this host");
|
|
275
|
+
const manifest = readManifest({ sandboxDir: config.sandboxDir, jobId: request.runId, fs });
|
|
276
|
+
if (!manifest || manifest.jobId !== request.runId) return refused("sandbox-absent", "the kept workspace is gone; refresh the Sandboxes tab");
|
|
277
|
+
const deadline = sandboxDeadline(manifest, config.sandboxRetentionHours);
|
|
278
|
+
if (request.expect && (request.expect.until !== deadline.until || request.expect.pin !== (deadline.source === "pin") || request.expect.generation !== sandboxManifestGeneration(manifest))) return refused("sandbox-changed", "the kept workspace changed after the preview; refresh and preview it again");
|
|
279
|
+
const result = pin({ sandboxDir: config.sandboxDir, jobId: request.runId, pinDays: config.sandboxPinDays, fs, now });
|
|
280
|
+
// Issue #761: a manifest a newer pi-dispatch kept is not rewritten here, and the words say which release can pin it.
|
|
281
|
+
if (result.reason === "manifest-newer") return refused("manifest-newer", "the kept workspace was kept by a newer pi-dispatch, so this host does not rewrite it; pin it with that release");
|
|
282
|
+
if (!result.pinned) return refused("sandbox-pin-failed", "the workspace could not be pinned on its host; refresh and check the host's retention directory");
|
|
283
|
+
return { id: request.runId, host: config.workerName, until: Date.parse(result.keepUntil), pinned: true, generation: sandboxManifestGeneration(manifest) };
|
|
284
|
+
};
|
|
285
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/** Checked references for a human's sandbox; no job or agent accepts this broader checkout surface (#771). */
|
|
2
|
+
import { FORGE_KINDS } from "./forges.mjs";
|
|
3
|
+
import { isDeterminateFetchFailure, isTransientStatus } from "./transient.mjs";
|
|
4
|
+
|
|
5
|
+
export const SANDBOX_SHA_RE = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/;
|
|
6
|
+
const refuse = (refused, message) => ({ refused, message });
|
|
7
|
+
|
|
8
|
+
export function sandboxRepoProblem(repo, kind) {
|
|
9
|
+
if (!FORGE_KINDS.includes(kind)) return "choose a forge-backed trigger; a local-folder trigger cannot clone a repository";
|
|
10
|
+
if (typeof repo !== "string" || repo.length > 300 || repo !== repo.trim() || /[\u0000-\u001f\u007f\\:@?#%]/.test(repo)) return "repository must be a plain forge repository path, without a URL or credential";
|
|
11
|
+
const parts = repo.split("/");
|
|
12
|
+
if (parts.some((p) => !p || p === "." || p === ".." || p.startsWith("-") || p.endsWith(".git"))) return "repository has an invalid path segment";
|
|
13
|
+
if ((kind === "gitlab" && parts.length < 2) || (kind !== "gitlab" && parts.length !== 2)) return kind === "gitlab" ? "GitLab repository must name its full group/project path" : "repository must have exactly two path segments";
|
|
14
|
+
if (parts.some((p) => !/^[\p{L}\p{N}._ -]+$/u.test(p))) return "repository has an unsupported path character";
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** pr:42 is deliberately distinct from a branch named 42. Fully qualified heads/tags are also accepted. */
|
|
19
|
+
export function parseSandboxRef(ref = "HEAD", kind = "github") {
|
|
20
|
+
if (typeof ref !== "string" || ref.length === 0 || ref.length > 200 || ref !== ref.trim()) return refuse("sandbox-ref-invalid", "ref must be HEAD, a branch or tag, a full commit id, or pr:<number>");
|
|
21
|
+
const pull = /^(?:pr|mr):([1-9]\d{0,9})$/.exec(ref);
|
|
22
|
+
if (pull) {
|
|
23
|
+
if (!FORGE_KINDS.includes(kind)) return refuse("sandbox-ref-invalid", "pull request refs need a forge-backed trigger");
|
|
24
|
+
return { type: "pull_request", number: Number(pull[1]), ref };
|
|
25
|
+
}
|
|
26
|
+
if (SANDBOX_SHA_RE.test(ref)) return { type: "sha", fetchRef: ref, ref };
|
|
27
|
+
// This is Git's ref-name grammar, with '-' and ':' additionally closed before the value reaches fetch's argv.
|
|
28
|
+
if (ref.startsWith("-") || /[\u0000-\u0020\u007f~^:?*\[\\]/.test(ref) || ref.includes("..") || ref.includes("@{") || ref === "@" || ref.endsWith(".") || ref.split("/").some((p) => !p || p.startsWith(".") || p.endsWith(".lock"))) return refuse("sandbox-ref-invalid", "ref is not a safe Git reference; use a branch, tag, full commit id, or pr:<number>");
|
|
29
|
+
return { type: "ref", fetchRef: ref, ref };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** A tokenless origin made only from host configuration and a checked repository, never a request-supplied URL. */
|
|
33
|
+
export function sandboxRemoteUrl(kind, repo, config) {
|
|
34
|
+
const problem = sandboxRepoProblem(repo, kind);
|
|
35
|
+
if (problem) throw new Error(problem);
|
|
36
|
+
const parts = repo.split("/").map(encodeURIComponent);
|
|
37
|
+
let root;
|
|
38
|
+
if (kind === "github") root = "https://github.com";
|
|
39
|
+
else root = kind === "azure" ? config?.azure?.orgUrl : config?.[kind]?.apiUrl;
|
|
40
|
+
let url;
|
|
41
|
+
try { url = new URL(root); } catch { throw new Error("the selected forge has no valid configured URL"); }
|
|
42
|
+
if (!["https:", "http:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw new Error("the selected forge requires a credential-free http(s) URL");
|
|
43
|
+
const base = url.href.replace(/\/+$/, "");
|
|
44
|
+
return kind === "azure" ? `${base}/${parts[0]}/_git/${parts[1]}` : `${base}/${parts.join("/")}.git`;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** GitHub, GitLab and Forgejo serve a PR head on the base repository. Azure exposes its source commit through API. */
|
|
48
|
+
export async function resolveSandboxRef({ kind, repo, ref, config, token, fetchFn = fetch, signal } = {}) {
|
|
49
|
+
const parsed = parseSandboxRef(ref, kind);
|
|
50
|
+
if (parsed.refused) return parsed;
|
|
51
|
+
if (parsed.type !== "pull_request") return { fetchRef: parsed.fetchRef };
|
|
52
|
+
if (kind !== "azure") return { fetchRef: kind === "gitlab" ? `refs/merge-requests/${parsed.number}/head` : `refs/pull/${parsed.number}/head` };
|
|
53
|
+
const [project, repository] = repo.split("/").map(encodeURIComponent);
|
|
54
|
+
const root = new URL(config.azure.orgUrl).href.replace(/\/+$/, "");
|
|
55
|
+
let response;
|
|
56
|
+
try {
|
|
57
|
+
response = await fetchFn(`${root}/${project}/_apis/git/repositories/${repository}/pullrequests/${parsed.number}?api-version=7.1`, { headers: { Authorization: `Basic ${Buffer.from(`:${token}`).toString("base64")}`, accept: "application/json" }, redirect: "error", signal: signal ?? AbortSignal.timeout(30_000) });
|
|
58
|
+
} catch (error) {
|
|
59
|
+
if (isDeterminateFetchFailure(error)) return refuse("sandbox-ref-unavailable", "the forge connection configuration prevents resolving this pull request");
|
|
60
|
+
throw new Error("sandbox pull request lookup did not answer", { cause: error });
|
|
61
|
+
}
|
|
62
|
+
if (!response.ok) {
|
|
63
|
+
if (isTransientStatus(response.status, (name) => response.headers?.get?.(name), "")) throw new Error("sandbox pull request lookup temporarily failed");
|
|
64
|
+
return refuse("sandbox-ref-unavailable", "the selected pull request could not be read with this host's forge credential");
|
|
65
|
+
}
|
|
66
|
+
const data = await response.json();
|
|
67
|
+
const sha = data?.lastMergeSourceCommit?.commitId;
|
|
68
|
+
if (!SANDBOX_SHA_RE.test(sha ?? "")) return refuse("sandbox-ref-unavailable", "the pull request does not report a source commit");
|
|
69
|
+
return { fetchRef: sha };
|
|
70
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { fork, spawn } from "node:child_process";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { userInfo } from "node:os";
|
|
4
|
+
import { execDockerBounded } from "./backend-local.mjs";
|
|
5
|
+
import { makeRedisClient } from "./connection.mjs";
|
|
6
|
+
import { networkNameFor, removeJobNetwork } from "./egress.mjs";
|
|
7
|
+
import { stopSandbox } from "./sandbox.mjs";
|
|
8
|
+
import { SANDBOX_SESSION_LABEL, SANDBOX_SESSION_POLL_MS, parseSandboxSession, publishSandboxNotice, sandboxBounded, sandboxNoticeKey, sandboxSessionAccount, sandboxSessionKey, sandboxSessionTick } from "./sandbox-sessions.mjs";
|
|
9
|
+
|
|
10
|
+
/** Inspect distinguishes a missing container from a daemon that could not answer. No stderr is published. */
|
|
11
|
+
export async function inspectSandboxSession({ bin, name, run = execDockerBounded }) {
|
|
12
|
+
let answer;
|
|
13
|
+
try { answer = await run(["inspect", name], { bin, timeoutMs: 5_000, withStderr: true }); } catch { return { unknown: true }; }
|
|
14
|
+
if (answer.code !== 0) return /no such (?:container|object)|no container with (?:name|ID)/i.test(answer.stderr ?? "") ? { present: false } : { unknown: true };
|
|
15
|
+
try {
|
|
16
|
+
const list = JSON.parse(answer.stdout);
|
|
17
|
+
const c = Array.isArray(list) ? list[0] : null;
|
|
18
|
+
if (!/^[a-f0-9]{64}$/.test(c?.Id ?? "")) return { unknown: true };
|
|
19
|
+
return { present: true, running: c.State?.Running === true, containerId: c.Id, nonce: c.Config?.Labels?.[SANDBOX_SESSION_LABEL] ?? null, networks: Object.keys(c.NetworkSettings?.Networks ?? {}), proxy: c.Config?.Labels?.["org.pi-dispatch.sandbox-proxy"] ?? null };
|
|
20
|
+
} catch { return { unknown: true }; }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** A private monitor owns its network after ready. It survives detach and the terminal's death. */
|
|
24
|
+
export function makeSandboxMonitor({ env = process.env, config, by, spawnMonitor = fork, now = Date.now } = {}) {
|
|
25
|
+
return async ({ resolved, args, network, runtime, admittedRuntime = null, existing = null }) => {
|
|
26
|
+
const nonce = existing?.nonce ?? randomBytes(16).toString("hex");
|
|
27
|
+
// The image is last in both sandbox builders. Put a label into the runtime argv, never an env variable.
|
|
28
|
+
if (args) args.splice(args.length - 1, 0, "--label", `${SANDBOX_SESSION_LABEL}=${nonce}`, ...(network ? ["--label", `org.pi-dispatch.sandbox-proxy=${config.egressProxy}`] : []));
|
|
29
|
+
const child = spawnMonitor(new URL("./sandbox-session-monitor.mjs", import.meta.url), [], { detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"], env });
|
|
30
|
+
const account = sandboxSessionAccount(by ?? (() => { try { return userInfo().username; } catch { return "operator"; } })());
|
|
31
|
+
const metadata = { host: config.workerName, id: String(resolved.manifest.jobId), venue: resolved.venue, name: resolved.name, nonce, by: account, since: existing?.since ?? now(), network, proxy: existing?.proxy ?? config.egressProxy, bin: runtime, valkeyUrl: config.valkeyUrl, admittedRuntime };
|
|
32
|
+
try {
|
|
33
|
+
await sandboxBounded(new Promise((resolve, reject) => {
|
|
34
|
+
child.once("error", reject);
|
|
35
|
+
child.once("exit", () => reject(new Error("sandbox monitor exited before ready")));
|
|
36
|
+
child.once("message", (message) => message?.ready === true ? resolve() : reject(new Error("sandbox tracking is unavailable")));
|
|
37
|
+
child.send(metadata);
|
|
38
|
+
}), 5_000);
|
|
39
|
+
} catch {
|
|
40
|
+
child.kill();
|
|
41
|
+
throw new Error("Sandbox tracking is unavailable. Check Valkey before opening a shell.");
|
|
42
|
+
}
|
|
43
|
+
child.unref();
|
|
44
|
+
child.channel?.unref();
|
|
45
|
+
return { ownsNetwork: true, finish: () => { if (child.connected) child.send({ foregroundDone: true }); } };
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Reattach can repair a dead monitor only after the runtime confirms the immutable labelled incarnation. */
|
|
50
|
+
export async function ensureSandboxMonitor({ resolved, runtime, state, admittedRuntime = null, config, env = process.env, by, now = Date.now, makeClient = makeRedisClient, monitor = makeSandboxMonitor({ env, config, by, now }) }) {
|
|
51
|
+
let redis;
|
|
52
|
+
try {
|
|
53
|
+
redis = makeClient(config.valkeyUrl, { failFast: true, lazyConnect: true, disconnectTimeoutMs: 0 });
|
|
54
|
+
redis.on("error", () => {});
|
|
55
|
+
await sandboxBounded(redis.connect(), 3_000);
|
|
56
|
+
const current = parseSandboxSession(await sandboxBounded(redis.get(sandboxSessionKey(config.workerName, resolved.manifest.jobId))));
|
|
57
|
+
if (current?.nonce === state.nonce && current?.containerId === state.containerId) return { tracked: true };
|
|
58
|
+
const network = state.networks?.includes(networkNameFor(resolved.name)) ? networkNameFor(resolved.name) : null;
|
|
59
|
+
if (network && !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(state.proxy ?? "")) throw new Error("original egress proxy is unknown");
|
|
60
|
+
await monitor({ resolved, runtime, args: null, network, admittedRuntime, existing: { nonce: state.nonce, since: current?.since ?? now(), proxy: state.proxy } });
|
|
61
|
+
return { tracked: true, recovered: true };
|
|
62
|
+
} finally { redis?.disconnect(); }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function watchSandboxSession({ metadata, redis, inspect = (name) => inspectSandboxSession({ bin: metadata.bin, name }), stop = (name) => stopSandbox({ bin: metadata.bin, name }), cleanup = async () => {}, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), foregroundDone = () => false, report = () => {} }) {
|
|
66
|
+
let session = null;
|
|
67
|
+
let absent = 0;
|
|
68
|
+
for (;;) {
|
|
69
|
+
try {
|
|
70
|
+
if (!session) {
|
|
71
|
+
const found = await inspect(metadata.name);
|
|
72
|
+
if (found.unknown) { await sleep(SANDBOX_SESSION_POLL_MS); continue; }
|
|
73
|
+
if (!found.present) {
|
|
74
|
+
if (++absent >= 30 || foregroundDone()) break;
|
|
75
|
+
await sleep(SANDBOX_SESSION_POLL_MS); continue;
|
|
76
|
+
}
|
|
77
|
+
if (found.nonce !== metadata.nonce) return { refused: "container-changed" };
|
|
78
|
+
session = { v: 1, host: metadata.host, id: metadata.id, nonce: metadata.nonce, containerId: found.containerId, venue: metadata.venue, by: metadata.by, since: metadata.since, at: now() };
|
|
79
|
+
}
|
|
80
|
+
const result = await sandboxSessionTick({ redis, session, inspect, stop, now });
|
|
81
|
+
if (result.unknown) await report("runtime-unreadable");
|
|
82
|
+
if (result.gone) break;
|
|
83
|
+
if (result.refused) return result;
|
|
84
|
+
} catch {
|
|
85
|
+
// No acknowledgement on a failed read or stop. The lease expires and the next good tick can recover.
|
|
86
|
+
await report("runtime-unreadable");
|
|
87
|
+
}
|
|
88
|
+
await sleep(SANDBOX_SESSION_POLL_MS);
|
|
89
|
+
}
|
|
90
|
+
// A new open cannot reuse this network until it is removed; still inspect the name so a replacement is kept.
|
|
91
|
+
const last = await inspect(metadata.name);
|
|
92
|
+
if (!last.unknown && !last.present) {
|
|
93
|
+
try { if (await cleanup() === false) await report("network-kept"); }
|
|
94
|
+
catch { await report("network-cleanup-failed"); }
|
|
95
|
+
} else if (metadata.network) await report("network-kept");
|
|
96
|
+
return { gone: true };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export async function runSandboxSessionMonitor({ metadata, redis, ready = () => {}, foregroundDone = () => false, watchOptions = {}, removeNetwork = removeJobNetwork }) {
|
|
100
|
+
await sandboxBounded(redis.connect().then(() => redis.ping()), 3_000);
|
|
101
|
+
// Clearing an old diagnostic is best effort and precedes the launch handshake. An outage here must not
|
|
102
|
+
// kill a monitor after it told the parent that a shell could start (#771).
|
|
103
|
+
await sandboxBounded(redis.del(sandboxNoticeKey(metadata.host, metadata.id)), 500).catch(() => {});
|
|
104
|
+
ready({ ready: true });
|
|
105
|
+
let said = null;
|
|
106
|
+
const report = async (reason) => { if (said === reason) return; try { await publishSandboxNotice(redis, { host: metadata.host, id: metadata.id, reason }); said = reason; } catch {} };
|
|
107
|
+
return watchSandboxSession({ metadata, redis, foregroundDone, report, ...watchOptions, cleanup: async () => {
|
|
108
|
+
if (metadata.network) return removeNetwork(spawn, { network: metadata.network, proxy: metadata.proxy, bin: metadata.bin, ...(metadata.admittedRuntime ? { readRuntime: async () => metadata.admittedRuntime } : {}), onRefused: () => {} });
|
|
109
|
+
return true;
|
|
110
|
+
} });
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export async function monitorMain() {
|
|
114
|
+
let finished = false;
|
|
115
|
+
process.on("message", (m) => { if (m?.foregroundDone) finished = true; });
|
|
116
|
+
const metadata = await new Promise((resolve) => process.once("message", resolve));
|
|
117
|
+
let redis;
|
|
118
|
+
try {
|
|
119
|
+
redis = makeRedisClient(metadata.valkeyUrl, { noOfflineQueue: true, lazyConnect: true, disconnectTimeoutMs: 0 });
|
|
120
|
+
redis.on("error", () => {});
|
|
121
|
+
await runSandboxSessionMonitor({ metadata, redis, foregroundDone: () => finished, ready: (message) => { if (process.connected) process.send?.(message); } });
|
|
122
|
+
} catch { if (process.connected) process.send?.({ ready: false }); }
|
|
123
|
+
finally { redis?.disconnect(); if (process.connected) process.disconnect?.(); }
|
|
124
|
+
}
|