@edgehero/pi-dispatch 4.3.1 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +84 -3
- package/README.md +21 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +118 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +136 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +190 -29
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +325 -0
- package/src/console-client.mjs +848 -0
- package/src/console-devices.mjs +372 -0
- package/src/console-install.mjs +239 -0
- package/src/console-open.mjs +99 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1657 -196
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +96 -6
- package/src/env-file.mjs +6 -4
- package/src/envelope.mjs +5 -3
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-host.mjs +42 -5
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +33 -3
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +734 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +4 -2
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +12 -2
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1550 -173
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +264 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +36 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +780 -38
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +402 -53
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
|
@@ -0,0 +1,1750 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable workspace checkpoints (issue #645): snapshot a paused job's `/workspace` into a per-run bare git
|
|
3
|
+
* store on the worker host, and restore it into a fresh base checkout after a crash or a move.
|
|
4
|
+
*
|
|
5
|
+
* THE THREAT MODEL IS THE WHOLE DESIGN. The tree being snapshotted was written by the agent, so every
|
|
6
|
+
* byte, every name and every file TYPE in it is attacker-chosen, and the snapshot runs on the HOST,
|
|
7
|
+
* outside the container. Four consequences, each of which rules out an obvious implementation:
|
|
8
|
+
*
|
|
9
|
+
* 1. NO GIT COMMAND EVER RUNS ON THE AGENT'S REPOSITORY. `git add -A && git commit` in `/workspace` is the
|
|
10
|
+
* two-line version, and it runs the agent's clean filters, its `core.fsmonitor`, its signing program,
|
|
11
|
+
* follows its `gitdir:` redirect and its `objects/info/alternates` into any repository on the host.
|
|
12
|
+
* Hardening flags close the ones we know of; not reading the config at all closes the ones we do not.
|
|
13
|
+
* So the tree is walked IN NODE, hashed in Node, and streamed into a bare store we own through one
|
|
14
|
+
* `git fast-import`. The agent's `.git` is parsed from FILES for a branch name and a tip, nothing more.
|
|
15
|
+
* Two bounded exceptions, each argued where it lives: `primeBase`'s `fetch` from the workspace's `.git`,
|
|
16
|
+
* which runs BEFORE the container starts, while that `.git` is still the fresh clone the worker itself
|
|
17
|
+
* made; and `check-ignore` in the byte-cap tier (`dropIgnored`), which is given the work tree but not
|
|
18
|
+
* its `.git`, and reads nothing but ignore files.
|
|
19
|
+
* 2. NOTHING IS FOLLOWED. Every stat is `lstat`, every open is `O_NOFOLLOW`, a symlink is stored as what
|
|
20
|
+
* it is (a mode 120000 blob holding the target) and a FIFO is skipped, never opened blocking.
|
|
21
|
+
* `O_NOFOLLOW` guards only the LAST component, so a directory swapped for a link between the walk's
|
|
22
|
+
* lstat and the open would still be followed. The caller pauses the container first, which is the
|
|
23
|
+
* precondition; on Linux the open is ALSO checked after the fact: `/proc/self/fd/<fd>` must resolve
|
|
24
|
+
* inside the workspace's realpath, or the snapshot is abandoned as `tree-changed`, so an unpaused or
|
|
25
|
+
* misbehaving caller cannot make it read a host file. Where `/proc/self/fd` does not exist (macOS, a
|
|
26
|
+
* development host) only the pause precondition and the dev/inode check stand, and that is the
|
|
27
|
+
* residual: an unpaused agent racing a directory swap there could get a host file read and stored.
|
|
28
|
+
* 3. IT NEVER THROWS. The caller has paused a PAID container to call this; a throw that unwinds through
|
|
29
|
+
* the pause handling is a job lost for a checkpoint that was optional. Every public method returns
|
|
30
|
+
* `{ skipped: reason }` (could not, try later) or `{ refused: reason }` (must not, a policy answer).
|
|
31
|
+
* 4. IT NEVER BLOCKS THE EVENT LOOP. The worker's heartbeat, its lock renewals and the container's own
|
|
32
|
+
* output pump share this loop; a synchronous walk of 200,000 files stalls them long enough for BullMQ
|
|
33
|
+
* to call the job stalled and hand it to another worker. So `fs/promises` throughout, and every git
|
|
34
|
+
* is an async spawn with an AbortSignal deadline and SIGKILL.
|
|
35
|
+
*
|
|
36
|
+
* WHY A BARE GIT STORE AND NOT A TARBALL: content addressing gives deduplication across checkpoints for
|
|
37
|
+
* free (a 2 GiB repository whose agent touched three files costs three blobs), and the restore can ask
|
|
38
|
+
* git for exactly the changed paths. Why not `git worktree`/`git stash` in the agent's repository: rule 1.
|
|
39
|
+
*
|
|
40
|
+
* Layout (`ck` is a hash of the run id, never the id itself: `sanitizeJobId` maps `..` to `..`, so a
|
|
41
|
+
* sanitised name is not a safe directory name):
|
|
42
|
+
*
|
|
43
|
+
* <storeDir>/ 0700
|
|
44
|
+
* <ck>/ 0700, ck = sha256("checkpoint\0" + runId).hex[0..32]
|
|
45
|
+
* repo.git/ bare, refs/pi-dispatch/{base,checkpoint,cp/<seq>}
|
|
46
|
+
* home/ empty, HOME and XDG_CONFIG_HOME for every store git
|
|
47
|
+
* lock exclusive create, "<pid> <tag>\n"
|
|
48
|
+
* base-too-large marker: primeBase judged the base over the cap
|
|
49
|
+
* CURRENT "<seq>\n", the commit point
|
|
50
|
+
* cp-<seq>/manifest.json 0600
|
|
51
|
+
* cp-<seq>/transcript.jsonl
|
|
52
|
+
*/
|
|
53
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
54
|
+
import { spawn } from "node:child_process";
|
|
55
|
+
import { constants as FS } from "node:fs";
|
|
56
|
+
import * as nodeFsp from "node:fs/promises";
|
|
57
|
+
import { hostname as osHostname } from "node:os";
|
|
58
|
+
import { join } from "node:path";
|
|
59
|
+
import { setImmediate as yieldTick } from "node:timers/promises";
|
|
60
|
+
import { pathToFileURL } from "node:url";
|
|
61
|
+
import { foldCase, isDotGitSegment, MAX_DEPTH, MAX_PATH_BYTES, validPath, validSegment } from "./checkpoint-paths.mjs";
|
|
62
|
+
import { STORE_GIT_FLAGS, storeGitEnv } from "./git-hardening.mjs";
|
|
63
|
+
import { sanitizeJobId } from "./run-history.mjs";
|
|
64
|
+
|
|
65
|
+
const GiB = 1024 ** 3;
|
|
66
|
+
const MiB = 1024 ** 2;
|
|
67
|
+
|
|
68
|
+
export const DEFAULT_CHECKPOINT_LIMITS = Object.freeze({
|
|
69
|
+
maxBaseBytes: 2 * GiB,
|
|
70
|
+
maxNewBytes: 512 * MiB,
|
|
71
|
+
maxFiles: 200_000,
|
|
72
|
+
maxFileBytes: 64 * MiB,
|
|
73
|
+
maxStoreBytes: 20 * GiB,
|
|
74
|
+
// The transcript copy's cap, judged against the caller's offset BEFORE a byte is copied. Without it a
|
|
75
|
+
// sparse session file (`truncate -s 100G`, one line appended) makes the copy write 100 GB of zeros onto
|
|
76
|
+
// the host while the paid container is frozen. 256 MiB is far above any real pi session (a long run's
|
|
77
|
+
// JSONL is tens of MiB; a model's context window bounds what one turn can add), equal to half the byte
|
|
78
|
+
// cap, and with at most two checkpoints kept per run it bounds one run's transcripts to 512 MiB.
|
|
79
|
+
maxTranscriptBytes: 256 * MiB,
|
|
80
|
+
snapshotTimeoutMs: 60_000,
|
|
81
|
+
gitTimeoutMs: 60_000,
|
|
82
|
+
ttlMs: 7 * 24 * 3600 * 1000,
|
|
83
|
+
lockStaleMs: 3600 * 1000,
|
|
84
|
+
repackEvery: 20,
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The manifest version this build WRITES (issue #761): one of `CHECKPOINT_MANIFEST_READS`, the newest or the one before
|
|
89
|
+
* it. A manifest is closed at every level, so one of a version above the newest this build reads (a newer release's: a
|
|
90
|
+
* new key, a new `v`) cannot be read here; `readManifest` then says `newer` rather than only `invalid`, and every caller
|
|
91
|
+
* that would DELETE on an unreadable checkpoint (the recovery pickup's cleanup, the salvage's removal of the lost
|
|
92
|
+
* attempt's directory) keeps it and waits instead.
|
|
93
|
+
*/
|
|
94
|
+
export const CHECKPOINT_MANIFEST_VERSION = 1;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Every manifest version this build reads: exactly the newest and the one before it, with the version written among
|
|
98
|
+
* them. The one before keeps a checkpoint the previous release took restoring for one checkpoint lifetime
|
|
99
|
+
* (`DEFAULT_CHECKPOINT_LIMITS.ttlMs`) after an upgrade. A newest above the version written is a release that READS the
|
|
100
|
+
* next version before any release writes it, so the release that then writes it shares a fleet, and a rollback, with
|
|
101
|
+
* readers of it: readers ship first, the writer a release later. A test pins both rules, so a bump that forgets the
|
|
102
|
+
* previous reader, or writes a version it does not read, fails there.
|
|
103
|
+
*/
|
|
104
|
+
export const CHECKPOINT_MANIFEST_READS = Object.freeze([1]);
|
|
105
|
+
|
|
106
|
+
const MAX_LINK_BYTES = 4096;
|
|
107
|
+
const HEX = { sha1: /^[0-9a-f]{40}$/, sha256: /^[0-9a-f]{64}$/ };
|
|
108
|
+
const ANY_HEX_OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;
|
|
109
|
+
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
|
110
|
+
const HOST_RE = /^[A-Za-z0-9._-]{1,64}$/;
|
|
111
|
+
const RUN_ID_RE = /^[A-Za-z0-9._-]{1,200}$/;
|
|
112
|
+
/** piVersion, imageDigest, venue, provider, model: short tokens, never prose (the manifest holds no attacker string). */
|
|
113
|
+
const TOKEN_RE = /^[A-Za-z0-9][A-Za-z0-9._:@/+-]{0,127}$/;
|
|
114
|
+
const BRANCH_RE = /^[A-Za-z0-9_-][A-Za-z0-9._-]*(?:\/[A-Za-z0-9_-][A-Za-z0-9._-]*)*$/;
|
|
115
|
+
const KINDS = new Set(["live", "exit", "salvage"]);
|
|
116
|
+
const QUIESCED = new Set(["frozen", "dead"]);
|
|
117
|
+
const GIT_STATES = new Set(["parsed", "unreadable", "none"]);
|
|
118
|
+
const META_FIELDS = ["piVersion", "imageDigest", "venue", "provider", "model"];
|
|
119
|
+
const COMMITTER = "pi-dispatch <checkpoint@pi-dispatch.invalid>";
|
|
120
|
+
const TRANSIENT_STORE_CODES = new Set(["EIO", "EAGAIN", "EMFILE", "ENFILE", "ETIMEDOUT", "EINTR"]);
|
|
121
|
+
|
|
122
|
+
/** Infrastructure misses must be retried; a damaged or expired artifact is a determinate refusal. */
|
|
123
|
+
export function retryableStoreError(error) {
|
|
124
|
+
const code = typeof error === "string" ? error : error?.code;
|
|
125
|
+
return TRANSIENT_STORE_CODES.has(code) ? code : null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** The directory name for a run: a hash, so no run id can name `..`, a slash or another run's directory. */
|
|
129
|
+
export function checkpointKey(runId) {
|
|
130
|
+
return createHash("sha256").update(`checkpoint\0${runId}`).digest("hex").slice(0, 32);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The plain spawn. Tests swap it for a spy that records every argv and env (the flags bolt). */
|
|
134
|
+
export function defaultSpawnGit(args, { env, cwd, signal }) {
|
|
135
|
+
return spawn("git", args, { env, cwd, signal, killSignal: "SIGKILL", stdio: ["pipe", "pipe", "pipe"] });
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function isBranchName(b) {
|
|
139
|
+
if (typeof b !== "string" || b.length === 0 || b.length > 200 || !BRANCH_RE.test(b)) return false;
|
|
140
|
+
return !b.split("/").some((seg) => seg.endsWith(".lock") || seg.includes(".."));
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function token(v) {
|
|
144
|
+
return typeof v === "string" && TOKEN_RE.test(v) ? v : null;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function isInt(v, min = 0, max = Number.MAX_SAFE_INTEGER) {
|
|
148
|
+
return Number.isSafeInteger(v) && v >= min && v <= max;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* C-quote a path for fast-import, ALWAYS. fast-import takes an unquoted path too, but then a name starting
|
|
153
|
+
* with `"` means something else and a newline ends the command; quoting every path removes the need to
|
|
154
|
+
* decide which names are safe bare. Bytes outside printable ASCII go out as octal, which fast-import
|
|
155
|
+
* unquotes back to the exact bytes.
|
|
156
|
+
*/
|
|
157
|
+
export function cQuote(path) {
|
|
158
|
+
let out = "\"";
|
|
159
|
+
for (const b of Buffer.from(path, "utf8")) {
|
|
160
|
+
if (b === 0x22) out += "\\\"";
|
|
161
|
+
else if (b === 0x5c) out += "\\\\";
|
|
162
|
+
else if (b < 0x20 || b >= 0x7f) out += `\\${b.toString(8).padStart(3, "0")}`;
|
|
163
|
+
else out += String.fromCharCode(b);
|
|
164
|
+
}
|
|
165
|
+
return `${out}"`;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** git's blob id, computed in Node, so the id a cache holds is exactly the id the store will have. */
|
|
169
|
+
function blobOid(objectFormat, bytes) {
|
|
170
|
+
return createHash(objectFormat === "sha256" ? "sha256" : "sha1")
|
|
171
|
+
.update(`blob ${bytes.length}\0`)
|
|
172
|
+
.update(bytes)
|
|
173
|
+
.digest("hex");
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Does `buf` contain any of `needles`? Byte comparison, so an encoding never hides a match. */
|
|
177
|
+
function containsAny(buf, needles) {
|
|
178
|
+
return needles.some((n) => buf.includes(n));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* A chunked reader over a child's stdout for the `cat-file --batch` protocol: one header line, then
|
|
183
|
+
* exactly N bytes. Chunks are kept as a list and joined once per `take`, because concatenating on every
|
|
184
|
+
* `data` event is quadratic on a 64 MiB blob.
|
|
185
|
+
*/
|
|
186
|
+
function byteReader(stream) {
|
|
187
|
+
const chunks = [];
|
|
188
|
+
let length = 0;
|
|
189
|
+
let ended = false;
|
|
190
|
+
let wake = null;
|
|
191
|
+
const poke = () => {
|
|
192
|
+
const w = wake;
|
|
193
|
+
wake = null;
|
|
194
|
+
w?.();
|
|
195
|
+
};
|
|
196
|
+
stream.on("data", (c) => {
|
|
197
|
+
chunks.push(c);
|
|
198
|
+
length += c.length;
|
|
199
|
+
poke();
|
|
200
|
+
});
|
|
201
|
+
stream.on("end", () => {
|
|
202
|
+
ended = true;
|
|
203
|
+
poke();
|
|
204
|
+
});
|
|
205
|
+
stream.on("error", () => {
|
|
206
|
+
ended = true;
|
|
207
|
+
poke();
|
|
208
|
+
});
|
|
209
|
+
const wait = () => new Promise((r) => (wake = r));
|
|
210
|
+
const take = (n) => {
|
|
211
|
+
const all = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks, length);
|
|
212
|
+
const out = all.subarray(0, n);
|
|
213
|
+
const rest = all.subarray(n);
|
|
214
|
+
chunks.length = 0;
|
|
215
|
+
length = rest.length;
|
|
216
|
+
if (rest.length) chunks.push(rest);
|
|
217
|
+
return out;
|
|
218
|
+
};
|
|
219
|
+
return {
|
|
220
|
+
async line() {
|
|
221
|
+
for (;;) {
|
|
222
|
+
if (chunks.length > 1) {
|
|
223
|
+
const all = Buffer.concat(chunks, length);
|
|
224
|
+
chunks.length = 0;
|
|
225
|
+
chunks.push(all);
|
|
226
|
+
}
|
|
227
|
+
const nl = length ? chunks[0].indexOf(0x0a) : -1;
|
|
228
|
+
if (nl !== -1) return take(nl + 1).subarray(0, nl).toString("latin1");
|
|
229
|
+
if (ended) throw new Error("eof");
|
|
230
|
+
await wait();
|
|
231
|
+
}
|
|
232
|
+
},
|
|
233
|
+
async bytes(n) {
|
|
234
|
+
while (length < n) {
|
|
235
|
+
if (ended) throw new Error("eof");
|
|
236
|
+
await wait();
|
|
237
|
+
}
|
|
238
|
+
return Buffer.from(take(n));
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* @param {object} deps
|
|
245
|
+
* @param {string} deps.storeDir the checkpoint root on the worker host (made 0700)
|
|
246
|
+
* @param {Function} [deps.spawnGit] `(argv, { env, cwd, signal }) => ChildProcess`
|
|
247
|
+
* @param {object} [deps.fs] a `node:fs/promises`-shaped object (the crash-injection seam)
|
|
248
|
+
* @param {() => number} [deps.now] milliseconds; the lock's staleness and the sweep's TTL read it
|
|
249
|
+
* @param {Function} [deps.log] `(event, fields)`; fields are names, counts and codes, never a path or value
|
|
250
|
+
* @param {object} [deps.limits] overrides of `DEFAULT_CHECKPOINT_LIMITS`
|
|
251
|
+
* @param {string} [deps.hostName] recorded in the manifest
|
|
252
|
+
* @param {Function} [deps.statfs] `(dir) => { bavail, bsize, blocks }`
|
|
253
|
+
* @param {string|null} [deps.procFdDir] where the open-file links live (Linux); a test points it at a fake
|
|
254
|
+
* @param {number[]} [deps.manifestReads] the manifest versions `readManifest` reads (`CHECKPOINT_MANIFEST_READS`); a
|
|
255
|
+
* seam for a test of the version policy with a two-version table, never set by a wiring
|
|
256
|
+
*/
|
|
257
|
+
export function makeWorkspaceCheckpointer({
|
|
258
|
+
storeDir,
|
|
259
|
+
spawnGit = defaultSpawnGit,
|
|
260
|
+
fs = nodeFsp,
|
|
261
|
+
now = Date.now,
|
|
262
|
+
log = () => {},
|
|
263
|
+
limits = {},
|
|
264
|
+
// Fork archives retain bounded entry-addressable history; crash recovery still keeps only two.
|
|
265
|
+
keepHistory = false,
|
|
266
|
+
hostName = osHostname(),
|
|
267
|
+
statfs = (dir) => nodeFsp.statfs(dir),
|
|
268
|
+
procFdDir = "/proc/self/fd",
|
|
269
|
+
manifestReads = CHECKPOINT_MANIFEST_READS,
|
|
270
|
+
} = {}) {
|
|
271
|
+
const L = { ...DEFAULT_CHECKPOINT_LIMITS, ...limits };
|
|
272
|
+
const manifestVersions = new Set(manifestReads);
|
|
273
|
+
// A version above every one this build reads is a newer release's; one below the oldest (or between two it reads) is
|
|
274
|
+
// a version this build stopped reading, which is a manifest it cannot use, not one to wait for.
|
|
275
|
+
const newestManifest = Math.max(...manifestVersions);
|
|
276
|
+
const host = HOST_RE.test(String(hostName)) ? String(hostName) : String(hostName).replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 64) || "unknown";
|
|
277
|
+
/** Per run, in memory: the stat cache and the object format. Lost on restart, which only costs re-reading. */
|
|
278
|
+
const runs = new Map();
|
|
279
|
+
|
|
280
|
+
function runState(runId) {
|
|
281
|
+
let s = runs.get(runId);
|
|
282
|
+
if (!s) {
|
|
283
|
+
s = { cache: new Map(), objectFormat: null };
|
|
284
|
+
runs.set(runId, s);
|
|
285
|
+
}
|
|
286
|
+
return s;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function runPaths(runId) {
|
|
290
|
+
const run = join(storeDir, checkpointKey(runId));
|
|
291
|
+
return {
|
|
292
|
+
run,
|
|
293
|
+
repo: join(run, "repo.git"),
|
|
294
|
+
home: join(run, "home"),
|
|
295
|
+
lock: join(run, "lock"),
|
|
296
|
+
current: join(run, "CURRENT"),
|
|
297
|
+
tooLarge: join(run, "base-too-large"),
|
|
298
|
+
cp: (seq) => join(run, `cp-${seq}`),
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// ── git ────────────────────────────────────────────────────────────────────────────────────────────
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Start one store git. EVERY spawn goes through here, which is what lets one test pin that every argv
|
|
306
|
+
* starts with `STORE_GIT_FLAGS` and every env is `storeGitEnv` exactly. `workTree` exists for one
|
|
307
|
+
* caller (`dropIgnored`) and is the only way `GIT_WORK_TREE` can reach a git.
|
|
308
|
+
*/
|
|
309
|
+
function startGit(paths, args, { signal, workTree = null, cwd = paths.run } = {}) {
|
|
310
|
+
const env = storeGitEnv({ gitDir: paths.repo, home: paths.home });
|
|
311
|
+
// env-internal GIT_WORK_TREE: written into the store git's closed env here, never read from the worker's environment.
|
|
312
|
+
if (workTree) env.GIT_WORK_TREE = workTree;
|
|
313
|
+
const timeout = AbortSignal.timeout(L.gitTimeoutMs);
|
|
314
|
+
const sig = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
315
|
+
const child = spawnGit([...STORE_GIT_FLAGS, ...args], { env, cwd, signal: sig });
|
|
316
|
+
const out = [];
|
|
317
|
+
const err = [];
|
|
318
|
+
let errBytes = 0;
|
|
319
|
+
child.stdout?.on("data", (c) => out.push(c));
|
|
320
|
+
child.stderr?.on("data", (c) => {
|
|
321
|
+
// Kept for a test's eyes only and bounded: stderr can name agent paths, so it is never logged.
|
|
322
|
+
if (errBytes < 65536) err.push(c);
|
|
323
|
+
errBytes += c.length;
|
|
324
|
+
});
|
|
325
|
+
child.stdin?.on("error", () => {});
|
|
326
|
+
const done = new Promise((resolve) => {
|
|
327
|
+
let settled = false;
|
|
328
|
+
const settle = (code) => {
|
|
329
|
+
if (settled) return;
|
|
330
|
+
settled = true;
|
|
331
|
+
resolve({ code, stdout: Buffer.concat(out), stderr: Buffer.concat(err).toString("utf8"), aborted: sig.aborted });
|
|
332
|
+
};
|
|
333
|
+
child.on("error", () => {
|
|
334
|
+
// A spawn that never started emits no `close`; one that did is settled by `close` below.
|
|
335
|
+
if (child.pid === undefined) settle(-1);
|
|
336
|
+
});
|
|
337
|
+
child.on("close", (code) => settle(code ?? -1));
|
|
338
|
+
});
|
|
339
|
+
return { child, done };
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/** Run a store git to completion. Resolves, never rejects. */
|
|
343
|
+
async function runGit(paths, args, { input = null, ...opts } = {}) {
|
|
344
|
+
let started;
|
|
345
|
+
try {
|
|
346
|
+
started = startGit(paths, args, opts);
|
|
347
|
+
} catch {
|
|
348
|
+
return { code: -1, stdout: Buffer.alloc(0), stderr: "", aborted: false };
|
|
349
|
+
}
|
|
350
|
+
started.child.stdin?.end(input ?? undefined);
|
|
351
|
+
return started.done;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// ── small fs helpers ────────────────────────────────────────────────────────────────────────────────
|
|
355
|
+
|
|
356
|
+
const rand = () => randomBytes(6).toString("hex");
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Read a small REGULAR file without following a link and without blocking on a FIFO: `O_NOFOLLOW`
|
|
360
|
+
* refuses a final-component link, `O_NONBLOCK` makes a FIFO open return at once, and the fstat then
|
|
361
|
+
* refuses anything that is not a regular file of at most `max` bytes.
|
|
362
|
+
*/
|
|
363
|
+
async function readRegular(path, max) {
|
|
364
|
+
let fh;
|
|
365
|
+
try {
|
|
366
|
+
fh = await fs.open(path, FS.O_RDONLY | FS.O_NOFOLLOW | FS.O_NONBLOCK);
|
|
367
|
+
} catch (err) {
|
|
368
|
+
return { error: err?.code ?? "open" };
|
|
369
|
+
}
|
|
370
|
+
try {
|
|
371
|
+
const st = await fh.stat();
|
|
372
|
+
if (!st.isFile()) return { error: "not-regular" };
|
|
373
|
+
if (st.size > max) return { error: "too-large" };
|
|
374
|
+
const buf = Buffer.alloc(st.size);
|
|
375
|
+
let off = 0;
|
|
376
|
+
while (off < st.size) {
|
|
377
|
+
const { bytesRead } = await fh.read(buf, off, st.size - off, off);
|
|
378
|
+
if (bytesRead === 0) break;
|
|
379
|
+
off += bytesRead;
|
|
380
|
+
}
|
|
381
|
+
return { buf: buf.subarray(0, off) };
|
|
382
|
+
} catch (err) {
|
|
383
|
+
return { error: err?.code ?? "read" };
|
|
384
|
+
} finally {
|
|
385
|
+
await fh.close().catch(() => {});
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
async function lstatOrNull(path, opts) {
|
|
390
|
+
try {
|
|
391
|
+
return await fs.lstat(path, opts);
|
|
392
|
+
} catch (err) {
|
|
393
|
+
if (err?.code === "ENOENT") return null;
|
|
394
|
+
throw err;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** Make a directory ours (0700), or refuse one that exists as anything but a real directory. */
|
|
399
|
+
async function ensureDir(path) {
|
|
400
|
+
try {
|
|
401
|
+
await fs.mkdir(path, { mode: 0o700 });
|
|
402
|
+
} catch (err) {
|
|
403
|
+
if (err?.code !== "EEXIST") throw err;
|
|
404
|
+
}
|
|
405
|
+
const st = await fs.lstat(path);
|
|
406
|
+
if (!st.isDirectory()) throw Object.assign(new Error("not a directory"), { code: "ENOTDIR" });
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
async function fsyncDir(dir) {
|
|
410
|
+
const fh = await fs.open(dir, "r");
|
|
411
|
+
try {
|
|
412
|
+
await fh.sync();
|
|
413
|
+
} finally {
|
|
414
|
+
await fh.close().catch(() => {});
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Write `name` in `dir` the crash-safe way: a fresh `<name>.<rand>.incoming` (exclusive create, so a
|
|
420
|
+
* planted link at the temp name cannot redirect the write), fsync, rename over, fsync the directory.
|
|
421
|
+
* A crash before the rename leaves the old file; after it, the new one, complete.
|
|
422
|
+
*/
|
|
423
|
+
async function atomicWrite(dir, name, data, mode = 0o600) {
|
|
424
|
+
const tmp = join(dir, `${name}.${rand()}.incoming`);
|
|
425
|
+
try {
|
|
426
|
+
const fh = await fs.open(tmp, "wx", mode);
|
|
427
|
+
try {
|
|
428
|
+
await fh.writeFile(data);
|
|
429
|
+
await fh.sync();
|
|
430
|
+
} finally {
|
|
431
|
+
await fh.close().catch(() => {});
|
|
432
|
+
}
|
|
433
|
+
await fs.rename(tmp, join(dir, name));
|
|
434
|
+
} catch (err) {
|
|
435
|
+
await fs.unlink(tmp).catch(() => {});
|
|
436
|
+
throw err;
|
|
437
|
+
}
|
|
438
|
+
await fsyncDir(dir);
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// ── the lock ────────────────────────────────────────────────────────────────────────────────────────
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Exclusive create IS the lock, as in `session-store.mjs`'s `takeLock` (mirrored, not imported: that
|
|
445
|
+
* module is sync and keyed differently). The difference is the takeover. A stale lock (older than
|
|
446
|
+
* `lockStaleMs` by the injected clock) is taken over by RENAMING it to a unique name, which is atomic,
|
|
447
|
+
* so of two processes judging the same stale lock exactly one moves it. The mover then checks that what
|
|
448
|
+
* it moved still holds the bytes it judged stale: a rival may have taken the lock over in between and
|
|
449
|
+
* written a NEW tag, and that live lock is put back (`link`, which never overwrites) rather than
|
|
450
|
+
* removed. Only then is the moved file unlinked and a fresh lock created with `wx`. An unlink by path,
|
|
451
|
+
* the obvious version, can delete a rival's brand new lock on the strength of a judgement about its
|
|
452
|
+
* predecessor.
|
|
453
|
+
*/
|
|
454
|
+
async function takeLock(paths) {
|
|
455
|
+
const tag = randomBytes(16).toString("hex");
|
|
456
|
+
const body = `${process.pid} ${tag}\n`;
|
|
457
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
458
|
+
try {
|
|
459
|
+
const fh = await fs.open(paths.lock, "wx", 0o600);
|
|
460
|
+
try {
|
|
461
|
+
await fh.writeFile(body);
|
|
462
|
+
} finally {
|
|
463
|
+
await fh.close().catch(() => {});
|
|
464
|
+
}
|
|
465
|
+
return { tag };
|
|
466
|
+
} catch (err) {
|
|
467
|
+
if (err?.code !== "EEXIST") return { error: err?.code ?? "lock" };
|
|
468
|
+
const seen = await readRegular(paths.lock, 256);
|
|
469
|
+
const st = await lstatOrNull(paths.lock).catch(() => null);
|
|
470
|
+
if (!seen.buf || !st) continue; // released between our create and our read: retry answers
|
|
471
|
+
const ageMs = now() - st.mtimeMs;
|
|
472
|
+
if (ageMs <= L.lockStaleMs) return { locked: true };
|
|
473
|
+
const moved = await moveAside(paths.lock, "stale");
|
|
474
|
+
if (!moved) continue; // someone else moved it first; the retry create answers who won
|
|
475
|
+
const held = await readRegular(moved, 256);
|
|
476
|
+
if (!held.buf || !held.buf.equals(seen.buf)) {
|
|
477
|
+
await putBack(moved, paths.lock);
|
|
478
|
+
return { locked: true };
|
|
479
|
+
}
|
|
480
|
+
await fs.unlink(moved).catch(() => {});
|
|
481
|
+
log("checkpoint_lock_stale_taken", { ageMs: Math.round(ageMs) });
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return { locked: true };
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** Rename `path` to a unique sibling; the new name, or null when it was already gone. */
|
|
488
|
+
async function moveAside(path, why) {
|
|
489
|
+
const to = `${path}.${why}.${rand()}`;
|
|
490
|
+
try {
|
|
491
|
+
await fs.rename(path, to);
|
|
492
|
+
return to;
|
|
493
|
+
} catch {
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/** Return a lock we moved but do not own. `link` fails if a new lock appeared meanwhile, which then wins. */
|
|
499
|
+
async function putBack(moved, path) {
|
|
500
|
+
await fs.link(moved, path).catch(() => {});
|
|
501
|
+
await fs.unlink(moved).catch(() => {});
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/** Release only a lock that still holds OUR tag, by the same move, verify, then unlink. */
|
|
505
|
+
async function releaseLock(paths, tag) {
|
|
506
|
+
const moved = await moveAside(paths.lock, "release");
|
|
507
|
+
if (!moved) return;
|
|
508
|
+
const held = await readRegular(moved, 256);
|
|
509
|
+
if (held.buf && held.buf.toString("latin1").includes(tag)) await fs.unlink(moved).catch(() => {});
|
|
510
|
+
else await putBack(moved, paths.lock);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
async function withLock(paths, fn) {
|
|
514
|
+
const lock = await takeLock(paths);
|
|
515
|
+
if (lock.locked) return { skipped: "locked" };
|
|
516
|
+
if (lock.error) return { skipped: "lock-failed" };
|
|
517
|
+
try {
|
|
518
|
+
return await fn();
|
|
519
|
+
} finally {
|
|
520
|
+
await releaseLock(paths, lock.tag).catch(() => {});
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// ── the walk ────────────────────────────────────────────────────────────────────────────────────────
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* Walk `root` without following anything. Returns the regular files and symlinks worth storing, with
|
|
528
|
+
* the counts of what was not, or `{ skipped }` when the tree is not one we can describe.
|
|
529
|
+
*
|
|
530
|
+
* What is excluded, and why it is COUNTED rather than silent: a dotgit segment at ANY depth (a nested
|
|
531
|
+
* repository's files are captured, its `.git` is not, and `sub/.GIT` is `.git` on a Mac); FIFOs,
|
|
532
|
+
* sockets and devices (no content to store, and opening a FIFO blocks); a name `validSegment` refuses;
|
|
533
|
+
* a directory or file we may not read (EACCES/EPERM, `partial`, and never chmod'ed: changing modes in
|
|
534
|
+
* the agent's tree is a write to it); a file over `maxFileBytes`. The counts reach the manifest so a
|
|
535
|
+
* restore that is missing something says how much.
|
|
536
|
+
*/
|
|
537
|
+
async function walkTree(root, signal, maxFiles = L.maxFiles) {
|
|
538
|
+
const rootSt = await lstatOrNull(root);
|
|
539
|
+
if (!rootSt?.isDirectory()) return { skipped: "workspace-not-directory" };
|
|
540
|
+
const files = [];
|
|
541
|
+
const excluded = { unreadable: 0, special: 0, dotgit: 0, badName: 0, tooLarge: 0 };
|
|
542
|
+
let partial = false;
|
|
543
|
+
let unsafeIgnoreFile = false;
|
|
544
|
+
const stack = [{ rel: "", depth: 0 }];
|
|
545
|
+
while (stack.length) {
|
|
546
|
+
if (signal?.aborted) return { skipped: "timeout" };
|
|
547
|
+
const { rel, depth } = stack.pop();
|
|
548
|
+
let dirents;
|
|
549
|
+
try {
|
|
550
|
+
dirents = await fs.readdir(rel ? join(root, rel) : root, { withFileTypes: true, encoding: "buffer" });
|
|
551
|
+
} catch (err) {
|
|
552
|
+
if (err?.code === "EACCES" || err?.code === "EPERM") {
|
|
553
|
+
excluded.unreadable++;
|
|
554
|
+
partial = true;
|
|
555
|
+
continue;
|
|
556
|
+
}
|
|
557
|
+
// The container is paused: a directory that vanished or turned into a file under us means
|
|
558
|
+
// something other than the agent is writing, and a snapshot of a moving tree is not one.
|
|
559
|
+
if (err?.code === "ENOENT" || err?.code === "ENOTDIR") return { skipped: "tree-changed" };
|
|
560
|
+
throw err;
|
|
561
|
+
}
|
|
562
|
+
for (const d of dirents) {
|
|
563
|
+
// Per ENTRY, not per directory: one directory of 200,000 names would otherwise run past the
|
|
564
|
+
// deadline and grow `files` without bound before either check got a turn.
|
|
565
|
+
if (signal?.aborted) return { skipped: "timeout" };
|
|
566
|
+
const nameBuf = d.name;
|
|
567
|
+
if (isDotGitSegment(nameBuf)) {
|
|
568
|
+
excluded.dotgit++;
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
if (!validSegment(nameBuf)) {
|
|
572
|
+
excluded.badName++;
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
const name = nameBuf.toString("utf8");
|
|
576
|
+
const path = rel ? `${rel}/${name}` : name;
|
|
577
|
+
if (Buffer.byteLength(path) > MAX_PATH_BYTES) {
|
|
578
|
+
excluded.badName++;
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
let st;
|
|
582
|
+
try {
|
|
583
|
+
st = await fs.lstat(join(root, path), { bigint: true });
|
|
584
|
+
} catch (err) {
|
|
585
|
+
if (err?.code === "EACCES" || err?.code === "EPERM") {
|
|
586
|
+
excluded.unreadable++;
|
|
587
|
+
partial = true;
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
if (err?.code === "ENOENT") return { skipped: "tree-changed" };
|
|
591
|
+
throw err;
|
|
592
|
+
}
|
|
593
|
+
if (st.isDirectory()) {
|
|
594
|
+
// Children of a directory at MAX_DEPTH would be deeper than any path `validPath` accepts,
|
|
595
|
+
// so the restore could never write them: counted as bad names, not silently lost.
|
|
596
|
+
if (depth + 1 >= MAX_DEPTH) excluded.badName++;
|
|
597
|
+
else stack.push({ rel: path, depth: depth + 1 });
|
|
598
|
+
} else if (st.isSymbolicLink()) {
|
|
599
|
+
files.push({ path, kind: "link", st });
|
|
600
|
+
if (files.length > maxFiles) return { skipped: "over-file-cap" };
|
|
601
|
+
} else if (st.isFile()) {
|
|
602
|
+
if (st.size > BigInt(L.maxFileBytes)) {
|
|
603
|
+
excluded.tooLarge++;
|
|
604
|
+
continue;
|
|
605
|
+
}
|
|
606
|
+
files.push({ path, kind: "file", st });
|
|
607
|
+
if (files.length > maxFiles) return { skipped: "over-file-cap" };
|
|
608
|
+
} else {
|
|
609
|
+
excluded.special++;
|
|
610
|
+
// git opens `.gitignore` with a plain blocking open: a FIFO there would hang `check-ignore`
|
|
611
|
+
// until its deadline. Remembered so the ignore tier is never attempted on such a tree.
|
|
612
|
+
if (name === ".gitignore") unsafeIgnoreFile = true;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
617
|
+
return { files, excluded, partial, unsafeIgnoreFile };
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* Read exactly the file the walk saw: `O_NOFOLLOW` (a link swapped in since the lstat is refused),
|
|
622
|
+
* `O_NONBLOCK` (a FIFO swapped in does not hang the open), then the fstat must show a regular file with
|
|
623
|
+
* the walk's dev and inode, and exactly `size` bytes must arrive. Anything else means the tree moved
|
|
624
|
+
* under a paused container, and the whole snapshot is abandoned rather than recording a mix.
|
|
625
|
+
*/
|
|
626
|
+
async function readExact(abs, st, rootReal = null) {
|
|
627
|
+
let fh;
|
|
628
|
+
try {
|
|
629
|
+
fh = await fs.open(abs, FS.O_RDONLY | FS.O_NOFOLLOW | FS.O_NONBLOCK);
|
|
630
|
+
} catch (err) {
|
|
631
|
+
if (err?.code === "EACCES" || err?.code === "EPERM") return { unreadable: true };
|
|
632
|
+
return { changed: true };
|
|
633
|
+
}
|
|
634
|
+
try {
|
|
635
|
+
const now2 = await fh.stat({ bigint: true });
|
|
636
|
+
if (!now2.isFile() || now2.dev !== st.dev || now2.ino !== st.ino) return { changed: true };
|
|
637
|
+
if (rootReal && !(await openedInside(fh, rootReal))) return { changed: true };
|
|
638
|
+
const size = Number(st.size);
|
|
639
|
+
const buf = Buffer.allocUnsafe(size);
|
|
640
|
+
let off = 0;
|
|
641
|
+
while (off < size) {
|
|
642
|
+
const { bytesRead } = await fh.read(buf, off, size - off, off);
|
|
643
|
+
if (bytesRead === 0) break;
|
|
644
|
+
off += bytesRead;
|
|
645
|
+
}
|
|
646
|
+
if (off !== size) return { changed: true };
|
|
647
|
+
return { buf };
|
|
648
|
+
} catch {
|
|
649
|
+
return { changed: true };
|
|
650
|
+
} finally {
|
|
651
|
+
await fh.close().catch(() => {});
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Linux only: where did this open REALLY land? `/proc/self/fd/<fd>` names the opened file's path as the
|
|
657
|
+
* kernel resolved it, through every intermediate link `O_NOFOLLOW` does not cover. Outside the
|
|
658
|
+
* workspace's realpath is a swap under us. Where the proc link does not exist the answer is "unknown",
|
|
659
|
+
* which passes: the header records that residual and the pause precondition that covers it.
|
|
660
|
+
*/
|
|
661
|
+
let procFd = procFdDir;
|
|
662
|
+
async function openedInside(fh, rootReal) {
|
|
663
|
+
if (!procFd) return true;
|
|
664
|
+
let where;
|
|
665
|
+
try {
|
|
666
|
+
where = await fs.readlink(`${procFd}/${fh.fd}`);
|
|
667
|
+
} catch (err) {
|
|
668
|
+
if (err?.code === "ENOENT" || err?.code === "ENOTDIR") {
|
|
669
|
+
procFd = null; // no procfs on this host: stop asking
|
|
670
|
+
return true;
|
|
671
|
+
}
|
|
672
|
+
return false;
|
|
673
|
+
}
|
|
674
|
+
return String(where).startsWith(`${rootReal}/`);
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
async function readLinkTarget(abs) {
|
|
678
|
+
try {
|
|
679
|
+
const target = await fs.readlink(abs, { encoding: "buffer" });
|
|
680
|
+
if (target.length === 0 || target.length > MAX_LINK_BYTES) return { tooLarge: true };
|
|
681
|
+
return { buf: target };
|
|
682
|
+
} catch (err) {
|
|
683
|
+
if (err?.code === "EACCES" || err?.code === "EPERM") return { unreadable: true };
|
|
684
|
+
return { changed: true };
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
function cacheMatches(hit, st) {
|
|
689
|
+
return hit && hit.dev === st.dev && hit.ino === st.ino && hit.size === st.size && hit.mtimeNs === st.mtimeNs && hit.ctimeNs === st.ctimeNs;
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
function cacheEntry(st, oid) {
|
|
693
|
+
return { dev: st.dev, ino: st.ino, size: st.size, mtimeNs: st.mtimeNs, ctimeNs: st.ctimeNs, oid };
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// ── the agent's git state, from files ──────────────────────────────────────────────────────────────
|
|
697
|
+
|
|
698
|
+
const NO_GIT = Object.freeze({ state: "none", branch: null, tip: null, detached: false });
|
|
699
|
+
const UNREADABLE_GIT = Object.freeze({ state: "unreadable", branch: null, tip: null, detached: false });
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Where the agent's repository points, parsed from files and never from a git command (rule 1). Only
|
|
703
|
+
* the CALLER's branch (the harness-minted issue branch) is ever looked up: a name read from HEAD is
|
|
704
|
+
* agent-chosen, so it is compared, never used as a path and never recorded. A `.git` that is a file
|
|
705
|
+
* (a `gitdir:` redirect) or a link is `unreadable`: following it is reading another repository.
|
|
706
|
+
*/
|
|
707
|
+
async function parseGitState(workspace, branch) {
|
|
708
|
+
const gitDir = join(workspace, ".git");
|
|
709
|
+
let st;
|
|
710
|
+
try {
|
|
711
|
+
st = await fs.lstat(gitDir);
|
|
712
|
+
} catch (err) {
|
|
713
|
+
return err?.code === "ENOENT" ? NO_GIT : UNREADABLE_GIT;
|
|
714
|
+
}
|
|
715
|
+
if (!st.isDirectory()) return UNREADABLE_GIT;
|
|
716
|
+
const head = await readRegular(join(gitDir, "HEAD"), 256);
|
|
717
|
+
if (!head.buf) return UNREADABLE_GIT;
|
|
718
|
+
const text = head.buf.toString("latin1");
|
|
719
|
+
const detached = /^([0-9a-f]{40}|[0-9a-f]{64})\n?$/.exec(text);
|
|
720
|
+
if (detached) return { state: "parsed", branch: null, tip: detached[1], detached: true };
|
|
721
|
+
const sym = /^ref: refs\/heads\/([^\n]+)\n?$/.exec(text);
|
|
722
|
+
if (!sym) return UNREADABLE_GIT;
|
|
723
|
+
if (!isBranchName(branch) || sym[1] !== branch) return { state: "parsed", branch: null, tip: null, detached: false };
|
|
724
|
+
return { state: "parsed", branch, tip: await lookupBranchTip(gitDir, branch), detached: false };
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
async function lookupBranchTip(gitDir, branch) {
|
|
728
|
+
const parts = ["refs", "heads", ...branch.split("/")];
|
|
729
|
+
let dir = gitDir;
|
|
730
|
+
let loose = true;
|
|
731
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
732
|
+
dir = join(dir, parts[i]);
|
|
733
|
+
const st = await lstatOrNull(dir).catch(() => null);
|
|
734
|
+
if (!st?.isDirectory()) {
|
|
735
|
+
loose = false;
|
|
736
|
+
break;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
if (loose) {
|
|
740
|
+
const ref = await readRegular(join(dir, parts.at(-1)), 256);
|
|
741
|
+
const m = ref.buf && /^([0-9a-f]{40}|[0-9a-f]{64})\n?$/.exec(ref.buf.toString("latin1"));
|
|
742
|
+
if (m) return m[1];
|
|
743
|
+
}
|
|
744
|
+
const packed = await readRegular(join(gitDir, "packed-refs"), MiB);
|
|
745
|
+
if (!packed.buf) return null;
|
|
746
|
+
const want = ` refs/heads/${branch}`;
|
|
747
|
+
for (const line of packed.buf.toString("latin1").split("\n")) {
|
|
748
|
+
if (line.endsWith(want)) {
|
|
749
|
+
const oid = line.slice(0, line.length - want.length);
|
|
750
|
+
if (ANY_HEX_OID.test(oid)) return oid;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
return null;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
// ── store setup ─────────────────────────────────────────────────────────────────────────────────────
|
|
757
|
+
|
|
758
|
+
async function ensureRun(paths) {
|
|
759
|
+
await ensureDir(storeDir);
|
|
760
|
+
await fs.chmod(storeDir, 0o700).catch(() => {});
|
|
761
|
+
await ensureDir(paths.run);
|
|
762
|
+
await ensureDir(paths.home);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
/**
|
|
766
|
+
* `git init --bare` once, with the config appended once. `uploadpack.allowAnySHA1InWant` lets the
|
|
767
|
+
* caller's restore fetch the base by its sha (`baseSourceUrl`) whatever refs the store advertises; the
|
|
768
|
+
* store is ours and holds only this run's objects, so "any sha" is no wider than "this repository".
|
|
769
|
+
*/
|
|
770
|
+
async function initStore(paths, objectFormat, signal) {
|
|
771
|
+
if (await lstatOrNull(join(paths.repo, "HEAD"))) return true;
|
|
772
|
+
const r = await runGit(paths, ["init", "--bare", "--quiet", `--object-format=${objectFormat}`], { signal });
|
|
773
|
+
if (r.code !== 0) return false;
|
|
774
|
+
await fs.appendFile(
|
|
775
|
+
join(paths.repo, "config"),
|
|
776
|
+
"[core]\n\tlogAllRefUpdates = false\n[gc]\n\tauto = 0\n[uploadpack]\n\tallowAnySHA1InWant = true\n",
|
|
777
|
+
);
|
|
778
|
+
return true;
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
async function storeObjectFormat(paths, state, signal) {
|
|
782
|
+
if (state.objectFormat) return state.objectFormat;
|
|
783
|
+
const r = await runGit(paths, ["rev-parse", "--show-object-format"], { signal });
|
|
784
|
+
const fmt = r.stdout.toString("latin1").trim();
|
|
785
|
+
if (r.code !== 0 || !HEX[fmt]) return null;
|
|
786
|
+
state.objectFormat = fmt;
|
|
787
|
+
return fmt;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
async function storeBytes(dir = storeDir) {
|
|
791
|
+
let total = 0;
|
|
792
|
+
const stack = [dir];
|
|
793
|
+
while (stack.length) {
|
|
794
|
+
const d = stack.pop();
|
|
795
|
+
let names;
|
|
796
|
+
try {
|
|
797
|
+
names = await fs.readdir(d);
|
|
798
|
+
} catch {
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
801
|
+
for (const n of names) {
|
|
802
|
+
const st = await lstatOrNull(join(d, n)).catch(() => null);
|
|
803
|
+
if (!st) continue;
|
|
804
|
+
if (st.isDirectory()) stack.push(join(d, n));
|
|
805
|
+
else total += st.size;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
return total;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
async function readCurrent(paths) {
|
|
812
|
+
const cur = await readRegular(paths.current, 32);
|
|
813
|
+
if (keepHistory && retryableStoreError(cur.error)) return { unavailable: cur.error };
|
|
814
|
+
if (cur.error === "ENOENT") return { absent: true };
|
|
815
|
+
if (!cur.buf) return { invalid: "current-unreadable" };
|
|
816
|
+
const m = /^([1-9][0-9]{0,8})\n$/.exec(cur.buf.toString("latin1"));
|
|
817
|
+
if (!m) return { invalid: "current-malformed" };
|
|
818
|
+
return { seq: Number(m[1]) };
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// ── primeBase ───────────────────────────────────────────────────────────────────────────────────────
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* Seed the store from the FRESH clone, before the container starts: the one moment the workspace's
|
|
825
|
+
* `.git` is still the worker's own, so the `fetch` from it reads a repository nobody hostile has
|
|
826
|
+
* touched (and with the store flags anyway). The stat cache it seeds is what makes a later snapshot
|
|
827
|
+
* cheap: a file whose dev, inode, size, mtime and ctime all match is not re-read.
|
|
828
|
+
*
|
|
829
|
+
* Only oids that exist in the base TREE are cached. The checkout may have converted a file on its way
|
|
830
|
+
* out (eol, ident, an encoding): its on-disk id is then not in the store, and fast-import refuses a
|
|
831
|
+
* `M` naming a missing object. Hashing the disk bytes and keeping only ids the base tree holds is
|
|
832
|
+
* exact either way.
|
|
833
|
+
*/
|
|
834
|
+
async function primeBase({ runId, workspace, baseSha, objectFormat } = {}) {
|
|
835
|
+
try {
|
|
836
|
+
if (typeof runId !== "string" || !runId || !HEX[objectFormat] || !HEX[objectFormat].test(String(baseSha))) {
|
|
837
|
+
return { skipped: "bad-args" };
|
|
838
|
+
}
|
|
839
|
+
const paths = runPaths(runId);
|
|
840
|
+
await ensureRun(paths);
|
|
841
|
+
return await withLock(paths, async () => {
|
|
842
|
+
const signal = AbortSignal.timeout(L.snapshotTimeoutMs);
|
|
843
|
+
const walk = await walkTree(workspace, signal);
|
|
844
|
+
if (walk.skipped) return { skipped: walk.skipped };
|
|
845
|
+
const rootReal = await fs.realpath(workspace);
|
|
846
|
+
const baseBytes = walk.files.reduce((n, f) => n + Number(f.st.size), 0);
|
|
847
|
+
if (baseBytes > L.maxBaseBytes) {
|
|
848
|
+
await atomicWrite(paths.run, "base-too-large", "1\n");
|
|
849
|
+
log("checkpoint_base_skipped", { reason: "base-too-large", baseBytes });
|
|
850
|
+
return { skipped: "base-too-large" };
|
|
851
|
+
}
|
|
852
|
+
if (!(await initStore(paths, objectFormat, signal))) return { skipped: "store-init-failed" };
|
|
853
|
+
const url = pathToFileURL(join(workspace, ".git")).href;
|
|
854
|
+
const fetched = await runGit(
|
|
855
|
+
paths,
|
|
856
|
+
["-c", "protocol.file.allow=always", "fetch", "--quiet", "--depth=1", "--no-tags", url, `+${baseSha}:refs/pi-dispatch/base`],
|
|
857
|
+
{ signal },
|
|
858
|
+
);
|
|
859
|
+
if (fetched.code !== 0) return { skipped: fetched.aborted ? "timeout" : "base-fetch-failed" };
|
|
860
|
+
const tree = await runGit(paths, ["ls-tree", "-r", "-z", "--full-tree", "refs/pi-dispatch/base"], { signal });
|
|
861
|
+
if (tree.code !== 0) return { skipped: "base-fetch-failed" };
|
|
862
|
+
const baseOids = new Set();
|
|
863
|
+
let baseHasExtensions = false;
|
|
864
|
+
for (const rec of tree.stdout.toString("latin1").split("\0")) {
|
|
865
|
+
const m = /^\d+ blob ([0-9a-f]+)\t/.exec(rec);
|
|
866
|
+
if (m) baseOids.add(m[1]);
|
|
867
|
+
if (/^\d+ (?:blob|commit) [0-9a-f]+\t\.pi\/extensions(?:\/|$)/i.test(rec)) baseHasExtensions = true;
|
|
868
|
+
}
|
|
869
|
+
const state = runState(runId);
|
|
870
|
+
state.objectFormat = objectFormat;
|
|
871
|
+
state.cache.clear();
|
|
872
|
+
for (const f of walk.files) {
|
|
873
|
+
if (signal.aborted) return { skipped: "timeout" };
|
|
874
|
+
if (f.kind !== "file") continue;
|
|
875
|
+
const got = await readExact(join(workspace, f.path), f.st, rootReal);
|
|
876
|
+
if (!got.buf) continue;
|
|
877
|
+
const oid = blobOid(objectFormat, got.buf);
|
|
878
|
+
if (baseOids.has(oid)) state.cache.set(f.path, cacheEntry(f.st, oid));
|
|
879
|
+
}
|
|
880
|
+
log("checkpoint_base_primed", { files: walk.files.length, cached: state.cache.size });
|
|
881
|
+
return { ok: true, files: walk.files.length, cached: state.cache.size, baseHasExtensions };
|
|
882
|
+
});
|
|
883
|
+
} catch (err) {
|
|
884
|
+
log("checkpoint_failed", { phase: "prime", code: err?.code ?? "internal" });
|
|
885
|
+
return { skipped: "internal-error" };
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// ── snapshot ────────────────────────────────────────────────────────────────────────────────────────
|
|
890
|
+
|
|
891
|
+
/**
|
|
892
|
+
* Tier 2 of the byte cap: drop every path the repository's ignore rules match (build output, caches,
|
|
893
|
+
* `node_modules`), which is usually what blew the cap.
|
|
894
|
+
*
|
|
895
|
+
* THE ONE GIT GIVEN THE WORK TREE, and why that is safe. `check-ignore --no-index` reads exactly the
|
|
896
|
+
* ignore FILES (`.gitignore` along each path, the store's `info/exclude`, and a global excludes file
|
|
897
|
+
* that cannot exist under `storeGitEnv`'s empty HOME): it runs no filter, reads no attributes, never
|
|
898
|
+
* touches an index (`--no-index`), and `GIT_DIR` is still the store, so the agent's `.git/config`,
|
|
899
|
+
* hooks and fsmonitor are never read. A `.gitignore` that is a FIFO would hang it, so the walk refuses
|
|
900
|
+
* this tier for such a tree. The rejected alternative, a Node reimplementation of gitignore matching,
|
|
901
|
+
* is a second dialect of a subtle format: it would drop files git keeps, which is data loss.
|
|
902
|
+
*/
|
|
903
|
+
async function dropIgnored(paths, workspace, files, signal) {
|
|
904
|
+
const input = Buffer.from(files.map((f) => `${f.path}\0`).join(""), "utf8");
|
|
905
|
+
// cwd is the run's empty `home/`, NOT the workspace: git resolves a work tree it is not inside from
|
|
906
|
+
// `GIT_WORK_TREE` alone and takes the paths relative to its top (measured on git 2.39), and a cwd
|
|
907
|
+
// inside the agent's tree is one more place git could look for something to read.
|
|
908
|
+
const r = await runGit(paths, ["check-ignore", "--no-index", "--stdin", "-z"], { input, signal, workTree: workspace, cwd: paths.home });
|
|
909
|
+
// Exit 1 is "nothing ignored", which is an answer, not a failure; anything else keeps nothing back,
|
|
910
|
+
// so the caller refuses over the cap rather than storing everything it was asked to cut.
|
|
911
|
+
if (r.code !== 0 && r.code !== 1) {
|
|
912
|
+
log("checkpoint_ignore_tier", { outcome: "failed" });
|
|
913
|
+
return null;
|
|
914
|
+
}
|
|
915
|
+
const ignored = new Set(r.stdout.toString("utf8").split("\0").filter(Boolean));
|
|
916
|
+
log("checkpoint_ignore_tier", { outcome: r.code === 1 ? "none-ignored" : "dropped", dropped: ignored.size });
|
|
917
|
+
return files.filter((f) => !ignored.has(f.path));
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* Free space must cover twice the byte cap plus the transcript about to be copied, AND 5% of the volume: a checkpoint that fills the disk takes
|
|
922
|
+
* the worker's logs, the queue's AOF and every other job's workspace down with it, which is a far
|
|
923
|
+
* worse outcome than one checkpoint skipped.
|
|
924
|
+
*/
|
|
925
|
+
async function diskOk(transcriptBytes = 0) {
|
|
926
|
+
try {
|
|
927
|
+
const s = await statfs(storeDir);
|
|
928
|
+
const free = Number(s.bavail) * Number(s.bsize);
|
|
929
|
+
const total = Number(s.blocks) * Number(s.bsize);
|
|
930
|
+
return free >= 2 * L.maxNewBytes + transcriptBytes && free >= 0.05 * total;
|
|
931
|
+
} catch {
|
|
932
|
+
return false;
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
/** Stream one checkpoint commit into `git fast-import`. Returns `{ commit, newOids }` or a refusal. */
|
|
937
|
+
async function importTree(paths, ctx) {
|
|
938
|
+
const { workspace, rootReal, files, cachedOid, baseSha, seq, secrets, objectFormat, signal, counts } = ctx;
|
|
939
|
+
const { child, done } = startGit(paths, ["fast-import", "--quiet", "--done", "--force", "--date-format=raw"], { signal });
|
|
940
|
+
let failed = false;
|
|
941
|
+
const closed = done.then(() => {
|
|
942
|
+
failed = true;
|
|
943
|
+
});
|
|
944
|
+
const write = async (buf) => {
|
|
945
|
+
if (failed) throw new Error("fast-import exited");
|
|
946
|
+
// Backpressure: wait for `drain`, but never past the child's death (a dead git never drains).
|
|
947
|
+
if (!child.stdin.write(buf)) await Promise.race([new Promise((r) => child.stdin.once("drain", r)), closed]);
|
|
948
|
+
if (failed) throw new Error("fast-import exited");
|
|
949
|
+
};
|
|
950
|
+
const abort = async (result) => {
|
|
951
|
+
child.kill("SIGKILL");
|
|
952
|
+
await done;
|
|
953
|
+
await cleanupImport(paths);
|
|
954
|
+
return result;
|
|
955
|
+
};
|
|
956
|
+
const newOids = new Map();
|
|
957
|
+
try {
|
|
958
|
+
const msg = `pi-dispatch checkpoint ${seq}\n`;
|
|
959
|
+
const sec = Math.floor(now() / 1000);
|
|
960
|
+
await write(
|
|
961
|
+
`feature done\ncommit refs/pi-dispatch/checkpoint\nmark :1\ncommitter ${COMMITTER} ${sec} +0000\n` +
|
|
962
|
+
`data ${Buffer.byteLength(msg)}\n${msg}from ${baseSha}\ndeleteall\n`,
|
|
963
|
+
);
|
|
964
|
+
for (const f of files) {
|
|
965
|
+
if (signal.aborted) return await abort({ skipped: "timeout" });
|
|
966
|
+
const abs = join(workspace, f.path);
|
|
967
|
+
const q = cQuote(f.path);
|
|
968
|
+
if (f.kind === "link") {
|
|
969
|
+
const t = await readLinkTarget(abs);
|
|
970
|
+
if (t.changed) return await abort({ skipped: "tree-changed" });
|
|
971
|
+
if (t.unreadable) {
|
|
972
|
+
counts.skipped.unreadable++;
|
|
973
|
+
counts.partial = true;
|
|
974
|
+
continue;
|
|
975
|
+
}
|
|
976
|
+
if (t.tooLarge) {
|
|
977
|
+
counts.skipped.tooLarge++;
|
|
978
|
+
continue;
|
|
979
|
+
}
|
|
980
|
+
if (containsAny(t.buf, secrets)) return await abort({ refused: "job-secret-in-tree" });
|
|
981
|
+
await write(`M 120000 inline ${q}\ndata ${t.buf.length}\n`);
|
|
982
|
+
await write(t.buf);
|
|
983
|
+
await write("\n");
|
|
984
|
+
counts.files++;
|
|
985
|
+
continue;
|
|
986
|
+
}
|
|
987
|
+
const mode = (f.st.mode & 0o111n) !== 0n ? "100755" : "100644";
|
|
988
|
+
const known = cachedOid.get(f.path);
|
|
989
|
+
if (known) {
|
|
990
|
+
await write(`M ${mode} ${known} ${q}\n`);
|
|
991
|
+
counts.files++;
|
|
992
|
+
counts.bytes += Number(f.st.size);
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
const got = await readExact(abs, f.st, rootReal);
|
|
996
|
+
if (got.unreadable) {
|
|
997
|
+
counts.skipped.unreadable++;
|
|
998
|
+
counts.partial = true;
|
|
999
|
+
continue;
|
|
1000
|
+
}
|
|
1001
|
+
if (!got.buf) return await abort({ skipped: "tree-changed" });
|
|
1002
|
+
if (containsAny(got.buf, secrets)) return await abort({ refused: "job-secret-in-tree" });
|
|
1003
|
+
newOids.set(f.path, cacheEntry(f.st, blobOid(objectFormat, got.buf)));
|
|
1004
|
+
await write(`M ${mode} inline ${q}\ndata ${got.buf.length}\n`);
|
|
1005
|
+
await write(got.buf);
|
|
1006
|
+
await write("\n");
|
|
1007
|
+
counts.files++;
|
|
1008
|
+
counts.bytes += got.buf.length;
|
|
1009
|
+
counts.newBytes += got.buf.length;
|
|
1010
|
+
}
|
|
1011
|
+
await write(`\nreset refs/pi-dispatch/cp/${seq}\nfrom :1\n\ndone\n`);
|
|
1012
|
+
child.stdin.end();
|
|
1013
|
+
} catch {
|
|
1014
|
+
return await abort({ skipped: signal.aborted ? "timeout" : "import-failed" });
|
|
1015
|
+
}
|
|
1016
|
+
const r = await done;
|
|
1017
|
+
if (r.code !== 0) {
|
|
1018
|
+
await cleanupImport(paths);
|
|
1019
|
+
return { skipped: r.aborted ? "timeout" : "import-failed" };
|
|
1020
|
+
}
|
|
1021
|
+
const rev = await runGit(paths, ["rev-parse", "--verify", "--quiet", `refs/pi-dispatch/cp/${seq}^{commit}`], { signal });
|
|
1022
|
+
const commit = rev.stdout.toString("latin1").trim();
|
|
1023
|
+
if (rev.code !== 0 || !HEX[objectFormat].test(commit)) return { skipped: "import-failed" };
|
|
1024
|
+
return { commit, newOids };
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
/**
|
|
1028
|
+
* After a killed fast-import: its half-written pack (`objects/pack/tmp_*`) and its crash report
|
|
1029
|
+
* (`fast_import_crash_<pid>`, which quotes recent commands, so agent path names) are removed. A
|
|
1030
|
+
* refused snapshot must leave no trace of the bytes it refused, which is the whole point of refusing.
|
|
1031
|
+
*/
|
|
1032
|
+
async function cleanupImport(paths) {
|
|
1033
|
+
const packDir = join(paths.repo, "objects", "pack");
|
|
1034
|
+
for (const [dir, prefix] of [
|
|
1035
|
+
[packDir, "tmp_"],
|
|
1036
|
+
[paths.repo, "fast_import_crash_"],
|
|
1037
|
+
]) {
|
|
1038
|
+
const names = await fs.readdir(dir).catch(() => []);
|
|
1039
|
+
for (const n of names) if (n.startsWith(prefix)) await fs.rm(join(dir, n), { force: true }).catch(() => {});
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/** Reserve `cp-<seq>/`, clearing temp files a crashed attempt at the same seq left behind. */
|
|
1044
|
+
async function reserveCp(paths, seq) {
|
|
1045
|
+
const dir = paths.cp(seq);
|
|
1046
|
+
await ensureDir(dir);
|
|
1047
|
+
for (const n of await fs.readdir(dir)) if (n.endsWith(".incoming")) await fs.rm(join(dir, n), { force: true });
|
|
1048
|
+
return dir;
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* Copy bytes [0, offset) of the transcript into `cp-<seq>/transcript.jsonl`: the source opened
|
|
1053
|
+
* `O_NOFOLLOW` (the worker's session file, but a link there must not pull in some other file), the copy
|
|
1054
|
+
* fsynced and renamed into place. The offset is the caller's: the transcript keeps growing while the
|
|
1055
|
+
* container runs, and the checkpoint must describe the bytes that match the workspace, not later ones.
|
|
1056
|
+
*/
|
|
1057
|
+
async function copyTranscript(dir, srcFile, offset, signal) {
|
|
1058
|
+
const tmp = join(dir, `transcript.jsonl.${rand()}.incoming`);
|
|
1059
|
+
const hash = createHash("sha256");
|
|
1060
|
+
let bytes = 0;
|
|
1061
|
+
try {
|
|
1062
|
+
const out = await fs.open(tmp, "wx", 0o600);
|
|
1063
|
+
try {
|
|
1064
|
+
if (srcFile && offset > 0) {
|
|
1065
|
+
const src = await fs.open(srcFile, FS.O_RDONLY | FS.O_NOFOLLOW | FS.O_NONBLOCK);
|
|
1066
|
+
try {
|
|
1067
|
+
if (!(await src.stat()).isFile()) throw Object.assign(new Error("transcript not a file"), { code: "ENOTREG" });
|
|
1068
|
+
const chunk = Buffer.allocUnsafe(MiB);
|
|
1069
|
+
while (bytes < offset) {
|
|
1070
|
+
// The snapshot's deadline covers the copy too: bounded 1 MiB chunks, the signal checked
|
|
1071
|
+
// before each, so a slow or huge source cannot hold the container frozen past it.
|
|
1072
|
+
if (signal?.aborted) throw Object.assign(new Error("transcript copy timed out"), { code: "ETIMEDOUT" });
|
|
1073
|
+
const { bytesRead } = await src.read(chunk, 0, Math.min(chunk.length, offset - bytes), bytes);
|
|
1074
|
+
if (bytesRead === 0) break;
|
|
1075
|
+
hash.update(chunk.subarray(0, bytesRead));
|
|
1076
|
+
await out.write(chunk, 0, bytesRead);
|
|
1077
|
+
bytes += bytesRead;
|
|
1078
|
+
}
|
|
1079
|
+
} finally {
|
|
1080
|
+
await src.close().catch(() => {});
|
|
1081
|
+
}
|
|
1082
|
+
if (bytes !== offset) throw Object.assign(new Error("transcript short"), { code: "ESHORT" });
|
|
1083
|
+
}
|
|
1084
|
+
await out.sync();
|
|
1085
|
+
} finally {
|
|
1086
|
+
await out.close().catch(() => {});
|
|
1087
|
+
}
|
|
1088
|
+
await fs.rename(tmp, join(dir, "transcript.jsonl"));
|
|
1089
|
+
} catch (err) {
|
|
1090
|
+
await fs.unlink(tmp).catch(() => {});
|
|
1091
|
+
throw err;
|
|
1092
|
+
}
|
|
1093
|
+
return { bytes, sha256: hash.digest("hex") };
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/** The public transcript helper: reserve `cp-<seq>/` and copy into it. Never throws. */
|
|
1097
|
+
async function writeTranscript(runId, seq, srcFile, offset) {
|
|
1098
|
+
try {
|
|
1099
|
+
if (typeof runId !== "string" || !runId || !isInt(seq, 1, 999_999_999) || !isInt(offset)) return { skipped: "bad-args" };
|
|
1100
|
+
const paths = runPaths(runId);
|
|
1101
|
+
if (!(await lstatOrNull(paths.run))?.isDirectory()) return { skipped: "not-primed" };
|
|
1102
|
+
if (offset > L.maxTranscriptBytes) return { skipped: "transcript-too-large" };
|
|
1103
|
+
const signal = AbortSignal.timeout(L.snapshotTimeoutMs);
|
|
1104
|
+
const dir = await reserveCp(paths, seq);
|
|
1105
|
+
try {
|
|
1106
|
+
return { ok: true, ...(await copyTranscript(dir, srcFile, offset, signal)) };
|
|
1107
|
+
} catch (err) {
|
|
1108
|
+
if (signal.aborted) return { skipped: "timeout" };
|
|
1109
|
+
throw err;
|
|
1110
|
+
}
|
|
1111
|
+
} catch (err) {
|
|
1112
|
+
log("checkpoint_failed", { phase: "transcript", code: err?.code ?? "internal" });
|
|
1113
|
+
return { skipped: "transcript-failed" };
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Take a checkpoint of a PAUSED container's workspace (the caller paused it; this never does).
|
|
1119
|
+
*
|
|
1120
|
+
* Order: free refusals and caps first (lock, seq, disk, store size, walk, file cap, byte cap with its
|
|
1121
|
+
* ignore tier), then the one expensive step (reading new bytes into fast-import, refusing on a job
|
|
1122
|
+
* secret), then the commit point: transcript, manifest, CURRENT, each fsynced and renamed, so a crash
|
|
1123
|
+
* at any step leaves the previous CURRENT pointing at a complete checkpoint.
|
|
1124
|
+
*/
|
|
1125
|
+
async function snapshot({
|
|
1126
|
+
runId,
|
|
1127
|
+
workspace,
|
|
1128
|
+
baseSha,
|
|
1129
|
+
seq,
|
|
1130
|
+
branch = null,
|
|
1131
|
+
secretValues = [],
|
|
1132
|
+
attempt = 0,
|
|
1133
|
+
kind = "live",
|
|
1134
|
+
quiesced = "frozen",
|
|
1135
|
+
transcript = {},
|
|
1136
|
+
meta = {},
|
|
1137
|
+
} = {}) {
|
|
1138
|
+
try {
|
|
1139
|
+
if (typeof runId !== "string" || !runId || !isInt(seq, 1, 999_999_999) || !ANY_HEX_OID.test(String(baseSha))) return { skipped: "bad-args" };
|
|
1140
|
+
if (!KINDS.has(kind) || !QUIESCED.has(quiesced) || !isInt(attempt, 0, 10_000)) return { skipped: "bad-args" };
|
|
1141
|
+
const paths = runPaths(runId);
|
|
1142
|
+
if (!(await lstatOrNull(paths.run))?.isDirectory()) return { skipped: "not-primed" };
|
|
1143
|
+
if (await lstatOrNull(paths.tooLarge)) return { skipped: "base-too-large" };
|
|
1144
|
+
if (!(await lstatOrNull(join(paths.repo, "HEAD")))) return { skipped: "not-primed" };
|
|
1145
|
+
return await withLock(paths, () =>
|
|
1146
|
+
snapshotLocked(paths, { runId, workspace, baseSha, seq, branch, secretValues, attempt, kind, quiesced, transcript, meta }),
|
|
1147
|
+
);
|
|
1148
|
+
} catch (err) {
|
|
1149
|
+
log("checkpoint_failed", { phase: "snapshot", code: err?.code ?? "internal" });
|
|
1150
|
+
return { skipped: "internal-error" };
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
async function snapshotLocked(paths, args) {
|
|
1155
|
+
const { runId, workspace, baseSha, seq, branch, secretValues, attempt, kind, quiesced, transcript, meta } = args;
|
|
1156
|
+
const signal = AbortSignal.timeout(L.snapshotTimeoutMs);
|
|
1157
|
+
const cur = await readCurrent(paths);
|
|
1158
|
+
if (cur.seq && seq <= cur.seq) return { skipped: "stale-seq" };
|
|
1159
|
+
// Judged on the caller's OFFSET, before anything is read or written: the offset is the number of bytes
|
|
1160
|
+
// the copy would write, whatever the source's apparent size (a sparse file reads as zeros).
|
|
1161
|
+
const trOffset = isInt(transcript?.offset) ? transcript.offset : 0;
|
|
1162
|
+
if (trOffset > L.maxTranscriptBytes) return { skipped: "transcript-too-large" };
|
|
1163
|
+
const state = runState(runId);
|
|
1164
|
+
const objectFormat = await storeObjectFormat(paths, state, signal);
|
|
1165
|
+
if (!objectFormat || !HEX[objectFormat].test(baseSha)) return { skipped: "store-unreadable" };
|
|
1166
|
+
if (!(await diskOk(trOffset))) return { skipped: "disk-low" };
|
|
1167
|
+
if ((await storeBytes()) + trOffset > L.maxStoreBytes) return { skipped: "store-full" };
|
|
1168
|
+
|
|
1169
|
+
const walk = await walkTree(workspace, signal);
|
|
1170
|
+
if (walk.skipped) return { skipped: walk.skipped };
|
|
1171
|
+
let files = walk.files;
|
|
1172
|
+
const rootReal = await fs.realpath(workspace);
|
|
1173
|
+
|
|
1174
|
+
const cachedOid = new Map();
|
|
1175
|
+
const newBytesOf = (list) => list.reduce((n, f) => (cachedOid.has(f.path) ? n : n + Number(f.st.size)), 0);
|
|
1176
|
+
for (const f of files) {
|
|
1177
|
+
const hit = f.kind === "file" ? state.cache.get(f.path) : null;
|
|
1178
|
+
if (cacheMatches(hit, f.st)) cachedOid.set(f.path, hit.oid);
|
|
1179
|
+
}
|
|
1180
|
+
let ignoredDropped = false;
|
|
1181
|
+
if (newBytesOf(files) > L.maxNewBytes) {
|
|
1182
|
+
if (walk.unsafeIgnoreFile) return { skipped: "over-byte-cap" };
|
|
1183
|
+
const kept = await dropIgnored(paths, workspace, files, signal);
|
|
1184
|
+
if (!kept) return { skipped: signal.aborted ? "timeout" : "over-byte-cap" };
|
|
1185
|
+
files = kept;
|
|
1186
|
+
ignoredDropped = true;
|
|
1187
|
+
if (newBytesOf(files) > L.maxNewBytes) return { skipped: "over-byte-cap" };
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
const git = await parseGitState(workspace, branch);
|
|
1191
|
+
// Scanned: every stored PATH (a file named after a token leaks it as surely as one containing it, and
|
|
1192
|
+
// a directory name is a substring of every path under it), every symlink target, and every byte READ
|
|
1193
|
+
// for this snapshot. Not re-scanned: the CONTENT of a file served from the stat cache, which is
|
|
1194
|
+
// either base content (the repository's own, already on the forge) or bytes a previous snapshot of
|
|
1195
|
+
// this run read and scanned; writing a secret into one changes its ctime, so it is re-read.
|
|
1196
|
+
const secrets = [...new Set(secretValues)].filter((s) => typeof s === "string" && s.length > 0).map((s) => Buffer.from(s, "utf8"));
|
|
1197
|
+
if (files.some((f) => containsAny(Buffer.from(f.path, "utf8"), secrets))) {
|
|
1198
|
+
log("checkpoint_snapshot_skipped", { seq, reason: "job-secret-in-tree" });
|
|
1199
|
+
return { refused: "job-secret-in-tree" };
|
|
1200
|
+
}
|
|
1201
|
+
const counts = { files: 0, bytes: 0, newBytes: 0, skipped: { ...walk.excluded }, partial: walk.partial };
|
|
1202
|
+
const imported = await importTree(paths, { workspace, rootReal, files, cachedOid, baseSha, seq, secrets, objectFormat, signal, counts });
|
|
1203
|
+
if (!imported.commit) {
|
|
1204
|
+
log("checkpoint_snapshot_skipped", { seq, reason: imported.refused ?? imported.skipped });
|
|
1205
|
+
return imported;
|
|
1206
|
+
}
|
|
1207
|
+
for (const [p, entry] of imported.newOids) state.cache.set(p, entry);
|
|
1208
|
+
if (seq % L.repackEvery === 0) await runGit(paths, ["repack", "-adq"], { signal });
|
|
1209
|
+
|
|
1210
|
+
// ── the commit point ──
|
|
1211
|
+
const dir = await reserveCp(paths, seq);
|
|
1212
|
+
let tr;
|
|
1213
|
+
try {
|
|
1214
|
+
tr = await copyTranscript(dir, transcript?.srcFile ?? null, trOffset, signal);
|
|
1215
|
+
} catch (err) {
|
|
1216
|
+
if (signal.aborted) return { skipped: "timeout" };
|
|
1217
|
+
throw err;
|
|
1218
|
+
}
|
|
1219
|
+
const manifest = {
|
|
1220
|
+
v: CHECKPOINT_MANIFEST_VERSION,
|
|
1221
|
+
runId: sanitizeJobId(runId).slice(0, 200),
|
|
1222
|
+
seq,
|
|
1223
|
+
attempt,
|
|
1224
|
+
host,
|
|
1225
|
+
kind,
|
|
1226
|
+
quiesced,
|
|
1227
|
+
at: Math.floor(now()),
|
|
1228
|
+
transcript: tr,
|
|
1229
|
+
workspace: {
|
|
1230
|
+
commit: imported.commit,
|
|
1231
|
+
baseSha,
|
|
1232
|
+
objectFormat,
|
|
1233
|
+
partial: counts.partial,
|
|
1234
|
+
ignoredDropped,
|
|
1235
|
+
counts: { files: counts.files, bytes: counts.bytes, newBytes: counts.newBytes, skipped: counts.skipped },
|
|
1236
|
+
git,
|
|
1237
|
+
},
|
|
1238
|
+
};
|
|
1239
|
+
for (const field of META_FIELDS) {
|
|
1240
|
+
manifest[field] = token(meta?.[field]);
|
|
1241
|
+
// Fail open and say which: a malformed token is dropped to null, and its NAME is logged.
|
|
1242
|
+
if (meta?.[field] != null && manifest[field] === null) log("checkpoint_meta_dropped", { field });
|
|
1243
|
+
}
|
|
1244
|
+
await atomicWrite(dir, "manifest.json", `${JSON.stringify(manifest)}\n`);
|
|
1245
|
+
await atomicWrite(paths.run, "CURRENT", `${seq}\n`);
|
|
1246
|
+
log("checkpoint_snapshot", { seq, files: counts.files, newBytes: counts.newBytes, partial: counts.partial, ignoredDropped });
|
|
1247
|
+
if (!keepHistory) await pruneOld(paths, seq, cur.seq ?? null);
|
|
1248
|
+
return { ok: true, seq, manifest };
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* Keep the checkpoint `CURRENT` now names and the one it replaced; remove every other `cp-<seq>/` and
|
|
1253
|
+
* every other `refs/pi-dispatch/cp/<seq>` (so the next repack can drop their objects). Without it every
|
|
1254
|
+
* checkpoint's transcript copy stays forever, and a long run's store grows by one transcript per
|
|
1255
|
+
* snapshot until `store-full` stops checkpointing altogether.
|
|
1256
|
+
*
|
|
1257
|
+
* CRASH-SAFE BY ORDER: it runs only after `CURRENT`'s rename and directory fsync, and it never touches
|
|
1258
|
+
* the two it keeps, so a crash anywhere in here leaves `CURRENT` naming a complete checkpoint; what is
|
|
1259
|
+
* left over is pruned by the next snapshot. The previous one is kept as the fallback an operator (or a
|
|
1260
|
+
* later recovery change) can reach when the newest turns out to be the bad one. A failure here is
|
|
1261
|
+
* logged and does not fail the snapshot, which has already committed. It runs under the snapshot's lock.
|
|
1262
|
+
*/
|
|
1263
|
+
async function pruneOld(paths, seq, previous) {
|
|
1264
|
+
const keep = new Set([seq, previous].filter((n) => n !== null));
|
|
1265
|
+
try {
|
|
1266
|
+
const refs = await runGit(paths, ["for-each-ref", "--format=%(refname)", "refs/pi-dispatch/cp/"]);
|
|
1267
|
+
const doomed = refs.stdout
|
|
1268
|
+
.toString("latin1")
|
|
1269
|
+
.split("\n")
|
|
1270
|
+
.filter((r) => /^refs\/pi-dispatch\/cp\/[1-9][0-9]*$/.test(r) && !keep.has(Number(r.slice("refs/pi-dispatch/cp/".length))));
|
|
1271
|
+
if (doomed.length) {
|
|
1272
|
+
const del = await runGit(paths, ["update-ref", "--stdin"], { input: doomed.map((r) => `delete ${r}\n`).join("") });
|
|
1273
|
+
if (del.code !== 0) throw Object.assign(new Error("update-ref failed"), { code: "EGIT" });
|
|
1274
|
+
}
|
|
1275
|
+
let removed = 0;
|
|
1276
|
+
for (const n of await fs.readdir(paths.run)) {
|
|
1277
|
+
const m = /^cp-([1-9][0-9]*)$/.exec(n);
|
|
1278
|
+
if (!m || keep.has(Number(m[1]))) continue;
|
|
1279
|
+
await fs.rm(join(paths.run, n), { recursive: true, force: true });
|
|
1280
|
+
removed++;
|
|
1281
|
+
}
|
|
1282
|
+
if (removed || doomed.length) log("checkpoint_pruned", { removed, refs: doomed.length });
|
|
1283
|
+
} catch (err) {
|
|
1284
|
+
log("checkpoint_prune_failed", { code: err?.code ?? "internal" });
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
// ── readManifest ────────────────────────────────────────────────────────────────────────────────────
|
|
1289
|
+
|
|
1290
|
+
/**
|
|
1291
|
+
* A plain object with EXACTLY these keys. Closed at every level, not only the top: an open nested
|
|
1292
|
+
* object is where a path or a payload string would ride along unvalidated into everything that reads
|
|
1293
|
+
* a manifest.
|
|
1294
|
+
*/
|
|
1295
|
+
function closed(o, keys) {
|
|
1296
|
+
if (!o || typeof o !== "object" || Array.isArray(o)) return false;
|
|
1297
|
+
const have = Object.keys(o).sort();
|
|
1298
|
+
const want = [...keys].sort();
|
|
1299
|
+
return have.length === want.length && have.every((k, i) => k === want[i]);
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
/**
|
|
1303
|
+
* Every field typed, every string an enum, hex, a host name or a short token. A reason, or null.
|
|
1304
|
+
*
|
|
1305
|
+
* `newer` (issue #761) is judged BEFORE the closed key set: a newer release may add keys, and read as `keys` its
|
|
1306
|
+
* manifest would look broken, where it is only unreadable HERE and is kept for a worker that reads it. Only an integer
|
|
1307
|
+
* `v` above every version this build reads is newer; `0`, `"1"` or `1.5` is a broken manifest (`v`).
|
|
1308
|
+
*/
|
|
1309
|
+
function manifestProblem(m) {
|
|
1310
|
+
const keys = ["v", "runId", "seq", "attempt", "host", "kind", "quiesced", "at", "transcript", "workspace", ...META_FIELDS];
|
|
1311
|
+
if (!m || typeof m !== "object" || Array.isArray(m)) return "not-object";
|
|
1312
|
+
if (Number.isSafeInteger(m.v) && m.v > newestManifest) return "newer";
|
|
1313
|
+
if (!closed(m, keys)) return "keys";
|
|
1314
|
+
if (!Number.isSafeInteger(m.v) || !manifestVersions.has(m.v)) return "v";
|
|
1315
|
+
if (typeof m.runId !== "string" || !RUN_ID_RE.test(m.runId)) return "runId";
|
|
1316
|
+
if (!isInt(m.seq, 1, 999_999_999)) return "seq";
|
|
1317
|
+
if (!isInt(m.attempt, 0, 10_000)) return "attempt";
|
|
1318
|
+
if (typeof m.host !== "string" || !HOST_RE.test(m.host)) return "host";
|
|
1319
|
+
if (!KINDS.has(m.kind)) return "kind";
|
|
1320
|
+
if (!QUIESCED.has(m.quiesced)) return "quiesced";
|
|
1321
|
+
if (!isInt(m.at)) return "at";
|
|
1322
|
+
const t = m.transcript;
|
|
1323
|
+
if (!closed(t, ["bytes", "sha256"]) || !isInt(t.bytes) || typeof t.sha256 !== "string" || !SHA256_HEX.test(t.sha256)) return "transcript";
|
|
1324
|
+
const w = m.workspace;
|
|
1325
|
+
if (!closed(w, ["commit", "baseSha", "objectFormat", "partial", "ignoredDropped", "counts", "git"])) return "workspace";
|
|
1326
|
+
if (!HEX[w.objectFormat]) return "objectFormat";
|
|
1327
|
+
if (typeof w.commit !== "string" || !HEX[w.objectFormat].test(w.commit)) return "commit";
|
|
1328
|
+
if (typeof w.baseSha !== "string" || !HEX[w.objectFormat].test(w.baseSha)) return "baseSha";
|
|
1329
|
+
if (typeof w.partial !== "boolean" || typeof w.ignoredDropped !== "boolean") return "flags";
|
|
1330
|
+
const c = w.counts;
|
|
1331
|
+
if (!closed(c, ["files", "bytes", "newBytes", "skipped"]) || !isInt(c.files) || !isInt(c.bytes) || !isInt(c.newBytes)) return "counts";
|
|
1332
|
+
const skippedKeys = ["unreadable", "special", "dotgit", "badName", "tooLarge"];
|
|
1333
|
+
if (!closed(c.skipped, skippedKeys) || !skippedKeys.every((k) => isInt(c.skipped[k]))) return "counts";
|
|
1334
|
+
const g = w.git;
|
|
1335
|
+
if (!closed(g, ["state", "branch", "tip", "detached"]) || !GIT_STATES.has(g.state) || typeof g.detached !== "boolean") return "git";
|
|
1336
|
+
if (g.branch !== null && !isBranchName(g.branch)) return "git";
|
|
1337
|
+
if (g.tip !== null && !(typeof g.tip === "string" && ANY_HEX_OID.test(g.tip))) return "git";
|
|
1338
|
+
for (const f of META_FIELDS) if (m[f] !== null && token(m[f]) === null) return f;
|
|
1339
|
+
return null;
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
async function hashFile(path) {
|
|
1343
|
+
const fh = await fs.open(path, FS.O_RDONLY | FS.O_NOFOLLOW | FS.O_NONBLOCK);
|
|
1344
|
+
try {
|
|
1345
|
+
if (!(await fh.stat()).isFile()) return null;
|
|
1346
|
+
const hash = createHash("sha256");
|
|
1347
|
+
const chunk = Buffer.allocUnsafe(MiB);
|
|
1348
|
+
let bytes = 0;
|
|
1349
|
+
for (;;) {
|
|
1350
|
+
const { bytesRead } = await fh.read(chunk, 0, chunk.length, bytes);
|
|
1351
|
+
if (bytesRead === 0) break;
|
|
1352
|
+
hash.update(chunk.subarray(0, bytesRead));
|
|
1353
|
+
bytes += bytesRead;
|
|
1354
|
+
}
|
|
1355
|
+
return { bytes, sha256: hash.digest("hex") };
|
|
1356
|
+
} finally {
|
|
1357
|
+
await fh.close().catch(() => {});
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/**
|
|
1362
|
+
* The checkpoint CURRENT names, validated: `{ ok, manifest, transcriptPath }`, `{ absent }` when there
|
|
1363
|
+
* is none, `{ invalid, reason }` when what is there cannot be trusted. The transcript's bytes are hashed
|
|
1364
|
+
* against the manifest, because the resume feeds it to a paid model and a truncated one resumes a
|
|
1365
|
+
* different conversation. A manifest a newer pi-dispatch wrote is `{ invalid, reason: "field:newer", newer }`
|
|
1366
|
+
* (issue #761): still `invalid` to every caller that only asks whether it can use it, and `newer` to the ones that
|
|
1367
|
+
* would otherwise delete what they could not read.
|
|
1368
|
+
*/
|
|
1369
|
+
async function readManifest(runId, { seq } = {}) {
|
|
1370
|
+
try {
|
|
1371
|
+
if (typeof runId !== "string" || !runId) return { invalid: true, reason: "bad-args" };
|
|
1372
|
+
const paths = runPaths(runId);
|
|
1373
|
+
// lstat, and a real directory: a link at the run's name is not this run's store, whatever it points at.
|
|
1374
|
+
if (!(await lstatOrNull(paths.run))?.isDirectory()) return { absent: true };
|
|
1375
|
+
if (seq !== undefined && (!keepHistory || !isInt(seq, 1, 999_999_999))) return { invalid: true, reason: "bad-seq" };
|
|
1376
|
+
const cur = seq === undefined ? await readCurrent(paths) : { seq };
|
|
1377
|
+
if (cur.unavailable) return cur;
|
|
1378
|
+
if (cur.absent) return { absent: true };
|
|
1379
|
+
if (cur.invalid) return { invalid: true, reason: cur.invalid };
|
|
1380
|
+
const raw = await readRegular(join(paths.cp(cur.seq), "manifest.json"), MiB);
|
|
1381
|
+
if (keepHistory && retryableStoreError(raw.error)) return { unavailable: raw.error };
|
|
1382
|
+
if (!raw.buf) return { invalid: true, reason: "manifest-unreadable" };
|
|
1383
|
+
let manifest;
|
|
1384
|
+
try {
|
|
1385
|
+
manifest = JSON.parse(raw.buf.toString("utf8"));
|
|
1386
|
+
} catch {
|
|
1387
|
+
return { invalid: true, reason: "manifest-json" };
|
|
1388
|
+
}
|
|
1389
|
+
const problem = manifestProblem(manifest);
|
|
1390
|
+
if (problem === "newer") return { invalid: true, reason: "field:newer", newer: true };
|
|
1391
|
+
if (problem) return { invalid: true, reason: `field:${problem}` };
|
|
1392
|
+
if (manifest.seq !== cur.seq) return { invalid: true, reason: "seq-mismatch" };
|
|
1393
|
+
const transcriptPath = join(paths.cp(cur.seq), "transcript.jsonl");
|
|
1394
|
+
const got = await hashFile(transcriptPath).catch((error) => {
|
|
1395
|
+
if (keepHistory && retryableStoreError(error)) throw error;
|
|
1396
|
+
return null;
|
|
1397
|
+
});
|
|
1398
|
+
if (!got || got.bytes !== manifest.transcript.bytes || got.sha256 !== manifest.transcript.sha256) {
|
|
1399
|
+
return { invalid: true, reason: "transcript-mismatch" };
|
|
1400
|
+
}
|
|
1401
|
+
return { ok: true, manifest, transcriptPath };
|
|
1402
|
+
} catch (err) {
|
|
1403
|
+
log("checkpoint_failed", { phase: "read", code: err?.code ?? "internal" });
|
|
1404
|
+
if (keepHistory && retryableStoreError(err)) return { unavailable: err.code };
|
|
1405
|
+
return { invalid: true, reason: "internal-error" };
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
// ── restore ─────────────────────────────────────────────────────────────────────────────────────────
|
|
1410
|
+
|
|
1411
|
+
/** `file://` URL of a run's store, for the caller's `fetch --depth=1 <url> <baseSha>` (needs `protocol.file.allow=always`). */
|
|
1412
|
+
function baseSourceUrl(runId) {
|
|
1413
|
+
return pathToFileURL(runPaths(runId).repo).href;
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
/** A fork may select the base without any snapshot yet; prove its clone source still exists pre-spend. */
|
|
1417
|
+
async function readBase(runId) {
|
|
1418
|
+
try {
|
|
1419
|
+
if (typeof runId !== "string" || !runId) return { absent: true };
|
|
1420
|
+
const paths = runPaths(runId);
|
|
1421
|
+
if (!(await lstatOrNull(paths.run))?.isDirectory() || !(await lstatOrNull(paths.repo))?.isDirectory()) return { absent: true };
|
|
1422
|
+
const result = await runGit(paths, ["rev-parse", "--verify", "refs/pi-dispatch/base"]);
|
|
1423
|
+
const baseSha = result.stdout.toString("utf8").trim();
|
|
1424
|
+
if (result.aborted || result.code === -1) return { unavailable: result.aborted ? "ETIMEDOUT" : "git-unavailable" };
|
|
1425
|
+
if (result.code !== 0 || !ANY_HEX_OID.test(baseSha)) return { absent: true };
|
|
1426
|
+
// A loose ref survives deletion of its objects. Verify both objects before the worker mints
|
|
1427
|
+
// credentials for a clone that otherwise could only fail on this same missing base.
|
|
1428
|
+
for (const kind of ["commit", "tree"]) {
|
|
1429
|
+
const object = await runGit(paths, ["cat-file", "-e", `${baseSha}^{${kind}}`]);
|
|
1430
|
+
if (object.aborted || object.code === -1) return { unavailable: object.aborted ? "ETIMEDOUT" : "git-unavailable" };
|
|
1431
|
+
if (object.code !== 0) return { absent: true };
|
|
1432
|
+
}
|
|
1433
|
+
return { ok: true, baseSha };
|
|
1434
|
+
} catch (error) {
|
|
1435
|
+
return retryableStoreError(error) ? { unavailable: error.code } : { absent: true };
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
/** Parse `diff-tree -r -z --raw` output into `{ srcMode, dstMode, dstOid, status, path }` records. */
|
|
1440
|
+
function parseRaw(buf) {
|
|
1441
|
+
const parts = buf.toString("utf8").split("\0");
|
|
1442
|
+
const out = [];
|
|
1443
|
+
for (let i = 0; i + 1 < parts.length; i += 2) {
|
|
1444
|
+
const m = /^:(\d{6}) (\d{6}) ([0-9a-f]+) ([0-9a-f]+) ([ADMT])$/.exec(parts[i]);
|
|
1445
|
+
if (!m) return null;
|
|
1446
|
+
out.push({ srcMode: m[1], dstMode: m[2], dstOid: m[4], status: m[5], path: parts[i + 1] });
|
|
1447
|
+
}
|
|
1448
|
+
return out;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
function prefixes(path) {
|
|
1452
|
+
const segs = path.split("/");
|
|
1453
|
+
const out = [];
|
|
1454
|
+
for (let i = 1; i < segs.length; i++) out.push(segs.slice(0, i).join("/"));
|
|
1455
|
+
return out;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
/**
|
|
1459
|
+
* Judge the whole change list BEFORE touching the disk: every path through `validPath` (the same rule
|
|
1460
|
+
* the snapshot applied), no gitlink in the result (the snapshot never writes one, so one there is a
|
|
1461
|
+
* forged store), no file under a path the result makes a symlink (the write would land through the
|
|
1462
|
+
* link), and no two spellings of one name under case folding (one would silently overwrite the other).
|
|
1463
|
+
*
|
|
1464
|
+
* A gitlink on the BASE side being deleted is not refused and not acted on: the snapshot cannot record
|
|
1465
|
+
* a submodule, so its absence from the snapshot tree describes the snapshot, not a change to make.
|
|
1466
|
+
*/
|
|
1467
|
+
function judgeChanges(changes, commitTree) {
|
|
1468
|
+
const OK_MODES = new Set(["100644", "100755", "120000"]);
|
|
1469
|
+
const links = new Set();
|
|
1470
|
+
const folds = new Map();
|
|
1471
|
+
for (const { path, mode } of commitTree) {
|
|
1472
|
+
if (mode === "120000") links.add(path);
|
|
1473
|
+
for (const p of [...prefixes(path), path]) {
|
|
1474
|
+
const k = foldCase(p);
|
|
1475
|
+
if (!folds.has(k)) folds.set(k, new Set());
|
|
1476
|
+
folds.get(k).add(p);
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
for (const c of changes) {
|
|
1480
|
+
if (!validPath(c.path)) return "tampered";
|
|
1481
|
+
// The mode check IS the gitlink check: 160000 is not a mode a snapshot writes, so a gitlink on the
|
|
1482
|
+
// result side is refused here, before any blob is asked for.
|
|
1483
|
+
if (c.status !== "D" && !OK_MODES.has(c.dstMode)) return "tampered";
|
|
1484
|
+
if (c.status === "D") continue;
|
|
1485
|
+
for (const p of prefixes(c.path)) if (links.has(p)) return "path-through-symlink";
|
|
1486
|
+
for (const p of [...prefixes(c.path), c.path]) if ((folds.get(foldCase(p))?.size ?? 0) > 1) return "case-collision";
|
|
1487
|
+
}
|
|
1488
|
+
return null;
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
class Refusal extends Error {
|
|
1492
|
+
constructor(reason) {
|
|
1493
|
+
super(reason);
|
|
1494
|
+
this.reason = reason;
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
/**
|
|
1499
|
+
* Every existing ancestor of `path` under `root` must lstat as a real directory; missing ones are made
|
|
1500
|
+
* one segment at a time (0755). A symlink anywhere on the way refuses: `mkdir -p` would follow it.
|
|
1501
|
+
*/
|
|
1502
|
+
async function ensureParents(root, path, create) {
|
|
1503
|
+
let cur = root;
|
|
1504
|
+
for (const seg of prefixes(path).map((p) => p.split("/").at(-1))) {
|
|
1505
|
+
cur = join(cur, seg);
|
|
1506
|
+
const st = await lstatOrNull(cur);
|
|
1507
|
+
if (!st) {
|
|
1508
|
+
if (!create) return;
|
|
1509
|
+
await fs.mkdir(cur, { mode: 0o755 });
|
|
1510
|
+
continue;
|
|
1511
|
+
}
|
|
1512
|
+
if (st.isSymbolicLink()) throw new Refusal("path-through-symlink");
|
|
1513
|
+
if (!st.isDirectory()) throw new Refusal("path-conflict");
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
/** Remove now-empty parents of a deleted path, the way a git checkout would; stops at the first non-empty. */
|
|
1518
|
+
async function pruneEmpty(root, path) {
|
|
1519
|
+
for (const p of prefixes(path).reverse()) {
|
|
1520
|
+
try {
|
|
1521
|
+
await fs.rmdir(join(root, p));
|
|
1522
|
+
} catch {
|
|
1523
|
+
return;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
/** One `cat-file --batch` for every blob a restore writes: one process, request, header, bytes. */
|
|
1529
|
+
function blobReader(paths, signal) {
|
|
1530
|
+
const { child, done } = startGit(paths, ["cat-file", "--batch"], { signal });
|
|
1531
|
+
child.stdout.removeAllListeners("data");
|
|
1532
|
+
const reader = byteReader(child.stdout);
|
|
1533
|
+
return {
|
|
1534
|
+
async read(oid) {
|
|
1535
|
+
child.stdin.write(`${oid}\n`);
|
|
1536
|
+
const header = await reader.line();
|
|
1537
|
+
const m = /^([0-9a-f]+) blob (\d+)$/.exec(header);
|
|
1538
|
+
if (!m || m[1] !== oid) throw new Refusal("tampered");
|
|
1539
|
+
const size = Number(m[2]);
|
|
1540
|
+
if (size > L.maxFileBytes) throw new Refusal("tampered");
|
|
1541
|
+
const bytes = await reader.bytes(size + 1);
|
|
1542
|
+
return bytes.subarray(0, size);
|
|
1543
|
+
},
|
|
1544
|
+
async close() {
|
|
1545
|
+
child.stdin.end();
|
|
1546
|
+
await done;
|
|
1547
|
+
},
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/**
|
|
1552
|
+
* The commit the caller's checkout is at, from FILES: `HEAD` as a sha, or `ref: refs/heads/<name>`
|
|
1553
|
+
* resolved through the loose ref or `packed-refs`. The restore applies a diff against `baseSha`, so a
|
|
1554
|
+
* checkout at any other commit would come out as neither tree. Parsed rather than `git rev-parse`,
|
|
1555
|
+
* although the target is a fresh host-made clone where rev-parse would be acceptable, for two reasons:
|
|
1556
|
+
* every store git keeps `GIT_DIR` pointing at the STORE (the bolt the tests hold), and the parser
|
|
1557
|
+
* already exists for the snapshot side. `null` when it cannot be read.
|
|
1558
|
+
*/
|
|
1559
|
+
async function checkoutHead(dir) {
|
|
1560
|
+
const gitDir = join(dir, ".git");
|
|
1561
|
+
if (!(await lstatOrNull(gitDir).catch(() => null))?.isDirectory()) return null;
|
|
1562
|
+
const head = await readRegular(join(gitDir, "HEAD"), 256);
|
|
1563
|
+
if (!head.buf) return null;
|
|
1564
|
+
const text = head.buf.toString("latin1");
|
|
1565
|
+
const sha = /^([0-9a-f]{40}|[0-9a-f]{64})\n?$/.exec(text);
|
|
1566
|
+
if (sha) return sha[1];
|
|
1567
|
+
const sym = /^ref: refs\/heads\/([^\n]+)\n?$/.exec(text);
|
|
1568
|
+
return sym && isBranchName(sym[1]) ? lookupBranchTip(gitDir, sym[1]) : null;
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
/**
|
|
1572
|
+
* Apply a checkpoint to `intoWorkspace`, which already holds a base checkout at the manifest's
|
|
1573
|
+
* `baseSha`. Only the changed paths are touched. Order: deletions and type changes (so a path that was a
|
|
1574
|
+
* file and is now a directory is free), then regular files, then symlinks LAST, so no write of this
|
|
1575
|
+
* restore can ever traverse a link this restore created. Files are opened `O_NOFOLLOW` with `O_TRUNC`
|
|
1576
|
+
* and their mode set explicitly (an existing file keeps its old mode through `O_CREAT`).
|
|
1577
|
+
*
|
|
1578
|
+
* A refusal found on DISK (a symlinked ancestor the base checkout holds) can come after some writes;
|
|
1579
|
+
* the caller discards a refused workspace, it never resumes in one.
|
|
1580
|
+
*/
|
|
1581
|
+
async function restore({ runId, manifest, intoWorkspace } = {}) {
|
|
1582
|
+
try {
|
|
1583
|
+
if (typeof runId !== "string" || !runId) return { skipped: "bad-args" };
|
|
1584
|
+
if (manifestProblem(manifest)) return { refused: "bad-manifest" };
|
|
1585
|
+
// A manifest from ANOTHER run's store applied here would restore that run's tree into this one.
|
|
1586
|
+
if (manifest.runId !== sanitizeJobId(runId).slice(0, 200)) return { refused: "run-mismatch" };
|
|
1587
|
+
const paths = runPaths(runId);
|
|
1588
|
+
if (!(await lstatOrNull(paths.run))?.isDirectory()) return { skipped: "not-primed" };
|
|
1589
|
+
if (!(await lstatOrNull(join(paths.repo, "HEAD")))) return { skipped: "not-primed" };
|
|
1590
|
+
if (!(await lstatOrNull(intoWorkspace))?.isDirectory()) return { skipped: "workspace-not-directory" };
|
|
1591
|
+
if ((await checkoutHead(intoWorkspace)) !== manifest.workspace.baseSha) return { refused: "base-mismatch" };
|
|
1592
|
+
return await withLock(paths, () => restoreLocked(paths, manifest, intoWorkspace));
|
|
1593
|
+
} catch (err) {
|
|
1594
|
+
if (err instanceof Refusal) return { refused: err.reason };
|
|
1595
|
+
log("checkpoint_failed", { phase: "restore", code: err?.code ?? "internal" });
|
|
1596
|
+
return { skipped: "internal-error" };
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
async function restoreLocked(paths, manifest, root) {
|
|
1601
|
+
const signal = AbortSignal.timeout(L.snapshotTimeoutMs);
|
|
1602
|
+
const { baseSha, commit } = manifest.workspace;
|
|
1603
|
+
const diff = await runGit(paths, ["diff-tree", "-r", "-z", "--raw", "--no-abbrev", "--no-renames", baseSha, commit], { signal });
|
|
1604
|
+
if (diff.code !== 0) return { skipped: "store-unreadable" };
|
|
1605
|
+
const changes = parseRaw(diff.stdout);
|
|
1606
|
+
if (!changes) return { refused: "tampered" };
|
|
1607
|
+
const tree = await runGit(paths, ["ls-tree", "-r", "-z", "--full-tree", commit], { signal });
|
|
1608
|
+
if (tree.code !== 0) return { skipped: "store-unreadable" };
|
|
1609
|
+
const commitTree = [];
|
|
1610
|
+
for (const rec of tree.stdout.toString("utf8").split("\0")) {
|
|
1611
|
+
const m = /^(\d{6}) \w+ [0-9a-f]+\t(.+)$/s.exec(rec);
|
|
1612
|
+
if (m) commitTree.push({ mode: m[1], path: m[2] });
|
|
1613
|
+
}
|
|
1614
|
+
const verdict = judgeChanges(changes, commitTree);
|
|
1615
|
+
if (verdict) {
|
|
1616
|
+
log("checkpoint_restore_refused", { reason: verdict });
|
|
1617
|
+
return { refused: verdict };
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
let removed = 0;
|
|
1621
|
+
let written = 0;
|
|
1622
|
+
for (const c of changes) {
|
|
1623
|
+
if (c.status !== "D" && c.status !== "T") continue;
|
|
1624
|
+
if (c.srcMode === "160000") continue;
|
|
1625
|
+
await ensureParents(root, c.path, false);
|
|
1626
|
+
const abs = join(root, c.path);
|
|
1627
|
+
const st = await lstatOrNull(abs);
|
|
1628
|
+
if (st?.isDirectory()) throw new Refusal("path-conflict");
|
|
1629
|
+
if (st) {
|
|
1630
|
+
await fs.unlink(abs);
|
|
1631
|
+
removed++;
|
|
1632
|
+
}
|
|
1633
|
+
if (c.status === "D") await pruneEmpty(root, c.path);
|
|
1634
|
+
}
|
|
1635
|
+
const blobs = blobReader(paths, signal);
|
|
1636
|
+
try {
|
|
1637
|
+
const ordered = changes.filter((c) => c.status !== "D");
|
|
1638
|
+
for (const c of [...ordered.filter((c) => c.dstMode !== "120000"), ...ordered.filter((c) => c.dstMode === "120000")]) {
|
|
1639
|
+
if (signal.aborted) return { skipped: "timeout" };
|
|
1640
|
+
await ensureParents(root, c.path, true);
|
|
1641
|
+
const abs = join(root, c.path);
|
|
1642
|
+
const bytes = await blobs.read(c.dstOid);
|
|
1643
|
+
const existing = await lstatOrNull(abs);
|
|
1644
|
+
if (existing?.isDirectory()) throw new Refusal("path-conflict");
|
|
1645
|
+
if (c.dstMode === "120000") {
|
|
1646
|
+
if (bytes.length === 0 || bytes.length > MAX_LINK_BYTES || bytes.includes(0)) throw new Refusal("tampered");
|
|
1647
|
+
if (existing) await fs.unlink(abs);
|
|
1648
|
+
await fs.symlink(bytes, abs);
|
|
1649
|
+
} else {
|
|
1650
|
+
const mode = c.dstMode === "100755" ? 0o755 : 0o644;
|
|
1651
|
+
let fh;
|
|
1652
|
+
try {
|
|
1653
|
+
fh = await fs.open(abs, FS.O_WRONLY | FS.O_CREAT | FS.O_TRUNC | FS.O_NOFOLLOW, mode);
|
|
1654
|
+
} catch (err) {
|
|
1655
|
+
if (err?.code === "ELOOP") throw new Refusal("path-through-symlink");
|
|
1656
|
+
throw err;
|
|
1657
|
+
}
|
|
1658
|
+
try {
|
|
1659
|
+
await fh.writeFile(bytes);
|
|
1660
|
+
await fh.chmod(mode);
|
|
1661
|
+
} finally {
|
|
1662
|
+
await fh.close().catch(() => {});
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
written++;
|
|
1666
|
+
}
|
|
1667
|
+
} finally {
|
|
1668
|
+
await blobs.close().catch(() => {});
|
|
1669
|
+
}
|
|
1670
|
+
log("checkpoint_restored", { seq: manifest.seq, written, removed });
|
|
1671
|
+
return { written, removed };
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
// ── remove, sweep, usage ────────────────────────────────────────────────────────────────────────────
|
|
1675
|
+
|
|
1676
|
+
async function remove(runId) {
|
|
1677
|
+
try {
|
|
1678
|
+
if (typeof runId !== "string" || !runId) return { skipped: "bad-args" };
|
|
1679
|
+
await fs.rm(runPaths(runId).run, { recursive: true, force: true });
|
|
1680
|
+
runs.delete(runId);
|
|
1681
|
+
return { removed: true };
|
|
1682
|
+
} catch (err) {
|
|
1683
|
+
log("checkpoint_failed", { phase: "remove", code: err?.code ?? "internal" });
|
|
1684
|
+
return { skipped: "remove-failed" };
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
/**
|
|
1689
|
+
* Remove run stores older than `ttlMs`. Only names that ARE run keys (32 hex) are judged, and only
|
|
1690
|
+
* when they lstat as a real directory: a link with a key's name is left alone, never followed into
|
|
1691
|
+
* whatever it points at. Age is the newest manifest's mtime (a run checkpointing yesterday is young
|
|
1692
|
+
* whatever its directory's mtime says), falling back to the directory's own. `active` names runs the
|
|
1693
|
+
* caller still holds; one yield between entries keeps a large store from monopolising the loop.
|
|
1694
|
+
*/
|
|
1695
|
+
async function sweep({ active = [] } = {}) {
|
|
1696
|
+
try {
|
|
1697
|
+
const keep = new Set([...active].map((id) => checkpointKey(String(id))));
|
|
1698
|
+
let names;
|
|
1699
|
+
try {
|
|
1700
|
+
names = await fs.readdir(storeDir);
|
|
1701
|
+
} catch (err) {
|
|
1702
|
+
if (err?.code === "ENOENT") return { removed: 0, kept: 0 };
|
|
1703
|
+
throw err;
|
|
1704
|
+
}
|
|
1705
|
+
let removed = 0;
|
|
1706
|
+
let kept = 0;
|
|
1707
|
+
for (const name of names) {
|
|
1708
|
+
await yieldTick();
|
|
1709
|
+
if (!/^[0-9a-f]{32}$/.test(name)) continue;
|
|
1710
|
+
const dir = join(storeDir, name);
|
|
1711
|
+
const st = await lstatOrNull(dir).catch(() => null);
|
|
1712
|
+
if (!st?.isDirectory() || keep.has(name)) {
|
|
1713
|
+
kept++;
|
|
1714
|
+
continue;
|
|
1715
|
+
}
|
|
1716
|
+
let newest = st.mtimeMs;
|
|
1717
|
+
let sawManifest = false;
|
|
1718
|
+
for (const n of await fs.readdir(dir).catch(() => [])) {
|
|
1719
|
+
if (!/^cp-[1-9][0-9]*$/.test(n)) continue;
|
|
1720
|
+
const m = await lstatOrNull(join(dir, n, "manifest.json")).catch(() => null);
|
|
1721
|
+
if (m?.isFile()) {
|
|
1722
|
+
newest = sawManifest ? Math.max(newest, m.mtimeMs) : m.mtimeMs;
|
|
1723
|
+
sawManifest = true;
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
if (now() - newest > L.ttlMs) {
|
|
1727
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
1728
|
+
removed++;
|
|
1729
|
+
} else {
|
|
1730
|
+
kept++;
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
if (removed) log("checkpoint_swept", { removed, kept });
|
|
1734
|
+
return { removed, kept };
|
|
1735
|
+
} catch (err) {
|
|
1736
|
+
log("checkpoint_failed", { phase: "sweep", code: err?.code ?? "internal" });
|
|
1737
|
+
return { skipped: "sweep-failed" };
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
async function usage() {
|
|
1742
|
+
try {
|
|
1743
|
+
return { bytes: await storeBytes() };
|
|
1744
|
+
} catch {
|
|
1745
|
+
return { skipped: "usage-failed" };
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
return { primeBase, snapshot, restore, remove, sweep, usage, readManifest, writeTranscript, baseSourceUrl, readBase };
|
|
1750
|
+
}
|