cool-workflow 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.gemini-plugin/mcp.json +10 -0
- package/.gemini-plugin/plugin.json +40 -0
- package/.opencode-plugin/mcp.json +10 -0
- package/.opencode-plugin/plugin.json +40 -0
- package/README.md +204 -48
- package/apps/architecture-review/app.json +1 -1
- package/apps/architecture-review-fast/app.json +1 -1
- package/apps/end-to-end-golden-path/app.json +1 -1
- package/apps/pr-review-fix-ci/app.json +1 -1
- package/apps/release-cut/app.json +1 -1
- package/apps/research-synthesis/app.json +1 -1
- package/dist/cli/dispatch.js +32 -4
- package/dist/cli/entry.js +11 -19
- package/dist/cli/global-flags.js +113 -0
- package/dist/cli/parseargv.js +7 -1
- package/dist/core/capability-data.js +337 -8
- package/dist/core/format/help.js +73 -3
- package/dist/core/format/recovery-hint.js +32 -0
- package/dist/core/multi-agent/collaboration.js +35 -6
- package/dist/core/multi-agent/runtime.js +7 -0
- package/dist/core/multi-agent/trust-policy.js +7 -1
- package/dist/core/pipeline/contract.js +7 -0
- package/dist/core/pipeline/error-feedback.js +2 -2
- package/dist/core/state/run-paths.js +3 -30
- package/dist/core/trust/evidence-grounding.js +13 -1
- package/dist/core/util/cli-args.js +22 -0
- package/dist/core/version.js +1 -1
- package/dist/mcp/dispatch.js +22 -2
- package/dist/mcp/server.js +215 -36
- package/dist/mcp/tool-process.js +181 -0
- package/dist/mcp-server.js +28 -1
- package/dist/shell/agent-config.js +11 -2
- package/dist/shell/audit-cli.js +1 -1
- package/dist/shell/commit.js +8 -2
- package/dist/shell/coordinator-io.js +73 -1
- package/dist/shell/drive.js +195 -128
- package/dist/shell/error-feedback-io.js +6 -0
- package/dist/shell/execution-backend/agent.js +201 -26
- package/dist/shell/execution-backend/container.js +44 -10
- package/dist/shell/execution-backend/local.js +33 -0
- package/dist/shell/fs-atomic.js +96 -12
- package/dist/shell/ledger-cli.js +9 -2
- package/dist/shell/metrics-cli.js +1 -1
- package/dist/shell/multi-agent-cli.js +9 -2
- package/dist/shell/multi-agent-operator-ux.js +1 -1
- package/dist/shell/observability.js +7 -4
- package/dist/shell/onramp.js +48 -5
- package/dist/shell/operator-ux.js +1 -1
- package/dist/shell/perf-trace.js +136 -0
- package/dist/shell/pipeline.js +3 -2
- package/dist/shell/reclamation-io.js +10 -9
- package/dist/shell/report-view-cli.js +2 -0
- package/dist/shell/run-export-cli.js +5 -2
- package/dist/shell/run-export.js +293 -31
- package/dist/shell/run-registry-io.js +8 -0
- package/dist/shell/run-store.js +183 -3
- package/dist/shell/scheduler-io.js +101 -10
- package/dist/shell/state-explosion-cli.js +4 -1
- package/dist/shell/telemetry-demo.js +1 -1
- package/dist/shell/telemetry-ledger-io.js +36 -24
- package/dist/shell/trust-audit.js +178 -36
- package/dist/shell/workbench-host.js +121 -10
- package/dist/shell/workbench.js +83 -6
- package/dist/shell/worker-isolation.js +1 -1
- package/dist/wiring/capability-table/basics.js +5 -0
- package/dist/wiring/capability-table/exec-backend.js +40 -22
- package/dist/wiring/capability-table/parity.js +10 -5
- package/dist/wiring/capability-table/pipeline.js +32 -0
- package/dist/wiring/capability-table/registry-core.js +26 -3
- package/dist/wiring/capability-table/reporting.js +7 -1
- package/dist/wiring/capability-table/scheduling-registry.js +8 -2
- package/dist/wiring/capability-table/trust-ledger.js +54 -12
- package/docs/agent-delegation-drive.7.md +17 -0
- package/docs/cli-mcp-parity.7.md +46 -5
- package/docs/contract-migration-tooling.7.md +4 -0
- package/docs/control-plane-scheduling.7.md +38 -0
- package/docs/cross-agent-ledger.7.md +20 -8
- package/docs/durable-state-and-locking.7.md +4 -0
- package/docs/evidence-adoption-reasoning-chain.7.md +4 -0
- package/docs/execution-backends.7.md +4 -0
- package/docs/fix.7.md +4 -4
- package/docs/getting-started.md +40 -32
- package/docs/index.md +17 -0
- package/docs/launch/demo.tape +4 -3
- package/docs/mcp-app-surface.7.md +27 -0
- package/docs/multi-agent-cli-mcp-surface.7.md +4 -0
- package/docs/multi-agent-eval-replay-harness.7.md +4 -0
- package/docs/multi-agent-operator-ux.7.md +4 -0
- package/docs/node-snapshot-diff-replay.7.md +4 -0
- package/docs/observability-cost-accounting.7.md +4 -0
- package/docs/project-index.md +53 -9
- package/docs/real-execution-backends.7.md +4 -0
- package/docs/release-and-migration.7.md +4 -0
- package/docs/release-tooling.7.md +12 -0
- package/docs/routine.7.md +22 -0
- package/docs/run-registry-control-plane.7.md +37 -9
- package/docs/run-retention-reclamation.7.md +12 -4
- package/docs/state-explosion-management.7.md +4 -0
- package/docs/team-collaboration.7.md +4 -0
- package/docs/trust-audit-anchor.7.md +4 -0
- package/docs/web-desktop-workbench.7.md +72 -2
- package/docs/workflow-app-framework.7.md +31 -0
- package/manifest/plugin.manifest.json +1 -1
- package/manifest/source-context-profiles.json +4 -3
- package/package.json +6 -3
- package/scripts/agents/agent-adapter-core.js +26 -0
- package/scripts/agents/claude-p-agent.js +4 -1
- package/scripts/agents/codex-agent.js +4 -0
- package/scripts/agents/cw-attest-wrap.js +1 -1
- package/scripts/agents/gemini-agent.js +4 -0
- package/scripts/agents/opencode-agent.js +5 -0
- package/scripts/block-unapproved-tag.js +160 -0
- package/scripts/canonical-apps.js +4 -4
- package/scripts/children/batch-delegate-child.js +72 -4
- package/scripts/children/http-batch-delegate-child.js +132 -0
- package/scripts/children/http-delegate-child.js +8 -0
- package/scripts/dogfood-release.js +2 -2
- package/scripts/fake-date-for-reproduction.js +1 -1
- package/scripts/gen-manifests.js +32 -61
- package/scripts/golden-path.js +4 -4
- package/scripts/lang-policy-check.js +82 -0
- package/scripts/parity-check.js +42 -23
- package/scripts/purity-baseline.json +0 -3
- package/scripts/purity-gate.js +16 -3
- package/scripts/release-check.js +2 -1
- package/scripts/release-flow.js +169 -41
- package/scripts/release-gate.js +208 -0
- package/scripts/release-oneclick.js +38 -7
- package/scripts/release-tags.js +47 -0
- package/scripts/schema-version-inventory.json +31 -0
- package/scripts/validate-run-state-schema.js +95 -4
- package/scripts/verdict-keygen.js +2 -2
- package/scripts/verify-bump-reproduction.js +193 -0
- package/scripts/verify-release-verdict.js +139 -0
- package/scripts/verify-verdict-signature.js +1 -1
- package/ui/workbench/app.css +50 -10
- package/ui/workbench/app.js +269 -34
- package/ui/workbench/index.html +4 -2
- package/ui/workbench/inspection.js +51 -0
- package/ui/workbench/navigation.js +44 -0
- package/docs/agent-framework.md +0 -177
- package/docs/designs/handoff-ledger.md +0 -145
- package/docs/dogfood/resume-drive-real-agent-2026-06-14.md +0 -40
- package/docs/launch/launch-kit.md +0 -195
- package/docs/launch/pre-launch-checklist.md +0 -53
- package/docs/readme-v0.1.87-full.md +0 -301
- package/docs/routines.md +0 -101
- package/docs/scheduled-tasks.md +0 -80
- package/scripts/agents/claude-p-agent.sh +0 -9
- package/scripts/block-unapproved-tag.sh +0 -75
- package/scripts/release-gate.sh +0 -94
- package/scripts/verify-bump-reproduction.sh +0 -148
- package/scripts/verify-container-selfref.js +0 -64
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
// MILESTONE 5 (docs/rebuild/PLAN.md build order, step 5). Byte-exact port of
|
|
6
6
|
// plugins/cool-workflow/src/execution-backend/agent.ts. This module holds
|
|
7
7
|
// the PURE data-transform helpers (invocation resolution, arg substitution,
|
|
8
|
-
// secret redaction, report parsing) plus the batch delegate-child spawn
|
|
8
|
+
// secret redaction, report parsing) plus the batch delegate-child spawn — both
|
|
9
|
+
// the CLI-binary batch (runAgentBatchOutcomes) and, added later, its
|
|
10
|
+
// HTTP-endpoint sibling (runEndpointBatchOutcomes) so `--concurrency N` gives
|
|
11
|
+
// endpoint-mode agents real concurrency instead of a serial per-task spawn.
|
|
9
12
|
//
|
|
10
13
|
// THE RED LINE: CW spawns the agent and records its attested output. It
|
|
11
14
|
// NEVER imports a model SDK, holds an API key, or constructs a model API
|
|
@@ -49,6 +52,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
49
52
|
};
|
|
50
53
|
})();
|
|
51
54
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
+
exports.reapRecordedVendor = reapRecordedVendor;
|
|
52
56
|
exports.buildAgentChildEnv = buildAgentChildEnv;
|
|
53
57
|
exports.resolveAgentInvocation = resolveAgentInvocation;
|
|
54
58
|
exports.stripSecretArgs = stripSecretArgs;
|
|
@@ -59,15 +63,19 @@ exports.recordedAgentHandle = recordedAgentHandle;
|
|
|
59
63
|
exports.extractEndpointResult = extractEndpointResult;
|
|
60
64
|
exports.agentHandle = agentHandle;
|
|
61
65
|
exports.prepareAgentSpawn = prepareAgentSpawn;
|
|
66
|
+
exports.prepareEndpointJob = prepareEndpointJob;
|
|
62
67
|
exports.reconcileBatchOutcomes = reconcileBatchOutcomes;
|
|
63
68
|
exports.runAgentBatchOutcomes = runAgentBatchOutcomes;
|
|
69
|
+
exports.runEndpointBatchOutcomes = runEndpointBatchOutcomes;
|
|
64
70
|
exports.shouldStreamAgentStderr = shouldStreamAgentStderr;
|
|
65
71
|
exports.runAgentProcess = runAgentProcess;
|
|
66
72
|
const fs = __importStar(require("node:fs"));
|
|
67
73
|
const path = __importStar(require("node:path"));
|
|
74
|
+
const os = __importStar(require("node:os"));
|
|
68
75
|
const node_child_process_1 = require("node:child_process");
|
|
69
76
|
const local_1 = require("./local");
|
|
70
77
|
const envelopes_1 = require("./envelopes");
|
|
78
|
+
const perf_trace_1 = require("../perf-trace");
|
|
71
79
|
function messageOf(error) {
|
|
72
80
|
return error instanceof Error ? error.message : String(error);
|
|
73
81
|
}
|
|
@@ -85,6 +93,46 @@ const AGENT_PROVIDER_KEY_ENV_RE = /^(CW_|ANTHROPIC_|OPENAI_|GEMINI_|DEEPSEEK_|CO
|
|
|
85
93
|
* call it, so the allowlist cannot drift into a second silent copy again.
|
|
86
94
|
* Returns the forwarded var NAMES too (never values) for the
|
|
87
95
|
* worker.agent-env trust-audit event. */
|
|
96
|
+
/** A unique sidecar path a shipped wrapper writes its vendor child's PID to
|
|
97
|
+
* (via the env CW_AGENT_VENDOR_PIDFILE we set below), so cw can reap the
|
|
98
|
+
* vendor if it has to SIGKILL the wrapper on a timeout. */
|
|
99
|
+
function vendorPidFilePath() {
|
|
100
|
+
return path.join(os.tmpdir(), `cw-agent-vendor-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.pid`);
|
|
101
|
+
}
|
|
102
|
+
/** Reap the vendor process a shipped wrapper recorded, then remove the sidecar.
|
|
103
|
+
* A wrapper that exited cleanly already removed the file, so this is a no-op
|
|
104
|
+
* then; only a wrapper cw SIGKILLed on a timeout leaves a live vendor PID
|
|
105
|
+
* here. Best-effort and race-tolerant: a missing file, an unparseable/too-low
|
|
106
|
+
* PID, or an already-gone process are all silently fine. Scope: kills the
|
|
107
|
+
* vendor process itself (the token spender), not a deeper grandchild tree,
|
|
108
|
+
* and only for the shipped vendor wrappers (claude, codex, gemini, and
|
|
109
|
+
* opencode/deepseek) -- an arbitrary CW_AGENT_COMMAND records no PID and is
|
|
110
|
+
* not covered. Returns true only if it
|
|
111
|
+
* actually signalled a process. Exported for tests; agent.ts is not part of
|
|
112
|
+
* the package public surface (index.ts). */
|
|
113
|
+
function reapRecordedVendor(pidFile) {
|
|
114
|
+
let raw;
|
|
115
|
+
try {
|
|
116
|
+
raw = fs.readFileSync(pidFile, "utf8");
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
return false; // no sidecar => the wrapper cleaned up (or never recorded one)
|
|
120
|
+
}
|
|
121
|
+
try {
|
|
122
|
+
fs.unlinkSync(pidFile);
|
|
123
|
+
}
|
|
124
|
+
catch { /* already gone */ }
|
|
125
|
+
const pid = Number(raw.trim());
|
|
126
|
+
if (!Number.isInteger(pid) || pid <= 1)
|
|
127
|
+
return false;
|
|
128
|
+
try {
|
|
129
|
+
process.kill(pid, "SIGKILL");
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return false; // already exited, or a pid we may not signal
|
|
134
|
+
}
|
|
135
|
+
}
|
|
88
136
|
function buildAgentChildEnv(policy, baseEnv = process.env) {
|
|
89
137
|
const env = (0, local_1.buildChildEnv)(policy, baseEnv);
|
|
90
138
|
// deny must win here too: buildChildEnv already applied it above, but the
|
|
@@ -98,6 +146,14 @@ function buildAgentChildEnv(policy, baseEnv = process.env) {
|
|
|
98
146
|
for (const key of Object.keys(baseEnv)) {
|
|
99
147
|
if (denied.has(key))
|
|
100
148
|
continue;
|
|
149
|
+
// Parent-only CW secrets are never re-added, even though they match the
|
|
150
|
+
// CW_ arm of AGENT_PROVIDER_KEY_ENV_RE and even if the operator forgot to
|
|
151
|
+
// deny them. buildChildEnv already stripped them above; this keeps the
|
|
152
|
+
// re-add loop from putting them back (and out of the trust-audit
|
|
153
|
+
// `forwarded` list). CW_AGENT_ATTEST_PRIVKEY is intentionally NOT in that
|
|
154
|
+
// set, so the attest wrapper still gets its signing key.
|
|
155
|
+
if (local_1.CW_NEVER_FORWARD_ENV.has(key))
|
|
156
|
+
continue;
|
|
101
157
|
if (AGENT_PROVIDER_KEY_ENV_RE.test(key)) {
|
|
102
158
|
env[key] = baseEnv[key];
|
|
103
159
|
forwarded.push(key);
|
|
@@ -297,9 +353,9 @@ function agentHandle(request, env = process.env) {
|
|
|
297
353
|
},
|
|
298
354
|
};
|
|
299
355
|
}
|
|
300
|
-
/** Resolve a request to a spawn-style batch job, or undefined when
|
|
301
|
-
* is endpoint-configured
|
|
302
|
-
* path). */
|
|
356
|
+
/** Resolve a request to a spawn-style (CLI-binary) batch job, or undefined when
|
|
357
|
+
* the agent is endpoint-configured (see prepareEndpointJob) or unconfigured
|
|
358
|
+
* (those settle through the serial path). */
|
|
303
359
|
function prepareAgentSpawn(request) {
|
|
304
360
|
const resolved = resolveAgentInvocation(request);
|
|
305
361
|
if (!resolved.binary)
|
|
@@ -312,6 +368,27 @@ function prepareAgentSpawn(request) {
|
|
|
312
368
|
timeoutMs: resolved.timeoutMs || 600000,
|
|
313
369
|
};
|
|
314
370
|
}
|
|
371
|
+
/** Resolve a request to an endpoint (HTTP-delegate) batch job, or undefined when
|
|
372
|
+
* the agent has a CLI binary (see prepareAgentSpawn) or is unconfigured. The
|
|
373
|
+
* POST body mirrors runAgentEndpoint's exactly: {manifest, prompt, model,
|
|
374
|
+
* resultPath, sandboxProfileId} — sandboxProfileId comes from
|
|
375
|
+
* request.sandboxPolicy.id, which runBackend sets to the same policy the serial
|
|
376
|
+
* path passes (registry.ts: `const policy = request.sandboxPolicy`), so the two
|
|
377
|
+
* paths post identical bytes. */
|
|
378
|
+
function prepareEndpointJob(request) {
|
|
379
|
+
const resolved = resolveAgentInvocation(request);
|
|
380
|
+
if (resolved.binary || !resolved.endpoint)
|
|
381
|
+
return undefined;
|
|
382
|
+
const manifest = request.manifest;
|
|
383
|
+
const job = JSON.stringify({
|
|
384
|
+
manifest,
|
|
385
|
+
prompt: manifest?.prompt,
|
|
386
|
+
model: resolved.model,
|
|
387
|
+
resultPath: manifest?.resultPath,
|
|
388
|
+
sandboxProfileId: request.sandboxPolicy.id,
|
|
389
|
+
});
|
|
390
|
+
return { endpoint: resolved.endpoint, job, timeoutMs: resolved.timeoutMs || 600000 };
|
|
391
|
+
}
|
|
315
392
|
// __dirname is dist/shell/execution-backend at runtime — THREE levels up
|
|
316
393
|
// (execution-backend -> shell -> dist -> package root) reaches scripts/,
|
|
317
394
|
// a sibling of dist/, not two (that was a bug: it resolved to a
|
|
@@ -360,12 +437,44 @@ function reconcileBatchOutcomes(jobs, child) {
|
|
|
360
437
|
* jobs. See SPEC/execution-backend.md's rebuild-risk note on maxBuffer
|
|
361
438
|
* scaling with job count and the no-`encoding` (raw Buffer) requirement. */
|
|
362
439
|
function runAgentBatchOutcomes(jobs) {
|
|
440
|
+
if (!jobs.length)
|
|
441
|
+
return [];
|
|
442
|
+
return (0, perf_trace_1.withPerfTraceGroup)("agent-wait", () => {
|
|
443
|
+
const maxTimeout = Math.max(...jobs.map((job) => job.timeoutMs));
|
|
444
|
+
let child;
|
|
445
|
+
try {
|
|
446
|
+
child = (0, node_child_process_1.spawnSync)(process.execPath, [BATCH_DELEGATE_CHILD_SCRIPT], {
|
|
447
|
+
input: JSON.stringify(jobs),
|
|
448
|
+
maxBuffer: 34 * 1024 * 1024 * jobs.length,
|
|
449
|
+
timeout: maxTimeout + 30000,
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
catch (error) {
|
|
453
|
+
child = { error: error instanceof Error ? error : new Error(String(error)), status: null, stdout: null };
|
|
454
|
+
}
|
|
455
|
+
return reconcileBatchOutcomes(jobs, child);
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
// Same package-root depth fix as BATCH_DELEGATE_CHILD_SCRIPT above.
|
|
459
|
+
const HTTP_BATCH_DELEGATE_CHILD_SCRIPT = path.resolve(__dirname, "..", "..", "..", "scripts", "children", "http-batch-delegate-child.js");
|
|
460
|
+
/** Run a batch of ENDPOINT (HTTP-delegate) jobs concurrently; outcomes
|
|
461
|
+
* index-align with jobs. The endpoint sibling of runAgentBatchOutcomes: one
|
|
462
|
+
* spawnSync of http-batch-delegate-child.js POSTs all N at once from a single
|
|
463
|
+
* process, so `--concurrency N` with an endpoint agent runs N delegations in
|
|
464
|
+
* parallel instead of serially. The child streams the SAME NDJSON line shape as
|
|
465
|
+
* batch-delegate-child.js — `{i, spawnError?, exitCode, stdout}` — so the same
|
|
466
|
+
* reconcileBatchOutcomes reads it (byte-boundary split on a Buffer, index-aligned,
|
|
467
|
+
* every job that produced no full line failed closed). Env is inherited (the
|
|
468
|
+
* serial runAgentEndpoint likewise runs the child with the full process env for
|
|
469
|
+
* any endpoint auth); no sandbox child-env is built here, matching the serial
|
|
470
|
+
* endpoint path. */
|
|
471
|
+
function runEndpointBatchOutcomes(jobs) {
|
|
363
472
|
if (!jobs.length)
|
|
364
473
|
return [];
|
|
365
474
|
const maxTimeout = Math.max(...jobs.map((job) => job.timeoutMs));
|
|
366
475
|
let child;
|
|
367
476
|
try {
|
|
368
|
-
child = (0, node_child_process_1.spawnSync)(process.execPath, [
|
|
477
|
+
child = (0, node_child_process_1.spawnSync)(process.execPath, [HTTP_BATCH_DELEGATE_CHILD_SCRIPT], {
|
|
369
478
|
input: JSON.stringify(jobs),
|
|
370
479
|
maxBuffer: 34 * 1024 * 1024 * jobs.length,
|
|
371
480
|
timeout: maxTimeout + 30000,
|
|
@@ -415,10 +524,16 @@ function runAgentProcess(descriptor, policy, request, label, attestation) {
|
|
|
415
524
|
else {
|
|
416
525
|
const streamStderr = shouldStreamAgentStderr(process.env, Boolean(process.stderr.isTTY));
|
|
417
526
|
const built = buildAgentChildEnv(policy);
|
|
418
|
-
const childEnv = built.env;
|
|
419
527
|
forwardedEnvVars = built.forwarded;
|
|
420
528
|
const timeoutMs = resolved.timeoutMs || 600000;
|
|
421
|
-
|
|
529
|
+
// A shipped wrapper (claude/codex/gemini) records its vendor child's PID
|
|
530
|
+
// here so we can reap the vendor if the timeout below SIGKILLs the
|
|
531
|
+
// wrapper -- a SIGKILL is uncatchable, so the wrapper cannot forward the
|
|
532
|
+
// stop itself. Not a forwarded secret; it is cw plumbing, so it stays out
|
|
533
|
+
// of `built.forwarded` / the trust-audit forwarded list.
|
|
534
|
+
const vendorPidFile = vendorPidFilePath();
|
|
535
|
+
const childEnv = { ...built.env, CW_AGENT_VENDOR_PIDFILE: vendorPidFile };
|
|
536
|
+
const child = (0, perf_trace_1.withPerfTraceGroup)("agent-wait", () => (0, node_child_process_1.spawnSync)(resolved.binary, realArgs, {
|
|
422
537
|
cwd: request.cwd,
|
|
423
538
|
env: childEnv,
|
|
424
539
|
encoding: "utf8",
|
|
@@ -432,19 +547,37 @@ function runAgentProcess(descriptor, policy, request, label, attestation) {
|
|
|
432
547
|
// anyway — so a hard kill loses nothing and cannot wedge cw.
|
|
433
548
|
killSignal: "SIGKILL",
|
|
434
549
|
stdio: ["ignore", "pipe", streamStderr ? "inherit" : "pipe"],
|
|
435
|
-
});
|
|
550
|
+
}));
|
|
436
551
|
// A timeout surfaces as child.error with code ETIMEDOUT (status null,
|
|
437
552
|
// signal set). Classify it FIRST: without this, the generic spawnError
|
|
438
553
|
// branch below reported "agent process failed to spawn: ...ETIMEDOUT"
|
|
439
554
|
// and the null-exit-code "timed out or killed" branch was dead code
|
|
440
555
|
// for real timeouts.
|
|
441
556
|
if (child.error && child.error.code === "ETIMEDOUT") {
|
|
557
|
+
// spawnSync SIGKILLed the wrapper (uncatchable, so the wrapper could
|
|
558
|
+
// not forward the stop to its vendor child). Reap the vendor a shipped
|
|
559
|
+
// wrapper recorded, so a timed-out agent's vendor process does not live
|
|
560
|
+
// on as an orphan and keep spending. Scoped: reaps the vendor process
|
|
561
|
+
// itself for the shipped wrappers (claude, codex, gemini, opencode/
|
|
562
|
+
// deepseek), not a deeper grandchild tree nor an arbitrary
|
|
563
|
+
// CW_AGENT_COMMAND (which records no PID). We do
|
|
564
|
+
// NOT use `detached:true` -- that would make the wrapper its own group
|
|
565
|
+
// leader, so an interactive Ctrl-C / a group SIGINT to cw would no
|
|
566
|
+
// longer reach it (strictly worse); reaping from the recorded PID keeps
|
|
567
|
+
// the wrapper in cw's group.
|
|
568
|
+
reapRecordedVendor(vendorPidFile);
|
|
442
569
|
const handleOut = recordedAgentHandle(resolved.binary, undefined, recordedArgs, resolved.model, "unreported", undefined, undefined, forwardedEnvVars);
|
|
443
570
|
return (0, envelopes_1.refusedEnvelope)(descriptor, policy, label, "delegation-failed", `agent process timed out after ${timeoutMs}ms and was killed (SIGKILL)`, {
|
|
444
571
|
...attestation,
|
|
445
572
|
handle: handleOut,
|
|
446
573
|
});
|
|
447
574
|
}
|
|
575
|
+
// Non-timeout paths: a wrapper that ran to any normal end removed its own
|
|
576
|
+
// sidecar on exit; clear any stray file best-effort so tmp cannot grow.
|
|
577
|
+
try {
|
|
578
|
+
fs.unlinkSync(vendorPidFile);
|
|
579
|
+
}
|
|
580
|
+
catch { /* usually already gone */ }
|
|
448
581
|
outcome = {
|
|
449
582
|
...(child.error ? { spawnError: messageOf(child.error) } : {}),
|
|
450
583
|
exitCode: typeof child.status === "number" ? child.status : null,
|
|
@@ -473,12 +606,70 @@ function runAgentProcess(descriptor, policy, request, label, attestation) {
|
|
|
473
606
|
}
|
|
474
607
|
return (0, envelopes_1.refusedEnvelope)(descriptor, policy, label, "delegation-target-missing", `Backend ${descriptor.id} has no command-template or endpoint configured`, attestation);
|
|
475
608
|
}
|
|
609
|
+
/** Settle a completed endpoint delegation (a numeric exitCode + stdout) into
|
|
610
|
+
* its envelope: parse the agent report, write result.md as transport when the
|
|
611
|
+
* runner returned a body and none exists yet, record the handle. Shared by the
|
|
612
|
+
* serial spawn path and the concurrent prepared-outcome path so the two never
|
|
613
|
+
* drift. The result-write stays HERE (the cw shell process), never in the batch
|
|
614
|
+
* child, so the child does no filesystem writes into the run dir. */
|
|
615
|
+
function settleEndpointResult(descriptor, label, endpoint, resolvedModel, manifest, attestation, exitCode, stdout) {
|
|
616
|
+
const report = parseAgentReport(stdout);
|
|
617
|
+
if (manifest?.resultPath && report.usage === undefined) {
|
|
618
|
+
const body = extractEndpointResult(stdout);
|
|
619
|
+
if (body && !fs.existsSync(manifest.resultPath)) {
|
|
620
|
+
try {
|
|
621
|
+
fs.writeFileSync(manifest.resultPath, body, "utf8");
|
|
622
|
+
}
|
|
623
|
+
catch {
|
|
624
|
+
/* the accept layer will fail closed on a missing result.md */
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
const reportedModel = report.model && report.model.trim() ? report.model.trim() : "unreported";
|
|
629
|
+
const handleOut = recordedAgentHandle(undefined, endpoint, [], resolvedModel, reportedModel, report.usage, report.usageSignature);
|
|
630
|
+
return (0, envelopes_1.delegatedEnvelope)(descriptor, label, handleOut, { ...attestation, handle: handleOut }, "agent-endpoint", [endpoint], exitCode, stdout);
|
|
631
|
+
}
|
|
476
632
|
/** Agent HTTP endpoint variant — POSTs the worker manifest/prompt to a
|
|
477
633
|
* configured agent endpoint via the shared Node delegate child. Byte-exact
|
|
478
|
-
* port of src/execution-backend.ts:1002-1062
|
|
634
|
+
* port of src/execution-backend.ts:1002-1062, plus a prepared-outcome branch:
|
|
635
|
+
* in a concurrent round the POST already ran in the batch child
|
|
636
|
+
* (runEndpointBatchOutcomes), so settle that pre-collected outcome instead of
|
|
637
|
+
* spawning again. The serial (no prepared outcome) path is unchanged. */
|
|
479
638
|
function runAgentEndpoint(descriptor, policy, request, label, resolved, attestation) {
|
|
480
639
|
const endpoint = resolved.endpoint;
|
|
481
640
|
const manifest = request.manifest;
|
|
641
|
+
const baseHandle = recordedAgentHandle(undefined, endpoint, [], resolved.model, "unreported");
|
|
642
|
+
// Concurrent round: the batch child already POSTed and settled this job. Any
|
|
643
|
+
// failure (network, non-2xx, poll, unparseable, timeout, no exitCode) is a
|
|
644
|
+
// spawnError; a success carries a numeric exitCode + stdout.
|
|
645
|
+
if (request.preparedAgentOutcome) {
|
|
646
|
+
const prep = request.preparedAgentOutcome;
|
|
647
|
+
if (prep.spawnError) {
|
|
648
|
+
// Match the SERIAL path's refusal wording so a concurrent round records the
|
|
649
|
+
// same reason bytes as a serial one. A per-job HTTP failure (non-2xx, poll,
|
|
650
|
+
// timeout, no exitCode) is the serial `{error}` branch — `agent endpoint
|
|
651
|
+
// error:`. A whole-batch-child process failure surfaces as
|
|
652
|
+
// reconcileBatchOutcomes' `batch delegate failed:` fallback — the serial
|
|
653
|
+
// analog is `child.error` — `agent endpoint delegation failed:`.
|
|
654
|
+
const summary = prep.spawnError.startsWith("batch delegate failed:")
|
|
655
|
+
? `agent endpoint delegation failed: ${prep.spawnError}`
|
|
656
|
+
: `agent endpoint error: ${prep.spawnError}`;
|
|
657
|
+
return (0, envelopes_1.refusedEnvelope)(descriptor, policy, label, "delegation-failed", summary, {
|
|
658
|
+
...attestation,
|
|
659
|
+
handle: baseHandle,
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
// Defensive: the batch child always pairs a null exitCode with a spawnError,
|
|
663
|
+
// so this is unreachable in practice — kept as a fail-closed backstop with
|
|
664
|
+
// the same wording the serial no-exitCode refusal uses.
|
|
665
|
+
if (typeof prep.exitCode !== "number") {
|
|
666
|
+
return (0, envelopes_1.refusedEnvelope)(descriptor, policy, label, "delegation-failed", `agent endpoint error: no exitCode reported`, {
|
|
667
|
+
...attestation,
|
|
668
|
+
handle: baseHandle,
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
return settleEndpointResult(descriptor, label, endpoint, resolved.model, manifest, attestation, prep.exitCode, prep.stdout);
|
|
672
|
+
}
|
|
482
673
|
const job = JSON.stringify({
|
|
483
674
|
manifest,
|
|
484
675
|
prompt: manifest?.prompt,
|
|
@@ -493,7 +684,6 @@ function runAgentEndpoint(descriptor, policy, request, label, resolved, attestat
|
|
|
493
684
|
timeout: resolved.timeoutMs || 600000,
|
|
494
685
|
maxBuffer: 32 * 1024 * 1024,
|
|
495
686
|
});
|
|
496
|
-
const baseHandle = recordedAgentHandle(undefined, endpoint, [], resolved.model, "unreported");
|
|
497
687
|
if (child.error) {
|
|
498
688
|
return (0, envelopes_1.refusedEnvelope)(descriptor, policy, label, "delegation-failed", `agent endpoint delegation failed: ${messageOf(child.error)}`, {
|
|
499
689
|
...attestation,
|
|
@@ -516,20 +706,5 @@ function runAgentEndpoint(descriptor, policy, request, label, resolved, attestat
|
|
|
516
706
|
handle: baseHandle,
|
|
517
707
|
});
|
|
518
708
|
}
|
|
519
|
-
|
|
520
|
-
const report = parseAgentReport(stdout);
|
|
521
|
-
if (manifest?.resultPath && report.usage === undefined) {
|
|
522
|
-
const body = extractEndpointResult(stdout);
|
|
523
|
-
if (body && !fs.existsSync(manifest.resultPath)) {
|
|
524
|
-
try {
|
|
525
|
-
fs.writeFileSync(manifest.resultPath, body, "utf8");
|
|
526
|
-
}
|
|
527
|
-
catch {
|
|
528
|
-
/* the accept layer will fail closed on a missing result.md */
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
const reportedModel = report.model && report.model.trim() ? report.model.trim() : "unreported";
|
|
533
|
-
const handleOut = recordedAgentHandle(undefined, endpoint, [], resolved.model, reportedModel, report.usage, report.usageSignature);
|
|
534
|
-
return (0, envelopes_1.delegatedEnvelope)(descriptor, label, handleOut, { ...attestation, handle: handleOut }, "agent-endpoint", [endpoint], parsed.exitCode, stdout);
|
|
709
|
+
return settleEndpointResult(descriptor, label, endpoint, resolved.model, manifest, attestation, parsed.exitCode, String(parsed.stdout || ""));
|
|
535
710
|
}
|
|
@@ -7,14 +7,49 @@
|
|
|
7
7
|
//
|
|
8
8
|
// Evidence: SPEC/execution-backend.md "container driver (runContainer)".
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.buildContainerEnvArgs = buildContainerEnvArgs;
|
|
10
11
|
exports.containerHandle = containerHandle;
|
|
11
12
|
exports.runContainer = runContainer;
|
|
12
13
|
const node_child_process_1 = require("node:child_process");
|
|
13
14
|
const probes_1 = require("./probes");
|
|
15
|
+
const local_1 = require("./local");
|
|
14
16
|
const envelopes_1 = require("./envelopes");
|
|
15
17
|
function messageOf(error) {
|
|
16
18
|
return error instanceof Error ? error.message : String(error);
|
|
17
19
|
}
|
|
20
|
+
/** buildContainerEnvArgs — build the `-e NAME=value` args passed into the
|
|
21
|
+
* container run command. Pulled out as its own pure function so it can be
|
|
22
|
+
* checked without a real docker/podman on the box (see local.ts's
|
|
23
|
+
* buildChildEnv for the same shape of test-only split).
|
|
24
|
+
*
|
|
25
|
+
* deny must win here too: an operator who sets inherit:true plus a deny
|
|
26
|
+
* list (a valid, normalized combination — see sandbox-profile.ts's
|
|
27
|
+
* normalizeEnv) means "everything EXCEPT these". Before this fix, deny was
|
|
28
|
+
* never read here, so a secret like AWS_SECRET_ACCESS_KEY that the operator
|
|
29
|
+
* named in deny still got copied into the container's `-e` args.
|
|
30
|
+
*
|
|
31
|
+
* This backend builds its `-e` args straight from baseEnv rather than through
|
|
32
|
+
* buildChildEnv, so it must apply CW_NEVER_FORWARD_ENV itself — otherwise
|
|
33
|
+
* inherit:true would still copy CW's own parent-only secrets (the release
|
|
34
|
+
* signing key, the workbench token) into the container. */
|
|
35
|
+
function buildContainerEnvArgs(policy, baseEnv = process.env) {
|
|
36
|
+
const args = [];
|
|
37
|
+
if (policy.env.inherit || (policy.env.expose && policy.env.expose.length)) {
|
|
38
|
+
const deny = new Set(policy.env.deny || []);
|
|
39
|
+
for (const name of policy.env.inherit ? Object.keys(baseEnv) : policy.env.expose || []) {
|
|
40
|
+
if (name === "PATH" || name === "HOME")
|
|
41
|
+
continue;
|
|
42
|
+
if (deny.has(name))
|
|
43
|
+
continue;
|
|
44
|
+
if (local_1.CW_NEVER_FORWARD_ENV.has(name))
|
|
45
|
+
continue; // parent-only secrets never enter the container
|
|
46
|
+
const value = baseEnv[name];
|
|
47
|
+
if (value !== undefined)
|
|
48
|
+
args.push("-e", `${name}=${value}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return args;
|
|
52
|
+
}
|
|
18
53
|
function containerHandle(request, env = process.env) {
|
|
19
54
|
const delegation = request.delegation || {};
|
|
20
55
|
const image = delegation.image || (env.CW_CONTAINER_IMAGE || "").trim() || undefined;
|
|
@@ -46,20 +81,19 @@ function runContainer(descriptor, policy, request, label, handle, attestation) {
|
|
|
46
81
|
if (policy.network.mode !== "any")
|
|
47
82
|
runArgs.push("--network", "none");
|
|
48
83
|
runArgs.push("-v", `${cwd}:${cwd}:ro`, "-w", cwd);
|
|
49
|
-
|
|
50
|
-
for (const name of policy.env.inherit ? Object.keys(process.env) : policy.env.expose || []) {
|
|
51
|
-
if (name === "PATH" || name === "HOME")
|
|
52
|
-
continue;
|
|
53
|
-
const value = process.env[name];
|
|
54
|
-
if (value !== undefined)
|
|
55
|
-
runArgs.push("-e", `${name}=${value}`);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
84
|
+
runArgs.push(...buildContainerEnvArgs(policy));
|
|
58
85
|
runArgs.push(handle.ref, command, ...args);
|
|
59
86
|
// An unset timeoutMs must not mean "no timeout" — spawnSync would then
|
|
60
87
|
// block forever on a hung container with no kill path. 600000 matches the
|
|
61
88
|
// agent backend's own default fallback (execution-backend/agent.ts).
|
|
62
|
-
|
|
89
|
+
// killSignal SIGKILL, not the default SIGTERM: a container runtime that
|
|
90
|
+
// ignores SIGTERM would leave this blocking spawnSync waiting forever (no
|
|
91
|
+
// second-stage escalation is possible from inside a sync call), so the
|
|
92
|
+
// timeout above would not actually bound anything. SIGKILL is uncatchable,
|
|
93
|
+
// so the runtime always dies and spawnSync returns; a timed-out run is
|
|
94
|
+
// classified the same either way (status null, ETIMEDOUT message does not
|
|
95
|
+
// name the signal, result.signal is never recorded). Mirrors agent.ts.
|
|
96
|
+
const result = (0, node_child_process_1.spawnSync)(runtime, runArgs, { cwd, encoding: "utf8", timeout: request.timeoutMs || 600000, maxBuffer: 32 * 1024 * 1024, killSignal: "SIGKILL" });
|
|
63
97
|
if (result.error) {
|
|
64
98
|
return (0, envelopes_1.refusedEnvelope)(descriptor, policy, label, "delegation-failed", `${runtime} run failed: ${messageOf(result.error)}`, attestation);
|
|
65
99
|
}
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
//
|
|
9
9
|
// Evidence: SPEC/execution-backend.md "Local execution (executeLocal)".
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.CW_NEVER_FORWARD_ENV = void 0;
|
|
11
12
|
exports.buildChildEnv = buildChildEnv;
|
|
12
13
|
exports.checkShellGuard = checkShellGuard;
|
|
13
14
|
exports.runtimeNoteFor = runtimeNoteFor;
|
|
@@ -18,6 +19,20 @@ const probes_1 = require("./probes");
|
|
|
18
19
|
function messageOf(error) {
|
|
19
20
|
return error instanceof Error ? error.message : String(error);
|
|
20
21
|
}
|
|
22
|
+
/** CW's OWN secrets that must NEVER reach ANY child process — on any backend
|
|
23
|
+
* (local/container/remote/agent), regardless of policy.env.inherit / expose /
|
|
24
|
+
* deny. Each is read only by parent-side tooling, so no spawned child ever
|
|
25
|
+
* needs it; stripping them here, unconditionally, is a fail-closed backstop
|
|
26
|
+
* that does not depend on an operator remembering to list them in
|
|
27
|
+
* policy.env.deny. Do NOT add the agent-attest private key
|
|
28
|
+
* (CW_AGENT_ATTEST_PRIVKEY) here: the attest wrapper
|
|
29
|
+
* (scripts/agents/cw-attest-wrap.js) is spawned AS the agent and must receive
|
|
30
|
+
* it to sign telemetry. */
|
|
31
|
+
exports.CW_NEVER_FORWARD_ENV = new Set([
|
|
32
|
+
"CW_RELEASE_VERDICT_PRIVKEY", // release verdict signing key — release-flow.js only
|
|
33
|
+
"CW_WORKBENCH_TOKEN", // workbench HTTP bearer token — workbench-host.ts only
|
|
34
|
+
"CW_BENCH_TRACE_FILE", // benchmark trace path — parent drive only
|
|
35
|
+
]);
|
|
21
36
|
function buildChildEnv(policy, baseEnv = process.env) {
|
|
22
37
|
// deny must win regardless of inherit: a custom profile combining
|
|
23
38
|
// inherit:true with deny:[...] (a valid, normalized combination — see
|
|
@@ -38,6 +53,10 @@ function buildChildEnv(policy, baseEnv = process.env) {
|
|
|
38
53
|
for (const name of policy.env.deny || []) {
|
|
39
54
|
delete env[name];
|
|
40
55
|
}
|
|
56
|
+
// Fail-closed backstop: parent-only secrets never cross into a child, even
|
|
57
|
+
// when inherit:true forwarded them above or an operator exposed one by name.
|
|
58
|
+
for (const name of exports.CW_NEVER_FORWARD_ENV)
|
|
59
|
+
delete env[name];
|
|
41
60
|
return env;
|
|
42
61
|
}
|
|
43
62
|
/** Shell injection guard (SPEC/execution-backend.md "Local execution"): for
|
|
@@ -75,6 +94,20 @@ function executeLocal(descriptor, request, label, attestation, spawnStyle) {
|
|
|
75
94
|
// agent backend's own default fallback (execution-backend/agent.ts).
|
|
76
95
|
timeout: request.timeoutMs || 600000,
|
|
77
96
|
maxBuffer: 32 * 1024 * 1024,
|
|
97
|
+
// SIGKILL, not the default SIGTERM: a child that ignores SIGTERM would
|
|
98
|
+
// leave this blocking spawnSync waiting forever (no second-stage
|
|
99
|
+
// escalation is possible from inside a sync call), so the `timeout` above
|
|
100
|
+
// would not actually bound anything. SIGKILL is uncatchable, so the child
|
|
101
|
+
// always dies and spawnSync returns. A timed-out run is classified as
|
|
102
|
+
// failed regardless of the signal (status stays null, the ETIMEDOUT
|
|
103
|
+
// message does not name the signal, and this backend never records
|
|
104
|
+
// result.signal). The exact captured stdout/stderr on a timeout is not
|
|
105
|
+
// guaranteed identical to the old SIGTERM path: a child that catches
|
|
106
|
+
// SIGTERM could flush a last partial write or exit with a code before it
|
|
107
|
+
// dies, while under SIGKILL it cannot — but that output belongs to a run
|
|
108
|
+
// we already treat as failed, so the hard kill loses nothing that counts.
|
|
109
|
+
// Mirrors execution-backend/agent.ts's fix.
|
|
110
|
+
killSignal: "SIGKILL",
|
|
78
111
|
};
|
|
79
112
|
if (spawnStyle === "shell") {
|
|
80
113
|
checkShellGuard(command, args);
|
package/dist/shell/fs-atomic.js
CHANGED
|
@@ -48,9 +48,12 @@ exports.assertSafeRunId = assertSafeRunId;
|
|
|
48
48
|
exports.realResolve = realResolve;
|
|
49
49
|
exports.isContainedPath = isContainedPath;
|
|
50
50
|
exports.durableAppendFileSync = durableAppendFileSync;
|
|
51
|
+
exports.logEndsWithNewline = logEndsWithNewline;
|
|
52
|
+
exports.nextBackoffMs = nextBackoffMs;
|
|
51
53
|
exports.withFileLock = withFileLock;
|
|
52
54
|
const fs = __importStar(require("node:fs"));
|
|
53
55
|
const path = __importStar(require("node:path"));
|
|
56
|
+
const perf_trace_1 = require("./perf-trace");
|
|
54
57
|
// ---------------------------------------------------------------------------
|
|
55
58
|
// writeJson — the ONE JSON-on-disk byte format: JSON.stringify(value, null, 2)
|
|
56
59
|
// + "\n", written via temp-file + optional fsync + rename (atomic).
|
|
@@ -102,6 +105,7 @@ function writeBytesAtomic(file, contents, durable) {
|
|
|
102
105
|
catch {
|
|
103
106
|
/* directory fsync is best-effort (not supported on every platform) */
|
|
104
107
|
}
|
|
108
|
+
(0, perf_trace_1.recordPerfDurableWrite)(Buffer.byteLength(contents, "utf8"));
|
|
105
109
|
}
|
|
106
110
|
}
|
|
107
111
|
/** Atomic, optionally-durable JSON write — see `writeBytesAtomic`. */
|
|
@@ -195,19 +199,57 @@ function durableAppendFileSync(file, data) {
|
|
|
195
199
|
finally {
|
|
196
200
|
fs.closeSync(fd);
|
|
197
201
|
}
|
|
202
|
+
(0, perf_trace_1.recordPerfDurableWrite)(Buffer.byteLength(data, "utf8"));
|
|
203
|
+
}
|
|
204
|
+
/** True when `file`'s final byte is "\n", given its already-known `size`.
|
|
205
|
+
* A COMPLETED `durableAppendFileSync` always leaves the file ending in
|
|
206
|
+
* "\n" (every append is `<line>\n`), so a non-newline last byte means the
|
|
207
|
+
* previous append was torn by a crash — its bytes were never a confirmed
|
|
208
|
+
* record. Reads ONLY the last byte at `size-1`, so callers on an append
|
|
209
|
+
* hot path stay O(1) and never re-read the whole file. A read failure
|
|
210
|
+
* returns false (treat as "not newline-terminated") — the safe side: an
|
|
211
|
+
* extra leading newline is harmless for an NDJSON reader that skips blank
|
|
212
|
+
* lines, while a MISSED torn boundary would merge two records into one
|
|
213
|
+
* unparseable line. Shared by trust-audit's events.jsonl and the
|
|
214
|
+
* blackboard's messages.jsonl append paths — same file shape, same
|
|
215
|
+
* torn-tail risk, one implementation. */
|
|
216
|
+
function logEndsWithNewline(file, size) {
|
|
217
|
+
if (size <= 0)
|
|
218
|
+
return false;
|
|
219
|
+
let fd;
|
|
220
|
+
try {
|
|
221
|
+
fd = fs.openSync(file, "r");
|
|
222
|
+
const buf = Buffer.alloc(1);
|
|
223
|
+
fs.readSync(fd, buf, 0, 1, size - 1);
|
|
224
|
+
return buf[0] === 0x0a; // "\n"
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
finally {
|
|
230
|
+
if (fd !== undefined)
|
|
231
|
+
fs.closeSync(fd);
|
|
232
|
+
}
|
|
198
233
|
}
|
|
199
234
|
// ---------------------------------------------------------------------------
|
|
200
235
|
// withFileLock — portable advisory cross-process lock.
|
|
201
236
|
//
|
|
202
237
|
// Lock file: `<targetPath>.lock`, created by hard-linking a per-attempt temp
|
|
203
238
|
// file onto the lock path (single-winner `link(2)`), body `"<pid>@<ISO>\n"`.
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
// (Atomics.wait busy-safe sleep
|
|
209
|
-
//
|
|
210
|
-
//
|
|
239
|
+
// On EEXIST, a lock whose mtime is older than FILE_LOCK_STALE_MS (30_000) is
|
|
240
|
+
// stolen — judged and deleted only while holding the single-winner
|
|
241
|
+
// `<lock>.steal` guard (see stealStaleLockUnderGuard) — and retried AT ONCE
|
|
242
|
+
// (so is a lock that vanished between the EEXIST and the stat). Otherwise the
|
|
243
|
+
// thread sleeps a short, growing backoff (Atomics.wait busy-safe sleep:
|
|
244
|
+
// FILE_LOCK_BACKOFF_BASE_MS doubling toward FILE_LOCK_BACKOFF_MAX_MS, with
|
|
245
|
+
// jitter so many processes contending on ONE lock do not retry in lock-step)
|
|
246
|
+
// and tries again. The WHOLE acquire is bounded by wall-clock, not a try
|
|
247
|
+
// count: after FILE_LOCK_ACQUIRE_BUDGET_MS (~6s, kept well under the 30s steal
|
|
248
|
+
// window so a fresh orphan lock fails fast rather than being waited out) with
|
|
249
|
+
// no lock, it throws `could not acquire file lock for <targetPath>`. The tiny
|
|
250
|
+
// early sleeps re-grab a briefly-held lock in a few ms; the wall-clock bound
|
|
251
|
+
// (not the try count) is what keeps the worst case at ~6s. Any non-EEXIST
|
|
252
|
+
// link error is rethrown.
|
|
211
253
|
//
|
|
212
254
|
// Right before fn() the lock mtime is refreshed (utimesSync, best-effort).
|
|
213
255
|
// After fn() returns, the lock body is re-read: if it no longer starts with
|
|
@@ -227,15 +269,37 @@ const FILE_LOCK_FORCE_STALE_MS = FILE_LOCK_STALE_MS * 10;
|
|
|
227
269
|
// guarded window (which is a handful of syscalls, microseconds for any live
|
|
228
270
|
// process) — remove it so stealing cannot wedge forever.
|
|
229
271
|
const FILE_LOCK_STEAL_GUARD_STALE_MS = FILE_LOCK_STALE_MS;
|
|
272
|
+
// Acquire-retry pacing (all internal to withFileLock — no wire/protocol
|
|
273
|
+
// meaning). The wall-clock BUDGET, not a fixed try count, bounds how long a
|
|
274
|
+
// contended acquire blocks the calling thread; kept at ~6s so it stays well
|
|
275
|
+
// under the 30s steal window (a fresh orphan lock is failed fast, not waited
|
|
276
|
+
// out — a later command past 30s steals it). Between misses the thread sleeps
|
|
277
|
+
// a backoff that starts at BASE and doubles to MAX, so a briefly-held lock is
|
|
278
|
+
// re-grabbed in a few ms while a genuinely busy one is not hot-spun. Jitter on
|
|
279
|
+
// each sleep (applied at the call site) de-syncs many contenders on one lock.
|
|
280
|
+
// MAX_ATTEMPTS is only a loop-termination backstop for the (never-observed)
|
|
281
|
+
// case of a clock that fails to advance; the budget is the real bound.
|
|
282
|
+
const FILE_LOCK_ACQUIRE_BUDGET_MS = 6_000;
|
|
283
|
+
const FILE_LOCK_BACKOFF_BASE_MS = 1;
|
|
284
|
+
const FILE_LOCK_BACKOFF_MAX_MS = 50;
|
|
285
|
+
const FILE_LOCK_MAX_ATTEMPTS = 10_000;
|
|
230
286
|
// Lock paths this process holds right now. A nested withFileLock on the
|
|
231
287
|
// SAME target runs its fn directly (re-entrant) instead of waiting on its
|
|
232
|
-
// own lock file until the
|
|
288
|
+
// own lock file until the acquire budget runs out — that lets a whole
|
|
233
289
|
// load -> change -> save cycle hold one lock while the save path inside
|
|
234
290
|
// it keeps its own withFileLock call unchanged.
|
|
235
291
|
const HELD_LOCKS = new Set();
|
|
236
292
|
function sleepSync(ms) {
|
|
237
293
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
238
294
|
}
|
|
295
|
+
/** Next backoff ceiling (ms) given the previous one: double it, capped at
|
|
296
|
+
* FILE_LOCK_BACKOFF_MAX_MS. Pure and deterministic (the per-sleep jitter is
|
|
297
|
+
* applied separately at the call site), so the retry schedule is
|
|
298
|
+
* unit-testable without any timing. Exported for that test only — fs-atomic
|
|
299
|
+
* is not part of the public index.ts surface. */
|
|
300
|
+
function nextBackoffMs(previous) {
|
|
301
|
+
return Math.min(previous * 2, FILE_LOCK_BACKOFF_MAX_MS);
|
|
302
|
+
}
|
|
239
303
|
/** Whether the pid recorded in a lock body ("<pid>@<ISO>\n") is still a
|
|
240
304
|
* live process on this machine. Locks here are local-advisory only (no
|
|
241
305
|
* network-shared run directories in this tool's model), so
|
|
@@ -375,7 +439,9 @@ function withFileLock(targetPath, fn) {
|
|
|
375
439
|
fs.mkdirSync(path.dirname(lock), { recursive: true });
|
|
376
440
|
const pid = String(process.pid);
|
|
377
441
|
let acquired = false;
|
|
378
|
-
|
|
442
|
+
const deadline = Date.now() + FILE_LOCK_ACQUIRE_BUDGET_MS;
|
|
443
|
+
let backoff = FILE_LOCK_BACKOFF_BASE_MS;
|
|
444
|
+
for (let attempt = 0; attempt < FILE_LOCK_MAX_ATTEMPTS && !acquired; attempt++) {
|
|
379
445
|
// Acquire via `linkSync`, not `open(lock, "wx")`: directly testing the
|
|
380
446
|
// retry loop under contention showed `open(O_CREAT|O_EXCL)` is NOT
|
|
381
447
|
// reliably single-winner on every Node version this runs on — under
|
|
@@ -421,16 +487,34 @@ function withFileLock(targetPath, fn) {
|
|
|
421
487
|
// The steal itself runs under a single-winner guard lock — see
|
|
422
488
|
// stealStaleLockUnderGuard for why judging staleness and deleting
|
|
423
489
|
// the lock without one can never be made safe from here.
|
|
424
|
-
if (stealStaleLockUnderGuard(lock, pid, attempt))
|
|
490
|
+
if (stealStaleLockUnderGuard(lock, pid, attempt)) {
|
|
491
|
+
// Verdict rendered and the lock likely just freed — retry AT ONCE
|
|
492
|
+
// and reset the backoff so we grab it before another waiter does.
|
|
493
|
+
backoff = FILE_LOCK_BACKOFF_BASE_MS;
|
|
425
494
|
continue;
|
|
495
|
+
}
|
|
426
496
|
// Guard was busy — another process is mid-steal. Back off like
|
|
427
|
-
// ordinary contention instead of hot-spinning
|
|
497
|
+
// ordinary contention instead of hot-spinning.
|
|
428
498
|
}
|
|
429
499
|
}
|
|
430
500
|
catch {
|
|
501
|
+
// The lock vanished between the EEXIST and the stat — it is likely free
|
|
502
|
+
// now, so retry AT ONCE (no sleep) with a reset backoff.
|
|
503
|
+
backoff = FILE_LOCK_BACKOFF_BASE_MS;
|
|
431
504
|
continue;
|
|
432
505
|
}
|
|
433
|
-
|
|
506
|
+
// Give up on wall-clock, never on a raw try count: this keeps the worst-
|
|
507
|
+
// case block at ~6s even though the per-sleep backoff grew. Bounding by a
|
|
508
|
+
// fixed try count instead (as the old flat-25ms loop's 240 tries did)
|
|
509
|
+
// would have let the growing backoff roughly double that worst case.
|
|
510
|
+
if (Date.now() >= deadline)
|
|
511
|
+
break;
|
|
512
|
+
// Sleep a jittered span in [BASE, backoff] — never 0, so never a busy
|
|
513
|
+
// spin — then grow the backoff toward the cap. The jitter de-syncs many
|
|
514
|
+
// processes contending on the SAME lock so they do not retry in lock-step.
|
|
515
|
+
const span = backoff - FILE_LOCK_BACKOFF_BASE_MS;
|
|
516
|
+
sleepSync(FILE_LOCK_BACKOFF_BASE_MS + Math.floor(Math.random() * (span + 1)));
|
|
517
|
+
backoff = nextBackoffMs(backoff);
|
|
434
518
|
}
|
|
435
519
|
if (!acquired)
|
|
436
520
|
throw new Error(`could not acquire file lock for ${targetPath}`);
|