intentdna 1.9.0 → 1.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +227 -87
- package/dist/cli/commands/run-lifecycle.d.ts +61 -11
- package/dist/cli/commands/run-lifecycle.js +184 -50
- package/dist/cli/commands/run-observer.d.ts +15 -0
- package/dist/cli/commands/run-observer.js +210 -0
- package/dist/cli/commands/run.d.ts +16 -20
- package/dist/cli/commands/run.js +461 -659
- package/dist/cli/commands/sync.js +16 -12
- package/dist/cli/index.js +14 -5
- package/dist/compiler/compile.d.ts +1 -0
- package/dist/compiler/compile.js +1 -1
- package/dist/compiler/controller.d.ts +16 -0
- package/dist/compiler/controller.js +693 -0
- package/dist/compiler/input-resolver.d.ts +2 -0
- package/dist/compiler/input-resolver.js +22 -6
- package/dist/hooks/cli.d.ts +45 -0
- package/dist/hooks/cli.js +263 -31
- package/dist/hooks/host-event.d.ts +57 -0
- package/dist/hooks/host-event.js +187 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +1 -0
- package/dist/hooks/protocol.d.ts +7 -0
- package/dist/hooks/schema.d.ts +1 -0
- package/dist/hooks/schema.js +89 -1
- package/dist/hooks/state.d.ts +23 -1
- package/dist/hooks/state.js +9 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/index.js +2 -0
- package/dist/mcp/tools-run.d.ts +4 -0
- package/dist/mcp/tools-run.js +419 -0
- package/dist/mcp/tools-state.d.ts +1 -1
- package/dist/mcp/tools-state.js +13 -241
- package/dist/runtime/artifact-store.d.ts +146 -0
- package/dist/runtime/artifact-store.js +1436 -0
- package/dist/runtime/canonical-attempt-outcome.d.ts +12 -0
- package/dist/runtime/canonical-attempt-outcome.js +195 -0
- package/dist/runtime/canonical-json.d.ts +10 -0
- package/dist/runtime/canonical-json.js +105 -0
- package/dist/runtime/canonical-run-application.d.ts +93 -0
- package/dist/runtime/canonical-run-application.js +229 -0
- package/dist/runtime/canonical-run-service.d.ts +454 -0
- package/dist/runtime/canonical-run-service.js +4760 -0
- package/dist/runtime/canonical-runtime-composition.d.ts +22 -0
- package/dist/runtime/canonical-runtime-composition.js +34 -0
- package/dist/runtime/canonical-runtime-projection.d.ts +3 -0
- package/dist/runtime/canonical-runtime-projection.js +4 -0
- package/dist/runtime/canonical-target-compiler.d.ts +46 -0
- package/dist/runtime/canonical-target-compiler.js +514 -0
- package/dist/runtime/claude-agent-identity.d.ts +2 -0
- package/dist/runtime/claude-agent-identity.js +21 -0
- package/dist/runtime/executable-run-plan.d.ts +328 -0
- package/dist/runtime/executable-run-plan.js +1485 -0
- package/dist/runtime/execution-authority.d.ts +83 -0
- package/dist/runtime/execution-authority.js +98 -0
- package/dist/runtime/handoff-resolver.js +4 -1
- package/dist/runtime/harness-pull-adapter.d.ts +146 -0
- package/dist/runtime/harness-pull-adapter.js +315 -0
- package/dist/runtime/index.d.ts +39 -14
- package/dist/runtime/index.js +29 -7
- package/dist/runtime/local-execution-authority.d.ts +194 -0
- package/dist/runtime/local-execution-authority.js +2897 -0
- package/dist/runtime/local-execution-reconciliation.d.ts +20 -0
- package/dist/runtime/local-execution-reconciliation.js +107 -0
- package/dist/runtime/local-execution-supervisor-script.d.ts +7 -0
- package/dist/runtime/local-execution-supervisor-script.js +793 -0
- package/dist/runtime/local-provider-sandbox.d.ts +42 -0
- package/dist/runtime/local-provider-sandbox.js +154 -0
- package/dist/runtime/local-verifier-execution.d.ts +24 -0
- package/dist/runtime/local-verifier-execution.js +290 -0
- package/dist/runtime/local-windows-execution-supervisor-script.d.ts +6 -0
- package/dist/runtime/local-windows-execution-supervisor-script.js +788 -0
- package/dist/runtime/plan-store.d.ts +30 -0
- package/dist/runtime/plan-store.js +231 -0
- package/dist/runtime/process-tree.d.ts +9 -0
- package/dist/runtime/process-tree.js +97 -24
- package/dist/runtime/providers/codex.js +2 -2
- package/dist/runtime/push-driver.d.ts +184 -0
- package/dist/runtime/push-driver.js +1002 -0
- package/dist/runtime/result-store.d.ts +34 -3
- package/dist/runtime/result-store.js +1073 -2
- package/dist/runtime/run-binding.d.ts +68 -0
- package/dist/runtime/run-binding.js +278 -0
- package/dist/runtime/run-contracts.d.ts +575 -0
- package/dist/runtime/run-contracts.js +58 -7
- package/dist/runtime/run-controller.d.ts +1 -6
- package/dist/runtime/run-controller.js +0 -41
- package/dist/runtime/run-store.d.ts +85 -7
- package/dist/runtime/run-store.js +2528 -186
- package/dist/runtime/skill-adapter.d.ts +18 -7
- package/dist/runtime/skill-adapter.js +100 -742
- package/dist/runtime/structured-output-validator.d.ts +44 -0
- package/dist/runtime/structured-output-validator.js +171 -0
- package/dist/runtime/verifier-command-binding.d.ts +22 -0
- package/dist/runtime/verifier-command-binding.js +162 -0
- package/dist/runtime/verifier.d.ts +46 -6
- package/dist/runtime/verifier.js +355 -53
- package/dist/runtime/windows-job-keeper.d.ts +47 -0
- package/dist/runtime/windows-job-keeper.js +231 -0
- package/dist/runtime/worker-executor.d.ts +17 -1
- package/dist/runtime/worker-executor.js +26 -5
- package/dist/runtime/workflow-plan-adapter.d.ts +19 -0
- package/dist/runtime/workflow-plan-adapter.js +502 -6
- package/dist/runtime/workflow-runtime-manifest.d.ts +1 -0
- package/dist/runtime/workflow-runtime-manifest.js +5 -0
- package/dist/runtime/workspace-isolation.d.ts +29 -2
- package/dist/runtime/workspace-isolation.js +488 -11
- package/dist/runtime/workspace-observation.d.ts +44 -0
- package/dist/runtime/workspace-observation.js +216 -0
- package/dist/schema/controller-registry.d.ts +0 -7
- package/dist/schema/controller-registry.js +0 -8
- package/dist/schema/types.d.ts +84 -10
- package/dist/schema/types.js +2 -0
- package/dist/schema/validate.js +151 -1
- package/dist/schema/validators/controllers.d.ts +1 -1
- package/dist/schema/validators/controllers.js +126 -41
- package/dist/schema/workflow-authoring-contract.js +2 -0
- package/dist/schema/yaml-parser.js +1 -1
- package/dist/templates/flutter-rewrite-new.dna.yaml +1 -0
- package/dist/templates/flutter-rewrite.dna.yaml +87 -60
- package/dist/templates/safe-refactoring.dna.yaml +1 -0
- package/native/windows-job-keeper/README.md +80 -0
- package/native/windows-job-keeper/bin/aarch64/intentdna-windows-job-keeper.exe +0 -0
- package/native/windows-job-keeper/bin/x86_64/intentdna-windows-job-keeper.exe +0 -0
- package/native/windows-job-keeper/keeper.c +979 -0
- package/native/windows-job-keeper/manifest.json +33 -0
- package/package.json +6 -2
- package/scripts/build-windows-job-keeper.mjs +172 -0
- package/scripts/verify-windows-job-keeper.mjs +184 -0
|
@@ -0,0 +1,2897 @@
|
|
|
1
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { chmodSync, closeSync, constants as fsConstants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, } from "node:fs";
|
|
4
|
+
import { connect } from "node:net";
|
|
5
|
+
import { basename, dirname, join, relative, resolve, win32 as win32Path } from "node:path";
|
|
6
|
+
import { canonicalizeJson } from "./canonical-json.js";
|
|
7
|
+
import { validateCanonicalAttemptOutcome } from "./canonical-attempt-outcome.js";
|
|
8
|
+
import { createWorkerAlreadyStoppedEvidence, createWorkerStopReceipt, workerAlreadyStoppedEvidenceDigest, workerAlreadyStoppedEvidenceMatchesStop, workerNonlaunchAttestationDigest, workerStopReceiptDigest, } from "./execution-authority.js";
|
|
9
|
+
import { LOCAL_EXECUTION_SUPERVISOR_DIGEST, LOCAL_EXECUTION_SUPERVISOR_SOURCE, } from "./local-execution-supervisor-script.js";
|
|
10
|
+
import { LOCAL_WINDOWS_EXECUTION_SUPERVISOR_DIGEST, LOCAL_WINDOWS_EXECUTION_SUPERVISOR_SOURCE, } from "./local-windows-execution-supervisor-script.js";
|
|
11
|
+
import { resolveProcessCommand, } from "./process-tree.js";
|
|
12
|
+
import { resolveWindowsJobKeeperArtifact, WindowsJobKeeperArtifactError, } from "./windows-job-keeper.js";
|
|
13
|
+
const RECORD_SCHEMA_VERSION = "intentdna.local_execution_record.v1";
|
|
14
|
+
const CONTROL_SCHEMA_VERSION = "intentdna.local_execution_control.v1";
|
|
15
|
+
const AUTHORITY_SCHEMA_VERSION = "intentdna.local_execution_authority.v1";
|
|
16
|
+
const PREPARATION_SCHEMA_VERSION = "intentdna.local_execution_preparation.v1";
|
|
17
|
+
const PREPARATION_HANDLE_SCHEMA_VERSION = "intentdna.local_execution_preparation_handle.v1";
|
|
18
|
+
const NONLAUNCH_EVIDENCE_SCHEMA_VERSION = "intentdna.local_nonlaunch_evidence.v1";
|
|
19
|
+
const DIRECTORY_MODE = 0o700;
|
|
20
|
+
const PRIVATE_FILE_MODE = 0o600;
|
|
21
|
+
const SCRIPT_FILE_MODE = 0o500;
|
|
22
|
+
const DEFAULT_CONTROL_TIMEOUT_MS = 10_000;
|
|
23
|
+
const DEFAULT_POLL_INTERVAL_MS = 10;
|
|
24
|
+
const DEFAULT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
|
|
25
|
+
const MAX_CONTROL_RESPONSE_BYTES = 1024 * 1024;
|
|
26
|
+
export class LocalExecutionAuthorityError extends Error {
|
|
27
|
+
code;
|
|
28
|
+
constructor(code, message, options) {
|
|
29
|
+
super(message, options);
|
|
30
|
+
this.name = "LocalExecutionAuthorityError";
|
|
31
|
+
this.code = code;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function durableLockOwnersEqual(left, right) {
|
|
35
|
+
return left !== null && canonicalizeJson(left) === canonicalizeJson(right);
|
|
36
|
+
}
|
|
37
|
+
function durableLockOwnerDigest(owner) {
|
|
38
|
+
return createHash("sha256").update(canonicalizeJson(owner), "utf8").digest("hex");
|
|
39
|
+
}
|
|
40
|
+
function recoveryIdentity(request) {
|
|
41
|
+
return {
|
|
42
|
+
run_id: request.run_id,
|
|
43
|
+
step_id: request.step_id,
|
|
44
|
+
activation_id: request.activation_id,
|
|
45
|
+
attempt_id: request.attempt_id,
|
|
46
|
+
claim_epoch: request.execution_claim_epoch,
|
|
47
|
+
worker_session_id: request.executor.worker_session_id,
|
|
48
|
+
started_at: request.started_at,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function requirePreparationRequest(request) {
|
|
52
|
+
for (const [name, value] of Object.entries({
|
|
53
|
+
run_id: request.run_id,
|
|
54
|
+
step_id: request.step_id,
|
|
55
|
+
activation_id: request.activation_id,
|
|
56
|
+
attempt_id: request.attempt_id,
|
|
57
|
+
worker_session_id: request.worker_session_id,
|
|
58
|
+
})) {
|
|
59
|
+
if (typeof value !== "string" || value.length === 0 || value.includes("\0")) {
|
|
60
|
+
throw new TypeError(`${name} must be a non-empty string without NUL bytes`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (!Number.isSafeInteger(request.execution_claim_epoch) || request.execution_claim_epoch <= 0) {
|
|
64
|
+
throw new TypeError("execution_claim_epoch must be a positive safe integer");
|
|
65
|
+
}
|
|
66
|
+
if (request.execution_instance_id !== undefined
|
|
67
|
+
&& (request.execution_instance_id.length === 0 || request.execution_instance_id.includes("\0"))) {
|
|
68
|
+
throw new TypeError("execution_instance_id must be a non-empty string without NUL bytes");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function preparationCoreIdentity(value) {
|
|
72
|
+
return {
|
|
73
|
+
run_id: value.run_id,
|
|
74
|
+
step_id: value.step_id,
|
|
75
|
+
activation_id: value.activation_id,
|
|
76
|
+
attempt_id: value.attempt_id,
|
|
77
|
+
execution_claim_epoch: value.execution_claim_epoch,
|
|
78
|
+
worker_session_id: value.worker_session_id,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
function preparationDigest(value) {
|
|
82
|
+
return digest({
|
|
83
|
+
schema_version: PREPARATION_SCHEMA_VERSION,
|
|
84
|
+
identity: preparationCoreIdentity(value),
|
|
85
|
+
execution_instance_id: value.execution_instance_id,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
function preparedHandle(preparation) {
|
|
89
|
+
return {
|
|
90
|
+
schema_version: PREPARATION_HANDLE_SCHEMA_VERSION,
|
|
91
|
+
...preparationCoreIdentity(preparation.identity),
|
|
92
|
+
execution_instance_id: preparation.execution_instance_id,
|
|
93
|
+
preparation_digest: preparation.preparation_digest,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function digest(value) {
|
|
97
|
+
return `sha256:${createHash("sha256").update(canonicalizeJson(value), "utf8").digest("hex")}`;
|
|
98
|
+
}
|
|
99
|
+
function textDigest(value) {
|
|
100
|
+
return `sha256:${createHash("sha256").update(value, "utf8").digest("hex")}`;
|
|
101
|
+
}
|
|
102
|
+
function digestBytes(value) {
|
|
103
|
+
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
104
|
+
}
|
|
105
|
+
function isErrno(error, code) {
|
|
106
|
+
return error instanceof Error
|
|
107
|
+
&& "code" in error
|
|
108
|
+
&& error.code === code;
|
|
109
|
+
}
|
|
110
|
+
function requirePositiveInteger(value, label) {
|
|
111
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
112
|
+
throw new LocalExecutionAuthorityError("invalid_configuration", `${label} must be a positive safe integer`);
|
|
113
|
+
}
|
|
114
|
+
return value;
|
|
115
|
+
}
|
|
116
|
+
function isCanonicalTimestamp(value) {
|
|
117
|
+
return typeof value === "string"
|
|
118
|
+
&& Number.isFinite(Date.parse(value))
|
|
119
|
+
&& new Date(Date.parse(value)).toISOString() === value;
|
|
120
|
+
}
|
|
121
|
+
function isCanonicalDigest(value) {
|
|
122
|
+
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value);
|
|
123
|
+
}
|
|
124
|
+
function isPositiveSafeInteger(value) {
|
|
125
|
+
return Number.isSafeInteger(value) && value > 0;
|
|
126
|
+
}
|
|
127
|
+
function isCanonicalWorkerOutcome(value, workerSessionId) {
|
|
128
|
+
try {
|
|
129
|
+
validateCanonicalAttemptOutcome({ kind: "worker", worker_session_id: workerSessionId }, value);
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function requireIdentity(identity) {
|
|
137
|
+
for (const [name, value] of Object.entries({
|
|
138
|
+
run_id: identity.run_id,
|
|
139
|
+
step_id: identity.step_id,
|
|
140
|
+
activation_id: identity.activation_id,
|
|
141
|
+
attempt_id: identity.attempt_id,
|
|
142
|
+
worker_session_id: identity.worker_session_id,
|
|
143
|
+
started_at: identity.started_at,
|
|
144
|
+
})) {
|
|
145
|
+
if (typeof value !== "string" || value.length === 0 || value.includes("\0")) {
|
|
146
|
+
throw new TypeError(`${name} must be a non-empty string without NUL bytes`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (!Number.isSafeInteger(identity.claim_epoch) || identity.claim_epoch <= 0) {
|
|
150
|
+
throw new TypeError("claim_epoch must be a positive safe integer");
|
|
151
|
+
}
|
|
152
|
+
if (!isCanonicalTimestamp(identity.started_at)) {
|
|
153
|
+
throw new TypeError("started_at must be a canonical RFC 3339 timestamp");
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function coreIdentity(value) {
|
|
157
|
+
return {
|
|
158
|
+
run_id: value.run_id,
|
|
159
|
+
step_id: value.step_id,
|
|
160
|
+
activation_id: value.activation_id,
|
|
161
|
+
attempt_id: value.attempt_id,
|
|
162
|
+
claim_epoch: value.claim_epoch,
|
|
163
|
+
worker_session_id: value.worker_session_id,
|
|
164
|
+
started_at: value.started_at,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
function identityDigest(value) {
|
|
168
|
+
return digest(coreIdentity(value));
|
|
169
|
+
}
|
|
170
|
+
function attemptDigest(value) {
|
|
171
|
+
return digest({
|
|
172
|
+
schema_version: "intentdna.local_execution_attempt_key.v1",
|
|
173
|
+
run_id: value.run_id,
|
|
174
|
+
step_id: value.step_id,
|
|
175
|
+
attempt_id: value.attempt_id,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
function executionRecordDigest(value) {
|
|
179
|
+
return digest({
|
|
180
|
+
schema_version: "intentdna.local_execution_record_key.v2",
|
|
181
|
+
run_id: value.run_id,
|
|
182
|
+
step_id: value.step_id,
|
|
183
|
+
attempt_id: value.attempt_id,
|
|
184
|
+
claim_epoch: value.claim_epoch,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
function requestIdentity(value) {
|
|
188
|
+
return {
|
|
189
|
+
run_id: value.run_id,
|
|
190
|
+
step_id: value.step_id,
|
|
191
|
+
activation_id: value.activation_id,
|
|
192
|
+
attempt_id: value.attempt_id,
|
|
193
|
+
claim_epoch: value.claim_epoch,
|
|
194
|
+
worker_session_id: value.executor.worker_session_id,
|
|
195
|
+
started_at: "started_at" in value ? value.started_at : value.reported_execution.started_at,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
function processMatchesCanonical(actual, canonical) {
|
|
199
|
+
return canonical.kind === "worker_process"
|
|
200
|
+
&& canonicalizeJson(actual) === canonicalizeJson(canonical.process);
|
|
201
|
+
}
|
|
202
|
+
function terminalOutcomeToCanonical(process) {
|
|
203
|
+
if (process.kind === "success") {
|
|
204
|
+
return {
|
|
205
|
+
kind: "worker_process",
|
|
206
|
+
status: "succeeded",
|
|
207
|
+
reason_code: null,
|
|
208
|
+
message: null,
|
|
209
|
+
process,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
if (process.kind === "cancelled") {
|
|
213
|
+
return {
|
|
214
|
+
kind: "worker_process",
|
|
215
|
+
status: "cancelled",
|
|
216
|
+
reason_code: "cancelled",
|
|
217
|
+
message: process.reason,
|
|
218
|
+
process,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
kind: "worker_process",
|
|
223
|
+
status: "failed",
|
|
224
|
+
reason_code: process.kind,
|
|
225
|
+
message: process.kind === "malformed_result" || process.kind === "launch_error"
|
|
226
|
+
? process.error
|
|
227
|
+
: null,
|
|
228
|
+
process,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
function ensureRealPrivateDirectory(path) {
|
|
232
|
+
mkdirSync(path, { recursive: true, mode: DIRECTORY_MODE });
|
|
233
|
+
const metadata = lstatSync(path);
|
|
234
|
+
if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
|
|
235
|
+
throw new LocalExecutionAuthorityError("corrupt_state", `authority path is not a real directory: ${path}`);
|
|
236
|
+
}
|
|
237
|
+
chmodSync(path, DIRECTORY_MODE);
|
|
238
|
+
}
|
|
239
|
+
function syncDirectory(path) {
|
|
240
|
+
let descriptor = null;
|
|
241
|
+
try {
|
|
242
|
+
descriptor = openSync(path, "r");
|
|
243
|
+
fsyncSync(descriptor);
|
|
244
|
+
}
|
|
245
|
+
catch (error) {
|
|
246
|
+
if (!isErrno(error, "EINVAL")
|
|
247
|
+
&& !isErrno(error, "ENOTSUP")
|
|
248
|
+
&& !isErrno(error, "EPERM")
|
|
249
|
+
&& !isErrno(error, "EACCES")
|
|
250
|
+
&& !isErrno(error, "EISDIR"))
|
|
251
|
+
throw error;
|
|
252
|
+
}
|
|
253
|
+
finally {
|
|
254
|
+
if (descriptor !== null)
|
|
255
|
+
closeSync(descriptor);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
function replaceFileSync(temporary, path) {
|
|
259
|
+
for (let attempt = 0;; attempt += 1) {
|
|
260
|
+
try {
|
|
261
|
+
renameSync(temporary, path);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
const retryable = process.platform === "win32"
|
|
266
|
+
&& ["EACCES", "EBUSY", "EPERM"].some((code) => isErrno(error, code));
|
|
267
|
+
if (!retryable || attempt >= 9) {
|
|
268
|
+
try {
|
|
269
|
+
unlinkSync(temporary);
|
|
270
|
+
}
|
|
271
|
+
catch { }
|
|
272
|
+
throw error;
|
|
273
|
+
}
|
|
274
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25 * (attempt + 1));
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
function atomicWriteJson(path, value) {
|
|
279
|
+
const directory = dirname(path);
|
|
280
|
+
const temporary = join(directory, `.${basename(path)}.tmp.${process.pid}.${randomUUID()}`);
|
|
281
|
+
const descriptor = openSync(temporary, "wx", PRIVATE_FILE_MODE);
|
|
282
|
+
try {
|
|
283
|
+
writeFileSync(descriptor, `${canonicalizeJson(value)}\n`, "utf8");
|
|
284
|
+
fsyncSync(descriptor);
|
|
285
|
+
}
|
|
286
|
+
finally {
|
|
287
|
+
closeSync(descriptor);
|
|
288
|
+
}
|
|
289
|
+
replaceFileSync(temporary, path);
|
|
290
|
+
syncDirectory(directory);
|
|
291
|
+
}
|
|
292
|
+
function readJson(path) {
|
|
293
|
+
try {
|
|
294
|
+
const metadata = lstatSync(path);
|
|
295
|
+
if (!metadata.isFile()
|
|
296
|
+
|| metadata.isSymbolicLink()
|
|
297
|
+
|| (process.platform !== "win32" && (metadata.mode & 0o077) !== 0)) {
|
|
298
|
+
throw new LocalExecutionAuthorityError("corrupt_state", `authority file is not private: ${path}`);
|
|
299
|
+
}
|
|
300
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
301
|
+
}
|
|
302
|
+
catch (error) {
|
|
303
|
+
if (error instanceof LocalExecutionAuthorityError)
|
|
304
|
+
throw error;
|
|
305
|
+
throw new LocalExecutionAuthorityError("corrupt_state", `cannot read authority file ${path}`, { cause: error });
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
function isRecord(value) {
|
|
309
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
310
|
+
}
|
|
311
|
+
function hasExactKeys(value, keys) {
|
|
312
|
+
const actual = Object.keys(value).sort();
|
|
313
|
+
const expected = [...keys].sort();
|
|
314
|
+
return actual.length === expected.length
|
|
315
|
+
&& actual.every((key, index) => key === expected[index]);
|
|
316
|
+
}
|
|
317
|
+
function isDurablePosixProvider(value) {
|
|
318
|
+
return isRecord(value)
|
|
319
|
+
&& hasExactKeys(value, ["pid", "process_group_id", "child_pid", "launched_at"])
|
|
320
|
+
&& Number.isSafeInteger(value.pid)
|
|
321
|
+
&& value.pid > 0
|
|
322
|
+
&& value.process_group_id === value.pid
|
|
323
|
+
&& Number.isSafeInteger(value.child_pid)
|
|
324
|
+
&& value.child_pid > 0
|
|
325
|
+
&& isCanonicalTimestamp(value.launched_at);
|
|
326
|
+
}
|
|
327
|
+
function isPositiveFiletime(value) {
|
|
328
|
+
return typeof value === "string" && /^[1-9][0-9]*$/.test(value);
|
|
329
|
+
}
|
|
330
|
+
function isDurableWindowsProvider(value) {
|
|
331
|
+
return isRecord(value)
|
|
332
|
+
&& hasExactKeys(value, [
|
|
333
|
+
"pid",
|
|
334
|
+
"keeper_pid",
|
|
335
|
+
"keeper_creation_filetime",
|
|
336
|
+
"root_pid",
|
|
337
|
+
"root_creation_filetime",
|
|
338
|
+
"job_identifier",
|
|
339
|
+
"containment_nonce",
|
|
340
|
+
"contained_at",
|
|
341
|
+
"released_at",
|
|
342
|
+
])
|
|
343
|
+
&& Number.isSafeInteger(value.pid)
|
|
344
|
+
&& value.pid > 0
|
|
345
|
+
&& value.pid === value.root_pid
|
|
346
|
+
&& Number.isSafeInteger(value.keeper_pid)
|
|
347
|
+
&& value.keeper_pid > 0
|
|
348
|
+
&& isPositiveFiletime(value.keeper_creation_filetime)
|
|
349
|
+
&& Number.isSafeInteger(value.root_pid)
|
|
350
|
+
&& value.root_pid > 0
|
|
351
|
+
&& isPositiveFiletime(value.root_creation_filetime)
|
|
352
|
+
&& isCanonicalDigest(value.job_identifier)
|
|
353
|
+
&& typeof value.containment_nonce === "string"
|
|
354
|
+
&& /^[0-9a-f]{64}$/.test(value.containment_nonce)
|
|
355
|
+
&& isCanonicalTimestamp(value.contained_at)
|
|
356
|
+
&& (value.released_at === null || isCanonicalTimestamp(value.released_at));
|
|
357
|
+
}
|
|
358
|
+
function isDurableWindowsJobCompletion(value) {
|
|
359
|
+
return isRecord(value)
|
|
360
|
+
&& hasExactKeys(value, [
|
|
361
|
+
"schema_version",
|
|
362
|
+
"kind",
|
|
363
|
+
"reason",
|
|
364
|
+
"root_exit_observed",
|
|
365
|
+
"root_exit_code",
|
|
366
|
+
"windows_signal",
|
|
367
|
+
"active_processes",
|
|
368
|
+
"process_tree_drained",
|
|
369
|
+
])
|
|
370
|
+
&& value.schema_version === "intentdna.windows_job_keeper_fact.v1"
|
|
371
|
+
&& value.kind === "completed"
|
|
372
|
+
&& ["root_exit", "terminate", "parent_eof", "protocol_error", "drain_timeout", "output_limit"].includes(String(value.reason))
|
|
373
|
+
&& typeof value.root_exit_observed === "boolean"
|
|
374
|
+
&& Number.isSafeInteger(value.root_exit_code)
|
|
375
|
+
&& value.root_exit_code >= 0
|
|
376
|
+
&& value.root_exit_code <= 0xffff_ffff
|
|
377
|
+
&& value.windows_signal === null
|
|
378
|
+
&& value.active_processes === 0
|
|
379
|
+
&& value.process_tree_drained === true;
|
|
380
|
+
}
|
|
381
|
+
function isDurableRawCompletion(value) {
|
|
382
|
+
return isRecord(value)
|
|
383
|
+
&& hasExactKeys(value, [
|
|
384
|
+
"stdout",
|
|
385
|
+
"stderr",
|
|
386
|
+
"max_output_bytes",
|
|
387
|
+
"timeout_ms",
|
|
388
|
+
"status",
|
|
389
|
+
"signal",
|
|
390
|
+
"timed_out",
|
|
391
|
+
"cancelled",
|
|
392
|
+
"cancellation_reason",
|
|
393
|
+
"process_limit_exceeded",
|
|
394
|
+
"output_limit_exceeded",
|
|
395
|
+
"termination_reason",
|
|
396
|
+
"process_id",
|
|
397
|
+
"process_tree_drained",
|
|
398
|
+
"completed_at",
|
|
399
|
+
"error",
|
|
400
|
+
...(value.windows_job_completion === undefined ? [] : ["windows_job_completion"]),
|
|
401
|
+
])
|
|
402
|
+
&& isDurableStreamEvidence(value.stdout)
|
|
403
|
+
&& isDurableStreamEvidence(value.stderr)
|
|
404
|
+
&& Number.isSafeInteger(value.max_output_bytes)
|
|
405
|
+
&& value.max_output_bytes > 0
|
|
406
|
+
&& (value.timeout_ms === null
|
|
407
|
+
|| (Number.isSafeInteger(value.timeout_ms) && value.timeout_ms > 0))
|
|
408
|
+
&& (value.status === null || Number.isSafeInteger(value.status))
|
|
409
|
+
&& (value.signal === null || typeof value.signal === "string")
|
|
410
|
+
&& typeof value.timed_out === "boolean"
|
|
411
|
+
&& typeof value.cancelled === "boolean"
|
|
412
|
+
&& (value.cancellation_reason === null || typeof value.cancellation_reason === "string")
|
|
413
|
+
&& typeof value.process_limit_exceeded === "boolean"
|
|
414
|
+
&& typeof value.output_limit_exceeded === "boolean"
|
|
415
|
+
&& (value.termination_reason === null
|
|
416
|
+
|| ["timeout", "cancellation", "process_limit", "output_limit"].includes(String(value.termination_reason)))
|
|
417
|
+
&& (value.process_id === null || (Number.isSafeInteger(value.process_id) && value.process_id > 0))
|
|
418
|
+
&& value.process_tree_drained === true
|
|
419
|
+
&& isCanonicalTimestamp(value.completed_at)
|
|
420
|
+
&& (value.error === null || (isRecord(value.error)
|
|
421
|
+
&& hasExactKeys(value.error, ["code", "message"])
|
|
422
|
+
&& typeof value.error.code === "string"
|
|
423
|
+
&& value.error.code.length > 0
|
|
424
|
+
&& typeof value.error.message === "string"
|
|
425
|
+
&& value.error.message.length > 0))
|
|
426
|
+
&& rawCompletionFlagsAreConsistent(value);
|
|
427
|
+
}
|
|
428
|
+
function isDurableStreamEvidence(value) {
|
|
429
|
+
return isRecord(value)
|
|
430
|
+
&& hasExactKeys(value, ["path", "byte_count", "sha256", "device", "inode", "birthtime_ns"])
|
|
431
|
+
&& typeof value.path === "string"
|
|
432
|
+
&& value.path.length > 0
|
|
433
|
+
&& Number.isSafeInteger(value.byte_count)
|
|
434
|
+
&& value.byte_count >= 0
|
|
435
|
+
&& isCanonicalDigest(value.sha256)
|
|
436
|
+
&& typeof value.device === "string"
|
|
437
|
+
&& /^(0|[1-9][0-9]*)$/.test(value.device)
|
|
438
|
+
&& typeof value.inode === "string"
|
|
439
|
+
&& /^(0|[1-9][0-9]*)$/.test(value.inode)
|
|
440
|
+
&& typeof value.birthtime_ns === "string"
|
|
441
|
+
&& /^(0|[1-9][0-9]*)$/.test(value.birthtime_ns);
|
|
442
|
+
}
|
|
443
|
+
function rawCompletionFlagsAreConsistent(raw) {
|
|
444
|
+
if (raw.stdout.byte_count > raw.max_output_bytes || raw.stderr.byte_count > raw.max_output_bytes) {
|
|
445
|
+
return false;
|
|
446
|
+
}
|
|
447
|
+
if ((raw.status !== null && raw.signal !== null) || (raw.error !== null && raw.termination_reason !== null)) {
|
|
448
|
+
return false;
|
|
449
|
+
}
|
|
450
|
+
if (raw.cancelled !== (raw.termination_reason === "cancellation")
|
|
451
|
+
|| raw.timed_out !== (raw.termination_reason === "timeout")
|
|
452
|
+
|| raw.process_limit_exceeded !== (raw.termination_reason === "process_limit")
|
|
453
|
+
|| raw.output_limit_exceeded !== (raw.termination_reason === "output_limit")) {
|
|
454
|
+
return false;
|
|
455
|
+
}
|
|
456
|
+
if ((raw.cancelled ? true : raw.cancellation_reason === null) === false)
|
|
457
|
+
return false;
|
|
458
|
+
if (raw.timed_out && raw.timeout_ms === null)
|
|
459
|
+
return false;
|
|
460
|
+
if (raw.output_limit_exceeded
|
|
461
|
+
&& raw.stdout.byte_count < raw.max_output_bytes
|
|
462
|
+
&& raw.stderr.byte_count < raw.max_output_bytes)
|
|
463
|
+
return false;
|
|
464
|
+
if (raw.termination_reason === null && raw.error === null) {
|
|
465
|
+
if (raw.signal !== null)
|
|
466
|
+
return raw.status === null;
|
|
467
|
+
return raw.status !== null;
|
|
468
|
+
}
|
|
469
|
+
return true;
|
|
470
|
+
}
|
|
471
|
+
function windowsCanonicalStopMode(value) {
|
|
472
|
+
if (!isRecord(value)
|
|
473
|
+
|| value.kind !== "worker_process"
|
|
474
|
+
|| !isRecord(value.process))
|
|
475
|
+
return null;
|
|
476
|
+
if (value.status === "failed"
|
|
477
|
+
&& value.process.kind === "timeout"
|
|
478
|
+
&& hasExactKeys(value.process, ["kind", "exit_code", "signal", "timeout_ms"])
|
|
479
|
+
&& value.process.exit_code === null
|
|
480
|
+
&& value.process.signal === null
|
|
481
|
+
&& Number.isSafeInteger(value.process.timeout_ms)
|
|
482
|
+
&& value.process.timeout_ms > 0)
|
|
483
|
+
return { kind: "timeout", reason: null };
|
|
484
|
+
if (value.status === "cancelled"
|
|
485
|
+
&& value.process.kind === "cancelled"
|
|
486
|
+
&& hasExactKeys(value.process, ["kind", "exit_code", "signal", "reason"])
|
|
487
|
+
&& value.process.exit_code === null
|
|
488
|
+
&& value.process.signal === null
|
|
489
|
+
&& (value.process.reason === null || typeof value.process.reason === "string"))
|
|
490
|
+
return { kind: "cancellation", reason: value.process.reason };
|
|
491
|
+
return null;
|
|
492
|
+
}
|
|
493
|
+
function windowsStopExpectation(value) {
|
|
494
|
+
if (value === null) {
|
|
495
|
+
return {
|
|
496
|
+
timedOut: false,
|
|
497
|
+
cancelled: false,
|
|
498
|
+
outputLimited: false,
|
|
499
|
+
cancellationReason: null,
|
|
500
|
+
terminationReason: null,
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
if (value.kind === "timeout") {
|
|
504
|
+
if (!hasExactKeys(value, ["kind", "reason"])
|
|
505
|
+
|| value.reason !== "timeout")
|
|
506
|
+
return null;
|
|
507
|
+
return {
|
|
508
|
+
timedOut: true,
|
|
509
|
+
cancelled: false,
|
|
510
|
+
outputLimited: false,
|
|
511
|
+
cancellationReason: null,
|
|
512
|
+
terminationReason: "timeout",
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
if (value.kind === "local_cancel") {
|
|
516
|
+
if (!hasExactKeys(value, ["kind", "reason"])
|
|
517
|
+
|| (value.reason !== null && typeof value.reason !== "string"))
|
|
518
|
+
return null;
|
|
519
|
+
return {
|
|
520
|
+
timedOut: false,
|
|
521
|
+
cancelled: true,
|
|
522
|
+
outputLimited: false,
|
|
523
|
+
cancellationReason: value.reason ?? null,
|
|
524
|
+
terminationReason: "cancellation",
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
if (value.kind === "output_limit") {
|
|
528
|
+
if (!hasExactKeys(value, ["kind", "reason"])
|
|
529
|
+
|| value.reason !== "output_limit")
|
|
530
|
+
return null;
|
|
531
|
+
return {
|
|
532
|
+
timedOut: false,
|
|
533
|
+
cancelled: false,
|
|
534
|
+
outputLimited: true,
|
|
535
|
+
cancellationReason: null,
|
|
536
|
+
terminationReason: "output_limit",
|
|
537
|
+
};
|
|
538
|
+
}
|
|
539
|
+
if (value.kind !== "canonical_stop"
|
|
540
|
+
|| !hasExactKeys(value, [
|
|
541
|
+
"kind",
|
|
542
|
+
"stop_idempotency_key",
|
|
543
|
+
"requested_at",
|
|
544
|
+
"requested_outcome",
|
|
545
|
+
])
|
|
546
|
+
|| !isCanonicalDigest(value.stop_idempotency_key)
|
|
547
|
+
|| !isCanonicalTimestamp(value.requested_at))
|
|
548
|
+
return null;
|
|
549
|
+
const mode = windowsCanonicalStopMode(value.requested_outcome);
|
|
550
|
+
return mode === null ? null : {
|
|
551
|
+
timedOut: mode.kind === "timeout",
|
|
552
|
+
cancelled: mode.kind === "cancellation",
|
|
553
|
+
outputLimited: false,
|
|
554
|
+
cancellationReason: mode.reason,
|
|
555
|
+
terminationReason: mode.kind,
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
function windowsCompletionMatchesRecord(record) {
|
|
559
|
+
if (record.backend.kind !== "windows_job" || record.raw_completion === null)
|
|
560
|
+
return true;
|
|
561
|
+
const raw = record.raw_completion;
|
|
562
|
+
const provider = record.provider;
|
|
563
|
+
const fact = raw.windows_job_completion;
|
|
564
|
+
const expected = windowsStopExpectation(record.stop_fact);
|
|
565
|
+
if (!isDurableWindowsProvider(provider)
|
|
566
|
+
|| provider.released_at === null
|
|
567
|
+
|| !isDurableWindowsJobCompletion(fact)
|
|
568
|
+
|| expected === null
|
|
569
|
+
|| raw.process_id !== provider.root_pid
|
|
570
|
+
|| raw.signal !== null
|
|
571
|
+
|| raw.error !== null
|
|
572
|
+
|| raw.completed_at < provider.released_at
|
|
573
|
+
|| fact.reason === "parent_eof"
|
|
574
|
+
|| fact.reason === "protocol_error")
|
|
575
|
+
return false;
|
|
576
|
+
const stopped = expected.timedOut || expected.cancelled || expected.outputLimited;
|
|
577
|
+
return raw.timed_out === expected.timedOut
|
|
578
|
+
&& raw.cancelled === expected.cancelled
|
|
579
|
+
&& raw.cancellation_reason === expected.cancellationReason
|
|
580
|
+
&& raw.process_limit_exceeded === false
|
|
581
|
+
&& raw.output_limit_exceeded === expected.outputLimited
|
|
582
|
+
&& (stopped
|
|
583
|
+
? fact.reason === (expected.outputLimited ? "output_limit" : "terminate")
|
|
584
|
+
&& raw.status === null
|
|
585
|
+
&& raw.termination_reason === expected.terminationReason
|
|
586
|
+
: (fact.reason === "root_exit" || fact.reason === "drain_timeout")
|
|
587
|
+
&& fact.root_exit_observed
|
|
588
|
+
&& raw.status === fact.root_exit_code
|
|
589
|
+
&& raw.termination_reason === null);
|
|
590
|
+
}
|
|
591
|
+
function windowsPhaseMatchesRecord(record) {
|
|
592
|
+
if (record.backend.kind !== "windows_job")
|
|
593
|
+
return true;
|
|
594
|
+
const supervisorReady = record.supervisor.pid !== null && record.supervisor.ready_at !== null;
|
|
595
|
+
if (record.phase === "never_launched" || record.phase === "prepared" || record.phase === "launching") {
|
|
596
|
+
return !supervisorReady
|
|
597
|
+
&& record.provider === null
|
|
598
|
+
&& record.raw_completion === null
|
|
599
|
+
&& record.stop_fact === null;
|
|
600
|
+
}
|
|
601
|
+
if (record.phase === "running") {
|
|
602
|
+
return supervisorReady && record.raw_completion === null;
|
|
603
|
+
}
|
|
604
|
+
if (["raw_completed", "normalized_completed", "receipted"].includes(record.phase)) {
|
|
605
|
+
return supervisorReady
|
|
606
|
+
&& isDurableWindowsProvider(record.provider)
|
|
607
|
+
&& record.provider.released_at !== null
|
|
608
|
+
&& record.raw_completion !== null;
|
|
609
|
+
}
|
|
610
|
+
return record.phase === "unsupported";
|
|
611
|
+
}
|
|
612
|
+
function isDurableBackend(value) {
|
|
613
|
+
if (!isRecord(value) || typeof value.kind !== "string")
|
|
614
|
+
return false;
|
|
615
|
+
if (value.kind === "posix_process_group") {
|
|
616
|
+
return hasExactKeys(value, ["kind"]);
|
|
617
|
+
}
|
|
618
|
+
return value.kind === "windows_job"
|
|
619
|
+
&& hasExactKeys(value, [
|
|
620
|
+
"kind",
|
|
621
|
+
"architecture",
|
|
622
|
+
"helper_path",
|
|
623
|
+
"helper_digest",
|
|
624
|
+
"helper_source_digest",
|
|
625
|
+
"helper_protocol_version",
|
|
626
|
+
"containment_nonce",
|
|
627
|
+
])
|
|
628
|
+
&& ["x86_64", "aarch64"].includes(String(value.architecture))
|
|
629
|
+
&& typeof value.helper_path === "string"
|
|
630
|
+
&& value.helper_path.length > 0
|
|
631
|
+
&& isCanonicalDigest(value.helper_digest)
|
|
632
|
+
&& isCanonicalDigest(value.helper_source_digest)
|
|
633
|
+
&& value.helper_protocol_version === "IDNA_WINDOWS_JOB_KEEPER_V1"
|
|
634
|
+
&& typeof value.containment_nonce === "string"
|
|
635
|
+
&& /^[0-9a-f]{64}$/.test(value.containment_nonce);
|
|
636
|
+
}
|
|
637
|
+
function isDurableSupervisor(value, backend) {
|
|
638
|
+
if (!isRecord(value) || !hasExactKeys(value, [
|
|
639
|
+
"kind",
|
|
640
|
+
"socket_path",
|
|
641
|
+
"script_digest",
|
|
642
|
+
"pid",
|
|
643
|
+
"ready_at",
|
|
644
|
+
]))
|
|
645
|
+
return false;
|
|
646
|
+
const expectedKind = backend.kind === "windows_job" ? "windows" : "posix";
|
|
647
|
+
const expectedDigest = backend.kind === "windows_job"
|
|
648
|
+
? LOCAL_WINDOWS_EXECUTION_SUPERVISOR_DIGEST
|
|
649
|
+
: LOCAL_EXECUTION_SUPERVISOR_DIGEST;
|
|
650
|
+
return value.kind === expectedKind
|
|
651
|
+
&& typeof value.socket_path === "string"
|
|
652
|
+
&& value.socket_path.length > 0
|
|
653
|
+
&& value.script_digest === expectedDigest
|
|
654
|
+
&& (value.pid === null || (Number.isSafeInteger(value.pid) && value.pid > 0))
|
|
655
|
+
&& (value.ready_at === null || isCanonicalTimestamp(value.ready_at));
|
|
656
|
+
}
|
|
657
|
+
function nonlaunchSourceEvidenceDigest(record) {
|
|
658
|
+
const { nonlaunch_attestation: _attestation, ...durableEvidence } = record;
|
|
659
|
+
return digest({
|
|
660
|
+
schema_version: NONLAUNCH_EVIDENCE_SCHEMA_VERSION,
|
|
661
|
+
durable_record: durableEvidence,
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
function alreadyStoppedSourceEvidenceDigest(record) {
|
|
665
|
+
return digest({
|
|
666
|
+
schema_version: "intentdna.local_already_stopped_source_evidence.v1",
|
|
667
|
+
identity_digest: record.identity_digest,
|
|
668
|
+
execution_instance_id: record.execution_instance_id,
|
|
669
|
+
authority_instance_id: record.authority_instance_id,
|
|
670
|
+
backend: record.backend,
|
|
671
|
+
supervisor: record.supervisor,
|
|
672
|
+
provider: record.provider,
|
|
673
|
+
raw_completion: record.raw_completion,
|
|
674
|
+
normalized_process_outcome: record.normalized_process_outcome,
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
function stopReceiptSourceEvidenceDigest(record) {
|
|
678
|
+
return digest({
|
|
679
|
+
identity_digest: record.identity_digest,
|
|
680
|
+
supervisor: record.supervisor,
|
|
681
|
+
provider: record.provider,
|
|
682
|
+
raw_completion: record.raw_completion,
|
|
683
|
+
normalized_process_outcome: record.normalized_process_outcome,
|
|
684
|
+
stop_fact: record.stop_fact,
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
function isDurableStopFact(value, workerSessionId) {
|
|
688
|
+
if (!isRecord(value) || typeof value.kind !== "string")
|
|
689
|
+
return false;
|
|
690
|
+
if (value.kind === "canonical_stop") {
|
|
691
|
+
return hasExactKeys(value, [
|
|
692
|
+
"kind",
|
|
693
|
+
"stop_idempotency_key",
|
|
694
|
+
"requested_at",
|
|
695
|
+
"requested_outcome",
|
|
696
|
+
])
|
|
697
|
+
&& isCanonicalDigest(value.stop_idempotency_key)
|
|
698
|
+
&& isCanonicalTimestamp(value.requested_at)
|
|
699
|
+
&& isCanonicalWorkerOutcome(value.requested_outcome, workerSessionId);
|
|
700
|
+
}
|
|
701
|
+
if (!hasExactKeys(value, ["kind", "reason"]))
|
|
702
|
+
return false;
|
|
703
|
+
if (value.kind === "local_cancel") {
|
|
704
|
+
return value.reason === null || typeof value.reason === "string";
|
|
705
|
+
}
|
|
706
|
+
return (value.kind === "timeout" && value.reason === "timeout")
|
|
707
|
+
|| (value.kind === "process_limit" && value.reason === "process_limit")
|
|
708
|
+
|| (value.kind === "output_limit" && value.reason === "output_limit");
|
|
709
|
+
}
|
|
710
|
+
function isDurableStopReceipt(value, record) {
|
|
711
|
+
if (!isRecord(value)
|
|
712
|
+
|| !hasExactKeys(value, ["receipt_digest", "payload"])
|
|
713
|
+
|| !isCanonicalDigest(value.receipt_digest)
|
|
714
|
+
|| !isRecord(value.payload)
|
|
715
|
+
|| !hasExactKeys(value.payload, [
|
|
716
|
+
"schema_version",
|
|
717
|
+
"run_id",
|
|
718
|
+
"step_id",
|
|
719
|
+
"activation_id",
|
|
720
|
+
"attempt_id",
|
|
721
|
+
"claim_epoch",
|
|
722
|
+
"worker_session_id",
|
|
723
|
+
"execution_instance_id",
|
|
724
|
+
"authority_kind",
|
|
725
|
+
"authority_instance_id",
|
|
726
|
+
"fencing_epoch",
|
|
727
|
+
"started_at",
|
|
728
|
+
"stopped_at",
|
|
729
|
+
"terminal_outcome",
|
|
730
|
+
"process_tree_drained",
|
|
731
|
+
"source_evidence_digest",
|
|
732
|
+
]))
|
|
733
|
+
return false;
|
|
734
|
+
const receipt = value;
|
|
735
|
+
const payload = receipt.payload;
|
|
736
|
+
if (payload.schema_version !== "intentdna.worker_stop_receipt.v1"
|
|
737
|
+
|| payload.run_id !== record.identity.run_id
|
|
738
|
+
|| payload.step_id !== record.identity.step_id
|
|
739
|
+
|| payload.activation_id !== record.identity.activation_id
|
|
740
|
+
|| payload.attempt_id !== record.identity.attempt_id
|
|
741
|
+
|| payload.claim_epoch !== record.identity.claim_epoch
|
|
742
|
+
|| payload.worker_session_id !== record.identity.worker_session_id
|
|
743
|
+
|| payload.execution_instance_id !== record.execution_instance_id
|
|
744
|
+
|| payload.authority_kind !== "local_process"
|
|
745
|
+
|| payload.authority_instance_id !== record.authority_instance_id
|
|
746
|
+
|| !isPositiveSafeInteger(payload.fencing_epoch)
|
|
747
|
+
|| payload.fencing_epoch !== record.fencing_epoch
|
|
748
|
+
|| payload.started_at !== record.identity.started_at
|
|
749
|
+
|| !isCanonicalTimestamp(payload.stopped_at)
|
|
750
|
+
|| payload.stopped_at < payload.started_at
|
|
751
|
+
|| record.raw_completion === null
|
|
752
|
+
|| payload.stopped_at < record.raw_completion.completed_at
|
|
753
|
+
|| !isCanonicalWorkerOutcome(payload.terminal_outcome, record.identity.worker_session_id)
|
|
754
|
+
|| payload.process_tree_drained !== true
|
|
755
|
+
|| !isCanonicalDigest(payload.source_evidence_digest)
|
|
756
|
+
|| payload.source_evidence_digest !== stopReceiptSourceEvidenceDigest(record)
|
|
757
|
+
|| receipt.receipt_digest !== workerStopReceiptDigest(payload))
|
|
758
|
+
return false;
|
|
759
|
+
if (record.stop_fact?.kind === "canonical_stop") {
|
|
760
|
+
return payload.stopped_at >= record.stop_fact.requested_at
|
|
761
|
+
&& canonicalizeJson(payload.terminal_outcome)
|
|
762
|
+
=== canonicalizeJson(record.stop_fact.requested_outcome);
|
|
763
|
+
}
|
|
764
|
+
return record.normalized_process_outcome !== null
|
|
765
|
+
&& processMatchesCanonical(record.normalized_process_outcome, payload.terminal_outcome);
|
|
766
|
+
}
|
|
767
|
+
function parseRecord(value) {
|
|
768
|
+
if (!isRecord(value)
|
|
769
|
+
|| !hasExactKeys(value, [
|
|
770
|
+
"schema_version",
|
|
771
|
+
"revision",
|
|
772
|
+
"phase",
|
|
773
|
+
"identity",
|
|
774
|
+
"identity_digest",
|
|
775
|
+
"execution_instance_id",
|
|
776
|
+
"challenge",
|
|
777
|
+
"authority_instance_id",
|
|
778
|
+
"backend",
|
|
779
|
+
"supervisor",
|
|
780
|
+
"provider",
|
|
781
|
+
"raw_completion",
|
|
782
|
+
"normalized_process_outcome",
|
|
783
|
+
"stop_fact",
|
|
784
|
+
"stop_receipt",
|
|
785
|
+
"already_stopped_evidence",
|
|
786
|
+
"nonlaunch_attestation",
|
|
787
|
+
"fencing_epoch",
|
|
788
|
+
"supervisor_error",
|
|
789
|
+
"created_at",
|
|
790
|
+
"updated_at",
|
|
791
|
+
])
|
|
792
|
+
|| value.schema_version !== RECORD_SCHEMA_VERSION) {
|
|
793
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution authority record has an invalid schema");
|
|
794
|
+
}
|
|
795
|
+
const record = value;
|
|
796
|
+
const nonlaunchAttestation = record.nonlaunch_attestation;
|
|
797
|
+
requireIdentity(record.identity);
|
|
798
|
+
const hasFencedEvidence = record.stop_receipt !== null
|
|
799
|
+
|| record.already_stopped_evidence !== null
|
|
800
|
+
|| record.nonlaunch_attestation !== null;
|
|
801
|
+
if (record.identity_digest !== identityDigest(record.identity)
|
|
802
|
+
|| record.execution_instance_id !== record.identity.execution_instance_id
|
|
803
|
+
|| !Number.isSafeInteger(record.revision)
|
|
804
|
+
|| record.revision <= 0
|
|
805
|
+
|| !["never_launched", "prepared", "launching", "running", "raw_completed", "normalized_completed", "receipted", "unsupported"].includes(record.phase)
|
|
806
|
+
|| typeof record.challenge !== "string"
|
|
807
|
+
|| record.challenge.length < 32
|
|
808
|
+
|| typeof record.authority_instance_id !== "string"
|
|
809
|
+
|| record.authority_instance_id.length === 0
|
|
810
|
+
|| !isDurableBackend(record.backend)
|
|
811
|
+
|| !isDurableSupervisor(record.supervisor, record.backend)
|
|
812
|
+
|| ((record.supervisor.pid === null) !== (record.supervisor.ready_at === null))
|
|
813
|
+
|| (record.raw_completion !== null && !isDurableRawCompletion(record.raw_completion))
|
|
814
|
+
|| (record.stop_fact !== null
|
|
815
|
+
&& !isDurableStopFact(record.stop_fact, record.identity.worker_session_id))
|
|
816
|
+
|| (hasFencedEvidence
|
|
817
|
+
? !isPositiveSafeInteger(record.fencing_epoch)
|
|
818
|
+
: record.fencing_epoch !== null)
|
|
819
|
+
|| !isCanonicalTimestamp(record.created_at)
|
|
820
|
+
|| !isCanonicalTimestamp(record.updated_at)
|
|
821
|
+
|| record.updated_at < record.created_at) {
|
|
822
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution authority record failed identity validation");
|
|
823
|
+
}
|
|
824
|
+
if ((record.phase === "never_launched" && (nonlaunchAttestation === null
|
|
825
|
+
|| record.provider !== null
|
|
826
|
+
|| record.raw_completion !== null
|
|
827
|
+
|| record.supervisor.pid !== null
|
|
828
|
+
|| nonlaunchAttestation.attestation_digest
|
|
829
|
+
!== workerNonlaunchAttestationDigest(nonlaunchAttestation.payload)
|
|
830
|
+
|| nonlaunchAttestation.payload.execution_instance_id !== record.execution_instance_id
|
|
831
|
+
|| nonlaunchAttestation.payload.authority_instance_id !== record.authority_instance_id
|
|
832
|
+
|| nonlaunchAttestation.payload.run_id !== record.identity.run_id
|
|
833
|
+
|| nonlaunchAttestation.payload.step_id !== record.identity.step_id
|
|
834
|
+
|| nonlaunchAttestation.payload.activation_id !== record.identity.activation_id
|
|
835
|
+
|| nonlaunchAttestation.payload.attempt_id !== record.identity.attempt_id
|
|
836
|
+
|| nonlaunchAttestation.payload.claim_epoch !== record.identity.claim_epoch
|
|
837
|
+
|| nonlaunchAttestation.payload.worker_session_id !== record.identity.worker_session_id
|
|
838
|
+
|| nonlaunchAttestation.payload.started_at !== record.identity.started_at
|
|
839
|
+
|| nonlaunchAttestation.payload.launch_state !== "never_launched"
|
|
840
|
+
|| nonlaunchAttestation.payload.authority_kind !== "local_process"
|
|
841
|
+
|| nonlaunchAttestation.payload.schema_version !== "intentdna.worker_nonlaunch_attestation.v1"
|
|
842
|
+
|| !Number.isSafeInteger(nonlaunchAttestation.payload.fencing_epoch)
|
|
843
|
+
|| nonlaunchAttestation.payload.fencing_epoch <= 0
|
|
844
|
+
|| nonlaunchAttestation.payload.fencing_epoch !== record.fencing_epoch
|
|
845
|
+
|| !isCanonicalTimestamp(nonlaunchAttestation.payload.attested_at)
|
|
846
|
+
|| !isCanonicalDigest(nonlaunchAttestation.payload.failure_idempotency_key)
|
|
847
|
+
|| !isCanonicalDigest(nonlaunchAttestation.payload.source_evidence_digest)
|
|
848
|
+
|| nonlaunchAttestation.payload.source_evidence_digest
|
|
849
|
+
!== nonlaunchSourceEvidenceDigest(record)))
|
|
850
|
+
|| (record.phase !== "never_launched" && nonlaunchAttestation !== null)
|
|
851
|
+
|| (record.already_stopped_evidence !== null && (record.raw_completion === null
|
|
852
|
+
|| record.stop_receipt !== null
|
|
853
|
+
|| record.fencing_epoch === null
|
|
854
|
+
|| record.already_stopped_evidence.evidence_digest
|
|
855
|
+
!== workerAlreadyStoppedEvidenceDigest(record.already_stopped_evidence.payload)
|
|
856
|
+
|| record.already_stopped_evidence.payload.schema_version
|
|
857
|
+
!== "intentdna.worker_already_stopped_evidence.v1"
|
|
858
|
+
|| record.already_stopped_evidence.payload.run_id !== record.identity.run_id
|
|
859
|
+
|| record.already_stopped_evidence.payload.step_id !== record.identity.step_id
|
|
860
|
+
|| record.already_stopped_evidence.payload.activation_id !== record.identity.activation_id
|
|
861
|
+
|| record.already_stopped_evidence.payload.attempt_id !== record.identity.attempt_id
|
|
862
|
+
|| record.already_stopped_evidence.payload.claim_epoch !== record.identity.claim_epoch
|
|
863
|
+
|| record.already_stopped_evidence.payload.worker_session_id !== record.identity.worker_session_id
|
|
864
|
+
|| record.already_stopped_evidence.payload.execution_instance_id !== record.execution_instance_id
|
|
865
|
+
|| record.already_stopped_evidence.payload.authority_instance_id !== record.authority_instance_id
|
|
866
|
+
|| record.already_stopped_evidence.payload.authority_kind !== "local_process"
|
|
867
|
+
|| record.already_stopped_evidence.payload.fencing_epoch !== record.fencing_epoch
|
|
868
|
+
|| record.already_stopped_evidence.payload.started_at !== record.identity.started_at
|
|
869
|
+
|| record.already_stopped_evidence.payload.completed_at !== record.raw_completion.completed_at
|
|
870
|
+
|| canonicalizeJson(record.already_stopped_evidence.payload.observed_outcome)
|
|
871
|
+
!== canonicalizeJson(terminalOutcomeToCanonical(rawCompletionToOutcome(record.raw_completion)))
|
|
872
|
+
|| record.already_stopped_evidence.payload.process_tree_drained !== true
|
|
873
|
+
|| !isCanonicalDigest(record.already_stopped_evidence.payload.stop_idempotency_key)
|
|
874
|
+
|| !isCanonicalTimestamp(record.already_stopped_evidence.payload.stop_requested_at)
|
|
875
|
+
|| !["cancel", "execution_deadline", "peer_terminal"].includes(record.already_stopped_evidence.payload.stop_purpose)
|
|
876
|
+
|| record.already_stopped_evidence.payload.source_evidence_digest
|
|
877
|
+
!== alreadyStoppedSourceEvidenceDigest(record)))
|
|
878
|
+
|| (record.stop_receipt !== null && record.already_stopped_evidence !== null)
|
|
879
|
+
|| (record.stop_receipt !== null && !isDurableStopReceipt(record.stop_receipt, record))
|
|
880
|
+
|| (record.provider !== null && (record.backend.kind === "windows_job"
|
|
881
|
+
? !isDurableWindowsProvider(record.provider)
|
|
882
|
+
: !isDurablePosixProvider(record.provider)))
|
|
883
|
+
|| (record.backend.kind === "windows_job" && record.provider !== null
|
|
884
|
+
&& (!isDurableWindowsProvider(record.provider)
|
|
885
|
+
|| record.provider.containment_nonce !== record.backend.containment_nonce))
|
|
886
|
+
|| (record.phase === "prepared" && record.provider !== null)
|
|
887
|
+
|| (record.phase === "raw_completed" && record.raw_completion === null)
|
|
888
|
+
|| (record.raw_completion !== null && record.raw_completion.process_tree_drained !== true)
|
|
889
|
+
|| !windowsPhaseMatchesRecord(record)
|
|
890
|
+
|| !windowsCompletionMatchesRecord(record)
|
|
891
|
+
|| (record.phase === "normalized_completed" && (record.raw_completion === null || record.normalized_process_outcome === null))
|
|
892
|
+
|| (record.phase === "receipted" && (record.raw_completion === null
|
|
893
|
+
|| (record.stop_receipt === null && record.already_stopped_evidence === null)
|
|
894
|
+
|| record.fencing_epoch === null))) {
|
|
895
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution authority phase lacks required evidence");
|
|
896
|
+
}
|
|
897
|
+
return record;
|
|
898
|
+
}
|
|
899
|
+
function processIsAlive(pid) {
|
|
900
|
+
try {
|
|
901
|
+
process.kill(pid, 0);
|
|
902
|
+
return true;
|
|
903
|
+
}
|
|
904
|
+
catch (error) {
|
|
905
|
+
return isErrno(error, "EPERM");
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
function processBirthIdentity(pid, platform) {
|
|
909
|
+
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
910
|
+
return null;
|
|
911
|
+
if (platform === "linux") {
|
|
912
|
+
try {
|
|
913
|
+
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
914
|
+
const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/);
|
|
915
|
+
const startTicks = fields[19];
|
|
916
|
+
return startTicks !== undefined && /^[0-9]+$/.test(startTicks)
|
|
917
|
+
? `linux-start-ticks:${startTicks}`
|
|
918
|
+
: null;
|
|
919
|
+
}
|
|
920
|
+
catch {
|
|
921
|
+
return null;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
if (platform !== "win32") {
|
|
925
|
+
const result = spawnSync("ps", ["-p", String(pid), "-o", "lstart="], {
|
|
926
|
+
encoding: "utf8",
|
|
927
|
+
timeout: 1_000,
|
|
928
|
+
windowsHide: true,
|
|
929
|
+
});
|
|
930
|
+
const started = result.status === 0 ? result.stdout.trim().replace(/\s+/g, " ") : "";
|
|
931
|
+
return started === "" ? null : `${platform}-lstart:${started}`;
|
|
932
|
+
}
|
|
933
|
+
return pid === process.pid ? `node-time-origin-ms:${Math.floor(performance.timeOrigin)}` : null;
|
|
934
|
+
}
|
|
935
|
+
function durableLockOwner(platform) {
|
|
936
|
+
const processBirthId = processBirthIdentity(process.pid, platform);
|
|
937
|
+
if (processBirthId === null) {
|
|
938
|
+
throw new LocalExecutionAuthorityError("unsupported", "cannot establish execution lock owner identity");
|
|
939
|
+
}
|
|
940
|
+
return {
|
|
941
|
+
schema_version: "intentdna.execution_lock_owner.v1",
|
|
942
|
+
pid: process.pid,
|
|
943
|
+
process_birth_id: processBirthId,
|
|
944
|
+
generation: randomUUID(),
|
|
945
|
+
acquired_at: Date.now(),
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
function readDurableLockOwner(path) {
|
|
949
|
+
try {
|
|
950
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
951
|
+
if (!isRecord(value)
|
|
952
|
+
|| !hasExactKeys(value, ["schema_version", "pid", "process_birth_id", "generation", "acquired_at"])
|
|
953
|
+
|| value.schema_version !== "intentdna.execution_lock_owner.v1"
|
|
954
|
+
|| !Number.isSafeInteger(value.pid)
|
|
955
|
+
|| value.pid <= 0
|
|
956
|
+
|| typeof value.process_birth_id !== "string"
|
|
957
|
+
|| value.process_birth_id.length === 0
|
|
958
|
+
|| typeof value.generation !== "string"
|
|
959
|
+
|| value.generation.length === 0
|
|
960
|
+
|| !Number.isSafeInteger(value.acquired_at)
|
|
961
|
+
|| value.acquired_at < 0)
|
|
962
|
+
return null;
|
|
963
|
+
return value;
|
|
964
|
+
}
|
|
965
|
+
catch {
|
|
966
|
+
return null;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
function publishDurableLock(lockPath, owner) {
|
|
970
|
+
const candidatePath = `${lockPath}.candidate.${process.pid}.${randomUUID()}`;
|
|
971
|
+
mkdirSync(candidatePath, { mode: DIRECTORY_MODE });
|
|
972
|
+
try {
|
|
973
|
+
const descriptor = openSync(join(candidatePath, "owner.json"), "wx", PRIVATE_FILE_MODE);
|
|
974
|
+
try {
|
|
975
|
+
writeFileSync(descriptor, `${canonicalizeJson(owner)}\n`, "utf8");
|
|
976
|
+
fsyncSync(descriptor);
|
|
977
|
+
}
|
|
978
|
+
finally {
|
|
979
|
+
closeSync(descriptor);
|
|
980
|
+
}
|
|
981
|
+
syncDirectory(candidatePath);
|
|
982
|
+
try {
|
|
983
|
+
renameSync(candidatePath, lockPath);
|
|
984
|
+
}
|
|
985
|
+
catch (error) {
|
|
986
|
+
if (isErrno(error, "EEXIST") || isErrno(error, "ENOTEMPTY")
|
|
987
|
+
|| (isErrno(error, "EPERM") && existsSync(lockPath)))
|
|
988
|
+
return false;
|
|
989
|
+
throw error;
|
|
990
|
+
}
|
|
991
|
+
syncDirectory(dirname(lockPath));
|
|
992
|
+
return true;
|
|
993
|
+
}
|
|
994
|
+
finally {
|
|
995
|
+
rmSync(candidatePath, { recursive: true, force: true });
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
function releaseDurableLock(lockPath, owner) {
|
|
999
|
+
const ownerPath = join(lockPath, "owner.json");
|
|
1000
|
+
if (!existsSync(ownerPath))
|
|
1001
|
+
return;
|
|
1002
|
+
const current = readDurableLockOwner(ownerPath);
|
|
1003
|
+
if (current === null) {
|
|
1004
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution authority lock owner is invalid during release");
|
|
1005
|
+
}
|
|
1006
|
+
if (!durableLockOwnersEqual(current, owner))
|
|
1007
|
+
return;
|
|
1008
|
+
const releasedPath = `${lockPath}.released.${durableLockOwnerDigest(owner)}.${randomUUID()}`;
|
|
1009
|
+
try {
|
|
1010
|
+
renameSync(lockPath, releasedPath);
|
|
1011
|
+
}
|
|
1012
|
+
catch (error) {
|
|
1013
|
+
if (isErrno(error, "ENOENT"))
|
|
1014
|
+
return;
|
|
1015
|
+
throw error;
|
|
1016
|
+
}
|
|
1017
|
+
syncDirectory(dirname(lockPath));
|
|
1018
|
+
const releasedOwner = readDurableLockOwner(join(releasedPath, "owner.json"));
|
|
1019
|
+
if (!durableLockOwnersEqual(releasedOwner, owner)) {
|
|
1020
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution authority lock ownership changed during release");
|
|
1021
|
+
}
|
|
1022
|
+
rmSync(releasedPath, { recursive: true, force: true });
|
|
1023
|
+
syncDirectory(dirname(lockPath));
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Retires one observed stale generation without exposing a later generation to
|
|
1027
|
+
* delayed contenders. The non-empty, generation-specific tombstone is retained
|
|
1028
|
+
* so a contender that observed the same owner can never rename a successor over it.
|
|
1029
|
+
* This is exported only for lock-protocol regression tests, not from the runtime barrel.
|
|
1030
|
+
*/
|
|
1031
|
+
export function retireObservedStaleDurableLock(lockPath, observed) {
|
|
1032
|
+
const quarantine = `${lockPath}.stale.${durableLockOwnerDigest(observed)}`;
|
|
1033
|
+
try {
|
|
1034
|
+
renameSync(lockPath, quarantine);
|
|
1035
|
+
}
|
|
1036
|
+
catch (error) {
|
|
1037
|
+
if (isErrno(error, "ENOENT")
|
|
1038
|
+
|| isErrno(error, "EEXIST")
|
|
1039
|
+
|| isErrno(error, "ENOTEMPTY")
|
|
1040
|
+
|| ((isErrno(error, "EPERM") || isErrno(error, "EACCES")) && existsSync(quarantine)))
|
|
1041
|
+
return false;
|
|
1042
|
+
throw error;
|
|
1043
|
+
}
|
|
1044
|
+
syncDirectory(dirname(lockPath));
|
|
1045
|
+
const retiredOwner = readDurableLockOwner(join(quarantine, "owner.json"));
|
|
1046
|
+
if (!durableLockOwnersEqual(retiredOwner, observed)) {
|
|
1047
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution authority lock ownership changed during stale takeover");
|
|
1048
|
+
}
|
|
1049
|
+
return true;
|
|
1050
|
+
}
|
|
1051
|
+
function staleDurableLock(owner, platform) {
|
|
1052
|
+
if (platform === "win32" || owner === null)
|
|
1053
|
+
return false;
|
|
1054
|
+
if (!processIsAlive(owner.pid))
|
|
1055
|
+
return true;
|
|
1056
|
+
const actualBirthId = processBirthIdentity(owner.pid, platform);
|
|
1057
|
+
return actualBirthId !== null && actualBirthId !== owner.process_birth_id;
|
|
1058
|
+
}
|
|
1059
|
+
function providerProcessId(record) {
|
|
1060
|
+
if (record.provider === null)
|
|
1061
|
+
return null;
|
|
1062
|
+
return record.backend.kind === "windows_job"
|
|
1063
|
+
? isDurableWindowsProvider(record.provider) ? record.provider.root_pid : null
|
|
1064
|
+
: isDurablePosixProvider(record.provider) ? record.provider.process_group_id : null;
|
|
1065
|
+
}
|
|
1066
|
+
function providerMatchesProcessId(record, processId) {
|
|
1067
|
+
return providerProcessId(record) === processId;
|
|
1068
|
+
}
|
|
1069
|
+
function windowsVerbatimCommandLine(executable, args) {
|
|
1070
|
+
if (executable.includes("\0") || executable.includes('"')) {
|
|
1071
|
+
throw new LocalExecutionAuthorityError("unsupported", "Windows executable path cannot be represented verbatim");
|
|
1072
|
+
}
|
|
1073
|
+
const executableToken = /\s/.test(executable) ? `"${executable}"` : executable;
|
|
1074
|
+
const value = [executableToken, ...args].join(" ");
|
|
1075
|
+
if (value.length === 0 || value.length > 32_766 || value.includes("\0")) {
|
|
1076
|
+
throw new LocalExecutionAuthorityError("unsupported", "Windows verbatim command line exceeds the platform limit");
|
|
1077
|
+
}
|
|
1078
|
+
return value;
|
|
1079
|
+
}
|
|
1080
|
+
function sleep(milliseconds) {
|
|
1081
|
+
return new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds));
|
|
1082
|
+
}
|
|
1083
|
+
function rawCompletionToOutcome(raw) {
|
|
1084
|
+
if (raw.cancelled) {
|
|
1085
|
+
return {
|
|
1086
|
+
kind: "cancelled",
|
|
1087
|
+
exit_code: raw.status,
|
|
1088
|
+
signal: raw.signal,
|
|
1089
|
+
reason: raw.cancellation_reason,
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
if (raw.timed_out) {
|
|
1093
|
+
return {
|
|
1094
|
+
kind: "timeout",
|
|
1095
|
+
exit_code: raw.status,
|
|
1096
|
+
signal: raw.signal,
|
|
1097
|
+
timeout_ms: raw.timeout_ms,
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
if (raw.error !== null) {
|
|
1101
|
+
return {
|
|
1102
|
+
kind: "launch_error",
|
|
1103
|
+
exit_code: null,
|
|
1104
|
+
signal: null,
|
|
1105
|
+
error: raw.error.message,
|
|
1106
|
+
};
|
|
1107
|
+
}
|
|
1108
|
+
if (raw.output_limit_exceeded || raw.process_limit_exceeded) {
|
|
1109
|
+
return {
|
|
1110
|
+
kind: "malformed_result",
|
|
1111
|
+
exit_code: raw.status,
|
|
1112
|
+
signal: null,
|
|
1113
|
+
error: raw.output_limit_exceeded
|
|
1114
|
+
? "provider output limit exceeded"
|
|
1115
|
+
: "provider process limit exceeded",
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
if (raw.signal !== null) {
|
|
1119
|
+
return { kind: "signal", exit_code: null, signal: raw.signal };
|
|
1120
|
+
}
|
|
1121
|
+
if (raw.status === 0)
|
|
1122
|
+
return { kind: "success", exit_code: 0, signal: null };
|
|
1123
|
+
return { kind: "non_zero_exit", exit_code: raw.status ?? -1, signal: null };
|
|
1124
|
+
}
|
|
1125
|
+
function rawCompletionToResult(raw, stdout, stderr) {
|
|
1126
|
+
return {
|
|
1127
|
+
stdout,
|
|
1128
|
+
stderr,
|
|
1129
|
+
status: raw.status,
|
|
1130
|
+
signal: raw.signal,
|
|
1131
|
+
timedOut: raw.timed_out,
|
|
1132
|
+
cancelled: raw.cancelled,
|
|
1133
|
+
cancellationReason: raw.cancellation_reason,
|
|
1134
|
+
processLimitExceeded: raw.process_limit_exceeded,
|
|
1135
|
+
outputLimitExceeded: raw.output_limit_exceeded,
|
|
1136
|
+
terminationReason: raw.termination_reason,
|
|
1137
|
+
processId: raw.process_id,
|
|
1138
|
+
processTreeDrained: raw.process_tree_drained,
|
|
1139
|
+
...(raw.error === null
|
|
1140
|
+
? {}
|
|
1141
|
+
: {
|
|
1142
|
+
error: Object.assign(new Error(raw.error.message), { code: raw.error.code }),
|
|
1143
|
+
}),
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1146
|
+
export class LocalAttemptExecutionAuthority {
|
|
1147
|
+
rootDirectory;
|
|
1148
|
+
preparationsDirectory;
|
|
1149
|
+
transitionsDirectory;
|
|
1150
|
+
recordsDirectory;
|
|
1151
|
+
socketsDirectory;
|
|
1152
|
+
streamsDirectory;
|
|
1153
|
+
controlNamespace;
|
|
1154
|
+
authorityInstanceId;
|
|
1155
|
+
controlTimeoutMs;
|
|
1156
|
+
pollIntervalMs;
|
|
1157
|
+
platform;
|
|
1158
|
+
architecture;
|
|
1159
|
+
windowsResolverOptions;
|
|
1160
|
+
registrations = new Map();
|
|
1161
|
+
constructor(options = {}) {
|
|
1162
|
+
this.platform = options.platform ?? process.platform;
|
|
1163
|
+
this.architecture = options.architecture ?? process.arch;
|
|
1164
|
+
this.windowsResolverOptions = {
|
|
1165
|
+
platform: this.platform,
|
|
1166
|
+
architecture: this.architecture,
|
|
1167
|
+
...(options.helper_path === undefined ? {} : { helper_path: options.helper_path }),
|
|
1168
|
+
...(options.expected_digest === undefined ? {} : { expected_digest: options.expected_digest }),
|
|
1169
|
+
};
|
|
1170
|
+
this.controlTimeoutMs = requirePositiveInteger(options.control_timeout_ms ?? DEFAULT_CONTROL_TIMEOUT_MS, "control_timeout_ms");
|
|
1171
|
+
this.pollIntervalMs = requirePositiveInteger(options.poll_interval_ms ?? DEFAULT_POLL_INTERVAL_MS, "poll_interval_ms");
|
|
1172
|
+
if (options.root_directory === undefined || options.root_directory.length === 0) {
|
|
1173
|
+
throw new LocalExecutionAuthorityError("invalid_configuration", "root_directory is required for durable local execution authority");
|
|
1174
|
+
}
|
|
1175
|
+
this.rootDirectory = resolve(options.root_directory);
|
|
1176
|
+
this.preparationsDirectory = join(this.rootDirectory, "preparations");
|
|
1177
|
+
this.transitionsDirectory = join(this.rootDirectory, "attempt-transitions");
|
|
1178
|
+
this.recordsDirectory = join(this.rootDirectory, "records");
|
|
1179
|
+
this.controlNamespace = createHash("sha256")
|
|
1180
|
+
.update(this.rootDirectory, "utf8")
|
|
1181
|
+
.digest("hex")
|
|
1182
|
+
.slice(0, 24);
|
|
1183
|
+
this.socketsDirectory = this.platform === "win32"
|
|
1184
|
+
? ""
|
|
1185
|
+
: join("/tmp", `intentdna-exec-${process.getuid?.() ?? "user"}-${this.controlNamespace}`);
|
|
1186
|
+
this.streamsDirectory = join(this.rootDirectory, "streams");
|
|
1187
|
+
for (const directory of [
|
|
1188
|
+
this.rootDirectory,
|
|
1189
|
+
this.preparationsDirectory,
|
|
1190
|
+
this.transitionsDirectory,
|
|
1191
|
+
this.recordsDirectory,
|
|
1192
|
+
this.streamsDirectory,
|
|
1193
|
+
...(this.platform === "win32" ? [] : [this.socketsDirectory]),
|
|
1194
|
+
])
|
|
1195
|
+
ensureRealPrivateDirectory(directory);
|
|
1196
|
+
this.ensureSupervisorScript();
|
|
1197
|
+
this.authorityInstanceId = this.loadOrCreateMetadata(options.authority_instance_id).authority_instance_id;
|
|
1198
|
+
}
|
|
1199
|
+
get authority_instance_id() {
|
|
1200
|
+
return this.authorityInstanceId;
|
|
1201
|
+
}
|
|
1202
|
+
resolveWindowsArtifact() {
|
|
1203
|
+
let artifact;
|
|
1204
|
+
try {
|
|
1205
|
+
artifact = resolveWindowsJobKeeperArtifact(this.windowsResolverOptions);
|
|
1206
|
+
}
|
|
1207
|
+
catch (error) {
|
|
1208
|
+
if (error instanceof WindowsJobKeeperArtifactError) {
|
|
1209
|
+
throw new LocalExecutionAuthorityError("unsupported", error.message, { cause: error });
|
|
1210
|
+
}
|
|
1211
|
+
throw error;
|
|
1212
|
+
}
|
|
1213
|
+
if (artifact === null) {
|
|
1214
|
+
throw new LocalExecutionAuthorityError("unsupported", "Windows Job keeper is unavailable");
|
|
1215
|
+
}
|
|
1216
|
+
return artifact;
|
|
1217
|
+
}
|
|
1218
|
+
ensureSupported() {
|
|
1219
|
+
if (this.platform === "win32")
|
|
1220
|
+
this.resolveWindowsArtifact();
|
|
1221
|
+
}
|
|
1222
|
+
assertExecutionSupported() {
|
|
1223
|
+
this.ensureSupported();
|
|
1224
|
+
}
|
|
1225
|
+
supervisorScriptPath() {
|
|
1226
|
+
const digestValue = this.platform === "win32"
|
|
1227
|
+
? LOCAL_WINDOWS_EXECUTION_SUPERVISOR_DIGEST
|
|
1228
|
+
: LOCAL_EXECUTION_SUPERVISOR_DIGEST;
|
|
1229
|
+
const version = digestValue.slice("sha256:".length);
|
|
1230
|
+
return join(this.rootDirectory, `supervisor.${this.platform === "win32" ? "windows." : ""}${version}.cjs`);
|
|
1231
|
+
}
|
|
1232
|
+
ensureSupervisorScript() {
|
|
1233
|
+
const path = this.supervisorScriptPath();
|
|
1234
|
+
const expectedSource = this.platform === "win32"
|
|
1235
|
+
? LOCAL_WINDOWS_EXECUTION_SUPERVISOR_SOURCE
|
|
1236
|
+
: LOCAL_EXECUTION_SUPERVISOR_SOURCE;
|
|
1237
|
+
const expectedDigest = this.platform === "win32"
|
|
1238
|
+
? LOCAL_WINDOWS_EXECUTION_SUPERVISOR_DIGEST
|
|
1239
|
+
: LOCAL_EXECUTION_SUPERVISOR_DIGEST;
|
|
1240
|
+
const validateExisting = () => {
|
|
1241
|
+
const metadata = lstatSync(path);
|
|
1242
|
+
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
|
1243
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution supervisor is not a regular file");
|
|
1244
|
+
}
|
|
1245
|
+
const source = readFileSync(path, "utf8");
|
|
1246
|
+
if (textDigest(source) !== expectedDigest) {
|
|
1247
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "execution supervisor digest changed");
|
|
1248
|
+
}
|
|
1249
|
+
chmodSync(path, SCRIPT_FILE_MODE);
|
|
1250
|
+
};
|
|
1251
|
+
if (existsSync(path)) {
|
|
1252
|
+
validateExisting();
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
let descriptor;
|
|
1256
|
+
try {
|
|
1257
|
+
descriptor = openSync(path, "wx", PRIVATE_FILE_MODE);
|
|
1258
|
+
}
|
|
1259
|
+
catch (error) {
|
|
1260
|
+
if (!isErrno(error, "EEXIST"))
|
|
1261
|
+
throw error;
|
|
1262
|
+
validateExisting();
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
try {
|
|
1266
|
+
writeFileSync(descriptor, expectedSource, "utf8");
|
|
1267
|
+
fsyncSync(descriptor);
|
|
1268
|
+
}
|
|
1269
|
+
finally {
|
|
1270
|
+
closeSync(descriptor);
|
|
1271
|
+
}
|
|
1272
|
+
chmodSync(path, SCRIPT_FILE_MODE);
|
|
1273
|
+
syncDirectory(this.rootDirectory);
|
|
1274
|
+
}
|
|
1275
|
+
loadOrCreateMetadata(requested) {
|
|
1276
|
+
const path = join(this.rootDirectory, "authority.json");
|
|
1277
|
+
const load = () => {
|
|
1278
|
+
const value = readJson(path);
|
|
1279
|
+
if (!isRecord(value)
|
|
1280
|
+
|| value.schema_version !== AUTHORITY_SCHEMA_VERSION
|
|
1281
|
+
|| typeof value.authority_instance_id !== "string"
|
|
1282
|
+
|| value.authority_instance_id.length === 0) {
|
|
1283
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "local authority metadata is invalid");
|
|
1284
|
+
}
|
|
1285
|
+
const metadata = value;
|
|
1286
|
+
if (requested !== undefined && requested !== metadata.authority_instance_id) {
|
|
1287
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "configured authority_instance_id conflicts with the durable authority root");
|
|
1288
|
+
}
|
|
1289
|
+
return metadata;
|
|
1290
|
+
};
|
|
1291
|
+
if (existsSync(path)) {
|
|
1292
|
+
return load();
|
|
1293
|
+
}
|
|
1294
|
+
const metadata = {
|
|
1295
|
+
schema_version: AUTHORITY_SCHEMA_VERSION,
|
|
1296
|
+
authority_instance_id: requested ?? `local-execution-authority:${randomUUID()}`,
|
|
1297
|
+
created_at: new Date().toISOString(),
|
|
1298
|
+
};
|
|
1299
|
+
try {
|
|
1300
|
+
const descriptor = openSync(path, "wx", PRIVATE_FILE_MODE);
|
|
1301
|
+
try {
|
|
1302
|
+
writeFileSync(descriptor, `${canonicalizeJson(metadata)}\n`, "utf8");
|
|
1303
|
+
fsyncSync(descriptor);
|
|
1304
|
+
}
|
|
1305
|
+
finally {
|
|
1306
|
+
closeSync(descriptor);
|
|
1307
|
+
}
|
|
1308
|
+
syncDirectory(this.rootDirectory);
|
|
1309
|
+
return metadata;
|
|
1310
|
+
}
|
|
1311
|
+
catch (error) {
|
|
1312
|
+
if (!isErrno(error, "EEXIST"))
|
|
1313
|
+
throw error;
|
|
1314
|
+
return load();
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
recordPath(identity) {
|
|
1318
|
+
return join(this.recordsDirectory, `${executionRecordDigest(identity).slice("sha256:".length)}.json`);
|
|
1319
|
+
}
|
|
1320
|
+
hasOtherAttemptRecord(identity) {
|
|
1321
|
+
const expectedPath = this.recordPath(identity);
|
|
1322
|
+
return readdirSync(this.recordsDirectory, { withFileTypes: true }).some((entry) => {
|
|
1323
|
+
if (!entry.isFile() || entry.isSymbolicLink() || !entry.name.endsWith(".json"))
|
|
1324
|
+
return false;
|
|
1325
|
+
const path = join(this.recordsDirectory, entry.name);
|
|
1326
|
+
if (path === expectedPath)
|
|
1327
|
+
return false;
|
|
1328
|
+
const record = this.readRecord(path);
|
|
1329
|
+
return record.identity.run_id === identity.run_id
|
|
1330
|
+
&& record.identity.step_id === identity.step_id
|
|
1331
|
+
&& record.identity.attempt_id === identity.attempt_id;
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
preparationPath(identity) {
|
|
1335
|
+
return join(this.preparationsDirectory, `${attemptDigest(identity).slice("sha256:".length)}.json`);
|
|
1336
|
+
}
|
|
1337
|
+
transitionPath(identity) {
|
|
1338
|
+
return join(this.transitionsDirectory, attemptDigest(identity).slice("sha256:".length));
|
|
1339
|
+
}
|
|
1340
|
+
controlEndpoint(identity) {
|
|
1341
|
+
const token = attemptDigest(identity).slice("sha256:".length, "sha256:".length + 24);
|
|
1342
|
+
if (this.platform === "win32") {
|
|
1343
|
+
return `\\\\.\\pipe\\intentdna-exec-${this.controlNamespace}-${token}`;
|
|
1344
|
+
}
|
|
1345
|
+
const socketPath = join(this.socketsDirectory, `${token}.sock`);
|
|
1346
|
+
if (Buffer.byteLength(socketPath, "utf8") > 100) {
|
|
1347
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution control socket path is too long");
|
|
1348
|
+
}
|
|
1349
|
+
return socketPath;
|
|
1350
|
+
}
|
|
1351
|
+
backend() {
|
|
1352
|
+
if (this.platform !== "win32")
|
|
1353
|
+
return { kind: "posix_process_group" };
|
|
1354
|
+
const artifact = this.resolveWindowsArtifact();
|
|
1355
|
+
return {
|
|
1356
|
+
kind: "windows_job",
|
|
1357
|
+
architecture: artifact.architecture,
|
|
1358
|
+
helper_path: artifact.path,
|
|
1359
|
+
helper_digest: artifact.digest,
|
|
1360
|
+
helper_source_digest: artifact.source_digest,
|
|
1361
|
+
helper_protocol_version: artifact.protocol_version,
|
|
1362
|
+
containment_nonce: randomUUID().replaceAll("-", "") + randomUUID().replaceAll("-", ""),
|
|
1363
|
+
};
|
|
1364
|
+
}
|
|
1365
|
+
supervisor(endpoint) {
|
|
1366
|
+
return {
|
|
1367
|
+
kind: this.platform === "win32" ? "windows" : "posix",
|
|
1368
|
+
socket_path: endpoint,
|
|
1369
|
+
script_digest: this.platform === "win32"
|
|
1370
|
+
? LOCAL_WINDOWS_EXECUTION_SUPERVISOR_DIGEST
|
|
1371
|
+
: LOCAL_EXECUTION_SUPERVISOR_DIGEST,
|
|
1372
|
+
pid: null,
|
|
1373
|
+
ready_at: null,
|
|
1374
|
+
};
|
|
1375
|
+
}
|
|
1376
|
+
resolveAbsoluteWindowsExecutable(command, environment, cwd) {
|
|
1377
|
+
const direct = win32Path.isAbsolute(command)
|
|
1378
|
+
? command
|
|
1379
|
+
: (command.includes("\\") || command.includes("/"))
|
|
1380
|
+
? win32Path.resolve(cwd, command)
|
|
1381
|
+
: null;
|
|
1382
|
+
if (direct !== null && existsSync(direct))
|
|
1383
|
+
return resolve(direct);
|
|
1384
|
+
const resolved = resolveProcessCommand(command, [], "win32", environment, cwd).command;
|
|
1385
|
+
if (win32Path.isAbsolute(resolved) && existsSync(resolved))
|
|
1386
|
+
return resolve(resolved);
|
|
1387
|
+
throw new LocalExecutionAuthorityError("unsupported", `Windows provider executable could not be resolved to an absolute path: ${command}`);
|
|
1388
|
+
}
|
|
1389
|
+
readRecord(path) {
|
|
1390
|
+
return parseRecord(readJson(path));
|
|
1391
|
+
}
|
|
1392
|
+
readPreparation(path) {
|
|
1393
|
+
const value = readJson(path);
|
|
1394
|
+
if (!isRecord(value) || !hasExactKeys(value, [
|
|
1395
|
+
"schema_version",
|
|
1396
|
+
"identity",
|
|
1397
|
+
"preparation_digest",
|
|
1398
|
+
"execution_instance_id",
|
|
1399
|
+
"challenge",
|
|
1400
|
+
"authority_instance_id",
|
|
1401
|
+
"backend",
|
|
1402
|
+
"supervisor",
|
|
1403
|
+
"created_at",
|
|
1404
|
+
]) || value.schema_version !== PREPARATION_SCHEMA_VERSION || !isRecord(value.identity)) {
|
|
1405
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "local execution preparation has an invalid schema");
|
|
1406
|
+
}
|
|
1407
|
+
const preparation = value;
|
|
1408
|
+
requirePreparationRequest(preparation.identity);
|
|
1409
|
+
if (preparation.identity.execution_instance_id !== preparation.execution_instance_id
|
|
1410
|
+
|| preparation.preparation_digest !== preparationDigest(preparation.identity)
|
|
1411
|
+
|| typeof preparation.challenge !== "string"
|
|
1412
|
+
|| preparation.challenge.length < 32
|
|
1413
|
+
|| typeof preparation.authority_instance_id !== "string"
|
|
1414
|
+
|| preparation.authority_instance_id.length === 0
|
|
1415
|
+
|| !isDurableBackend(preparation.backend)
|
|
1416
|
+
|| !isDurableSupervisor(preparation.supervisor, preparation.backend)
|
|
1417
|
+
|| preparation.supervisor.pid !== null
|
|
1418
|
+
|| preparation.supervisor.ready_at !== null
|
|
1419
|
+
|| !isCanonicalTimestamp(preparation.created_at)) {
|
|
1420
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "local execution preparation failed validation");
|
|
1421
|
+
}
|
|
1422
|
+
return preparation;
|
|
1423
|
+
}
|
|
1424
|
+
readVerifiedStream(evidence, expectedPath, maxOutputBytes) {
|
|
1425
|
+
const expected = resolve(expectedPath);
|
|
1426
|
+
const actual = resolve(evidence.path);
|
|
1427
|
+
const relativePath = relative(this.streamsDirectory, actual);
|
|
1428
|
+
if (actual !== expected
|
|
1429
|
+
|| relativePath === ""
|
|
1430
|
+
|| relativePath === ".."
|
|
1431
|
+
|| relativePath.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)) {
|
|
1432
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "durable stream path escapes its execution identity");
|
|
1433
|
+
}
|
|
1434
|
+
const pathMetadata = lstatSync(actual, { bigint: true });
|
|
1435
|
+
if (!pathMetadata.isFile() || pathMetadata.isSymbolicLink()) {
|
|
1436
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "durable stream is not a regular file");
|
|
1437
|
+
}
|
|
1438
|
+
const descriptor = openSync(actual, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0));
|
|
1439
|
+
try {
|
|
1440
|
+
const before = fstatSync(descriptor, { bigint: true });
|
|
1441
|
+
if (!before.isFile()
|
|
1442
|
+
|| before.dev.toString() !== evidence.device
|
|
1443
|
+
|| before.ino.toString() !== evidence.inode
|
|
1444
|
+
|| before.birthtimeNs.toString() !== evidence.birthtime_ns
|
|
1445
|
+
|| before.size !== BigInt(evidence.byte_count)
|
|
1446
|
+
|| before.size > BigInt(maxOutputBytes)) {
|
|
1447
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "durable stream identity does not match its evidence");
|
|
1448
|
+
}
|
|
1449
|
+
const bytes = readFileSync(descriptor);
|
|
1450
|
+
const after = fstatSync(descriptor, { bigint: true });
|
|
1451
|
+
const currentPath = lstatSync(actual, { bigint: true });
|
|
1452
|
+
if (bytes.byteLength !== evidence.byte_count
|
|
1453
|
+
|| after.dev !== before.dev
|
|
1454
|
+
|| after.ino !== before.ino
|
|
1455
|
+
|| after.birthtimeNs !== before.birthtimeNs
|
|
1456
|
+
|| after.size !== before.size
|
|
1457
|
+
|| currentPath.dev !== before.dev
|
|
1458
|
+
|| currentPath.ino !== before.ino
|
|
1459
|
+
|| currentPath.birthtimeNs !== before.birthtimeNs
|
|
1460
|
+
|| digestBytes(bytes) !== evidence.sha256) {
|
|
1461
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "durable stream changed after completion");
|
|
1462
|
+
}
|
|
1463
|
+
return bytes.toString("utf8");
|
|
1464
|
+
}
|
|
1465
|
+
finally {
|
|
1466
|
+
closeSync(descriptor);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
rawCompletionResult(record) {
|
|
1470
|
+
const raw = record.raw_completion;
|
|
1471
|
+
if (raw === null) {
|
|
1472
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "durable raw completion is missing");
|
|
1473
|
+
}
|
|
1474
|
+
const streamToken = record.identity_digest.slice("sha256:".length, "sha256:".length + 24);
|
|
1475
|
+
const stdout = this.readVerifiedStream(raw.stdout, join(this.streamsDirectory, `${streamToken}.stdout`), raw.max_output_bytes);
|
|
1476
|
+
const stderr = this.readVerifiedStream(raw.stderr, join(this.streamsDirectory, `${streamToken}.stderr`), raw.max_output_bytes);
|
|
1477
|
+
return {
|
|
1478
|
+
...rawCompletionToResult(raw, stdout, stderr),
|
|
1479
|
+
startedAt: record.identity.started_at,
|
|
1480
|
+
completedAt: raw.completed_at,
|
|
1481
|
+
stdoutDigest: raw.stdout.sha256,
|
|
1482
|
+
stdoutBytes: raw.stdout.byte_count,
|
|
1483
|
+
stderrDigest: raw.stderr.sha256,
|
|
1484
|
+
stderrBytes: raw.stderr.byte_count,
|
|
1485
|
+
};
|
|
1486
|
+
}
|
|
1487
|
+
windowsArtifactMatchesBackend(artifact, backend) {
|
|
1488
|
+
return artifact.architecture === backend.architecture
|
|
1489
|
+
&& artifact.path === backend.helper_path
|
|
1490
|
+
&& artifact.digest === backend.helper_digest
|
|
1491
|
+
&& artifact.source_digest === backend.helper_source_digest
|
|
1492
|
+
&& artifact.protocol_version === backend.helper_protocol_version;
|
|
1493
|
+
}
|
|
1494
|
+
async markUnsupported(path, message) {
|
|
1495
|
+
return this.updateRecord(path, (current) => {
|
|
1496
|
+
if (current.phase === "unsupported"
|
|
1497
|
+
|| current.phase === "never_launched"
|
|
1498
|
+
|| current.raw_completion !== null)
|
|
1499
|
+
return current;
|
|
1500
|
+
return {
|
|
1501
|
+
...current,
|
|
1502
|
+
phase: "unsupported",
|
|
1503
|
+
supervisor_error: message,
|
|
1504
|
+
revision: current.revision + 1,
|
|
1505
|
+
updated_at: new Date().toISOString(),
|
|
1506
|
+
};
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
async failWindowsAuthentication(path, message, cause) {
|
|
1510
|
+
const updated = await this.markUnsupported(path, message);
|
|
1511
|
+
if (updated.raw_completion !== null) {
|
|
1512
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "Windows execution identity changed while completion was committed", { cause });
|
|
1513
|
+
}
|
|
1514
|
+
throw new LocalExecutionAuthorityError("identity_conflict", message, { cause });
|
|
1515
|
+
}
|
|
1516
|
+
async ensureWindowsBackendTrusted(record, path) {
|
|
1517
|
+
if (record.backend.kind !== "windows_job")
|
|
1518
|
+
return;
|
|
1519
|
+
try {
|
|
1520
|
+
const artifact = this.resolveWindowsArtifact();
|
|
1521
|
+
if (!this.windowsArtifactMatchesBackend(artifact, record.backend)) {
|
|
1522
|
+
throw new LocalExecutionAuthorityError("unsupported", "Windows Job keeper trust identity does not match the durable execution record");
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
catch (error) {
|
|
1526
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1527
|
+
const updated = await this.markUnsupported(path, message);
|
|
1528
|
+
if (updated.raw_completion !== null)
|
|
1529
|
+
return;
|
|
1530
|
+
throw error instanceof LocalExecutionAuthorityError
|
|
1531
|
+
? error
|
|
1532
|
+
: new LocalExecutionAuthorityError("unsupported", message, { cause: error });
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
async authenticateWindowsSupervisor(initial, path) {
|
|
1536
|
+
if (initial.backend.kind !== "windows_job" || initial.raw_completion !== null)
|
|
1537
|
+
return initial;
|
|
1538
|
+
await this.ensureWindowsBackendTrusted(initial, path);
|
|
1539
|
+
let lastError = null;
|
|
1540
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
1541
|
+
const before = this.readRecord(path);
|
|
1542
|
+
if (before.raw_completion !== null || before.phase === "unsupported")
|
|
1543
|
+
return before;
|
|
1544
|
+
if (before.phase !== "running" || before.supervisor.pid === null || before.supervisor.ready_at === null) {
|
|
1545
|
+
lastError = new Error("Windows execution supervisor is not durably ready");
|
|
1546
|
+
break;
|
|
1547
|
+
}
|
|
1548
|
+
try {
|
|
1549
|
+
const response = await this.sendControl(before, { action: "probe" });
|
|
1550
|
+
const after = this.readRecord(path);
|
|
1551
|
+
if (after.raw_completion !== null || after.phase === "unsupported")
|
|
1552
|
+
return after;
|
|
1553
|
+
if (after.revision !== before.revision)
|
|
1554
|
+
continue;
|
|
1555
|
+
const identity = response.identity;
|
|
1556
|
+
const expected = {
|
|
1557
|
+
record_revision: after.revision,
|
|
1558
|
+
execution_instance_id: after.execution_instance_id,
|
|
1559
|
+
supervisor_pid: after.supervisor.pid,
|
|
1560
|
+
supervisor_ready_at: after.supervisor.ready_at,
|
|
1561
|
+
containment_nonce: after.backend.kind === "windows_job"
|
|
1562
|
+
? after.backend.containment_nonce
|
|
1563
|
+
: "",
|
|
1564
|
+
backend: after.backend,
|
|
1565
|
+
provider: after.provider,
|
|
1566
|
+
};
|
|
1567
|
+
if (!isRecord(identity)
|
|
1568
|
+
|| !hasExactKeys(identity, Object.keys(expected))
|
|
1569
|
+
|| canonicalizeJson(identity) !== canonicalizeJson(expected)) {
|
|
1570
|
+
throw new Error("Windows execution supervisor identity does not match durable containment evidence");
|
|
1571
|
+
}
|
|
1572
|
+
return after;
|
|
1573
|
+
}
|
|
1574
|
+
catch (error) {
|
|
1575
|
+
lastError = error;
|
|
1576
|
+
break;
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
const message = lastError instanceof Error
|
|
1580
|
+
? lastError.message
|
|
1581
|
+
: "Windows execution supervisor identity could not be authenticated";
|
|
1582
|
+
return this.failWindowsAuthentication(path, message, lastError);
|
|
1583
|
+
}
|
|
1584
|
+
readRecoveryRecord(request) {
|
|
1585
|
+
const identity = recoveryIdentity(request);
|
|
1586
|
+
const path = this.recordPath(identity);
|
|
1587
|
+
if (!existsSync(path)) {
|
|
1588
|
+
if (this.hasOtherAttemptRecord(identity)) {
|
|
1589
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "recovery identity does not match the original execution authority claim");
|
|
1590
|
+
}
|
|
1591
|
+
return null;
|
|
1592
|
+
}
|
|
1593
|
+
const record = this.readRecord(path);
|
|
1594
|
+
if (canonicalizeJson(coreIdentity(record.identity)) !== canonicalizeJson(coreIdentity(identity))) {
|
|
1595
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "recovery identity does not match the original execution authority");
|
|
1596
|
+
}
|
|
1597
|
+
return record;
|
|
1598
|
+
}
|
|
1599
|
+
requireRequestRecord(request) {
|
|
1600
|
+
const identity = requestIdentity(request);
|
|
1601
|
+
const record = this.readRecord(this.recordPath(identity));
|
|
1602
|
+
if (canonicalizeJson(coreIdentity(record.identity)) !== canonicalizeJson(coreIdentity(identity))) {
|
|
1603
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution authority identity does not match");
|
|
1604
|
+
}
|
|
1605
|
+
const requestedExecutionInstance = request.execution_instance_id;
|
|
1606
|
+
if (requestedExecutionInstance !== undefined
|
|
1607
|
+
&& requestedExecutionInstance !== record.execution_instance_id) {
|
|
1608
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution instance identity does not match");
|
|
1609
|
+
}
|
|
1610
|
+
return record;
|
|
1611
|
+
}
|
|
1612
|
+
async withLock(targetPath, operation) {
|
|
1613
|
+
const lockPath = `${targetPath}.lock`;
|
|
1614
|
+
const ownerPath = join(lockPath, "owner.json");
|
|
1615
|
+
const owner = durableLockOwner(this.platform);
|
|
1616
|
+
const deadline = Date.now() + this.controlTimeoutMs;
|
|
1617
|
+
for (;;) {
|
|
1618
|
+
if (publishDurableLock(lockPath, owner))
|
|
1619
|
+
break;
|
|
1620
|
+
const observed = readDurableLockOwner(ownerPath);
|
|
1621
|
+
if (staleDurableLock(observed, this.platform)
|
|
1622
|
+
&& observed !== null
|
|
1623
|
+
&& retireObservedStaleDurableLock(lockPath, observed)) {
|
|
1624
|
+
continue;
|
|
1625
|
+
}
|
|
1626
|
+
if (Date.now() >= deadline) {
|
|
1627
|
+
throw new LocalExecutionAuthorityError("unsupported", `execution authority lock is unavailable for ${targetPath}`);
|
|
1628
|
+
}
|
|
1629
|
+
await sleep(this.pollIntervalMs);
|
|
1630
|
+
}
|
|
1631
|
+
try {
|
|
1632
|
+
return await operation();
|
|
1633
|
+
}
|
|
1634
|
+
finally {
|
|
1635
|
+
releaseDurableLock(lockPath, owner);
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
withLockSync(targetPath, operation) {
|
|
1639
|
+
const lockPath = `${targetPath}.lock`;
|
|
1640
|
+
const ownerPath = join(lockPath, "owner.json");
|
|
1641
|
+
const owner = durableLockOwner(this.platform);
|
|
1642
|
+
const deadline = Date.now() + this.controlTimeoutMs;
|
|
1643
|
+
for (;;) {
|
|
1644
|
+
if (publishDurableLock(lockPath, owner))
|
|
1645
|
+
break;
|
|
1646
|
+
const observed = readDurableLockOwner(ownerPath);
|
|
1647
|
+
if (staleDurableLock(observed, this.platform)
|
|
1648
|
+
&& observed !== null
|
|
1649
|
+
&& retireObservedStaleDurableLock(lockPath, observed)) {
|
|
1650
|
+
continue;
|
|
1651
|
+
}
|
|
1652
|
+
if (Date.now() >= deadline) {
|
|
1653
|
+
throw new LocalExecutionAuthorityError("unsupported", `execution authority lock is unavailable for ${targetPath}`);
|
|
1654
|
+
}
|
|
1655
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, this.pollIntervalMs);
|
|
1656
|
+
}
|
|
1657
|
+
try {
|
|
1658
|
+
return operation();
|
|
1659
|
+
}
|
|
1660
|
+
finally {
|
|
1661
|
+
releaseDurableLock(lockPath, owner);
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
async updateRecord(path, operation) {
|
|
1665
|
+
return this.withLock(path, () => {
|
|
1666
|
+
const current = this.readRecord(path);
|
|
1667
|
+
const next = operation(current);
|
|
1668
|
+
if (next === current)
|
|
1669
|
+
return current;
|
|
1670
|
+
if (next.execution_instance_id !== current.execution_instance_id
|
|
1671
|
+
|| next.challenge !== current.challenge
|
|
1672
|
+
|| next.revision !== current.revision + 1) {
|
|
1673
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "invalid execution record transition");
|
|
1674
|
+
}
|
|
1675
|
+
atomicWriteJson(path, next);
|
|
1676
|
+
return next;
|
|
1677
|
+
});
|
|
1678
|
+
}
|
|
1679
|
+
nextFencingEpoch() {
|
|
1680
|
+
const path = join(this.rootDirectory, "fencing.epoch");
|
|
1681
|
+
return this.withLock(path, () => {
|
|
1682
|
+
let current = 0;
|
|
1683
|
+
if (existsSync(path)) {
|
|
1684
|
+
const metadata = lstatSync(path);
|
|
1685
|
+
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
|
1686
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "fencing epoch is not a regular file");
|
|
1687
|
+
}
|
|
1688
|
+
current = Number(readFileSync(path, "utf8"));
|
|
1689
|
+
if (!Number.isSafeInteger(current) || current < 0) {
|
|
1690
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "fencing epoch is invalid");
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
const next = current + 1;
|
|
1694
|
+
if (!Number.isSafeInteger(next)) {
|
|
1695
|
+
throw new LocalExecutionAuthorityError("unsupported", "fencing epoch is exhausted");
|
|
1696
|
+
}
|
|
1697
|
+
const temporary = `${path}.tmp.${process.pid}.${randomUUID()}`;
|
|
1698
|
+
const descriptor = openSync(temporary, "wx", PRIVATE_FILE_MODE);
|
|
1699
|
+
try {
|
|
1700
|
+
writeFileSync(descriptor, String(next), "utf8");
|
|
1701
|
+
fsyncSync(descriptor);
|
|
1702
|
+
}
|
|
1703
|
+
finally {
|
|
1704
|
+
closeSync(descriptor);
|
|
1705
|
+
}
|
|
1706
|
+
replaceFileSync(temporary, path);
|
|
1707
|
+
syncDirectory(this.rootDirectory);
|
|
1708
|
+
return next;
|
|
1709
|
+
});
|
|
1710
|
+
}
|
|
1711
|
+
nextFencingEpochSync() {
|
|
1712
|
+
const path = join(this.rootDirectory, "fencing.epoch");
|
|
1713
|
+
return this.withLockSync(path, () => {
|
|
1714
|
+
let current = 0;
|
|
1715
|
+
if (existsSync(path)) {
|
|
1716
|
+
const metadata = lstatSync(path);
|
|
1717
|
+
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
|
1718
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "fencing epoch is not a regular file");
|
|
1719
|
+
}
|
|
1720
|
+
current = Number(readFileSync(path, "utf8"));
|
|
1721
|
+
if (!Number.isSafeInteger(current) || current < 0) {
|
|
1722
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "fencing epoch is invalid");
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
const next = current + 1;
|
|
1726
|
+
if (!Number.isSafeInteger(next)) {
|
|
1727
|
+
throw new LocalExecutionAuthorityError("unsupported", "fencing epoch is exhausted");
|
|
1728
|
+
}
|
|
1729
|
+
atomicWriteJson(path, next);
|
|
1730
|
+
return next;
|
|
1731
|
+
});
|
|
1732
|
+
}
|
|
1733
|
+
createLaunchBinding(record) {
|
|
1734
|
+
if (record.phase !== "prepared") {
|
|
1735
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution is no longer prepared for launch");
|
|
1736
|
+
}
|
|
1737
|
+
const recordPath = this.recordPath(record.identity);
|
|
1738
|
+
const existing = this.registrations.get(record.identity_digest);
|
|
1739
|
+
if (existing !== undefined) {
|
|
1740
|
+
return this.bindingForRegistration(existing);
|
|
1741
|
+
}
|
|
1742
|
+
let resolveProcessStarted;
|
|
1743
|
+
let rejectProcessStarted;
|
|
1744
|
+
const processStarted = new Promise((resolvePromise, rejectPromise) => {
|
|
1745
|
+
resolveProcessStarted = resolvePromise;
|
|
1746
|
+
rejectProcessStarted = rejectPromise;
|
|
1747
|
+
});
|
|
1748
|
+
const registration = {
|
|
1749
|
+
identity: record.identity,
|
|
1750
|
+
recordPath,
|
|
1751
|
+
abortController: new AbortController(),
|
|
1752
|
+
processStarted,
|
|
1753
|
+
resolveProcessStarted,
|
|
1754
|
+
rejectProcessStarted,
|
|
1755
|
+
};
|
|
1756
|
+
this.registrations.set(record.identity_digest, registration);
|
|
1757
|
+
return this.bindingForRegistration(registration);
|
|
1758
|
+
}
|
|
1759
|
+
bindingForRegistration(registration) {
|
|
1760
|
+
return {
|
|
1761
|
+
cancellation_signal: registration.abortController.signal,
|
|
1762
|
+
process_started: registration.processStarted,
|
|
1763
|
+
processStarted: (processId) => {
|
|
1764
|
+
if (!Number.isSafeInteger(processId) || processId <= 0) {
|
|
1765
|
+
const error = new LocalExecutionAuthorityError("corrupt_state", "execution process id is invalid");
|
|
1766
|
+
registration.rejectProcessStarted(error);
|
|
1767
|
+
throw error;
|
|
1768
|
+
}
|
|
1769
|
+
const current = this.readRecord(registration.recordPath);
|
|
1770
|
+
if (!providerMatchesProcessId(current, processId)) {
|
|
1771
|
+
const error = new LocalExecutionAuthorityError("identity_conflict", "execution process does not match the durable supervisor record");
|
|
1772
|
+
registration.rejectProcessStarted(error);
|
|
1773
|
+
throw error;
|
|
1774
|
+
}
|
|
1775
|
+
registration.resolveProcessStarted(processId);
|
|
1776
|
+
},
|
|
1777
|
+
processTreeCompleted: (observation) => this.recordNormalizedCompletion(registration.recordPath, observation),
|
|
1778
|
+
executeProcessTree: (command, args, processOptions) => this.executeSupervised(registration, command, args, processOptions),
|
|
1779
|
+
};
|
|
1780
|
+
}
|
|
1781
|
+
prepareExecution(request) {
|
|
1782
|
+
requirePreparationRequest(request);
|
|
1783
|
+
this.ensureSupported();
|
|
1784
|
+
const path = this.preparationPath(request);
|
|
1785
|
+
return this.withLockSync(this.transitionPath(request), () => {
|
|
1786
|
+
const recordPath = this.recordPath({
|
|
1787
|
+
...request,
|
|
1788
|
+
claim_epoch: request.execution_claim_epoch,
|
|
1789
|
+
started_at: "1970-01-01T00:00:00.000Z",
|
|
1790
|
+
});
|
|
1791
|
+
const boundRecord = existsSync(recordPath) ? this.readRecord(recordPath) : null;
|
|
1792
|
+
if (existsSync(path)) {
|
|
1793
|
+
const current = this.readPreparation(path);
|
|
1794
|
+
const sameAttempt = current.identity.run_id === request.run_id
|
|
1795
|
+
&& current.identity.step_id === request.step_id
|
|
1796
|
+
&& current.identity.activation_id === request.activation_id
|
|
1797
|
+
&& current.identity.attempt_id === request.attempt_id;
|
|
1798
|
+
if (!sameAttempt || request.execution_claim_epoch < current.identity.execution_claim_epoch) {
|
|
1799
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution attempt already has a different durable preparation");
|
|
1800
|
+
}
|
|
1801
|
+
if (request.execution_claim_epoch > current.identity.execution_claim_epoch) {
|
|
1802
|
+
if (boundRecord !== null) {
|
|
1803
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "a bound execution preparation cannot be superseded");
|
|
1804
|
+
}
|
|
1805
|
+
unlinkSync(path);
|
|
1806
|
+
}
|
|
1807
|
+
else if (canonicalizeJson(preparationCoreIdentity(current.identity))
|
|
1808
|
+
!== canonicalizeJson(preparationCoreIdentity(request))) {
|
|
1809
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "same-epoch preparation has a different worker identity");
|
|
1810
|
+
}
|
|
1811
|
+
else {
|
|
1812
|
+
if (request.execution_instance_id !== undefined
|
|
1813
|
+
&& request.execution_instance_id !== current.execution_instance_id) {
|
|
1814
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution instance conflicts with the durable preparation");
|
|
1815
|
+
}
|
|
1816
|
+
return preparedHandle(current);
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
if (boundRecord !== null) {
|
|
1820
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution attempt is already bound and cannot be prepared again");
|
|
1821
|
+
}
|
|
1822
|
+
const executionInstanceId = request.execution_instance_id ?? `local-process:${randomUUID()}`;
|
|
1823
|
+
const fullIdentity = {
|
|
1824
|
+
...request,
|
|
1825
|
+
execution_instance_id: executionInstanceId,
|
|
1826
|
+
};
|
|
1827
|
+
const preparation = {
|
|
1828
|
+
schema_version: PREPARATION_SCHEMA_VERSION,
|
|
1829
|
+
identity: fullIdentity,
|
|
1830
|
+
preparation_digest: preparationDigest(fullIdentity),
|
|
1831
|
+
execution_instance_id: executionInstanceId,
|
|
1832
|
+
challenge: randomUUID().replaceAll("-", "") + randomUUID().replaceAll("-", ""),
|
|
1833
|
+
authority_instance_id: this.authorityInstanceId,
|
|
1834
|
+
backend: this.backend(),
|
|
1835
|
+
supervisor: this.supervisor(this.controlEndpoint({
|
|
1836
|
+
...request,
|
|
1837
|
+
claim_epoch: request.execution_claim_epoch,
|
|
1838
|
+
started_at: "1970-01-01T00:00:00.000Z",
|
|
1839
|
+
})),
|
|
1840
|
+
created_at: new Date().toISOString(),
|
|
1841
|
+
};
|
|
1842
|
+
atomicWriteJson(path, preparation);
|
|
1843
|
+
return preparedHandle(preparation);
|
|
1844
|
+
});
|
|
1845
|
+
}
|
|
1846
|
+
recoverPreparedExecution(request) {
|
|
1847
|
+
requirePreparationRequest(request);
|
|
1848
|
+
this.ensureSupported();
|
|
1849
|
+
const path = this.preparationPath(request);
|
|
1850
|
+
if (!existsSync(path))
|
|
1851
|
+
return null;
|
|
1852
|
+
const preparation = this.readPreparation(path);
|
|
1853
|
+
if (canonicalizeJson(preparationCoreIdentity(preparation.identity))
|
|
1854
|
+
!== canonicalizeJson(preparationCoreIdentity(request))) {
|
|
1855
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "prepared execution recovery identity does not match");
|
|
1856
|
+
}
|
|
1857
|
+
return preparedHandle(preparation);
|
|
1858
|
+
}
|
|
1859
|
+
recoverExecutionForPreparation(request) {
|
|
1860
|
+
requirePreparationRequest(request);
|
|
1861
|
+
this.ensureSupported();
|
|
1862
|
+
const recordPath = this.recordPath({
|
|
1863
|
+
...request,
|
|
1864
|
+
claim_epoch: request.execution_claim_epoch,
|
|
1865
|
+
started_at: "1970-01-01T00:00:00.000Z",
|
|
1866
|
+
});
|
|
1867
|
+
if (!existsSync(recordPath))
|
|
1868
|
+
return null;
|
|
1869
|
+
const record = this.readRecord(recordPath);
|
|
1870
|
+
const recordedPreparationIdentity = {
|
|
1871
|
+
run_id: record.identity.run_id,
|
|
1872
|
+
step_id: record.identity.step_id,
|
|
1873
|
+
activation_id: record.identity.activation_id,
|
|
1874
|
+
attempt_id: record.identity.attempt_id,
|
|
1875
|
+
execution_claim_epoch: record.identity.claim_epoch,
|
|
1876
|
+
worker_session_id: record.identity.worker_session_id,
|
|
1877
|
+
execution_instance_id: record.identity.execution_instance_id,
|
|
1878
|
+
};
|
|
1879
|
+
if (canonicalizeJson(preparationCoreIdentity(recordedPreparationIdentity))
|
|
1880
|
+
!== canonicalizeJson(preparationCoreIdentity(request))) {
|
|
1881
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "durable execution identity does not match the verifier preparation");
|
|
1882
|
+
}
|
|
1883
|
+
return this.recoverExecution({
|
|
1884
|
+
run_id: record.identity.run_id,
|
|
1885
|
+
step_id: record.identity.step_id,
|
|
1886
|
+
activation_id: record.identity.activation_id,
|
|
1887
|
+
attempt_id: record.identity.attempt_id,
|
|
1888
|
+
execution_claim_epoch: record.identity.claim_epoch,
|
|
1889
|
+
executor: { kind: "worker", worker_session_id: record.identity.worker_session_id },
|
|
1890
|
+
started_at: record.identity.started_at,
|
|
1891
|
+
});
|
|
1892
|
+
}
|
|
1893
|
+
listPreparedExecutions() {
|
|
1894
|
+
const entries = readdirSync(this.preparationsDirectory, { withFileTypes: true })
|
|
1895
|
+
.filter((entry) => !/^\..+\.json\.tmp\.\d+\.[0-9a-f-]+$/u.test(entry.name))
|
|
1896
|
+
.sort((left, right) => left.name.localeCompare(right.name));
|
|
1897
|
+
return entries.map((entry) => {
|
|
1898
|
+
if (!entry.isFile() || entry.isSymbolicLink() || !entry.name.endsWith(".json")) {
|
|
1899
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "local execution preparations directory contains an unexpected entry");
|
|
1900
|
+
}
|
|
1901
|
+
const preparation = this.readPreparation(join(this.preparationsDirectory, entry.name));
|
|
1902
|
+
return {
|
|
1903
|
+
handle: preparedHandle(preparation),
|
|
1904
|
+
created_at: preparation.created_at,
|
|
1905
|
+
};
|
|
1906
|
+
});
|
|
1907
|
+
}
|
|
1908
|
+
abandonPreparedExecution(handle) {
|
|
1909
|
+
requirePreparationRequest(handle);
|
|
1910
|
+
if (handle.schema_version !== PREPARATION_HANDLE_SCHEMA_VERSION
|
|
1911
|
+
|| handle.preparation_digest !== preparationDigest(handle)) {
|
|
1912
|
+
throw new TypeError("local execution preparation handle is invalid");
|
|
1913
|
+
}
|
|
1914
|
+
const path = this.preparationPath(handle);
|
|
1915
|
+
this.withLockSync(this.transitionPath(handle), () => {
|
|
1916
|
+
const recordPath = this.recordPath({
|
|
1917
|
+
...handle,
|
|
1918
|
+
claim_epoch: handle.execution_claim_epoch,
|
|
1919
|
+
started_at: "1970-01-01T00:00:00.000Z",
|
|
1920
|
+
});
|
|
1921
|
+
if (!existsSync(path)) {
|
|
1922
|
+
if (existsSync(recordPath)) {
|
|
1923
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "bound execution preparation cannot be abandoned");
|
|
1924
|
+
}
|
|
1925
|
+
return;
|
|
1926
|
+
}
|
|
1927
|
+
const preparation = this.readPreparation(path);
|
|
1928
|
+
if (canonicalizeJson(preparedHandle(preparation)) !== canonicalizeJson(handle)) {
|
|
1929
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "prepared execution handle does not match durable authority state");
|
|
1930
|
+
}
|
|
1931
|
+
if (existsSync(recordPath)) {
|
|
1932
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "bound execution preparation cannot be abandoned");
|
|
1933
|
+
}
|
|
1934
|
+
unlinkSync(path);
|
|
1935
|
+
syncDirectory(this.preparationsDirectory);
|
|
1936
|
+
});
|
|
1937
|
+
}
|
|
1938
|
+
bindExecution(handle, request) {
|
|
1939
|
+
if (handle.schema_version !== PREPARATION_HANDLE_SCHEMA_VERSION
|
|
1940
|
+
|| !isCanonicalTimestamp(request.started_at)) {
|
|
1941
|
+
throw new TypeError("local execution bind request is invalid");
|
|
1942
|
+
}
|
|
1943
|
+
const path = this.preparationPath(handle);
|
|
1944
|
+
return this.withLockSync(this.transitionPath(handle), () => {
|
|
1945
|
+
const identity = {
|
|
1946
|
+
run_id: handle.run_id,
|
|
1947
|
+
step_id: handle.step_id,
|
|
1948
|
+
activation_id: handle.activation_id,
|
|
1949
|
+
attempt_id: handle.attempt_id,
|
|
1950
|
+
claim_epoch: handle.execution_claim_epoch,
|
|
1951
|
+
worker_session_id: handle.worker_session_id,
|
|
1952
|
+
started_at: request.started_at,
|
|
1953
|
+
execution_instance_id: handle.execution_instance_id,
|
|
1954
|
+
};
|
|
1955
|
+
const recordPath = this.recordPath(identity);
|
|
1956
|
+
if (existsSync(recordPath)) {
|
|
1957
|
+
const current = this.readRecord(recordPath);
|
|
1958
|
+
if (canonicalizeJson(coreIdentity(current.identity)) !== canonicalizeJson(coreIdentity(identity))
|
|
1959
|
+
|| current.execution_instance_id !== handle.execution_instance_id) {
|
|
1960
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "bound execution record conflicts with the prepared handle");
|
|
1961
|
+
}
|
|
1962
|
+
if (existsSync(path)) {
|
|
1963
|
+
const lingering = this.readPreparation(path);
|
|
1964
|
+
if (canonicalizeJson(preparedHandle(lingering)) !== canonicalizeJson(handle)) {
|
|
1965
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "lingering preparation conflicts with the bound execution record");
|
|
1966
|
+
}
|
|
1967
|
+
unlinkSync(path);
|
|
1968
|
+
syncDirectory(this.preparationsDirectory);
|
|
1969
|
+
}
|
|
1970
|
+
return this.createLaunchBinding(current);
|
|
1971
|
+
}
|
|
1972
|
+
const preparation = this.readPreparation(path);
|
|
1973
|
+
if (canonicalizeJson(preparedHandle(preparation)) !== canonicalizeJson(handle)) {
|
|
1974
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "prepared execution handle does not match durable authority state");
|
|
1975
|
+
}
|
|
1976
|
+
const binding = this.registerExecution(identity, preparation);
|
|
1977
|
+
unlinkSync(path);
|
|
1978
|
+
syncDirectory(this.preparationsDirectory);
|
|
1979
|
+
return binding;
|
|
1980
|
+
});
|
|
1981
|
+
}
|
|
1982
|
+
registerExecution(identity, prepared) {
|
|
1983
|
+
requireIdentity(identity);
|
|
1984
|
+
this.ensureSupported();
|
|
1985
|
+
if (prepared === undefined && this.hasOtherAttemptRecord(identity)) {
|
|
1986
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution attempt already has durable authority under a different claim");
|
|
1987
|
+
}
|
|
1988
|
+
if (prepared === undefined) {
|
|
1989
|
+
const replay = () => {
|
|
1990
|
+
const recordPath = this.recordPath(identity);
|
|
1991
|
+
if (!existsSync(recordPath))
|
|
1992
|
+
return null;
|
|
1993
|
+
const current = this.readRecord(recordPath);
|
|
1994
|
+
if (canonicalizeJson(coreIdentity(current.identity)) !== canonicalizeJson(coreIdentity(identity))
|
|
1995
|
+
|| (identity.execution_instance_id !== undefined
|
|
1996
|
+
&& current.execution_instance_id !== identity.execution_instance_id)) {
|
|
1997
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution attempt already has a different durable identity");
|
|
1998
|
+
}
|
|
1999
|
+
if (current.phase === "prepared")
|
|
2000
|
+
return this.createLaunchBinding(current);
|
|
2001
|
+
if (current.phase === "never_launched") {
|
|
2002
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution was durably attested as never launched");
|
|
2003
|
+
}
|
|
2004
|
+
if (current.phase === "unsupported") {
|
|
2005
|
+
throw new LocalExecutionAuthorityError("unsupported", current.supervisor_error ?? "execution supervisor evidence is unavailable");
|
|
2006
|
+
}
|
|
2007
|
+
return this.recoverExecution({
|
|
2008
|
+
run_id: identity.run_id,
|
|
2009
|
+
step_id: identity.step_id,
|
|
2010
|
+
activation_id: identity.activation_id,
|
|
2011
|
+
attempt_id: identity.attempt_id,
|
|
2012
|
+
execution_claim_epoch: identity.claim_epoch,
|
|
2013
|
+
executor: { kind: "worker", worker_session_id: identity.worker_session_id },
|
|
2014
|
+
started_at: identity.started_at,
|
|
2015
|
+
}).binding;
|
|
2016
|
+
};
|
|
2017
|
+
const existing = replay();
|
|
2018
|
+
if (existing !== null)
|
|
2019
|
+
return existing;
|
|
2020
|
+
try {
|
|
2021
|
+
const handle = this.prepareExecution({
|
|
2022
|
+
run_id: identity.run_id,
|
|
2023
|
+
step_id: identity.step_id,
|
|
2024
|
+
activation_id: identity.activation_id,
|
|
2025
|
+
attempt_id: identity.attempt_id,
|
|
2026
|
+
execution_claim_epoch: identity.claim_epoch,
|
|
2027
|
+
worker_session_id: identity.worker_session_id,
|
|
2028
|
+
...(identity.execution_instance_id === undefined
|
|
2029
|
+
? {}
|
|
2030
|
+
: { execution_instance_id: identity.execution_instance_id }),
|
|
2031
|
+
});
|
|
2032
|
+
return this.bindExecution(handle, { started_at: identity.started_at });
|
|
2033
|
+
}
|
|
2034
|
+
catch (error) {
|
|
2035
|
+
if (error instanceof LocalExecutionAuthorityError && error.code === "identity_conflict") {
|
|
2036
|
+
const raced = replay();
|
|
2037
|
+
if (raced !== null)
|
|
2038
|
+
return raced;
|
|
2039
|
+
}
|
|
2040
|
+
throw error;
|
|
2041
|
+
}
|
|
2042
|
+
}
|
|
2043
|
+
const backend = prepared?.backend ?? this.backend();
|
|
2044
|
+
const executionInstanceId = identity.execution_instance_id ?? `local-process:${randomUUID()}`;
|
|
2045
|
+
const requiredIdentity = {
|
|
2046
|
+
...identity,
|
|
2047
|
+
execution_instance_id: executionInstanceId,
|
|
2048
|
+
};
|
|
2049
|
+
const recordPath = this.recordPath(identity);
|
|
2050
|
+
const controlEndpoint = this.controlEndpoint(identity);
|
|
2051
|
+
const now = new Date().toISOString();
|
|
2052
|
+
const record = {
|
|
2053
|
+
schema_version: RECORD_SCHEMA_VERSION,
|
|
2054
|
+
revision: 1,
|
|
2055
|
+
phase: "prepared",
|
|
2056
|
+
identity: requiredIdentity,
|
|
2057
|
+
identity_digest: identityDigest(identity),
|
|
2058
|
+
execution_instance_id: executionInstanceId,
|
|
2059
|
+
challenge: prepared?.challenge
|
|
2060
|
+
?? randomUUID().replaceAll("-", "") + randomUUID().replaceAll("-", ""),
|
|
2061
|
+
authority_instance_id: prepared?.authority_instance_id ?? this.authorityInstanceId,
|
|
2062
|
+
backend,
|
|
2063
|
+
supervisor: prepared?.supervisor ?? this.supervisor(controlEndpoint),
|
|
2064
|
+
provider: null,
|
|
2065
|
+
raw_completion: null,
|
|
2066
|
+
normalized_process_outcome: null,
|
|
2067
|
+
stop_fact: null,
|
|
2068
|
+
stop_receipt: null,
|
|
2069
|
+
already_stopped_evidence: null,
|
|
2070
|
+
nonlaunch_attestation: null,
|
|
2071
|
+
fencing_epoch: null,
|
|
2072
|
+
supervisor_error: null,
|
|
2073
|
+
created_at: now,
|
|
2074
|
+
updated_at: now,
|
|
2075
|
+
};
|
|
2076
|
+
this.withLockSync(recordPath, () => {
|
|
2077
|
+
if (existsSync(recordPath)) {
|
|
2078
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution identity is already registered");
|
|
2079
|
+
}
|
|
2080
|
+
const descriptor = openSync(recordPath, "wx", PRIVATE_FILE_MODE);
|
|
2081
|
+
try {
|
|
2082
|
+
writeFileSync(descriptor, `${canonicalizeJson(record)}\n`, "utf8");
|
|
2083
|
+
fsyncSync(descriptor);
|
|
2084
|
+
}
|
|
2085
|
+
finally {
|
|
2086
|
+
closeSync(descriptor);
|
|
2087
|
+
}
|
|
2088
|
+
syncDirectory(this.recordsDirectory);
|
|
2089
|
+
});
|
|
2090
|
+
return this.createLaunchBinding(record);
|
|
2091
|
+
}
|
|
2092
|
+
async probeRecovery(request) {
|
|
2093
|
+
requireIdentity(recoveryIdentity(request));
|
|
2094
|
+
let record = this.readRecoveryRecord(request);
|
|
2095
|
+
if (record !== null
|
|
2096
|
+
&& record.backend.kind === "windows_job"
|
|
2097
|
+
&& record.raw_completion === null
|
|
2098
|
+
&& record.phase !== "never_launched"
|
|
2099
|
+
&& record.phase !== "unsupported") {
|
|
2100
|
+
record = await this.authenticateWindowsSupervisor(record, this.recordPath(record.identity));
|
|
2101
|
+
}
|
|
2102
|
+
return { raw_completion_durable: record?.raw_completion !== null && record?.raw_completion !== undefined };
|
|
2103
|
+
}
|
|
2104
|
+
async attestNeverLaunched(request) {
|
|
2105
|
+
const identity = recoveryIdentity(request);
|
|
2106
|
+
requireIdentity(identity);
|
|
2107
|
+
if (!isCanonicalDigest(request.failure_idempotency_key)) {
|
|
2108
|
+
throw new TypeError("failure_idempotency_key must be a canonical SHA-256 digest");
|
|
2109
|
+
}
|
|
2110
|
+
if (!isCanonicalTimestamp(request.requested_at)) {
|
|
2111
|
+
throw new TypeError("requested_at must be a canonical RFC 3339 timestamp");
|
|
2112
|
+
}
|
|
2113
|
+
const path = this.recordPath(identity);
|
|
2114
|
+
const preparationPath = this.preparationPath(identity);
|
|
2115
|
+
return this.withLockSync(this.transitionPath(identity), () => {
|
|
2116
|
+
let current = null;
|
|
2117
|
+
let preparation = null;
|
|
2118
|
+
if (existsSync(preparationPath)) {
|
|
2119
|
+
preparation = this.readPreparation(preparationPath);
|
|
2120
|
+
if (canonicalizeJson(preparationCoreIdentity(preparation.identity))
|
|
2121
|
+
!== canonicalizeJson({
|
|
2122
|
+
run_id: identity.run_id,
|
|
2123
|
+
step_id: identity.step_id,
|
|
2124
|
+
activation_id: identity.activation_id,
|
|
2125
|
+
attempt_id: identity.attempt_id,
|
|
2126
|
+
execution_claim_epoch: identity.claim_epoch,
|
|
2127
|
+
worker_session_id: identity.worker_session_id,
|
|
2128
|
+
})) {
|
|
2129
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "nonlaunch identity does not match the durable preparation");
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
if (existsSync(path)) {
|
|
2133
|
+
current = this.readRecord(path);
|
|
2134
|
+
if (canonicalizeJson(coreIdentity(current.identity)) !== canonicalizeJson(coreIdentity(identity))) {
|
|
2135
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "nonlaunch identity does not match durable authority state");
|
|
2136
|
+
}
|
|
2137
|
+
if (current.phase === "never_launched") {
|
|
2138
|
+
const attestation = current.nonlaunch_attestation;
|
|
2139
|
+
if (attestation.payload.failure_idempotency_key !== request.failure_idempotency_key) {
|
|
2140
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution already has a different nonlaunch failure");
|
|
2141
|
+
}
|
|
2142
|
+
if (preparation !== null) {
|
|
2143
|
+
if (preparation.execution_instance_id !== current.execution_instance_id) {
|
|
2144
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "never-launched record conflicts with its lingering preparation");
|
|
2145
|
+
}
|
|
2146
|
+
unlinkSync(preparationPath);
|
|
2147
|
+
syncDirectory(this.preparationsDirectory);
|
|
2148
|
+
}
|
|
2149
|
+
return attestation;
|
|
2150
|
+
}
|
|
2151
|
+
if (current.phase !== "prepared" || current.supervisor.pid !== null || current.provider !== null) {
|
|
2152
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution launch ownership has already begun");
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
if (current === null && preparation === null) {
|
|
2156
|
+
if (this.hasOtherAttemptRecord(identity)) {
|
|
2157
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution attempt already has durable authority under a different claim");
|
|
2158
|
+
}
|
|
2159
|
+
throw new LocalExecutionAuthorityError("unsupported", "never-launched state cannot be attested without durable preparation evidence");
|
|
2160
|
+
}
|
|
2161
|
+
const fencingEpoch = this.nextFencingEpochSync();
|
|
2162
|
+
const now = new Date().toISOString();
|
|
2163
|
+
const executionInstanceId = current?.execution_instance_id
|
|
2164
|
+
?? preparation?.execution_instance_id
|
|
2165
|
+
?? `local-process:${randomUUID()}`;
|
|
2166
|
+
const authorityInstanceId = current?.authority_instance_id
|
|
2167
|
+
?? preparation?.authority_instance_id
|
|
2168
|
+
?? this.authorityInstanceId;
|
|
2169
|
+
const requiredIdentity = {
|
|
2170
|
+
...identity,
|
|
2171
|
+
execution_instance_id: executionInstanceId,
|
|
2172
|
+
};
|
|
2173
|
+
const backend = current?.backend ?? preparation?.backend ?? this.backend();
|
|
2174
|
+
const controlEndpoint = this.controlEndpoint(identity);
|
|
2175
|
+
const tombstone = current === null ? {
|
|
2176
|
+
schema_version: RECORD_SCHEMA_VERSION,
|
|
2177
|
+
revision: 1,
|
|
2178
|
+
phase: "never_launched",
|
|
2179
|
+
identity: requiredIdentity,
|
|
2180
|
+
identity_digest: identityDigest(identity),
|
|
2181
|
+
execution_instance_id: executionInstanceId,
|
|
2182
|
+
challenge: preparation?.challenge
|
|
2183
|
+
?? randomUUID().replaceAll("-", "") + randomUUID().replaceAll("-", ""),
|
|
2184
|
+
authority_instance_id: authorityInstanceId,
|
|
2185
|
+
backend,
|
|
2186
|
+
supervisor: preparation?.supervisor ?? this.supervisor(controlEndpoint),
|
|
2187
|
+
provider: null,
|
|
2188
|
+
raw_completion: null,
|
|
2189
|
+
normalized_process_outcome: null,
|
|
2190
|
+
stop_fact: null,
|
|
2191
|
+
stop_receipt: null,
|
|
2192
|
+
already_stopped_evidence: null,
|
|
2193
|
+
nonlaunch_attestation: null,
|
|
2194
|
+
fencing_epoch: fencingEpoch,
|
|
2195
|
+
supervisor_error: null,
|
|
2196
|
+
created_at: now,
|
|
2197
|
+
updated_at: now,
|
|
2198
|
+
} : {
|
|
2199
|
+
...current,
|
|
2200
|
+
phase: "never_launched",
|
|
2201
|
+
nonlaunch_attestation: null,
|
|
2202
|
+
fencing_epoch: fencingEpoch,
|
|
2203
|
+
revision: current.revision + 1,
|
|
2204
|
+
updated_at: now,
|
|
2205
|
+
};
|
|
2206
|
+
const payload = {
|
|
2207
|
+
schema_version: "intentdna.worker_nonlaunch_attestation.v1",
|
|
2208
|
+
run_id: identity.run_id,
|
|
2209
|
+
step_id: identity.step_id,
|
|
2210
|
+
activation_id: identity.activation_id,
|
|
2211
|
+
attempt_id: identity.attempt_id,
|
|
2212
|
+
claim_epoch: identity.claim_epoch,
|
|
2213
|
+
worker_session_id: identity.worker_session_id,
|
|
2214
|
+
execution_instance_id: executionInstanceId,
|
|
2215
|
+
authority_kind: "local_process",
|
|
2216
|
+
authority_instance_id: authorityInstanceId,
|
|
2217
|
+
fencing_epoch: fencingEpoch,
|
|
2218
|
+
started_at: identity.started_at,
|
|
2219
|
+
attested_at: now,
|
|
2220
|
+
failure_idempotency_key: request.failure_idempotency_key,
|
|
2221
|
+
launch_state: "never_launched",
|
|
2222
|
+
source_evidence_digest: nonlaunchSourceEvidenceDigest(tombstone),
|
|
2223
|
+
};
|
|
2224
|
+
const attestation = JSON.parse(canonicalizeJson({
|
|
2225
|
+
attestation_digest: workerNonlaunchAttestationDigest(payload),
|
|
2226
|
+
payload,
|
|
2227
|
+
}));
|
|
2228
|
+
const next = { ...tombstone, nonlaunch_attestation: attestation };
|
|
2229
|
+
atomicWriteJson(path, next);
|
|
2230
|
+
if (preparation !== null) {
|
|
2231
|
+
unlinkSync(preparationPath);
|
|
2232
|
+
syncDirectory(this.preparationsDirectory);
|
|
2233
|
+
}
|
|
2234
|
+
return attestation;
|
|
2235
|
+
});
|
|
2236
|
+
}
|
|
2237
|
+
recoverExecution(request) {
|
|
2238
|
+
const identity = recoveryIdentity(request);
|
|
2239
|
+
requireIdentity(identity);
|
|
2240
|
+
this.ensureSupported();
|
|
2241
|
+
const record = this.readRecoveryRecord(request);
|
|
2242
|
+
if (record === null) {
|
|
2243
|
+
throw new LocalExecutionAuthorityError("unsupported", "durable execution authority record is unavailable");
|
|
2244
|
+
}
|
|
2245
|
+
if (record.phase === "prepared") {
|
|
2246
|
+
return { kind: "launch", binding: this.createLaunchBinding(record) };
|
|
2247
|
+
}
|
|
2248
|
+
if (record.phase === "never_launched") {
|
|
2249
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution was durably attested as never launched");
|
|
2250
|
+
}
|
|
2251
|
+
if (record.phase === "unsupported") {
|
|
2252
|
+
throw new LocalExecutionAuthorityError("unsupported", record.supervisor_error ?? "execution supervisor evidence is unavailable");
|
|
2253
|
+
}
|
|
2254
|
+
const recordPath = this.recordPath(identity);
|
|
2255
|
+
const authentication = record.backend.kind === "windows_job" && record.raw_completion === null
|
|
2256
|
+
? this.authenticateWindowsSupervisor(record, recordPath)
|
|
2257
|
+
: Promise.resolve(record);
|
|
2258
|
+
const processStarted = (async () => {
|
|
2259
|
+
await authentication;
|
|
2260
|
+
for (;;) {
|
|
2261
|
+
const current = this.readRecord(recordPath);
|
|
2262
|
+
if (current.provider?.pid !== null && current.provider !== null)
|
|
2263
|
+
return current.provider.pid;
|
|
2264
|
+
if (current.raw_completion !== null) {
|
|
2265
|
+
if (current.raw_completion.process_id !== null)
|
|
2266
|
+
return current.raw_completion.process_id;
|
|
2267
|
+
throw new LocalExecutionAuthorityError("unsupported", "durable execution completed without a provider identity");
|
|
2268
|
+
}
|
|
2269
|
+
if (current.phase === "unsupported") {
|
|
2270
|
+
throw new LocalExecutionAuthorityError("unsupported", current.supervisor_error ?? "execution supervisor evidence is unavailable");
|
|
2271
|
+
}
|
|
2272
|
+
await sleep(this.pollIntervalMs);
|
|
2273
|
+
}
|
|
2274
|
+
})();
|
|
2275
|
+
const registration = {
|
|
2276
|
+
identity: record.identity,
|
|
2277
|
+
recordPath,
|
|
2278
|
+
abortController: new AbortController(),
|
|
2279
|
+
processStarted,
|
|
2280
|
+
resolveProcessStarted: () => undefined,
|
|
2281
|
+
rejectProcessStarted: () => undefined,
|
|
2282
|
+
};
|
|
2283
|
+
registration.processStarted.catch(() => undefined);
|
|
2284
|
+
return { kind: "observe", binding: {
|
|
2285
|
+
cancellation_signal: registration.abortController.signal,
|
|
2286
|
+
process_started: registration.processStarted,
|
|
2287
|
+
processStarted: (processId) => {
|
|
2288
|
+
const current = this.readRecord(registration.recordPath);
|
|
2289
|
+
if (!providerMatchesProcessId(current, processId)) {
|
|
2290
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "recovered execution process does not match durable guardian identity");
|
|
2291
|
+
}
|
|
2292
|
+
},
|
|
2293
|
+
processTreeCompleted: (observation) => this.recordNormalizedCompletion(registration.recordPath, observation),
|
|
2294
|
+
executeProcessTree: (_command, _args, options) => this.observeRecoveredExecution(registration, options),
|
|
2295
|
+
} };
|
|
2296
|
+
}
|
|
2297
|
+
async executeSupervised(registration, command, args, options) {
|
|
2298
|
+
if (options.spawnImpl !== undefined || options.platform !== undefined) {
|
|
2299
|
+
throw new LocalExecutionAuthorityError("unsupported", "custom process launch adapters are unsupported");
|
|
2300
|
+
}
|
|
2301
|
+
if (options.maxProcessCount !== undefined) {
|
|
2302
|
+
throw new LocalExecutionAuthorityError("unsupported", "durable supervisor process limits are not implemented");
|
|
2303
|
+
}
|
|
2304
|
+
if (options.killSignal === "SIGKILL" || options.killSignal === "SIGSTOP") {
|
|
2305
|
+
throw new LocalExecutionAuthorityError("unsupported", `durable supervisor cannot preserve guardian identity with ${options.killSignal}`);
|
|
2306
|
+
}
|
|
2307
|
+
const record = await this.withLock(this.transitionPath(registration.identity), () => this.updateRecord(registration.recordPath, (current) => {
|
|
2308
|
+
if (current.phase !== "prepared") {
|
|
2309
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "another execution binding already owns provider launch");
|
|
2310
|
+
}
|
|
2311
|
+
return {
|
|
2312
|
+
...current,
|
|
2313
|
+
phase: "launching",
|
|
2314
|
+
revision: current.revision + 1,
|
|
2315
|
+
updated_at: new Date().toISOString(),
|
|
2316
|
+
};
|
|
2317
|
+
}));
|
|
2318
|
+
const streamToken = record.identity_digest.slice("sha256:".length, "sha256:".length + 24);
|
|
2319
|
+
const stdoutPath = join(this.streamsDirectory, `${streamToken}.stdout`);
|
|
2320
|
+
const stderrPath = join(this.streamsDirectory, `${streamToken}.stderr`);
|
|
2321
|
+
const gatePath = join(this.streamsDirectory, `${streamToken}.gate`);
|
|
2322
|
+
const stdinPath = join(this.streamsDirectory, `${streamToken}.stdin`);
|
|
2323
|
+
for (const path of [stdoutPath, stderrPath, gatePath, stdinPath]) {
|
|
2324
|
+
try {
|
|
2325
|
+
unlinkSync(path);
|
|
2326
|
+
}
|
|
2327
|
+
catch (error) {
|
|
2328
|
+
if (!isErrno(error, "ENOENT"))
|
|
2329
|
+
throw error;
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
const environment = Object.fromEntries(Object.entries(options.env ?? process.env)
|
|
2333
|
+
.filter((entry) => entry[1] !== undefined));
|
|
2334
|
+
const cwd = resolve(options.cwd ?? process.cwd());
|
|
2335
|
+
const timeoutMs = options.timeoutMs !== undefined
|
|
2336
|
+
&& Number.isFinite(options.timeoutMs)
|
|
2337
|
+
&& options.timeoutMs > 0
|
|
2338
|
+
? Math.floor(options.timeoutMs)
|
|
2339
|
+
: null;
|
|
2340
|
+
const maxOutputBytes = options.maxOutputBytes === undefined
|
|
2341
|
+
? DEFAULT_MAX_OUTPUT_BYTES
|
|
2342
|
+
: requirePositiveInteger(Math.floor(options.maxOutputBytes), "maxOutputBytes");
|
|
2343
|
+
const naturalDrainGraceMs = options.sigkillGraceMs !== undefined
|
|
2344
|
+
&& Number.isFinite(options.sigkillGraceMs)
|
|
2345
|
+
&& options.sigkillGraceMs >= 0
|
|
2346
|
+
? Math.floor(options.sigkillGraceMs)
|
|
2347
|
+
: 1_000;
|
|
2348
|
+
const forcedDrainGraceMs = options.cancellationGraceMs !== undefined
|
|
2349
|
+
&& Number.isFinite(options.cancellationGraceMs)
|
|
2350
|
+
&& options.cancellationGraceMs >= 0
|
|
2351
|
+
? Math.floor(options.cancellationGraceMs)
|
|
2352
|
+
: 1_000;
|
|
2353
|
+
let configuration;
|
|
2354
|
+
if (record.backend.kind === "windows_job") {
|
|
2355
|
+
const spec = resolveProcessCommand(command, args, "win32", environment, cwd);
|
|
2356
|
+
const executable = this.resolveAbsoluteWindowsExecutable(spec.command, environment, cwd);
|
|
2357
|
+
configuration = {
|
|
2358
|
+
execution_instance_id: record.execution_instance_id,
|
|
2359
|
+
challenge: record.challenge,
|
|
2360
|
+
containment_nonce: record.backend.containment_nonce,
|
|
2361
|
+
control_endpoint: record.supervisor.socket_path,
|
|
2362
|
+
helper_path: record.backend.helper_path,
|
|
2363
|
+
helper_digest: record.backend.helper_digest,
|
|
2364
|
+
executable,
|
|
2365
|
+
args: [...spec.args],
|
|
2366
|
+
command_line: spec.windowsVerbatimArguments
|
|
2367
|
+
? windowsVerbatimCommandLine(executable, spec.args)
|
|
2368
|
+
: null,
|
|
2369
|
+
cwd,
|
|
2370
|
+
env: environment,
|
|
2371
|
+
stdin_path: options.stdin === undefined ? null : stdinPath,
|
|
2372
|
+
stdin: options.stdin ?? "",
|
|
2373
|
+
stdout_path: stdoutPath,
|
|
2374
|
+
stderr_path: stderrPath,
|
|
2375
|
+
max_output_bytes: maxOutputBytes,
|
|
2376
|
+
timeout_ms: timeoutMs,
|
|
2377
|
+
natural_drain_grace_ms: naturalDrainGraceMs,
|
|
2378
|
+
forced_drain_grace_ms: forcedDrainGraceMs,
|
|
2379
|
+
};
|
|
2380
|
+
}
|
|
2381
|
+
else {
|
|
2382
|
+
configuration = {
|
|
2383
|
+
execution_instance_id: record.execution_instance_id,
|
|
2384
|
+
challenge: record.challenge,
|
|
2385
|
+
socket_path: record.supervisor.socket_path,
|
|
2386
|
+
stdout_path: stdoutPath,
|
|
2387
|
+
stderr_path: stderrPath,
|
|
2388
|
+
max_output_bytes: maxOutputBytes,
|
|
2389
|
+
gate_path: gatePath,
|
|
2390
|
+
command,
|
|
2391
|
+
args: [...args],
|
|
2392
|
+
cwd,
|
|
2393
|
+
env: environment,
|
|
2394
|
+
stdin: options.stdin ?? null,
|
|
2395
|
+
timeout_ms: timeoutMs,
|
|
2396
|
+
kill_signal: options.killSignal ?? "SIGTERM",
|
|
2397
|
+
cancellation_grace_ms: options.cancellationGraceMs ?? 1_000,
|
|
2398
|
+
timeout_grace_ms: options.sigkillGraceMs ?? 1_000,
|
|
2399
|
+
windows_verbatim_arguments: false,
|
|
2400
|
+
auto_release_gate: true,
|
|
2401
|
+
};
|
|
2402
|
+
}
|
|
2403
|
+
const child = spawn(process.execPath, [this.supervisorScriptPath(), registration.recordPath], {
|
|
2404
|
+
detached: true,
|
|
2405
|
+
stdio: ["pipe", "ignore", "ignore"],
|
|
2406
|
+
windowsHide: true,
|
|
2407
|
+
});
|
|
2408
|
+
child.unref();
|
|
2409
|
+
child.stdin?.end(JSON.stringify(configuration));
|
|
2410
|
+
let supervisorExit = null;
|
|
2411
|
+
child.once("exit", (status, signal) => { supervisorExit = { status, signal }; });
|
|
2412
|
+
let processObserved = false;
|
|
2413
|
+
let cancellationSent = false;
|
|
2414
|
+
const cancellationSignal = options.cancellationSignal;
|
|
2415
|
+
const cancel = async () => {
|
|
2416
|
+
if (cancellationSent)
|
|
2417
|
+
return;
|
|
2418
|
+
let current = this.readRecord(registration.recordPath);
|
|
2419
|
+
if (current.raw_completion !== null)
|
|
2420
|
+
return;
|
|
2421
|
+
if (current.supervisor.ready_at === null)
|
|
2422
|
+
return;
|
|
2423
|
+
if (current.backend.kind === "windows_job") {
|
|
2424
|
+
current = await this.authenticateWindowsSupervisor(current, registration.recordPath);
|
|
2425
|
+
if (current.raw_completion !== null)
|
|
2426
|
+
return;
|
|
2427
|
+
}
|
|
2428
|
+
await this.sendControl(current, {
|
|
2429
|
+
action: "stop",
|
|
2430
|
+
reason: typeof cancellationSignal?.reason === "string"
|
|
2431
|
+
? cancellationSignal.reason
|
|
2432
|
+
: options.cancellationReason ?? null,
|
|
2433
|
+
});
|
|
2434
|
+
cancellationSent = true;
|
|
2435
|
+
};
|
|
2436
|
+
const onAbort = () => { void cancel().catch(() => undefined); };
|
|
2437
|
+
cancellationSignal?.addEventListener("abort", onAbort, { once: true });
|
|
2438
|
+
if (cancellationSignal?.aborted)
|
|
2439
|
+
onAbort();
|
|
2440
|
+
try {
|
|
2441
|
+
const deadline = timeoutMs === null
|
|
2442
|
+
? Number.POSITIVE_INFINITY
|
|
2443
|
+
: Date.now()
|
|
2444
|
+
+ Math.max(this.controlTimeoutMs, timeoutMs)
|
|
2445
|
+
+ Math.max(forcedDrainGraceMs, naturalDrainGraceMs)
|
|
2446
|
+
+ 5_000;
|
|
2447
|
+
for (;;) {
|
|
2448
|
+
const current = this.readRecord(registration.recordPath);
|
|
2449
|
+
if (cancellationSignal?.aborted && !cancellationSent && current.supervisor.ready_at !== null) {
|
|
2450
|
+
await cancel();
|
|
2451
|
+
continue;
|
|
2452
|
+
}
|
|
2453
|
+
if (!processObserved && current.provider?.pid !== null && current.provider?.pid !== undefined) {
|
|
2454
|
+
processObserved = true;
|
|
2455
|
+
options.onProcessStart?.(current.provider.pid);
|
|
2456
|
+
}
|
|
2457
|
+
if (current.phase === "unsupported") {
|
|
2458
|
+
throw new LocalExecutionAuthorityError("unsupported", current.supervisor_error ?? "execution supervisor failed without trusted evidence");
|
|
2459
|
+
}
|
|
2460
|
+
if (["raw_completed", "normalized_completed", "receipted"].includes(current.phase)) {
|
|
2461
|
+
if (current.raw_completion === null) {
|
|
2462
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "supervisor completed without raw evidence");
|
|
2463
|
+
}
|
|
2464
|
+
const result = this.rawCompletionResult(current);
|
|
2465
|
+
if (result.stdout !== "")
|
|
2466
|
+
options.onStdout?.(result.stdout);
|
|
2467
|
+
if (result.stderr !== "")
|
|
2468
|
+
options.onStderr?.(result.stderr);
|
|
2469
|
+
if (result.terminationReason !== null)
|
|
2470
|
+
options.onTerminationRequested?.(result.terminationReason);
|
|
2471
|
+
return result;
|
|
2472
|
+
}
|
|
2473
|
+
const supervisorPid = current.supervisor.pid ?? child.pid ?? null;
|
|
2474
|
+
if (record.backend.kind === "windows_job" && supervisorExit !== null) {
|
|
2475
|
+
const detail = supervisorExit;
|
|
2476
|
+
const message = `Windows execution supervisor exited before trusted completion (status ${String(detail.status)}, signal ${String(detail.signal)})`;
|
|
2477
|
+
await this.markUnsupported(registration.recordPath, message);
|
|
2478
|
+
throw new LocalExecutionAuthorityError("unsupported", message);
|
|
2479
|
+
}
|
|
2480
|
+
if (record.backend.kind !== "windows_job" && supervisorPid !== null && !processIsAlive(supervisorPid)) {
|
|
2481
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor exited before publishing trusted completion evidence");
|
|
2482
|
+
}
|
|
2483
|
+
if (Date.now() >= deadline) {
|
|
2484
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor did not publish completion before the authority deadline");
|
|
2485
|
+
}
|
|
2486
|
+
await sleep(this.pollIntervalMs);
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
finally {
|
|
2490
|
+
cancellationSignal?.removeEventListener("abort", onAbort);
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
async observeRecoveredExecution(registration, options) {
|
|
2494
|
+
const initial = this.readRecord(registration.recordPath);
|
|
2495
|
+
if (initial.backend.kind === "windows_job" && initial.raw_completion === null) {
|
|
2496
|
+
await this.authenticateWindowsSupervisor(initial, registration.recordPath);
|
|
2497
|
+
}
|
|
2498
|
+
let cancellationSent = false;
|
|
2499
|
+
const cancel = async () => {
|
|
2500
|
+
if (cancellationSent)
|
|
2501
|
+
return;
|
|
2502
|
+
const current = this.readRecord(registration.recordPath);
|
|
2503
|
+
if (current.raw_completion !== null)
|
|
2504
|
+
return;
|
|
2505
|
+
const authenticated = current.backend.kind === "windows_job"
|
|
2506
|
+
? await this.authenticateWindowsSupervisor(current, registration.recordPath)
|
|
2507
|
+
: current;
|
|
2508
|
+
if (authenticated.raw_completion !== null)
|
|
2509
|
+
return;
|
|
2510
|
+
await this.sendControl(authenticated, {
|
|
2511
|
+
action: "stop",
|
|
2512
|
+
reason: typeof options.cancellationSignal?.reason === "string"
|
|
2513
|
+
? options.cancellationSignal.reason
|
|
2514
|
+
: options.cancellationReason ?? null,
|
|
2515
|
+
});
|
|
2516
|
+
cancellationSent = true;
|
|
2517
|
+
};
|
|
2518
|
+
const onAbort = () => { void cancel().catch(() => undefined); };
|
|
2519
|
+
options.cancellationSignal?.addEventListener("abort", onAbort, { once: true });
|
|
2520
|
+
if (options.cancellationSignal?.aborted)
|
|
2521
|
+
onAbort();
|
|
2522
|
+
let processObserved = false;
|
|
2523
|
+
let nextWindowsProbeAt = 0;
|
|
2524
|
+
try {
|
|
2525
|
+
for (;;) {
|
|
2526
|
+
const current = this.readRecord(registration.recordPath);
|
|
2527
|
+
if (current.phase === "unsupported") {
|
|
2528
|
+
throw new LocalExecutionAuthorityError("unsupported", current.supervisor_error ?? "recovered execution supervisor failed");
|
|
2529
|
+
}
|
|
2530
|
+
if (current.raw_completion !== null) {
|
|
2531
|
+
const result = this.rawCompletionResult(current);
|
|
2532
|
+
if (result.stdout !== "")
|
|
2533
|
+
options.onStdout?.(result.stdout);
|
|
2534
|
+
if (result.stderr !== "")
|
|
2535
|
+
options.onStderr?.(result.stderr);
|
|
2536
|
+
if (result.terminationReason !== null)
|
|
2537
|
+
options.onTerminationRequested?.(result.terminationReason);
|
|
2538
|
+
return result;
|
|
2539
|
+
}
|
|
2540
|
+
if (!processObserved && current.provider?.pid !== null && current.provider !== null) {
|
|
2541
|
+
processObserved = true;
|
|
2542
|
+
options.onProcessStart?.(current.provider.pid);
|
|
2543
|
+
}
|
|
2544
|
+
if (current.backend.kind === "windows_job" && Date.now() >= nextWindowsProbeAt) {
|
|
2545
|
+
await this.authenticateWindowsSupervisor(current, registration.recordPath);
|
|
2546
|
+
nextWindowsProbeAt = Date.now() + 250;
|
|
2547
|
+
}
|
|
2548
|
+
if (current.backend.kind !== "windows_job"
|
|
2549
|
+
&& (current.supervisor.pid === null || !processIsAlive(current.supervisor.pid))) {
|
|
2550
|
+
throw new LocalExecutionAuthorityError("unsupported", "recovered execution supervisor exited without durable completion");
|
|
2551
|
+
}
|
|
2552
|
+
await sleep(this.pollIntervalMs);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
finally {
|
|
2556
|
+
options.cancellationSignal?.removeEventListener("abort", onAbort);
|
|
2557
|
+
}
|
|
2558
|
+
}
|
|
2559
|
+
async recordNormalizedCompletion(path, observation) {
|
|
2560
|
+
await this.updateRecord(path, (current) => {
|
|
2561
|
+
const matchesDurableEvidence = current.raw_completion !== null
|
|
2562
|
+
&& observation.process_tree_drained
|
|
2563
|
+
&& current.raw_completion.process_tree_drained
|
|
2564
|
+
&& observation.started_at === current.identity.started_at
|
|
2565
|
+
&& observation.completed_at >= current.raw_completion.completed_at
|
|
2566
|
+
&& observation.process_id === current.raw_completion.process_id;
|
|
2567
|
+
if (current.phase === "receipted") {
|
|
2568
|
+
if (!matchesDurableEvidence
|
|
2569
|
+
|| (current.normalized_process_outcome !== null
|
|
2570
|
+
&& canonicalizeJson(current.normalized_process_outcome) !== canonicalizeJson(observation.outcome))) {
|
|
2571
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution completion conflicts with the durable receipt");
|
|
2572
|
+
}
|
|
2573
|
+
return current;
|
|
2574
|
+
}
|
|
2575
|
+
if (current.phase === "normalized_completed" && current.normalized_process_outcome !== null) {
|
|
2576
|
+
if (!matchesDurableEvidence
|
|
2577
|
+
|| canonicalizeJson(current.normalized_process_outcome) !== canonicalizeJson(observation.outcome)) {
|
|
2578
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "execution completion was already normalized differently");
|
|
2579
|
+
}
|
|
2580
|
+
return current;
|
|
2581
|
+
}
|
|
2582
|
+
if (current.phase !== "raw_completed" || current.raw_completion === null) {
|
|
2583
|
+
throw new LocalExecutionAuthorityError("unsupported", "raw supervisor completion is unavailable");
|
|
2584
|
+
}
|
|
2585
|
+
if (!matchesDurableEvidence) {
|
|
2586
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "normalized completion does not match durable supervisor evidence");
|
|
2587
|
+
}
|
|
2588
|
+
return {
|
|
2589
|
+
...current,
|
|
2590
|
+
phase: "normalized_completed",
|
|
2591
|
+
normalized_process_outcome: observation.outcome,
|
|
2592
|
+
revision: current.revision + 1,
|
|
2593
|
+
updated_at: new Date().toISOString(),
|
|
2594
|
+
};
|
|
2595
|
+
});
|
|
2596
|
+
}
|
|
2597
|
+
sendControl(record, request) {
|
|
2598
|
+
const socketPath = record.supervisor.socket_path;
|
|
2599
|
+
if (record.backend.kind !== "windows_job") {
|
|
2600
|
+
let metadata;
|
|
2601
|
+
try {
|
|
2602
|
+
metadata = lstatSync(socketPath);
|
|
2603
|
+
}
|
|
2604
|
+
catch (error) {
|
|
2605
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor control socket is unavailable", { cause: error });
|
|
2606
|
+
}
|
|
2607
|
+
if (!metadata.isSocket() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
|
2608
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor control socket is not trusted");
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
return new Promise((resolvePromise, rejectPromise) => {
|
|
2612
|
+
const socket = connect(socketPath);
|
|
2613
|
+
const framedControl = record.backend.kind === "windows_job";
|
|
2614
|
+
let response = "";
|
|
2615
|
+
const timer = setTimeout(() => {
|
|
2616
|
+
socket.destroy();
|
|
2617
|
+
rejectPromise(new LocalExecutionAuthorityError("unsupported", "execution supervisor control timed out"));
|
|
2618
|
+
}, this.controlTimeoutMs);
|
|
2619
|
+
timer.unref?.();
|
|
2620
|
+
const fail = (error) => {
|
|
2621
|
+
clearTimeout(timer);
|
|
2622
|
+
rejectPromise(new LocalExecutionAuthorityError("unsupported", "execution supervisor control failed", { cause: error }));
|
|
2623
|
+
};
|
|
2624
|
+
socket.once("error", fail);
|
|
2625
|
+
socket.setEncoding("utf8");
|
|
2626
|
+
socket.on("data", (chunk) => {
|
|
2627
|
+
response += chunk;
|
|
2628
|
+
if (Buffer.byteLength(response, "utf8") > MAX_CONTROL_RESPONSE_BYTES) {
|
|
2629
|
+
socket.destroy(new Error("execution control response too large"));
|
|
2630
|
+
}
|
|
2631
|
+
});
|
|
2632
|
+
socket.once("end", () => {
|
|
2633
|
+
clearTimeout(timer);
|
|
2634
|
+
try {
|
|
2635
|
+
if (response.trim().length === 0) {
|
|
2636
|
+
try {
|
|
2637
|
+
const durable = this.readRecord(this.recordPath(record.identity));
|
|
2638
|
+
const matchesStop = durable.raw_completion !== null
|
|
2639
|
+
&& (request.stop_idempotency_key === undefined
|
|
2640
|
+
|| (durable.stop_fact?.kind === "canonical_stop"
|
|
2641
|
+
&& durable.stop_fact.stop_idempotency_key === request.stop_idempotency_key
|
|
2642
|
+
&& canonicalizeJson(durable.stop_fact.requested_outcome)
|
|
2643
|
+
=== canonicalizeJson(request.requested_outcome)));
|
|
2644
|
+
if (matchesStop) {
|
|
2645
|
+
resolvePromise({ ok: true, raw_completion: durable.raw_completion });
|
|
2646
|
+
return;
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
catch {
|
|
2650
|
+
// The control response remains authoritative unless durable completion proves the stop.
|
|
2651
|
+
}
|
|
2652
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor control response was empty");
|
|
2653
|
+
}
|
|
2654
|
+
let responseText = response;
|
|
2655
|
+
if (framedControl) {
|
|
2656
|
+
const newline = response.indexOf("\n");
|
|
2657
|
+
if (newline < 0 || response.slice(newline + 1).length !== 0) {
|
|
2658
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor control response was incomplete or invalid");
|
|
2659
|
+
}
|
|
2660
|
+
responseText = response.slice(0, newline);
|
|
2661
|
+
}
|
|
2662
|
+
let value;
|
|
2663
|
+
try {
|
|
2664
|
+
value = JSON.parse(responseText);
|
|
2665
|
+
}
|
|
2666
|
+
catch (error) {
|
|
2667
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor control response was incomplete or invalid", { cause: error });
|
|
2668
|
+
}
|
|
2669
|
+
if (value === null || typeof value !== "object" || typeof value.ok !== "boolean") {
|
|
2670
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor control response was incomplete or invalid");
|
|
2671
|
+
}
|
|
2672
|
+
if (!value.ok) {
|
|
2673
|
+
throw new LocalExecutionAuthorityError("identity_conflict", value.error ?? "execution supervisor rejected the control request");
|
|
2674
|
+
}
|
|
2675
|
+
resolvePromise(value);
|
|
2676
|
+
}
|
|
2677
|
+
catch (error) {
|
|
2678
|
+
rejectPromise(error);
|
|
2679
|
+
}
|
|
2680
|
+
});
|
|
2681
|
+
socket.once("connect", () => {
|
|
2682
|
+
const payload = JSON.stringify({
|
|
2683
|
+
schema_version: CONTROL_SCHEMA_VERSION,
|
|
2684
|
+
execution_instance_id: record.execution_instance_id,
|
|
2685
|
+
challenge: record.challenge,
|
|
2686
|
+
...(record.backend.kind === "windows_job"
|
|
2687
|
+
? { containment_nonce: record.backend.containment_nonce }
|
|
2688
|
+
: {}),
|
|
2689
|
+
...request,
|
|
2690
|
+
});
|
|
2691
|
+
if (framedControl)
|
|
2692
|
+
socket.write(`${payload}\n`);
|
|
2693
|
+
else
|
|
2694
|
+
socket.end(payload);
|
|
2695
|
+
});
|
|
2696
|
+
});
|
|
2697
|
+
}
|
|
2698
|
+
async issueReceipt(path, requestedOutcome, stoppedAt, expectedReceipt) {
|
|
2699
|
+
return this.withLock(path, async () => {
|
|
2700
|
+
const current = this.readRecord(path);
|
|
2701
|
+
if (current.stop_receipt !== null) {
|
|
2702
|
+
if (canonicalizeJson(current.stop_receipt.payload.terminal_outcome) !== canonicalizeJson(requestedOutcome)
|
|
2703
|
+
|| current.stop_receipt.payload.stopped_at !== stoppedAt
|
|
2704
|
+
|| (expectedReceipt !== null && canonicalizeJson(current.stop_receipt) !== canonicalizeJson(expectedReceipt))) {
|
|
2705
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "durable stop receipt conflicts with the request");
|
|
2706
|
+
}
|
|
2707
|
+
return {
|
|
2708
|
+
stop_receipt: current.stop_receipt,
|
|
2709
|
+
authority_instance_id: current.authority_instance_id,
|
|
2710
|
+
fencing_epoch: current.fencing_epoch,
|
|
2711
|
+
};
|
|
2712
|
+
}
|
|
2713
|
+
if (current.raw_completion === null) {
|
|
2714
|
+
throw new LocalExecutionAuthorityError("unsupported", "trusted process completion evidence is unavailable");
|
|
2715
|
+
}
|
|
2716
|
+
if (current.raw_completion.process_tree_drained !== true) {
|
|
2717
|
+
throw new LocalExecutionAuthorityError("corrupt_state", "trusted process-tree drain evidence is unavailable");
|
|
2718
|
+
}
|
|
2719
|
+
const fencingEpoch = await this.nextFencingEpoch();
|
|
2720
|
+
const receipt = createWorkerStopReceipt({
|
|
2721
|
+
schema_version: "intentdna.worker_stop_receipt.v1",
|
|
2722
|
+
run_id: current.identity.run_id,
|
|
2723
|
+
step_id: current.identity.step_id,
|
|
2724
|
+
activation_id: current.identity.activation_id,
|
|
2725
|
+
attempt_id: current.identity.attempt_id,
|
|
2726
|
+
claim_epoch: current.identity.claim_epoch,
|
|
2727
|
+
worker_session_id: current.identity.worker_session_id,
|
|
2728
|
+
execution_instance_id: current.execution_instance_id,
|
|
2729
|
+
authority_kind: "local_process",
|
|
2730
|
+
authority_instance_id: current.authority_instance_id,
|
|
2731
|
+
fencing_epoch: fencingEpoch,
|
|
2732
|
+
started_at: current.identity.started_at,
|
|
2733
|
+
stopped_at: stoppedAt,
|
|
2734
|
+
terminal_outcome: requestedOutcome,
|
|
2735
|
+
process_tree_drained: true,
|
|
2736
|
+
source_evidence_digest: stopReceiptSourceEvidenceDigest(current),
|
|
2737
|
+
});
|
|
2738
|
+
if (expectedReceipt !== null && canonicalizeJson(receipt) !== canonicalizeJson(expectedReceipt)) {
|
|
2739
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "expected stop receipt does not match durable evidence");
|
|
2740
|
+
}
|
|
2741
|
+
const updated = {
|
|
2742
|
+
...current,
|
|
2743
|
+
phase: "receipted",
|
|
2744
|
+
stop_receipt: receipt,
|
|
2745
|
+
fencing_epoch: fencingEpoch,
|
|
2746
|
+
revision: current.revision + 1,
|
|
2747
|
+
updated_at: new Date().toISOString(),
|
|
2748
|
+
};
|
|
2749
|
+
atomicWriteJson(path, updated);
|
|
2750
|
+
return {
|
|
2751
|
+
stop_receipt: receipt,
|
|
2752
|
+
authority_instance_id: updated.authority_instance_id,
|
|
2753
|
+
fencing_epoch: fencingEpoch,
|
|
2754
|
+
};
|
|
2755
|
+
});
|
|
2756
|
+
}
|
|
2757
|
+
async issueAlreadyStoppedEvidence(path, request) {
|
|
2758
|
+
return this.withLock(path, async () => {
|
|
2759
|
+
const current = this.readRecord(path);
|
|
2760
|
+
if (current.already_stopped_evidence !== null) {
|
|
2761
|
+
if (!workerAlreadyStoppedEvidenceMatchesStop(current.already_stopped_evidence, request)) {
|
|
2762
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "durable already-stopped evidence conflicts with the request");
|
|
2763
|
+
}
|
|
2764
|
+
return current.already_stopped_evidence;
|
|
2765
|
+
}
|
|
2766
|
+
if (request.expected_receipt !== null) {
|
|
2767
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "expected stop receipt cannot match a naturally completed execution");
|
|
2768
|
+
}
|
|
2769
|
+
if (current.stop_receipt !== null || current.stop_fact?.kind === "canonical_stop") {
|
|
2770
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "canonical stop already owns the durable terminal transition");
|
|
2771
|
+
}
|
|
2772
|
+
if (current.raw_completion === null) {
|
|
2773
|
+
throw new LocalExecutionAuthorityError("unsupported", "trusted natural completion evidence is unavailable");
|
|
2774
|
+
}
|
|
2775
|
+
const fencingEpoch = await this.nextFencingEpoch();
|
|
2776
|
+
const evidence = createWorkerAlreadyStoppedEvidence({
|
|
2777
|
+
schema_version: "intentdna.worker_already_stopped_evidence.v1",
|
|
2778
|
+
run_id: current.identity.run_id,
|
|
2779
|
+
step_id: current.identity.step_id,
|
|
2780
|
+
activation_id: current.identity.activation_id,
|
|
2781
|
+
attempt_id: current.identity.attempt_id,
|
|
2782
|
+
claim_epoch: current.identity.claim_epoch,
|
|
2783
|
+
worker_session_id: current.identity.worker_session_id,
|
|
2784
|
+
execution_instance_id: current.execution_instance_id,
|
|
2785
|
+
authority_kind: "local_process",
|
|
2786
|
+
authority_instance_id: current.authority_instance_id,
|
|
2787
|
+
fencing_epoch: fencingEpoch,
|
|
2788
|
+
started_at: current.identity.started_at,
|
|
2789
|
+
completed_at: current.raw_completion.completed_at,
|
|
2790
|
+
observed_outcome: terminalOutcomeToCanonical(rawCompletionToOutcome(current.raw_completion)),
|
|
2791
|
+
process_tree_drained: true,
|
|
2792
|
+
stop_idempotency_key: request.stop_idempotency_key,
|
|
2793
|
+
stop_requested_at: request.requested_at,
|
|
2794
|
+
stop_purpose: request.purpose,
|
|
2795
|
+
stop_requested_outcome: request.requested_outcome,
|
|
2796
|
+
source_evidence_digest: alreadyStoppedSourceEvidenceDigest(current),
|
|
2797
|
+
});
|
|
2798
|
+
const updated = {
|
|
2799
|
+
...current,
|
|
2800
|
+
phase: "receipted",
|
|
2801
|
+
already_stopped_evidence: evidence,
|
|
2802
|
+
fencing_epoch: fencingEpoch,
|
|
2803
|
+
revision: current.revision + 1,
|
|
2804
|
+
updated_at: new Date().toISOString(),
|
|
2805
|
+
};
|
|
2806
|
+
atomicWriteJson(path, updated);
|
|
2807
|
+
return evidence;
|
|
2808
|
+
});
|
|
2809
|
+
}
|
|
2810
|
+
async withStoppedExecution(request, operation) {
|
|
2811
|
+
const record = this.requireRequestRecord(request);
|
|
2812
|
+
if (record.phase === "unsupported" || record.raw_completion === null) {
|
|
2813
|
+
throw new LocalExecutionAuthorityError("unsupported", "trusted execution completion is unavailable");
|
|
2814
|
+
}
|
|
2815
|
+
if (record.normalized_process_outcome === null
|
|
2816
|
+
|| !processMatchesCanonical(record.normalized_process_outcome, request.reported_execution.outcome)
|
|
2817
|
+
|| request.reported_execution.started_at !== record.identity.started_at
|
|
2818
|
+
|| request.reported_execution.completed_at < record.raw_completion.completed_at) {
|
|
2819
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "reported execution does not match durable evidence");
|
|
2820
|
+
}
|
|
2821
|
+
const authority = await this.issueReceipt(this.recordPath(requestIdentity(request)), request.reported_execution.outcome, request.reported_execution.completed_at, request.expected_receipt);
|
|
2822
|
+
return operation(authority);
|
|
2823
|
+
}
|
|
2824
|
+
async requestStop(request) {
|
|
2825
|
+
const identity = requestIdentity(request);
|
|
2826
|
+
const path = this.recordPath(identity);
|
|
2827
|
+
const nonlaunchRequest = {
|
|
2828
|
+
run_id: request.run_id,
|
|
2829
|
+
step_id: request.step_id,
|
|
2830
|
+
activation_id: request.activation_id,
|
|
2831
|
+
attempt_id: request.attempt_id,
|
|
2832
|
+
execution_claim_epoch: request.claim_epoch,
|
|
2833
|
+
executor: request.executor,
|
|
2834
|
+
started_at: request.started_at,
|
|
2835
|
+
failure_idempotency_key: request.stop_idempotency_key,
|
|
2836
|
+
requested_at: request.requested_at,
|
|
2837
|
+
};
|
|
2838
|
+
if (!existsSync(path)) {
|
|
2839
|
+
return {
|
|
2840
|
+
kind: "never_launched",
|
|
2841
|
+
nonlaunch_attestation: await this.attestNeverLaunched(nonlaunchRequest),
|
|
2842
|
+
};
|
|
2843
|
+
}
|
|
2844
|
+
let record = this.requireRequestRecord(request);
|
|
2845
|
+
if (record.phase === "never_launched" || (record.phase === "prepared"
|
|
2846
|
+
&& record.supervisor.pid === null
|
|
2847
|
+
&& record.provider === null
|
|
2848
|
+
&& record.raw_completion === null)) {
|
|
2849
|
+
if (request.expected_receipt !== null) {
|
|
2850
|
+
throw new LocalExecutionAuthorityError("identity_conflict", "expected stop receipt cannot match a never-launched execution");
|
|
2851
|
+
}
|
|
2852
|
+
return {
|
|
2853
|
+
kind: "never_launched",
|
|
2854
|
+
nonlaunch_attestation: await this.attestNeverLaunched(nonlaunchRequest),
|
|
2855
|
+
};
|
|
2856
|
+
}
|
|
2857
|
+
if (record.already_stopped_evidence !== null) {
|
|
2858
|
+
const evidence = await this.issueAlreadyStoppedEvidence(path, request);
|
|
2859
|
+
return { kind: "already_stopped", drain_evidence: evidence };
|
|
2860
|
+
}
|
|
2861
|
+
if (record.stop_receipt !== null) {
|
|
2862
|
+
const authority = await this.issueReceipt(path, request.requested_outcome, record.stop_receipt.payload.stopped_at, request.expected_receipt);
|
|
2863
|
+
return { kind: "stopped_by_request", authority };
|
|
2864
|
+
}
|
|
2865
|
+
if (record.phase === "unsupported") {
|
|
2866
|
+
throw new LocalExecutionAuthorityError("unsupported", "execution supervisor evidence is unavailable");
|
|
2867
|
+
}
|
|
2868
|
+
if (record.raw_completion !== null && record.stop_fact?.kind !== "canonical_stop") {
|
|
2869
|
+
const evidence = await this.issueAlreadyStoppedEvidence(path, request);
|
|
2870
|
+
return { kind: "already_stopped", drain_evidence: evidence };
|
|
2871
|
+
}
|
|
2872
|
+
if (record.raw_completion === null) {
|
|
2873
|
+
if (record.backend.kind === "windows_job") {
|
|
2874
|
+
record = await this.authenticateWindowsSupervisor(record, path);
|
|
2875
|
+
}
|
|
2876
|
+
}
|
|
2877
|
+
if (record.raw_completion === null) {
|
|
2878
|
+
await this.sendControl(record, {
|
|
2879
|
+
action: "stop",
|
|
2880
|
+
stop_idempotency_key: request.stop_idempotency_key,
|
|
2881
|
+
requested_at: request.requested_at,
|
|
2882
|
+
requested_outcome: request.requested_outcome,
|
|
2883
|
+
});
|
|
2884
|
+
record = this.readRecord(path);
|
|
2885
|
+
}
|
|
2886
|
+
if (record.raw_completion === null
|
|
2887
|
+
|| record.stop_fact?.kind !== "canonical_stop"
|
|
2888
|
+
|| record.stop_fact.stop_idempotency_key !== request.stop_idempotency_key
|
|
2889
|
+
|| canonicalizeJson(record.stop_fact.requested_outcome) !== canonicalizeJson(request.requested_outcome)) {
|
|
2890
|
+
throw new LocalExecutionAuthorityError("unsupported", "canonical stop evidence was not durably published");
|
|
2891
|
+
}
|
|
2892
|
+
const authority = await this.issueReceipt(path, request.requested_outcome, record.raw_completion.completed_at < request.requested_at
|
|
2893
|
+
? request.requested_at
|
|
2894
|
+
: record.raw_completion.completed_at, request.expected_receipt);
|
|
2895
|
+
return { kind: "stopped_by_request", authority };
|
|
2896
|
+
}
|
|
2897
|
+
}
|