@tomflow/proflow-execution-runtime 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/conformance.json +4 -0
- package/dist/deployment/adapter.d.ts +458 -0
- package/dist/deployment/adapter.js +265 -0
- package/dist/deployment/descriptor.d.ts +105 -0
- package/dist/deployment/descriptor.js +123 -0
- package/dist/deployment/requirements.d.ts +5 -0
- package/dist/deployment/requirements.js +5 -0
- package/dist/deployment/verification.d.ts +3 -0
- package/dist/deployment/verification.js +7 -0
- package/dist/src/approval-owner.d.ts +40 -0
- package/dist/src/approval-owner.js +278 -0
- package/dist/src/browser-vision-client.d.ts +15 -0
- package/dist/src/browser-vision-client.js +79 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +50 -0
- package/dist/src/executor-port.d.ts +5 -0
- package/dist/src/executor-port.js +1 -0
- package/dist/src/executors/local-adapter.d.ts +4 -0
- package/dist/src/executors/local-adapter.js +55 -0
- package/dist/src/formal-process.d.ts +20 -0
- package/dist/src/formal-process.js +161 -0
- package/dist/src/index.d.ts +122 -0
- package/dist/src/index.js +983 -0
- package/dist/src/model-decision-client.d.ts +10 -0
- package/dist/src/model-decision-client.js +244 -0
- package/dist/src/service.d.ts +67 -0
- package/dist/src/service.js +473 -0
- package/package.json +56 -0
- package/proflow.module.json +139 -0
|
@@ -0,0 +1,983 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { appendFile, mkdir } from "node:fs/promises";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { DatabaseSync } from "node:sqlite";
|
|
5
|
+
import { browserCapabilityIds, EXECUTION_CONTRACT_VERSION, executionErrorCodes, parseCancelExecutionRequest, parseExecuteCapabilityRequest, parseExecutionArtifactRecord, parseExecutionRecord, parseExecutionRef, parseReadExecutionOutputRequest, } from "@tomflow/proflow-execution-contracts";
|
|
6
|
+
import { createExecutionApprovalOwner } from "./approval-owner.js";
|
|
7
|
+
export const moduleRef = "execution-runtime";
|
|
8
|
+
export class ExecutionRuntimeError extends Error {
|
|
9
|
+
code;
|
|
10
|
+
retryable;
|
|
11
|
+
constructor(code, message, retryable = false) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = "ExecutionRuntimeError";
|
|
14
|
+
this.code = code;
|
|
15
|
+
this.retryable = retryable;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
const sensitive = /(?:authorization|bearer|api[_-]?key|token|password|cookie|private[_-]?key|secret)/i;
|
|
19
|
+
const browserIds = new Set(browserCapabilityIds);
|
|
20
|
+
function canonical(value) {
|
|
21
|
+
if (Array.isArray(value))
|
|
22
|
+
return value.map(canonical);
|
|
23
|
+
if (typeof value !== "object" || value === null)
|
|
24
|
+
return value;
|
|
25
|
+
return Object.fromEntries(Object.entries(value)
|
|
26
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
27
|
+
.map(([key, item]) => [key, canonical(item)]));
|
|
28
|
+
}
|
|
29
|
+
function sha(value) {
|
|
30
|
+
return `sha256:${createHash("sha256")
|
|
31
|
+
.update(JSON.stringify(canonical(value)))
|
|
32
|
+
.digest("hex")}`;
|
|
33
|
+
}
|
|
34
|
+
const secretFlagArg = /^--?(?:token|password|api[_-]?key|authorization|bearer|cookie|secret|private[_-]?key)(?:=(.*))?$/i;
|
|
35
|
+
/** Scrubs credential-shaped fragments inside plain strings (JSON bodies, headers). */
|
|
36
|
+
function scrubEmbeddedSecrets(value) {
|
|
37
|
+
return value
|
|
38
|
+
.replace(/Bearer\s+([A-Za-z0-9._~+/-]+=*)/gi, (_match, token) => typeof token === "string" ? `Bearer [SECRET_REF:${sha(token)}]` : _match)
|
|
39
|
+
.replace(/((?:authorization|api[_-]?key|token|password|cookie|secret|private[_-]?key)\s*[:=]\s*)([^\s,;"'}]+)/gi, (_match, prefix, secretValue) => typeof prefix === "string" && typeof secretValue === "string"
|
|
40
|
+
? `${prefix}[SECRET_REF:${sha(secretValue)}]`
|
|
41
|
+
: _match)
|
|
42
|
+
.replace(/("(?:password|token|secret|api[_-]?key|authorization|cookie|private[_-]?key)"\s*:\s*")((?:[^"\\]|\\.)*)(")/gi, (_match, prefix, secretValue, suffix) => typeof prefix === "string" &&
|
|
43
|
+
typeof secretValue === "string" &&
|
|
44
|
+
typeof suffix === "string"
|
|
45
|
+
? `${prefix}[SECRET_REF:${sha(secretValue)}]${suffix}`
|
|
46
|
+
: _match);
|
|
47
|
+
}
|
|
48
|
+
function protectedValue(value, key = "") {
|
|
49
|
+
if (sensitive.test(key))
|
|
50
|
+
return `[SECRET_REF:${sha(value)}]`;
|
|
51
|
+
if (Array.isArray(value)) {
|
|
52
|
+
const items = [];
|
|
53
|
+
for (let i = 0; i < value.length; i++) {
|
|
54
|
+
const item = value[i];
|
|
55
|
+
if (typeof item !== "string") {
|
|
56
|
+
items.push(protectedValue(item));
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const flag = secretFlagArg.exec(item);
|
|
60
|
+
if (!flag) {
|
|
61
|
+
items.push(protectedValue(item));
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (flag[1] !== undefined && flag[1].length > 0) {
|
|
65
|
+
// `--token=value` — keep the flag, hash the value.
|
|
66
|
+
items.push(`${item.slice(0, item.length - flag[1].length)}[SECRET_REF:${sha(flag[1])}]`);
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
items.push(item);
|
|
70
|
+
if (i + 1 < value.length) {
|
|
71
|
+
// `--token value` — the following positional arg is the secret.
|
|
72
|
+
items.push(`[SECRET_REF:${sha(value[i + 1])}]`);
|
|
73
|
+
i++;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return items;
|
|
78
|
+
}
|
|
79
|
+
if (typeof value !== "object" || value === null) {
|
|
80
|
+
return typeof value === "string" ? scrubEmbeddedSecrets(value) : value;
|
|
81
|
+
}
|
|
82
|
+
return Object.fromEntries(Object.entries(value).map(([childKey, item]) => [
|
|
83
|
+
childKey,
|
|
84
|
+
protectedValue(item, childKey),
|
|
85
|
+
]));
|
|
86
|
+
}
|
|
87
|
+
export function executionInputFingerprint(input) {
|
|
88
|
+
const request = parseExecuteCapabilityRequest(input);
|
|
89
|
+
return sha({
|
|
90
|
+
capability: request.capability,
|
|
91
|
+
input: protectedValue(request.input),
|
|
92
|
+
callerRef: request.callerRef,
|
|
93
|
+
taskId: request.taskId,
|
|
94
|
+
nodeId: request.nodeId,
|
|
95
|
+
roleRef: request.roleRef,
|
|
96
|
+
workerRef: request.workerRef,
|
|
97
|
+
projectRoot: request.projectRoot,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
function materializeArtifactRefs(artifacts) {
|
|
101
|
+
return artifacts.map((artifact) => ({
|
|
102
|
+
ref: artifact.ref,
|
|
103
|
+
kind: artifact.kind ?? "output",
|
|
104
|
+
bytes: artifact.bytes,
|
|
105
|
+
...(artifact.hash === undefined ? {} : { hash: artifact.hash }),
|
|
106
|
+
...(artifact.mime === undefined ? {} : { mime: artifact.mime }),
|
|
107
|
+
}));
|
|
108
|
+
}
|
|
109
|
+
function runtimeError(error) {
|
|
110
|
+
if (error instanceof ExecutionRuntimeError)
|
|
111
|
+
return error;
|
|
112
|
+
if (typeof error === "object" && error !== null) {
|
|
113
|
+
const code = Reflect.get(error, "code");
|
|
114
|
+
const message = Reflect.get(error, "message");
|
|
115
|
+
const retryable = Reflect.get(error, "retryable");
|
|
116
|
+
if (typeof code === "string" &&
|
|
117
|
+
executionErrorCodes.includes(code) &&
|
|
118
|
+
typeof message === "string")
|
|
119
|
+
return new ExecutionRuntimeError(code, message, retryable === true);
|
|
120
|
+
}
|
|
121
|
+
return new ExecutionRuntimeError("EXECUTION_FAILED", error instanceof Error ? error.message : "execution failed");
|
|
122
|
+
}
|
|
123
|
+
const defaultPolicy = {
|
|
124
|
+
decide(request) {
|
|
125
|
+
const readOnly = request.capability === "file.read" ||
|
|
126
|
+
request.capability === "file.searchText" ||
|
|
127
|
+
request.capability === "artifact.external-file.materialize" ||
|
|
128
|
+
request.capability === "artifact.context-pack.materialize" ||
|
|
129
|
+
request.capability === "artifact.patch-proposal.materialize" ||
|
|
130
|
+
request.capability === "git.status" ||
|
|
131
|
+
request.capability === "git.diff" ||
|
|
132
|
+
request.capability === "project.info" ||
|
|
133
|
+
request.capability === "code.findSymbol" ||
|
|
134
|
+
request.capability === "code.findReferences" ||
|
|
135
|
+
request.capability === "process.status" ||
|
|
136
|
+
request.capability === "browser.observe" ||
|
|
137
|
+
request.capability === "browser.screenshot" ||
|
|
138
|
+
request.capability === "browser.verify" ||
|
|
139
|
+
(request.capability === "network.request" &&
|
|
140
|
+
(request.input.method === "GET" || request.input.method === "HEAD"));
|
|
141
|
+
if (readOnly)
|
|
142
|
+
return {
|
|
143
|
+
decision: "ALLOW",
|
|
144
|
+
decisionPath: "deterministic",
|
|
145
|
+
approvalRequired: false,
|
|
146
|
+
};
|
|
147
|
+
// Frozen policy: every ordinary real side effect is semantically judged by
|
|
148
|
+
// FAST first. REASON/Human are escalation paths, never a blanket gate.
|
|
149
|
+
return {
|
|
150
|
+
decision: "REVIEW",
|
|
151
|
+
decisionPath: "fast",
|
|
152
|
+
approvalRequired: false,
|
|
153
|
+
};
|
|
154
|
+
},
|
|
155
|
+
};
|
|
156
|
+
class Semaphore {
|
|
157
|
+
#available;
|
|
158
|
+
#maxQueued;
|
|
159
|
+
#waiters = [];
|
|
160
|
+
constructor(count, maxQueued) {
|
|
161
|
+
this.#available = count;
|
|
162
|
+
this.#maxQueued = maxQueued;
|
|
163
|
+
}
|
|
164
|
+
async acquire() {
|
|
165
|
+
if (this.#available > 0)
|
|
166
|
+
this.#available -= 1;
|
|
167
|
+
else {
|
|
168
|
+
if (this.#waiters.length >= this.#maxQueued)
|
|
169
|
+
throw new ExecutionRuntimeError("EXECUTOR_UNAVAILABLE", "execution queue is full", true);
|
|
170
|
+
await new Promise((resolveWait) => this.#waiters.push(resolveWait));
|
|
171
|
+
}
|
|
172
|
+
return () => {
|
|
173
|
+
const next = this.#waiters.shift();
|
|
174
|
+
if (next)
|
|
175
|
+
next();
|
|
176
|
+
else
|
|
177
|
+
this.#available += 1;
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
export async function createExecutionRuntime(options) {
|
|
182
|
+
await mkdir(dirname(options.databasePath), { recursive: true, mode: 0o700 });
|
|
183
|
+
const database = new DatabaseSync(options.databasePath);
|
|
184
|
+
database.exec("PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA busy_timeout=2500;");
|
|
185
|
+
database.exec(`
|
|
186
|
+
CREATE TABLE IF NOT EXISTS executions (
|
|
187
|
+
execution_ref TEXT PRIMARY KEY, caller_ref TEXT NOT NULL, capability TEXT NOT NULL,
|
|
188
|
+
idempotency_key TEXT NOT NULL, input_fingerprint TEXT NOT NULL, request_json TEXT NOT NULL,
|
|
189
|
+
record_json TEXT NOT NULL, precondition_json TEXT, created_at TEXT NOT NULL,
|
|
190
|
+
UNIQUE(caller_ref, capability, idempotency_key)
|
|
191
|
+
);
|
|
192
|
+
CREATE TABLE IF NOT EXISTS execution_artifacts (
|
|
193
|
+
execution_ref TEXT NOT NULL, artifact_ref TEXT NOT NULL, stream TEXT NOT NULL, path TEXT NOT NULL,
|
|
194
|
+
PRIMARY KEY(execution_ref, artifact_ref), FOREIGN KEY(execution_ref) REFERENCES executions(execution_ref)
|
|
195
|
+
);
|
|
196
|
+
CREATE TABLE IF NOT EXISTS artifact_registry (
|
|
197
|
+
artifact_ref TEXT PRIMARY KEY, kind TEXT NOT NULL, owner_caller_ref TEXT NOT NULL,
|
|
198
|
+
task_id TEXT, node_id TEXT, role_ref TEXT, worker_ref TEXT, path TEXT NOT NULL,
|
|
199
|
+
record_json TEXT NOT NULL, created_at TEXT NOT NULL
|
|
200
|
+
);
|
|
201
|
+
CREATE INDEX IF NOT EXISTS artifact_registry_scope_idx ON artifact_registry(task_id, owner_caller_ref, created_at);
|
|
202
|
+
CREATE TABLE IF NOT EXISTS execution_observer_signals (
|
|
203
|
+
signal_ref TEXT PRIMARY KEY, execution_ref TEXT NOT NULL, kind TEXT NOT NULL,
|
|
204
|
+
record_json TEXT NOT NULL, created_at TEXT NOT NULL, acknowledged_at TEXT,
|
|
205
|
+
FOREIGN KEY(execution_ref) REFERENCES executions(execution_ref)
|
|
206
|
+
);
|
|
207
|
+
CREATE INDEX IF NOT EXISTS execution_observer_signals_pending_idx
|
|
208
|
+
ON execution_observer_signals(acknowledged_at, created_at);
|
|
209
|
+
`);
|
|
210
|
+
const now = options.now ?? (() => new Date());
|
|
211
|
+
const idFactory = options.idFactory ?? randomUUID;
|
|
212
|
+
const approvalOwner = createExecutionApprovalOwner({
|
|
213
|
+
databasePath: options.databasePath,
|
|
214
|
+
...(options.now ? { now: options.now } : {}),
|
|
215
|
+
...(options.idFactory ? { idFactory: options.idFactory } : {}),
|
|
216
|
+
});
|
|
217
|
+
const approvalDraftTtlMs = options.approvalDraftTtlMs ?? 15 * 60_000;
|
|
218
|
+
if (!Number.isInteger(approvalDraftTtlMs) || approvalDraftTtlMs < 60_000)
|
|
219
|
+
throw new TypeError("approvalDraftTtlMs must be an integer >= 60000");
|
|
220
|
+
const semaphore = new Semaphore(Math.max(1, options.maxConcurrent ?? 4), Math.max(0, options.maxQueued ?? 100));
|
|
221
|
+
const controllers = new Map();
|
|
222
|
+
const logPath = join(dirname(options.databasePath), "logs", "execution-runtime", "events.jsonl");
|
|
223
|
+
await mkdir(dirname(logPath), { recursive: true, mode: 0o700 });
|
|
224
|
+
let closed = false;
|
|
225
|
+
const log = async (record, event, errorCode) => {
|
|
226
|
+
await appendFile(logPath, `${JSON.stringify({ timestamp: now().toISOString(), component: "execution-runtime", executionRef: record.executionRef, correlationId: record.correlationId, taskId: record.taskId, nodeId: record.nodeId, runNo: record.runNo, roleRef: record.roleRef, workerRef: record.workerRef, capability: record.capability, phase: record.status, event, ...(errorCode ? { errorCode } : {}) })}\n`, { mode: 0o600 });
|
|
227
|
+
};
|
|
228
|
+
const emitObserverSignal = (record, kind) => {
|
|
229
|
+
if (!record.taskId || !record.nodeId || !record.workerRef)
|
|
230
|
+
return;
|
|
231
|
+
const signalRef = `execution-signal:${createHash("sha256")
|
|
232
|
+
.update(JSON.stringify({
|
|
233
|
+
executionRef: record.executionRef,
|
|
234
|
+
kind,
|
|
235
|
+
status: record.status,
|
|
236
|
+
sideEffectState: record.sideEffectState,
|
|
237
|
+
}))
|
|
238
|
+
.digest("hex")}`;
|
|
239
|
+
const signal = {
|
|
240
|
+
signalRef,
|
|
241
|
+
kind,
|
|
242
|
+
executionRef: record.executionRef,
|
|
243
|
+
taskId: record.taskId,
|
|
244
|
+
nodeId: record.nodeId,
|
|
245
|
+
...(record.runNo === undefined ? {} : { runNo: record.runNo }),
|
|
246
|
+
...(record.roleRef === undefined ? {} : { roleRef: record.roleRef }),
|
|
247
|
+
workerRef: record.workerRef,
|
|
248
|
+
status: record.status,
|
|
249
|
+
sideEffectState: record.sideEffectState,
|
|
250
|
+
...(record.error?.code === undefined
|
|
251
|
+
? {}
|
|
252
|
+
: { errorCode: record.error.code }),
|
|
253
|
+
createdAt: now().toISOString(),
|
|
254
|
+
};
|
|
255
|
+
database
|
|
256
|
+
.prepare("INSERT OR IGNORE INTO execution_observer_signals(signal_ref, execution_ref, kind, record_json, created_at, acknowledged_at) VALUES (?, ?, ?, ?, ?, NULL)")
|
|
257
|
+
.run(signalRef, record.executionRef, kind, JSON.stringify(signal), signal.createdAt);
|
|
258
|
+
};
|
|
259
|
+
const listExecutionObserverSignals = (limit = 50) => {
|
|
260
|
+
const bounded = Math.max(1, Math.min(100, Math.trunc(limit)));
|
|
261
|
+
return database
|
|
262
|
+
.prepare("SELECT record_json FROM execution_observer_signals WHERE acknowledged_at IS NULL ORDER BY created_at, signal_ref LIMIT ?")
|
|
263
|
+
.all(bounded).map((row) => JSON.parse(String(row.record_json)));
|
|
264
|
+
};
|
|
265
|
+
const acknowledgeExecutionObserverSignal = (signalRef) => {
|
|
266
|
+
const result = database
|
|
267
|
+
.prepare("UPDATE execution_observer_signals SET acknowledged_at=? WHERE signal_ref=? AND acknowledged_at IS NULL")
|
|
268
|
+
.run(now().toISOString(), signalRef);
|
|
269
|
+
return { acknowledged: result.changes === 1 };
|
|
270
|
+
};
|
|
271
|
+
const getRow = (executionRef) => database
|
|
272
|
+
.prepare("SELECT * FROM executions WHERE execution_ref=?")
|
|
273
|
+
.get(executionRef);
|
|
274
|
+
const fromRow = (row) => parseExecutionRecord(JSON.parse(String(row.record_json)));
|
|
275
|
+
const save = (record, precondition) => {
|
|
276
|
+
database
|
|
277
|
+
.prepare("UPDATE executions SET record_json=?, precondition_json=COALESCE(?, precondition_json) WHERE execution_ref=?")
|
|
278
|
+
.run(JSON.stringify(record), precondition === undefined
|
|
279
|
+
? null
|
|
280
|
+
: JSON.stringify(protectedValue(precondition)), record.executionRef);
|
|
281
|
+
};
|
|
282
|
+
const failRecord = (record, error, sideEffectState = "NOT_APPLIED") => parseExecutionRecord({
|
|
283
|
+
...record,
|
|
284
|
+
status: sideEffectState === "UNKNOWN" ? "UNKNOWN" : "FAILED",
|
|
285
|
+
sideEffectState,
|
|
286
|
+
retryable: sideEffectState === "UNKNOWN" ? false : error.retryable,
|
|
287
|
+
error: {
|
|
288
|
+
code: sideEffectState === "UNKNOWN" ? "UNKNOWN_SIDE_EFFECT" : error.code,
|
|
289
|
+
message: error.message,
|
|
290
|
+
retryable: sideEffectState === "UNKNOWN" ? false : error.retryable,
|
|
291
|
+
},
|
|
292
|
+
finishedAt: now().toISOString(),
|
|
293
|
+
updatedAt: now().toISOString(),
|
|
294
|
+
});
|
|
295
|
+
async function admit(request, inputFingerprint, executionRef) {
|
|
296
|
+
if (options.identity && !(await options.identity.authorize(request)))
|
|
297
|
+
throw new ExecutionRuntimeError("IDENTITY_INVALID", "caller, Task, Agent, workspace, or Browser identity is not authoritative");
|
|
298
|
+
const policy = options.policy ?? defaultPolicy;
|
|
299
|
+
const policyDecision = await policy.decide(request);
|
|
300
|
+
if (policyDecision.decision === "DENY")
|
|
301
|
+
throw new ExecutionRuntimeError("POLICY_DENIED", policyDecision.reason ?? "deterministic policy denied execution");
|
|
302
|
+
let decisionPath = policyDecision.decisionPath;
|
|
303
|
+
let approvalRequired = policyDecision.approvalRequired === true;
|
|
304
|
+
if (policyDecision.decision === "REVIEW") {
|
|
305
|
+
if (!options.modelDecision)
|
|
306
|
+
throw new ExecutionRuntimeError("DECISION_UNRESOLVED", "review requires a model decision");
|
|
307
|
+
const model = options.modelDecision;
|
|
308
|
+
const modelDecision = await model.decide(request, {
|
|
309
|
+
executionRef,
|
|
310
|
+
inputFingerprint,
|
|
311
|
+
});
|
|
312
|
+
decisionPath = modelDecision.decisionPath;
|
|
313
|
+
approvalRequired =
|
|
314
|
+
approvalRequired || modelDecision.approvalRequired === true;
|
|
315
|
+
if (approvalRequired)
|
|
316
|
+
decisionPath = "human";
|
|
317
|
+
if (modelDecision.decision === "DENY")
|
|
318
|
+
throw new ExecutionRuntimeError("POLICY_DENIED", modelDecision.reason ?? "model decision denied execution");
|
|
319
|
+
}
|
|
320
|
+
const validateApproval = async (precondition) => {
|
|
321
|
+
if (!approvalRequired)
|
|
322
|
+
return;
|
|
323
|
+
if (!request.approvalRef)
|
|
324
|
+
throw new ExecutionRuntimeError("APPROVAL_REQUIRED", "bound approval is required");
|
|
325
|
+
const approval = options.approval ?? approvalOwner;
|
|
326
|
+
if (!(await approval.validate({
|
|
327
|
+
approvalRef: request.approvalRef,
|
|
328
|
+
executionRef,
|
|
329
|
+
callerRef: request.callerRef,
|
|
330
|
+
...(request.taskId ? { taskId: request.taskId } : {}),
|
|
331
|
+
capability: request.capability,
|
|
332
|
+
inputFingerprint,
|
|
333
|
+
request,
|
|
334
|
+
...(request.projectRoot ? { projectRoot: request.projectRoot } : {}),
|
|
335
|
+
...(precondition ? { precondition } : {}),
|
|
336
|
+
now: now().toISOString(),
|
|
337
|
+
})))
|
|
338
|
+
throw new ExecutionRuntimeError("APPROVAL_INVALID", "approval binding, expiry or status is invalid");
|
|
339
|
+
};
|
|
340
|
+
await validateApproval();
|
|
341
|
+
return {
|
|
342
|
+
policy: "ALLOW",
|
|
343
|
+
decisionPath,
|
|
344
|
+
approval: approvalRequired
|
|
345
|
+
? "VALID"
|
|
346
|
+
: "NOT_REQUIRED",
|
|
347
|
+
validateApproval,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
function createPending(request, inputFingerprint) {
|
|
351
|
+
const stamp = now().toISOString();
|
|
352
|
+
return parseExecutionRecord({
|
|
353
|
+
contract: "execution",
|
|
354
|
+
contractVersion: EXECUTION_CONTRACT_VERSION,
|
|
355
|
+
executionRef: request.executionRef ?? `execution:${idFactory()}`,
|
|
356
|
+
capability: request.capability,
|
|
357
|
+
callerRef: request.callerRef,
|
|
358
|
+
...(request.correlationId
|
|
359
|
+
? { correlationId: request.correlationId }
|
|
360
|
+
: {}),
|
|
361
|
+
...(request.taskId ? { taskId: request.taskId } : {}),
|
|
362
|
+
...(request.nodeId ? { nodeId: request.nodeId } : {}),
|
|
363
|
+
...(request.runNo ? { runNo: request.runNo } : {}),
|
|
364
|
+
...(request.roleRef ? { roleRef: request.roleRef } : {}),
|
|
365
|
+
...(request.workerRef ? { workerRef: request.workerRef } : {}),
|
|
366
|
+
idempotencyKey: request.idempotencyKey,
|
|
367
|
+
inputFingerprint,
|
|
368
|
+
status: "PENDING",
|
|
369
|
+
sideEffectState: "NOT_STARTED",
|
|
370
|
+
retryable: false,
|
|
371
|
+
evidence: [],
|
|
372
|
+
attemptCount: 0,
|
|
373
|
+
createdAt: stamp,
|
|
374
|
+
updatedAt: stamp,
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
function registerArtifact(input) {
|
|
378
|
+
const record = parseExecutionArtifactRecord(input.record);
|
|
379
|
+
const existing = database
|
|
380
|
+
.prepare("SELECT record_json, path FROM artifact_registry WHERE artifact_ref=?")
|
|
381
|
+
.get(record.artifactRef);
|
|
382
|
+
if (existing) {
|
|
383
|
+
const existingRecord = parseExecutionArtifactRecord(JSON.parse(String(existing.record_json)));
|
|
384
|
+
if (JSON.stringify(existingRecord) !== JSON.stringify(record) ||
|
|
385
|
+
String(existing.path) !== input.path)
|
|
386
|
+
throw new ExecutionRuntimeError("IDEMPOTENCY_CONFLICT", "artifactRef is immutable and was already registered with different content");
|
|
387
|
+
return existingRecord;
|
|
388
|
+
}
|
|
389
|
+
database
|
|
390
|
+
.prepare(`INSERT INTO artifact_registry
|
|
391
|
+
(artifact_ref, kind, owner_caller_ref, task_id, node_id, role_ref, worker_ref, path, record_json, created_at)
|
|
392
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
393
|
+
.run(record.artifactRef, record.kind, record.ownerCallerRef, record.taskId ?? null, record.nodeId ?? null, record.roleRef ?? null, record.workerRef ?? null, input.path, JSON.stringify(record), record.createdAt);
|
|
394
|
+
return record;
|
|
395
|
+
}
|
|
396
|
+
function getArtifactStorage(artifactRef) {
|
|
397
|
+
const row = database
|
|
398
|
+
.prepare("SELECT record_json, path FROM artifact_registry WHERE artifact_ref=?")
|
|
399
|
+
.get(artifactRef);
|
|
400
|
+
if (!row)
|
|
401
|
+
throw new ExecutionRuntimeError("INVALID_REQUEST", "artifact was not found");
|
|
402
|
+
return {
|
|
403
|
+
record: parseExecutionArtifactRecord(JSON.parse(String(row.record_json))),
|
|
404
|
+
path: String(row.path),
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function getArtifactRecordInternal(artifactRef) {
|
|
408
|
+
return getArtifactStorage(artifactRef).record;
|
|
409
|
+
}
|
|
410
|
+
const getArtifactSummary = () => {
|
|
411
|
+
const total = database
|
|
412
|
+
.prepare("SELECT COUNT(*) AS count FROM artifact_registry")
|
|
413
|
+
.get();
|
|
414
|
+
const kinds = database
|
|
415
|
+
.prepare("SELECT kind, COUNT(*) AS count FROM artifact_registry GROUP BY kind ORDER BY kind")
|
|
416
|
+
.all();
|
|
417
|
+
const latest = database
|
|
418
|
+
.prepare("SELECT MAX(created_at) AS latest FROM artifact_registry")
|
|
419
|
+
.get();
|
|
420
|
+
const pending = database
|
|
421
|
+
.prepare("SELECT COUNT(*) AS count FROM execution_observer_signals WHERE acknowledged_at IS NULL")
|
|
422
|
+
.get();
|
|
423
|
+
const byKind = {};
|
|
424
|
+
for (const row of kinds)
|
|
425
|
+
byKind[String(row.kind)] = Number(row.count);
|
|
426
|
+
return {
|
|
427
|
+
totalArtifacts: Number(total?.count ?? 0),
|
|
428
|
+
byKind,
|
|
429
|
+
latestCreatedAt: latest?.latest === undefined || latest.latest === null
|
|
430
|
+
? null
|
|
431
|
+
: String(latest.latest),
|
|
432
|
+
pendingObserverSignals: Number(pending?.count ?? 0),
|
|
433
|
+
};
|
|
434
|
+
};
|
|
435
|
+
function getArtifactRecord(input) {
|
|
436
|
+
const artifact = getArtifactRecordInternal(input.artifactRef);
|
|
437
|
+
if (artifact.ownerCallerRef !== input.callerRef)
|
|
438
|
+
throw new ExecutionRuntimeError("IDENTITY_INVALID", "caller does not own the Artifact");
|
|
439
|
+
for (const [label, expected, actual] of [
|
|
440
|
+
["task", artifact.taskId, input.taskId],
|
|
441
|
+
["node", artifact.nodeId, input.nodeId],
|
|
442
|
+
["role", artifact.roleRef, input.roleRef],
|
|
443
|
+
["worker", artifact.workerRef, input.workerRef],
|
|
444
|
+
]) {
|
|
445
|
+
if (expected !== undefined && expected !== actual)
|
|
446
|
+
throw new ExecutionRuntimeError("SCOPE_DENIED", `Artifact ${label} scope does not match the read request`);
|
|
447
|
+
}
|
|
448
|
+
return artifact;
|
|
449
|
+
}
|
|
450
|
+
function assertPatchArtifactScope(request) {
|
|
451
|
+
if (request.capability !== "patch.apply")
|
|
452
|
+
return;
|
|
453
|
+
const artifact = getArtifactRecordInternal(request.input.artifactRef);
|
|
454
|
+
if (artifact.kind !== "patch-proposal")
|
|
455
|
+
throw new ExecutionRuntimeError("PRECONDITION_FAILED", "patch.apply requires a durable patch-proposal Artifact");
|
|
456
|
+
if (artifact.ownerCallerRef !== request.callerRef)
|
|
457
|
+
throw new ExecutionRuntimeError("IDENTITY_INVALID", "caller does not own the patch proposal Artifact");
|
|
458
|
+
for (const [label, expected, actual] of [
|
|
459
|
+
["task", artifact.taskId, request.taskId],
|
|
460
|
+
["node", artifact.nodeId, request.nodeId],
|
|
461
|
+
["role", artifact.roleRef, request.roleRef],
|
|
462
|
+
["worker", artifact.workerRef, request.workerRef],
|
|
463
|
+
]) {
|
|
464
|
+
if (expected !== undefined && expected !== actual)
|
|
465
|
+
throw new ExecutionRuntimeError("SCOPE_DENIED", `patch proposal ${label} scope does not match the Execution Request`);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
options.localExecutor.bindPatchArtifactResolver?.(async (artifactRef) => {
|
|
469
|
+
try {
|
|
470
|
+
const { record, path } = getArtifactStorage(artifactRef);
|
|
471
|
+
if (record.kind !== "patch-proposal")
|
|
472
|
+
return undefined;
|
|
473
|
+
const baseHash = record.metadata?.baseHash;
|
|
474
|
+
const baseRef = record.metadata?.baseRef;
|
|
475
|
+
const hash = record.hash;
|
|
476
|
+
if (typeof baseHash !== "string" ||
|
|
477
|
+
typeof baseRef !== "string" ||
|
|
478
|
+
typeof hash !== "string")
|
|
479
|
+
return undefined;
|
|
480
|
+
return {
|
|
481
|
+
artifactRef: record.artifactRef,
|
|
482
|
+
kind: "patch-proposal",
|
|
483
|
+
path,
|
|
484
|
+
hash,
|
|
485
|
+
baseHash,
|
|
486
|
+
baseRef,
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
catch {
|
|
490
|
+
return undefined;
|
|
491
|
+
}
|
|
492
|
+
});
|
|
493
|
+
function lookupExecutionIntent(input) {
|
|
494
|
+
const request = parseExecuteCapabilityRequest(input);
|
|
495
|
+
const row = database
|
|
496
|
+
.prepare("SELECT * FROM executions WHERE caller_ref=? AND capability=? AND idempotency_key=?")
|
|
497
|
+
.get(request.callerRef, request.capability, request.idempotencyKey);
|
|
498
|
+
if (!row)
|
|
499
|
+
throw new ExecutionRuntimeError("INVALID_REQUEST", "execution intent was not found");
|
|
500
|
+
if (String(row.input_fingerprint) !== executionInputFingerprint(request))
|
|
501
|
+
throw new ExecutionRuntimeError("IDEMPOTENCY_CONFLICT", "execution intent lookup input does not match durable input");
|
|
502
|
+
return fromRow(row);
|
|
503
|
+
}
|
|
504
|
+
async function executeCapability(input) {
|
|
505
|
+
if (closed)
|
|
506
|
+
throw new ExecutionRuntimeError("EXECUTOR_UNAVAILABLE", "execution runtime is closed");
|
|
507
|
+
const request = parseExecuteCapabilityRequest(input);
|
|
508
|
+
assertPatchArtifactScope(request);
|
|
509
|
+
const inputFingerprint = executionInputFingerprint(request);
|
|
510
|
+
const existing = database
|
|
511
|
+
.prepare("SELECT * FROM executions WHERE caller_ref=? AND capability=? AND idempotency_key=?")
|
|
512
|
+
.get(request.callerRef, request.capability, request.idempotencyKey);
|
|
513
|
+
let record;
|
|
514
|
+
if (existing) {
|
|
515
|
+
if (String(existing.input_fingerprint) !== inputFingerprint)
|
|
516
|
+
throw new ExecutionRuntimeError("IDEMPOTENCY_CONFLICT", "idempotency key was already used with different critical input");
|
|
517
|
+
const prior = fromRow(existing);
|
|
518
|
+
const safelyResumable = (prior.status === "PENDING" &&
|
|
519
|
+
prior.sideEffectState === "NOT_STARTED") ||
|
|
520
|
+
(prior.status === "FAILED" && prior.sideEffectState === "NOT_APPLIED");
|
|
521
|
+
if (!safelyResumable)
|
|
522
|
+
return prior;
|
|
523
|
+
const { error: _error, result: _result, finishedAt: _finishedAt, startedAt: _startedAt, effectStartedAt: _effectStartedAt, decisionPath: _decisionPath, approvalRef: _approvalRef, ...resumable } = prior;
|
|
524
|
+
record = parseExecutionRecord({
|
|
525
|
+
...resumable,
|
|
526
|
+
status: "PENDING",
|
|
527
|
+
sideEffectState: "NOT_STARTED",
|
|
528
|
+
retryable: false,
|
|
529
|
+
updatedAt: now().toISOString(),
|
|
530
|
+
});
|
|
531
|
+
database
|
|
532
|
+
.prepare("UPDATE executions SET request_json=?, record_json=?, precondition_json=NULL WHERE execution_ref=?")
|
|
533
|
+
.run(JSON.stringify(protectedValue(request)), JSON.stringify(record), record.executionRef);
|
|
534
|
+
await log(record, "EXECUTION_REDECISION_REQUESTED");
|
|
535
|
+
}
|
|
536
|
+
else {
|
|
537
|
+
record = createPending(request, inputFingerprint);
|
|
538
|
+
try {
|
|
539
|
+
database
|
|
540
|
+
.prepare("INSERT INTO executions VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?)")
|
|
541
|
+
.run(record.executionRef, request.callerRef, request.capability, request.idempotencyKey, inputFingerprint, JSON.stringify(protectedValue(request)), JSON.stringify(record), record.createdAt);
|
|
542
|
+
await log(record, "INTENT_PERSISTED");
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
const raced = database
|
|
546
|
+
.prepare("SELECT * FROM executions WHERE caller_ref=? AND capability=? AND idempotency_key=?")
|
|
547
|
+
.get(request.callerRef, request.capability, request.idempotencyKey);
|
|
548
|
+
if (!raced || String(raced.input_fingerprint) !== inputFingerprint)
|
|
549
|
+
throw new ExecutionRuntimeError("IDEMPOTENCY_CONFLICT", "concurrent idempotency conflict");
|
|
550
|
+
return fromRow(raced);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
let admission;
|
|
554
|
+
try {
|
|
555
|
+
admission = await admit(request, inputFingerprint, record.executionRef);
|
|
556
|
+
}
|
|
557
|
+
catch (error) {
|
|
558
|
+
const admissionError = runtimeError(error);
|
|
559
|
+
if (admissionError.code === "APPROVAL_REQUIRED" &&
|
|
560
|
+
!request.approvalRef &&
|
|
561
|
+
!options.approval) {
|
|
562
|
+
const executorForObserve = browserIds.has(request.capability)
|
|
563
|
+
? options.browserExecutor
|
|
564
|
+
: options.localExecutor;
|
|
565
|
+
let precondition;
|
|
566
|
+
try {
|
|
567
|
+
precondition = executorForObserve?.observePrecondition
|
|
568
|
+
? await executorForObserve.observePrecondition(request)
|
|
569
|
+
: undefined;
|
|
570
|
+
}
|
|
571
|
+
catch (error) {
|
|
572
|
+
record = failRecord(record, runtimeError(error));
|
|
573
|
+
save(record);
|
|
574
|
+
await log(record, "ADMISSION_REJECTED", record.error?.code);
|
|
575
|
+
return record;
|
|
576
|
+
}
|
|
577
|
+
const approval = approvalOwner.ensurePendingApproval({
|
|
578
|
+
executionRef: record.executionRef,
|
|
579
|
+
actorRef: "execution-runtime:policy",
|
|
580
|
+
expiresAt: new Date(now().getTime() + approvalDraftTtlMs).toISOString(),
|
|
581
|
+
...(precondition ? { precondition } : {}),
|
|
582
|
+
});
|
|
583
|
+
record = parseExecutionRecord({
|
|
584
|
+
...record,
|
|
585
|
+
approvalRef: approval.approvalRef,
|
|
586
|
+
updatedAt: now().toISOString(),
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
record = failRecord(record, admissionError);
|
|
590
|
+
save(record);
|
|
591
|
+
await log(record, "ADMISSION_REJECTED", record.error?.code);
|
|
592
|
+
return record;
|
|
593
|
+
}
|
|
594
|
+
let release;
|
|
595
|
+
try {
|
|
596
|
+
release = await semaphore.acquire();
|
|
597
|
+
}
|
|
598
|
+
catch (error) {
|
|
599
|
+
record = failRecord(record, runtimeError(error));
|
|
600
|
+
save(record);
|
|
601
|
+
await log(record, "QUEUE_REJECTED", record.error?.code);
|
|
602
|
+
return record;
|
|
603
|
+
}
|
|
604
|
+
const admittedRow = getRow(record.executionRef);
|
|
605
|
+
if (admittedRow) {
|
|
606
|
+
const admittedRecord = fromRow(admittedRow);
|
|
607
|
+
if (admittedRecord.status !== "PENDING") {
|
|
608
|
+
release();
|
|
609
|
+
return admittedRecord;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
const controller = new AbortController();
|
|
613
|
+
controllers.set(record.executionRef, controller);
|
|
614
|
+
let timedOut = false;
|
|
615
|
+
const runtimeTimeout = request.timeoutMs === undefined
|
|
616
|
+
? undefined
|
|
617
|
+
: setTimeout(() => {
|
|
618
|
+
timedOut = true;
|
|
619
|
+
controller.abort();
|
|
620
|
+
}, request.timeoutMs);
|
|
621
|
+
try {
|
|
622
|
+
await admission.validateApproval();
|
|
623
|
+
if (controller.signal.aborted)
|
|
624
|
+
throw new ExecutionRuntimeError(timedOut ? "TIMEOUT" : "CANCELLED", timedOut
|
|
625
|
+
? "execution runtime timeout elapsed during admission"
|
|
626
|
+
: "execution was cancelled during admission");
|
|
627
|
+
// CAS: only transition to RUNNING if the record is still PENDING.
|
|
628
|
+
// A concurrent cancel/timeout finalizes PENDING records, so a
|
|
629
|
+
// stale RUNNING write would silently resurrect a cancelled intent.
|
|
630
|
+
const casRow = getRow(record.executionRef);
|
|
631
|
+
if (casRow) {
|
|
632
|
+
const casRecord = fromRow(casRow);
|
|
633
|
+
if (casRecord.status !== "PENDING")
|
|
634
|
+
return casRecord;
|
|
635
|
+
}
|
|
636
|
+
record = parseExecutionRecord({
|
|
637
|
+
...record,
|
|
638
|
+
status: "RUNNING",
|
|
639
|
+
sideEffectState: "NOT_STARTED",
|
|
640
|
+
decisionPath: admission.decisionPath,
|
|
641
|
+
...(request.approvalRef ? { approvalRef: request.approvalRef } : {}),
|
|
642
|
+
attemptCount: record.attemptCount + 1,
|
|
643
|
+
startedAt: now().toISOString(),
|
|
644
|
+
updatedAt: now().toISOString(),
|
|
645
|
+
});
|
|
646
|
+
save(record);
|
|
647
|
+
await log(record, "EXECUTION_STARTED");
|
|
648
|
+
const executor = browserIds.has(request.capability)
|
|
649
|
+
? options.browserExecutor
|
|
650
|
+
: options.localExecutor;
|
|
651
|
+
if (!executor)
|
|
652
|
+
throw new ExecutionRuntimeError("EXECUTOR_UNAVAILABLE", "no executor is registered for capability");
|
|
653
|
+
let precondition;
|
|
654
|
+
const result = await executor.execute({
|
|
655
|
+
request,
|
|
656
|
+
admission,
|
|
657
|
+
signal: controller.signal,
|
|
658
|
+
async onEffectStarted(value) {
|
|
659
|
+
if (controller.signal.aborted)
|
|
660
|
+
throw new ExecutionRuntimeError(timedOut ? "TIMEOUT" : "CANCELLED", timedOut
|
|
661
|
+
? "execution runtime timeout elapsed before effect"
|
|
662
|
+
: "execution was cancelled before effect");
|
|
663
|
+
await admission.validateApproval(value);
|
|
664
|
+
if (controller.signal.aborted)
|
|
665
|
+
throw new ExecutionRuntimeError(timedOut ? "TIMEOUT" : "CANCELLED", "execution was cancelled while revalidating approval before effect");
|
|
666
|
+
if (request.approvalRef && !options.approval)
|
|
667
|
+
approvalOwner.consume(request.approvalRef);
|
|
668
|
+
precondition = value;
|
|
669
|
+
record = parseExecutionRecord({
|
|
670
|
+
...record,
|
|
671
|
+
status: "RUNNING",
|
|
672
|
+
sideEffectState: "STARTED",
|
|
673
|
+
effectStartedAt: now().toISOString(),
|
|
674
|
+
updatedAt: now().toISOString(),
|
|
675
|
+
});
|
|
676
|
+
save(record, value);
|
|
677
|
+
await log(record, "EFFECT_STARTED");
|
|
678
|
+
},
|
|
679
|
+
});
|
|
680
|
+
if (controller.signal.aborted)
|
|
681
|
+
throw new ExecutionRuntimeError(timedOut ? "TIMEOUT" : "CANCELLED", timedOut
|
|
682
|
+
? "execution runtime timeout elapsed"
|
|
683
|
+
: "execution was cancelled");
|
|
684
|
+
if (!result.successful && result.effectApplied)
|
|
685
|
+
throw new ExecutionRuntimeError("EXECUTION_FAILED", "effectful executor reported an unsuccessful result");
|
|
686
|
+
database.exec("BEGIN IMMEDIATE");
|
|
687
|
+
try {
|
|
688
|
+
for (const artifact of result.artifacts) {
|
|
689
|
+
registerArtifact({
|
|
690
|
+
path: artifact.path,
|
|
691
|
+
record: {
|
|
692
|
+
contract: "execution.artifact",
|
|
693
|
+
contractVersion: EXECUTION_CONTRACT_VERSION,
|
|
694
|
+
artifactRef: artifact.ref,
|
|
695
|
+
kind: artifact.kind ?? "output",
|
|
696
|
+
ownerCallerRef: request.callerRef,
|
|
697
|
+
...(request.taskId ? { taskId: request.taskId } : {}),
|
|
698
|
+
...(request.nodeId ? { nodeId: request.nodeId } : {}),
|
|
699
|
+
...(request.roleRef ? { roleRef: request.roleRef } : {}),
|
|
700
|
+
...(request.workerRef ? { workerRef: request.workerRef } : {}),
|
|
701
|
+
...(artifact.hash ? { hash: artifact.hash } : {}),
|
|
702
|
+
...(artifact.mime ? { mime: artifact.mime } : {}),
|
|
703
|
+
bytes: artifact.bytes,
|
|
704
|
+
metadata: {
|
|
705
|
+
stream: artifact.stream,
|
|
706
|
+
executionRef: record.executionRef,
|
|
707
|
+
...(artifact.metadata ?? {}),
|
|
708
|
+
},
|
|
709
|
+
createdAt: now().toISOString(),
|
|
710
|
+
},
|
|
711
|
+
});
|
|
712
|
+
const existingRelation = database
|
|
713
|
+
.prepare("SELECT stream, path FROM execution_artifacts WHERE execution_ref=? AND artifact_ref=?")
|
|
714
|
+
.get(record.executionRef, artifact.ref);
|
|
715
|
+
if (existingRelation) {
|
|
716
|
+
if (String(existingRelation.stream) !== artifact.stream ||
|
|
717
|
+
String(existingRelation.path) !== artifact.path)
|
|
718
|
+
throw new ExecutionRuntimeError("IDEMPOTENCY_CONFLICT", "Execution Artifact relation is immutable");
|
|
719
|
+
}
|
|
720
|
+
else
|
|
721
|
+
database
|
|
722
|
+
.prepare("INSERT INTO execution_artifacts VALUES (?, ?, ?, ?)")
|
|
723
|
+
.run(record.executionRef, artifact.ref, artifact.stream, artifact.path);
|
|
724
|
+
}
|
|
725
|
+
database.exec("COMMIT");
|
|
726
|
+
}
|
|
727
|
+
catch (error) {
|
|
728
|
+
database.exec("ROLLBACK");
|
|
729
|
+
throw error;
|
|
730
|
+
}
|
|
731
|
+
if (!result.successful) {
|
|
732
|
+
const resultData = result.result.data;
|
|
733
|
+
const exitCode = typeof resultData.exitCode === "number"
|
|
734
|
+
? resultData.exitCode
|
|
735
|
+
: undefined;
|
|
736
|
+
record = parseExecutionRecord({
|
|
737
|
+
...record,
|
|
738
|
+
status: "FAILED",
|
|
739
|
+
sideEffectState: "NOT_APPLIED",
|
|
740
|
+
retryable: false,
|
|
741
|
+
evidence: result.evidence,
|
|
742
|
+
evidenceRefs: result.evidence.map((item) => item.evidenceRef),
|
|
743
|
+
artifactRefs: materializeArtifactRefs(result.artifacts).map((item) => item.ref),
|
|
744
|
+
error: {
|
|
745
|
+
code: "EXECUTION_FAILED",
|
|
746
|
+
message: exitCode !== undefined
|
|
747
|
+
? `verification command exited with code ${exitCode}`
|
|
748
|
+
: "verification command returned a known non-zero result",
|
|
749
|
+
retryable: false,
|
|
750
|
+
},
|
|
751
|
+
finishedAt: now().toISOString(),
|
|
752
|
+
updatedAt: now().toISOString(),
|
|
753
|
+
});
|
|
754
|
+
save(record);
|
|
755
|
+
await log(record, "EXECUTION_FAILED", "EXECUTION_FAILED");
|
|
756
|
+
return record;
|
|
757
|
+
}
|
|
758
|
+
record = parseExecutionRecord({
|
|
759
|
+
...record,
|
|
760
|
+
status: "SUCCEEDED",
|
|
761
|
+
sideEffectState: result.effectApplied ? "APPLIED" : "NOT_APPLIED",
|
|
762
|
+
retryable: false,
|
|
763
|
+
result: result.result,
|
|
764
|
+
evidence: result.evidence,
|
|
765
|
+
evidenceRefs: result.evidence.map((item) => item.evidenceRef),
|
|
766
|
+
artifactRefs: materializeArtifactRefs(result.artifacts).map((item) => item.ref),
|
|
767
|
+
finishedAt: now().toISOString(),
|
|
768
|
+
updatedAt: now().toISOString(),
|
|
769
|
+
});
|
|
770
|
+
save(record, precondition);
|
|
771
|
+
await log(record, "EXECUTION_SUCCEEDED");
|
|
772
|
+
return record;
|
|
773
|
+
}
|
|
774
|
+
catch (error) {
|
|
775
|
+
const failure = timedOut
|
|
776
|
+
? new ExecutionRuntimeError("TIMEOUT", "execution runtime timeout elapsed")
|
|
777
|
+
: runtimeError(error);
|
|
778
|
+
const row = getRow(record.executionRef);
|
|
779
|
+
if (row)
|
|
780
|
+
record = fromRow(row);
|
|
781
|
+
if (record.sideEffectState === "STARTED") {
|
|
782
|
+
const executor = browserIds.has(request.capability)
|
|
783
|
+
? options.browserExecutor
|
|
784
|
+
: options.localExecutor;
|
|
785
|
+
const preconditionRaw = row?.precondition_json;
|
|
786
|
+
if (executor && preconditionRaw) {
|
|
787
|
+
try {
|
|
788
|
+
const reconciliation = await executor.reconcile(request, JSON.parse(String(preconditionRaw)));
|
|
789
|
+
if (reconciliation.state === "NOT_APPLIED")
|
|
790
|
+
record = failRecord(record, failure);
|
|
791
|
+
else if (reconciliation.state === "APPLIED" &&
|
|
792
|
+
reconciliation.result)
|
|
793
|
+
record = parseExecutionRecord({
|
|
794
|
+
...record,
|
|
795
|
+
status: "SUCCEEDED",
|
|
796
|
+
sideEffectState: "APPLIED",
|
|
797
|
+
retryable: false,
|
|
798
|
+
result: reconciliation.result,
|
|
799
|
+
evidence: reconciliation.evidence,
|
|
800
|
+
evidenceRefs: reconciliation.evidence.map((item) => item.evidenceRef),
|
|
801
|
+
finishedAt: now().toISOString(),
|
|
802
|
+
updatedAt: now().toISOString(),
|
|
803
|
+
});
|
|
804
|
+
else
|
|
805
|
+
record = failRecord(record, new ExecutionRuntimeError("UNKNOWN_SIDE_EFFECT", "effect started but reality could not be reconstructed"), "UNKNOWN");
|
|
806
|
+
}
|
|
807
|
+
catch {
|
|
808
|
+
record = failRecord(record, new ExecutionRuntimeError("UNKNOWN_SIDE_EFFECT", "effect reconciliation failed"), "UNKNOWN");
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
else
|
|
812
|
+
record = failRecord(record, new ExecutionRuntimeError("UNKNOWN_SIDE_EFFECT", "effect executor unavailable during reconciliation"), "UNKNOWN");
|
|
813
|
+
}
|
|
814
|
+
else
|
|
815
|
+
record = failRecord(record, failure);
|
|
816
|
+
save(record);
|
|
817
|
+
await log(record, "EXECUTION_TERMINATED", record.error?.code);
|
|
818
|
+
return record;
|
|
819
|
+
}
|
|
820
|
+
finally {
|
|
821
|
+
if (runtimeTimeout !== undefined)
|
|
822
|
+
clearTimeout(runtimeTimeout);
|
|
823
|
+
controllers.delete(record.executionRef);
|
|
824
|
+
release();
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
function getExecutionRecord(executionRef) {
|
|
828
|
+
const row = getRow(executionRef);
|
|
829
|
+
if (!row)
|
|
830
|
+
throw new ExecutionRuntimeError("INVALID_REQUEST", "execution was not found");
|
|
831
|
+
return fromRow(row);
|
|
832
|
+
}
|
|
833
|
+
function getExecution(input) {
|
|
834
|
+
return getExecutionRecord(parseExecutionRef(input));
|
|
835
|
+
}
|
|
836
|
+
function getExecutionForCaller(input) {
|
|
837
|
+
const record = getExecutionRecord(parseExecutionRef(input.executionRef));
|
|
838
|
+
if (record.callerRef !== input.callerRef)
|
|
839
|
+
throw new ExecutionRuntimeError("IDENTITY_INVALID", "only the bound caller may read execution");
|
|
840
|
+
return record;
|
|
841
|
+
}
|
|
842
|
+
async function readExecutionOutputRecord(input, record) {
|
|
843
|
+
const request = parseReadExecutionOutputRequest(input);
|
|
844
|
+
if (record.executionRef !== request.executionRef)
|
|
845
|
+
throw new ExecutionRuntimeError("INVALID_REQUEST", "execution output scope mismatch");
|
|
846
|
+
const row = database
|
|
847
|
+
.prepare("SELECT artifact_ref FROM execution_artifacts WHERE execution_ref=? AND stream=? ORDER BY artifact_ref LIMIT 1")
|
|
848
|
+
.get(request.executionRef, request.stream);
|
|
849
|
+
if (!row)
|
|
850
|
+
throw new ExecutionRuntimeError("INVALID_REQUEST", "execution output stream was not found");
|
|
851
|
+
const executor = browserIds.has(record.capability)
|
|
852
|
+
? options.browserExecutor
|
|
853
|
+
: options.localExecutor;
|
|
854
|
+
if (!executor)
|
|
855
|
+
throw new ExecutionRuntimeError("EXECUTOR_UNAVAILABLE", "output executor is unavailable");
|
|
856
|
+
const offset = request.offset ?? 0;
|
|
857
|
+
const output = await executor.readArtifact(String(row.artifact_ref), offset, request.limit ?? 16_384);
|
|
858
|
+
return {
|
|
859
|
+
contract: "execution",
|
|
860
|
+
contractVersion: EXECUTION_CONTRACT_VERSION,
|
|
861
|
+
executionRef: request.executionRef,
|
|
862
|
+
stream: request.stream,
|
|
863
|
+
chunk: output.chunk,
|
|
864
|
+
offset,
|
|
865
|
+
nextOffset: output.nextOffset,
|
|
866
|
+
eof: output.eof,
|
|
867
|
+
artifactRef: String(row.artifact_ref),
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
async function readExecutionOutput(input) {
|
|
871
|
+
const request = parseReadExecutionOutputRequest(input);
|
|
872
|
+
return readExecutionOutputRecord(request, getExecutionRecord(request.executionRef));
|
|
873
|
+
}
|
|
874
|
+
async function readExecutionOutputForCaller(input, callerRef) {
|
|
875
|
+
const request = parseReadExecutionOutputRequest(input);
|
|
876
|
+
return readExecutionOutputRecord(request, getExecutionForCaller({ executionRef: request.executionRef, callerRef }));
|
|
877
|
+
}
|
|
878
|
+
async function cancelExecution(input) {
|
|
879
|
+
const request = parseCancelExecutionRequest(input);
|
|
880
|
+
let record = getExecutionRecord(request.executionRef);
|
|
881
|
+
if (record.callerRef !== request.callerRef)
|
|
882
|
+
throw new ExecutionRuntimeError("IDENTITY_INVALID", "only the bound caller may cancel execution");
|
|
883
|
+
controllers.get(request.executionRef)?.abort();
|
|
884
|
+
if (record.status === "PENDING") {
|
|
885
|
+
record = failRecord(record, new ExecutionRuntimeError("CANCELLED", request.reason));
|
|
886
|
+
save(record);
|
|
887
|
+
await log(record, "EXECUTION_CANCELLED", "CANCELLED");
|
|
888
|
+
}
|
|
889
|
+
return record;
|
|
890
|
+
}
|
|
891
|
+
async function recoverIncomplete() {
|
|
892
|
+
const rows = database.prepare("SELECT * FROM executions").all();
|
|
893
|
+
for (const row of rows) {
|
|
894
|
+
let record = fromRow(row);
|
|
895
|
+
if (record.status !== "PENDING" &&
|
|
896
|
+
record.status !== "RUNNING" &&
|
|
897
|
+
record.status !== "UNKNOWN")
|
|
898
|
+
continue;
|
|
899
|
+
if (record.status === "PENDING" &&
|
|
900
|
+
record.sideEffectState === "NOT_STARTED") {
|
|
901
|
+
await log(record, "RECOVERY_PENDING_REQUIRES_REDECISION");
|
|
902
|
+
emitObserverSignal(record, "RECOVERY_RESUME");
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
if (record.status !== "UNKNOWN" && record.sideEffectState !== "STARTED") {
|
|
906
|
+
record = failRecord(record, new ExecutionRuntimeError("CANCELLED", "runtime restarted before effect start"));
|
|
907
|
+
save(record);
|
|
908
|
+
await log(record, "RECOVERY_RECONCILED", record.error?.code);
|
|
909
|
+
emitObserverSignal(record, "RECOVERY_RESUME");
|
|
910
|
+
continue;
|
|
911
|
+
}
|
|
912
|
+
const request = parseExecuteCapabilityRequest(JSON.parse(String(row.request_json)));
|
|
913
|
+
const executor = browserIds.has(request.capability)
|
|
914
|
+
? options.browserExecutor
|
|
915
|
+
: options.localExecutor;
|
|
916
|
+
if (!executor || !row.precondition_json) {
|
|
917
|
+
record = failRecord(record, new ExecutionRuntimeError("UNKNOWN_SIDE_EFFECT", "restart recovery lacks verifier"), "UNKNOWN");
|
|
918
|
+
save(record);
|
|
919
|
+
await log(record, "RECOVERY_RECONCILED", record.error?.code);
|
|
920
|
+
emitObserverSignal(record, "UNKNOWN_REALITY");
|
|
921
|
+
continue;
|
|
922
|
+
}
|
|
923
|
+
try {
|
|
924
|
+
const reconciled = await executor.reconcile(request, JSON.parse(String(row.precondition_json)));
|
|
925
|
+
if (reconciled.state === "NOT_APPLIED")
|
|
926
|
+
record = failRecord(record, new ExecutionRuntimeError("EXECUTION_FAILED", "recovery verified no effect"));
|
|
927
|
+
else if (reconciled.state === "APPLIED" && reconciled.result) {
|
|
928
|
+
const { error: _previousError, ...recoverable } = record;
|
|
929
|
+
record = parseExecutionRecord({
|
|
930
|
+
...recoverable,
|
|
931
|
+
status: "SUCCEEDED",
|
|
932
|
+
sideEffectState: "APPLIED",
|
|
933
|
+
retryable: false,
|
|
934
|
+
result: reconciled.result,
|
|
935
|
+
evidence: reconciled.evidence,
|
|
936
|
+
evidenceRefs: reconciled.evidence.map((item) => item.evidenceRef),
|
|
937
|
+
finishedAt: now().toISOString(),
|
|
938
|
+
updatedAt: now().toISOString(),
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
else
|
|
942
|
+
record = failRecord(record, new ExecutionRuntimeError("UNKNOWN_SIDE_EFFECT", "restart reconciliation unresolved"), "UNKNOWN");
|
|
943
|
+
}
|
|
944
|
+
catch {
|
|
945
|
+
record = failRecord(record, new ExecutionRuntimeError("UNKNOWN_SIDE_EFFECT", "restart verifier failed"), "UNKNOWN");
|
|
946
|
+
}
|
|
947
|
+
save(record);
|
|
948
|
+
await log(record, "RECOVERY_RECONCILED", record.error?.code);
|
|
949
|
+
emitObserverSignal(record, record.status === "UNKNOWN" ? "UNKNOWN_REALITY" : "RECOVERY_RESUME");
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
await recoverIncomplete();
|
|
953
|
+
return Object.freeze({
|
|
954
|
+
executeCapability,
|
|
955
|
+
lookupExecutionIntent,
|
|
956
|
+
listExecutionObserverSignals,
|
|
957
|
+
acknowledgeExecutionObserverSignal,
|
|
958
|
+
getExecution,
|
|
959
|
+
getExecutionForCaller,
|
|
960
|
+
readExecutionOutput,
|
|
961
|
+
readExecutionOutputForCaller,
|
|
962
|
+
cancelExecution,
|
|
963
|
+
requestExecutionApproval: approvalOwner.requestApproval,
|
|
964
|
+
decideExecutionApproval: approvalOwner.decideApproval,
|
|
965
|
+
revokeExecutionApproval: approvalOwner.revokeApproval,
|
|
966
|
+
getExecutionApproval: approvalOwner.getApproval,
|
|
967
|
+
listExecutionApprovals: approvalOwner.listApprovals,
|
|
968
|
+
registerArtifact,
|
|
969
|
+
getArtifactRecord,
|
|
970
|
+
getArtifactSummary,
|
|
971
|
+
recoverIncomplete,
|
|
972
|
+
databasePath: options.databasePath,
|
|
973
|
+
logPath,
|
|
974
|
+
close() {
|
|
975
|
+
closed = true;
|
|
976
|
+
for (const controller of controllers.values())
|
|
977
|
+
controller.abort();
|
|
978
|
+
approvalOwner.close();
|
|
979
|
+
database.close();
|
|
980
|
+
},
|
|
981
|
+
});
|
|
982
|
+
}
|
|
983
|
+
export { createExecutionApprovalOwner, ExecutionApprovalOwnerError, } from "./approval-owner.js";
|