@zackbart/connecta 0.25.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CHANGELOG.md +387 -0
- package/README.md +32 -15
- package/SECURITY.md +6 -5
- package/bin/connecta.mjs +5 -1
- package/dist/activity.d.ts +10 -3
- package/dist/activity.js +10 -1
- package/dist/artifacts/connector.d.ts +55 -0
- package/dist/artifacts/connector.js +724 -0
- package/dist/artifacts/document.js +80 -0
- package/dist/artifacts/guide.js +140 -0
- package/dist/artifacts/html-scan.js +329 -0
- package/dist/artifacts/json.js +68 -0
- package/dist/artifacts/kv-store.d.ts +20 -0
- package/dist/artifacts/kv-store.js +140 -0
- package/dist/artifacts/markdown.js +478 -0
- package/dist/artifacts/module.d.ts +37 -0
- package/dist/artifacts/module.js +60 -0
- package/dist/artifacts/operations.d.ts +195 -0
- package/dist/artifacts/operations.js +944 -0
- package/dist/artifacts/refresh.d.ts +53 -0
- package/dist/artifacts/refresh.js +235 -0
- package/dist/artifacts/routes.js +217 -0
- package/dist/artifacts/types.d.ts +214 -0
- package/dist/artifacts/types.js +22 -0
- package/dist/artifacts/validate.d.ts +62 -0
- package/dist/artifacts/validate.js +653 -0
- package/dist/artifacts.d.ts +6 -0
- package/dist/artifacts.js +9 -0
- package/dist/auth/downstream-oauth.js +397 -48
- package/dist/auth/oauth-start-reset.js +20 -0
- package/dist/branding.d.ts +87 -0
- package/dist/branding.js +170 -1
- package/dist/catalog-service.js +39 -29
- package/dist/catalog.js +42 -8
- package/dist/connector-access.d.ts +18 -4
- package/dist/connector-access.js +65 -20
- package/dist/connector-caller.d.ts +8 -0
- package/dist/connector-caller.js +19 -0
- package/dist/connectors/remote-mcp.js +48 -9
- package/dist/errors.d.ts +16 -1
- package/dist/errors.js +28 -0
- package/dist/execute.js +431 -75
- package/dist/executor-admission.d.ts +11 -1
- package/dist/executor-admission.js +79 -14
- package/dist/executors/quickjs.js +4 -2
- package/dist/index.d.ts +62 -6
- package/dist/index.js +230 -6
- package/dist/invocation.js +56 -10
- package/dist/meta-tools.js +7 -2
- package/dist/module-contracts.d.ts +21 -1
- package/dist/oauth-callback-outcome.js +43 -18
- package/dist/operator-ui/generated.js +2 -2
- package/dist/operator-ui/model.d.ts +62 -5
- package/dist/operator-ui/view.js +372 -28
- package/dist/page-styles.js +7 -0
- package/dist/program-source.js +116 -0
- package/dist/registry.d.ts +42 -6
- package/dist/registry.js +164 -60
- package/dist/resumable.js +1291 -0
- package/dist/routes/activity.js +6 -69
- package/dist/routes/actor-labels.js +114 -0
- package/dist/routes/credentials.js +27 -5
- package/dist/routes/mcp.js +120 -59
- package/dist/routes/oauth-management.js +87 -20
- package/dist/routes/oauth.js +54 -87
- package/dist/routes/operator.js +1 -0
- package/dist/routes/shared.d.ts +23 -1
- package/dist/routes/shared.js +20 -2
- package/dist/routes/ui-data.js +2 -1
- package/dist/routes/ui.js +19 -4
- package/dist/run-journal.js +347 -0
- package/dist/runtime/shared-read.js +106 -0
- package/dist/server.js +30 -2
- package/dist/skills.js +30 -11
- package/dist/timeout.js +26 -0
- package/dist/tool-safety.d.ts +35 -0
- package/dist/tool-safety.js +31 -0
- package/dist/types.d.ts +23 -0
- package/dist/ui.d.ts +17 -5
- package/dist/ui.js +51 -68
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/documentation/architecture.md +214 -27
- package/documentation/auth.md +186 -2
- package/documentation/code-mode.md +236 -63
- package/documentation/meta-tools.md +55 -29
- package/ethos.md +12 -10
- package/examples/worker/AGENTS.md +6 -0
- package/examples/worker/README.md +174 -10
- package/examples/worker/src/d1-activity-row.ts +14 -2
- package/examples/worker/src/d1-activity.ts +4 -2
- package/examples/worker/src/index.ts +36 -1
- package/examples/worker/src/r2-artifact-blobs.ts +29 -0
- package/examples/worker/wrangler.jsonc +5 -1
- package/package.json +6 -2
- package/templates/node/AGENTS.md +9 -2
- package/templates/node/README.md +31 -3
- package/templates/node/package.json +1 -1
- package/templates/node/src/index.ts +27 -0
|
@@ -0,0 +1,1291 @@
|
|
|
1
|
+
// Resumable writes: a program pauses host-side at its first unapproved write
|
|
2
|
+
// and resumes by replay through `resume_execution` (#565).
|
|
3
|
+
//
|
|
4
|
+
// Three pieces live here. `RunState` is one play of one run — the host side of
|
|
5
|
+
// the sandbox's host calls, numbering them, answering replayed ones from the
|
|
6
|
+
// journal, gating writes, and stopping the run when one needs a human.
|
|
7
|
+
// `ReplayState` is the journal a resumed play answers from. And the
|
|
8
|
+
// `resume_execution` handler claims a paused run and plays it again.
|
|
9
|
+
//
|
|
10
|
+
// The guarantees, and where each is kept:
|
|
11
|
+
//
|
|
12
|
+
// - A write needing approval is never sent by `execute_code`. The gate sits in
|
|
13
|
+
// the shared invocation path after validation and before admission, and
|
|
14
|
+
// answers `pause` for it (`gate`).
|
|
15
|
+
// - An approved write is sent at most once, across restarts and racing
|
|
16
|
+
// resumes. Only the holder of the run's claim sends, and it marks each live
|
|
17
|
+
// write `sending` in the header by compare-and-set before dispatch
|
|
18
|
+
// (`writeAhead`). A second resume loses the claim; a takeover after a crash
|
|
19
|
+
// finds the `sending` mark and reports the outcome unknown rather than
|
|
20
|
+
// replaying past it.
|
|
21
|
+
// - A write whose outcome is unknown is never sent again: the run fails there
|
|
22
|
+
// and a failed run is never replayed (`settleWrite`). A failure outranks a
|
|
23
|
+
// pause that arrived first (`stopWith`), and no play ends paused or
|
|
24
|
+
// completed over a write that is `unknown` or `sending`.
|
|
25
|
+
// - Replay only ever picks up at a pause. A play that sent writes and then
|
|
26
|
+
// ended any other way — the executor failed, the claim lapsed — fails
|
|
27
|
+
// `execution_interrupted`, because what it read after those writes was never
|
|
28
|
+
// journaled and a replay would read it afresh (`finishExecutorFailure`, the
|
|
29
|
+
// takeover in `resumeExecution`).
|
|
30
|
+
// - Replay reaches nothing downstream. Recorded calls are answered from the
|
|
31
|
+
// journal before the invocation path, so they load no catalog, take no
|
|
32
|
+
// permit, call no connector, and record no activity (`lookup`).
|
|
33
|
+
// - Replay never mixes reads across runs: a journal expires at a fixed time
|
|
34
|
+
// from its first pause, and a program that stops matching its journal fails
|
|
35
|
+
// typed (`execution_diverged`).
|
|
36
|
+
import { Deferred, Effect } from "effect";
|
|
37
|
+
import { z } from "zod";
|
|
38
|
+
import { advertisedSchema } from "./advertised-schema.js";
|
|
39
|
+
import { echoedCallArgs, framingError } from "./errors.js";
|
|
40
|
+
import { InvocationFailure, } from "./invocation.js";
|
|
41
|
+
import { jsonResult } from "./meta-tools.js";
|
|
42
|
+
import { splitAddress } from "./registry.js";
|
|
43
|
+
import { runEdge } from "./runtime/run.js";
|
|
44
|
+
import { underAnySignal } from "./timeout.js";
|
|
45
|
+
import { isApprovalExempt, NO_EXEMPTIONS, } from "./tool-safety.js";
|
|
46
|
+
import { canonicalJson, classifyWriteOutcome, formatToken, MAX_FINAL_CHARS, MAX_JOURNAL_BYTES, hasProtoKey, MAX_PENDING_ARGS_BYTES, parseToken, randomId, RunJournal, sha256Hex, utf8Bytes, } from "./run-journal.js";
|
|
47
|
+
/**
|
|
48
|
+
* The floor under the deadline on each paused-run storage call. The deadline
|
|
49
|
+
* is the host-call deadline or this, whichever is longer: a write's
|
|
50
|
+
* reservation and its settling cannot be interrupted, so without one a
|
|
51
|
+
* storage call that never answers would hold the play — and the drain behind
|
|
52
|
+
* it — forever.
|
|
53
|
+
*/
|
|
54
|
+
export const MIN_STORAGE_TIMEOUT_MS = 5_000;
|
|
55
|
+
export const DEFAULT_MAX_WRITES = 10;
|
|
56
|
+
export const DEFAULT_PAUSED_RUN_TTL_SECONDS = 1_800;
|
|
57
|
+
/**
|
|
58
|
+
* The journal a resumed play answers from.
|
|
59
|
+
*
|
|
60
|
+
* Calls sharing a key answer in the order they were first issued, so two
|
|
61
|
+
* identical reads around a write replay as the two different answers they
|
|
62
|
+
* got. Until the approved pending call is re-issued, every call must find a
|
|
63
|
+
* record: the pause was decided only after every earlier-numbered call had
|
|
64
|
+
* been decided and journaled, so a call with no record before that point is a
|
|
65
|
+
* program that took another path.
|
|
66
|
+
*/
|
|
67
|
+
class ReplayState {
|
|
68
|
+
pendingKey;
|
|
69
|
+
queues = new Map();
|
|
70
|
+
writes = new Set();
|
|
71
|
+
pendingReissued = false;
|
|
72
|
+
constructor(entries, pendingKey) {
|
|
73
|
+
this.pendingKey = pendingKey;
|
|
74
|
+
for (const entry of [...entries].sort((a, b) => a.seq - b.seq)) {
|
|
75
|
+
const queue = this.queues.get(entry.key) ?? [];
|
|
76
|
+
queue.push(entry);
|
|
77
|
+
this.queues.set(entry.key, queue);
|
|
78
|
+
if (entry.write)
|
|
79
|
+
this.writes.add(entry);
|
|
80
|
+
}
|
|
81
|
+
// A run with no pending call (none survives a completed pause) has
|
|
82
|
+
// nothing to wait for.
|
|
83
|
+
if (pendingKey === undefined)
|
|
84
|
+
this.pendingReissued = true;
|
|
85
|
+
}
|
|
86
|
+
lookup(key) {
|
|
87
|
+
const entry = this.queues.get(key)?.shift();
|
|
88
|
+
if (entry) {
|
|
89
|
+
// Never the pending call, even when its key matches: the pending call
|
|
90
|
+
// is not journaled, so a hit is an earlier identical call.
|
|
91
|
+
this.writes.delete(entry);
|
|
92
|
+
return { kind: "hit", entry };
|
|
93
|
+
}
|
|
94
|
+
if (!this.pendingReissued && key === this.pendingKey) {
|
|
95
|
+
this.pendingReissued = true;
|
|
96
|
+
return { kind: "live" };
|
|
97
|
+
}
|
|
98
|
+
if (!this.pendingReissued) {
|
|
99
|
+
return {
|
|
100
|
+
kind: "diverged",
|
|
101
|
+
reason: "it made a call its journal has no record of before repeating the approved write",
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
return { kind: "live" };
|
|
105
|
+
}
|
|
106
|
+
/** Divergence (b) and (c), checked when the program settles. */
|
|
107
|
+
unfinished() {
|
|
108
|
+
if (!this.pendingReissued) {
|
|
109
|
+
return "it finished without repeating the approved write";
|
|
110
|
+
}
|
|
111
|
+
if (this.writes.size > 0) {
|
|
112
|
+
return "it finished without repeating a write it had already sent";
|
|
113
|
+
}
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
function failure(code, message, extra = {}) {
|
|
118
|
+
return { code, message, retryable: false, ...extra };
|
|
119
|
+
}
|
|
120
|
+
function guestFailure(code, message) {
|
|
121
|
+
return new InvocationFailure({ code, message, retryable: false });
|
|
122
|
+
}
|
|
123
|
+
const RERUN = {
|
|
124
|
+
tool: "execute_code",
|
|
125
|
+
purpose: "Run the task again from the start; nothing from this run will be replayed.",
|
|
126
|
+
};
|
|
127
|
+
/** RERUN, for a run whose writes may have landed. */
|
|
128
|
+
const RERUN_AFTER_CHECK = {
|
|
129
|
+
tool: "execute_code",
|
|
130
|
+
purpose: "Check what the writes counted in writes did first: running the task again from the start sends them again.",
|
|
131
|
+
};
|
|
132
|
+
/**
|
|
133
|
+
* A replay that stopped matching its journal. It stops where it noticed; what
|
|
134
|
+
* it had already sent is counted in `writes`, and nothing is replayed again.
|
|
135
|
+
*/
|
|
136
|
+
function diverged(reason) {
|
|
137
|
+
return failure("execution_diverged", `The resumed program diverged from its journal: ${reason}. The run stopped there and will not be replayed; any writes it sent are counted in writes.`, { nextAction: RERUN });
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* The run stopped where replay cannot pick it up: storage failed under it, or
|
|
141
|
+
* a play that had already sent writes ended without pausing. Not retryable —
|
|
142
|
+
* a re-run repeats whatever the run already sent, which `writes` counts.
|
|
143
|
+
*/
|
|
144
|
+
function interrupted(reason) {
|
|
145
|
+
return failure("execution_interrupted", `${reason} The run stopped there and cannot be resumed; nothing further was sent. Writes it already sent are counted in writes and are not undone, so check them before running the task again.`, { nextAction: RERUN });
|
|
146
|
+
}
|
|
147
|
+
function storageFailure() {
|
|
148
|
+
return interrupted("Paused-run storage failed.");
|
|
149
|
+
}
|
|
150
|
+
/** A write whose outcome nobody knows, found when the play ends. */
|
|
151
|
+
function unknownOutcome() {
|
|
152
|
+
return failure("write_outcome_unknown", "A write this run sent has no known outcome, so the run stopped and will not be replayed; that write will not be sent again. Check its target before doing anything that depends on it.");
|
|
153
|
+
}
|
|
154
|
+
function countWrites(writes) {
|
|
155
|
+
return {
|
|
156
|
+
succeeded: writes.filter((write) => write.state === "ok").length,
|
|
157
|
+
failed: writes.filter((write) => write.state === "failed").length,
|
|
158
|
+
unknown: writes.filter((write) => write.state === "unknown" || write.state === "sending").length,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* A failure with the run's write counts. "Run it again" is only plain advice
|
|
163
|
+
* when nothing landed: once a write succeeded, or may have, the advice says
|
|
164
|
+
* to check first, because a re-run sends it again.
|
|
165
|
+
*/
|
|
166
|
+
function counted(runFailure, writes) {
|
|
167
|
+
const landed = writes.succeeded + writes.unknown > 0;
|
|
168
|
+
return landed && runFailure.nextAction === RERUN
|
|
169
|
+
? { ...runFailure, nextAction: RERUN_AFTER_CHECK, writes }
|
|
170
|
+
: { ...runFailure, writes };
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* One play of one run: the host-side state behind a program's host calls.
|
|
174
|
+
*
|
|
175
|
+
* A fresh play (`execute_code`) starts with an empty journal and no header;
|
|
176
|
+
* it writes both only if it pauses. A replay (`resume_execution`) starts from
|
|
177
|
+
* a claimed header and its entries, answers recorded calls from them, and
|
|
178
|
+
* sends live only what comes after.
|
|
179
|
+
*/
|
|
180
|
+
export class RunState {
|
|
181
|
+
journal;
|
|
182
|
+
program;
|
|
183
|
+
environment;
|
|
184
|
+
source;
|
|
185
|
+
settings;
|
|
186
|
+
claim;
|
|
187
|
+
replay;
|
|
188
|
+
/** Completed when the play first stops early: a pause, or a typed failure. */
|
|
189
|
+
stopped = Deferred.makeUnsafe();
|
|
190
|
+
/**
|
|
191
|
+
* Why the play stopped. A failure replaces a pause — a write that turned
|
|
192
|
+
* unknown beside a concurrent pause must not be buried under it, or the
|
|
193
|
+
* pause would persist and a replay could send that write again.
|
|
194
|
+
*/
|
|
195
|
+
stop;
|
|
196
|
+
/** Set once the program settled: a write gated after this is not sent. */
|
|
197
|
+
closed = false;
|
|
198
|
+
seq = 0;
|
|
199
|
+
decisions = [];
|
|
200
|
+
writesSpent = 0;
|
|
201
|
+
/** Live calls this play completed that the journal does not hold yet. */
|
|
202
|
+
unpersisted = [];
|
|
203
|
+
/** Live writes this play dispatched: seq → reserved entry slot. */
|
|
204
|
+
liveWrites = new Map();
|
|
205
|
+
/** Writes between approval and dispatch, so a drain can wait for them. */
|
|
206
|
+
gating = new Set();
|
|
207
|
+
/** Writes a fresh play sent, for the header of a later pause. */
|
|
208
|
+
freshWrites = [];
|
|
209
|
+
/** Calls a gate refused because the play had stopped. Not journaled. */
|
|
210
|
+
unrecorded = new Set();
|
|
211
|
+
inFlight = new Map();
|
|
212
|
+
headerTurn = Promise.resolve();
|
|
213
|
+
constructor(journal, program, environment, source, settings, claim, replay) {
|
|
214
|
+
this.journal = journal;
|
|
215
|
+
this.program = program;
|
|
216
|
+
this.environment = environment;
|
|
217
|
+
this.source = source;
|
|
218
|
+
this.settings = settings;
|
|
219
|
+
this.claim = claim;
|
|
220
|
+
this.replay = replay;
|
|
221
|
+
}
|
|
222
|
+
/** A fresh run. Its journal exists only once it pauses. */
|
|
223
|
+
static fresh(storage, program, environment, settings, storageTimeoutMs) {
|
|
224
|
+
return new RunState(new RunJournal(storage, randomId(), storageTimeoutMs), program, environment, "execute_code", settings, undefined, undefined);
|
|
225
|
+
}
|
|
226
|
+
/** A claimed replay of a paused run. */
|
|
227
|
+
static replaying(options) {
|
|
228
|
+
const { header } = options.claim;
|
|
229
|
+
return new RunState(options.journal, options.program, { clockMs: header.clock, seed: header.seed }, "resume_execution", options.settings, options.claim, new ReplayState(options.entries, header.pending?.key));
|
|
230
|
+
}
|
|
231
|
+
// --- numbering and the gate sequencer ---------------------------------
|
|
232
|
+
/**
|
|
233
|
+
* Number a host call. Called synchronously when the program makes it, so
|
|
234
|
+
* the numbering is the program's own issue order.
|
|
235
|
+
*/
|
|
236
|
+
begin() {
|
|
237
|
+
const seq = this.seq++;
|
|
238
|
+
this.decisions[seq] = Deferred.makeUnsafe();
|
|
239
|
+
return seq;
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Record that call `seq` has made its gate decision: dispatched, refused,
|
|
243
|
+
* answered from the journal, or ended. Idempotent.
|
|
244
|
+
*/
|
|
245
|
+
decide(seq) {
|
|
246
|
+
const decision = this.decisions[seq];
|
|
247
|
+
if (decision && !Deferred.isDoneUnsafe(decision)) {
|
|
248
|
+
Deferred.doneUnsafe(decision, Effect.void);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Every lower-numbered call's decision. A write decides only after these,
|
|
253
|
+
* so of two writes issued together the first always pauses first, however
|
|
254
|
+
* their catalogs happen to resolve — which is what makes the pause point
|
|
255
|
+
* reproducible on replay.
|
|
256
|
+
*/
|
|
257
|
+
decidedBefore(seq) {
|
|
258
|
+
const waits = this.decisions.slice(0, seq).filter((decision) => !Deferred.isDoneUnsafe(decision));
|
|
259
|
+
return waits.length === 0
|
|
260
|
+
? Effect.void
|
|
261
|
+
: Effect.forEach(waits, (decision) => Deferred.await(decision), {
|
|
262
|
+
discard: true,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
/** Track a call until it settles, so a stop can wait for what is on the wire. */
|
|
266
|
+
track(seq, settled) {
|
|
267
|
+
this.inFlight.set(seq, settled);
|
|
268
|
+
void settled.then(() => this.inFlight.delete(seq), () => this.inFlight.delete(seq));
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* The program settled. A write that reaches the gate from now on is not
|
|
272
|
+
* sent: nobody is left to see its answer, and a mark it left behind would
|
|
273
|
+
* claim a write the run never made.
|
|
274
|
+
*/
|
|
275
|
+
close() {
|
|
276
|
+
this.closed = true;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Wait for in-flight calls: every one after a stop, since each will be
|
|
280
|
+
* journaled; only writes otherwise — past approval, or already on the wire
|
|
281
|
+
* — since abandoning a read costs nothing but abandoning a write turns its
|
|
282
|
+
* outcome unknown. Each is bounded by its own host-call deadline.
|
|
283
|
+
*/
|
|
284
|
+
drain(scope) {
|
|
285
|
+
return Effect.promise(async () => {
|
|
286
|
+
for (;;) {
|
|
287
|
+
const waiting = [...this.inFlight.entries()]
|
|
288
|
+
.filter(([seq]) => scope === "all" || this.liveWrites.has(seq) || this.gating.has(seq))
|
|
289
|
+
.map(([, settled]) => settled);
|
|
290
|
+
if (waiting.length === 0)
|
|
291
|
+
return;
|
|
292
|
+
await Promise.allSettled(waiting);
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
// --- stopping ----------------------------------------------------------
|
|
297
|
+
/** The failure a call made after the play stopped receives, if it has. */
|
|
298
|
+
halted() {
|
|
299
|
+
const stop = this.stop;
|
|
300
|
+
if (!stop)
|
|
301
|
+
return undefined;
|
|
302
|
+
return stop.kind === "paused"
|
|
303
|
+
? guestFailure("execution_paused", `The run paused at ${stop.pending.address} to wait for approval, so no further call is made. This program's result will be discarded.`)
|
|
304
|
+
: guestFailure(stop.failure.code, stop.failure.message);
|
|
305
|
+
}
|
|
306
|
+
stopWith(stop) {
|
|
307
|
+
if (this.stop && !(this.stop.kind === "paused" && stop.kind === "failed")) {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
const first = this.stop === undefined;
|
|
311
|
+
this.stop = stop;
|
|
312
|
+
if (first)
|
|
313
|
+
Deferred.doneUnsafe(this.stopped, Effect.succeed(stop));
|
|
314
|
+
}
|
|
315
|
+
/** Divergence (a): stop, and hand the calling program the typed failure. */
|
|
316
|
+
diverge(reason) {
|
|
317
|
+
this.stopWith({
|
|
318
|
+
kind: "failed",
|
|
319
|
+
failure: diverged(reason),
|
|
320
|
+
});
|
|
321
|
+
return this.halted() ?? guestFailure("execution_diverged", reason);
|
|
322
|
+
}
|
|
323
|
+
// --- replay ------------------------------------------------------------
|
|
324
|
+
/** Answer a call from the journal, or say it goes live. */
|
|
325
|
+
lookup(key) {
|
|
326
|
+
if (!this.replay)
|
|
327
|
+
return { kind: "live" };
|
|
328
|
+
const found = this.replay.lookup(key);
|
|
329
|
+
// Replayed writes spend the write budget again, as replayed calls spend
|
|
330
|
+
// the host-call budget: per-play totals equal per-run totals.
|
|
331
|
+
if (found.kind === "hit" && found.entry.write)
|
|
332
|
+
this.writesSpent++;
|
|
333
|
+
return found;
|
|
334
|
+
}
|
|
335
|
+
/** Journal a completed live call that was not a write. */
|
|
336
|
+
record(seq, op, key, outcome) {
|
|
337
|
+
if (this.unrecorded.has(seq) || this.liveWrites.has(seq))
|
|
338
|
+
return;
|
|
339
|
+
this.unpersisted.push({ seq, op, key, outcome });
|
|
340
|
+
}
|
|
341
|
+
// --- the write gate ----------------------------------------------------
|
|
342
|
+
/** Whether an approval on file covers this write, and which one. */
|
|
343
|
+
approvalFor(address, args) {
|
|
344
|
+
const approvals = this.claim?.header.approvals ?? [];
|
|
345
|
+
const toolIndex = approvals.findIndex((approval) => approval.address === address && approval.scope === "tool");
|
|
346
|
+
if (toolIndex >= 0) {
|
|
347
|
+
return { approval: approvals[toolIndex], index: toolIndex };
|
|
348
|
+
}
|
|
349
|
+
const canonical = canonicalJson(args);
|
|
350
|
+
const callIndex = approvals.findIndex((approval) => approval.address === address &&
|
|
351
|
+
approval.scope === "call" &&
|
|
352
|
+
!approval.consumed &&
|
|
353
|
+
approval.argsCanonical === canonical);
|
|
354
|
+
return callIndex >= 0
|
|
355
|
+
? { approval: approvals[callIndex], index: callIndex }
|
|
356
|
+
: undefined;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Decide one consequential call. Runs inside the invocation path, after
|
|
360
|
+
* the call resolved and validated and before admission.
|
|
361
|
+
*/
|
|
362
|
+
gate(seq, key, target, args) {
|
|
363
|
+
const decided = Effect.sync(() => this.decide(seq));
|
|
364
|
+
return Effect.gen({ self: this }, function* () {
|
|
365
|
+
yield* this.decidedBefore(seq);
|
|
366
|
+
const halted = this.halted();
|
|
367
|
+
if (halted) {
|
|
368
|
+
this.unrecorded.add(seq);
|
|
369
|
+
return { kind: "refuse", error: halted.details, activity: "none" };
|
|
370
|
+
}
|
|
371
|
+
if (this.closed) {
|
|
372
|
+
this.unrecorded.add(seq);
|
|
373
|
+
return {
|
|
374
|
+
kind: "refuse",
|
|
375
|
+
error: {
|
|
376
|
+
code: "cancelled",
|
|
377
|
+
message: "The program had already returned, so this write was not sent.",
|
|
378
|
+
retryable: false,
|
|
379
|
+
},
|
|
380
|
+
activity: "none",
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
const address = `${target.connector.id}.${target.toolName}`;
|
|
384
|
+
// A config exemption (#566) dispatches like an approval would, with
|
|
385
|
+
// everything an approved write gets — the write budget, the
|
|
386
|
+
// write-ahead mark, the journal — except the pause.
|
|
387
|
+
const exempt = isApprovalExempt(this.settings.approval ?? NO_EXEMPTIONS, target.connector, target.toolName, target.definition);
|
|
388
|
+
if (!exempt &&
|
|
389
|
+
(args === null || typeof args !== "object" || Array.isArray(args))) {
|
|
390
|
+
// Neither approval route takes arguments that are not an object, so
|
|
391
|
+
// there is nothing a pause could ask a human to repeat.
|
|
392
|
+
return {
|
|
393
|
+
kind: "refuse",
|
|
394
|
+
error: framingError("destructive_tool_requires_approval", `Tool "${address}" is not explicitly read-only, and its arguments are not an object, so it cannot pause for approval.`),
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
if (this.writesSpent >= this.settings.maxWrites) {
|
|
398
|
+
return {
|
|
399
|
+
kind: "refuse",
|
|
400
|
+
error: {
|
|
401
|
+
code: "budget_exceeded",
|
|
402
|
+
message: `execute_code write budget exceeded (${this.settings.maxWrites} writes maximum, execute.maxWrites); ${address} was not sent`,
|
|
403
|
+
retryable: false,
|
|
404
|
+
},
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
const approved = exempt ? { index: -1 } : this.approvalFor(address, args);
|
|
408
|
+
if (approved)
|
|
409
|
+
return yield* this.reserve(seq, address, approved.index);
|
|
410
|
+
// Not approved: this is where the run pauses — unless the write cannot
|
|
411
|
+
// be shown to a human faithfully. A `__proto__` key survives JSON but
|
|
412
|
+
// not every object a host or schema layer builds from it, so the write
|
|
413
|
+
// a human approved could differ from the one sent.
|
|
414
|
+
if (hasProtoKey(args)) {
|
|
415
|
+
return {
|
|
416
|
+
kind: "refuse",
|
|
417
|
+
error: {
|
|
418
|
+
code: "invalid_args",
|
|
419
|
+
message: `The arguments for ${address} contain a "__proto__" key, which cannot be shown and repeated faithfully for approval, so the write was not sent.`,
|
|
420
|
+
retryable: false,
|
|
421
|
+
},
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
// A write too large to hold for a human stops the run instead, before
|
|
425
|
+
// anything is kept.
|
|
426
|
+
let argsText;
|
|
427
|
+
try {
|
|
428
|
+
argsText = canonicalJson(args);
|
|
429
|
+
}
|
|
430
|
+
catch {
|
|
431
|
+
argsText = "";
|
|
432
|
+
}
|
|
433
|
+
if (!argsText || utf8Bytes(argsText) > MAX_PENDING_ARGS_BYTES) {
|
|
434
|
+
const tooLarge = failure("pending_write_too_large", `The write to ${address} has arguments over ${MAX_PENDING_ARGS_BYTES} bytes, too large to hold for approval. Nothing was sent. Call it through call_destructive_tool instead.`, {
|
|
435
|
+
nextAction: {
|
|
436
|
+
tool: "call_destructive_tool",
|
|
437
|
+
arguments: { address },
|
|
438
|
+
purpose: "Ask the MCP host to approve this one call directly.",
|
|
439
|
+
},
|
|
440
|
+
});
|
|
441
|
+
this.stopWith({ kind: "failed", failure: tooLarge });
|
|
442
|
+
this.unrecorded.add(seq);
|
|
443
|
+
return {
|
|
444
|
+
kind: "refuse",
|
|
445
|
+
error: { code: tooLarge.code, message: tooLarge.message, retryable: false },
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
this.stopWith({ kind: "paused", pending: { address, args, key } });
|
|
449
|
+
this.unrecorded.add(seq);
|
|
450
|
+
return {
|
|
451
|
+
kind: "refuse",
|
|
452
|
+
error: this.halted().details,
|
|
453
|
+
activity: "paused",
|
|
454
|
+
};
|
|
455
|
+
}).pipe(Effect.ensuring(decided));
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Reserve an approved write: the write-ahead mark, then the bookkeeping
|
|
459
|
+
* that makes it a live write. Uninterruptible as one, so a host-call
|
|
460
|
+
* deadline cannot land between a `sending` mark and the record that the
|
|
461
|
+
* write is this play's — which would leave a reserved journal slot nothing
|
|
462
|
+
* ever fills. A deadline that fires here takes effect just after, before
|
|
463
|
+
* dispatch, and the write settles as `failed`: marked, never sent. Every
|
|
464
|
+
* storage call has a deadline of its own (`RunJournal`), so the wait here —
|
|
465
|
+
* and the drain behind it — is bounded; a mark that times out is a lost
|
|
466
|
+
* one, and the write is not sent.
|
|
467
|
+
*/
|
|
468
|
+
reserve(seq, address, approvalIndex) {
|
|
469
|
+
this.gating.add(seq);
|
|
470
|
+
return Effect.uninterruptible(this.writeAhead(address, approvalIndex).pipe(Effect.map((reserved) => {
|
|
471
|
+
this.gating.delete(seq);
|
|
472
|
+
if (reserved.kind === "lost") {
|
|
473
|
+
this.stopWith({ kind: "failed", failure: reserved.failure });
|
|
474
|
+
this.unrecorded.add(seq);
|
|
475
|
+
return {
|
|
476
|
+
kind: "refuse",
|
|
477
|
+
error: {
|
|
478
|
+
code: reserved.failure.code,
|
|
479
|
+
message: reserved.failure.message,
|
|
480
|
+
retryable: false,
|
|
481
|
+
},
|
|
482
|
+
activity: "none",
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
this.writesSpent++;
|
|
486
|
+
this.liveWrites.set(seq, reserved.slot);
|
|
487
|
+
return { kind: "dispatch" };
|
|
488
|
+
})));
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* Before a claimed play sends a write: reserve its journal slot, mark it
|
|
492
|
+
* `sending`, and spend a call-scoped approval, in one compare-and-set on
|
|
493
|
+
* the header. If the claim moved, nothing is sent. A fresh play holds no
|
|
494
|
+
* header, so there is nothing to mark — and no journal that could replay
|
|
495
|
+
* past the write.
|
|
496
|
+
*/
|
|
497
|
+
writeAhead(address, approvalIndex) {
|
|
498
|
+
if (!this.claim)
|
|
499
|
+
return Effect.succeed({ kind: "reserved", slot: undefined });
|
|
500
|
+
let slot = -1;
|
|
501
|
+
return this.mutateHeader((header) => {
|
|
502
|
+
slot = header.entries;
|
|
503
|
+
const approvals = header.approvals.map((approval, index) => index === approvalIndex && approval.scope === "call"
|
|
504
|
+
? { ...approval, consumed: true }
|
|
505
|
+
: approval);
|
|
506
|
+
return {
|
|
507
|
+
...header,
|
|
508
|
+
approvals,
|
|
509
|
+
entries: header.entries + 1,
|
|
510
|
+
writes: [...header.writes, { entry: slot, address, state: "sending" }],
|
|
511
|
+
};
|
|
512
|
+
}).pipe(Effect.map((result) => result === "ok"
|
|
513
|
+
? { kind: "reserved", slot }
|
|
514
|
+
: {
|
|
515
|
+
kind: "lost",
|
|
516
|
+
failure: result === "lost"
|
|
517
|
+
? failure("execution_claim_lost", `Another resume_execution took this run over, so this one stopped before sending ${address}.`)
|
|
518
|
+
: storageFailure(),
|
|
519
|
+
}));
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* After a live write answered, or failed to: journal it, and stop the run
|
|
523
|
+
* if nobody can say whether it landed. Returns the failure the program
|
|
524
|
+
* sees in place of the call's own when the run stops here.
|
|
525
|
+
*/
|
|
526
|
+
settleWrite(seq, key, target, outcome) {
|
|
527
|
+
return Effect.gen({ self: this }, function* () {
|
|
528
|
+
if (!this.liveWrites.has(seq))
|
|
529
|
+
return undefined;
|
|
530
|
+
const state = writeStateOf(outcome);
|
|
531
|
+
const entry = {
|
|
532
|
+
seq,
|
|
533
|
+
op: "call",
|
|
534
|
+
key,
|
|
535
|
+
write: true,
|
|
536
|
+
outcome: outcome.ok
|
|
537
|
+
? { ok: true, value: outcome.value }
|
|
538
|
+
: { ok: false, error: outcome.error },
|
|
539
|
+
};
|
|
540
|
+
const slot = this.liveWrites.get(seq);
|
|
541
|
+
const unknownFailure = state === "unknown"
|
|
542
|
+
? failure("write_outcome_unknown", `The write to ${target.address} was sent, but no answer came back, so whether it happened is unknown. It will not be sent again. Check its target before doing anything that depends on it.`, {
|
|
543
|
+
address: target.address,
|
|
544
|
+
...echoedCallArgs(target.args),
|
|
545
|
+
})
|
|
546
|
+
: undefined;
|
|
547
|
+
if (this.claim && slot !== undefined) {
|
|
548
|
+
const claim = this.claim;
|
|
549
|
+
const written = yield* Effect.tryPromise(() => this.journal.writeEntry(slot, entry, claim.header.expiresAt)).pipe(Effect.as(true), Effect.catch(() => Effect.succeed(false)));
|
|
550
|
+
const recorded = written ? state : "unknown";
|
|
551
|
+
const settled = yield* this.mutateHeader((header) => ({
|
|
552
|
+
...header,
|
|
553
|
+
bytes: header.bytes + utf8Bytes(RunJournal.entryText(entry)),
|
|
554
|
+
writes: header.writes.map((write) => write.entry === slot ? { ...write, state: recorded } : write),
|
|
555
|
+
}));
|
|
556
|
+
if (settled !== "ok") {
|
|
557
|
+
// The write happened, or may have; only the record of it is lost.
|
|
558
|
+
// Say which, from what this play knows, rather than "nothing sent".
|
|
559
|
+
this.noteLocally(slot, recorded);
|
|
560
|
+
const lost = settled === "lost"
|
|
561
|
+
? failure("execution_claim_lost", `The write to ${target.address} was sent (${describeState(recorded)}), but another resume_execution took the run over before it could be recorded, so this play stopped there.`)
|
|
562
|
+
: interrupted(`The write to ${target.address} was sent (${describeState(recorded)}), but paused-run storage failed before it could be recorded.`);
|
|
563
|
+
this.stopWith({ kind: "failed", failure: lost });
|
|
564
|
+
return guestFailure(lost.code, lost.message);
|
|
565
|
+
}
|
|
566
|
+
if (!written && !unknownFailure) {
|
|
567
|
+
// The write's outcome is known here but could not be journaled, so
|
|
568
|
+
// no later replay could answer it: stop rather than lose it.
|
|
569
|
+
const lost = interrupted(`The write to ${target.address} was sent (${describeState(state)}), but its result could not be journaled.`);
|
|
570
|
+
this.stopWith({ kind: "failed", failure: lost });
|
|
571
|
+
return guestFailure(lost.code, lost.message);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
else {
|
|
575
|
+
this.unpersisted.push(entry);
|
|
576
|
+
this.freshWrites.push({
|
|
577
|
+
entry: -1,
|
|
578
|
+
address: target.address,
|
|
579
|
+
state: state,
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
if (unknownFailure) {
|
|
583
|
+
this.stopWith({ kind: "failed", failure: unknownFailure });
|
|
584
|
+
return guestFailure(unknownFailure.code, unknownFailure.message);
|
|
585
|
+
}
|
|
586
|
+
return undefined;
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
/** Whether `seq` was dispatched as a live write. */
|
|
590
|
+
isLiveWrite(seq) {
|
|
591
|
+
return this.liveWrites.has(seq);
|
|
592
|
+
}
|
|
593
|
+
// --- the header ----------------------------------------------------------
|
|
594
|
+
/**
|
|
595
|
+
* Compare-and-set the claimed header, one change at a time. A write that
|
|
596
|
+
* finds the header moved has lost the claim: someone else holds the run.
|
|
597
|
+
*/
|
|
598
|
+
mutateHeader(update) {
|
|
599
|
+
return Effect.promise(() => {
|
|
600
|
+
const turn = this.headerTurn.then(async () => {
|
|
601
|
+
const claim = this.claim;
|
|
602
|
+
if (!claim)
|
|
603
|
+
return "lost";
|
|
604
|
+
const next = update(structuredClone(claim.header));
|
|
605
|
+
next.version = claim.header.version + 1;
|
|
606
|
+
let raw;
|
|
607
|
+
try {
|
|
608
|
+
raw = await this.journal.casHeader(claim.raw, next);
|
|
609
|
+
}
|
|
610
|
+
catch {
|
|
611
|
+
return "storage";
|
|
612
|
+
}
|
|
613
|
+
if (raw === undefined)
|
|
614
|
+
return "lost";
|
|
615
|
+
this.claim = { ...claim, raw, header: next };
|
|
616
|
+
return "ok";
|
|
617
|
+
});
|
|
618
|
+
this.headerTurn = turn.catch(() => { });
|
|
619
|
+
return turn;
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* Record a write's state in this play's copy of the header only, after the
|
|
624
|
+
* claim was lost: nothing can be stored any more, but the counts this play
|
|
625
|
+
* reports should still say what it knows.
|
|
626
|
+
*/
|
|
627
|
+
noteLocally(slot, state) {
|
|
628
|
+
const claim = this.claim;
|
|
629
|
+
if (!claim)
|
|
630
|
+
return;
|
|
631
|
+
this.claim = {
|
|
632
|
+
...claim,
|
|
633
|
+
header: {
|
|
634
|
+
...claim.header,
|
|
635
|
+
writes: claim.header.writes.map((write) => write.entry === slot ? { ...write, state } : write),
|
|
636
|
+
},
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* A fresh play's writes, each pointing at its entry. They were journaled
|
|
641
|
+
* in completion order with everything else, in the same order they were
|
|
642
|
+
* settled, so the n-th write entry is the n-th write.
|
|
643
|
+
*/
|
|
644
|
+
freshWriteSlots() {
|
|
645
|
+
const slots = [];
|
|
646
|
+
this.unpersisted.forEach((entry, slot) => {
|
|
647
|
+
const write = this.freshWrites[slots.length];
|
|
648
|
+
if (entry.write && write)
|
|
649
|
+
slots.push({ ...write, entry: slot });
|
|
650
|
+
});
|
|
651
|
+
return slots;
|
|
652
|
+
}
|
|
653
|
+
writeCounts() {
|
|
654
|
+
return countWrites(this.claim?.header.writes ?? this.freshWrites);
|
|
655
|
+
}
|
|
656
|
+
/** Whether this run has sent any write, in this play or an earlier one. */
|
|
657
|
+
sentAny() {
|
|
658
|
+
const counts = this.writeCounts();
|
|
659
|
+
return counts.succeeded + counts.failed + counts.unknown > 0;
|
|
660
|
+
}
|
|
661
|
+
// --- how a play ends -----------------------------------------------------
|
|
662
|
+
/**
|
|
663
|
+
* The play stopped: persist the pause, or the failure, and say so. Called
|
|
664
|
+
* after in-flight calls drained, so everything the program started is in
|
|
665
|
+
* `unpersisted` or already journaled. The stop consulted is the current
|
|
666
|
+
* one — a failure that arrived during the drain has replaced a pause — and
|
|
667
|
+
* a pause is never persisted over a write whose outcome is unknown.
|
|
668
|
+
*/
|
|
669
|
+
finishStopped() {
|
|
670
|
+
const stop = this.stop;
|
|
671
|
+
if (!stop)
|
|
672
|
+
return Effect.die(new Error("finishStopped without a stop"));
|
|
673
|
+
if (stop.kind === "failed")
|
|
674
|
+
return this.finishFailed(stop.failure);
|
|
675
|
+
if (this.writeCounts().unknown > 0)
|
|
676
|
+
return this.finishFailed(unknownOutcome());
|
|
677
|
+
return this.persistPause(stop.pending);
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* The program settled on its own. Its own result is only the answer when
|
|
681
|
+
* nothing else happened: a stop that arrived while writes drained (a
|
|
682
|
+
* concurrent pause, or an unawaited write that turned unknown) wins, as does
|
|
683
|
+
* any write left without a known outcome.
|
|
684
|
+
*/
|
|
685
|
+
finishSettled(result) {
|
|
686
|
+
return Effect.gen({ self: this }, function* () {
|
|
687
|
+
if (this.stop)
|
|
688
|
+
return yield* this.finishStopped();
|
|
689
|
+
const unfinished = this.replay?.unfinished();
|
|
690
|
+
if (unfinished)
|
|
691
|
+
return yield* this.finishFailed(diverged(unfinished));
|
|
692
|
+
if (this.writeCounts().unknown > 0) {
|
|
693
|
+
return yield* this.finishFailed(unknownOutcome());
|
|
694
|
+
}
|
|
695
|
+
// A program that failed after sending writes: its error, with the
|
|
696
|
+
// counts, so nobody mistakes the run for one safe to repeat.
|
|
697
|
+
const reported = result.isError && this.sentAny()
|
|
698
|
+
? withWrites(result, this.writeCounts())
|
|
699
|
+
: result;
|
|
700
|
+
if (!this.claim)
|
|
701
|
+
return reported;
|
|
702
|
+
const text = reported.content[0]?.type === "text" ? reported.content[0].text ?? "" : "";
|
|
703
|
+
const final = {
|
|
704
|
+
isError: reported.isError === true,
|
|
705
|
+
text: reported.content.length > 1
|
|
706
|
+
? JSON.stringify({
|
|
707
|
+
completed: true,
|
|
708
|
+
note: "The run completed and its emitted content blocks were delivered once; they were not kept for a repeated resume_execution.",
|
|
709
|
+
})
|
|
710
|
+
: text.length > MAX_FINAL_CHARS
|
|
711
|
+
? JSON.stringify({
|
|
712
|
+
completed: true,
|
|
713
|
+
note: "The run completed and its result was returned once; it was too large to keep for a repeated resume_execution.",
|
|
714
|
+
})
|
|
715
|
+
: text,
|
|
716
|
+
};
|
|
717
|
+
// An erroring program ends the run `failed`: it is not replayed, and
|
|
718
|
+
// its answer — error and counts — is what a repeated resume returns.
|
|
719
|
+
const moved = yield* this.mutateHeader((header) => {
|
|
720
|
+
const { claim: _released, ...rest } = header;
|
|
721
|
+
return {
|
|
722
|
+
...rest,
|
|
723
|
+
state: reported.isError ? "failed" : "completed",
|
|
724
|
+
final,
|
|
725
|
+
};
|
|
726
|
+
});
|
|
727
|
+
if (moved !== "ok") {
|
|
728
|
+
// A timed-out CAS may have committed before storage reported an
|
|
729
|
+
// error. If its final answer is readable, completion is durable and
|
|
730
|
+
// the same resume can return it again. Otherwise never report a
|
|
731
|
+
// successful completion that the header cannot repeat.
|
|
732
|
+
const read = yield* Effect.tryPromise(() => this.journal.readHeader()).pipe(Effect.catch(() => Effect.succeed(undefined)));
|
|
733
|
+
if (read?.header.final?.text === final.text &&
|
|
734
|
+
read.header.final.isError === final.isError &&
|
|
735
|
+
read.header.state === (reported.isError ? "failed" : "completed"))
|
|
736
|
+
return reported;
|
|
737
|
+
if (moved === "lost") {
|
|
738
|
+
return errorEnvelope(counted(failure("execution_claim_lost", "Another resume_execution took this run over before its final answer could be recorded."), this.writeCounts()));
|
|
739
|
+
}
|
|
740
|
+
return yield* this.finishFailed(interrupted("The program finished, but paused-run storage failed before its final answer could be recorded."));
|
|
741
|
+
}
|
|
742
|
+
return reported;
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* The executor itself failed — admission, cancellation, the watchdog — not
|
|
747
|
+
* the program. If this play sent no write, the run goes back to the same
|
|
748
|
+
* pause, its claim released and this attempt's unused approval withdrawn,
|
|
749
|
+
* and the resume can be tried again: nothing it read mattered. If it sent a
|
|
750
|
+
* write, it fails: what it read between that write and the failure was
|
|
751
|
+
* never journaled, so a replay would read afresh and could build a later
|
|
752
|
+
* write from different data than the earlier one.
|
|
753
|
+
*/
|
|
754
|
+
finishExecutorFailure(result) {
|
|
755
|
+
return Effect.gen({ self: this }, function* () {
|
|
756
|
+
if (!this.claim) {
|
|
757
|
+
return this.sentAny() ? withWrites(result, this.writeCounts()) : result;
|
|
758
|
+
}
|
|
759
|
+
if (this.writeCounts().unknown > 0) {
|
|
760
|
+
return yield* this.finishFailed(unknownOutcome());
|
|
761
|
+
}
|
|
762
|
+
if (this.liveWrites.size > 0) {
|
|
763
|
+
return yield* this.finishFailed(interrupted(`The sandbox failed partway through a resumed run that had already sent writes (${errorText(result)}).`));
|
|
764
|
+
}
|
|
765
|
+
const nonce = this.claim.header.nonce;
|
|
766
|
+
yield* this.mutateHeader((header) => {
|
|
767
|
+
const { claim: _released, ...rest } = header;
|
|
768
|
+
return {
|
|
769
|
+
...rest,
|
|
770
|
+
state: "paused",
|
|
771
|
+
// Withdraw this attempt's approval of the current pause, unless it
|
|
772
|
+
// was a call approval already spent: the retry must not be handed a
|
|
773
|
+
// fresh one.
|
|
774
|
+
approvals: header.approvals.filter((approval) => approval.nonce !== nonce ||
|
|
775
|
+
(approval.scope === "call" && approval.consumed === true)),
|
|
776
|
+
};
|
|
777
|
+
});
|
|
778
|
+
return result;
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
finishFailed(runFailure) {
|
|
782
|
+
return Effect.gen({ self: this }, function* () {
|
|
783
|
+
const withCounts = this.claim || this.sentAny()
|
|
784
|
+
? counted(runFailure, this.writeCounts())
|
|
785
|
+
: runFailure;
|
|
786
|
+
const result = errorEnvelope(withCounts);
|
|
787
|
+
if (this.claim) {
|
|
788
|
+
const text = result.content[0]?.text ?? "";
|
|
789
|
+
yield* this.mutateHeader((header) => {
|
|
790
|
+
const { claim: _released, ...rest } = header;
|
|
791
|
+
return {
|
|
792
|
+
...rest,
|
|
793
|
+
state: "failed",
|
|
794
|
+
final: { isError: true, text },
|
|
795
|
+
};
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
return result;
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
/** Write the journal and the header, then hand back the pending result. */
|
|
802
|
+
persistPause(pending) {
|
|
803
|
+
return Effect.gen({ self: this }, function* () {
|
|
804
|
+
const nonce = randomId();
|
|
805
|
+
const now = Date.now();
|
|
806
|
+
const claim = this.claim;
|
|
807
|
+
const expiresAt = claim?.header.expiresAt ??
|
|
808
|
+
now + this.settings.ttlSeconds * 1_000;
|
|
809
|
+
if (now >= expiresAt) {
|
|
810
|
+
// Only a claimed play can outlive the deadline it replays under. A
|
|
811
|
+
// pause now would be born expired, and "run it again" is the wrong
|
|
812
|
+
// advice for a run that has sent writes.
|
|
813
|
+
return yield* this.finishFailed(failure("execution_expired", "The run reached another write after its paused-run lifetime ended, so it cannot pause again. Writes it already sent are counted in writes and are not undone; check them before running the rest of the task again.", { nextAction: RERUN }));
|
|
814
|
+
}
|
|
815
|
+
const newBytes = this.unpersisted.reduce((sum, entry) => sum + utf8Bytes(RunJournal.entryText(entry)), 0);
|
|
816
|
+
const bytes = (claim ? claim.header.bytes : utf8Bytes(this.program)) + newBytes;
|
|
817
|
+
if (bytes > MAX_JOURNAL_BYTES) {
|
|
818
|
+
return yield* this.finishFailed(failure("journal_too_large", `This run recorded more than ${MAX_JOURNAL_BYTES} bytes of source and host calls, too much to hold while it waits for approval, so it stopped here without pausing. ${this.sentAny()
|
|
819
|
+
? "Writes it already sent are counted in writes and are not undone; check them before running the task again."
|
|
820
|
+
: "Nothing was sent."} Read less before the first write, or split the work.`, { nextAction: RERUN }));
|
|
821
|
+
}
|
|
822
|
+
const firstSlot = claim?.header.entries ?? 0;
|
|
823
|
+
const written = yield* Effect.tryPromise(async () => {
|
|
824
|
+
if (!claim)
|
|
825
|
+
await this.journal.writeSource(this.program, expiresAt);
|
|
826
|
+
for (let index = 0; index < this.unpersisted.length; index++) {
|
|
827
|
+
const entry = this.unpersisted[index];
|
|
828
|
+
if (entry)
|
|
829
|
+
await this.journal.writeEntry(firstSlot + index, entry, expiresAt);
|
|
830
|
+
}
|
|
831
|
+
}).pipe(Effect.as(true), Effect.catch(() => Effect.succeed(false)));
|
|
832
|
+
if (!written)
|
|
833
|
+
return yield* this.finishFailed(storageFailure());
|
|
834
|
+
const entries = firstSlot + this.unpersisted.length;
|
|
835
|
+
if (claim) {
|
|
836
|
+
const moved = yield* this.mutateHeader((header) => {
|
|
837
|
+
const { claim: _released, ...rest } = header;
|
|
838
|
+
return {
|
|
839
|
+
...rest,
|
|
840
|
+
state: "paused",
|
|
841
|
+
nonce,
|
|
842
|
+
pending,
|
|
843
|
+
entries,
|
|
844
|
+
playFrom: entries,
|
|
845
|
+
bytes,
|
|
846
|
+
};
|
|
847
|
+
});
|
|
848
|
+
if (moved !== "ok") {
|
|
849
|
+
return errorEnvelope(counted(moved === "lost"
|
|
850
|
+
? failure("execution_claim_lost", "Another resume_execution took this run over before its next pause could be recorded, so this play stopped there.")
|
|
851
|
+
: storageFailure(), this.writeCounts()));
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
else {
|
|
855
|
+
const header = {
|
|
856
|
+
v: 1,
|
|
857
|
+
state: "paused",
|
|
858
|
+
version: 1,
|
|
859
|
+
nonce,
|
|
860
|
+
pool: this.settings.pool,
|
|
861
|
+
createdAt: now,
|
|
862
|
+
expiresAt,
|
|
863
|
+
clock: this.environment.clockMs,
|
|
864
|
+
seed: [...this.environment.seed],
|
|
865
|
+
programHash: yield* Effect.promise(() => sha256Hex(this.program)),
|
|
866
|
+
pending,
|
|
867
|
+
approvals: [],
|
|
868
|
+
writes: this.freshWriteSlots(),
|
|
869
|
+
entries,
|
|
870
|
+
playFrom: entries,
|
|
871
|
+
bytes,
|
|
872
|
+
};
|
|
873
|
+
const stored = yield* Effect.tryPromise(() => this.journal.casHeader(null, header)).pipe(Effect.catch(() => Effect.succeed(undefined)));
|
|
874
|
+
if (stored === undefined)
|
|
875
|
+
return yield* this.finishFailed(storageFailure());
|
|
876
|
+
}
|
|
877
|
+
this.unpersisted.length = 0;
|
|
878
|
+
return pausedResult(formatToken({ runId: this.journal.runId, nonce, expiresAt }), pending, expiresAt);
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
/** What a dispatched write's outcome says about whether it landed. */
|
|
883
|
+
export function writeStateOf(outcome) {
|
|
884
|
+
return classifyWriteOutcome(outcome.ok
|
|
885
|
+
? { ok: true, dispatched: outcome.dispatched }
|
|
886
|
+
: {
|
|
887
|
+
ok: false,
|
|
888
|
+
dispatched: outcome.dispatched,
|
|
889
|
+
error: outcome.error,
|
|
890
|
+
...(outcome.answered !== undefined ? { answered: outcome.answered } : {}),
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
/**
|
|
894
|
+
* The writes a program sends under a config exemption (#566) when nothing
|
|
895
|
+
* can pause: resumable writes are off, so there is no run state, but an
|
|
896
|
+
* exempt write is still a write. The play closes this when the program
|
|
897
|
+
* settles — a write that reaches the gate after is not sent — and drains
|
|
898
|
+
* what is on the wire before the run's scope aborts it, so the write
|
|
899
|
+
* finishes and its activity says how it did. As with a resumable run, the
|
|
900
|
+
* result never hides a write whose outcome is unknown, and a program that
|
|
901
|
+
* fails after writing reports the counts.
|
|
902
|
+
*/
|
|
903
|
+
export class ExemptWrites {
|
|
904
|
+
closed = false;
|
|
905
|
+
inFlight = new Set();
|
|
906
|
+
states = [];
|
|
907
|
+
/** Whether the program has settled: a write gated now is not sent. */
|
|
908
|
+
get isClosed() {
|
|
909
|
+
return this.closed;
|
|
910
|
+
}
|
|
911
|
+
close() {
|
|
912
|
+
this.closed = true;
|
|
913
|
+
}
|
|
914
|
+
/** A write is being dispatched; the function returned records how it ended. */
|
|
915
|
+
begin() {
|
|
916
|
+
let done = () => { };
|
|
917
|
+
const settled = new Promise((resolve) => {
|
|
918
|
+
done = resolve;
|
|
919
|
+
});
|
|
920
|
+
this.inFlight.add(settled);
|
|
921
|
+
let recorded = false;
|
|
922
|
+
return (state) => {
|
|
923
|
+
if (recorded)
|
|
924
|
+
return;
|
|
925
|
+
recorded = true;
|
|
926
|
+
this.states.push(state);
|
|
927
|
+
this.inFlight.delete(settled);
|
|
928
|
+
done();
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
/** Wait for dispatched writes, each bounded by its host-call deadline. */
|
|
932
|
+
drain() {
|
|
933
|
+
return Effect.promise(async () => {
|
|
934
|
+
while (this.inFlight.size > 0)
|
|
935
|
+
await Promise.all(this.inFlight);
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
finish(result) {
|
|
939
|
+
if (this.states.length === 0)
|
|
940
|
+
return result;
|
|
941
|
+
const writes = countWrites(this.states.map((state) => ({ state })));
|
|
942
|
+
if (writes.unknown > 0) {
|
|
943
|
+
return errorEnvelope({
|
|
944
|
+
code: "write_outcome_unknown",
|
|
945
|
+
message: "A write this program sent has no known outcome, so that is the result rather than what the program returned. It will not be sent again. Check its target before doing anything that depends on it.",
|
|
946
|
+
retryable: false,
|
|
947
|
+
writes,
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
return result.isError ? withWrites(result, writes) : result;
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
function describeState(state) {
|
|
954
|
+
return state === "ok"
|
|
955
|
+
? "it succeeded"
|
|
956
|
+
: state === "failed"
|
|
957
|
+
? "it failed"
|
|
958
|
+
: "its outcome is unknown";
|
|
959
|
+
}
|
|
960
|
+
/** The first part of an error result's text, for a message that wraps it. */
|
|
961
|
+
function errorText(result) {
|
|
962
|
+
const structured = result.structuredContent?.error;
|
|
963
|
+
const text = typeof structured?.message === "string"
|
|
964
|
+
? structured.message
|
|
965
|
+
: result.content[0]?.text ?? "";
|
|
966
|
+
return text.length > 300 ? `${text.slice(0, 300)}…` : text;
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* An error result from a run that sent writes, carrying their counts: a
|
|
970
|
+
* program that fails after writing is not a run that is safe to repeat.
|
|
971
|
+
*/
|
|
972
|
+
function withWrites(result, writes) {
|
|
973
|
+
const structured = result.structuredContent;
|
|
974
|
+
const error = structured?.error;
|
|
975
|
+
const annotated = errorEnvelope({
|
|
976
|
+
...(error !== null && typeof error === "object"
|
|
977
|
+
? error
|
|
978
|
+
: {
|
|
979
|
+
code: "executor_failed",
|
|
980
|
+
message: result.content[0]?.text ?? "",
|
|
981
|
+
retryable: false,
|
|
982
|
+
}),
|
|
983
|
+
writes,
|
|
984
|
+
note: "This run attempted writes, counted in writes; those that succeeded are not undone. Check them before running the task again.",
|
|
985
|
+
});
|
|
986
|
+
if (structured && error !== null && typeof error === "object") {
|
|
987
|
+
const { error: _replaced, ...rest } = structured;
|
|
988
|
+
annotated.structuredContent = { ...rest, error: annotated.structuredContent?.error };
|
|
989
|
+
annotated.content = [{ type: "text", text: JSON.stringify(annotated.structuredContent) }];
|
|
990
|
+
}
|
|
991
|
+
return annotated;
|
|
992
|
+
}
|
|
993
|
+
/** The `execute_code` / `resume_execution` result for a paused run. */
|
|
994
|
+
function pausedResult(token, pending, expiresAt) {
|
|
995
|
+
return jsonResult({
|
|
996
|
+
paused: {
|
|
997
|
+
address: pending.address,
|
|
998
|
+
args: pending.args,
|
|
999
|
+
token,
|
|
1000
|
+
expiresAt: new Date(expiresAt).toISOString(),
|
|
1001
|
+
nextAction: {
|
|
1002
|
+
tool: "resume_execution",
|
|
1003
|
+
arguments: {
|
|
1004
|
+
token,
|
|
1005
|
+
address: pending.address,
|
|
1006
|
+
args: pending.args,
|
|
1007
|
+
approval: "call",
|
|
1008
|
+
},
|
|
1009
|
+
},
|
|
1010
|
+
hint: 'Not sent. To run it, call resume_execution repeating this address and args exactly; approval "tool" also covers later calls to this tool for the rest of the run. Every read before this point is replayed from the journal, not repeated. If resume_execution is denied, that is the human\'s answer: report it rather than sending the write another way.',
|
|
1011
|
+
},
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
function errorEnvelope(error) {
|
|
1015
|
+
const result = jsonResult({ error });
|
|
1016
|
+
result.isError = true;
|
|
1017
|
+
return result;
|
|
1018
|
+
}
|
|
1019
|
+
function resumeError(code, message, extra = {}, retryable = false) {
|
|
1020
|
+
return errorEnvelope({ code, message, retryable, ...extra });
|
|
1021
|
+
}
|
|
1022
|
+
function resumeAction(header, runId) {
|
|
1023
|
+
const pending = header.pending;
|
|
1024
|
+
if (header.state !== "paused" || !pending)
|
|
1025
|
+
return undefined;
|
|
1026
|
+
return {
|
|
1027
|
+
tool: "resume_execution",
|
|
1028
|
+
arguments: {
|
|
1029
|
+
token: formatToken({ runId, nonce: header.nonce, expiresAt: header.expiresAt }),
|
|
1030
|
+
address: pending.address,
|
|
1031
|
+
args: pending.args,
|
|
1032
|
+
approval: "call",
|
|
1033
|
+
},
|
|
1034
|
+
};
|
|
1035
|
+
}
|
|
1036
|
+
/**
|
|
1037
|
+
* The `resume_execution` handler: check the token and the exact repetition,
|
|
1038
|
+
* claim the run, and play it again.
|
|
1039
|
+
*
|
|
1040
|
+
* Everything that can refuse — an unknown or expired token, a stale one, a
|
|
1041
|
+
* mismatch, a live claim — refuses before the claim, so a refusal spends
|
|
1042
|
+
* nothing and changes nothing. The claim is one compare-and-set from `paused`
|
|
1043
|
+
* (or `running` under a lapsed claim) to `running`; of two resumes racing for
|
|
1044
|
+
* it, exactly one wins. A run that can no longer be replayed is failed in
|
|
1045
|
+
* that same compare-and-set, with its answer, so a repeat gets it too.
|
|
1046
|
+
*/
|
|
1047
|
+
export function resumeExecution(args, ctx) {
|
|
1048
|
+
return Effect.gen(function* () {
|
|
1049
|
+
const { settings, storage } = ctx;
|
|
1050
|
+
if (!settings || !storage.compareAndSet) {
|
|
1051
|
+
return resumeError("resumable_writes_unavailable", "This deployment does not pause programs at writes, so there is nothing to resume. Send a write through call_destructive_tool.");
|
|
1052
|
+
}
|
|
1053
|
+
const token = parseToken(args.token);
|
|
1054
|
+
const notFound = resumeError("execution_not_found", "No paused run matches this token. Run the task again with execute_code.", { nextAction: RERUN });
|
|
1055
|
+
const unavailable = resumeError("unavailable", "Paused-run storage could not be read or written. Nothing was approved or sent. Retry shortly.", {}, true);
|
|
1056
|
+
const claimedFirst = resumeError("execution_in_progress", "Another resume_execution claimed this run first. Retry after it finishes to see its result.", {}, true);
|
|
1057
|
+
if (!token)
|
|
1058
|
+
return notFound;
|
|
1059
|
+
const journal = new RunJournal(storage, token.runId, ctx.storageTimeoutMs);
|
|
1060
|
+
const read = yield* Effect.tryPromise(() => journal.readHeader()).pipe(Effect.catch(() => Effect.succeed(null)));
|
|
1061
|
+
if (read === null)
|
|
1062
|
+
return unavailable;
|
|
1063
|
+
const now = Date.now();
|
|
1064
|
+
if (!read) {
|
|
1065
|
+
return now >= token.expiresAt
|
|
1066
|
+
? resumeError("execution_expired", "This paused run expired. Its reads are stale, so it will not be replayed: run the task again with execute_code.", { nextAction: RERUN })
|
|
1067
|
+
: notFound;
|
|
1068
|
+
}
|
|
1069
|
+
const { header } = read;
|
|
1070
|
+
if (header.pool !== settings.pool)
|
|
1071
|
+
return notFound;
|
|
1072
|
+
if (token.nonce !== header.nonce) {
|
|
1073
|
+
const next = now < header.expiresAt ? resumeAction(header, token.runId) : undefined;
|
|
1074
|
+
return resumeError("execution_token_stale", "This token names an earlier pause of the run, which has moved on.", next ? { nextAction: next } : {});
|
|
1075
|
+
}
|
|
1076
|
+
const pending = header.pending;
|
|
1077
|
+
if (!pending ||
|
|
1078
|
+
args.address !== pending.address ||
|
|
1079
|
+
args.argsHadProtoKey === true ||
|
|
1080
|
+
hasProtoKey(args.args) ||
|
|
1081
|
+
canonicalJson(args.args) !== canonicalJson(pending.args)) {
|
|
1082
|
+
return resumeError("approval_mismatch", "resume_execution must repeat the paused write's address and args exactly. Nothing was approved or sent.", pending ? { nextAction: resumeAction(header, token.runId) } : {});
|
|
1083
|
+
}
|
|
1084
|
+
if (header.state === "completed" || header.state === "failed") {
|
|
1085
|
+
// The same resume, repeated after the run ended: the same answer —
|
|
1086
|
+
// before the expiry check, because "expired, run it again" is the
|
|
1087
|
+
// wrong answer for a run that sent writes and then ended.
|
|
1088
|
+
const final = header.final;
|
|
1089
|
+
if (!final)
|
|
1090
|
+
return notFound;
|
|
1091
|
+
let parsed;
|
|
1092
|
+
try {
|
|
1093
|
+
parsed = JSON.parse(final.text);
|
|
1094
|
+
}
|
|
1095
|
+
catch {
|
|
1096
|
+
parsed = undefined;
|
|
1097
|
+
}
|
|
1098
|
+
const repeated = jsonResult(parsed, final.text);
|
|
1099
|
+
if (final.isError)
|
|
1100
|
+
repeated.isError = true;
|
|
1101
|
+
return repeated;
|
|
1102
|
+
}
|
|
1103
|
+
if (header.state === "running" && (header.claim?.until ?? 0) > now) {
|
|
1104
|
+
// Someone is playing the run right now — past its deadline or not, a
|
|
1105
|
+
// live claim may still be sending, and only its result is the answer.
|
|
1106
|
+
return resumeError("execution_in_progress", "Another resume_execution is playing this run right now. Retry after it finishes to see its result.", {}, true);
|
|
1107
|
+
}
|
|
1108
|
+
// What the run can no longer be replayed past. A write marked `sending`
|
|
1109
|
+
// may or may not have landed. A write sent by a play that ended without
|
|
1110
|
+
// pausing — its claim lapsed after a crash or restart — was followed by
|
|
1111
|
+
// reads nobody journaled, so a replay would read afresh and could build a
|
|
1112
|
+
// later write from different data. A paused header should hold neither,
|
|
1113
|
+
// but a replay past either could send a write twice, so both refuse —
|
|
1114
|
+
// before the expiry check, because the answer is the writes' counts, not
|
|
1115
|
+
// "expired, run it again".
|
|
1116
|
+
const unsettled = header.writes.some((write) => write.state === "sending" || write.state === "unknown");
|
|
1117
|
+
const unjournaled = header.state === "running" &&
|
|
1118
|
+
header.writes.some((write) => write.entry >= header.playFrom);
|
|
1119
|
+
if (unsettled || unjournaled) {
|
|
1120
|
+
const writes = header.writes.map((write) => write.state === "sending" ? { ...write, state: "unknown" } : write);
|
|
1121
|
+
return yield* failRun(journal, read.raw, header, writes, unsettled
|
|
1122
|
+
? failure("write_outcome_unknown", "An earlier play of this run left a write whose outcome is unknown, so the run will not be replayed and that write will not be sent again. Check its target before doing anything that depends on it.")
|
|
1123
|
+
: interrupted("An earlier resume of this run stopped partway after sending writes, and what it read between them was not kept."), { unavailable, claimedFirst });
|
|
1124
|
+
}
|
|
1125
|
+
if (now >= header.expiresAt) {
|
|
1126
|
+
if (header.writes.length === 0) {
|
|
1127
|
+
return resumeError("execution_expired", "This paused run expired. Its reads are stale, so it will not be replayed: run the task again with execute_code.", { nextAction: RERUN });
|
|
1128
|
+
}
|
|
1129
|
+
// The header outlives the run as a tombstone for exactly this answer.
|
|
1130
|
+
return errorEnvelope(counted(failure("execution_expired", "This paused run expired, so it will not be replayed. Writes it already sent are counted in writes and are not undone; check them before running the task again.", { nextAction: RERUN }), countWrites(header.writes)));
|
|
1131
|
+
}
|
|
1132
|
+
// Read the journal before claiming, so a storage hiccup refuses and
|
|
1133
|
+
// changes nothing, and a journal that is gone fails the run in the same
|
|
1134
|
+
// compare-and-set that would have claimed it.
|
|
1135
|
+
const loaded = yield* Effect.tryPromise(async () => ({
|
|
1136
|
+
program: await journal.readSource(header.programHash),
|
|
1137
|
+
entries: await journal.readEntries(header.entries),
|
|
1138
|
+
})).pipe(Effect.catch(() => Effect.succeed(undefined)));
|
|
1139
|
+
if (!loaded)
|
|
1140
|
+
return unavailable;
|
|
1141
|
+
if (!loaded.program || !loaded.entries) {
|
|
1142
|
+
// Missing is unrecoverable: a partial journal cannot be replayed.
|
|
1143
|
+
return yield* failRun(journal, read.raw, header, header.writes, interrupted("This run's journal could not be read back in full."), { unavailable, claimedFirst });
|
|
1144
|
+
}
|
|
1145
|
+
const scope = args.approval ?? "call";
|
|
1146
|
+
const claimId = randomId();
|
|
1147
|
+
// One approval per pause: this resume's. An approval of this pause still
|
|
1148
|
+
// on file was given to an attempt that sent nothing — one that sent a
|
|
1149
|
+
// write failed above — so this resume's scope replaces it rather than
|
|
1150
|
+
// being ignored. A spent one is kept, and then none is added: approving
|
|
1151
|
+
// the same call twice must not leave a second, unspent approval behind
|
|
1152
|
+
// for an identical later call to use.
|
|
1153
|
+
const kept = header.approvals.filter((approval) => approval.nonce !== header.nonce ||
|
|
1154
|
+
(approval.scope === "call" && approval.consumed === true));
|
|
1155
|
+
const approvedNow = !kept.some((approval) => approval.nonce === header.nonce);
|
|
1156
|
+
const claimed = {
|
|
1157
|
+
...header,
|
|
1158
|
+
state: "running",
|
|
1159
|
+
version: header.version + 1,
|
|
1160
|
+
claim: { id: claimId, until: now + ctx.claimMs },
|
|
1161
|
+
approvals: approvedNow
|
|
1162
|
+
? [...kept, approvalOf(pending, scope, header.nonce)]
|
|
1163
|
+
: kept,
|
|
1164
|
+
};
|
|
1165
|
+
const raw = yield* Effect.tryPromise(() => journal.casHeader(read.raw, claimed)).pipe(Effect.catch(() => Effect.succeed(null)));
|
|
1166
|
+
if (raw === null)
|
|
1167
|
+
return unavailable;
|
|
1168
|
+
if (raw === undefined)
|
|
1169
|
+
return claimedFirst;
|
|
1170
|
+
if (approvedNow)
|
|
1171
|
+
recordApproval(ctx.activity, pending.address, scope);
|
|
1172
|
+
return yield* ctx.run(RunState.replaying({
|
|
1173
|
+
journal,
|
|
1174
|
+
program: loaded.program,
|
|
1175
|
+
claim: { id: claimId, raw, header: claimed },
|
|
1176
|
+
entries: loaded.entries,
|
|
1177
|
+
settings,
|
|
1178
|
+
}));
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1181
|
+
/**
|
|
1182
|
+
* End a run that can no longer be replayed, in one compare-and-set that also
|
|
1183
|
+
* stores its answer: a repeated resume gets the same one, and no window is
|
|
1184
|
+
* left in which the run is `failed` with nothing to say.
|
|
1185
|
+
*/
|
|
1186
|
+
function failRun(journal, expected, header, writes, runFailure, refusals) {
|
|
1187
|
+
return Effect.gen(function* () {
|
|
1188
|
+
const result = errorEnvelope(counted(runFailure, countWrites(writes)));
|
|
1189
|
+
const { claim: _lapsed, ...rest } = header;
|
|
1190
|
+
const stored = yield* Effect.tryPromise(() => journal.casHeader(expected, {
|
|
1191
|
+
...rest,
|
|
1192
|
+
state: "failed",
|
|
1193
|
+
version: header.version + 1,
|
|
1194
|
+
writes,
|
|
1195
|
+
final: { isError: true, text: result.content[0]?.text ?? "" },
|
|
1196
|
+
})).pipe(Effect.catch(() => Effect.succeed(null)));
|
|
1197
|
+
if (stored === null)
|
|
1198
|
+
return refusals.unavailable;
|
|
1199
|
+
if (stored === undefined)
|
|
1200
|
+
return refusals.claimedFirst;
|
|
1201
|
+
return result;
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
function approvalOf(pending, scope, nonce) {
|
|
1205
|
+
return scope === "tool"
|
|
1206
|
+
? { address: pending.address, scope, nonce }
|
|
1207
|
+
: {
|
|
1208
|
+
address: pending.address,
|
|
1209
|
+
scope,
|
|
1210
|
+
argsCanonical: canonicalJson(pending.args),
|
|
1211
|
+
nonce,
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
/** One payload-free `approved` event: the address and the scope, nothing else. */
|
|
1215
|
+
function recordApproval(activity, address, approval) {
|
|
1216
|
+
const parts = splitAddress(address);
|
|
1217
|
+
if (!activity || !parts)
|
|
1218
|
+
return;
|
|
1219
|
+
activity.recordTool?.(activity, {
|
|
1220
|
+
connectorId: parts.connectorId,
|
|
1221
|
+
toolName: parts.toolName,
|
|
1222
|
+
address,
|
|
1223
|
+
source: "resume_execution",
|
|
1224
|
+
outcome: "approved",
|
|
1225
|
+
durationMs: 0,
|
|
1226
|
+
attempts: 0,
|
|
1227
|
+
approval,
|
|
1228
|
+
});
|
|
1229
|
+
}
|
|
1230
|
+
const RESUME_DESC = 'Approve and run the write a paused execute_code program stopped at. Repeat the paused result\'s token, address, and args exactly. The program replays from its journal — earlier reads are answered from it, not repeated — sends the write once, and continues to its result or its next pause. approval "tool" also approves later calls to that tool in this run. A mismatch approves and sends nothing. reason is for the human reviewer and is not sent downstream.';
|
|
1231
|
+
/**
|
|
1232
|
+
* The resume input, plus the one fact validation loses: `z.record` copies
|
|
1233
|
+
* `args` and drops a `__proto__` key as it does, so a repetition carrying one
|
|
1234
|
+
* would compare equal to the paused write. The flag rides on the validated
|
|
1235
|
+
* value for the handler, which answers `approval_mismatch`.
|
|
1236
|
+
*/
|
|
1237
|
+
function flaggingProtoArgs(schema) {
|
|
1238
|
+
const standard = schema["~standard"];
|
|
1239
|
+
return {
|
|
1240
|
+
"~standard": {
|
|
1241
|
+
...standard,
|
|
1242
|
+
validate: (value) => {
|
|
1243
|
+
const args = value !== null && typeof value === "object"
|
|
1244
|
+
? value.args
|
|
1245
|
+
: undefined;
|
|
1246
|
+
const flagged = hasProtoKey(args);
|
|
1247
|
+
const result = standard.validate(value);
|
|
1248
|
+
const flag = (outcome) => flagged && !outcome.issues
|
|
1249
|
+
? { value: { ...outcome.value, argsHadProtoKey: true } }
|
|
1250
|
+
: outcome;
|
|
1251
|
+
return result instanceof Promise ? result.then(flag) : flag(result);
|
|
1252
|
+
},
|
|
1253
|
+
},
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
// Listed on every deployment, so the surface never changes shape; one whose
|
|
1257
|
+
// storage cannot claim atomically says plainly that nothing will pause.
|
|
1258
|
+
const RESUME_OFF_DESC = "Resumes a program paused at a write. This deployment does not pause programs — execute_code refuses tools not annotated readOnlyHint: true — so there is nothing to resume. Send a write through call_destructive_tool.";
|
|
1259
|
+
// Built once at module scope, like the other meta-tool inputs.
|
|
1260
|
+
const RESUME_INPUT = flaggingProtoArgs(advertisedSchema(z.strictObject({
|
|
1261
|
+
token: z.string().max(512),
|
|
1262
|
+
address: z.string(),
|
|
1263
|
+
args: z.record(z.string(), z.unknown()),
|
|
1264
|
+
approval: z.enum(["call", "tool"]).optional(),
|
|
1265
|
+
// Dropped in the handler exactly as call_destructive_tool drops it.
|
|
1266
|
+
reason: z.string().max(500).optional(),
|
|
1267
|
+
})));
|
|
1268
|
+
/** Register `resume_execution`, the approval point for a paused program. */
|
|
1269
|
+
export function registerResumeTool(server, registry, ctx) {
|
|
1270
|
+
server.registerTool("resume_execution", {
|
|
1271
|
+
description: ctx.settings ? RESUME_DESC : RESUME_OFF_DESC,
|
|
1272
|
+
inputSchema: RESUME_INPUT,
|
|
1273
|
+
// The one program tool that sends writes, so the host's permission
|
|
1274
|
+
// prompt shows it — with the exact write in its arguments.
|
|
1275
|
+
annotations: {
|
|
1276
|
+
readOnlyHint: false,
|
|
1277
|
+
destructiveHint: true,
|
|
1278
|
+
openWorldHint: true,
|
|
1279
|
+
},
|
|
1280
|
+
}, (args, extra) => {
|
|
1281
|
+
const { reason: _hostContext, ...resume } = args;
|
|
1282
|
+
return underAnySignal([extra.mcpReq.signal, ctx.requestSignal], (signal) => runEdge(resumeExecution(resume, {
|
|
1283
|
+
storage: registry.resultsStorage(),
|
|
1284
|
+
settings: ctx.settings,
|
|
1285
|
+
run: (runState) => ctx.runner.replay(runState, signal),
|
|
1286
|
+
claimMs: ctx.runner.claimMs,
|
|
1287
|
+
storageTimeoutMs: ctx.runner.storageTimeoutMs,
|
|
1288
|
+
activity: ctx.activity,
|
|
1289
|
+
})));
|
|
1290
|
+
});
|
|
1291
|
+
}
|