@cotal-ai/runtime 0.41.3 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fork.d.ts +7 -5
- package/dist/fork.d.ts.map +1 -1
- package/dist/fork.js +10 -7
- package/dist/fork.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/mesh-handler.d.ts +454 -42
- package/dist/mesh-handler.d.ts.map +1 -1
- package/dist/mesh-handler.js +1747 -111
- package/dist/mesh-handler.js.map +1 -1
- package/dist/migrate.d.ts +28 -1
- package/dist/migrate.d.ts.map +1 -1
- package/dist/migrate.js +92 -12
- package/dist/migrate.js.map +1 -1
- package/dist/resolve-checkpoint.d.ts +14 -7
- package/dist/resolve-checkpoint.d.ts.map +1 -1
- package/dist/resolve-checkpoint.js +23 -6
- package/dist/resolve-checkpoint.js.map +1 -1
- package/dist/run-command.d.ts.map +1 -1
- package/dist/run-command.js +35 -20
- package/dist/run-command.js.map +1 -1
- package/dist/run-context.d.ts +11 -7
- package/dist/run-context.d.ts.map +1 -1
- package/dist/run-context.js +18 -8
- package/dist/run-context.js.map +1 -1
- package/dist/run-driver.d.ts +36 -0
- package/dist/run-driver.d.ts.map +1 -1
- package/dist/run-driver.js +204 -26
- package/dist/run-driver.js.map +1 -1
- package/package.json +5 -5
package/dist/mesh-handler.js
CHANGED
|
@@ -17,8 +17,10 @@
|
|
|
17
17
|
* again.
|
|
18
18
|
*/
|
|
19
19
|
import { createHash } from "node:crypto";
|
|
20
|
-
import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, checkpointSettleSubject, epfStreamName, eptStreamName, eptSubject, chatStream, chatSubject, isConcreteChannel, assertSafePattern, runNoticeId, writeRunNotice, } from "@cotal-ai/core";
|
|
21
|
-
import {
|
|
20
|
+
import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, checkpointSettleSubject, epfStreamName, eptStreamName, eptSubject, chatStream, chatSubject, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, liveKvEntries, IncompleteKvScan, openMembersRegistry, openChannelRegistry, writeChannelConfig, readMember, commitMember, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, readGoalResult, readGoalStatus, resolveService, listRunNotices, markRunNoticeConsumed, listRunNoticesForRun, readRunRecord, writeRunStatus, EpEnvelopeError, } from "@cotal-ai/core";
|
|
21
|
+
import { renderRunContext } from "./run-context.js";
|
|
22
|
+
import { Kvm } from "@nats-io/kv";
|
|
23
|
+
import { parseDuration, Cancelled, EffectError, Journal, EffectRefused, askSchemaShape, conformsToAskSchema, journalEntryKeyString, stepKeyString, } from "@cotal-ai/lang";
|
|
22
24
|
/**
|
|
23
25
|
* A checkpoint resumed with no answer to read.
|
|
24
26
|
*
|
|
@@ -42,18 +44,20 @@ export class CheckpointAnswerMissing extends Error {
|
|
|
42
44
|
}
|
|
43
45
|
}
|
|
44
46
|
/**
|
|
45
|
-
* The one subject the
|
|
46
|
-
*
|
|
47
|
+
* The one subject the remaining seam is gated by: `turn` and `wait(replied)` both ride the turn
|
|
48
|
+
* machinery an agent handle answers through (`spawn`, `conclave`, `ask`, `monitor` and
|
|
49
|
+
* `wait(down)` perform). Named once so the refusals cannot drift apart.
|
|
47
50
|
*/
|
|
48
|
-
const ACTION_MACHINERY = "the durable-action machinery an agent handle comes from";
|
|
49
51
|
export class MeshHandler {
|
|
52
|
+
nc;
|
|
50
53
|
kv;
|
|
51
54
|
js;
|
|
52
55
|
jsm;
|
|
53
56
|
binding;
|
|
54
57
|
watcher;
|
|
55
58
|
clock;
|
|
56
|
-
constructor(kv, js, jsm, binding, watcher, clock = () => Date.now()) {
|
|
59
|
+
constructor(nc, kv, js, jsm, binding, watcher, clock = () => Date.now()) {
|
|
60
|
+
this.nc = nc;
|
|
57
61
|
this.kv = kv;
|
|
58
62
|
this.js = js;
|
|
59
63
|
this.jsm = jsm;
|
|
@@ -61,6 +65,90 @@ export class MeshHandler {
|
|
|
61
65
|
this.watcher = watcher;
|
|
62
66
|
this.clock = clock;
|
|
63
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* The resolved manager service, memoized as a PROMISE so concurrent branches share one describe
|
|
70
|
+
* round-trip — and dropped on failure, so a resolve that lost to a manager restart is retried by
|
|
71
|
+
* the next effect instead of poisoning every spawn for the handler's lifetime.
|
|
72
|
+
*/
|
|
73
|
+
managerService;
|
|
74
|
+
manager() {
|
|
75
|
+
this.managerService ??= resolveService(this.nc, this.binding.space, this.binding.endpoint, this.binding.caller)
|
|
76
|
+
.catch((e) => {
|
|
77
|
+
this.managerService = undefined;
|
|
78
|
+
throw e;
|
|
79
|
+
});
|
|
80
|
+
return this.managerService;
|
|
81
|
+
}
|
|
82
|
+
/** The branded goal-fact context over this handler's own connection, memoized the same way. */
|
|
83
|
+
actions;
|
|
84
|
+
actionCtx() {
|
|
85
|
+
this.actions ??= actionContext(this.nc, this.binding.space).catch((e) => {
|
|
86
|
+
this.actions = undefined;
|
|
87
|
+
throw e;
|
|
88
|
+
});
|
|
89
|
+
return this.actions;
|
|
90
|
+
}
|
|
91
|
+
// The three registries a conclave touches, memoized like the service resolves above. All three
|
|
92
|
+
// are OPENED, never created: the provisioner pre-creates them at `cotal up` (auth mode) and the
|
|
93
|
+
// endpoints create presence/channels lazily in open mode — a mesh where one is genuinely absent
|
|
94
|
+
// was never provisioned for durable membership, and that fails loud rather than self-provisions.
|
|
95
|
+
presenceKvOpen;
|
|
96
|
+
presenceRegistry() {
|
|
97
|
+
this.presenceKvOpen ??= new Kvm(this.nc).open(presenceBucket(this.binding.space)).catch((e) => {
|
|
98
|
+
this.presenceKvOpen = undefined;
|
|
99
|
+
throw e;
|
|
100
|
+
});
|
|
101
|
+
return this.presenceKvOpen;
|
|
102
|
+
}
|
|
103
|
+
membersKvOpen;
|
|
104
|
+
membersRegistry() {
|
|
105
|
+
this.membersKvOpen ??= openMembersRegistry(this.nc, this.binding.space).catch((e) => {
|
|
106
|
+
this.membersKvOpen = undefined;
|
|
107
|
+
throw e;
|
|
108
|
+
});
|
|
109
|
+
return this.membersKvOpen;
|
|
110
|
+
}
|
|
111
|
+
channelsKvOpen;
|
|
112
|
+
channelRegistry() {
|
|
113
|
+
this.channelsKvOpen ??= openChannelRegistry(this.nc, this.binding.space).catch((e) => {
|
|
114
|
+
this.channelsKvOpen = undefined;
|
|
115
|
+
throw e;
|
|
116
|
+
});
|
|
117
|
+
return this.channelsKvOpen;
|
|
118
|
+
}
|
|
119
|
+
/** The chat stream's current last sequence — a conclave join/leave cursor (SPEC §7 interval). */
|
|
120
|
+
async chatFrontier() {
|
|
121
|
+
return (await this.jsm.streams.info(chatStream(this.binding.space))).state.last_seq;
|
|
122
|
+
}
|
|
123
|
+
/** The open conclaves this process performed, keyed by the scope's request id, so the close that
|
|
124
|
+
* follows the body reads the SAME plan the open executed. A crash loses the map and loses
|
|
125
|
+
* nothing: the plan was bound as the entry's external state before a single row was written,
|
|
126
|
+
* and a re-entered open repopulates the map from `ctx.resume`. */
|
|
127
|
+
conclaves = new Map();
|
|
128
|
+
/** The run's roster: every agent this run spawned, by name — the handle a handoff resolves to,
|
|
129
|
+
* and the owner/actor address a `turn` targets (absent when the spawn's acceptance floor was
|
|
130
|
+
* never served; a `turn` on such an agent refuses loudly rather than guessing an address).
|
|
131
|
+
* Seeded live by `spawn`, rebuilt at adoption from the journal's settled spawn entries. */
|
|
132
|
+
roster = new Map();
|
|
133
|
+
/** Turns this run has dispatched per handle composite, live and seeded — what a `turns` permit is spent against. */
|
|
134
|
+
turnsTaken = new Map();
|
|
135
|
+
/** Handle composites a `monitor` registered, live and seeded — what makes `down(agent)` observable. */
|
|
136
|
+
monitored = new Set();
|
|
137
|
+
/** The most recent unhonored handoff yield per SCOPE (lang §5.3): the goal-chain linkage memo.
|
|
138
|
+
* `"ambiguous"` when two pending handoffs in one scope named the same agent — ambiguity records
|
|
139
|
+
* no linkage at all. Spent (deleted) by the next `turn` in the scope, honored or not: honoring
|
|
140
|
+
* is immediate-only. Rebuilt at adoption by replaying the same two rules over the journal. */
|
|
141
|
+
handoffMemos = new Map();
|
|
142
|
+
/** This run's turn goals per handle composite ("name#uid") — the observable `wait(replied)`
|
|
143
|
+
* rides. Fed by `turn` at submission and adoption, reseeded from journal turn entries. */
|
|
144
|
+
turnGoals = new Map();
|
|
145
|
+
/** The last agent this run spawned into each logical worktree — what the L4008 guard reads.
|
|
146
|
+
* Fed by `spawn` and reseeded from journal spawn results at adoption. */
|
|
147
|
+
worktreeHolders = new Map();
|
|
148
|
+
/** Seats a committed migration handed to this program under `--adopt`, keyed by persona and
|
|
149
|
+
* handed out in journal order: the next `spawn` of that persona receives the recorded seat
|
|
150
|
+
* instead of minting one. Fed by {@link adoptMigratedSeats}; spent by `spawn`. */
|
|
151
|
+
adoptable = new Map();
|
|
64
152
|
now() {
|
|
65
153
|
return this.clock();
|
|
66
154
|
}
|
|
@@ -75,7 +163,301 @@ export class MeshHandler {
|
|
|
75
163
|
* cannot advance it.
|
|
76
164
|
*/
|
|
77
165
|
async adopted(entries) {
|
|
78
|
-
|
|
166
|
+
const folded = foldEntries(entries);
|
|
167
|
+
this.seedRunMemos(folded);
|
|
168
|
+
return await rearmOutstandingPauses({ kv: this.kv, js: this.js, jsm: this.jsm }, this.binding, folded);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Hand this program the seats a committed migration kept for it under `--adopt` (spec §11.2).
|
|
172
|
+
* The orphaned spawn's settled entry is the whole hand-over: the next `spawn` of its persona
|
|
173
|
+
* returns that entry's handle, binds that entry's floor, and submits nothing, so the seat keeps
|
|
174
|
+
* its identity, its worktree and its turn history across the edit. Called by the driver on the
|
|
175
|
+
* migrated run before the program runs; a map with no entry for a persona changes nothing.
|
|
176
|
+
*/
|
|
177
|
+
adoptMigratedSeats(seats) {
|
|
178
|
+
for (const [persona, entries] of seats)
|
|
179
|
+
this.adoptable.set(persona, [...(this.adoptable.get(persona) ?? []), ...entries]);
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Rebuild the in-memory run memos an adopted run needs to keep performing `turn`: the roster
|
|
183
|
+
* (from every settled ok `spawn`, its result the handle and its bound floor the address), the
|
|
184
|
+
* worktree holders (a settled spawn holds its tree by identity, a spawn still in flight by the
|
|
185
|
+
* reservation its bound request took) and the handoff memos (replaying, in journal order, the
|
|
186
|
+
* same two rules the live path applies — a turn's begin spends its scope's memo, a settled
|
|
187
|
+
* handoff yield writes one, a second pending handoff to the same name in one scope makes it
|
|
188
|
+
* ambiguous). Deterministic from the journal alone.
|
|
189
|
+
*/
|
|
190
|
+
seedRunMemos(entries) {
|
|
191
|
+
for (const e of entries) {
|
|
192
|
+
if (e.kind === "spawn" && e.state === "pending") {
|
|
193
|
+
const x = e.external;
|
|
194
|
+
if (typeof x?.goalId === "string" && typeof x?.worktree === "string")
|
|
195
|
+
this.worktreeHolders.set(x.worktree, { pending: x.goalId });
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
if (e.kind === "spawn" && e.state === "settled" && e.status === "ok" && e.result !== undefined) {
|
|
199
|
+
const handle = e.result;
|
|
200
|
+
if (typeof handle.agent !== "string")
|
|
201
|
+
continue; // a garbled result seeds nothing; the turn that needs it refuses loudly
|
|
202
|
+
const { name, uid } = parseAgentHandle(handle.agent);
|
|
203
|
+
const ext = e.external;
|
|
204
|
+
this.roster.set(name, {
|
|
205
|
+
handle,
|
|
206
|
+
uid,
|
|
207
|
+
...(typeof ext?.owner === "string" ? { owner: ext.owner } : {}),
|
|
208
|
+
...(typeof ext?.actor === "string" ? { actor: ext.actor } : {}),
|
|
209
|
+
...(ext?.permits !== undefined ? { permits: readPermits(ext.permits, handle.persona) } : {}),
|
|
210
|
+
...(typeof ext?.spawnedAt === "number" ? { spawnedAt: ext.spawnedAt } : {}),
|
|
211
|
+
});
|
|
212
|
+
if (typeof handle.worktree === "string")
|
|
213
|
+
this.worktreeHolders.set(handle.worktree, { name, uid });
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
if (e.kind === "monitor" && e.state === "settled" && e.status === "ok") {
|
|
217
|
+
const m = e.external;
|
|
218
|
+
if (typeof m?.agent === "string")
|
|
219
|
+
this.monitored.add(m.agent);
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
if (e.kind !== "turn")
|
|
223
|
+
continue;
|
|
224
|
+
const x = e.external;
|
|
225
|
+
// Every relayed turn spent one of its agent's turns, whatever it came to.
|
|
226
|
+
if (typeof x?.name === "string" && typeof x?.uid === "string")
|
|
227
|
+
this.turnsTaken.set(`${x.name}#${x.uid}`, (this.turnsTaken.get(`${x.name}#${x.uid}`) ?? 0) + 1);
|
|
228
|
+
if (typeof x?.name === "string" && typeof x?.uid === "string" && typeof x?.goalId === "string" && (e.state === "pending" || e.status === "ok"))
|
|
229
|
+
this.recordTurnGoal(`${x.name}#${x.uid}`, x.goalId);
|
|
230
|
+
this.handoffMemos.delete(e.scope); // its begin spent whatever was pending, honored or not
|
|
231
|
+
if (e.state !== "settled" || e.status !== "ok" || e.result === undefined)
|
|
232
|
+
continue;
|
|
233
|
+
const r = e.result;
|
|
234
|
+
if (r.status !== "handoff" || r.to === undefined || typeof r.to.agent !== "string")
|
|
235
|
+
continue;
|
|
236
|
+
this.recordHandoffMemo(e.scope, parseAgentHandle(r.to.agent).name, e.requestId ?? "");
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* The runtime half of "two agents MUST NOT share a worktree concurrently" (spec 6.5; the
|
|
241
|
+
* validator owns the literal case as L3022). The registry records who holds each logical
|
|
242
|
+
* worktree this run spawned into — the live seat, or the spawn still bringing one up — and a
|
|
243
|
+
* new spawn CLAIMS the tree here before anything is submitted: a tree nobody holds is taken in
|
|
244
|
+
* the same synchronous step that read it, so two branches spawning into one computed id cannot
|
|
245
|
+
* both pass; a tree held by a spawn in flight refuses; a tree held by a seat is admitted only
|
|
246
|
+
* when that holder is no longer live on presence, and re-read after the liveness wait, because
|
|
247
|
+
* a sibling may have claimed it meanwhile. Sequential reuse is legal, concurrency is data loss.
|
|
248
|
+
* The liveness read is the same witness `wait(down)` and a conclave join use, so "still
|
|
249
|
+
* holding" and "down" are one definition — a discharged loser or a crashed seat releases its
|
|
250
|
+
* tree the moment its presence row is gone, with no bookkeeping of its own.
|
|
251
|
+
*/
|
|
252
|
+
async claimWorktree(worktree, persona, goalId) {
|
|
253
|
+
const holder = this.worktreeHolders.get(worktree);
|
|
254
|
+
if (holder === undefined) {
|
|
255
|
+
this.worktreeHolders.set(worktree, { pending: goalId });
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if ("pending" in holder)
|
|
259
|
+
throw new EffectError("L4008", "worktree", `spawn(${persona}) would put a second agent into the worktree "${worktree}" while spawn goal ${holder.pending} is still bringing one up in it; two agents MUST NOT share a worktree concurrently (L3022/L4008)`);
|
|
260
|
+
let rows;
|
|
261
|
+
for (let attempt = 1;; attempt += 1) {
|
|
262
|
+
try {
|
|
263
|
+
rows = await this.presenceRows();
|
|
264
|
+
break;
|
|
265
|
+
}
|
|
266
|
+
catch (e) {
|
|
267
|
+
// A scan that keeps coming back incomplete is a broken presence plane, and a guard that
|
|
268
|
+
// never answers would park the spawn for good: bounded, then the scan's own error.
|
|
269
|
+
if (!(e instanceof IncompleteKvScan) || attempt >= WORKTREE_SCAN_ATTEMPTS)
|
|
270
|
+
throw e;
|
|
271
|
+
await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (rows.some((p) => p.card?.name === holder.name && p.lifecycleUid === holder.uid))
|
|
275
|
+
throw new EffectError("L4008", "worktree", `spawn(${persona}) would put a second agent into the worktree "${worktree}" while ${holder.name}#${holder.uid} is live in it; two agents MUST NOT share a worktree concurrently (L3022/L4008)`);
|
|
276
|
+
const now = this.worktreeHolders.get(worktree);
|
|
277
|
+
if (now !== undefined && now !== holder)
|
|
278
|
+
throw new EffectError("L4008", "worktree", `spawn(${persona}) would put a second agent into the worktree "${worktree}", which ${"pending" in now ? `spawn goal ${now.pending}` : `${now.name}#${now.uid}`} claimed while this spawn read its previous holder's liveness; two agents MUST NOT share a worktree concurrently (L3022/L4008)`);
|
|
279
|
+
this.worktreeHolders.set(worktree, { pending: goalId });
|
|
280
|
+
}
|
|
281
|
+
/** One turn goal registered under its handle composite — what `wait(replied)` observes. */
|
|
282
|
+
recordTurnGoal(handle, goalId) {
|
|
283
|
+
const set = this.turnGoals.get(handle);
|
|
284
|
+
if (set === undefined)
|
|
285
|
+
this.turnGoals.set(handle, new Set([goalId]));
|
|
286
|
+
else
|
|
287
|
+
set.add(goalId);
|
|
288
|
+
}
|
|
289
|
+
/** One handoff yield's memo write: most-recent-wins across different names, ambiguous when a
|
|
290
|
+
* pending handoff to the SAME name is already waiting (lang §5.3 — ambiguity records nothing). */
|
|
291
|
+
recordHandoffMemo(scope, to, fromGoalId) {
|
|
292
|
+
const prev = this.handoffMemos.get(scope);
|
|
293
|
+
this.handoffMemos.set(scope, prev !== undefined && (prev === "ambiguous" || prev.to === to) ? "ambiguous" : { to, fromGoalId });
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* End the external state of a cancelled scope's LOSERS: the world half of the discharge the
|
|
297
|
+
* scope entry's `cancel.issued` records (§7.6, and the driver's `dischargeCancellations` is the
|
|
298
|
+
* caller). The entries handed in are the losers' subtrees; what has external state to end is the
|
|
299
|
+
* three pause kinds — a pause's timer is claimed so its armed schedule cannot fire into a run
|
|
300
|
+
* that moved on, and a wait's durable consumer is deleted because a cancelled wait replays as
|
|
301
|
+
* cancelled and nothing will ever read its position.
|
|
302
|
+
*
|
|
303
|
+
* IDEMPOTENT BY THE PLANE: `cancelTimer` declines a pause that is not waiting, the consumer
|
|
304
|
+
* delete tolerates one already gone, and both tolerate a loser that cleaned up after itself on
|
|
305
|
+
* the live path — this is the durable backstop for the process that died before its own cleanup
|
|
306
|
+
* landed, and the flip to `issued: true` happens only after it returns.
|
|
307
|
+
*/
|
|
308
|
+
async discharge(entries) {
|
|
309
|
+
for (const e of entries) {
|
|
310
|
+
if (e.requestId === undefined)
|
|
311
|
+
continue;
|
|
312
|
+
if (e.kind === "spawn") {
|
|
313
|
+
await this.dischargeSpawn(e);
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (e.kind === "conclave") {
|
|
317
|
+
await this.dischargeConclave(e);
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
if (e.kind === "notify") {
|
|
321
|
+
await this.dischargeNotify(e);
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
if (e.kind === "turn") {
|
|
325
|
+
// A cancelled turn is never a reply, whatever the seat yields to the relay later.
|
|
326
|
+
const x = e.external;
|
|
327
|
+
if (typeof x?.name === "string" && typeof x?.uid === "string" && typeof x?.goalId === "string")
|
|
328
|
+
this.turnGoals.get(`${x.name}#${x.uid}`)?.delete(x.goalId);
|
|
329
|
+
}
|
|
330
|
+
if (e.kind !== "sleep" && e.kind !== "checkpoint" && e.kind !== "wait" && e.kind !== "ask" && e.kind !== "turn")
|
|
331
|
+
continue;
|
|
332
|
+
// An ask's armed timer is its CURRENT attempt's, whose token is bound as `askToken`; a
|
|
333
|
+
// crash before the first bind leaves attempt 1, which is the request id itself.
|
|
334
|
+
const current = e.kind === "ask" && typeof e.external?.askToken === "string"
|
|
335
|
+
? e.external.askToken
|
|
336
|
+
: e.requestId;
|
|
337
|
+
await this.cancelTimer({ endpoint: this.binding.endpoint, token: current });
|
|
338
|
+
if (e.kind === "wait") {
|
|
339
|
+
await this.cancelTimer({ endpoint: this.binding.endpoint, token: derivedToken(e.requestId, "wait-timeout") });
|
|
340
|
+
try {
|
|
341
|
+
await this.jsm.consumers.delete(chatStream(this.binding.space), waitConsumerName(e.requestId));
|
|
342
|
+
}
|
|
343
|
+
catch { /* never created, or already deleted — nothing is held either way */ }
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Withdraw a cancelled `notify`'s undelivered notices.
|
|
349
|
+
*
|
|
350
|
+
* A notice is world state exactly as a live seat or an armed timer is: it sits on the run
|
|
351
|
+
* waiting to be rendered ahead of its addressee's next turn. A branch the run cancelled decided
|
|
352
|
+
* nothing, and a decision the run withdrew must not arrive at an agent afterwards — which is
|
|
353
|
+
* what happened, because the discharge released seats, memberships and timers and walked past
|
|
354
|
+
* this one kind.
|
|
355
|
+
*
|
|
356
|
+
* Withdrawn is spelled as CONSUMED, by the discharge rather than by a turn. The notice's status
|
|
357
|
+
* is a closed record with one meaning — this notice will not be delivered again — and that is
|
|
358
|
+
* the fact being recorded; `by` says which cancelled step withdrew it, so a reader tracing a
|
|
359
|
+
* notice finds the withdrawal instead of a delivery that never happened. It also puts the
|
|
360
|
+
* migration verdict where it belongs: an orphaned `notify` is rejected only while its notice is
|
|
361
|
+
* still owed to somebody, and this one is not owed to anybody any more.
|
|
362
|
+
*
|
|
363
|
+
* IDEMPOTENT BY THE PLANE, like the rest of the discharge: the consumption write is create-only,
|
|
364
|
+
* so a notice a turn already carried, or that an earlier discharge pass already withdrew, is
|
|
365
|
+
* left exactly as it is.
|
|
366
|
+
*/
|
|
367
|
+
async dischargeNotify(e) {
|
|
368
|
+
const step = journalEntryKeyString(e);
|
|
369
|
+
const notices = await listRunNoticesForRun(this.kv, this.binding.endpoint, this.binding.runId);
|
|
370
|
+
for (const n of notices) {
|
|
371
|
+
if (n.spec.step !== step || n.consumed !== undefined)
|
|
372
|
+
continue;
|
|
373
|
+
try {
|
|
374
|
+
await markRunNoticeConsumed(this.kv, this.binding.endpoint, this.binding.runId, n.spec.addressee, n.noticeId, `discharge:${step}`, this.now());
|
|
375
|
+
}
|
|
376
|
+
catch (err) {
|
|
377
|
+
// A turn carried it between the read and the write: the create-only status is the arbiter
|
|
378
|
+
// and it has spoken. Anything else is the store, and a discharge that could not finish
|
|
379
|
+
// must not be flipped to `issued`.
|
|
380
|
+
if (!(err instanceof EpEnvelopeError && err.code === "conflict"))
|
|
381
|
+
throw err;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Release a cancelled spawn's AGENT (§8.6.4): the world half a loser `spawn` leaves behind is a
|
|
387
|
+
* seat the run will never address, so the discharge despawns it. The goal's identity re-derives
|
|
388
|
+
* from the entry alone — the request id IS the goalId (the pinned envelope id) unless the entry
|
|
389
|
+
* bound another (an adopted seat's), and the caller triple is run-stable — so a crash before the
|
|
390
|
+
* acceptance was even bound still finds its goal.
|
|
391
|
+
*
|
|
392
|
+
* The terminal is what says whether a seat exists. No goal at all: the submission never landed,
|
|
393
|
+
* nothing to release. Accepted but not terminal: the manager owes a terminal within the accepted
|
|
394
|
+
* readiness window, so this waits it out (bounded by the recorded window plus one poll of slack)
|
|
395
|
+
* and THROWS if none lands — an unfinished discharge must not be flipped to `issued`, and the
|
|
396
|
+
* driver's next sweep retries idempotently. `succeeded` and `uncertain` both despawn (an
|
|
397
|
+
* uncertain readiness verdict leaves the process running); `failed` was reaped by the manager
|
|
398
|
+
* and `cancelled` means a despawn already drove the teardown, so both are already released.
|
|
399
|
+
*/
|
|
400
|
+
async dischargeSpawn(e) {
|
|
401
|
+
// The bound goal when the entry carries one (a spawn a migration handed a seat binds the
|
|
402
|
+
// ORPHANED spawn's goal, under which its terminal and its seat live); the request id otherwise.
|
|
403
|
+
const goalId = typeof e.external?.goalId === "string" ? e.external.goalId : e.requestId;
|
|
404
|
+
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
405
|
+
const actx = await this.actionCtx();
|
|
406
|
+
let fact = await readGoalResult(actx, ref);
|
|
407
|
+
if (fact === undefined) {
|
|
408
|
+
if ((await readGoalStatus(actx, ref)) === undefined)
|
|
409
|
+
return;
|
|
410
|
+
const window = typeof e.external?.readinessDeadlineMs === "number" ? e.external.readinessDeadlineMs : DISCHARGE_TERMINAL_BOUND_MS;
|
|
411
|
+
const deadline = this.now() + window + GOAL_POLL_MS;
|
|
412
|
+
for (;;) {
|
|
413
|
+
fact = await readGoalResult(actx, ref);
|
|
414
|
+
if (fact !== undefined)
|
|
415
|
+
break;
|
|
416
|
+
if (this.now() >= deadline)
|
|
417
|
+
throw new Error(`the cancelled spawn goal "${goalId}" is accepted but reached no terminal within its ${window}ms readiness window; its agent cannot be released yet, and the discharge stays open to retry`);
|
|
418
|
+
await new Promise((r) => setTimeout(r, GOAL_POLL_MS).unref());
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
if (fact.state !== "succeeded" && fact.state !== "uncertain")
|
|
422
|
+
return;
|
|
423
|
+
const target = spawnDespawnTarget(e.external, fact);
|
|
424
|
+
if (target === undefined) {
|
|
425
|
+
// An `uncertain` terminal carries no identity, and this entry bound none (the crash landed
|
|
426
|
+
// between the acceptance and the bind). The seat — if one came up — is not addressable from
|
|
427
|
+
// here, and no retry will ever learn more, so throwing would wedge every future sweep of
|
|
428
|
+
// this run behind an answer that cannot arrive. Name the leak for the operator instead.
|
|
429
|
+
console.error(`! discharge: the cancelled spawn goal "${goalId}" settled ${fact.state} with no readable agent identity; if its seat is up it must be despawned by hand (cotal ps)`);
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
432
|
+
const service = await this.manager();
|
|
433
|
+
const reply = await invokeCommand(this.nc, this.binding.space, service, "despawn", { graceful: true }, {
|
|
434
|
+
target: { mode: "owner", ...target },
|
|
435
|
+
deadlineMs: SPAWN_ACCEPT_DEADLINE_MS,
|
|
436
|
+
});
|
|
437
|
+
// Tolerated refusals are the two "already gone" shapes: `not-found` (no such agent), and
|
|
438
|
+
// `expired` (the target's lifecycle mapping is gone or rotated — this despawn pins one
|
|
439
|
+
// incarnation, and an incarnation the mapping no longer names is not running).
|
|
440
|
+
const code = reply.reply.ok === false ? reply.reply.error?.code : undefined;
|
|
441
|
+
if (code !== undefined && code !== "not-found" && code !== "expired")
|
|
442
|
+
throw new Error(`the cancelled spawn's agent could not be despawned: ${reply.reply.error?.message ?? "refused"}`);
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Release a cancelled conclave's MEMBERSHIP (spec §7.5): a cancelled body performs no new
|
|
446
|
+
* effect, so it never closed its room, and the release travels this recovery path like every
|
|
447
|
+
* other branch-local resource. The entry's own `closed` fact is the gate — a conclave whose
|
|
448
|
+
* body failed but whose close acknowledged holds nothing. An entry with no bound plan created
|
|
449
|
+
* nothing (rows are written only after the bind), so there is nothing to release and no retry
|
|
450
|
+
* that could learn more. Failures are raised: an unfinished release must not be flipped to
|
|
451
|
+
* `issued`, and the driver's next sweep retries idempotently.
|
|
452
|
+
*/
|
|
453
|
+
async dischargeConclave(e) {
|
|
454
|
+
if (e.closed === true)
|
|
455
|
+
return;
|
|
456
|
+
const plan = e.external;
|
|
457
|
+
if (!isConclavePlan(plan))
|
|
458
|
+
return;
|
|
459
|
+
await this.releaseConclave(plan);
|
|
460
|
+
this.conclaves.delete(e.requestId);
|
|
79
461
|
}
|
|
80
462
|
/**
|
|
81
463
|
* `sleep` is a checkpoint nobody answers.
|
|
@@ -87,6 +469,8 @@ export class MeshHandler {
|
|
|
87
469
|
* and `null` is the whole answer.
|
|
88
470
|
*/
|
|
89
471
|
async sleep(req, ctx) {
|
|
472
|
+
if (ctx.signal.cancelled)
|
|
473
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
90
474
|
const ref = { endpoint: this.binding.endpoint, token: ctx.requestId };
|
|
91
475
|
const now = this.now();
|
|
92
476
|
const deadline = now + parseDuration(req.duration);
|
|
@@ -97,7 +481,7 @@ export class MeshHandler {
|
|
|
97
481
|
// A request emitted from here would carry the caller's coordinates and could arm a stale
|
|
98
482
|
// generation the writer would then have to be trusted to ignore.
|
|
99
483
|
await this.arm(ref, deadline);
|
|
100
|
-
await this.settle(ref);
|
|
484
|
+
await this.settle(ref, ctx.signal);
|
|
101
485
|
return null;
|
|
102
486
|
}
|
|
103
487
|
/**
|
|
@@ -112,13 +496,41 @@ export class MeshHandler {
|
|
|
112
496
|
* It returns the RAW outcome and never the program's result. Whether an expiry throws or returns
|
|
113
497
|
* is `onExpiry`, which is computed from today's source on the live path and the replay path
|
|
114
498
|
* alike; deciding it here would bake one answer into the journal.
|
|
499
|
+
*
|
|
500
|
+
* WHAT IT ASKS IS BOUND ON THE ENTRY, and that is not decoration. A pause the reference calls
|
|
501
|
+
* "a durable pause a human or an agent resolves from anywhere" is answered by address (`run`
|
|
502
|
+
* plus step key), and the prompt lived only in the source: everything durable held the input
|
|
503
|
+
* HASH, so whoever was asked saw a token and had to go find the program to learn the question.
|
|
504
|
+
* `to` rides with it for the same reason — an escalation names an addressee, and an addressee
|
|
505
|
+
* nothing records is an addressee nobody can be shown.
|
|
115
506
|
*/
|
|
116
507
|
async checkpoint(req, ctx) {
|
|
508
|
+
if (ctx.signal.cancelled)
|
|
509
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
117
510
|
const ref = { endpoint: this.binding.endpoint, token: ctx.requestId };
|
|
118
|
-
|
|
119
|
-
|
|
511
|
+
// ONE deadline per attempt, decided when the attempt is first bound and read back from the
|
|
512
|
+
// entry on every re-entry, exactly as `ask` does. Recomputing `now + timeout` on a resume
|
|
513
|
+
// handed the relay an instant the pause did not have: `arm` attaches to the recorded spec and
|
|
514
|
+
// keeps ITS deadline, so the seat's hold and the checkpoint plane denied at different times.
|
|
515
|
+
const recorded = ctx.resume?.deadlineAt;
|
|
516
|
+
const deadline = typeof recorded === "number"
|
|
517
|
+
? recorded
|
|
518
|
+
: this.now() + parseDuration(req.timeout ?? this.binding.defaultCheckpointTimeout);
|
|
519
|
+
// Once per attempt, before the pause exists: a crash between the bind and the mint leaves a
|
|
520
|
+
// pending entry that says what it was going to ask, which is the harmless direction.
|
|
521
|
+
if (ctx.resume?.asks === undefined)
|
|
522
|
+
await ctx.bind({ asks: req.prompt, deadlineAt: deadline, ...(req.to !== undefined ? { addressee: req.to } : {}) });
|
|
523
|
+
// AN ESCALATION IS ADDRESSED, so where the addressee is an agent of this run it is TOLD.
|
|
524
|
+
// Attempt 0 has no addressee (the reference allows `to` only with `onExpiry: "escalate"`,
|
|
525
|
+
// and the escalation is the second mint), and a `to` that names no agent of this run is a
|
|
526
|
+
// person: nothing to relay to, and the pause is answerable from anywhere by design. Both
|
|
527
|
+
// are visible at the operator surface, which is what the bind above is for. Told BEFORE the
|
|
528
|
+
// pause is armed, the order `ask` uses: a relay this endpoint refuses outright then fails the
|
|
529
|
+
// step with nothing armed behind it, instead of leaving a timer to fire into a failed step.
|
|
530
|
+
if (ctx.attempt > 0 && req.to !== undefined)
|
|
531
|
+
await this.relayEscalation(req, ctx, ref.token, deadline);
|
|
120
532
|
await this.arm(ref, deadline);
|
|
121
|
-
const settled = await this.settle(ref);
|
|
533
|
+
const settled = await this.settle(ref, ctx.signal);
|
|
122
534
|
if (settled.settle === "expired")
|
|
123
535
|
return { outcome: "expired", at: settled.ts };
|
|
124
536
|
// The settle NAMES its answer, and the record is read under that name rather than by looking
|
|
@@ -150,14 +562,17 @@ export class MeshHandler {
|
|
|
150
562
|
* resumed 20-minute wait with 30 seconds left has 30 seconds left. A timeout resolves `null` and
|
|
151
563
|
* never throws: `??` is `otherwise`.
|
|
152
564
|
*
|
|
153
|
-
* `
|
|
154
|
-
* `
|
|
565
|
+
* `down(agent)` and `replied(agent)` are agent-addressed events with no channel, so they branch
|
|
566
|
+
* to `waitDown` and `waitReplied` before any of the channel machinery.
|
|
155
567
|
*/
|
|
156
568
|
async wait(req, ctx) {
|
|
569
|
+
if (ctx.signal.cancelled)
|
|
570
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
157
571
|
const ev = req.event;
|
|
158
|
-
if (ev.event === "replied"
|
|
159
|
-
|
|
160
|
-
|
|
572
|
+
if (ev.event === "replied")
|
|
573
|
+
return await this.waitReplied(ev, req, ctx);
|
|
574
|
+
if (ev.event === "down")
|
|
575
|
+
return await this.waitDown(ev, req, ctx);
|
|
161
576
|
if (!isConcreteChannel(ev.channel)) {
|
|
162
577
|
throw new Error(`wait() cannot await a wildcard channel ("${ev.channel}"); an await names one channel`);
|
|
163
578
|
}
|
|
@@ -165,8 +580,14 @@ export class MeshHandler {
|
|
|
165
580
|
// rather than looking again: the consumer has already acked it, so looking again would wait for
|
|
166
581
|
// a second event the program never asked for.
|
|
167
582
|
const bound = ctx.resume?.chatSeq;
|
|
168
|
-
if (typeof bound === "number")
|
|
583
|
+
if (typeof bound === "number") {
|
|
584
|
+
// The match is recorded, so this wait is over — and the deadlines the PREVIOUS attempt armed
|
|
585
|
+
// are still live, because the crash came before it could claim them. Claim them here for the
|
|
586
|
+
// same reason the live match path does: this is the ending, it just happened last time.
|
|
587
|
+
await this.cancelTimer({ endpoint: this.binding.endpoint, token: ctx.requestId });
|
|
588
|
+
await this.cancelTimer({ endpoint: this.binding.endpoint, token: derivedToken(ctx.requestId, "wait-timeout") });
|
|
169
589
|
return await this.messageAt(bound);
|
|
590
|
+
}
|
|
170
591
|
const timeoutAt = req.timeout === undefined ? undefined : this.now() + parseDuration(req.timeout);
|
|
171
592
|
const idleFor = ev.event === "idle" ? parseDuration(ev.duration) : undefined;
|
|
172
593
|
const matcher = ev.event === "message" && ev.matches !== undefined ? compileMatch(ev.matches) : undefined;
|
|
@@ -194,15 +615,37 @@ export class MeshHandler {
|
|
|
194
615
|
let over = false;
|
|
195
616
|
try {
|
|
196
617
|
for (;;) {
|
|
618
|
+
// THE CANCELLATION IS OBSERVED ON THE SAME CADENCE AS THE DEADLINE: once per poll, so a
|
|
619
|
+
// race decided against this branch ends its wait within one fetch rather than never
|
|
620
|
+
// (§7.6 — a cancelled branch performs no new work, and a wait mid-poll is this branch's
|
|
621
|
+
// one in-flight effect). A cancelled wait is OVER, not abandoned: its timers are claimed
|
|
622
|
+
// here and its consumer is deleted by the cleanup below, because a `cancelled` settle
|
|
623
|
+
// replays as cancelled and nothing will ever re-attach to this position.
|
|
624
|
+
if (ctx.signal.cancelled) {
|
|
625
|
+
if (primary !== undefined)
|
|
626
|
+
await this.cancelTimer(primary);
|
|
627
|
+
if (outer !== undefined)
|
|
628
|
+
await this.cancelTimer(outer);
|
|
629
|
+
over = true;
|
|
630
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
631
|
+
}
|
|
197
632
|
// The deadline is durable and authoritative — a checkpoint's settle fact — and this is only
|
|
198
633
|
// the OBSERVATION of it, so the cost of polling is lateness bounded by one poll rather than
|
|
199
634
|
// a wait that outlives its deadline.
|
|
635
|
+
// A WAIT THAT IS OVER CLAIMS BOTH OF ITS DEADLINES, on every path that ends it. An idle
|
|
636
|
+
// wait with a timeout arms two, and each expiry used to claim only the one it read: the
|
|
637
|
+
// sibling stayed armed, fired into a run that had moved on, and sat there as an unclaimed
|
|
638
|
+
// settle until the run's own discharge swept it. The match path below already did this.
|
|
200
639
|
const ended = await this.expired(outer ?? (idleFor === undefined ? primary : undefined));
|
|
201
640
|
if (ended !== undefined) {
|
|
641
|
+
if (outer !== undefined && primary !== undefined)
|
|
642
|
+
await this.cancelTimer(primary);
|
|
202
643
|
over = true;
|
|
203
644
|
return null;
|
|
204
645
|
}
|
|
205
646
|
if (idleFor !== undefined && (await this.expired(primary)) !== undefined) {
|
|
647
|
+
if (outer !== undefined)
|
|
648
|
+
await this.cancelTimer(outer);
|
|
206
649
|
over = true;
|
|
207
650
|
return { channel: ev.channel, at: this.now() };
|
|
208
651
|
}
|
|
@@ -234,10 +677,13 @@ export class MeshHandler {
|
|
|
234
677
|
}
|
|
235
678
|
}
|
|
236
679
|
finally {
|
|
237
|
-
// A THROW IS NOT AN ENDING
|
|
238
|
-
// worthless; a throw leaves the step pending, and
|
|
239
|
-
// of where this run reached on the channel.
|
|
240
|
-
// refuse one (L5010, RunSuperseded), so a
|
|
680
|
+
// A THROW IS NOT AN ENDING — with one exception, and it marks itself. The three returns above
|
|
681
|
+
// are the wait being over and its position worthless; a throw leaves the step pending, and
|
|
682
|
+
// the consumer's position is the only record of where this run reached on the channel.
|
|
683
|
+
// `ctx.bind` is a journal append and a journal can refuse one (L5010, RunSuperseded), so a
|
|
684
|
+
// throw here is ordinary operation and not only a bug. The exception is `Cancelled`, which
|
|
685
|
+
// sets `over` before it leaves: a cancelled wait settles `cancelled` and replays as
|
|
686
|
+
// cancelled, so its position answers nothing ever again.
|
|
241
687
|
// Keeping the consumer costs one durable on an abandoned run, which is what a host crash
|
|
242
688
|
// already costs; reaping on inactivity instead could delete a live wait's position while its
|
|
243
689
|
// host was down.
|
|
@@ -249,6 +695,148 @@ export class MeshHandler {
|
|
|
249
695
|
}
|
|
250
696
|
}
|
|
251
697
|
}
|
|
698
|
+
/**
|
|
699
|
+
* `wait(down(agent))` — the death of one incarnation, read off presence liveness.
|
|
700
|
+
*
|
|
701
|
+
* The handle pins `<name>#<lifecycleUid>` and DOWN is a fact about the INCARNATION. The mesh's
|
|
702
|
+
* liveness witness is the presence row a seat heartbeats — TTL'd out of the bucket when the
|
|
703
|
+
* heartbeats stop — and it is the same source conclave membership resolves members through, so
|
|
704
|
+
* "down" here and "down or gone" at a conclave join are one definition. No live row carrying
|
|
705
|
+
* the name AND this incarnation is the death, with the reason split by what the name shows now:
|
|
706
|
+
* `"lapsed"` when nothing live holds the name any more, `"superseded"` when a live row holds it
|
|
707
|
+
* under a DIFFERENT incarnation — this incarnation dead with a successor already up.
|
|
708
|
+
*
|
|
709
|
+
* NOTHING BINDS, because a death is re-observable where a matched message is not: a lifecycle
|
|
710
|
+
* uid is minted once per incarnation and never heartbeats again after its row lapses, so a
|
|
711
|
+
* crash between the observation and the settle re-observes the same death on resume — at worst
|
|
712
|
+
* with the reason upgraded from `"lapsed"` to `"superseded"` by a successor that appeared in
|
|
713
|
+
* between. `at` is the time of OBSERVATION: presence records no time of death, and inventing
|
|
714
|
+
* one would be a value the planes cannot back.
|
|
715
|
+
*
|
|
716
|
+
* The TIMEOUT is the same mediated pause every wait arms — minted once with an absolute
|
|
717
|
+
* deadline under the step's own request id, so a resume ATTACHES to the recorded deadline
|
|
718
|
+
* rather than restarting the clock — and it resolves `null`, never a throw: `??` is
|
|
719
|
+
* `otherwise`. The discharge and the adoption re-arm already speak this wait's tokens (the
|
|
720
|
+
* `wait` kind arms under its request id), so a cancelled or adopted down-wait needs nothing of
|
|
721
|
+
* its own.
|
|
722
|
+
*/
|
|
723
|
+
async waitDown(ev, req, ctx) {
|
|
724
|
+
const { name, uid } = parseAgentHandle(ev.agent);
|
|
725
|
+
// `down(agent)` is an event AFTER `monitor` registered interest (cotal-lang 6.5); a wait on an
|
|
726
|
+
// agent nobody monitored would be a registration this run never recorded.
|
|
727
|
+
if (!this.monitored.has(`${name}#${uid}`))
|
|
728
|
+
throw new Error(`wait(down(${name}#${uid})) observes a monitored agent, and this run never performed monitor() on that handle; register interest first (cotal-lang 6.5)`);
|
|
729
|
+
const primary = req.timeout === undefined
|
|
730
|
+
? undefined
|
|
731
|
+
: { endpoint: this.binding.endpoint, token: ctx.requestId };
|
|
732
|
+
if (primary !== undefined)
|
|
733
|
+
await this.arm(primary, this.now() + parseDuration(req.timeout));
|
|
734
|
+
for (;;) {
|
|
735
|
+
if (ctx.signal.cancelled) {
|
|
736
|
+
if (primary !== undefined)
|
|
737
|
+
await this.cancelTimer(primary);
|
|
738
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
739
|
+
}
|
|
740
|
+
const ended = await this.expired(primary);
|
|
741
|
+
if (ended !== undefined)
|
|
742
|
+
return null;
|
|
743
|
+
let rows;
|
|
744
|
+
try {
|
|
745
|
+
rows = await this.presenceRows();
|
|
746
|
+
}
|
|
747
|
+
catch (e) {
|
|
748
|
+
// `liveKvEntries` REFUSES a pass cut short mid-scan instead of returning a partial view —
|
|
749
|
+
// and a partial view is exactly what must not decide a death. Its own contract says retry,
|
|
750
|
+
// so the next poll is the retry; a link that stays down keeps surfacing here rather than
|
|
751
|
+
// as a false DOWN, and the deadline above still ends the wait.
|
|
752
|
+
if (e instanceof IncompleteKvScan) {
|
|
753
|
+
await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
throw e;
|
|
757
|
+
}
|
|
758
|
+
if (!rows.some((p) => p.card?.name === name && p.lifecycleUid === uid)) {
|
|
759
|
+
const reason = rows.some((p) => p.card?.name === name) ? "superseded" : "lapsed";
|
|
760
|
+
if (primary !== undefined)
|
|
761
|
+
await this.cancelTimer(primary);
|
|
762
|
+
return { agent: ev.agent, reason, at: this.now() };
|
|
763
|
+
}
|
|
764
|
+
await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* `wait(replied(agent))` — the agent finished a reply, observed over THIS RUN's turn terminals.
|
|
769
|
+
*
|
|
770
|
+
* The honest observable a run holds for "finished a reply" is the terminal fact of a turn it
|
|
771
|
+
* relayed itself: a `succeeded` turn goal IS a completed reply, durable on the goal plane. The
|
|
772
|
+
* wait resolves off the run's turn-goal registry for the handle — fed by `turn` at submission
|
|
773
|
+
* and reseeded from the journal on adoption — and REPLIED IS A LEVEL, exactly as `down` is: a
|
|
774
|
+
* reply that already exists resolves the wait at once (the latest, by the yield's own `at`
|
|
775
|
+
* stamp), and a wait that begins before any reply parks for the next terminal. A denied or
|
|
776
|
+
* cancelled turn is not a reply — the agent never finished one — so it leaves the wait parked.
|
|
777
|
+
* A handle outside the run's roster with no recorded turn can never be observed (only this
|
|
778
|
+
* run's turns are), so it refuses loudly instead of parking on nothing.
|
|
779
|
+
*
|
|
780
|
+
* NOTHING BINDS, for `down`'s reason: goal terminals are durable facts, so a crash between the
|
|
781
|
+
* observation and the settle re-observes the same reply on resume — at worst a NEWER reply
|
|
782
|
+
* completed in between. The value is the observation record, `down`-shaped:
|
|
783
|
+
* `{ agent, status, note?, at }`, where `at` is the yield's own stamp. The TIMEOUT is the same
|
|
784
|
+
* mediated pause every wait arms, resolving `null`, never a throw.
|
|
785
|
+
*/
|
|
786
|
+
async waitReplied(ev, req, ctx) {
|
|
787
|
+
const { name, uid } = parseAgentHandle(ev.agent);
|
|
788
|
+
const handle = `${name}#${uid}`;
|
|
789
|
+
if (this.turnGoals.get(handle) === undefined) {
|
|
790
|
+
const entry = this.roster.get(name);
|
|
791
|
+
if (entry === undefined || entry.uid !== uid)
|
|
792
|
+
throw new Error(`wait(replied(${handle})) addresses an agent that is not in this run's roster; a reply is observed on a turn this run relayed`);
|
|
793
|
+
}
|
|
794
|
+
const primary = req.timeout === undefined
|
|
795
|
+
? undefined
|
|
796
|
+
: { endpoint: this.binding.endpoint, token: ctx.requestId };
|
|
797
|
+
if (primary !== undefined)
|
|
798
|
+
await this.arm(primary, this.now() + parseDuration(req.timeout));
|
|
799
|
+
const actx = await this.actionCtx();
|
|
800
|
+
for (;;) {
|
|
801
|
+
if (ctx.signal.cancelled) {
|
|
802
|
+
if (primary !== undefined)
|
|
803
|
+
await this.cancelTimer(primary);
|
|
804
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
805
|
+
}
|
|
806
|
+
const ended = await this.expired(primary);
|
|
807
|
+
if (ended !== undefined)
|
|
808
|
+
return null;
|
|
809
|
+
const goals = this.turnGoals.get(handle);
|
|
810
|
+
let latest;
|
|
811
|
+
for (const goalId of goals ?? []) {
|
|
812
|
+
const fact = await readGoalResult(actx, { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId });
|
|
813
|
+
if (fact === undefined)
|
|
814
|
+
continue;
|
|
815
|
+
// Denied (deadline) or cancelled: the agent never finished a reply. Stop reading it.
|
|
816
|
+
if (fact.state !== "succeeded") {
|
|
817
|
+
goals.delete(goalId);
|
|
818
|
+
continue;
|
|
819
|
+
}
|
|
820
|
+
const d = fact.data;
|
|
821
|
+
if ((d?.status !== "done" && d?.status !== "blocked" && d?.status !== "handoff") || typeof d.at !== "number")
|
|
822
|
+
throw new Error(`wait(replied(${handle})) observed a malformed turn terminal (${JSON.stringify(fact.data)}); a garbled yield never resolves a wait`);
|
|
823
|
+
// A handoff the turn refuses (an addressee outside the run, or across worktrees) is the
|
|
824
|
+
// turn's own failure, so it is not a reply here either: both observers read one verdict.
|
|
825
|
+
if (d.status === "handoff" && this.handoffRefusal(this.roster.get(name)?.handle.worktree, d.to) !== undefined) {
|
|
826
|
+
goals.delete(goalId);
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
if (latest === undefined || d.at > latest.at)
|
|
830
|
+
latest = { status: d.status, ...(typeof d.note === "string" ? { note: d.note } : {}), at: d.at };
|
|
831
|
+
}
|
|
832
|
+
if (latest !== undefined) {
|
|
833
|
+
if (primary !== undefined)
|
|
834
|
+
await this.cancelTimer(primary);
|
|
835
|
+
return { agent: ev.agent, ...latest };
|
|
836
|
+
}
|
|
837
|
+
await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
|
|
838
|
+
}
|
|
839
|
+
}
|
|
252
840
|
/**
|
|
253
841
|
* `notify` writes one bounded decision record per addressee, onto the run.
|
|
254
842
|
*
|
|
@@ -260,13 +848,23 @@ export class MeshHandler {
|
|
|
260
848
|
* **One call to N agents is N records, and a retry lands on its own.** The id is derived from the
|
|
261
849
|
* step's request id and the addressee, so a crash between the second and third write is repaired
|
|
262
850
|
* by re-running the call: the first two creates find their own bytes and return, the third
|
|
263
|
-
* happens. Nothing is written twice and nothing needs a memo of how far it got.
|
|
851
|
+
* happens. Nothing is written twice and nothing needs a memo of how far it got. That holds only
|
|
852
|
+
* because the instant every notice carries is bound with the step rather than read from the
|
|
853
|
+
* clock again, which is the one field a second pass could otherwise change.
|
|
264
854
|
*
|
|
265
855
|
* The fact's bound is the language's and is enforced BEFORE this is reached (L3043 at the effect
|
|
266
856
|
* boundary), so a fact that could not be rendered as one table row cannot arrive here.
|
|
267
857
|
*/
|
|
268
858
|
async notify(req, ctx) {
|
|
269
|
-
|
|
859
|
+
// THE INSTANT IS BOUND, never re-read. `writeRunNotice` is create-only and compares canonical
|
|
860
|
+
// bytes, so a second pass carrying a fresh clock reading hits `conflict` on the notice the
|
|
861
|
+
// first pass already wrote, and every retry after that hits it identically: the run wedges on
|
|
862
|
+
// the one effect whose whole design is that re-running it is safe. A crash between the write
|
|
863
|
+
// and the settling append is ordinary, so the retry has to rewrite the same bytes.
|
|
864
|
+
const recorded = ctx.resume?.at;
|
|
865
|
+
const at = typeof recorded === "number" ? recorded : this.now();
|
|
866
|
+
if (typeof recorded !== "number")
|
|
867
|
+
await ctx.bind({ at });
|
|
270
868
|
const step = stepKeyString(ctx.key);
|
|
271
869
|
for (const agent of req.agents) {
|
|
272
870
|
const noticeId = runNoticeId(ctx.requestId, agent.agent);
|
|
@@ -281,65 +879,840 @@ export class MeshHandler {
|
|
|
281
879
|
}
|
|
282
880
|
return null;
|
|
283
881
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
882
|
+
/**
|
|
883
|
+
* `spawn` is the manager's spawn ACTION, submitted under the step's own identity.
|
|
884
|
+
*
|
|
885
|
+
* **The request id is the goalId.** The envelope id is pinned to `ctx.requestId`, and the
|
|
886
|
+
* manager binds its goal under the envelope id — so a resumed run that re-submits is served the
|
|
887
|
+
* RECORDED acceptance (same fingerprint, same goal) instead of allocating a second seat, and the
|
|
888
|
+
* goal's terminal fact sits on a subject this run can re-derive from nothing but its journal.
|
|
889
|
+
*
|
|
890
|
+
* The ACCEPTANCE is bound as the entry's external state before the terminal is awaited, so a
|
|
891
|
+
* crash mid-await resumes straight into the poll — it must NOT re-invoke, because by then the
|
|
892
|
+
* manager may have restarted and a fresh submission would be judged against a live seat rather
|
|
893
|
+
* than served from its acceptance cache. The terminal await itself is a read of a durable fact,
|
|
894
|
+
* deliberately not `submitAndFollowGoal`: a live progress subscription dies with the process,
|
|
895
|
+
* and the fact is the thing a resume can still read.
|
|
896
|
+
*
|
|
897
|
+
* `permits`, `supervise` and `onFork` are POLICY, not identity (§6.4): they ride the journalled
|
|
898
|
+
* request and are enforced where they bind (`permits` at `turn`, `supervise` by `monitor`, an
|
|
899
|
+
* `onFork` at fork adoption) — nothing about them travels in the submission.
|
|
900
|
+
*/
|
|
901
|
+
async spawn(req, ctx) {
|
|
902
|
+
if (ctx.signal.cancelled)
|
|
903
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
904
|
+
const goalId = ctx.requestId;
|
|
905
|
+
// A recorded goalId is a previous attempt's ACCEPTANCE: the submission landed and its
|
|
906
|
+
// identity was bound before the crash. Go straight back to the terminal. An entry that says
|
|
907
|
+
// `adoptedFrom` names the goal the ORPHANED spawn submitted, not this step's own request id:
|
|
908
|
+
// that goal is the terminal this step reads on every pass, and no pass ever submits one.
|
|
909
|
+
const recorded = ctx.resume;
|
|
910
|
+
let ext = typeof recorded?.goalId === "string" && (recorded.goalId === goalId || typeof recorded.adoptedFrom === "string")
|
|
911
|
+
? recorded
|
|
912
|
+
: undefined;
|
|
913
|
+
// A budget this host cannot meter is refused before anything is submitted: accepting it and
|
|
914
|
+
// enforcing nothing would be the silent no-op the effect table exists to prevent.
|
|
915
|
+
const permits = req.permits !== undefined ? readPermits(req.permits, req.persona) : undefined;
|
|
916
|
+
// Same rule, for the other policy option. `supervise` is "a declarative restart policy" in the
|
|
917
|
+
// reference and nothing more: it names no keys, no restart semantics and no code, so there is
|
|
918
|
+
// nothing here to enforce and no way to invent it without writing language semantics into a
|
|
919
|
+
// host. Accepting it and restarting nothing is the silent no-op `readPermits` refuses by name.
|
|
920
|
+
if (req.supervise !== undefined)
|
|
921
|
+
throw new Error(`spawn(${req.persona}): supervise is a restart policy this host does not implement, and a policy it cannot enforce is refused rather than ignored`);
|
|
922
|
+
// A seat a migration kept for this persona (§11.2): the orphaned spawn's GOAL becomes this
|
|
923
|
+
// step's own, bound with that spawn's floor and the step it came from. From here the two kinds
|
|
924
|
+
// of spawn are one path — the terminal is read under the bound goal, the handle comes from it,
|
|
925
|
+
// a resume re-reads it, and a discharge despawns by it — so the hand-over is spent exactly
|
|
926
|
+
// once and this step never mints a second seat for a persona it was handed one for.
|
|
927
|
+
const adopted = ext === undefined ? this.adoptable.get(req.persona)?.shift() : undefined;
|
|
928
|
+
if (adopted !== undefined) {
|
|
929
|
+
const handle = adopted.result;
|
|
930
|
+
const floor = adopted.external;
|
|
931
|
+
if (typeof handle?.agent !== "string" || typeof adopted.requestId !== "string")
|
|
932
|
+
throw new Error(`spawn(${req.persona}): the migrated seat's entry carries no readable handle or goal; a garbled hand-over is refused rather than minted over`);
|
|
933
|
+
if (req.worktree !== undefined && handle.worktree !== req.worktree)
|
|
934
|
+
throw new Error(`spawn(${req.persona}) asks for worktree "${req.worktree}" but the seat the migration hands it holds ${handle.worktree === undefined ? "none" : `"${handle.worktree}"`}; an adopted seat keeps its tree, so the edited program names that tree or none`);
|
|
935
|
+
ext = {
|
|
936
|
+
goalId: adopted.requestId,
|
|
937
|
+
...(floor !== undefined ? pickAcceptanceFloor(floor) : {}),
|
|
938
|
+
...(handle.worktree !== undefined ? { worktree: handle.worktree } : {}),
|
|
939
|
+
...(req.onFork !== undefined ? { onFork: req.onFork } : {}),
|
|
940
|
+
...(req.permits !== undefined ? { permits: req.permits } : {}),
|
|
941
|
+
spawnedAt: typeof floor?.spawnedAt === "number" ? floor.spawnedAt : this.now(),
|
|
942
|
+
adoptedFrom: journalEntryKeyString(adopted),
|
|
943
|
+
};
|
|
944
|
+
await ctx.bind(ext);
|
|
945
|
+
}
|
|
946
|
+
// The goal this step reads its terminal under: its own submission, or the adopted spawn's.
|
|
947
|
+
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId: typeof ext?.goalId === "string" ? ext.goalId : goalId };
|
|
948
|
+
if (ext === undefined && req.worktree !== undefined)
|
|
949
|
+
await this.claimWorktree(req.worktree, req.persona, goalId);
|
|
950
|
+
try {
|
|
951
|
+
if (ext === undefined) {
|
|
952
|
+
let reply;
|
|
953
|
+
try {
|
|
954
|
+
const service = await this.manager();
|
|
955
|
+
reply = await invokeCommand(this.nc, this.binding.space, service, "spawn", spawnArgs(req), {
|
|
956
|
+
id: goalId,
|
|
957
|
+
deadlineMs: SPAWN_ACCEPT_DEADLINE_MS,
|
|
958
|
+
});
|
|
959
|
+
}
|
|
960
|
+
catch (err) {
|
|
961
|
+
// The invoke did not come back — which does not prove nothing happened: the request may
|
|
962
|
+
// have been accepted while the reply was lost. The goal record is the arbiter: a durable
|
|
963
|
+
// trace under this goalId means the submission landed, so proceed to its terminal; none
|
|
964
|
+
// means it never did, and the raised error is the honest outcome.
|
|
965
|
+
if ((await readGoalStatus(await this.actionCtx(), ref)) === undefined)
|
|
966
|
+
throw err;
|
|
967
|
+
}
|
|
968
|
+
if (reply !== undefined && reply.reply.ok === false) {
|
|
969
|
+
// Refused AT ACCEPT: the manager bound no goal and provisioned nothing, so this is the
|
|
970
|
+
// effect's own failure, catchable as such — and never L4002, because no agent existed to
|
|
971
|
+
// be down. A seat budget the endpoint will not extend is a permit the run does not hold
|
|
972
|
+
// (L4001); every other refusal (persona unknown, name taken) is the host declining the
|
|
973
|
+
// request (L4000), the manager's own code carried in the detail.
|
|
974
|
+
const err = reply.reply.error;
|
|
975
|
+
throw new EffectError(err?.code === "resource-exhausted" ? "L4001" : "L4000", "spawn", `spawn(${req.persona}) was refused by the ${this.binding.endpoint} endpoint: ${err?.message ?? "refused with no message"}`, err?.code !== undefined ? { code: err.code } : undefined);
|
|
976
|
+
}
|
|
977
|
+
const floor = reply === undefined ? undefined : reply.reply.data;
|
|
978
|
+
if (floor !== undefined && floor.goalId !== goalId)
|
|
979
|
+
throw new Error(`the spawn acceptance names goal "${String(floor.goalId)}" but this submission pinned "${goalId}"; a mismatched acceptance never authorizes (SPEC 13.6)`);
|
|
980
|
+
// BIND BEFORE AWAITING: the goalId is re-derivable, but the bind is what tells a resume the
|
|
981
|
+
// submission LANDED — and the identity floor beside it is what a discharge despawns by when
|
|
982
|
+
// the terminal alone carries none (an `uncertain` verdict). The worktree rides along so an
|
|
983
|
+
// adoption re-takes this spawn's reservation; `onFork` so a fork can read the policy.
|
|
984
|
+
ext = {
|
|
985
|
+
goalId,
|
|
986
|
+
...(floor !== undefined ? pickAcceptanceFloor(floor) : {}),
|
|
987
|
+
...(req.worktree !== undefined ? { worktree: req.worktree } : {}),
|
|
988
|
+
...(req.onFork !== undefined ? { onFork: req.onFork } : {}),
|
|
989
|
+
...(req.permits !== undefined ? { permits: req.permits } : {}),
|
|
990
|
+
spawnedAt: this.now(),
|
|
991
|
+
};
|
|
992
|
+
await ctx.bind(ext);
|
|
993
|
+
}
|
|
994
|
+
const fact = await this.goalTerminal(ref, ctx.signal);
|
|
995
|
+
const handle = spawnHandleOf(req, ext, fact, this.binding.endpoint);
|
|
996
|
+
// Register the run-roster entry `turn` addresses and a handoff resolves to. The owner/actor
|
|
997
|
+
// address prefers the bound floor and falls back to the terminal's own recorded identity —
|
|
998
|
+
// the same discipline the discharge uses (see spawnDespawnTarget).
|
|
999
|
+
const address = spawnDespawnTarget(ext, fact);
|
|
1000
|
+
this.roster.set(parseAgentHandle(handle.agent).name, {
|
|
1001
|
+
handle,
|
|
1002
|
+
uid: parseAgentHandle(handle.agent).uid,
|
|
1003
|
+
...(address !== undefined ? { owner: address.owner, actor: address.actor } : {}),
|
|
1004
|
+
...(permits !== undefined ? { permits } : {}),
|
|
1005
|
+
...(typeof ext.spawnedAt === "number" ? { spawnedAt: ext.spawnedAt } : {}),
|
|
1006
|
+
});
|
|
1007
|
+
if (typeof handle.worktree === "string")
|
|
1008
|
+
this.worktreeHolders.set(handle.worktree, { name: parseAgentHandle(handle.agent).name, uid: parseAgentHandle(handle.agent).uid });
|
|
1009
|
+
return handle;
|
|
1010
|
+
}
|
|
1011
|
+
catch (e) {
|
|
1012
|
+
const tree = typeof ext?.worktree === "string" ? ext.worktree : req.worktree;
|
|
1013
|
+
if (tree !== undefined)
|
|
1014
|
+
this.releaseWorktree(tree, ref.goalId, ext, e instanceof Cancelled);
|
|
1015
|
+
throw e;
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
/**
|
|
1019
|
+
* A spawn that ends without a handle gives its tree back. The one exception is a cancellation
|
|
1020
|
+
* while parked on an ACCEPTED submission: the seat may well be up in the tree until its
|
|
1021
|
+
* discharge lands, so the reservation becomes a hold by the accepted identity, which presence
|
|
1022
|
+
* liveness then releases the way it releases any holder.
|
|
1023
|
+
*/
|
|
1024
|
+
releaseWorktree(worktree, goalId, ext, parked) {
|
|
1025
|
+
const holder = this.worktreeHolders.get(worktree);
|
|
1026
|
+
if (holder === undefined || !("pending" in holder) || holder.pending !== goalId)
|
|
1027
|
+
return;
|
|
1028
|
+
if (parked && typeof ext?.name === "string" && typeof ext.uid === "string") {
|
|
1029
|
+
this.worktreeHolders.set(worktree, { name: ext.name, uid: ext.uid });
|
|
1030
|
+
return;
|
|
1031
|
+
}
|
|
1032
|
+
this.worktreeHolders.delete(worktree);
|
|
1033
|
+
}
|
|
1034
|
+
/** Poll the goal's durable terminal fact, observing cancellation on the poll cadence — the same
|
|
1035
|
+
* discipline as `wait`: a race decided against this branch stops parking within one poll, and
|
|
1036
|
+
* the seat its acceptance may have produced is the DISCHARGE's to release, not this branch's. */
|
|
1037
|
+
async goalTerminal(ref, signal) {
|
|
1038
|
+
const actx = await this.actionCtx();
|
|
1039
|
+
for (;;) {
|
|
1040
|
+
if (signal.cancelled)
|
|
1041
|
+
throw new Cancelled(signal.reason ?? "cancelled");
|
|
1042
|
+
const fact = await readGoalResult(actx, ref);
|
|
1043
|
+
if (fact !== undefined)
|
|
1044
|
+
return fact;
|
|
1045
|
+
await new Promise((r) => setTimeout(r, GOAL_POLL_MS).unref());
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* `turn` wakes ONE AGENT for one turn, over the manager's relay (a seat is not an endpoint):
|
|
1050
|
+
* the manager accepts the goal, parks the rendered context durably, the seat pulls and yields
|
|
1051
|
+
* under its own self reach, and the yield is this goal's terminal. The request id is the goalId,
|
|
1052
|
+
* the same recovery discipline as `spawn`: a resumed run re-enters the poll, never re-submits.
|
|
1053
|
+
*
|
|
1054
|
+
* THREE AUTHORITIES END IT, one per ending. The seat's yield is the manager's `succeeded`
|
|
1055
|
+
* terminal, carrying the TurnResult. The DEADLINE is double-covered: the manager arms a
|
|
1056
|
+
* goal-bound hold (its expiry commits `failed` reason `turn-deadline`), and this client arms its
|
|
1057
|
+
* OWN pause under the step's request id — the L4003 authority that survives a dead manager.
|
|
1058
|
+
* DEATH likewise: the manager's reap hook fails pending turns `agent-down`, and this client
|
|
1059
|
+
* watches presence itself (the L4002 authority when the manager died with the seat).
|
|
1060
|
+
*
|
|
1061
|
+
* Handoff honoring (lang §5.3) happens HERE: the scope's pending memo is spent at every turn's
|
|
1062
|
+
* begin, and when this turn targets its `to`, the link rides the submission (`handoffFrom`, the
|
|
1063
|
+
* manager mirrors it into the terminal) and the bound external state. A handoff YIELD is
|
|
1064
|
+
* validated against the run roster: an addressee outside it is L4005, one in a different
|
|
1065
|
+
* worktree is L4004.
|
|
1066
|
+
*/
|
|
1067
|
+
async turn(req, ctx) {
|
|
1068
|
+
if (ctx.signal.cancelled)
|
|
1069
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1070
|
+
const { name, uid } = parseAgentHandle(req.agent.agent);
|
|
1071
|
+
const goalId = ctx.requestId;
|
|
1072
|
+
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
1073
|
+
const primary = { endpoint: this.binding.endpoint, token: goalId };
|
|
1074
|
+
const scope = scopeOf(ctx.key);
|
|
1075
|
+
let ext = ctx.resume?.goalId === goalId ? ctx.resume : undefined;
|
|
1076
|
+
if (ext === undefined) {
|
|
1077
|
+
const seat = this.seatAddress(name, uid);
|
|
1078
|
+
if (seat.address === undefined) {
|
|
1079
|
+
if (seat.why === "not-in-roster")
|
|
1080
|
+
throw new Error(`turn(${name}#${uid}) addresses an agent that is not in this run's roster; a turn wakes an agent this run spawned`);
|
|
1081
|
+
throw new Error(`turn(${name}#${uid}) has no address: the spawn's acceptance floor was never served, so the seat's owner/actor coordinates are unknown`);
|
|
1082
|
+
}
|
|
1083
|
+
const { owner, actor } = seat.address;
|
|
1084
|
+
// The permits are the roster entry's, not the address's: a budget is spent per spawn, and
|
|
1085
|
+
// the address is only where this turn is sent.
|
|
1086
|
+
const entry = this.roster.get(name);
|
|
1087
|
+
// Spend the scope's handoff memo at BEGIN, honored or not — honoring is immediate-only.
|
|
1088
|
+
const memo = this.handoffMemos.get(scope);
|
|
1089
|
+
this.handoffMemos.delete(scope);
|
|
1090
|
+
const handoffFrom = memo !== undefined && memo !== "ambiguous" && memo.to === name ? memo.fromGoalId : undefined;
|
|
1091
|
+
// Render the seat's context: every unconsumed notice addressed to it, as one durable payload.
|
|
1092
|
+
const step = stepKeyString(ctx.key);
|
|
1093
|
+
// Addressed by the HANDLE COMPOSITE, exactly as `notify` filed them (an incarnation is the
|
|
1094
|
+
// addressee, and a successor under the name is somebody else).
|
|
1095
|
+
const notices = (await listRunNotices(this.kv, this.binding.endpoint, this.binding.runId, req.agent.agent))
|
|
1096
|
+
.filter((n) => n.consumed === undefined);
|
|
1097
|
+
const context = renderRunContext({ run: this.binding.runId, step, notices });
|
|
1098
|
+
const deadlineMs = parseDuration(req.deadline ?? this.binding.defaultCheckpointTimeout);
|
|
1099
|
+
// The agent's permits are budgets this run spends at the call that would exceed them
|
|
1100
|
+
// (cotal-lang 6.5, L4001): one turn per `turns`, and a turn whose deadline runs past the
|
|
1101
|
+
// agent's remaining wall clock is already over it. Counted on the fresh path only: a resume
|
|
1102
|
+
// re-enters a turn the seed already counted from its journal entry.
|
|
1103
|
+
const taken = this.turnsTaken.get(`${name}#${uid}`) ?? 0;
|
|
1104
|
+
if (entry.permits?.turns !== undefined && taken >= entry.permits.turns)
|
|
1105
|
+
throw new EffectError("L4001", "permit-turns", `turn(${name}#${uid}) would be that agent's turn ${taken + 1}, past its permit of ${entry.permits.turns}`);
|
|
1106
|
+
if (entry.permits?.wallClockMs !== undefined && entry.spawnedAt !== undefined) {
|
|
1107
|
+
const remaining = entry.spawnedAt + entry.permits.wallClockMs - this.now();
|
|
1108
|
+
if (remaining <= 0)
|
|
1109
|
+
throw new EffectError("L4001", "permit-wall-clock", `turn(${name}#${uid}): the agent's wall-clock permit (${entry.permits.wallClockMs}ms from its spawn) is spent`);
|
|
1110
|
+
if (deadlineMs > remaining)
|
|
1111
|
+
throw new EffectError("L4001", "permit-wall-clock", `turn(${name}#${uid}): a ${deadlineMs}ms deadline runs past the agent's remaining wall clock (${remaining}ms of its ${entry.permits.wallClockMs}ms permit)`);
|
|
1112
|
+
}
|
|
1113
|
+
const payload = JSON.stringify({ run: this.binding.runId, step, context, noticeIds: notices.map((n) => n.noticeId) });
|
|
1114
|
+
const submit = async () => invokeCommand(this.nc, this.binding.space, await this.manager(), "turn", { payload, deadlineMs, ...(handoffFrom !== undefined ? { handoffFrom } : {}) }, {
|
|
1115
|
+
id: goalId,
|
|
1116
|
+
deadlineMs: TURN_ACCEPT_DEADLINE_MS,
|
|
1117
|
+
target: { mode: "owner", owner, actor, lifecycleUid: uid },
|
|
1118
|
+
});
|
|
1119
|
+
let reply;
|
|
1120
|
+
try {
|
|
1121
|
+
reply = await submit();
|
|
1122
|
+
}
|
|
1123
|
+
catch (err) {
|
|
1124
|
+
// The invoke did not come back — the goal record is the arbiter, exactly as in `spawn`.
|
|
1125
|
+
if ((await readGoalStatus(await this.actionCtx(), ref)) === undefined)
|
|
1126
|
+
throw err;
|
|
1127
|
+
}
|
|
1128
|
+
// ONE reading of a refused acceptance, for both submits: a seat that dies between the first
|
|
1129
|
+
// submit and the re-read is the same dead seat, and the resubmit path classing it as an
|
|
1130
|
+
// ordinary refusal made an L4002 the program can catch arrive as an uncatchable L4000.
|
|
1131
|
+
const refusal = (r, resubmitted) => {
|
|
1132
|
+
const err = r.reply.error;
|
|
1133
|
+
// `expired` is the serve boundary's "target is not a live managed agent": the seat is gone.
|
|
1134
|
+
if (err?.code === "expired")
|
|
1135
|
+
return new EffectError("L4002", "turn", `turn(${name}#${uid}) found the agent down before the relay began: ${err.message}`);
|
|
1136
|
+
return new Error(resubmitted
|
|
1137
|
+
? `turn(${name}#${uid}) landed but its acceptance could not be re-read: ${err?.message ?? "refused with no message"}`
|
|
1138
|
+
: `turn(${name}#${uid}) was refused by the ${this.binding.endpoint} endpoint: ${err?.message ?? "refused with no message"}`);
|
|
1139
|
+
};
|
|
1140
|
+
if (reply !== undefined && reply.reply.ok === false)
|
|
1141
|
+
throw refusal(reply, false);
|
|
1142
|
+
if (reply === undefined) {
|
|
1143
|
+
// The acceptance carries the deadline authority, so a lost reply is asked for again: the
|
|
1144
|
+
// same goalId under the same fingerprint is served from the manager's acceptance cache.
|
|
1145
|
+
reply = await submit();
|
|
1146
|
+
if (reply.reply.ok === false)
|
|
1147
|
+
throw refusal(reply, true);
|
|
1148
|
+
}
|
|
1149
|
+
const floor = reply.reply.data;
|
|
1150
|
+
if (typeof floor?.deadlineAt !== "number")
|
|
1151
|
+
throw new Error(`turn(${name}#${uid}) was accepted with no readable deadline (${JSON.stringify(reply.reply.data)}); a garbled acceptance never authorizes (SPEC 13.6)`);
|
|
1152
|
+
// ONE deadline for the turn: the acceptance's `deadlineAt` is the instant the manager's own
|
|
1153
|
+
// hold denies at, and the client's pause is armed on that same instant. A second clock read
|
|
1154
|
+
// before the accept round-trip would fire first by the round-trip plus skew and throw away a
|
|
1155
|
+
// yield the manager's window still admitted.
|
|
1156
|
+
const deadlineAt = floor.deadlineAt;
|
|
1157
|
+
ext = {
|
|
1158
|
+
goalId, name, owner, actor, uid, deadlineAt,
|
|
1159
|
+
noticeIds: notices.map((n) => n.noticeId),
|
|
1160
|
+
...(handoffFrom !== undefined ? { handoffFrom } : {}),
|
|
1161
|
+
};
|
|
1162
|
+
await ctx.bind(ext);
|
|
1163
|
+
// Spent once the turn is RECORDED, which is the same event the adoption seed counts. Spending
|
|
1164
|
+
// it before the submit charged a permit for a turn the seat never saw (a dead seat's L4002,
|
|
1165
|
+
// an endpoint refusal), and a recovering run would then read a different number off the
|
|
1166
|
+
// journal than the live one held.
|
|
1167
|
+
this.turnsTaken.set(`${name}#${uid}`, taken + 1);
|
|
1168
|
+
}
|
|
1169
|
+
this.recordTurnGoal(`${name}#${uid}`, goalId);
|
|
1170
|
+
const deadlineAt = ext.deadlineAt;
|
|
1171
|
+
if (typeof deadlineAt !== "number")
|
|
1172
|
+
throw new Error(`turn(${name}#${uid}) resumed with no recorded deadline; a garbled external state never authorizes`);
|
|
1173
|
+
// The client's OWN deadline authority: minted on the first pass, attached to on re-entry.
|
|
1174
|
+
await this.arm(primary, deadlineAt);
|
|
1175
|
+
const actx = await this.actionCtx();
|
|
1176
|
+
try {
|
|
1177
|
+
for (;;) {
|
|
1178
|
+
if (ctx.signal.cancelled) {
|
|
1179
|
+
await this.cancelTimer(primary);
|
|
1180
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1181
|
+
}
|
|
1182
|
+
const fact = await readGoalResult(actx, ref);
|
|
1183
|
+
if (fact !== undefined) {
|
|
1184
|
+
await this.cancelTimer(primary);
|
|
1185
|
+
return await this.turnOutcome(req, fact, ext, scope, name, uid);
|
|
1186
|
+
}
|
|
1187
|
+
const ended = await this.expired(primary);
|
|
1188
|
+
if (ended !== undefined)
|
|
1189
|
+
throw new EffectError("L4003", "turn-deadline", `turn(${name}#${uid}) deadline elapsed before a yield`);
|
|
1190
|
+
let rows;
|
|
1191
|
+
try {
|
|
1192
|
+
rows = await this.presenceRows();
|
|
1193
|
+
}
|
|
1194
|
+
catch (e) {
|
|
1195
|
+
if (e instanceof IncompleteKvScan) {
|
|
1196
|
+
await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
|
|
1197
|
+
continue;
|
|
1198
|
+
}
|
|
1199
|
+
throw e;
|
|
1200
|
+
}
|
|
1201
|
+
if (!rows.some((pr) => pr.card?.name === name && pr.lifecycleUid === uid)) {
|
|
1202
|
+
const reason = rows.some((pr) => pr.card?.name === name) ? "superseded" : "lapsed";
|
|
1203
|
+
await this.cancelTimer(primary);
|
|
1204
|
+
throw new EffectError("L4002", "turn", `turn(${name}#${uid}) found the agent down (${reason}) before a yield`);
|
|
1205
|
+
}
|
|
1206
|
+
await new Promise((r) => setTimeout(r, WAIT_POLL_MS).unref());
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
catch (e) {
|
|
1210
|
+
// A turn that ends on the run's side without a yield the run accepted (its deadline, the
|
|
1211
|
+
// seat's death, a refused handoff, a cancellation) is not a reply, whatever the relay
|
|
1212
|
+
// records later: `wait(replied)` reads the verdict this branch raised, never a stale yield.
|
|
1213
|
+
this.turnGoals.get(`${name}#${uid}`)?.delete(goalId);
|
|
1214
|
+
throw e;
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
/** Map a turn goal's terminal onto the effect's contract: `succeeded` carries the TurnResult
|
|
1218
|
+
* (a handoff addressee resolved against the roster — L4005 outside it, L4004 across
|
|
1219
|
+
* worktrees), `failed` splits on the manager's recorded reason, `cancelled` unwinds. The
|
|
1220
|
+
* consumed notices are marked HERE, by the goal that carried them, tolerating the re-entry
|
|
1221
|
+
* conflict (a crash between the terminal and the mark re-marks on resume). */
|
|
1222
|
+
/**
|
|
1223
|
+
* A yielded handoff's refusal class: L4005 when the addressee is not in this run's roster,
|
|
1224
|
+
* L4004 when it sits in a different worktree than the seat handing off, undefined when the
|
|
1225
|
+
* handoff is honorable. One reading for the turn that raises it and the `wait(replied)` that
|
|
1226
|
+
* must not count it, so the two observers never diverge on a spoofed or misdirected `to`.
|
|
1227
|
+
*/
|
|
1228
|
+
handoffRefusal(fromWorktree, to) {
|
|
1229
|
+
const toName = typeof to === "string" ? to : "";
|
|
1230
|
+
const target = this.roster.get(toName);
|
|
1231
|
+
if (toName.length === 0 || target === undefined)
|
|
1232
|
+
return "L4005";
|
|
1233
|
+
if ((target.handle.worktree ?? undefined) !== (fromWorktree ?? undefined))
|
|
1234
|
+
return "L4004";
|
|
1235
|
+
return undefined;
|
|
1236
|
+
}
|
|
1237
|
+
async turnOutcome(req, fact, ext, scope, name, uid) {
|
|
1238
|
+
if (fact.state === "cancelled")
|
|
1239
|
+
throw new Cancelled(`the turn goal for ${name}#${uid} was cancelled`);
|
|
1240
|
+
if (fact.state === "failed") {
|
|
1241
|
+
const d = fact.data;
|
|
1242
|
+
// The one reasoned failure a relay commits. A dead target has no early terminal of its own
|
|
1243
|
+
// on the goal plane (SPEC 13.6 item 7): the client's presence watch in the poll loop is the
|
|
1244
|
+
// first L4002 authority, and a death that rode to the deadline arrives on this same deny,
|
|
1245
|
+
// MARKED by the relay (`agentDownAt`) — read here as the agent-down failure it is, never as
|
|
1246
|
+
// a deadline the program could have set longer.
|
|
1247
|
+
if (d?.reason === "turn-deadline") {
|
|
1248
|
+
if (typeof d.agentDownAt === "number")
|
|
1249
|
+
throw new EffectError("L4002", "turn", `turn(${name}#${uid}) found the agent down (it left presence at ${new Date(d.agentDownAt).toISOString()}) before a yield; the deadline terminal carries its death`);
|
|
1250
|
+
throw new EffectError("L4003", "turn-deadline", `turn(${name}#${uid}) deadline elapsed before a yield`);
|
|
1251
|
+
}
|
|
1252
|
+
throw new Error(`turn(${name}#${uid}) failed at the ${this.binding.endpoint} endpoint: ${typeof d?.error === "string" ? d.error : JSON.stringify(fact.data)}`);
|
|
1253
|
+
}
|
|
1254
|
+
if (fact.state !== "succeeded")
|
|
1255
|
+
throw new Error(`turn(${name}#${uid}) settled ${fact.state}; a turn's yield is a succeeded terminal or a reasoned failure, never this`);
|
|
1256
|
+
const d = fact.data;
|
|
1257
|
+
if ((d?.status !== "done" && d?.status !== "blocked" && d?.status !== "handoff") || typeof d.at !== "number")
|
|
1258
|
+
throw new Error(`turn(${name}#${uid}) succeeded with a malformed TurnResult (${JSON.stringify(fact.data)}); a garbled terminal never yields a result`);
|
|
1259
|
+
let to;
|
|
1260
|
+
if (d.status === "handoff") {
|
|
1261
|
+
const toName = typeof d.to === "string" ? d.to : "";
|
|
1262
|
+
const refusal = this.handoffRefusal(req.agent.worktree, d.to);
|
|
1263
|
+
if (refusal === "L4005")
|
|
1264
|
+
throw new EffectError("L4005", "turn-handoff", `turn(${name}#${uid}) yielded a handoff to "${toName}", which is not in this run's roster`);
|
|
1265
|
+
const target = this.roster.get(toName);
|
|
1266
|
+
if (refusal === "L4004")
|
|
1267
|
+
throw new EffectError("L4004", "turn-handoff", `turn(${name}#${uid}) yielded a handoff to "${toName}" across worktrees (${req.agent.worktree ?? "none"} -> ${target.handle.worktree ?? "none"}); you cannot hand someone a working tree they are not in`);
|
|
1268
|
+
to = target.handle;
|
|
1269
|
+
}
|
|
1270
|
+
// The notices this turn carried are consumed by THIS goal — the create-only CAS arbitrates,
|
|
1271
|
+
// and a re-entry's conflict reads as "already recorded", never as a failure.
|
|
1272
|
+
const noticeIds = Array.isArray(ext.noticeIds) ? ext.noticeIds.filter((n) => typeof n === "string") : [];
|
|
1273
|
+
for (const noticeId of noticeIds) {
|
|
1274
|
+
try {
|
|
1275
|
+
await markRunNoticeConsumed(this.kv, this.binding.endpoint, this.binding.runId, req.agent.agent, noticeId, String(ext.goalId), this.now());
|
|
1276
|
+
}
|
|
1277
|
+
catch (e) {
|
|
1278
|
+
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
1279
|
+
throw e;
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
if (d.status === "handoff" && to !== undefined)
|
|
1283
|
+
this.recordHandoffMemo(scope, parseAgentHandle(to.agent).name, String(ext.goalId));
|
|
1284
|
+
return {
|
|
1285
|
+
status: d.status,
|
|
1286
|
+
...(to !== undefined ? { to } : {}),
|
|
1287
|
+
...(typeof d.note === "string" ? { note: d.note } : {}),
|
|
1288
|
+
at: d.at,
|
|
1289
|
+
};
|
|
1290
|
+
}
|
|
1291
|
+
/**
|
|
1292
|
+
* `ask` is a schema-checked pause the ADDRESSED AGENT is expected to answer.
|
|
1293
|
+
*
|
|
1294
|
+
* Each attempt is one checkpoint-plane pause — the same mint, settle and answer record a
|
|
1295
|
+
* `checkpoint` rides, answered through the run driver's `resolveCheckpoint` (`cotal run
|
|
1296
|
+
* answer`), which is how "the agent publishes a record" reaches a holder-bound plane: the agent,
|
|
1297
|
+
* or anyone the run's ACL admits on its behalf, answers through the driver exactly as a human
|
|
1298
|
+
* answers a checkpoint. What `ask` adds is the handler-side contract of §6.5: the schema is read
|
|
1299
|
+
* as the SHORTHAND (refused L4022 when it cannot be), every answer is checked against it, a
|
|
1300
|
+
* non-conforming answer costs one attempt and the refusal reason is bound onto the entry for the
|
|
1301
|
+
* answerer to read, and exhausted attempts are the effect's own catchable L4006.
|
|
1302
|
+
*
|
|
1303
|
+
* **One absolute deadline for the whole ask**, computed once and bound with the first attempt: a
|
|
1304
|
+
* re-ask does not restart the clock, or a stream of non-conforming answers could stretch the
|
|
1305
|
+
* pause forever. The deadline elapsing with no conforming answer is the ask's own L4006, the
|
|
1306
|
+
* same outcome exhausted attempts name: the budget it ran out of is the ask's, and L4003 belongs
|
|
1307
|
+
* to a `turn` whose own deadline elapsed.
|
|
1308
|
+
*
|
|
1309
|
+
* **Attempt N's token derives from the step's request id** (attempt 1 IS the request id), and
|
|
1310
|
+
* the CURRENT attempt's token is bound as `askToken` before its pause is armed — so a resume
|
|
1311
|
+
* re-enters the attempt in flight, an answer judged non-conforming just before a crash is
|
|
1312
|
+
* re-judged identically from its durable settle, and the discharge and the adoption re-arm
|
|
1313
|
+
* address the one timer that is actually armed.
|
|
1314
|
+
*/
|
|
1315
|
+
async ask(req, ctx) {
|
|
1316
|
+
if (ctx.signal.cancelled)
|
|
1317
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1318
|
+
const shape = askSchemaShape(req.schema);
|
|
1319
|
+
if (shape === null) {
|
|
1320
|
+
throw new EffectError("L4022", "ask-schema-unreadable", `L4022 Unreadable ask schema\n\n step ${stepKeyString(ctx.key)}\n\nThe schema is not the shorthand this handler enforces: a record mapping each field name to one of "string", "number", "boolean", "array", "record", "null".\n\nFix: write the shorthand, for example { steps: "array" }, or pass {} to accept any record.`);
|
|
1321
|
+
}
|
|
1322
|
+
const attempts = req.attempts ?? 1;
|
|
1323
|
+
// The seat the ask is told to, resolved before anything binds: an ask addresses an agent this
|
|
1324
|
+
// run spawned, and one it cannot tell opens no attempt.
|
|
1325
|
+
const seat = this.askSeat(req, ctx);
|
|
1326
|
+
const resume = askResume(ctx.resume);
|
|
1327
|
+
const deadlineAt = resume?.deadlineAt
|
|
1328
|
+
?? this.now() + parseDuration(req.deadline ?? this.binding.defaultCheckpointTimeout);
|
|
1329
|
+
let attempt = resume?.attempt ?? 1;
|
|
1330
|
+
// The resumed attempt is already bound; every attempt this call opens binds before it arms.
|
|
1331
|
+
let bindOwed = resume === undefined;
|
|
1332
|
+
let refused = resume?.refused;
|
|
1333
|
+
for (;;) {
|
|
1334
|
+
const token = askAttemptToken(ctx.requestId, attempt);
|
|
1335
|
+
if (bindOwed) {
|
|
1336
|
+
await ctx.bind({
|
|
1337
|
+
attempt,
|
|
1338
|
+
askToken: token,
|
|
1339
|
+
deadlineAt,
|
|
1340
|
+
...(refused !== undefined ? { refused } : {}),
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
bindOwed = true;
|
|
1344
|
+
// The addressed agent is TOLD: the attempt rides the turn relay under the attempt's own
|
|
1345
|
+
// token, carrying the token, the schema, the attempt count, the deadline and the previous
|
|
1346
|
+
// refusal, so the seat can answer through `cotal run answer`. Idempotent by goal id: a
|
|
1347
|
+
// re-entry finds the relay already accepted and submits nothing.
|
|
1348
|
+
await this.relayAsk(seat, ctx, token, { attempt, attempts, deadlineAt, ...(refused !== undefined ? { refused } : {}) });
|
|
1349
|
+
const ref = { endpoint: this.binding.endpoint, token };
|
|
1350
|
+
await this.arm(ref, deadlineAt);
|
|
1351
|
+
const settled = await this.settle(ref, ctx.signal);
|
|
1352
|
+
if (settled.settle === "expired") {
|
|
1353
|
+
// The deadline is the ask's whole budget of time: passing it with no conforming record
|
|
1354
|
+
// is the same outcome exhausted attempts name (L4006), never a turn's deadline (L4003).
|
|
1355
|
+
throw new EffectError("L4006", "ask-deadline", `ask(${stepKeyString(ctx.key)}) produced no conforming record before its recorded deadline: attempt ${attempt} of ${attempts} was still open when it passed`);
|
|
1356
|
+
}
|
|
1357
|
+
const answer = settled.answerId === undefined
|
|
1358
|
+
? undefined
|
|
1359
|
+
: await readCheckpointAnswer(this.kv, this.binding.endpoint, token, settled.answerId);
|
|
1360
|
+
if (answer === undefined)
|
|
1361
|
+
throw new CheckpointAnswerMissing(token, settled.answerId);
|
|
1362
|
+
if (conformsToAskSchema(answer.value, shape))
|
|
1363
|
+
return answer.value;
|
|
1364
|
+
const why = askNonconformance(answer.value, shape);
|
|
1365
|
+
if (attempt >= attempts) {
|
|
1366
|
+
throw new EffectError("L4006", "ask-nonconforming", `L4006 ask never produced a conforming record\n\n step ${stepKeyString(ctx.key)}\n\n${attempts} repl${attempts === 1 ? "y was" : "ies were"} checked against the schema and none conformed; the last was refused: ${why}.\n\nFix: have the agent publish a record matching the schema, widen the schema, or raise attempts.`);
|
|
1367
|
+
}
|
|
1368
|
+
refused = why;
|
|
1369
|
+
attempt += 1;
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
/**
|
|
1373
|
+
* One ask attempt's relay to the addressed seat: a `turn` goal under the attempt's token whose
|
|
1374
|
+
* payload carries the ask (token, schema, attempt, deadline, the previous refusal), rendered by
|
|
1375
|
+
* the seat's connector as the request to answer. The goal is the seat's to yield when it has
|
|
1376
|
+
* answered; the ask itself settles on the checkpoint plane, so the relay's terminal is never
|
|
1377
|
+
* read here, and it is never a reply `wait(replied)` observes. A seat the serve boundary reports
|
|
1378
|
+
* gone is the agent-down failure (L4002).
|
|
1379
|
+
*/
|
|
1380
|
+
askSeat(req, ctx) {
|
|
1381
|
+
const { name, uid } = parseAgentHandle(req.agent.agent);
|
|
1382
|
+
const seat = this.seatAddress(name, uid);
|
|
1383
|
+
const step = stepKeyString(ctx.key);
|
|
1384
|
+
if (seat.address === undefined) {
|
|
1385
|
+
if (seat.why === "not-in-roster")
|
|
1386
|
+
throw new Error(`ask(${step}) addresses ${name}#${uid}, which is not in this run's roster; an ask is relayed to an agent this run spawned`);
|
|
1387
|
+
throw new Error(`ask(${step}) has no address for ${name}#${uid}: the spawn's acceptance floor was never served, so the seat's owner/actor coordinates are unknown`);
|
|
1388
|
+
}
|
|
1389
|
+
return { ...seat.address, schema: req.schema };
|
|
1390
|
+
}
|
|
1391
|
+
/**
|
|
1392
|
+
* Where a relay to an agent of this run is addressed: the roster entry under the handle's name,
|
|
1393
|
+
* at the handle's incarnation, with the owner/actor coordinates its acceptance floor served.
|
|
1394
|
+
* One resolver for the three relays (`turn`, `ask`, an escalation), because the three
|
|
1395
|
+
* dispositions differ and the lookup does not: a turn and an ask refuse, an escalation stands.
|
|
1396
|
+
*/
|
|
1397
|
+
seatAddress(name, uid) {
|
|
1398
|
+
const entry = this.roster.get(name);
|
|
1399
|
+
if (entry === undefined || (uid !== undefined && entry.uid !== uid))
|
|
1400
|
+
return { why: "not-in-roster" };
|
|
1401
|
+
if (entry.owner === undefined || entry.actor === undefined)
|
|
1402
|
+
return { why: "no-floor" };
|
|
1403
|
+
return { address: { name, uid: entry.uid, owner: entry.owner, actor: entry.actor } };
|
|
1404
|
+
}
|
|
1405
|
+
async relayAsk(seat, ctx, token, ask) {
|
|
1406
|
+
const step = stepKeyString(ctx.key);
|
|
1407
|
+
const context = renderRunContext({ run: this.binding.runId, step, notices: [] });
|
|
1408
|
+
const payload = JSON.stringify({
|
|
1409
|
+
run: this.binding.runId, step, context, noticeIds: [],
|
|
1410
|
+
ask: { token, schema: seat.schema, attempt: ask.attempt, attempts: ask.attempts, deadlineAt: ask.deadlineAt, ...(ask.refused !== undefined ? { refused: ask.refused } : {}) },
|
|
1411
|
+
});
|
|
1412
|
+
await this.relayToSeat(seat, token, payload, ask.deadlineAt, "ask", step);
|
|
1413
|
+
}
|
|
1414
|
+
/**
|
|
1415
|
+
* An escalated checkpoint, delivered to the agent it names.
|
|
1416
|
+
*
|
|
1417
|
+
* `to` is legal only with `onExpiry: "escalate"` and the escalation is the SECOND mint, so this
|
|
1418
|
+
* runs once per chain at most. The addressee is resolved through the run's own roster: a name
|
|
1419
|
+
* this run spawned is a seat and is told, and anything else is a person — the pause is
|
|
1420
|
+
* answerable from anywhere by design, its addressee is on the entry, and refusing a program
|
|
1421
|
+
* that escalates to a human would be this host narrowing an option the reference leaves open.
|
|
1422
|
+
*
|
|
1423
|
+
* The relay is the ask's, because the two are the same act: one turn on the seat, carrying what
|
|
1424
|
+
* is asked and the token to answer under, settling on the checkpoint plane rather than on the
|
|
1425
|
+
* relay's own terminal.
|
|
1426
|
+
*/
|
|
1427
|
+
async relayEscalation(req, ctx, token, deadlineAt) {
|
|
1428
|
+
const to = req.to;
|
|
1429
|
+
const step = stepKeyString(ctx.key);
|
|
1430
|
+
// A seat is a roster entry whose acceptance floor was served: that is what carries the
|
|
1431
|
+
// owner/actor coordinates a relay is addressed by. Anything else — a name this run never
|
|
1432
|
+
// spawned, or one whose floor never landed — is not a seat, and the escalation stays the
|
|
1433
|
+
// durable pause it already is, addressed on its entry and answerable from anywhere.
|
|
1434
|
+
const seat = this.seatAddress(to);
|
|
1435
|
+
if (seat.address === undefined)
|
|
1436
|
+
return;
|
|
1437
|
+
const context = renderRunContext({ run: this.binding.runId, step, notices: [] });
|
|
1438
|
+
const payload = JSON.stringify({
|
|
1439
|
+
run: this.binding.runId, step, context, noticeIds: [],
|
|
1440
|
+
checkpoint: { token, prompt: req.prompt, schema: req.schema ?? null, deadlineAt, escalatedTo: to },
|
|
1441
|
+
});
|
|
1442
|
+
try {
|
|
1443
|
+
await this.relayToSeat(seat.address, token, payload, deadlineAt, "checkpoint", step);
|
|
1444
|
+
}
|
|
1445
|
+
catch (e) {
|
|
1446
|
+
// THE PAUSE OUTLIVES ITS ADDRESSEE. The reference defines no failure for an escalation whose
|
|
1447
|
+
// addressee cannot be reached: it "mints exactly one further checkpoint addressed to `to`",
|
|
1448
|
+
// and a checkpoint is "a durable pause a human or an agent resolves from anywhere". A seat
|
|
1449
|
+
// the endpoint reports gone is therefore the same case as a `to` that names a person: nobody
|
|
1450
|
+
// is told, the addressee is on the entry, and anyone may still answer. L4002 is the turn's
|
|
1451
|
+
// vocabulary, and this is not a turn the program asked for. Said once, not swallowed.
|
|
1452
|
+
if (!(e instanceof EffectError && e.code === "L4002"))
|
|
1453
|
+
throw e;
|
|
1454
|
+
console.error(`run ${this.binding.runId}: the escalation at ${step} could not be told to ${to} (${e.message}); the pause stands, answerable from anywhere`);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
/**
|
|
1458
|
+
* Submit one relay to a seat as a `turn` goal, and read its acceptance.
|
|
1459
|
+
*
|
|
1460
|
+
* IDEMPOTENT BY THE GOAL RECORD, which is also the arbiter when the invoke does not come back:
|
|
1461
|
+
* a relay that landed and lost its reply is a relay that landed, and re-submitting it would put
|
|
1462
|
+
* a second turn on the seat for one request. A seat the serve boundary reports gone is the
|
|
1463
|
+
* agent-down failure (L4002); every other refusal is this endpoint's and is uncatchable.
|
|
1464
|
+
*/
|
|
1465
|
+
async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) {
|
|
1466
|
+
const { name, uid } = seat;
|
|
1467
|
+
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
1468
|
+
const actx = await this.actionCtx();
|
|
1469
|
+
if ((await readGoalStatus(actx, ref)) !== undefined)
|
|
1470
|
+
return;
|
|
1471
|
+
let reply;
|
|
1472
|
+
try {
|
|
1473
|
+
reply = await invokeCommand(this.nc, this.binding.space, await this.manager(), "turn", { payload, deadlineMs: Math.max(1_000, deadlineAt - this.now()) }, {
|
|
1474
|
+
id: goalId,
|
|
1475
|
+
deadlineMs: TURN_ACCEPT_DEADLINE_MS,
|
|
1476
|
+
target: { mode: "owner", owner: seat.owner, actor: seat.actor, lifecycleUid: uid },
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1479
|
+
catch (err) {
|
|
1480
|
+
// The invoke did not come back — the goal record is the arbiter, exactly as in `turn`.
|
|
1481
|
+
if ((await readGoalStatus(actx, ref)) === undefined)
|
|
1482
|
+
throw err;
|
|
1483
|
+
return;
|
|
1484
|
+
}
|
|
1485
|
+
if (reply.reply.ok === false) {
|
|
1486
|
+
const err = reply.reply.error;
|
|
1487
|
+
if (err?.code === "expired")
|
|
1488
|
+
throw new EffectError("L4002", kind, `${kind}(${step}) found ${name}#${uid} down before its relay began: ${err.message}`);
|
|
1489
|
+
throw new Error(`${kind}(${step}) was refused by the ${this.binding.endpoint} endpoint: ${err?.message ?? "refused with no message"}`);
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
/**
|
|
1493
|
+
* `monitor` registers interest: after it, `down(agent)` is an event a branch can await (§5.9).
|
|
1494
|
+
*
|
|
1495
|
+
* THE REGISTRATION IS THE JOURNAL ENTRY, and that is the whole mechanism. Death is a STATE on
|
|
1496
|
+
* this mesh — the monitored incarnation's presence row gone (see `waitDown`) — not a message
|
|
1497
|
+
* that must find a standing mailbox, so there is no subscription to create, nothing to arm, and
|
|
1498
|
+
* nothing for a discharge or a migration to release: `wait(down(...))` reads the same fact
|
|
1499
|
+
* whenever it is asked, and the migration table's row for `monitor` ("nothing outlives it")
|
|
1500
|
+
* stays true. What the step performs is the validation a registration owes: the value must BE
|
|
1501
|
+
* an agent handle, refused loudly when it is not, because a wait parked on a malformed handle
|
|
1502
|
+
* would poll for a death nothing can ever report.
|
|
1503
|
+
*
|
|
1504
|
+
* Monitoring an agent that is ALREADY dead succeeds — Erlang's monitor of a dead process
|
|
1505
|
+
* delivers its DOWN rather than failing, and the rescue idiom (race work against
|
|
1506
|
+
* `wait(down(...))`) needs exactly that: the death is observed by the wait, immediately.
|
|
1507
|
+
*/
|
|
1508
|
+
async monitor(req, ctx) {
|
|
1509
|
+
if (ctx.signal.cancelled)
|
|
1510
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1511
|
+
const { name, uid } = parseAgentHandle(req.agent.agent);
|
|
1512
|
+
// The entry carries the handle it registered, so an adopted run rebuilds the same registry.
|
|
1513
|
+
await ctx.bind({ agent: `${name}#${uid}` });
|
|
1514
|
+
this.monitored.add(`${name}#${uid}`);
|
|
1515
|
+
return null;
|
|
1516
|
+
}
|
|
1517
|
+
/**
|
|
1518
|
+
* `conclave` open: mint (or take) the channel and join the members as durable membership rows.
|
|
1519
|
+
*
|
|
1520
|
+
* **The plan is the recovery identity.** Everything the close and the discharge need — the
|
|
1521
|
+
* channel, whether this open minted its registry row, and per member the resolved principal,
|
|
1522
|
+
* incarnation, generation and whether THIS conclave created the row — is computed first, bound
|
|
1523
|
+
* as the entry's external state, and only then executed. A crash before the bind created
|
|
1524
|
+
* nothing (rows are written after it); a crash after it re-enters with `ctx.resume` carrying
|
|
1525
|
+
* the plan, and the execute converges idempotently without re-resolving anything: the members
|
|
1526
|
+
* may have died since, and the recorded plan — not the world's current shape — is what a
|
|
1527
|
+
* release answers to.
|
|
1528
|
+
*
|
|
1529
|
+
* **The channel is handler-derived** when the program names none: `conclave-` plus a digest of
|
|
1530
|
+
* the step's own request id, so a resumed run re-derives the same room instead of opening a
|
|
1531
|
+
* second one. A program-named channel is taken as-is (validated concrete) and its registry row
|
|
1532
|
+
* is left alone — naming a room is not creating one, and the close must not tear down a channel
|
|
1533
|
+
* the run merely borrowed.
|
|
1534
|
+
*
|
|
1535
|
+
* **Members resolve through presence.** A handle carries `<name>#<lifecycleUid>`; the row that
|
|
1536
|
+
* maps it to the principal a membership record needs is the seat's own self-published presence —
|
|
1537
|
+
* the same source DM name-resolution reads, and the witness the manager's readiness gate
|
|
1538
|
+
* requires before a spawn reports `succeeded`. A member with no matching row is down or gone,
|
|
1539
|
+
* which is the effect's own catchable failure (L4002), not a handler fault.
|
|
1540
|
+
*
|
|
1541
|
+
* A member already durably in the channel (a pinned pre-existing room) is planned `joined:
|
|
1542
|
+
* false`: the conclave neither re-joins nor — at close — evicts a membership it did not create.
|
|
1543
|
+
*/
|
|
1544
|
+
async openConclave(req, ctx) {
|
|
1545
|
+
if (ctx.signal.cancelled)
|
|
1546
|
+
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1547
|
+
let plan;
|
|
1548
|
+
if (isConclavePlan(ctx.resume)) {
|
|
1549
|
+
plan = ctx.resume;
|
|
1550
|
+
}
|
|
1551
|
+
else {
|
|
1552
|
+
plan = await this.planConclave(req, ctx);
|
|
1553
|
+
await ctx.bind({ ...plan });
|
|
1554
|
+
}
|
|
1555
|
+
await this.executeConclavePlan(plan);
|
|
1556
|
+
this.conclaves.set(ctx.requestId, plan);
|
|
1557
|
+
return { channel: plan.channel };
|
|
315
1558
|
}
|
|
316
1559
|
/**
|
|
317
|
-
*
|
|
1560
|
+
* `conclave` close: release exactly what the recorded plan says the open created — tombstone
|
|
1561
|
+
* the memberships planned `joined: true`, delete the registry row when this conclave minted the
|
|
1562
|
+
* channel. A member that left and rejoined on its own since carries a newer generation, and the
|
|
1563
|
+
* stale-write guard makes this leave a no-op for it: the membership is theirs now.
|
|
318
1564
|
*
|
|
319
|
-
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
* to explain than a primitive that is not here yet.
|
|
1565
|
+
* Idempotent by the plane (a tombstone at or below an existing leave cursor is a no-op), so the
|
|
1566
|
+
* re-entry that retries an unacknowledged close converges. The failure mode is the interpreter's
|
|
1567
|
+
* `CloseOwed`: a close that throws leaves the entry pending, which IS the durable "a close is
|
|
1568
|
+
* still owed".
|
|
324
1569
|
*/
|
|
325
|
-
async
|
|
326
|
-
|
|
1570
|
+
async closeConclave(_req, ctx) {
|
|
1571
|
+
const plan = this.conclaves.get(ctx.requestId) ?? (isConclavePlan(ctx.resume) ? ctx.resume : undefined);
|
|
1572
|
+
if (plan === undefined)
|
|
1573
|
+
throw new Error(`conclave close for "${ctx.requestId}" has no recorded plan: the open that owes this close bound none`);
|
|
1574
|
+
await this.releaseConclave(plan);
|
|
1575
|
+
this.conclaves.delete(ctx.requestId);
|
|
1576
|
+
return null;
|
|
1577
|
+
}
|
|
1578
|
+
/** Compute the conclave plan from the world: derive or validate the channel, resolve every
|
|
1579
|
+
* member handle to its principal, and read each existing membership row so the join generation
|
|
1580
|
+
* and the created-by-us fact are decided BEFORE anything is written. Duplicate handles collapse
|
|
1581
|
+
* to one planned member — one identity is one membership row. */
|
|
1582
|
+
async planConclave(req, ctx) {
|
|
1583
|
+
const derived = req.channel === undefined;
|
|
1584
|
+
const channel = derived
|
|
1585
|
+
? `conclave-${createHash("sha256").update(ctx.requestId).digest("hex").slice(0, 12)}`
|
|
1586
|
+
: assertConclaveChannel(req.channel);
|
|
1587
|
+
const members = [];
|
|
1588
|
+
if (req.members.length > 0) {
|
|
1589
|
+
const presence = await this.presenceRows();
|
|
1590
|
+
const membersKv = await this.membersRegistry();
|
|
1591
|
+
const seen = new Set();
|
|
1592
|
+
for (const m of req.members) {
|
|
1593
|
+
const { name, uid } = parseAgentHandle(m.agent);
|
|
1594
|
+
if (seen.has(`${name}#${uid}`))
|
|
1595
|
+
continue;
|
|
1596
|
+
seen.add(`${name}#${uid}`);
|
|
1597
|
+
const principal = resolveMemberPrincipal(presence, m.agent, name, uid);
|
|
1598
|
+
const existing = await readMember(membersKv, channel, principal, uid);
|
|
1599
|
+
const open = existing !== undefined && existing.record.state === "durable-active" && existing.record.leaveCursor === undefined;
|
|
1600
|
+
members.push({
|
|
1601
|
+
agent: m.agent,
|
|
1602
|
+
principal,
|
|
1603
|
+
uid,
|
|
1604
|
+
generation: open ? existing.record.generation : (existing?.record.generation ?? 0) + 1,
|
|
1605
|
+
joined: !open,
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
return { channel, registered: derived, members };
|
|
1610
|
+
}
|
|
1611
|
+
/** Execute a conclave plan against the registries, idempotently: the registry row is a merge
|
|
1612
|
+
* the re-entry rewrites byte-identical, and a member row is committed only where the plan's
|
|
1613
|
+
* generation is NEWER than what is stored — a re-entry must not roll a landed row's join
|
|
1614
|
+
* cursor forward (the members were mid-conversation when the host died), and a row the world
|
|
1615
|
+
* moved past (an independent leave or rejoin) is theirs, not this conclave's. */
|
|
1616
|
+
async executeConclavePlan(plan) {
|
|
1617
|
+
if (plan.registered) {
|
|
1618
|
+
await writeChannelConfig(await this.channelRegistry(), plan.channel, {
|
|
1619
|
+
description: `a workflow conclave of run ${this.binding.runId}`,
|
|
1620
|
+
});
|
|
1621
|
+
}
|
|
1622
|
+
const joins = plan.members.filter((m) => m.joined);
|
|
1623
|
+
if (joins.length === 0)
|
|
1624
|
+
return;
|
|
1625
|
+
const membersKv = await this.membersRegistry();
|
|
1626
|
+
const joinCursor = await this.chatFrontier();
|
|
1627
|
+
for (const m of joins) {
|
|
1628
|
+
const existing = await readMember(membersKv, plan.channel, m.principal, m.uid);
|
|
1629
|
+
if (existing !== undefined && existing.record.generation >= m.generation)
|
|
1630
|
+
continue;
|
|
1631
|
+
try {
|
|
1632
|
+
await commitMember(membersKv, {
|
|
1633
|
+
channel: plan.channel,
|
|
1634
|
+
owner: m.principal,
|
|
1635
|
+
lifecycleUid: m.uid,
|
|
1636
|
+
state: "durable-active",
|
|
1637
|
+
joinCursor,
|
|
1638
|
+
generation: m.generation,
|
|
1639
|
+
// No activation catch-up is owed: eligibility starts at the join cursor captured here,
|
|
1640
|
+
// so the completeness the flag reports holds by construction — nothing before the join
|
|
1641
|
+
// is in this membership's interval.
|
|
1642
|
+
activated: true,
|
|
1643
|
+
writerIdentity: `${this.binding.caller.owner}.${this.binding.caller.actor}`,
|
|
1644
|
+
updatedAt: this.now(),
|
|
1645
|
+
});
|
|
1646
|
+
}
|
|
1647
|
+
catch (e) {
|
|
1648
|
+
// A concurrent newer write won between the read and the commit — same verdict as the
|
|
1649
|
+
// read-side skip above: the membership is the newer writer's now.
|
|
1650
|
+
if (!(e instanceof StaleMembershipWrite))
|
|
1651
|
+
throw e;
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
/** The world half of ending a conclave, shared by the close and the discharge: tombstone the
|
|
1656
|
+
* memberships this conclave created, then delete the registry row it minted. Idempotent, and
|
|
1657
|
+
* tolerant of exactly one foreign move — a NEWER generation on a row (the member left and
|
|
1658
|
+
* rejoined on its own), which the stale-write guard reports and this leave must not evict. */
|
|
1659
|
+
async releaseConclave(plan) {
|
|
1660
|
+
const joined = plan.members.filter((m) => m.joined);
|
|
1661
|
+
if (joined.length > 0) {
|
|
1662
|
+
const membersKv = await this.membersRegistry();
|
|
1663
|
+
const leaveCursor = await this.chatFrontier();
|
|
1664
|
+
const writer = `${this.binding.caller.owner}.${this.binding.caller.actor}`;
|
|
1665
|
+
for (const m of joined) {
|
|
1666
|
+
try {
|
|
1667
|
+
await tombstoneMember(membersKv, plan.channel, m.principal, m.uid, leaveCursor, writer, m.generation);
|
|
1668
|
+
}
|
|
1669
|
+
catch (e) {
|
|
1670
|
+
if (!(e instanceof StaleMembershipWrite))
|
|
1671
|
+
throw e;
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
if (plan.registered) {
|
|
1676
|
+
await (await this.channelRegistry()).delete(plan.channel);
|
|
1677
|
+
}
|
|
327
1678
|
}
|
|
328
|
-
|
|
329
|
-
|
|
1679
|
+
/** Every live presence row that decodes. Foreign bytes in the bucket are skipped — a peer's
|
|
1680
|
+
* malformed self-publish must not break another agent's name resolution — and what an ABSENT
|
|
1681
|
+
* row means belongs to the caller: `resolveMemberPrincipal` refuses the join loudly, and
|
|
1682
|
+
* `waitDown` reads it as the death it is waiting for. */
|
|
1683
|
+
async presenceRows() {
|
|
1684
|
+
const rows = [];
|
|
1685
|
+
for (const e of await liveKvEntries(await this.presenceRegistry())) {
|
|
1686
|
+
try {
|
|
1687
|
+
rows.push(e.json());
|
|
1688
|
+
}
|
|
1689
|
+
catch {
|
|
1690
|
+
// not a presence row
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
return rows;
|
|
330
1694
|
}
|
|
331
1695
|
/**
|
|
332
1696
|
* Arm this pause, or ATTACH to the one already recorded under the same token.
|
|
333
1697
|
*
|
|
334
|
-
* A mint is idempotent only if the whole spec is identical, so the recorded
|
|
335
|
-
* authority
|
|
336
|
-
*
|
|
337
|
-
*
|
|
1698
|
+
* A mint is idempotent only if the whole spec is identical, so the recorded spec is the
|
|
1699
|
+
* authority for BOTH halves of the mint's identity and a resume may not recompute either. The
|
|
1700
|
+
* deadline, because `now() + duration` is a different deadline a second later, and the plane
|
|
1701
|
+
* reads a different deadline as a different intent. And the HOLDER, because a resumed run does
|
|
1702
|
+
* not necessarily arrive under the principal that minted its pause: the CLI mints a fresh holder
|
|
1703
|
+
* per invocation, and a cross-host adoption is a different principal by definition. Only the
|
|
1704
|
+
* recorded holder itself may mint again — completing its own crash between the spec and the
|
|
1705
|
+
* status, where the identical spec is exactly what makes the retry idempotent. Everyone else
|
|
1706
|
+
* attaches. Measured before the repair, through the CLI: one `resume` of a checkpoint-parked
|
|
1707
|
+
* run re-minted under its own fresh holder, the plane refused ("a token is minted once"), and
|
|
1708
|
+
* the interpreter recorded that infrastructure refusal as the step's own failure (L4000) — a
|
|
1709
|
+
* stranded run whose journal blames the program.
|
|
338
1710
|
*
|
|
339
1711
|
* An already-passed deadline cannot be minted at all, correctly, because a due pause is not being
|
|
340
1712
|
* armed. It needs its schedule re-emitted at the status's current generation, which is the
|
|
341
|
-
* reconciler's job
|
|
342
|
-
* is
|
|
1713
|
+
* reconciler's job — the same operation an attaching successor needs, so the two share the exit.
|
|
1714
|
+
* A spec with no status that this holder cannot complete is unrepairable from here, so it is
|
|
1715
|
+
* raised rather than waited on.
|
|
343
1716
|
*/
|
|
344
1717
|
async arm(ref, deadline) {
|
|
345
1718
|
// Over already: an expiry or an answer landed while this host was away. Nothing to arm, and the
|
|
@@ -348,23 +1721,43 @@ export class MeshHandler {
|
|
|
348
1721
|
return;
|
|
349
1722
|
const prior = await readCheckpointSpec(this.kv, ref);
|
|
350
1723
|
const now = this.now();
|
|
351
|
-
|
|
352
|
-
|
|
1724
|
+
if (prior === undefined) {
|
|
1725
|
+
// The first arm: this driver's own mint, under its own holder, at the deadline the caller
|
|
1726
|
+
// computed from the duration it was just handed.
|
|
1727
|
+
await mintCheckpoint(this.kv, this.js, this.binding.space, {
|
|
1728
|
+
ref,
|
|
1729
|
+
instanceId: this.binding.instanceId,
|
|
1730
|
+
epoch: this.binding.epoch,
|
|
1731
|
+
holder: this.binding.holder,
|
|
1732
|
+
deadline,
|
|
1733
|
+
now,
|
|
1734
|
+
});
|
|
1735
|
+
return;
|
|
1736
|
+
}
|
|
1737
|
+
const mine = prior.holder.id === this.binding.holder.id
|
|
1738
|
+
&& prior.holder.lifecycleUid === this.binding.holder.lifecycleUid;
|
|
1739
|
+
if (mine && prior.initialDeadline > now) {
|
|
1740
|
+
// A retry of this holder's own mint: idempotent-if-identical, and the one path that can
|
|
1741
|
+
// repair a crash between the spec and the status, because only an identical spec completes.
|
|
353
1742
|
await mintCheckpoint(this.kv, this.js, this.binding.space, {
|
|
354
1743
|
ref,
|
|
355
1744
|
instanceId: this.binding.instanceId,
|
|
356
1745
|
epoch: this.binding.epoch,
|
|
357
1746
|
holder: this.binding.holder,
|
|
358
|
-
deadline:
|
|
1747
|
+
deadline: prior.initialDeadline,
|
|
359
1748
|
now,
|
|
360
1749
|
});
|
|
361
1750
|
return;
|
|
362
1751
|
}
|
|
1752
|
+
// ATTACH: the pause exists and is not this attempt's to mint — another holder's pause, or this
|
|
1753
|
+
// holder's own come back overdue (a due pause is being collected, not armed). Both need the
|
|
1754
|
+
// same thing: the schedule re-emitted at THIS instance's coordinates, so the fire lands where
|
|
1755
|
+
// this driver is listening. The settle the caller waits on next is coordinate-free.
|
|
363
1756
|
const status = await readCheckpointStatus(this.kv, ref);
|
|
364
1757
|
if (status === undefined) {
|
|
365
|
-
throw new Error(`checkpoint "${ref.token}" carries a spec with no status
|
|
366
|
-
+
|
|
367
|
-
+ `
|
|
1758
|
+
throw new Error(`checkpoint "${ref.token}" carries a spec with no status`
|
|
1759
|
+
+ `${mine ? "" : ` held by ${prior.holder.id}, and only its own holder's identical re-mint can repair a half-minted pause`}; `
|
|
1760
|
+
+ `this pause has to be reconciled on the plane before the run can go on`);
|
|
368
1761
|
}
|
|
369
1762
|
await reconcileCheckpointSchedule(this.kv, this.js, this.jsm, this.binding.space, {
|
|
370
1763
|
ref,
|
|
@@ -402,7 +1795,15 @@ export class MeshHandler {
|
|
|
402
1795
|
const subject = eptSubject(this.binding.space, ref.endpoint, this.binding.instanceId, this.binding.epoch, ref.token, "fire");
|
|
403
1796
|
const fired = await this.jsm.streams
|
|
404
1797
|
.getMessage(eptStreamName(this.binding.space), { last_by_subj: subject })
|
|
405
|
-
.catch(() =>
|
|
1798
|
+
.catch((e) => {
|
|
1799
|
+
// 10037 is the stream saying there is no message on that subject, which IS the ordinary
|
|
1800
|
+
// "the timer has not fired yet" answer. Every other error is the broker refusing or
|
|
1801
|
+
// unreachable, and reading those as "not yet" is how a `wait` with a timeout outlives its
|
|
1802
|
+
// own durable deadline in silence, which is the failure this method exists to prevent.
|
|
1803
|
+
if (e?.code === 10037)
|
|
1804
|
+
return null;
|
|
1805
|
+
throw e;
|
|
1806
|
+
});
|
|
406
1807
|
if (fired === null || fired === undefined)
|
|
407
1808
|
return false;
|
|
408
1809
|
const verdict = await handleCheckpointFire(this.kv, this.js, this.jsm, this.binding.space, {
|
|
@@ -437,15 +1838,31 @@ export class MeshHandler {
|
|
|
437
1838
|
if (st?.value.state !== "waiting")
|
|
438
1839
|
return;
|
|
439
1840
|
try {
|
|
1841
|
+
// The presenter is the ARMING holder read off the pause's own record, as everywhere on this
|
|
1842
|
+
// plane: a timer minted by a predecessor is still this run's to end after a takeover, and a
|
|
1843
|
+
// successor presenting its own identity would be refused (resume is holder-bound, SPEC
|
|
1844
|
+
// 13.10). A spec that cannot be read leaves the timer to its own deadline, which the catch
|
|
1845
|
+
// below already tolerates: it fires, settles expired, and nobody is reading that token.
|
|
1846
|
+
const spec = await readCheckpointSpec(this.kv, ref);
|
|
1847
|
+
if (spec === undefined)
|
|
1848
|
+
return;
|
|
440
1849
|
await resumeCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
|
|
441
1850
|
ref,
|
|
442
|
-
presenter:
|
|
1851
|
+
presenter: spec.holder,
|
|
443
1852
|
now: this.now(),
|
|
444
1853
|
});
|
|
445
1854
|
}
|
|
446
|
-
catch {
|
|
447
|
-
//
|
|
448
|
-
// caller has its answer either way, and a
|
|
1855
|
+
catch (e) {
|
|
1856
|
+
// IT SETTLED UNDERNEATH US is one outcome, and it is the expected one: the deadline won a
|
|
1857
|
+
// race it was always allowed to win, the caller has its answer either way, and a claimed
|
|
1858
|
+
// timer is not a fact anyone reads. A BROKER THAT REFUSED OR WENT AWAY is a different thing
|
|
1859
|
+
// wearing the same silence, and this catch used to swallow both. It still must not replace
|
|
1860
|
+
// the caller's answer — a timer left armed fires, settles expired, and nobody reads that
|
|
1861
|
+
// token — so the second one is loud on the way past instead.
|
|
1862
|
+
const settledAlready = e instanceof EpEnvelopeError
|
|
1863
|
+
&& (e.code === "conflict" || e.code === "failed-precondition");
|
|
1864
|
+
if (!settledAlready)
|
|
1865
|
+
console.error(`run ${this.binding.runId}: the pause ${ref.token} could not be claimed (${e.message}); it will fire and settle expired with nobody reading it`);
|
|
449
1866
|
}
|
|
450
1867
|
}
|
|
451
1868
|
/** The message at a recorded stream sequence — the re-bind path after a crash. */
|
|
@@ -467,7 +1884,7 @@ export class MeshHandler {
|
|
|
467
1884
|
* optimization, it is the difference between resuming and waiting forever for an event that is
|
|
468
1885
|
* already in the past.
|
|
469
1886
|
*/
|
|
470
|
-
async settle(ref) {
|
|
1887
|
+
async settle(ref, signal) {
|
|
471
1888
|
const already = await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
472
1889
|
if (already !== undefined)
|
|
473
1890
|
return already;
|
|
@@ -483,12 +1900,38 @@ export class MeshHandler {
|
|
|
483
1900
|
// A pump that ENDED is not an answer, only one that FAILED is: a failure means this process
|
|
484
1901
|
// cannot expire the pause, so it is raised rather than absorbed and the step stays pending.
|
|
485
1902
|
pump.then(() => new Promise(() => { })),
|
|
1903
|
+
// A cancelled branch stops parking NOW (§7.6): the rejection decides the race first, so a
|
|
1904
|
+
// `checkpoint` cannot mistake its own claim below for an answer, and then the claim ends
|
|
1905
|
+
// the pause in the world. The claim is not awaited here — a claim that loses its own race
|
|
1906
|
+
// is tolerated by `cancelTimer`, and the discharge sweep at the run's completion re-claims
|
|
1907
|
+
// idempotently for the case where this process died before the claim landed.
|
|
1908
|
+
...(signal === undefined ? [] : [this.settleCancelled(ref, signal)]),
|
|
486
1909
|
]);
|
|
487
1910
|
}
|
|
488
1911
|
finally {
|
|
489
1912
|
wait.over = true;
|
|
490
1913
|
}
|
|
491
1914
|
}
|
|
1915
|
+
/** Reject with `Cancelled` the moment this branch's signal fires, then claim the pause so its
|
|
1916
|
+
* armed schedule cannot fire into a run that has moved on. The claim also writes the one-use
|
|
1917
|
+
* settle, which is what lets an abandoned `awaitSettle` poll loop see a fact and end. */
|
|
1918
|
+
settleCancelled(ref, signal) {
|
|
1919
|
+
return new Promise((_, reject) => {
|
|
1920
|
+
let fired = false;
|
|
1921
|
+
const fire = (reason) => {
|
|
1922
|
+
if (fired)
|
|
1923
|
+
return;
|
|
1924
|
+
fired = true;
|
|
1925
|
+
reject(new Cancelled(reason ?? "cancelled"));
|
|
1926
|
+
void this.cancelTimer(ref).catch(() => undefined);
|
|
1927
|
+
};
|
|
1928
|
+
if (signal.cancelled) {
|
|
1929
|
+
fire(signal.reason);
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
signal.onCancel(fire);
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
492
1935
|
/** Take this deadline's fire for as long as somebody is waiting on it. */
|
|
493
1936
|
async pumpFires(ref, wait) {
|
|
494
1937
|
while (!wait.over) {
|
|
@@ -524,6 +1967,181 @@ export function waitConsumerConfig(space, requestId, channel) {
|
|
|
524
1967
|
/** How long one poll of a wait's consumer blocks. The deadline itself is durable; this is only how
|
|
525
1968
|
* late its observation can be, and a shorter poll buys latency at the cost of fetch traffic. */
|
|
526
1969
|
const WAIT_POLL_MS = 2_000;
|
|
1970
|
+
/**
|
|
1971
|
+
* Read a spawn's `permits` as the budgets this host can enforce: `turns`, a positive integer of
|
|
1972
|
+
* turns the run may dispatch to the agent, and `wallClock`, a duration from the spawn after which
|
|
1973
|
+
* no turn is admitted. Anything else (tokens, spend) is a budget this host has no meter for, and a
|
|
1974
|
+
* budget it cannot enforce is refused loudly rather than accepted as a silent no-op.
|
|
1975
|
+
*/
|
|
1976
|
+
function readPermits(raw, persona) {
|
|
1977
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw))
|
|
1978
|
+
throw new Error(`spawn(${persona}): permits must be a record of budgets, got ${JSON.stringify(raw)}`);
|
|
1979
|
+
const out = {};
|
|
1980
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
1981
|
+
if (key === "turns") {
|
|
1982
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1)
|
|
1983
|
+
throw new Error(`spawn(${persona}): permits.turns must be a positive integer, got ${JSON.stringify(value)}`);
|
|
1984
|
+
out.turns = value;
|
|
1985
|
+
}
|
|
1986
|
+
else if (key === "wallClock") {
|
|
1987
|
+
if (typeof value !== "string")
|
|
1988
|
+
throw new Error(`spawn(${persona}): permits.wallClock must be a duration string, got ${JSON.stringify(value)}`);
|
|
1989
|
+
out.wallClockMs = parseDuration(value);
|
|
1990
|
+
}
|
|
1991
|
+
else {
|
|
1992
|
+
throw new Error(`spawn(${persona}): permits.${key} is a budget this host has no meter for; it meters turns and wallClock, and a budget it cannot enforce is refused rather than ignored`);
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
return out;
|
|
1996
|
+
}
|
|
1997
|
+
/** How many incomplete presence scans the worktree guard tolerates before it fails loudly. */
|
|
1998
|
+
/**
|
|
1999
|
+
* The keyed view of an append log: the last record written for each step, in the order the run
|
|
2000
|
+
* first reached it.
|
|
2001
|
+
*
|
|
2002
|
+
* A DRIVER HANDS OVER AN APPEND LOG, never a keyed journal. Settling appends a second record rather
|
|
2003
|
+
* than editing the first, so every completed step is in the list twice and a retried one is in it
|
|
2004
|
+
* more times still. Seeding from the raw list charged one turn to `permits.turns` once per RECORD,
|
|
2005
|
+
* so an agent with two turns was refused its second the moment its run recovered; it also left a
|
|
2006
|
+
* failed spawn holding a worktree reservation forever, and re-registered turn goals for turns the
|
|
2007
|
+
* run had cancelled. `fork.ts` carries the same warning about the same list, from the same defect.
|
|
2008
|
+
*
|
|
2009
|
+
* The fold is the journal's own, so "the state of this step" has one definition here and there.
|
|
2010
|
+
*/
|
|
2011
|
+
function foldEntries(entries) {
|
|
2012
|
+
const first = entries[0];
|
|
2013
|
+
if (first === undefined)
|
|
2014
|
+
return entries;
|
|
2015
|
+
return new Journal({ run: first.run, entries, readOnly: true }).entries();
|
|
2016
|
+
}
|
|
2017
|
+
const WORKTREE_SCAN_ATTEMPTS = 5;
|
|
2018
|
+
/** How often an action's durable terminal fact is looked for. Same argument as `WAIT_POLL_MS`. */
|
|
2019
|
+
const GOAL_POLL_MS = 2_000;
|
|
2020
|
+
/** How long one acceptance round-trip may take. The ACCEPT is synchronous and cheap on the far
|
|
2021
|
+
* side (the manager replies at identity mint, before any provision), so this bounds a lost
|
|
2022
|
+
* broker, not the spawn itself — the spawn's own outcome rides the goal terminal. */
|
|
2023
|
+
const SPAWN_ACCEPT_DEADLINE_MS = 30_000;
|
|
2024
|
+
/** Bound on the manager's synchronous `turn` ACCEPT reply (the relay registration, not the yield). */
|
|
2025
|
+
const TURN_ACCEPT_DEADLINE_MS = 30_000;
|
|
2026
|
+
/** A step key's enclosing scope: the journal's own rendering (`entry.scope`), re-derived so the
|
|
2027
|
+
* live path and the adoption rebuild key the handoff memos identically. */
|
|
2028
|
+
function scopeOf(key) {
|
|
2029
|
+
const k = stepKeyString(key);
|
|
2030
|
+
return k.slice(0, k.lastIndexOf("/"));
|
|
2031
|
+
}
|
|
2032
|
+
/** The terminal wait a discharge grants a goal whose entry recorded no readiness window (the
|
|
2033
|
+
* crash-before-bind case). Matches the manager's default readiness budget. */
|
|
2034
|
+
const DISCHARGE_TERMINAL_BOUND_MS = 30_000;
|
|
2035
|
+
/** The manager `spawn` args a {@link SpawnRequest} submits: persona names the persona file
|
|
2036
|
+
* (`name`), `join` becomes the seat's channel subscriptions. Policy fields do not travel. */
|
|
2037
|
+
function spawnArgs(req) {
|
|
2038
|
+
return {
|
|
2039
|
+
name: req.persona,
|
|
2040
|
+
...(req.model !== undefined ? { model: req.model } : {}),
|
|
2041
|
+
...(req.variant !== undefined ? { variant: req.variant } : {}),
|
|
2042
|
+
...(req.role !== undefined ? { role: req.role } : {}),
|
|
2043
|
+
...(req.join !== undefined && req.join.length > 0 ? { subscribe: req.join.map((c) => c.channel) } : {}),
|
|
2044
|
+
};
|
|
2045
|
+
}
|
|
2046
|
+
/** The acceptance-floor fields worth binding: the allocated identity a discharge despawns by, and
|
|
2047
|
+
* the readiness window that bounds its wait for a terminal. Copied field-by-field so a widened
|
|
2048
|
+
* acceptance never smuggles unknown keys into the journal. */
|
|
2049
|
+
function pickAcceptanceFloor(floor) {
|
|
2050
|
+
const out = {};
|
|
2051
|
+
for (const k of ["name", "owner", "actor", "uid", "readinessDeadlineMs"]) {
|
|
2052
|
+
if (floor[k] !== undefined)
|
|
2053
|
+
out[k] = floor[k];
|
|
2054
|
+
}
|
|
2055
|
+
return out;
|
|
2056
|
+
}
|
|
2057
|
+
/** The despawn target for a discharged spawn: the bound acceptance floor when the entry carries
|
|
2058
|
+
* one, else the identity the SUCCEEDED terminal itself records (`id` is the `owner.actor`
|
|
2059
|
+
* principal, `lifecycleUid` the incarnation). `undefined` when neither names an agent. */
|
|
2060
|
+
function spawnDespawnTarget(external, fact) {
|
|
2061
|
+
if (typeof external?.owner === "string" && typeof external.actor === "string" && typeof external.uid === "string")
|
|
2062
|
+
return { owner: external.owner, actor: external.actor, lifecycleUid: external.uid };
|
|
2063
|
+
const d = fact.data;
|
|
2064
|
+
if (typeof d?.id !== "string" || typeof d.lifecycleUid !== "string")
|
|
2065
|
+
return undefined;
|
|
2066
|
+
const dot = d.id.indexOf(".");
|
|
2067
|
+
if (dot <= 0 || dot === d.id.length - 1)
|
|
2068
|
+
return undefined;
|
|
2069
|
+
return { owner: d.id.slice(0, dot), actor: d.id.slice(dot + 1), lifecycleUid: d.lifecycleUid };
|
|
2070
|
+
}
|
|
2071
|
+
/**
|
|
2072
|
+
* The program's value from a spawn terminal. `succeeded` yields the agent handle — `agent` is the
|
|
2073
|
+
* `<name>#<lifecycleUid>` composite, one string that addresses the NAME the mesh knows the seat by
|
|
2074
|
+
* while pinning WHICH incarnation this run spawned (a respawned namesake is not this handle).
|
|
2075
|
+
* Every other state throws the catchable spawn failure (L4002) carrying the terminal's own reason:
|
|
2076
|
+
* `failed` and `uncertain` are the manager's readiness verdicts, `cancelled` means a despawn ended
|
|
2077
|
+
* the launch under it. Each of them leaves the program with an agent that is not up, which is
|
|
2078
|
+
* what L4002 names; each replays identically because the fact is durable. A refusal at accept is
|
|
2079
|
+
* a different thing (no agent was ever allocated) and is classed at the accept, in `spawn`. The
|
|
2080
|
+
* handle's worktree is the BOUND one: the request's for a spawn this step submitted, the seat's own
|
|
2081
|
+
* for one a migration handed it (an adopted seat keeps its tree whether or not the edit names it).
|
|
2082
|
+
*/
|
|
2083
|
+
function spawnHandleOf(req, ext, fact, endpoint) {
|
|
2084
|
+
if (fact.state !== "succeeded") {
|
|
2085
|
+
const d = fact.data;
|
|
2086
|
+
const why = typeof d?.error === "string" ? d.error
|
|
2087
|
+
: typeof d?.reason === "string" ? d.reason
|
|
2088
|
+
: d?.cancelledBy !== undefined ? `cancelled by ${JSON.stringify(d.cancelledBy)}`
|
|
2089
|
+
: `the ${endpoint} endpoint recorded no reason`;
|
|
2090
|
+
throw new EffectError("L4002", "spawn", `spawn(${req.persona}) ${fact.state}: ${why}`, { state: fact.state });
|
|
2091
|
+
}
|
|
2092
|
+
const d = fact.data;
|
|
2093
|
+
if (typeof d?.name !== "string" || typeof d.lifecycleUid !== "string")
|
|
2094
|
+
throw new Error(`the spawn goal's succeeded terminal carries no readable agent identity (${JSON.stringify(fact.data)}); a garbled terminal never yields a handle (SPEC 13.6)`);
|
|
2095
|
+
return {
|
|
2096
|
+
agent: `${d.name}#${d.lifecycleUid}`,
|
|
2097
|
+
persona: req.persona,
|
|
2098
|
+
...(typeof ext.worktree === "string" ? { worktree: ext.worktree } : {}),
|
|
2099
|
+
...(typeof d.role === "string" ? { role: d.role } : {}),
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
/** Whether a recorded external is a conclave plan. A journal is bytes from an earlier process, so
|
|
2103
|
+
* the shape is checked rather than trusted — a malformed external reads as "nothing was bound". */
|
|
2104
|
+
function isConclavePlan(v) {
|
|
2105
|
+
if (typeof v !== "object" || v === null)
|
|
2106
|
+
return false;
|
|
2107
|
+
const p = v;
|
|
2108
|
+
if (typeof p.channel !== "string" || typeof p.registered !== "boolean" || !Array.isArray(p.members))
|
|
2109
|
+
return false;
|
|
2110
|
+
return p.members.every((m) => {
|
|
2111
|
+
if (typeof m !== "object" || m === null)
|
|
2112
|
+
return false;
|
|
2113
|
+
const r = m;
|
|
2114
|
+
return typeof r.agent === "string" && typeof r.principal === "string" && typeof r.uid === "string"
|
|
2115
|
+
&& typeof r.generation === "number" && typeof r.joined === "boolean";
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
/** Split an agent handle `<name>#<lifecycleUid>` on its LAST `#` — the uid alphabet
|
|
2119
|
+
* (`[a-z0-9]{26,32}`) cannot carry one, a name could. */
|
|
2120
|
+
function parseAgentHandle(agent) {
|
|
2121
|
+
const i = agent.lastIndexOf("#");
|
|
2122
|
+
if (i <= 0 || i === agent.length - 1)
|
|
2123
|
+
throw new Error(`"${agent}" is not an agent handle of the form <name>#<lifecycleUid>`);
|
|
2124
|
+
return { name: agent.slice(0, i), uid: agent.slice(i + 1) };
|
|
2125
|
+
}
|
|
2126
|
+
/** The one presence row that carries the handle's name AND incarnation, as a principal. None is
|
|
2127
|
+
* the effect's own catchable failure — the agent is down or gone (L4002) — and more than one is
|
|
2128
|
+
* an ambiguity no membership row may be written under. */
|
|
2129
|
+
function resolveMemberPrincipal(rows, agent, name, uid) {
|
|
2130
|
+
const matches = rows.filter((p) => p.card?.name === name && p.lifecycleUid === uid && typeof p.card?.id === "string");
|
|
2131
|
+
if (matches.length === 1)
|
|
2132
|
+
return matches[0].card.id;
|
|
2133
|
+
if (matches.length === 0)
|
|
2134
|
+
throw new EffectError("L4002", "conclave", `conclave member "${agent}" is not present on the mesh: no live presence row carries that name and incarnation, so the agent is down or gone`);
|
|
2135
|
+
throw new Error(`conclave member "${agent}" is ambiguous: ${matches.length} presence rows claim that name and incarnation`);
|
|
2136
|
+
}
|
|
2137
|
+
/** A program-named conclave channel: valid channel grammar AND concrete — membership rows and a
|
|
2138
|
+
* room to talk in are per-channel things a wildcard cannot name. */
|
|
2139
|
+
function assertConclaveChannel(channel) {
|
|
2140
|
+
assertValidChannel(channel);
|
|
2141
|
+
if (!isConcreteChannel(channel))
|
|
2142
|
+
throw new Error(`conclave channel "${channel}" is a wildcard; a conclave joins its members to one concrete channel`);
|
|
2143
|
+
return channel;
|
|
2144
|
+
}
|
|
527
2145
|
/** How often a pause that nobody will answer looks for the broker's fire. Same argument as
|
|
528
2146
|
* `WAIT_POLL_MS`: the deadline is durable and this is only how late its observation can be. */
|
|
529
2147
|
const FIRE_POLL_MS = 2_000;
|
|
@@ -532,6 +2150,40 @@ const FIRE_POLL_MS = 2_000;
|
|
|
532
2150
|
function derivedToken(requestId, purpose) {
|
|
533
2151
|
return createHash("sha256").update(`${requestId}:${purpose}`, "utf8").digest("base64url").slice(0, 43);
|
|
534
2152
|
}
|
|
2153
|
+
/** An ask attempt's pause token: attempt 1 IS the step's request id; a re-ask derives its own. */
|
|
2154
|
+
function askAttemptToken(requestId, attempt) {
|
|
2155
|
+
return attempt === 1 ? requestId : derivedToken(requestId, `ask-attempt-${attempt}`);
|
|
2156
|
+
}
|
|
2157
|
+
/** The recorded ask progress a resume re-enters at, or undefined for a fresh first attempt. The
|
|
2158
|
+
* external is bytes from an earlier process, so the shape is checked rather than trusted. */
|
|
2159
|
+
function askResume(v) {
|
|
2160
|
+
if (v === undefined)
|
|
2161
|
+
return undefined;
|
|
2162
|
+
if (!(typeof v.attempt === "number" && v.attempt >= 1 && typeof v.deadlineAt === "number"))
|
|
2163
|
+
return undefined;
|
|
2164
|
+
// The refusal is bound with the attempt it belongs to. A resume that dropped it would re-ask the
|
|
2165
|
+
// seat with no reason its last answer failed, which is the one thing the re-ask exists to say.
|
|
2166
|
+
return {
|
|
2167
|
+
attempt: v.attempt,
|
|
2168
|
+
deadlineAt: v.deadlineAt,
|
|
2169
|
+
...(typeof v.refused === "string" ? { refused: v.refused } : {}),
|
|
2170
|
+
};
|
|
2171
|
+
}
|
|
2172
|
+
/** WHY a reply does not conform, per declared field — the refusal an answerer reads off the entry
|
|
2173
|
+
* before answering again. Judged with the same single-field check that judged the reply, so the
|
|
2174
|
+
* description can never disagree with the verdict. */
|
|
2175
|
+
function askNonconformance(value, shape) {
|
|
2176
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
2177
|
+
return "the reply is not a record";
|
|
2178
|
+
const record = value;
|
|
2179
|
+
const bad = [];
|
|
2180
|
+
for (const [field, kind] of Object.entries(shape)) {
|
|
2181
|
+
if (conformsToAskSchema({ [field]: record[field] }, { [field]: kind }))
|
|
2182
|
+
continue;
|
|
2183
|
+
bad.push(field in record ? `"${field}" wants ${kind}` : `"${field}" is missing (wants ${kind})`);
|
|
2184
|
+
}
|
|
2185
|
+
return bad.join("; ");
|
|
2186
|
+
}
|
|
535
2187
|
/**
|
|
536
2188
|
* A `matches` pattern, admitted through the repo's bounded-regex subset before it is compiled.
|
|
537
2189
|
*
|
|
@@ -575,30 +2227,6 @@ function matchesEvent(msg, from, matcher) {
|
|
|
575
2227
|
.join("\n");
|
|
576
2228
|
return matcher.test(text);
|
|
577
2229
|
}
|
|
578
|
-
/**
|
|
579
|
-
* An effect whose durable substrate has not landed on this host.
|
|
580
|
-
*
|
|
581
|
-
* An honest two-exit, and deliberately not a fake success: the simulator implements these, so a
|
|
582
|
-
* program that uses them can be written, validated and dry-run today — but a DURABLE run refuses
|
|
583
|
-
* rather than performing them on a plane that could not recover them. A run that "succeeded" at an
|
|
584
|
-
* effect nothing can replay would be a lie the journal then carries forever.
|
|
585
|
-
*/
|
|
586
|
-
export class NotYetDurable extends EffectRefused {
|
|
587
|
-
effect;
|
|
588
|
-
needs;
|
|
589
|
-
constructor(effect, needs) {
|
|
590
|
-
super(
|
|
591
|
-
// L5016, and it is load-bearing rather than decorative: the interpreter settles the entry
|
|
592
|
-
// `refused` under the code the refusal carries, so the journal says which thing happened —
|
|
593
|
-
// "no substrate on this host" rather than "the handler broke".
|
|
594
|
-
"L5016", `${effect} is not durable on this host yet: it rides ${needs}, which has not landed. ` +
|
|
595
|
-
`The simulator performs it, so the program can be tested and dry-run; a durable run refuses ` +
|
|
596
|
-
`rather than performing an effect it could not recover after a crash.`);
|
|
597
|
-
this.effect = effect;
|
|
598
|
-
this.needs = needs;
|
|
599
|
-
this.name = "NotYetDurable";
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
2230
|
/**
|
|
603
2231
|
* RE-ARM the timers of every pause this run is still holding, under THIS driver's coordinates.
|
|
604
2232
|
*
|
|
@@ -636,10 +2264,10 @@ export async function rearmOutstandingPauses(deps, binding, entries) {
|
|
|
636
2264
|
* later record wins — a step that settled has a settled entry after its pending one, and reading
|
|
637
2265
|
* only the first would re-arm timers for pauses that are already over.
|
|
638
2266
|
*
|
|
639
|
-
* THE KINDS ARE THE
|
|
640
|
-
*
|
|
641
|
-
*
|
|
642
|
-
* epoch fires.
|
|
2267
|
+
* THE KINDS ARE THE FIVE THAT ARM A TIMER, and `wait`, `ask` and `turn` are three of them. None of
|
|
2268
|
+
* the three mints a pause that looks like its own, but a wait's idle window and timeout, an ask
|
|
2269
|
+
* attempt's deadline, and a turn's deadline authority are mediated deadlines exactly as `sleep`'s
|
|
2270
|
+
* is, and one adopted at a new epoch would otherwise wait on a deadline no live epoch fires.
|
|
643
2271
|
*
|
|
644
2272
|
* An idle wait with a timeout arms TWO, and the second is DERIVED rather than recorded, so it is
|
|
645
2273
|
* re-derived here for the same reason the live path derives it: a resume that had to remember it
|
|
@@ -647,6 +2275,11 @@ export async function rearmOutstandingPauses(deps, binding, entries) {
|
|
|
647
2275
|
* is harmless by construction — the reconciler reads the checkpoint's status first and re-emits
|
|
648
2276
|
* nothing when there is none — and the alternative, reading the request shape back out of the
|
|
649
2277
|
* entry to decide, would make the repair depend on a field a replay is not guaranteed to carry.
|
|
2278
|
+
* An ask's armed pause is its CURRENT attempt's, whose token is bound as `askToken`; before the
|
|
2279
|
+
* first bind it is attempt 1, which is the request id itself. A `turn`'s is under its goal id,
|
|
2280
|
+
* which is the request id: that pause is the client-side L4003 authority the run keeps for a
|
|
2281
|
+
* manager that dies, so leaving it armed at the predecessor's coordinates would go dark in exactly
|
|
2282
|
+
* the window recovery opens.
|
|
650
2283
|
*/
|
|
651
2284
|
export function outstandingPauseTokens(entries) {
|
|
652
2285
|
const last = new Map();
|
|
@@ -656,8 +2289,11 @@ export function outstandingPauseTokens(entries) {
|
|
|
656
2289
|
for (const e of last.values()) {
|
|
657
2290
|
if (e.state !== "pending" || e.requestId === undefined)
|
|
658
2291
|
continue;
|
|
659
|
-
|
|
2292
|
+
// `turn` arms its client-side deadline authority under the step's request id (the goal id).
|
|
2293
|
+
if (e.kind === "sleep" || e.kind === "checkpoint" || e.kind === "turn")
|
|
660
2294
|
tokens.push(e.requestId);
|
|
2295
|
+
else if (e.kind === "ask")
|
|
2296
|
+
tokens.push(typeof e.external?.askToken === "string" ? e.external.askToken : e.requestId);
|
|
661
2297
|
else if (e.kind === "wait")
|
|
662
2298
|
tokens.push(e.requestId, derivedToken(e.requestId, "wait-timeout"));
|
|
663
2299
|
}
|