@cotal-ai/manager 0.76.0 → 0.77.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/manager.d.ts +19 -4
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +122 -82
- package/dist/manager.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -245,14 +245,22 @@ function parseTurnNote(raw) {
|
|
|
245
245
|
if (typeof n.payload !== "string" || typeof n.deadlineAt !== "number"
|
|
246
246
|
|| typeof n.holdEpoch !== "number" || typeof n.owner !== "string")
|
|
247
247
|
return undefined;
|
|
248
|
+
if (n.holdToken !== undefined && typeof n.holdToken !== "string")
|
|
249
|
+
return undefined;
|
|
248
250
|
if (n.handoffFrom !== undefined && typeof n.handoffFrom !== "string")
|
|
249
251
|
return undefined;
|
|
250
|
-
return { payload: n.payload, deadlineAt: n.deadlineAt, holdEpoch: n.holdEpoch, owner: n.owner, ...(typeof n.handoffFrom === "string" ? { handoffFrom: n.handoffFrom } : {}) };
|
|
252
|
+
return { payload: n.payload, deadlineAt: n.deadlineAt, holdEpoch: n.holdEpoch, owner: n.owner, ...(typeof n.holdToken === "string" ? { holdToken: n.holdToken } : {}), ...(typeof n.handoffFrom === "string" ? { handoffFrom: n.handoffFrom } : {}) };
|
|
253
|
+
}
|
|
254
|
+
/** A turn relay's key in the manager's maps: the caller triple and goal id the goal plane stores
|
|
255
|
+
* the turn under, because a goal id is the caller's own (SPEC 13.6 item 1) and two callers may
|
|
256
|
+
* pick the same one. Every part is a subject token, so the dotted join is unambiguous. */
|
|
257
|
+
function turnKey(ref) {
|
|
258
|
+
return `${ref.caller.owner}.${ref.caller.actor}.${ref.caller.uid}.${ref.goalId}`;
|
|
251
259
|
}
|
|
252
|
-
/** A turn hold's token, DERIVED
|
|
253
|
-
*
|
|
254
|
-
function turnHoldToken(
|
|
255
|
-
return createHash("sha256").update(`${
|
|
260
|
+
/** A turn hold's token, DERIVED (same recipe as the runtime's pause tokens) from the relay key, so
|
|
261
|
+
* two callers' turns under one goal id never share a hold. */
|
|
262
|
+
function turnHoldToken(key) {
|
|
263
|
+
return createHash("sha256").update(`${key}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
|
|
256
264
|
}
|
|
257
265
|
/**
|
|
258
266
|
* Event channels in `channels` that do NOT belong to `{owner, actor}`.
|
|
@@ -466,9 +474,9 @@ export class Manager {
|
|
|
466
474
|
* drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
|
|
467
475
|
agentGoals = new Map();
|
|
468
476
|
/**
|
|
469
|
-
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin)
|
|
470
|
-
* same-
|
|
471
|
-
* the goal-index entry (its acceptance floor + note).
|
|
477
|
+
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin), by
|
|
478
|
+
* {@link turnKey}: a same-goal retry serves the identical acceptance; cross-incarnation retries
|
|
479
|
+
* rebuild from the goal-index entry (its acceptance floor + note).
|
|
472
480
|
*
|
|
473
481
|
* One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
|
|
474
482
|
* from, and — once the turn settles — the answer a RETRIED yield is served from.
|
|
@@ -489,7 +497,7 @@ export class Manager {
|
|
|
489
497
|
* window would hear `not-found` again.
|
|
490
498
|
*/
|
|
491
499
|
turnAcceptances = new Map();
|
|
492
|
-
/** Every relayed turn awaiting its seat's yield, by
|
|
500
|
+
/** Every relayed turn awaiting its seat's yield, by {@link turnKey}. The seat's `turn-pending` pull
|
|
493
501
|
* scans it; the deadline sweep drives expiry; a seat reap stamps its entries `seatDiedAt`, which
|
|
494
502
|
* the deadline terminal carries as `agentDownAt` so the run reads that death as L4002.
|
|
495
503
|
* Rebuilt at boot from goal-index entries carrying a turn note. */
|
|
@@ -4657,6 +4665,29 @@ export class Manager {
|
|
|
4657
4665
|
const { reason } = resolveHarness(connector.name, connector.requires ?? []);
|
|
4658
4666
|
return reason ? { refusal: reason } : {};
|
|
4659
4667
|
}
|
|
4668
|
+
/** The launch fields the host supplies rather than the seat. Spawn and resume both take them from
|
|
4669
|
+
* here, so a resumed seat launches under the same host policy, `eventsRequired` included, as the
|
|
4670
|
+
* same seat spawned fresh. */
|
|
4671
|
+
hostLaunchFields(agent, shareTools, harness) {
|
|
4672
|
+
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
4673
|
+
return {
|
|
4674
|
+
space: this.space,
|
|
4675
|
+
servers: this.servers,
|
|
4676
|
+
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
4677
|
+
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
4678
|
+
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
4679
|
+
mcpServers: connectorServers(cotalConfig, agent, shareTools),
|
|
4680
|
+
// The operator's spawn-env policy travels the same route: absent means no extras (the OS
|
|
4681
|
+
// allow-list + operator knobs + connector-declared inputs), present means those names too.
|
|
4682
|
+
// A connector never reads the config itself.
|
|
4683
|
+
envAllow: spawnEnvAllow(cotalConfig),
|
|
4684
|
+
resolvedBinaries: harness.binaries,
|
|
4685
|
+
eventsRequired: this.eventsRequired,
|
|
4686
|
+
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
4687
|
+
// (possibly per-agent) launch cwd.
|
|
4688
|
+
workspaceRoot: this.workspaceRoot,
|
|
4689
|
+
};
|
|
4690
|
+
}
|
|
4660
4691
|
/** The refusal for a launch choice the connector cannot honour. Spawn and resume both ask before
|
|
4661
4692
|
* any reserve or mint, so a rule changed here holds on both paths. */
|
|
4662
4693
|
capabilityRefusal(connector, launch) {
|
|
@@ -4668,6 +4699,18 @@ export class Manager {
|
|
|
4668
4699
|
return `${connector.name} connector does not support exact session continuity (continuity: exact)`;
|
|
4669
4700
|
return undefined;
|
|
4670
4701
|
}
|
|
4702
|
+
/** The refusal for an event-plane choice the registration policy or the connector cannot meet.
|
|
4703
|
+
* Spawn and resume both ask before a seat's launch is built, so a seat retained without the
|
|
4704
|
+
* plane cannot come back on a space that now requires it. An absent choice arms the plane. */
|
|
4705
|
+
eventsRefusal(connector, events) {
|
|
4706
|
+
if (this.eventsRequired && events === false)
|
|
4707
|
+
return `space "${this.space}" requires the event plane by registration policy; --no-events (events: false on the start op) is not allowed`;
|
|
4708
|
+
if (events !== false && !connector.eventChannel)
|
|
4709
|
+
return this.eventsRequired
|
|
4710
|
+
? `space "${this.space}" requires the event plane by registration policy, but connector "${connector.name}" does not publish one`
|
|
4711
|
+
: `connector "${connector.name}" does not publish an AG-UI event plane; pass --no-events (events: false on the start op) to launch it without one`;
|
|
4712
|
+
return undefined;
|
|
4713
|
+
}
|
|
4671
4714
|
/** Return connector-provided model catalogs for selector UIs. Optional by connector: a host with no
|
|
4672
4715
|
* local model-list API reports `supported:false` rather than blocking the manager. A connector that
|
|
4673
4716
|
* fails to import shows an `error:` row (from manifest enumeration) and never blocks the others. */
|
|
@@ -5172,13 +5215,10 @@ export class Manager {
|
|
|
5172
5215
|
// the principal's owner is resolved further down, so deriving it from anything in scope here
|
|
5173
5216
|
// would mean guessing at the identity the child will actually connect as. It is added at the
|
|
5174
5217
|
// accept seam below, where the allocated triple exists.
|
|
5175
|
-
|
|
5176
|
-
|
|
5218
|
+
const eventsRefused = this.eventsRefusal(connector, opts.events);
|
|
5219
|
+
if (eventsRefused)
|
|
5220
|
+
return { ok: false, error: eventsRefused };
|
|
5177
5221
|
const events = this.eventsRequired || opts.events !== false;
|
|
5178
|
-
if (events && !connector.eventChannel)
|
|
5179
|
-
return { ok: false, error: this.eventsRequired
|
|
5180
|
-
? `space "${this.space}" requires the event plane by registration policy, but connector "${connector.name}" does not publish one`
|
|
5181
|
-
: `connector "${connector.name}" does not publish an AG-UI event plane; pass --no-events (events: false on the start op) to launch it without one` };
|
|
5182
5222
|
// F2 (Unit B): a STATIC managed spawn REFUSES endpoint capabilities, fail-closed IN CODE (not
|
|
5183
5223
|
// a doc note): the static terminal has no obligation-drain/frontier steps yet, so an accepted-
|
|
5184
5224
|
// but-uncompleted endpoint obligation could execute AFTER its uid is declared retired. The
|
|
@@ -5416,15 +5456,6 @@ export class Manager {
|
|
|
5416
5456
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(credsPath, this.space), credsPath);
|
|
5417
5457
|
provisioned = { id: identity.id, name, lifecycleUid, secretPaths: { creds: credsPath }, ...(custody ? { runtime: custody } : {}) }; // footprint now exists — the finally rolls it back if the spawn throws
|
|
5418
5458
|
}
|
|
5419
|
-
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
5420
|
-
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
5421
|
-
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
5422
|
-
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
5423
|
-
const mcpServers = connectorServers(cotalConfig, agent, opts.shareTools);
|
|
5424
|
-
// The operator's spawn-env policy travels the same route: absent means no extras (the OS
|
|
5425
|
-
// allow-list + operator knobs + connector-declared inputs), present means those names too.
|
|
5426
|
-
// A connector never reads the config itself.
|
|
5427
|
-
const envAllow = spawnEnvAllow(cotalConfig);
|
|
5428
5459
|
// Per-agent cwd overrides the manager's shared workspace root, so agents can be rooted at
|
|
5429
5460
|
// arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
|
|
5430
5461
|
// agent shares the workspace root (the prior, unchanged behavior).
|
|
@@ -5437,7 +5468,7 @@ export class Manager {
|
|
|
5437
5468
|
: undefined;
|
|
5438
5469
|
const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
|
|
5439
5470
|
const launchOpts = {
|
|
5440
|
-
|
|
5471
|
+
...this.hostLaunchFields(agent, opts.shareTools, harness),
|
|
5441
5472
|
name,
|
|
5442
5473
|
role,
|
|
5443
5474
|
// User mode: the principal IS the identity (the endpoint derives card.id from owner+actor);
|
|
@@ -5450,7 +5481,6 @@ export class Manager {
|
|
|
5450
5481
|
// at the broker, never silently).
|
|
5451
5482
|
lifecycleUid,
|
|
5452
5483
|
acceptedToken: issued?.acceptedToken,
|
|
5453
|
-
servers: this.servers,
|
|
5454
5484
|
configPath,
|
|
5455
5485
|
model,
|
|
5456
5486
|
variant,
|
|
@@ -5472,13 +5502,6 @@ export class Manager {
|
|
|
5472
5502
|
allowPublish,
|
|
5473
5503
|
capabilities,
|
|
5474
5504
|
events,
|
|
5475
|
-
eventsRequired: this.eventsRequired,
|
|
5476
|
-
mcpServers,
|
|
5477
|
-
envAllow,
|
|
5478
|
-
resolvedBinaries: harness.binaries,
|
|
5479
|
-
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
5480
|
-
// (possibly per-agent) launch cwd.
|
|
5481
|
-
workspaceRoot: this.workspaceRoot,
|
|
5482
5505
|
cwd,
|
|
5483
5506
|
};
|
|
5484
5507
|
// A delegated seat gets no local launch at all: no launch-material file, control token or
|
|
@@ -6150,6 +6173,9 @@ export class Manager {
|
|
|
6150
6173
|
const unsupported = this.capabilityRefusal(connector, { variant: entry.launch.variant, prompt: entry.launch.prompt, exact });
|
|
6151
6174
|
if (unsupported)
|
|
6152
6175
|
return { ok: false, error: unsupported };
|
|
6176
|
+
const eventsRefused = this.eventsRefusal(connector, entry.launch.events);
|
|
6177
|
+
if (eventsRefused)
|
|
6178
|
+
return { ok: false, error: eventsRefused };
|
|
6153
6179
|
let authority;
|
|
6154
6180
|
try {
|
|
6155
6181
|
authority = await this.validateRetainedAuthority(entry);
|
|
@@ -6158,11 +6184,8 @@ export class Manager {
|
|
|
6158
6184
|
return { ok: false, error: rejectionText(e) };
|
|
6159
6185
|
}
|
|
6160
6186
|
try {
|
|
6161
|
-
const resumeConfig = loadCotalConfig(this.workspaceRoot);
|
|
6162
|
-
const mcpServers = connectorServers(resumeConfig, entry.launch.connector, entry.launch.shareTools);
|
|
6163
|
-
const envAllow = spawnEnvAllow(resumeConfig);
|
|
6164
6187
|
const launchOpts = {
|
|
6165
|
-
|
|
6188
|
+
...this.hostLaunchFields(entry.launch.connector, entry.launch.shareTools, harness),
|
|
6166
6189
|
name: entry.name,
|
|
6167
6190
|
role: entry.role,
|
|
6168
6191
|
id: authority.id,
|
|
@@ -6176,7 +6199,6 @@ export class Manager {
|
|
|
6176
6199
|
lifecycleUid: entry.identity.lifecycleUid,
|
|
6177
6200
|
backfillFloor: entry.backfillFloor,
|
|
6178
6201
|
acceptedToken: entry.identity.mode === "static" ? entry.identity.issued?.acceptedToken : undefined,
|
|
6179
|
-
servers: this.servers,
|
|
6180
6202
|
configPath: entry.launch.source.configPath,
|
|
6181
6203
|
model: entry.launch.model,
|
|
6182
6204
|
variant: entry.launch.variant,
|
|
@@ -6189,10 +6211,6 @@ export class Manager {
|
|
|
6189
6211
|
allowPublish: entry.launch.allowPublish,
|
|
6190
6212
|
capabilities: entry.launch.capabilities,
|
|
6191
6213
|
events: entry.launch.events,
|
|
6192
|
-
mcpServers,
|
|
6193
|
-
envAllow,
|
|
6194
|
-
resolvedBinaries: harness.binaries,
|
|
6195
|
-
workspaceRoot: this.workspaceRoot,
|
|
6196
6214
|
cwd: entry.launch.cwd,
|
|
6197
6215
|
};
|
|
6198
6216
|
const spec = connector.buildLaunch(launchOpts);
|
|
@@ -7685,7 +7703,7 @@ export class Manager {
|
|
|
7685
7703
|
// at a time), so all are reconciled. The `iid` field is the hook item-3's multi-instance sweep
|
|
7686
7704
|
// filters on (skip a goal whose accepting `iid` is a still-LIVE sibling — never settle its goal).
|
|
7687
7705
|
for (const entry of entries) {
|
|
7688
|
-
if (this.goalAcceptances.has(entry.ref.goalId) || this.turnAcceptances.has(entry.ref
|
|
7706
|
+
if (this.goalAcceptances.has(entry.ref.goalId) || this.turnAcceptances.has(turnKey(entry.ref)))
|
|
7689
7707
|
continue; // never settle a goal THIS incarnation drives
|
|
7690
7708
|
try {
|
|
7691
7709
|
// A note marks a TURN entry: its relay is rebuilt (the hold is its bounded ending), never
|
|
@@ -8072,14 +8090,17 @@ export class Manager {
|
|
|
8072
8090
|
const goalId = ctx.request.id;
|
|
8073
8091
|
const { fingerprint } = submissionFingerprint(ctx.request, ctx.subject);
|
|
8074
8092
|
const ref = goalRefOf(ctx.subject, goalId);
|
|
8093
|
+
const key = turnKey(ref);
|
|
8075
8094
|
const executor = { lifecycleUid: this.managerInstanceId, epoch: this.serviceServe?.grant.epoch ?? 0 };
|
|
8076
8095
|
const acceptedAt = Date.now();
|
|
8077
|
-
const prior = this.turnAcceptances.get(
|
|
8096
|
+
const prior = this.turnAcceptances.get(key)?.acceptance;
|
|
8078
8097
|
if (prior !== undefined) {
|
|
8079
8098
|
if (prior.fingerprint !== fingerprint)
|
|
8080
8099
|
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted under a different submission; one goalId never carries two specs (SPEC 13.6)`);
|
|
8081
8100
|
return prior;
|
|
8082
8101
|
}
|
|
8102
|
+
// Refused here, before anything durable, so the caller's goal id stays free for another seat.
|
|
8103
|
+
this.assertSeatTurnFree(a, t, goalId);
|
|
8083
8104
|
const raw = (ctx.request.args ?? {});
|
|
8084
8105
|
const payload = String(raw.payload);
|
|
8085
8106
|
const deadlineAt = raw.deadlineAt === undefined ? acceptedAt + Number(raw.deadlineMs) : Number(raw.deadlineAt);
|
|
@@ -8087,7 +8108,8 @@ export class Manager {
|
|
|
8087
8108
|
throw new EpEnvelopeError("deadline-exceeded", `goal "${goalId}" names a deadline already past (${new Date(deadlineAt).toISOString()}); a turn with no time left to serve it is never accepted (SPEC 13.6)`);
|
|
8088
8109
|
const deadlineMs = deadlineAt - acceptedAt;
|
|
8089
8110
|
const handoffFrom = raw.handoffFrom === undefined ? undefined : String(raw.handoffFrom);
|
|
8090
|
-
const
|
|
8111
|
+
const holdToken = turnHoldToken(key);
|
|
8112
|
+
const note = JSON.stringify({ payload, deadlineAt, holdEpoch: executor.epoch, owner: t.owner, holdToken, ...(handoffFrom !== undefined ? { handoffFrom } : {}) });
|
|
8091
8113
|
// A goal that already ENDED is never accepted again, whatever the retry carries: the bind is
|
|
8092
8114
|
// create-only and outlives an unwind (the failed terminal, the cleared index), so a retry of an
|
|
8093
8115
|
// unwound accept would otherwise re-record an index entry over a tombstone and, before that
|
|
@@ -8137,7 +8159,7 @@ export class Manager {
|
|
|
8137
8159
|
// none. Measured on an auth mesh: minted over the goal-writer, the schedule publish was
|
|
8138
8160
|
// broker-denied and every accept unwound.
|
|
8139
8161
|
await mintCheckpoint(gw.ctx.kv, jetstream(serve.nc), this.space, {
|
|
8140
|
-
ref: { endpoint: ref.endpoint, token:
|
|
8162
|
+
ref: { endpoint: ref.endpoint, token: holdToken },
|
|
8141
8163
|
instanceId: this.managerInstanceId,
|
|
8142
8164
|
epoch: executor.epoch,
|
|
8143
8165
|
goal: { caller: { owner: ctx.subject.caller.owner, actor: ctx.subject.caller.actor, uid: ctx.subject.caller.uid }, goalId },
|
|
@@ -8145,6 +8167,9 @@ export class Manager {
|
|
|
8145
8167
|
deadline: deadlineAt,
|
|
8146
8168
|
now: acceptedAt,
|
|
8147
8169
|
});
|
|
8170
|
+
// Again after the awaits, with no await before the maps are written: another caller's
|
|
8171
|
+
// accept of this id to this seat may have finished meanwhile, and this unwinds the loser.
|
|
8172
|
+
this.assertSeatTurnFree(a, t, goalId);
|
|
8148
8173
|
}
|
|
8149
8174
|
catch (e) {
|
|
8150
8175
|
// The accept is inline (no launch closure to fail later), so a post-bind throw unwinds HERE:
|
|
@@ -8174,13 +8199,13 @@ export class Manager {
|
|
|
8174
8199
|
ref, goalId,
|
|
8175
8200
|
seat: { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid },
|
|
8176
8201
|
payload, acceptedAt, deadlineAt,
|
|
8177
|
-
holdToken
|
|
8202
|
+
holdToken, holdEpoch: executor.epoch,
|
|
8178
8203
|
...(handoffFrom !== undefined ? { handoffFrom } : {}),
|
|
8179
8204
|
};
|
|
8180
|
-
this.pendingTurns.set(
|
|
8205
|
+
this.pendingTurns.set(key, pending);
|
|
8181
8206
|
this.ensureTurnSweep();
|
|
8182
8207
|
const acceptance = { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid, goalId, fingerprint, deadlineAt, executor };
|
|
8183
|
-
this.turnAcceptances.set(
|
|
8208
|
+
this.turnAcceptances.set(key, { acceptance, ref });
|
|
8184
8209
|
this.emitGoalProgress(ref, executor.epoch, { phase: "relayed" });
|
|
8185
8210
|
return acceptance;
|
|
8186
8211
|
}
|
|
@@ -8209,6 +8234,21 @@ export class Manager {
|
|
|
8209
8234
|
.map((p) => ({ goalId: p.goalId, payload: p.payload, acceptedAt: p.acceptedAt, deadlineAt: p.deadlineAt }));
|
|
8210
8235
|
return { turns };
|
|
8211
8236
|
}
|
|
8237
|
+
/** A seat names a turn by goal id alone when it yields, so one incarnation holding two callers'
|
|
8238
|
+
* turns under one id could not say which of the two it answered. */
|
|
8239
|
+
assertSeatTurnFree(a, t, goalId) {
|
|
8240
|
+
if (this.seatTurnKey({ owner: t.owner, actor: t.actor, uid: t.lifecycleUid }, goalId) !== undefined)
|
|
8241
|
+
throw new EpEnvelopeError("failed-precondition", `seat "${a.name}" already holds another caller's turn "${goalId}"; a seat yields by goal id, so it is never handed two turns under one (SPEC 13.6)`);
|
|
8242
|
+
}
|
|
8243
|
+
/** The relay key of the turn addressed to seat incarnation `c` under `goalId`, pending or kept
|
|
8244
|
+
* for a retried yield. The accept path never hands one incarnation two turns under one goal
|
|
8245
|
+
* id, so at most one matches. */
|
|
8246
|
+
seatTurnKey(c, goalId) {
|
|
8247
|
+
for (const [key, { acceptance: a }] of this.turnAcceptances)
|
|
8248
|
+
if (a.goalId === goalId && a.owner === c.owner && a.actor === c.actor && a.uid === c.uid)
|
|
8249
|
+
return key;
|
|
8250
|
+
return undefined;
|
|
8251
|
+
}
|
|
8212
8252
|
/** The seat's yield: claim the hold (one-use; expiry fails closed), then commit the goal
|
|
8213
8253
|
* `succeeded` carrying the TurnResult. A yield AFTER the deadline drives the deadline terminal
|
|
8214
8254
|
* instead and refuses — the expiry outcome stands, never a late success over it. */
|
|
@@ -8227,17 +8267,17 @@ export class Manager {
|
|
|
8227
8267
|
const yieldNote = raw.note === undefined ? undefined : String(raw.note);
|
|
8228
8268
|
if (status === "handoff" && (to === undefined || to.length === 0))
|
|
8229
8269
|
throw new EpEnvelopeError("failed-precondition", `a handoff yield names its addressee ("to"); a handoff to nobody relays nothing`);
|
|
8230
|
-
const
|
|
8270
|
+
const key = this.seatTurnKey(c, goalId);
|
|
8271
|
+
if (key === undefined)
|
|
8272
|
+
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
8273
|
+
const p = this.pendingTurns.get(key);
|
|
8231
8274
|
if (!p) {
|
|
8232
8275
|
// A YIELD WHOSE REPLY WAS LOST IS NOT A YIELD THAT FAILED. The commit deleted the pending
|
|
8233
8276
|
// turn, so a retry found nothing and heard `not-found` — which a seat reads as "drop it",
|
|
8234
8277
|
// reporting failure for work the run already has. The answer the first reply carried is
|
|
8235
8278
|
// served again instead, to the addressee it was addressed to and nobody else.
|
|
8236
|
-
const settled = this.turnAcceptances.get(
|
|
8279
|
+
const settled = this.turnAcceptances.get(key);
|
|
8237
8280
|
if (settled !== undefined) {
|
|
8238
|
-
const a = settled.acceptance;
|
|
8239
|
-
if (a.owner !== c.owner || a.actor !== c.actor || a.uid !== c.uid)
|
|
8240
|
-
throw new EpEnvelopeError("permission-denied", `turn "${goalId}" was addressed to ${a.owner}.${a.actor}/${a.uid}; a yield is the addressee's own (SPEC 13.6)`);
|
|
8241
8281
|
if (settled.settled !== undefined)
|
|
8242
8282
|
return { goalId, state: settled.settled.state };
|
|
8243
8283
|
// The in-memory answer is not yet remembered (pending deleted before the terminal CAS,
|
|
@@ -8246,15 +8286,13 @@ export class Manager {
|
|
|
8246
8286
|
if (settled.ref !== undefined) {
|
|
8247
8287
|
const ended = await readGoalResult(gw.ctx, settled.ref);
|
|
8248
8288
|
if (ended !== undefined) {
|
|
8249
|
-
this.rememberSettledTurn(
|
|
8289
|
+
this.rememberSettledTurn(key, ended.state, ended.ts);
|
|
8250
8290
|
return { goalId, state: ended.state };
|
|
8251
8291
|
}
|
|
8252
8292
|
}
|
|
8253
8293
|
}
|
|
8254
8294
|
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
8255
8295
|
}
|
|
8256
|
-
if (p.seat.owner !== c.owner || p.seat.actor !== c.actor || p.seat.uid !== c.uid)
|
|
8257
|
-
throw new EpEnvelopeError("permission-denied", `turn "${goalId}" is addressed to ${p.seat.owner}.${p.seat.actor}/${p.seat.uid}; a yield is the addressee's own (SPEC 13.6)`);
|
|
8258
8296
|
const cpRef = { endpoint: p.ref.endpoint, token: p.holdToken };
|
|
8259
8297
|
try {
|
|
8260
8298
|
await resumeCheckpoint(gw.ctx.kv, gw.ctx.js, gw.ctx.jsm, this.space, { ref: cpRef, presenter: { id: MANAGER_ENDPOINT, lifecycleUid: this.managerInstanceId }, now: Date.now() });
|
|
@@ -8286,9 +8324,9 @@ export class Manager {
|
|
|
8286
8324
|
});
|
|
8287
8325
|
this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
8288
8326
|
await clearGoalIndex(gw.ctx, p.ref);
|
|
8289
|
-
this.pendingTurns.delete(
|
|
8290
|
-
this.markTurnLatchDropped(
|
|
8291
|
-
this.rememberSettledTurn(
|
|
8327
|
+
this.pendingTurns.delete(key);
|
|
8328
|
+
this.markTurnLatchDropped(key, at);
|
|
8329
|
+
this.rememberSettledTurn(key, fact.state, at);
|
|
8292
8330
|
this.maybeStopTurnSweep();
|
|
8293
8331
|
return { goalId, state: fact.state };
|
|
8294
8332
|
}
|
|
@@ -8311,15 +8349,15 @@ export class Manager {
|
|
|
8311
8349
|
const raw = (ctx.request.args ?? {});
|
|
8312
8350
|
const goalId = String(raw.goalId);
|
|
8313
8351
|
const ref = goalRefOf(ctx.subject, goalId);
|
|
8314
|
-
const
|
|
8315
|
-
const
|
|
8316
|
-
if (p === undefined
|
|
8352
|
+
const key = turnKey(ref);
|
|
8353
|
+
const p = this.pendingTurns.get(key);
|
|
8354
|
+
if (p === undefined) {
|
|
8317
8355
|
const ended = await readGoalResult(gw.ctx, ref);
|
|
8318
8356
|
if (ended !== undefined)
|
|
8319
8357
|
throw goalAlreadyTerminal(goalId, ended);
|
|
8320
8358
|
throw new EpEnvelopeError("failed-precondition", `no turn "${goalId}" of this caller is pending on this manager; cancel withdraws a relayed turn (SPEC 13.6)`);
|
|
8321
8359
|
}
|
|
8322
|
-
this.pendingTurns.delete(
|
|
8360
|
+
this.pendingTurns.delete(key);
|
|
8323
8361
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
8324
8362
|
let fact;
|
|
8325
8363
|
try {
|
|
@@ -8332,14 +8370,14 @@ export class Manager {
|
|
|
8332
8370
|
// a yield that has dropped the entry remembered its answer, and one still committing drops
|
|
8333
8371
|
// the entry it finds. An ended relay is stamped so its acceptance still ages off.
|
|
8334
8372
|
if ((await readGoalResult(gw.ctx, ref)) !== undefined)
|
|
8335
|
-
this.markTurnLatchDropped(
|
|
8336
|
-
else if (this.turnAcceptances.get(
|
|
8337
|
-
this.pendingTurns.set(
|
|
8373
|
+
this.markTurnLatchDropped(key);
|
|
8374
|
+
else if (this.turnAcceptances.get(key)?.settled === undefined)
|
|
8375
|
+
this.pendingTurns.set(key, p);
|
|
8338
8376
|
throw e;
|
|
8339
8377
|
}
|
|
8340
8378
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
8341
|
-
this.markTurnLatchDropped(
|
|
8342
|
-
this.rememberSettledTurn(
|
|
8379
|
+
this.markTurnLatchDropped(key);
|
|
8380
|
+
this.rememberSettledTurn(key, fact.state, Date.now());
|
|
8343
8381
|
await clearGoalIndex(gw.ctx, ref);
|
|
8344
8382
|
this.maybeStopTurnSweep();
|
|
8345
8383
|
return { goalId, state: fact.state };
|
|
@@ -8352,9 +8390,10 @@ export class Manager {
|
|
|
8352
8390
|
const gw = this.goalWriter;
|
|
8353
8391
|
if (!gw)
|
|
8354
8392
|
return;
|
|
8355
|
-
|
|
8393
|
+
const key = turnKey(p.ref);
|
|
8394
|
+
if (!this.pendingTurns.delete(key))
|
|
8356
8395
|
return;
|
|
8357
|
-
this.markTurnLatchDropped(
|
|
8396
|
+
this.markTurnLatchDropped(key);
|
|
8358
8397
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
8359
8398
|
try {
|
|
8360
8399
|
await this.assertGoalWriterEpochCurrent(epoch);
|
|
@@ -8364,7 +8403,7 @@ export class Manager {
|
|
|
8364
8403
|
// (the delete at the top of this function) and after this CAS has to name this terminal.
|
|
8365
8404
|
// Clearing first, then remembering, left a window where maybeStopTurnSweep saw no pending
|
|
8366
8405
|
// turn and no settled answer, wiped the acceptance, and rememberSettledTurn no-op'd.
|
|
8367
|
-
this.rememberSettledTurn(
|
|
8406
|
+
this.rememberSettledTurn(key, fact.state, Date.now());
|
|
8368
8407
|
await clearGoalIndex(gw.ctx, p.ref);
|
|
8369
8408
|
}
|
|
8370
8409
|
catch (e) {
|
|
@@ -8406,16 +8445,16 @@ export class Manager {
|
|
|
8406
8445
|
}
|
|
8407
8446
|
/** The answer a retried yield is served, held for {@link TURN_ANSWER_RETENTION_MS}. Only for a
|
|
8408
8447
|
* turn this incarnation accepted: an entry it has no acceptance for is one it cannot vouch for. */
|
|
8409
|
-
rememberSettledTurn(
|
|
8410
|
-
const entry = this.turnAcceptances.get(
|
|
8448
|
+
rememberSettledTurn(key, state, at) {
|
|
8449
|
+
const entry = this.turnAcceptances.get(key);
|
|
8411
8450
|
if (entry === undefined)
|
|
8412
8451
|
return;
|
|
8413
8452
|
entry.settled = { state, at };
|
|
8414
8453
|
}
|
|
8415
8454
|
/** Stamp when the pending latch dropped, so an unsettled leftover ages off that instant rather
|
|
8416
8455
|
* than the original deadline. Idempotent: the first drop is the one the retry window starts at. */
|
|
8417
|
-
markTurnLatchDropped(
|
|
8418
|
-
const entry = this.turnAcceptances.get(
|
|
8456
|
+
markTurnLatchDropped(key, at = Date.now()) {
|
|
8457
|
+
const entry = this.turnAcceptances.get(key);
|
|
8419
8458
|
if (entry === undefined || entry.leftoverSince !== undefined)
|
|
8420
8459
|
return;
|
|
8421
8460
|
entry.leftoverSince = at;
|
|
@@ -8449,13 +8488,13 @@ export class Manager {
|
|
|
8449
8488
|
// An UNSETTLED acceptance whose pending latch is already gone is the in-flight (or failed)
|
|
8450
8489
|
// deadline commit: keep it for the same window so a late yield can still read the durable
|
|
8451
8490
|
// terminal, then drop it so a commit that never remembered cannot pin the sweep forever.
|
|
8452
|
-
for (const [
|
|
8453
|
-
if (this.pendingTurns.has(
|
|
8491
|
+
for (const [key, e] of [...this.turnAcceptances.entries()]) {
|
|
8492
|
+
if (this.pendingTurns.has(key))
|
|
8454
8493
|
continue;
|
|
8455
8494
|
// Unsettled leftovers age off the latch-drop stamp, never `deadlineAt`.
|
|
8456
8495
|
const at = e.settled?.at ?? e.leftoverSince;
|
|
8457
8496
|
if (at !== undefined && now - at >= TURN_ANSWER_RETENTION_MS)
|
|
8458
|
-
this.turnAcceptances.delete(
|
|
8497
|
+
this.turnAcceptances.delete(key);
|
|
8459
8498
|
}
|
|
8460
8499
|
// The prune above is the one event after which the sweep may have nothing left to do, and
|
|
8461
8500
|
// the settle-time callers cannot see it: they run when an answer has just been remembered.
|
|
@@ -8515,7 +8554,7 @@ export class Manager {
|
|
|
8515
8554
|
payload: parsed.payload,
|
|
8516
8555
|
acceptedAt: spec.value.acceptedAt,
|
|
8517
8556
|
deadlineAt: parsed.deadlineAt,
|
|
8518
|
-
holdToken: turnHoldToken(entry.ref.goalId), holdEpoch: parsed.holdEpoch,
|
|
8557
|
+
holdToken: parsed.holdToken ?? turnHoldToken(entry.ref.goalId), holdEpoch: parsed.holdEpoch,
|
|
8519
8558
|
...(parsed.handoffFrom !== undefined ? { handoffFrom: parsed.handoffFrom } : {}),
|
|
8520
8559
|
};
|
|
8521
8560
|
// The hold is the relay's bounded ending, minted AFTER the index entry and the goal record,
|
|
@@ -8525,13 +8564,14 @@ export class Manager {
|
|
|
8525
8564
|
console.error(`turn reconcile ${p.goalId}: its deadline hold was never minted (the accept crashed before it); left unsettled`);
|
|
8526
8565
|
return;
|
|
8527
8566
|
}
|
|
8528
|
-
|
|
8567
|
+
const key = turnKey(p.ref);
|
|
8568
|
+
this.pendingTurns.set(key, p);
|
|
8529
8569
|
// THE ACCEPTANCE TOO, or the adopted turn's answer is never remembered: `rememberSettledTurn`
|
|
8530
8570
|
// writes only against an acceptance this incarnation holds, and adoption rebuilt the pending
|
|
8531
8571
|
// relay without one, so a yield whose reply was lost after a restart still heard `not-found`
|
|
8532
8572
|
// on the very path adoption exists for. The acceptance is rebuilt from the same records the
|
|
8533
8573
|
// relay was: the spec holds the fingerprint, the index holds the seat, the note the deadline.
|
|
8534
|
-
this.turnAcceptances.set(
|
|
8574
|
+
this.turnAcceptances.set(key, {
|
|
8535
8575
|
acceptance: {
|
|
8536
8576
|
name: p.seat.name, owner: p.seat.owner, actor: p.seat.actor, uid: p.seat.uid, goalId: p.goalId,
|
|
8537
8577
|
fingerprint: spec.value.fingerprint, deadlineAt: p.deadlineAt,
|