@cotal-ai/manager 0.41.3 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/manager.js CHANGED
@@ -18,7 +18,7 @@ import { controlSession } from "./control-session.js";
18
18
  import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
19
19
  // Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
20
20
  // manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
21
- import { jetstreamManager } from "@nats-io/jetstream";
21
+ import { jetstream, jetstreamManager } from "@nats-io/jetstream";
22
22
  import { Kvm } from "@nats-io/kv";
23
23
  import { recordsBucket, epAuthBucket, ensureAuthorityStores, ensureContractStore, createEndpointStreams, contractStoreContext, publishContractArtifact, contractArtifactCanonicalBytes, standaloneConnectOpts, STATIC_SLOT_PREFIX, rawDigest, STANDING_RENEWABLE_TTL_SEC as MANAGED_STATIC_TTL_SEC, newArtifactSigner, RotatingSigner, generationAnchor, sessionsBucket, SESSION_GRANT_MAX_TTL_MS, } from "@cotal-ai/core";
24
24
  // P2 item 6: the manager's ONE §13.6 session plane — offer mint + one-use redeem + PTY-bridge
@@ -28,7 +28,7 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
28
28
  // P2 item 1 (1a-serve): the manager as an ordinary v0.4 `service` endpoint — the §13.1
29
29
  // endpoint-serve credential subsystem (gate provisioning, registration barrier, mint fence) plus
30
30
  // the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
31
- import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, registerServiceInstance, deregisterServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, renderLifecycleBlocked, } from "@cotal-ai/core";
31
+ import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, registerServiceInstance, deregisterServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, renderLifecycleBlocked, } from "@cotal-ai/core";
32
32
  import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
33
33
  import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, } from "./static-lifecycle.js";
34
34
  /** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
@@ -38,6 +38,14 @@ const MAX_AGENTS = 50;
38
38
  * before living this long leaves a cooling stamp that still counts toward the ceiling until it
39
39
  * expires — so churn (spawn↔despawn or spawn↔fast-exit) can't outrun the concurrency bound. */
40
40
  const MIN_LIFETIME = 10_000;
41
+ /** Cadence of the turn-deadline sweep while turns are pending: the poll that turns an elapsed
42
+ * hold's fire into its expired settle and commits the deadline terminal. Deadlines are
43
+ * minutes-scale; a few seconds of lateness on the commit is invisible to the run. */
44
+ const TURN_SWEEP_MS = 5_000;
45
+ /** How long a settled turn's answer is kept so a RETRIED yield hears it instead of `not-found`.
46
+ * The window a lost reply is retried in is seconds; five minutes is generous for that and short
47
+ * enough that the map drains between bursts rather than growing for the process lifetime. */
48
+ const TURN_ANSWER_RETENTION_MS = 5 * 60_000;
41
49
  /** Backstop for the detached-launch readiness race (#159 B1). `startAgent` waits on two REAL outcomes —
42
50
  * the assigned id joining the mesh (presence) = started, the child process exiting = failed — NOT a
43
51
  * liveness-inferring timer. This is only the last-resort bound for "neither happened in time": the launch
@@ -131,6 +139,29 @@ const FREE_SLOT_CAUSE_TEXT = {
131
139
  "session-bind-failed": "this manager stopped it: its host session could not be bound at launch",
132
140
  "resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
133
141
  };
142
+ function parseTurnNote(raw) {
143
+ let o;
144
+ try {
145
+ o = JSON.parse(raw);
146
+ }
147
+ catch {
148
+ return undefined;
149
+ }
150
+ if (o === null || typeof o !== "object")
151
+ return undefined;
152
+ const n = o;
153
+ if (typeof n.payload !== "string" || typeof n.deadlineAt !== "number"
154
+ || typeof n.holdEpoch !== "number" || typeof n.owner !== "string")
155
+ return undefined;
156
+ if (n.handoffFrom !== undefined && typeof n.handoffFrom !== "string")
157
+ return undefined;
158
+ return { payload: n.payload, deadlineAt: n.deadlineAt, holdEpoch: n.holdEpoch, owner: n.owner, ...(typeof n.handoffFrom === "string" ? { handoffFrom: n.handoffFrom } : {}) };
159
+ }
160
+ /** A turn hold's token, DERIVED from the goal id (same recipe as the runtime's pause tokens): a
161
+ * same-goalId retry or a successor incarnation re-derives the identical token with no lookup. */
162
+ function turnHoldToken(goalId) {
163
+ return createHash("sha256").update(`${goalId}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
164
+ }
134
165
  /** One ep request/reply round-trip on the caller's OWN reply-plane filter (§13.2). The responder
135
166
  * derives the reply subject from the authenticated request, so there is no caller-selected reply
136
167
  * target to honour; the caller binds the answer off the reply SUBJECT — endpoint and nonce, both
@@ -330,6 +361,27 @@ export class Manager {
330
361
  /** P2 item 2 (M4): the live spawn goal ref for each managed agent name, so a despawn MID-GOAL
331
362
  * drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
332
363
  agentGoals = new Map();
364
+ /** The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
365
+ * same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
366
+ * the goal-index entry (its acceptance floor + note). */
367
+ /**
368
+ * One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
369
+ * from, and — once the turn settles — the answer a RETRIED yield is served from.
370
+ *
371
+ * A yield's reply can be lost, and the retry used to find the pending turn already deleted and
372
+ * hear `not-found`, which a seat reads as "drop it": failure reported for work that committed.
373
+ * The settled answer lives here for a bounded window instead, and the sweep drops it after,
374
+ * which is also what keeps this map from being one entry per turn for the process lifetime.
375
+ */
376
+ turnAcceptances = new Map();
377
+ /** Every relayed turn awaiting its seat's yield, by goal id. The seat's `turn-pending` pull
378
+ * scans it; the deadline sweep drives expiry; a seat reap stamps its entries `seatDiedAt`, which
379
+ * the deadline terminal carries as `agentDownAt` so the run reads that death as L4002.
380
+ * Rebuilt at boot from goal-index entries carrying a turn note. */
381
+ pendingTurns = new Map();
382
+ /** The deadline sweep behind {@link pendingTurns}: takes the hold's fire into an expired
383
+ * settle and commits the deadline terminal. Unref'd; idle when no turn is pending. */
384
+ turnSweepTimer;
333
385
  /** Process start, for the served `status` uptime. */
334
386
  startedAtMs = Date.now();
335
387
  /** Connector harness paths resolved ONCE at boot. Missing binaries do not stop unrelated manager
@@ -1755,6 +1807,20 @@ export class Manager {
1755
1807
  throw new EpEnvelopeError("permission-denied", denied);
1756
1808
  return this.inputAuthorized(a, args(ctx));
1757
1809
  }),
1810
+ // The turn relay (§8 durable actions): `turn` shares despawn/input's reach policy — the
1811
+ // caller must hold owner-equality or admin over the TARGET seat — written out for the same
1812
+ // reason `input` is (a shared policy, not a shared body).
1813
+ turn: (ctx) => this.serveGated(ctx, async () => {
1814
+ const a = targetAgent(ctx);
1815
+ const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
1816
+ if (denied)
1817
+ throw new EpEnvelopeError("permission-denied", denied);
1818
+ return this.serveTurnGoal(ctx, a);
1819
+ }),
1820
+ // The seat's own half of the relay (`manager.self`, stop-self's tier): a seat pulls the
1821
+ // turns addressed to ITS incarnation and yields them; no reach beyond itself exists here.
1822
+ turnPending: (ctx) => this.serveGated(ctx, () => this.turnPendingFor(ctx.subject.caller)),
1823
+ turnYield: (ctx) => this.serveGated(ctx, () => this.serveTurnYield(ctx.subject.caller, args(ctx))),
1758
1824
  stopSelf: (ctx) => this.serveGated(ctx, () => unwrap(this.opStopSelf(callerOf(ctx), args(ctx)))),
1759
1825
  definePersona: (ctx) => this.serveGated(ctx, () => unwrap(this.opDefinePersona(args(ctx), callerOf(ctx), false))),
1760
1826
  listPersonas: (ctx) => this.serveGated(ctx, () => unwrap(this.opListPersonas(callerOf(ctx), false))),
@@ -2138,6 +2204,10 @@ export class Manager {
2138
2204
  // `target-despawn` reason. Fires once per agent on every free path (despawn / self-stop / reap /
2139
2205
  // exit) via the `agents` guard above; a no-op when no plane or no live session for the target.
2140
2206
  this.sessionPlane?.endForTarget(a.name, a.lifecycleUid, "target-despawn");
2207
+ // The turn relay's reap hook: every pending turn addressed to THIS incarnation is stamped
2208
+ // with the death, and its deadline terminal carries it as `agentDownAt`. The addressee is gone
2209
+ // and no successor may answer for it; the terminal is still the deadline's, not a new reason.
2210
+ this.failSeatTurns(a.name, a.lifecycleUid);
2141
2211
  if (floor && Date.now() - a.startedAt < MIN_LIFETIME)
2142
2212
  this.cooling.push(a.startedAt + MIN_LIFETIME);
2143
2213
  // #29 piece 3: on a USER mesh the name is RESERVED PENDING RETIREMENT — despawn started this
@@ -5045,14 +5115,19 @@ export class Manager {
5045
5115
  // Single-manager item 2: EVERY inherited entry belongs to a DEAD predecessor (only one manager
5046
5116
  // at a time), so all are reconciled. The `iid` field is the hook item-3's multi-instance sweep
5047
5117
  // filters on (skip a goal whose accepting `iid` is a still-LIVE sibling — never settle its goal).
5048
- for (const { ref, iid } of entries) {
5049
- if (this.goalAcceptances.has(ref.goalId))
5118
+ for (const entry of entries) {
5119
+ if (this.goalAcceptances.has(entry.ref.goalId) || this.turnAcceptances.has(entry.ref.goalId))
5050
5120
  continue; // never settle a goal THIS incarnation drives
5051
5121
  try {
5052
- await this.reconcileOneGoal(ref, iid);
5122
+ // A note marks a TURN entry: its relay is rebuilt (the hold is its bounded ending), never
5123
+ // settled uncertain — the spawn arm's readiness window is the wrong ending for a relay.
5124
+ if (entry.note !== undefined)
5125
+ await this.adoptTurnGoal(entry);
5126
+ else
5127
+ await this.reconcileOneGoal(entry.ref, entry.iid);
5053
5128
  }
5054
5129
  catch (e) {
5055
- console.error(`! goal reconcile for ${ref.goalId}: ${e.message}`);
5130
+ console.error(`! goal reconcile for ${entry.ref.goalId}: ${e.message}`);
5056
5131
  }
5057
5132
  }
5058
5133
  if (entries.length)
@@ -5399,6 +5474,423 @@ export class Manager {
5399
5474
  return { name: d.name, owner: DEV_OWNER, actor: d.id, uid: d.lifecycleUid, goalId, fingerprint, readinessDeadlineMs, executor };
5400
5475
  throw new EpEnvelopeError("unavailable", `goal "${goalId}" is already accepted but its allocated identity is not readable (no acceptance floor, and no terminal carrying one); retry (SPEC 13.6)`);
5401
5476
  }
5477
+ // ---- the turn relay (§8 durable actions) ------------------------------------------------------
5478
+ //
5479
+ // A seat is NOT an endpoint: a run's `turn(agent, payload)` rides the manager, which accepts it
5480
+ // as a goal (the spawn-as-action pattern minus the launch closure), parks the payload durably on
5481
+ // the goal-index entry's note, and lets the seat PULL it (`turn-pending`) and answer it
5482
+ // (`turn-yield`) under its own `manager.self` reach. The deadline is a goal-bound HOLD minted at
5483
+ // accept: the delivery daemon's timer writer pumps its schedule, the fire settles it `expired`,
5484
+ // and THIS manager commits the deny (`hold-expired` is the predicate core verifies). Three
5485
+ // endings, one each: yield -> succeeded (the TurnResult rides the terminal's data), deadline ->
5486
+ // failed `turn-deadline`, and a seat death before that is stamped by the reap hook so the same
5487
+ // terminal carries `agentDownAt` (the run reads it as L4002). The adoption sweep stamps nothing.
5488
+ /** Accept one `turn` goal against a live managed seat. Mirrors {@link serveSpawnGoal}'s accept
5489
+ * path (idempotent retry map, index-CAS-before-bind, create-only bindGoal) but runs it INLINE:
5490
+ * there is no launch to hand off, so the accept either completes durably or unwinds its own
5491
+ * bound goal with a `failed` terminal before refusing. */
5492
+ async serveTurnGoal(ctx, a) {
5493
+ const gw = this.goalWriter;
5494
+ if (!gw)
5495
+ throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing; turn-as-action cannot accept (SPEC 13.6)");
5496
+ const serve = this.serviceServe;
5497
+ if (!serve)
5498
+ throw new EpEnvelopeError("unavailable", "the manager service endpoint is not serving; a turn's deadline hold is armed over its connection (SPEC 13.9)");
5499
+ if (!this.goalReconcileDone)
5500
+ throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; retry shortly (SPEC 13.6)");
5501
+ const t = ctx.request.target; // targeted command: the serve boundary enforced presence + currency
5502
+ const goalId = ctx.request.id;
5503
+ const { fingerprint } = submissionFingerprint(ctx.request, ctx.subject);
5504
+ const ref = goalRefOf(ctx.subject, goalId);
5505
+ const executor = { lifecycleUid: this.managerInstanceId, epoch: this.serviceServe?.grant.epoch ?? 0 };
5506
+ const acceptedAt = Date.now();
5507
+ const prior = this.turnAcceptances.get(goalId)?.acceptance;
5508
+ if (prior !== undefined) {
5509
+ if (prior.fingerprint !== fingerprint)
5510
+ throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted under a different submission; one goalId never carries two specs (SPEC 13.6)`);
5511
+ return prior;
5512
+ }
5513
+ const raw = (ctx.request.args ?? {});
5514
+ const payload = String(raw.payload);
5515
+ const deadlineMs = Number(raw.deadlineMs);
5516
+ const handoffFrom = raw.handoffFrom === undefined ? undefined : String(raw.handoffFrom);
5517
+ const deadlineAt = acceptedAt + deadlineMs;
5518
+ const note = JSON.stringify({ payload, deadlineAt, holdEpoch: executor.epoch, owner: t.owner, ...(handoffFrom !== undefined ? { handoffFrom } : {}) });
5519
+ // A goal that already ENDED is never accepted again, whatever the retry carries: the bind is
5520
+ // create-only and outlives an unwind (the failed terminal, the cleared index), so a retry of an
5521
+ // unwound accept would otherwise re-record an index entry over a tombstone and, before that
5522
+ // check existed, be served an acceptance rebuilt from it — a live deadline on a turn no seat
5523
+ // would ever be shown. The terminal is the durable answer, and it is read before anything is
5524
+ // written.
5525
+ const ended = await readGoalResult(gw.ctx, ref);
5526
+ if (ended !== undefined)
5527
+ throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" already ended ${ended.state}; a bound goal is never accepted again (SPEC 13.6)`);
5528
+ // Index-CAS-before-bind, exactly as spawn: the entry (floor + note) is the durable relay
5529
+ // record a successor rebuilds from, so it must exist before the acceptance is servable.
5530
+ const idx = await recordGoalIndex(gw.ctx, ref, executor.lifecycleUid, { name: a.name, actor: t.actor, uid: t.lifecycleUid, readinessDeadlineMs: deadlineMs }, note);
5531
+ if (!idx.recorded && idx.existing.iid !== executor.lifecycleUid)
5532
+ throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted by instance "${idx.existing.iid}"; that instance owns its relay and this attempt provisions nothing (SPEC 13.6)`);
5533
+ const b = await bindGoal(gw.ctx, ref, fingerprint);
5534
+ if (!b.bound) {
5535
+ if (b.existing.fingerprint !== fingerprint)
5536
+ throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" is already bound to a different submission; one goalId never carries two specs (SPEC 13.6)`);
5537
+ // Same submission, bound, not ended, and not in the acceptance map. Every relay this
5538
+ // incarnation holds is in that map (the accept path and the boot sweep both write it), so
5539
+ // this is a goal nobody is relaying: an earlier attempt bound it and could not finish its
5540
+ // unwind, or a predecessor's relay was left unadopted. Nothing here can rebuild a relay a
5541
+ // seat will be shown, so the caller is refused rather than served an acceptance no pending
5542
+ // entry backs; the index entry this attempt recorded is withdrawn with it.
5543
+ if (idx.recorded)
5544
+ await clearGoalIndex(gw.ctx, ref);
5545
+ throw new EpEnvelopeError("unavailable", `goal "${goalId}" is bound but no relay of it is pending on this instance; retry (SPEC 13.6)`);
5546
+ }
5547
+ try {
5548
+ await createGoal(gw.ctx, ref, {
5549
+ fingerprint,
5550
+ command: ctx.subject.command,
5551
+ caller: { id: `${ctx.subject.caller.owner}.${ctx.subject.caller.actor}`, lifecycleUid: ctx.subject.caller.uid },
5552
+ acceptedEpoch: executor.epoch,
5553
+ requestId: goalId,
5554
+ sourceSeq: 0,
5555
+ acceptedAt,
5556
+ readinessDeadlineMs: deadlineMs,
5557
+ // The §13.6 target pin: the seat INCARNATION this payload is addressed to. A successor
5558
+ // under the same name is a different addressee; its uid differs and the pin holds that.
5559
+ target: { owner: t.owner, actor: t.actor, lifecycleUid: t.lifecycleUid, mappingRevision: 0 },
5560
+ });
5561
+ // The deadline, as a goal-bound hold on the plane: its EXPIRED settle is the one predicate
5562
+ // `commitGoalResult` accepts for the deny. The record rides the goal-writer's KV; the
5563
+ // `.schedule` request rides the SERVE connection, because the timer row is the serving
5564
+ // instance's own (`ept.<e>.<iid>.<epoch>.*.schedule`, SPEC 13.9) and the goal-writer holds
5565
+ // none. Measured on an auth mesh: minted over the goal-writer, the schedule publish was
5566
+ // broker-denied and every accept unwound.
5567
+ await mintCheckpoint(gw.ctx.kv, jetstream(serve.nc), this.space, {
5568
+ ref: { endpoint: ref.endpoint, token: turnHoldToken(goalId) },
5569
+ instanceId: this.managerInstanceId,
5570
+ epoch: executor.epoch,
5571
+ goal: { caller: { owner: ctx.subject.caller.owner, actor: ctx.subject.caller.actor, uid: ctx.subject.caller.uid }, goalId },
5572
+ holder: { id: MANAGER_ENDPOINT, lifecycleUid: this.managerInstanceId },
5573
+ deadline: deadlineAt,
5574
+ now: acceptedAt,
5575
+ });
5576
+ }
5577
+ catch (e) {
5578
+ // The accept is inline (no launch closure to fail later), so a post-bind throw unwinds HERE:
5579
+ // commit the failed terminal this attempt owns, clear the index, and refuse the accept — an
5580
+ // accepted-but-unanswered goal must never be left for the boot sweep to find (H1's rule).
5581
+ const msg = e?.message ?? String(e);
5582
+ try {
5583
+ await this.assertGoalWriterEpochCurrent(executor.epoch);
5584
+ // The goal is TARGET-PINNED, so its completion proves the seat's currency exactly as the
5585
+ // yield's does (managed-seat epochs are 0 within an incarnation; the resolver answers from
5586
+ // the live agents map). Committed without it, core refused the terminal and the unwind
5587
+ // left a bound goal with no ending for every retry to find.
5588
+ await commitGoalResult(gw.ctx, {
5589
+ ref, now: Date.now(), cause: "complete", state: "failed", data: { error: msg },
5590
+ committer: { instanceId: this.managerInstanceId, epoch: executor.epoch },
5591
+ executor: { lifecycleUid: t.lifecycleUid, epoch: 0 },
5592
+ resolveCurrentEpoch: (target) => this.agents.get(a.name)?.lifecycleUid === target.lifecycleUid ? 0 : null,
5593
+ });
5594
+ await clearGoalIndex(gw.ctx, ref);
5595
+ }
5596
+ catch (e2) {
5597
+ console.error(`! turn accept unwind for ${goalId}: ${e2.message}`);
5598
+ }
5599
+ throw e instanceof EpEnvelopeError ? e : new EpEnvelopeError("internal", `turn accept for ${goalId} failed: ${msg}`);
5600
+ }
5601
+ const pending = {
5602
+ ref, goalId,
5603
+ seat: { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid },
5604
+ payload, acceptedAt, deadlineAt,
5605
+ holdToken: turnHoldToken(goalId), holdEpoch: executor.epoch,
5606
+ ...(handoffFrom !== undefined ? { handoffFrom } : {}),
5607
+ };
5608
+ this.pendingTurns.set(goalId, pending);
5609
+ this.ensureTurnSweep();
5610
+ const acceptance = { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid, goalId, fingerprint, deadlineAt, executor };
5611
+ this.turnAcceptances.set(goalId, { acceptance });
5612
+ this.emitGoalProgress(ref, executor.epoch, { phase: "relayed" });
5613
+ return acceptance;
5614
+ }
5615
+ /** The seat's pull: every pending turn addressed to the CALLER's incarnation, oldest first.
5616
+ * The uid is part of the address — a successor seat never receives a predecessor's turn. */
5617
+ turnPendingFor(c) {
5618
+ // AN EMPTY LIST IS AN ANSWER, so it must not be given before the index is rebuilt. Between
5619
+ // registration and `reconcileGoalIndex`, `pendingTurns` holds nothing a predecessor accepted,
5620
+ // and a seat that pulls in that window is told authoritatively that it has no turn. The seat
5621
+ // treats a refusal as "keep what you hold" and an empty list as "you hold nothing", so the
5622
+ // window has to refuse, the same way `serveTurnGoal` does.
5623
+ if (!this.goalReconcileDone)
5624
+ throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; the pending-turn index is not rebuilt yet, retry (SPEC 13.6)");
5625
+ // Two turns on one seat are serialized at the dispatch (cotal-lang 6.5): only the oldest
5626
+ // unsettled one is served, and the next surfaces once it yields or its deadline denies, so a
5627
+ // seat is never shown two turns at once.
5628
+ // An ELAPSED turn is not servable: its hold is expirable and its run has already thrown from
5629
+ // its own pause, so handing it to the seat buys work whose yield is refused. It also cannot be
5630
+ // allowed to stay the head, or one expired turn dams every later turn for that seat.
5631
+ const now = Date.now();
5632
+ const turns = [...this.pendingTurns.values()]
5633
+ .filter((p) => p.seat.owner === c.owner && p.seat.actor === c.actor && p.seat.uid === c.uid)
5634
+ .filter((p) => now < p.deadlineAt)
5635
+ .sort((x, y) => x.acceptedAt - y.acceptedAt)
5636
+ .slice(0, 1)
5637
+ .map((p) => ({ goalId: p.goalId, payload: p.payload, acceptedAt: p.acceptedAt, deadlineAt: p.deadlineAt }));
5638
+ return { turns };
5639
+ }
5640
+ /** The seat's yield: claim the hold (one-use; expiry fails closed), then commit the goal
5641
+ * `succeeded` carrying the TurnResult. A yield AFTER the deadline drives the deadline terminal
5642
+ * instead and refuses — the expiry outcome stands, never a late success over it. */
5643
+ async serveTurnYield(c, raw) {
5644
+ // Same boot window as the pull, and checked first because it is the earlier boot phase: before
5645
+ // the index is rebuilt a predecessor's accepted turn is not in `pendingTurns`, and `not-found`
5646
+ // would tell the seat to drop work it is holding.
5647
+ if (!this.goalReconcileDone)
5648
+ throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; this yield's turn may not be indexed yet, retry (SPEC 13.6)");
5649
+ const gw = this.goalWriter;
5650
+ if (!gw)
5651
+ throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing (SPEC 13.6)");
5652
+ const goalId = String(raw.goalId);
5653
+ const status = String(raw.status);
5654
+ const to = raw.to === undefined ? undefined : String(raw.to);
5655
+ const yieldNote = raw.note === undefined ? undefined : String(raw.note);
5656
+ if (status === "handoff" && (to === undefined || to.length === 0))
5657
+ throw new EpEnvelopeError("failed-precondition", `a handoff yield names its addressee ("to"); a handoff to nobody relays nothing`);
5658
+ const p = this.pendingTurns.get(goalId);
5659
+ if (!p) {
5660
+ // A YIELD WHOSE REPLY WAS LOST IS NOT A YIELD THAT FAILED. The commit deleted the pending
5661
+ // turn, so a retry found nothing and heard `not-found` — which a seat reads as "drop it",
5662
+ // reporting failure for work the run already has. The answer the first reply carried is
5663
+ // served again instead, to the addressee it was addressed to and nobody else.
5664
+ const settled = this.turnAcceptances.get(goalId);
5665
+ if (settled?.settled !== undefined) {
5666
+ const a = settled.acceptance;
5667
+ if (a.owner !== c.owner || a.actor !== c.actor || a.uid !== c.uid)
5668
+ throw new EpEnvelopeError("permission-denied", `turn "${goalId}" was addressed to ${a.owner}.${a.actor}/${a.uid}; a yield is the addressee's own (SPEC 13.6)`);
5669
+ return { goalId, state: settled.settled.state };
5670
+ }
5671
+ throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
5672
+ }
5673
+ if (p.seat.owner !== c.owner || p.seat.actor !== c.actor || p.seat.uid !== c.uid)
5674
+ throw new EpEnvelopeError("permission-denied", `turn "${goalId}" is addressed to ${p.seat.owner}.${p.seat.actor}/${p.seat.uid}; a yield is the addressee's own (SPEC 13.6)`);
5675
+ const cpRef = { endpoint: p.ref.endpoint, token: p.holdToken };
5676
+ try {
5677
+ await resumeCheckpoint(gw.ctx.kv, gw.ctx.js, gw.ctx.jsm, this.space, { ref: cpRef, presenter: { id: MANAGER_ENDPOINT, lifecycleUid: this.managerInstanceId }, now: Date.now() });
5678
+ }
5679
+ catch (e) {
5680
+ const settle = await readCheckpointSettle(gw.ctx.jsm, this.space, cpRef).catch(() => undefined);
5681
+ if (settle?.settle === "expired") {
5682
+ await this.commitTurnDeadline(p);
5683
+ throw new EpEnvelopeError("failed-precondition", `turn "${goalId}" elapsed its deadline before the yield; the deadline terminal stands (SPEC 13.6)`);
5684
+ }
5685
+ // Already resumed BY THIS MANAGER (the hold is holder-bound): a prior yield attempt claimed
5686
+ // it and then failed to commit. Fall through and commit — the claim is ours to finish.
5687
+ if (settle?.settle !== "resumed")
5688
+ throw e;
5689
+ }
5690
+ const epoch = this.serviceServe?.grant.epoch ?? 0;
5691
+ await this.assertGoalWriterEpochCurrent(epoch);
5692
+ const at = Date.now();
5693
+ const data = { status, ...(to !== undefined ? { to } : {}), ...(yieldNote !== undefined ? { note: yieldNote } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}), at };
5694
+ // The goal is TARGET-PINNED, so this completion proves the EXECUTOR's (the seat's) fresh
5695
+ // currency, not the manager's: managed-seat epochs are 0 within an incarnation (the same
5696
+ // convention resolveTarget serves), and the resolver answers from the live agents map — a
5697
+ // seat that died between its yield call and this commit refuses `expired` here.
5698
+ const { fact } = await commitGoalResult(gw.ctx, {
5699
+ ref: p.ref, now: at, cause: "complete", state: "succeeded", data,
5700
+ committer: { instanceId: this.managerInstanceId, epoch },
5701
+ executor: { lifecycleUid: p.seat.uid, epoch: 0 },
5702
+ resolveCurrentEpoch: (target) => this.agents.get(p.seat.name)?.lifecycleUid === target.lifecycleUid ? 0 : null,
5703
+ });
5704
+ this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
5705
+ await clearGoalIndex(gw.ctx, p.ref);
5706
+ this.pendingTurns.delete(goalId);
5707
+ this.rememberSettledTurn(goalId, fact.state, at);
5708
+ this.maybeStopTurnSweep();
5709
+ return { goalId, state: fact.state };
5710
+ }
5711
+ /** Commit the deadline terminal for one pending turn whose hold settled EXPIRED: the deny's
5712
+ * predicate is that recorded settle, verified by core against the spec's goal binding. The map
5713
+ * delete is the idempotency latch (yield-loss and sweep race here); a commit failure after it
5714
+ * converges through the goal index at the next boot, the same narrow leg spawn leaves open. */
5715
+ async commitTurnDeadline(p) {
5716
+ const gw = this.goalWriter;
5717
+ if (!gw)
5718
+ return;
5719
+ if (!this.pendingTurns.delete(p.goalId))
5720
+ return;
5721
+ const epoch = this.serviceServe?.grant.epoch ?? 0;
5722
+ try {
5723
+ await this.assertGoalWriterEpochCurrent(epoch);
5724
+ const { fact } = await commitGoalResult(gw.ctx, { ref: p.ref, now: Date.now(), cause: "deny", denial: { kind: "hold-expired", token: p.holdToken }, data: { reason: "turn-deadline", ...(p.seatDiedAt !== undefined ? { agentDownAt: p.seatDiedAt } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}) }, committer: { instanceId: this.managerInstanceId, epoch } });
5725
+ this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
5726
+ await clearGoalIndex(gw.ctx, p.ref);
5727
+ // The SAME order the yield path keeps: remember, then ask whether the sweep may stop. Asked
5728
+ // first, the stop saw no pending turn and no settled answer, cleared the acceptance map, and
5729
+ // the remember below found nothing to write to; a yield whose reply was lost then heard
5730
+ // `not-found` for the last expired turn, which the seat reads as "drop it".
5731
+ this.rememberSettledTurn(p.goalId, fact.state, Date.now());
5732
+ }
5733
+ catch (e) {
5734
+ console.error(`! turn deadline terminal for ${p.goalId}: ${e.message}`);
5735
+ }
5736
+ this.maybeStopTurnSweep();
5737
+ }
5738
+ /** The reap hook: a pending turn addressed to the reaped INCARNATION is MARKED dead, never
5739
+ * settled early — no honest terminal exists for it (a completion must prove the executor's
5740
+ * live currency, and the hold is only expirable once due), so the entry rides to its deadline
5741
+ * and the deny then records both facts. The addressee's own client observes the death from
5742
+ * presence long before that, which is the run's L4002. */
5743
+ failSeatTurns(name, uid) {
5744
+ for (const p of this.pendingTurns.values()) {
5745
+ if (p.seat.name !== name || p.seat.uid !== uid || p.seatDiedAt !== undefined)
5746
+ continue;
5747
+ p.seatDiedAt = Date.now();
5748
+ console.error(`turn ${p.goalId}: addressed seat ${name}/${uid} died before yielding; the deadline terminal will record it`);
5749
+ }
5750
+ }
5751
+ /** Settle one DUE hold as its owner: the manager minted it, so once the deadline has passed it
5752
+ * expires the pause itself ({@link expireCheckpoint}, the guard-hold precedent) rather than
5753
+ * waiting on the broker's fire — which it holds no grant to read, and which a hold minted under
5754
+ * a predecessor's epoch would carry on a subject this incarnation never subscribes. Idempotent
5755
+ * by the plane: a yield that claimed the hold first is observed as `resumed` and left to its
5756
+ * own commit, and an already-expired hold returns its recorded settle. */
5757
+ async expireTurnHold(p) {
5758
+ const gw = this.goalWriter;
5759
+ if (!gw)
5760
+ return undefined;
5761
+ return expireCheckpoint(gw.ctx.kv, gw.ctx.js, gw.ctx.jsm, this.space, { ref: { endpoint: p.ref.endpoint, token: p.holdToken }, now: Date.now() });
5762
+ }
5763
+ ensureTurnSweep() {
5764
+ if (this.turnSweepTimer !== undefined)
5765
+ return;
5766
+ const t = setInterval(() => { void this.sweepTurnDeadlines().catch((e) => console.error(`! turn deadline sweep: ${e.message}`)); }, TURN_SWEEP_MS);
5767
+ t.unref?.();
5768
+ this.turnSweepTimer = t;
5769
+ }
5770
+ /** The answer a retried yield is served, held for {@link TURN_ANSWER_RETENTION_MS}. Only for a
5771
+ * turn this incarnation accepted: an entry it has no acceptance for is one it cannot vouch for. */
5772
+ rememberSettledTurn(goalId, state, at) {
5773
+ const entry = this.turnAcceptances.get(goalId);
5774
+ if (entry === undefined)
5775
+ return;
5776
+ entry.settled = { state, at };
5777
+ }
5778
+ /** Stop the sweep only when it has nothing left to do. It drives elapsed turns to their deadline
5779
+ * terminals AND drops settled answers once their retry window has passed, so a stop while the
5780
+ * second is still owed would leave one entry per turn for the process lifetime. */
5781
+ maybeStopTurnSweep() {
5782
+ if (this.pendingTurns.size > 0)
5783
+ return;
5784
+ for (const e of this.turnAcceptances.values())
5785
+ if (e.settled !== undefined)
5786
+ return;
5787
+ this.turnAcceptances.clear();
5788
+ this.stopTurnSweep();
5789
+ }
5790
+ stopTurnSweep() {
5791
+ if (this.turnSweepTimer === undefined)
5792
+ return;
5793
+ clearInterval(this.turnSweepTimer);
5794
+ this.turnSweepTimer = undefined;
5795
+ }
5796
+ /** Drive every elapsed pending turn to its deadline terminal. Only entries at/past their own
5797
+ * `deadlineAt` are read at all; a `resumed` settle is a yield mid-commit and is left alone. */
5798
+ async sweepTurnDeadlines() {
5799
+ const now = Date.now();
5800
+ // A settled answer is kept only as long as a lost reply could still be retried under it.
5801
+ for (const [goalId, e] of [...this.turnAcceptances.entries()])
5802
+ if (e.settled !== undefined && now - e.settled.at >= TURN_ANSWER_RETENTION_MS)
5803
+ this.turnAcceptances.delete(goalId);
5804
+ // The prune above is the one event after which the sweep may have nothing left to do, and
5805
+ // the settle-time callers cannot see it: they run when an answer has just been remembered.
5806
+ this.maybeStopTurnSweep();
5807
+ for (const p of [...this.pendingTurns.values()]) {
5808
+ if (now < p.deadlineAt)
5809
+ continue;
5810
+ try {
5811
+ const settle = await this.expireTurnHold(p);
5812
+ if (settle?.settle === "expired")
5813
+ await this.commitTurnDeadline(p);
5814
+ }
5815
+ catch (e) {
5816
+ console.error(`! turn deadline sweep for ${p.goalId}: ${e.message}`);
5817
+ }
5818
+ }
5819
+ }
5820
+ /** Adopt one inherited TURN goal at boot (the reconcile sweep's turn branch): a non-terminal
5821
+ * entry carrying a parseable note rebuilds its pending relay — never an `uncertain` settle,
5822
+ * because the deadline hold IS this goal's bounded ending and it survived the restart. No
5823
+ * liveness verdict is made here (the agents map is empty during reconcile, so absence means
5824
+ * "not yet re-registered", never "dead"); a garbled note or a missing floor is logged and
5825
+ * left, the dead-pointer honesty rule. */
5826
+ async adoptTurnGoal(entry) {
5827
+ const gw = this.goalWriter;
5828
+ if (!gw)
5829
+ return;
5830
+ const status = await readGoalStatus(gw.ctx, entry.ref);
5831
+ if (status === undefined)
5832
+ return; // index points at no goal record: a dead pointer, left for honesty
5833
+ if (GOAL_TERMINAL_STATES.includes(status.value.state)) {
5834
+ await clearGoalIndex(gw.ctx, entry.ref);
5835
+ return;
5836
+ }
5837
+ if (entry.iid !== this.managerInstanceId) {
5838
+ console.error(`turn reconcile ${entry.ref.goalId}: accepted by instance "${entry.iid}", not this incarnation "${this.managerInstanceId}"; left for its owner (never a cross-instance settle)`);
5839
+ return;
5840
+ }
5841
+ const parsed = entry.note !== undefined ? parseTurnNote(entry.note) : undefined;
5842
+ if (entry.allocated === undefined || parsed === undefined) {
5843
+ console.error(`turn reconcile ${entry.ref.goalId}: the relay record is garbled (no floor, or an unparseable note); left unsettled`);
5844
+ return;
5845
+ }
5846
+ const spec = await readGoalSpec(gw.ctx, entry.ref);
5847
+ if (spec === undefined) {
5848
+ // NOW IS NOT WHEN THIS WAS ACCEPTED. `acceptedAt` is what orders a seat's queue, and a turn
5849
+ // stamped with the boot instant sorts BEHIND every turn accepted since — a predecessor's
5850
+ // oldest turn served last, which is the one thing the ordering exists to prevent. A goal
5851
+ // with an index entry and no spec is a relay record that is not readable, and this is the
5852
+ // same verdict the garbled-note branch above reaches: left unsettled, said out loud.
5853
+ console.error(`turn reconcile ${entry.ref.goalId}: the goal spec is unreadable, so its acceptance time is unknown; left unsettled rather than re-stamped with the boot instant`);
5854
+ return;
5855
+ }
5856
+ const p = {
5857
+ ref: entry.ref, goalId: entry.ref.goalId,
5858
+ seat: { name: entry.allocated.name, owner: parsed.owner, actor: entry.allocated.actor, uid: entry.allocated.uid },
5859
+ payload: parsed.payload,
5860
+ acceptedAt: spec.value.acceptedAt,
5861
+ deadlineAt: parsed.deadlineAt,
5862
+ holdToken: turnHoldToken(entry.ref.goalId), holdEpoch: parsed.holdEpoch,
5863
+ ...(parsed.handoffFrom !== undefined ? { handoffFrom: parsed.handoffFrom } : {}),
5864
+ };
5865
+ // The hold is the relay's bounded ending, minted AFTER the index entry and the goal record,
5866
+ // so a crash between them leaves a goal with no hold: nothing could ever settle it, and the
5867
+ // sweep would report the same missing pause every tick. Left unsettled, said once.
5868
+ if ((await readCheckpointSpec(gw.ctx.kv, { endpoint: p.ref.endpoint, token: p.holdToken })) === undefined) {
5869
+ console.error(`turn reconcile ${p.goalId}: its deadline hold was never minted (the accept crashed before it); left unsettled`);
5870
+ return;
5871
+ }
5872
+ this.pendingTurns.set(p.goalId, p);
5873
+ // THE ACCEPTANCE TOO, or the adopted turn's answer is never remembered: `rememberSettledTurn`
5874
+ // writes only against an acceptance this incarnation holds, and adoption rebuilt the pending
5875
+ // relay without one, so a yield whose reply was lost after a restart still heard `not-found`
5876
+ // on the very path adoption exists for. The acceptance is rebuilt from the same records the
5877
+ // relay was: the spec holds the fingerprint, the index holds the seat, the note the deadline.
5878
+ this.turnAcceptances.set(p.goalId, {
5879
+ acceptance: {
5880
+ name: p.seat.name, owner: p.seat.owner, actor: p.seat.actor, uid: p.seat.uid, goalId: p.goalId,
5881
+ fingerprint: spec.value.fingerprint, deadlineAt: p.deadlineAt,
5882
+ executor: { lifecycleUid: this.managerInstanceId, epoch: p.holdEpoch },
5883
+ },
5884
+ });
5885
+ // NO LIVENESS VERDICT AT BOOT. The agents map is empty here: seats are re-registered later,
5886
+ // by the resume path, so `not in the map` at this moment means "not yet re-registered", never
5887
+ // "dead". Stamping death from it marked EVERY adopted turn as a dead seat, and its deadline
5888
+ // terminal then carried `agentDownAt`, so the run raised L4002 for a seat that was alive the
5889
+ // whole time where the reference says L4003. The reap hook is the only honest writer of that
5890
+ // fact: it fires when a managed seat actually dies.
5891
+ console.error(`turn reconcile ${p.goalId}: pending relay to ${p.seat.name}/${p.seat.uid} adopted (deadline ${p.deadlineAt})`);
5892
+ this.ensureTurnSweep();
5893
+ }
5402
5894
  /** M4 (settle race): a despawn MID-GOAL drives the goal's cancel terminal - transition to
5403
5895
  * `cancelling`, then commit the `cancel` cause on the goal-writer connection. First-terminal-fact
5404
5896
  * wins: if the readiness outcome already committed (succeeded/failed/uncertain) the transition or