@cotal-ai/runtime 0.169.0 → 0.170.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,7 +19,7 @@
19
19
  import { createHash } from "node:crypto";
20
20
  import { isAbsolute } from "node:path";
21
21
  import { realpathSync } from "node:fs";
22
- import { mintCheckpoint, heartbeatCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointAnswer, readCheckpointStatus, readCheckpointSpec, reconcileCheckpointSchedule, handleCheckpointFire, eptStreamName, eptSubject, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, isPresenceRow, principalKey, parseSubject, isFromSubjectSender, liveKvEntries, IncompleteKvScan, openMembersRegistry, openChannelRegistry, openAclRegistry, readAcl, channelInAllow, memberKey, writeChannelConfig, readMember, commitMemberRevision, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, invokeRepairingSplit, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, readGoalSpec, readGoalIndex, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, EpEnvelopeError, replyTargetUnmapped, unansweredRequest, assertAdmittedPublish, assertAdmittedSubscribe, isStreamNotFound, } from "@cotal-ai/core";
22
+ import { readCheckpointSettle, readCheckpointAnswer, reconcileCheckpointSchedule, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, parseSubject, isFromSubjectSender, IncompleteKvScan, openMembersRegistry, openChannelRegistry, openAclRegistry, readAcl, channelInAllow, memberKey, writeChannelConfig, readMember, commitMemberRevision, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, invokeRepairingSplit, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, EpEnvelopeError, replyTargetUnmapped, unansweredRequest, assertAdmittedPublish, assertAdmittedSubscribe, isStreamNotFound, } from "@cotal-ai/core";
23
23
  import { renderRunContext } from "./run-context.js";
24
24
  import { migrationSeats } from "./migrate.js";
25
25
  import { Kvm } from "@nats-io/kv";
@@ -28,6 +28,8 @@ import { PauseOverdue } from "./run-pause-host.js";
28
28
  import { loopLag, servedDespiteStarvation } from "./host-starvation.js";
29
29
  import { askAttemptToken, observeToken, pauseTokens, waitTimeoutToken, waitTimers } from "./pause-tokens.js";
30
30
  import { delay } from "./delay.js";
31
+ import { FIRE_POLL_MS, PauseTimers, decodeMessage } from "./mesh-pause-timers.js";
32
+ import { SeatWatch } from "./mesh-seat-watch.js";
31
33
  /**
32
34
  * A checkpoint resumed with no answer to read.
33
35
  *
@@ -1926,161 +1928,6 @@ export class MeshHandler {
1926
1928
  });
1927
1929
  await this.relayToSeat(seat, token, payload, ask.deadlineAt, "ask", step);
1928
1930
  }
1929
- /**
1930
- * An escalated checkpoint, delivered to the agent it names.
1931
- *
1932
- * `to` is legal only with `onExpiry: "escalate"` and the escalation is the SECOND mint, so this
1933
- * runs once per chain at most. The addressee is resolved through the run's own roster: a name
1934
- * this run spawned is a seat and is told, and anything else is a person — the pause is
1935
- * answerable from anywhere by design, its addressee is on the entry, and refusing a program
1936
- * that escalates to a human would be this host narrowing an option the reference leaves open.
1937
- *
1938
- * The relay is the ask's, because the two are the same act: one turn on the seat, carrying what
1939
- * is asked and the token to answer under, settling on the checkpoint plane rather than on the
1940
- * relay's own terminal.
1941
- */
1942
- async relayEscalation(req, ctx, token, deadlineAt) {
1943
- const to = req.to;
1944
- const step = stepKeyString(ctx.key);
1945
- // A seat is a roster entry whose acceptance floor was served: that is what carries the
1946
- // owner/actor coordinates a relay is addressed by. Anything else — a name this run never
1947
- // spawned, or one whose floor never landed — is not a seat, and the escalation stays the
1948
- // durable pause it already is, addressed on its entry and answerable from anywhere.
1949
- const seat = this.seatAddress(to);
1950
- if (seat.address === undefined)
1951
- return;
1952
- const context = renderRunContext({ run: this.binding.runId, step, notices: [] });
1953
- const payload = JSON.stringify({
1954
- run: this.binding.runId, step, context, noticeIds: [],
1955
- checkpoint: { token, prompt: req.prompt, schema: req.schema ?? null, deadlineAt, escalatedTo: to },
1956
- });
1957
- try {
1958
- await this.relayToSeat(seat.address, token, payload, deadlineAt, "checkpoint", step);
1959
- }
1960
- catch (e) {
1961
- // THE PAUSE OUTLIVES ITS ADDRESSEE. The reference defines no failure for an escalation whose
1962
- // addressee cannot be reached: it "mints exactly one further checkpoint addressed to `to`",
1963
- // and a checkpoint is "a durable pause a human or an agent resolves from anywhere". A seat
1964
- // the endpoint reports gone is therefore the same case as a `to` that names a person: nobody
1965
- // is told, the addressee is on the entry, and anyone may still answer. L4002 is the turn's
1966
- // vocabulary, and this is not a turn the program asked for. Said once, not swallowed.
1967
- if (!(e instanceof EffectError && e.code === "L4002"))
1968
- throw e;
1969
- console.error(`run ${this.binding.runId}: the escalation at ${step} could not be told to ${to} (${e.message}); the pause stands, answerable from anywhere`);
1970
- }
1971
- }
1972
- /**
1973
- * Submit one relay to a seat as a `turn` goal, and read its acceptance.
1974
- *
1975
- * IDEMPOTENT BY THE GOAL RECORD, which is also the arbiter when the invoke does not come back:
1976
- * a relay that landed and lost its reply is a relay that landed, and re-submitting it would put
1977
- * a second turn on the seat for one request. A seat the serve boundary reports gone is the
1978
- * agent-down failure (L4002); every other refusal is this endpoint's and is uncatchable.
1979
- *
1980
- * The relay carries the pause's own `deadlineAt` and the manager holds it to that instant: a
1981
- * duration counted from the manager's acceptance would outlive the pause by the submit's trip
1982
- * plus skew and show the seat an attempt whose answer is refused. A relay the manager refuses as
1983
- * already past, and that is past on this clock too, has nobody left to tell, because the pause
1984
- * denies at that same instant; the arm that follows meets the passed deadline and decides the
1985
- * outcome.
1986
- */
1987
- async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) {
1988
- const { name, uid } = seat;
1989
- const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
1990
- const actx = await this.actionCtx();
1991
- if ((await readGoalStatus(actx, ref)) !== undefined)
1992
- return;
1993
- const service = await this.relayManagerBeforeSubmit(seat.host, "turn", ref, kind, `${kind}(${step})`);
1994
- let reply;
1995
- try {
1996
- reply = await this.invokeManager(service, "turn", { payload, deadlineAt }, {
1997
- id: goalId,
1998
- deadlineMs: TURN_ACCEPT_DEADLINE_MS,
1999
- target: { mode: "owner", owner: seat.owner, actor: seat.actor, lifecycleUid: uid },
2000
- });
2001
- }
2002
- catch (err) {
2003
- // The invoke did not come back — the goal record is the arbiter, exactly as in `turn`.
2004
- if ((await readGoalStatus(actx, ref)) === undefined)
2005
- throw err;
2006
- return;
2007
- }
2008
- if (reply.reply.ok === false) {
2009
- const err = reply.reply.error;
2010
- if (err.code === "expired")
2011
- throw new EffectError("L4002", kind, `${kind}(${step}): ${name}#${uid} ${seatDownReading(err, seat.host)} before its relay began: ${err.message}`);
2012
- if (err.code === "deadline-exceeded" && this.now() >= deadlineAt)
2013
- return;
2014
- throw new Error(`${kind}(${step}) was refused by the ${this.binding.endpoint} endpoint: ${err.message}`);
2015
- }
2016
- }
2017
- /**
2018
- * Withdraw the relay a cancelled step sent to a seat, through the manager's reserved `cancel`
2019
- * (SPEC 13.6 item 4), so the seat is not shown it again. A token no relay rode has no goal and
2020
- * nothing to withdraw, and a relay that already ended is the outcome either way: the manager
2021
- * refuses it with the cached terminal, so a repeated withdrawal is a no-op. Any other refusal
2022
- * throws under the code it was refused with, which leaves a discharge open to retry.
2023
- *
2024
- * Only the manager that accepted the relay holds it, and the class rail reaches any member, so a
2025
- * refusal is final only from the accepter its index entry names. One from another member is
2026
- * re-issued on the class rail, as the spawn discharge's despawn is.
2027
- */
2028
- async withdrawRelay(goalId) {
2029
- const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
2030
- const actx = await this.actionCtx();
2031
- if ((await readGoalStatus(actx, ref)) === undefined)
2032
- return;
2033
- const accepter = (await readGoalIndex(actx, ref))?.iid;
2034
- for (let attempt = 0;; attempt += 1) {
2035
- const reply = await this.invokeManager(await this.manager("cancel"), "cancel", { goalId }, { deadlineMs: TURN_ACCEPT_DEADLINE_MS });
2036
- if (reply.reply.ok !== false || (await readGoalResult(actx, ref)) !== undefined)
2037
- return;
2038
- if (accepter === undefined || reply.responder.instanceId === accepter) {
2039
- const err = reply.reply.error;
2040
- throw new EpEnvelopeError(err.code, `the relay "${goalId}" could not be withdrawn from its seat: ${err.message}`);
2041
- }
2042
- if (attempt === MANAGER_ROUTE_ATTEMPTS)
2043
- throw new Error(`the relay "${goalId}" was accepted by manager instance ${accepter}, but none of ${attempt + 1} cancels was answered by it; the discharge stays open to retry`);
2044
- this.managerServices.delete("cancel");
2045
- }
2046
- }
2047
- /**
2048
- * Withdraw a cancelled step's relay before the cancellation leaves the step, so the scope it
2049
- * settles shows the seat nothing the run withdrew. A withdrawal that fails is tried again rather
2050
- * than left to the discharge, which runs only when the run ends: until then the relay would stay
2051
- * first on the seat and hold back the run's next turn to it. The retries stop when the relay's
2052
- * deadline passes, because the manager no longer serves it then, and a failure that outlasts it
2053
- * is the step's error. A refusal no retry can change is the step's error at once, since retrying
2054
- * it would only hold the cancelled step in its scope until that deadline: of the refusals, only
2055
- * `unavailable` (a manager still reconciling at boot) and `deadline-exceeded` (a lost reply) are
2056
- * tried again.
2057
- *
2058
- * That deadline is read from the relay's goal, where the accepting manager recorded the instant
2059
- * it stops serving the relay; a withdrawal holds only the goal id. No deadline is known until that
2060
- * goal is read, and the relay may be served until then, so a failed read is tried again as well.
2061
- * A goal record that can never yield a deadline is thrown.
2062
- */
2063
- async withdrawCancelledRelay(goalId, deadlineAt) {
2064
- for (;;) {
2065
- try {
2066
- if (deadlineAt === undefined) {
2067
- const spec = (await readGoalSpec(await this.actionCtx(), { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId }))?.value;
2068
- if (spec === undefined)
2069
- return;
2070
- if (spec.readinessDeadlineMs === undefined)
2071
- throw new EpEnvelopeError("internal", `the relay "${goalId}" was accepted with no deadline on its goal; a garbled acceptance never authorizes (SPEC 13.6)`);
2072
- deadlineAt = spec.acceptedAt + spec.readinessDeadlineMs;
2073
- }
2074
- return await this.withdrawRelay(goalId);
2075
- }
2076
- catch (e) {
2077
- const final = e instanceof EpEnvelopeError && e.code !== "unavailable" && e.code !== "deadline-exceeded";
2078
- if (deadlineAt === undefined ? e instanceof EpEnvelopeError : final || this.now() >= deadlineAt)
2079
- throw e;
2080
- }
2081
- await delay(WAIT_POLL_MS);
2082
- }
2083
- }
2084
1931
  /**
2085
1932
  * `monitor` registers interest: after it, `down(agent)` is an event a branch can await (§5.9).
2086
1933
  *
@@ -2304,465 +2151,49 @@ export class MeshHandler {
2304
2151
  await (await this.channelRegistry()).delete(plan.channel);
2305
2152
  }
2306
2153
  }
2307
- /** Every live presence row {@link isPresenceRow} accepts under its KV key. Foreign bytes in
2308
- * the bucket are skipped — a peer's malformed self-publish must not break another agent's name
2309
- * resolution — and what an ABSENT row means belongs to the caller: `resolveMemberPrincipal`
2310
- * refuses the join loudly, and `waitDown` reads it as the death it is waiting for. */
2311
- async presenceRows() {
2312
- const rows = [];
2313
- for (const e of await liveKvEntries(await this.presenceRegistry())) {
2314
- let row;
2315
- try {
2316
- row = e.json();
2317
- }
2318
- catch {
2319
- continue;
2320
- }
2321
- if (isPresenceRow(row, e.key))
2322
- rows.push(row);
2323
- }
2324
- return rows;
2325
- }
2326
- /** One poll of the presence death watch `wait(down)` and `turn` share. `undefined` means this
2327
- * incarnation is still live, or a lapse has not yet stood for {@link LAPSE_CONFIRM_MS}. A missing
2328
- * incarnation is `"superseded"` when a live row holds the name and `"lapsed"` when nothing does.
2329
- * `watch` is the caller's own window, written here and read on the next poll. An incomplete scan
2330
- * decides nothing and returns `undefined`; every other read failure is raised. */
2331
- async presenceDeath(name, uid, watch, decided) {
2332
- let rows;
2333
- const readAt = this.now();
2334
- try {
2335
- rows = await this.presenceRows();
2336
- }
2337
- catch (e) {
2338
- // `liveKvEntries` REFUSES a pass cut short mid-scan instead of returning a partial view —
2339
- // and a partial view is exactly what must not decide a death. Its own contract says retry,
2340
- // so the next poll is the retry; a link that stays down keeps surfacing here rather than
2341
- // as a false DOWN, and the caller's deadline still ends the wait.
2342
- if (e instanceof IncompleteKvScan)
2343
- return undefined;
2344
- throw e;
2345
- }
2346
- if (!rows.some((p) => p.card.name === name && p.lifecycleUid === uid)) {
2347
- const reason = rows.some((p) => p.card.name === name) ? "superseded" : "lapsed";
2348
- watch.lapsedSince = lapseWindow(watch.lapsedSince, watch.lastReadAt ?? 0, this.now());
2349
- if (reason === "superseded" || readAt - watch.lapsedSince >= LAPSE_CONFIRM_MS) {
2350
- if (reason === "lapsed" && !(await this.lapseStands(name, uid, decided)))
2351
- watch.lapsedSince = undefined;
2352
- else {
2353
- watch.lastReadAt = readAt;
2354
- return reason;
2355
- }
2356
- }
2357
- }
2358
- else
2359
- watch.lapsedSince = undefined;
2360
- watch.lastReadAt = readAt;
2361
- return undefined;
2362
- }
2363
- /** Whether a lapse of `name#uid` confirmed from presence stands as the death, asked of the manager
2364
- * first. A lapse is only the seat's heartbeats going quiet: on a loaded host a live seat's presence
2365
- * writer stalls past {@link LAPSE_CONFIRM_MS} while its process goes on working (#2807), and the
2366
- * manager owns that process. Only the manager holding the seat has a running row under its
2367
- * incarnation, so that row keeps it alive whoever reports it. A miss decides only from the manager
2368
- * that allocated it, the committer of its spawn terminal: another member of the class rail knows
2369
- * only its own seats, so the class rail is asked again, as a spawn's discharge asks it. An allocator
2370
- * that does not answer leaves presence as the only witness, which is the case presence is the
2371
- * authority for: the manager died with the seat. The asking can outlast the caller's own
2372
- * cancellation, deadline or terminal, so once `decided` holds nothing stands and no refusal is
2373
- * raised: that authority decides at the caller's next check. */
2374
- async lapseStands(name, uid, decided) {
2375
- const entry = this.roster.get(name);
2376
- const spawned = entry?.uid === uid && entry.goalId !== undefined
2377
- ? await readGoalResult(await this.actionCtx(), { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId: entry.goalId })
2378
- : undefined;
2379
- const allocator = spawned?.committer?.instanceId;
2380
- for (let attempt = 0; attempt <= MANAGER_ROUTE_ATTEMPTS; attempt += 1) {
2381
- if (await decided())
2382
- return false;
2383
- let answer;
2384
- try {
2385
- answer = await this.invokeManager(await this.manager("inspect"), "inspect", { name }, {});
2386
- }
2387
- catch (e) {
2388
- if (unansweredRequest(e))
2389
- break;
2390
- if (await decided())
2391
- return false;
2392
- throw e;
2393
- }
2394
- const { reply, responder } = answer;
2395
- const row = reply.ok ? reply.data : undefined;
2396
- if (row?.lifecycleUid === uid && row.status === "running")
2397
- return false;
2398
- if (responder.instanceId === allocator && !replyRefusedBeforeEffect(reply.error)) {
2399
- if (reply.ok || reply.error?.code === "not-found")
2400
- break;
2401
- if (await decided())
2402
- return false;
2403
- throw new Error(`inspect(${name}) was refused by manager instance ${allocator}, which allocated ${name}#${uid}: ${reply.error?.message ?? "refused with no message"}`);
2404
- }
2405
- this.managerServices.delete("inspect");
2406
- }
2407
- return !(await decided());
2408
- }
2409
- /**
2410
- * Arm this pause, or ATTACH to the one already recorded under the same token.
2411
- *
2412
- * A mint is idempotent only if the whole spec is identical, so the recorded spec is the
2413
- * authority for BOTH halves of the mint's identity and a resume may not recompute either. The
2414
- * deadline, because `now() + duration` is a different deadline a second later, and the plane
2415
- * reads a different deadline as a different intent. And the HOLDER, because a resumed run does
2416
- * not necessarily arrive under the principal that minted its pause: the CLI mints a fresh holder
2417
- * per invocation, and a cross-host adoption is a different principal by definition. Only the
2418
- * recorded holder itself may mint again — completing its own crash between the spec and the
2419
- * status, where the identical spec is exactly what makes the retry idempotent. Everyone else
2420
- * attaches. Measured before the repair, through the CLI: one `resume` of a checkpoint-parked
2421
- * run re-minted under its own fresh holder, the plane refused ("a token is minted once"), and
2422
- * the interpreter recorded that infrastructure refusal as the step's own failure (L4000) — a
2423
- * stranded run whose journal blames the program.
2424
- *
2425
- * An already-passed deadline cannot be minted at all, correctly, because a due pause is not being
2426
- * armed. It needs its schedule re-emitted at the status's current generation, which is the
2427
- * reconciler's job — the same operation an attaching successor needs, so the two share the exit.
2428
- * A first arm whose deadline has passed has nothing to attach to and raises `PauseOverdue`.
2429
- * A spec with no status that this holder cannot complete is unrepairable from here, so it is
2430
- * raised rather than waited on.
2431
- */
2432
- async arm(ref, deadline) {
2433
- // #1508: every read and write below rides a client-side deadline, and a deadline that elapsed
2434
- // because THIS PROCESS was off the CPU is not evidence about the plane. See `host-starvation`.
2435
- return await servedDespiteStarvation(() => this.armOnce(ref, deadline), this.lag, `arming the pause ${ref.token}`, this.onStarved);
2436
- }
2437
- /** `arm`, answering `true` when the deadline passed before anything was minted for it. Only a
2438
- * first arm finds that (`PauseOverdue`), and there is then no pause to settle, so a caller whose
2439
- * deadline is its own outcome takes `true` as that outcome. Cancellation still wins. */
2440
- async armUnlessDue(ref, deadline, signal) {
2441
- try {
2442
- await this.arm(ref, deadline);
2443
- return false;
2444
- }
2445
- catch (error) {
2446
- if (error instanceof PauseOverdue) {
2447
- if (signal.cancelled)
2448
- throw new Cancelled(signal.reason ?? "cancelled");
2449
- return true;
2450
- }
2451
- throw error;
2452
- }
2453
- }
2454
- async armOnce(ref, deadline) {
2455
- if (this.services)
2456
- return this.services.pauses.arm(ref.token, deadline);
2457
- // Over already: an expiry or an answer landed while this host was away. Nothing to arm, and the
2458
- // caller reads the fact next.
2459
- if ((await readCheckpointSettle(this.jsm, this.binding.space, ref)) !== undefined)
2460
- return;
2461
- const prior = await readCheckpointSpec(this.kv, ref);
2462
- const now = this.now();
2463
- if (prior === undefined) {
2464
- // The first arm: this driver's own mint, under its own holder, at the deadline the caller
2465
- // computed from the duration it was just handed.
2466
- if (deadline <= now)
2467
- throw new PauseOverdue(ref.token, deadline, now);
2468
- await mintCheckpoint(this.kv, this.js, this.binding.space, {
2469
- ref,
2470
- instanceId: this.binding.instanceId,
2471
- epoch: this.binding.epoch,
2472
- holder: this.binding.holder,
2473
- deadline,
2474
- now,
2475
- });
2476
- return;
2477
- }
2478
- const mine = prior.holder.id === this.binding.holder.id
2479
- && prior.holder.lifecycleUid === this.binding.holder.lifecycleUid;
2480
- if (mine && prior.initialDeadline > now) {
2481
- // A retry of this holder's own mint: idempotent-if-identical, and the one path that can
2482
- // repair a crash between the spec and the status, because only an identical spec completes.
2483
- await mintCheckpoint(this.kv, this.js, this.binding.space, {
2484
- ref,
2485
- instanceId: this.binding.instanceId,
2486
- epoch: this.binding.epoch,
2487
- holder: this.binding.holder,
2488
- deadline: prior.initialDeadline,
2489
- now,
2490
- });
2491
- return;
2492
- }
2493
- // ATTACH: the pause exists and is not this attempt's to mint — another holder's pause, or this
2494
- // holder's own come back overdue (a due pause is being collected, not armed). Both need the
2495
- // same thing: the schedule re-emitted at THIS instance's coordinates, so the fire lands where
2496
- // this driver is listening. The settle the caller waits on next is coordinate-free.
2497
- const status = await readCheckpointStatus(this.kv, ref);
2498
- if (status === undefined) {
2499
- throw new Error(`checkpoint "${ref.token}" carries a spec with no status`
2500
- + `${mine ? "" : ` held by ${prior.holder.id}, and only its own holder's identical re-mint can repair a half-minted pause`}; `
2501
- + `this pause has to be reconciled on the plane before the run can go on`);
2502
- }
2503
- await reconcileCheckpointSchedule(this.kv, this.js, this.jsm, this.binding.space, {
2504
- ref,
2505
- instanceId: this.binding.instanceId,
2506
- epoch: this.binding.epoch,
2507
- });
2508
- }
2509
- /** Push a live deadline out — the idle window restarting. The heartbeat CAS-advances the
2510
- * generation before replacing the timer, so a fire from the old one no-ops rather than racing. */
2511
- async push(ref, deadline) {
2512
- if (this.services)
2513
- return this.services.pauses.heartbeat(ref.token, deadline);
2514
- await heartbeatCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
2515
- ref,
2516
- instanceId: this.binding.instanceId,
2517
- epoch: this.binding.epoch,
2518
- deadline,
2519
- now: this.now(),
2520
- });
2521
- }
2522
- /**
2523
- * Take the broker's `.fire` for this deadline, if it has published one, and let the checkpoint
2524
- * plane judge it. Answers whether the fire SETTLED the pause.
2525
- *
2526
- * A fire is a MESSAGE and a settlement is a FACT; `handleCheckpointFire` is what turns one into
2527
- * the other, and a pause nobody answers ends no other way.
2528
- *
2529
- * READ BY SUBJECT rather than by a subscription, because the fire is a record of a deadline
2530
- * passing and callers here are already polling: `last_by_subj` gives the current one whether it
2531
- * arrived a moment ago or while this host was down, and a resume lands in the second case.
2532
- *
2533
- * IDEMPOTENT BY THE PLANE rather than by bookkeeping here: a fire handed over twice finds the
2534
- * pause settled and declines. The one repeated verdict with a side effect is `re-armed` under
2535
- * owner-behind clock skew, and over-emission is idempotent at the timer writer.
2536
- */
2537
- async takeFire(ref) {
2538
- if (this.services)
2539
- return this.services.pauses.takeFire(ref.token);
2540
- const subject = eptSubject(this.binding.space, ref.endpoint, this.binding.instanceId, this.binding.epoch, ref.token, "fire");
2541
- const fired = await this.jsm.streams
2542
- .getMessage(eptStreamName(this.binding.space), { last_by_subj: subject })
2543
- .catch((e) => {
2544
- // 10037 is the stream saying there is no message on that subject, which IS the ordinary
2545
- // "the timer has not fired yet" answer. Every other error is the broker refusing or
2546
- // unreachable, and reading those as "not yet" is how a `wait` with a timeout outlives its
2547
- // own durable deadline in silence, which is the failure this method exists to prevent.
2548
- if (e?.code === 10037)
2549
- return null;
2550
- throw e;
2551
- });
2552
- if (fired === null || fired === undefined)
2553
- return false;
2554
- const verdict = await handleCheckpointFire(this.kv, this.js, this.jsm, this.binding.space, {
2555
- ref,
2556
- instanceId: this.binding.instanceId,
2557
- epoch: this.binding.epoch,
2558
- msg: { subject, ...(fired.header !== undefined ? { headers: fired.header } : {}), data: fired.data },
2559
- now: this.now(),
2560
- });
2561
- return verdict.acted;
2562
- }
2563
- /** Has this deadline settled? `undefined` for "no deadline" and for "not yet".
2564
- *
2565
- * A `wait` observes its deadlines by polling this, so the fire is taken on the same poll: the
2566
- * settle fact a `wait` is reading for is one this process has to produce, and reading for it
2567
- * without ever producing it is how a `wait` with a timeout waited past its timeout forever. */
2568
- async expired(ref) {
2569
- if (ref === undefined)
2570
- return undefined;
2571
- if (this.services) {
2572
- const settled = await this.services.pauses.readSettle(ref.token);
2573
- if (settled !== undefined)
2574
- return settled;
2575
- if (!(await this.services.pauses.takeFire(ref.token)))
2576
- return undefined;
2577
- return this.services.pauses.readSettle(ref.token);
2578
- }
2579
- const settled = await readCheckpointSettle(this.jsm, this.binding.space, ref);
2580
- if (settled !== undefined)
2581
- return settled;
2582
- if (!(await this.takeFire(ref)))
2583
- return undefined;
2584
- return await readCheckpointSettle(this.jsm, this.binding.space, ref);
2585
- }
2586
- /** End a deadline that is no longer waited on, by claiming its one-use settlement with no answer.
2587
- * A timer left armed would fire into a run that has moved on; claiming it is how the plane says
2588
- * "this pause is over" without a second mechanism for cancellation. */
2589
- async cancelTimer(ref) {
2590
- if (this.services)
2591
- return this.services.pauses.claim(ref.token);
2592
- const st = await readCheckpointStatus(this.kv, ref);
2593
- if (st?.value.state !== "waiting")
2594
- return;
2595
- try {
2596
- // The presenter is the ARMING holder read off the pause's own record, as everywhere on this
2597
- // plane: a timer minted by a predecessor is still this run's to end after a takeover, and a
2598
- // successor presenting its own identity would be refused (resume is holder-bound, SPEC
2599
- // 13.10). A spec that cannot be read leaves the timer to its own deadline, which the catch
2600
- // below already tolerates: it fires, settles expired, and nobody is reading that token.
2601
- const spec = await readCheckpointSpec(this.kv, ref);
2602
- if (spec === undefined)
2603
- return;
2604
- await resumeCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
2605
- ref,
2606
- presenter: spec.holder,
2607
- now: this.now(),
2608
- });
2609
- }
2610
- catch (e) {
2611
- // IT SETTLED UNDERNEATH US is one outcome, and it is the expected one: the deadline won a
2612
- // race it was always allowed to win, the caller has its answer either way, and a claimed
2613
- // timer is not a fact anyone reads. A BROKER THAT REFUSED OR WENT AWAY is a different thing
2614
- // wearing the same silence, and this catch used to swallow both. It still must not replace
2615
- // the caller's answer — a timer left armed fires, settles expired, and nobody reads that
2616
- // token — so the second one is loud on the way past instead.
2617
- const settledAlready = e instanceof EpEnvelopeError
2618
- && (e.code === "conflict" || e.code === "failed-precondition");
2619
- if (!settledAlready)
2620
- console.error(`run ${this.binding.runId}: the pause ${ref.token} could not be claimed (${e.message}); it will fire and settle expired with nobody reading it`);
2621
- }
2622
- }
2623
- /** The message at a recorded stream sequence — the re-bind path after a crash. */
2624
- async messageAt(seq, requestId) {
2625
- if (this.services) {
2626
- const message = decodeMessage(await this.services.waits.messageAt(requestId, seq));
2627
- if (message === undefined)
2628
- throw new Error(`wait ${requestId}'s recorded message does not decode`);
2629
- return message;
2630
- }
2631
- const m = await this.jsm.streams.getMessage(chatStream(this.binding.space), { seq });
2632
- if (m === null || m === undefined) {
2633
- throw new Error(`the message this wait matched (sequence ${seq}) is no longer on the channel's stream; a recorded match cannot be re-read`);
2634
- }
2635
- const msg = decodeMessage(m.data);
2636
- if (msg === undefined)
2637
- throw new Error(`the message at sequence ${seq} did not decode; a recorded match cannot be re-read`);
2638
- return msg;
2639
- }
2640
- /**
2641
- * Wait for this token's one-use settlement.
2642
- *
2643
- * A settle that ALREADY landed is the ordinary case on a resume: the run crashed while paused and
2644
- * the timer fired, or somebody answered, without it. Reading before waiting is not an
2645
- * optimization, it is the difference between resuming and waiting forever for an event that is
2646
- * already in the past.
2647
- */
2648
- async settle(ref, signal, relayed = false) {
2649
- // #1508, and the load-bearing half: this is where a `sleep` spends its whole duration, so this
2650
- // is where a starved host was reporting its own scheduling as the effect's failure. The pause
2651
- // and its timer are durable facts on the plane; re-reading them observes the same world, and a
2652
- // sleep whose deadline passed while this process was blocked settles `ok`, late.
2653
- try {
2654
- return await servedDespiteStarvation(() => this.settleOnce(ref, signal), this.lag, `waiting on the pause ${ref.token}`, this.onStarved);
2655
- }
2656
- catch (e) {
2657
- // An `ask` attempt or an escalation may have relayed this token to a seat.
2658
- if (e instanceof Cancelled && relayed)
2659
- await this.withdrawCancelledRelay(ref.token);
2660
- throw e;
2661
- }
2662
- }
2663
- async settleOnce(ref, signal) {
2664
- const already = this.services ? await this.services.pauses.readSettle(ref.token) : await readCheckpointSettle(this.jsm, this.binding.space, ref);
2665
- if (already !== undefined)
2666
- return already;
2667
- // The watcher waits for the FACT. For a pause with an answer the fact arrives because somebody
2668
- // answered; for a pause nobody answers - a `sleep`, or a `checkpoint` whose timeout wins - it
2669
- // arrives only because this process took the fire and the plane wrote it. So the two run
2670
- // together for the life of one wait, and the pump ends when the wait does.
2671
- const wait = { over: false };
2672
- const pump = this.pumpFires(ref, wait);
2673
- try {
2674
- return await Promise.race([
2675
- this.watcher.awaitSettle(ref),
2676
- // A pump that ENDED is not an answer, only one that FAILED is: a failure means this process
2677
- // cannot expire the pause, so it is raised rather than absorbed and the step stays pending.
2678
- pump.then(() => new Promise(() => { })),
2679
- // A cancelled branch stops parking NOW (§7.6): the rejection decides the race first, so a
2680
- // `checkpoint` cannot mistake its own claim below for an answer, and then the claim ends
2681
- // the pause in the world. The claim is not awaited here — a claim that loses its own race
2682
- // is tolerated by `cancelTimer`, and the discharge sweep at the run's completion re-claims
2683
- // idempotently for the case where this process died before the claim landed.
2684
- ...(signal === undefined ? [] : [this.settleCancelled(ref, signal)]),
2685
- ]);
2686
- }
2687
- finally {
2688
- wait.over = true;
2689
- // THE PUMP IS DRAINED BEFORE THE SETTLE RETURNS (#1460). The race above has already decided
2690
- // — the fact, the failure, or the cancellation is this call's answer — but the pump's
2691
- // in-flight `takeFire` is a journal replay under the run's takeover id, created, read and
2692
- // deleted inside one `replayRunJournal` call with nothing serialising it across two
2693
- // authorities for the same run and lease. A settle that returned past it let a completed
2694
- // `driveRun` hold the replay open after it resolved, and the next reader under the same
2695
- // takeover hit `RunJournalReplayRaced` about a driver that did not exist. So only the RETURN
2696
- // is delayed, until the pump has ended: `wait.over` stops it after its current `takeFire`
2697
- // without starting another, and awaiting it here means no pump fire is left in flight on any
2698
- // exit. A pump that FAILED raises through this await on its way out, exactly as it did
2699
- // through the race; the settle's own answer is decided either way and is what this returns.
2700
- await pump;
2701
- }
2702
- }
2703
- /** Reject with `Cancelled` the moment this branch's signal fires, then claim the pause so its
2704
- * armed schedule cannot fire into a run that has moved on. The claim also writes the one-use
2705
- * settle, which is what lets an abandoned `awaitSettle` poll loop see a fact and end. */
2706
- settleCancelled(ref, signal) {
2707
- return new Promise((_, reject) => {
2708
- let fired = false;
2709
- const fire = (reason) => {
2710
- if (fired)
2711
- return;
2712
- fired = true;
2713
- reject(new Cancelled(reason ?? "cancelled"));
2714
- void this.cancelTimer(ref).catch(() => undefined);
2715
- };
2716
- if (signal.cancelled) {
2717
- fire(signal.reason);
2718
- return;
2719
- }
2720
- signal.onCancel(fire);
2721
- });
2722
- }
2723
- /** Take this deadline's fire for as long as somebody is waiting on it. */
2724
- async pumpFires(ref, wait) {
2725
- while (!wait.over) {
2726
- await this.takeFire(ref);
2727
- // THE FLAG IS RE-READ AFTER THE FIRE, before the poll sleeps (#1460): `takeFire` is a journal
2728
- // replay under the run's takeover id, so the wait can have ENDED while this one was in
2729
- // flight — and a pump that slept another poll on a wait that is over keeps exactly one more
2730
- // fire in flight past the drain `settleOnce` awaits. Checked here, the pump ends right after
2731
- // its current fire without starting another.
2732
- if (wait.over)
2733
- break;
2734
- await delay(FIRE_POLL_MS);
2735
- }
2736
- }
2154
+ /** The pause timers own no handler state: each input is read through the handler at its use. */
2155
+ pauseTimers = new PauseTimers({
2156
+ binding: () => this.binding, kv: () => this.kv, js: () => this.js, jsm: () => this.jsm, watcher: () => this.watcher,
2157
+ services: () => this.services, lag: () => this.lag, onStarved: (note) => this.onStarved(note), now: () => this.now(),
2158
+ withdrawCancelledRelay: (goalId, deadlineAt) => this.withdrawCancelledRelay(goalId, deadlineAt),
2159
+ });
2160
+ async arm(ref, deadline) { return this.pauseTimers.arm(ref, deadline); }
2161
+ async armUnlessDue(ref, deadline, signal) { return this.pauseTimers.armUnlessDue(ref, deadline, signal); }
2162
+ async push(ref, deadline) { return this.pauseTimers.push(ref, deadline); }
2163
+ async expired(ref) { return this.pauseTimers.expired(ref); }
2164
+ async cancelTimer(ref) { return this.pauseTimers.cancelTimer(ref); }
2165
+ async messageAt(seq, requestId) { return this.pauseTimers.messageAt(seq, requestId); }
2166
+ async settle(ref, signal, relayed = false) { return this.pauseTimers.settle(ref, signal, relayed); }
2167
+ /** The relay and presence watch own no handler state: each input, the handler's own members
2168
+ * included, is read through the handler at its use. */
2169
+ seatWatch = new SeatWatch({
2170
+ self: () => this,
2171
+ binding: () => this.binding,
2172
+ roster: () => this.roster,
2173
+ managerServices: () => this.managerServices,
2174
+ now: () => this.now(),
2175
+ actionCtx: () => this.actionCtx(),
2176
+ presenceRegistry: () => this.presenceRegistry(),
2177
+ seatAddress: (name, uid) => this.seatAddress(name, uid),
2178
+ manager: (command, instanceId) => this.manager(command, instanceId),
2179
+ invokeManager: (service, command, args, opts) => this.invokeManager(service, command, args, opts),
2180
+ relayManagerBeforeSubmit: (host, command, ref, kind, label) => this.relayManagerBeforeSubmit(host, command, ref, kind, label),
2181
+ });
2182
+ async relayEscalation(req, ctx, token, deadlineAt) { return this.seatWatch.relayEscalation(req, ctx, token, deadlineAt); }
2183
+ async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) { return this.seatWatch.relayToSeat(seat, goalId, payload, deadlineAt, kind, step); }
2184
+ async withdrawRelay(goalId) { return this.seatWatch.withdrawRelay(goalId); }
2185
+ async withdrawCancelledRelay(goalId, deadlineAt) { return this.seatWatch.withdrawCancelledRelay(goalId, deadlineAt); }
2186
+ async presenceRows() { return this.seatWatch.presenceRows(); }
2187
+ async presenceDeath(name, uid, watch, decided) { return this.seatWatch.presenceDeath(name, uid, watch, decided); }
2737
2188
  }
2738
2189
  export { waitConsumerName, waitConsumerConfig } from "@cotal-ai/core";
2739
2190
  /** How long one poll of a wait's consumer blocks. The deadline itself is durable; this is only how
2740
2191
  * late its observation can be, and a shorter poll buys latency at the cost of fetch traffic. */
2741
- const WAIT_POLL_MS = 2_000;
2742
- /**
2743
- * How long an incarnation's presence row must stay absent before `turn` and `wait(down)` read it as
2744
- * `lapsed`. A row is gone 6s after its last heartbeat, and a live seat whose connector stalls for
2745
- * longer (host load, a reconnect) comes back under the same lifecycle uid on its next heartbeat. One
2746
- * absent read failed a turn L4002 while the seat went on working (#2344). Five liveness windows lets
2747
- * a stalled seat renew, and stays short against a turn's deadline. A `superseded` read needs no
2748
- * wait: a different incarnation already holds the name. The window is measured from the end of the
2749
- * read that opened it to the start of the read that closes it, over reads {@link lapseWindow} chains.
2750
- */
2751
- const LAPSE_CONFIRM_MS = 30_000;
2752
- /**
2753
- * The longest span from the start of one presence read to the end of the next across which an
2754
- * absent row still counts as continuously absent: the presence bucket's 6s TTL. A heartbeat keeps
2755
- * the row live at least that long, so a renewal cannot fall between two reads that close. A slower
2756
- * read or a run of failed scans leaves the gap unobserved, and the lapse window starts over.
2757
- */
2758
- const PRESENCE_GAP_MS = 6_000;
2759
- /** The start of the lapse window after an absent read that ended at `now`: kept when the previous
2760
- * read began within {@link PRESENCE_GAP_MS}, otherwise this read opens a new one. */
2761
- const lapseWindow = (lapsedSince, lastReadAt, now) => lapsedSince === undefined || now - lastReadAt > PRESENCE_GAP_MS ? now : lapsedSince;
2192
+ export const WAIT_POLL_MS = 2_000;
2762
2193
  /** What an `expired` turn refusal shows. The placed host speaks for its own seat. A class-rail
2763
2194
  * target-unmapped refusal says only that no member the call reached maps the seat: in a space with
2764
2195
  * more than one manager a live seat on a member it missed answers the same as a dead one. */
2765
- function seatDownReading(err, host) {
2196
+ export function seatDownReading(err, host) {
2766
2197
  if (host !== undefined)
2767
2198
  return `found the agent down on its placed host ${host}`;
2768
2199
  return replyTargetUnmapped(err)
@@ -2857,14 +2288,14 @@ const SPAWN_ACCEPT_DEADLINE_MS = 30_000;
2857
2288
  /** Total unanswered attempts per seat. Separate from class routing so an outage is bounded. */
2858
2289
  const SPAWN_DISCHARGE_UNANSWERED_ATTEMPTS = 3;
2859
2290
  /** Bound on the manager's synchronous `turn` ACCEPT reply (the relay registration, not the yield). */
2860
- const TURN_ACCEPT_DEADLINE_MS = 30_000;
2291
+ export const TURN_ACCEPT_DEADLINE_MS = 30_000;
2861
2292
  /** How many class-rail calls a despawn, a relay withdrawal or a lapse's inspect sends before it stops
2862
2293
  * waiting for the one manager that holds its goal (the spawn's allocator, the relay's accepter) to
2863
2294
  * answer. Each is one {@link MeshHandler.invokeManager} call, whose describe and invoke both land on
2864
2295
  * that manager with probability 1/m^2 per trip in a space of m managers, so one call reaches it with
2865
2296
  * probability (1 - ((m-1)/m)^17) / m: about 1/2 for m = 2 and 0.25 for m = 4. All 65 then miss
2866
2297
  * with probability about 2^-65 and 9e-9. */
2867
- const MANAGER_ROUTE_ATTEMPTS = 64;
2298
+ export const MANAGER_ROUTE_ATTEMPTS = 64;
2868
2299
  /** A step key's enclosing scope: the journal's own rendering (`entry.scope`), re-derived so the
2869
2300
  * live path and the adoption rebuild key the handoff memos identically. */
2870
2301
  function scopeOf(key) {
@@ -3065,9 +2496,6 @@ function assertConclaveChannel(channel) {
3065
2496
  throw new Error(`conclave channel "${channel}" is a wildcard; a conclave joins its members to one concrete channel`);
3066
2497
  return channel;
3067
2498
  }
3068
- /** How often a pause that nobody will answer looks for the broker's fire. Same argument as
3069
- * `WAIT_POLL_MS`: the deadline is durable and this is only how late its observation can be. */
3070
- const FIRE_POLL_MS = 2_000;
3071
2499
  /** The recorded ask progress a resume re-enters at, or undefined for a fresh first attempt. The
3072
2500
  * external is bytes from an earlier process, so the shape is checked rather than trusted. */
3073
2501
  function askResume(v) {
@@ -3128,17 +2556,6 @@ function isWaitEvent(ev) {
3128
2556
  return typeof e.channel === "string";
3129
2557
  return (e.event === "replied" || e.event === "down") && typeof e.agent === "string";
3130
2558
  }
3131
- function decodeMessage(data) {
3132
- try {
3133
- const v = JSON.parse(new TextDecoder().decode(data));
3134
- return v !== null && typeof v === "object" ? v : undefined;
3135
- }
3136
- catch {
3137
- // Someone else's malformed publish is not this run's failure. It does not match, and the wait
3138
- // goes on waiting — which is what would happen if the message had never been sent.
3139
- return undefined;
3140
- }
3141
- }
3142
2559
  /** Does this message answer the await? It must come from the principal its subject names (SPEC
3143
2560
  * §5), and when the wait names a `sender` principal, the subject's sender must be that principal.
3144
2561
  * The payload's `from.name` is whatever the sender wrote, so it is never compared. */