@cotal-ai/runtime 0.169.0 → 0.171.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/mesh-handler.d.ts +53 -239
- package/dist/mesh-handler.d.ts.map +1 -1
- package/dist/mesh-handler.js +80 -1093
- package/dist/mesh-handler.js.map +1 -1
- package/dist/mesh-pause-timers.d.ts +119 -0
- package/dist/mesh-pause-timers.d.ts.map +1 -0
- package/dist/mesh-pause-timers.js +369 -0
- package/dist/mesh-pause-timers.js.map +1 -0
- package/dist/mesh-seat-watch.d.ts +142 -0
- package/dist/mesh-seat-watch.d.ts.map +1 -0
- package/dist/mesh-seat-watch.js +309 -0
- package/dist/mesh-seat-watch.js.map +1 -0
- package/dist/mesh-turn-ask.d.ts +181 -0
- package/dist/mesh-turn-ask.d.ts.map +1 -0
- package/dist/mesh-turn-ask.js +514 -0
- package/dist/mesh-turn-ask.js.map +1 -0
- package/dist/offline-command.d.ts.map +1 -1
- package/dist/offline-command.js +42 -24
- package/dist/offline-command.js.map +1 -1
- package/dist/run-command.d.ts.map +1 -1
- package/dist/run-command.js +13 -6
- package/dist/run-command.js.map +1 -1
- package/dist/run-driver.d.ts +1 -0
- package/dist/run-driver.d.ts.map +1 -1
- package/dist/run-driver.js +20 -15
- package/dist/run-driver.js.map +1 -1
- package/dist/run-host.d.ts.map +1 -1
- package/dist/run-host.js +2 -1
- package/dist/run-host.js.map +1 -1
- package/package.json +4 -4
package/dist/mesh-handler.js
CHANGED
|
@@ -19,15 +19,17 @@
|
|
|
19
19
|
import { createHash } from "node:crypto";
|
|
20
20
|
import { isAbsolute } from "node:path";
|
|
21
21
|
import { realpathSync } from "node:fs";
|
|
22
|
-
import {
|
|
23
|
-
import { renderRunContext } from "./run-context.js";
|
|
22
|
+
import { readCheckpointSettle, readCheckpointAnswer, reconcileCheckpointSchedule, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, parseSubject, isFromSubjectSender, IncompleteKvScan, openMembersRegistry, openChannelRegistry, openAclRegistry, readAcl, channelInAllow, memberKey, writeChannelConfig, readMember, commitMemberRevision, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, invokeRepairingSplit, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, resolveService, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, EpEnvelopeError, replyTargetUnmapped, unansweredRequest, assertAdmittedPublish, assertAdmittedSubscribe, isStreamNotFound, } from "@cotal-ai/core";
|
|
24
23
|
import { migrationSeats } from "./migrate.js";
|
|
25
24
|
import { Kvm } from "@nats-io/kv";
|
|
26
|
-
import { parseDuration, Cancelled, EffectError, Journal,
|
|
25
|
+
import { parseDuration, Cancelled, EffectError, Journal, digest, journalEntryKeyString, stepKeyString, } from "@cotal-ai/lang";
|
|
27
26
|
import { PauseOverdue } from "./run-pause-host.js";
|
|
28
|
-
import { loopLag
|
|
29
|
-
import {
|
|
27
|
+
import { loopLag } from "./host-starvation.js";
|
|
28
|
+
import { observeToken, pauseTokens, waitTimeoutToken, waitTimers } from "./pause-tokens.js";
|
|
30
29
|
import { delay } from "./delay.js";
|
|
30
|
+
import { FIRE_POLL_MS, PauseTimers, decodeMessage } from "./mesh-pause-timers.js";
|
|
31
|
+
import { SeatWatch } from "./mesh-seat-watch.js";
|
|
32
|
+
import { TurnAsk } from "./mesh-turn-ask.js";
|
|
31
33
|
/**
|
|
32
34
|
* A checkpoint resumed with no answer to read.
|
|
33
35
|
*
|
|
@@ -1498,589 +1500,6 @@ export class MeshHandler {
|
|
|
1498
1500
|
this.roster.set(name, { handle, owner: caller.owner, actor: caller.actor, uid: caller.uid, host: admitted.instanceId });
|
|
1499
1501
|
return handle;
|
|
1500
1502
|
}
|
|
1501
|
-
/**
|
|
1502
|
-
* `turn` wakes ONE AGENT for one turn, over the manager's relay (a seat is not an endpoint):
|
|
1503
|
-
* the manager accepts the goal, parks the rendered context durably, the seat pulls and yields
|
|
1504
|
-
* under its own self reach, and the yield is this goal's terminal. The request id is the goalId,
|
|
1505
|
-
* the same recovery discipline as `spawn`: a resumed run re-enters the poll, never re-submits.
|
|
1506
|
-
*
|
|
1507
|
-
* THREE AUTHORITIES END IT, one per ending. The seat's yield is the manager's `succeeded`
|
|
1508
|
-
* terminal, carrying the TurnResult. The DEADLINE is double-covered: the manager arms a
|
|
1509
|
-
* goal-bound hold (its expiry commits `failed` reason `turn-deadline`), and this client arms its
|
|
1510
|
-
* OWN pause under the step's request id — the L4003 authority that survives a dead manager.
|
|
1511
|
-
* DEATH: the manager's reap hook marks pending turns; death rides the deadline terminal as
|
|
1512
|
-
* `agentDownAt`. This client's presence watch is the early L4002 authority even while the
|
|
1513
|
-
* manager runs. The watch reads death the way `wait(down)` does: a superseded incarnation at
|
|
1514
|
-
* once, a lapsed row only once it has stayed absent for {@link LAPSE_CONFIRM_MS} and its manager
|
|
1515
|
-
* no longer runs the seat.
|
|
1516
|
-
*
|
|
1517
|
-
* Handoff honoring (lang §5.3) happens HERE: the scope's pending memo is spent at every turn's
|
|
1518
|
-
* begin, and when this turn targets its `to`, the link rides the submission (`handoffFrom`, the
|
|
1519
|
-
* manager mirrors it into the terminal) and the bound external state. A handoff YIELD is
|
|
1520
|
-
* validated against the run roster: an addressee outside it is L4005, one in a different
|
|
1521
|
-
* worktree is L4004.
|
|
1522
|
-
*/
|
|
1523
|
-
async turn(req, ctx) {
|
|
1524
|
-
if (ctx.signal.cancelled)
|
|
1525
|
-
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1526
|
-
const { name, uid } = parseAgentHandle(req.agent.agent);
|
|
1527
|
-
const goalId = ctx.requestId;
|
|
1528
|
-
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
1529
|
-
const primary = { endpoint: this.binding.endpoint, token: goalId };
|
|
1530
|
-
const scope = scopeOf(ctx.key);
|
|
1531
|
-
let ext = ctx.resume?.goalId === goalId ? ctx.resume : undefined;
|
|
1532
|
-
if (ext === undefined) {
|
|
1533
|
-
const seat = this.seatAddress(name, uid);
|
|
1534
|
-
if (seat.address === undefined) {
|
|
1535
|
-
if (seat.why === "not-in-roster")
|
|
1536
|
-
throw new Error(`turn(${name}#${uid}) addresses an agent that is not in this run's roster; a turn wakes an agent this run spawned`);
|
|
1537
|
-
throw new Error(`turn(${name}#${uid}) has no address: the spawn's acceptance floor was never served, so the seat's owner/actor coordinates are unknown`);
|
|
1538
|
-
}
|
|
1539
|
-
const { owner, actor, host } = seat.address;
|
|
1540
|
-
// The permits are the roster entry's, not the address's: a budget is spent per spawn, and
|
|
1541
|
-
// the address is only where this turn is sent.
|
|
1542
|
-
const entry = this.roster.get(name);
|
|
1543
|
-
// Spend the scope's handoff memo at BEGIN, honored or not — honoring is immediate-only.
|
|
1544
|
-
const memo = this.handoffMemos.get(scope);
|
|
1545
|
-
this.handoffMemos.delete(scope);
|
|
1546
|
-
const handoffFrom = memo !== undefined && memo !== "ambiguous" && memo.to === name ? memo.fromGoalId : undefined;
|
|
1547
|
-
// Render the seat's context: every unconsumed notice addressed to it, as one durable payload.
|
|
1548
|
-
const step = stepKeyString(ctx.key);
|
|
1549
|
-
// Addressed by the HANDLE COMPOSITE, exactly as `notify` filed them (an incarnation is the
|
|
1550
|
-
// addressee, and a successor under the name is somebody else).
|
|
1551
|
-
const notices = (await listRunNotices(this.kv, this.binding.endpoint, this.binding.runId, req.agent.agent))
|
|
1552
|
-
.filter((n) => n.consumed === undefined);
|
|
1553
|
-
const context = renderRunContext({ run: this.binding.runId, step, notices });
|
|
1554
|
-
const deadlineMs = parseDuration(req.deadline ?? this.binding.defaultCheckpointTimeout);
|
|
1555
|
-
// The agent's permits are budgets this run spends at the call that would exceed them
|
|
1556
|
-
// (cotal-lang 6.5, L4001): one turn per `turns`, and a turn whose deadline runs past the
|
|
1557
|
-
// agent's remaining wall clock is already over it. Counted on the fresh path only: a resume
|
|
1558
|
-
// re-enters a turn the seed already counted from its journal entry.
|
|
1559
|
-
const taken = this.turnsTaken.get(`${name}#${uid}`) ?? 0;
|
|
1560
|
-
if (entry.permits?.turns !== undefined && taken >= entry.permits.turns)
|
|
1561
|
-
throw new EffectError("L4001", "permit-turns", `turn(${name}#${uid}) would be that agent's turn ${taken + 1}, past its permit of ${entry.permits.turns}`);
|
|
1562
|
-
if (entry.permits?.wallClockMs !== undefined && entry.spawnedAt !== undefined) {
|
|
1563
|
-
const remaining = entry.spawnedAt + entry.permits.wallClockMs - this.now();
|
|
1564
|
-
if (remaining <= 0)
|
|
1565
|
-
throw new EffectError("L4001", "permit-wall-clock", `turn(${name}#${uid}): the agent's wall-clock permit (${entry.permits.wallClockMs}ms from its spawn) is spent`);
|
|
1566
|
-
if (deadlineMs > remaining)
|
|
1567
|
-
throw new EffectError("L4001", "permit-wall-clock", `turn(${name}#${uid}): a ${deadlineMs}ms deadline runs past the agent's remaining wall clock (${remaining}ms of its ${entry.permits.wallClockMs}ms permit)`);
|
|
1568
|
-
}
|
|
1569
|
-
const payload = JSON.stringify({ run: this.binding.runId, step, context, noticeIds: notices.map((n) => n.noticeId) });
|
|
1570
|
-
let submitted = false;
|
|
1571
|
-
const submit = async () => {
|
|
1572
|
-
const service = submitted ? await this.manager("turn", host)
|
|
1573
|
-
: await this.relayManagerBeforeSubmit(host, "turn", ref, "turn", `turn(${name}#${uid})`);
|
|
1574
|
-
// After invocation may have run, retain the existing goal reconciliation.
|
|
1575
|
-
submitted = true;
|
|
1576
|
-
return this.invokeManager(service, "turn", { payload, deadlineMs, ...(handoffFrom !== undefined ? { handoffFrom } : {}) }, {
|
|
1577
|
-
id: goalId,
|
|
1578
|
-
deadlineMs: TURN_ACCEPT_DEADLINE_MS,
|
|
1579
|
-
target: { mode: "owner", owner, actor, lifecycleUid: uid },
|
|
1580
|
-
});
|
|
1581
|
-
};
|
|
1582
|
-
let reply;
|
|
1583
|
-
try {
|
|
1584
|
-
reply = await submit();
|
|
1585
|
-
}
|
|
1586
|
-
catch (err) {
|
|
1587
|
-
// The invoke did not come back — the goal record is the arbiter, exactly as in `spawn`.
|
|
1588
|
-
if ((await readGoalStatus(await this.actionCtx(), ref)) === undefined)
|
|
1589
|
-
throw err;
|
|
1590
|
-
}
|
|
1591
|
-
// ONE reading of a refused acceptance, for both submits: a seat that dies between the first
|
|
1592
|
-
// submit and the re-read is the same dead seat, and the resubmit path classing it as an
|
|
1593
|
-
// ordinary refusal made an L4002 the program can catch arrive as an uncatchable L4000.
|
|
1594
|
-
const refusal = (r, resubmitted) => {
|
|
1595
|
-
const err = r.reply.error;
|
|
1596
|
-
// `expired` is the serve boundary's "target is not a live managed agent". From the placed
|
|
1597
|
-
// host that is the seat's own manager answering about it. From the class rail it is only
|
|
1598
|
-
// the members this call reached, so the message does not claim more than that.
|
|
1599
|
-
if (err?.code === "expired")
|
|
1600
|
-
return new EffectError("L4002", "turn", `turn(${name}#${uid}) ${seatDownReading(err, host)} before the relay began: ${err.message}`);
|
|
1601
|
-
return new Error(resubmitted
|
|
1602
|
-
? `turn(${name}#${uid}) landed but its acceptance could not be re-read: ${err?.message ?? "refused with no message"}`
|
|
1603
|
-
: `turn(${name}#${uid}) was refused by the ${this.binding.endpoint} endpoint: ${err?.message ?? "refused with no message"}`);
|
|
1604
|
-
};
|
|
1605
|
-
if (reply !== undefined && reply.reply.ok === false)
|
|
1606
|
-
throw refusal(reply, false);
|
|
1607
|
-
if (reply === undefined) {
|
|
1608
|
-
// The acceptance carries the deadline authority, so a lost reply is asked for again: the
|
|
1609
|
-
// same goalId under the same fingerprint is served from the manager's acceptance cache.
|
|
1610
|
-
reply = await submit();
|
|
1611
|
-
if (reply.reply.ok === false)
|
|
1612
|
-
throw refusal(reply, true);
|
|
1613
|
-
}
|
|
1614
|
-
const floor = reply.reply.data;
|
|
1615
|
-
if (typeof floor?.deadlineAt !== "number")
|
|
1616
|
-
throw new Error(`turn(${name}#${uid}) was accepted with no readable deadline (${JSON.stringify(reply.reply.data)}); a garbled acceptance never authorizes (SPEC 13.6)`);
|
|
1617
|
-
// ONE deadline for the turn: the acceptance's `deadlineAt` is the instant the manager's own
|
|
1618
|
-
// hold denies at, and the client's pause is armed on that same instant. A second clock read
|
|
1619
|
-
// before the accept round-trip would fire first by the round-trip plus skew and throw away a
|
|
1620
|
-
// yield the manager's window still admitted.
|
|
1621
|
-
const deadlineAt = floor.deadlineAt;
|
|
1622
|
-
ext = {
|
|
1623
|
-
goalId, name, owner, actor, uid, deadlineAt,
|
|
1624
|
-
noticeIds: notices.map((n) => n.noticeId),
|
|
1625
|
-
...(handoffFrom !== undefined ? { handoffFrom } : {}),
|
|
1626
|
-
};
|
|
1627
|
-
await ctx.bind(ext);
|
|
1628
|
-
// Spent once the turn is RECORDED, which is the same event the adoption seed counts. Spending
|
|
1629
|
-
// it before the submit charged a permit for a turn the seat never saw (a dead seat's L4002,
|
|
1630
|
-
// an endpoint refusal), and a recovering run would then read a different number off the
|
|
1631
|
-
// journal than the live one held.
|
|
1632
|
-
this.turnsTaken.set(`${name}#${uid}`, taken + 1);
|
|
1633
|
-
}
|
|
1634
|
-
this.recordTurnGoal(`${name}#${uid}`, goalId);
|
|
1635
|
-
const deadlineAt = ext.deadlineAt;
|
|
1636
|
-
if (typeof deadlineAt !== "number")
|
|
1637
|
-
throw new Error(`turn(${name}#${uid}) resumed with no recorded deadline; a garbled external state never authorizes`);
|
|
1638
|
-
// The client's OWN deadline authority: minted on the first pass, attached to on re-entry.
|
|
1639
|
-
await this.arm(primary, deadlineAt);
|
|
1640
|
-
const actx = await this.actionCtx();
|
|
1641
|
-
const watch = {};
|
|
1642
|
-
// Only re-enter the read, never the relay submission or its outcome processing. Each read
|
|
1643
|
-
// gets the pause plane's bounded timeout/starvation policy and names itself in diagnostics.
|
|
1644
|
-
const poll = (what, read) => servedDespiteStarvation(async () => {
|
|
1645
|
-
if (ctx.signal.cancelled)
|
|
1646
|
-
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1647
|
-
return await read();
|
|
1648
|
-
}, this.lag, `turn(${name}#${uid}) ${what}`, this.onStarved);
|
|
1649
|
-
const result = () => poll("readGoalResult", () => readGoalResult(actx, ref));
|
|
1650
|
-
const expired = () => poll("expired", () => this.expired(primary));
|
|
1651
|
-
const decided = async () => ctx.signal.cancelled || (await result()) !== undefined || (await expired()) !== undefined;
|
|
1652
|
-
try {
|
|
1653
|
-
for (;;) {
|
|
1654
|
-
if (ctx.signal.cancelled)
|
|
1655
|
-
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1656
|
-
const fact = await result();
|
|
1657
|
-
if (fact !== undefined) {
|
|
1658
|
-
await this.cancelTimer(primary);
|
|
1659
|
-
return await this.turnOutcome(req, fact, ext, scope, name, uid);
|
|
1660
|
-
}
|
|
1661
|
-
const ended = await expired();
|
|
1662
|
-
if (ended !== undefined)
|
|
1663
|
-
throw new EffectError("L4003", "turn-deadline", `turn(${name}#${uid}) deadline elapsed before a yield`);
|
|
1664
|
-
const reason = await poll("presenceDeath", () => this.presenceDeath(name, uid, watch, decided));
|
|
1665
|
-
if (reason !== undefined) {
|
|
1666
|
-
await this.cancelTimer(primary);
|
|
1667
|
-
throw new EffectError("L4002", "turn", `turn(${name}#${uid}) found the agent down (${reason}) before a yield`);
|
|
1668
|
-
}
|
|
1669
|
-
await delay(WAIT_POLL_MS);
|
|
1670
|
-
}
|
|
1671
|
-
}
|
|
1672
|
-
catch (e) {
|
|
1673
|
-
// A turn that ends on the run's side without a yield the run accepted (its deadline, the
|
|
1674
|
-
// seat's death, a refused handoff, a cancellation) is not a reply, whatever the relay
|
|
1675
|
-
// records later: `wait(replied)` reads the verdict this branch raised, never a stale yield.
|
|
1676
|
-
this.turnGoals.get(`${name}#${uid}`)?.delete(goalId);
|
|
1677
|
-
// In particular a read that exhausted its retry budget must not leave a live relay that
|
|
1678
|
-
// tells the seat its later yield continued a turn the run already abandoned. The ordinary
|
|
1679
|
-
// turn verdicts already have an authority: death, the shared deadline, or a yielded handoff.
|
|
1680
|
-
// Do not ask a dead manager to confirm the client's independent deadline or death verdict.
|
|
1681
|
-
const terminal = e instanceof EffectError && ["L4002", "L4003", "L4004", "L4005"].includes(e.code);
|
|
1682
|
-
if (!terminal) {
|
|
1683
|
-
// Neither cleanup failure may prevent the other resource from being retired.
|
|
1684
|
-
const cleanup = await Promise.allSettled([
|
|
1685
|
-
this.cancelTimer(primary),
|
|
1686
|
-
this.withdrawCancelledRelay(goalId, deadlineAt),
|
|
1687
|
-
]);
|
|
1688
|
-
const failures = cleanup.flatMap((r, i) => r.status === "rejected"
|
|
1689
|
-
? [{ resource: i === 0 ? "primary timer claim" : "relay withdrawal", error: r.reason }]
|
|
1690
|
-
: []);
|
|
1691
|
-
if (failures.length > 0) {
|
|
1692
|
-
const message = `${e instanceof Error ? e.message : String(e)}; turn cleanup failed: `
|
|
1693
|
-
+ failures.map((f) => `${f.resource}: ${f.error instanceof Error ? f.error.message : String(f.error)}`).join("; ");
|
|
1694
|
-
const cause = new AggregateError([e, ...failures.map((f) => f.error)], message);
|
|
1695
|
-
if (e instanceof EffectError) {
|
|
1696
|
-
const error = new EffectError(e.code, e.kind, message, e.detail);
|
|
1697
|
-
error.cause = cause;
|
|
1698
|
-
throw error;
|
|
1699
|
-
}
|
|
1700
|
-
throw cause;
|
|
1701
|
-
}
|
|
1702
|
-
}
|
|
1703
|
-
throw e;
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
1706
|
-
/**
|
|
1707
|
-
* A yielded handoff's refusal class: L4005 when the addressee is not in this run's roster,
|
|
1708
|
-
* L4004 when it sits in a different worktree than the seat handing off, undefined when the
|
|
1709
|
-
* handoff is honorable. One reading for the turn that raises it and the `wait(replied)` that
|
|
1710
|
-
* must not count it, so the two observers never diverge on a spoofed or misdirected `to`.
|
|
1711
|
-
*/
|
|
1712
|
-
handoffRefusal(fromWorktree, to) {
|
|
1713
|
-
const toName = typeof to === "string" ? to : "";
|
|
1714
|
-
const target = this.roster.get(toName);
|
|
1715
|
-
if (toName.length === 0 || target === undefined)
|
|
1716
|
-
return "L4005";
|
|
1717
|
-
if ((target.handle.worktree ?? undefined) !== (fromWorktree ?? undefined))
|
|
1718
|
-
return "L4004";
|
|
1719
|
-
return undefined;
|
|
1720
|
-
}
|
|
1721
|
-
/** Map a turn goal's terminal onto the effect's contract: `succeeded` carries the TurnResult
|
|
1722
|
-
* (a handoff addressee resolved against the roster — L4005 outside it, L4004 across
|
|
1723
|
-
* worktrees), `failed` splits on the manager's recorded reason, `cancelled` unwinds. The
|
|
1724
|
-
* consumed notices are marked HERE, by the goal that carried them, tolerating the re-entry
|
|
1725
|
-
* conflict (a crash between the terminal and the mark re-marks on resume). */
|
|
1726
|
-
async turnOutcome(req, fact, ext, scope, name, uid) {
|
|
1727
|
-
if (fact.state === "cancelled")
|
|
1728
|
-
throw new Cancelled(`the turn goal for ${name}#${uid} was cancelled`);
|
|
1729
|
-
if (fact.state === "failed") {
|
|
1730
|
-
const d = fact.data;
|
|
1731
|
-
// The one reasoned failure a relay commits. A dead target has no early terminal of its own
|
|
1732
|
-
// on the goal plane (SPEC 13.6 item 7): the client's presence watch in the poll loop is the
|
|
1733
|
-
// first L4002 authority, and a death that rode to the deadline arrives on this same deny,
|
|
1734
|
-
// MARKED by the relay (`agentDownAt`) — read here as the agent-down failure it is, never as
|
|
1735
|
-
// a deadline the program could have set longer.
|
|
1736
|
-
if (d?.reason === "turn-deadline") {
|
|
1737
|
-
if (typeof d.agentDownAt === "number")
|
|
1738
|
-
throw new EffectError("L4002", "turn", `turn(${name}#${uid}) found the agent down (it left presence at ${new Date(d.agentDownAt).toISOString()}) before a yield; the deadline terminal carries its death`);
|
|
1739
|
-
throw new EffectError("L4003", "turn-deadline", `turn(${name}#${uid}) deadline elapsed before a yield`);
|
|
1740
|
-
}
|
|
1741
|
-
throw new Error(`turn(${name}#${uid}) failed at the ${this.binding.endpoint} endpoint: ${typeof d?.error === "string" ? d.error : JSON.stringify(fact.data)}`);
|
|
1742
|
-
}
|
|
1743
|
-
if (fact.state !== "succeeded")
|
|
1744
|
-
throw new Error(`turn(${name}#${uid}) settled ${fact.state}; a turn's yield is a succeeded terminal or a reasoned failure, never this`);
|
|
1745
|
-
const d = fact.data;
|
|
1746
|
-
if ((d?.status !== "done" && d?.status !== "blocked" && d?.status !== "handoff") || typeof d.at !== "number")
|
|
1747
|
-
throw new Error(`turn(${name}#${uid}) succeeded with a malformed TurnResult (${JSON.stringify(fact.data)}); a garbled terminal never yields a result`);
|
|
1748
|
-
let to;
|
|
1749
|
-
if (d.status === "handoff") {
|
|
1750
|
-
const toName = typeof d.to === "string" ? d.to : "";
|
|
1751
|
-
const refusal = this.handoffRefusal(req.agent.worktree, d.to);
|
|
1752
|
-
if (refusal === "L4005")
|
|
1753
|
-
throw new EffectError("L4005", "turn-handoff", `turn(${name}#${uid}) yielded a handoff to "${toName}", which is not in this run's roster`);
|
|
1754
|
-
const target = this.roster.get(toName);
|
|
1755
|
-
if (refusal === "L4004")
|
|
1756
|
-
throw new EffectError("L4004", "turn-handoff", `turn(${name}#${uid}) yielded a handoff to "${toName}" across worktrees (${req.agent.worktree ?? "none"} -> ${target.handle.worktree ?? "none"}); you cannot hand someone a working tree they are not in`);
|
|
1757
|
-
to = target.handle;
|
|
1758
|
-
}
|
|
1759
|
-
// The notices this turn carried are consumed by THIS goal — the create-only CAS arbitrates,
|
|
1760
|
-
// and a re-entry's conflict reads as "already recorded", never as a failure.
|
|
1761
|
-
const noticeIds = Array.isArray(ext.noticeIds) ? ext.noticeIds.filter((n) => typeof n === "string") : [];
|
|
1762
|
-
for (const noticeId of noticeIds) {
|
|
1763
|
-
try {
|
|
1764
|
-
await markRunNoticeConsumed(this.kv, this.binding.endpoint, this.binding.runId, req.agent.agent, noticeId, String(ext.goalId), this.now());
|
|
1765
|
-
}
|
|
1766
|
-
catch (e) {
|
|
1767
|
-
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
1768
|
-
throw e;
|
|
1769
|
-
}
|
|
1770
|
-
}
|
|
1771
|
-
if (d.status === "handoff" && to !== undefined)
|
|
1772
|
-
this.recordHandoffMemo(scope, parseAgentHandle(to.agent).name, String(ext.goalId));
|
|
1773
|
-
return {
|
|
1774
|
-
status: d.status,
|
|
1775
|
-
...(to !== undefined ? { to } : {}),
|
|
1776
|
-
...(typeof d.note === "string" ? { note: d.note } : {}),
|
|
1777
|
-
at: d.at,
|
|
1778
|
-
};
|
|
1779
|
-
}
|
|
1780
|
-
/**
|
|
1781
|
-
* `ask` is a schema-checked pause the ADDRESSED AGENT is expected to answer.
|
|
1782
|
-
*
|
|
1783
|
-
* Each attempt is one checkpoint-plane pause — the same mint, settle and answer record a
|
|
1784
|
-
* `checkpoint` rides, answered through the run driver's `resolveCheckpoint` (`cotal run
|
|
1785
|
-
* answer`), which is how "the agent publishes a record" reaches a holder-bound plane: the agent,
|
|
1786
|
-
* or anyone the run's ACL admits on its behalf, answers through the driver exactly as a human
|
|
1787
|
-
* answers a checkpoint. What `ask` adds is the handler-side contract of §6.5: the schema is read
|
|
1788
|
-
* as the SHORTHAND (refused L4022 when it cannot be), every answer is checked against it, a
|
|
1789
|
-
* non-conforming answer costs one attempt and the refusal reason is bound onto the entry for the
|
|
1790
|
-
* answerer to read, and exhausted attempts are the effect's own catchable L4006.
|
|
1791
|
-
*
|
|
1792
|
-
* **One absolute deadline for the whole ask**, computed once and bound with the first attempt: a
|
|
1793
|
-
* re-ask does not restart the clock, or a stream of non-conforming answers could stretch the
|
|
1794
|
-
* pause forever. The deadline elapsing with no conforming answer is the ask's own L4006, the
|
|
1795
|
-
* same outcome exhausted attempts name: the budget it ran out of is the ask's, and L4003 belongs
|
|
1796
|
-
* to a `turn` whose own deadline elapsed.
|
|
1797
|
-
*
|
|
1798
|
-
* **Attempt N's token derives from the step's request id** (attempt 1 IS the request id), and
|
|
1799
|
-
* the CURRENT attempt's token is bound as `askToken` before its pause is armed — so a resume
|
|
1800
|
-
* re-enters the attempt in flight, an answer judged non-conforming just before a crash is
|
|
1801
|
-
* re-judged identically from its durable settle, and the discharge and the adoption re-arm
|
|
1802
|
-
* address the one timer that is actually armed.
|
|
1803
|
-
*/
|
|
1804
|
-
async ask(req, ctx) {
|
|
1805
|
-
if (ctx.signal.cancelled)
|
|
1806
|
-
throw new Cancelled(ctx.signal.reason ?? "cancelled");
|
|
1807
|
-
const shape = askSchemaShape(req.schema);
|
|
1808
|
-
if (shape === null) {
|
|
1809
|
-
throw new EffectError("L4022", "ask-schema-unreadable", `L4022 Unreadable ask schema\n\n step ${stepKeyString(ctx.key)}\n\nThe schema is not the shorthand this handler enforces: a record mapping each field name to one of "string", "number", "boolean", "array", "record", "null".\n\nFix: write the shorthand, for example { steps: "array" }, or pass {} to accept any record.`);
|
|
1810
|
-
}
|
|
1811
|
-
const attempts = req.attempts ?? 1;
|
|
1812
|
-
// The seat the ask is told to, resolved before anything binds: an ask addresses an agent this
|
|
1813
|
-
// run spawned, and one it cannot tell opens no attempt.
|
|
1814
|
-
const seat = this.askSeat(req, ctx);
|
|
1815
|
-
const resume = askResume(ctx.resume);
|
|
1816
|
-
const deadlineAt = resume?.deadlineAt
|
|
1817
|
-
?? this.now() + parseDuration(req.deadline ?? this.binding.defaultCheckpointTimeout);
|
|
1818
|
-
let attempt = resume?.attempt ?? 1;
|
|
1819
|
-
// The resumed attempt is already bound; every attempt this call opens binds before it arms.
|
|
1820
|
-
let bindOwed = resume === undefined;
|
|
1821
|
-
let refused = resume?.refused;
|
|
1822
|
-
for (;;) {
|
|
1823
|
-
const token = askAttemptToken(ctx.requestId, attempt);
|
|
1824
|
-
if (bindOwed) {
|
|
1825
|
-
await ctx.bind({
|
|
1826
|
-
attempt,
|
|
1827
|
-
askToken: token,
|
|
1828
|
-
deadlineAt,
|
|
1829
|
-
...(refused !== undefined ? { refused } : {}),
|
|
1830
|
-
});
|
|
1831
|
-
}
|
|
1832
|
-
bindOwed = true;
|
|
1833
|
-
// The addressed agent is TOLD: the attempt rides the turn relay under the attempt's own
|
|
1834
|
-
// token, carrying the token, the schema, the attempt count, the deadline and the previous
|
|
1835
|
-
// refusal, so the seat can answer through `cotal run answer`. Idempotent by goal id: a
|
|
1836
|
-
// re-entry finds the relay already accepted and submits nothing.
|
|
1837
|
-
await this.relayAsk(seat, ctx, token, { attempt, attempts, deadlineAt, ...(refused !== undefined ? { refused } : {}) });
|
|
1838
|
-
const ref = { endpoint: this.binding.endpoint, token };
|
|
1839
|
-
// The deadline is the ask's whole budget of time: passing it with no conforming record
|
|
1840
|
-
// is the same outcome exhausted attempts name (L4006), never a turn's deadline (L4003),
|
|
1841
|
-
// and that holds when it passed while the attempt was bound and relayed, before its pause
|
|
1842
|
-
// could be minted.
|
|
1843
|
-
const expired = () => new EffectError("L4006", "ask-deadline", `ask(${stepKeyString(ctx.key)}) produced no conforming record before its recorded deadline: attempt ${attempt} of ${attempts} was still open when it passed`);
|
|
1844
|
-
try {
|
|
1845
|
-
await this.arm(ref, deadlineAt);
|
|
1846
|
-
}
|
|
1847
|
-
catch (e) {
|
|
1848
|
-
if (e instanceof PauseOverdue)
|
|
1849
|
-
throw expired();
|
|
1850
|
-
throw e;
|
|
1851
|
-
}
|
|
1852
|
-
const settled = await this.settle(ref, ctx.signal, true);
|
|
1853
|
-
if (settled.settle === "expired")
|
|
1854
|
-
throw expired();
|
|
1855
|
-
const answer = settled.answerId === undefined
|
|
1856
|
-
? undefined
|
|
1857
|
-
: this.services ? await this.services.pauses.readAnswer(token) : await readCheckpointAnswer(this.kv, this.binding.endpoint, token, settled.answerId);
|
|
1858
|
-
if (answer === undefined)
|
|
1859
|
-
throw new CheckpointAnswerMissing(token, settled.answerId);
|
|
1860
|
-
if (conformsToAskSchema(answer.value, shape))
|
|
1861
|
-
return answer.value;
|
|
1862
|
-
const why = askNonconformance(answer.value, shape);
|
|
1863
|
-
if (attempt >= attempts) {
|
|
1864
|
-
throw new EffectError("L4006", "ask-nonconforming", `L4006 ask never produced a conforming record\n\n step ${stepKeyString(ctx.key)}\n\n${attempts} repl${attempts === 1 ? "y was" : "ies were"} checked against the schema and none conformed; the last was refused: ${why}.\n\nFix: have the agent publish a record matching the schema, widen the schema, or raise attempts.`);
|
|
1865
|
-
}
|
|
1866
|
-
refused = why;
|
|
1867
|
-
attempt += 1;
|
|
1868
|
-
}
|
|
1869
|
-
}
|
|
1870
|
-
/**
|
|
1871
|
-
* One ask attempt's relay to the addressed seat: a `turn` goal under the attempt's token whose
|
|
1872
|
-
* payload carries the ask (token, schema, attempt, deadline, the previous refusal), rendered by
|
|
1873
|
-
* the seat's connector as the request to answer. The goal is the seat's to yield when it has
|
|
1874
|
-
* answered; the ask itself settles on the checkpoint plane, so the relay's terminal is never
|
|
1875
|
-
* read here, and it is never a reply `wait(replied)` observes. A seat the serve boundary reports
|
|
1876
|
-
* gone is the agent-down failure (L4002).
|
|
1877
|
-
*/
|
|
1878
|
-
askSeat(req, ctx) {
|
|
1879
|
-
const { name, uid } = parseAgentHandle(req.agent.agent);
|
|
1880
|
-
const seat = this.seatAddress(name, uid);
|
|
1881
|
-
const step = stepKeyString(ctx.key);
|
|
1882
|
-
if (seat.address === undefined) {
|
|
1883
|
-
if (seat.why === "not-in-roster")
|
|
1884
|
-
throw new Error(`ask(${step}) addresses ${name}#${uid}, which is not in this run's roster; an ask is relayed to an agent this run spawned`);
|
|
1885
|
-
throw new Error(`ask(${step}) has no address for ${name}#${uid}: the spawn's acceptance floor was never served, so the seat's owner/actor coordinates are unknown`);
|
|
1886
|
-
}
|
|
1887
|
-
return { ...seat.address, schema: req.schema };
|
|
1888
|
-
}
|
|
1889
|
-
/**
|
|
1890
|
-
* Where a relay to an agent of this run is addressed: the roster entry under the handle's name,
|
|
1891
|
-
* at the handle's incarnation, with the owner/actor coordinates its acceptance floor served.
|
|
1892
|
-
* One resolver for the three relays (`turn`, `ask`, an escalation), because the three
|
|
1893
|
-
* dispositions differ and the lookup does not: a turn and an ask refuse, an escalation stands.
|
|
1894
|
-
*/
|
|
1895
|
-
seatAddress(name, uid) {
|
|
1896
|
-
const entry = this.roster.get(name);
|
|
1897
|
-
if (entry === undefined || (uid !== undefined && entry.uid !== uid))
|
|
1898
|
-
return { why: "not-in-roster" };
|
|
1899
|
-
if (entry.owner === undefined || entry.actor === undefined)
|
|
1900
|
-
return { why: "no-floor" };
|
|
1901
|
-
return { address: { name, uid: entry.uid, owner: entry.owner, actor: entry.actor, ...(entry.host !== undefined ? { host: entry.host } : {}) } };
|
|
1902
|
-
}
|
|
1903
|
-
/**
|
|
1904
|
-
* The principal a `message` wait's `from` handle names, as `<owner>.<actor>`: the coordinates
|
|
1905
|
-
* this run's roster recorded when it spawned the seat or bound the participant. A message
|
|
1906
|
-
* answers only when its subject's sender is that principal. A handle outside the roster, or one
|
|
1907
|
-
* whose coordinates were never served, refuses: matching it on a display name would let any peer
|
|
1908
|
-
* that names itself after the agent answer.
|
|
1909
|
-
*/
|
|
1910
|
-
waitSender(agent) {
|
|
1911
|
-
const { name, uid } = parseAgentHandle(agent);
|
|
1912
|
-
const seat = this.seatAddress(name, uid);
|
|
1913
|
-
if (seat.address === undefined) {
|
|
1914
|
-
if (seat.why === "not-in-roster")
|
|
1915
|
-
throw new Error(`wait(message(..., { from: ${name}#${uid} })) names an agent that is not in this run's roster; \`from\` takes a handle this run spawned or bound with participant()`);
|
|
1916
|
-
throw new Error(`wait(message(..., { from: ${name}#${uid} })) has no sender to match: the spawn's acceptance floor was never served, so the seat's owner/actor coordinates are unknown`);
|
|
1917
|
-
}
|
|
1918
|
-
return principalKey(seat.address.owner, seat.address.actor).key;
|
|
1919
|
-
}
|
|
1920
|
-
async relayAsk(seat, ctx, token, ask) {
|
|
1921
|
-
const step = stepKeyString(ctx.key);
|
|
1922
|
-
const context = renderRunContext({ run: this.binding.runId, step, notices: [] });
|
|
1923
|
-
const payload = JSON.stringify({
|
|
1924
|
-
run: this.binding.runId, step, context, noticeIds: [],
|
|
1925
|
-
ask: { token, schema: seat.schema, attempt: ask.attempt, attempts: ask.attempts, deadlineAt: ask.deadlineAt, ...(ask.refused !== undefined ? { refused: ask.refused } : {}) },
|
|
1926
|
-
});
|
|
1927
|
-
await this.relayToSeat(seat, token, payload, ask.deadlineAt, "ask", step);
|
|
1928
|
-
}
|
|
1929
|
-
/**
|
|
1930
|
-
* An escalated checkpoint, delivered to the agent it names.
|
|
1931
|
-
*
|
|
1932
|
-
* `to` is legal only with `onExpiry: "escalate"` and the escalation is the SECOND mint, so this
|
|
1933
|
-
* runs once per chain at most. The addressee is resolved through the run's own roster: a name
|
|
1934
|
-
* this run spawned is a seat and is told, and anything else is a person — the pause is
|
|
1935
|
-
* answerable from anywhere by design, its addressee is on the entry, and refusing a program
|
|
1936
|
-
* that escalates to a human would be this host narrowing an option the reference leaves open.
|
|
1937
|
-
*
|
|
1938
|
-
* The relay is the ask's, because the two are the same act: one turn on the seat, carrying what
|
|
1939
|
-
* is asked and the token to answer under, settling on the checkpoint plane rather than on the
|
|
1940
|
-
* relay's own terminal.
|
|
1941
|
-
*/
|
|
1942
|
-
async relayEscalation(req, ctx, token, deadlineAt) {
|
|
1943
|
-
const to = req.to;
|
|
1944
|
-
const step = stepKeyString(ctx.key);
|
|
1945
|
-
// A seat is a roster entry whose acceptance floor was served: that is what carries the
|
|
1946
|
-
// owner/actor coordinates a relay is addressed by. Anything else — a name this run never
|
|
1947
|
-
// spawned, or one whose floor never landed — is not a seat, and the escalation stays the
|
|
1948
|
-
// durable pause it already is, addressed on its entry and answerable from anywhere.
|
|
1949
|
-
const seat = this.seatAddress(to);
|
|
1950
|
-
if (seat.address === undefined)
|
|
1951
|
-
return;
|
|
1952
|
-
const context = renderRunContext({ run: this.binding.runId, step, notices: [] });
|
|
1953
|
-
const payload = JSON.stringify({
|
|
1954
|
-
run: this.binding.runId, step, context, noticeIds: [],
|
|
1955
|
-
checkpoint: { token, prompt: req.prompt, schema: req.schema ?? null, deadlineAt, escalatedTo: to },
|
|
1956
|
-
});
|
|
1957
|
-
try {
|
|
1958
|
-
await this.relayToSeat(seat.address, token, payload, deadlineAt, "checkpoint", step);
|
|
1959
|
-
}
|
|
1960
|
-
catch (e) {
|
|
1961
|
-
// THE PAUSE OUTLIVES ITS ADDRESSEE. The reference defines no failure for an escalation whose
|
|
1962
|
-
// addressee cannot be reached: it "mints exactly one further checkpoint addressed to `to`",
|
|
1963
|
-
// and a checkpoint is "a durable pause a human or an agent resolves from anywhere". A seat
|
|
1964
|
-
// the endpoint reports gone is therefore the same case as a `to` that names a person: nobody
|
|
1965
|
-
// is told, the addressee is on the entry, and anyone may still answer. L4002 is the turn's
|
|
1966
|
-
// vocabulary, and this is not a turn the program asked for. Said once, not swallowed.
|
|
1967
|
-
if (!(e instanceof EffectError && e.code === "L4002"))
|
|
1968
|
-
throw e;
|
|
1969
|
-
console.error(`run ${this.binding.runId}: the escalation at ${step} could not be told to ${to} (${e.message}); the pause stands, answerable from anywhere`);
|
|
1970
|
-
}
|
|
1971
|
-
}
|
|
1972
|
-
/**
|
|
1973
|
-
* Submit one relay to a seat as a `turn` goal, and read its acceptance.
|
|
1974
|
-
*
|
|
1975
|
-
* IDEMPOTENT BY THE GOAL RECORD, which is also the arbiter when the invoke does not come back:
|
|
1976
|
-
* a relay that landed and lost its reply is a relay that landed, and re-submitting it would put
|
|
1977
|
-
* a second turn on the seat for one request. A seat the serve boundary reports gone is the
|
|
1978
|
-
* agent-down failure (L4002); every other refusal is this endpoint's and is uncatchable.
|
|
1979
|
-
*
|
|
1980
|
-
* The relay carries the pause's own `deadlineAt` and the manager holds it to that instant: a
|
|
1981
|
-
* duration counted from the manager's acceptance would outlive the pause by the submit's trip
|
|
1982
|
-
* plus skew and show the seat an attempt whose answer is refused. A relay the manager refuses as
|
|
1983
|
-
* already past, and that is past on this clock too, has nobody left to tell, because the pause
|
|
1984
|
-
* denies at that same instant; the arm that follows meets the passed deadline and decides the
|
|
1985
|
-
* outcome.
|
|
1986
|
-
*/
|
|
1987
|
-
async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) {
|
|
1988
|
-
const { name, uid } = seat;
|
|
1989
|
-
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
1990
|
-
const actx = await this.actionCtx();
|
|
1991
|
-
if ((await readGoalStatus(actx, ref)) !== undefined)
|
|
1992
|
-
return;
|
|
1993
|
-
const service = await this.relayManagerBeforeSubmit(seat.host, "turn", ref, kind, `${kind}(${step})`);
|
|
1994
|
-
let reply;
|
|
1995
|
-
try {
|
|
1996
|
-
reply = await this.invokeManager(service, "turn", { payload, deadlineAt }, {
|
|
1997
|
-
id: goalId,
|
|
1998
|
-
deadlineMs: TURN_ACCEPT_DEADLINE_MS,
|
|
1999
|
-
target: { mode: "owner", owner: seat.owner, actor: seat.actor, lifecycleUid: uid },
|
|
2000
|
-
});
|
|
2001
|
-
}
|
|
2002
|
-
catch (err) {
|
|
2003
|
-
// The invoke did not come back — the goal record is the arbiter, exactly as in `turn`.
|
|
2004
|
-
if ((await readGoalStatus(actx, ref)) === undefined)
|
|
2005
|
-
throw err;
|
|
2006
|
-
return;
|
|
2007
|
-
}
|
|
2008
|
-
if (reply.reply.ok === false) {
|
|
2009
|
-
const err = reply.reply.error;
|
|
2010
|
-
if (err.code === "expired")
|
|
2011
|
-
throw new EffectError("L4002", kind, `${kind}(${step}): ${name}#${uid} ${seatDownReading(err, seat.host)} before its relay began: ${err.message}`);
|
|
2012
|
-
if (err.code === "deadline-exceeded" && this.now() >= deadlineAt)
|
|
2013
|
-
return;
|
|
2014
|
-
throw new Error(`${kind}(${step}) was refused by the ${this.binding.endpoint} endpoint: ${err.message}`);
|
|
2015
|
-
}
|
|
2016
|
-
}
|
|
2017
|
-
/**
|
|
2018
|
-
* Withdraw the relay a cancelled step sent to a seat, through the manager's reserved `cancel`
|
|
2019
|
-
* (SPEC 13.6 item 4), so the seat is not shown it again. A token no relay rode has no goal and
|
|
2020
|
-
* nothing to withdraw, and a relay that already ended is the outcome either way: the manager
|
|
2021
|
-
* refuses it with the cached terminal, so a repeated withdrawal is a no-op. Any other refusal
|
|
2022
|
-
* throws under the code it was refused with, which leaves a discharge open to retry.
|
|
2023
|
-
*
|
|
2024
|
-
* Only the manager that accepted the relay holds it, and the class rail reaches any member, so a
|
|
2025
|
-
* refusal is final only from the accepter its index entry names. One from another member is
|
|
2026
|
-
* re-issued on the class rail, as the spawn discharge's despawn is.
|
|
2027
|
-
*/
|
|
2028
|
-
async withdrawRelay(goalId) {
|
|
2029
|
-
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
2030
|
-
const actx = await this.actionCtx();
|
|
2031
|
-
if ((await readGoalStatus(actx, ref)) === undefined)
|
|
2032
|
-
return;
|
|
2033
|
-
const accepter = (await readGoalIndex(actx, ref))?.iid;
|
|
2034
|
-
for (let attempt = 0;; attempt += 1) {
|
|
2035
|
-
const reply = await this.invokeManager(await this.manager("cancel"), "cancel", { goalId }, { deadlineMs: TURN_ACCEPT_DEADLINE_MS });
|
|
2036
|
-
if (reply.reply.ok !== false || (await readGoalResult(actx, ref)) !== undefined)
|
|
2037
|
-
return;
|
|
2038
|
-
if (accepter === undefined || reply.responder.instanceId === accepter) {
|
|
2039
|
-
const err = reply.reply.error;
|
|
2040
|
-
throw new EpEnvelopeError(err.code, `the relay "${goalId}" could not be withdrawn from its seat: ${err.message}`);
|
|
2041
|
-
}
|
|
2042
|
-
if (attempt === MANAGER_ROUTE_ATTEMPTS)
|
|
2043
|
-
throw new Error(`the relay "${goalId}" was accepted by manager instance ${accepter}, but none of ${attempt + 1} cancels was answered by it; the discharge stays open to retry`);
|
|
2044
|
-
this.managerServices.delete("cancel");
|
|
2045
|
-
}
|
|
2046
|
-
}
|
|
2047
|
-
/**
|
|
2048
|
-
* Withdraw a cancelled step's relay before the cancellation leaves the step, so the scope it
|
|
2049
|
-
* settles shows the seat nothing the run withdrew. A withdrawal that fails is tried again rather
|
|
2050
|
-
* than left to the discharge, which runs only when the run ends: until then the relay would stay
|
|
2051
|
-
* first on the seat and hold back the run's next turn to it. The retries stop when the relay's
|
|
2052
|
-
* deadline passes, because the manager no longer serves it then, and a failure that outlasts it
|
|
2053
|
-
* is the step's error. A refusal no retry can change is the step's error at once, since retrying
|
|
2054
|
-
* it would only hold the cancelled step in its scope until that deadline: of the refusals, only
|
|
2055
|
-
* `unavailable` (a manager still reconciling at boot) and `deadline-exceeded` (a lost reply) are
|
|
2056
|
-
* tried again.
|
|
2057
|
-
*
|
|
2058
|
-
* That deadline is read from the relay's goal, where the accepting manager recorded the instant
|
|
2059
|
-
* it stops serving the relay; a withdrawal holds only the goal id. No deadline is known until that
|
|
2060
|
-
* goal is read, and the relay may be served until then, so a failed read is tried again as well.
|
|
2061
|
-
* A goal record that can never yield a deadline is thrown.
|
|
2062
|
-
*/
|
|
2063
|
-
async withdrawCancelledRelay(goalId, deadlineAt) {
|
|
2064
|
-
for (;;) {
|
|
2065
|
-
try {
|
|
2066
|
-
if (deadlineAt === undefined) {
|
|
2067
|
-
const spec = (await readGoalSpec(await this.actionCtx(), { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId }))?.value;
|
|
2068
|
-
if (spec === undefined)
|
|
2069
|
-
return;
|
|
2070
|
-
if (spec.readinessDeadlineMs === undefined)
|
|
2071
|
-
throw new EpEnvelopeError("internal", `the relay "${goalId}" was accepted with no deadline on its goal; a garbled acceptance never authorizes (SPEC 13.6)`);
|
|
2072
|
-
deadlineAt = spec.acceptedAt + spec.readinessDeadlineMs;
|
|
2073
|
-
}
|
|
2074
|
-
return await this.withdrawRelay(goalId);
|
|
2075
|
-
}
|
|
2076
|
-
catch (e) {
|
|
2077
|
-
const final = e instanceof EpEnvelopeError && e.code !== "unavailable" && e.code !== "deadline-exceeded";
|
|
2078
|
-
if (deadlineAt === undefined ? e instanceof EpEnvelopeError : final || this.now() >= deadlineAt)
|
|
2079
|
-
throw e;
|
|
2080
|
-
}
|
|
2081
|
-
await delay(WAIT_POLL_MS);
|
|
2082
|
-
}
|
|
2083
|
-
}
|
|
2084
1503
|
/**
|
|
2085
1504
|
* `monitor` registers interest: after it, `down(agent)` is an event a branch can await (§5.9).
|
|
2086
1505
|
*
|
|
@@ -2304,465 +1723,83 @@ export class MeshHandler {
|
|
|
2304
1723
|
await (await this.channelRegistry()).delete(plan.channel);
|
|
2305
1724
|
}
|
|
2306
1725
|
}
|
|
2307
|
-
/**
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
const entry = this.roster.get(name);
|
|
2376
|
-
const spawned = entry?.uid === uid && entry.goalId !== undefined
|
|
2377
|
-
? await readGoalResult(await this.actionCtx(), { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId: entry.goalId })
|
|
2378
|
-
: undefined;
|
|
2379
|
-
const allocator = spawned?.committer?.instanceId;
|
|
2380
|
-
for (let attempt = 0; attempt <= MANAGER_ROUTE_ATTEMPTS; attempt += 1) {
|
|
2381
|
-
if (await decided())
|
|
2382
|
-
return false;
|
|
2383
|
-
let answer;
|
|
2384
|
-
try {
|
|
2385
|
-
answer = await this.invokeManager(await this.manager("inspect"), "inspect", { name }, {});
|
|
2386
|
-
}
|
|
2387
|
-
catch (e) {
|
|
2388
|
-
if (unansweredRequest(e))
|
|
2389
|
-
break;
|
|
2390
|
-
if (await decided())
|
|
2391
|
-
return false;
|
|
2392
|
-
throw e;
|
|
2393
|
-
}
|
|
2394
|
-
const { reply, responder } = answer;
|
|
2395
|
-
const row = reply.ok ? reply.data : undefined;
|
|
2396
|
-
if (row?.lifecycleUid === uid && row.status === "running")
|
|
2397
|
-
return false;
|
|
2398
|
-
if (responder.instanceId === allocator && !replyRefusedBeforeEffect(reply.error)) {
|
|
2399
|
-
if (reply.ok || reply.error?.code === "not-found")
|
|
2400
|
-
break;
|
|
2401
|
-
if (await decided())
|
|
2402
|
-
return false;
|
|
2403
|
-
throw new Error(`inspect(${name}) was refused by manager instance ${allocator}, which allocated ${name}#${uid}: ${reply.error?.message ?? "refused with no message"}`);
|
|
2404
|
-
}
|
|
2405
|
-
this.managerServices.delete("inspect");
|
|
2406
|
-
}
|
|
2407
|
-
return !(await decided());
|
|
2408
|
-
}
|
|
2409
|
-
/**
|
|
2410
|
-
* Arm this pause, or ATTACH to the one already recorded under the same token.
|
|
2411
|
-
*
|
|
2412
|
-
* A mint is idempotent only if the whole spec is identical, so the recorded spec is the
|
|
2413
|
-
* authority for BOTH halves of the mint's identity and a resume may not recompute either. The
|
|
2414
|
-
* deadline, because `now() + duration` is a different deadline a second later, and the plane
|
|
2415
|
-
* reads a different deadline as a different intent. And the HOLDER, because a resumed run does
|
|
2416
|
-
* not necessarily arrive under the principal that minted its pause: the CLI mints a fresh holder
|
|
2417
|
-
* per invocation, and a cross-host adoption is a different principal by definition. Only the
|
|
2418
|
-
* recorded holder itself may mint again — completing its own crash between the spec and the
|
|
2419
|
-
* status, where the identical spec is exactly what makes the retry idempotent. Everyone else
|
|
2420
|
-
* attaches. Measured before the repair, through the CLI: one `resume` of a checkpoint-parked
|
|
2421
|
-
* run re-minted under its own fresh holder, the plane refused ("a token is minted once"), and
|
|
2422
|
-
* the interpreter recorded that infrastructure refusal as the step's own failure (L4000) — a
|
|
2423
|
-
* stranded run whose journal blames the program.
|
|
2424
|
-
*
|
|
2425
|
-
* An already-passed deadline cannot be minted at all, correctly, because a due pause is not being
|
|
2426
|
-
* armed. It needs its schedule re-emitted at the status's current generation, which is the
|
|
2427
|
-
* reconciler's job — the same operation an attaching successor needs, so the two share the exit.
|
|
2428
|
-
* A first arm whose deadline has passed has nothing to attach to and raises `PauseOverdue`.
|
|
2429
|
-
* A spec with no status that this holder cannot complete is unrepairable from here, so it is
|
|
2430
|
-
* raised rather than waited on.
|
|
2431
|
-
*/
|
|
2432
|
-
async arm(ref, deadline) {
|
|
2433
|
-
// #1508: every read and write below rides a client-side deadline, and a deadline that elapsed
|
|
2434
|
-
// because THIS PROCESS was off the CPU is not evidence about the plane. See `host-starvation`.
|
|
2435
|
-
return await servedDespiteStarvation(() => this.armOnce(ref, deadline), this.lag, `arming the pause ${ref.token}`, this.onStarved);
|
|
2436
|
-
}
|
|
2437
|
-
/** `arm`, answering `true` when the deadline passed before anything was minted for it. Only a
|
|
2438
|
-
* first arm finds that (`PauseOverdue`), and there is then no pause to settle, so a caller whose
|
|
2439
|
-
* deadline is its own outcome takes `true` as that outcome. Cancellation still wins. */
|
|
2440
|
-
async armUnlessDue(ref, deadline, signal) {
|
|
2441
|
-
try {
|
|
2442
|
-
await this.arm(ref, deadline);
|
|
2443
|
-
return false;
|
|
2444
|
-
}
|
|
2445
|
-
catch (error) {
|
|
2446
|
-
if (error instanceof PauseOverdue) {
|
|
2447
|
-
if (signal.cancelled)
|
|
2448
|
-
throw new Cancelled(signal.reason ?? "cancelled");
|
|
2449
|
-
return true;
|
|
2450
|
-
}
|
|
2451
|
-
throw error;
|
|
2452
|
-
}
|
|
2453
|
-
}
|
|
2454
|
-
async armOnce(ref, deadline) {
|
|
2455
|
-
if (this.services)
|
|
2456
|
-
return this.services.pauses.arm(ref.token, deadline);
|
|
2457
|
-
// Over already: an expiry or an answer landed while this host was away. Nothing to arm, and the
|
|
2458
|
-
// caller reads the fact next.
|
|
2459
|
-
if ((await readCheckpointSettle(this.jsm, this.binding.space, ref)) !== undefined)
|
|
2460
|
-
return;
|
|
2461
|
-
const prior = await readCheckpointSpec(this.kv, ref);
|
|
2462
|
-
const now = this.now();
|
|
2463
|
-
if (prior === undefined) {
|
|
2464
|
-
// The first arm: this driver's own mint, under its own holder, at the deadline the caller
|
|
2465
|
-
// computed from the duration it was just handed.
|
|
2466
|
-
if (deadline <= now)
|
|
2467
|
-
throw new PauseOverdue(ref.token, deadline, now);
|
|
2468
|
-
await mintCheckpoint(this.kv, this.js, this.binding.space, {
|
|
2469
|
-
ref,
|
|
2470
|
-
instanceId: this.binding.instanceId,
|
|
2471
|
-
epoch: this.binding.epoch,
|
|
2472
|
-
holder: this.binding.holder,
|
|
2473
|
-
deadline,
|
|
2474
|
-
now,
|
|
2475
|
-
});
|
|
2476
|
-
return;
|
|
2477
|
-
}
|
|
2478
|
-
const mine = prior.holder.id === this.binding.holder.id
|
|
2479
|
-
&& prior.holder.lifecycleUid === this.binding.holder.lifecycleUid;
|
|
2480
|
-
if (mine && prior.initialDeadline > now) {
|
|
2481
|
-
// A retry of this holder's own mint: idempotent-if-identical, and the one path that can
|
|
2482
|
-
// repair a crash between the spec and the status, because only an identical spec completes.
|
|
2483
|
-
await mintCheckpoint(this.kv, this.js, this.binding.space, {
|
|
2484
|
-
ref,
|
|
2485
|
-
instanceId: this.binding.instanceId,
|
|
2486
|
-
epoch: this.binding.epoch,
|
|
2487
|
-
holder: this.binding.holder,
|
|
2488
|
-
deadline: prior.initialDeadline,
|
|
2489
|
-
now,
|
|
2490
|
-
});
|
|
2491
|
-
return;
|
|
2492
|
-
}
|
|
2493
|
-
// ATTACH: the pause exists and is not this attempt's to mint — another holder's pause, or this
|
|
2494
|
-
// holder's own come back overdue (a due pause is being collected, not armed). Both need the
|
|
2495
|
-
// same thing: the schedule re-emitted at THIS instance's coordinates, so the fire lands where
|
|
2496
|
-
// this driver is listening. The settle the caller waits on next is coordinate-free.
|
|
2497
|
-
const status = await readCheckpointStatus(this.kv, ref);
|
|
2498
|
-
if (status === undefined) {
|
|
2499
|
-
throw new Error(`checkpoint "${ref.token}" carries a spec with no status`
|
|
2500
|
-
+ `${mine ? "" : ` held by ${prior.holder.id}, and only its own holder's identical re-mint can repair a half-minted pause`}; `
|
|
2501
|
-
+ `this pause has to be reconciled on the plane before the run can go on`);
|
|
2502
|
-
}
|
|
2503
|
-
await reconcileCheckpointSchedule(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2504
|
-
ref,
|
|
2505
|
-
instanceId: this.binding.instanceId,
|
|
2506
|
-
epoch: this.binding.epoch,
|
|
2507
|
-
});
|
|
2508
|
-
}
|
|
2509
|
-
/** Push a live deadline out — the idle window restarting. The heartbeat CAS-advances the
|
|
2510
|
-
* generation before replacing the timer, so a fire from the old one no-ops rather than racing. */
|
|
2511
|
-
async push(ref, deadline) {
|
|
2512
|
-
if (this.services)
|
|
2513
|
-
return this.services.pauses.heartbeat(ref.token, deadline);
|
|
2514
|
-
await heartbeatCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2515
|
-
ref,
|
|
2516
|
-
instanceId: this.binding.instanceId,
|
|
2517
|
-
epoch: this.binding.epoch,
|
|
2518
|
-
deadline,
|
|
2519
|
-
now: this.now(),
|
|
2520
|
-
});
|
|
2521
|
-
}
|
|
2522
|
-
/**
|
|
2523
|
-
* Take the broker's `.fire` for this deadline, if it has published one, and let the checkpoint
|
|
2524
|
-
* plane judge it. Answers whether the fire SETTLED the pause.
|
|
2525
|
-
*
|
|
2526
|
-
* A fire is a MESSAGE and a settlement is a FACT; `handleCheckpointFire` is what turns one into
|
|
2527
|
-
* the other, and a pause nobody answers ends no other way.
|
|
2528
|
-
*
|
|
2529
|
-
* READ BY SUBJECT rather than by a subscription, because the fire is a record of a deadline
|
|
2530
|
-
* passing and callers here are already polling: `last_by_subj` gives the current one whether it
|
|
2531
|
-
* arrived a moment ago or while this host was down, and a resume lands in the second case.
|
|
2532
|
-
*
|
|
2533
|
-
* IDEMPOTENT BY THE PLANE rather than by bookkeeping here: a fire handed over twice finds the
|
|
2534
|
-
* pause settled and declines. The one repeated verdict with a side effect is `re-armed` under
|
|
2535
|
-
* owner-behind clock skew, and over-emission is idempotent at the timer writer.
|
|
2536
|
-
*/
|
|
2537
|
-
async takeFire(ref) {
|
|
2538
|
-
if (this.services)
|
|
2539
|
-
return this.services.pauses.takeFire(ref.token);
|
|
2540
|
-
const subject = eptSubject(this.binding.space, ref.endpoint, this.binding.instanceId, this.binding.epoch, ref.token, "fire");
|
|
2541
|
-
const fired = await this.jsm.streams
|
|
2542
|
-
.getMessage(eptStreamName(this.binding.space), { last_by_subj: subject })
|
|
2543
|
-
.catch((e) => {
|
|
2544
|
-
// 10037 is the stream saying there is no message on that subject, which IS the ordinary
|
|
2545
|
-
// "the timer has not fired yet" answer. Every other error is the broker refusing or
|
|
2546
|
-
// unreachable, and reading those as "not yet" is how a `wait` with a timeout outlives its
|
|
2547
|
-
// own durable deadline in silence, which is the failure this method exists to prevent.
|
|
2548
|
-
if (e?.code === 10037)
|
|
2549
|
-
return null;
|
|
2550
|
-
throw e;
|
|
2551
|
-
});
|
|
2552
|
-
if (fired === null || fired === undefined)
|
|
2553
|
-
return false;
|
|
2554
|
-
const verdict = await handleCheckpointFire(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2555
|
-
ref,
|
|
2556
|
-
instanceId: this.binding.instanceId,
|
|
2557
|
-
epoch: this.binding.epoch,
|
|
2558
|
-
msg: { subject, ...(fired.header !== undefined ? { headers: fired.header } : {}), data: fired.data },
|
|
2559
|
-
now: this.now(),
|
|
2560
|
-
});
|
|
2561
|
-
return verdict.acted;
|
|
2562
|
-
}
|
|
2563
|
-
/** Has this deadline settled? `undefined` for "no deadline" and for "not yet".
|
|
2564
|
-
*
|
|
2565
|
-
* A `wait` observes its deadlines by polling this, so the fire is taken on the same poll: the
|
|
2566
|
-
* settle fact a `wait` is reading for is one this process has to produce, and reading for it
|
|
2567
|
-
* without ever producing it is how a `wait` with a timeout waited past its timeout forever. */
|
|
2568
|
-
async expired(ref) {
|
|
2569
|
-
if (ref === undefined)
|
|
2570
|
-
return undefined;
|
|
2571
|
-
if (this.services) {
|
|
2572
|
-
const settled = await this.services.pauses.readSettle(ref.token);
|
|
2573
|
-
if (settled !== undefined)
|
|
2574
|
-
return settled;
|
|
2575
|
-
if (!(await this.services.pauses.takeFire(ref.token)))
|
|
2576
|
-
return undefined;
|
|
2577
|
-
return this.services.pauses.readSettle(ref.token);
|
|
2578
|
-
}
|
|
2579
|
-
const settled = await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
2580
|
-
if (settled !== undefined)
|
|
2581
|
-
return settled;
|
|
2582
|
-
if (!(await this.takeFire(ref)))
|
|
2583
|
-
return undefined;
|
|
2584
|
-
return await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
2585
|
-
}
|
|
2586
|
-
/** End a deadline that is no longer waited on, by claiming its one-use settlement with no answer.
|
|
2587
|
-
* A timer left armed would fire into a run that has moved on; claiming it is how the plane says
|
|
2588
|
-
* "this pause is over" without a second mechanism for cancellation. */
|
|
2589
|
-
async cancelTimer(ref) {
|
|
2590
|
-
if (this.services)
|
|
2591
|
-
return this.services.pauses.claim(ref.token);
|
|
2592
|
-
const st = await readCheckpointStatus(this.kv, ref);
|
|
2593
|
-
if (st?.value.state !== "waiting")
|
|
2594
|
-
return;
|
|
2595
|
-
try {
|
|
2596
|
-
// The presenter is the ARMING holder read off the pause's own record, as everywhere on this
|
|
2597
|
-
// plane: a timer minted by a predecessor is still this run's to end after a takeover, and a
|
|
2598
|
-
// successor presenting its own identity would be refused (resume is holder-bound, SPEC
|
|
2599
|
-
// 13.10). A spec that cannot be read leaves the timer to its own deadline, which the catch
|
|
2600
|
-
// below already tolerates: it fires, settles expired, and nobody is reading that token.
|
|
2601
|
-
const spec = await readCheckpointSpec(this.kv, ref);
|
|
2602
|
-
if (spec === undefined)
|
|
2603
|
-
return;
|
|
2604
|
-
await resumeCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2605
|
-
ref,
|
|
2606
|
-
presenter: spec.holder,
|
|
2607
|
-
now: this.now(),
|
|
2608
|
-
});
|
|
2609
|
-
}
|
|
2610
|
-
catch (e) {
|
|
2611
|
-
// IT SETTLED UNDERNEATH US is one outcome, and it is the expected one: the deadline won a
|
|
2612
|
-
// race it was always allowed to win, the caller has its answer either way, and a claimed
|
|
2613
|
-
// timer is not a fact anyone reads. A BROKER THAT REFUSED OR WENT AWAY is a different thing
|
|
2614
|
-
// wearing the same silence, and this catch used to swallow both. It still must not replace
|
|
2615
|
-
// the caller's answer — a timer left armed fires, settles expired, and nobody reads that
|
|
2616
|
-
// token — so the second one is loud on the way past instead.
|
|
2617
|
-
const settledAlready = e instanceof EpEnvelopeError
|
|
2618
|
-
&& (e.code === "conflict" || e.code === "failed-precondition");
|
|
2619
|
-
if (!settledAlready)
|
|
2620
|
-
console.error(`run ${this.binding.runId}: the pause ${ref.token} could not be claimed (${e.message}); it will fire and settle expired with nobody reading it`);
|
|
2621
|
-
}
|
|
2622
|
-
}
|
|
2623
|
-
/** The message at a recorded stream sequence — the re-bind path after a crash. */
|
|
2624
|
-
async messageAt(seq, requestId) {
|
|
2625
|
-
if (this.services) {
|
|
2626
|
-
const message = decodeMessage(await this.services.waits.messageAt(requestId, seq));
|
|
2627
|
-
if (message === undefined)
|
|
2628
|
-
throw new Error(`wait ${requestId}'s recorded message does not decode`);
|
|
2629
|
-
return message;
|
|
2630
|
-
}
|
|
2631
|
-
const m = await this.jsm.streams.getMessage(chatStream(this.binding.space), { seq });
|
|
2632
|
-
if (m === null || m === undefined) {
|
|
2633
|
-
throw new Error(`the message this wait matched (sequence ${seq}) is no longer on the channel's stream; a recorded match cannot be re-read`);
|
|
2634
|
-
}
|
|
2635
|
-
const msg = decodeMessage(m.data);
|
|
2636
|
-
if (msg === undefined)
|
|
2637
|
-
throw new Error(`the message at sequence ${seq} did not decode; a recorded match cannot be re-read`);
|
|
2638
|
-
return msg;
|
|
2639
|
-
}
|
|
2640
|
-
/**
|
|
2641
|
-
* Wait for this token's one-use settlement.
|
|
2642
|
-
*
|
|
2643
|
-
* A settle that ALREADY landed is the ordinary case on a resume: the run crashed while paused and
|
|
2644
|
-
* the timer fired, or somebody answered, without it. Reading before waiting is not an
|
|
2645
|
-
* optimization, it is the difference between resuming and waiting forever for an event that is
|
|
2646
|
-
* already in the past.
|
|
2647
|
-
*/
|
|
2648
|
-
async settle(ref, signal, relayed = false) {
|
|
2649
|
-
// #1508, and the load-bearing half: this is where a `sleep` spends its whole duration, so this
|
|
2650
|
-
// is where a starved host was reporting its own scheduling as the effect's failure. The pause
|
|
2651
|
-
// and its timer are durable facts on the plane; re-reading them observes the same world, and a
|
|
2652
|
-
// sleep whose deadline passed while this process was blocked settles `ok`, late.
|
|
2653
|
-
try {
|
|
2654
|
-
return await servedDespiteStarvation(() => this.settleOnce(ref, signal), this.lag, `waiting on the pause ${ref.token}`, this.onStarved);
|
|
2655
|
-
}
|
|
2656
|
-
catch (e) {
|
|
2657
|
-
// An `ask` attempt or an escalation may have relayed this token to a seat.
|
|
2658
|
-
if (e instanceof Cancelled && relayed)
|
|
2659
|
-
await this.withdrawCancelledRelay(ref.token);
|
|
2660
|
-
throw e;
|
|
2661
|
-
}
|
|
2662
|
-
}
|
|
2663
|
-
async settleOnce(ref, signal) {
|
|
2664
|
-
const already = this.services ? await this.services.pauses.readSettle(ref.token) : await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
2665
|
-
if (already !== undefined)
|
|
2666
|
-
return already;
|
|
2667
|
-
// The watcher waits for the FACT. For a pause with an answer the fact arrives because somebody
|
|
2668
|
-
// answered; for a pause nobody answers - a `sleep`, or a `checkpoint` whose timeout wins - it
|
|
2669
|
-
// arrives only because this process took the fire and the plane wrote it. So the two run
|
|
2670
|
-
// together for the life of one wait, and the pump ends when the wait does.
|
|
2671
|
-
const wait = { over: false };
|
|
2672
|
-
const pump = this.pumpFires(ref, wait);
|
|
2673
|
-
try {
|
|
2674
|
-
return await Promise.race([
|
|
2675
|
-
this.watcher.awaitSettle(ref),
|
|
2676
|
-
// A pump that ENDED is not an answer, only one that FAILED is: a failure means this process
|
|
2677
|
-
// cannot expire the pause, so it is raised rather than absorbed and the step stays pending.
|
|
2678
|
-
pump.then(() => new Promise(() => { })),
|
|
2679
|
-
// A cancelled branch stops parking NOW (§7.6): the rejection decides the race first, so a
|
|
2680
|
-
// `checkpoint` cannot mistake its own claim below for an answer, and then the claim ends
|
|
2681
|
-
// the pause in the world. The claim is not awaited here — a claim that loses its own race
|
|
2682
|
-
// is tolerated by `cancelTimer`, and the discharge sweep at the run's completion re-claims
|
|
2683
|
-
// idempotently for the case where this process died before the claim landed.
|
|
2684
|
-
...(signal === undefined ? [] : [this.settleCancelled(ref, signal)]),
|
|
2685
|
-
]);
|
|
2686
|
-
}
|
|
2687
|
-
finally {
|
|
2688
|
-
wait.over = true;
|
|
2689
|
-
// THE PUMP IS DRAINED BEFORE THE SETTLE RETURNS (#1460). The race above has already decided
|
|
2690
|
-
// — the fact, the failure, or the cancellation is this call's answer — but the pump's
|
|
2691
|
-
// in-flight `takeFire` is a journal replay under the run's takeover id, created, read and
|
|
2692
|
-
// deleted inside one `replayRunJournal` call with nothing serialising it across two
|
|
2693
|
-
// authorities for the same run and lease. A settle that returned past it let a completed
|
|
2694
|
-
// `driveRun` hold the replay open after it resolved, and the next reader under the same
|
|
2695
|
-
// takeover hit `RunJournalReplayRaced` about a driver that did not exist. So only the RETURN
|
|
2696
|
-
// is delayed, until the pump has ended: `wait.over` stops it after its current `takeFire`
|
|
2697
|
-
// without starting another, and awaiting it here means no pump fire is left in flight on any
|
|
2698
|
-
// exit. A pump that FAILED raises through this await on its way out, exactly as it did
|
|
2699
|
-
// through the race; the settle's own answer is decided either way and is what this returns.
|
|
2700
|
-
await pump;
|
|
2701
|
-
}
|
|
2702
|
-
}
|
|
2703
|
-
/** Reject with `Cancelled` the moment this branch's signal fires, then claim the pause so its
|
|
2704
|
-
* armed schedule cannot fire into a run that has moved on. The claim also writes the one-use
|
|
2705
|
-
* settle, which is what lets an abandoned `awaitSettle` poll loop see a fact and end. */
|
|
2706
|
-
settleCancelled(ref, signal) {
|
|
2707
|
-
return new Promise((_, reject) => {
|
|
2708
|
-
let fired = false;
|
|
2709
|
-
const fire = (reason) => {
|
|
2710
|
-
if (fired)
|
|
2711
|
-
return;
|
|
2712
|
-
fired = true;
|
|
2713
|
-
reject(new Cancelled(reason ?? "cancelled"));
|
|
2714
|
-
void this.cancelTimer(ref).catch(() => undefined);
|
|
2715
|
-
};
|
|
2716
|
-
if (signal.cancelled) {
|
|
2717
|
-
fire(signal.reason);
|
|
2718
|
-
return;
|
|
2719
|
-
}
|
|
2720
|
-
signal.onCancel(fire);
|
|
2721
|
-
});
|
|
2722
|
-
}
|
|
2723
|
-
/** Take this deadline's fire for as long as somebody is waiting on it. */
|
|
2724
|
-
async pumpFires(ref, wait) {
|
|
2725
|
-
while (!wait.over) {
|
|
2726
|
-
await this.takeFire(ref);
|
|
2727
|
-
// THE FLAG IS RE-READ AFTER THE FIRE, before the poll sleeps (#1460): `takeFire` is a journal
|
|
2728
|
-
// replay under the run's takeover id, so the wait can have ENDED while this one was in
|
|
2729
|
-
// flight — and a pump that slept another poll on a wait that is over keeps exactly one more
|
|
2730
|
-
// fire in flight past the drain `settleOnce` awaits. Checked here, the pump ends right after
|
|
2731
|
-
// its current fire without starting another.
|
|
2732
|
-
if (wait.over)
|
|
2733
|
-
break;
|
|
2734
|
-
await delay(FIRE_POLL_MS);
|
|
2735
|
-
}
|
|
2736
|
-
}
|
|
1726
|
+
/** The pause timers own no handler state: each input is read through the handler at its use. */
|
|
1727
|
+
pauseTimers = new PauseTimers({
|
|
1728
|
+
binding: () => this.binding, kv: () => this.kv, js: () => this.js, jsm: () => this.jsm, watcher: () => this.watcher,
|
|
1729
|
+
services: () => this.services, lag: () => this.lag, onStarved: (note) => this.onStarved(note), now: () => this.now(),
|
|
1730
|
+
withdrawCancelledRelay: (goalId, deadlineAt) => this.withdrawCancelledRelay(goalId, deadlineAt),
|
|
1731
|
+
});
|
|
1732
|
+
async arm(ref, deadline) { return this.pauseTimers.arm(ref, deadline); }
|
|
1733
|
+
async armUnlessDue(ref, deadline, signal) { return this.pauseTimers.armUnlessDue(ref, deadline, signal); }
|
|
1734
|
+
async push(ref, deadline) { return this.pauseTimers.push(ref, deadline); }
|
|
1735
|
+
async expired(ref) { return this.pauseTimers.expired(ref); }
|
|
1736
|
+
async cancelTimer(ref) { return this.pauseTimers.cancelTimer(ref); }
|
|
1737
|
+
async messageAt(seq, requestId) { return this.pauseTimers.messageAt(seq, requestId); }
|
|
1738
|
+
async settle(ref, signal, relayed = false) { return this.pauseTimers.settle(ref, signal, relayed); }
|
|
1739
|
+
/** The relay and presence watch own no handler state: each input, the handler's own members
|
|
1740
|
+
* included, is read through the handler at its use. */
|
|
1741
|
+
seatWatch = new SeatWatch({
|
|
1742
|
+
self: () => this,
|
|
1743
|
+
binding: () => this.binding,
|
|
1744
|
+
roster: () => this.roster,
|
|
1745
|
+
managerServices: () => this.managerServices,
|
|
1746
|
+
now: () => this.now(),
|
|
1747
|
+
actionCtx: () => this.actionCtx(),
|
|
1748
|
+
presenceRegistry: () => this.presenceRegistry(),
|
|
1749
|
+
seatAddress: (name, uid) => this.seatAddress(name, uid),
|
|
1750
|
+
manager: (command, instanceId) => this.manager(command, instanceId),
|
|
1751
|
+
invokeManager: (service, command, args, opts) => this.invokeManager(service, command, args, opts),
|
|
1752
|
+
relayManagerBeforeSubmit: (host, command, ref, kind, label) => this.relayManagerBeforeSubmit(host, command, ref, kind, label),
|
|
1753
|
+
});
|
|
1754
|
+
async relayEscalation(req, ctx, token, deadlineAt) { return this.seatWatch.relayEscalation(req, ctx, token, deadlineAt); }
|
|
1755
|
+
async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) { return this.seatWatch.relayToSeat(seat, goalId, payload, deadlineAt, kind, step); }
|
|
1756
|
+
async withdrawRelay(goalId) { return this.seatWatch.withdrawRelay(goalId); }
|
|
1757
|
+
async withdrawCancelledRelay(goalId, deadlineAt) { return this.seatWatch.withdrawCancelledRelay(goalId, deadlineAt); }
|
|
1758
|
+
async presenceRows() { return this.seatWatch.presenceRows(); }
|
|
1759
|
+
async presenceDeath(name, uid, watch, decided) { return this.seatWatch.presenceDeath(name, uid, watch, decided); }
|
|
1760
|
+
/** The turn and ask clients own no handler state: the run-memory maps they write are mutated in
|
|
1761
|
+
* place through getters, and each input, the handler's own members included, is read through
|
|
1762
|
+
* the handler at its use. */
|
|
1763
|
+
turnAsk = new TurnAsk({
|
|
1764
|
+
self: () => this,
|
|
1765
|
+
binding: () => this.binding,
|
|
1766
|
+
kv: () => this.kv,
|
|
1767
|
+
services: () => this.services,
|
|
1768
|
+
lag: () => this.lag,
|
|
1769
|
+
onStarved: () => this.onStarved,
|
|
1770
|
+
roster: () => this.roster,
|
|
1771
|
+
turnsTaken: () => this.turnsTaken,
|
|
1772
|
+
handoffMemos: () => this.handoffMemos,
|
|
1773
|
+
turnGoals: () => this.turnGoals,
|
|
1774
|
+
now: () => this.now(),
|
|
1775
|
+
actionCtx: () => this.actionCtx(),
|
|
1776
|
+
manager: (command, instanceId) => this.manager(command, instanceId),
|
|
1777
|
+
invokeManager: (service, command, args, opts) => this.invokeManager(service, command, args, opts),
|
|
1778
|
+
relayManagerBeforeSubmit: (host, command, ref, kind, label) => this.relayManagerBeforeSubmit(host, command, ref, kind, label),
|
|
1779
|
+
arm: (ref, deadline) => this.arm(ref, deadline),
|
|
1780
|
+
expired: (ref) => this.expired(ref),
|
|
1781
|
+
cancelTimer: (ref) => this.cancelTimer(ref),
|
|
1782
|
+
settle: (ref, signal, relayed) => this.settle(ref, signal, relayed),
|
|
1783
|
+
presenceDeath: (name, uid, watch, decided) => this.presenceDeath(name, uid, watch, decided),
|
|
1784
|
+
relayToSeat: (seat, goalId, payload, deadlineAt, kind, step) => this.relayToSeat(seat, goalId, payload, deadlineAt, kind, step),
|
|
1785
|
+
withdrawCancelledRelay: (goalId, deadlineAt) => this.withdrawCancelledRelay(goalId, deadlineAt),
|
|
1786
|
+
recordTurnGoal: (handle, goalId) => this.recordTurnGoal(handle, goalId),
|
|
1787
|
+
recordHandoffMemo: (scope, to, fromGoalId) => this.recordHandoffMemo(scope, to, fromGoalId),
|
|
1788
|
+
});
|
|
1789
|
+
async turn(req, ctx) { return this.turnAsk.turn(req, ctx); }
|
|
1790
|
+
async ask(req, ctx) { return this.turnAsk.ask(req, ctx); }
|
|
1791
|
+
handoffRefusal(fromWorktree, to) { return this.turnAsk.handoffRefusal(fromWorktree, to); }
|
|
1792
|
+
seatAddress(name, uid) { return this.turnAsk.seatAddress(name, uid); }
|
|
1793
|
+
waitSender(agent) { return this.turnAsk.waitSender(agent); }
|
|
2737
1794
|
}
|
|
2738
1795
|
export { waitConsumerName, waitConsumerConfig } from "@cotal-ai/core";
|
|
2739
1796
|
/** How long one poll of a wait's consumer blocks. The deadline itself is durable; this is only how
|
|
2740
1797
|
* late its observation can be, and a shorter poll buys latency at the cost of fetch traffic. */
|
|
2741
|
-
const WAIT_POLL_MS = 2_000;
|
|
2742
|
-
/**
|
|
2743
|
-
* How long an incarnation's presence row must stay absent before `turn` and `wait(down)` read it as
|
|
2744
|
-
* `lapsed`. A row is gone 6s after its last heartbeat, and a live seat whose connector stalls for
|
|
2745
|
-
* longer (host load, a reconnect) comes back under the same lifecycle uid on its next heartbeat. One
|
|
2746
|
-
* absent read failed a turn L4002 while the seat went on working (#2344). Five liveness windows lets
|
|
2747
|
-
* a stalled seat renew, and stays short against a turn's deadline. A `superseded` read needs no
|
|
2748
|
-
* wait: a different incarnation already holds the name. The window is measured from the end of the
|
|
2749
|
-
* read that opened it to the start of the read that closes it, over reads {@link lapseWindow} chains.
|
|
2750
|
-
*/
|
|
2751
|
-
const LAPSE_CONFIRM_MS = 30_000;
|
|
2752
|
-
/**
|
|
2753
|
-
* The longest span from the start of one presence read to the end of the next across which an
|
|
2754
|
-
* absent row still counts as continuously absent: the presence bucket's 6s TTL. A heartbeat keeps
|
|
2755
|
-
* the row live at least that long, so a renewal cannot fall between two reads that close. A slower
|
|
2756
|
-
* read or a run of failed scans leaves the gap unobserved, and the lapse window starts over.
|
|
2757
|
-
*/
|
|
2758
|
-
const PRESENCE_GAP_MS = 6_000;
|
|
2759
|
-
/** The start of the lapse window after an absent read that ended at `now`: kept when the previous
|
|
2760
|
-
* read began within {@link PRESENCE_GAP_MS}, otherwise this read opens a new one. */
|
|
2761
|
-
const lapseWindow = (lapsedSince, lastReadAt, now) => lapsedSince === undefined || now - lastReadAt > PRESENCE_GAP_MS ? now : lapsedSince;
|
|
1798
|
+
export const WAIT_POLL_MS = 2_000;
|
|
2762
1799
|
/** What an `expired` turn refusal shows. The placed host speaks for its own seat. A class-rail
|
|
2763
1800
|
* target-unmapped refusal says only that no member the call reached maps the seat: in a space with
|
|
2764
1801
|
* more than one manager a live seat on a member it missed answers the same as a dead one. */
|
|
2765
|
-
function seatDownReading(err, host) {
|
|
1802
|
+
export function seatDownReading(err, host) {
|
|
2766
1803
|
if (host !== undefined)
|
|
2767
1804
|
return `found the agent down on its placed host ${host}`;
|
|
2768
1805
|
return replyTargetUnmapped(err)
|
|
@@ -2857,20 +1894,14 @@ const SPAWN_ACCEPT_DEADLINE_MS = 30_000;
|
|
|
2857
1894
|
/** Total unanswered attempts per seat. Separate from class routing so an outage is bounded. */
|
|
2858
1895
|
const SPAWN_DISCHARGE_UNANSWERED_ATTEMPTS = 3;
|
|
2859
1896
|
/** Bound on the manager's synchronous `turn` ACCEPT reply (the relay registration, not the yield). */
|
|
2860
|
-
const TURN_ACCEPT_DEADLINE_MS = 30_000;
|
|
1897
|
+
export const TURN_ACCEPT_DEADLINE_MS = 30_000;
|
|
2861
1898
|
/** How many class-rail calls a despawn, a relay withdrawal or a lapse's inspect sends before it stops
|
|
2862
1899
|
* waiting for the one manager that holds its goal (the spawn's allocator, the relay's accepter) to
|
|
2863
1900
|
* answer. Each is one {@link MeshHandler.invokeManager} call, whose describe and invoke both land on
|
|
2864
1901
|
* that manager with probability 1/m^2 per trip in a space of m managers, so one call reaches it with
|
|
2865
1902
|
* probability (1 - ((m-1)/m)^17) / m: about 1/2 for m = 2 and 0.25 for m = 4. All 65 then miss
|
|
2866
1903
|
* with probability about 2^-65 and 9e-9. */
|
|
2867
|
-
const MANAGER_ROUTE_ATTEMPTS = 64;
|
|
2868
|
-
/** A step key's enclosing scope: the journal's own rendering (`entry.scope`), re-derived so the
|
|
2869
|
-
* live path and the adoption rebuild key the handoff memos identically. */
|
|
2870
|
-
function scopeOf(key) {
|
|
2871
|
-
const k = stepKeyString(key);
|
|
2872
|
-
return k.slice(0, k.lastIndexOf("/"));
|
|
2873
|
-
}
|
|
1904
|
+
export const MANAGER_ROUTE_ATTEMPTS = 64;
|
|
2874
1905
|
/** The terminal wait a discharge grants a goal whose entry recorded no readiness window (the
|
|
2875
1906
|
* crash-before-bind case). Matches the manager's default readiness budget. */
|
|
2876
1907
|
const DISCHARGE_TERMINAL_BOUND_MS = 30_000;
|
|
@@ -3040,7 +2071,7 @@ function isConclavePlan(v) {
|
|
|
3040
2071
|
}
|
|
3041
2072
|
/** Split an agent handle `<name>#<lifecycleUid>` on its LAST `#` — the uid alphabet
|
|
3042
2073
|
* (`[a-z0-9]{26,32}`) cannot carry one, a name could. */
|
|
3043
|
-
function parseAgentHandle(agent) {
|
|
2074
|
+
export function parseAgentHandle(agent) {
|
|
3044
2075
|
const i = agent.lastIndexOf("#");
|
|
3045
2076
|
if (i <= 0 || i === agent.length - 1)
|
|
3046
2077
|
throw new Error(`"${agent}" is not an agent handle of the form <name>#<lifecycleUid>`);
|
|
@@ -3065,39 +2096,6 @@ function assertConclaveChannel(channel) {
|
|
|
3065
2096
|
throw new Error(`conclave channel "${channel}" is a wildcard; a conclave joins its members to one concrete channel`);
|
|
3066
2097
|
return channel;
|
|
3067
2098
|
}
|
|
3068
|
-
/** How often a pause that nobody will answer looks for the broker's fire. Same argument as
|
|
3069
|
-
* `WAIT_POLL_MS`: the deadline is durable and this is only how late its observation can be. */
|
|
3070
|
-
const FIRE_POLL_MS = 2_000;
|
|
3071
|
-
/** The recorded ask progress a resume re-enters at, or undefined for a fresh first attempt. The
|
|
3072
|
-
* external is bytes from an earlier process, so the shape is checked rather than trusted. */
|
|
3073
|
-
function askResume(v) {
|
|
3074
|
-
if (v === undefined)
|
|
3075
|
-
return undefined;
|
|
3076
|
-
if (!(typeof v.attempt === "number" && v.attempt >= 1 && typeof v.deadlineAt === "number"))
|
|
3077
|
-
return undefined;
|
|
3078
|
-
// The refusal is bound with the attempt it belongs to. A resume that dropped it would re-ask the
|
|
3079
|
-
// seat with no reason its last answer failed, which is the one thing the re-ask exists to say.
|
|
3080
|
-
return {
|
|
3081
|
-
attempt: v.attempt,
|
|
3082
|
-
deadlineAt: v.deadlineAt,
|
|
3083
|
-
...(typeof v.refused === "string" ? { refused: v.refused } : {}),
|
|
3084
|
-
};
|
|
3085
|
-
}
|
|
3086
|
-
/** WHY a reply does not conform, per declared field — the refusal an answerer reads off the entry
|
|
3087
|
-
* before answering again. Judged with the same single-field check that judged the reply, so the
|
|
3088
|
-
* description can never disagree with the verdict. */
|
|
3089
|
-
function askNonconformance(value, shape) {
|
|
3090
|
-
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
3091
|
-
return "the reply is not a record";
|
|
3092
|
-
const record = value;
|
|
3093
|
-
const bad = [];
|
|
3094
|
-
for (const [field, kind] of Object.entries(shape)) {
|
|
3095
|
-
if (conformsToAskSchema({ [field]: record[field] }, { [field]: kind }))
|
|
3096
|
-
continue;
|
|
3097
|
-
bad.push(field in record ? `"${field}" wants ${kind}` : `"${field}" is missing (wants ${kind})`);
|
|
3098
|
-
}
|
|
3099
|
-
return bad.join("; ");
|
|
3100
|
-
}
|
|
3101
2099
|
/**
|
|
3102
2100
|
* A `matches` pattern, admitted through the repo's bounded-regex subset before it is compiled.
|
|
3103
2101
|
*
|
|
@@ -3128,17 +2126,6 @@ function isWaitEvent(ev) {
|
|
|
3128
2126
|
return typeof e.channel === "string";
|
|
3129
2127
|
return (e.event === "replied" || e.event === "down") && typeof e.agent === "string";
|
|
3130
2128
|
}
|
|
3131
|
-
function decodeMessage(data) {
|
|
3132
|
-
try {
|
|
3133
|
-
const v = JSON.parse(new TextDecoder().decode(data));
|
|
3134
|
-
return v !== null && typeof v === "object" ? v : undefined;
|
|
3135
|
-
}
|
|
3136
|
-
catch {
|
|
3137
|
-
// Someone else's malformed publish is not this run's failure. It does not match, and the wait
|
|
3138
|
-
// goes on waiting — which is what would happen if the message had never been sent.
|
|
3139
|
-
return undefined;
|
|
3140
|
-
}
|
|
3141
|
-
}
|
|
3142
2129
|
/** Does this message answer the await? It must come from the principal its subject names (SPEC
|
|
3143
2130
|
* §5), and when the wait names a `sender` principal, the subject's sender must be that principal.
|
|
3144
2131
|
* The payload's `from.name` is whatever the sender wrote, so it is never compared. */
|