@cotal-ai/runtime 0.168.0 → 0.170.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/mesh-handler.d.ts +38 -153
- package/dist/mesh-handler.d.ts.map +1 -1
- package/dist/mesh-handler.js +41 -624
- package/dist/mesh-handler.js.map +1 -1
- package/dist/mesh-pause-timers.d.ts +119 -0
- package/dist/mesh-pause-timers.d.ts.map +1 -0
- package/dist/mesh-pause-timers.js +369 -0
- package/dist/mesh-pause-timers.js.map +1 -0
- package/dist/mesh-seat-watch.d.ts +142 -0
- package/dist/mesh-seat-watch.d.ts.map +1 -0
- package/dist/mesh-seat-watch.js +309 -0
- package/dist/mesh-seat-watch.js.map +1 -0
- package/dist/offline-command.d.ts.map +1 -1
- package/dist/offline-command.js +24 -22
- package/dist/offline-command.js.map +1 -1
- package/dist/run-command.d.ts.map +1 -1
- package/dist/run-command.js +21 -27
- package/dist/run-command.js.map +1 -1
- package/dist/run-driver.d.ts +1 -0
- package/dist/run-driver.d.ts.map +1 -1
- package/dist/run-driver.js +20 -15
- package/dist/run-driver.js.map +1 -1
- package/dist/run-host.d.ts.map +1 -1
- package/dist/run-host.js +2 -1
- package/dist/run-host.js.map +1 -1
- package/package.json +4 -4
package/dist/mesh-handler.js
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
import { createHash } from "node:crypto";
|
|
20
20
|
import { isAbsolute } from "node:path";
|
|
21
21
|
import { realpathSync } from "node:fs";
|
|
22
|
-
import {
|
|
22
|
+
import { readCheckpointSettle, readCheckpointAnswer, reconcileCheckpointSchedule, chatStream, waitConsumerName, waitConsumerConfig, isConcreteChannel, assertSafePattern, assertValidChannel, presenceBucket, principalKey, parseSubject, isFromSubjectSender, IncompleteKvScan, openMembersRegistry, openChannelRegistry, openAclRegistry, readAcl, channelInAllow, memberKey, writeChannelConfig, readMember, commitMemberRevision, tombstoneMember, StaleMembershipWrite, runNoticeId, writeRunNotice, actionContext, invokeCommand, invokeRepairingSplit, replyRefusedBeforeEffect, readGoalResult, readGoalStatus, resolveService, listRunNotices, listRunMigrations, markRunNoticeConsumed, listRunNoticesForRun, EpEnvelopeError, replyTargetUnmapped, unansweredRequest, assertAdmittedPublish, assertAdmittedSubscribe, isStreamNotFound, } from "@cotal-ai/core";
|
|
23
23
|
import { renderRunContext } from "./run-context.js";
|
|
24
24
|
import { migrationSeats } from "./migrate.js";
|
|
25
25
|
import { Kvm } from "@nats-io/kv";
|
|
@@ -28,6 +28,8 @@ import { PauseOverdue } from "./run-pause-host.js";
|
|
|
28
28
|
import { loopLag, servedDespiteStarvation } from "./host-starvation.js";
|
|
29
29
|
import { askAttemptToken, observeToken, pauseTokens, waitTimeoutToken, waitTimers } from "./pause-tokens.js";
|
|
30
30
|
import { delay } from "./delay.js";
|
|
31
|
+
import { FIRE_POLL_MS, PauseTimers, decodeMessage } from "./mesh-pause-timers.js";
|
|
32
|
+
import { SeatWatch } from "./mesh-seat-watch.js";
|
|
31
33
|
/**
|
|
32
34
|
* A checkpoint resumed with no answer to read.
|
|
33
35
|
*
|
|
@@ -1926,161 +1928,6 @@ export class MeshHandler {
|
|
|
1926
1928
|
});
|
|
1927
1929
|
await this.relayToSeat(seat, token, payload, ask.deadlineAt, "ask", step);
|
|
1928
1930
|
}
|
|
1929
|
-
/**
|
|
1930
|
-
* An escalated checkpoint, delivered to the agent it names.
|
|
1931
|
-
*
|
|
1932
|
-
* `to` is legal only with `onExpiry: "escalate"` and the escalation is the SECOND mint, so this
|
|
1933
|
-
* runs once per chain at most. The addressee is resolved through the run's own roster: a name
|
|
1934
|
-
* this run spawned is a seat and is told, and anything else is a person — the pause is
|
|
1935
|
-
* answerable from anywhere by design, its addressee is on the entry, and refusing a program
|
|
1936
|
-
* that escalates to a human would be this host narrowing an option the reference leaves open.
|
|
1937
|
-
*
|
|
1938
|
-
* The relay is the ask's, because the two are the same act: one turn on the seat, carrying what
|
|
1939
|
-
* is asked and the token to answer under, settling on the checkpoint plane rather than on the
|
|
1940
|
-
* relay's own terminal.
|
|
1941
|
-
*/
|
|
1942
|
-
async relayEscalation(req, ctx, token, deadlineAt) {
|
|
1943
|
-
const to = req.to;
|
|
1944
|
-
const step = stepKeyString(ctx.key);
|
|
1945
|
-
// A seat is a roster entry whose acceptance floor was served: that is what carries the
|
|
1946
|
-
// owner/actor coordinates a relay is addressed by. Anything else — a name this run never
|
|
1947
|
-
// spawned, or one whose floor never landed — is not a seat, and the escalation stays the
|
|
1948
|
-
// durable pause it already is, addressed on its entry and answerable from anywhere.
|
|
1949
|
-
const seat = this.seatAddress(to);
|
|
1950
|
-
if (seat.address === undefined)
|
|
1951
|
-
return;
|
|
1952
|
-
const context = renderRunContext({ run: this.binding.runId, step, notices: [] });
|
|
1953
|
-
const payload = JSON.stringify({
|
|
1954
|
-
run: this.binding.runId, step, context, noticeIds: [],
|
|
1955
|
-
checkpoint: { token, prompt: req.prompt, schema: req.schema ?? null, deadlineAt, escalatedTo: to },
|
|
1956
|
-
});
|
|
1957
|
-
try {
|
|
1958
|
-
await this.relayToSeat(seat.address, token, payload, deadlineAt, "checkpoint", step);
|
|
1959
|
-
}
|
|
1960
|
-
catch (e) {
|
|
1961
|
-
// THE PAUSE OUTLIVES ITS ADDRESSEE. The reference defines no failure for an escalation whose
|
|
1962
|
-
// addressee cannot be reached: it "mints exactly one further checkpoint addressed to `to`",
|
|
1963
|
-
// and a checkpoint is "a durable pause a human or an agent resolves from anywhere". A seat
|
|
1964
|
-
// the endpoint reports gone is therefore the same case as a `to` that names a person: nobody
|
|
1965
|
-
// is told, the addressee is on the entry, and anyone may still answer. L4002 is the turn's
|
|
1966
|
-
// vocabulary, and this is not a turn the program asked for. Said once, not swallowed.
|
|
1967
|
-
if (!(e instanceof EffectError && e.code === "L4002"))
|
|
1968
|
-
throw e;
|
|
1969
|
-
console.error(`run ${this.binding.runId}: the escalation at ${step} could not be told to ${to} (${e.message}); the pause stands, answerable from anywhere`);
|
|
1970
|
-
}
|
|
1971
|
-
}
|
|
1972
|
-
/**
|
|
1973
|
-
* Submit one relay to a seat as a `turn` goal, and read its acceptance.
|
|
1974
|
-
*
|
|
1975
|
-
* IDEMPOTENT BY THE GOAL RECORD, which is also the arbiter when the invoke does not come back:
|
|
1976
|
-
* a relay that landed and lost its reply is a relay that landed, and re-submitting it would put
|
|
1977
|
-
* a second turn on the seat for one request. A seat the serve boundary reports gone is the
|
|
1978
|
-
* agent-down failure (L4002); every other refusal is this endpoint's and is uncatchable.
|
|
1979
|
-
*
|
|
1980
|
-
* The relay carries the pause's own `deadlineAt` and the manager holds it to that instant: a
|
|
1981
|
-
* duration counted from the manager's acceptance would outlive the pause by the submit's trip
|
|
1982
|
-
* plus skew and show the seat an attempt whose answer is refused. A relay the manager refuses as
|
|
1983
|
-
* already past, and that is past on this clock too, has nobody left to tell, because the pause
|
|
1984
|
-
* denies at that same instant; the arm that follows meets the passed deadline and decides the
|
|
1985
|
-
* outcome.
|
|
1986
|
-
*/
|
|
1987
|
-
async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) {
|
|
1988
|
-
const { name, uid } = seat;
|
|
1989
|
-
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
1990
|
-
const actx = await this.actionCtx();
|
|
1991
|
-
if ((await readGoalStatus(actx, ref)) !== undefined)
|
|
1992
|
-
return;
|
|
1993
|
-
const service = await this.relayManagerBeforeSubmit(seat.host, "turn", ref, kind, `${kind}(${step})`);
|
|
1994
|
-
let reply;
|
|
1995
|
-
try {
|
|
1996
|
-
reply = await this.invokeManager(service, "turn", { payload, deadlineAt }, {
|
|
1997
|
-
id: goalId,
|
|
1998
|
-
deadlineMs: TURN_ACCEPT_DEADLINE_MS,
|
|
1999
|
-
target: { mode: "owner", owner: seat.owner, actor: seat.actor, lifecycleUid: uid },
|
|
2000
|
-
});
|
|
2001
|
-
}
|
|
2002
|
-
catch (err) {
|
|
2003
|
-
// The invoke did not come back — the goal record is the arbiter, exactly as in `turn`.
|
|
2004
|
-
if ((await readGoalStatus(actx, ref)) === undefined)
|
|
2005
|
-
throw err;
|
|
2006
|
-
return;
|
|
2007
|
-
}
|
|
2008
|
-
if (reply.reply.ok === false) {
|
|
2009
|
-
const err = reply.reply.error;
|
|
2010
|
-
if (err.code === "expired")
|
|
2011
|
-
throw new EffectError("L4002", kind, `${kind}(${step}): ${name}#${uid} ${seatDownReading(err, seat.host)} before its relay began: ${err.message}`);
|
|
2012
|
-
if (err.code === "deadline-exceeded" && this.now() >= deadlineAt)
|
|
2013
|
-
return;
|
|
2014
|
-
throw new Error(`${kind}(${step}) was refused by the ${this.binding.endpoint} endpoint: ${err.message}`);
|
|
2015
|
-
}
|
|
2016
|
-
}
|
|
2017
|
-
/**
|
|
2018
|
-
* Withdraw the relay a cancelled step sent to a seat, through the manager's reserved `cancel`
|
|
2019
|
-
* (SPEC 13.6 item 4), so the seat is not shown it again. A token no relay rode has no goal and
|
|
2020
|
-
* nothing to withdraw, and a relay that already ended is the outcome either way: the manager
|
|
2021
|
-
* refuses it with the cached terminal, so a repeated withdrawal is a no-op. Any other refusal
|
|
2022
|
-
* throws under the code it was refused with, which leaves a discharge open to retry.
|
|
2023
|
-
*
|
|
2024
|
-
* Only the manager that accepted the relay holds it, and the class rail reaches any member, so a
|
|
2025
|
-
* refusal is final only from the accepter its index entry names. One from another member is
|
|
2026
|
-
* re-issued on the class rail, as the spawn discharge's despawn is.
|
|
2027
|
-
*/
|
|
2028
|
-
async withdrawRelay(goalId) {
|
|
2029
|
-
const ref = { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId };
|
|
2030
|
-
const actx = await this.actionCtx();
|
|
2031
|
-
if ((await readGoalStatus(actx, ref)) === undefined)
|
|
2032
|
-
return;
|
|
2033
|
-
const accepter = (await readGoalIndex(actx, ref))?.iid;
|
|
2034
|
-
for (let attempt = 0;; attempt += 1) {
|
|
2035
|
-
const reply = await this.invokeManager(await this.manager("cancel"), "cancel", { goalId }, { deadlineMs: TURN_ACCEPT_DEADLINE_MS });
|
|
2036
|
-
if (reply.reply.ok !== false || (await readGoalResult(actx, ref)) !== undefined)
|
|
2037
|
-
return;
|
|
2038
|
-
if (accepter === undefined || reply.responder.instanceId === accepter) {
|
|
2039
|
-
const err = reply.reply.error;
|
|
2040
|
-
throw new EpEnvelopeError(err.code, `the relay "${goalId}" could not be withdrawn from its seat: ${err.message}`);
|
|
2041
|
-
}
|
|
2042
|
-
if (attempt === MANAGER_ROUTE_ATTEMPTS)
|
|
2043
|
-
throw new Error(`the relay "${goalId}" was accepted by manager instance ${accepter}, but none of ${attempt + 1} cancels was answered by it; the discharge stays open to retry`);
|
|
2044
|
-
this.managerServices.delete("cancel");
|
|
2045
|
-
}
|
|
2046
|
-
}
|
|
2047
|
-
/**
|
|
2048
|
-
* Withdraw a cancelled step's relay before the cancellation leaves the step, so the scope it
|
|
2049
|
-
* settles shows the seat nothing the run withdrew. A withdrawal that fails is tried again rather
|
|
2050
|
-
* than left to the discharge, which runs only when the run ends: until then the relay would stay
|
|
2051
|
-
* first on the seat and hold back the run's next turn to it. The retries stop when the relay's
|
|
2052
|
-
* deadline passes, because the manager no longer serves it then, and a failure that outlasts it
|
|
2053
|
-
* is the step's error. A refusal no retry can change is the step's error at once, since retrying
|
|
2054
|
-
* it would only hold the cancelled step in its scope until that deadline: of the refusals, only
|
|
2055
|
-
* `unavailable` (a manager still reconciling at boot) and `deadline-exceeded` (a lost reply) are
|
|
2056
|
-
* tried again.
|
|
2057
|
-
*
|
|
2058
|
-
* That deadline is read from the relay's goal, where the accepting manager recorded the instant
|
|
2059
|
-
* it stops serving the relay; a withdrawal holds only the goal id. No deadline is known until that
|
|
2060
|
-
* goal is read, and the relay may be served until then, so a failed read is tried again as well.
|
|
2061
|
-
* A goal record that can never yield a deadline is thrown.
|
|
2062
|
-
*/
|
|
2063
|
-
async withdrawCancelledRelay(goalId, deadlineAt) {
|
|
2064
|
-
for (;;) {
|
|
2065
|
-
try {
|
|
2066
|
-
if (deadlineAt === undefined) {
|
|
2067
|
-
const spec = (await readGoalSpec(await this.actionCtx(), { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId }))?.value;
|
|
2068
|
-
if (spec === undefined)
|
|
2069
|
-
return;
|
|
2070
|
-
if (spec.readinessDeadlineMs === undefined)
|
|
2071
|
-
throw new EpEnvelopeError("internal", `the relay "${goalId}" was accepted with no deadline on its goal; a garbled acceptance never authorizes (SPEC 13.6)`);
|
|
2072
|
-
deadlineAt = spec.acceptedAt + spec.readinessDeadlineMs;
|
|
2073
|
-
}
|
|
2074
|
-
return await this.withdrawRelay(goalId);
|
|
2075
|
-
}
|
|
2076
|
-
catch (e) {
|
|
2077
|
-
const final = e instanceof EpEnvelopeError && e.code !== "unavailable" && e.code !== "deadline-exceeded";
|
|
2078
|
-
if (deadlineAt === undefined ? e instanceof EpEnvelopeError : final || this.now() >= deadlineAt)
|
|
2079
|
-
throw e;
|
|
2080
|
-
}
|
|
2081
|
-
await delay(WAIT_POLL_MS);
|
|
2082
|
-
}
|
|
2083
|
-
}
|
|
2084
1931
|
/**
|
|
2085
1932
|
* `monitor` registers interest: after it, `down(agent)` is an event a branch can await (§5.9).
|
|
2086
1933
|
*
|
|
@@ -2304,465 +2151,49 @@ export class MeshHandler {
|
|
|
2304
2151
|
await (await this.channelRegistry()).delete(plan.channel);
|
|
2305
2152
|
}
|
|
2306
2153
|
}
|
|
2307
|
-
/**
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
// as a false DOWN, and the caller's deadline still ends the wait.
|
|
2342
|
-
if (e instanceof IncompleteKvScan)
|
|
2343
|
-
return undefined;
|
|
2344
|
-
throw e;
|
|
2345
|
-
}
|
|
2346
|
-
if (!rows.some((p) => p.card.name === name && p.lifecycleUid === uid)) {
|
|
2347
|
-
const reason = rows.some((p) => p.card.name === name) ? "superseded" : "lapsed";
|
|
2348
|
-
watch.lapsedSince = lapseWindow(watch.lapsedSince, watch.lastReadAt ?? 0, this.now());
|
|
2349
|
-
if (reason === "superseded" || readAt - watch.lapsedSince >= LAPSE_CONFIRM_MS) {
|
|
2350
|
-
if (reason === "lapsed" && !(await this.lapseStands(name, uid, decided)))
|
|
2351
|
-
watch.lapsedSince = undefined;
|
|
2352
|
-
else {
|
|
2353
|
-
watch.lastReadAt = readAt;
|
|
2354
|
-
return reason;
|
|
2355
|
-
}
|
|
2356
|
-
}
|
|
2357
|
-
}
|
|
2358
|
-
else
|
|
2359
|
-
watch.lapsedSince = undefined;
|
|
2360
|
-
watch.lastReadAt = readAt;
|
|
2361
|
-
return undefined;
|
|
2362
|
-
}
|
|
2363
|
-
/** Whether a lapse of `name#uid` confirmed from presence stands as the death, asked of the manager
|
|
2364
|
-
* first. A lapse is only the seat's heartbeats going quiet: on a loaded host a live seat's presence
|
|
2365
|
-
* writer stalls past {@link LAPSE_CONFIRM_MS} while its process goes on working (#2807), and the
|
|
2366
|
-
* manager owns that process. Only the manager holding the seat has a running row under its
|
|
2367
|
-
* incarnation, so that row keeps it alive whoever reports it. A miss decides only from the manager
|
|
2368
|
-
* that allocated it, the committer of its spawn terminal: another member of the class rail knows
|
|
2369
|
-
* only its own seats, so the class rail is asked again, as a spawn's discharge asks it. An allocator
|
|
2370
|
-
* that does not answer leaves presence as the only witness, which is the case presence is the
|
|
2371
|
-
* authority for: the manager died with the seat. The asking can outlast the caller's own
|
|
2372
|
-
* cancellation, deadline or terminal, so once `decided` holds nothing stands and no refusal is
|
|
2373
|
-
* raised: that authority decides at the caller's next check. */
|
|
2374
|
-
async lapseStands(name, uid, decided) {
|
|
2375
|
-
const entry = this.roster.get(name);
|
|
2376
|
-
const spawned = entry?.uid === uid && entry.goalId !== undefined
|
|
2377
|
-
? await readGoalResult(await this.actionCtx(), { endpoint: this.binding.endpoint, caller: this.binding.caller, goalId: entry.goalId })
|
|
2378
|
-
: undefined;
|
|
2379
|
-
const allocator = spawned?.committer?.instanceId;
|
|
2380
|
-
for (let attempt = 0; attempt <= MANAGER_ROUTE_ATTEMPTS; attempt += 1) {
|
|
2381
|
-
if (await decided())
|
|
2382
|
-
return false;
|
|
2383
|
-
let answer;
|
|
2384
|
-
try {
|
|
2385
|
-
answer = await this.invokeManager(await this.manager("inspect"), "inspect", { name }, {});
|
|
2386
|
-
}
|
|
2387
|
-
catch (e) {
|
|
2388
|
-
if (unansweredRequest(e))
|
|
2389
|
-
break;
|
|
2390
|
-
if (await decided())
|
|
2391
|
-
return false;
|
|
2392
|
-
throw e;
|
|
2393
|
-
}
|
|
2394
|
-
const { reply, responder } = answer;
|
|
2395
|
-
const row = reply.ok ? reply.data : undefined;
|
|
2396
|
-
if (row?.lifecycleUid === uid && row.status === "running")
|
|
2397
|
-
return false;
|
|
2398
|
-
if (responder.instanceId === allocator && !replyRefusedBeforeEffect(reply.error)) {
|
|
2399
|
-
if (reply.ok || reply.error?.code === "not-found")
|
|
2400
|
-
break;
|
|
2401
|
-
if (await decided())
|
|
2402
|
-
return false;
|
|
2403
|
-
throw new Error(`inspect(${name}) was refused by manager instance ${allocator}, which allocated ${name}#${uid}: ${reply.error?.message ?? "refused with no message"}`);
|
|
2404
|
-
}
|
|
2405
|
-
this.managerServices.delete("inspect");
|
|
2406
|
-
}
|
|
2407
|
-
return !(await decided());
|
|
2408
|
-
}
|
|
2409
|
-
/**
|
|
2410
|
-
* Arm this pause, or ATTACH to the one already recorded under the same token.
|
|
2411
|
-
*
|
|
2412
|
-
* A mint is idempotent only if the whole spec is identical, so the recorded spec is the
|
|
2413
|
-
* authority for BOTH halves of the mint's identity and a resume may not recompute either. The
|
|
2414
|
-
* deadline, because `now() + duration` is a different deadline a second later, and the plane
|
|
2415
|
-
* reads a different deadline as a different intent. And the HOLDER, because a resumed run does
|
|
2416
|
-
* not necessarily arrive under the principal that minted its pause: the CLI mints a fresh holder
|
|
2417
|
-
* per invocation, and a cross-host adoption is a different principal by definition. Only the
|
|
2418
|
-
* recorded holder itself may mint again — completing its own crash between the spec and the
|
|
2419
|
-
* status, where the identical spec is exactly what makes the retry idempotent. Everyone else
|
|
2420
|
-
* attaches. Measured before the repair, through the CLI: one `resume` of a checkpoint-parked
|
|
2421
|
-
* run re-minted under its own fresh holder, the plane refused ("a token is minted once"), and
|
|
2422
|
-
* the interpreter recorded that infrastructure refusal as the step's own failure (L4000) — a
|
|
2423
|
-
* stranded run whose journal blames the program.
|
|
2424
|
-
*
|
|
2425
|
-
* An already-passed deadline cannot be minted at all, correctly, because a due pause is not being
|
|
2426
|
-
* armed. It needs its schedule re-emitted at the status's current generation, which is the
|
|
2427
|
-
* reconciler's job — the same operation an attaching successor needs, so the two share the exit.
|
|
2428
|
-
* A first arm whose deadline has passed has nothing to attach to and raises `PauseOverdue`.
|
|
2429
|
-
* A spec with no status that this holder cannot complete is unrepairable from here, so it is
|
|
2430
|
-
* raised rather than waited on.
|
|
2431
|
-
*/
|
|
2432
|
-
async arm(ref, deadline) {
|
|
2433
|
-
// #1508: every read and write below rides a client-side deadline, and a deadline that elapsed
|
|
2434
|
-
// because THIS PROCESS was off the CPU is not evidence about the plane. See `host-starvation`.
|
|
2435
|
-
return await servedDespiteStarvation(() => this.armOnce(ref, deadline), this.lag, `arming the pause ${ref.token}`, this.onStarved);
|
|
2436
|
-
}
|
|
2437
|
-
/** `arm`, answering `true` when the deadline passed before anything was minted for it. Only a
|
|
2438
|
-
* first arm finds that (`PauseOverdue`), and there is then no pause to settle, so a caller whose
|
|
2439
|
-
* deadline is its own outcome takes `true` as that outcome. Cancellation still wins. */
|
|
2440
|
-
async armUnlessDue(ref, deadline, signal) {
|
|
2441
|
-
try {
|
|
2442
|
-
await this.arm(ref, deadline);
|
|
2443
|
-
return false;
|
|
2444
|
-
}
|
|
2445
|
-
catch (error) {
|
|
2446
|
-
if (error instanceof PauseOverdue) {
|
|
2447
|
-
if (signal.cancelled)
|
|
2448
|
-
throw new Cancelled(signal.reason ?? "cancelled");
|
|
2449
|
-
return true;
|
|
2450
|
-
}
|
|
2451
|
-
throw error;
|
|
2452
|
-
}
|
|
2453
|
-
}
|
|
2454
|
-
async armOnce(ref, deadline) {
|
|
2455
|
-
if (this.services)
|
|
2456
|
-
return this.services.pauses.arm(ref.token, deadline);
|
|
2457
|
-
// Over already: an expiry or an answer landed while this host was away. Nothing to arm, and the
|
|
2458
|
-
// caller reads the fact next.
|
|
2459
|
-
if ((await readCheckpointSettle(this.jsm, this.binding.space, ref)) !== undefined)
|
|
2460
|
-
return;
|
|
2461
|
-
const prior = await readCheckpointSpec(this.kv, ref);
|
|
2462
|
-
const now = this.now();
|
|
2463
|
-
if (prior === undefined) {
|
|
2464
|
-
// The first arm: this driver's own mint, under its own holder, at the deadline the caller
|
|
2465
|
-
// computed from the duration it was just handed.
|
|
2466
|
-
if (deadline <= now)
|
|
2467
|
-
throw new PauseOverdue(ref.token, deadline, now);
|
|
2468
|
-
await mintCheckpoint(this.kv, this.js, this.binding.space, {
|
|
2469
|
-
ref,
|
|
2470
|
-
instanceId: this.binding.instanceId,
|
|
2471
|
-
epoch: this.binding.epoch,
|
|
2472
|
-
holder: this.binding.holder,
|
|
2473
|
-
deadline,
|
|
2474
|
-
now,
|
|
2475
|
-
});
|
|
2476
|
-
return;
|
|
2477
|
-
}
|
|
2478
|
-
const mine = prior.holder.id === this.binding.holder.id
|
|
2479
|
-
&& prior.holder.lifecycleUid === this.binding.holder.lifecycleUid;
|
|
2480
|
-
if (mine && prior.initialDeadline > now) {
|
|
2481
|
-
// A retry of this holder's own mint: idempotent-if-identical, and the one path that can
|
|
2482
|
-
// repair a crash between the spec and the status, because only an identical spec completes.
|
|
2483
|
-
await mintCheckpoint(this.kv, this.js, this.binding.space, {
|
|
2484
|
-
ref,
|
|
2485
|
-
instanceId: this.binding.instanceId,
|
|
2486
|
-
epoch: this.binding.epoch,
|
|
2487
|
-
holder: this.binding.holder,
|
|
2488
|
-
deadline: prior.initialDeadline,
|
|
2489
|
-
now,
|
|
2490
|
-
});
|
|
2491
|
-
return;
|
|
2492
|
-
}
|
|
2493
|
-
// ATTACH: the pause exists and is not this attempt's to mint — another holder's pause, or this
|
|
2494
|
-
// holder's own come back overdue (a due pause is being collected, not armed). Both need the
|
|
2495
|
-
// same thing: the schedule re-emitted at THIS instance's coordinates, so the fire lands where
|
|
2496
|
-
// this driver is listening. The settle the caller waits on next is coordinate-free.
|
|
2497
|
-
const status = await readCheckpointStatus(this.kv, ref);
|
|
2498
|
-
if (status === undefined) {
|
|
2499
|
-
throw new Error(`checkpoint "${ref.token}" carries a spec with no status`
|
|
2500
|
-
+ `${mine ? "" : ` held by ${prior.holder.id}, and only its own holder's identical re-mint can repair a half-minted pause`}; `
|
|
2501
|
-
+ `this pause has to be reconciled on the plane before the run can go on`);
|
|
2502
|
-
}
|
|
2503
|
-
await reconcileCheckpointSchedule(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2504
|
-
ref,
|
|
2505
|
-
instanceId: this.binding.instanceId,
|
|
2506
|
-
epoch: this.binding.epoch,
|
|
2507
|
-
});
|
|
2508
|
-
}
|
|
2509
|
-
/** Push a live deadline out — the idle window restarting. The heartbeat CAS-advances the
|
|
2510
|
-
* generation before replacing the timer, so a fire from the old one no-ops rather than racing. */
|
|
2511
|
-
async push(ref, deadline) {
|
|
2512
|
-
if (this.services)
|
|
2513
|
-
return this.services.pauses.heartbeat(ref.token, deadline);
|
|
2514
|
-
await heartbeatCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2515
|
-
ref,
|
|
2516
|
-
instanceId: this.binding.instanceId,
|
|
2517
|
-
epoch: this.binding.epoch,
|
|
2518
|
-
deadline,
|
|
2519
|
-
now: this.now(),
|
|
2520
|
-
});
|
|
2521
|
-
}
|
|
2522
|
-
/**
|
|
2523
|
-
* Take the broker's `.fire` for this deadline, if it has published one, and let the checkpoint
|
|
2524
|
-
* plane judge it. Answers whether the fire SETTLED the pause.
|
|
2525
|
-
*
|
|
2526
|
-
* A fire is a MESSAGE and a settlement is a FACT; `handleCheckpointFire` is what turns one into
|
|
2527
|
-
* the other, and a pause nobody answers ends no other way.
|
|
2528
|
-
*
|
|
2529
|
-
* READ BY SUBJECT rather than by a subscription, because the fire is a record of a deadline
|
|
2530
|
-
* passing and callers here are already polling: `last_by_subj` gives the current one whether it
|
|
2531
|
-
* arrived a moment ago or while this host was down, and a resume lands in the second case.
|
|
2532
|
-
*
|
|
2533
|
-
* IDEMPOTENT BY THE PLANE rather than by bookkeeping here: a fire handed over twice finds the
|
|
2534
|
-
* pause settled and declines. The one repeated verdict with a side effect is `re-armed` under
|
|
2535
|
-
* owner-behind clock skew, and over-emission is idempotent at the timer writer.
|
|
2536
|
-
*/
|
|
2537
|
-
async takeFire(ref) {
|
|
2538
|
-
if (this.services)
|
|
2539
|
-
return this.services.pauses.takeFire(ref.token);
|
|
2540
|
-
const subject = eptSubject(this.binding.space, ref.endpoint, this.binding.instanceId, this.binding.epoch, ref.token, "fire");
|
|
2541
|
-
const fired = await this.jsm.streams
|
|
2542
|
-
.getMessage(eptStreamName(this.binding.space), { last_by_subj: subject })
|
|
2543
|
-
.catch((e) => {
|
|
2544
|
-
// 10037 is the stream saying there is no message on that subject, which IS the ordinary
|
|
2545
|
-
// "the timer has not fired yet" answer. Every other error is the broker refusing or
|
|
2546
|
-
// unreachable, and reading those as "not yet" is how a `wait` with a timeout outlives its
|
|
2547
|
-
// own durable deadline in silence, which is the failure this method exists to prevent.
|
|
2548
|
-
if (e?.code === 10037)
|
|
2549
|
-
return null;
|
|
2550
|
-
throw e;
|
|
2551
|
-
});
|
|
2552
|
-
if (fired === null || fired === undefined)
|
|
2553
|
-
return false;
|
|
2554
|
-
const verdict = await handleCheckpointFire(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2555
|
-
ref,
|
|
2556
|
-
instanceId: this.binding.instanceId,
|
|
2557
|
-
epoch: this.binding.epoch,
|
|
2558
|
-
msg: { subject, ...(fired.header !== undefined ? { headers: fired.header } : {}), data: fired.data },
|
|
2559
|
-
now: this.now(),
|
|
2560
|
-
});
|
|
2561
|
-
return verdict.acted;
|
|
2562
|
-
}
|
|
2563
|
-
/** Has this deadline settled? `undefined` for "no deadline" and for "not yet".
|
|
2564
|
-
*
|
|
2565
|
-
* A `wait` observes its deadlines by polling this, so the fire is taken on the same poll: the
|
|
2566
|
-
* settle fact a `wait` is reading for is one this process has to produce, and reading for it
|
|
2567
|
-
* without ever producing it is how a `wait` with a timeout waited past its timeout forever. */
|
|
2568
|
-
async expired(ref) {
|
|
2569
|
-
if (ref === undefined)
|
|
2570
|
-
return undefined;
|
|
2571
|
-
if (this.services) {
|
|
2572
|
-
const settled = await this.services.pauses.readSettle(ref.token);
|
|
2573
|
-
if (settled !== undefined)
|
|
2574
|
-
return settled;
|
|
2575
|
-
if (!(await this.services.pauses.takeFire(ref.token)))
|
|
2576
|
-
return undefined;
|
|
2577
|
-
return this.services.pauses.readSettle(ref.token);
|
|
2578
|
-
}
|
|
2579
|
-
const settled = await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
2580
|
-
if (settled !== undefined)
|
|
2581
|
-
return settled;
|
|
2582
|
-
if (!(await this.takeFire(ref)))
|
|
2583
|
-
return undefined;
|
|
2584
|
-
return await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
2585
|
-
}
|
|
2586
|
-
/** End a deadline that is no longer waited on, by claiming its one-use settlement with no answer.
|
|
2587
|
-
* A timer left armed would fire into a run that has moved on; claiming it is how the plane says
|
|
2588
|
-
* "this pause is over" without a second mechanism for cancellation. */
|
|
2589
|
-
async cancelTimer(ref) {
|
|
2590
|
-
if (this.services)
|
|
2591
|
-
return this.services.pauses.claim(ref.token);
|
|
2592
|
-
const st = await readCheckpointStatus(this.kv, ref);
|
|
2593
|
-
if (st?.value.state !== "waiting")
|
|
2594
|
-
return;
|
|
2595
|
-
try {
|
|
2596
|
-
// The presenter is the ARMING holder read off the pause's own record, as everywhere on this
|
|
2597
|
-
// plane: a timer minted by a predecessor is still this run's to end after a takeover, and a
|
|
2598
|
-
// successor presenting its own identity would be refused (resume is holder-bound, SPEC
|
|
2599
|
-
// 13.10). A spec that cannot be read leaves the timer to its own deadline, which the catch
|
|
2600
|
-
// below already tolerates: it fires, settles expired, and nobody is reading that token.
|
|
2601
|
-
const spec = await readCheckpointSpec(this.kv, ref);
|
|
2602
|
-
if (spec === undefined)
|
|
2603
|
-
return;
|
|
2604
|
-
await resumeCheckpoint(this.kv, this.js, this.jsm, this.binding.space, {
|
|
2605
|
-
ref,
|
|
2606
|
-
presenter: spec.holder,
|
|
2607
|
-
now: this.now(),
|
|
2608
|
-
});
|
|
2609
|
-
}
|
|
2610
|
-
catch (e) {
|
|
2611
|
-
// IT SETTLED UNDERNEATH US is one outcome, and it is the expected one: the deadline won a
|
|
2612
|
-
// race it was always allowed to win, the caller has its answer either way, and a claimed
|
|
2613
|
-
// timer is not a fact anyone reads. A BROKER THAT REFUSED OR WENT AWAY is a different thing
|
|
2614
|
-
// wearing the same silence, and this catch used to swallow both. It still must not replace
|
|
2615
|
-
// the caller's answer — a timer left armed fires, settles expired, and nobody reads that
|
|
2616
|
-
// token — so the second one is loud on the way past instead.
|
|
2617
|
-
const settledAlready = e instanceof EpEnvelopeError
|
|
2618
|
-
&& (e.code === "conflict" || e.code === "failed-precondition");
|
|
2619
|
-
if (!settledAlready)
|
|
2620
|
-
console.error(`run ${this.binding.runId}: the pause ${ref.token} could not be claimed (${e.message}); it will fire and settle expired with nobody reading it`);
|
|
2621
|
-
}
|
|
2622
|
-
}
|
|
2623
|
-
/** The message at a recorded stream sequence — the re-bind path after a crash. */
|
|
2624
|
-
async messageAt(seq, requestId) {
|
|
2625
|
-
if (this.services) {
|
|
2626
|
-
const message = decodeMessage(await this.services.waits.messageAt(requestId, seq));
|
|
2627
|
-
if (message === undefined)
|
|
2628
|
-
throw new Error(`wait ${requestId}'s recorded message does not decode`);
|
|
2629
|
-
return message;
|
|
2630
|
-
}
|
|
2631
|
-
const m = await this.jsm.streams.getMessage(chatStream(this.binding.space), { seq });
|
|
2632
|
-
if (m === null || m === undefined) {
|
|
2633
|
-
throw new Error(`the message this wait matched (sequence ${seq}) is no longer on the channel's stream; a recorded match cannot be re-read`);
|
|
2634
|
-
}
|
|
2635
|
-
const msg = decodeMessage(m.data);
|
|
2636
|
-
if (msg === undefined)
|
|
2637
|
-
throw new Error(`the message at sequence ${seq} did not decode; a recorded match cannot be re-read`);
|
|
2638
|
-
return msg;
|
|
2639
|
-
}
|
|
2640
|
-
/**
|
|
2641
|
-
* Wait for this token's one-use settlement.
|
|
2642
|
-
*
|
|
2643
|
-
* A settle that ALREADY landed is the ordinary case on a resume: the run crashed while paused and
|
|
2644
|
-
* the timer fired, or somebody answered, without it. Reading before waiting is not an
|
|
2645
|
-
* optimization, it is the difference between resuming and waiting forever for an event that is
|
|
2646
|
-
* already in the past.
|
|
2647
|
-
*/
|
|
2648
|
-
async settle(ref, signal, relayed = false) {
|
|
2649
|
-
// #1508, and the load-bearing half: this is where a `sleep` spends its whole duration, so this
|
|
2650
|
-
// is where a starved host was reporting its own scheduling as the effect's failure. The pause
|
|
2651
|
-
// and its timer are durable facts on the plane; re-reading them observes the same world, and a
|
|
2652
|
-
// sleep whose deadline passed while this process was blocked settles `ok`, late.
|
|
2653
|
-
try {
|
|
2654
|
-
return await servedDespiteStarvation(() => this.settleOnce(ref, signal), this.lag, `waiting on the pause ${ref.token}`, this.onStarved);
|
|
2655
|
-
}
|
|
2656
|
-
catch (e) {
|
|
2657
|
-
// An `ask` attempt or an escalation may have relayed this token to a seat.
|
|
2658
|
-
if (e instanceof Cancelled && relayed)
|
|
2659
|
-
await this.withdrawCancelledRelay(ref.token);
|
|
2660
|
-
throw e;
|
|
2661
|
-
}
|
|
2662
|
-
}
|
|
2663
|
-
async settleOnce(ref, signal) {
|
|
2664
|
-
const already = this.services ? await this.services.pauses.readSettle(ref.token) : await readCheckpointSettle(this.jsm, this.binding.space, ref);
|
|
2665
|
-
if (already !== undefined)
|
|
2666
|
-
return already;
|
|
2667
|
-
// The watcher waits for the FACT. For a pause with an answer the fact arrives because somebody
|
|
2668
|
-
// answered; for a pause nobody answers - a `sleep`, or a `checkpoint` whose timeout wins - it
|
|
2669
|
-
// arrives only because this process took the fire and the plane wrote it. So the two run
|
|
2670
|
-
// together for the life of one wait, and the pump ends when the wait does.
|
|
2671
|
-
const wait = { over: false };
|
|
2672
|
-
const pump = this.pumpFires(ref, wait);
|
|
2673
|
-
try {
|
|
2674
|
-
return await Promise.race([
|
|
2675
|
-
this.watcher.awaitSettle(ref),
|
|
2676
|
-
// A pump that ENDED is not an answer, only one that FAILED is: a failure means this process
|
|
2677
|
-
// cannot expire the pause, so it is raised rather than absorbed and the step stays pending.
|
|
2678
|
-
pump.then(() => new Promise(() => { })),
|
|
2679
|
-
// A cancelled branch stops parking NOW (§7.6): the rejection decides the race first, so a
|
|
2680
|
-
// `checkpoint` cannot mistake its own claim below for an answer, and then the claim ends
|
|
2681
|
-
// the pause in the world. The claim is not awaited here — a claim that loses its own race
|
|
2682
|
-
// is tolerated by `cancelTimer`, and the discharge sweep at the run's completion re-claims
|
|
2683
|
-
// idempotently for the case where this process died before the claim landed.
|
|
2684
|
-
...(signal === undefined ? [] : [this.settleCancelled(ref, signal)]),
|
|
2685
|
-
]);
|
|
2686
|
-
}
|
|
2687
|
-
finally {
|
|
2688
|
-
wait.over = true;
|
|
2689
|
-
// THE PUMP IS DRAINED BEFORE THE SETTLE RETURNS (#1460). The race above has already decided
|
|
2690
|
-
// — the fact, the failure, or the cancellation is this call's answer — but the pump's
|
|
2691
|
-
// in-flight `takeFire` is a journal replay under the run's takeover id, created, read and
|
|
2692
|
-
// deleted inside one `replayRunJournal` call with nothing serialising it across two
|
|
2693
|
-
// authorities for the same run and lease. A settle that returned past it let a completed
|
|
2694
|
-
// `driveRun` hold the replay open after it resolved, and the next reader under the same
|
|
2695
|
-
// takeover hit `RunJournalReplayRaced` about a driver that did not exist. So only the RETURN
|
|
2696
|
-
// is delayed, until the pump has ended: `wait.over` stops it after its current `takeFire`
|
|
2697
|
-
// without starting another, and awaiting it here means no pump fire is left in flight on any
|
|
2698
|
-
// exit. A pump that FAILED raises through this await on its way out, exactly as it did
|
|
2699
|
-
// through the race; the settle's own answer is decided either way and is what this returns.
|
|
2700
|
-
await pump;
|
|
2701
|
-
}
|
|
2702
|
-
}
|
|
2703
|
-
/** Reject with `Cancelled` the moment this branch's signal fires, then claim the pause so its
|
|
2704
|
-
* armed schedule cannot fire into a run that has moved on. The claim also writes the one-use
|
|
2705
|
-
* settle, which is what lets an abandoned `awaitSettle` poll loop see a fact and end. */
|
|
2706
|
-
settleCancelled(ref, signal) {
|
|
2707
|
-
return new Promise((_, reject) => {
|
|
2708
|
-
let fired = false;
|
|
2709
|
-
const fire = (reason) => {
|
|
2710
|
-
if (fired)
|
|
2711
|
-
return;
|
|
2712
|
-
fired = true;
|
|
2713
|
-
reject(new Cancelled(reason ?? "cancelled"));
|
|
2714
|
-
void this.cancelTimer(ref).catch(() => undefined);
|
|
2715
|
-
};
|
|
2716
|
-
if (signal.cancelled) {
|
|
2717
|
-
fire(signal.reason);
|
|
2718
|
-
return;
|
|
2719
|
-
}
|
|
2720
|
-
signal.onCancel(fire);
|
|
2721
|
-
});
|
|
2722
|
-
}
|
|
2723
|
-
/** Take this deadline's fire for as long as somebody is waiting on it. */
|
|
2724
|
-
async pumpFires(ref, wait) {
|
|
2725
|
-
while (!wait.over) {
|
|
2726
|
-
await this.takeFire(ref);
|
|
2727
|
-
// THE FLAG IS RE-READ AFTER THE FIRE, before the poll sleeps (#1460): `takeFire` is a journal
|
|
2728
|
-
// replay under the run's takeover id, so the wait can have ENDED while this one was in
|
|
2729
|
-
// flight — and a pump that slept another poll on a wait that is over keeps exactly one more
|
|
2730
|
-
// fire in flight past the drain `settleOnce` awaits. Checked here, the pump ends right after
|
|
2731
|
-
// its current fire without starting another.
|
|
2732
|
-
if (wait.over)
|
|
2733
|
-
break;
|
|
2734
|
-
await delay(FIRE_POLL_MS);
|
|
2735
|
-
}
|
|
2736
|
-
}
|
|
2154
|
+
/** The pause timers own no handler state: each input is read through the handler at its use. */
|
|
2155
|
+
pauseTimers = new PauseTimers({
|
|
2156
|
+
binding: () => this.binding, kv: () => this.kv, js: () => this.js, jsm: () => this.jsm, watcher: () => this.watcher,
|
|
2157
|
+
services: () => this.services, lag: () => this.lag, onStarved: (note) => this.onStarved(note), now: () => this.now(),
|
|
2158
|
+
withdrawCancelledRelay: (goalId, deadlineAt) => this.withdrawCancelledRelay(goalId, deadlineAt),
|
|
2159
|
+
});
|
|
2160
|
+
async arm(ref, deadline) { return this.pauseTimers.arm(ref, deadline); }
|
|
2161
|
+
async armUnlessDue(ref, deadline, signal) { return this.pauseTimers.armUnlessDue(ref, deadline, signal); }
|
|
2162
|
+
async push(ref, deadline) { return this.pauseTimers.push(ref, deadline); }
|
|
2163
|
+
async expired(ref) { return this.pauseTimers.expired(ref); }
|
|
2164
|
+
async cancelTimer(ref) { return this.pauseTimers.cancelTimer(ref); }
|
|
2165
|
+
async messageAt(seq, requestId) { return this.pauseTimers.messageAt(seq, requestId); }
|
|
2166
|
+
async settle(ref, signal, relayed = false) { return this.pauseTimers.settle(ref, signal, relayed); }
|
|
2167
|
+
/** The relay and presence watch own no handler state: each input, the handler's own members
|
|
2168
|
+
* included, is read through the handler at its use. */
|
|
2169
|
+
seatWatch = new SeatWatch({
|
|
2170
|
+
self: () => this,
|
|
2171
|
+
binding: () => this.binding,
|
|
2172
|
+
roster: () => this.roster,
|
|
2173
|
+
managerServices: () => this.managerServices,
|
|
2174
|
+
now: () => this.now(),
|
|
2175
|
+
actionCtx: () => this.actionCtx(),
|
|
2176
|
+
presenceRegistry: () => this.presenceRegistry(),
|
|
2177
|
+
seatAddress: (name, uid) => this.seatAddress(name, uid),
|
|
2178
|
+
manager: (command, instanceId) => this.manager(command, instanceId),
|
|
2179
|
+
invokeManager: (service, command, args, opts) => this.invokeManager(service, command, args, opts),
|
|
2180
|
+
relayManagerBeforeSubmit: (host, command, ref, kind, label) => this.relayManagerBeforeSubmit(host, command, ref, kind, label),
|
|
2181
|
+
});
|
|
2182
|
+
async relayEscalation(req, ctx, token, deadlineAt) { return this.seatWatch.relayEscalation(req, ctx, token, deadlineAt); }
|
|
2183
|
+
async relayToSeat(seat, goalId, payload, deadlineAt, kind, step) { return this.seatWatch.relayToSeat(seat, goalId, payload, deadlineAt, kind, step); }
|
|
2184
|
+
async withdrawRelay(goalId) { return this.seatWatch.withdrawRelay(goalId); }
|
|
2185
|
+
async withdrawCancelledRelay(goalId, deadlineAt) { return this.seatWatch.withdrawCancelledRelay(goalId, deadlineAt); }
|
|
2186
|
+
async presenceRows() { return this.seatWatch.presenceRows(); }
|
|
2187
|
+
async presenceDeath(name, uid, watch, decided) { return this.seatWatch.presenceDeath(name, uid, watch, decided); }
|
|
2737
2188
|
}
|
|
2738
2189
|
export { waitConsumerName, waitConsumerConfig } from "@cotal-ai/core";
|
|
2739
2190
|
/** How long one poll of a wait's consumer blocks. The deadline itself is durable; this is only how
|
|
2740
2191
|
* late its observation can be, and a shorter poll buys latency at the cost of fetch traffic. */
|
|
2741
|
-
const WAIT_POLL_MS = 2_000;
|
|
2742
|
-
/**
|
|
2743
|
-
* How long an incarnation's presence row must stay absent before `turn` and `wait(down)` read it as
|
|
2744
|
-
* `lapsed`. A row is gone 6s after its last heartbeat, and a live seat whose connector stalls for
|
|
2745
|
-
* longer (host load, a reconnect) comes back under the same lifecycle uid on its next heartbeat. One
|
|
2746
|
-
* absent read failed a turn L4002 while the seat went on working (#2344). Five liveness windows lets
|
|
2747
|
-
* a stalled seat renew, and stays short against a turn's deadline. A `superseded` read needs no
|
|
2748
|
-
* wait: a different incarnation already holds the name. The window is measured from the end of the
|
|
2749
|
-
* read that opened it to the start of the read that closes it, over reads {@link lapseWindow} chains.
|
|
2750
|
-
*/
|
|
2751
|
-
const LAPSE_CONFIRM_MS = 30_000;
|
|
2752
|
-
/**
|
|
2753
|
-
* The longest span from the start of one presence read to the end of the next across which an
|
|
2754
|
-
* absent row still counts as continuously absent: the presence bucket's 6s TTL. A heartbeat keeps
|
|
2755
|
-
* the row live at least that long, so a renewal cannot fall between two reads that close. A slower
|
|
2756
|
-
* read or a run of failed scans leaves the gap unobserved, and the lapse window starts over.
|
|
2757
|
-
*/
|
|
2758
|
-
const PRESENCE_GAP_MS = 6_000;
|
|
2759
|
-
/** The start of the lapse window after an absent read that ended at `now`: kept when the previous
|
|
2760
|
-
* read began within {@link PRESENCE_GAP_MS}, otherwise this read opens a new one. */
|
|
2761
|
-
const lapseWindow = (lapsedSince, lastReadAt, now) => lapsedSince === undefined || now - lastReadAt > PRESENCE_GAP_MS ? now : lapsedSince;
|
|
2192
|
+
export const WAIT_POLL_MS = 2_000;
|
|
2762
2193
|
/** What an `expired` turn refusal shows. The placed host speaks for its own seat. A class-rail
|
|
2763
2194
|
* target-unmapped refusal says only that no member the call reached maps the seat: in a space with
|
|
2764
2195
|
* more than one manager a live seat on a member it missed answers the same as a dead one. */
|
|
2765
|
-
function seatDownReading(err, host) {
|
|
2196
|
+
export function seatDownReading(err, host) {
|
|
2766
2197
|
if (host !== undefined)
|
|
2767
2198
|
return `found the agent down on its placed host ${host}`;
|
|
2768
2199
|
return replyTargetUnmapped(err)
|
|
@@ -2857,14 +2288,14 @@ const SPAWN_ACCEPT_DEADLINE_MS = 30_000;
|
|
|
2857
2288
|
/** Total unanswered attempts per seat. Separate from class routing so an outage is bounded. */
|
|
2858
2289
|
const SPAWN_DISCHARGE_UNANSWERED_ATTEMPTS = 3;
|
|
2859
2290
|
/** Bound on the manager's synchronous `turn` ACCEPT reply (the relay registration, not the yield). */
|
|
2860
|
-
const TURN_ACCEPT_DEADLINE_MS = 30_000;
|
|
2291
|
+
export const TURN_ACCEPT_DEADLINE_MS = 30_000;
|
|
2861
2292
|
/** How many class-rail calls a despawn, a relay withdrawal or a lapse's inspect sends before it stops
|
|
2862
2293
|
* waiting for the one manager that holds its goal (the spawn's allocator, the relay's accepter) to
|
|
2863
2294
|
* answer. Each is one {@link MeshHandler.invokeManager} call, whose describe and invoke both land on
|
|
2864
2295
|
* that manager with probability 1/m^2 per trip in a space of m managers, so one call reaches it with
|
|
2865
2296
|
* probability (1 - ((m-1)/m)^17) / m: about 1/2 for m = 2 and 0.25 for m = 4. All 65 then miss
|
|
2866
2297
|
* with probability about 2^-65 and 9e-9. */
|
|
2867
|
-
const MANAGER_ROUTE_ATTEMPTS = 64;
|
|
2298
|
+
export const MANAGER_ROUTE_ATTEMPTS = 64;
|
|
2868
2299
|
/** A step key's enclosing scope: the journal's own rendering (`entry.scope`), re-derived so the
|
|
2869
2300
|
* live path and the adoption rebuild key the handoff memos identically. */
|
|
2870
2301
|
function scopeOf(key) {
|
|
@@ -3065,9 +2496,6 @@ function assertConclaveChannel(channel) {
|
|
|
3065
2496
|
throw new Error(`conclave channel "${channel}" is a wildcard; a conclave joins its members to one concrete channel`);
|
|
3066
2497
|
return channel;
|
|
3067
2498
|
}
|
|
3068
|
-
/** How often a pause that nobody will answer looks for the broker's fire. Same argument as
|
|
3069
|
-
* `WAIT_POLL_MS`: the deadline is durable and this is only how late its observation can be. */
|
|
3070
|
-
const FIRE_POLL_MS = 2_000;
|
|
3071
2499
|
/** The recorded ask progress a resume re-enters at, or undefined for a fresh first attempt. The
|
|
3072
2500
|
* external is bytes from an earlier process, so the shape is checked rather than trusted. */
|
|
3073
2501
|
function askResume(v) {
|
|
@@ -3128,17 +2556,6 @@ function isWaitEvent(ev) {
|
|
|
3128
2556
|
return typeof e.channel === "string";
|
|
3129
2557
|
return (e.event === "replied" || e.event === "down") && typeof e.agent === "string";
|
|
3130
2558
|
}
|
|
3131
|
-
function decodeMessage(data) {
|
|
3132
|
-
try {
|
|
3133
|
-
const v = JSON.parse(new TextDecoder().decode(data));
|
|
3134
|
-
return v !== null && typeof v === "object" ? v : undefined;
|
|
3135
|
-
}
|
|
3136
|
-
catch {
|
|
3137
|
-
// Someone else's malformed publish is not this run's failure. It does not match, and the wait
|
|
3138
|
-
// goes on waiting — which is what would happen if the message had never been sent.
|
|
3139
|
-
return undefined;
|
|
3140
|
-
}
|
|
3141
|
-
}
|
|
3142
2559
|
/** Does this message answer the await? It must come from the principal its subject names (SPEC
|
|
3143
2560
|
* §5), and when the wait names a `sender` principal, the subject's sender must be that principal.
|
|
3144
2561
|
* The payload's `from.name` is whatever the sender wrote, so it is never compared. */
|