@cotal-ai/manager 0.133.13 → 0.135.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/manager-service-contract.d.ts +9 -3
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +45 -3
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +143 -9
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +745 -66
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts +1 -0
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +3 -1
- package/dist/resume.js.map +1 -1
- package/dist/runtime/pty.d.ts +1 -0
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +3 -1
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -5,7 +5,7 @@ import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, s
|
|
|
5
5
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
6
6
|
import { parse as parseYaml } from "yaml";
|
|
7
7
|
import Headless from "@xterm/headless";
|
|
8
|
-
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, MANAGER_LEASE_TTL_MS, awaitLeaseExpiry, leaseRemainingMs, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, fenceLifecycleAcl, DeprovisionError, addDeprovisionAccounting, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, identityFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, bindIssuedStores, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, oneLine, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, digestIdToken, } from "@cotal-ai/core";
|
|
8
|
+
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, MANAGER_LEASE_TTL_MS, awaitLeaseExpiry, leaseRemainingMs, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, fenceLifecycleAcl, DeprovisionError, addDeprovisionAccounting, isConcreteChannel, isChannelFamily, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, identityFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, bindIssuedStores, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, oneLine, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, digestIdToken, renderOnceTask, } from "@cotal-ai/core";
|
|
9
9
|
import { loadLaunchPersonaFile, PersonaSourceError } from "@cotal-ai/workspace";
|
|
10
10
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimIdentityRecord, claimManagerSiblingIdentities, connectorInstallHint, DEFAULT_SEAT_CAP, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, probeLiveness, remintDaemonCreds, resolveAgentType, resolveOnPath, runtimeInstallHint, createManagerInstanceIdentity, SEAT_CAP_UNLIMITED, spaceKey, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
11
11
|
import { createRuntime, exitProof, isCustodialRuntime, pollProvedExit, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, watchProvedExit, } from "./runtime/index.js";
|
|
@@ -34,7 +34,7 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
|
|
|
34
34
|
// The manager as an ordinary v0.4 `service` endpoint — the §13.1
|
|
35
35
|
// endpoint-serve credential subsystem (gate provisioning, registration barrier, mint fence) plus
|
|
36
36
|
// the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
|
|
37
|
-
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServingInstance, deregisterServiceInstance, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, requestGoalCancel, goalAlreadyTerminal, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, turnNoticeSubject, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, seatCapRefused, lifecycleBlockedFrom, renderLifecycleBlocked, controlReplyFromThrown, } from "@cotal-ai/core";
|
|
37
|
+
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServingInstance, deregisterServiceInstance, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, requestGoalCancel, goalAlreadyTerminal, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, MAX_SCHEDULE_MS, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, turnNoticeSubject, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, seatCapRefused, lifecycleBlockedFrom, renderLifecycleBlocked, controlReplyFromThrown, } from "@cotal-ai/core";
|
|
38
38
|
import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
|
|
39
39
|
import { measureManagerVitals } from "./vitals.js";
|
|
40
40
|
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, ownedBySibling, observeStaticSlot, listStaticSlotObservations, walkStaticSlots, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
|
|
@@ -53,6 +53,14 @@ const TURN_SWEEP_MS = 5_000;
|
|
|
53
53
|
* The window a lost reply is retried in is seconds; five minutes is generous for that and short
|
|
54
54
|
* enough that the map drains between bursts rather than growing for the process lifetime. */
|
|
55
55
|
const TURN_ANSWER_RETENTION_MS = 5 * 60_000;
|
|
56
|
+
/** A `once` spawn's task deadline when the caller names none: one hour from the spawn's accept. */
|
|
57
|
+
const ONCE_DEFAULT_DEADLINE_MS = 60 * 60_000;
|
|
58
|
+
/** The bound on one {@link Connector.retireAfterExit} call. An expired bound is not proof: the debt
|
|
59
|
+
* stays and the turn sweep asks again. */
|
|
60
|
+
const ONCE_RETIRE_MS = 60_000;
|
|
61
|
+
/** The goal index's bound on a relay note (core `GOAL_INDEX_NOTE_MAX`). A once task's note must
|
|
62
|
+
* fit it, and is refused at admission when it cannot. */
|
|
63
|
+
const TURN_NOTE_MAX = 65536;
|
|
56
64
|
/** Backstop for the detached-launch readiness race (#159 B1). `startAgent` waits on two REAL outcomes —
|
|
57
65
|
* the assigned id joining the mesh (presence) = started, the child process exiting = failed — NOT a
|
|
58
66
|
* liveness-inferring timer. This is only the last-resort bound for "neither happened in time": the launch
|
|
@@ -62,6 +70,14 @@ const TURN_ANSWER_RETENTION_MS = 5 * 60_000;
|
|
|
62
70
|
* launch-parity smoke can assert every launch client's request timeout OUTLIVES this window — the tier
|
|
63
71
|
* rule forbids the clients importing it directly. */
|
|
64
72
|
export const READINESS_TIMEOUT_MS = 30_000;
|
|
73
|
+
/** How long a once seat whose connector needs a confirmed startup waits for its startup watch to end,
|
|
74
|
+
* from its launch: the watch's 15s for the confirm prompt and 15s for its notice, plus 5s for the
|
|
75
|
+
* last read. The readiness window applies when it is longer. */
|
|
76
|
+
const STARTUP_OUTCOME_MS = 35_000;
|
|
77
|
+
/** The cancel reason of an inherited once task whose spawn has no succeeded terminal. */
|
|
78
|
+
const NEVER_SERVED = "never served: the manager restarted before the spawn's readiness verdict";
|
|
79
|
+
/** Said by every settle of an inherited once task whose seat this manager does not hold. */
|
|
80
|
+
const UNHELD_SEAT = "the manager does not hold the seat; it may still be running";
|
|
65
81
|
/** How often the readiness race asks a runtime that streams no exit whether its child has exited, and
|
|
66
82
|
* how long it waits past the window for the proof of its last read. An operator waits on this reply,
|
|
67
83
|
* so it asks more often than the exit watch that outlives readiness. */
|
|
@@ -297,6 +313,18 @@ function stopCauseText(cause) {
|
|
|
297
313
|
case "stopped-self": return "it stopped itself (self-stop)";
|
|
298
314
|
case "stopped-reap": return `this manager stopped it because its parent ${cause.parent} left (recursive reap)`;
|
|
299
315
|
case "stopped-shutdown": return "this manager stopped it on manager shutdown";
|
|
316
|
+
case "stopped-once": return `this manager stopped it: its once task ended (${cause.reason})`;
|
|
317
|
+
default: return cause;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
/** The reason a once task is cancelled with when a stop of this kind ends its seat first. */
|
|
321
|
+
function onceStopReason(cause) {
|
|
322
|
+
switch (cause.kind) {
|
|
323
|
+
case "stopped-requested": return "despawned";
|
|
324
|
+
case "stopped-self": return "stopped-itself";
|
|
325
|
+
case "stopped-reap": return "parent-left";
|
|
326
|
+
case "stopped-shutdown": return "manager-shutdown";
|
|
327
|
+
case "stopped-once": return cause.reason;
|
|
300
328
|
default: return cause;
|
|
301
329
|
}
|
|
302
330
|
}
|
|
@@ -315,14 +343,47 @@ function parseTurnNote(raw) {
|
|
|
315
343
|
if (o === null || typeof o !== "object")
|
|
316
344
|
return undefined;
|
|
317
345
|
const n = o;
|
|
318
|
-
if (
|
|
319
|
-
|
|
346
|
+
if (n.once !== undefined && n.once !== true)
|
|
347
|
+
return undefined;
|
|
348
|
+
if (typeof n.payload !== "string" || typeof n.holdEpoch !== "number")
|
|
349
|
+
return undefined;
|
|
350
|
+
if (typeof n.deadlineAt !== "number" && !(n.once === true && n.deadlineAt === undefined))
|
|
320
351
|
return undefined;
|
|
321
352
|
if (n.holdToken !== undefined && typeof n.holdToken !== "string")
|
|
322
353
|
return undefined;
|
|
323
354
|
if (n.handoffFrom !== undefined && typeof n.handoffFrom !== "string")
|
|
324
355
|
return undefined;
|
|
325
|
-
|
|
356
|
+
if (n.retire !== undefined && (n.once !== true || typeof n.retire !== "string"))
|
|
357
|
+
return undefined;
|
|
358
|
+
if (n.spawn !== undefined && (n.once !== true || typeof n.spawn !== "string"))
|
|
359
|
+
return undefined;
|
|
360
|
+
return {
|
|
361
|
+
payload: n.payload, holdEpoch: n.holdEpoch,
|
|
362
|
+
...(typeof n.deadlineAt === "number" ? { deadlineAt: n.deadlineAt } : {}),
|
|
363
|
+
...(typeof n.holdToken === "string" ? { holdToken: n.holdToken } : {}),
|
|
364
|
+
...(typeof n.handoffFrom === "string" ? { handoffFrom: n.handoffFrom } : {}),
|
|
365
|
+
...(n.once === true ? { once: true } : {}),
|
|
366
|
+
...(typeof n.retire === "string" ? { retire: n.retire } : {}),
|
|
367
|
+
...(typeof n.spawn === "string" ? { spawn: n.spawn } : {}),
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
function retireOf(d) {
|
|
371
|
+
return d.retire === undefined ? undefined : { connector: d.retire, name: d.seat.name, uid: d.seat.uid };
|
|
372
|
+
}
|
|
373
|
+
/** The relay record of a `once` task. The prompt is only ever the string value of `context`, so a
|
|
374
|
+
* prompt shaped like a run payload stays text and cannot add an `ask` or a `checkpoint`; the
|
|
375
|
+
* `once` block is the manager's, rendered by the seat beside the prompt. */
|
|
376
|
+
function onceTurnNote(o) {
|
|
377
|
+
const payload = JSON.stringify({ context: o.prompt, once: { spawner: o.spawner, ...(o.deadlineAt !== undefined ? { deadlineAt: o.deadlineAt } : {}) } });
|
|
378
|
+
const note = JSON.stringify({
|
|
379
|
+
payload, holdEpoch: o.holdEpoch,
|
|
380
|
+
...(o.deadlineAt !== undefined ? { deadlineAt: o.deadlineAt } : {}),
|
|
381
|
+
...(o.holdToken !== undefined ? { holdToken: o.holdToken } : {}),
|
|
382
|
+
once: true,
|
|
383
|
+
...(o.retire !== undefined ? { retire: o.retire } : {}),
|
|
384
|
+
spawn: o.spawn,
|
|
385
|
+
});
|
|
386
|
+
return { payload, note };
|
|
326
387
|
}
|
|
327
388
|
/** A goal's key in the manager's maps: the caller triple and goal id the goal plane stores the
|
|
328
389
|
* goal under, because a goal id is the caller's own (SPEC 13.6 item 1) and two callers may pick
|
|
@@ -598,6 +659,21 @@ export class Manager {
|
|
|
598
659
|
/** The deadline sweep behind {@link pendingTurns}: takes the hold's fire into an expired
|
|
599
660
|
* settle and commits the deadline terminal. Unref'd; idle when no turn is pending. */
|
|
600
661
|
turnSweepTimer;
|
|
662
|
+
/** Once tasks whose ending is owed, by {@link goalKey}: the task can no longer run (its seat's
|
|
663
|
+
* exit is proved, or it was never served), so it is cancelled if it has no terminal and its index
|
|
664
|
+
* entry is dropped. A settle that fails stays here and the turn sweep retries it. */
|
|
665
|
+
onceSettles = new Map();
|
|
666
|
+
/** Once tasks whose seat's exit is watched: that exit, not the spawn's outcome, settles the debt. */
|
|
667
|
+
onceWatched = new Set();
|
|
668
|
+
/** Once tasks a predecessor accepted, by seat uid, until this incarnation binds the seat or
|
|
669
|
+
* proves it gone. `readinessMs` bounds how long a seat no reconcile here can account for is
|
|
670
|
+
* waited for. */
|
|
671
|
+
onceInherited = new Map();
|
|
672
|
+
/** Inherited once seats this manager does not hold whose task has a terminal, by lifecycle uid.
|
|
673
|
+
* Every pull from one is answered `stop`, and its notice repeats until a pull has carried it. */
|
|
674
|
+
onceStops = new Map();
|
|
675
|
+
/** When the goal reconcile finished: the start of an inherited once task's wait for its seat. */
|
|
676
|
+
goalReconcileDoneAt = 0;
|
|
601
677
|
/** Process start, for the served `status` uptime. */
|
|
602
678
|
startedAtMs = Date.now();
|
|
603
679
|
/** Connector harness paths resolved ONCE at boot. Missing binaries do not stop unrelated manager
|
|
@@ -2063,6 +2139,8 @@ export class Manager {
|
|
|
2063
2139
|
failures.push({ name: a.name, id: a.id, error: "a delegated seat is not resumed by a later manager; stop it before preserving" });
|
|
2064
2140
|
if (a.seatHome)
|
|
2065
2141
|
failures.push({ name: a.name, id: a.id, error: "a carried resume seat's private home is not kept for a later manager; stop it before preserving" });
|
|
2142
|
+
if (a.once)
|
|
2143
|
+
failures.push({ name: a.name, id: a.id, error: "a once seat runs its one task in this process, and a later manager would start it again; let its task end or stop it before preserving" });
|
|
2066
2144
|
}
|
|
2067
2145
|
const unverifiedStops = this.unverifiedStops.filter((stopped) => {
|
|
2068
2146
|
try {
|
|
@@ -3478,6 +3556,7 @@ export class Manager {
|
|
|
3478
3556
|
// turns addressed to ITS incarnation and yields them; no reach beyond itself exists here.
|
|
3479
3557
|
turnPending: (ctx) => this.serveGated(ctx, () => this.turnPendingFor(ctx.subject.caller)),
|
|
3480
3558
|
turnYield: (ctx) => this.serveGated(ctx, () => this.serveTurnYield(ctx.subject.caller, args(ctx))),
|
|
3559
|
+
turnUnconfirmed: (ctx) => this.serveGated(ctx, () => this.serveTurnUnconfirmed(ctx.subject.caller, args(ctx))),
|
|
3481
3560
|
// The reserved goal cancel (SPEC 13.6 item 4), served for the relay: the goal ref is the
|
|
3482
3561
|
// authenticated caller's own, so a run withdraws only the turns it submitted.
|
|
3483
3562
|
cancel: (ctx) => this.serveGated(ctx, () => this.serveTurnCancel(ctx)),
|
|
@@ -3743,6 +3822,9 @@ export class Manager {
|
|
|
3743
3822
|
* wait proves exit: nobody asked for those children to be gone, so they stay managed until the
|
|
3744
3823
|
* runtime says otherwise, and a runtime that cannot prove exit records an unverified stop. */
|
|
3745
3824
|
trackStoppedHandle(a, floor, cause, requireAuthoritativeExit = false) {
|
|
3825
|
+
// What a once seat's task is cancelled with if this stop ends it before the task has a terminal.
|
|
3826
|
+
if (a.once)
|
|
3827
|
+
a.once.stopReason ??= onceStopReason(cause);
|
|
3746
3828
|
if (!a.handle.waitForExit) {
|
|
3747
3829
|
// Preserve ordinary external-runtime stop behavior, but retain enough evidence for a later
|
|
3748
3830
|
// maintenance prepare to fail if that runtime still cannot prove the surface disappeared.
|
|
@@ -4596,7 +4678,7 @@ export class Manager {
|
|
|
4596
4678
|
try {
|
|
4597
4679
|
const r = await this.ep.requestDeliveryAdmin("lifecycleMemberships", { principal: principalKey(t.owner, t.actor).key, lifecycleUid: t.lifecycleUid }, 5_000);
|
|
4598
4680
|
const data = r.data;
|
|
4599
|
-
if (!r.ok || data?.complete !== true || !Array.isArray(data.channels) || !data.channels.every((c) => typeof c === "string" && isConcreteChannel(c)))
|
|
4681
|
+
if (!r.ok || data?.complete !== true || !Array.isArray(data.channels) || !data.channels.every((c) => typeof c === "string" && (isConcreteChannel(c) || isChannelFamily(c))))
|
|
4600
4682
|
return { complete: false, channels: [], reason: r.ok ? "malformed inventory reply" : (r.error ?? "refused") };
|
|
4601
4683
|
return { complete: true, channels: data.channels };
|
|
4602
4684
|
}
|
|
@@ -5049,6 +5131,21 @@ export class Manager {
|
|
|
5049
5131
|
return Promise.resolve({ ok: false, error: "supervise.windowMs: expected a positive integer" });
|
|
5050
5132
|
supervise = { restarts: rec.restarts, windowMs: rec.windowMs };
|
|
5051
5133
|
}
|
|
5134
|
+
let once;
|
|
5135
|
+
if (args.once !== undefined) {
|
|
5136
|
+
const raw = args.once;
|
|
5137
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw))
|
|
5138
|
+
return Promise.resolve({ ok: false, error: "once: expected { deadlineMs? }" });
|
|
5139
|
+
const rec = raw;
|
|
5140
|
+
const extra = Object.keys(rec).filter((k) => k !== "deadlineMs");
|
|
5141
|
+
if (extra.length > 0)
|
|
5142
|
+
return Promise.resolve({ ok: false, error: `once: unknown key ${extra[0]}; it takes deadlineMs` });
|
|
5143
|
+
// The default is applied here, so every caller of the wire gets the same one.
|
|
5144
|
+
const d = rec.deadlineMs ?? ONCE_DEFAULT_DEADLINE_MS;
|
|
5145
|
+
if (d !== -1 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1 || Date.now() + d > MAX_SCHEDULE_MS))
|
|
5146
|
+
return Promise.resolve({ ok: false, error: "once.deadlineMs: expected -1 (no deadline) or a positive whole number of milliseconds that ends by the deadline scheduler's last instant (the end of year 9999)" });
|
|
5147
|
+
once = { deadlineMs: d === -1 ? undefined : d };
|
|
5148
|
+
}
|
|
5052
5149
|
// ACL overrides arrive as string arrays or not at all — a malformed value is a bad request,
|
|
5053
5150
|
// not something to coerce (no fallbacks).
|
|
5054
5151
|
const strList = (v, flag) => {
|
|
@@ -5059,11 +5156,23 @@ export class Manager {
|
|
|
5059
5156
|
return v;
|
|
5060
5157
|
};
|
|
5061
5158
|
let subscribe, allowSubscribe, allowPublish, shareTools;
|
|
5159
|
+
let join, callerAcl;
|
|
5062
5160
|
try {
|
|
5063
5161
|
subscribe = strList(args.subscribe, "subscribe");
|
|
5064
5162
|
allowSubscribe = strList(args.allowSubscribe, "allowSubscribe");
|
|
5065
5163
|
allowPublish = strList(args.allowPublish, "allowPublish");
|
|
5066
5164
|
shareTools = strList(args.shareTools, "shareTools");
|
|
5165
|
+
join = strList(args.join, "join");
|
|
5166
|
+
if (args.callerAcl !== undefined) {
|
|
5167
|
+
const rec = args.callerAcl;
|
|
5168
|
+
if (rec === null || typeof rec !== "object" || Array.isArray(rec))
|
|
5169
|
+
throw new Error("callerAcl: expected an object");
|
|
5170
|
+
const sub = strList(rec.allowSubscribe, "callerAcl.allowSubscribe");
|
|
5171
|
+
const pub = strList(rec.allowPublish, "callerAcl.allowPublish");
|
|
5172
|
+
if (!sub || !pub)
|
|
5173
|
+
throw new Error("callerAcl: allowSubscribe and allowPublish are both required");
|
|
5174
|
+
callerAcl = { allowSubscribe: sub, allowPublish: pub };
|
|
5175
|
+
}
|
|
5067
5176
|
}
|
|
5068
5177
|
catch (e) {
|
|
5069
5178
|
return Promise.resolve({ ok: false, error: rejectionText(e) });
|
|
@@ -5120,8 +5229,11 @@ export class Manager {
|
|
|
5120
5229
|
subscribe,
|
|
5121
5230
|
allowSubscribe,
|
|
5122
5231
|
allowPublish,
|
|
5232
|
+
join,
|
|
5233
|
+
callerAcl,
|
|
5123
5234
|
shareTools,
|
|
5124
5235
|
...(supervise !== undefined ? { supervise } : {}),
|
|
5236
|
+
...(once !== undefined ? { once } : {}),
|
|
5125
5237
|
...(typeof args.runtime === "string" ? { runtime: args.runtime.trim() } : {}),
|
|
5126
5238
|
route,
|
|
5127
5239
|
}, caller, hooks, authenticatedCaller);
|
|
@@ -5295,6 +5407,49 @@ export class Manager {
|
|
|
5295
5407
|
return `${connector.name} connector does not support exact session continuity (continuity: exact)`;
|
|
5296
5408
|
return undefined;
|
|
5297
5409
|
}
|
|
5410
|
+
/** The refusal for a `once` spawn this manager cannot run as one task: every check is made before
|
|
5411
|
+
* anything is allocated, so a refused spawn leaves nothing behind. */
|
|
5412
|
+
onceRefusal(connector, runtime, launch, spawner, hooks) {
|
|
5413
|
+
if (!hooks?.onOnce)
|
|
5414
|
+
return "once: a once task rides the spawn action, which this launch door is not";
|
|
5415
|
+
if (launch.prompt === undefined || launch.prompt.trim() === "")
|
|
5416
|
+
return "once: needs a prompt, which is the task";
|
|
5417
|
+
if (launch.supervise !== undefined)
|
|
5418
|
+
return "once: a once seat is never restarted, so it cannot carry supervise";
|
|
5419
|
+
if (!connector.supportsTurnRelay)
|
|
5420
|
+
return `once: the ${connector.name} connector cannot take a once task${connector.turnRelayRefusal ? `: ${connector.turnRelayRefusal}` : ""}`;
|
|
5421
|
+
if (connector.onceNeedsConfirmed !== undefined && !runtime.reportsStartupOutcome)
|
|
5422
|
+
return `once: runtime "${runtime.kind}" does not report how a seat's startup ended, which a once seat on the ${connector.name} connector needs`;
|
|
5423
|
+
if (runtime.spawnDelegated)
|
|
5424
|
+
return `once: runtime "${runtime.kind}" starts seats outside this host, where this manager cannot prove the seat's exit`;
|
|
5425
|
+
const operator = connector.waitsForOperator?.(launch.launchOptions);
|
|
5426
|
+
if (operator !== undefined)
|
|
5427
|
+
return `once: these launch options make the seat wait at ${operator} before it starts, so nothing could take its task`;
|
|
5428
|
+
// The task's relay record, measured with the widest deadline it could carry.
|
|
5429
|
+
const { note, payload } = onceTurnNote({
|
|
5430
|
+
prompt: launch.prompt, spawner: this.spawnerName(spawner),
|
|
5431
|
+
...(launch.once.deadlineMs !== undefined ? { deadlineAt: MAX_SCHEDULE_MS, holdToken: turnHoldToken("") } : {}),
|
|
5432
|
+
holdEpoch: this.serviceServe?.grant.epoch ?? 0,
|
|
5433
|
+
...(connector.retireAfterExit ? { retire: connector.name } : {}),
|
|
5434
|
+
// The longest id token core accepts (endpoint-subjects `assertIdToken`).
|
|
5435
|
+
spawn: "x".repeat(64),
|
|
5436
|
+
});
|
|
5437
|
+
if (note.length > TURN_NOTE_MAX)
|
|
5438
|
+
return `once: the prompt is too long for one task (its relay record is ${note.length} characters, over ${TURN_NOTE_MAX})`;
|
|
5439
|
+
if (connector.turnFrameLimit !== undefined) {
|
|
5440
|
+
// The text the seat shows, rendered by the seat's own function with the widest values it could
|
|
5441
|
+
// carry: the task's goal id is a digest token, and a deadline renders as one ISO instant.
|
|
5442
|
+
const text = renderOnceTask({ goalId: digestIdToken("once"), payload, spawner: this.spawnerName(spawner), ...(launch.once.deadlineMs !== undefined ? { deadlineAt: MAX_SCHEDULE_MS } : {}) });
|
|
5443
|
+
if (text.length > connector.turnFrameLimit)
|
|
5444
|
+
return `once: the prompt is too long for one ${connector.name} turn (the task renders to ${text.length} characters, over ${connector.turnFrameLimit})`;
|
|
5445
|
+
}
|
|
5446
|
+
return undefined;
|
|
5447
|
+
}
|
|
5448
|
+
/** How a once task names who sent it: the spawner's roster name, else its principal. */
|
|
5449
|
+
spawnerName(spawner) {
|
|
5450
|
+
const id = spawner ?? this.ep.ref().id;
|
|
5451
|
+
return this.ep.getRoster().find((p) => p.card.id === id)?.card.name ?? id;
|
|
5452
|
+
}
|
|
5298
5453
|
/** The refusal for an event-plane choice the registration policy or the connector cannot meet.
|
|
5299
5454
|
* Spawn and resume both ask before a seat's launch is built, so a seat retained without the
|
|
5300
5455
|
* plane cannot come back on a space that now requires it. An absent choice arms the plane. */
|
|
@@ -5558,6 +5713,8 @@ export class Manager {
|
|
|
5558
5713
|
if (!policy.success)
|
|
5559
5714
|
return { ok: false, error: `supervise.${policy.error.issues[0]?.path.join(".")}: expected a positive integer` };
|
|
5560
5715
|
}
|
|
5716
|
+
if (opts.once)
|
|
5717
|
+
opts.once = { deadlineMs: opts.once.deadlineMs };
|
|
5561
5718
|
let runtime;
|
|
5562
5719
|
try {
|
|
5563
5720
|
await this.loadSeatRuntime(opts.runtime);
|
|
@@ -5719,8 +5876,8 @@ export class Manager {
|
|
|
5719
5876
|
if (opts.resolved) {
|
|
5720
5877
|
// A manifest launch is the access + identity authority: imperative overrides arriving
|
|
5721
5878
|
// alongside `resolved` are a caller contract error, not something to merge (no fallbacks).
|
|
5722
|
-
if (opts.subscribe || opts.allowSubscribe || opts.allowPublish || opts.prompt || opts.shareTools || opts.identity)
|
|
5723
|
-
return { ok: false, error: "a manifest launch (resolved) rejects imperative overrides (identity/subscribe/allow*/prompt/shareTools)" };
|
|
5879
|
+
if (opts.subscribe || opts.allowSubscribe || opts.allowPublish || opts.join || opts.callerAcl || opts.prompt || opts.shareTools || opts.identity)
|
|
5880
|
+
return { ok: false, error: "a manifest launch (resolved) rejects imperative overrides (identity/subscribe/allow*/join/callerAcl/prompt/shareTools)" };
|
|
5724
5881
|
const r = opts.resolved;
|
|
5725
5882
|
identityName = r.name;
|
|
5726
5883
|
role = opts.role ?? r.role;
|
|
@@ -5746,9 +5903,20 @@ export class Manager {
|
|
|
5746
5903
|
// Flags > persona file — the same precedence as foreground `cotal spawn`, so the two launch
|
|
5747
5904
|
// paths of the merged grammar can't diverge. One source feeds BOTH the minted creds and the
|
|
5748
5905
|
// connector env below.
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
|
|
5906
|
+
// A persona that names no channel field at all is scoped to its spawner's ACL (`callerAcl`,
|
|
5907
|
+
// when the door sent one). The spawner's own event channel is dropped from it: that plane is
|
|
5908
|
+
// foreign to the child, and the guard below refuses any spawn that asks for one. The joined
|
|
5909
|
+
// set is never copied from the spawner: a helper starts on what its persona lists plus what
|
|
5910
|
+
// the caller passes in `join`, and nothing else.
|
|
5911
|
+
const personaNamesChannels = def.subscribe !== undefined || def.allowSubscribe !== undefined || def.allowPublish !== undefined;
|
|
5912
|
+
const scoped = !personaNamesChannels && opts.callerAcl ? {
|
|
5913
|
+
allowSubscribe: opts.callerAcl.allowSubscribe.filter((ch) => eventChannelPrincipal(ch) === null),
|
|
5914
|
+
allowPublish: opts.callerAcl.allowPublish.filter((ch) => eventChannelPrincipal(ch) === null),
|
|
5915
|
+
} : undefined;
|
|
5916
|
+
const base = opts.subscribe ?? def.subscribe;
|
|
5917
|
+
subscribe = opts.join?.length ? [...new Set([...(base ?? []), ...opts.join])] : base;
|
|
5918
|
+
allowSubscribe = opts.allowSubscribe ?? def.allowSubscribe ?? scoped?.allowSubscribe ?? [];
|
|
5919
|
+
allowPublish = opts.allowPublish ?? def.allowPublish ?? scoped?.allowPublish;
|
|
5752
5920
|
capabilities = def.capabilities;
|
|
5753
5921
|
// #651: fold the persona's model into the launch record, mirroring the variant line below
|
|
5754
5922
|
// and the manifest branch above. Without this, a persona-file model (the common pin source)
|
|
@@ -5835,6 +6003,9 @@ export class Manager {
|
|
|
5835
6003
|
const unsupported = this.capabilityRefusal(connector, { variant, prompt, exact });
|
|
5836
6004
|
if (unsupported)
|
|
5837
6005
|
return { ok: false, error: unsupported };
|
|
6006
|
+
const onceRefused = opts.once && this.onceRefusal(connector, runtime, { ...opts, launchOptions, prompt }, spawner, hooks);
|
|
6007
|
+
if (onceRefused)
|
|
6008
|
+
return { ok: false, error: onceRefused };
|
|
5838
6009
|
let reopenSession;
|
|
5839
6010
|
if (exact) {
|
|
5840
6011
|
try {
|
|
@@ -5998,8 +6169,20 @@ export class Manager {
|
|
|
5998
6169
|
// The hosted enrollment arm accepts only once the host has answered: the floor names the uid
|
|
5999
6170
|
// the agent runs at, and before the host picks it there is no such uid to name.
|
|
6000
6171
|
const hostedEnrollment = this.userMode && (opts.delegatedIntent !== undefined || this.remoteAuthority?.enrollManagedAgent !== undefined);
|
|
6001
|
-
|
|
6172
|
+
// A once spawn's task is accepted with the spawn, before anything launches.
|
|
6173
|
+
let onceTask;
|
|
6174
|
+
const acceptOnce = async (seat) => {
|
|
6175
|
+
if (opts.once === undefined)
|
|
6176
|
+
return;
|
|
6177
|
+
onceTask = await hooks.onOnce({
|
|
6178
|
+
seat: { name, ...seat }, prompt: prompt, spawner: this.spawnerName(spawner), deadlineMs: opts.once.deadlineMs, readinessDeadlineMs: readinessTimeoutMs,
|
|
6179
|
+
...(connector.retireAfterExit ? { retire: connector.name } : {}),
|
|
6180
|
+
});
|
|
6181
|
+
};
|
|
6182
|
+
if (!hostedEnrollment) {
|
|
6002
6183
|
await hooks?.onAccepted?.({ name, identity, lifecycleUid, agentTriple });
|
|
6184
|
+
await acceptOnce(agentTriple);
|
|
6185
|
+
}
|
|
6003
6186
|
// In auth mode, mint the agent's creds from the space signing key and write them where the
|
|
6004
6187
|
// spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
|
|
6005
6188
|
// subscribe/allowSubscribe (read) + allowPublish (post, default-deny).
|
|
@@ -6047,6 +6230,7 @@ export class Manager {
|
|
|
6047
6230
|
if (hostedEnrollment) {
|
|
6048
6231
|
try {
|
|
6049
6232
|
await hooks?.onAccepted?.({ name, identity, lifecycleUid, agentTriple: { owner: prep.owner, actor: name, uid: lifecycleUid } });
|
|
6233
|
+
await acceptOnce({ owner: prep.owner, actor: name, uid: lifecycleUid });
|
|
6050
6234
|
}
|
|
6051
6235
|
catch (e) {
|
|
6052
6236
|
// A refused accept rolls the enrollment back: the host's grant through `provisioned`, and
|
|
@@ -6083,7 +6267,7 @@ export class Manager {
|
|
|
6083
6267
|
// evidence activates only once the row exists, and the row exists only after the evidence
|
|
6084
6268
|
// was staged. The agent learns the generation by reading the accepted row under its token.
|
|
6085
6269
|
// The generation's horizon bounds every renewal of this incarnation's credential.
|
|
6086
|
-
issued = { generation: mintGeneration(now + STATIC_GENERATION_HORIZON_SEC), acceptedToken: mintAcceptedToken() };
|
|
6270
|
+
issued = { generation: mintGeneration(now + STATIC_GENERATION_HORIZON_SEC), acceptedToken: mintAcceptedToken(), listing: true };
|
|
6087
6271
|
const issuedAgent = { name, id: identity.id, lifecycleUid };
|
|
6088
6272
|
const creds = await this.withIssuer((i) => this.withProvisioner((prov) => provisionAgent(prov, this.auth, identity, {
|
|
6089
6273
|
subscribe,
|
|
@@ -6142,8 +6326,10 @@ export class Manager {
|
|
|
6142
6326
|
resume: opts.resume,
|
|
6143
6327
|
reopenSession,
|
|
6144
6328
|
// Initial prompt: the `--prompt` flag, or the manifest entry's `prompt:` on a resolved launch.
|
|
6145
|
-
// A reopened session already had its kickoff turn, so it is not submitted again.
|
|
6146
|
-
prompt
|
|
6329
|
+
// A reopened session already had its kickoff turn, so it is not submitted again. A once
|
|
6330
|
+
// seat gets its prompt as its one relayed task instead.
|
|
6331
|
+
prompt: reopenSession === undefined && opts.once === undefined ? prompt : undefined,
|
|
6332
|
+
...(opts.once !== undefined ? { once: true } : {}),
|
|
6147
6333
|
// The SAME access set the creds were minted from (above) — forwarded so the session's
|
|
6148
6334
|
// runtime read/post set matches its credentials. Without this a manifest-spawned agent
|
|
6149
6335
|
// (materialized persona has no access frontmatter) has no channel set to read and joins
|
|
@@ -6178,6 +6364,9 @@ export class Manager {
|
|
|
6178
6364
|
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
6179
6365
|
handle = await this.spawnCustodied(runtime, name, spec, cwd, custody);
|
|
6180
6366
|
}
|
|
6367
|
+
if (onceTask && !handle.waitForExit)
|
|
6368
|
+
throw new Error(`once: runtime "${handle.kind}" cannot prove the seat's exit, which is what ends its task when the seat goes first`);
|
|
6369
|
+
const launchedAt = Date.now();
|
|
6181
6370
|
hooks?.onLaunched?.(); // The "launched" progress edge (process spawned, pre-presence)
|
|
6182
6371
|
const managed = {
|
|
6183
6372
|
name,
|
|
@@ -6215,8 +6404,9 @@ export class Manager {
|
|
|
6215
6404
|
model,
|
|
6216
6405
|
variant,
|
|
6217
6406
|
// The kickoff prompt is part of the launch form: a stale-restart re-submits it, so the
|
|
6218
|
-
// recorded launch must carry it for the restart gate (and preserve) to see it.
|
|
6219
|
-
prompt,
|
|
6407
|
+
// recorded launch must carry it for the restart gate (and preserve) to see it. A once
|
|
6408
|
+
// seat's prompt is its task, which no relaunch may submit again.
|
|
6409
|
+
prompt: onceTask ? undefined : prompt,
|
|
6220
6410
|
subscribe,
|
|
6221
6411
|
allowSubscribe,
|
|
6222
6412
|
allowPublish,
|
|
@@ -6232,8 +6422,10 @@ export class Manager {
|
|
|
6232
6422
|
? Object.keys(opts.launchOptions).sort()
|
|
6233
6423
|
: undefined,
|
|
6234
6424
|
},
|
|
6235
|
-
// A delegated agent is never restarted (SPEC 13.16),
|
|
6236
|
-
|
|
6425
|
+
// A delegated agent is never restarted (SPEC 13.16), and a once seat never runs its task
|
|
6426
|
+
// twice, so neither slot arms recovery.
|
|
6427
|
+
...(onceTask ? { once: { ref: onceTask.ref, ...(connector.retireAfterExit ? { retire: connector.name } : {}) } } : {}),
|
|
6428
|
+
...(spec && (connector.supportsSessionContinuation || opts.supervise !== undefined) && !opts.delegatedIntent && !onceTask
|
|
6237
6429
|
? {
|
|
6238
6430
|
restart: {
|
|
6239
6431
|
opts: launchOpts,
|
|
@@ -6262,6 +6454,8 @@ export class Manager {
|
|
|
6262
6454
|
}
|
|
6263
6455
|
this.agents.set(name, managed);
|
|
6264
6456
|
this.reservedLive.add(name);
|
|
6457
|
+
if (managed.once)
|
|
6458
|
+
this.watchOnceExit(managed);
|
|
6265
6459
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
6266
6460
|
// orphan-rollback in `finally` no longer applies to it.
|
|
6267
6461
|
provisioned = undefined;
|
|
@@ -6286,6 +6480,8 @@ export class Manager {
|
|
|
6286
6480
|
const error = reopenSession === undefined ? readiness.detail
|
|
6287
6481
|
: `${readiness.detail} (continuity: exact was reopening recorded session ${reopenSession}; if the harness no longer has it, ` +
|
|
6288
6482
|
`remove ${join(this.workspaceRoot, ".cotal", "continuity", `${name}.json`)} to start a new session)`;
|
|
6483
|
+
if (managed.once)
|
|
6484
|
+
managed.once.stopReason ??= "launch-failed";
|
|
6289
6485
|
await hooks?.onOutcome?.({ kind: "failed", data: { error } });
|
|
6290
6486
|
return { ok: false, error };
|
|
6291
6487
|
}
|
|
@@ -6295,9 +6491,36 @@ export class Manager {
|
|
|
6295
6491
|
this.watchExit(managed);
|
|
6296
6492
|
if (managed.launch.carried)
|
|
6297
6493
|
void this.refuseLateCarriedFork(managed, readinessTimeoutMs);
|
|
6494
|
+
// A once seat is never left running with nothing to do, and its stop does not wait on the
|
|
6495
|
+
// spawn's terminal, whose write can stall. The spawn keeps that terminal: the stop lets go
|
|
6496
|
+
// of the spawn goal first. Its task was never served and is cancelled with the verdict.
|
|
6497
|
+
if (managed.once) {
|
|
6498
|
+
this.agentGoals.delete(managed.name);
|
|
6499
|
+
this.stopOnceSeat(managed, "readiness-timeout");
|
|
6500
|
+
}
|
|
6298
6501
|
await hooks?.onOutcome?.({ kind: "uncertain", data: { reason: readiness.detail, ...(readiness.running ? { running: true } : {}) } });
|
|
6299
6502
|
return { ok: false, error: readiness.detail };
|
|
6300
6503
|
}
|
|
6504
|
+
// A once seat whose connector needs a confirmed startup is served only on that outcome. Any
|
|
6505
|
+
// other ends it like a seat that never joined: stopped before the failed terminal, and its
|
|
6506
|
+
// task, never served, cancelled with the verdict.
|
|
6507
|
+
if (onceTask && connector.onceNeedsConfirmed !== undefined) {
|
|
6508
|
+
const startup = await this.awaitStartupOutcome(managed, launchedAt, readinessTimeoutMs);
|
|
6509
|
+
if (managed.terminalizing) {
|
|
6510
|
+
hooks?.onTerminalDeferred?.();
|
|
6511
|
+
return { ok: false, error: `${name} was stopped before its startup ended` };
|
|
6512
|
+
}
|
|
6513
|
+
if (startup !== "confirmed") {
|
|
6514
|
+
const watch = { ready: "opened its input screen without the prompt", failed: "failed", exited: "saw the seat exit", pending: "had not ended" }[startup];
|
|
6515
|
+
const error = `${name} joined, but ${connector.onceNeedsConfirmed}: its startup watch ${watch}, so it cannot take a once task`;
|
|
6516
|
+
this.watchExit(managed);
|
|
6517
|
+
this.agentGoals.delete(managed.name);
|
|
6518
|
+
managed.once.stopReason ??= "launch-failed";
|
|
6519
|
+
this.stopOnceSeat(managed, "launch-failed");
|
|
6520
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error } });
|
|
6521
|
+
return { ok: false, error };
|
|
6522
|
+
}
|
|
6523
|
+
}
|
|
6301
6524
|
if (managed.launch.carried) {
|
|
6302
6525
|
const refused = await this.refuseCarriedFork(managed, readinessTimeoutMs);
|
|
6303
6526
|
if (refused !== undefined && managed.terminalizing) {
|
|
@@ -6367,8 +6590,18 @@ export class Manager {
|
|
|
6367
6590
|
// "no role", so dropping the key preserves the reply (surfaced by readiness:live).
|
|
6368
6591
|
// A host that granted other channels than this spawn asked for: the receipt carries both.
|
|
6369
6592
|
const acl = enrolled ? enrolledAcl({ allowSubscribe, allowPublish }, enrolled.material) : undefined;
|
|
6370
|
-
const okData = { name, agent, id: managed.id, mode: handle.kind, streamsTerminal: runtime.streamsTerminal, lifecycleUid, ...(role !== undefined ? { role } : {}), ...(opts.eventsNotice !== undefined ? { eventsNotice: opts.eventsNotice } : {}), ...(acl ? { acl } : {}), ...(readiness.operator !== undefined ? { waitingFor: readiness.operator } : {}) };
|
|
6371
|
-
|
|
6593
|
+
const okData = { name, agent, id: managed.id, mode: handle.kind, streamsTerminal: runtime.streamsTerminal, lifecycleUid, ...(role !== undefined ? { role } : {}), ...(opts.eventsNotice !== undefined ? { eventsNotice: opts.eventsNotice } : {}), ...(acl ? { acl } : {}), ...(readiness.operator !== undefined ? { waitingFor: readiness.operator } : {}), ...(onceTask ? { once: { goalId: onceTask.ref.goalId, ...(onceTask.deadlineAt !== undefined ? { deadlineAt: onceTask.deadlineAt } : {}) } } : {}) };
|
|
6594
|
+
// The task is served once the succeeded terminal is written, so a served task always has it
|
|
6595
|
+
// and a successor reads it as the readiness verdict. Every failure arm above ran before it. A
|
|
6596
|
+
// terminal this call did not commit never served the task: the seat is stopped, which cancels it.
|
|
6597
|
+
const committed = await hooks?.onOutcome?.({ kind: "succeeded", data: okData });
|
|
6598
|
+
if (onceTask && committed === true)
|
|
6599
|
+
this.openOnceDelivery(onceTask.ref);
|
|
6600
|
+
else if (onceTask) {
|
|
6601
|
+
this.agentGoals.delete(managed.name);
|
|
6602
|
+
managed.once.stopReason ??= "launch-failed";
|
|
6603
|
+
this.stopOnceSeat(managed, "launch-failed");
|
|
6604
|
+
}
|
|
6372
6605
|
return { ok: true, data: okData };
|
|
6373
6606
|
}
|
|
6374
6607
|
catch (e) {
|
|
@@ -8544,6 +8777,7 @@ export class Manager {
|
|
|
8544
8777
|
}
|
|
8545
8778
|
finally {
|
|
8546
8779
|
this.goalReconcileDone = true;
|
|
8780
|
+
this.goalReconcileDoneAt = Date.now();
|
|
8547
8781
|
// A seat that pulled during the window was refused, and a predecessor may have died before
|
|
8548
8782
|
// its notice went out, so every seat holding a rebuilt turn is told once.
|
|
8549
8783
|
const seats = new Map([...this.pendingTurns.values()].map((p) => [`${p.seat.owner}.${p.seat.actor}.${p.seat.uid}`, p.seat]));
|
|
@@ -8729,6 +8963,8 @@ export class Manager {
|
|
|
8729
8963
|
let effectStarted = false;
|
|
8730
8964
|
// Set by onUndone: the host's code for a later refusal that left nothing behind.
|
|
8731
8965
|
let undone;
|
|
8966
|
+
// The task of a once spawn, once accepted.
|
|
8967
|
+
let onceRef;
|
|
8732
8968
|
// The terminal commits OFF-handler on the goal-writer connection (manager-only authority; a
|
|
8733
8969
|
// caller cannot publish it). TWO COMPOSED FENCES (defense in depth): (a) reads THIS
|
|
8734
8970
|
// incarnation's OWN gate epoch and REFUSES a superseded commit (the currency belt), and (b)
|
|
@@ -8745,8 +8981,19 @@ export class Manager {
|
|
|
8745
8981
|
// incarnation or a sibling. Refusing here rather than at each unwind site is the point --
|
|
8746
8982
|
// there is exactly one commit path, so this fences every route into it, present and future.
|
|
8747
8983
|
if (!ownsGoal)
|
|
8748
|
-
return;
|
|
8984
|
+
return false;
|
|
8749
8985
|
terminalEntered = true; // entered, not succeeded — see the catch below
|
|
8986
|
+
// A once spawn that did not succeed never served its task (delivery opens only with a
|
|
8987
|
+
// succeeded verdict), so the task is cancelled now rather than at the seat's exit. A launched
|
|
8988
|
+
// seat's debt is still settled by its exit, which proves the seat gone first; a spawn that
|
|
8989
|
+
// never launched one owes nothing more.
|
|
8990
|
+
if (o.kind !== "succeeded" && onceRef !== undefined) {
|
|
8991
|
+
const reason = o.kind === "uncertain" ? "readiness-timeout" : "launch-failed";
|
|
8992
|
+
if (this.onceWatched.has(goalKey(onceRef)))
|
|
8993
|
+
void this.commitOnceCancel(onceRef, reason).catch((e) => console.error(`! once task ${onceRef.goalId} (${reason}): ${rejectionText(e)}; its seat's exit cancels it`));
|
|
8994
|
+
else
|
|
8995
|
+
void this.settleOnceTurn(onceRef, reason);
|
|
8996
|
+
}
|
|
8750
8997
|
try {
|
|
8751
8998
|
await this.assertGoalWriterEpochCurrent(epoch); // a superseded corpse never commits
|
|
8752
8999
|
let fact;
|
|
@@ -8768,6 +9015,7 @@ export class Manager {
|
|
|
8768
9015
|
await clearGoalIndex(gw.ctx, ref); // terminal reached - the successor never reconciles it
|
|
8769
9016
|
if (acceptance)
|
|
8770
9017
|
this.agentGoals.delete(acceptance.name); // goal terminal - no cancel path left
|
|
9018
|
+
return true;
|
|
8771
9019
|
}
|
|
8772
9020
|
catch (e) {
|
|
8773
9021
|
// THE NARROWER LEG, LEFT OPEN DELIBERATELY. If the COMMIT ITSELF throws (the currency belt
|
|
@@ -8778,6 +9026,7 @@ export class Manager {
|
|
|
8778
9026
|
// next boot, which is why the index is NOT cleared above on this path. Do not "close" it
|
|
8779
9027
|
// here with a retry loop.
|
|
8780
9028
|
console.error(`! goal terminal commit for ${goalId} failed: ${rejectionText(e)}`);
|
|
9029
|
+
return false;
|
|
8781
9030
|
}
|
|
8782
9031
|
};
|
|
8783
9032
|
const bg = run({
|
|
@@ -8850,6 +9099,18 @@ export class Manager {
|
|
|
8850
9099
|
onTerminalDeferred: () => { terminalEntered = true; },
|
|
8851
9100
|
onEffect: () => { effectStarted = true; },
|
|
8852
9101
|
onUndone: (code) => { undone = code; },
|
|
9102
|
+
onOnce: async (task) => {
|
|
9103
|
+
try {
|
|
9104
|
+
const accepted = await this.acceptOnceTurn({ spawnRef: ref, fingerprint, command: ctx.subject.command, executor, acceptedAt, ...task });
|
|
9105
|
+
onceRef = accepted.ref;
|
|
9106
|
+
return accepted;
|
|
9107
|
+
}
|
|
9108
|
+
catch (e) {
|
|
9109
|
+
// The spawn goal is already bound, so its refusal is a terminal: nothing has launched.
|
|
9110
|
+
await onOutcome({ kind: "failed", data: { error: `the once task was not accepted: ${rejectionText(e)}` } });
|
|
9111
|
+
throw e;
|
|
9112
|
+
}
|
|
9113
|
+
},
|
|
8853
9114
|
});
|
|
8854
9115
|
bg.then((reply) => {
|
|
8855
9116
|
// Refused BEFORE onAccepted (hard-pin collision, capacity, persona-not-found) — no goal bound.
|
|
@@ -8970,6 +9231,8 @@ export class Manager {
|
|
|
8970
9231
|
}
|
|
8971
9232
|
// Refused here, before anything durable, so the caller's goal id stays free for another seat.
|
|
8972
9233
|
this.assertSeatTurnFree(name, t, goalId);
|
|
9234
|
+
if (target.kind === "managed" && target.agent.once)
|
|
9235
|
+
throw new EpEnvelopeError("failed-precondition", `seat "${name}" is a once seat; it takes only the task it was spawned with`, undefined, "not-executed");
|
|
8973
9236
|
const raw = (ctx.request.args ?? {});
|
|
8974
9237
|
const payload = String(raw.payload);
|
|
8975
9238
|
const deadlineAt = raw.deadlineAt === undefined ? acceptedAt + Number(raw.deadlineMs) : Number(raw.deadlineAt);
|
|
@@ -9089,21 +9352,30 @@ export class Manager {
|
|
|
9089
9352
|
// window has to refuse, the same way `serveTurnGoal` does.
|
|
9090
9353
|
if (!this.goalReconcileDone)
|
|
9091
9354
|
throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; the pending-turn index is not rebuilt yet, retry (SPEC 13.6)");
|
|
9355
|
+
// A once seat this manager does not hold, whose task has ended, is told to end itself. Nothing
|
|
9356
|
+
// else rides that answer, and the settled task is never served again.
|
|
9357
|
+
const stop = this.onceStops.get(c.uid);
|
|
9358
|
+
if (stop !== undefined && stop.owner === c.owner && stop.actor === c.actor) {
|
|
9359
|
+
stop.told = true;
|
|
9360
|
+
return { turns: [], notices: true, stop: true };
|
|
9361
|
+
}
|
|
9092
9362
|
// Two turns on one seat are serialized at the dispatch (cotal-lang 6.5): only the oldest
|
|
9093
9363
|
// unsettled one is served, and the next surfaces once it yields or its deadline denies, so a
|
|
9094
9364
|
// seat is never shown two turns at once.
|
|
9095
9365
|
// An ELAPSED turn is not servable: its hold is expirable and its run has already thrown from
|
|
9096
9366
|
// its own pause, so handing it to the seat buys work whose yield is refused. It also cannot be
|
|
9097
|
-
// allowed to stay the head, or one expired turn dams every later turn for that seat.
|
|
9367
|
+
// allowed to stay the head, or one expired turn dams every later turn for that seat. A once
|
|
9368
|
+
// task is not served before its spawn's readiness verdict, and one with no deadline never
|
|
9369
|
+
// elapses.
|
|
9098
9370
|
const now = Date.now();
|
|
9099
9371
|
const head = [...this.pendingTurns.values()]
|
|
9100
9372
|
.filter((p) => p.seat.owner === c.owner && p.seat.actor === c.actor && p.seat.uid === c.uid)
|
|
9101
|
-
.filter((p) => now < p.deadlineAt)
|
|
9373
|
+
.filter((p) => (p.deadlineAt === undefined || now < p.deadlineAt) && p.once?.deliverable !== false)
|
|
9102
9374
|
.sort((x, y) => x.acceptedAt - y.acceptedAt)
|
|
9103
9375
|
.slice(0, 1);
|
|
9104
9376
|
for (const p of head)
|
|
9105
9377
|
p.pulledAt ??= now;
|
|
9106
|
-
const turns = head.map((p) => ({ goalId: p.goalId, payload: p.payload, acceptedAt: p.acceptedAt, deadlineAt: p.deadlineAt }));
|
|
9378
|
+
const turns = head.map((p) => ({ goalId: p.goalId, payload: p.payload, acceptedAt: p.acceptedAt, ...(p.deadlineAt !== undefined ? { deadlineAt: p.deadlineAt } : {}), ...(p.once ? { once: true } : {}) }));
|
|
9107
9379
|
return { turns, notices: true };
|
|
9108
9380
|
}
|
|
9109
9381
|
/** Tell a seat that its `turn-pending` answer changed (SPEC 14.3 turn notice), over the serve
|
|
@@ -9164,28 +9436,66 @@ export class Manager {
|
|
|
9164
9436
|
if (key === undefined)
|
|
9165
9437
|
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
9166
9438
|
const p = this.pendingTurns.get(key);
|
|
9167
|
-
if (!p)
|
|
9168
|
-
|
|
9169
|
-
|
|
9170
|
-
|
|
9171
|
-
|
|
9172
|
-
|
|
9173
|
-
|
|
9174
|
-
|
|
9175
|
-
|
|
9176
|
-
|
|
9177
|
-
|
|
9178
|
-
|
|
9179
|
-
|
|
9180
|
-
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9184
|
-
|
|
9439
|
+
if (!p)
|
|
9440
|
+
return this.settledTurnAnswer(key, goalId);
|
|
9441
|
+
await this.claimTurnHold(p);
|
|
9442
|
+
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
9443
|
+
await this.assertGoalWriterEpochCurrent(epoch);
|
|
9444
|
+
const at = Date.now();
|
|
9445
|
+
const data = { status, ...(to !== undefined ? { to } : {}), ...(yieldNote !== undefined ? { note: yieldNote } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}), at };
|
|
9446
|
+
// The goal is TARGET-PINNED, so this completion proves the EXECUTOR's (the seat's) fresh
|
|
9447
|
+
// currency, not the manager's: managed-seat epochs are 0 within an incarnation (the same
|
|
9448
|
+
// convention resolveTarget serves), and the resolver answers from the live agents map — a
|
|
9449
|
+
// seat that died between its yield call and this commit refuses `expired` here.
|
|
9450
|
+
const { fact } = await commitGoalResult(gw.ctx, {
|
|
9451
|
+
ref: p.ref, now: at, cause: "complete", state: "succeeded", data,
|
|
9452
|
+
committer: { instanceId: this.managerInstanceId, epoch },
|
|
9453
|
+
executor: { lifecycleUid: p.seat.uid, epoch: 0 },
|
|
9454
|
+
resolveCurrentEpoch: (target) => this.turnExecutorEpoch(p.seat.name, p.seat.owner, p.seat.actor, target.lifecycleUid),
|
|
9455
|
+
});
|
|
9456
|
+
this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
9457
|
+
// A once task's index entry is its seat's debt, dropped at the seat's proved exit.
|
|
9458
|
+
if (!p.once)
|
|
9459
|
+
await clearGoalIndex(gw.ctx, p.ref);
|
|
9460
|
+
this.pendingTurns.delete(key);
|
|
9461
|
+
this.noticeSeatTurns(p.seat);
|
|
9462
|
+
this.markTurnLatchDropped(key, at);
|
|
9463
|
+
this.rememberSettledTurn(key, fact.state, at);
|
|
9464
|
+
if (p.once)
|
|
9465
|
+
this.endOnceSeat(p, `yielded ${status}`);
|
|
9466
|
+
this.maybeStopTurnSweep();
|
|
9467
|
+
return { goalId, state: fact.state };
|
|
9468
|
+
}
|
|
9469
|
+
/** A seat's answer for a turn no longer pending. A REPLY THAT WAS LOST IS NOT AN ANSWER THAT
|
|
9470
|
+
* FAILED: the commit deleted the pending turn, so a retry found nothing and heard `not-found`,
|
|
9471
|
+
* which a seat reads as "drop it", reporting failure for work the run already has. The answer
|
|
9472
|
+
* the first reply carried is served again instead, to the addressee it was addressed to and
|
|
9473
|
+
* nobody else. */
|
|
9474
|
+
async settledTurnAnswer(key, goalId) {
|
|
9475
|
+
const settled = this.turnAcceptances.get(key);
|
|
9476
|
+
if (settled !== undefined) {
|
|
9477
|
+
if (settled.settled !== undefined)
|
|
9478
|
+
return { goalId, state: settled.settled.state };
|
|
9479
|
+
// The in-memory answer is not yet remembered (pending deleted before the terminal CAS,
|
|
9480
|
+
// or a concurrent sweep dropped `settled` while the commit was still in flight). The
|
|
9481
|
+
// durable terminal is the one the run already has; name it, never `not-found`.
|
|
9482
|
+
if (settled.ref !== undefined && this.goalWriter) {
|
|
9483
|
+
const ended = await readGoalResult(this.goalWriter.ctx, settled.ref);
|
|
9484
|
+
if (ended !== undefined) {
|
|
9485
|
+
this.rememberSettledTurn(key, ended.state, ended.ts);
|
|
9486
|
+
return { goalId, state: ended.state };
|
|
9185
9487
|
}
|
|
9186
9488
|
}
|
|
9187
|
-
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
9188
9489
|
}
|
|
9490
|
+
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
9491
|
+
}
|
|
9492
|
+
/** Claim a pending turn's deadline hold before the seat's answer commits: one use, and a hold
|
|
9493
|
+
* that already expired drives the deadline terminal instead and refuses the answer. A once task
|
|
9494
|
+
* with no deadline has no hold. */
|
|
9495
|
+
async claimTurnHold(p) {
|
|
9496
|
+
if (p.holdToken === undefined)
|
|
9497
|
+
return;
|
|
9498
|
+
const gw = this.goalWriter;
|
|
9189
9499
|
const cpRef = { endpoint: p.ref.endpoint, token: p.holdToken };
|
|
9190
9500
|
try {
|
|
9191
9501
|
await resumeCheckpoint(gw.ctx.kv, gw.ctx.js, gw.ctx.jsm, this.space, { ref: cpRef, presenter: { id: MANAGER_ENDPOINT, lifecycleUid: this.managerInstanceId }, now: Date.now() });
|
|
@@ -9194,33 +9504,53 @@ export class Manager {
|
|
|
9194
9504
|
const settle = await readCheckpointSettle(gw.ctx.jsm, this.space, cpRef).catch(() => undefined);
|
|
9195
9505
|
if (settle?.settle === "expired") {
|
|
9196
9506
|
await this.commitTurnDeadline(p);
|
|
9197
|
-
throw new EpEnvelopeError("failed-precondition", `turn "${goalId}" elapsed its deadline before the
|
|
9507
|
+
throw new EpEnvelopeError("failed-precondition", `turn "${p.goalId}" elapsed its deadline before the seat answered; the deadline terminal stands (SPEC 13.6)`);
|
|
9198
9508
|
}
|
|
9199
|
-
// Already resumed BY THIS MANAGER (the hold is holder-bound): a prior
|
|
9200
|
-
//
|
|
9509
|
+
// Already resumed BY THIS MANAGER (the hold is holder-bound): a prior attempt claimed it
|
|
9510
|
+
// and then failed to commit. Fall through and commit; the claim is ours to finish.
|
|
9201
9511
|
if (settle?.settle !== "resumed")
|
|
9202
9512
|
throw e;
|
|
9203
9513
|
}
|
|
9514
|
+
}
|
|
9515
|
+
/** A once seat's report that its task's handoff to its harness was not confirmed. The seat sealed
|
|
9516
|
+
* the task before it reported, so nothing hands it over again; the task fails
|
|
9517
|
+
* `delivery-unconfirmed` and the seat is stopped. The commit proves the seat's live currency, as
|
|
9518
|
+
* a yield's does: a seat already gone commits nothing here, and its exit cancels the task. Only
|
|
9519
|
+
* the addressed seat may report, only for its once task; a repeat after the terminal is answered
|
|
9520
|
+
* with it. */
|
|
9521
|
+
async serveTurnUnconfirmed(c, raw) {
|
|
9522
|
+
if (this.execution === "none")
|
|
9523
|
+
throw new EpEnvelopeError("unimplemented", this.executionRefusal());
|
|
9524
|
+
if (!this.goalReconcileDone)
|
|
9525
|
+
throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; this report's turn may not be indexed yet, retry (SPEC 13.6)");
|
|
9526
|
+
const gw = this.goalWriter;
|
|
9527
|
+
if (!gw)
|
|
9528
|
+
throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing (SPEC 13.6)");
|
|
9529
|
+
const goalId = String(raw.goalId);
|
|
9530
|
+
const key = this.seatTurnKey(c, goalId);
|
|
9531
|
+
if (key === undefined)
|
|
9532
|
+
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
9533
|
+
const p = this.pendingTurns.get(key);
|
|
9534
|
+
if (!p)
|
|
9535
|
+
return this.settledTurnAnswer(key, goalId);
|
|
9536
|
+
if (!p.once)
|
|
9537
|
+
throw new EpEnvelopeError("failed-precondition", `turn "${goalId}" is a run's turn; turn-unconfirmed reports only a once task, and a run's turn is offered again`);
|
|
9538
|
+
await this.claimTurnHold(p);
|
|
9204
9539
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
9205
9540
|
await this.assertGoalWriterEpochCurrent(epoch);
|
|
9206
9541
|
const at = Date.now();
|
|
9207
|
-
const data = { status, ...(to !== undefined ? { to } : {}), ...(yieldNote !== undefined ? { note: yieldNote } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}), at };
|
|
9208
|
-
// The goal is TARGET-PINNED, so this completion proves the EXECUTOR's (the seat's) fresh
|
|
9209
|
-
// currency, not the manager's: managed-seat epochs are 0 within an incarnation (the same
|
|
9210
|
-
// convention resolveTarget serves), and the resolver answers from the live agents map — a
|
|
9211
|
-
// seat that died between its yield call and this commit refuses `expired` here.
|
|
9212
9542
|
const { fact } = await commitGoalResult(gw.ctx, {
|
|
9213
|
-
ref: p.ref, now: at, cause: "complete", state: "
|
|
9543
|
+
ref: p.ref, now: at, cause: "complete", state: "failed", data: { reason: "delivery-unconfirmed", at },
|
|
9214
9544
|
committer: { instanceId: this.managerInstanceId, epoch },
|
|
9215
9545
|
executor: { lifecycleUid: p.seat.uid, epoch: 0 },
|
|
9216
9546
|
resolveCurrentEpoch: (target) => this.turnExecutorEpoch(p.seat.name, p.seat.owner, p.seat.actor, target.lifecycleUid),
|
|
9217
9547
|
});
|
|
9218
9548
|
this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
9219
|
-
await clearGoalIndex(gw.ctx, p.ref);
|
|
9220
9549
|
this.pendingTurns.delete(key);
|
|
9221
9550
|
this.noticeSeatTurns(p.seat);
|
|
9222
9551
|
this.markTurnLatchDropped(key, at);
|
|
9223
9552
|
this.rememberSettledTurn(key, fact.state, at);
|
|
9553
|
+
this.endOnceSeat(p, "delivery-unconfirmed");
|
|
9224
9554
|
this.maybeStopTurnSweep();
|
|
9225
9555
|
return { goalId, state: fact.state };
|
|
9226
9556
|
}
|
|
@@ -9275,7 +9605,10 @@ export class Manager {
|
|
|
9275
9605
|
this.noticeSeatTurns(p.seat);
|
|
9276
9606
|
this.markTurnLatchDropped(key);
|
|
9277
9607
|
this.rememberSettledTurn(key, fact.state, Date.now());
|
|
9278
|
-
|
|
9608
|
+
if (p.once)
|
|
9609
|
+
this.endOnceSeat(p, "cancelled");
|
|
9610
|
+
else
|
|
9611
|
+
await clearGoalIndex(gw.ctx, ref);
|
|
9279
9612
|
this.maybeStopTurnSweep();
|
|
9280
9613
|
return { goalId, state: fact.state };
|
|
9281
9614
|
}
|
|
@@ -9295,17 +9628,27 @@ export class Manager {
|
|
|
9295
9628
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
9296
9629
|
try {
|
|
9297
9630
|
await this.assertGoalWriterEpochCurrent(epoch);
|
|
9298
|
-
const { fact } = await commitGoalResult(gw.ctx, { ref: p.ref, now: Date.now(), cause: "deny", denial: { kind: "hold-expired", token: p.holdToken }, data: { reason: "turn-deadline", ...(p.seatDiedAt !== undefined ? { agentDownAt: p.seatDiedAt } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}) }, committer: { instanceId: this.managerInstanceId, epoch } });
|
|
9631
|
+
const { fact } = await commitGoalResult(gw.ctx, { ref: p.ref, now: Date.now(), cause: "deny", denial: { kind: "hold-expired", token: p.holdToken }, data: { reason: "turn-deadline", ...(p.seatDiedAt !== undefined ? { agentDownAt: p.seatDiedAt } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}), ...(this.onceSeatUnheld(p) ? { seat: UNHELD_SEAT } : {}) }, committer: { instanceId: this.managerInstanceId, epoch } });
|
|
9299
9632
|
this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
9300
9633
|
// Remember BEFORE the index clear: a late yield that arrives after the pending latch
|
|
9301
9634
|
// (the delete at the top of this function) and after this CAS has to name this terminal.
|
|
9302
9635
|
// Clearing first, then remembering, left a window where maybeStopTurnSweep saw no pending
|
|
9303
9636
|
// turn and no settled answer, wiped the acceptance, and rememberSettledTurn no-op'd.
|
|
9304
9637
|
this.rememberSettledTurn(key, fact.state, Date.now());
|
|
9305
|
-
|
|
9638
|
+
if (p.once)
|
|
9639
|
+
this.endOnceSeat(p, "turn-deadline");
|
|
9640
|
+
else
|
|
9641
|
+
await clearGoalIndex(gw.ctx, p.ref);
|
|
9306
9642
|
}
|
|
9307
9643
|
catch (e) {
|
|
9308
9644
|
console.error(`! turn deadline terminal for ${p.goalId}: ${rejectionText(e)}`);
|
|
9645
|
+
// A once seat stops at its deadline whether or not the terminal landed, and the task stays
|
|
9646
|
+
// pending so the turn sweep writes the terminal again while this manager runs.
|
|
9647
|
+
if (p.once) {
|
|
9648
|
+
this.pendingTurns.set(key, p);
|
|
9649
|
+
this.endOnceSeat(p, "turn-deadline");
|
|
9650
|
+
this.ensureTurnSweep();
|
|
9651
|
+
}
|
|
9309
9652
|
}
|
|
9310
9653
|
this.maybeStopTurnSweep();
|
|
9311
9654
|
}
|
|
@@ -9361,7 +9704,7 @@ export class Manager {
|
|
|
9361
9704
|
* terminals AND drops settled answers once their retry window has passed, so a stop while the
|
|
9362
9705
|
* second is still owed would leave one entry per turn for the process lifetime. */
|
|
9363
9706
|
maybeStopTurnSweep() {
|
|
9364
|
-
if (this.pendingTurns.size > 0)
|
|
9707
|
+
if (this.pendingTurns.size > 0 || this.onceSettles.size > 0 || this.onceInherited.size > 0 || [...this.onceStops.values()].some((s) => !s.told))
|
|
9365
9708
|
return;
|
|
9366
9709
|
// An acceptance with no `settled` yet is the in-flight deadline commit: pending was the
|
|
9367
9710
|
// latch and is already gone, the terminal CAS has not returned, and a concurrent sweep
|
|
@@ -9401,7 +9744,7 @@ export class Manager {
|
|
|
9401
9744
|
// Repeat the notice until it does; a seat that pulled is not told again for that turn.
|
|
9402
9745
|
const heads = new Map();
|
|
9403
9746
|
for (const p of this.pendingTurns.values()) {
|
|
9404
|
-
if (now >= p.deadlineAt)
|
|
9747
|
+
if ((p.deadlineAt !== undefined && now >= p.deadlineAt) || p.once?.deliverable === false)
|
|
9405
9748
|
continue;
|
|
9406
9749
|
const k = `${p.seat.owner}.${p.seat.actor}.${p.seat.uid}`;
|
|
9407
9750
|
const h = heads.get(k);
|
|
@@ -9411,8 +9754,16 @@ export class Manager {
|
|
|
9411
9754
|
for (const p of heads.values())
|
|
9412
9755
|
if (p.pulledAt === undefined)
|
|
9413
9756
|
this.noticeSeatTurns(p.seat);
|
|
9757
|
+
for (const stop of this.onceStops.values())
|
|
9758
|
+
if (!stop.told)
|
|
9759
|
+
this.noticeSeatTurns(stop);
|
|
9760
|
+
for (const owed of [...this.onceSettles.values()])
|
|
9761
|
+
if (!owed.flight)
|
|
9762
|
+
void this.settleOnceTurn(owed.ref, owed.reason);
|
|
9763
|
+
if (this.onceInherited.size > 0)
|
|
9764
|
+
await this.settleInheritedOnce().catch((e) => console.error(`! inherited once tasks: ${rejectionText(e)}`));
|
|
9414
9765
|
for (const p of [...this.pendingTurns.values()]) {
|
|
9415
|
-
if (now < p.deadlineAt)
|
|
9766
|
+
if (p.deadlineAt === undefined || now < p.deadlineAt)
|
|
9416
9767
|
continue;
|
|
9417
9768
|
try {
|
|
9418
9769
|
const settle = await this.expireTurnHold(p);
|
|
@@ -9437,15 +9788,21 @@ export class Manager {
|
|
|
9437
9788
|
const status = await readGoalStatus(gw.ctx, entry.ref);
|
|
9438
9789
|
if (status === undefined)
|
|
9439
9790
|
return; // index points at no goal record: a dead pointer, left for honesty
|
|
9791
|
+
const parsed = entry.note !== undefined ? parseTurnNote(entry.note) : undefined;
|
|
9792
|
+
const ownOnce = parsed?.once === true && entry.iid === this.managerInstanceId && entry.allocated?.owner !== undefined;
|
|
9440
9793
|
if (GOAL_TERMINAL_STATES.includes(status.value.state)) {
|
|
9441
|
-
|
|
9794
|
+
// A once task's entry is its seat's debt until that seat's exit is proved, so it outlives
|
|
9795
|
+
// the terminal: the stop the predecessor owed may never have landed.
|
|
9796
|
+
if (ownOnce)
|
|
9797
|
+
await this.inheritOnce(entry.ref, entry.allocated, parsed);
|
|
9798
|
+
else
|
|
9799
|
+
await clearGoalIndex(gw.ctx, entry.ref);
|
|
9442
9800
|
return;
|
|
9443
9801
|
}
|
|
9444
9802
|
if (entry.iid !== this.managerInstanceId) {
|
|
9445
9803
|
console.error(`turn reconcile ${entry.ref.goalId}: accepted by instance "${entry.iid}", not this incarnation "${this.managerInstanceId}"; left for its owner (never a cross-instance settle)`);
|
|
9446
9804
|
return;
|
|
9447
9805
|
}
|
|
9448
|
-
const parsed = entry.note !== undefined ? parseTurnNote(entry.note) : undefined;
|
|
9449
9806
|
if (entry.allocated?.owner === undefined || parsed === undefined) {
|
|
9450
9807
|
console.error(`turn reconcile ${entry.ref.goalId}: the relay record is garbled (no floor naming the seat's owner, or an unparseable note); left unsettled`);
|
|
9451
9808
|
return;
|
|
@@ -9465,17 +9822,33 @@ export class Manager {
|
|
|
9465
9822
|
seat: { name: entry.allocated.name, owner: entry.allocated.owner, actor: entry.allocated.actor, uid: entry.allocated.uid },
|
|
9466
9823
|
payload: parsed.payload,
|
|
9467
9824
|
acceptedAt: spec.value.acceptedAt,
|
|
9468
|
-
|
|
9469
|
-
holdToken: parsed.holdToken ?? turnHoldToken(entry.ref.goalId)
|
|
9825
|
+
holdEpoch: parsed.holdEpoch,
|
|
9826
|
+
...(parsed.deadlineAt !== undefined ? { deadlineAt: parsed.deadlineAt, holdToken: parsed.holdToken ?? turnHoldToken(entry.ref.goalId) } : {}),
|
|
9470
9827
|
...(parsed.handoffFrom !== undefined ? { handoffFrom: parsed.handoffFrom } : {}),
|
|
9828
|
+
// An inherited once task is served again only once its seat is bound to this incarnation.
|
|
9829
|
+
...(parsed.once ? { once: { deliverable: false } } : {}),
|
|
9471
9830
|
};
|
|
9472
9831
|
// The hold is the relay's bounded ending, minted AFTER the index entry and the goal record,
|
|
9473
9832
|
// so a crash between them leaves a goal with no hold: nothing could ever settle it, and the
|
|
9474
9833
|
// sweep would report the same missing pause every tick. Left unsettled, said once.
|
|
9475
|
-
if ((await readCheckpointSpec(gw.ctx.kv, { endpoint: p.ref.endpoint, token: p.holdToken })) === undefined) {
|
|
9834
|
+
if (p.holdToken !== undefined && (await readCheckpointSpec(gw.ctx.kv, { endpoint: p.ref.endpoint, token: p.holdToken })) === undefined) {
|
|
9835
|
+
// A once task's hold is minted before its seat launches, so without one no seat ever ran it:
|
|
9836
|
+
// the task is cancelled and its entry dropped, with nothing launched to prove gone.
|
|
9837
|
+
if (parsed.once) {
|
|
9838
|
+
await this.settleOnceTurn(p.ref, "launch-failed");
|
|
9839
|
+
return;
|
|
9840
|
+
}
|
|
9476
9841
|
console.error(`turn reconcile ${p.goalId}: its deadline hold was never minted (the accept crashed before it); left unsettled`);
|
|
9477
9842
|
return;
|
|
9478
9843
|
}
|
|
9844
|
+
// A once task whose spawn has no succeeded terminal was never served, whatever seat exists:
|
|
9845
|
+
// it is cancelled, its row is not rebuilt so a later pull finds nothing, and its seat is
|
|
9846
|
+
// settled as an inherited debt.
|
|
9847
|
+
if (parsed.once && !(await this.onceServed(entry.ref, parsed))) {
|
|
9848
|
+
await this.commitOnceCancel(entry.ref, NEVER_SERVED);
|
|
9849
|
+
await this.inheritOnce(entry.ref, entry.allocated, parsed);
|
|
9850
|
+
return;
|
|
9851
|
+
}
|
|
9479
9852
|
const key = goalKey(p.ref);
|
|
9480
9853
|
this.pendingTurns.set(key, p);
|
|
9481
9854
|
// THE ACCEPTANCE TOO, or the adopted turn's answer is never remembered: `rememberSettledTurn`
|
|
@@ -9497,7 +9870,312 @@ export class Manager {
|
|
|
9497
9870
|
// terminal then carried `agentDownAt`, so the run raised L4002 for a seat that was alive the
|
|
9498
9871
|
// whole time where the reference says L4003. The reap hook is the only honest writer of that
|
|
9499
9872
|
// fact: it fires when a managed seat actually dies.
|
|
9500
|
-
console.error(`turn reconcile ${p.goalId}: pending relay to ${p.seat.name}/${p.seat.uid} adopted (deadline ${p.deadlineAt})`);
|
|
9873
|
+
console.error(`turn reconcile ${p.goalId}: pending relay to ${p.seat.name}/${p.seat.uid} adopted (deadline ${p.deadlineAt ?? "none"})`);
|
|
9874
|
+
if (parsed.once)
|
|
9875
|
+
await this.inheritOnce(entry.ref, entry.allocated, parsed);
|
|
9876
|
+
this.ensureTurnSweep();
|
|
9877
|
+
}
|
|
9878
|
+
// ---- once (a spawn whose prompt is its one relayed task) ----------------------------------------
|
|
9879
|
+
//
|
|
9880
|
+
// A once spawn's prompt is not a kickoff: the manager accepts it as one turn addressed to the
|
|
9881
|
+
// allocated incarnation, under the spawner's own caller triple and a goal id derived from the
|
|
9882
|
+
// spawn's, so the spawner reads the task's outcome with `goal-result`. The seat is served it from
|
|
9883
|
+
// the spawn's readiness verdict on. The task's terminal (yield, deadline) stops the seat, and the
|
|
9884
|
+
// seat's proved exit cancels a task that has none. Until that exit, the task's index entry is the
|
|
9885
|
+
// seat's debt: a successor finds it there.
|
|
9886
|
+
/** Accept a once spawn's task, inline in the spawn's accept seam: nothing has launched yet, so a
|
|
9887
|
+
* refusal here fails the spawn and leaves no seat. Mirrors {@link serveTurnGoal}'s accept, with
|
|
9888
|
+
* the goal ref and the seat given rather than read off a request. A task with no deadline mints
|
|
9889
|
+
* no hold; its goal still records the spawn's readiness budget, which a successor needs to settle
|
|
9890
|
+
* it. */
|
|
9891
|
+
async acceptOnceTurn(o) {
|
|
9892
|
+
const gw = this.goalWriter;
|
|
9893
|
+
if (!gw)
|
|
9894
|
+
throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing; a once task cannot be accepted (SPEC 13.6)");
|
|
9895
|
+
const serve = this.serviceServe;
|
|
9896
|
+
if (o.deadlineMs !== undefined && !serve)
|
|
9897
|
+
throw new EpEnvelopeError("unavailable", "the manager service endpoint is not serving; a once task's deadline hold is armed over its connection (SPEC 13.9)");
|
|
9898
|
+
const ref = { endpoint: o.spawnRef.endpoint, caller: { ...o.spawnRef.caller }, goalId: digestIdToken(`${o.spawnRef.goalId}:once`) };
|
|
9899
|
+
const key = goalKey(ref);
|
|
9900
|
+
const deadlineAt = o.deadlineMs === undefined ? undefined : o.acceptedAt + o.deadlineMs;
|
|
9901
|
+
if (deadlineAt !== undefined && deadlineAt > MAX_SCHEDULE_MS)
|
|
9902
|
+
throw new EpEnvelopeError("failed-precondition", `once: the deadline ends after the deadline scheduler's last instant (the end of year 9999)`);
|
|
9903
|
+
const holdToken = deadlineAt === undefined ? undefined : turnHoldToken(key);
|
|
9904
|
+
const { payload, note } = onceTurnNote({ prompt: o.prompt, spawner: o.spawner, holdEpoch: o.executor.epoch, ...(deadlineAt !== undefined ? { deadlineAt, holdToken } : {}), ...(o.retire !== undefined ? { retire: o.retire } : {}), spawn: o.spawnRef.goalId });
|
|
9905
|
+
const { seat } = o;
|
|
9906
|
+
const idx = await recordGoalIndex(gw.ctx, ref, o.executor.lifecycleUid, { name: seat.name, owner: seat.owner, actor: seat.actor, uid: seat.uid, readinessDeadlineMs: o.readinessDeadlineMs }, note);
|
|
9907
|
+
if (!idx.recorded)
|
|
9908
|
+
throw new EpEnvelopeError("failed-precondition", `once: the task goal "${ref.goalId}" is already recorded; a spawn accepts its task once (SPEC 13.6)`);
|
|
9909
|
+
const b = await bindGoal(gw.ctx, ref, o.fingerprint);
|
|
9910
|
+
if (!b.bound) {
|
|
9911
|
+
await clearGoalIndex(gw.ctx, ref);
|
|
9912
|
+
throw new EpEnvelopeError("failed-precondition", `once: the task goal "${ref.goalId}" is already bound; a spawn accepts its task once (SPEC 13.6)`);
|
|
9913
|
+
}
|
|
9914
|
+
try {
|
|
9915
|
+
await createGoal(gw.ctx, ref, {
|
|
9916
|
+
fingerprint: o.fingerprint,
|
|
9917
|
+
command: o.command,
|
|
9918
|
+
caller: { id: `${ref.caller.owner}.${ref.caller.actor}`, lifecycleUid: ref.caller.uid },
|
|
9919
|
+
acceptedEpoch: o.executor.epoch,
|
|
9920
|
+
requestId: ref.goalId,
|
|
9921
|
+
sourceSeq: 0,
|
|
9922
|
+
acceptedAt: o.acceptedAt,
|
|
9923
|
+
readinessDeadlineMs: o.readinessDeadlineMs,
|
|
9924
|
+
target: { owner: seat.owner, actor: seat.actor, lifecycleUid: seat.uid, mappingRevision: 0 },
|
|
9925
|
+
});
|
|
9926
|
+
if (deadlineAt !== undefined)
|
|
9927
|
+
await mintCheckpoint(gw.ctx.kv, jetstream(serve.nc), this.space, {
|
|
9928
|
+
ref: { endpoint: ref.endpoint, token: holdToken },
|
|
9929
|
+
instanceId: this.managerInstanceId,
|
|
9930
|
+
epoch: o.executor.epoch,
|
|
9931
|
+
goal: { caller: { ...ref.caller }, goalId: ref.goalId },
|
|
9932
|
+
holder: { id: MANAGER_ENDPOINT, lifecycleUid: this.managerInstanceId },
|
|
9933
|
+
deadline: deadlineAt,
|
|
9934
|
+
now: o.acceptedAt,
|
|
9935
|
+
});
|
|
9936
|
+
}
|
|
9937
|
+
catch (e) {
|
|
9938
|
+
// Nothing launched and nothing was served, so the task is withdrawn with `cancel`.
|
|
9939
|
+
try {
|
|
9940
|
+
await transitionGoal(gw.ctx, ref, "cancelling", { fields: { cancelMode: "terminate" } });
|
|
9941
|
+
await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "cancel", data: { cancelledBy: "manager", reason: "accept-failed" }, committer: { instanceId: this.managerInstanceId, epoch: o.executor.epoch } });
|
|
9942
|
+
await clearGoalIndex(gw.ctx, ref);
|
|
9943
|
+
}
|
|
9944
|
+
catch (e2) {
|
|
9945
|
+
console.error(`! once task accept unwind for ${ref.goalId}: ${rejectionText(e2)}`);
|
|
9946
|
+
}
|
|
9947
|
+
throw e;
|
|
9948
|
+
}
|
|
9949
|
+
this.pendingTurns.set(key, {
|
|
9950
|
+
ref, goalId: ref.goalId, seat: { ...seat }, payload, acceptedAt: o.acceptedAt, holdEpoch: o.executor.epoch,
|
|
9951
|
+
...(deadlineAt !== undefined ? { deadlineAt, holdToken } : {}),
|
|
9952
|
+
once: { deliverable: false },
|
|
9953
|
+
});
|
|
9954
|
+
this.turnAcceptances.set(key, {
|
|
9955
|
+
acceptance: { name: seat.name, owner: seat.owner, actor: seat.actor, uid: seat.uid, goalId: ref.goalId, fingerprint: o.fingerprint, ...(deadlineAt !== undefined ? { deadlineAt } : {}), executor: o.executor },
|
|
9956
|
+
ref,
|
|
9957
|
+
});
|
|
9958
|
+
this.ensureTurnSweep();
|
|
9959
|
+
this.emitGoalProgress(ref, o.executor.epoch, { phase: "relayed" });
|
|
9960
|
+
return { ref, ...(deadlineAt !== undefined ? { deadlineAt } : {}) };
|
|
9961
|
+
}
|
|
9962
|
+
/** How a once seat's startup watch ended, waited for until {@link STARTUP_OUTCOME_MS} after its
|
|
9963
|
+
* launch or the readiness window, whichever is longer. A watch that has not ended by then, or a
|
|
9964
|
+
* seat being stopped, answers what the handle reports at that moment. */
|
|
9965
|
+
async awaitStartupOutcome(a, launchedAt, readinessTimeoutMs) {
|
|
9966
|
+
const until = launchedAt + Math.max(readinessTimeoutMs, STARTUP_OUTCOME_MS);
|
|
9967
|
+
for (;;) {
|
|
9968
|
+
const outcome = a.handle.startupOutcome?.() ?? "pending";
|
|
9969
|
+
if (outcome !== "pending" || a.terminalizing || Date.now() >= until)
|
|
9970
|
+
return outcome;
|
|
9971
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
9972
|
+
}
|
|
9973
|
+
}
|
|
9974
|
+
/** Serve a once task from now on: its spawn's succeeded terminal is written. */
|
|
9975
|
+
openOnceDelivery(ref) {
|
|
9976
|
+
const p = this.pendingTurns.get(goalKey(ref));
|
|
9977
|
+
if (p?.once === undefined || p.once.deliverable)
|
|
9978
|
+
return;
|
|
9979
|
+
p.once.deliverable = true;
|
|
9980
|
+
this.noticeSeatTurns(p.seat);
|
|
9981
|
+
}
|
|
9982
|
+
/** Stop the seat of a once task that just ended, found by its incarnation and never by its name.
|
|
9983
|
+
* After the readiness verdict the spawn goal's outcome belongs to that verdict, so the stop lets
|
|
9984
|
+
* go of the spawn goal first and never cancels it. Before the verdict it is a deliberate stop,
|
|
9985
|
+
* which cancels the spawn like a despawn does. */
|
|
9986
|
+
endOnceSeat(p, reason) {
|
|
9987
|
+
const a = this.findManagedByTarget({ owner: p.seat.owner, actor: p.seat.actor, lifecycleUid: p.seat.uid });
|
|
9988
|
+
if (a?.once === undefined || a.terminalizing)
|
|
9989
|
+
return;
|
|
9990
|
+
if (p.once?.deliverable)
|
|
9991
|
+
this.agentGoals.delete(a.name);
|
|
9992
|
+
this.stopOnceSeat(a, reason);
|
|
9993
|
+
}
|
|
9994
|
+
/** A once seat's helpers end with its task. A natural exit reaps them in {@link onAgentExit}; a
|
|
9995
|
+
* stop frees the slot first, so this stop reaps them itself. */
|
|
9996
|
+
stopOnceSeat(a, reason) {
|
|
9997
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
9998
|
+
this.stopHandle(a, true);
|
|
9999
|
+
this.trackStoppedHandle(a, false, { kind: "stopped-once", reason });
|
|
10000
|
+
void this.cancelAgentGoal(a.name, "graceful");
|
|
10001
|
+
}
|
|
10002
|
+
/** The once task's exit hook, on the handle the seat was launched with: whichever path ends the
|
|
10003
|
+
* seat, its proved exit cancels a task that has no terminal yet and discharges the seat's debt. An
|
|
10004
|
+
* exit that cannot be proved leaves both. */
|
|
10005
|
+
watchOnceExit(a) {
|
|
10006
|
+
const once = a.once;
|
|
10007
|
+
this.onceWatched.add(goalKey(once.ref));
|
|
10008
|
+
const retire = once.retire === undefined ? undefined : { connector: once.retire, name: a.name, uid: a.lifecycleUid };
|
|
10009
|
+
// A startup watch that failed stopped the seat itself, possibly before the launch read that outcome.
|
|
10010
|
+
void a.handle.waitForExit().then(() => this.settleOnceTurn(once.ref, once.stopReason ?? (a.handle.startupOutcome?.() === "failed" ? "launch-failed" : "seat-exited"), retire), (e) => console.error(`! ${a.name}: its exit could not be proved (${rejectionText(e)}); its once task stays owed`));
|
|
10011
|
+
}
|
|
10012
|
+
/** End a once task that can no longer run: once its connector, when it has a `retire` hook, proves
|
|
10013
|
+
* the seat's launched processes gone, cancel the task with `reason` if it has no terminal yet,
|
|
10014
|
+
* then drop its index entry. One flight per task; a failure keeps it in {@link onceSettles} and
|
|
10015
|
+
* the turn sweep retries it. */
|
|
10016
|
+
settleOnceTurn(ref, reason, retire) {
|
|
10017
|
+
const key = goalKey(ref);
|
|
10018
|
+
const owed = this.onceSettles.get(key) ?? { ref, reason, ...(retire !== undefined ? { retire } : {}) };
|
|
10019
|
+
if (owed.flight)
|
|
10020
|
+
return owed.flight;
|
|
10021
|
+
this.onceSettles.set(key, owed);
|
|
10022
|
+
const flight = this.runOnceSettle(key, owed).finally(() => { owed.flight = undefined; });
|
|
10023
|
+
owed.flight = flight;
|
|
10024
|
+
return flight;
|
|
10025
|
+
}
|
|
10026
|
+
async runOnceSettle(key, owed) {
|
|
10027
|
+
const gw = this.goalWriter;
|
|
10028
|
+
if (!gw)
|
|
10029
|
+
return;
|
|
10030
|
+
// The pending entry is the latch the yield and the deadline race on; a task settled here is
|
|
10031
|
+
// served to no one again.
|
|
10032
|
+
const p = this.pendingTurns.get(key);
|
|
10033
|
+
// A task past its deadline owes the deadline terminal, which the turn sweep writes again until
|
|
10034
|
+
// it lands; the settle waits for it rather than record a cancel in its place.
|
|
10035
|
+
const denyOwed = p?.deadlineAt !== undefined && p.deadlineAt <= Date.now();
|
|
10036
|
+
if (p && !denyOwed) {
|
|
10037
|
+
this.pendingTurns.delete(key);
|
|
10038
|
+
this.markTurnLatchDropped(key);
|
|
10039
|
+
this.noticeSeatTurns(p.seat);
|
|
10040
|
+
}
|
|
10041
|
+
try {
|
|
10042
|
+
if (denyOwed)
|
|
10043
|
+
throw new Error("its deadline terminal is not written yet");
|
|
10044
|
+
// A seat whose exit is proved may have left processes of its own running the task: the task
|
|
10045
|
+
// is not recorded as ended, nor its debt cleared, before they are proved gone.
|
|
10046
|
+
const retired = owed.retire === undefined ? undefined : await this.retireOnceSeat(owed.retire);
|
|
10047
|
+
await this.commitOnceCancel(owed.ref, owed.reason);
|
|
10048
|
+
await clearGoalIndex(gw.ctx, owed.ref);
|
|
10049
|
+
this.onceSettles.delete(key);
|
|
10050
|
+
this.onceWatched.delete(key);
|
|
10051
|
+
await retired?.discard().catch((e) => console.error(`! once task ${owed.ref.goalId}: its retirement receipt was not discarded (${rejectionText(e)})`));
|
|
10052
|
+
}
|
|
10053
|
+
catch (e) {
|
|
10054
|
+
console.error(`! once task ${owed.ref.goalId} (${owed.reason}): ${rejectionText(e)}; retried by the turn sweep`);
|
|
10055
|
+
this.ensureTurnSweep();
|
|
10056
|
+
}
|
|
10057
|
+
this.maybeStopTurnSweep();
|
|
10058
|
+
}
|
|
10059
|
+
/** Cancel a once task with `reason` unless it already has a terminal. */
|
|
10060
|
+
async commitOnceCancel(ref, reason) {
|
|
10061
|
+
const gw = this.goalWriter;
|
|
10062
|
+
if ((await readGoalResult(gw.ctx, ref)) !== undefined)
|
|
10063
|
+
return;
|
|
10064
|
+
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
10065
|
+
await this.assertGoalWriterEpochCurrent(epoch);
|
|
10066
|
+
if ((await readGoalStatus(gw.ctx, ref))?.value.state !== "cancelling")
|
|
10067
|
+
await transitionGoal(gw.ctx, ref, "cancelling", { fields: { cancelMode: "terminate" } });
|
|
10068
|
+
const { fact } = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "cancel", data: { cancelledBy: "manager", reason }, committer: { instanceId: this.managerInstanceId, epoch } });
|
|
10069
|
+
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
10070
|
+
this.rememberSettledTurn(goalKey(ref), fact.state, Date.now());
|
|
10071
|
+
}
|
|
10072
|
+
/** Ask the seat's connector to prove its launched processes gone, within {@link ONCE_RETIRE_MS}.
|
|
10073
|
+
* Throws without that proof. */
|
|
10074
|
+
async retireOnceSeat(r) {
|
|
10075
|
+
const connector = await this.resolveConnector(r.connector);
|
|
10076
|
+
if (!connector.retireAfterExit)
|
|
10077
|
+
throw new Error(`the ${r.connector} connector cannot prove seat ${r.name}'s launched processes gone`);
|
|
10078
|
+
return withTimeout(connector.retireAfterExit({ space: this.space, name: r.name, lifecycleUid: r.uid, workspaceRoot: this.workspaceRoot }), ONCE_RETIRE_MS, `seat ${r.name}'s launched processes were not proved gone within ${ONCE_RETIRE_MS}ms`);
|
|
10079
|
+
}
|
|
10080
|
+
/** Record a once task a predecessor accepted, and bind its seat if this incarnation already
|
|
10081
|
+
* adopted it. The static adoption and the goal reconcile run in either order, so each side
|
|
10082
|
+
* binds what the other has already found. */
|
|
10083
|
+
async inheritOnce(ref, allocated, note) {
|
|
10084
|
+
this.onceInherited.set(allocated.uid, {
|
|
10085
|
+
ref, seat: { name: allocated.name, owner: allocated.owner, actor: allocated.actor, uid: allocated.uid },
|
|
10086
|
+
...(note.deadlineAt !== undefined ? { deadlineAt: note.deadlineAt } : {}),
|
|
10087
|
+
readinessMs: allocated.readinessDeadlineMs ?? this.readinessTimeoutMs,
|
|
10088
|
+
...(note.retire !== undefined ? { retire: note.retire } : {}),
|
|
10089
|
+
});
|
|
10090
|
+
const a = this.findManagedByTarget({ owner: allocated.owner, actor: allocated.actor, lifecycleUid: allocated.uid });
|
|
10091
|
+
if (a !== undefined)
|
|
10092
|
+
await this.bindOnceSeat(a);
|
|
10093
|
+
this.ensureTurnSweep();
|
|
10094
|
+
}
|
|
10095
|
+
/** An inherited once task whose seat this manager cannot hold: it adopts no seats (an open or
|
|
10096
|
+
* user-mode mesh) and the seat is not bound here. */
|
|
10097
|
+
onceSeatUnheld(p) {
|
|
10098
|
+
return p.once !== undefined && this.onceInherited.has(p.seat.uid) && !(this.auth !== undefined && !this.userMode);
|
|
10099
|
+
}
|
|
10100
|
+
/** Whether a once task was served: its spawn's succeeded terminal is written. */
|
|
10101
|
+
async onceServed(ref, note) {
|
|
10102
|
+
if (note.spawn === undefined)
|
|
10103
|
+
return false;
|
|
10104
|
+
const spawn = await readGoalResult(this.goalWriter.ctx, { endpoint: ref.endpoint, caller: ref.caller, goalId: note.spawn });
|
|
10105
|
+
return spawn?.state === "succeeded";
|
|
10106
|
+
}
|
|
10107
|
+
/** Make an adopted seat a once seat again: its exit settles its task, and the task is served to
|
|
10108
|
+
* it. A task that ended before the restart owes the stop the predecessor may not have made. */
|
|
10109
|
+
async bindOnceSeat(a) {
|
|
10110
|
+
const inherited = this.onceInherited.get(a.lifecycleUid);
|
|
10111
|
+
if (inherited === undefined || a.once !== undefined)
|
|
10112
|
+
return;
|
|
10113
|
+
this.onceInherited.delete(a.lifecycleUid);
|
|
10114
|
+
a.once = { ref: inherited.ref, ...(inherited.retire !== undefined ? { retire: inherited.retire } : {}) };
|
|
10115
|
+
this.watchOnceExit(a);
|
|
10116
|
+
const p = this.pendingTurns.get(goalKey(inherited.ref));
|
|
10117
|
+
if (p?.once) {
|
|
10118
|
+
p.once.deliverable = true;
|
|
10119
|
+
this.noticeSeatTurns(p.seat);
|
|
10120
|
+
return;
|
|
10121
|
+
}
|
|
10122
|
+
const ended = this.goalWriter ? await readGoalResult(this.goalWriter.ctx, inherited.ref) : undefined;
|
|
10123
|
+
if (ended !== undefined)
|
|
10124
|
+
this.stopOnceSeat(a, "turn-ended");
|
|
10125
|
+
}
|
|
10126
|
+
/** Inherited once tasks whose seat is not bound here. On a static mesh the boot reconcile adopts
|
|
10127
|
+
* every seat it can and reaps the rest by their custody reference, so once it has finished with
|
|
10128
|
+
* a seat's slot, an unbound seat is proved gone. A manager that adopts no seats cannot reach the
|
|
10129
|
+
* old seat or prove its exit: it does not serve the task to it, a task with a deadline ends at
|
|
10130
|
+
* that deadline, and one without is settled `uncertain` after one readiness window. Its debt is
|
|
10131
|
+
* never cleared here, since nothing proves that seat gone. */
|
|
10132
|
+
async settleInheritedOnce() {
|
|
10133
|
+
const gw = this.goalWriter;
|
|
10134
|
+
if (!gw || !this.goalReconcileDone)
|
|
10135
|
+
return;
|
|
10136
|
+
const adopts = this.auth !== undefined && !this.userMode;
|
|
10137
|
+
for (const [uid, d] of [...this.onceInherited]) {
|
|
10138
|
+
const live = this.findManagedByTarget({ owner: d.seat.owner, actor: d.seat.actor, lifecycleUid: uid });
|
|
10139
|
+
if (live !== undefined) {
|
|
10140
|
+
await this.bindOnceSeat(live);
|
|
10141
|
+
continue;
|
|
10142
|
+
}
|
|
10143
|
+
if (adopts) {
|
|
10144
|
+
const reconciling = this.staticReconcileSweepsInFlight > 0
|
|
10145
|
+
|| [...this.staticReconcileItems.values()].some((i) => i.lifecycleUid === uid && i.disposition !== "recovered");
|
|
10146
|
+
if (reconciling)
|
|
10147
|
+
continue;
|
|
10148
|
+
this.onceInherited.delete(uid);
|
|
10149
|
+
void this.settleOnceTurn(d.ref, "seat-exited", retireOf(d));
|
|
10150
|
+
continue;
|
|
10151
|
+
}
|
|
10152
|
+
if ((await readGoalResult(gw.ctx, d.ref)) !== undefined) {
|
|
10153
|
+
this.onceInherited.delete(uid);
|
|
10154
|
+
this.keepUnprovedOnceDebt(d);
|
|
10155
|
+
continue;
|
|
10156
|
+
}
|
|
10157
|
+
if (d.deadlineAt !== undefined || Date.now() - this.goalReconcileDoneAt < d.readinessMs)
|
|
10158
|
+
continue;
|
|
10159
|
+
this.onceInherited.delete(uid);
|
|
10160
|
+
const key = goalKey(d.ref);
|
|
10161
|
+
this.pendingTurns.delete(key);
|
|
10162
|
+
this.markTurnLatchDropped(key);
|
|
10163
|
+
try {
|
|
10164
|
+
await settleGoalUncertain(gw.ctx, { ref: d.ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch: this.serviceServe?.grant.epoch ?? 0 }, reason: `the manager restarted and could not reach the seat; ${UNHELD_SEAT}, and the task may have run in part or in full` });
|
|
10165
|
+
this.keepUnprovedOnceDebt(d);
|
|
10166
|
+
}
|
|
10167
|
+
catch (e) {
|
|
10168
|
+
console.error(`! inherited once task ${d.ref.goalId}: ${rejectionText(e)}`);
|
|
10169
|
+
}
|
|
10170
|
+
}
|
|
10171
|
+
}
|
|
10172
|
+
/** An inherited task's seat this manager cannot reach: its terminal stands, and its index entry,
|
|
10173
|
+
* the seat's debt, stays recorded, because nothing here can prove that seat gone. The seat is
|
|
10174
|
+
* told to end itself on its next pull, and its notice repeats until a pull carries that. */
|
|
10175
|
+
keepUnprovedOnceDebt(d) {
|
|
10176
|
+
console.error(`! once task ${d.ref.goalId}: this manager cannot reach seat ${d.seat.name}/${d.seat.uid} to prove it gone; its debt stays in the goal index, and the seat is told to end itself`);
|
|
10177
|
+
this.onceStops.set(d.seat.uid, { ...d.seat });
|
|
10178
|
+
this.noticeSeatTurns(d.seat);
|
|
9501
10179
|
this.ensureTurnSweep();
|
|
9502
10180
|
}
|
|
9503
10181
|
/** Settle race: a despawn MID-GOAL drives the goal's cancel terminal - transition to
|
|
@@ -9750,6 +10428,7 @@ export class Manager {
|
|
|
9750
10428
|
this.agents.set(entry.name, managed);
|
|
9751
10429
|
this.watchExit(managed);
|
|
9752
10430
|
console.error(`! ${entry.name}: adopted the seat its previous manager left (${row.runtime.kind}:${row.runtime.id})`);
|
|
10431
|
+
await this.bindOnceSeat(managed);
|
|
9753
10432
|
// A credential that came close to its expiry while no manager ran is renewed now, not at the
|
|
9754
10433
|
// next pass.
|
|
9755
10434
|
const health = inspectCredHealth(stored).state;
|