@cotal-ai/manager 0.77.0 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +25 -70
- package/dist/commands.js.map +1 -1
- package/dist/manager-service-contract.d.ts +6 -1
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +8 -2
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +26 -9
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +193 -131
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +1 -0
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +16 -1
- package/dist/remote-authority.js.map +1 -1
- package/dist/runtime/index.d.ts +7 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +45 -31
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +17 -27
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -6,7 +6,7 @@ import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
|
6
6
|
import Headless from "@xterm/headless";
|
|
7
7
|
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimIdentityRecord, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceKey, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
|
-
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, watchProvedExit, } from "./runtime/index.js";
|
|
9
|
+
import { createRuntime, exitProof, isCustodialRuntime, pollProvedExit, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, watchProvedExit, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
11
|
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers, withScopedEndpoint } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
@@ -54,6 +54,10 @@ const TURN_ANSWER_RETENTION_MS = 5 * 60_000;
|
|
|
54
54
|
* launch-parity smoke can assert every launch client's request timeout OUTLIVES this window — the tier
|
|
55
55
|
* rule forbids the clients importing it directly. */
|
|
56
56
|
export const READINESS_TIMEOUT_MS = 30_000;
|
|
57
|
+
/** How often the readiness race asks a runtime that streams no exit whether its child has exited, and
|
|
58
|
+
* how long it waits past the window for the proof of its last read. An operator waits on this reply,
|
|
59
|
+
* so it asks more often than the exit watch that outlives readiness. */
|
|
60
|
+
const READINESS_EXIT_POLL_MS = 1_000;
|
|
57
61
|
/** Managed same-session crash recovery follows the Codex host precedent: three restarts are allowed
|
|
58
62
|
* inside a rolling two-minute window; the fourth crash is a loop and retires the seat loud. */
|
|
59
63
|
const SESSION_RESTART_LIMIT = 3;
|
|
@@ -245,14 +249,22 @@ function parseTurnNote(raw) {
|
|
|
245
249
|
if (typeof n.payload !== "string" || typeof n.deadlineAt !== "number"
|
|
246
250
|
|| typeof n.holdEpoch !== "number" || typeof n.owner !== "string")
|
|
247
251
|
return undefined;
|
|
252
|
+
if (n.holdToken !== undefined && typeof n.holdToken !== "string")
|
|
253
|
+
return undefined;
|
|
248
254
|
if (n.handoffFrom !== undefined && typeof n.handoffFrom !== "string")
|
|
249
255
|
return undefined;
|
|
250
|
-
return { payload: n.payload, deadlineAt: n.deadlineAt, holdEpoch: n.holdEpoch, owner: n.owner, ...(typeof n.handoffFrom === "string" ? { handoffFrom: n.handoffFrom } : {}) };
|
|
256
|
+
return { payload: n.payload, deadlineAt: n.deadlineAt, holdEpoch: n.holdEpoch, owner: n.owner, ...(typeof n.holdToken === "string" ? { holdToken: n.holdToken } : {}), ...(typeof n.handoffFrom === "string" ? { handoffFrom: n.handoffFrom } : {}) };
|
|
257
|
+
}
|
|
258
|
+
/** A turn relay's key in the manager's maps: the caller triple and goal id the goal plane stores
|
|
259
|
+
* the turn under, because a goal id is the caller's own (SPEC 13.6 item 1) and two callers may
|
|
260
|
+
* pick the same one. Every part is a subject token, so the dotted join is unambiguous. */
|
|
261
|
+
function turnKey(ref) {
|
|
262
|
+
return `${ref.caller.owner}.${ref.caller.actor}.${ref.caller.uid}.${ref.goalId}`;
|
|
251
263
|
}
|
|
252
|
-
/** A turn hold's token, DERIVED
|
|
253
|
-
*
|
|
254
|
-
function turnHoldToken(
|
|
255
|
-
return createHash("sha256").update(`${
|
|
264
|
+
/** A turn hold's token, DERIVED (same recipe as the runtime's pause tokens) from the relay key, so
|
|
265
|
+
* two callers' turns under one goal id never share a hold. */
|
|
266
|
+
function turnHoldToken(key) {
|
|
267
|
+
return createHash("sha256").update(`${key}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
|
|
256
268
|
}
|
|
257
269
|
/**
|
|
258
270
|
* Event channels in `channels` that do NOT belong to `{owner, actor}`.
|
|
@@ -466,9 +478,9 @@ export class Manager {
|
|
|
466
478
|
* drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
|
|
467
479
|
agentGoals = new Map();
|
|
468
480
|
/**
|
|
469
|
-
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin)
|
|
470
|
-
* same-
|
|
471
|
-
* the goal-index entry (its acceptance floor + note).
|
|
481
|
+
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin), by
|
|
482
|
+
* {@link turnKey}: a same-goal retry serves the identical acceptance; cross-incarnation retries
|
|
483
|
+
* rebuild from the goal-index entry (its acceptance floor + note).
|
|
472
484
|
*
|
|
473
485
|
* One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
|
|
474
486
|
* from, and — once the turn settles — the answer a RETRIED yield is served from.
|
|
@@ -489,7 +501,7 @@ export class Manager {
|
|
|
489
501
|
* window would hear `not-found` again.
|
|
490
502
|
*/
|
|
491
503
|
turnAcceptances = new Map();
|
|
492
|
-
/** Every relayed turn awaiting its seat's yield, by
|
|
504
|
+
/** Every relayed turn awaiting its seat's yield, by {@link turnKey}. The seat's `turn-pending` pull
|
|
493
505
|
* scans it; the deadline sweep drives expiry; a seat reap stamps its entries `seatDiedAt`, which
|
|
494
506
|
* the deadline terminal carries as `agentDownAt` so the run reads that death as L4002.
|
|
495
507
|
* Rebuilt at boot from goal-index entries carrying a turn note. */
|
|
@@ -2802,11 +2814,11 @@ export class Manager {
|
|
|
2802
2814
|
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2803
2815
|
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2804
2816
|
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2805
|
-
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2806
|
-
*
|
|
2817
|
+
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone:
|
|
2818
|
+
* the deploy path is its only consumer and stamps the caller's own owner,
|
|
2807
2819
|
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2808
2820
|
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2809
|
-
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity
|
|
2821
|
+
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity): (1)
|
|
2810
2822
|
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2811
2823
|
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2812
2824
|
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
@@ -2992,7 +3004,7 @@ export class Manager {
|
|
|
2992
3004
|
return r.data;
|
|
2993
3005
|
}),
|
|
2994
3006
|
purge: (ctx) => this.serveGated(ctx, () => adminGated(ctx, async () => unwrap(await this.opPurge(args(ctx), callerOf(ctx))))),
|
|
2995
|
-
// launch is OWNER-EQUALITY on the ep door for every caller
|
|
3007
|
+
// launch is OWNER-EQUALITY on the ep door for every caller: the deploy
|
|
2996
3008
|
// path is the only launch consumer and its spec stamps the CALLER's own owner, so
|
|
2997
3009
|
// owner-equality always holds for a legitimate deploy; cross-owner launch was a ctl
|
|
2998
3010
|
// admin-tier INCIDENTAL never exercised by a real flow (static is single-owner, so the flag
|
|
@@ -3000,7 +3012,7 @@ export class Manager {
|
|
|
3000
3012
|
// NOT reflect the deployer VIEW's privileged-tier attenuation - an admin user's stolen
|
|
3001
3013
|
// deployer bearer would then bypass owner-equality (operator launch) despite the view holding
|
|
3002
3014
|
// no admin rows. Uniform owner-equality removes that divergence in the least-privilege
|
|
3003
|
-
// direction
|
|
3015
|
+
// direction.
|
|
3004
3016
|
// P2 item 2 (ruling 3): manifest `launch` is an ACTION through the SAME chokepoint as spawn -
|
|
3005
3017
|
// the manifest resolve + owner-equality authz run in opLaunch's accept path, then the goal
|
|
3006
3018
|
// drives progress + terminal. The acceptance floor is the allocated identity + goal coords.
|
|
@@ -4657,6 +4669,29 @@ export class Manager {
|
|
|
4657
4669
|
const { reason } = resolveHarness(connector.name, connector.requires ?? []);
|
|
4658
4670
|
return reason ? { refusal: reason } : {};
|
|
4659
4671
|
}
|
|
4672
|
+
/** The launch fields the host supplies rather than the seat. Spawn and resume both take them from
|
|
4673
|
+
* here, so a resumed seat launches under the same host policy, `eventsRequired` included, as the
|
|
4674
|
+
* same seat spawned fresh. */
|
|
4675
|
+
hostLaunchFields(agent, shareTools, harness) {
|
|
4676
|
+
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
4677
|
+
return {
|
|
4678
|
+
space: this.space,
|
|
4679
|
+
servers: this.servers,
|
|
4680
|
+
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
4681
|
+
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
4682
|
+
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
4683
|
+
mcpServers: connectorServers(cotalConfig, agent, shareTools),
|
|
4684
|
+
// The operator's spawn-env policy travels the same route: absent means no extras (the OS
|
|
4685
|
+
// allow-list + operator knobs + connector-declared inputs), present means those names too.
|
|
4686
|
+
// A connector never reads the config itself.
|
|
4687
|
+
envAllow: spawnEnvAllow(cotalConfig),
|
|
4688
|
+
resolvedBinaries: harness.binaries,
|
|
4689
|
+
eventsRequired: this.eventsRequired,
|
|
4690
|
+
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
4691
|
+
// (possibly per-agent) launch cwd.
|
|
4692
|
+
workspaceRoot: this.workspaceRoot,
|
|
4693
|
+
};
|
|
4694
|
+
}
|
|
4660
4695
|
/** The refusal for a launch choice the connector cannot honour. Spawn and resume both ask before
|
|
4661
4696
|
* any reserve or mint, so a rule changed here holds on both paths. */
|
|
4662
4697
|
capabilityRefusal(connector, launch) {
|
|
@@ -4668,6 +4703,18 @@ export class Manager {
|
|
|
4668
4703
|
return `${connector.name} connector does not support exact session continuity (continuity: exact)`;
|
|
4669
4704
|
return undefined;
|
|
4670
4705
|
}
|
|
4706
|
+
/** The refusal for an event-plane choice the registration policy or the connector cannot meet.
|
|
4707
|
+
* Spawn and resume both ask before a seat's launch is built, so a seat retained without the
|
|
4708
|
+
* plane cannot come back on a space that now requires it. An absent choice arms the plane. */
|
|
4709
|
+
eventsRefusal(connector, events) {
|
|
4710
|
+
if (this.eventsRequired && events === false)
|
|
4711
|
+
return `space "${this.space}" requires the event plane by registration policy; --no-events (events: false on the start op) is not allowed`;
|
|
4712
|
+
if (events !== false && !connector.eventChannel)
|
|
4713
|
+
return this.eventsRequired
|
|
4714
|
+
? `space "${this.space}" requires the event plane by registration policy, but connector "${connector.name}" does not publish one`
|
|
4715
|
+
: `connector "${connector.name}" does not publish an AG-UI event plane; pass --no-events (events: false on the start op) to launch it without one`;
|
|
4716
|
+
return undefined;
|
|
4717
|
+
}
|
|
4671
4718
|
/** Return connector-provided model catalogs for selector UIs. Optional by connector: a host with no
|
|
4672
4719
|
* local model-list API reports `supported:false` rather than blocking the manager. A connector that
|
|
4673
4720
|
* fails to import shows an `error:` row (from manifest enumeration) and never blocks the others. */
|
|
@@ -5172,24 +5219,11 @@ export class Manager {
|
|
|
5172
5219
|
// the principal's owner is resolved further down, so deriving it from anything in scope here
|
|
5173
5220
|
// would mean guessing at the identity the child will actually connect as. It is added at the
|
|
5174
5221
|
// accept seam below, where the allocated triple exists.
|
|
5175
|
-
|
|
5176
|
-
|
|
5222
|
+
const eventsRefused = this.eventsRefusal(connector, opts.events);
|
|
5223
|
+
if (eventsRefused)
|
|
5224
|
+
return { ok: false, error: eventsRefused };
|
|
5177
5225
|
const events = this.eventsRequired || opts.events !== false;
|
|
5178
|
-
|
|
5179
|
-
return { ok: false, error: this.eventsRequired
|
|
5180
|
-
? `space "${this.space}" requires the event plane by registration policy, but connector "${connector.name}" does not publish one`
|
|
5181
|
-
: `connector "${connector.name}" does not publish an AG-UI event plane; pass --no-events (events: false on the start op) to launch it without one` };
|
|
5182
|
-
// F2 (Unit B): a STATIC managed spawn REFUSES endpoint capabilities, fail-closed IN CODE (not
|
|
5183
|
-
// a doc note): the static terminal has no obligation-drain/frontier steps yet, so an accepted-
|
|
5184
|
-
// but-uncompleted endpoint obligation could execute AFTER its uid is declared retired. The
|
|
5185
|
-
// refusal sits at spawn-accept, before any provisioning, over the same records a persona or
|
|
5186
|
-
// manifest self-claim would ride in on — capabilities cannot slip past it into the grant path.
|
|
5187
|
-
if (this.auth && !this.userMode) {
|
|
5188
|
-
const claims = [opts, (opts.resolved ?? {})];
|
|
5189
|
-
if (claims.some((c) => c.endpointCapabilities !== undefined))
|
|
5190
|
-
return { ok: false, error: "a static managed spawn refuses endpointCapabilities (Unit B F2): the static lifecycle terminal carries no obligation-drain/frontier steps, so endpoint-rail grants are not containable in static mode" };
|
|
5191
|
-
}
|
|
5192
|
-
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
5226
|
+
// The roster the allocation consults must reflect the initial presence snapshot,
|
|
5193
5227
|
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
5194
5228
|
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
5195
5229
|
// empty mesh) before allocating; the broker/auth remain the authority downstream. Deliberately
|
|
@@ -5221,7 +5255,7 @@ export class Manager {
|
|
|
5221
5255
|
// exists; cleared when a live slot takes ownership. If it survives to `finally`, the spawn threw
|
|
5222
5256
|
// AFTER provisioning (buildLaunch / runtime.spawn) — the orphan-rollback tears it down. Carries
|
|
5223
5257
|
// `userOwner` for a user-mode spawn so that rollback runs the revoke+shred branch, not just the
|
|
5224
|
-
// static durable teardown
|
|
5258
|
+
// static durable teardown, which would leak the managed grant + files.
|
|
5225
5259
|
// The RESERVED custody reference rides the rollback object: a spawn that launched the seat and
|
|
5226
5260
|
// then threw (the activation CAS refuses a slot that is not this spawn's intent) reaches the
|
|
5227
5261
|
// `finally` with the handle only in a local, so without this the terminal has nothing to reap
|
|
@@ -5416,15 +5450,6 @@ export class Manager {
|
|
|
5416
5450
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(credsPath, this.space), credsPath);
|
|
5417
5451
|
provisioned = { id: identity.id, name, lifecycleUid, secretPaths: { creds: credsPath }, ...(custody ? { runtime: custody } : {}) }; // footprint now exists — the finally rolls it back if the spawn throws
|
|
5418
5452
|
}
|
|
5419
|
-
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
5420
|
-
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
5421
|
-
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
5422
|
-
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
5423
|
-
const mcpServers = connectorServers(cotalConfig, agent, opts.shareTools);
|
|
5424
|
-
// The operator's spawn-env policy travels the same route: absent means no extras (the OS
|
|
5425
|
-
// allow-list + operator knobs + connector-declared inputs), present means those names too.
|
|
5426
|
-
// A connector never reads the config itself.
|
|
5427
|
-
const envAllow = spawnEnvAllow(cotalConfig);
|
|
5428
5453
|
// Per-agent cwd overrides the manager's shared workspace root, so agents can be rooted at
|
|
5429
5454
|
// arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
|
|
5430
5455
|
// agent shares the workspace root (the prior, unchanged behavior).
|
|
@@ -5437,7 +5462,7 @@ export class Manager {
|
|
|
5437
5462
|
: undefined;
|
|
5438
5463
|
const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
|
|
5439
5464
|
const launchOpts = {
|
|
5440
|
-
|
|
5465
|
+
...this.hostLaunchFields(agent, opts.shareTools, harness),
|
|
5441
5466
|
name,
|
|
5442
5467
|
role,
|
|
5443
5468
|
// User mode: the principal IS the identity (the endpoint derives card.id from owner+actor);
|
|
@@ -5450,7 +5475,6 @@ export class Manager {
|
|
|
5450
5475
|
// at the broker, never silently).
|
|
5451
5476
|
lifecycleUid,
|
|
5452
5477
|
acceptedToken: issued?.acceptedToken,
|
|
5453
|
-
servers: this.servers,
|
|
5454
5478
|
configPath,
|
|
5455
5479
|
model,
|
|
5456
5480
|
variant,
|
|
@@ -5472,13 +5496,6 @@ export class Manager {
|
|
|
5472
5496
|
allowPublish,
|
|
5473
5497
|
capabilities,
|
|
5474
5498
|
events,
|
|
5475
|
-
eventsRequired: this.eventsRequired,
|
|
5476
|
-
mcpServers,
|
|
5477
|
-
envAllow,
|
|
5478
|
-
resolvedBinaries: harness.binaries,
|
|
5479
|
-
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
5480
|
-
// (possibly per-agent) launch cwd.
|
|
5481
|
-
workspaceRoot: this.workspaceRoot,
|
|
5482
5499
|
cwd,
|
|
5483
5500
|
};
|
|
5484
5501
|
// A delegated seat gets no local launch at all: no launch-material file, control token or
|
|
@@ -6150,6 +6167,9 @@ export class Manager {
|
|
|
6150
6167
|
const unsupported = this.capabilityRefusal(connector, { variant: entry.launch.variant, prompt: entry.launch.prompt, exact });
|
|
6151
6168
|
if (unsupported)
|
|
6152
6169
|
return { ok: false, error: unsupported };
|
|
6170
|
+
const eventsRefused = this.eventsRefusal(connector, entry.launch.events);
|
|
6171
|
+
if (eventsRefused)
|
|
6172
|
+
return { ok: false, error: eventsRefused };
|
|
6153
6173
|
let authority;
|
|
6154
6174
|
try {
|
|
6155
6175
|
authority = await this.validateRetainedAuthority(entry);
|
|
@@ -6158,11 +6178,8 @@ export class Manager {
|
|
|
6158
6178
|
return { ok: false, error: rejectionText(e) };
|
|
6159
6179
|
}
|
|
6160
6180
|
try {
|
|
6161
|
-
const resumeConfig = loadCotalConfig(this.workspaceRoot);
|
|
6162
|
-
const mcpServers = connectorServers(resumeConfig, entry.launch.connector, entry.launch.shareTools);
|
|
6163
|
-
const envAllow = spawnEnvAllow(resumeConfig);
|
|
6164
6181
|
const launchOpts = {
|
|
6165
|
-
|
|
6182
|
+
...this.hostLaunchFields(entry.launch.connector, entry.launch.shareTools, harness),
|
|
6166
6183
|
name: entry.name,
|
|
6167
6184
|
role: entry.role,
|
|
6168
6185
|
id: authority.id,
|
|
@@ -6176,7 +6193,6 @@ export class Manager {
|
|
|
6176
6193
|
lifecycleUid: entry.identity.lifecycleUid,
|
|
6177
6194
|
backfillFloor: entry.backfillFloor,
|
|
6178
6195
|
acceptedToken: entry.identity.mode === "static" ? entry.identity.issued?.acceptedToken : undefined,
|
|
6179
|
-
servers: this.servers,
|
|
6180
6196
|
configPath: entry.launch.source.configPath,
|
|
6181
6197
|
model: entry.launch.model,
|
|
6182
6198
|
variant: entry.launch.variant,
|
|
@@ -6189,10 +6205,6 @@ export class Manager {
|
|
|
6189
6205
|
allowPublish: entry.launch.allowPublish,
|
|
6190
6206
|
capabilities: entry.launch.capabilities,
|
|
6191
6207
|
events: entry.launch.events,
|
|
6192
|
-
mcpServers,
|
|
6193
|
-
envAllow,
|
|
6194
|
-
resolvedBinaries: harness.binaries,
|
|
6195
|
-
workspaceRoot: this.workspaceRoot,
|
|
6196
6208
|
cwd: entry.launch.cwd,
|
|
6197
6209
|
};
|
|
6198
6210
|
const spec = connector.buildLaunch(launchOpts);
|
|
@@ -6385,15 +6397,17 @@ export class Manager {
|
|
|
6385
6397
|
* Presence is keyed on the EXACT freshly-minted id, never the name — a fresh id has no prior record, so
|
|
6386
6398
|
* any live presence for it is from THIS launch (stale/same-name records can't false-start it). The
|
|
6387
6399
|
* `"presence"` event is only a wake; the roster is re-read as the source of truth (subscribe-then-check
|
|
6388
|
-
* catches a join/exit that landed before we subscribed).
|
|
6389
|
-
*
|
|
6400
|
+
* catches a join/exit that landed before we subscribed). A runtime whose `attach()` throws (an external
|
|
6401
|
+
* surface) streams no output or exit, so its exit leg polls the runtime's exit proof ({@link pollProvedExit}),
|
|
6402
|
+
* reads it once more when the window closes, and its failure carries no last output; one that cannot prove an
|
|
6403
|
+
* exit either races presence-vs-backstop only. */
|
|
6390
6404
|
async awaitReadiness(a, readinessTimeoutMs, opts = {}) {
|
|
6391
6405
|
let session;
|
|
6392
6406
|
try {
|
|
6393
6407
|
session = a.handle.attach();
|
|
6394
6408
|
}
|
|
6395
6409
|
catch {
|
|
6396
|
-
/* external surfaces stream no exit
|
|
6410
|
+
/* external surfaces stream no output or exit; their exit is polled below */
|
|
6397
6411
|
}
|
|
6398
6412
|
const s = session;
|
|
6399
6413
|
// Presence cards carry the wire PRINCIPAL dot-form (`<owner>.<actor>`), never a raw nkey — match
|
|
@@ -6445,7 +6459,7 @@ export class Manager {
|
|
|
6445
6459
|
// the backlog reads async — the process is known dead, that's a failure, not an unknown. Reap through
|
|
6446
6460
|
// onAgentExit so a child the launcher spawned in the window is reaped too.
|
|
6447
6461
|
const onExit = () => {
|
|
6448
|
-
if (done
|
|
6462
|
+
if (done)
|
|
6449
6463
|
return;
|
|
6450
6464
|
clearTimeout(timer);
|
|
6451
6465
|
// `stopHandle` sets `terminalizing` synchronously before it kills, and `onAgentExit` sets it
|
|
@@ -6456,7 +6470,8 @@ export class Manager {
|
|
|
6456
6470
|
// waitForExit may close the attach stream before this snapshot
|
|
6457
6471
|
let tail = "";
|
|
6458
6472
|
try {
|
|
6459
|
-
|
|
6473
|
+
if (s)
|
|
6474
|
+
tail = await this.tail(s);
|
|
6460
6475
|
}
|
|
6461
6476
|
catch { }
|
|
6462
6477
|
if (opts.reapOnExit !== false)
|
|
@@ -6494,12 +6509,28 @@ export class Manager {
|
|
|
6494
6509
|
finish({ ok: false, detail: `${a.name} exited on launch${tail ? ` - last output: ${tail}` : ""}${exitDetail}` });
|
|
6495
6510
|
})();
|
|
6496
6511
|
};
|
|
6497
|
-
|
|
6512
|
+
const uncertain = () => finish({
|
|
6498
6513
|
ok: false,
|
|
6499
6514
|
uncertain: true,
|
|
6500
6515
|
detail: `${a.name} (${a.id}): launch status uncertain - no process exit and no mesh presence within ${Math.round(readinessTimeoutMs / 1000)}s; it may still be booting or stuck before connector startup. Inspect with \`cotal attach ${a.name}\` / \`cotal ps\`; do not stop it solely because this bounded wait elapsed.`,
|
|
6501
|
-
})
|
|
6502
|
-
|
|
6516
|
+
});
|
|
6517
|
+
// The exit poll read the runtime up to a poll ago, and a window can be shorter than a poll, so
|
|
6518
|
+
// the window closes on a fresh read: a child that exited by then is reported as exited. Nothing
|
|
6519
|
+
// bounds how long a runtime's proof takes, so it gets one more poll before the launch is uncertain.
|
|
6520
|
+
timer = setTimeout(() => {
|
|
6521
|
+
const proof = s ? undefined : exitProof(a.handle);
|
|
6522
|
+
if (!proof) {
|
|
6523
|
+
uncertain();
|
|
6524
|
+
return;
|
|
6525
|
+
}
|
|
6526
|
+
timer = setTimeout(uncertain, READINESS_EXIT_POLL_MS);
|
|
6527
|
+
proof.then(onExit, uncertain);
|
|
6528
|
+
}, readinessTimeoutMs);
|
|
6529
|
+
// The poll stops once the race is done, so it needs no unsubscribe.
|
|
6530
|
+
if (s)
|
|
6531
|
+
unsubExit = s.onExit(onExit);
|
|
6532
|
+
else
|
|
6533
|
+
pollProvedExit(a.handle, onExit, () => !done, READINESS_EXIT_POLL_MS);
|
|
6503
6534
|
this.ep.on("presence", onPresence);
|
|
6504
6535
|
// Subscribe-then-check (TOCTOU): a join or an exit that already landed before we subscribed.
|
|
6505
6536
|
if (s && a.handle.status() === "exited")
|
|
@@ -6820,11 +6851,12 @@ export class Manager {
|
|
|
6820
6851
|
const failures = [...this.staticReconcileItems.values()]
|
|
6821
6852
|
.map(({ owner: _owner, timer: _timer, flight: _flight, retryFlight: _retryFlight, ...item }) => ({ ...item }))
|
|
6822
6853
|
.sort((a, b) => a.alias.localeCompare(b.alias));
|
|
6823
|
-
const state =
|
|
6824
|
-
failures.some((item) => item.disposition === "
|
|
6825
|
-
failures.some((item) => item.disposition === "
|
|
6826
|
-
|
|
6827
|
-
|
|
6854
|
+
const state = this.staticReconcileLastSweep?.error !== undefined ? "failed" :
|
|
6855
|
+
failures.some((item) => item.disposition === "retrying") ? "retrying" :
|
|
6856
|
+
failures.some((item) => item.disposition === "retry-scheduled") ? "retry-wait" :
|
|
6857
|
+
failures.some((item) => item.disposition === "refused" || item.disposition === "refused-foreign" || item.disposition === "retry-exhausted") ? "failed" :
|
|
6858
|
+
this.staticReconcileSweepsInFlight > 0 ? "running" :
|
|
6859
|
+
failures.some((item) => item.disposition === "recovered") ? "recovered" : "idle";
|
|
6828
6860
|
return {
|
|
6829
6861
|
state,
|
|
6830
6862
|
...(this.staticReconcileLastSweep ? { lastSweep: { ...this.staticReconcileLastSweep } } : {}),
|
|
@@ -7685,7 +7717,7 @@ export class Manager {
|
|
|
7685
7717
|
// at a time), so all are reconciled. The `iid` field is the hook item-3's multi-instance sweep
|
|
7686
7718
|
// filters on (skip a goal whose accepting `iid` is a still-LIVE sibling — never settle its goal).
|
|
7687
7719
|
for (const entry of entries) {
|
|
7688
|
-
if (this.goalAcceptances.has(entry.ref.goalId) || this.turnAcceptances.has(entry.ref
|
|
7720
|
+
if (this.goalAcceptances.has(entry.ref.goalId) || this.turnAcceptances.has(turnKey(entry.ref)))
|
|
7689
7721
|
continue; // never settle a goal THIS incarnation drives
|
|
7690
7722
|
try {
|
|
7691
7723
|
// A note marks a TURN entry: its relay is rebuilt (the hold is its bounded ending), never
|
|
@@ -7926,7 +7958,7 @@ export class Manager {
|
|
|
7926
7958
|
// exists that early — the goal spec does not carry it and the terminal does not exist yet.
|
|
7927
7959
|
// A same-goalId attempt that loses the bind while the winner is still in flight can then
|
|
7928
7960
|
// serve what the winner actually allocated instead of inventing an empty identity.
|
|
7929
|
-
const idx = await recordGoalIndex(gw.ctx, ref, executor.lifecycleUid, { name, actor: agentTriple.actor, uid: agentTriple.uid, readinessDeadlineMs: readinessTimeoutMs });
|
|
7961
|
+
const idx = await recordGoalIndex(gw.ctx, ref, executor.lifecycleUid, { name, owner: agentTriple.owner, actor: agentTriple.actor, uid: agentTriple.uid, readinessDeadlineMs: readinessTimeoutMs });
|
|
7930
7962
|
// A create loss is an idempotent retry ONLY for the same incarnation. A FOREIGN iid means a
|
|
7931
7963
|
// sibling instance accepted this goalId (the live vector is a client retry over ANYCAST, not
|
|
7932
7964
|
// a journal consumer): this attempt provisions nothing and answers with the winner's floor,
|
|
@@ -8012,12 +8044,12 @@ export class Manager {
|
|
|
8012
8044
|
* wrote before its own ack). Refuses `unavailable` rather than inventing one — see
|
|
8013
8045
|
* {@link cachedSpawnAcceptance} for why an empty identity is never an acceptable answer. */
|
|
8014
8046
|
async acceptanceFromIndex(entry, ref, goalId, fingerprint, executor) {
|
|
8015
|
-
if (entry.allocated === undefined)
|
|
8016
|
-
throw new EpEnvelopeError("unavailable", `goal "${goalId}" was accepted by instance "${entry.iid}", which persisted no acceptance floor; its allocated identity is not readable from here (SPEC 13.6)`);
|
|
8047
|
+
if (entry.allocated?.owner === undefined)
|
|
8048
|
+
throw new EpEnvelopeError("unavailable", `goal "${goalId}" was accepted by instance "${entry.iid}", which persisted no acceptance floor naming its owner; its allocated identity is not readable from here (SPEC 13.6)`);
|
|
8017
8049
|
const readinessDeadlineMs = entry.allocated.readinessDeadlineMs ?? (await readGoalSpec(this.goalWriter.ctx, ref))?.value.readinessDeadlineMs;
|
|
8018
8050
|
if (readinessDeadlineMs === undefined)
|
|
8019
8051
|
throw new EpEnvelopeError("unavailable", `goal "${goalId}" is accepted but its readiness deadline is not readable; a synchronous follower cannot bound its wait honestly (SPEC 13.6)`);
|
|
8020
|
-
return { name: entry.allocated.name, owner:
|
|
8052
|
+
return { name: entry.allocated.name, owner: entry.allocated.owner, actor: entry.allocated.actor, uid: entry.allocated.uid, goalId, fingerprint, readinessDeadlineMs, executor };
|
|
8021
8053
|
}
|
|
8022
8054
|
/** Reconstruct a cached acceptance for a same-goalId retry NOT in the live map (a prior incarnation
|
|
8023
8055
|
* accepted it, or a concurrent winner whose map write this reader has not yet observed).
|
|
@@ -8040,8 +8072,11 @@ export class Manager {
|
|
|
8040
8072
|
const d = (result?.data ?? {});
|
|
8041
8073
|
const spec = await readGoalSpec(this.goalWriter.ctx, ref);
|
|
8042
8074
|
const readinessDeadlineMs = spec?.value.readinessDeadlineMs;
|
|
8043
|
-
|
|
8044
|
-
|
|
8075
|
+
// The terminal names the agent by its slot id: the `owner.actor` principal in user mode, the
|
|
8076
|
+
// bare nkey under DEV_OWNER on a static mesh.
|
|
8077
|
+
const principal = typeof d.id !== "string" || d.id.length === 0 ? null : this.userMode ? parsePrincipalKey(d.id) : { owner: DEV_OWNER, actor: d.id };
|
|
8078
|
+
if (typeof d.name === "string" && d.name.length > 0 && principal !== null && typeof d.lifecycleUid === "string" && d.lifecycleUid.length > 0 && readinessDeadlineMs !== undefined)
|
|
8079
|
+
return { name: d.name, owner: principal.owner, actor: principal.actor, uid: d.lifecycleUid, goalId, fingerprint, readinessDeadlineMs, executor };
|
|
8045
8080
|
throw new EpEnvelopeError("unavailable", `goal "${goalId}" is already accepted but its allocated identity is not readable (no acceptance floor, and no terminal carrying one); retry (SPEC 13.6)`);
|
|
8046
8081
|
}
|
|
8047
8082
|
// ---- the turn relay (§8 durable actions) ------------------------------------------------------
|
|
@@ -8072,14 +8107,17 @@ export class Manager {
|
|
|
8072
8107
|
const goalId = ctx.request.id;
|
|
8073
8108
|
const { fingerprint } = submissionFingerprint(ctx.request, ctx.subject);
|
|
8074
8109
|
const ref = goalRefOf(ctx.subject, goalId);
|
|
8110
|
+
const key = turnKey(ref);
|
|
8075
8111
|
const executor = { lifecycleUid: this.managerInstanceId, epoch: this.serviceServe?.grant.epoch ?? 0 };
|
|
8076
8112
|
const acceptedAt = Date.now();
|
|
8077
|
-
const prior = this.turnAcceptances.get(
|
|
8113
|
+
const prior = this.turnAcceptances.get(key)?.acceptance;
|
|
8078
8114
|
if (prior !== undefined) {
|
|
8079
8115
|
if (prior.fingerprint !== fingerprint)
|
|
8080
8116
|
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted under a different submission; one goalId never carries two specs (SPEC 13.6)`);
|
|
8081
8117
|
return prior;
|
|
8082
8118
|
}
|
|
8119
|
+
// Refused here, before anything durable, so the caller's goal id stays free for another seat.
|
|
8120
|
+
this.assertSeatTurnFree(a, t, goalId);
|
|
8083
8121
|
const raw = (ctx.request.args ?? {});
|
|
8084
8122
|
const payload = String(raw.payload);
|
|
8085
8123
|
const deadlineAt = raw.deadlineAt === undefined ? acceptedAt + Number(raw.deadlineMs) : Number(raw.deadlineAt);
|
|
@@ -8087,7 +8125,8 @@ export class Manager {
|
|
|
8087
8125
|
throw new EpEnvelopeError("deadline-exceeded", `goal "${goalId}" names a deadline already past (${new Date(deadlineAt).toISOString()}); a turn with no time left to serve it is never accepted (SPEC 13.6)`);
|
|
8088
8126
|
const deadlineMs = deadlineAt - acceptedAt;
|
|
8089
8127
|
const handoffFrom = raw.handoffFrom === undefined ? undefined : String(raw.handoffFrom);
|
|
8090
|
-
const
|
|
8128
|
+
const holdToken = turnHoldToken(key);
|
|
8129
|
+
const note = JSON.stringify({ payload, deadlineAt, holdEpoch: executor.epoch, owner: t.owner, holdToken, ...(handoffFrom !== undefined ? { handoffFrom } : {}) });
|
|
8091
8130
|
// A goal that already ENDED is never accepted again, whatever the retry carries: the bind is
|
|
8092
8131
|
// create-only and outlives an unwind (the failed terminal, the cleared index), so a retry of an
|
|
8093
8132
|
// unwound accept would otherwise re-record an index entry over a tombstone and, before that
|
|
@@ -8137,7 +8176,7 @@ export class Manager {
|
|
|
8137
8176
|
// none. Measured on an auth mesh: minted over the goal-writer, the schedule publish was
|
|
8138
8177
|
// broker-denied and every accept unwound.
|
|
8139
8178
|
await mintCheckpoint(gw.ctx.kv, jetstream(serve.nc), this.space, {
|
|
8140
|
-
ref: { endpoint: ref.endpoint, token:
|
|
8179
|
+
ref: { endpoint: ref.endpoint, token: holdToken },
|
|
8141
8180
|
instanceId: this.managerInstanceId,
|
|
8142
8181
|
epoch: executor.epoch,
|
|
8143
8182
|
goal: { caller: { owner: ctx.subject.caller.owner, actor: ctx.subject.caller.actor, uid: ctx.subject.caller.uid }, goalId },
|
|
@@ -8145,6 +8184,9 @@ export class Manager {
|
|
|
8145
8184
|
deadline: deadlineAt,
|
|
8146
8185
|
now: acceptedAt,
|
|
8147
8186
|
});
|
|
8187
|
+
// Again after the awaits, with no await before the maps are written: another caller's
|
|
8188
|
+
// accept of this id to this seat may have finished meanwhile, and this unwinds the loser.
|
|
8189
|
+
this.assertSeatTurnFree(a, t, goalId);
|
|
8148
8190
|
}
|
|
8149
8191
|
catch (e) {
|
|
8150
8192
|
// The accept is inline (no launch closure to fail later), so a post-bind throw unwinds HERE:
|
|
@@ -8174,13 +8216,13 @@ export class Manager {
|
|
|
8174
8216
|
ref, goalId,
|
|
8175
8217
|
seat: { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid },
|
|
8176
8218
|
payload, acceptedAt, deadlineAt,
|
|
8177
|
-
holdToken
|
|
8219
|
+
holdToken, holdEpoch: executor.epoch,
|
|
8178
8220
|
...(handoffFrom !== undefined ? { handoffFrom } : {}),
|
|
8179
8221
|
};
|
|
8180
|
-
this.pendingTurns.set(
|
|
8222
|
+
this.pendingTurns.set(key, pending);
|
|
8181
8223
|
this.ensureTurnSweep();
|
|
8182
8224
|
const acceptance = { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid, goalId, fingerprint, deadlineAt, executor };
|
|
8183
|
-
this.turnAcceptances.set(
|
|
8225
|
+
this.turnAcceptances.set(key, { acceptance, ref });
|
|
8184
8226
|
this.emitGoalProgress(ref, executor.epoch, { phase: "relayed" });
|
|
8185
8227
|
return acceptance;
|
|
8186
8228
|
}
|
|
@@ -8209,6 +8251,21 @@ export class Manager {
|
|
|
8209
8251
|
.map((p) => ({ goalId: p.goalId, payload: p.payload, acceptedAt: p.acceptedAt, deadlineAt: p.deadlineAt }));
|
|
8210
8252
|
return { turns };
|
|
8211
8253
|
}
|
|
8254
|
+
/** A seat names a turn by goal id alone when it yields, so one incarnation holding two callers'
|
|
8255
|
+
* turns under one id could not say which of the two it answered. */
|
|
8256
|
+
assertSeatTurnFree(a, t, goalId) {
|
|
8257
|
+
if (this.seatTurnKey({ owner: t.owner, actor: t.actor, uid: t.lifecycleUid }, goalId) !== undefined)
|
|
8258
|
+
throw new EpEnvelopeError("failed-precondition", `seat "${a.name}" already holds another caller's turn "${goalId}"; a seat yields by goal id, so it is never handed two turns under one (SPEC 13.6)`);
|
|
8259
|
+
}
|
|
8260
|
+
/** The relay key of the turn addressed to seat incarnation `c` under `goalId`, pending or kept
|
|
8261
|
+
* for a retried yield. The accept path never hands one incarnation two turns under one goal
|
|
8262
|
+
* id, so at most one matches. */
|
|
8263
|
+
seatTurnKey(c, goalId) {
|
|
8264
|
+
for (const [key, { acceptance: a }] of this.turnAcceptances)
|
|
8265
|
+
if (a.goalId === goalId && a.owner === c.owner && a.actor === c.actor && a.uid === c.uid)
|
|
8266
|
+
return key;
|
|
8267
|
+
return undefined;
|
|
8268
|
+
}
|
|
8212
8269
|
/** The seat's yield: claim the hold (one-use; expiry fails closed), then commit the goal
|
|
8213
8270
|
* `succeeded` carrying the TurnResult. A yield AFTER the deadline drives the deadline terminal
|
|
8214
8271
|
* instead and refuses — the expiry outcome stands, never a late success over it. */
|
|
@@ -8227,17 +8284,17 @@ export class Manager {
|
|
|
8227
8284
|
const yieldNote = raw.note === undefined ? undefined : String(raw.note);
|
|
8228
8285
|
if (status === "handoff" && (to === undefined || to.length === 0))
|
|
8229
8286
|
throw new EpEnvelopeError("failed-precondition", `a handoff yield names its addressee ("to"); a handoff to nobody relays nothing`);
|
|
8230
|
-
const
|
|
8287
|
+
const key = this.seatTurnKey(c, goalId);
|
|
8288
|
+
if (key === undefined)
|
|
8289
|
+
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
8290
|
+
const p = this.pendingTurns.get(key);
|
|
8231
8291
|
if (!p) {
|
|
8232
8292
|
// A YIELD WHOSE REPLY WAS LOST IS NOT A YIELD THAT FAILED. The commit deleted the pending
|
|
8233
8293
|
// turn, so a retry found nothing and heard `not-found` — which a seat reads as "drop it",
|
|
8234
8294
|
// reporting failure for work the run already has. The answer the first reply carried is
|
|
8235
8295
|
// served again instead, to the addressee it was addressed to and nobody else.
|
|
8236
|
-
const settled = this.turnAcceptances.get(
|
|
8296
|
+
const settled = this.turnAcceptances.get(key);
|
|
8237
8297
|
if (settled !== undefined) {
|
|
8238
|
-
const a = settled.acceptance;
|
|
8239
|
-
if (a.owner !== c.owner || a.actor !== c.actor || a.uid !== c.uid)
|
|
8240
|
-
throw new EpEnvelopeError("permission-denied", `turn "${goalId}" was addressed to ${a.owner}.${a.actor}/${a.uid}; a yield is the addressee's own (SPEC 13.6)`);
|
|
8241
8298
|
if (settled.settled !== undefined)
|
|
8242
8299
|
return { goalId, state: settled.settled.state };
|
|
8243
8300
|
// The in-memory answer is not yet remembered (pending deleted before the terminal CAS,
|
|
@@ -8246,15 +8303,13 @@ export class Manager {
|
|
|
8246
8303
|
if (settled.ref !== undefined) {
|
|
8247
8304
|
const ended = await readGoalResult(gw.ctx, settled.ref);
|
|
8248
8305
|
if (ended !== undefined) {
|
|
8249
|
-
this.rememberSettledTurn(
|
|
8306
|
+
this.rememberSettledTurn(key, ended.state, ended.ts);
|
|
8250
8307
|
return { goalId, state: ended.state };
|
|
8251
8308
|
}
|
|
8252
8309
|
}
|
|
8253
8310
|
}
|
|
8254
8311
|
throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
|
|
8255
8312
|
}
|
|
8256
|
-
if (p.seat.owner !== c.owner || p.seat.actor !== c.actor || p.seat.uid !== c.uid)
|
|
8257
|
-
throw new EpEnvelopeError("permission-denied", `turn "${goalId}" is addressed to ${p.seat.owner}.${p.seat.actor}/${p.seat.uid}; a yield is the addressee's own (SPEC 13.6)`);
|
|
8258
8313
|
const cpRef = { endpoint: p.ref.endpoint, token: p.holdToken };
|
|
8259
8314
|
try {
|
|
8260
8315
|
await resumeCheckpoint(gw.ctx.kv, gw.ctx.js, gw.ctx.jsm, this.space, { ref: cpRef, presenter: { id: MANAGER_ENDPOINT, lifecycleUid: this.managerInstanceId }, now: Date.now() });
|
|
@@ -8286,9 +8341,9 @@ export class Manager {
|
|
|
8286
8341
|
});
|
|
8287
8342
|
this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
8288
8343
|
await clearGoalIndex(gw.ctx, p.ref);
|
|
8289
|
-
this.pendingTurns.delete(
|
|
8290
|
-
this.markTurnLatchDropped(
|
|
8291
|
-
this.rememberSettledTurn(
|
|
8344
|
+
this.pendingTurns.delete(key);
|
|
8345
|
+
this.markTurnLatchDropped(key, at);
|
|
8346
|
+
this.rememberSettledTurn(key, fact.state, at);
|
|
8292
8347
|
this.maybeStopTurnSweep();
|
|
8293
8348
|
return { goalId, state: fact.state };
|
|
8294
8349
|
}
|
|
@@ -8311,15 +8366,15 @@ export class Manager {
|
|
|
8311
8366
|
const raw = (ctx.request.args ?? {});
|
|
8312
8367
|
const goalId = String(raw.goalId);
|
|
8313
8368
|
const ref = goalRefOf(ctx.subject, goalId);
|
|
8314
|
-
const
|
|
8315
|
-
const
|
|
8316
|
-
if (p === undefined
|
|
8369
|
+
const key = turnKey(ref);
|
|
8370
|
+
const p = this.pendingTurns.get(key);
|
|
8371
|
+
if (p === undefined) {
|
|
8317
8372
|
const ended = await readGoalResult(gw.ctx, ref);
|
|
8318
8373
|
if (ended !== undefined)
|
|
8319
8374
|
throw goalAlreadyTerminal(goalId, ended);
|
|
8320
8375
|
throw new EpEnvelopeError("failed-precondition", `no turn "${goalId}" of this caller is pending on this manager; cancel withdraws a relayed turn (SPEC 13.6)`);
|
|
8321
8376
|
}
|
|
8322
|
-
this.pendingTurns.delete(
|
|
8377
|
+
this.pendingTurns.delete(key);
|
|
8323
8378
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
8324
8379
|
let fact;
|
|
8325
8380
|
try {
|
|
@@ -8332,14 +8387,14 @@ export class Manager {
|
|
|
8332
8387
|
// a yield that has dropped the entry remembered its answer, and one still committing drops
|
|
8333
8388
|
// the entry it finds. An ended relay is stamped so its acceptance still ages off.
|
|
8334
8389
|
if ((await readGoalResult(gw.ctx, ref)) !== undefined)
|
|
8335
|
-
this.markTurnLatchDropped(
|
|
8336
|
-
else if (this.turnAcceptances.get(
|
|
8337
|
-
this.pendingTurns.set(
|
|
8390
|
+
this.markTurnLatchDropped(key);
|
|
8391
|
+
else if (this.turnAcceptances.get(key)?.settled === undefined)
|
|
8392
|
+
this.pendingTurns.set(key, p);
|
|
8338
8393
|
throw e;
|
|
8339
8394
|
}
|
|
8340
8395
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
8341
|
-
this.markTurnLatchDropped(
|
|
8342
|
-
this.rememberSettledTurn(
|
|
8396
|
+
this.markTurnLatchDropped(key);
|
|
8397
|
+
this.rememberSettledTurn(key, fact.state, Date.now());
|
|
8343
8398
|
await clearGoalIndex(gw.ctx, ref);
|
|
8344
8399
|
this.maybeStopTurnSweep();
|
|
8345
8400
|
return { goalId, state: fact.state };
|
|
@@ -8352,9 +8407,10 @@ export class Manager {
|
|
|
8352
8407
|
const gw = this.goalWriter;
|
|
8353
8408
|
if (!gw)
|
|
8354
8409
|
return;
|
|
8355
|
-
|
|
8410
|
+
const key = turnKey(p.ref);
|
|
8411
|
+
if (!this.pendingTurns.delete(key))
|
|
8356
8412
|
return;
|
|
8357
|
-
this.markTurnLatchDropped(
|
|
8413
|
+
this.markTurnLatchDropped(key);
|
|
8358
8414
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
8359
8415
|
try {
|
|
8360
8416
|
await this.assertGoalWriterEpochCurrent(epoch);
|
|
@@ -8364,7 +8420,7 @@ export class Manager {
|
|
|
8364
8420
|
// (the delete at the top of this function) and after this CAS has to name this terminal.
|
|
8365
8421
|
// Clearing first, then remembering, left a window where maybeStopTurnSweep saw no pending
|
|
8366
8422
|
// turn and no settled answer, wiped the acceptance, and rememberSettledTurn no-op'd.
|
|
8367
|
-
this.rememberSettledTurn(
|
|
8423
|
+
this.rememberSettledTurn(key, fact.state, Date.now());
|
|
8368
8424
|
await clearGoalIndex(gw.ctx, p.ref);
|
|
8369
8425
|
}
|
|
8370
8426
|
catch (e) {
|
|
@@ -8406,16 +8462,16 @@ export class Manager {
|
|
|
8406
8462
|
}
|
|
8407
8463
|
/** The answer a retried yield is served, held for {@link TURN_ANSWER_RETENTION_MS}. Only for a
|
|
8408
8464
|
* turn this incarnation accepted: an entry it has no acceptance for is one it cannot vouch for. */
|
|
8409
|
-
rememberSettledTurn(
|
|
8410
|
-
const entry = this.turnAcceptances.get(
|
|
8465
|
+
rememberSettledTurn(key, state, at) {
|
|
8466
|
+
const entry = this.turnAcceptances.get(key);
|
|
8411
8467
|
if (entry === undefined)
|
|
8412
8468
|
return;
|
|
8413
8469
|
entry.settled = { state, at };
|
|
8414
8470
|
}
|
|
8415
8471
|
/** Stamp when the pending latch dropped, so an unsettled leftover ages off that instant rather
|
|
8416
8472
|
* than the original deadline. Idempotent: the first drop is the one the retry window starts at. */
|
|
8417
|
-
markTurnLatchDropped(
|
|
8418
|
-
const entry = this.turnAcceptances.get(
|
|
8473
|
+
markTurnLatchDropped(key, at = Date.now()) {
|
|
8474
|
+
const entry = this.turnAcceptances.get(key);
|
|
8419
8475
|
if (entry === undefined || entry.leftoverSince !== undefined)
|
|
8420
8476
|
return;
|
|
8421
8477
|
entry.leftoverSince = at;
|
|
@@ -8449,13 +8505,13 @@ export class Manager {
|
|
|
8449
8505
|
// An UNSETTLED acceptance whose pending latch is already gone is the in-flight (or failed)
|
|
8450
8506
|
// deadline commit: keep it for the same window so a late yield can still read the durable
|
|
8451
8507
|
// terminal, then drop it so a commit that never remembered cannot pin the sweep forever.
|
|
8452
|
-
for (const [
|
|
8453
|
-
if (this.pendingTurns.has(
|
|
8508
|
+
for (const [key, e] of [...this.turnAcceptances.entries()]) {
|
|
8509
|
+
if (this.pendingTurns.has(key))
|
|
8454
8510
|
continue;
|
|
8455
8511
|
// Unsettled leftovers age off the latch-drop stamp, never `deadlineAt`.
|
|
8456
8512
|
const at = e.settled?.at ?? e.leftoverSince;
|
|
8457
8513
|
if (at !== undefined && now - at >= TURN_ANSWER_RETENTION_MS)
|
|
8458
|
-
this.turnAcceptances.delete(
|
|
8514
|
+
this.turnAcceptances.delete(key);
|
|
8459
8515
|
}
|
|
8460
8516
|
// The prune above is the one event after which the sweep may have nothing left to do, and
|
|
8461
8517
|
// the settle-time callers cannot see it: they run when an answer has just been remembered.
|
|
@@ -8515,7 +8571,7 @@ export class Manager {
|
|
|
8515
8571
|
payload: parsed.payload,
|
|
8516
8572
|
acceptedAt: spec.value.acceptedAt,
|
|
8517
8573
|
deadlineAt: parsed.deadlineAt,
|
|
8518
|
-
holdToken: turnHoldToken(entry.ref.goalId), holdEpoch: parsed.holdEpoch,
|
|
8574
|
+
holdToken: parsed.holdToken ?? turnHoldToken(entry.ref.goalId), holdEpoch: parsed.holdEpoch,
|
|
8519
8575
|
...(parsed.handoffFrom !== undefined ? { handoffFrom: parsed.handoffFrom } : {}),
|
|
8520
8576
|
};
|
|
8521
8577
|
// The hold is the relay's bounded ending, minted AFTER the index entry and the goal record,
|
|
@@ -8525,13 +8581,14 @@ export class Manager {
|
|
|
8525
8581
|
console.error(`turn reconcile ${p.goalId}: its deadline hold was never minted (the accept crashed before it); left unsettled`);
|
|
8526
8582
|
return;
|
|
8527
8583
|
}
|
|
8528
|
-
|
|
8584
|
+
const key = turnKey(p.ref);
|
|
8585
|
+
this.pendingTurns.set(key, p);
|
|
8529
8586
|
// THE ACCEPTANCE TOO, or the adopted turn's answer is never remembered: `rememberSettledTurn`
|
|
8530
8587
|
// writes only against an acceptance this incarnation holds, and adoption rebuilt the pending
|
|
8531
8588
|
// relay without one, so a yield whose reply was lost after a restart still heard `not-found`
|
|
8532
8589
|
// on the very path adoption exists for. The acceptance is rebuilt from the same records the
|
|
8533
8590
|
// relay was: the spec holds the fingerprint, the index holds the seat, the note the deadline.
|
|
8534
|
-
this.turnAcceptances.set(
|
|
8591
|
+
this.turnAcceptances.set(key, {
|
|
8535
8592
|
acceptance: {
|
|
8536
8593
|
name: p.seat.name, owner: p.seat.owner, actor: p.seat.actor, uid: p.seat.uid, goalId: p.goalId,
|
|
8537
8594
|
fingerprint: spec.value.fingerprint, deadlineAt: p.deadlineAt,
|
|
@@ -9066,25 +9123,30 @@ export class Manager {
|
|
|
9066
9123
|
failed: 0,
|
|
9067
9124
|
};
|
|
9068
9125
|
this.staticReconcileLastSweep = sweep;
|
|
9069
|
-
const identity = newIdentity();
|
|
9070
|
-
const creds = await mintCreds(this.auth, identity, "provisioner");
|
|
9071
|
-
const nc = await this.dial({ ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
9072
9126
|
const slotRows = [];
|
|
9127
|
+
// A sweep that stops before it plans any alias records why on the sweep itself: otherwise
|
|
9128
|
+
// `status` cannot tell it from a sweep that collected and found nothing.
|
|
9073
9129
|
try {
|
|
9074
|
-
const
|
|
9075
|
-
const
|
|
9076
|
-
await
|
|
9077
|
-
|
|
9078
|
-
|
|
9079
|
-
|
|
9130
|
+
const identity = newIdentity();
|
|
9131
|
+
const creds = await mintCreds(this.auth, identity, "provisioner");
|
|
9132
|
+
const nc = await this.dial({ ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
9133
|
+
try {
|
|
9134
|
+
const jsm = await jetstreamManager(nc);
|
|
9135
|
+
const kvm = new Kvm(nc);
|
|
9136
|
+
await ensureAuthorityStores(jsm, kvm, this.space);
|
|
9137
|
+
const recordsKv = await kvm.open(recordsBucket(this.space));
|
|
9138
|
+
for (const slot of await walkStaticSlots(recordsKv, DEV_OWNER))
|
|
9139
|
+
slotRows.push(slot.row);
|
|
9140
|
+
}
|
|
9141
|
+
finally {
|
|
9142
|
+
await nc.drain().catch(() => nc.close());
|
|
9143
|
+
}
|
|
9080
9144
|
}
|
|
9081
9145
|
catch (error) {
|
|
9082
9146
|
sweep.completedAt = new Date().toISOString();
|
|
9147
|
+
sweep.error = rejectionText(error);
|
|
9083
9148
|
throw error;
|
|
9084
9149
|
}
|
|
9085
|
-
finally {
|
|
9086
|
-
await nc.drain().catch(() => nc.close());
|
|
9087
|
-
}
|
|
9088
9150
|
let retiredSlots = 0;
|
|
9089
9151
|
let preservedForeign = 0;
|
|
9090
9152
|
let preservedAdopted = 0;
|