@gajae-code/coding-agent 0.17.4 → 0.17.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/dist/types/config/autorouting-tier-map.d.ts +1031 -3
- package/dist/types/config/file-lock.d.ts +4 -2
- package/dist/types/config/model-registry.d.ts +3 -1
- package/dist/types/config/model-resolver.d.ts +1 -1
- package/dist/types/exec/bash-executor.d.ts +2 -0
- package/dist/types/extensibility/gjc-plugins/runtime-adapters.d.ts +11 -2
- package/dist/types/gjc-runtime/state-writer.d.ts +14 -1
- package/dist/types/harness-control-plane/control-endpoint.d.ts +4 -1
- package/dist/types/harness-control-plane/finalize.d.ts +14 -3
- package/dist/types/main.d.ts +1 -1
- package/dist/types/runtime-mcp/loader.d.ts +2 -0
- package/dist/types/runtime-mcp/manager.d.ts +13 -6
- package/dist/types/sdk/broker/broker.d.ts +4 -2
- package/dist/types/sdk/broker/ensure.d.ts +3 -0
- package/dist/types/sdk/broker/identity.d.ts +2 -0
- package/dist/types/sdk/broker/managed-task-dag.d.ts +1335 -0
- package/dist/types/sdk/broker/managed-task-verification.d.ts +37 -0
- package/dist/types/sdk/broker/spawn-authority.d.ts +8 -0
- package/dist/types/sdk/broker/startup-failure.d.ts +4 -0
- package/dist/types/sdk/bus/telegram-daemon-contract.d.ts +5 -1
- package/dist/types/sdk/prompt-deadline-flush.d.ts +2 -0
- package/dist/types/sdk/session.d.ts +3 -3
- package/dist/types/session/agent-session.d.ts +7 -2
- package/dist/types/task/types.d.ts +2 -0
- package/dist/types/tools/bash-interactive.d.ts +1 -0
- package/dist/types/tools/fetch.d.ts +10 -1
- package/dist/types/tools/subagent.d.ts +8 -1
- package/package.json +7 -7
- package/src/cli/public-command-registry.ts +8 -2
- package/src/cli/setup-cli.ts +1 -0
- package/src/cli/stats-cli.ts +11 -1
- package/src/config/autorouting-tier-map.ts +529 -3
- package/src/config/file-lock.ts +59 -14
- package/src/config/model-discovery-manager.ts +4 -4
- package/src/config/model-profile-activation.ts +83 -23
- package/src/config/model-registry.ts +72 -12
- package/src/config/model-resolver.ts +6 -1
- package/src/coordinator-mcp/question-state.ts +4 -1
- package/src/coordinator-mcp/server.ts +14 -6
- package/src/exec/bash-executor.ts +3 -0
- package/src/extensibility/gjc-plugins/runtime-adapters.ts +46 -8
- package/src/gjc-runtime/state-writer.ts +317 -6
- package/src/harness-control-plane/control-endpoint.ts +24 -6
- package/src/harness-control-plane/finalize.ts +312 -25
- package/src/harness-control-plane/owner.ts +323 -89
- package/src/internal-urls/docs-index.generated.ts +6 -4
- package/src/main.ts +69 -19
- package/src/modes/acp/acp-agent.ts +581 -37
- package/src/modes/components/gjc-bundle-settings.ts +21 -1
- package/src/modes/components/tool-status-header.ts +72 -6
- package/src/modes/controllers/runtime-mcp-command-controller.ts +1 -1
- package/src/runtime-mcp/loader.ts +6 -3
- package/src/runtime-mcp/manager.ts +177 -75
- package/src/sdk/broker/broker.ts +1057 -150
- package/src/sdk/broker/ensure.ts +21 -3
- package/src/sdk/broker/identity.ts +13 -0
- package/src/sdk/broker/managed-task-dag.ts +1927 -0
- package/src/sdk/broker/managed-task-verification.ts +642 -0
- package/src/sdk/broker/spawn-authority.ts +140 -7
- package/src/sdk/broker/startup-failure.ts +80 -1
- package/src/sdk/broker/transport.ts +1 -0
- package/src/sdk/bus/chat-daemon-control.ts +2 -2
- package/src/sdk/bus/index.ts +22 -6
- package/src/sdk/bus/telegram-daemon-contract.ts +5 -1
- package/src/sdk/cli/session-cli.ts +18 -0
- package/src/sdk/host/session-runtime.test.ts +46 -0
- package/src/sdk/host/session-runtime.ts +13 -6
- package/src/sdk/lifecycle/service.ts +2 -0
- package/src/sdk/prompt-deadline-flush.ts +26 -4
- package/src/sdk/session.ts +587 -115
- package/src/session/agent-session.ts +40 -12
- package/src/session/internal/managed-session-scope.ts +10 -0
- package/src/session/internal/managed-session-storage.ts +30 -1
- package/src/setup/provider-presets.json +2 -1
- package/src/slash-commands/builtin-registry.ts +1 -1
- package/src/task/executor.ts +46 -17
- package/src/task/types.ts +2 -0
- package/src/tools/bash-interactive.ts +2 -0
- package/src/tools/bash.ts +20 -0
- package/src/tools/fetch.ts +21 -5
- package/src/tools/subagent-render.ts +24 -1
- package/src/tools/subagent.ts +27 -1
- package/src/utils/mupdf.ts +11 -4
package/src/sdk/broker/broker.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { logger, resolveEquivalentPath } from "@gajae-code/utils";
|
|
|
8
8
|
import packageJson from "../../../package.json" with { type: "json" };
|
|
9
9
|
import type { ModelProfileErrorDetails } from "../../config/model-profile-contract";
|
|
10
10
|
import { planLaunchWorktree } from "../../gjc-runtime/launch-worktree";
|
|
11
|
+
import { readExistingStateForMutation, withWorkflowStateLock } from "../../gjc-runtime/state-writer";
|
|
11
12
|
import { SdkClient, SdkClientError } from "../client";
|
|
12
13
|
import {
|
|
13
14
|
BROKER_RUNTIME_ABORT_CAPABILITY_FIELD,
|
|
@@ -41,6 +42,7 @@ import {
|
|
|
41
42
|
deriveIdempotencyIdentity,
|
|
42
43
|
deriveLegacyIdentity,
|
|
43
44
|
deriveLegacyTargetIdentity,
|
|
45
|
+
deriveScopedIdempotencyIdentity,
|
|
44
46
|
getBrokerIdentityKey,
|
|
45
47
|
} from "./identity";
|
|
46
48
|
import {
|
|
@@ -59,6 +61,36 @@ import {
|
|
|
59
61
|
type LifecycleState,
|
|
60
62
|
type TerminalReadBack,
|
|
61
63
|
} from "./lifecycle-ledger";
|
|
64
|
+
import {
|
|
65
|
+
cancelManagedTasks,
|
|
66
|
+
createManagedDomainBinding,
|
|
67
|
+
currentManagedRevision,
|
|
68
|
+
defineManagedTaskGraph,
|
|
69
|
+
inspectManagedAttemptByNativeIdentity,
|
|
70
|
+
loadManagedDomainBinding,
|
|
71
|
+
loadManagedEnrollmentRecord,
|
|
72
|
+
lookupManagedAttemptByNativeIdentity,
|
|
73
|
+
type ManagedAttemptRef,
|
|
74
|
+
type ManagedDomainBinding,
|
|
75
|
+
type ManagedEnrollmentRecord,
|
|
76
|
+
type ManagedTaskAttempt,
|
|
77
|
+
managedAttemptRefMatches,
|
|
78
|
+
managedIdentity,
|
|
79
|
+
managedNativeVector,
|
|
80
|
+
managedTaskDomainPath,
|
|
81
|
+
markManagedEnrollmentEstablished,
|
|
82
|
+
markManagedEnrollmentPending,
|
|
83
|
+
markManagedEnrollmentPublishing,
|
|
84
|
+
observeOrAdmitManagedTask,
|
|
85
|
+
recordManagedEnrollment,
|
|
86
|
+
recordManagedNativeObservation,
|
|
87
|
+
restoreManagedAttemptRefs,
|
|
88
|
+
reviseManagedTaskGraph,
|
|
89
|
+
transactManagedTaskDomain,
|
|
90
|
+
validateManagedTaskDomain,
|
|
91
|
+
withManagedNativeEffectAuthorized,
|
|
92
|
+
} from "./managed-task-dag";
|
|
93
|
+
import { reconcileRunningManagedVerification, verifyManagedTaskAttempt } from "./managed-task-verification";
|
|
62
94
|
import { createMasterCapabilityVerifier, readEndpoint } from "./master-capability";
|
|
63
95
|
import { sdkInternalRuntimeImage } from "./runtime";
|
|
64
96
|
import { type IndexedSession, isSessionAuthorityEligible, SessionIndex, type SessionList } from "./session-index";
|
|
@@ -215,7 +247,7 @@ export type BrokerCleanupEvidence = {
|
|
|
215
247
|
artifactsIdentity?: BrokerCleanupIdentity;
|
|
216
248
|
transcriptIdentity?: BrokerCleanupIdentity;
|
|
217
249
|
transcriptParentIdentity?: { dev: string; ino: string };
|
|
218
|
-
/** Identity-bound lifecycle
|
|
250
|
+
/** Identity-bound lifecycle metadataker retained when exact cleanup is deferred. */
|
|
219
251
|
metadataIdentity?: BrokerCleanupIdentity;
|
|
220
252
|
metadataPath?: string;
|
|
221
253
|
/** Monotonic append-only cleanup attempt. */
|
|
@@ -247,7 +279,7 @@ export type BrokerCleanupEvidence = {
|
|
|
247
279
|
/** Fully identity-bound startup-failure cleanup plan, persisted before any detach. */
|
|
248
280
|
lifecycleFiles?: BrokerLifecycleCleanupFile[];
|
|
249
281
|
lifecycleParentIdentity?: { dev: string; ino: string };
|
|
250
|
-
/** Delete metadata receipts authorize only the
|
|
282
|
+
/** Delete metadata receipts authorize only the canonicalker/ready sibling pair. */
|
|
251
283
|
lifecycleDeleteMetadata?: true;
|
|
252
284
|
};
|
|
253
285
|
export type BrokerResponse =
|
|
@@ -265,6 +297,17 @@ export type BrokerResponse =
|
|
|
265
297
|
durableEffects?: LifecycleDurableEffectsReceipt;
|
|
266
298
|
startupFailure?: LifecycleStartupFailureReceipt;
|
|
267
299
|
};
|
|
300
|
+
type VerifiedManagedOwner =
|
|
301
|
+
| {
|
|
302
|
+
kind: "verified";
|
|
303
|
+
ownerSessionId: string;
|
|
304
|
+
attestationEpoch: string;
|
|
305
|
+
controlRoot: string;
|
|
306
|
+
enrollmentId: string;
|
|
307
|
+
worktrees: string[];
|
|
308
|
+
aliases?: string[];
|
|
309
|
+
}
|
|
310
|
+
| { kind: "denied"; response: BrokerResponse };
|
|
268
311
|
const error = (code: BrokerErrorCode, message: string): BrokerResponse => ({ ok: false, error: { code, message } });
|
|
269
312
|
const spawnFailureError = (failure: SpawnSubstrateFailure): BrokerResponse => ({
|
|
270
313
|
ok: false,
|
|
@@ -362,6 +405,7 @@ type SpawnInFlight = {
|
|
|
362
405
|
resolve: (response: BrokerResponse) => void;
|
|
363
406
|
claimId?: string;
|
|
364
407
|
phase?: SpawnClaimV2["state"];
|
|
408
|
+
managedRequestHash?: string;
|
|
365
409
|
};
|
|
366
410
|
|
|
367
411
|
/** Complete five-leg child endpoint pin captured at registration. */
|
|
@@ -417,6 +461,9 @@ type SpawnAdmissionInput = {
|
|
|
417
461
|
cwd: string;
|
|
418
462
|
modelId?: string;
|
|
419
463
|
modelPreset?: string;
|
|
464
|
+
managedAttempt?: ManagedAttemptRef;
|
|
465
|
+
managedBinding?: ManagedDomainBinding;
|
|
466
|
+
managedNativeVector?: string;
|
|
420
467
|
};
|
|
421
468
|
|
|
422
469
|
function parseSpawnInput(
|
|
@@ -511,6 +558,23 @@ const SPAWN_HOST_REGISTRATION_TIMEOUT_MS = 10_000;
|
|
|
511
558
|
const SPAWN_HOST_REGISTRATION_POLL_MS = 50;
|
|
512
559
|
const SPAWN_PROMPT_EXCHANGE_TIMEOUT_MS = 10_000;
|
|
513
560
|
const MASTER_ORPHAN_GRACE_DEFAULT_MS = 120_000;
|
|
561
|
+
const MANAGED_CLOSE_WAIT_MS = 10_000;
|
|
562
|
+
|
|
563
|
+
function settlesWithin(promise: Promise<unknown>, timeoutMs: number): Promise<boolean> {
|
|
564
|
+
const settled = Promise.withResolvers<boolean>();
|
|
565
|
+
const timer = setTimeout(() => settled.resolve(false), timeoutMs);
|
|
566
|
+
void promise.then(
|
|
567
|
+
() => {
|
|
568
|
+
clearTimeout(timer);
|
|
569
|
+
settled.resolve(true);
|
|
570
|
+
},
|
|
571
|
+
() => {
|
|
572
|
+
clearTimeout(timer);
|
|
573
|
+
settled.resolve(true);
|
|
574
|
+
},
|
|
575
|
+
);
|
|
576
|
+
return settled.promise;
|
|
577
|
+
}
|
|
514
578
|
|
|
515
579
|
/** The five legs every usable pin must carry; a partial pin is missing authority. */
|
|
516
580
|
type CompleteSpawnPinAuthority = SpawnAuthorityV1 & {
|
|
@@ -1383,6 +1447,7 @@ const publicationObservationOverridesForTest = new WeakMap<Broker, BrokerPublica
|
|
|
1383
1447
|
const lockArtifactGraceOverridesForTest = new WeakMap<Broker, number>();
|
|
1384
1448
|
const livenessGraceOverridesForTest = new WeakMap<Broker, number>();
|
|
1385
1449
|
const heartbeatStallOverridesForTest = new WeakMap<Broker, PromiseWithResolvers<void>>();
|
|
1450
|
+
const managedCloseWaitOverridesForTest = new WeakMap<Broker, number>();
|
|
1386
1451
|
|
|
1387
1452
|
export class Broker {
|
|
1388
1453
|
readonly settings: ResolvedBrokerSettings;
|
|
@@ -1395,6 +1460,9 @@ export class Broker {
|
|
|
1395
1460
|
#chains = new Map<string, Promise<void>>();
|
|
1396
1461
|
#spawnInFlight = new Map<string, SpawnInFlight>();
|
|
1397
1462
|
#spawnTasks = new WeakMap<SpawnInFlight, string>();
|
|
1463
|
+
#managedAttempts = new Map<string, ManagedAttemptRef>();
|
|
1464
|
+
#managedFailedRoots = new Set<string>();
|
|
1465
|
+
#managedEnrollmentFailed = false;
|
|
1398
1466
|
#spawnAuthority: SpawnAuthorityStore | null = null;
|
|
1399
1467
|
#spawnPromptLayer: SpawnPromptLayer;
|
|
1400
1468
|
#spawnReapInFlight = false;
|
|
@@ -1510,122 +1578,12 @@ export class Broker {
|
|
|
1510
1578
|
admission.masterCapability = "";
|
|
1511
1579
|
}
|
|
1512
1580
|
if (!verified.allowed) return finish(error("spawn_failed", "master capability verification was denied"));
|
|
1513
|
-
const
|
|
1514
|
-
|
|
1515
|
-
if (!authority) return finish(error("unavailable", "spawn authority is unavailable"));
|
|
1516
|
-
const requestBindingMac = spawnBindingMac(
|
|
1517
|
-
key,
|
|
1518
|
-
admission,
|
|
1519
|
-
admission.modelId ?? null,
|
|
1520
|
-
admission.modelPreset ?? null,
|
|
1521
|
-
);
|
|
1522
|
-
const existing = authority.claim(lifecycleIdentity);
|
|
1523
|
-
const resolveModels = async (): Promise<BrokerResponse | undefined> => {
|
|
1524
|
-
// Validate the optional profile BEFORE any new durable claim or substrate
|
|
1525
|
-
// effect. Generic lifecycle create validates it; spawn must do the same.
|
|
1526
|
-
if (admission.modelPreset !== undefined) {
|
|
1527
|
-
const validated = validateBrokerModelPresetSync(this.settings.agentDir, admission.modelPreset);
|
|
1528
|
-
if (isBrokerResponse(validated)) return validated;
|
|
1529
|
-
admission.modelPreset = validated;
|
|
1530
|
-
}
|
|
1531
|
-
// Resolve the explicit model pin before a new durable claim. Raw
|
|
1532
|
-
// selectors must never reach a newly launched child.
|
|
1533
|
-
if (admission.modelId !== undefined) {
|
|
1534
|
-
const resolved = await this.#resolveModelPin(admission.modelId, { cwd: admission.cwd });
|
|
1535
|
-
if (!resolved.ok) return error("unknown_model", resolved.error);
|
|
1536
|
-
if (resolved.model === null)
|
|
1537
|
-
return error("unknown_model", "session.spawn modelId could not be resolved.");
|
|
1538
|
-
admission.modelId = resolved.model;
|
|
1539
|
-
}
|
|
1540
|
-
return undefined;
|
|
1541
|
-
};
|
|
1542
|
-
let decision: SpawnClaimDecision;
|
|
1543
|
-
if (existing !== undefined) {
|
|
1544
|
-
// An existing claim carries the raw-selector binding, so terminal and
|
|
1545
|
-
// replay responses do not depend on today's model/profile catalog.
|
|
1546
|
-
let existingBindingMac: string | undefined;
|
|
1547
|
-
if (existing.requestBindingMac === undefined) {
|
|
1548
|
-
// Claims written before raw-selector evidence was introduced only have
|
|
1549
|
-
// the canonical binding. Re-resolve that historical selector before
|
|
1550
|
-
// comparison; if the catalog no longer proves it, fail closed rather
|
|
1551
|
-
// than guessing or weakening idempotency conflict protection.
|
|
1552
|
-
const modelError = await resolveModels();
|
|
1553
|
-
if (modelError) return finish(modelError);
|
|
1554
|
-
existingBindingMac = spawnBindingMac(
|
|
1555
|
-
key,
|
|
1556
|
-
admission,
|
|
1557
|
-
admission.modelId ?? null,
|
|
1558
|
-
admission.modelPreset ?? null,
|
|
1559
|
-
);
|
|
1560
|
-
}
|
|
1561
|
-
decision = await authority.claimOrJoin(
|
|
1562
|
-
lifecycleIdentity,
|
|
1563
|
-
existingBindingMac,
|
|
1564
|
-
existingBindingMac ?? requestBindingMac,
|
|
1565
|
-
);
|
|
1566
|
-
if (decision.kind === "owner") {
|
|
1567
|
-
becameOwner = true;
|
|
1568
|
-
// A recovered pre-send claim may still need to launch. Preserve the
|
|
1569
|
-
// fail-closed model validation for that effectful path and reject a
|
|
1570
|
-
// catalog remap rather than launching a different model.
|
|
1571
|
-
const modelError = await resolveModels();
|
|
1572
|
-
if (modelError) return finish(modelError);
|
|
1573
|
-
const recoveredBindingMac = spawnBindingMac(
|
|
1574
|
-
key,
|
|
1575
|
-
admission,
|
|
1576
|
-
admission.modelId ?? null,
|
|
1577
|
-
admission.modelPreset ?? null,
|
|
1578
|
-
);
|
|
1579
|
-
if (recoveredBindingMac !== decision.claim.bindingMac)
|
|
1580
|
-
return finish(
|
|
1581
|
-
error("idempotency_conflict", "session.spawn model selection differs from its durable claim"),
|
|
1582
|
-
);
|
|
1583
|
-
}
|
|
1584
|
-
} else {
|
|
1585
|
-
const modelError = await resolveModels();
|
|
1586
|
-
if (modelError) {
|
|
1587
|
-
// A concurrent claimant may have durably established this exact raw
|
|
1588
|
-
// request while catalog loading was in flight. Re-check its opaque
|
|
1589
|
-
// binding before returning the validation failure.
|
|
1590
|
-
const raced = authority.claim(lifecycleIdentity);
|
|
1591
|
-
if (raced === undefined) return finish(modelError);
|
|
1592
|
-
decision = await authority.claimOrJoin(lifecycleIdentity, undefined, requestBindingMac);
|
|
1593
|
-
if (decision.kind === "owner") {
|
|
1594
|
-
becameOwner = true;
|
|
1595
|
-
return finish(modelError);
|
|
1596
|
-
}
|
|
1597
|
-
} else {
|
|
1598
|
-
const bindingMac = spawnBindingMac(
|
|
1599
|
-
key,
|
|
1600
|
-
admission,
|
|
1601
|
-
admission.modelId ?? null,
|
|
1602
|
-
admission.modelPreset ?? null,
|
|
1603
|
-
);
|
|
1604
|
-
decision = await authority.claimOrJoin(lifecycleIdentity, bindingMac, requestBindingMac);
|
|
1605
|
-
}
|
|
1606
|
-
}
|
|
1607
|
-
if (decision.kind === "idempotency_conflict")
|
|
1608
|
-
return finish(
|
|
1609
|
-
error("idempotency_conflict", "idempotency key conflicts with an existing session.spawn claim"),
|
|
1610
|
-
);
|
|
1611
|
-
if (decision.kind === "in_progress")
|
|
1612
|
-
return finish(error("spawn_in_progress", `session.spawn is ${decision.claim.state}`));
|
|
1613
|
-
if (decision.kind === "terminal_uncertain") {
|
|
1614
|
-
if (decision.claim.substrateProof !== undefined && decision.claim.authorityRef === undefined)
|
|
1615
|
-
return finish(await this.#reconcileUncertainRegistration(lifecycleIdentity, decision.claim));
|
|
1616
|
-
return finish(error("terminal_uncertain", "session.spawn outcome is uncertain"));
|
|
1617
|
-
}
|
|
1618
|
-
if (decision.kind === "terminal") return finish(this.#spawnTerminalResponse(decision.claim));
|
|
1619
|
-
if (decision.kind === "replay")
|
|
1620
|
-
return finish(await this.#reconcileSpawnReplay(lifecycleIdentity, decision.claim));
|
|
1621
|
-
becameOwner = true;
|
|
1622
|
-
inFlight.claimId = decision.claim.claimId;
|
|
1623
|
-
inFlight.phase = decision.claim.state;
|
|
1624
|
-
// Claim fsync precedes this audit mirror. Substrate work may start
|
|
1625
|
-
// only after the mirror succeeds; a startup reconciler rebuilds it claim-first.
|
|
1626
|
-
await this.ledger.begin(lifecycleIdentity, decision.claim.bindingMac);
|
|
1581
|
+
const managedDenied = await this.#denyOrdinaryManagedSpawn(lifecycleIdentity, idempotencyKey!);
|
|
1582
|
+
if (managedDenied) return finish(managedDenied);
|
|
1627
1583
|
return finish(
|
|
1628
|
-
await this.#
|
|
1584
|
+
await this.#driveVerifiedSpawn(lifecycleIdentity, admission, inFlight, owner => {
|
|
1585
|
+
becameOwner = owner;
|
|
1586
|
+
}),
|
|
1629
1587
|
);
|
|
1630
1588
|
} catch {
|
|
1631
1589
|
return finish(error("spawn_failed", "session.spawn admission could not be durably established"));
|
|
@@ -1636,6 +1594,641 @@ export class Broker {
|
|
|
1636
1594
|
if (becameOwner) await this.#spawnAuthority?.releaseOwner(lifecycleIdentity);
|
|
1637
1595
|
}
|
|
1638
1596
|
}
|
|
1597
|
+
async #denyOrdinaryManagedSpawn(lifecycleIdentity: string, callerKey: string): Promise<BrokerResponse | undefined> {
|
|
1598
|
+
if (this.#managedAttempts.has(lifecycleIdentity))
|
|
1599
|
+
return error("spawn_failed", "managed native identity cannot re-enter ordinary session.spawn");
|
|
1600
|
+
if (this.#managedEnrollmentFailed) return error("spawn_failed", "managed enrollment membership is unprovable");
|
|
1601
|
+
let enrolled: ManagedEnrollmentRecord;
|
|
1602
|
+
try {
|
|
1603
|
+
enrolled = await loadManagedEnrollmentRecord(this.settings.agentDir);
|
|
1604
|
+
} catch {
|
|
1605
|
+
return error("spawn_failed", "managed enrollment membership is unprovable");
|
|
1606
|
+
}
|
|
1607
|
+
if (enrolled.nativeIdentities.includes(lifecycleIdentity))
|
|
1608
|
+
return error("spawn_failed", "managed native identity cannot re-enter ordinary session.spawn");
|
|
1609
|
+
for (const root of enrolled.controlRoots) {
|
|
1610
|
+
if (this.#managedFailedRoots.has(root)) continue;
|
|
1611
|
+
try {
|
|
1612
|
+
const scopedIdentity = await deriveScopedIdempotencyIdentity(
|
|
1613
|
+
this.settings.agentDir,
|
|
1614
|
+
"session.spawn",
|
|
1615
|
+
callerKey,
|
|
1616
|
+
root,
|
|
1617
|
+
);
|
|
1618
|
+
if (this.#managedAttempts.has(scopedIdentity) || enrolled.nativeIdentities.includes(scopedIdentity))
|
|
1619
|
+
return error("spawn_failed", "managed native identity cannot re-enter ordinary session.spawn");
|
|
1620
|
+
const read = await readExistingStateForMutation(managedTaskDomainPath(root));
|
|
1621
|
+
if (read.kind !== "valid") continue;
|
|
1622
|
+
const state = validateManagedTaskDomain(read.value);
|
|
1623
|
+
const attempt =
|
|
1624
|
+
lookupManagedAttemptByNativeIdentity(state, lifecycleIdentity) ??
|
|
1625
|
+
lookupManagedAttemptByNativeIdentity(state, scopedIdentity) ??
|
|
1626
|
+
state.graphs.flatMap(graph => graph.attempts).find(item => item.native.key === callerKey);
|
|
1627
|
+
if (attempt) return error("spawn_failed", "managed native identity cannot re-enter ordinary session.spawn");
|
|
1628
|
+
} catch {}
|
|
1629
|
+
}
|
|
1630
|
+
return undefined;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
async #driveVerifiedSpawn(
|
|
1634
|
+
lifecycleIdentity: string,
|
|
1635
|
+
admission: SpawnAdmissionInput,
|
|
1636
|
+
inFlight: SpawnInFlight,
|
|
1637
|
+
setOwner: (owned: boolean) => void,
|
|
1638
|
+
): Promise<BrokerResponse> {
|
|
1639
|
+
const key = await getBrokerIdentityKey(this.settings.agentDir);
|
|
1640
|
+
const authority = this.#spawnAuthority;
|
|
1641
|
+
if (!authority) return error("unavailable", "spawn authority is unavailable");
|
|
1642
|
+
const requestBindingMac = spawnBindingMac(
|
|
1643
|
+
key,
|
|
1644
|
+
admission,
|
|
1645
|
+
admission.modelId ?? null,
|
|
1646
|
+
admission.modelPreset ?? null,
|
|
1647
|
+
);
|
|
1648
|
+
const existing = authority.claim(lifecycleIdentity);
|
|
1649
|
+
const resolveModels = async (): Promise<BrokerResponse | undefined> => {
|
|
1650
|
+
if (admission.modelPreset !== undefined) {
|
|
1651
|
+
const validated = validateBrokerModelPresetSync(this.settings.agentDir, admission.modelPreset);
|
|
1652
|
+
if (isBrokerResponse(validated)) return validated;
|
|
1653
|
+
admission.modelPreset = validated;
|
|
1654
|
+
}
|
|
1655
|
+
if (admission.modelId !== undefined) {
|
|
1656
|
+
const resolved = await this.#resolveModelPin(admission.modelId, { cwd: admission.cwd });
|
|
1657
|
+
if (!resolved.ok) return error("unknown_model", resolved.error);
|
|
1658
|
+
if (resolved.model === null) return error("unknown_model", "session.spawn modelId could not be resolved.");
|
|
1659
|
+
admission.modelId = resolved.model;
|
|
1660
|
+
}
|
|
1661
|
+
return undefined;
|
|
1662
|
+
};
|
|
1663
|
+
let decision: SpawnClaimDecision | undefined;
|
|
1664
|
+
if (existing !== undefined) {
|
|
1665
|
+
let existingBindingMac: string | undefined;
|
|
1666
|
+
if (existing.requestBindingMac === undefined) {
|
|
1667
|
+
const modelError = await resolveModels();
|
|
1668
|
+
if (modelError) return modelError;
|
|
1669
|
+
existingBindingMac = spawnBindingMac(
|
|
1670
|
+
key,
|
|
1671
|
+
admission,
|
|
1672
|
+
admission.modelId ?? null,
|
|
1673
|
+
admission.modelPreset ?? null,
|
|
1674
|
+
);
|
|
1675
|
+
}
|
|
1676
|
+
decision = await authority.claimOrJoin(
|
|
1677
|
+
lifecycleIdentity,
|
|
1678
|
+
existingBindingMac,
|
|
1679
|
+
existingBindingMac ?? requestBindingMac,
|
|
1680
|
+
);
|
|
1681
|
+
if (decision.kind === "owner") {
|
|
1682
|
+
setOwner(true);
|
|
1683
|
+
const modelError = await resolveModels();
|
|
1684
|
+
if (modelError) return modelError;
|
|
1685
|
+
const recoveredBindingMac = spawnBindingMac(
|
|
1686
|
+
key,
|
|
1687
|
+
admission,
|
|
1688
|
+
admission.modelId ?? null,
|
|
1689
|
+
admission.modelPreset ?? null,
|
|
1690
|
+
);
|
|
1691
|
+
if (recoveredBindingMac !== decision.claim.bindingMac)
|
|
1692
|
+
return error("idempotency_conflict", "session.spawn model selection differs from its durable claim");
|
|
1693
|
+
}
|
|
1694
|
+
} else {
|
|
1695
|
+
const modelError = await resolveModels();
|
|
1696
|
+
if (modelError) {
|
|
1697
|
+
const raced = authority.claim(lifecycleIdentity);
|
|
1698
|
+
if (raced === undefined) return modelError;
|
|
1699
|
+
decision = await authority.claimOrJoin(lifecycleIdentity, undefined, requestBindingMac);
|
|
1700
|
+
if (decision.kind === "owner") {
|
|
1701
|
+
setOwner(true);
|
|
1702
|
+
return modelError;
|
|
1703
|
+
}
|
|
1704
|
+
} else {
|
|
1705
|
+
const bindingMac = spawnBindingMac(
|
|
1706
|
+
key,
|
|
1707
|
+
admission,
|
|
1708
|
+
admission.modelId ?? null,
|
|
1709
|
+
admission.modelPreset ?? null,
|
|
1710
|
+
);
|
|
1711
|
+
decision = await authority.claimOrJoin(lifecycleIdentity, bindingMac, requestBindingMac);
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
if (!decision) return error("spawn_failed", "session.spawn admission could not be durably established");
|
|
1715
|
+
if (decision.kind === "managed_key_conflict")
|
|
1716
|
+
return error("idempotency_conflict", "idempotency key is reserved by a managed task.dag attempt");
|
|
1717
|
+
if (decision.kind === "idempotency_conflict")
|
|
1718
|
+
return error("idempotency_conflict", "idempotency key conflicts with an existing session.spawn claim");
|
|
1719
|
+
if (decision.kind === "in_progress")
|
|
1720
|
+
return error("spawn_in_progress", `session.spawn is ${decision.claim.state}`);
|
|
1721
|
+
if (decision.kind === "terminal_uncertain") {
|
|
1722
|
+
if (decision.claim.substrateProof !== undefined && decision.claim.authorityRef === undefined)
|
|
1723
|
+
return await this.#reconcileUncertainRegistration(lifecycleIdentity, decision.claim);
|
|
1724
|
+
return error("terminal_uncertain", "session.spawn outcome is uncertain");
|
|
1725
|
+
}
|
|
1726
|
+
if (decision.kind === "terminal") return this.#spawnTerminalResponse(decision.claim);
|
|
1727
|
+
if (decision.kind === "replay") return await this.#reconcileSpawnReplay(lifecycleIdentity, decision.claim);
|
|
1728
|
+
setOwner(true);
|
|
1729
|
+
inFlight.claimId = decision.claim.claimId;
|
|
1730
|
+
inFlight.phase = decision.claim.state;
|
|
1731
|
+
await this.ledger.begin(lifecycleIdentity, decision.claim.bindingMac);
|
|
1732
|
+
return await this.#driveSpawn(lifecycleIdentity, decision.claim, admission, inFlight, decision.recovery);
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
async #handleManagedTaskDag(
|
|
1736
|
+
input: Record<string, unknown>,
|
|
1737
|
+
idempotencyKey: string | undefined,
|
|
1738
|
+
): Promise<BrokerResponse> {
|
|
1739
|
+
if (input.action === "define") return this.#managedDefine(input);
|
|
1740
|
+
if (input.action === "advance") return this.#managedAdvance(input, idempotencyKey);
|
|
1741
|
+
if (input.action === "status") return this.#managedStatus(input);
|
|
1742
|
+
if (input.action === "revise") return this.#managedRevise(input);
|
|
1743
|
+
if (input.action === "cancel") return this.#managedCancel(input);
|
|
1744
|
+
if (input.action === "verify") return this.#managedVerify(input);
|
|
1745
|
+
return error("invalid_input", "unknown task.dag action");
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
async #verifiedManagedOwner(input: Record<string, unknown>): Promise<VerifiedManagedOwner> {
|
|
1749
|
+
const ownerSessionId = input.ownerSessionId;
|
|
1750
|
+
const attestationEpoch = input.attestationEpoch;
|
|
1751
|
+
const masterCapability = input.masterCapability;
|
|
1752
|
+
const controlRoot = input.controlRoot;
|
|
1753
|
+
const enrollmentId = input.enrollmentId;
|
|
1754
|
+
if (
|
|
1755
|
+
typeof ownerSessionId !== "string" ||
|
|
1756
|
+
!isCanonicalSessionId(ownerSessionId) ||
|
|
1757
|
+
typeof attestationEpoch !== "string" ||
|
|
1758
|
+
attestationEpoch.length === 0 ||
|
|
1759
|
+
attestationEpoch.length > 512 ||
|
|
1760
|
+
typeof masterCapability !== "string" ||
|
|
1761
|
+
masterCapability.length === 0 ||
|
|
1762
|
+
masterCapability.length > 16_384 ||
|
|
1763
|
+
typeof controlRoot !== "string" ||
|
|
1764
|
+
!path.isAbsolute(controlRoot) ||
|
|
1765
|
+
path.normalize(controlRoot) !== controlRoot ||
|
|
1766
|
+
typeof enrollmentId !== "string" ||
|
|
1767
|
+
enrollmentId.trim().length === 0 ||
|
|
1768
|
+
enrollmentId.length > 256
|
|
1769
|
+
)
|
|
1770
|
+
return { kind: "denied", response: error("invalid_input", "task.dag authorization is invalid") };
|
|
1771
|
+
if ("accepted" in input || "pass" in input || "receipt" in input)
|
|
1772
|
+
return { kind: "denied", response: error("invalid_input", "public verification material is not authority") };
|
|
1773
|
+
if (input.worktrees !== undefined) {
|
|
1774
|
+
if (!Array.isArray(input.worktrees) || input.worktrees.length === 0)
|
|
1775
|
+
return { kind: "denied", response: error("invalid_input", "worktrees are invalid") };
|
|
1776
|
+
if (
|
|
1777
|
+
input.worktrees.some(
|
|
1778
|
+
value => typeof value !== "string" || !path.isAbsolute(value) || path.normalize(value) !== value,
|
|
1779
|
+
)
|
|
1780
|
+
)
|
|
1781
|
+
return { kind: "denied", response: error("invalid_input", "worktrees are invalid") };
|
|
1782
|
+
}
|
|
1783
|
+
if (input.aliases !== undefined) {
|
|
1784
|
+
if (!Array.isArray(input.aliases))
|
|
1785
|
+
return { kind: "denied", response: error("invalid_input", "aliases are invalid") };
|
|
1786
|
+
if (
|
|
1787
|
+
input.aliases.some(
|
|
1788
|
+
value =>
|
|
1789
|
+
typeof value !== "string" || value.length === 0 || path.normalize(value) !== path.resolve(value),
|
|
1790
|
+
)
|
|
1791
|
+
)
|
|
1792
|
+
return { kind: "denied", response: error("invalid_input", "aliases are invalid") };
|
|
1793
|
+
}
|
|
1794
|
+
let allowed = false;
|
|
1795
|
+
try {
|
|
1796
|
+
allowed = (await this.verifyMasterCapability(ownerSessionId, masterCapability, attestationEpoch)).allowed;
|
|
1797
|
+
} finally {
|
|
1798
|
+
if (typeof input.masterCapability === "string") input.masterCapability = "";
|
|
1799
|
+
}
|
|
1800
|
+
if (!allowed)
|
|
1801
|
+
return { kind: "denied", response: error("spawn_failed", "master capability verification was denied") };
|
|
1802
|
+
const worktrees = Array.isArray(input.worktrees)
|
|
1803
|
+
? input.worktrees.filter((value): value is string => typeof value === "string" && value.length > 0)
|
|
1804
|
+
: [controlRoot];
|
|
1805
|
+
const aliases = Array.isArray(input.aliases)
|
|
1806
|
+
? input.aliases.filter((value): value is string => typeof value === "string" && value.length > 0)
|
|
1807
|
+
: undefined;
|
|
1808
|
+
return {
|
|
1809
|
+
kind: "verified",
|
|
1810
|
+
ownerSessionId,
|
|
1811
|
+
attestationEpoch,
|
|
1812
|
+
controlRoot,
|
|
1813
|
+
enrollmentId,
|
|
1814
|
+
worktrees: worktrees.length > 0 ? worktrees : [controlRoot],
|
|
1815
|
+
...(aliases && aliases.length > 0 ? { aliases } : {}),
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
#managedPublic(
|
|
1820
|
+
stateRevision: number,
|
|
1821
|
+
graphs: Array<{ id: string; revision: number; attempts: ManagedTaskAttempt[] }>,
|
|
1822
|
+
): Record<string, unknown> {
|
|
1823
|
+
return {
|
|
1824
|
+
stateRevision,
|
|
1825
|
+
graphs: graphs.map(graph => ({
|
|
1826
|
+
id: graph.id,
|
|
1827
|
+
revision: graph.revision,
|
|
1828
|
+
attempts: graph.attempts.length,
|
|
1829
|
+
nativeRefs: graph.attempts.map(attempt => attempt.native.identity),
|
|
1830
|
+
fences: graph.attempts.map(attempt => attempt.fence),
|
|
1831
|
+
validations: graph.attempts.map(attempt => attempt.validation),
|
|
1832
|
+
accepted: graph.attempts.map(attempt => attempt.accepted !== null),
|
|
1833
|
+
})),
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
async #managedDefine(input: Record<string, unknown>): Promise<BrokerResponse> {
|
|
1838
|
+
if (
|
|
1839
|
+
typeof input.expectedRevision !== "number" ||
|
|
1840
|
+
!Number.isSafeInteger(input.expectedRevision) ||
|
|
1841
|
+
input.expectedRevision < 0
|
|
1842
|
+
)
|
|
1843
|
+
return error("invalid_input", "expectedRevision is required");
|
|
1844
|
+
if (typeof input.graphId !== "string" || input.graphId.trim().length === 0)
|
|
1845
|
+
return error("invalid_input", "graphId is required");
|
|
1846
|
+
const auth = await this.#verifiedManagedOwner(input);
|
|
1847
|
+
if (auth.kind === "denied") return auth.response;
|
|
1848
|
+
try {
|
|
1849
|
+
const binding = await createManagedDomainBinding({
|
|
1850
|
+
controlRoot: auth.controlRoot,
|
|
1851
|
+
agentDir: this.settings.agentDir,
|
|
1852
|
+
enrollmentId: auth.enrollmentId,
|
|
1853
|
+
worktrees: auth.worktrees,
|
|
1854
|
+
...(auth.aliases ? { aliases: auth.aliases } : {}),
|
|
1855
|
+
});
|
|
1856
|
+
let rootWasEstablished = false;
|
|
1857
|
+
try {
|
|
1858
|
+
rootWasEstablished = (await loadManagedEnrollmentRecord(this.settings.agentDir)).establishedRoots.includes(
|
|
1859
|
+
binding.controlRoot,
|
|
1860
|
+
);
|
|
1861
|
+
} catch {
|
|
1862
|
+
throw new Error("native managed evidence exists");
|
|
1863
|
+
}
|
|
1864
|
+
// Publish the enrollment before the first domain snapshot. The index is
|
|
1865
|
+
// the recovery root list, so this ordering leaves only a recoverable
|
|
1866
|
+
// stale index entry if the process dies before the state publication.
|
|
1867
|
+
await recordManagedEnrollment(this.settings.agentDir, binding.controlRoot);
|
|
1868
|
+
const result = await transactManagedTaskDomain(
|
|
1869
|
+
{
|
|
1870
|
+
binding,
|
|
1871
|
+
expectedRevision: input.expectedRevision,
|
|
1872
|
+
...(input.expectedRevision === 0
|
|
1873
|
+
? {
|
|
1874
|
+
assertNoManagedEvidence: async () => {
|
|
1875
|
+
let enrolled: ManagedEnrollmentRecord;
|
|
1876
|
+
try {
|
|
1877
|
+
enrolled = await loadManagedEnrollmentRecord(this.settings.agentDir);
|
|
1878
|
+
} catch {
|
|
1879
|
+
throw new Error("native managed evidence exists");
|
|
1880
|
+
}
|
|
1881
|
+
if (
|
|
1882
|
+
rootWasEstablished ||
|
|
1883
|
+
enrolled.establishedRoots.includes(binding.controlRoot) ||
|
|
1884
|
+
enrolled.publishingRoots.includes(binding.controlRoot) ||
|
|
1885
|
+
(enrolled.byRoot[binding.controlRoot] ?? []).length > 0
|
|
1886
|
+
)
|
|
1887
|
+
throw new Error("native managed evidence exists");
|
|
1888
|
+
await recordManagedEnrollment(this.settings.agentDir, binding.controlRoot);
|
|
1889
|
+
},
|
|
1890
|
+
beforeFirstPublication: async () => {
|
|
1891
|
+
await markManagedEnrollmentPublishing(this.settings.agentDir, binding.controlRoot);
|
|
1892
|
+
},
|
|
1893
|
+
onFirstPublication: async () => {
|
|
1894
|
+
await markManagedEnrollmentEstablished(this.settings.agentDir, binding.controlRoot);
|
|
1895
|
+
},
|
|
1896
|
+
onFirstPublicationAborted: async () => {
|
|
1897
|
+
await markManagedEnrollmentPending(this.settings.agentDir, binding.controlRoot);
|
|
1898
|
+
},
|
|
1899
|
+
}
|
|
1900
|
+
: {}),
|
|
1901
|
+
},
|
|
1902
|
+
async state =>
|
|
1903
|
+
defineManagedTaskGraph(state, {
|
|
1904
|
+
id: input.graphId as string,
|
|
1905
|
+
owner: auth.ownerSessionId,
|
|
1906
|
+
nodes: input.nodes,
|
|
1907
|
+
}),
|
|
1908
|
+
);
|
|
1909
|
+
await markManagedEnrollmentEstablished(this.settings.agentDir, binding.controlRoot);
|
|
1910
|
+
return {
|
|
1911
|
+
ok: true,
|
|
1912
|
+
result: {
|
|
1913
|
+
...this.#managedPublic(result.state.state_revision, result.state.graphs),
|
|
1914
|
+
graphId: result.result.id,
|
|
1915
|
+
},
|
|
1916
|
+
};
|
|
1917
|
+
} catch (caught) {
|
|
1918
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed define failed");
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
async #managedAdvance(input: Record<string, unknown>, idempotencyKey: string | undefined): Promise<BrokerResponse> {
|
|
1923
|
+
if (!idempotencyKey || idempotencyKey.length > 512)
|
|
1924
|
+
return error("invalid_input", "idempotencyKey is required for task.dag advance");
|
|
1925
|
+
if (
|
|
1926
|
+
typeof input.expectedRevision !== "number" ||
|
|
1927
|
+
!Number.isSafeInteger(input.expectedRevision) ||
|
|
1928
|
+
input.expectedRevision < 0
|
|
1929
|
+
)
|
|
1930
|
+
return error("invalid_input", "expectedRevision is required");
|
|
1931
|
+
if (typeof input.graphId !== "string" || typeof input.nodeId !== "string")
|
|
1932
|
+
return error("invalid_input", "graphId and nodeId are required");
|
|
1933
|
+
const maxAdmissions = input.maxAdmissions === undefined ? 1 : input.maxAdmissions;
|
|
1934
|
+
if (typeof maxAdmissions !== "number" || !Number.isSafeInteger(maxAdmissions) || maxAdmissions < 1)
|
|
1935
|
+
return error("invalid_input", "maxAdmissions is invalid");
|
|
1936
|
+
const auth = await this.#verifiedManagedOwner(input);
|
|
1937
|
+
if (auth.kind === "denied") return auth.response;
|
|
1938
|
+
try {
|
|
1939
|
+
const binding = await createManagedDomainBinding({
|
|
1940
|
+
controlRoot: auth.controlRoot,
|
|
1941
|
+
agentDir: this.settings.agentDir,
|
|
1942
|
+
enrollmentId: auth.enrollmentId,
|
|
1943
|
+
worktrees: auth.worktrees,
|
|
1944
|
+
...(auth.aliases ? { aliases: auth.aliases } : {}),
|
|
1945
|
+
});
|
|
1946
|
+
const scopedIdentity = await deriveScopedIdempotencyIdentity(
|
|
1947
|
+
this.settings.agentDir,
|
|
1948
|
+
"session.spawn",
|
|
1949
|
+
idempotencyKey,
|
|
1950
|
+
binding.controlRoot,
|
|
1951
|
+
);
|
|
1952
|
+
const ordinaryAliases = await Promise.all([
|
|
1953
|
+
deriveIdempotencyIdentity(this.settings.agentDir, "session.spawn", idempotencyKey),
|
|
1954
|
+
deriveLegacyIdentity(this.settings.agentDir, "session.spawn", idempotencyKey),
|
|
1955
|
+
]);
|
|
1956
|
+
const authority = this.#spawnAuthority;
|
|
1957
|
+
if (!authority) return error("unavailable", "spawn authority is unavailable");
|
|
1958
|
+
const requestHash = managedIdentity({
|
|
1959
|
+
graphId: input.graphId,
|
|
1960
|
+
nodeId: input.nodeId,
|
|
1961
|
+
ownerSessionId: auth.ownerSessionId,
|
|
1962
|
+
controlRoot: binding.controlRoot,
|
|
1963
|
+
});
|
|
1964
|
+
const active = this.#spawnInFlight.get(scopedIdentity);
|
|
1965
|
+
if (active) {
|
|
1966
|
+
if (active.managedRequestHash !== requestHash)
|
|
1967
|
+
return error("idempotency_conflict", "idempotency key conflicts with a live managed task.dag advance");
|
|
1968
|
+
return await active.completion;
|
|
1969
|
+
}
|
|
1970
|
+
const reserved = await transactManagedTaskDomain(
|
|
1971
|
+
{ binding, expectedRevision: input.expectedRevision },
|
|
1972
|
+
async state => {
|
|
1973
|
+
const graph = state.graphs.find(item => item.id === input.graphId);
|
|
1974
|
+
if (!graph) throw new Error("graph owner mismatch");
|
|
1975
|
+
const node = graph.nodes.find(item => item.definition.id === input.nodeId);
|
|
1976
|
+
if (!node) throw new Error("node not ready");
|
|
1977
|
+
const workspace = path.resolve(node.definition.workspace);
|
|
1978
|
+
if (typeof input.cwd === "string" && input.cwd.length > 0 && path.resolve(input.cwd) !== workspace)
|
|
1979
|
+
throw new Error("cwd must match admitted workspace");
|
|
1980
|
+
const prior = state.graphs
|
|
1981
|
+
.flatMap(item => item.attempts)
|
|
1982
|
+
.find(
|
|
1983
|
+
attempt =>
|
|
1984
|
+
!attempt.retired &&
|
|
1985
|
+
attempt.native.key === idempotencyKey &&
|
|
1986
|
+
attempt.native.requestHash === requestHash &&
|
|
1987
|
+
attempt.nodeId === input.nodeId &&
|
|
1988
|
+
attempt.id === `attempt-${idempotencyKey}`,
|
|
1989
|
+
);
|
|
1990
|
+
const admitted = await observeOrAdmitManagedTask(state, {
|
|
1991
|
+
graphId: graph.id,
|
|
1992
|
+
owner: auth.ownerSessionId,
|
|
1993
|
+
nodeId: input.nodeId as string,
|
|
1994
|
+
attemptId: `attempt-${idempotencyKey}`,
|
|
1995
|
+
native: {
|
|
1996
|
+
key: idempotencyKey,
|
|
1997
|
+
identity: prior?.native.identity ?? scopedIdentity,
|
|
1998
|
+
requestHash,
|
|
1999
|
+
},
|
|
2000
|
+
});
|
|
2001
|
+
if (
|
|
2002
|
+
!(await authority.reserveManagedKeyAliases(
|
|
2003
|
+
ordinaryAliases,
|
|
2004
|
+
admitted.nativeIdentity,
|
|
2005
|
+
binding.controlRoot,
|
|
2006
|
+
))
|
|
2007
|
+
)
|
|
2008
|
+
throw new Error("idempotency key conflicts with an existing ordinary session.spawn claim");
|
|
2009
|
+
return admitted;
|
|
2010
|
+
},
|
|
2011
|
+
);
|
|
2012
|
+
const attemptRef = reserved.result;
|
|
2013
|
+
if (
|
|
2014
|
+
!managedAttemptRefMatches(
|
|
2015
|
+
attemptRef,
|
|
2016
|
+
reserved.state.graphs.flatMap(g => g.attempts).find(a => a.id === attemptRef.attemptId)!,
|
|
2017
|
+
binding,
|
|
2018
|
+
)
|
|
2019
|
+
)
|
|
2020
|
+
return error("spawn_failed", "managed attempt ref mismatch");
|
|
2021
|
+
const activeAfterAdmission = this.#spawnInFlight.get(attemptRef.nativeIdentity);
|
|
2022
|
+
if (activeAfterAdmission) {
|
|
2023
|
+
if (activeAfterAdmission.managedRequestHash !== requestHash)
|
|
2024
|
+
return error("idempotency_conflict", "idempotency key conflicts with a live managed task.dag advance");
|
|
2025
|
+
return await activeAfterAdmission.completion;
|
|
2026
|
+
}
|
|
2027
|
+
const node = reserved.state.graphs
|
|
2028
|
+
.find(graph => graph.id === attemptRef.graphId)
|
|
2029
|
+
?.nodes.find(item => item.definition.id === attemptRef.nodeId);
|
|
2030
|
+
const persisted = reserved.state.graphs
|
|
2031
|
+
.flatMap(graph => graph.attempts)
|
|
2032
|
+
?.find(item => item.id === attemptRef.attemptId);
|
|
2033
|
+
if (!node || !persisted) return error("spawn_failed", "admitted node missing");
|
|
2034
|
+
this.#managedAttempts.set(attemptRef.nativeIdentity, attemptRef);
|
|
2035
|
+
const spawnAdmission: SpawnAdmissionInput = {
|
|
2036
|
+
task: node.definition.task,
|
|
2037
|
+
masterCapability: "",
|
|
2038
|
+
ownerSessionId: auth.ownerSessionId,
|
|
2039
|
+
attestationEpoch: auth.attestationEpoch,
|
|
2040
|
+
cwd: path.resolve(node.definition.workspace),
|
|
2041
|
+
managedAttempt: attemptRef,
|
|
2042
|
+
managedBinding: binding,
|
|
2043
|
+
managedNativeVector: managedNativeVector(persisted),
|
|
2044
|
+
};
|
|
2045
|
+
const completion = Promise.withResolvers<BrokerResponse>();
|
|
2046
|
+
const inFlight: SpawnInFlight = {
|
|
2047
|
+
completion: completion.promise,
|
|
2048
|
+
resolve: completion.resolve,
|
|
2049
|
+
managedRequestHash: requestHash,
|
|
2050
|
+
};
|
|
2051
|
+
this.#spawnInFlight.set(attemptRef.nativeIdentity, inFlight);
|
|
2052
|
+
this.#spawnTasks.set(inFlight, spawnAdmission.task);
|
|
2053
|
+
let becameOwner = false;
|
|
2054
|
+
let completionResult: BrokerResponse = error("spawn_failed", "managed advance did not complete");
|
|
2055
|
+
try {
|
|
2056
|
+
await recordManagedEnrollment(this.settings.agentDir, binding.controlRoot, attemptRef.nativeIdentity);
|
|
2057
|
+
const driven = await this.#driveVerifiedSpawn(
|
|
2058
|
+
attemptRef.nativeIdentity,
|
|
2059
|
+
spawnAdmission,
|
|
2060
|
+
inFlight,
|
|
2061
|
+
owner => {
|
|
2062
|
+
becameOwner = owner;
|
|
2063
|
+
},
|
|
2064
|
+
);
|
|
2065
|
+
completionResult = driven;
|
|
2066
|
+
if (!driven.ok) return driven;
|
|
2067
|
+
const liveRevision = await currentManagedRevision(binding);
|
|
2068
|
+
completionResult = {
|
|
2069
|
+
ok: true,
|
|
2070
|
+
result: {
|
|
2071
|
+
...this.#managedPublic(liveRevision, reserved.state.graphs),
|
|
2072
|
+
attemptId: attemptRef.attemptId,
|
|
2073
|
+
nativeIdentity: attemptRef.nativeIdentity,
|
|
2074
|
+
spawn: driven.result,
|
|
2075
|
+
},
|
|
2076
|
+
};
|
|
2077
|
+
return completionResult;
|
|
2078
|
+
} catch (caught) {
|
|
2079
|
+
completionResult = error(
|
|
2080
|
+
"spawn_failed",
|
|
2081
|
+
caught instanceof Error ? caught.message : "managed advance failed",
|
|
2082
|
+
);
|
|
2083
|
+
throw caught;
|
|
2084
|
+
} finally {
|
|
2085
|
+
completion.resolve(completionResult);
|
|
2086
|
+
this.#spawnTasks.delete(inFlight);
|
|
2087
|
+
if (this.#spawnInFlight.get(attemptRef.nativeIdentity) === inFlight)
|
|
2088
|
+
this.#spawnInFlight.delete(attemptRef.nativeIdentity);
|
|
2089
|
+
if (becameOwner) await this.#spawnAuthority?.releaseOwner(attemptRef.nativeIdentity);
|
|
2090
|
+
}
|
|
2091
|
+
} catch (caught) {
|
|
2092
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed advance failed");
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
async #managedStatus(input: Record<string, unknown>): Promise<BrokerResponse> {
|
|
2097
|
+
const auth = await this.#verifiedManagedOwner(input);
|
|
2098
|
+
if (auth.kind === "denied") return auth.response;
|
|
2099
|
+
try {
|
|
2100
|
+
const binding = await createManagedDomainBinding({
|
|
2101
|
+
controlRoot: auth.controlRoot,
|
|
2102
|
+
agentDir: this.settings.agentDir,
|
|
2103
|
+
enrollmentId: auth.enrollmentId,
|
|
2104
|
+
worktrees: auth.worktrees,
|
|
2105
|
+
...(auth.aliases ? { aliases: auth.aliases } : {}),
|
|
2106
|
+
});
|
|
2107
|
+
const target = managedTaskDomainPath(binding.controlRoot);
|
|
2108
|
+
const state = await withWorkflowStateLock(
|
|
2109
|
+
target,
|
|
2110
|
+
async () => {
|
|
2111
|
+
const read = await readExistingStateForMutation(target);
|
|
2112
|
+
if (read.kind !== "valid") throw new Error("established authority missing");
|
|
2113
|
+
const parsed = validateManagedTaskDomain(read.value);
|
|
2114
|
+
if (managedIdentity(parsed.binding) !== managedIdentity(binding))
|
|
2115
|
+
throw new Error("domain binding mismatch");
|
|
2116
|
+
return parsed;
|
|
2117
|
+
},
|
|
2118
|
+
{ cwd: binding.controlRoot, privateDurable: { directory: path.dirname(target) } },
|
|
2119
|
+
);
|
|
2120
|
+
return { ok: true, result: this.#managedPublic(state.state_revision, state.graphs) };
|
|
2121
|
+
} catch (caught) {
|
|
2122
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed status failed");
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
async #managedRevise(input: Record<string, unknown>): Promise<BrokerResponse> {
|
|
2127
|
+
if (typeof input.expectedRevision !== "number" || !Number.isSafeInteger(input.expectedRevision))
|
|
2128
|
+
return error("invalid_input", "expectedRevision is required");
|
|
2129
|
+
if (typeof input.graphId !== "string") return error("invalid_input", "graphId is required");
|
|
2130
|
+
const auth = await this.#verifiedManagedOwner(input);
|
|
2131
|
+
if (auth.kind === "denied") return auth.response;
|
|
2132
|
+
if (this.#managedEnrollmentFailed || this.#managedFailedRoots.has(auth.controlRoot))
|
|
2133
|
+
return error("terminal_uncertain", "managed enrollment membership is unprovable; revision was not applied");
|
|
2134
|
+
try {
|
|
2135
|
+
const binding = await createManagedDomainBinding({
|
|
2136
|
+
controlRoot: auth.controlRoot,
|
|
2137
|
+
agentDir: this.settings.agentDir,
|
|
2138
|
+
enrollmentId: auth.enrollmentId,
|
|
2139
|
+
worktrees: auth.worktrees,
|
|
2140
|
+
...(auth.aliases ? { aliases: auth.aliases } : {}),
|
|
2141
|
+
});
|
|
2142
|
+
const result = await transactManagedTaskDomain(
|
|
2143
|
+
{ binding, expectedRevision: input.expectedRevision },
|
|
2144
|
+
async state => {
|
|
2145
|
+
const graph = state.graphs.find(item => item.id === input.graphId);
|
|
2146
|
+
if (!graph || graph.owner !== auth.ownerSessionId) throw new Error("graph owner mismatch");
|
|
2147
|
+
return reviseManagedTaskGraph(state, graph, input.nodes);
|
|
2148
|
+
},
|
|
2149
|
+
);
|
|
2150
|
+
if (!(await this.#closeManagedAttempts(binding.controlRoot, input.graphId as string, result.result)))
|
|
2151
|
+
return error("terminal_uncertain", "managed child close remains unresolved after graph revision");
|
|
2152
|
+
return {
|
|
2153
|
+
ok: true,
|
|
2154
|
+
result: {
|
|
2155
|
+
...this.#managedPublic(result.state.state_revision, result.state.graphs),
|
|
2156
|
+
affected: result.result,
|
|
2157
|
+
},
|
|
2158
|
+
};
|
|
2159
|
+
} catch (caught) {
|
|
2160
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed revise failed");
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
async #managedCancel(input: Record<string, unknown>): Promise<BrokerResponse> {
|
|
2165
|
+
if (typeof input.expectedRevision !== "number" || !Number.isSafeInteger(input.expectedRevision))
|
|
2166
|
+
return error("invalid_input", "expectedRevision is required");
|
|
2167
|
+
if (typeof input.graphId !== "string") return error("invalid_input", "graphId is required");
|
|
2168
|
+
const ids = Array.isArray(input.nodeIds)
|
|
2169
|
+
? input.nodeIds.filter((value): value is string => typeof value === "string")
|
|
2170
|
+
: [];
|
|
2171
|
+
if (ids.length === 0) return error("invalid_input", "nodeIds are required");
|
|
2172
|
+
const auth = await this.#verifiedManagedOwner(input);
|
|
2173
|
+
if (auth.kind === "denied") return auth.response;
|
|
2174
|
+
if (this.#managedEnrollmentFailed || this.#managedFailedRoots.has(auth.controlRoot))
|
|
2175
|
+
return error(
|
|
2176
|
+
"terminal_uncertain",
|
|
2177
|
+
"managed enrollment membership is unprovable; cancellation was not applied",
|
|
2178
|
+
);
|
|
2179
|
+
try {
|
|
2180
|
+
const binding = await createManagedDomainBinding({
|
|
2181
|
+
controlRoot: auth.controlRoot,
|
|
2182
|
+
agentDir: this.settings.agentDir,
|
|
2183
|
+
enrollmentId: auth.enrollmentId,
|
|
2184
|
+
worktrees: auth.worktrees,
|
|
2185
|
+
...(auth.aliases ? { aliases: auth.aliases } : {}),
|
|
2186
|
+
});
|
|
2187
|
+
const result = await transactManagedTaskDomain(
|
|
2188
|
+
{ binding, expectedRevision: input.expectedRevision },
|
|
2189
|
+
async state => {
|
|
2190
|
+
const graph = state.graphs.find(item => item.id === input.graphId);
|
|
2191
|
+
if (!graph || graph.owner !== auth.ownerSessionId) throw new Error("graph owner mismatch");
|
|
2192
|
+
return cancelManagedTasks(graph, ids);
|
|
2193
|
+
},
|
|
2194
|
+
);
|
|
2195
|
+
if (!(await this.#closeManagedAttempts(binding.controlRoot, input.graphId as string, result.result)))
|
|
2196
|
+
return error("terminal_uncertain", "managed child close remains unresolved after cancellation");
|
|
2197
|
+
return {
|
|
2198
|
+
ok: true,
|
|
2199
|
+
result: {
|
|
2200
|
+
...this.#managedPublic(result.state.state_revision, result.state.graphs),
|
|
2201
|
+
affected: result.result,
|
|
2202
|
+
},
|
|
2203
|
+
};
|
|
2204
|
+
} catch (caught) {
|
|
2205
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed cancel failed");
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
2208
|
+
async #managedVerify(input: Record<string, unknown>): Promise<BrokerResponse> {
|
|
2209
|
+
if (typeof input.graphId !== "string" || typeof input.nodeId !== "string")
|
|
2210
|
+
return error("invalid_input", "graphId and nodeId are required");
|
|
2211
|
+
const auth = await this.#verifiedManagedOwner(input);
|
|
2212
|
+
if (auth.kind === "denied") return auth.response;
|
|
2213
|
+
try {
|
|
2214
|
+
const binding = await createManagedDomainBinding({
|
|
2215
|
+
controlRoot: auth.controlRoot,
|
|
2216
|
+
agentDir: this.settings.agentDir,
|
|
2217
|
+
enrollmentId: auth.enrollmentId,
|
|
2218
|
+
worktrees: auth.worktrees,
|
|
2219
|
+
...(auth.aliases ? { aliases: auth.aliases } : {}),
|
|
2220
|
+
});
|
|
2221
|
+
const verified = await verifyManagedTaskAttempt({
|
|
2222
|
+
binding,
|
|
2223
|
+
graphId: input.graphId,
|
|
2224
|
+
nodeId: input.nodeId,
|
|
2225
|
+
owner: auth.ownerSessionId,
|
|
2226
|
+
});
|
|
2227
|
+
return { ok: true, result: { ...verified, commandsStarted: verified.started } };
|
|
2228
|
+
} catch (caught) {
|
|
2229
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed verify failed");
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
1639
2232
|
|
|
1640
2233
|
/** Safe, allowlisted spawn result projection; never carries request material. */
|
|
1641
2234
|
#spawnResult(code: "spawn_accepted" | "spawn_replayed", claim: SpawnClaimV2): Record<string, unknown> {
|
|
@@ -1751,14 +2344,39 @@ export class Broker {
|
|
|
1751
2344
|
...(admission.modelId === undefined ? {} : { modelId: admission.modelId }),
|
|
1752
2345
|
...(admission.modelPreset === undefined ? {} : { modelPreset: admission.modelPreset }),
|
|
1753
2346
|
});
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
2347
|
+
if (admission.managedAttempt && admission.managedBinding && admission.managedNativeVector) {
|
|
2348
|
+
try {
|
|
2349
|
+
current = await withManagedNativeEffectAuthorized(
|
|
2350
|
+
admission.managedBinding,
|
|
2351
|
+
admission.managedAttempt.nativeIdentity,
|
|
2352
|
+
admission.managedNativeVector,
|
|
2353
|
+
"launch",
|
|
2354
|
+
async () =>
|
|
2355
|
+
(
|
|
2356
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
2357
|
+
claimId: current.claimId,
|
|
2358
|
+
from: "prepared",
|
|
2359
|
+
to: "substrate_starting",
|
|
2360
|
+
childId: prep.childId,
|
|
2361
|
+
})
|
|
2362
|
+
).claim,
|
|
2363
|
+
);
|
|
2364
|
+
} catch (caught) {
|
|
2365
|
+
return error(
|
|
2366
|
+
"spawn_failed",
|
|
2367
|
+
caught instanceof Error ? caught.message : "managed native fence denied",
|
|
2368
|
+
);
|
|
2369
|
+
}
|
|
2370
|
+
} else {
|
|
2371
|
+
current = (
|
|
2372
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
2373
|
+
claimId: current.claimId,
|
|
2374
|
+
from: "prepared",
|
|
2375
|
+
to: "substrate_starting",
|
|
2376
|
+
childId: prep.childId,
|
|
2377
|
+
})
|
|
2378
|
+
).claim;
|
|
2379
|
+
}
|
|
1762
2380
|
inFlight.phase = current.state;
|
|
1763
2381
|
const launched = await provider.launch({
|
|
1764
2382
|
childSessionId: prep.childId,
|
|
@@ -1831,6 +2449,56 @@ export class Broker {
|
|
|
1831
2449
|
substrateProof: launched.proof,
|
|
1832
2450
|
})
|
|
1833
2451
|
).claim;
|
|
2452
|
+
inFlight.phase = current.state;
|
|
2453
|
+
if (admission.managedAttempt && admission.managedBinding) {
|
|
2454
|
+
let attempt: ManagedTaskAttempt | undefined;
|
|
2455
|
+
try {
|
|
2456
|
+
attempt = await inspectManagedAttemptByNativeIdentity(
|
|
2457
|
+
admission.managedBinding,
|
|
2458
|
+
admission.managedAttempt.nativeIdentity,
|
|
2459
|
+
);
|
|
2460
|
+
} catch {
|
|
2461
|
+
// A failed fence read cannot prove the launch is still authorized.
|
|
2462
|
+
}
|
|
2463
|
+
if (attempt?.fence !== "current" || attempt?.retired) {
|
|
2464
|
+
const release = await this.#releaseUnownedSubstrate(provider, launched.proof);
|
|
2465
|
+
let gone = release === "gone";
|
|
2466
|
+
if (release === "closed") {
|
|
2467
|
+
try {
|
|
2468
|
+
gone = (await provider.verify(launched.proof)) === "gone";
|
|
2469
|
+
} catch {
|
|
2470
|
+
gone = false;
|
|
2471
|
+
}
|
|
2472
|
+
}
|
|
2473
|
+
if (gone) {
|
|
2474
|
+
current = (
|
|
2475
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
2476
|
+
claimId: current.claimId,
|
|
2477
|
+
from: "substrate_starting",
|
|
2478
|
+
to: "closed",
|
|
2479
|
+
})
|
|
2480
|
+
).claim;
|
|
2481
|
+
inFlight.phase = current.state;
|
|
2482
|
+
launchedProof = undefined;
|
|
2483
|
+
return error("spawn_failed", "managed attempt was fenced while its child was launching");
|
|
2484
|
+
}
|
|
2485
|
+
current = (
|
|
2486
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
2487
|
+
claimId: current.claimId,
|
|
2488
|
+
from: "substrate_starting",
|
|
2489
|
+
to: "uncertain",
|
|
2490
|
+
failure: {
|
|
2491
|
+
substrateKind: launched.proof.substrateKind,
|
|
2492
|
+
code: "managed_child_close_unresolved",
|
|
2493
|
+
message: "managed child close could not be proven after its attempt was fenced",
|
|
2494
|
+
},
|
|
2495
|
+
})
|
|
2496
|
+
).claim;
|
|
2497
|
+
inFlight.phase = current.state;
|
|
2498
|
+
launchedProof = undefined;
|
|
2499
|
+
return error("terminal_uncertain", "managed child close is unresolved after its attempt was fenced");
|
|
2500
|
+
}
|
|
2501
|
+
}
|
|
1834
2502
|
const marker = { pid, incarnation, effectMarker: prep.effectMarker };
|
|
1835
2503
|
await writeEffectMarker(prep.stateRoot, prep.childId, marker);
|
|
1836
2504
|
const registration = await this.#spawnPromptLayer.awaitRegistration({
|
|
@@ -1976,26 +2644,59 @@ export class Broker {
|
|
|
1976
2644
|
}
|
|
1977
2645
|
const childId = current.childId;
|
|
1978
2646
|
const preparedSeed = current.seed;
|
|
2647
|
+
const preSendLeaseEpoch = current.preSendLease.epoch;
|
|
2648
|
+
const exactPinnedRegistration = pinnedRegistration;
|
|
1979
2649
|
// Dispatch without a complete pin must never consume the pre-send lease.
|
|
1980
|
-
if (
|
|
2650
|
+
if (exactPinnedRegistration === undefined)
|
|
1981
2651
|
return error("terminal_uncertain", "session.spawn endpoint pin is unavailable before dispatch");
|
|
1982
|
-
|
|
1983
|
-
await
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
)
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
2652
|
+
const dispatchSeed = async (): Promise<SpawnPromptDispatch> =>
|
|
2653
|
+
await this.#spawnPromptLayer.dispatch({
|
|
2654
|
+
sessionId: childId,
|
|
2655
|
+
task: admission.task,
|
|
2656
|
+
clientRef: preparedSeed.clientRef,
|
|
2657
|
+
pinned: exactPinnedRegistration,
|
|
2658
|
+
});
|
|
2659
|
+
let dispatched: SpawnPromptDispatch;
|
|
2660
|
+
if (admission.managedAttempt && admission.managedBinding && admission.managedNativeVector) {
|
|
2661
|
+
try {
|
|
2662
|
+
dispatched = await withManagedNativeEffectAuthorized(
|
|
2663
|
+
admission.managedBinding,
|
|
2664
|
+
admission.managedAttempt.nativeIdentity,
|
|
2665
|
+
admission.managedNativeVector,
|
|
2666
|
+
"seed",
|
|
2667
|
+
async () => {
|
|
2668
|
+
current = (
|
|
2669
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
2670
|
+
claimId: current.claimId,
|
|
2671
|
+
from: "seed_prepared",
|
|
2672
|
+
to: "dispatching",
|
|
2673
|
+
leaseEpoch: preSendLeaseEpoch,
|
|
2674
|
+
seed: { ...preparedSeed, phase: "dispatching" },
|
|
2675
|
+
})
|
|
2676
|
+
).claim;
|
|
2677
|
+
inFlight.phase = current.state;
|
|
2678
|
+
handedOff = true;
|
|
2679
|
+
return await dispatchSeed();
|
|
2680
|
+
},
|
|
2681
|
+
);
|
|
2682
|
+
} catch (caught) {
|
|
2683
|
+
if (handedOff) throw caught;
|
|
2684
|
+
return error("spawn_failed", caught instanceof Error ? caught.message : "managed native fence denied");
|
|
2685
|
+
}
|
|
2686
|
+
} else {
|
|
2687
|
+
current = (
|
|
2688
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
2689
|
+
claimId: current.claimId,
|
|
2690
|
+
from: "seed_prepared",
|
|
2691
|
+
to: "dispatching",
|
|
2692
|
+
leaseEpoch: current.preSendLease.epoch,
|
|
2693
|
+
seed: { ...preparedSeed, phase: "dispatching" },
|
|
2694
|
+
})
|
|
2695
|
+
).claim;
|
|
2696
|
+
inFlight.phase = current.state;
|
|
2697
|
+
handedOff = true;
|
|
2698
|
+
dispatched = await dispatchSeed();
|
|
2699
|
+
}
|
|
1999
2700
|
if (dispatched.kind === "accepted") {
|
|
2000
2701
|
current = (
|
|
2001
2702
|
await store.persistTransition(lifecycleIdentity, {
|
|
@@ -2104,6 +2805,10 @@ export class Broker {
|
|
|
2104
2805
|
}
|
|
2105
2806
|
}
|
|
2106
2807
|
}
|
|
2808
|
+
if (ambiguous && admission.managedBinding)
|
|
2809
|
+
await recordManagedNativeObservation(admission.managedBinding, lifecycleIdentity, "unknown").catch(
|
|
2810
|
+
() => undefined,
|
|
2811
|
+
);
|
|
2107
2812
|
return ambiguous
|
|
2108
2813
|
? error(
|
|
2109
2814
|
"terminal_uncertain",
|
|
@@ -2303,6 +3008,148 @@ export class Broker {
|
|
|
2303
3008
|
}
|
|
2304
3009
|
}
|
|
2305
3010
|
|
|
3011
|
+
async #restoreManagedAttempts(): Promise<void> {
|
|
3012
|
+
this.#managedAttempts.clear();
|
|
3013
|
+
this.#managedFailedRoots.clear();
|
|
3014
|
+
this.#managedEnrollmentFailed = false;
|
|
3015
|
+
try {
|
|
3016
|
+
const restored = await restoreManagedAttemptRefs(this.settings.agentDir);
|
|
3017
|
+
for (const ref of restored.refs) this.#managedAttempts.set(ref.nativeIdentity, ref);
|
|
3018
|
+
for (const root of restored.failedRoots) this.#managedFailedRoots.add(root);
|
|
3019
|
+
} catch {
|
|
3020
|
+
this.#managedEnrollmentFailed = true;
|
|
3021
|
+
}
|
|
3022
|
+
}
|
|
3023
|
+
|
|
3024
|
+
async #observeManagedNative(
|
|
3025
|
+
lifecycleIdentity: string,
|
|
3026
|
+
worker: "no-effect" | "authorized" | "unknown" | "closed",
|
|
3027
|
+
): Promise<void> {
|
|
3028
|
+
const ref = this.#managedAttempts.get(lifecycleIdentity);
|
|
3029
|
+
if (!ref) return;
|
|
3030
|
+
const binding = await loadManagedDomainBinding(ref.controlRoot);
|
|
3031
|
+
if (!binding) return;
|
|
3032
|
+
await recordManagedNativeObservation(binding, lifecycleIdentity, worker);
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
async #reconcileManagedNativeLifetimes(): Promise<void> {
|
|
3036
|
+
const store = this.#spawnAuthority;
|
|
3037
|
+
if (!store || this.#managedEnrollmentFailed) return;
|
|
3038
|
+
for (const ref of this.#managedAttempts.values()) {
|
|
3039
|
+
if (this.#managedFailedRoots.has(ref.controlRoot)) continue;
|
|
3040
|
+
const binding = await loadManagedDomainBinding(ref.controlRoot);
|
|
3041
|
+
if (!binding) throw new Error("managed attempt domain is unavailable during startup reconciliation");
|
|
3042
|
+
const attempt = await inspectManagedAttemptByNativeIdentity(binding, ref.nativeIdentity);
|
|
3043
|
+
if (!attempt) throw new Error("managed attempt authority is missing during startup reconciliation");
|
|
3044
|
+
const claim = store.claim(ref.nativeIdentity);
|
|
3045
|
+
if (attempt.fence !== "current" || attempt.retired) {
|
|
3046
|
+
if (!claim) {
|
|
3047
|
+
await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3048
|
+
continue;
|
|
3049
|
+
}
|
|
3050
|
+
if (claim.state === "prepared") {
|
|
3051
|
+
await store.persistTransition(ref.nativeIdentity, {
|
|
3052
|
+
claimId: claim.claimId,
|
|
3053
|
+
from: "prepared",
|
|
3054
|
+
to: "closed",
|
|
3055
|
+
});
|
|
3056
|
+
await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3057
|
+
continue;
|
|
3058
|
+
}
|
|
3059
|
+
if (claim.state === "closed" && !store.authority(ref.nativeIdentity) && !claim.substrateProof) {
|
|
3060
|
+
await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3061
|
+
continue;
|
|
3062
|
+
}
|
|
3063
|
+
if (claim.state === "pre_send_rejected" && !store.authority(ref.nativeIdentity) && !claim.substrateProof) {
|
|
3064
|
+
await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3065
|
+
continue;
|
|
3066
|
+
}
|
|
3067
|
+
const outcome = await this.#closeSpawnAuthority(ref.nativeIdentity);
|
|
3068
|
+
if (outcome !== "closed") {
|
|
3069
|
+
await this.#observeManagedNative(ref.nativeIdentity, "unknown").catch(() => undefined);
|
|
3070
|
+
throw new Error("fenced managed child could not be exactly closed during startup reconciliation");
|
|
3071
|
+
}
|
|
3072
|
+
await this.#observeManagedNative(ref.nativeIdentity, "closed");
|
|
3073
|
+
continue;
|
|
3074
|
+
}
|
|
3075
|
+
if (!claim) {
|
|
3076
|
+
await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3077
|
+
continue;
|
|
3078
|
+
}
|
|
3079
|
+
if (claim.state === "uncertain" || claim.state === "dispatching" || claim.state === "substrate_starting") {
|
|
3080
|
+
await this.#observeManagedNative(ref.nativeIdentity, "unknown");
|
|
3081
|
+
continue;
|
|
3082
|
+
}
|
|
3083
|
+
if (claim.state === "closed") {
|
|
3084
|
+
const authority = store.authority(ref.nativeIdentity);
|
|
3085
|
+
if (authority?.closeState === "closed" || (!authority && claim.substrateProof))
|
|
3086
|
+
await this.#observeManagedNative(ref.nativeIdentity, "closed");
|
|
3087
|
+
else if (!authority) await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3088
|
+
else await this.#observeManagedNative(ref.nativeIdentity, "unknown");
|
|
3089
|
+
continue;
|
|
3090
|
+
}
|
|
3091
|
+
if (claim.state === "pre_send_rejected") {
|
|
3092
|
+
const authority = store.authority(ref.nativeIdentity);
|
|
3093
|
+
if (!authority && claim.substrateProof) {
|
|
3094
|
+
const outcome = await this.#closeSpawnAuthority(ref.nativeIdentity);
|
|
3095
|
+
await this.#observeManagedNative(ref.nativeIdentity, outcome === "closed" ? "closed" : "unknown");
|
|
3096
|
+
} else if (!authority) await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3097
|
+
else await this.#observeManagedNative(ref.nativeIdentity, "unknown");
|
|
3098
|
+
continue;
|
|
3099
|
+
}
|
|
3100
|
+
if (claim.state === "prepared") continue;
|
|
3101
|
+
await this.#observeManagedNative(ref.nativeIdentity, "authorized");
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
|
|
3105
|
+
async #closeManagedAttempts(controlRoot: string, graphId: string, nodeIds: string[]): Promise<boolean> {
|
|
3106
|
+
const affected = [...this.#managedAttempts.values()].filter(
|
|
3107
|
+
ref => ref.controlRoot === controlRoot && ref.graphId === graphId && nodeIds.includes(ref.nodeId),
|
|
3108
|
+
);
|
|
3109
|
+
let allClosed = true;
|
|
3110
|
+
for (const ref of affected) {
|
|
3111
|
+
const inFlight = this.#spawnInFlight.get(ref.nativeIdentity);
|
|
3112
|
+
const waitMs = managedCloseWaitOverridesForTest.get(this) ?? MANAGED_CLOSE_WAIT_MS;
|
|
3113
|
+
if (inFlight && !(await settlesWithin(inFlight.completion, waitMs))) {
|
|
3114
|
+
this.#scheduleManagedCloseAfterFlight(ref, inFlight);
|
|
3115
|
+
await this.#observeManagedNative(ref.nativeIdentity, "unknown").catch(() => undefined);
|
|
3116
|
+
allClosed = false;
|
|
3117
|
+
continue;
|
|
3118
|
+
}
|
|
3119
|
+
const claim = this.#spawnAuthority?.claim(ref.nativeIdentity);
|
|
3120
|
+
const authority = this.#spawnAuthority?.authority(ref.nativeIdentity);
|
|
3121
|
+
if (!claim || (claim.state === "pre_send_rejected" && !authority && !claim.substrateProof)) {
|
|
3122
|
+
try {
|
|
3123
|
+
await this.#observeManagedNative(ref.nativeIdentity, "no-effect");
|
|
3124
|
+
} catch {
|
|
3125
|
+
await this.#observeManagedNative(ref.nativeIdentity, "unknown").catch(() => undefined);
|
|
3126
|
+
allClosed = false;
|
|
3127
|
+
}
|
|
3128
|
+
continue;
|
|
3129
|
+
}
|
|
3130
|
+
const outcome = await this.#closeSpawnAuthority(ref.nativeIdentity);
|
|
3131
|
+
if (outcome === "closed") await this.#observeManagedNative(ref.nativeIdentity, "closed");
|
|
3132
|
+
else {
|
|
3133
|
+
await this.#observeManagedNative(ref.nativeIdentity, "unknown").catch(() => undefined);
|
|
3134
|
+
allClosed = false;
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
return allClosed;
|
|
3138
|
+
}
|
|
3139
|
+
|
|
3140
|
+
#scheduleManagedCloseAfterFlight(ref: ManagedAttemptRef, inFlight: SpawnInFlight): void {
|
|
3141
|
+
void inFlight.completion
|
|
3142
|
+
.then(async () => {
|
|
3143
|
+
const outcome = await this.#closeSpawnAuthority(ref.nativeIdentity);
|
|
3144
|
+
if (outcome === "closed") await this.#observeManagedNative(ref.nativeIdentity, "closed");
|
|
3145
|
+
else await this.#observeManagedNative(ref.nativeIdentity, "unknown");
|
|
3146
|
+
})
|
|
3147
|
+
.catch(async caught => {
|
|
3148
|
+
logger.warn(`sdk broker: deferred managed child close failed for ${ref.nativeIdentity}: ${String(caught)}`);
|
|
3149
|
+
await this.#observeManagedNative(ref.nativeIdentity, "unknown").catch(() => undefined);
|
|
3150
|
+
});
|
|
3151
|
+
}
|
|
3152
|
+
|
|
2306
3153
|
async #awaitSpawnHostRegistration(input: {
|
|
2307
3154
|
childId: string;
|
|
2308
3155
|
cwd: string;
|
|
@@ -2494,7 +3341,10 @@ export class Broker {
|
|
|
2494
3341
|
const claim = store.claims().find(candidate => candidate.childId === sessionId);
|
|
2495
3342
|
if (!claim) return undefined;
|
|
2496
3343
|
const outcome = await this.#closeSpawnAuthority(claim.lifecycleIdentity);
|
|
2497
|
-
if (outcome === "closed")
|
|
3344
|
+
if (outcome === "closed") {
|
|
3345
|
+
await this.#observeManagedNative(claim.lifecycleIdentity, "closed").catch(() => undefined);
|
|
3346
|
+
return { ok: true, result: { code: "spawn_child_closed", sessionId } };
|
|
3347
|
+
}
|
|
2498
3348
|
if (outcome === "uncertain")
|
|
2499
3349
|
return error("terminal_uncertain", "session.close substrate identity could not be re-proven");
|
|
2500
3350
|
return error("close_refused", "session.close could not complete for the spawned child");
|
|
@@ -2511,8 +3361,47 @@ export class Broker {
|
|
|
2511
3361
|
try {
|
|
2512
3362
|
const claim = store.claim(lifecycleIdentity);
|
|
2513
3363
|
let authority = store.authority(lifecycleIdentity);
|
|
2514
|
-
if (!claim
|
|
2515
|
-
if (claim.state === "closed" && authority.closeState === "closed") return "closed";
|
|
3364
|
+
if (!claim) return "retained";
|
|
3365
|
+
if (claim.state === "closed" && (!authority || authority.closeState === "closed")) return "closed";
|
|
3366
|
+
if (!authority) {
|
|
3367
|
+
if (!claim.substrateProof || !claim.childId) return "retained";
|
|
3368
|
+
const provider = this.#spawnSubstrateProvider();
|
|
3369
|
+
const proof = claim.substrateProof;
|
|
3370
|
+
let verdict: "verified" | "mismatch" | "gone";
|
|
3371
|
+
try {
|
|
3372
|
+
verdict = await provider.verify(proof);
|
|
3373
|
+
} catch {
|
|
3374
|
+
verdict = "mismatch";
|
|
3375
|
+
}
|
|
3376
|
+
let gone = verdict === "gone";
|
|
3377
|
+
if (verdict === "verified") {
|
|
3378
|
+
try {
|
|
3379
|
+
gone = (await provider.close(proof)).ok && (await provider.verify(proof)) === "gone";
|
|
3380
|
+
} catch {
|
|
3381
|
+
gone = false;
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
3384
|
+
if (!gone) {
|
|
3385
|
+
if (claim.state !== "uncertain")
|
|
3386
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
3387
|
+
claimId: claim.claimId,
|
|
3388
|
+
from: claim.state,
|
|
3389
|
+
to: "uncertain",
|
|
3390
|
+
failure: {
|
|
3391
|
+
substrateKind: proof.substrateKind,
|
|
3392
|
+
code: "child_close_unresolved",
|
|
3393
|
+
message: "session.spawn child close could not be proven",
|
|
3394
|
+
},
|
|
3395
|
+
});
|
|
3396
|
+
return verdict === "mismatch" ? "uncertain" : "retained";
|
|
3397
|
+
}
|
|
3398
|
+
await store.persistTransition(lifecycleIdentity, {
|
|
3399
|
+
claimId: claim.claimId,
|
|
3400
|
+
from: claim.state,
|
|
3401
|
+
to: "closed",
|
|
3402
|
+
});
|
|
3403
|
+
return "closed";
|
|
3404
|
+
}
|
|
2516
3405
|
if (authority.closeState === "closed") return "closed";
|
|
2517
3406
|
const provider = this.#spawnSubstrateProvider();
|
|
2518
3407
|
const proof = spawnProofFromAuthority(authority);
|
|
@@ -2539,13 +3428,11 @@ export class Broker {
|
|
|
2539
3428
|
})
|
|
2540
3429
|
).authority!;
|
|
2541
3430
|
}
|
|
2542
|
-
if (verdict === "verified") {
|
|
3431
|
+
if (verdict === "verified" || verdict === "gone") {
|
|
2543
3432
|
const closedSubstrate = await provider.close(proof);
|
|
2544
3433
|
if (!closedSubstrate.ok) return "retained";
|
|
2545
|
-
// Signal delivery is not terminal evidence: only a second exact proof
|
|
2546
|
-
// that observes this substrate gone permits the durable close.
|
|
2547
3434
|
if ((await provider.verify(proof)) !== "gone") return "retained";
|
|
2548
|
-
}
|
|
3435
|
+
} else return "retained";
|
|
2549
3436
|
const at = Math.max(Date.now(), authority.updatedAt + 1);
|
|
2550
3437
|
const closedAuthority: SpawnAuthorityV1 = { ...authority, closeState: "closed", closedAt: at, updatedAt: at };
|
|
2551
3438
|
const currentClaim = store.claim(lifecycleIdentity);
|
|
@@ -2876,7 +3763,20 @@ export class Broker {
|
|
|
2876
3763
|
else if (mirror.requestHash !== claim.bindingMac)
|
|
2877
3764
|
throw new Error("Spawn claim lifecycle mirror binding differs from durable authority.");
|
|
2878
3765
|
}
|
|
3766
|
+
await this.#restoreManagedAttempts();
|
|
3767
|
+
if (this.#managedEnrollmentFailed || this.#managedFailedRoots.size > 0)
|
|
3768
|
+
throw new Error("Broker cannot establish complete managed enrollment membership.");
|
|
2879
3769
|
await this.#recoverSpawnClaims();
|
|
3770
|
+
await this.#reconcileManagedNativeLifetimes();
|
|
3771
|
+
const reconciledRoots = new Set<string>();
|
|
3772
|
+
for (const ref of this.#managedAttempts.values()) {
|
|
3773
|
+
if (this.#managedFailedRoots.has(ref.controlRoot) || reconciledRoots.has(ref.controlRoot)) continue;
|
|
3774
|
+
reconciledRoots.add(ref.controlRoot);
|
|
3775
|
+
try {
|
|
3776
|
+
const binding = await loadManagedDomainBinding(ref.controlRoot);
|
|
3777
|
+
if (binding) await reconcileRunningManagedVerification(binding);
|
|
3778
|
+
} catch {}
|
|
3779
|
+
}
|
|
2880
3780
|
const now = Date.now();
|
|
2881
3781
|
const incarnation = brokerProcessIncarnation(process.pid);
|
|
2882
3782
|
if (!incarnation) throw new Error("Broker process incarnation is unavailable.");
|
|
@@ -3697,6 +4597,7 @@ export class Broker {
|
|
|
3697
4597
|
// Spawn bypasses generic raw-input transforms. They must never observe task
|
|
3698
4598
|
// or capability fields, even transiently.
|
|
3699
4599
|
if (operation === "session.spawn") return this.#handleSpawn(input, idempotencyKey);
|
|
4600
|
+
if (operation === "task.dag") return this.#handleManagedTaskDag(input, idempotencyKey);
|
|
3700
4601
|
if (operation === "session.close") {
|
|
3701
4602
|
const spawnClose = await this.#maybeCloseSpawnChild(input);
|
|
3702
4603
|
if (spawnClose) return spawnClose;
|
|
@@ -4088,6 +4989,12 @@ export function setLivenessGraceForTest(broker: Broker, graceMs: number | undefi
|
|
|
4088
4989
|
else livenessGraceOverridesForTest.set(broker, graceMs);
|
|
4089
4990
|
}
|
|
4090
4991
|
|
|
4992
|
+
/** Test-only hook for shortening the bounded wait before a deferred managed close is queued. */
|
|
4993
|
+
export function setManagedCloseWaitForTest(broker: Broker, waitMs: number | undefined): void {
|
|
4994
|
+
if (waitMs === undefined) managedCloseWaitOverridesForTest.delete(broker);
|
|
4995
|
+
else managedCloseWaitOverridesForTest.set(broker, Math.max(0, waitMs));
|
|
4996
|
+
}
|
|
4997
|
+
|
|
4091
4998
|
/**
|
|
4092
4999
|
* Test-only hook for stalling the heartbeat write, reproducing a publication tick
|
|
4093
5000
|
* whose awaited IO does not settle. Clearing it releases the stalled tick the way
|