@tangle-network/agent-runtime 3.0.43 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5263 -0
- package/dist/{activation-CiUbW1QE.d.ts → activation-CtLdCiH3.d.ts} +2 -2
- package/dist/{activation-1YUy4GWu.js → activation-DGLWxFJu.js} +2 -2
- package/dist/{activation-1YUy4GWu.js.map → activation-DGLWxFJu.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +4 -4
- package/dist/candidate-execution/index.d.ts +1 -1
- package/dist/{coordination-manager-DXOwLu9T.js → coordination-manager-D74AZg7m.js} +7 -7
- package/dist/{coordination-manager-DXOwLu9T.js.map → coordination-manager-D74AZg7m.js.map} +1 -1
- package/dist/{coordination-map-routing-2_hUlTnz.js → coordination-map-routing-DVjW5Gx0.js} +2 -2
- package/dist/{coordination-map-routing-2_hUlTnz.js.map → coordination-map-routing-DVjW5Gx0.js.map} +1 -1
- package/dist/{delegate-CAvTMCk0.js → delegate-BVyFJ9AX.js} +2 -2
- package/dist/{delegate-CAvTMCk0.js.map → delegate-BVyFJ9AX.js.map} +1 -1
- package/dist/durable.d.ts +4 -2
- package/dist/durable.js +12 -184
- package/dist/durable.js.map +1 -1
- package/dist/{graph-C7Xthj-4.js → graph-r8zcRxbi.js} +6 -8
- package/dist/{graph-C7Xthj-4.js.map → graph-r8zcRxbi.js.map} +1 -1
- package/dist/{graph-registry-BoaUV5g8.js → graph-registry-B25FxBZN.js} +5 -5
- package/dist/graph-registry-B25FxBZN.js.map +1 -0
- package/dist/{graph-registry-EGeo_moB.js → graph-registry-C-hjRx2t.js} +1 -1
- package/dist/{improve-BZpoezFh.js → improve-D12GulmK.js} +7 -7
- package/dist/{improve-BZpoezFh.js.map → improve-D12GulmK.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +9 -9
- package/dist/intelligence.d.ts +3 -3
- package/dist/intelligence.js +5 -5
- package/dist/{jsonl-file-CbxMVnUj.js → jsonl-file-CeVYTIQJ.js} +45 -33
- package/dist/jsonl-file-CeVYTIQJ.js.map +1 -0
- package/dist/kernel.d.ts +2 -2
- package/dist/kernel.js +14 -13
- package/dist/{loop-runner-Dmjf2IY6.d.ts → loop-runner-BHBabRIw.d.ts} +2 -2
- package/dist/{loop-runner-BchibTTn.js → loop-runner-CYcN11I4.js} +2 -2
- package/dist/{loop-runner-BchibTTn.js.map → loop-runner-CYcN11I4.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +5 -5
- package/dist/profiles.d.ts +1 -1
- package/dist/{provision-supervisor-BtDnk11d.js → provision-supervisor-BiqHJ-d4.js} +5 -5
- package/dist/{provision-supervisor-BtDnk11d.js.map → provision-supervisor-BiqHJ-d4.js.map} +1 -1
- package/dist/{run-cancellation-DEW04dun.js → run-cancellation-D5R-7UQt.js} +3 -3
- package/dist/{run-cancellation-DEW04dun.js.map → run-cancellation-D5R-7UQt.js.map} +1 -1
- package/dist/run-state-DRh8BOFi.js +350 -0
- package/dist/run-state-DRh8BOFi.js.map +1 -0
- package/dist/{runtime-CKoChP6L.js → runtime-BX_9-DJB.js} +18 -17
- package/dist/runtime-BX_9-DJB.js.map +1 -0
- package/dist/{scope-Bg1uCsPR.js → scope-JcCkUy_z.js} +268 -165
- package/dist/scope-JcCkUy_z.js.map +1 -0
- package/dist/{search-executor-CJ5y2-b8.js → search-executor-CdmaS5r4.js} +3 -3
- package/dist/{search-executor-CJ5y2-b8.js.map → search-executor-CdmaS5r4.js.map} +1 -1
- package/dist/{server-zdmmyP7r.js → server-BY6OluUf.js} +3 -3
- package/dist/{server-zdmmyP7r.js.map → server-BY6OluUf.js.map} +1 -1
- package/dist/settle-record-BSDDf9Z4.js +130 -0
- package/dist/settle-record-BSDDf9Z4.js.map +1 -0
- package/dist/{structural-rollout-CH9b7K9B.js → structural-rollout-BjMUUXLn.js} +3 -3
- package/dist/{structural-rollout-CH9b7K9B.js.map → structural-rollout-BjMUUXLn.js.map} +1 -1
- package/dist/{supervise-472pyfgQ.js → supervise-rYEsVVcn.js} +254 -414
- package/dist/supervise-rYEsVVcn.js.map +1 -0
- package/dist/{supervision-discovery-CD350Iar.js → supervision-discovery-D95Ack4J.js} +4 -129
- package/dist/supervision-discovery-D95Ack4J.js.map +1 -0
- package/dist/{supervisor-BDOWzyCC.js → supervisor-B-bX69uL.js} +185 -120
- package/dist/supervisor-B-bX69uL.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +11 -11
- package/dist/{top-web-CNTJBqFy.js → top-web-D-wH9ipn.js} +13 -9
- package/dist/{top-web-CNTJBqFy.js.map → top-web-D-wH9ipn.js.map} +1 -1
- package/dist/tui/bin.js +3 -3
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +2 -2
- package/dist/{types-tG_tps7N.d.ts → types-DblV14Oe.d.ts} +204 -224
- package/package.json +7 -8
- package/dist/graph-registry-BoaUV5g8.js.map +0 -1
- package/dist/jsonl-file-CbxMVnUj.js.map +0 -1
- package/dist/runtime-CKoChP6L.js.map +0 -1
- package/dist/scope-Bg1uCsPR.js.map +0 -1
- package/dist/supervise-472pyfgQ.js.map +0 -1
- package/dist/supervision-discovery-CD350Iar.js.map +0 -1
- package/dist/supervisor-BDOWzyCC.js.map +0 -1
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { An as WORKER_TRACE_PROPAGATION, Ar as recordRuntimeOwnedDriveHarnessProviderEvidence, At as beginScopeRetainedOwnerDrive, B as beginScopeOwnerAttempt, Bn as createOtelExporter, Ci as executorFailure, Dt as prepareScopeRetainedOwnerTask, Et as consumeScopeRetainedOwnerResult, Fi as assertValidBudget, Fr as runtimeOwnedDriveHarnessProviderEvidence, Ft as scopeRetainedOwnerRestorePoint, G as scopeOwnerExecutorNodeContext, H as noteScopeOwnerPlannedExecution, Hn as generateSpanId, Ht as scopeRetainedOwnerContext, Ir as runtimeOwnedExecutorExecutionBinding, It as noteScopeRetainedOwnerCoordination, J as providerVisibleProfile, Jr as runtimeFlowBrain, K as coordinationMcpAlias, Kr as isRuntimeFlowBrain, Lr as runtimeOwnedExecutorMaterialization, Mi as newUsageTotals, Mt as scopeRetainedOwnerTurnEnded, Nt as assertRetainedOwnerCoordinationRekey, O as parseWorkerToolTraceArtifact, Ot as scopeRetainedOwnerPriorSpend, Pi as spendFromUsageTotals, Pt as reconcileScopeRetainedOwnerEnvironment, Q as assertRecursiveReservationPolicy, Rr as runtimeOwnedExecutorProviderEvidence, Rt as bindScopeRetainedOwnerEnvironmentId, Sr as authoredAgentIdentity, St as scopeRetainedOwnerResourceReader, Tt as abandonScopeRetainedOwnerInvocation, U as recordScopeOwnerMaterialization, Ut as scopeRetainedOwnerJournalRoot, Vr as runtimeOwnedScopeOwnerRuntime, W as restoreScopeOwnerAcceptedExecution, Y as runtimeGraphMetadataKey, _i as unavailableResumeAt, a as readRootStream, bt as prepareScopeRetainedOwnerSeatSwitch, ct as recordScopeOwnerBarrenPause, dr as isTerminalNodeStatus, er as persistHarnessTranscript, fa as RunCancellationReason, fi as cappedDoublingMs, fr as SEAT_RECHECK_MS, gi as unavailablePauseMs, gr as seatExecutionProfile, ha as runAbortable, hi as reportedResetAt, hr as authoredSeatStages, ia as recordValue, ji as meterUsageEvent, kr as providerAttemptEvidence, kt as scopeRetainedOwnerResult, li as retainedExecutorSeamKey, lt as recordScopeOwnerDriverAttempt, ma as linkAbort, mr as SeatPauseError, n as ROOT_STREAM_FILE, ot as meterRuntimeOwnedAccounting, pi as jitteredMs, pr as SeatChain, q as coordinationProfileToolPrefix, qr as runtimeFlowAgent, r as createRootStreamSink, ri as RetainedExecutionPendingError, si as registerRetainedExecutorPreparation, sr as createActivityLog, st as meterRuntimeOwnedProviderAttempt, tr as readHarnessTranscript, ur as isLiveNodeStatus, ut as recordScopeOwnerPause, xi as contentAddress, xr as attestRuntimeOwnedScopeOwner, xt as releaseScopeRetainedOwnerForSeatPause, zr as runtimeOwnedPendingExecutorMaterialization, zt as bindScopeRetainedOwnerProvider } from "./scope-JcCkUy_z.js";
|
|
2
2
|
import { f as RuntimeRunStateError, i as ConfigError, m as ValidationError } from "./errors-AzSuGI23.js";
|
|
3
|
-
import { $ as isAsyncIterable, Dt as teardownExecutor, G as chargedTokens,
|
|
4
|
-
import { d as detachedSnapshot, h as errorHttpStatus, m as errMessage, p as causeChain } from "./jsonl-file-
|
|
5
|
-
import { $t as promptModelProfileMaterialization, At as bridgeModelRouteRefusal, B as createRouterTranscript, C as createExecutor, Ct as admitContinuationPolicy, Et as createContinuationKeeper, F as codeModeArguments, I as codeModeModelOutput, Jt as controlProfileMaterialization, Kt as assertProfileMaterialization, L as createCodeModeTools, Qt as promptControlProfileMaterialization, Rt as bridgeRuntimeAttachmentsKey, S as captureReusableExecutorConfig, St as CONTINUATIONS_DIR, Tt as checkVerdictOf, U as resolveAgentEnvironmentProvider, V as routerBrain, X as createInbox, Xt as fullProfileMaterialization, Y as INBOX_RECEIPT_KEY, Yt as defineProfileMaterializationContract, Zt as profileMaterializationAxes$1, _t as CheckUnavailableError, a as assertRunParking, c as sleepScopeWait, ct as HarnessTurnFailedError, d as runFinalizer, dt as isContentFilterRefusal, en as renderUnsupported, et as PEER_MAIL_WIRE_KEY, f as runTree, ft as summarizeDriverAttempts, g as isDriverSpec, gt as preflightPublicCoordination, h as driverExecutorFactory, ht as upstreamUnavailableSignal, i as RunParkedError, in as worktreeCliProfileMaterialization, kt as bridgeAdmissionRefusal, l as withRunParking, lt as abandonablePending, m as driverChild, mt as undispatchedCredentialExit, n as createSupervisor, nn as sandboxProfileMaterialization, o as bindScopeParkingDrain, p as createRootHandle, pt as transientFailureKind, rn as unsupportedProfileDimensions, rt as isPeerMailEnvelope, st as DriverAttemptsExhaustedError, u as bestDelivered, ut as classifyDriverFailure, vt as gateFactoryOnDeliverable, w as snapshotExecutorConfig, wt as bestComposite, x as bindReusableExecutorExecutionId, z as toolResultText, zt as bridgeStopSignalKey } from "./supervisor-
|
|
3
|
+
import { $ as isAsyncIterable, Dt as teardownExecutor, G as chargedTokens, St as armDeadlineTimer, Tt as isDeadlineAbortReason, U as addSpend, Y as errorMessage, at as sleep, bt as DEFAULT_SUCCESSFUL_SHUTDOWN_MS, dt as zeroSpend, gt as withBudgetResources, ht as resourceTelemetry, lt as unmeteredSpend, pt as addResourceSpend, tt as promptCacheTokenClasses, ut as withTimeout } from "./workspace-streams-8ALHaZzx.js";
|
|
4
|
+
import { _ as errorText, d as detachedSnapshot, h as errorHttpStatus, m as errMessage, p as causeChain } from "./jsonl-file-CeVYTIQJ.js";
|
|
5
|
+
import { $t as promptModelProfileMaterialization, At as bridgeModelRouteRefusal, B as createRouterTranscript, C as createExecutor, Ct as admitContinuationPolicy, Et as createContinuationKeeper, F as codeModeArguments, I as codeModeModelOutput, Jt as controlProfileMaterialization, Kt as assertProfileMaterialization, L as createCodeModeTools, Qt as promptControlProfileMaterialization, Rt as bridgeRuntimeAttachmentsKey, S as captureReusableExecutorConfig, St as CONTINUATIONS_DIR, Tt as checkVerdictOf, U as resolveAgentEnvironmentProvider, V as routerBrain, X as createInbox, Xt as fullProfileMaterialization, Y as INBOX_RECEIPT_KEY, Yt as defineProfileMaterializationContract, Zt as profileMaterializationAxes$1, _t as CheckUnavailableError, a as assertRunParking, c as sleepScopeWait, ct as HarnessTurnFailedError, d as runFinalizer, dt as isContentFilterRefusal, en as renderUnsupported, et as PEER_MAIL_WIRE_KEY, f as runTree, ft as summarizeDriverAttempts, g as isDriverSpec, gt as preflightPublicCoordination, h as driverExecutorFactory, ht as upstreamUnavailableSignal, i as RunParkedError, in as worktreeCliProfileMaterialization, kt as bridgeAdmissionRefusal, l as withRunParking, lt as abandonablePending, m as driverChild, mt as undispatchedCredentialExit, n as createSupervisor, nn as sandboxProfileMaterialization, o as bindScopeParkingDrain, p as createRootHandle, pt as transientFailureKind, rn as unsupportedProfileDimensions, rt as isPeerMailEnvelope, st as DriverAttemptsExhaustedError, u as bestDelivered, ut as classifyDriverFailure, vt as gateFactoryOnDeliverable, w as snapshotExecutorConfig, wt as bestComposite, x as bindReusableExecutorExecutionId, z as toolResultText, zt as bridgeStopSignalKey } from "./supervisor-B-bX69uL.js";
|
|
6
6
|
import { _ as agentHarness, a as concreteProfileModel, c as profileModelExecutionSettings, l as profileProviderModel, n as assertModelAllowed, o as enforceTokenLimits, r as assertProfileModelsAllowed, s as profileBridgeWireModel, t as assertExecutableAgentProfile, v as harnessRunsAgent } from "./model-policy-JOcg3NRs.js";
|
|
7
7
|
import { n as createStdioToolServer } from "./tool-server-CZWd_Opp.js";
|
|
8
8
|
import { t as composeRuntimeHooks } from "./runtime-hooks-CsW2pfFk.js";
|
|
9
|
-
import { _ as workerAssignmentNamespace, c as writeRunCancellation, d as coordinationEventId, f as externalExecutionId, g as supervisionRunNamespace, h as rootCoordinationOwner, i as readRunCancellation, l as canonicalExecution, m as freezeDetachedProfile, p as freezeDetached, r as readRunCancelRequest, s as watchRunCancellation, t as applyRunCancellation, u as childCoordinationOwner } from "./run-cancellation-
|
|
10
|
-
import { c as createSteerAcknowledger, d as coordinationVerbNames, f as createFileRunContext, l as budgetOverspendText, m as withRunContext, n as createCoordinationToolsForManager, p as createInMemoryRunContext, s as createCancelAcknowledger, t as createCoordinationTools, u as normalizeAnalyzeOnSettle } from "./coordination-manager-
|
|
11
|
-
import { o as safeJsonText } from "./coordination-map-routing-
|
|
12
|
-
import { a as graphRegistry, c as loadGraph } from "./graph-registry-
|
|
9
|
+
import { _ as workerAssignmentNamespace, c as writeRunCancellation, d as coordinationEventId, f as externalExecutionId, g as supervisionRunNamespace, h as rootCoordinationOwner, i as readRunCancellation, l as canonicalExecution, m as freezeDetachedProfile, p as freezeDetached, r as readRunCancelRequest, s as watchRunCancellation, t as applyRunCancellation, u as childCoordinationOwner } from "./run-cancellation-D5R-7UQt.js";
|
|
10
|
+
import { c as createSteerAcknowledger, d as coordinationVerbNames, f as createFileRunContext, l as budgetOverspendText, m as withRunContext, n as createCoordinationToolsForManager, p as createInMemoryRunContext, s as createCancelAcknowledger, t as createCoordinationTools, u as normalizeAnalyzeOnSettle } from "./coordination-manager-D74AZg7m.js";
|
|
11
|
+
import { o as safeJsonText } from "./coordination-map-routing-DVjW5Gx0.js";
|
|
12
|
+
import { a as graphRegistry, c as loadGraph } from "./graph-registry-B25FxBZN.js";
|
|
13
13
|
import { ATTR, agentIdentityAttributes } from "@tangle-network/agent-trace-contract";
|
|
14
14
|
import { ProxySecretScopeSchema, ProxySecretsSchema, agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalCandidateJson, mergeProxySecretBindings, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
15
15
|
import { toolSpansToTraceAnalysisStore } from "@tangle-network/agent-eval";
|
|
@@ -96,8 +96,6 @@ const superviseOptionKeySet = /* @__PURE__ */ new Set([
|
|
|
96
96
|
"stopRule",
|
|
97
97
|
"watchWorkers",
|
|
98
98
|
"continuation",
|
|
99
|
-
"workerRetry",
|
|
100
|
-
"onWorkerRetry",
|
|
101
99
|
"workerSlots",
|
|
102
100
|
"recoverExecutor"
|
|
103
101
|
]);
|
|
@@ -164,7 +162,6 @@ const superviseExecutableOptionKeys = [
|
|
|
164
162
|
"onCoordinationEvent",
|
|
165
163
|
"onDriverAttempt",
|
|
166
164
|
"onProgressStop",
|
|
167
|
-
"onWorkerRetry",
|
|
168
165
|
"recoverExecutor",
|
|
169
166
|
"resolveDeliverable",
|
|
170
167
|
"resolveDriveHarness",
|
|
@@ -208,9 +205,9 @@ function assertNoUncapturedExecutableOption(decisionData) {
|
|
|
208
205
|
* detached and frozen; executable ports are copied as the exact references selected at intake.
|
|
209
206
|
* Service internals intentionally remain live, while replacing a callback/service on the caller's
|
|
210
207
|
* mutable options object can no longer change an in-flight run. */
|
|
211
|
-
function captureSuperviseOptions(opts) {
|
|
208
|
+
function captureSuperviseOptions(opts, mode = "execution") {
|
|
212
209
|
assertSuperviseOptionKeys(opts, "supervise");
|
|
213
|
-
const { backend, coordination, driverBackend, deliverable, router, compaction, watchWorkers, analysts, continuation,
|
|
210
|
+
const { backend, coordination, driverBackend, deliverable, router, compaction, watchWorkers, analysts, continuation, ...rest } = opts;
|
|
214
211
|
const references = {};
|
|
215
212
|
const decisionData = {};
|
|
216
213
|
for (const [key, value] of Object.entries(rest)) if (!superviseReferenceOptionKeys.has(key)) decisionData[key] = value;
|
|
@@ -227,8 +224,8 @@ function captureSuperviseOptions(opts) {
|
|
|
227
224
|
...onControlReady === void 0 ? {} : { onControlReady }
|
|
228
225
|
});
|
|
229
226
|
})();
|
|
230
|
-
const capturedBackend = backend === void 0 ? void 0 : snapshotExecutorConfig(backend);
|
|
231
|
-
const capturedDriverBackend = driverBackend === void 0 ? void 0 : snapshotExecutorConfig(driverBackend);
|
|
227
|
+
const capturedBackend = backend === void 0 ? void 0 : snapshotExecutorConfig(backend, mode);
|
|
228
|
+
const capturedDriverBackend = driverBackend === void 0 ? void 0 : snapshotExecutorConfig(driverBackend, mode);
|
|
232
229
|
const capturedDeliverable = deliverable === void 0 || typeof deliverable === "string" ? deliverable : captureDeliverable(deliverable, "supervise deliverable");
|
|
233
230
|
const capturedRouter = router === void 0 ? void 0 : (() => {
|
|
234
231
|
const { complete, ...routerData } = router;
|
|
@@ -250,15 +247,6 @@ function captureSuperviseOptions(opts) {
|
|
|
250
247
|
...detachedSnapshot({ maxFindingsPerWorker: watchWorkers.maxFindingsPerWorker }, "supervise worker-watch configuration"),
|
|
251
248
|
...watchWorkers.detectors === void 0 ? {} : { detectors: Object.freeze([...watchWorkers.detectors]) }
|
|
252
249
|
});
|
|
253
|
-
const capturedWorkerRetry = workerRetry === void 0 ? void 0 : Object.freeze({
|
|
254
|
-
...detachedSnapshot({
|
|
255
|
-
...workerRetry.enabled === void 0 ? {} : { enabled: workerRetry.enabled },
|
|
256
|
-
...workerRetry.maxTotalMs === void 0 ? {} : { maxTotalMs: workerRetry.maxTotalMs },
|
|
257
|
-
...workerRetry.initialBackoffMs === void 0 ? {} : { initialBackoffMs: workerRetry.initialBackoffMs },
|
|
258
|
-
...workerRetry.maxBackoffMs === void 0 ? {} : { maxBackoffMs: workerRetry.maxBackoffMs }
|
|
259
|
-
}, "supervise worker-retry configuration"),
|
|
260
|
-
...workerRetry.additionalPreSpawnSignatures === void 0 ? {} : { additionalPreSpawnSignatures: Object.freeze([...workerRetry.additionalPreSpawnSignatures]) }
|
|
261
|
-
});
|
|
262
250
|
const capturedAnalysts = analysts === void 0 || typeof analysts === "string" ? analysts : Object.freeze({
|
|
263
251
|
kinds: detachedSnapshot(analysts.kinds, "supervise analyst kinds"),
|
|
264
252
|
run: analysts.run,
|
|
@@ -283,8 +271,7 @@ function captureSuperviseOptions(opts) {
|
|
|
283
271
|
...capturedCompaction === void 0 ? {} : { compaction: capturedCompaction },
|
|
284
272
|
...capturedWatchWorkers === void 0 ? {} : { watchWorkers: capturedWatchWorkers },
|
|
285
273
|
...capturedAnalysts === void 0 ? {} : { analysts: capturedAnalysts },
|
|
286
|
-
...capturedContinuation === void 0 ? {} : { continuation: capturedContinuation }
|
|
287
|
-
...capturedWorkerRetry === void 0 ? {} : { workerRetry: capturedWorkerRetry }
|
|
274
|
+
...capturedContinuation === void 0 ? {} : { continuation: capturedContinuation }
|
|
288
275
|
});
|
|
289
276
|
}
|
|
290
277
|
// src/runtime/supervise/otel-spans.ts
|
|
@@ -2406,7 +2393,7 @@ function ledgerAnalyzes(ledger, edge, outcome, bytes, reason, workerId) {
|
|
|
2406
2393
|
*/
|
|
2407
2394
|
function exhaustedBudgetChannels(budget, nowMs) {
|
|
2408
2395
|
const spent = [];
|
|
2409
|
-
if (budget.tokensLeft <= 0) spent.push(`tokens: ${-budget.tokensLeft} over the cap`);
|
|
2396
|
+
if (budget.tokensCapped !== false && budget.tokensLeft <= 0) spent.push(`tokens: ${-budget.tokensLeft} over the cap`);
|
|
2410
2397
|
if (budget.usdCapped && budget.usdLeft <= 0) spent.push(`usd: ${(-budget.usdLeft).toFixed(4)} over the cap`);
|
|
2411
2398
|
for (const [name, resource] of Object.entries(budget.resources ?? {})) if (!resource.known) spent.push(`resource ${name}: unmetered`);
|
|
2412
2399
|
else if (resource.remaining <= 0) spent.push(`resource ${name}: ${-resource.remaining} ${resource.unit} over the cap`);
|
|
@@ -2415,7 +2402,7 @@ function exhaustedBudgetChannels(budget, nowMs) {
|
|
|
2415
2402
|
}
|
|
2416
2403
|
/** A per-child budget may not exceed the conserved pool it is reserved from. */
|
|
2417
2404
|
function assertPerWorkerWithinPool(perWorker, pool) {
|
|
2418
|
-
const over = (child, total, field) => child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null;
|
|
2405
|
+
const over = (child, total, field) => child !== void 0 && total !== void 0 && child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null;
|
|
2419
2406
|
for (const [name, resource] of Object.entries(pool.resources ?? {})) {
|
|
2420
2407
|
const child = perWorker.resources?.[name];
|
|
2421
2408
|
if (!child) throw new ValidationError(`supervise perWorker resource ${name}: child must declare its limit`);
|
|
@@ -2438,6 +2425,7 @@ function assertPerWorkerWithinPool(perWorker, pool) {
|
|
|
2438
2425
|
* next turn is refused as budget-exhausted while the child still holds the rest. */
|
|
2439
2426
|
function assertPerWorkerAdmittable(perWorker, pool, ownerShare) {
|
|
2440
2427
|
const unreservable = (child, total, kept, field) => {
|
|
2428
|
+
if (child === void 0 || total === void 0) return null;
|
|
2441
2429
|
if (ownerShare > 0) {
|
|
2442
2430
|
const reservable = total - kept;
|
|
2443
2431
|
return child > reservable ? `supervise perWorker.${field} (${child}) exceeds the reservable budget.${field} (${reservable}: ${total} less the owner's kept ${kept} at reservationPolicy.ownerShare ${ownerShare})` : null;
|
|
@@ -2445,24 +2433,26 @@ function assertPerWorkerAdmittable(perWorker, pool, ownerShare) {
|
|
|
2445
2433
|
return child >= total ? `supervise perWorker.${field} (${child}) equals the whole budget.${field} (${total}), so the first child takes every unit and the owner has none left for its next turn` : null;
|
|
2446
2434
|
};
|
|
2447
2435
|
const unadmittable = [
|
|
2448
|
-
unreservable(perWorker.maxTokens, pool.maxTokens, Math.ceil(pool.maxTokens * ownerShare), "maxTokens"),
|
|
2449
|
-
unreservable(perWorker.maxIterations, pool.maxIterations, Math.ceil(pool.maxIterations * ownerShare), "maxIterations"),
|
|
2436
|
+
unreservable(perWorker.maxTokens, pool.maxTokens, Math.ceil((pool.maxTokens ?? 0) * ownerShare), "maxTokens"),
|
|
2437
|
+
unreservable(perWorker.maxIterations, pool.maxIterations, Math.ceil((pool.maxIterations ?? 0) * ownerShare), "maxIterations"),
|
|
2450
2438
|
perWorker.maxUsd !== void 0 && pool.maxUsd !== void 0 && ownerShare > 0 ? unreservable(perWorker.maxUsd, pool.maxUsd, pool.maxUsd * ownerShare, "maxUsd") : null
|
|
2451
2439
|
].filter((x) => x !== null);
|
|
2452
2440
|
if (unadmittable.length > 0) throw new ValidationError(`${unadmittable.join("; ")} — lower perWorker on that channel so a child fits beside the owner's own turns; otherwise children are refused or starve at spawn hours into the run.`);
|
|
2453
2441
|
}
|
|
2454
|
-
/** The slice a child gets when its manager names none:
|
|
2455
|
-
*
|
|
2456
|
-
*
|
|
2457
|
-
|
|
2458
|
-
|
|
2442
|
+
/** The slice a child gets when its manager names none: the part of the manager's budget its
|
|
2443
|
+
* children may reserve, divided by the run's worker slots (four with none), so the slots, not the
|
|
2444
|
+
* budget, set how many run at once. A quarter share capped managers with 8-15 slots at 2-4
|
|
2445
|
+
* children (zro-w4a16-hopper-20261104b held 4 of 8; canary-soak-20261104a queued s5-s7 163-176 s). */
|
|
2446
|
+
function defaultPerWorker(budget, ownerShare, workerSlots) {
|
|
2447
|
+
const slots = typeof workerSlots === "number" ? workerSlots : workerSlots?.max;
|
|
2448
|
+
const share = (1 - ownerShare) / (slots !== void 0 && slots > 0 ? slots : 4);
|
|
2459
2449
|
return {
|
|
2460
2450
|
...budget.resources === void 0 ? {} : { resources: Object.fromEntries(Object.entries(budget.resources).map(([name, value]) => [name, {
|
|
2461
2451
|
unit: value.unit,
|
|
2462
2452
|
limit: Math.floor(value.limit * share)
|
|
2463
2453
|
}])) },
|
|
2464
|
-
maxIterations: Math.max(
|
|
2465
|
-
maxTokens: Math.max(
|
|
2454
|
+
...budget.maxIterations === void 0 ? {} : { maxIterations: Math.max(0, Math.floor(budget.maxIterations * share)) },
|
|
2455
|
+
...budget.maxTokens === void 0 ? {} : { maxTokens: Math.max(0, Math.floor(budget.maxTokens * share)) },
|
|
2466
2456
|
...budget.maxUsd !== void 0 ? { maxUsd: budget.maxUsd * share } : {}
|
|
2467
2457
|
};
|
|
2468
2458
|
}
|
|
@@ -2718,17 +2708,13 @@ async function runBrainLoop(opts) {
|
|
|
2718
2708
|
};
|
|
2719
2709
|
}
|
|
2720
2710
|
// src/runtime/supervise/driver-budget-guards.ts
|
|
2721
|
-
/**
|
|
2722
|
-
*
|
|
2723
|
-
* in-loop budget guard the turn cap alone never provided). Checks BOTH conserved channels: tokens
|
|
2724
|
-
* (can't afford a worker) and usd (a usd-capped pool whose ceiling the driver's own metered
|
|
2725
|
-
* inference has drained — `meter` debits usd, so without this a huge-token/small-usd pool would
|
|
2726
|
-
* overspend usd up to the turn tripwire). */
|
|
2711
|
+
/** Stop when no work is in flight and a capped channel cannot fund another worker.
|
|
2712
|
+
* Owner inference also debits the pool, so a turn limit alone cannot enforce its ceilings. */
|
|
2727
2713
|
function poolStarved(scope, perWorker, preserveOwnerTurns = false) {
|
|
2728
2714
|
const b = scope.budget;
|
|
2729
2715
|
if (scope.view.inFlight > 0) return false;
|
|
2730
|
-
const tokenStarved =
|
|
2731
|
-
const iterationStarved = b.iterationsLeft <= 0;
|
|
2716
|
+
const tokenStarved = b.tokensCapped !== false && (b.tokensLeft <= 0 || !preserveOwnerTurns && perWorker.maxTokens !== void 0 && b.tokensLeft < perWorker.maxTokens);
|
|
2717
|
+
const iterationStarved = b.iterationsCapped !== false && b.iterationsLeft <= 0;
|
|
2732
2718
|
const usdStarved = b.usdCapped && (b.usdLeft <= 0 || !preserveOwnerTurns && perWorker.maxUsd !== void 0 && b.usdLeft < perWorker.maxUsd);
|
|
2733
2719
|
const resourceStarved = Object.entries(b.resources ?? {}).some(([name, value]) => !value.known || value.remaining <= 0 || !preserveOwnerTurns && value.remaining < (perWorker.resources?.[name]?.limit ?? 0));
|
|
2734
2720
|
return tokenStarved || iterationStarved || usdStarved || resourceStarved;
|
|
@@ -2749,7 +2735,7 @@ const UNMETERED_DRIVER_TURNS = 16;
|
|
|
2749
2735
|
function ownTurnsUnbounded(scope, poolTokens, turns, maxTurns) {
|
|
2750
2736
|
const b = scope.budget;
|
|
2751
2737
|
if (maxTurns !== void 0 || b.deadlineMs > 0) return false;
|
|
2752
|
-
if (poolTokens > 0 && b.tokensLeft < -poolTokens) return true;
|
|
2738
|
+
if (b.tokensCapped !== false && poolTokens > 0 && b.tokensLeft < -poolTokens) return true;
|
|
2753
2739
|
return !b.tokensKnown && !b.usdCapped && turns > UNMETERED_DRIVER_TURNS;
|
|
2754
2740
|
}
|
|
2755
2741
|
/** The absolute wall-clock deadline (when the root set one) has passed. */
|
|
@@ -3590,23 +3576,24 @@ function createCoordinationCredentials(input) {
|
|
|
3590
3576
|
current = next;
|
|
3591
3577
|
staged = void 0;
|
|
3592
3578
|
};
|
|
3579
|
+
let installing = false;
|
|
3593
3580
|
const rotate = () => {
|
|
3594
|
-
if (
|
|
3581
|
+
if (installing) throw new ConfigError("coordination credential replacement is pending");
|
|
3582
|
+
if (staged) revoked.set(staged.token.toString(), staged.expiresAt);
|
|
3595
3583
|
commit(issue());
|
|
3596
3584
|
readerToken = randomToken();
|
|
3597
3585
|
controllerToken = randomToken();
|
|
3598
3586
|
};
|
|
3599
3587
|
const replace = async (install) => {
|
|
3600
|
-
if (
|
|
3601
|
-
|
|
3602
|
-
|
|
3588
|
+
if (installing) throw new ConfigError("coordination credential replacement is pending");
|
|
3589
|
+
if (!staged || staged.expiresAt <= Date.now()) staged = issue();
|
|
3590
|
+
const next = staged;
|
|
3591
|
+
installing = true;
|
|
3603
3592
|
try {
|
|
3604
3593
|
await install(next.headers);
|
|
3605
3594
|
commit(next, true);
|
|
3606
|
-
}
|
|
3607
|
-
|
|
3608
|
-
revoked.set(next.token.toString(), next.expiresAt);
|
|
3609
|
-
throw error;
|
|
3595
|
+
} finally {
|
|
3596
|
+
installing = false;
|
|
3610
3597
|
}
|
|
3611
3598
|
};
|
|
3612
3599
|
const matches = (supplied, expected) => supplied.length === expected.length && timingSafeEqual(supplied, expected);
|
|
@@ -4291,6 +4278,7 @@ function assertCoordinationTransport(options) {
|
|
|
4291
4278
|
}
|
|
4292
4279
|
/** Response hold cap, under the 60 s MCP tool-call timeout Claude Code and Codex apply. */
|
|
4293
4280
|
const RESPONSE_FENCE_CAP_MS = 45e3;
|
|
4281
|
+
const RENEWAL_RETRY_MAX_MS = 6e4;
|
|
4294
4282
|
/** The longest a fenced call (`submit_result`, node tools) holds its response before answering with
|
|
4295
4283
|
* a receipt: half of `requestTimeoutMs` (beating the transport's 504), at most {@link RESPONSE_FENCE_CAP_MS}. */
|
|
4296
4284
|
function coordinationResponseFenceMs(requestTimeoutMs) {
|
|
@@ -4350,6 +4338,7 @@ async function serveCoordinationMcpForManager(opts, lifetime) {
|
|
|
4350
4338
|
} });
|
|
4351
4339
|
if ((opts.authentication || opts.onAudit) && (!identity.runId.trim() || !identity.actorId.trim())) throw new ConfigError("authenticated coordination requires a trusted run and actor identity");
|
|
4352
4340
|
const state = openCoordinationListenerState();
|
|
4341
|
+
let renewing = false;
|
|
4353
4342
|
const stopUnreachable = (reason) => {
|
|
4354
4343
|
state.closed = true;
|
|
4355
4344
|
try {
|
|
@@ -4364,7 +4353,9 @@ async function serveCoordinationMcpForManager(opts, lifetime) {
|
|
|
4364
4353
|
lifetimeSignal: lifetime,
|
|
4365
4354
|
lifetimeExpiresAt: opts.scope.rootDeadlineAtMs,
|
|
4366
4355
|
isClosed: () => state.closed,
|
|
4367
|
-
onExpiredBearer: () =>
|
|
4356
|
+
onExpiredBearer: () => {
|
|
4357
|
+
if (!renewing) stopUnreachable("coordination credential expired in its holder: no renewal reached it");
|
|
4358
|
+
}
|
|
4368
4359
|
});
|
|
4369
4360
|
const ctx = {
|
|
4370
4361
|
opts,
|
|
@@ -4440,15 +4431,25 @@ async function serveCoordinationMcpForManager(opts, lifetime) {
|
|
|
4440
4431
|
await pending;
|
|
4441
4432
|
};
|
|
4442
4433
|
stopCredentialBinding = stop;
|
|
4434
|
+
let failures = 0;
|
|
4443
4435
|
const schedule = () => {
|
|
4436
|
+
failures = 0;
|
|
4437
|
+
renewing = false;
|
|
4444
4438
|
const expiry = credentials.expiresAt;
|
|
4445
4439
|
if (!active || state.closed || opts.scope.signal.aborted || lifetime?.aborted || opts.scope.rootDeadlineAtMs !== void 0 && expiry >= opts.scope.rootDeadlineAtMs) return;
|
|
4446
4440
|
cancelTimer = armDeadlineTimer(Math.max(1, Math.floor((expiry - Date.now()) / 2)), renew);
|
|
4447
4441
|
};
|
|
4442
|
+
const stopped = () => !active || state.closed || opts.scope.signal.aborted || lifetime?.aborted;
|
|
4443
|
+
const retry = (error) => {
|
|
4444
|
+
if (stopped()) return;
|
|
4445
|
+
const paced = error.retryAfterMs ?? 1e3 * 2 ** failures++;
|
|
4446
|
+
console.warn(`[coordination] credential renewal retries: ${errMessage(error)}`);
|
|
4447
|
+
cancelTimer = armDeadlineTimer(Math.min(paced, RENEWAL_RETRY_MAX_MS), renew);
|
|
4448
|
+
};
|
|
4448
4449
|
const renew = () => {
|
|
4449
|
-
if (
|
|
4450
|
-
|
|
4451
|
-
pending.then(schedule,
|
|
4450
|
+
if (stopped()) return;
|
|
4451
|
+
renewing = true;
|
|
4452
|
+
pending = credentials.replace(install).then(schedule, retry);
|
|
4452
4453
|
};
|
|
4453
4454
|
refreshBoundCredential = () => {
|
|
4454
4455
|
cancelTimer?.();
|
|
@@ -4864,8 +4865,8 @@ function observeManagerControls(config, session) {
|
|
|
4864
4865
|
// src/runtime/supervise/driver-retry-loop.ts
|
|
4865
4866
|
const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3;
|
|
4866
4867
|
const DEFAULT_MAX_ATTEMPTS = Number.POSITIVE_INFINITY;
|
|
4867
|
-
const DEFAULT_INITIAL_BACKOFF_MS
|
|
4868
|
-
const DEFAULT_MAX_BACKOFF_MS
|
|
4868
|
+
const DEFAULT_INITIAL_BACKOFF_MS = 2e3;
|
|
4869
|
+
const DEFAULT_MAX_BACKOFF_MS = 3e4;
|
|
4869
4870
|
const DEFAULT_TRANSIENT_OUTAGE_MS = 9e5;
|
|
4870
4871
|
const BARREN_PAUSE_FIRST_MS = 3e5;
|
|
4871
4872
|
const BARREN_PAUSE_MAX_MS = 36e5;
|
|
@@ -4915,7 +4916,7 @@ function abortStop(signal) {
|
|
|
4915
4916
|
/** The budget's own verdict on whether another attempt may run at all. */
|
|
4916
4917
|
function budgetStop(budget, atMs) {
|
|
4917
4918
|
if (budget.deadlineMs > 0 && atMs >= budget.deadlineMs) return "deadline";
|
|
4918
|
-
return budget.tokensLeft <= 0 || budget.iterationsLeft <= 0 || budget.usdCapped && (budget.usdLeft <= 0 || budget.usdKnown === false) || Object.values(budget.resources ?? {}).some((r) => !r.known || r.remaining <= 0) ? "budget-exhausted" : void 0;
|
|
4919
|
+
return budget.tokensCapped !== false && budget.tokensLeft <= 0 || budget.iterationsCapped !== false && budget.iterationsLeft <= 0 || budget.usdCapped && (budget.usdLeft <= 0 || budget.usdKnown === false) || Object.values(budget.resources ?? {}).some((r) => !r.known || r.remaining <= 0) ? "budget-exhausted" : void 0;
|
|
4919
4920
|
}
|
|
4920
4921
|
/** The cause an admission or continuation refusal carries: it has no failure of its own. */
|
|
4921
4922
|
function admissionError(stop) {
|
|
@@ -4934,23 +4935,23 @@ function madeProgress(before, after) {
|
|
|
4934
4935
|
if (contractOf(after) === "unmet") return false;
|
|
4935
4936
|
return after.poolTokensSpent > before.poolTokensSpent || after.settledCount > before.settledCount;
|
|
4936
4937
|
}
|
|
4937
|
-
async function defaultSleep
|
|
4938
|
+
async function defaultSleep(ms, signal) {
|
|
4938
4939
|
if (ms <= 0 || signal.aborted) return;
|
|
4939
4940
|
await sleep(ms, signal, false);
|
|
4940
4941
|
}
|
|
4941
4942
|
function driverRetryLimits(run) {
|
|
4942
4943
|
const policy = run.policy ?? {};
|
|
4943
|
-
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS
|
|
4944
|
+
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
|
|
4944
4945
|
return {
|
|
4945
4946
|
now: run.now ?? Date.now,
|
|
4946
|
-
sleep: run.sleep ?? defaultSleep
|
|
4947
|
+
sleep: run.sleep ?? defaultSleep,
|
|
4947
4948
|
random: run.random ?? Math.random,
|
|
4948
4949
|
policy,
|
|
4949
4950
|
retryEnabled: policy.enabled !== false,
|
|
4950
4951
|
maxConsecutive: Math.max(0, policy.maxConsecutiveFailures ?? DEFAULT_MAX_CONSECUTIVE_FAILURES),
|
|
4951
4952
|
maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS),
|
|
4952
4953
|
initialBackoff,
|
|
4953
|
-
maxBackoff: Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS
|
|
4954
|
+
maxBackoff: Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS),
|
|
4954
4955
|
transientOutage: Math.max(0, policy.transientOutageMs ?? DEFAULT_TRANSIENT_OUTAGE_MS)
|
|
4955
4956
|
};
|
|
4956
4957
|
}
|
|
@@ -5171,13 +5172,17 @@ var DriverRetryLoop = class {
|
|
|
5171
5172
|
async pause(failure) {
|
|
5172
5173
|
const { run, limits } = this;
|
|
5173
5174
|
const { error, progressed } = failure;
|
|
5175
|
+
const unavailableSignal = upstreamUnavailableSignal(error) ?? "unavailable";
|
|
5176
|
+
const rotatingSeat = run.seatChain === true && unavailableSignal === "subscription-usage-limit";
|
|
5174
5177
|
if (progressed) this.barrenReentries = 0;
|
|
5175
5178
|
const stop = (() => {
|
|
5176
5179
|
if (run.signal.aborted) return abortStop(run.signal);
|
|
5177
5180
|
if (!limits.retryEnabled) return "retry-disabled";
|
|
5178
|
-
|
|
5181
|
+
const byBudget = budgetStop(run.budget(), limits.now());
|
|
5182
|
+
if (byBudget) return byBudget;
|
|
5183
|
+
const deadline = run.budget().deadlineMs;
|
|
5184
|
+
return !rotatingSeat && deadline > 0 && (reportedResetAt(error, limits.now()) ?? 0) >= deadline ? "deadline" : void 0;
|
|
5179
5185
|
})();
|
|
5180
|
-
const unavailableSignal = upstreamUnavailableSignal(error) ?? "unavailable";
|
|
5181
5186
|
if (stop !== void 0) {
|
|
5182
5187
|
await this.stopOrPause(error, stop, {
|
|
5183
5188
|
...this.failureRecord(failure),
|
|
@@ -5188,9 +5193,9 @@ var DriverRetryLoop = class {
|
|
|
5188
5193
|
this.pauses += 1;
|
|
5189
5194
|
this.consecutivePauses = progressed ? 1 : this.consecutivePauses + 1;
|
|
5190
5195
|
const pausedAt = limits.now();
|
|
5191
|
-
const resetAt = unavailableResumeAt(
|
|
5196
|
+
const resetAt = rotatingSeat ? void 0 : unavailableResumeAt(reportedResetAt(error, limits.now()), pausedAt);
|
|
5192
5197
|
const deadlineMs = run.budget().deadlineMs;
|
|
5193
|
-
const pause = resetAt === void 0 ? jitteredMs(unavailablePauseMs(this.consecutivePauses, limits.policy), limits.random) : Math.max(1e3, Math.min(resetAt, deadlineMs > 0 ? deadlineMs : resetAt) - pausedAt);
|
|
5198
|
+
const pause = rotatingSeat ? 0 : resetAt === void 0 ? jitteredMs(unavailablePauseMs(this.consecutivePauses, limits.policy), limits.random) : Math.max(1e3, Math.min(resetAt, deadlineMs > 0 ? deadlineMs : resetAt) - pausedAt);
|
|
5194
5199
|
const exitCode = undispatchedCredentialExit(error);
|
|
5195
5200
|
const resumeAt = resetAt ?? (exitCode === void 0 ? void 0 : pausedAt + pause);
|
|
5196
5201
|
await this.emit({
|
|
@@ -5249,14 +5254,14 @@ var DriverRetryLoop = class {
|
|
|
5249
5254
|
failureStop(failure, failedAt) {
|
|
5250
5255
|
const { run, limits } = this;
|
|
5251
5256
|
const { error, classification, progressed, before, after } = failure;
|
|
5252
|
-
const
|
|
5257
|
+
const kind = classification === "transient" ? transientFailureKind(error) : void 0;
|
|
5258
|
+
const idle = kind === "infra" && !progressed && after.poolTokensSpent === before.poolTokensSpent && after.settledCount === before.settledCount;
|
|
5253
5259
|
if (!idle) this.outageSince = void 0;
|
|
5254
5260
|
else this.outageSince ??= failedAt;
|
|
5255
5261
|
const inOutage = idle && failedAt - (this.outageSince ?? failedAt) < limits.transientOutage;
|
|
5256
5262
|
if (!inOutage) this.chargedFailures += 1;
|
|
5257
5263
|
const sameFailure = failure.durationMs >= INSTANT_FAILURE_MS ? void 0 : extend(this.sameFailure, error, failedAt);
|
|
5258
5264
|
this.sameFailure = sameFailure;
|
|
5259
|
-
const kind = classification === "transient" ? transientFailureKind(error) : void 0;
|
|
5260
5265
|
if (progressed || kind === void 0 || isContentFilterRefusal(error)) this.sameIdle = void 0;
|
|
5261
5266
|
else if (failure.pending === void 0) this.sameIdle = extend(this.sameIdle, error, failedAt);
|
|
5262
5267
|
const sameIdle = this.sameIdle;
|
|
@@ -5439,96 +5444,24 @@ var DriverRetryLoop = class {
|
|
|
5439
5444
|
};
|
|
5440
5445
|
// src/runtime/supervise/driver-retry.ts
|
|
5441
5446
|
/**
|
|
5442
|
-
*
|
|
5443
|
-
*
|
|
5444
|
-
* One dropped connection, one SIGKILLed harness process or one upstream 5xx used to end a run of
|
|
5445
|
-
* arbitrary length with `reason: 'driver-failed'` and tear down every live child (#741). This
|
|
5446
|
-
* module runs the driver again on the SAME scope, coordination server and live children until the
|
|
5447
|
-
* budget or the deadline says stop. It restarts no children and replays no work: the bridge
|
|
5448
|
-
* backend reattaches the harness session because the execution id is bound durably per node.
|
|
5449
|
-
*
|
|
5450
|
-
* Failure classes (all conservative):
|
|
5451
|
-
* - TERMINAL: Runtime's own refusals (`ValidationError`/`ConfigError`, exhausted budget, the run's
|
|
5452
|
-
* own abort, a client-side transport status such as 401/404/422). Retrying re-runs a decision.
|
|
5453
|
-
* Fails at once.
|
|
5454
|
-
* - TRANSIENT: everything foreign (harness exit without a reason, a stream cut mid-turn, a 5xx, a
|
|
5455
|
-
* socket reset, an `AbortError` that neither the run's signal nor the manager's own stop
|
|
5456
|
-
* issued). A harness turn that RETURNED a failed outcome is the same accident reported as
|
|
5457
|
-
* a value, so the drive raises {@link HarnessTurnFailedError} and it is classified like any other.
|
|
5458
|
-
* - UNAVAILABLE: the upstream said "not now" (429, 503, 529, or the router's own codes: quota,
|
|
5459
|
-
* rate limit, overload). The driver PAUSES and re-enters; a pause consumes neither `maxAttempts`
|
|
5460
|
-
* nor the barren streak, so only the deadline, the budget and cancellation bound it. Measured
|
|
5461
|
-
* 2026-09-24 on play anomaly-referee-v3d: during the router's flash quota outage four of five
|
|
5462
|
-
* lead lanes ended `driver-failed` after 12 to 13 attempts with budget and an 8-hour deadline
|
|
5463
|
-
* almost untouched.
|
|
5464
|
-
*
|
|
5465
|
-
* A driver that dies INSTANTLY and repeatedly spends nothing, so the budget alone cannot bound it.
|
|
5466
|
-
* Progress is measured between attempts; a run of attempts that changes nothing stops at
|
|
5467
|
-
* `maxConsecutiveFailures` once it has outlasted the outage window below. A failure that made
|
|
5468
|
-
* progress resets that counter.
|
|
5469
|
-
*
|
|
5470
|
-
* WHAT COUNTS AS PROGRESS was got wrong first. Reading metered spend, settled children and an
|
|
5471
|
-
* accepted submission measured the filesystem and the meter, never the goal: across 1,422 settled
|
|
5472
|
-
* discovery-lab runs it retried the runs that produced NOTHING 629 of 827 times
|
|
5473
|
-
* (76.1%) and the runs that HAD left an artifact 21 of 399 times (5.3%). So when the caller
|
|
5474
|
-
* declares a completion check, spend and settlements alone are NOT progress while it is unmet; only
|
|
5475
|
-
* a delivery (an accepted submission, a child that passed the check, the contract turning met, or a
|
|
5476
|
-
* rise in the check's best composite) resets the barren counter. A caller that declares no check
|
|
5477
|
-
* reports `contract: 'none'` and keeps the historical reading.
|
|
5478
|
-
*
|
|
5479
|
-
* CONTINUATION. `budgetStop` used to be consulted only after a failure, so a driver that RETURNED
|
|
5480
|
-
* with the contract unmet ended the run silently (376 of 376 winning lab runs ended on this loop's
|
|
5481
|
-
* own `stop: 'completed'`). A completed drive whose contract is unmet is now a first-class moment:
|
|
5482
|
-
* the manager's `continuation` policy (`./continuation.ts`) re-enters the driver with Runtime's
|
|
5483
|
-
* continuation note, crossing the same budget, deadline and abort bounds. Successful
|
|
5484
|
-
* continuations do not consume the failure allowance. Progress paces the continuations: once
|
|
5485
|
-
* `continuation.maxBarren` re-entered drives in a row complete without progress, the loop PAUSES
|
|
5486
|
-
* before each further continuation, 5 minutes and doubling per barren drive up to 60, journals one
|
|
5487
|
-
* `barren-pause` row per pause for an operator alarm, and re-enters. A barren streak never ends the
|
|
5488
|
-
* run: the check, the deadline, the budget and cancellation do. Progress resets the streak and the
|
|
5489
|
-
* backoff. There is no count cap and no caller veto (the old prompt-owned loop logged 896 of 1,011
|
|
5490
|
-
* rounds with no progress). The barren bound
|
|
5491
|
-
* settled a 48-hour run `no-winner` after 14.3 hours, while its last two turns updated its pages
|
|
5492
|
-
* and waited on its own 2^20 prove.
|
|
5447
|
+
* Re-enter a driver on the same scope, coordination server and live children. The harness owns
|
|
5448
|
+
* session/environment reuse; this loop owns retry, continuation, budget and deadline decisions.
|
|
5493
5449
|
*
|
|
5494
|
-
*
|
|
5495
|
-
*
|
|
5496
|
-
*
|
|
5450
|
+
* Runtime refusals and cancellation stay terminal. Canonical producer failures distinguish caller,
|
|
5451
|
+
* capacity, infrastructure and work; legacy transport/harness evidence remains a fallback.
|
|
5452
|
+
* Capacity pauses do not consume failure allowances. A seat-bearing drive first lets its existing
|
|
5453
|
+
* chain rotate; the chain owns its exhausted-seat wait, while the run's bounds still apply.
|
|
5497
5454
|
*
|
|
5498
|
-
*
|
|
5499
|
-
*
|
|
5500
|
-
*
|
|
5501
|
-
* speaks and then stays barren is paced as any other. Measured 2026-10-19 on
|
|
5502
|
-
* magnetar-rust-20261019f: after a host reboot the root was driven with the bare original task and
|
|
5503
|
-
* no word of its three open workers, two barren continuations followed, and the barren bound of the
|
|
5504
|
-
* time settled the run `no-result-selected`.
|
|
5455
|
+
* Only idle infrastructure failures earn an outage window. Repeated deterministic failures with
|
|
5456
|
+
* no progress stop inside that window; work failures consume the normal attempt/barren bounds.
|
|
5457
|
+
* With a declared check, progress means delivery or an improved verdict, not spend alone.
|
|
5505
5458
|
*
|
|
5506
|
-
*
|
|
5507
|
-
*
|
|
5508
|
-
*
|
|
5509
|
-
* blanked) on every re-entry with no progress stop with `repeated-failure` inside the window: a
|
|
5510
|
-
* `PartialFailureError` box at the second, a `nested` manager's failed harness turn at the third,
|
|
5511
|
-
* and its other failures at the repeated-failure count and window.
|
|
5512
|
-
* A Platform restart answered
|
|
5513
|
-
* `HTTP 502 ... Platform key verification unavailable` from 03:04:34 to 03:09:41Z, and the
|
|
5514
|
-
* coordinator's twelve attempts ran out at 03:07:12Z with no token spent.
|
|
5459
|
+
* Completed drives with an unmet contract continue with Runtime's note. Barren continuations
|
|
5460
|
+
* wait with increasing backoff; progress resets their streak. A resumed run first receives its
|
|
5461
|
+
* open workers, and counts barren drives only after a model event in this process.
|
|
5515
5462
|
*
|
|
5516
|
-
*
|
|
5517
|
-
*
|
|
5518
|
-
*
|
|
5519
|
-
* STOP-PAUSE. A run started with `driverRetry.pauseRootStops` hands its ROOT a `stopPause`;
|
|
5520
|
-
* no other manager gets one. Terminal refusals end the run immediately; the remaining stops wait and alarm:
|
|
5521
|
-
* `no-progress`, `max-attempts`, `repeated-failure`, `pending-unresolved` and
|
|
5522
|
-
* `budget-exhausted` each emit one record with `replacesStop` and pause 5 minutes, doubling per
|
|
5523
|
-
* stop-pause in a row up to 60, never past a declared deadline and not jittered. An attempt that
|
|
5524
|
-
* made progress resets the streak. A cancel ends the pause and settles the run; a root steer ends
|
|
5525
|
-
* any pause at once, a budget pause included, because a steer is how an operator changes a live
|
|
5526
|
-
* run. Ending a pause admits nothing by itself: the next admission reads the pool again, a pool
|
|
5527
|
-
* still spent pauses again with its own row, and the steer stays queued until a turn reads it.
|
|
5528
|
-
* After the pause the per-window counters start over and the driver is re-entered: after
|
|
5529
|
-
* a failure as `driver-failure` with `pausedMs`, after an admission or continuation refusal as the
|
|
5530
|
-
* re-entry it was about to use. Runs must not fail early: over 259 runs, 8 of the 10 stops settled runs that
|
|
5531
|
-
* nobody solved or stopped, including 17 roots whose harness never streamed.
|
|
5463
|
+
* Opted-in root stop-pauses alarm and wait instead of settling recoverable stops. Cancellation
|
|
5464
|
+
* ends the wait; a steer wakes it but grants no admission. The next attempt rechecks the budget.
|
|
5532
5465
|
*/
|
|
5533
5466
|
function assertContinuationPolicy(continuation) {
|
|
5534
5467
|
if (continuation === void 0) return;
|
|
@@ -5928,6 +5861,7 @@ async function driveManagerTurns(inputs) {
|
|
|
5928
5861
|
sleep: (ms, signal) => sleepScopeWait(scope, ms, signal, "paused-root"),
|
|
5929
5862
|
...rootStopPause(config, session),
|
|
5930
5863
|
nested: (config.nodeContextSeed?.depth ?? 0) > 0,
|
|
5864
|
+
seatChain: config.driveHarness.seatChain === true && authoredSeatStages(config.profile) !== void 0,
|
|
5931
5865
|
resolvePending: (failure, failures) => abandonScopeRetainedOwnerInvocation(scope, failure, failures),
|
|
5932
5866
|
wait: {
|
|
5933
5867
|
open: () => !mcp.submittedResult() && !mcp.isStopped() && session.state.progressStopReason === void 0 && (session.controls.hasOpenWork() || session.state.nextTurnInbox.pendingAuthority() > 0),
|
|
@@ -6036,7 +5970,7 @@ function readManagerProgress(loop) {
|
|
|
6036
5970
|
const submitted = Boolean(mcp.submittedResult());
|
|
6037
5971
|
const composite = bestComposite(controls.checkReads());
|
|
6038
5972
|
return {
|
|
6039
|
-
poolTokensSpent: loop.baseTokensLeft - scope.budget.tokensLeft,
|
|
5973
|
+
poolTokensSpent: scope.budget.tokensCapped === false ? scope.budget.committedTokens ?? 0 : loop.baseTokensLeft - scope.budget.tokensLeft,
|
|
6040
5974
|
settledCount: settled.length,
|
|
6041
5975
|
submitted,
|
|
6042
5976
|
deliveredCount,
|
|
@@ -6766,7 +6700,7 @@ function automaticDriverBackendSupported(backend, coordination) {
|
|
|
6766
6700
|
function buildNestedManagerDeps(child) {
|
|
6767
6701
|
const { env, seat, admitted, spawnContext } = child;
|
|
6768
6702
|
const { options, ownerShare, authorizeDownFor, peerAuditFor } = env.setup;
|
|
6769
|
-
const nestedPerWorker = defaultPerWorker(spawnContext.budget, ownerShare);
|
|
6703
|
+
const nestedPerWorker = defaultPerWorker(spawnContext.budget, ownerShare, options.workerSlots);
|
|
6770
6704
|
const runMessageAuthority = authorizeDownFor(admitted.authorized, seat.depth + 1);
|
|
6771
6705
|
const authorizeNestedMessage = child.subtreeGraph && runMessageAuthority ? child.subtreeGraph.observeMessages(runMessageAuthority) : runMessageAuthority;
|
|
6772
6706
|
const policy = nestedSettlementPolicy(child);
|
|
@@ -6923,7 +6857,7 @@ async function storeCoordinationSecret(provider, secrets, name, bind, policy) {
|
|
|
6923
6857
|
if (!(date instanceof Date) || !Number.isFinite(date.getTime())) throw new ValidationError("provider secret receipt has no valid updatedAt");
|
|
6924
6858
|
return date.toISOString();
|
|
6925
6859
|
};
|
|
6926
|
-
let revision;
|
|
6860
|
+
let revision, adopted = false;
|
|
6927
6861
|
const prior = secrets.get(name);
|
|
6928
6862
|
if (prior?.binding === bind && prior.ready) return prior.policy;
|
|
6929
6863
|
if (prior?.binding === bind) throw new ValidationError("coordination binding is not ready");
|
|
@@ -6962,16 +6896,26 @@ async function storeCoordinationSecret(provider, secrets, name, bind, policy) {
|
|
|
6962
6896
|
const authority = (bindings) => canonicalCandidateDigest(bindings?.map(({ ttlSeconds: _, ...binding }) => binding));
|
|
6963
6897
|
if (stored.exposeInBox || authority(recovered) !== authority(policy.proxySecrets)) throw new ValidationError("stored coordination policy has different authority");
|
|
6964
6898
|
entry.policy = { proxySecrets: recovered };
|
|
6899
|
+
adopted = true;
|
|
6965
6900
|
}
|
|
6966
6901
|
}
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6902
|
+
try {
|
|
6903
|
+
revision = stamp(await store.update(name, value, {
|
|
6904
|
+
...entry.policy,
|
|
6905
|
+
expectedUpdatedAt
|
|
6906
|
+
}));
|
|
6907
|
+
} catch (error) {
|
|
6908
|
+
if (entry.ready) revision = await store.list().then((rows) => stamp(rows.find((row) => row.name === name)), () => revision);
|
|
6909
|
+
throw error;
|
|
6910
|
+
}
|
|
6971
6911
|
});
|
|
6972
6912
|
entry.ready = true;
|
|
6973
6913
|
return entry.policy;
|
|
6974
6914
|
} catch (cause) {
|
|
6915
|
+
if (adopted) {
|
|
6916
|
+
secrets.delete(name);
|
|
6917
|
+
throw cause;
|
|
6918
|
+
}
|
|
6975
6919
|
await entry.release().catch((cleanup) => {
|
|
6976
6920
|
throw new AggregateError([cause, cleanup], "coordination binding and cleanup failed");
|
|
6977
6921
|
});
|
|
@@ -7257,7 +7201,7 @@ function leafWorker(env, seat, admitted, spawnContext) {
|
|
|
7257
7201
|
const { options, deliverable, runNamespace } = env.setup;
|
|
7258
7202
|
const { childDeliverable } = admitted;
|
|
7259
7203
|
if (childDeliverable !== deliverable && !options.backend) throw new ValidationError("supervise: resolveDeliverable selected a per-spawn deliverable but there is no backend to derive that leaf from; makeLeafAgent owns its own completion check");
|
|
7260
|
-
return withModelSubstitution((childDeliverable === deliverable ? env.makeLeaf :
|
|
7204
|
+
return withModelSubstitution((childDeliverable === deliverable ? env.makeLeaf : workerFromBackend(options.backend, childDeliverable))(admitted.authorized, Object.freeze({
|
|
7261
7205
|
...admitted.authorizedContext,
|
|
7262
7206
|
assignmentId: workerAssignmentNamespace(runNamespace, seat.parentOwnerId, spawnContext.assignmentId)
|
|
7263
7207
|
})), admitted.modelSubstitution);
|
|
@@ -7376,232 +7320,6 @@ function registerRecovery(env, makeRecursiveWorker) {
|
|
|
7376
7320
|
});
|
|
7377
7321
|
env.recoveryFactories.set(makeRecursiveWorker, factory);
|
|
7378
7322
|
}
|
|
7379
|
-
// src/runtime/supervise/worker-retry.ts
|
|
7380
|
-
/**
|
|
7381
|
-
* WORKER-SPAWN RETRY — the second chance `driverRetry` gives the root, given to a leaf.
|
|
7382
|
-
*
|
|
7383
|
-
* `driverRetry` guards the root and nothing else. A worker that dies is typed into a `down`
|
|
7384
|
-
* settlement, which is the right shape when the worker RAN. It is the wrong shape when the worker
|
|
7385
|
-
* never started, and one failure produces exactly that:
|
|
7386
|
-
*
|
|
7387
|
-
* bridgeExecutor: bridge stream error: host-executor: acquire timeout after 60000ms
|
|
7388
|
-
* (in_flight=4/4, queued=1)
|
|
7389
|
-
*
|
|
7390
|
-
* Two live cells held 2 leads and up to 8
|
|
7391
|
-
* workers on one bridge whose host executor caps live harness children at 4. A worker spawn that
|
|
7392
|
-
* queued past the bridge's single 60 s acquire deadline died there, and the run lost that worker
|
|
7393
|
-
* permanently even though a slot freed seconds later. The bridge retries nothing.
|
|
7394
|
-
*
|
|
7395
|
-
* Retrying execution is dangerous by default, so this seam is FAIL-CLOSED on two independent
|
|
7396
|
-
* proofs that the attempt did no work:
|
|
7397
|
-
*
|
|
7398
|
-
* 1. The error carries a PRE-SPAWN signature — a refusal the backend emits before the harness
|
|
7399
|
-
* child or the box exists ({@link isPreSpawnExecutorFailure}). No provider call started.
|
|
7400
|
-
* 2. The attempt yielded NO execution event. An attempt that streamed anything may have metered
|
|
7401
|
-
* usage, and a blind re-run would double-spend, so it stays fatal whatever its message says.
|
|
7402
|
-
*
|
|
7403
|
-
* Either proof missing means no retry. This is deliberately NARROWER than "is this an
|
|
7404
|
-
* infrastructure hiccup": `ECONNRESET` is a transient transport failure and is NOT retried here,
|
|
7405
|
-
* because it can arrive mid-stream after a metered call.
|
|
7406
|
-
*
|
|
7407
|
-
* Re-entry re-invokes `execute` on the SAME executor object. Nothing is rebuilt, so the executor's
|
|
7408
|
-
* runtime-owned materialization attestation, its bound session id, and its inbox are all the ones
|
|
7409
|
-
* the kernel admitted. For the bridge backend a second `execute` is a fresh session stream over the
|
|
7410
|
-
* same durable execution id, and a saturation refusal happens before that session's child spawned.
|
|
7411
|
-
*
|
|
7412
|
-
* @experimental
|
|
7413
|
-
*/
|
|
7414
|
-
/**
|
|
7415
|
-
* The backend refusals that are emitted BEFORE any child process, box, or provider request exists.
|
|
7416
|
-
*
|
|
7417
|
-
* Each is a queue admission deadline: the executor asked for a slot, waited, and was refused
|
|
7418
|
-
* without anything being started. The three names are the pools cli-bridge and the container host
|
|
7419
|
-
* admit through — the host executor, one scoped per-run host executor, and the container pool.
|
|
7420
|
-
*
|
|
7421
|
-
* The pattern is anchored to the whole phrase, including the `after <n>ms` deadline, so a message
|
|
7422
|
-
* that merely mentions a queue does not qualify.
|
|
7423
|
-
*/
|
|
7424
|
-
const PRE_SPAWN_SIGNATURES = Object.freeze([/\b(?:host-executor|scoped-host-executor|container-pool): acquire timeout after \d+ms\b/u]);
|
|
7425
|
-
const DEFAULT_MAX_TOTAL_MS = 9e5;
|
|
7426
|
-
const DEFAULT_INITIAL_BACKOFF_MS = 5e3;
|
|
7427
|
-
const DEFAULT_MAX_BACKOFF_MS = 6e4;
|
|
7428
|
-
/**
|
|
7429
|
-
* True when `error` is a backend refusal issued before anything ran.
|
|
7430
|
-
*
|
|
7431
|
-
* This answers "did this fail BEFORE any provider call", which is a strictly narrower question
|
|
7432
|
-
* than "is this an infrastructure hiccup". The broader question admits `ECONNRESET` and a bare
|
|
7433
|
-
* `fetch failed`, both of which can arrive after a metered call, and answering the broad question
|
|
7434
|
-
* where the narrow one is required is what makes a retry double-spend.
|
|
7435
|
-
*/
|
|
7436
|
-
function isPreSpawnExecutorFailure(error, additionalSignatures = []) {
|
|
7437
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
7438
|
-
for (const signature of PRE_SPAWN_SIGNATURES) if (signature.test(message)) return true;
|
|
7439
|
-
for (const signature of additionalSignatures) if (signature.test(message)) return true;
|
|
7440
|
-
return false;
|
|
7441
|
-
}
|
|
7442
|
-
/** Read the policy, refusing a number a run cannot act on rather than silently clamping it. */
|
|
7443
|
-
function resolveWorkerSpawnRetry(policy) {
|
|
7444
|
-
if (policy === void 0 || policy.enabled === false) return void 0;
|
|
7445
|
-
const maxTotalMs = policy.maxTotalMs ?? DEFAULT_MAX_TOTAL_MS;
|
|
7446
|
-
const initialBackoffMs = policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
|
|
7447
|
-
const maxBackoffMs = policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
|
|
7448
|
-
for (const [name, value] of [
|
|
7449
|
-
["maxTotalMs", maxTotalMs],
|
|
7450
|
-
["initialBackoffMs", initialBackoffMs],
|
|
7451
|
-
["maxBackoffMs", maxBackoffMs]
|
|
7452
|
-
]) if (!Number.isFinite(value) || value < 0) throw new ValidationError(`workerRetry.${name} must be a nonnegative finite number of milliseconds`);
|
|
7453
|
-
const signatures = policy.additionalPreSpawnSignatures ?? [];
|
|
7454
|
-
for (const signature of signatures) if (!(signature instanceof RegExp)) throw new ValidationError("workerRetry.additionalPreSpawnSignatures must contain RegExp values");
|
|
7455
|
-
return Object.freeze({
|
|
7456
|
-
maxTotalMs,
|
|
7457
|
-
initialBackoffMs,
|
|
7458
|
-
maxBackoffMs: Math.max(initialBackoffMs, maxBackoffMs),
|
|
7459
|
-
signatures: Object.freeze([...signatures])
|
|
7460
|
-
});
|
|
7461
|
-
}
|
|
7462
|
-
/**
|
|
7463
|
-
* Wrap a worker seam so a leaf whose spawn is refused before it runs is re-entered instead of
|
|
7464
|
-
* settling dead.
|
|
7465
|
-
*
|
|
7466
|
-
* The wrapper composes the agent's `executorFactory`, which is where a leaf executor is built with
|
|
7467
|
-
* the real child signal and node context, and it never replaces the executor object the factory
|
|
7468
|
-
* returned: the retry lives in one `execute` wrapper that carries the inner executor's attestation
|
|
7469
|
-
* forward and delegates every other surface to it.
|
|
7470
|
-
*/
|
|
7471
|
-
function withWorkerSpawnRetry(make, policy, hooks = {}) {
|
|
7472
|
-
const resolved = resolveWorkerSpawnRetry(policy);
|
|
7473
|
-
if (resolved === void 0) return make;
|
|
7474
|
-
return (profile, spawnContext) => {
|
|
7475
|
-
const agent = make(profile, spawnContext);
|
|
7476
|
-
const spec = agent.executorSpec;
|
|
7477
|
-
if (spec === void 0) return agent;
|
|
7478
|
-
const seamHooks = {
|
|
7479
|
-
...hooks,
|
|
7480
|
-
...typeof profile.name === "string" ? { worker: profile.name } : {}
|
|
7481
|
-
};
|
|
7482
|
-
if (spec.executorFactory !== void 0) {
|
|
7483
|
-
const innerFactory = spec.executorFactory;
|
|
7484
|
-
const executorFactory = (builtSpec, ctx) => retryPreSpawnRefusals(innerFactory(builtSpec, ctx), resolved, seamHooks);
|
|
7485
|
-
return {
|
|
7486
|
-
...agent,
|
|
7487
|
-
executorSpec: {
|
|
7488
|
-
...spec,
|
|
7489
|
-
executorFactory
|
|
7490
|
-
}
|
|
7491
|
-
};
|
|
7492
|
-
}
|
|
7493
|
-
if (spec.executor !== void 0) {
|
|
7494
|
-
const executor = retryPreSpawnRefusals(spec.executor, resolved, seamHooks);
|
|
7495
|
-
return {
|
|
7496
|
-
...agent,
|
|
7497
|
-
executorSpec: {
|
|
7498
|
-
...spec,
|
|
7499
|
-
executor
|
|
7500
|
-
}
|
|
7501
|
-
};
|
|
7502
|
-
}
|
|
7503
|
-
return agent;
|
|
7504
|
-
};
|
|
7505
|
-
}
|
|
7506
|
-
/**
|
|
7507
|
-
* Re-enter `execute` on one executor while a pre-spawn refusal keeps proving nothing ran.
|
|
7508
|
-
*
|
|
7509
|
-
* Exported so an owner of `makeLeafAgent` — which builds its own executors — gets the same seam
|
|
7510
|
-
* without reimplementing the two proofs.
|
|
7511
|
-
*/
|
|
7512
|
-
function retryPreSpawnRefusals(inner, policy, hooks = {}) {
|
|
7513
|
-
const now = hooks.now ?? Date.now;
|
|
7514
|
-
const sleep = hooks.sleep ?? defaultSleep;
|
|
7515
|
-
/** The wait before the next attempt, or `null` when this failure stays fatal. */
|
|
7516
|
-
const nextWait = (error, attempt, yielded, startedAt, signal) => {
|
|
7517
|
-
if (yielded) return null;
|
|
7518
|
-
if (signal?.aborted) return null;
|
|
7519
|
-
if (!isPreSpawnExecutorFailure(error, policy.signatures)) return null;
|
|
7520
|
-
const backoff = Math.min(policy.maxBackoffMs, policy.initialBackoffMs * 2 ** (attempt - 1));
|
|
7521
|
-
if (now() - startedAt + backoff >= policy.maxTotalMs) return null;
|
|
7522
|
-
return backoff;
|
|
7523
|
-
};
|
|
7524
|
-
const report = (error, attempt, waitMs) => {
|
|
7525
|
-
hooks.onRetry?.({
|
|
7526
|
-
attempt,
|
|
7527
|
-
waitMs,
|
|
7528
|
-
...hooks.worker === void 0 ? {} : { worker: hooks.worker },
|
|
7529
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7530
|
-
});
|
|
7531
|
-
};
|
|
7532
|
-
const wrapped = {
|
|
7533
|
-
runtime: inner.runtime,
|
|
7534
|
-
...inner.budgetExempt !== void 0 ? { budgetExempt: inner.budgetExempt } : {},
|
|
7535
|
-
...inner.deliver ? { deliver: (message) => inner.deliver?.(message) } : {},
|
|
7536
|
-
...inner.progress ? { progress: () => inner.progress?.() } : {},
|
|
7537
|
-
...inner.traceSource ? { traceSource: () => inner.traceSource?.() } : {},
|
|
7538
|
-
...inner.metered ? { metered: () => inner.metered?.() } : {},
|
|
7539
|
-
...inner.accounting ? { accounting: () => inner.accounting?.() } : {},
|
|
7540
|
-
...inner.resultAccepted ? { resultAccepted: () => inner.resultAccepted?.() === true } : {},
|
|
7541
|
-
...inner.harnessTranscript ? { harnessTranscript: () => inner.harnessTranscript?.() } : {},
|
|
7542
|
-
...inner.harnessTranscriptSettled ? { harnessTranscriptSettled: () => inner.harnessTranscriptSettled?.() ?? Promise.resolve() } : {},
|
|
7543
|
-
...inner.interactive === void 0 ? {} : { interactive: () => interactiveOf(inner) },
|
|
7544
|
-
execute(task, signal) {
|
|
7545
|
-
const first = inner.execute(task, signal);
|
|
7546
|
-
const restart = () => inner.execute(task, signal);
|
|
7547
|
-
return isAsyncIterable(first) ? retryStream(first, restart, signal) : retryOneShot(first, restart, signal);
|
|
7548
|
-
},
|
|
7549
|
-
...inner.recover ? { recover: (task, signal) => inner.recover(task, signal) } : {},
|
|
7550
|
-
teardown: (grace) => inner.teardown(grace),
|
|
7551
|
-
...teardownSurfaces(inner),
|
|
7552
|
-
resultArtifact: () => inner.resultArtifact()
|
|
7553
|
-
};
|
|
7554
|
-
async function* retryStream(first, restart, signal) {
|
|
7555
|
-
const startedAt = now();
|
|
7556
|
-
let execution = first;
|
|
7557
|
-
for (let attempt = 1;; attempt += 1) {
|
|
7558
|
-
let yielded = false;
|
|
7559
|
-
try {
|
|
7560
|
-
for await (const event of execution) {
|
|
7561
|
-
yielded = true;
|
|
7562
|
-
yield event;
|
|
7563
|
-
}
|
|
7564
|
-
return;
|
|
7565
|
-
} catch (error) {
|
|
7566
|
-
const waitMs = nextWait(error, attempt, yielded, startedAt, signal);
|
|
7567
|
-
if (waitMs === null) throw error;
|
|
7568
|
-
report(error, attempt, waitMs);
|
|
7569
|
-
await sleep(waitMs, signal);
|
|
7570
|
-
const next = restart();
|
|
7571
|
-
if (!isAsyncIterable(next)) throw new ValidationError("retryPreSpawnRefusals: executor changed execution shape between attempts");
|
|
7572
|
-
execution = next;
|
|
7573
|
-
}
|
|
7574
|
-
}
|
|
7575
|
-
}
|
|
7576
|
-
async function retryOneShot(first, restart, signal) {
|
|
7577
|
-
const startedAt = now();
|
|
7578
|
-
let execution = first;
|
|
7579
|
-
for (let attempt = 1;; attempt += 1) try {
|
|
7580
|
-
return await execution;
|
|
7581
|
-
} catch (error) {
|
|
7582
|
-
const waitMs = nextWait(error, attempt, false, startedAt, signal);
|
|
7583
|
-
if (waitMs === null) throw error;
|
|
7584
|
-
report(error, attempt, waitMs);
|
|
7585
|
-
await sleep(waitMs, signal);
|
|
7586
|
-
const next = restart();
|
|
7587
|
-
if (isAsyncIterable(next)) throw new ValidationError("retryPreSpawnRefusals: executor changed execution shape between attempts");
|
|
7588
|
-
execution = next;
|
|
7589
|
-
}
|
|
7590
|
-
}
|
|
7591
|
-
return inheritRuntimeOwnedExecutorAttestation(inner, wrapped);
|
|
7592
|
-
}
|
|
7593
|
-
/** Forward the inner executor's interactive contract exactly. The method is present only when the
|
|
7594
|
-
* inner executor implements it, so the reachable call always has one to delegate to. */
|
|
7595
|
-
function interactiveOf(inner) {
|
|
7596
|
-
if (inner.interactive === void 0) throw new ValidationError("retryPreSpawnRefusals: inner executor exposes no interactive session");
|
|
7597
|
-
return inner.interactive();
|
|
7598
|
-
}
|
|
7599
|
-
/** The wait stays REF'd: a worker waiting out a saturated executor is live work, and an unref'd
|
|
7600
|
-
* timer lets the process exit mid-backoff when nothing else is pending. */
|
|
7601
|
-
function defaultSleep(ms, signal) {
|
|
7602
|
-
if (ms <= 0 || signal?.aborted) return Promise.resolve();
|
|
7603
|
-
return sleep(ms, signal);
|
|
7604
|
-
}
|
|
7605
7323
|
// src/runtime/supervise/supervise-recursive-workers.ts
|
|
7606
7324
|
function createSuperviseWorkers(setup, cancelDurableRun) {
|
|
7607
7325
|
const { options, canonicalProfile, rootExecution, guidanceComposer, deliverable, analysts, journal, runId, rootOwnerId, now } = setup;
|
|
@@ -7652,15 +7370,13 @@ function createSuperviseWorkers(setup, cancelDurableRun) {
|
|
|
7652
7370
|
let makeWorkerAgent = options.makeWorkerAgent;
|
|
7653
7371
|
if (!makeWorkerAgent) {
|
|
7654
7372
|
if (!options.backend && !options.makeLeafAgent) throw new ValidationError("supervise: provide opts.backend (where workers run), opts.makeLeafAgent, or opts.makeWorkerAgent");
|
|
7655
|
-
const withRetry = (make) => withWorkerSpawnRetry(make, options.workerRetry, { ...options.onWorkerRetry ? { onRetry: options.onWorkerRetry } : {} });
|
|
7656
7373
|
makeWorkerAgent = recursiveWorkerFactoryFor({
|
|
7657
7374
|
setup,
|
|
7658
7375
|
cancelDurableRun,
|
|
7659
7376
|
graphSubtasks,
|
|
7660
7377
|
graphOverlayFor,
|
|
7661
7378
|
recoveryFactories,
|
|
7662
|
-
|
|
7663
|
-
makeLeaf: withRetry(options.makeLeafAgent ?? workerFromBackend(options.backend, deliverable)),
|
|
7379
|
+
makeLeaf: options.makeLeafAgent ?? workerFromBackend(options.backend, deliverable),
|
|
7664
7380
|
securityPolicy: options.profileSecurity ?? DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY
|
|
7665
7381
|
}, {
|
|
7666
7382
|
parent: canonicalProfile,
|
|
@@ -8348,10 +8064,14 @@ var ManagerDriveSession = class {
|
|
|
8348
8064
|
}
|
|
8349
8065
|
const seatJournalRoot = scopeRetainedOwnerJournalRoot(scope) ?? seatSelection.rootId;
|
|
8350
8066
|
this.seatChain ??= new SeatChain(authoredProfile, scope.view.root, seatJournalRoot, seatSelection.journal, seatSelection.select, now, (segment, phase) => seatSelection.onSegment(scope.view.root, segment, phase), async (selection, previous) => {
|
|
8351
|
-
if (previous
|
|
8352
|
-
|
|
8353
|
-
|
|
8354
|
-
|
|
8067
|
+
if (!previous || previous.seat === selection.seat) return;
|
|
8068
|
+
const sameEnvironment = previous.provider === selection.stage.provider && previous.harness === selection.stage.harness;
|
|
8069
|
+
try {
|
|
8070
|
+
const checkpointAt = await prepareScopeRetainedOwnerSeatSwitch(scope, sameEnvironment ? "same-environment" : "replace-environment");
|
|
8071
|
+
if (!sameEnvironment) this.switchedProviderCheckpointAt = checkpointAt;
|
|
8072
|
+
} catch (error) {
|
|
8073
|
+
scope.signal.throwIfAborted();
|
|
8074
|
+
throw new SeatPauseError(new Date(now() + SEAT_RECHECK_MS).toISOString(), `seat switch waits: ${errorText(error instanceof Error ? error.message : error)}`);
|
|
8355
8075
|
}
|
|
8356
8076
|
});
|
|
8357
8077
|
const usageLimit = reentry?.reason.reason === "upstream-unavailable" && reentry.reason.signal === "subscription-usage-limit";
|
|
@@ -8374,13 +8094,14 @@ var ManagerDriveSession = class {
|
|
|
8374
8094
|
if (afterRecheck) await seatSelection.onPause(scope.view.root, "", "resumed");
|
|
8375
8095
|
return seat;
|
|
8376
8096
|
} catch (error) {
|
|
8377
|
-
if (!(error instanceof
|
|
8097
|
+
if (!(error instanceof SeatPauseError)) throw error;
|
|
8378
8098
|
if (error.resumeAt !== pausedUntil) {
|
|
8379
8099
|
pausedUntil = error.resumeAt;
|
|
8380
8100
|
await seatSelection.journal.appendEvent(seatJournalRoot, {
|
|
8381
8101
|
kind: "seat-paused",
|
|
8382
8102
|
id: scope.view.root,
|
|
8383
8103
|
resumeAt: error.resumeAt,
|
|
8104
|
+
...error.detail === void 0 ? {} : { detail: error.detail },
|
|
8384
8105
|
seq: Date.parse(error.resumeAt),
|
|
8385
8106
|
at: new Date(now()).toISOString()
|
|
8386
8107
|
});
|
|
@@ -8400,7 +8121,7 @@ var ManagerDriveSession = class {
|
|
|
8400
8121
|
async function executeDriverAttempt(input) {
|
|
8401
8122
|
const { state, scope, retainedOwner, turnCap } = input;
|
|
8402
8123
|
if (!(scope.view.inFlight > 0)) {
|
|
8403
|
-
const spent = exhaustedBudgetChannels(scope.budget, input.now()) ?? (scope.budget.iterationsLeft <= 0 ? "iterations: none left" : void 0);
|
|
8124
|
+
const spent = exhaustedBudgetChannels(scope.budget, input.now()) ?? (scope.budget.iterationsCapped !== false && scope.budget.iterationsLeft <= 0 ? "iterations: none left" : void 0);
|
|
8404
8125
|
if (spent) throw new ValidationError(`driveHarnessFromBackend: supervisor budget exhausted (${spent})`);
|
|
8405
8126
|
}
|
|
8406
8127
|
const identity = admitDriverProfiles(input);
|
|
@@ -8649,7 +8370,7 @@ function driveHarnessFromBackend(backend, executionId, now = Date.now, maxTurns,
|
|
|
8649
8370
|
const failures = (await Promise.allSettled([manager.closeSeatSegment(reason), ...[...state.coordinationSecrets?.values() ?? []].map((secret) => secret.release())])).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
|
|
8650
8371
|
if (failures.length) throw new AggregateError(failures, "manager settlement cleanup failed");
|
|
8651
8372
|
};
|
|
8652
|
-
return attestRuntimeOwnedScopeOwner(drive, manager.ownerRuntime);
|
|
8373
|
+
return attestRuntimeOwnedScopeOwner(Object.assign(drive, seatSelection ? { seatChain: true } : {}), manager.ownerRuntime);
|
|
8653
8374
|
}
|
|
8654
8375
|
async function bindDriveRetainedOwner(manager, { scope, profile, authoredProfile }) {
|
|
8655
8376
|
const { state, activeSeat } = manager;
|
|
@@ -8851,7 +8572,7 @@ function providerSpawnPreflight(provider) {
|
|
|
8851
8572
|
const validate = provider.validateProfile?.bind(provider);
|
|
8852
8573
|
if (validate === void 0) return void 0;
|
|
8853
8574
|
return async (profile) => {
|
|
8854
|
-
const result = await validate(profile);
|
|
8575
|
+
const result = await validate(providerVisibleProfile(profile));
|
|
8855
8576
|
const errors = result.issues.filter((issue) => issue.level === "error");
|
|
8856
8577
|
if (result.ok && errors.length === 0) return void 0;
|
|
8857
8578
|
const named = errors.map((issue) => `${issue.code}${issue.path === void 0 ? "" : ` at ${issue.path}`}: ${issue.message}`);
|
|
@@ -8861,6 +8582,23 @@ function providerSpawnPreflight(provider) {
|
|
|
8861
8582
|
};
|
|
8862
8583
|
};
|
|
8863
8584
|
}
|
|
8585
|
+
/** Refuse a Runtime-managed child whose model controls its executor would refuse at its first
|
|
8586
|
+
* attempt (an unknown `model.metadata` key, an ambiguous `maxTokens`): the same check, at the call. */
|
|
8587
|
+
function modelSettingsSpawnPreflight(runtimeOwnsManager, path) {
|
|
8588
|
+
if (!runtimeOwnsManager) return void 0;
|
|
8589
|
+
return async (profile) => {
|
|
8590
|
+
try {
|
|
8591
|
+
profileModelExecutionSettings(profile, "spawn_worker", path);
|
|
8592
|
+
return;
|
|
8593
|
+
} catch (error) {
|
|
8594
|
+
if (!(error instanceof ConfigError)) throw error;
|
|
8595
|
+
return {
|
|
8596
|
+
cause: "unprovisionable-profile",
|
|
8597
|
+
detail: error.message
|
|
8598
|
+
};
|
|
8599
|
+
}
|
|
8600
|
+
};
|
|
8601
|
+
}
|
|
8864
8602
|
/** Refuse a Runtime-managed child whose declared tools cannot exist on its execution path. */
|
|
8865
8603
|
function profileToolSpawnPreflight(runtimeOwnsManager, canResolveProductTools) {
|
|
8866
8604
|
return async (profile) => {
|
|
@@ -8933,21 +8671,34 @@ async function assertSqlRunProvider(options) {
|
|
|
8933
8671
|
if (!capabilities.retainedControl || !capabilities.streaming.turnIdempotency || !capabilities.streaming.replay || !provider.get) throw new ValidationError("supervise: SQL runContext requires retainedControl, replay, turn idempotency, and provider.get");
|
|
8934
8672
|
}
|
|
8935
8673
|
// src/runtime/supervise/supervise-run-setup.ts
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
assertValidBudget(options.budget, "supervise budget");
|
|
8674
|
+
/** Canonical root identity is needed for control operations before executable admission. */
|
|
8675
|
+
function prepareSuperviseIdentity(profile, task, opts, mode = "execution") {
|
|
8676
|
+
const options = captureSuperviseOptions(opts, mode);
|
|
8940
8677
|
const guidanceComposer = profileGuidanceComposer(options.profileGuidance);
|
|
8941
8678
|
const { applyTraceTools, composeSpawnProfile } = traceToolPolicy(options, guidanceComposer);
|
|
8942
|
-
const peerAuditFor = peerAuditRegistry(options);
|
|
8943
8679
|
const registeredProfile = applyTraceTools(superviseRootProfile(profile, options.profileGuidance));
|
|
8944
8680
|
if (options.segment !== void 0) admitSegment(options.segment);
|
|
8945
8681
|
const canonicalProfile = options.segment === void 0 ? registeredProfile : withRootSegment(registeredProfile, options.segment);
|
|
8946
|
-
if (!testBrain || !isRuntimeFlowBrain(testBrain)) assertExecutableAgentProfile(canonicalProfile, "supervise root");
|
|
8947
8682
|
const canonicalTask = freezeDetached(task);
|
|
8683
|
+
return {
|
|
8684
|
+
options,
|
|
8685
|
+
guidanceComposer,
|
|
8686
|
+
applyTraceTools,
|
|
8687
|
+
composeSpawnProfile,
|
|
8688
|
+
registeredProfile,
|
|
8689
|
+
canonicalProfile,
|
|
8690
|
+
canonicalTask,
|
|
8691
|
+
rootExecution: canonicalExecution(registeredProfile, canonicalTask, options.execution, "supervise root")
|
|
8692
|
+
};
|
|
8693
|
+
}
|
|
8694
|
+
function prepareSuperviseRun(profile, task, opts, testBrain) {
|
|
8695
|
+
const { options, guidanceComposer, applyTraceTools, composeSpawnProfile, registeredProfile, canonicalProfile, canonicalTask, rootExecution } = prepareSuperviseIdentity(profile, task, opts);
|
|
8696
|
+
assertSqlRunOptions(options);
|
|
8697
|
+
assertValidBudget(options.budget, "supervise budget");
|
|
8698
|
+
const peerAuditFor = peerAuditRegistry(options);
|
|
8699
|
+
if (!testBrain || !isRuntimeFlowBrain(testBrain)) assertExecutableAgentProfile(canonicalProfile, "supervise root");
|
|
8948
8700
|
assertWorkerSeams(options);
|
|
8949
8701
|
const authorizeDownFor = downMessageAuthorizer(options);
|
|
8950
|
-
const rootExecution = canonicalExecution(registeredProfile, canonicalTask, options.execution, "supervise root");
|
|
8951
8702
|
assertRunModelsAllowed(canonicalProfile, options);
|
|
8952
8703
|
const { deliverable, finalizer, analysts, profileTable } = resolveRunRegistry(options);
|
|
8953
8704
|
assertCoordinationBinding(options.coordination);
|
|
@@ -9077,7 +8828,6 @@ function peerAuditRegistry(options) {
|
|
|
9077
8828
|
function assertWorkerSeams(options) {
|
|
9078
8829
|
if (options.makeWorkerAgent && options.authorizeSpawn) throw new ValidationError("supervise: authorizeSpawn cannot be combined with caller-owned makeWorkerAgent; use makeLeafAgent to override leaf execution inside the authorized path, or use backend-derived workers");
|
|
9079
8830
|
if (options.makeWorkerAgent && options.makeLeafAgent) throw new ValidationError("supervise: makeWorkerAgent replaces the worker path and makeLeafAgent overrides a leaf inside it; pass one");
|
|
9080
|
-
if (options.makeWorkerAgent && (options.workerRetry || options.onWorkerRetry)) throw new ValidationError("supervise: workerRetry applies only to backend-derived workers; compose withWorkerSpawnRetry onto a caller-owned makeWorkerAgent explicitly");
|
|
9081
8831
|
if (options.makeWorkerAgent && options.resolveDeliverable) throw new ValidationError("supervise: resolveDeliverable applies only to backend-derived workers; wrap a caller-owned makeWorkerAgent with its completion checks explicitly");
|
|
9082
8832
|
}
|
|
9083
8833
|
function downMessageAuthorizer(options) {
|
|
@@ -9123,7 +8873,7 @@ function resolveRunRegistry(options) {
|
|
|
9123
8873
|
function admitPerWorker(options) {
|
|
9124
8874
|
assertRecursiveReservationPolicy(options.reservationPolicy);
|
|
9125
8875
|
const ownerShare = options.reservationPolicy?.ownerShare ?? 0;
|
|
9126
|
-
const perWorker = options.perWorker ?? defaultPerWorker(options.budget, ownerShare);
|
|
8876
|
+
const perWorker = options.perWorker ?? defaultPerWorker(options.budget, ownerShare, options.workerSlots);
|
|
9127
8877
|
assertValidBudget(perWorker, "supervise perWorker");
|
|
9128
8878
|
assertPerWorkerWithinPool(perWorker, options.budget);
|
|
9129
8879
|
if (options.perWorker !== void 0) assertPerWorkerAdmittable(perWorker, options.budget, ownerShare);
|
|
@@ -9154,7 +8904,13 @@ function managerDriverPlan(options, canonicalProfile, externalRoot) {
|
|
|
9154
8904
|
if (authoredSeatStages(canonicalProfile) && !hasCustomDriveHarness && !options.selectSeat) throw new ValidationError("supervise: AgentProfile.seats requires selectSeat before execution");
|
|
9155
8905
|
const automaticDriver = managerBackend !== void 0 && automaticDriverBackendSupported(managerBackend, options.coordination);
|
|
9156
8906
|
const spawnResourceRoot = hasCustomDriveHarness ? void 0 : rootSpawnResourceRoot(managerBackend);
|
|
9157
|
-
const spawnPreflight = composeSpawnPreflights(
|
|
8907
|
+
const spawnPreflight = composeSpawnPreflights(async (profile, context) => {
|
|
8908
|
+
const external = isExternalSupervisor(profile);
|
|
8909
|
+
const manager = declaredRuntimeToolNames(profile).length > 0 || external && authoredSeatStages(profile) !== void 0;
|
|
8910
|
+
const custom = manager ? external && hasCustomDriveHarness : options.makeLeafAgent !== void 0;
|
|
8911
|
+
const backend = manager ? external ? managerBackend : void 0 : options.backend;
|
|
8912
|
+
return modelSettingsSpawnPreflight(options.makeWorkerAgent === void 0 && !custom, backend?.backend === "provider" ? "provider" : "router")?.(profile, context);
|
|
8913
|
+
}, profileToolSpawnPreflight(options.makeWorkerAgent === void 0, options.resolveSupervisorTools !== void 0), coordinationChannelSpawnPreflight(options.makeWorkerAgent === void 0, hasCustomDriveHarness, managerBackend, options.coordination), options.backend?.backend === "bridge" ? bridgeSpawnPreflight(options.backend) : void 0, options.backend?.backend === "provider" && typeof options.backend.provider === "object" ? providerSpawnPreflight(options.backend.provider) : void 0);
|
|
9158
8914
|
const driverMaterialization = hasCustomDriveHarness ? options.driveHarnessMaterialization ?? fullProfileMaterialization : managerBackend && automaticDriver ? backendProfileMaterialization(managerBackend) : void 0;
|
|
9159
8915
|
if (externalRoot && !hasCustomDriveHarness && !automaticDriver) throw new ValidationError(`supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge, a provider with authenticated coordination.publicUrl and runtime MCP attachments, or an explicit driveHarness with reachable coordination transport`);
|
|
9160
8916
|
return {
|
|
@@ -9185,6 +8941,8 @@ function spawnProfileComposer(options, driver, childrenCanSubmit, composeSpawnPr
|
|
|
9185
8941
|
}
|
|
9186
8942
|
// src/runtime/supervise/supervise-run-lifecycle.ts
|
|
9187
8943
|
function superviseInternal(profile, task, opts, testBrain) {
|
|
8944
|
+
const cancelled = pendingCancellationRun(profile, task, opts);
|
|
8945
|
+
if (cancelled !== void 0) return cancelled;
|
|
9188
8946
|
const setup = prepareSuperviseRun(profile, task, opts, testBrain);
|
|
9189
8947
|
const { options } = setup;
|
|
9190
8948
|
const workers = createSuperviseWorkers(setup, (reason, request) => cancelDurableRun(reason, request));
|
|
@@ -9202,6 +8960,88 @@ function superviseInternal(profile, task, opts, testBrain) {
|
|
|
9202
8960
|
rootContinuation: void 0
|
|
9203
8961
|
});
|
|
9204
8962
|
}
|
|
8963
|
+
/** The aborted kernel entry owns replay and resource custody without executable admission. */
|
|
8964
|
+
function pendingCancellationRun(profile, task, opts) {
|
|
8965
|
+
const runDir = opts.runDir;
|
|
8966
|
+
if (runDir === void 0 || opts.runContext?.durability === "sql") return void 0;
|
|
8967
|
+
const request = readRunCancelRequest(runDir);
|
|
8968
|
+
if (request === void 0) return void 0;
|
|
8969
|
+
const acknowledged = readRunCancellation(runDir, request.operationId);
|
|
8970
|
+
if (acknowledged !== void 0 && acknowledged.effect !== "cancel_requested") return void 0;
|
|
8971
|
+
const { options, registeredProfile, canonicalProfile, canonicalTask, rootExecution } = prepareSuperviseIdentity(profile, task, opts, "control");
|
|
8972
|
+
const now = options.now ?? Date.now;
|
|
8973
|
+
const controller = new AbortController();
|
|
8974
|
+
applyRunCancellation(runDir, (reason, request) => {
|
|
8975
|
+
controller.abort(request === void 0 ? new Error(reason) : new RunCancellationReason(request.source, reason, request.operationId, request.operator));
|
|
8976
|
+
}, () => new Date(now()).toISOString(), "observer");
|
|
8977
|
+
const context = options.runContext ?? createFileRunContext(runDir, { withDriver: true });
|
|
8978
|
+
return (async () => {
|
|
8979
|
+
if (!controller.signal.aborted) throw new Error("run cancellation changed before control entry");
|
|
8980
|
+
const runId = options.runId ?? context.runId ?? "supervise";
|
|
8981
|
+
const journal = options.journal ?? context.journal;
|
|
8982
|
+
const prior = await journal.loadTree(runId);
|
|
8983
|
+
const root = prior?.find((event) => event.kind === "spawned" && event.id === runId);
|
|
8984
|
+
const reopened = [...prior ?? []].reverse().find((event) => event.kind === "reopened" && event.id === runId && event.budget !== void 0);
|
|
8985
|
+
const budget = options.budget ?? (reopened?.kind === "reopened" ? reopened.budget : void 0) ?? (root?.kind === "spawned" ? root.budget : void 0);
|
|
8986
|
+
if (budget === void 0) throw new Error("cancelled run has no declared or recorded budget");
|
|
8987
|
+
assertValidBudget(budget, "supervise budget");
|
|
8988
|
+
const rootAgent = {
|
|
8989
|
+
name: canonicalProfile.name ?? "supervise",
|
|
8990
|
+
async act() {
|
|
8991
|
+
throw new Error("cancelled run cannot enter execution");
|
|
8992
|
+
}
|
|
8993
|
+
};
|
|
8994
|
+
const backend = options.driverBackend ?? (options.rootDriverFromBackend === false ? void 0 : options.backend);
|
|
8995
|
+
if (backend?.backend === "provider" && !options.driveHarness && !options.resolveDriveHarness) {
|
|
8996
|
+
const provider = typeof backend.provider === "string" ? backend.registry?.get(backend.provider) : backend.provider;
|
|
8997
|
+
const priorProviders = new Set((prior ?? []).flatMap((event) => event.id === runId && event.kind === "execution-admitted" ? [event.admission.phase === "dispatched" ? event.admission.controlRef.provider : event.admission.provider] : []));
|
|
8998
|
+
if (provider !== void 0 && priorProviders.size === 1 && priorProviders.has(provider.name)) attestRuntimeOwnedScopeOwner(rootAgent, root?.kind === "spawned" ? root.runtime : provider.name, {
|
|
8999
|
+
provider,
|
|
9000
|
+
...backend.workspaceRetention === void 0 ? {} : { checkpointCapture: {
|
|
9001
|
+
port: backend.workspaceRetention,
|
|
9002
|
+
profile: providerVisibleProfile(canonicalProfile)
|
|
9003
|
+
} }
|
|
9004
|
+
});
|
|
9005
|
+
}
|
|
9006
|
+
const result = await createSupervisor().run(rootAgent, canonicalTask, {
|
|
9007
|
+
budget,
|
|
9008
|
+
runId,
|
|
9009
|
+
journal,
|
|
9010
|
+
blobs: options.blobs ?? context.blobs,
|
|
9011
|
+
executors: context.executors,
|
|
9012
|
+
rootIdentity: rootExecution.identity,
|
|
9013
|
+
rootMaterialization: {
|
|
9014
|
+
runtime: root?.kind === "spawned" ? root.runtime : "inline",
|
|
9015
|
+
declaration: "deferred",
|
|
9016
|
+
authoredProfile: registeredProfile
|
|
9017
|
+
},
|
|
9018
|
+
...options.segment === void 0 ? {} : { rootSegment: {
|
|
9019
|
+
registered: rootSegmentOf(registeredProfile),
|
|
9020
|
+
current: {
|
|
9021
|
+
...rootSegmentOf(canonicalProfile),
|
|
9022
|
+
...options.segment.stack === void 0 ? {} : { stack: { ...options.segment.stack } }
|
|
9023
|
+
},
|
|
9024
|
+
reason: options.segment.reason
|
|
9025
|
+
} },
|
|
9026
|
+
...options.recoverExecutor === void 0 ? {} : { recoverExecutor: options.recoverExecutor },
|
|
9027
|
+
...options.reservationPolicy === void 0 ? {} : { reservationPolicy: options.reservationPolicy },
|
|
9028
|
+
...options.workerSlots === void 0 ? {} : { workerSlots: options.workerSlots },
|
|
9029
|
+
...options.teardownConfirmMs === void 0 ? {} : { teardownConfirmMs: options.teardownConfirmMs },
|
|
9030
|
+
...context.resume === true || options.resume === true ? { resume: true } : {},
|
|
9031
|
+
retainedAtSettlement: options.retainedAtSettlement ?? "release",
|
|
9032
|
+
hooks: options.hooks,
|
|
9033
|
+
now,
|
|
9034
|
+
signal: options.signal === void 0 ? controller.signal : AbortSignal.any([controller.signal, options.signal]),
|
|
9035
|
+
interactiveBindingDir: runDir
|
|
9036
|
+
});
|
|
9037
|
+
recordRunCancellationOutcome(runDir, result, now);
|
|
9038
|
+
return {
|
|
9039
|
+
...result,
|
|
9040
|
+
rootProviderModel: rootProviderModelEvidence([]),
|
|
9041
|
+
...rootExecution.ref === void 0 ? {} : { execution: rootExecution.ref }
|
|
9042
|
+
};
|
|
9043
|
+
})();
|
|
9044
|
+
}
|
|
9205
9045
|
async function startSupervisedRun(run) {
|
|
9206
9046
|
const { setup, runControl, cancelDurableRun } = run;
|
|
9207
9047
|
const { options, ctx, log, runId, rootOwnerId, canonicalTask } = setup;
|
|
@@ -9440,6 +9280,6 @@ function superviseWithContext(profile, task, opts, brain) {
|
|
|
9440
9280
|
...signal === void 0 ? {} : { signal }
|
|
9441
9281
|
}, brain));
|
|
9442
9282
|
}
|
|
9443
|
-
export {
|
|
9283
|
+
export { dumbContinuationPassPrompt as A, areaUnderCurve as B, defaultEdgeTraversalCap as C, createPromptRegistry as D, analyzesFindingsReportPrompt as E, allWorkersStalled as F, plateauLength as H, anyOf as I, noProgressFor as L, kernelPromptRegistry as M, naiveContinuationPrompt as N, delegatesWorkerBriefPrompt as O, promptHandle as P, plateau as R, validateFlowGraph as S, validateGraphTopology as T, renderAnytimeTable as U, bestSoFar as V, createSupervisorSpanRecorder as W, GraphRouteError as _, superviseRootProfile as a, journaledGateDecider as b, supervisorAgentWithTestBrain as c, composeReentryTask as d, serveCoordinationMcp as f, createGraphOverlay as g, runBrainLoop as h, profileGuidanceComposer as i, formatPromptHandle as j, dumbContinuationFailPrompt as k, runToolSpans as l, driverAgent as m, superviseFleetCapabilities as n, workerFromBackend as o, coordinationControlToolNames as p, superviseWithTestBrain as r, supervisorAgent as s, supervise as t, UNPROVEN_CONTINUITY as u, createFlowRun as v, graphRootProfile as w, routeFlowOf as x, flowRootProfile as y, anytimeReport as z };
|
|
9444
9284
|
|
|
9445
|
-
//# sourceMappingURL=supervise-
|
|
9285
|
+
//# sourceMappingURL=supervise-rYEsVVcn.js.map
|