@tangle-network/agent-runtime 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5263 -0
- package/dist/{activation-B49A14V5.d.ts → activation-CtLdCiH3.d.ts} +2 -2
- package/dist/{activation-BiQYCayd.js → activation-DGLWxFJu.js} +2 -2
- package/dist/{activation-BiQYCayd.js.map → activation-DGLWxFJu.js.map} +1 -1
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +4 -4
- package/dist/candidate-execution/index.d.ts +1 -1
- package/dist/{coordination-manager-C-ZoeZ73.js → coordination-manager-D74AZg7m.js} +7 -7
- package/dist/{coordination-manager-C-ZoeZ73.js.map → coordination-manager-D74AZg7m.js.map} +1 -1
- package/dist/{coordination-map-routing-SnsZp287.js → coordination-map-routing-DVjW5Gx0.js} +2 -2
- package/dist/{coordination-map-routing-SnsZp287.js.map → coordination-map-routing-DVjW5Gx0.js.map} +1 -1
- package/dist/{delegate-d9HcOFC5.js → delegate-BVyFJ9AX.js} +2 -2
- package/dist/{delegate-d9HcOFC5.js.map → delegate-BVyFJ9AX.js.map} +1 -1
- package/dist/durable.d.ts +4 -2
- package/dist/durable.js +12 -184
- package/dist/durable.js.map +1 -1
- package/dist/{graph-C_Z0Ua10.js → graph-r8zcRxbi.js} +6 -8
- package/dist/{graph-C_Z0Ua10.js.map → graph-r8zcRxbi.js.map} +1 -1
- package/dist/{graph-registry-CI1o16DH.js → graph-registry-B25FxBZN.js} +5 -5
- package/dist/graph-registry-B25FxBZN.js.map +1 -0
- package/dist/{graph-registry-BU7pcrUW.js → graph-registry-C-hjRx2t.js} +1 -1
- package/dist/{improve-B4v4S8jY.js → improve-D12GulmK.js} +7 -7
- package/dist/{improve-B4v4S8jY.js.map → improve-D12GulmK.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +9 -9
- package/dist/intelligence.d.ts +3 -3
- package/dist/intelligence.js +5 -5
- package/dist/{jsonl-file-CvPuwM0a.js → jsonl-file-CeVYTIQJ.js} +41 -32
- package/dist/jsonl-file-CeVYTIQJ.js.map +1 -0
- package/dist/kernel.d.ts +2 -2
- package/dist/kernel.js +14 -13
- package/dist/{loop-runner-DKvA0JJr.d.ts → loop-runner-BHBabRIw.d.ts} +2 -2
- package/dist/{loop-runner-DMZkwlND.js → loop-runner-CYcN11I4.js} +2 -2
- package/dist/{loop-runner-DMZkwlND.js.map → loop-runner-CYcN11I4.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +5 -5
- package/dist/profiles.d.ts +1 -1
- package/dist/{provision-supervisor-Cjs4SQ9Q.js → provision-supervisor-BiqHJ-d4.js} +5 -5
- package/dist/{provision-supervisor-Cjs4SQ9Q.js.map → provision-supervisor-BiqHJ-d4.js.map} +1 -1
- package/dist/{run-cancellation-CGgk54Md.js → run-cancellation-D5R-7UQt.js} +3 -3
- package/dist/{run-cancellation-CGgk54Md.js.map → run-cancellation-D5R-7UQt.js.map} +1 -1
- package/dist/run-state-DRh8BOFi.js +350 -0
- package/dist/run-state-DRh8BOFi.js.map +1 -0
- package/dist/{runtime-BmxmsaNz.js → runtime-BX_9-DJB.js} +18 -17
- package/dist/runtime-BX_9-DJB.js.map +1 -0
- package/dist/{scope-CJ7xtrs8.js → scope-JcCkUy_z.js} +151 -106
- package/dist/scope-JcCkUy_z.js.map +1 -0
- package/dist/{search-executor-DSB6XAl1.js → search-executor-CdmaS5r4.js} +3 -3
- package/dist/{search-executor-DSB6XAl1.js.map → search-executor-CdmaS5r4.js.map} +1 -1
- package/dist/{server-FWT2Nt_w.js → server-BY6OluUf.js} +3 -3
- package/dist/{server-FWT2Nt_w.js.map → server-BY6OluUf.js.map} +1 -1
- package/dist/settle-record-BSDDf9Z4.js +130 -0
- package/dist/settle-record-BSDDf9Z4.js.map +1 -0
- package/dist/{structural-rollout-Bl23HSOd.js → structural-rollout-BjMUUXLn.js} +3 -3
- package/dist/{structural-rollout-Bl23HSOd.js.map → structural-rollout-BjMUUXLn.js.map} +1 -1
- package/dist/{supervise-MAzd0hOk.js → supervise-rYEsVVcn.js} +172 -384
- package/dist/supervise-rYEsVVcn.js.map +1 -0
- package/dist/{supervision-discovery-6EtvKCsU.js → supervision-discovery-D95Ack4J.js} +4 -129
- package/dist/supervision-discovery-D95Ack4J.js.map +1 -0
- package/dist/{supervisor-CwANBb9-.js → supervisor-B-bX69uL.js} +178 -120
- package/dist/supervisor-B-bX69uL.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +11 -11
- package/dist/{top-web-CZP3RRhD.js → top-web-D-wH9ipn.js} +13 -9
- package/dist/{top-web-CZP3RRhD.js.map → top-web-D-wH9ipn.js.map} +1 -1
- package/dist/tui/bin.js +3 -3
- package/dist/tui/index.d.ts +1 -1
- package/dist/tui/index.js +2 -2
- package/dist/{types-ooyUaxq_.d.ts → types-DblV14Oe.d.ts} +187 -222
- package/package.json +7 -8
- package/dist/graph-registry-CI1o16DH.js.map +0 -1
- package/dist/jsonl-file-CvPuwM0a.js.map +0 -1
- package/dist/runtime-BmxmsaNz.js.map +0 -1
- package/dist/scope-CJ7xtrs8.js.map +0 -1
- package/dist/supervise-MAzd0hOk.js.map +0 -1
- package/dist/supervision-discovery-6EtvKCsU.js.map +0 -1
- package/dist/supervisor-CwANBb9-.js.map +0 -1
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { An as WORKER_TRACE_PROPAGATION, Ar as recordRuntimeOwnedDriveHarnessProviderEvidence, At as beginScopeRetainedOwnerDrive, B as beginScopeOwnerAttempt, Bn as createOtelExporter, Ci as executorFailure, Dt as prepareScopeRetainedOwnerTask, Et as consumeScopeRetainedOwnerResult, Fi as assertValidBudget, Fr as runtimeOwnedDriveHarnessProviderEvidence, Ft as scopeRetainedOwnerRestorePoint, G as scopeOwnerExecutorNodeContext, H as noteScopeOwnerPlannedExecution, Hn as generateSpanId, Ht as scopeRetainedOwnerContext, Ir as runtimeOwnedExecutorExecutionBinding, It as noteScopeRetainedOwnerCoordination, J as providerVisibleProfile, Jr as runtimeFlowBrain, K as coordinationMcpAlias, Kr as isRuntimeFlowBrain, Lr as runtimeOwnedExecutorMaterialization, Mi as newUsageTotals, Mt as scopeRetainedOwnerTurnEnded, Nt as assertRetainedOwnerCoordinationRekey, O as parseWorkerToolTraceArtifact, Ot as scopeRetainedOwnerPriorSpend, Pi as spendFromUsageTotals, Pt as reconcileScopeRetainedOwnerEnvironment, Q as assertRecursiveReservationPolicy, Rr as runtimeOwnedExecutorProviderEvidence, Rt as bindScopeRetainedOwnerEnvironmentId, Sr as authoredAgentIdentity, St as scopeRetainedOwnerResourceReader, Tt as abandonScopeRetainedOwnerInvocation, U as recordScopeOwnerMaterialization, Ut as scopeRetainedOwnerJournalRoot, Vr as runtimeOwnedScopeOwnerRuntime, W as restoreScopeOwnerAcceptedExecution, Y as runtimeGraphMetadataKey, _i as unavailableResumeAt, a as readRootStream, bt as prepareScopeRetainedOwnerSeatSwitch, ct as recordScopeOwnerBarrenPause, dr as isTerminalNodeStatus, er as persistHarnessTranscript, fa as RunCancellationReason, fi as cappedDoublingMs, fr as SEAT_RECHECK_MS, gi as unavailablePauseMs, gr as seatExecutionProfile, ha as runAbortable, hi as reportedResetAt, hr as authoredSeatStages, ia as recordValue, ji as meterUsageEvent, kr as providerAttemptEvidence, kt as scopeRetainedOwnerResult, li as retainedExecutorSeamKey, lt as recordScopeOwnerDriverAttempt, ma as linkAbort, mr as SeatPauseError, n as ROOT_STREAM_FILE, ot as meterRuntimeOwnedAccounting, pi as jitteredMs, pr as SeatChain, q as coordinationProfileToolPrefix, qr as runtimeFlowAgent, r as createRootStreamSink, ri as RetainedExecutionPendingError, si as registerRetainedExecutorPreparation, sr as createActivityLog, st as meterRuntimeOwnedProviderAttempt, tr as readHarnessTranscript, ur as isLiveNodeStatus, ut as recordScopeOwnerPause, xi as contentAddress, xr as attestRuntimeOwnedScopeOwner, xt as releaseScopeRetainedOwnerForSeatPause, zr as runtimeOwnedPendingExecutorMaterialization, zt as bindScopeRetainedOwnerProvider } from "./scope-JcCkUy_z.js";
|
|
2
2
|
import { f as RuntimeRunStateError, i as ConfigError, m as ValidationError } from "./errors-AzSuGI23.js";
|
|
3
|
-
import { $ as isAsyncIterable, Dt as teardownExecutor, G as chargedTokens,
|
|
4
|
-
import { _ as errorText, d as detachedSnapshot, h as errorHttpStatus, m as errMessage, p as causeChain } from "./jsonl-file-
|
|
5
|
-
import { $t as promptModelProfileMaterialization, At as bridgeModelRouteRefusal, B as createRouterTranscript, C as createExecutor, Ct as admitContinuationPolicy, Et as createContinuationKeeper, F as codeModeArguments, I as codeModeModelOutput, Jt as controlProfileMaterialization, Kt as assertProfileMaterialization, L as createCodeModeTools, Qt as promptControlProfileMaterialization, Rt as bridgeRuntimeAttachmentsKey, S as captureReusableExecutorConfig, St as CONTINUATIONS_DIR, Tt as checkVerdictOf, U as resolveAgentEnvironmentProvider, V as routerBrain, X as createInbox, Xt as fullProfileMaterialization, Y as INBOX_RECEIPT_KEY, Yt as defineProfileMaterializationContract, Zt as profileMaterializationAxes$1, _t as CheckUnavailableError, a as assertRunParking, c as sleepScopeWait, ct as HarnessTurnFailedError, d as runFinalizer, dt as isContentFilterRefusal, en as renderUnsupported, et as PEER_MAIL_WIRE_KEY, f as runTree, ft as summarizeDriverAttempts, g as isDriverSpec, gt as preflightPublicCoordination, h as driverExecutorFactory, ht as upstreamUnavailableSignal, i as RunParkedError, in as worktreeCliProfileMaterialization, kt as bridgeAdmissionRefusal, l as withRunParking, lt as abandonablePending, m as driverChild, mt as undispatchedCredentialExit, n as createSupervisor, nn as sandboxProfileMaterialization, o as bindScopeParkingDrain, p as createRootHandle, pt as transientFailureKind, rn as unsupportedProfileDimensions, rt as isPeerMailEnvelope, st as DriverAttemptsExhaustedError, u as bestDelivered, ut as classifyDriverFailure, vt as gateFactoryOnDeliverable, w as snapshotExecutorConfig, wt as bestComposite, x as bindReusableExecutorExecutionId, z as toolResultText, zt as bridgeStopSignalKey } from "./supervisor-
|
|
3
|
+
import { $ as isAsyncIterable, Dt as teardownExecutor, G as chargedTokens, St as armDeadlineTimer, Tt as isDeadlineAbortReason, U as addSpend, Y as errorMessage, at as sleep, bt as DEFAULT_SUCCESSFUL_SHUTDOWN_MS, dt as zeroSpend, gt as withBudgetResources, ht as resourceTelemetry, lt as unmeteredSpend, pt as addResourceSpend, tt as promptCacheTokenClasses, ut as withTimeout } from "./workspace-streams-8ALHaZzx.js";
|
|
4
|
+
import { _ as errorText, d as detachedSnapshot, h as errorHttpStatus, m as errMessage, p as causeChain } from "./jsonl-file-CeVYTIQJ.js";
|
|
5
|
+
import { $t as promptModelProfileMaterialization, At as bridgeModelRouteRefusal, B as createRouterTranscript, C as createExecutor, Ct as admitContinuationPolicy, Et as createContinuationKeeper, F as codeModeArguments, I as codeModeModelOutput, Jt as controlProfileMaterialization, Kt as assertProfileMaterialization, L as createCodeModeTools, Qt as promptControlProfileMaterialization, Rt as bridgeRuntimeAttachmentsKey, S as captureReusableExecutorConfig, St as CONTINUATIONS_DIR, Tt as checkVerdictOf, U as resolveAgentEnvironmentProvider, V as routerBrain, X as createInbox, Xt as fullProfileMaterialization, Y as INBOX_RECEIPT_KEY, Yt as defineProfileMaterializationContract, Zt as profileMaterializationAxes$1, _t as CheckUnavailableError, a as assertRunParking, c as sleepScopeWait, ct as HarnessTurnFailedError, d as runFinalizer, dt as isContentFilterRefusal, en as renderUnsupported, et as PEER_MAIL_WIRE_KEY, f as runTree, ft as summarizeDriverAttempts, g as isDriverSpec, gt as preflightPublicCoordination, h as driverExecutorFactory, ht as upstreamUnavailableSignal, i as RunParkedError, in as worktreeCliProfileMaterialization, kt as bridgeAdmissionRefusal, l as withRunParking, lt as abandonablePending, m as driverChild, mt as undispatchedCredentialExit, n as createSupervisor, nn as sandboxProfileMaterialization, o as bindScopeParkingDrain, p as createRootHandle, pt as transientFailureKind, rn as unsupportedProfileDimensions, rt as isPeerMailEnvelope, st as DriverAttemptsExhaustedError, u as bestDelivered, ut as classifyDriverFailure, vt as gateFactoryOnDeliverable, w as snapshotExecutorConfig, wt as bestComposite, x as bindReusableExecutorExecutionId, z as toolResultText, zt as bridgeStopSignalKey } from "./supervisor-B-bX69uL.js";
|
|
6
6
|
import { _ as agentHarness, a as concreteProfileModel, c as profileModelExecutionSettings, l as profileProviderModel, n as assertModelAllowed, o as enforceTokenLimits, r as assertProfileModelsAllowed, s as profileBridgeWireModel, t as assertExecutableAgentProfile, v as harnessRunsAgent } from "./model-policy-JOcg3NRs.js";
|
|
7
7
|
import { n as createStdioToolServer } from "./tool-server-CZWd_Opp.js";
|
|
8
8
|
import { t as composeRuntimeHooks } from "./runtime-hooks-CsW2pfFk.js";
|
|
9
|
-
import { _ as workerAssignmentNamespace, c as writeRunCancellation, d as coordinationEventId, f as externalExecutionId, g as supervisionRunNamespace, h as rootCoordinationOwner, i as readRunCancellation, l as canonicalExecution, m as freezeDetachedProfile, p as freezeDetached, r as readRunCancelRequest, s as watchRunCancellation, t as applyRunCancellation, u as childCoordinationOwner } from "./run-cancellation-
|
|
10
|
-
import { c as createSteerAcknowledger, d as coordinationVerbNames, f as createFileRunContext, l as budgetOverspendText, m as withRunContext, n as createCoordinationToolsForManager, p as createInMemoryRunContext, s as createCancelAcknowledger, t as createCoordinationTools, u as normalizeAnalyzeOnSettle } from "./coordination-manager-
|
|
11
|
-
import { o as safeJsonText } from "./coordination-map-routing-
|
|
12
|
-
import { a as graphRegistry, c as loadGraph } from "./graph-registry-
|
|
9
|
+
import { _ as workerAssignmentNamespace, c as writeRunCancellation, d as coordinationEventId, f as externalExecutionId, g as supervisionRunNamespace, h as rootCoordinationOwner, i as readRunCancellation, l as canonicalExecution, m as freezeDetachedProfile, p as freezeDetached, r as readRunCancelRequest, s as watchRunCancellation, t as applyRunCancellation, u as childCoordinationOwner } from "./run-cancellation-D5R-7UQt.js";
|
|
10
|
+
import { c as createSteerAcknowledger, d as coordinationVerbNames, f as createFileRunContext, l as budgetOverspendText, m as withRunContext, n as createCoordinationToolsForManager, p as createInMemoryRunContext, s as createCancelAcknowledger, t as createCoordinationTools, u as normalizeAnalyzeOnSettle } from "./coordination-manager-D74AZg7m.js";
|
|
11
|
+
import { o as safeJsonText } from "./coordination-map-routing-DVjW5Gx0.js";
|
|
12
|
+
import { a as graphRegistry, c as loadGraph } from "./graph-registry-B25FxBZN.js";
|
|
13
13
|
import { ATTR, agentIdentityAttributes } from "@tangle-network/agent-trace-contract";
|
|
14
14
|
import { ProxySecretScopeSchema, ProxySecretsSchema, agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalCandidateJson, mergeProxySecretBindings, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
15
15
|
import { toolSpansToTraceAnalysisStore } from "@tangle-network/agent-eval";
|
|
@@ -96,8 +96,6 @@ const superviseOptionKeySet = /* @__PURE__ */ new Set([
|
|
|
96
96
|
"stopRule",
|
|
97
97
|
"watchWorkers",
|
|
98
98
|
"continuation",
|
|
99
|
-
"workerRetry",
|
|
100
|
-
"onWorkerRetry",
|
|
101
99
|
"workerSlots",
|
|
102
100
|
"recoverExecutor"
|
|
103
101
|
]);
|
|
@@ -164,7 +162,6 @@ const superviseExecutableOptionKeys = [
|
|
|
164
162
|
"onCoordinationEvent",
|
|
165
163
|
"onDriverAttempt",
|
|
166
164
|
"onProgressStop",
|
|
167
|
-
"onWorkerRetry",
|
|
168
165
|
"recoverExecutor",
|
|
169
166
|
"resolveDeliverable",
|
|
170
167
|
"resolveDriveHarness",
|
|
@@ -208,9 +205,9 @@ function assertNoUncapturedExecutableOption(decisionData) {
|
|
|
208
205
|
* detached and frozen; executable ports are copied as the exact references selected at intake.
|
|
209
206
|
* Service internals intentionally remain live, while replacing a callback/service on the caller's
|
|
210
207
|
* mutable options object can no longer change an in-flight run. */
|
|
211
|
-
function captureSuperviseOptions(opts) {
|
|
208
|
+
function captureSuperviseOptions(opts, mode = "execution") {
|
|
212
209
|
assertSuperviseOptionKeys(opts, "supervise");
|
|
213
|
-
const { backend, coordination, driverBackend, deliverable, router, compaction, watchWorkers, analysts, continuation,
|
|
210
|
+
const { backend, coordination, driverBackend, deliverable, router, compaction, watchWorkers, analysts, continuation, ...rest } = opts;
|
|
214
211
|
const references = {};
|
|
215
212
|
const decisionData = {};
|
|
216
213
|
for (const [key, value] of Object.entries(rest)) if (!superviseReferenceOptionKeys.has(key)) decisionData[key] = value;
|
|
@@ -227,8 +224,8 @@ function captureSuperviseOptions(opts) {
|
|
|
227
224
|
...onControlReady === void 0 ? {} : { onControlReady }
|
|
228
225
|
});
|
|
229
226
|
})();
|
|
230
|
-
const capturedBackend = backend === void 0 ? void 0 : snapshotExecutorConfig(backend);
|
|
231
|
-
const capturedDriverBackend = driverBackend === void 0 ? void 0 : snapshotExecutorConfig(driverBackend);
|
|
227
|
+
const capturedBackend = backend === void 0 ? void 0 : snapshotExecutorConfig(backend, mode);
|
|
228
|
+
const capturedDriverBackend = driverBackend === void 0 ? void 0 : snapshotExecutorConfig(driverBackend, mode);
|
|
232
229
|
const capturedDeliverable = deliverable === void 0 || typeof deliverable === "string" ? deliverable : captureDeliverable(deliverable, "supervise deliverable");
|
|
233
230
|
const capturedRouter = router === void 0 ? void 0 : (() => {
|
|
234
231
|
const { complete, ...routerData } = router;
|
|
@@ -250,15 +247,6 @@ function captureSuperviseOptions(opts) {
|
|
|
250
247
|
...detachedSnapshot({ maxFindingsPerWorker: watchWorkers.maxFindingsPerWorker }, "supervise worker-watch configuration"),
|
|
251
248
|
...watchWorkers.detectors === void 0 ? {} : { detectors: Object.freeze([...watchWorkers.detectors]) }
|
|
252
249
|
});
|
|
253
|
-
const capturedWorkerRetry = workerRetry === void 0 ? void 0 : Object.freeze({
|
|
254
|
-
...detachedSnapshot({
|
|
255
|
-
...workerRetry.enabled === void 0 ? {} : { enabled: workerRetry.enabled },
|
|
256
|
-
...workerRetry.maxTotalMs === void 0 ? {} : { maxTotalMs: workerRetry.maxTotalMs },
|
|
257
|
-
...workerRetry.initialBackoffMs === void 0 ? {} : { initialBackoffMs: workerRetry.initialBackoffMs },
|
|
258
|
-
...workerRetry.maxBackoffMs === void 0 ? {} : { maxBackoffMs: workerRetry.maxBackoffMs }
|
|
259
|
-
}, "supervise worker-retry configuration"),
|
|
260
|
-
...workerRetry.additionalPreSpawnSignatures === void 0 ? {} : { additionalPreSpawnSignatures: Object.freeze([...workerRetry.additionalPreSpawnSignatures]) }
|
|
261
|
-
});
|
|
262
250
|
const capturedAnalysts = analysts === void 0 || typeof analysts === "string" ? analysts : Object.freeze({
|
|
263
251
|
kinds: detachedSnapshot(analysts.kinds, "supervise analyst kinds"),
|
|
264
252
|
run: analysts.run,
|
|
@@ -283,8 +271,7 @@ function captureSuperviseOptions(opts) {
|
|
|
283
271
|
...capturedCompaction === void 0 ? {} : { compaction: capturedCompaction },
|
|
284
272
|
...capturedWatchWorkers === void 0 ? {} : { watchWorkers: capturedWatchWorkers },
|
|
285
273
|
...capturedAnalysts === void 0 ? {} : { analysts: capturedAnalysts },
|
|
286
|
-
...capturedContinuation === void 0 ? {} : { continuation: capturedContinuation }
|
|
287
|
-
...capturedWorkerRetry === void 0 ? {} : { workerRetry: capturedWorkerRetry }
|
|
274
|
+
...capturedContinuation === void 0 ? {} : { continuation: capturedContinuation }
|
|
288
275
|
});
|
|
289
276
|
}
|
|
290
277
|
// src/runtime/supervise/otel-spans.ts
|
|
@@ -2406,7 +2393,7 @@ function ledgerAnalyzes(ledger, edge, outcome, bytes, reason, workerId) {
|
|
|
2406
2393
|
*/
|
|
2407
2394
|
function exhaustedBudgetChannels(budget, nowMs) {
|
|
2408
2395
|
const spent = [];
|
|
2409
|
-
if (budget.tokensLeft <= 0) spent.push(`tokens: ${-budget.tokensLeft} over the cap`);
|
|
2396
|
+
if (budget.tokensCapped !== false && budget.tokensLeft <= 0) spent.push(`tokens: ${-budget.tokensLeft} over the cap`);
|
|
2410
2397
|
if (budget.usdCapped && budget.usdLeft <= 0) spent.push(`usd: ${(-budget.usdLeft).toFixed(4)} over the cap`);
|
|
2411
2398
|
for (const [name, resource] of Object.entries(budget.resources ?? {})) if (!resource.known) spent.push(`resource ${name}: unmetered`);
|
|
2412
2399
|
else if (resource.remaining <= 0) spent.push(`resource ${name}: ${-resource.remaining} ${resource.unit} over the cap`);
|
|
@@ -2415,7 +2402,7 @@ function exhaustedBudgetChannels(budget, nowMs) {
|
|
|
2415
2402
|
}
|
|
2416
2403
|
/** A per-child budget may not exceed the conserved pool it is reserved from. */
|
|
2417
2404
|
function assertPerWorkerWithinPool(perWorker, pool) {
|
|
2418
|
-
const over = (child, total, field) => child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null;
|
|
2405
|
+
const over = (child, total, field) => child !== void 0 && total !== void 0 && child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null;
|
|
2419
2406
|
for (const [name, resource] of Object.entries(pool.resources ?? {})) {
|
|
2420
2407
|
const child = perWorker.resources?.[name];
|
|
2421
2408
|
if (!child) throw new ValidationError(`supervise perWorker resource ${name}: child must declare its limit`);
|
|
@@ -2438,6 +2425,7 @@ function assertPerWorkerWithinPool(perWorker, pool) {
|
|
|
2438
2425
|
* next turn is refused as budget-exhausted while the child still holds the rest. */
|
|
2439
2426
|
function assertPerWorkerAdmittable(perWorker, pool, ownerShare) {
|
|
2440
2427
|
const unreservable = (child, total, kept, field) => {
|
|
2428
|
+
if (child === void 0 || total === void 0) return null;
|
|
2441
2429
|
if (ownerShare > 0) {
|
|
2442
2430
|
const reservable = total - kept;
|
|
2443
2431
|
return child > reservable ? `supervise perWorker.${field} (${child}) exceeds the reservable budget.${field} (${reservable}: ${total} less the owner's kept ${kept} at reservationPolicy.ownerShare ${ownerShare})` : null;
|
|
@@ -2445,8 +2433,8 @@ function assertPerWorkerAdmittable(perWorker, pool, ownerShare) {
|
|
|
2445
2433
|
return child >= total ? `supervise perWorker.${field} (${child}) equals the whole budget.${field} (${total}), so the first child takes every unit and the owner has none left for its next turn` : null;
|
|
2446
2434
|
};
|
|
2447
2435
|
const unadmittable = [
|
|
2448
|
-
unreservable(perWorker.maxTokens, pool.maxTokens, Math.ceil(pool.maxTokens * ownerShare), "maxTokens"),
|
|
2449
|
-
unreservable(perWorker.maxIterations, pool.maxIterations, Math.ceil(pool.maxIterations * ownerShare), "maxIterations"),
|
|
2436
|
+
unreservable(perWorker.maxTokens, pool.maxTokens, Math.ceil((pool.maxTokens ?? 0) * ownerShare), "maxTokens"),
|
|
2437
|
+
unreservable(perWorker.maxIterations, pool.maxIterations, Math.ceil((pool.maxIterations ?? 0) * ownerShare), "maxIterations"),
|
|
2450
2438
|
perWorker.maxUsd !== void 0 && pool.maxUsd !== void 0 && ownerShare > 0 ? unreservable(perWorker.maxUsd, pool.maxUsd, pool.maxUsd * ownerShare, "maxUsd") : null
|
|
2451
2439
|
].filter((x) => x !== null);
|
|
2452
2440
|
if (unadmittable.length > 0) throw new ValidationError(`${unadmittable.join("; ")} — lower perWorker on that channel so a child fits beside the owner's own turns; otherwise children are refused or starve at spawn hours into the run.`);
|
|
@@ -2463,8 +2451,8 @@ function defaultPerWorker(budget, ownerShare, workerSlots) {
|
|
|
2463
2451
|
unit: value.unit,
|
|
2464
2452
|
limit: Math.floor(value.limit * share)
|
|
2465
2453
|
}])) },
|
|
2466
|
-
maxIterations: Math.max(
|
|
2467
|
-
maxTokens: Math.max(
|
|
2454
|
+
...budget.maxIterations === void 0 ? {} : { maxIterations: Math.max(0, Math.floor(budget.maxIterations * share)) },
|
|
2455
|
+
...budget.maxTokens === void 0 ? {} : { maxTokens: Math.max(0, Math.floor(budget.maxTokens * share)) },
|
|
2468
2456
|
...budget.maxUsd !== void 0 ? { maxUsd: budget.maxUsd * share } : {}
|
|
2469
2457
|
};
|
|
2470
2458
|
}
|
|
@@ -2720,17 +2708,13 @@ async function runBrainLoop(opts) {
|
|
|
2720
2708
|
};
|
|
2721
2709
|
}
|
|
2722
2710
|
// src/runtime/supervise/driver-budget-guards.ts
|
|
2723
|
-
/**
|
|
2724
|
-
*
|
|
2725
|
-
* in-loop budget guard the turn cap alone never provided). Checks BOTH conserved channels: tokens
|
|
2726
|
-
* (can't afford a worker) and usd (a usd-capped pool whose ceiling the driver's own metered
|
|
2727
|
-
* inference has drained — `meter` debits usd, so without this a huge-token/small-usd pool would
|
|
2728
|
-
* overspend usd up to the turn tripwire). */
|
|
2711
|
+
/** Stop when no work is in flight and a capped channel cannot fund another worker.
|
|
2712
|
+
* Owner inference also debits the pool, so a turn limit alone cannot enforce its ceilings. */
|
|
2729
2713
|
function poolStarved(scope, perWorker, preserveOwnerTurns = false) {
|
|
2730
2714
|
const b = scope.budget;
|
|
2731
2715
|
if (scope.view.inFlight > 0) return false;
|
|
2732
|
-
const tokenStarved =
|
|
2733
|
-
const iterationStarved = b.iterationsLeft <= 0;
|
|
2716
|
+
const tokenStarved = b.tokensCapped !== false && (b.tokensLeft <= 0 || !preserveOwnerTurns && perWorker.maxTokens !== void 0 && b.tokensLeft < perWorker.maxTokens);
|
|
2717
|
+
const iterationStarved = b.iterationsCapped !== false && b.iterationsLeft <= 0;
|
|
2734
2718
|
const usdStarved = b.usdCapped && (b.usdLeft <= 0 || !preserveOwnerTurns && perWorker.maxUsd !== void 0 && b.usdLeft < perWorker.maxUsd);
|
|
2735
2719
|
const resourceStarved = Object.entries(b.resources ?? {}).some(([name, value]) => !value.known || value.remaining <= 0 || !preserveOwnerTurns && value.remaining < (perWorker.resources?.[name]?.limit ?? 0));
|
|
2736
2720
|
return tokenStarved || iterationStarved || usdStarved || resourceStarved;
|
|
@@ -2751,7 +2735,7 @@ const UNMETERED_DRIVER_TURNS = 16;
|
|
|
2751
2735
|
function ownTurnsUnbounded(scope, poolTokens, turns, maxTurns) {
|
|
2752
2736
|
const b = scope.budget;
|
|
2753
2737
|
if (maxTurns !== void 0 || b.deadlineMs > 0) return false;
|
|
2754
|
-
if (poolTokens > 0 && b.tokensLeft < -poolTokens) return true;
|
|
2738
|
+
if (b.tokensCapped !== false && poolTokens > 0 && b.tokensLeft < -poolTokens) return true;
|
|
2755
2739
|
return !b.tokensKnown && !b.usdCapped && turns > UNMETERED_DRIVER_TURNS;
|
|
2756
2740
|
}
|
|
2757
2741
|
/** The absolute wall-clock deadline (when the root set one) has passed. */
|
|
@@ -4881,8 +4865,8 @@ function observeManagerControls(config, session) {
|
|
|
4881
4865
|
// src/runtime/supervise/driver-retry-loop.ts
|
|
4882
4866
|
const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3;
|
|
4883
4867
|
const DEFAULT_MAX_ATTEMPTS = Number.POSITIVE_INFINITY;
|
|
4884
|
-
const DEFAULT_INITIAL_BACKOFF_MS
|
|
4885
|
-
const DEFAULT_MAX_BACKOFF_MS
|
|
4868
|
+
const DEFAULT_INITIAL_BACKOFF_MS = 2e3;
|
|
4869
|
+
const DEFAULT_MAX_BACKOFF_MS = 3e4;
|
|
4886
4870
|
const DEFAULT_TRANSIENT_OUTAGE_MS = 9e5;
|
|
4887
4871
|
const BARREN_PAUSE_FIRST_MS = 3e5;
|
|
4888
4872
|
const BARREN_PAUSE_MAX_MS = 36e5;
|
|
@@ -4932,7 +4916,7 @@ function abortStop(signal) {
|
|
|
4932
4916
|
/** The budget's own verdict on whether another attempt may run at all. */
|
|
4933
4917
|
function budgetStop(budget, atMs) {
|
|
4934
4918
|
if (budget.deadlineMs > 0 && atMs >= budget.deadlineMs) return "deadline";
|
|
4935
|
-
return budget.tokensLeft <= 0 || budget.iterationsLeft <= 0 || budget.usdCapped && (budget.usdLeft <= 0 || budget.usdKnown === false) || Object.values(budget.resources ?? {}).some((r) => !r.known || r.remaining <= 0) ? "budget-exhausted" : void 0;
|
|
4919
|
+
return budget.tokensCapped !== false && budget.tokensLeft <= 0 || budget.iterationsCapped !== false && budget.iterationsLeft <= 0 || budget.usdCapped && (budget.usdLeft <= 0 || budget.usdKnown === false) || Object.values(budget.resources ?? {}).some((r) => !r.known || r.remaining <= 0) ? "budget-exhausted" : void 0;
|
|
4936
4920
|
}
|
|
4937
4921
|
/** The cause an admission or continuation refusal carries: it has no failure of its own. */
|
|
4938
4922
|
function admissionError(stop) {
|
|
@@ -4951,23 +4935,23 @@ function madeProgress(before, after) {
|
|
|
4951
4935
|
if (contractOf(after) === "unmet") return false;
|
|
4952
4936
|
return after.poolTokensSpent > before.poolTokensSpent || after.settledCount > before.settledCount;
|
|
4953
4937
|
}
|
|
4954
|
-
async function defaultSleep
|
|
4938
|
+
async function defaultSleep(ms, signal) {
|
|
4955
4939
|
if (ms <= 0 || signal.aborted) return;
|
|
4956
4940
|
await sleep(ms, signal, false);
|
|
4957
4941
|
}
|
|
4958
4942
|
function driverRetryLimits(run) {
|
|
4959
4943
|
const policy = run.policy ?? {};
|
|
4960
|
-
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS
|
|
4944
|
+
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
|
|
4961
4945
|
return {
|
|
4962
4946
|
now: run.now ?? Date.now,
|
|
4963
|
-
sleep: run.sleep ?? defaultSleep
|
|
4947
|
+
sleep: run.sleep ?? defaultSleep,
|
|
4964
4948
|
random: run.random ?? Math.random,
|
|
4965
4949
|
policy,
|
|
4966
4950
|
retryEnabled: policy.enabled !== false,
|
|
4967
4951
|
maxConsecutive: Math.max(0, policy.maxConsecutiveFailures ?? DEFAULT_MAX_CONSECUTIVE_FAILURES),
|
|
4968
4952
|
maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS),
|
|
4969
4953
|
initialBackoff,
|
|
4970
|
-
maxBackoff: Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS
|
|
4954
|
+
maxBackoff: Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS),
|
|
4971
4955
|
transientOutage: Math.max(0, policy.transientOutageMs ?? DEFAULT_TRANSIENT_OUTAGE_MS)
|
|
4972
4956
|
};
|
|
4973
4957
|
}
|
|
@@ -5188,6 +5172,8 @@ var DriverRetryLoop = class {
|
|
|
5188
5172
|
async pause(failure) {
|
|
5189
5173
|
const { run, limits } = this;
|
|
5190
5174
|
const { error, progressed } = failure;
|
|
5175
|
+
const unavailableSignal = upstreamUnavailableSignal(error) ?? "unavailable";
|
|
5176
|
+
const rotatingSeat = run.seatChain === true && unavailableSignal === "subscription-usage-limit";
|
|
5191
5177
|
if (progressed) this.barrenReentries = 0;
|
|
5192
5178
|
const stop = (() => {
|
|
5193
5179
|
if (run.signal.aborted) return abortStop(run.signal);
|
|
@@ -5195,9 +5181,8 @@ var DriverRetryLoop = class {
|
|
|
5195
5181
|
const byBudget = budgetStop(run.budget(), limits.now());
|
|
5196
5182
|
if (byBudget) return byBudget;
|
|
5197
5183
|
const deadline = run.budget().deadlineMs;
|
|
5198
|
-
return deadline > 0 && (reportedResetAt(error) ?? 0) >= deadline ? "deadline" : void 0;
|
|
5184
|
+
return !rotatingSeat && deadline > 0 && (reportedResetAt(error, limits.now()) ?? 0) >= deadline ? "deadline" : void 0;
|
|
5199
5185
|
})();
|
|
5200
|
-
const unavailableSignal = upstreamUnavailableSignal(error) ?? "unavailable";
|
|
5201
5186
|
if (stop !== void 0) {
|
|
5202
5187
|
await this.stopOrPause(error, stop, {
|
|
5203
5188
|
...this.failureRecord(failure),
|
|
@@ -5208,9 +5193,9 @@ var DriverRetryLoop = class {
|
|
|
5208
5193
|
this.pauses += 1;
|
|
5209
5194
|
this.consecutivePauses = progressed ? 1 : this.consecutivePauses + 1;
|
|
5210
5195
|
const pausedAt = limits.now();
|
|
5211
|
-
const resetAt = unavailableResumeAt(reportedResetAt(error), pausedAt);
|
|
5196
|
+
const resetAt = rotatingSeat ? void 0 : unavailableResumeAt(reportedResetAt(error, limits.now()), pausedAt);
|
|
5212
5197
|
const deadlineMs = run.budget().deadlineMs;
|
|
5213
|
-
const pause = resetAt === void 0 ? jitteredMs(unavailablePauseMs(this.consecutivePauses, limits.policy), limits.random) : Math.max(1e3, Math.min(resetAt, deadlineMs > 0 ? deadlineMs : resetAt) - pausedAt);
|
|
5198
|
+
const pause = rotatingSeat ? 0 : resetAt === void 0 ? jitteredMs(unavailablePauseMs(this.consecutivePauses, limits.policy), limits.random) : Math.max(1e3, Math.min(resetAt, deadlineMs > 0 ? deadlineMs : resetAt) - pausedAt);
|
|
5214
5199
|
const exitCode = undispatchedCredentialExit(error);
|
|
5215
5200
|
const resumeAt = resetAt ?? (exitCode === void 0 ? void 0 : pausedAt + pause);
|
|
5216
5201
|
await this.emit({
|
|
@@ -5269,14 +5254,14 @@ var DriverRetryLoop = class {
|
|
|
5269
5254
|
failureStop(failure, failedAt) {
|
|
5270
5255
|
const { run, limits } = this;
|
|
5271
5256
|
const { error, classification, progressed, before, after } = failure;
|
|
5272
|
-
const
|
|
5257
|
+
const kind = classification === "transient" ? transientFailureKind(error) : void 0;
|
|
5258
|
+
const idle = kind === "infra" && !progressed && after.poolTokensSpent === before.poolTokensSpent && after.settledCount === before.settledCount;
|
|
5273
5259
|
if (!idle) this.outageSince = void 0;
|
|
5274
5260
|
else this.outageSince ??= failedAt;
|
|
5275
5261
|
const inOutage = idle && failedAt - (this.outageSince ?? failedAt) < limits.transientOutage;
|
|
5276
5262
|
if (!inOutage) this.chargedFailures += 1;
|
|
5277
5263
|
const sameFailure = failure.durationMs >= INSTANT_FAILURE_MS ? void 0 : extend(this.sameFailure, error, failedAt);
|
|
5278
5264
|
this.sameFailure = sameFailure;
|
|
5279
|
-
const kind = classification === "transient" ? transientFailureKind(error) : void 0;
|
|
5280
5265
|
if (progressed || kind === void 0 || isContentFilterRefusal(error)) this.sameIdle = void 0;
|
|
5281
5266
|
else if (failure.pending === void 0) this.sameIdle = extend(this.sameIdle, error, failedAt);
|
|
5282
5267
|
const sameIdle = this.sameIdle;
|
|
@@ -5459,96 +5444,24 @@ var DriverRetryLoop = class {
|
|
|
5459
5444
|
};
|
|
5460
5445
|
// src/runtime/supervise/driver-retry.ts
|
|
5461
5446
|
/**
|
|
5462
|
-
*
|
|
5463
|
-
*
|
|
5464
|
-
* One dropped connection, one SIGKILLed harness process or one upstream 5xx used to end a run of
|
|
5465
|
-
* arbitrary length with `reason: 'driver-failed'` and tear down every live child (#741). This
|
|
5466
|
-
* module runs the driver again on the SAME scope, coordination server and live children until the
|
|
5467
|
-
* budget or the deadline says stop. It restarts no children and replays no work: the bridge
|
|
5468
|
-
* backend reattaches the harness session because the execution id is bound durably per node.
|
|
5447
|
+
* Re-enter a driver on the same scope, coordination server and live children. The harness owns
|
|
5448
|
+
* session/environment reuse; this loop owns retry, continuation, budget and deadline decisions.
|
|
5469
5449
|
*
|
|
5470
|
-
*
|
|
5471
|
-
*
|
|
5472
|
-
*
|
|
5473
|
-
*
|
|
5474
|
-
* - TRANSIENT: everything foreign (harness exit without a reason, a stream cut mid-turn, a 5xx, a
|
|
5475
|
-
* socket reset, an `AbortError` that neither the run's signal nor the manager's own stop
|
|
5476
|
-
* issued). A harness turn that RETURNED a failed outcome is the same accident reported as
|
|
5477
|
-
* a value, so the drive raises {@link HarnessTurnFailedError} and it is classified like any other.
|
|
5478
|
-
* - UNAVAILABLE: the upstream said "not now" (429, 503, 529, or the router's own codes: quota,
|
|
5479
|
-
* rate limit, overload). The driver PAUSES and re-enters; a pause consumes neither `maxAttempts`
|
|
5480
|
-
* nor the barren streak, so only the deadline, the budget and cancellation bound it. Measured
|
|
5481
|
-
* 2026-09-24 on play anomaly-referee-v3d: during the router's flash quota outage four of five
|
|
5482
|
-
* lead lanes ended `driver-failed` after 12 to 13 attempts with budget and an 8-hour deadline
|
|
5483
|
-
* almost untouched.
|
|
5450
|
+
* Runtime refusals and cancellation stay terminal. Canonical producer failures distinguish caller,
|
|
5451
|
+
* capacity, infrastructure and work; legacy transport/harness evidence remains a fallback.
|
|
5452
|
+
* Capacity pauses do not consume failure allowances. A seat-bearing drive first lets its existing
|
|
5453
|
+
* chain rotate; the chain owns its exhausted-seat wait, while the run's bounds still apply.
|
|
5484
5454
|
*
|
|
5485
|
-
*
|
|
5486
|
-
*
|
|
5487
|
-
*
|
|
5488
|
-
* progress resets that counter.
|
|
5455
|
+
* Only idle infrastructure failures earn an outage window. Repeated deterministic failures with
|
|
5456
|
+
* no progress stop inside that window; work failures consume the normal attempt/barren bounds.
|
|
5457
|
+
* With a declared check, progress means delivery or an improved verdict, not spend alone.
|
|
5489
5458
|
*
|
|
5490
|
-
*
|
|
5491
|
-
*
|
|
5492
|
-
*
|
|
5493
|
-
* (76.1%) and the runs that HAD left an artifact 21 of 399 times (5.3%). So when the caller
|
|
5494
|
-
* declares a completion check, spend and settlements alone are NOT progress while it is unmet; only
|
|
5495
|
-
* a delivery (an accepted submission, a child that passed the check, the contract turning met, or a
|
|
5496
|
-
* rise in the check's best composite) resets the barren counter. A caller that declares no check
|
|
5497
|
-
* reports `contract: 'none'` and keeps the historical reading.
|
|
5459
|
+
* Completed drives with an unmet contract continue with Runtime's note. Barren continuations
|
|
5460
|
+
* wait with increasing backoff; progress resets their streak. A resumed run first receives its
|
|
5461
|
+
* open workers, and counts barren drives only after a model event in this process.
|
|
5498
5462
|
*
|
|
5499
|
-
*
|
|
5500
|
-
*
|
|
5501
|
-
* own `stop: 'completed'`). A completed drive whose contract is unmet is now a first-class moment:
|
|
5502
|
-
* the manager's `continuation` policy (`./continuation.ts`) re-enters the driver with Runtime's
|
|
5503
|
-
* continuation note, crossing the same budget, deadline and abort bounds. Successful
|
|
5504
|
-
* continuations do not consume the failure allowance. Progress paces the continuations: once
|
|
5505
|
-
* `continuation.maxBarren` re-entered drives in a row complete without progress, the loop PAUSES
|
|
5506
|
-
* before each further continuation, 5 minutes and doubling per barren drive up to 60, journals one
|
|
5507
|
-
* `barren-pause` row per pause for an operator alarm, and re-enters. A barren streak never ends the
|
|
5508
|
-
* run: the check, the deadline, the budget and cancellation do. Progress resets the streak and the
|
|
5509
|
-
* backoff. There is no count cap and no caller veto (the old prompt-owned loop logged 896 of 1,011
|
|
5510
|
-
* rounds with no progress). The barren bound
|
|
5511
|
-
* settled a 48-hour run `no-winner` after 14.3 hours, while its last two turns updated its pages
|
|
5512
|
-
* and waited on its own 2^20 prove.
|
|
5513
|
-
*
|
|
5514
|
-
* This loop decides WHETHER to re-enter, not WHERE the next drive runs: the drive harness owns
|
|
5515
|
-
* whether it continues the same session, environment or a replacement, and the caller composes the
|
|
5516
|
-
* re-entry task from it (`./reentry.ts`).
|
|
5517
|
-
*
|
|
5518
|
-
* A RE-ENTERED run is one this process resumed after an earlier process drove it. Its first drive
|
|
5519
|
-
* is entered as `run-reentered` with the workers that were open, and a drive counts toward
|
|
5520
|
-
* `maxBarren` only if it began after the root streamed a model event in this process; a root that
|
|
5521
|
-
* speaks and then stays barren is paced as any other. Measured 2026-10-19 on
|
|
5522
|
-
* magnetar-rust-20261019f: after a host reboot the root was driven with the bare original task and
|
|
5523
|
-
* no word of its three open workers, two barren continuations followed, and the barren bound of the
|
|
5524
|
-
* time settled the run `no-result-selected`.
|
|
5525
|
-
*
|
|
5526
|
-
* A TRANSIENT failure that spent nothing, settled nothing and made no progress is an outage until
|
|
5527
|
-
* its streak has lasted `transientOutageMs`: it consumes neither `maxAttempts` nor the barren
|
|
5528
|
-
* streak, like a pause; after it the failure bounds apply. Failures met alike (ids and times
|
|
5529
|
-
* blanked) on every re-entry with no progress stop with `repeated-failure` inside the window: a
|
|
5530
|
-
* `PartialFailureError` box at the second, a `nested` manager's failed harness turn at the third,
|
|
5531
|
-
* and its other failures at the repeated-failure count and window.
|
|
5532
|
-
* A Platform restart answered
|
|
5533
|
-
* `HTTP 502 ... Platform key verification unavailable` from 03:04:34 to 03:09:41Z, and the
|
|
5534
|
-
* coordinator's twelve attempts ran out at 03:07:12Z with no token spent.
|
|
5535
|
-
*
|
|
5536
|
-
* Every wait, a failure backoff or a pause, is jittered over its upper half so drivers that one
|
|
5537
|
-
* outage cut off together do not re-enter together.
|
|
5538
|
-
*
|
|
5539
|
-
* STOP-PAUSE. A run started with `driverRetry.pauseRootStops` hands its ROOT a `stopPause`;
|
|
5540
|
-
* no other manager gets one. Terminal refusals end the run immediately; the remaining stops wait and alarm:
|
|
5541
|
-
* `no-progress`, `max-attempts`, `repeated-failure`, `pending-unresolved` and
|
|
5542
|
-
* `budget-exhausted` each emit one record with `replacesStop` and pause 5 minutes, doubling per
|
|
5543
|
-
* stop-pause in a row up to 60, never past a declared deadline and not jittered. An attempt that
|
|
5544
|
-
* made progress resets the streak. A cancel ends the pause and settles the run; a root steer ends
|
|
5545
|
-
* any pause at once, a budget pause included, because a steer is how an operator changes a live
|
|
5546
|
-
* run. Ending a pause admits nothing by itself: the next admission reads the pool again, a pool
|
|
5547
|
-
* still spent pauses again with its own row, and the steer stays queued until a turn reads it.
|
|
5548
|
-
* After the pause the per-window counters start over and the driver is re-entered: after
|
|
5549
|
-
* a failure as `driver-failure` with `pausedMs`, after an admission or continuation refusal as the
|
|
5550
|
-
* re-entry it was about to use. Runs must not fail early: over 259 runs, 8 of the 10 stops settled runs that
|
|
5551
|
-
* nobody solved or stopped, including 17 roots whose harness never streamed.
|
|
5463
|
+
* Opted-in root stop-pauses alarm and wait instead of settling recoverable stops. Cancellation
|
|
5464
|
+
* ends the wait; a steer wakes it but grants no admission. The next attempt rechecks the budget.
|
|
5552
5465
|
*/
|
|
5553
5466
|
function assertContinuationPolicy(continuation) {
|
|
5554
5467
|
if (continuation === void 0) return;
|
|
@@ -5948,6 +5861,7 @@ async function driveManagerTurns(inputs) {
|
|
|
5948
5861
|
sleep: (ms, signal) => sleepScopeWait(scope, ms, signal, "paused-root"),
|
|
5949
5862
|
...rootStopPause(config, session),
|
|
5950
5863
|
nested: (config.nodeContextSeed?.depth ?? 0) > 0,
|
|
5864
|
+
seatChain: config.driveHarness.seatChain === true && authoredSeatStages(config.profile) !== void 0,
|
|
5951
5865
|
resolvePending: (failure, failures) => abandonScopeRetainedOwnerInvocation(scope, failure, failures),
|
|
5952
5866
|
wait: {
|
|
5953
5867
|
open: () => !mcp.submittedResult() && !mcp.isStopped() && session.state.progressStopReason === void 0 && (session.controls.hasOpenWork() || session.state.nextTurnInbox.pendingAuthority() > 0),
|
|
@@ -6056,7 +5970,7 @@ function readManagerProgress(loop) {
|
|
|
6056
5970
|
const submitted = Boolean(mcp.submittedResult());
|
|
6057
5971
|
const composite = bestComposite(controls.checkReads());
|
|
6058
5972
|
return {
|
|
6059
|
-
poolTokensSpent: loop.baseTokensLeft - scope.budget.tokensLeft,
|
|
5973
|
+
poolTokensSpent: scope.budget.tokensCapped === false ? scope.budget.committedTokens ?? 0 : loop.baseTokensLeft - scope.budget.tokensLeft,
|
|
6060
5974
|
settledCount: settled.length,
|
|
6061
5975
|
submitted,
|
|
6062
5976
|
deliveredCount,
|
|
@@ -7287,7 +7201,7 @@ function leafWorker(env, seat, admitted, spawnContext) {
|
|
|
7287
7201
|
const { options, deliverable, runNamespace } = env.setup;
|
|
7288
7202
|
const { childDeliverable } = admitted;
|
|
7289
7203
|
if (childDeliverable !== deliverable && !options.backend) throw new ValidationError("supervise: resolveDeliverable selected a per-spawn deliverable but there is no backend to derive that leaf from; makeLeafAgent owns its own completion check");
|
|
7290
|
-
return withModelSubstitution((childDeliverable === deliverable ? env.makeLeaf :
|
|
7204
|
+
return withModelSubstitution((childDeliverable === deliverable ? env.makeLeaf : workerFromBackend(options.backend, childDeliverable))(admitted.authorized, Object.freeze({
|
|
7291
7205
|
...admitted.authorizedContext,
|
|
7292
7206
|
assignmentId: workerAssignmentNamespace(runNamespace, seat.parentOwnerId, spawnContext.assignmentId)
|
|
7293
7207
|
})), admitted.modelSubstitution);
|
|
@@ -7406,232 +7320,6 @@ function registerRecovery(env, makeRecursiveWorker) {
|
|
|
7406
7320
|
});
|
|
7407
7321
|
env.recoveryFactories.set(makeRecursiveWorker, factory);
|
|
7408
7322
|
}
|
|
7409
|
-
// src/runtime/supervise/worker-retry.ts
|
|
7410
|
-
/**
|
|
7411
|
-
* WORKER-SPAWN RETRY — the second chance `driverRetry` gives the root, given to a leaf.
|
|
7412
|
-
*
|
|
7413
|
-
* `driverRetry` guards the root and nothing else. A worker that dies is typed into a `down`
|
|
7414
|
-
* settlement, which is the right shape when the worker RAN. It is the wrong shape when the worker
|
|
7415
|
-
* never started, and one failure produces exactly that:
|
|
7416
|
-
*
|
|
7417
|
-
* bridgeExecutor: bridge stream error: host-executor: acquire timeout after 60000ms
|
|
7418
|
-
* (in_flight=4/4, queued=1)
|
|
7419
|
-
*
|
|
7420
|
-
* Two live cells held 2 leads and up to 8
|
|
7421
|
-
* workers on one bridge whose host executor caps live harness children at 4. A worker spawn that
|
|
7422
|
-
* queued past the bridge's single 60 s acquire deadline died there, and the run lost that worker
|
|
7423
|
-
* permanently even though a slot freed seconds later. The bridge retries nothing.
|
|
7424
|
-
*
|
|
7425
|
-
* Retrying execution is dangerous by default, so this seam is FAIL-CLOSED on two independent
|
|
7426
|
-
* proofs that the attempt did no work:
|
|
7427
|
-
*
|
|
7428
|
-
* 1. The error carries a PRE-SPAWN signature — a refusal the backend emits before the harness
|
|
7429
|
-
* child or the box exists ({@link isPreSpawnExecutorFailure}). No provider call started.
|
|
7430
|
-
* 2. The attempt yielded NO execution event. An attempt that streamed anything may have metered
|
|
7431
|
-
* usage, and a blind re-run would double-spend, so it stays fatal whatever its message says.
|
|
7432
|
-
*
|
|
7433
|
-
* Either proof missing means no retry. This is deliberately NARROWER than "is this an
|
|
7434
|
-
* infrastructure hiccup": `ECONNRESET` is a transient transport failure and is NOT retried here,
|
|
7435
|
-
* because it can arrive mid-stream after a metered call.
|
|
7436
|
-
*
|
|
7437
|
-
* Re-entry re-invokes `execute` on the SAME executor object. Nothing is rebuilt, so the executor's
|
|
7438
|
-
* runtime-owned materialization attestation, its bound session id, and its inbox are all the ones
|
|
7439
|
-
* the kernel admitted. For the bridge backend a second `execute` is a fresh session stream over the
|
|
7440
|
-
* same durable execution id, and a saturation refusal happens before that session's child spawned.
|
|
7441
|
-
*
|
|
7442
|
-
* @experimental
|
|
7443
|
-
*/
|
|
7444
|
-
/**
|
|
7445
|
-
* The backend refusals that are emitted BEFORE any child process, box, or provider request exists.
|
|
7446
|
-
*
|
|
7447
|
-
* Each is a queue admission deadline: the executor asked for a slot, waited, and was refused
|
|
7448
|
-
* without anything being started. The three names are the pools cli-bridge and the container host
|
|
7449
|
-
* admit through — the host executor, one scoped per-run host executor, and the container pool.
|
|
7450
|
-
*
|
|
7451
|
-
* The pattern is anchored to the whole phrase, including the `after <n>ms` deadline, so a message
|
|
7452
|
-
* that merely mentions a queue does not qualify.
|
|
7453
|
-
*/
|
|
7454
|
-
const PRE_SPAWN_SIGNATURES = Object.freeze([/\b(?:host-executor|scoped-host-executor|container-pool): acquire timeout after \d+ms\b/u]);
|
|
7455
|
-
const DEFAULT_MAX_TOTAL_MS = 9e5;
|
|
7456
|
-
const DEFAULT_INITIAL_BACKOFF_MS = 5e3;
|
|
7457
|
-
const DEFAULT_MAX_BACKOFF_MS = 6e4;
|
|
7458
|
-
/**
|
|
7459
|
-
* True when `error` is a backend refusal issued before anything ran.
|
|
7460
|
-
*
|
|
7461
|
-
* This answers "did this fail BEFORE any provider call", which is a strictly narrower question
|
|
7462
|
-
* than "is this an infrastructure hiccup". The broader question admits `ECONNRESET` and a bare
|
|
7463
|
-
* `fetch failed`, both of which can arrive after a metered call, and answering the broad question
|
|
7464
|
-
* where the narrow one is required is what makes a retry double-spend.
|
|
7465
|
-
*/
|
|
7466
|
-
function isPreSpawnExecutorFailure(error, additionalSignatures = []) {
|
|
7467
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
7468
|
-
for (const signature of PRE_SPAWN_SIGNATURES) if (signature.test(message)) return true;
|
|
7469
|
-
for (const signature of additionalSignatures) if (signature.test(message)) return true;
|
|
7470
|
-
return false;
|
|
7471
|
-
}
|
|
7472
|
-
/** Read the policy, refusing a number a run cannot act on rather than silently clamping it. */
|
|
7473
|
-
function resolveWorkerSpawnRetry(policy) {
|
|
7474
|
-
if (policy === void 0 || policy.enabled === false) return void 0;
|
|
7475
|
-
const maxTotalMs = policy.maxTotalMs ?? DEFAULT_MAX_TOTAL_MS;
|
|
7476
|
-
const initialBackoffMs = policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
|
|
7477
|
-
const maxBackoffMs = policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
|
|
7478
|
-
for (const [name, value] of [
|
|
7479
|
-
["maxTotalMs", maxTotalMs],
|
|
7480
|
-
["initialBackoffMs", initialBackoffMs],
|
|
7481
|
-
["maxBackoffMs", maxBackoffMs]
|
|
7482
|
-
]) if (!Number.isFinite(value) || value < 0) throw new ValidationError(`workerRetry.${name} must be a nonnegative finite number of milliseconds`);
|
|
7483
|
-
const signatures = policy.additionalPreSpawnSignatures ?? [];
|
|
7484
|
-
for (const signature of signatures) if (!(signature instanceof RegExp)) throw new ValidationError("workerRetry.additionalPreSpawnSignatures must contain RegExp values");
|
|
7485
|
-
return Object.freeze({
|
|
7486
|
-
maxTotalMs,
|
|
7487
|
-
initialBackoffMs,
|
|
7488
|
-
maxBackoffMs: Math.max(initialBackoffMs, maxBackoffMs),
|
|
7489
|
-
signatures: Object.freeze([...signatures])
|
|
7490
|
-
});
|
|
7491
|
-
}
|
|
7492
|
-
/**
|
|
7493
|
-
* Wrap a worker seam so a leaf whose spawn is refused before it runs is re-entered instead of
|
|
7494
|
-
* settling dead.
|
|
7495
|
-
*
|
|
7496
|
-
* The wrapper composes the agent's `executorFactory`, which is where a leaf executor is built with
|
|
7497
|
-
* the real child signal and node context, and it never replaces the executor object the factory
|
|
7498
|
-
* returned: the retry lives in one `execute` wrapper that carries the inner executor's attestation
|
|
7499
|
-
* forward and delegates every other surface to it.
|
|
7500
|
-
*/
|
|
7501
|
-
function withWorkerSpawnRetry(make, policy, hooks = {}) {
|
|
7502
|
-
const resolved = resolveWorkerSpawnRetry(policy);
|
|
7503
|
-
if (resolved === void 0) return make;
|
|
7504
|
-
return (profile, spawnContext) => {
|
|
7505
|
-
const agent = make(profile, spawnContext);
|
|
7506
|
-
const spec = agent.executorSpec;
|
|
7507
|
-
if (spec === void 0) return agent;
|
|
7508
|
-
const seamHooks = {
|
|
7509
|
-
...hooks,
|
|
7510
|
-
...typeof profile.name === "string" ? { worker: profile.name } : {}
|
|
7511
|
-
};
|
|
7512
|
-
if (spec.executorFactory !== void 0) {
|
|
7513
|
-
const innerFactory = spec.executorFactory;
|
|
7514
|
-
const executorFactory = (builtSpec, ctx) => retryPreSpawnRefusals(innerFactory(builtSpec, ctx), resolved, seamHooks);
|
|
7515
|
-
return {
|
|
7516
|
-
...agent,
|
|
7517
|
-
executorSpec: {
|
|
7518
|
-
...spec,
|
|
7519
|
-
executorFactory
|
|
7520
|
-
}
|
|
7521
|
-
};
|
|
7522
|
-
}
|
|
7523
|
-
if (spec.executor !== void 0) {
|
|
7524
|
-
const executor = retryPreSpawnRefusals(spec.executor, resolved, seamHooks);
|
|
7525
|
-
return {
|
|
7526
|
-
...agent,
|
|
7527
|
-
executorSpec: {
|
|
7528
|
-
...spec,
|
|
7529
|
-
executor
|
|
7530
|
-
}
|
|
7531
|
-
};
|
|
7532
|
-
}
|
|
7533
|
-
return agent;
|
|
7534
|
-
};
|
|
7535
|
-
}
|
|
7536
|
-
/**
|
|
7537
|
-
* Re-enter `execute` on one executor while a pre-spawn refusal keeps proving nothing ran.
|
|
7538
|
-
*
|
|
7539
|
-
* Exported so an owner of `makeLeafAgent` — which builds its own executors — gets the same seam
|
|
7540
|
-
* without reimplementing the two proofs.
|
|
7541
|
-
*/
|
|
7542
|
-
function retryPreSpawnRefusals(inner, policy, hooks = {}) {
|
|
7543
|
-
const now = hooks.now ?? Date.now;
|
|
7544
|
-
const sleep = hooks.sleep ?? defaultSleep;
|
|
7545
|
-
/** The wait before the next attempt, or `null` when this failure stays fatal. */
|
|
7546
|
-
const nextWait = (error, attempt, yielded, startedAt, signal) => {
|
|
7547
|
-
if (yielded) return null;
|
|
7548
|
-
if (signal?.aborted) return null;
|
|
7549
|
-
if (!isPreSpawnExecutorFailure(error, policy.signatures)) return null;
|
|
7550
|
-
const backoff = Math.min(policy.maxBackoffMs, policy.initialBackoffMs * 2 ** (attempt - 1));
|
|
7551
|
-
if (now() - startedAt + backoff >= policy.maxTotalMs) return null;
|
|
7552
|
-
return backoff;
|
|
7553
|
-
};
|
|
7554
|
-
const report = (error, attempt, waitMs) => {
|
|
7555
|
-
hooks.onRetry?.({
|
|
7556
|
-
attempt,
|
|
7557
|
-
waitMs,
|
|
7558
|
-
...hooks.worker === void 0 ? {} : { worker: hooks.worker },
|
|
7559
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7560
|
-
});
|
|
7561
|
-
};
|
|
7562
|
-
const wrapped = {
|
|
7563
|
-
runtime: inner.runtime,
|
|
7564
|
-
...inner.budgetExempt !== void 0 ? { budgetExempt: inner.budgetExempt } : {},
|
|
7565
|
-
...inner.deliver ? { deliver: (message) => inner.deliver?.(message) } : {},
|
|
7566
|
-
...inner.progress ? { progress: () => inner.progress?.() } : {},
|
|
7567
|
-
...inner.traceSource ? { traceSource: () => inner.traceSource?.() } : {},
|
|
7568
|
-
...inner.metered ? { metered: () => inner.metered?.() } : {},
|
|
7569
|
-
...inner.accounting ? { accounting: () => inner.accounting?.() } : {},
|
|
7570
|
-
...inner.resultAccepted ? { resultAccepted: () => inner.resultAccepted?.() === true } : {},
|
|
7571
|
-
...inner.harnessTranscript ? { harnessTranscript: () => inner.harnessTranscript?.() } : {},
|
|
7572
|
-
...inner.harnessTranscriptSettled ? { harnessTranscriptSettled: () => inner.harnessTranscriptSettled?.() ?? Promise.resolve() } : {},
|
|
7573
|
-
...inner.interactive === void 0 ? {} : { interactive: () => interactiveOf(inner) },
|
|
7574
|
-
execute(task, signal) {
|
|
7575
|
-
const first = inner.execute(task, signal);
|
|
7576
|
-
const restart = () => inner.execute(task, signal);
|
|
7577
|
-
return isAsyncIterable(first) ? retryStream(first, restart, signal) : retryOneShot(first, restart, signal);
|
|
7578
|
-
},
|
|
7579
|
-
...inner.recover ? { recover: (task, signal) => inner.recover(task, signal) } : {},
|
|
7580
|
-
teardown: (grace) => inner.teardown(grace),
|
|
7581
|
-
...teardownSurfaces(inner),
|
|
7582
|
-
resultArtifact: () => inner.resultArtifact()
|
|
7583
|
-
};
|
|
7584
|
-
async function* retryStream(first, restart, signal) {
|
|
7585
|
-
const startedAt = now();
|
|
7586
|
-
let execution = first;
|
|
7587
|
-
for (let attempt = 1;; attempt += 1) {
|
|
7588
|
-
let yielded = false;
|
|
7589
|
-
try {
|
|
7590
|
-
for await (const event of execution) {
|
|
7591
|
-
yielded = true;
|
|
7592
|
-
yield event;
|
|
7593
|
-
}
|
|
7594
|
-
return;
|
|
7595
|
-
} catch (error) {
|
|
7596
|
-
const waitMs = nextWait(error, attempt, yielded, startedAt, signal);
|
|
7597
|
-
if (waitMs === null) throw error;
|
|
7598
|
-
report(error, attempt, waitMs);
|
|
7599
|
-
await sleep(waitMs, signal);
|
|
7600
|
-
const next = restart();
|
|
7601
|
-
if (!isAsyncIterable(next)) throw new ValidationError("retryPreSpawnRefusals: executor changed execution shape between attempts");
|
|
7602
|
-
execution = next;
|
|
7603
|
-
}
|
|
7604
|
-
}
|
|
7605
|
-
}
|
|
7606
|
-
async function retryOneShot(first, restart, signal) {
|
|
7607
|
-
const startedAt = now();
|
|
7608
|
-
let execution = first;
|
|
7609
|
-
for (let attempt = 1;; attempt += 1) try {
|
|
7610
|
-
return await execution;
|
|
7611
|
-
} catch (error) {
|
|
7612
|
-
const waitMs = nextWait(error, attempt, false, startedAt, signal);
|
|
7613
|
-
if (waitMs === null) throw error;
|
|
7614
|
-
report(error, attempt, waitMs);
|
|
7615
|
-
await sleep(waitMs, signal);
|
|
7616
|
-
const next = restart();
|
|
7617
|
-
if (isAsyncIterable(next)) throw new ValidationError("retryPreSpawnRefusals: executor changed execution shape between attempts");
|
|
7618
|
-
execution = next;
|
|
7619
|
-
}
|
|
7620
|
-
}
|
|
7621
|
-
return inheritRuntimeOwnedExecutorAttestation(inner, wrapped);
|
|
7622
|
-
}
|
|
7623
|
-
/** Forward the inner executor's interactive contract exactly. The method is present only when the
|
|
7624
|
-
* inner executor implements it, so the reachable call always has one to delegate to. */
|
|
7625
|
-
function interactiveOf(inner) {
|
|
7626
|
-
if (inner.interactive === void 0) throw new ValidationError("retryPreSpawnRefusals: inner executor exposes no interactive session");
|
|
7627
|
-
return inner.interactive();
|
|
7628
|
-
}
|
|
7629
|
-
/** The wait stays REF'd: a worker waiting out a saturated executor is live work, and an unref'd
|
|
7630
|
-
* timer lets the process exit mid-backoff when nothing else is pending. */
|
|
7631
|
-
function defaultSleep(ms, signal) {
|
|
7632
|
-
if (ms <= 0 || signal?.aborted) return Promise.resolve();
|
|
7633
|
-
return sleep(ms, signal);
|
|
7634
|
-
}
|
|
7635
7323
|
// src/runtime/supervise/supervise-recursive-workers.ts
|
|
7636
7324
|
function createSuperviseWorkers(setup, cancelDurableRun) {
|
|
7637
7325
|
const { options, canonicalProfile, rootExecution, guidanceComposer, deliverable, analysts, journal, runId, rootOwnerId, now } = setup;
|
|
@@ -7682,15 +7370,13 @@ function createSuperviseWorkers(setup, cancelDurableRun) {
|
|
|
7682
7370
|
let makeWorkerAgent = options.makeWorkerAgent;
|
|
7683
7371
|
if (!makeWorkerAgent) {
|
|
7684
7372
|
if (!options.backend && !options.makeLeafAgent) throw new ValidationError("supervise: provide opts.backend (where workers run), opts.makeLeafAgent, or opts.makeWorkerAgent");
|
|
7685
|
-
const withRetry = (make) => withWorkerSpawnRetry(make, options.workerRetry, { ...options.onWorkerRetry ? { onRetry: options.onWorkerRetry } : {} });
|
|
7686
7373
|
makeWorkerAgent = recursiveWorkerFactoryFor({
|
|
7687
7374
|
setup,
|
|
7688
7375
|
cancelDurableRun,
|
|
7689
7376
|
graphSubtasks,
|
|
7690
7377
|
graphOverlayFor,
|
|
7691
7378
|
recoveryFactories,
|
|
7692
|
-
|
|
7693
|
-
makeLeaf: withRetry(options.makeLeafAgent ?? workerFromBackend(options.backend, deliverable)),
|
|
7379
|
+
makeLeaf: options.makeLeafAgent ?? workerFromBackend(options.backend, deliverable),
|
|
7694
7380
|
securityPolicy: options.profileSecurity ?? DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY
|
|
7695
7381
|
}, {
|
|
7696
7382
|
parent: canonicalProfile,
|
|
@@ -8435,7 +8121,7 @@ var ManagerDriveSession = class {
|
|
|
8435
8121
|
async function executeDriverAttempt(input) {
|
|
8436
8122
|
const { state, scope, retainedOwner, turnCap } = input;
|
|
8437
8123
|
if (!(scope.view.inFlight > 0)) {
|
|
8438
|
-
const spent = exhaustedBudgetChannels(scope.budget, input.now()) ?? (scope.budget.iterationsLeft <= 0 ? "iterations: none left" : void 0);
|
|
8124
|
+
const spent = exhaustedBudgetChannels(scope.budget, input.now()) ?? (scope.budget.iterationsCapped !== false && scope.budget.iterationsLeft <= 0 ? "iterations: none left" : void 0);
|
|
8439
8125
|
if (spent) throw new ValidationError(`driveHarnessFromBackend: supervisor budget exhausted (${spent})`);
|
|
8440
8126
|
}
|
|
8441
8127
|
const identity = admitDriverProfiles(input);
|
|
@@ -8684,7 +8370,7 @@ function driveHarnessFromBackend(backend, executionId, now = Date.now, maxTurns,
|
|
|
8684
8370
|
const failures = (await Promise.allSettled([manager.closeSeatSegment(reason), ...[...state.coordinationSecrets?.values() ?? []].map((secret) => secret.release())])).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
|
|
8685
8371
|
if (failures.length) throw new AggregateError(failures, "manager settlement cleanup failed");
|
|
8686
8372
|
};
|
|
8687
|
-
return attestRuntimeOwnedScopeOwner(drive, manager.ownerRuntime);
|
|
8373
|
+
return attestRuntimeOwnedScopeOwner(Object.assign(drive, seatSelection ? { seatChain: true } : {}), manager.ownerRuntime);
|
|
8688
8374
|
}
|
|
8689
8375
|
async function bindDriveRetainedOwner(manager, { scope, profile, authoredProfile }) {
|
|
8690
8376
|
const { state, activeSeat } = manager;
|
|
@@ -8985,21 +8671,34 @@ async function assertSqlRunProvider(options) {
|
|
|
8985
8671
|
if (!capabilities.retainedControl || !capabilities.streaming.turnIdempotency || !capabilities.streaming.replay || !provider.get) throw new ValidationError("supervise: SQL runContext requires retainedControl, replay, turn idempotency, and provider.get");
|
|
8986
8672
|
}
|
|
8987
8673
|
// src/runtime/supervise/supervise-run-setup.ts
|
|
8988
|
-
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
assertValidBudget(options.budget, "supervise budget");
|
|
8674
|
+
/** Canonical root identity is needed for control operations before executable admission. */
|
|
8675
|
+
function prepareSuperviseIdentity(profile, task, opts, mode = "execution") {
|
|
8676
|
+
const options = captureSuperviseOptions(opts, mode);
|
|
8992
8677
|
const guidanceComposer = profileGuidanceComposer(options.profileGuidance);
|
|
8993
8678
|
const { applyTraceTools, composeSpawnProfile } = traceToolPolicy(options, guidanceComposer);
|
|
8994
|
-
const peerAuditFor = peerAuditRegistry(options);
|
|
8995
8679
|
const registeredProfile = applyTraceTools(superviseRootProfile(profile, options.profileGuidance));
|
|
8996
8680
|
if (options.segment !== void 0) admitSegment(options.segment);
|
|
8997
8681
|
const canonicalProfile = options.segment === void 0 ? registeredProfile : withRootSegment(registeredProfile, options.segment);
|
|
8998
|
-
if (!testBrain || !isRuntimeFlowBrain(testBrain)) assertExecutableAgentProfile(canonicalProfile, "supervise root");
|
|
8999
8682
|
const canonicalTask = freezeDetached(task);
|
|
8683
|
+
return {
|
|
8684
|
+
options,
|
|
8685
|
+
guidanceComposer,
|
|
8686
|
+
applyTraceTools,
|
|
8687
|
+
composeSpawnProfile,
|
|
8688
|
+
registeredProfile,
|
|
8689
|
+
canonicalProfile,
|
|
8690
|
+
canonicalTask,
|
|
8691
|
+
rootExecution: canonicalExecution(registeredProfile, canonicalTask, options.execution, "supervise root")
|
|
8692
|
+
};
|
|
8693
|
+
}
|
|
8694
|
+
function prepareSuperviseRun(profile, task, opts, testBrain) {
|
|
8695
|
+
const { options, guidanceComposer, applyTraceTools, composeSpawnProfile, registeredProfile, canonicalProfile, canonicalTask, rootExecution } = prepareSuperviseIdentity(profile, task, opts);
|
|
8696
|
+
assertSqlRunOptions(options);
|
|
8697
|
+
assertValidBudget(options.budget, "supervise budget");
|
|
8698
|
+
const peerAuditFor = peerAuditRegistry(options);
|
|
8699
|
+
if (!testBrain || !isRuntimeFlowBrain(testBrain)) assertExecutableAgentProfile(canonicalProfile, "supervise root");
|
|
9000
8700
|
assertWorkerSeams(options);
|
|
9001
8701
|
const authorizeDownFor = downMessageAuthorizer(options);
|
|
9002
|
-
const rootExecution = canonicalExecution(registeredProfile, canonicalTask, options.execution, "supervise root");
|
|
9003
8702
|
assertRunModelsAllowed(canonicalProfile, options);
|
|
9004
8703
|
const { deliverable, finalizer, analysts, profileTable } = resolveRunRegistry(options);
|
|
9005
8704
|
assertCoordinationBinding(options.coordination);
|
|
@@ -9129,7 +8828,6 @@ function peerAuditRegistry(options) {
|
|
|
9129
8828
|
function assertWorkerSeams(options) {
|
|
9130
8829
|
if (options.makeWorkerAgent && options.authorizeSpawn) throw new ValidationError("supervise: authorizeSpawn cannot be combined with caller-owned makeWorkerAgent; use makeLeafAgent to override leaf execution inside the authorized path, or use backend-derived workers");
|
|
9131
8830
|
if (options.makeWorkerAgent && options.makeLeafAgent) throw new ValidationError("supervise: makeWorkerAgent replaces the worker path and makeLeafAgent overrides a leaf inside it; pass one");
|
|
9132
|
-
if (options.makeWorkerAgent && (options.workerRetry || options.onWorkerRetry)) throw new ValidationError("supervise: workerRetry applies only to backend-derived workers; compose withWorkerSpawnRetry onto a caller-owned makeWorkerAgent explicitly");
|
|
9133
8831
|
if (options.makeWorkerAgent && options.resolveDeliverable) throw new ValidationError("supervise: resolveDeliverable applies only to backend-derived workers; wrap a caller-owned makeWorkerAgent with its completion checks explicitly");
|
|
9134
8832
|
}
|
|
9135
8833
|
function downMessageAuthorizer(options) {
|
|
@@ -9206,7 +8904,13 @@ function managerDriverPlan(options, canonicalProfile, externalRoot) {
|
|
|
9206
8904
|
if (authoredSeatStages(canonicalProfile) && !hasCustomDriveHarness && !options.selectSeat) throw new ValidationError("supervise: AgentProfile.seats requires selectSeat before execution");
|
|
9207
8905
|
const automaticDriver = managerBackend !== void 0 && automaticDriverBackendSupported(managerBackend, options.coordination);
|
|
9208
8906
|
const spawnResourceRoot = hasCustomDriveHarness ? void 0 : rootSpawnResourceRoot(managerBackend);
|
|
9209
|
-
const spawnPreflight = composeSpawnPreflights(
|
|
8907
|
+
const spawnPreflight = composeSpawnPreflights(async (profile, context) => {
|
|
8908
|
+
const external = isExternalSupervisor(profile);
|
|
8909
|
+
const manager = declaredRuntimeToolNames(profile).length > 0 || external && authoredSeatStages(profile) !== void 0;
|
|
8910
|
+
const custom = manager ? external && hasCustomDriveHarness : options.makeLeafAgent !== void 0;
|
|
8911
|
+
const backend = manager ? external ? managerBackend : void 0 : options.backend;
|
|
8912
|
+
return modelSettingsSpawnPreflight(options.makeWorkerAgent === void 0 && !custom, backend?.backend === "provider" ? "provider" : "router")?.(profile, context);
|
|
8913
|
+
}, profileToolSpawnPreflight(options.makeWorkerAgent === void 0, options.resolveSupervisorTools !== void 0), coordinationChannelSpawnPreflight(options.makeWorkerAgent === void 0, hasCustomDriveHarness, managerBackend, options.coordination), options.backend?.backend === "bridge" ? bridgeSpawnPreflight(options.backend) : void 0, options.backend?.backend === "provider" && typeof options.backend.provider === "object" ? providerSpawnPreflight(options.backend.provider) : void 0);
|
|
9210
8914
|
const driverMaterialization = hasCustomDriveHarness ? options.driveHarnessMaterialization ?? fullProfileMaterialization : managerBackend && automaticDriver ? backendProfileMaterialization(managerBackend) : void 0;
|
|
9211
8915
|
if (externalRoot && !hasCustomDriveHarness && !automaticDriver) throw new ValidationError(`supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge, a provider with authenticated coordination.publicUrl and runtime MCP attachments, or an explicit driveHarness with reachable coordination transport`);
|
|
9212
8916
|
return {
|
|
@@ -9237,6 +8941,8 @@ function spawnProfileComposer(options, driver, childrenCanSubmit, composeSpawnPr
|
|
|
9237
8941
|
}
|
|
9238
8942
|
// src/runtime/supervise/supervise-run-lifecycle.ts
|
|
9239
8943
|
function superviseInternal(profile, task, opts, testBrain) {
|
|
8944
|
+
const cancelled = pendingCancellationRun(profile, task, opts);
|
|
8945
|
+
if (cancelled !== void 0) return cancelled;
|
|
9240
8946
|
const setup = prepareSuperviseRun(profile, task, opts, testBrain);
|
|
9241
8947
|
const { options } = setup;
|
|
9242
8948
|
const workers = createSuperviseWorkers(setup, (reason, request) => cancelDurableRun(reason, request));
|
|
@@ -9254,6 +8960,88 @@ function superviseInternal(profile, task, opts, testBrain) {
|
|
|
9254
8960
|
rootContinuation: void 0
|
|
9255
8961
|
});
|
|
9256
8962
|
}
|
|
8963
|
+
/** The aborted kernel entry owns replay and resource custody without executable admission. */
|
|
8964
|
+
function pendingCancellationRun(profile, task, opts) {
|
|
8965
|
+
const runDir = opts.runDir;
|
|
8966
|
+
if (runDir === void 0 || opts.runContext?.durability === "sql") return void 0;
|
|
8967
|
+
const request = readRunCancelRequest(runDir);
|
|
8968
|
+
if (request === void 0) return void 0;
|
|
8969
|
+
const acknowledged = readRunCancellation(runDir, request.operationId);
|
|
8970
|
+
if (acknowledged !== void 0 && acknowledged.effect !== "cancel_requested") return void 0;
|
|
8971
|
+
const { options, registeredProfile, canonicalProfile, canonicalTask, rootExecution } = prepareSuperviseIdentity(profile, task, opts, "control");
|
|
8972
|
+
const now = options.now ?? Date.now;
|
|
8973
|
+
const controller = new AbortController();
|
|
8974
|
+
applyRunCancellation(runDir, (reason, request) => {
|
|
8975
|
+
controller.abort(request === void 0 ? new Error(reason) : new RunCancellationReason(request.source, reason, request.operationId, request.operator));
|
|
8976
|
+
}, () => new Date(now()).toISOString(), "observer");
|
|
8977
|
+
const context = options.runContext ?? createFileRunContext(runDir, { withDriver: true });
|
|
8978
|
+
return (async () => {
|
|
8979
|
+
if (!controller.signal.aborted) throw new Error("run cancellation changed before control entry");
|
|
8980
|
+
const runId = options.runId ?? context.runId ?? "supervise";
|
|
8981
|
+
const journal = options.journal ?? context.journal;
|
|
8982
|
+
const prior = await journal.loadTree(runId);
|
|
8983
|
+
const root = prior?.find((event) => event.kind === "spawned" && event.id === runId);
|
|
8984
|
+
const reopened = [...prior ?? []].reverse().find((event) => event.kind === "reopened" && event.id === runId && event.budget !== void 0);
|
|
8985
|
+
const budget = options.budget ?? (reopened?.kind === "reopened" ? reopened.budget : void 0) ?? (root?.kind === "spawned" ? root.budget : void 0);
|
|
8986
|
+
if (budget === void 0) throw new Error("cancelled run has no declared or recorded budget");
|
|
8987
|
+
assertValidBudget(budget, "supervise budget");
|
|
8988
|
+
const rootAgent = {
|
|
8989
|
+
name: canonicalProfile.name ?? "supervise",
|
|
8990
|
+
async act() {
|
|
8991
|
+
throw new Error("cancelled run cannot enter execution");
|
|
8992
|
+
}
|
|
8993
|
+
};
|
|
8994
|
+
const backend = options.driverBackend ?? (options.rootDriverFromBackend === false ? void 0 : options.backend);
|
|
8995
|
+
if (backend?.backend === "provider" && !options.driveHarness && !options.resolveDriveHarness) {
|
|
8996
|
+
const provider = typeof backend.provider === "string" ? backend.registry?.get(backend.provider) : backend.provider;
|
|
8997
|
+
const priorProviders = new Set((prior ?? []).flatMap((event) => event.id === runId && event.kind === "execution-admitted" ? [event.admission.phase === "dispatched" ? event.admission.controlRef.provider : event.admission.provider] : []));
|
|
8998
|
+
if (provider !== void 0 && priorProviders.size === 1 && priorProviders.has(provider.name)) attestRuntimeOwnedScopeOwner(rootAgent, root?.kind === "spawned" ? root.runtime : provider.name, {
|
|
8999
|
+
provider,
|
|
9000
|
+
...backend.workspaceRetention === void 0 ? {} : { checkpointCapture: {
|
|
9001
|
+
port: backend.workspaceRetention,
|
|
9002
|
+
profile: providerVisibleProfile(canonicalProfile)
|
|
9003
|
+
} }
|
|
9004
|
+
});
|
|
9005
|
+
}
|
|
9006
|
+
const result = await createSupervisor().run(rootAgent, canonicalTask, {
|
|
9007
|
+
budget,
|
|
9008
|
+
runId,
|
|
9009
|
+
journal,
|
|
9010
|
+
blobs: options.blobs ?? context.blobs,
|
|
9011
|
+
executors: context.executors,
|
|
9012
|
+
rootIdentity: rootExecution.identity,
|
|
9013
|
+
rootMaterialization: {
|
|
9014
|
+
runtime: root?.kind === "spawned" ? root.runtime : "inline",
|
|
9015
|
+
declaration: "deferred",
|
|
9016
|
+
authoredProfile: registeredProfile
|
|
9017
|
+
},
|
|
9018
|
+
...options.segment === void 0 ? {} : { rootSegment: {
|
|
9019
|
+
registered: rootSegmentOf(registeredProfile),
|
|
9020
|
+
current: {
|
|
9021
|
+
...rootSegmentOf(canonicalProfile),
|
|
9022
|
+
...options.segment.stack === void 0 ? {} : { stack: { ...options.segment.stack } }
|
|
9023
|
+
},
|
|
9024
|
+
reason: options.segment.reason
|
|
9025
|
+
} },
|
|
9026
|
+
...options.recoverExecutor === void 0 ? {} : { recoverExecutor: options.recoverExecutor },
|
|
9027
|
+
...options.reservationPolicy === void 0 ? {} : { reservationPolicy: options.reservationPolicy },
|
|
9028
|
+
...options.workerSlots === void 0 ? {} : { workerSlots: options.workerSlots },
|
|
9029
|
+
...options.teardownConfirmMs === void 0 ? {} : { teardownConfirmMs: options.teardownConfirmMs },
|
|
9030
|
+
...context.resume === true || options.resume === true ? { resume: true } : {},
|
|
9031
|
+
retainedAtSettlement: options.retainedAtSettlement ?? "release",
|
|
9032
|
+
hooks: options.hooks,
|
|
9033
|
+
now,
|
|
9034
|
+
signal: options.signal === void 0 ? controller.signal : AbortSignal.any([controller.signal, options.signal]),
|
|
9035
|
+
interactiveBindingDir: runDir
|
|
9036
|
+
});
|
|
9037
|
+
recordRunCancellationOutcome(runDir, result, now);
|
|
9038
|
+
return {
|
|
9039
|
+
...result,
|
|
9040
|
+
rootProviderModel: rootProviderModelEvidence([]),
|
|
9041
|
+
...rootExecution.ref === void 0 ? {} : { execution: rootExecution.ref }
|
|
9042
|
+
};
|
|
9043
|
+
})();
|
|
9044
|
+
}
|
|
9257
9045
|
async function startSupervisedRun(run) {
|
|
9258
9046
|
const { setup, runControl, cancelDurableRun } = run;
|
|
9259
9047
|
const { options, ctx, log, runId, rootOwnerId, canonicalTask } = setup;
|
|
@@ -9492,6 +9280,6 @@ function superviseWithContext(profile, task, opts, brain) {
|
|
|
9492
9280
|
...signal === void 0 ? {} : { signal }
|
|
9493
9281
|
}, brain));
|
|
9494
9282
|
}
|
|
9495
|
-
export {
|
|
9283
|
+
export { dumbContinuationPassPrompt as A, areaUnderCurve as B, defaultEdgeTraversalCap as C, createPromptRegistry as D, analyzesFindingsReportPrompt as E, allWorkersStalled as F, plateauLength as H, anyOf as I, noProgressFor as L, kernelPromptRegistry as M, naiveContinuationPrompt as N, delegatesWorkerBriefPrompt as O, promptHandle as P, plateau as R, validateFlowGraph as S, validateGraphTopology as T, renderAnytimeTable as U, bestSoFar as V, createSupervisorSpanRecorder as W, GraphRouteError as _, superviseRootProfile as a, journaledGateDecider as b, supervisorAgentWithTestBrain as c, composeReentryTask as d, serveCoordinationMcp as f, createGraphOverlay as g, runBrainLoop as h, profileGuidanceComposer as i, formatPromptHandle as j, dumbContinuationFailPrompt as k, runToolSpans as l, driverAgent as m, superviseFleetCapabilities as n, workerFromBackend as o, coordinationControlToolNames as p, superviseWithTestBrain as r, supervisorAgent as s, supervise as t, UNPROVEN_CONTINUITY as u, createFlowRun as v, graphRootProfile as w, routeFlowOf as x, flowRootProfile as y, anytimeReport as z };
|
|
9496
9284
|
|
|
9497
|
-
//# sourceMappingURL=supervise-
|
|
9285
|
+
//# sourceMappingURL=supervise-rYEsVVcn.js.map
|