@tangle-network/agent-runtime 4.0.0 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5279 -0
- package/dist/{activation-BiQYCayd.js → activation-BRfZOfwb.js} +4 -4
- package/dist/{activation-BiQYCayd.js.map → activation-BRfZOfwb.js.map} +1 -1
- package/dist/{activation-B49A14V5.d.ts → activation-DEKw9CDD.d.ts} +2 -2
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +4 -4
- package/dist/{bundle-CV4Ts0cS.js → bundle-vFFiCigS.js} +2 -2
- package/dist/{bundle-CV4Ts0cS.js.map → bundle-vFFiCigS.js.map} +1 -1
- package/dist/candidate-execution/index.d.ts +71 -4
- package/dist/candidate-execution/index.js +4 -4
- package/dist/{coordination-manager-C-ZoeZ73.js → coordination-manager-RSJg5GmY.js} +12 -11
- package/dist/coordination-manager-RSJg5GmY.js.map +1 -0
- package/dist/{coordination-map-routing-SnsZp287.js → coordination-map-routing-DVjW5Gx0.js} +2 -2
- package/dist/coordination-map-routing-DVjW5Gx0.js.map +1 -0
- package/dist/{delegate-d9HcOFC5.js → delegate-C0cprMUN.js} +2 -2
- package/dist/{delegate-d9HcOFC5.js.map → delegate-C0cprMUN.js.map} +1 -1
- package/dist/durable.d.ts +5 -3
- package/dist/durable.js +14 -186
- package/dist/durable.js.map +1 -1
- package/dist/{graph-C_Z0Ua10.js → graph-cEg-gquv.js} +6 -8
- package/dist/{graph-C_Z0Ua10.js.map → graph-cEg-gquv.js.map} +1 -1
- package/dist/{graph-registry-CI1o16DH.js → graph-registry-B25FxBZN.js} +5 -5
- package/dist/graph-registry-B25FxBZN.js.map +1 -0
- package/dist/{graph-registry-BU7pcrUW.js → graph-registry-C-hjRx2t.js} +1 -1
- package/dist/{improve-B4v4S8jY.js → improve-DNl4_6lP.js} +8 -8
- package/dist/{improve-B4v4S8jY.js.map → improve-DNl4_6lP.js.map} +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.js +11 -11
- package/dist/intelligence.d.ts +5 -5
- package/dist/intelligence.js +8 -8
- package/dist/{jsonl-file-CvPuwM0a.js → jsonl-file-CeVYTIQJ.js} +41 -32
- package/dist/jsonl-file-CeVYTIQJ.js.map +1 -0
- package/dist/kernel.d.ts +3 -3
- package/dist/kernel.js +14 -13
- package/dist/{loop-runner-DKvA0JJr.d.ts → loop-runner-CDad0CCY.d.ts} +2 -2
- package/dist/{loop-runner-DMZkwlND.js → loop-runner-DQJjowz0.js} +2 -2
- package/dist/{loop-runner-DMZkwlND.js.map → loop-runner-DQJjowz0.js.map} +1 -1
- package/dist/loop-runner-bin.d.ts +1 -1
- package/dist/loop-runner-bin.js +1 -1
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +6 -6
- package/dist/{prepare-DutbxpWr.js → prepare-B6SQA12v.js} +3 -3
- package/dist/{prepare-DutbxpWr.js.map → prepare-B6SQA12v.js.map} +1 -1
- package/dist/{profile-ClvuI8aY.js → profile-DN8Dw2Kp.js} +2 -2
- package/dist/{profile-ClvuI8aY.js.map → profile-DN8Dw2Kp.js.map} +1 -1
- package/dist/profiles.d.ts +1 -1
- package/dist/{protected-model-port-DEsNIYi1.d.ts → protected-model-port-uN4M48vT.d.ts} +2 -2
- package/dist/{provision-supervisor-Cjs4SQ9Q.js → provision-supervisor-CtAaJCUa.js} +6 -6
- package/dist/{provision-supervisor-Cjs4SQ9Q.js.map → provision-supervisor-CtAaJCUa.js.map} +1 -1
- package/dist/{run-cancellation-CGgk54Md.js → run-cancellation-BCuu6khW.js} +3 -3
- package/dist/{run-cancellation-CGgk54Md.js.map → run-cancellation-BCuu6khW.js.map} +1 -1
- package/dist/run-state-ByHiAjUs.js +351 -0
- package/dist/run-state-ByHiAjUs.js.map +1 -0
- package/dist/{runtime-BmxmsaNz.js → runtime-9EYvBcHG.js} +19 -18
- package/dist/runtime-9EYvBcHG.js.map +1 -0
- package/dist/runtime-hooks-CsW2pfFk.js.map +1 -1
- package/dist/{runtime-hooks-ZQqPw-0g.d.ts → runtime-hooks-DPghSHV7.d.ts} +11 -16
- package/dist/{scope-CJ7xtrs8.js → scope-DzhBJrRN.js} +3632 -3528
- package/dist/scope-DzhBJrRN.js.map +1 -0
- package/dist/{search-executor-DSB6XAl1.js → search-executor-BJRNavck.js} +4 -4
- package/dist/{search-executor-DSB6XAl1.js.map → search-executor-BJRNavck.js.map} +1 -1
- package/dist/{server-FWT2Nt_w.js → server-DH0P6OY_.js} +4 -4
- package/dist/{server-FWT2Nt_w.js.map → server-DH0P6OY_.js.map} +1 -1
- package/dist/settle-record-BSDDf9Z4.js +130 -0
- package/dist/settle-record-BSDDf9Z4.js.map +1 -0
- package/dist/{structural-rollout-Bl23HSOd.js → structural-rollout-DgvJLTO0.js} +4 -4
- package/dist/{structural-rollout-Bl23HSOd.js.map → structural-rollout-DgvJLTO0.js.map} +1 -1
- package/dist/{supervise-MAzd0hOk.js → supervise-Cqhkb2Oj.js} +402 -450
- package/dist/supervise-Cqhkb2Oj.js.map +1 -0
- package/dist/{supervision-discovery-6EtvKCsU.js → supervision-discovery-DI5GIFBJ.js} +5 -130
- package/dist/supervision-discovery-DI5GIFBJ.js.map +1 -0
- package/dist/{supervisor-CwANBb9-.js → supervisor-B6sl14OR.js} +408 -1069
- package/dist/supervisor-B6sl14OR.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +13 -13
- package/dist/tool-loop.d.ts +1 -1
- package/dist/{top-web-CZP3RRhD.js → top-web-DGoXGxH8.js} +13 -9
- package/dist/{top-web-CZP3RRhD.js.map → top-web-DGoXGxH8.js.map} +1 -1
- package/dist/tui/bin.js +3 -3
- package/dist/tui/index.d.ts +2 -2
- package/dist/tui/index.js +2 -2
- package/dist/{types-ooyUaxq_.d.ts → types-niKnm6ux.d.ts} +270 -461
- package/dist/{workspace-streams-DbKiDOWA.d.ts → workspace-archive-DS9E9tUr.d.ts} +2 -69
- package/dist/{workspace-streams-8ALHaZzx.js → workspace-streams-BqNM06y4.js} +34 -9
- package/dist/workspace-streams-BqNM06y4.js.map +1 -0
- package/package.json +9 -10
- package/dist/coordination-manager-C-ZoeZ73.js.map +0 -1
- package/dist/coordination-map-routing-SnsZp287.js.map +0 -1
- package/dist/graph-registry-CI1o16DH.js.map +0 -1
- package/dist/jsonl-file-CvPuwM0a.js.map +0 -1
- package/dist/runtime-BmxmsaNz.js.map +0 -1
- package/dist/scope-CJ7xtrs8.js.map +0 -1
- package/dist/supervise-MAzd0hOk.js.map +0 -1
- package/dist/supervision-discovery-6EtvKCsU.js.map +0 -1
- package/dist/supervisor-CwANBb9-.js.map +0 -1
- package/dist/workspace-streams-8ALHaZzx.js.map +0 -1
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { $
|
|
1
|
+
import { $r as RetainedExecutionPendingError, Ai as spendFromUsageTotals, At as scopeOwnerExecutorNodeContext, Bt as scopeRetainedOwnerRestorePoint, Ct as scopeRetainedOwnerResult, Di as meterUsageEvent, Dt as recordScopeOwnerMaterialization, En as WORKER_TRACE_PROPAGATION, Er as recordRuntimeOwnedDriveHarnessProviderEvidence, Et as noteScopeOwnerPlannedExecution, Fr as runtimeOwnedPendingExecutorMaterialization, Ft as beginScopeRetainedOwnerDrive, Gt as scopeRetainedOwnerContext, Hr as isRuntimeFlowBrain, In as createOtelExporter, K as assertRecursiveReservationPolicy, Kt as scopeRetainedOwnerJournalRoot, Lr as runtimeOwnedScopeOwnerRuntime, Lt as scopeRetainedOwnerTurnEnded, Mr as runtimeOwnedExecutorExecutionBinding, Mt as coordinationProfileToolPrefix, Nr as runtimeOwnedExecutorMaterialization, Nt as providerVisibleProfile, Oi as newUsageTotals, Ot as restoreScopeOwnerAcceptedExecution, Pr as runtimeOwnedExecutorProviderEvidence, Pt as runtimeGraphMetadataKey, Rn as generateSpanId, Rt as assertRetainedOwnerCoordinationRekey, Si as turnFailurePolicy, St as scopeRetainedOwnerPriorSpend, Tr as providerAttemptEvidence, Ur as runtimeFlowAgent, Ut as bindScopeRetainedOwnerEnvironmentId, Vt as noteScopeRetainedOwnerCoordination, Wr as runtimeFlowBrain, Wt as bindScopeRetainedOwnerProvider, Xn as persistHarnessTranscript, Zn as readHarnessTranscript, _i as contentAddress, _r as attestRuntimeOwnedScopeOwner, a as readRootStream, ai as retainedExecutorSeamKey, an as executorObservationSeamKey, at as recordScopeOwnerPause, bt as consumeScopeRetainedOwnerResult, ci as cappedDoublingMs, cr as SEAT_RECHECK_MS, di as reportedResetAt, dr as authoredSeatStages, ea as recordValue, et as meterRuntimeOwnedAccounting, fi as unavailablePauseMs, fr as seatExecutionProfile, gt as scopeRetainedOwnerResourceReader, ht as releaseScopeRetainedOwnerForSeatPause, it as recordScopeOwnerDriverAttempt, j as parseWorkerToolTraceArtifact, ji as assertValidBudget, jr as runtimeOwnedDriveHarnessProviderEvidence, jt as coordinationMcpAlias, kt as scopeOwnerExecutionObservation, la as linkAbort, li as jitteredMs, lr as SeatChain, mt as prepareScopeRetainedOwnerSeatSwitch, n as ROOT_STREAM_FILE, nt as pauseScopeSeatAdmission, or as isLiveNodeStatus, pi as unavailableResumeAt, r as createRootStreamSink, ri as registerRetainedExecutorPreparation, rr as createActivityLog, rt as recordScopeOwnerBarrenPause, sa as RunCancellationReason, sr as isTerminalNodeStatus, tt as meterRuntimeOwnedProviderAttempt, ua as runAbortable, ur as SeatPauseError, vr as authoredAgentIdentity, wt as beginScopeOwnerAttempt, xi as turnFailureOf, xt as prepareScopeRetainedOwnerTask, yi as executorFailure, yt as abandonScopeRetainedOwnerInvocation, zt as reconcileScopeRetainedOwnerEnvironment } from "./scope-DzhBJrRN.js";
|
|
2
2
|
import { f as RuntimeRunStateError, i as ConfigError, m as ValidationError } from "./errors-AzSuGI23.js";
|
|
3
|
-
import {
|
|
4
|
-
import { _ as errorText, d as detachedSnapshot, h as errorHttpStatus, m as errMessage, p as causeChain } from "./jsonl-file-
|
|
5
|
-
import { $t as promptModelProfileMaterialization, At as bridgeModelRouteRefusal, B as createRouterTranscript, C as createExecutor, Ct as admitContinuationPolicy, Et as createContinuationKeeper, F as codeModeArguments, I as codeModeModelOutput, Jt as controlProfileMaterialization, Kt as assertProfileMaterialization, L as createCodeModeTools, Qt as promptControlProfileMaterialization, Rt as bridgeRuntimeAttachmentsKey, S as captureReusableExecutorConfig, St as CONTINUATIONS_DIR, Tt as checkVerdictOf, U as resolveAgentEnvironmentProvider, V as routerBrain, X as createInbox, Xt as fullProfileMaterialization, Y as INBOX_RECEIPT_KEY, Yt as defineProfileMaterializationContract, Zt as profileMaterializationAxes$1, _t as CheckUnavailableError, a as assertRunParking, c as sleepScopeWait, ct as HarnessTurnFailedError, d as runFinalizer, dt as isContentFilterRefusal, en as renderUnsupported, et as PEER_MAIL_WIRE_KEY, f as runTree, ft as summarizeDriverAttempts, g as isDriverSpec, gt as preflightPublicCoordination, h as driverExecutorFactory, ht as upstreamUnavailableSignal, i as RunParkedError, in as worktreeCliProfileMaterialization, kt as bridgeAdmissionRefusal, l as withRunParking, lt as abandonablePending, m as driverChild, mt as undispatchedCredentialExit, n as createSupervisor, nn as sandboxProfileMaterialization, o as bindScopeParkingDrain, p as createRootHandle, pt as transientFailureKind, rn as unsupportedProfileDimensions, rt as isPeerMailEnvelope, st as DriverAttemptsExhaustedError, u as bestDelivered, ut as classifyDriverFailure, vt as gateFactoryOnDeliverable, w as snapshotExecutorConfig, wt as bestComposite, x as bindReusableExecutorExecutionId, z as toolResultText, zt as bridgeStopSignalKey } from "./supervisor-
|
|
3
|
+
import { Dt as teardownExecutor, H as addSpend, J as errorMessage, Q as isAsyncIterable, St as armDeadlineTimer, Tt as isDeadlineAbortReason, W as chargedTokens, bt as DEFAULT_SUCCESSFUL_SHUTDOWN_MS, ct as unmeteredSpend, dt as zeroSpend, et as promptCacheTokenClasses, gt as withBudgetResources, ht as resourceTelemetry, it as sleep, pt as addResourceSpend, ut as withTimeout } from "./workspace-streams-BqNM06y4.js";
|
|
4
|
+
import { _ as errorText, d as detachedSnapshot, h as errorHttpStatus, m as errMessage, p as causeChain, v as defaultRedactor } from "./jsonl-file-CeVYTIQJ.js";
|
|
5
|
+
import { $t as promptModelProfileMaterialization, At as bridgeModelRouteRefusal, B as createRouterTranscript, C as createExecutor, Ct as admitContinuationPolicy, Et as createContinuationKeeper, F as codeModeArguments, I as codeModeModelOutput, Jt as controlProfileMaterialization, Kt as assertProfileMaterialization, L as createCodeModeTools, Qt as promptControlProfileMaterialization, Rt as bridgeRuntimeAttachmentsKey, S as captureReusableExecutorConfig, St as CONTINUATIONS_DIR, Tt as checkVerdictOf, U as resolveAgentEnvironmentProvider, V as routerBrain, X as createInbox, Xt as fullProfileMaterialization, Y as INBOX_RECEIPT_KEY, Yt as defineProfileMaterializationContract, Zt as profileMaterializationAxes$1, _t as CheckUnavailableError, a as assertRunParking, c as sleepScopeWait, ct as HarnessTurnFailedError, d as runFinalizer, dt as isContentFilterRefusal, en as renderUnsupported, et as PEER_MAIL_WIRE_KEY, f as runTree, ft as summarizeDriverAttempts, g as isDriverSpec, gt as preflightPublicCoordination, h as driverExecutorFactory, ht as upstreamUnavailableSignal, i as RunParkedError, in as worktreeCliProfileMaterialization, kt as bridgeAdmissionRefusal, l as withRunParking, lt as abandonablePending, m as driverChild, mt as undispatchedCredentialExit, n as createSupervisor, nn as sandboxProfileMaterialization, o as bindScopeParkingDrain, p as createRootHandle, pt as transientFailureKind, rn as unsupportedProfileDimensions, rt as isPeerMailEnvelope, st as DriverAttemptsExhaustedError, u as bestDelivered, ut as classifyDriverFailure, vt as gateFactoryOnDeliverable, w as snapshotExecutorConfig, wt as bestComposite, x as bindReusableExecutorExecutionId, z as toolResultText, zt as bridgeStopSignalKey } from "./supervisor-B6sl14OR.js";
|
|
6
6
|
import { _ as agentHarness, a as concreteProfileModel, c as profileModelExecutionSettings, l as profileProviderModel, n as assertModelAllowed, o as enforceTokenLimits, r as assertProfileModelsAllowed, s as profileBridgeWireModel, t as assertExecutableAgentProfile, v as harnessRunsAgent } from "./model-policy-JOcg3NRs.js";
|
|
7
7
|
import { n as createStdioToolServer } from "./tool-server-CZWd_Opp.js";
|
|
8
8
|
import { t as composeRuntimeHooks } from "./runtime-hooks-CsW2pfFk.js";
|
|
9
|
-
import { _ as workerAssignmentNamespace, c as writeRunCancellation, d as coordinationEventId, f as externalExecutionId, g as supervisionRunNamespace, h as rootCoordinationOwner, i as readRunCancellation, l as canonicalExecution, m as freezeDetachedProfile, p as freezeDetached, r as readRunCancelRequest, s as watchRunCancellation, t as applyRunCancellation, u as childCoordinationOwner } from "./run-cancellation-
|
|
10
|
-
import { c as createSteerAcknowledger, d as coordinationVerbNames, f as createFileRunContext, l as budgetOverspendText, m as withRunContext, n as createCoordinationToolsForManager, p as createInMemoryRunContext, s as createCancelAcknowledger, t as createCoordinationTools, u as normalizeAnalyzeOnSettle } from "./coordination-manager-
|
|
11
|
-
import { o as safeJsonText } from "./coordination-map-routing-
|
|
12
|
-
import { a as graphRegistry, c as loadGraph } from "./graph-registry-
|
|
13
|
-
import { ATTR, agentIdentityAttributes } from "@tangle-network/agent-trace-contract";
|
|
9
|
+
import { _ as workerAssignmentNamespace, c as writeRunCancellation, d as coordinationEventId, f as externalExecutionId, g as supervisionRunNamespace, h as rootCoordinationOwner, i as readRunCancellation, l as canonicalExecution, m as freezeDetachedProfile, p as freezeDetached, r as readRunCancelRequest, s as watchRunCancellation, t as applyRunCancellation, u as childCoordinationOwner } from "./run-cancellation-BCuu6khW.js";
|
|
10
|
+
import { c as createSteerAcknowledger, d as coordinationVerbNames, f as createFileRunContext, l as budgetOverspendText, m as withRunContext, n as createCoordinationToolsForManager, p as createInMemoryRunContext, s as createCancelAcknowledger, t as createCoordinationTools, u as normalizeAnalyzeOnSettle } from "./coordination-manager-RSJg5GmY.js";
|
|
11
|
+
import { o as safeJsonText } from "./coordination-map-routing-DVjW5Gx0.js";
|
|
12
|
+
import { a as graphRegistry, c as loadGraph } from "./graph-registry-B25FxBZN.js";
|
|
13
|
+
import { ATTR, LINK_KIND_ATTR, agentIdentityAttributes } from "@tangle-network/agent-trace-contract";
|
|
14
14
|
import { ProxySecretScopeSchema, ProxySecretsSchema, agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalCandidateJson, mergeProxySecretBindings, validateAgentProfileSecurity } from "@tangle-network/agent-interface";
|
|
15
15
|
import { toolSpansToTraceAnalysisStore } from "@tangle-network/agent-eval";
|
|
16
16
|
import { isRecord } from "@tangle-network/agent-core";
|
|
@@ -96,8 +96,6 @@ const superviseOptionKeySet = /* @__PURE__ */ new Set([
|
|
|
96
96
|
"stopRule",
|
|
97
97
|
"watchWorkers",
|
|
98
98
|
"continuation",
|
|
99
|
-
"workerRetry",
|
|
100
|
-
"onWorkerRetry",
|
|
101
99
|
"workerSlots",
|
|
102
100
|
"recoverExecutor"
|
|
103
101
|
]);
|
|
@@ -164,7 +162,6 @@ const superviseExecutableOptionKeys = [
|
|
|
164
162
|
"onCoordinationEvent",
|
|
165
163
|
"onDriverAttempt",
|
|
166
164
|
"onProgressStop",
|
|
167
|
-
"onWorkerRetry",
|
|
168
165
|
"recoverExecutor",
|
|
169
166
|
"resolveDeliverable",
|
|
170
167
|
"resolveDriveHarness",
|
|
@@ -208,9 +205,9 @@ function assertNoUncapturedExecutableOption(decisionData) {
|
|
|
208
205
|
* detached and frozen; executable ports are copied as the exact references selected at intake.
|
|
209
206
|
* Service internals intentionally remain live, while replacing a callback/service on the caller's
|
|
210
207
|
* mutable options object can no longer change an in-flight run. */
|
|
211
|
-
function captureSuperviseOptions(opts) {
|
|
208
|
+
function captureSuperviseOptions(opts, mode = "execution") {
|
|
212
209
|
assertSuperviseOptionKeys(opts, "supervise");
|
|
213
|
-
const { backend, coordination, driverBackend, deliverable, router, compaction, watchWorkers, analysts, continuation,
|
|
210
|
+
const { backend, coordination, driverBackend, deliverable, router, compaction, watchWorkers, analysts, continuation, ...rest } = opts;
|
|
214
211
|
const references = {};
|
|
215
212
|
const decisionData = {};
|
|
216
213
|
for (const [key, value] of Object.entries(rest)) if (!superviseReferenceOptionKeys.has(key)) decisionData[key] = value;
|
|
@@ -227,8 +224,8 @@ function captureSuperviseOptions(opts) {
|
|
|
227
224
|
...onControlReady === void 0 ? {} : { onControlReady }
|
|
228
225
|
});
|
|
229
226
|
})();
|
|
230
|
-
const capturedBackend = backend === void 0 ? void 0 : snapshotExecutorConfig(backend);
|
|
231
|
-
const capturedDriverBackend = driverBackend === void 0 ? void 0 : snapshotExecutorConfig(driverBackend);
|
|
227
|
+
const capturedBackend = backend === void 0 ? void 0 : snapshotExecutorConfig(backend, mode);
|
|
228
|
+
const capturedDriverBackend = driverBackend === void 0 ? void 0 : snapshotExecutorConfig(driverBackend, mode);
|
|
232
229
|
const capturedDeliverable = deliverable === void 0 || typeof deliverable === "string" ? deliverable : captureDeliverable(deliverable, "supervise deliverable");
|
|
233
230
|
const capturedRouter = router === void 0 ? void 0 : (() => {
|
|
234
231
|
const { complete, ...routerData } = router;
|
|
@@ -250,15 +247,6 @@ function captureSuperviseOptions(opts) {
|
|
|
250
247
|
...detachedSnapshot({ maxFindingsPerWorker: watchWorkers.maxFindingsPerWorker }, "supervise worker-watch configuration"),
|
|
251
248
|
...watchWorkers.detectors === void 0 ? {} : { detectors: Object.freeze([...watchWorkers.detectors]) }
|
|
252
249
|
});
|
|
253
|
-
const capturedWorkerRetry = workerRetry === void 0 ? void 0 : Object.freeze({
|
|
254
|
-
...detachedSnapshot({
|
|
255
|
-
...workerRetry.enabled === void 0 ? {} : { enabled: workerRetry.enabled },
|
|
256
|
-
...workerRetry.maxTotalMs === void 0 ? {} : { maxTotalMs: workerRetry.maxTotalMs },
|
|
257
|
-
...workerRetry.initialBackoffMs === void 0 ? {} : { initialBackoffMs: workerRetry.initialBackoffMs },
|
|
258
|
-
...workerRetry.maxBackoffMs === void 0 ? {} : { maxBackoffMs: workerRetry.maxBackoffMs }
|
|
259
|
-
}, "supervise worker-retry configuration"),
|
|
260
|
-
...workerRetry.additionalPreSpawnSignatures === void 0 ? {} : { additionalPreSpawnSignatures: Object.freeze([...workerRetry.additionalPreSpawnSignatures]) }
|
|
261
|
-
});
|
|
262
250
|
const capturedAnalysts = analysts === void 0 || typeof analysts === "string" ? analysts : Object.freeze({
|
|
263
251
|
kinds: detachedSnapshot(analysts.kinds, "supervise analyst kinds"),
|
|
264
252
|
run: analysts.run,
|
|
@@ -283,35 +271,11 @@ function captureSuperviseOptions(opts) {
|
|
|
283
271
|
...capturedCompaction === void 0 ? {} : { compaction: capturedCompaction },
|
|
284
272
|
...capturedWatchWorkers === void 0 ? {} : { watchWorkers: capturedWatchWorkers },
|
|
285
273
|
...capturedAnalysts === void 0 ? {} : { analysts: capturedAnalysts },
|
|
286
|
-
...capturedContinuation === void 0 ? {} : { continuation: capturedContinuation }
|
|
287
|
-
...capturedWorkerRetry === void 0 ? {} : { workerRetry: capturedWorkerRetry }
|
|
274
|
+
...capturedContinuation === void 0 ? {} : { continuation: capturedContinuation }
|
|
288
275
|
});
|
|
289
276
|
}
|
|
290
277
|
// src/runtime/supervise/otel-spans.ts
|
|
291
|
-
/**
|
|
292
|
-
* Supervisor tree → OTLP spans. OPT-IN, off by default.
|
|
293
|
-
*
|
|
294
|
-
* WHY. A supervised tree is legible only by parsing this package's own spawn journal; a span
|
|
295
|
-
* carrying `parent_span_id` IS a tree, so emitting spans makes the supervisor readable by the same
|
|
296
|
-
* viewer as every other multi-agent shape, with no per-system reader.
|
|
297
|
-
*
|
|
298
|
-
* NOT the record of truth. The spawn journal remains the sole durable ledger for replay/resume and
|
|
299
|
-
* cost; nothing here is read back, and a run whose export fails is unaffected.
|
|
300
|
-
*
|
|
301
|
-
* HOW. A pure `RuntimeHooks` observer over events `Scope` ALREADY emits — `agent.spawn` (node
|
|
302
|
-
* opened), `agent.child` (node settled), `agent.turn` (driver turn metered). It adds no event,
|
|
303
|
-
* mutates no journal, changes no result. `Scope` re-seeds the hooks into every nested scope
|
|
304
|
-
* (`makeNestedScopeSeam`), so one observer sees the WHOLE recursion.
|
|
305
|
-
*
|
|
306
|
-
* SHAPE. One span per node, opened at spawn, closed at settle, parented to its parent node's
|
|
307
|
-
* span; the run root is the trace. Driver inference rides as an `LLM` child span under the node
|
|
308
|
-
* that metered it. Attributes reuse the stack's vocabulary (`openinference.span.kind`,
|
|
309
|
-
* `agent.name`, `llm.token_count.*`, `llm.cost_usd`, `tangle.cost.usd`), consumed by
|
|
310
|
-
* `@tangle-network/agent-eval`'s `src/trace/attribute-vocabulary.ts`.
|
|
311
|
-
*
|
|
312
|
-
* UNKNOWN IS NEVER ZERO. `tokensKnown === false` / `usdKnown === false` spans OMIT the token/cost
|
|
313
|
-
* attribute and set `tangle.supervise.tokens_known` / `cost_known` to `false`.
|
|
314
|
-
*/
|
|
278
|
+
/** Best-effort supervisor spans; the spawn journal owns replay and accounting. */
|
|
315
279
|
/** OTEL status codes (`UNSET` / `OK` / `ERROR`) — the numeric wire values `OtelSpan.status` carries. */
|
|
316
280
|
const STATUS_UNSET = 0;
|
|
317
281
|
const STATUS_OK = 1;
|
|
@@ -328,6 +292,7 @@ function otelSpan(traceId, spec) {
|
|
|
328
292
|
startTimeUnixNano: msToUnixNano(spec.startMs),
|
|
329
293
|
endTimeUnixNano: msToUnixNano(Math.max(spec.startMs, spec.endMs)),
|
|
330
294
|
attributes: toOtlpAttributes(spec.attrs),
|
|
295
|
+
...spec.links ? { links: spec.links } : {},
|
|
331
296
|
status: {
|
|
332
297
|
code: spec.status,
|
|
333
298
|
...spec.message ? { message: spec.message } : {}
|
|
@@ -378,9 +343,9 @@ function settleNode(t, event) {
|
|
|
378
343
|
name: node.name,
|
|
379
344
|
startMs: node.startMs,
|
|
380
345
|
endMs: event.timestamp,
|
|
381
|
-
attrs: settledAttributes(node.attrs, event),
|
|
346
|
+
attrs: settledAttributes(node.attrs, event, t.telemetry),
|
|
382
347
|
status: down ? STATUS_ERROR : STATUS_OK,
|
|
383
|
-
message: down ? str(p.reason) ?? "child down" : void 0
|
|
348
|
+
message: down ? t.telemetry.includeEvalDetails ? truncate(str(p.reason) ?? "child down") : "child down" : void 0
|
|
384
349
|
});
|
|
385
350
|
}
|
|
386
351
|
function turnSpan(t, event) {
|
|
@@ -399,18 +364,134 @@ function turnSpan(t, event) {
|
|
|
399
364
|
};
|
|
400
365
|
if (parentId) attrs["tangle.supervise.node.id"] = parentId;
|
|
401
366
|
const p = record(event.payload);
|
|
402
|
-
assignTurnDetail(attrs, p);
|
|
367
|
+
assignTurnDetail(attrs, p, t.telemetry);
|
|
403
368
|
const spend = assignSpend(attrs, p.spend);
|
|
404
369
|
return otelSpan(t.traceId, {
|
|
405
370
|
spanId: generateSpanId(),
|
|
406
371
|
parentSpanId: parent.spanId,
|
|
407
372
|
name: "gen_ai.client.inference",
|
|
408
|
-
startMs: event.timestamp,
|
|
409
|
-
endMs: event.timestamp
|
|
373
|
+
startMs: event.timestamp - (spend?.ms ?? 0),
|
|
374
|
+
endMs: event.timestamp,
|
|
410
375
|
attrs,
|
|
411
376
|
status: STATUS_OK
|
|
412
377
|
});
|
|
413
378
|
}
|
|
379
|
+
/** Runtime-owned metadata only: prompts, arguments, results and error messages stay private. */
|
|
380
|
+
function observationSpan(t, event) {
|
|
381
|
+
const p = record(event.payload);
|
|
382
|
+
const nodeId = event.target === "agent.steer" ? str(p.nodeId) ?? t.runId : event.parentId ?? t.runId;
|
|
383
|
+
const parent = parentSpan(t, nodeId);
|
|
384
|
+
const attrs = {
|
|
385
|
+
...t.base,
|
|
386
|
+
...t.identityOf.get(nodeId) ?? {},
|
|
387
|
+
[ATTR.parentConfidence]: parent.confidence,
|
|
388
|
+
"tangle.supervise.node.id": nodeId
|
|
389
|
+
};
|
|
390
|
+
const attempt = str(p.attemptId);
|
|
391
|
+
if (attempt) attrs[ATTR.attemptId] = attempt;
|
|
392
|
+
let name = event.target;
|
|
393
|
+
let links;
|
|
394
|
+
if (event.target === "agent.model") {
|
|
395
|
+
attrs[ATTR.spanKind] = "LLM";
|
|
396
|
+
name = "gen_ai.client.inference";
|
|
397
|
+
if (attempt) t.modeledAttempts.add(`${nodeId}:${attempt}`);
|
|
398
|
+
const model = str(p.model);
|
|
399
|
+
if (model) attrs[ATTR.model] = model;
|
|
400
|
+
for (const [field, key] of [
|
|
401
|
+
["tokensIn", ATTR.inputTokens],
|
|
402
|
+
["tokensOut", ATTR.outputTokens],
|
|
403
|
+
["costUsd", ATTR.costUsd]
|
|
404
|
+
]) if (typeof p[field] === "number" && Number.isFinite(p[field])) attrs[key] = p[field];
|
|
405
|
+
} else if (event.target === "agent.tool_call") {
|
|
406
|
+
attrs[ATTR.spanKind] = "TOOL";
|
|
407
|
+
const tool = str(p.toolName);
|
|
408
|
+
if (tool) attrs[ATTR.toolName] = tool;
|
|
409
|
+
const call = str(p.toolCallId);
|
|
410
|
+
if (call) attrs["tool.call_id"] = call;
|
|
411
|
+
if (typeof p.durationKnown === "boolean") attrs["tangle.tool.duration_known"] = p.durationKnown;
|
|
412
|
+
} else {
|
|
413
|
+
attrs[ATTR.spanKind] = "CHAIN";
|
|
414
|
+
if (event.target === "agent.steer") {
|
|
415
|
+
const target = str(p.nodeId);
|
|
416
|
+
if (target) attrs["tangle.supervise.steer.node_id"] = target;
|
|
417
|
+
const senderSpan = event.parentId === void 0 ? void 0 : t.spanIdOf.get(event.parentId);
|
|
418
|
+
if (senderSpan) links = [{
|
|
419
|
+
traceId: t.traceId,
|
|
420
|
+
spanId: senderSpan,
|
|
421
|
+
attributes: toOtlpAttributes({ [LINK_KIND_ATTR]: "steered_by" })
|
|
422
|
+
}];
|
|
423
|
+
} else {
|
|
424
|
+
const control = record(p.controlRef);
|
|
425
|
+
for (const key of [
|
|
426
|
+
"provider",
|
|
427
|
+
"environmentId",
|
|
428
|
+
"executionId",
|
|
429
|
+
"sessionId"
|
|
430
|
+
]) {
|
|
431
|
+
const value = str(control[key]);
|
|
432
|
+
if (value) attrs[`tangle.execution.${key}`] = value;
|
|
433
|
+
}
|
|
434
|
+
attrs["tangle.execution.phase"] = "dispatched";
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
const failed = p.status === "error";
|
|
438
|
+
if (typeof p.status === "string") attrs["tangle.observation.status"] = p.status;
|
|
439
|
+
const errorType = str(record(p.error).type);
|
|
440
|
+
if (errorType) attrs["error.type"] = truncate(errorType);
|
|
441
|
+
return otelSpan(t.traceId, {
|
|
442
|
+
spanId: generateSpanId(),
|
|
443
|
+
parentSpanId: parent.spanId,
|
|
444
|
+
name,
|
|
445
|
+
startMs: typeof p.startedAt === "number" ? p.startedAt : event.timestamp,
|
|
446
|
+
endMs: typeof p.endedAt === "number" ? p.endedAt : event.timestamp,
|
|
447
|
+
attrs,
|
|
448
|
+
status: failed ? STATUS_ERROR : p.status === "ok" ? STATUS_OK : STATUS_UNSET,
|
|
449
|
+
...links ? { links } : {}
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
/** One acknowledged driver outcome is execution evidence, never another model call. */
|
|
453
|
+
function attemptSpan(t, event) {
|
|
454
|
+
const p = record(event.payload), attempt = record(p.record);
|
|
455
|
+
const duration = attempt.durationMs;
|
|
456
|
+
if (!Number.isSafeInteger(attempt.attempt) || Number(attempt.attempt) < 1 || typeof duration !== "number" || !Number.isFinite(duration) || duration < 0 || !Number.isFinite(event.timestamp)) return void 0;
|
|
457
|
+
const nodeId = str(p.nodeId) ?? event.parentId ?? t.runId;
|
|
458
|
+
const parent = parentSpan(t, nodeId);
|
|
459
|
+
const failure = turnFailureOf(attempt);
|
|
460
|
+
const failed = failure !== void 0 || typeof attempt.error === "string";
|
|
461
|
+
const attrs = {
|
|
462
|
+
...t.base,
|
|
463
|
+
...t.identityOf.get(nodeId) ?? {},
|
|
464
|
+
[ATTR.spanKind]: "CHAIN",
|
|
465
|
+
[ATTR.parentConfidence]: parent.confidence,
|
|
466
|
+
[ATTR.outcome]: failed ? "error" : "ok",
|
|
467
|
+
"tangle.supervise.node.id": nodeId
|
|
468
|
+
};
|
|
469
|
+
const id = str(p.attemptId);
|
|
470
|
+
if (id) attrs[ATTR.attemptId] = id;
|
|
471
|
+
if (typeof attempt.attempt === "number") attrs["tangle.driver.attempt"] = attempt.attempt;
|
|
472
|
+
for (const key of [
|
|
473
|
+
"classification",
|
|
474
|
+
"failureKind",
|
|
475
|
+
"stop"
|
|
476
|
+
]) {
|
|
477
|
+
const value = str(attempt[key]);
|
|
478
|
+
if (value) attrs[`tangle.driver.${key}`] = value;
|
|
479
|
+
}
|
|
480
|
+
if (failure) assignFailure(attrs, {
|
|
481
|
+
...turnFailurePolicy(failure),
|
|
482
|
+
providerCode: failure.code
|
|
483
|
+
});
|
|
484
|
+
if (failed) attrs["error.type"] = failure?.kind ?? "driver-attempt";
|
|
485
|
+
return otelSpan(t.traceId, {
|
|
486
|
+
spanId: generateSpanId(),
|
|
487
|
+
parentSpanId: parent.spanId,
|
|
488
|
+
name: "agent.execution",
|
|
489
|
+
startMs: event.timestamp - duration,
|
|
490
|
+
endMs: event.timestamp,
|
|
491
|
+
attrs,
|
|
492
|
+
status: failed ? STATUS_ERROR : STATUS_OK
|
|
493
|
+
});
|
|
494
|
+
}
|
|
414
495
|
/** The spans that close a run, yielded one at a time. A node still open never settled: its span is
|
|
415
496
|
* closed UNSET and explicitly marked, never dropped or reported as clean. */
|
|
416
497
|
function* closingSpans(t, opts, endMs, outcome) {
|
|
@@ -436,12 +517,12 @@ function* closingSpans(t, opts, endMs, outcome) {
|
|
|
436
517
|
startMs: t.rootStartMs,
|
|
437
518
|
endMs,
|
|
438
519
|
attrs: {
|
|
439
|
-
...rootAttrs(t.base, opts.agentName ?? "supervisor", outcome),
|
|
520
|
+
...rootAttrs(t.base, opts.agentName ?? "supervisor", outcome, opts),
|
|
440
521
|
...t.rootIdentity,
|
|
441
522
|
...opts.parentSpanId ? { [ATTR.parentConfidence]: "explicit" } : {}
|
|
442
523
|
},
|
|
443
524
|
status: rootStatus(outcome),
|
|
444
|
-
message: rootMessage(outcome)
|
|
525
|
+
message: rootMessage(outcome, opts)
|
|
445
526
|
});
|
|
446
527
|
}
|
|
447
528
|
/** Build the span recorder for one run, or `undefined` when no exporter resolves (off by default; "configured but unreachable" also costs nothing). */
|
|
@@ -455,6 +536,7 @@ function createSupervisorSpanRecorder(opts) {
|
|
|
455
536
|
const rootStartMs = now();
|
|
456
537
|
const rootIdentity = agentIdentityAttributes(opts.agentIdentity);
|
|
457
538
|
const trace = {
|
|
539
|
+
telemetry: opts,
|
|
458
540
|
runId: opts.runId,
|
|
459
541
|
traceId,
|
|
460
542
|
rootSpanId,
|
|
@@ -465,6 +547,7 @@ function createSupervisorSpanRecorder(opts) {
|
|
|
465
547
|
...opts.attributes ?? {}
|
|
466
548
|
},
|
|
467
549
|
rootIdentity,
|
|
550
|
+
modeledAttempts: /* @__PURE__ */ new Set(),
|
|
468
551
|
identityOf: /* @__PURE__ */ new Map([[opts.runId, rootIdentity]]),
|
|
469
552
|
open: /* @__PURE__ */ new Map(),
|
|
470
553
|
spanIdOf: /* @__PURE__ */ new Map([[opts.runId, rootSpanId]])
|
|
@@ -480,12 +563,20 @@ function createSupervisorSpanRecorder(opts) {
|
|
|
480
563
|
hooks: { onEvent(event) {
|
|
481
564
|
if (finished) return;
|
|
482
565
|
try {
|
|
566
|
+
if (event.target === "agent.execution" && event.phase === "event") {
|
|
567
|
+
const span = attemptSpan(trace, event);
|
|
568
|
+
if (span) emit(span);
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
483
571
|
if (event.phase !== "after") return;
|
|
484
572
|
if (event.target === "agent.spawn") openNode(trace, event);
|
|
485
573
|
else if (event.target === "agent.child") {
|
|
486
574
|
const settled = settleNode(trace, event);
|
|
487
575
|
if (settled) emit(settled);
|
|
488
|
-
} else if (event.target === "agent.turn")
|
|
576
|
+
} else if (event.target === "agent.turn") {
|
|
577
|
+
const attempt = str(record(event.payload).attemptId);
|
|
578
|
+
if (!attempt || !trace.modeledAttempts.has(`${event.parentId}:${attempt}`)) emit(turnSpan(trace, event));
|
|
579
|
+
} else if (event.target === "agent.model" || event.target === "agent.tool_call" || event.target === "agent.execution" || event.target === "agent.steer") emit(observationSpan(trace, event));
|
|
489
580
|
} catch {}
|
|
490
581
|
} },
|
|
491
582
|
traceId,
|
|
@@ -545,13 +636,14 @@ function spawnAttributes(base, event, identity) {
|
|
|
545
636
|
return attrs;
|
|
546
637
|
}
|
|
547
638
|
/** A node span's attributes at `agent.child`: the spawn-time ones plus how it settled. */
|
|
548
|
-
function settledAttributes(opened, event) {
|
|
639
|
+
function settledAttributes(opened, event, telemetry) {
|
|
549
640
|
const p = record(event.payload);
|
|
550
641
|
const status = str(p.status);
|
|
551
642
|
const attrs = {
|
|
552
643
|
...opened,
|
|
553
644
|
"tangle.supervise.node.status": status ?? "done"
|
|
554
645
|
};
|
|
646
|
+
assignFailure(attrs, p.failure);
|
|
555
647
|
if (typeof event.stepIndex === "number") attrs["tangle.supervise.node.seq"] = event.stepIndex;
|
|
556
648
|
if (typeof p.outRef === "string") attrs["tangle.supervise.node.out_ref"] = p.outRef;
|
|
557
649
|
if (typeof p.valid === "boolean") attrs["tangle.supervise.verdict.valid"] = p.valid;
|
|
@@ -559,7 +651,7 @@ function settledAttributes(opened, event) {
|
|
|
559
651
|
if (status === "down") {
|
|
560
652
|
attrs["error.type"] = p.infra === true ? "infra" : "child-down";
|
|
561
653
|
const reason = str(p.reason);
|
|
562
|
-
if (reason) attrs["error.message"] = truncate(reason);
|
|
654
|
+
if (reason && telemetry.includeEvalDetails) attrs["error.message"] = truncate(reason);
|
|
563
655
|
if (typeof p.infra === "boolean") attrs["tangle.supervise.node.infra"] = p.infra;
|
|
564
656
|
}
|
|
565
657
|
const wokeBy = str(record(p.wait).settled);
|
|
@@ -569,7 +661,7 @@ function settledAttributes(opened, event) {
|
|
|
569
661
|
}
|
|
570
662
|
/** Free-form `Scope.meter` detail: named keys get the canonical attribute, the rest ride
|
|
571
663
|
* namespaced and bounded so a caller's extra context survives without unbounded span growth. */
|
|
572
|
-
function assignTurnDetail(attrs, payload) {
|
|
664
|
+
function assignTurnDetail(attrs, payload, telemetry) {
|
|
573
665
|
for (const [key, value] of Object.entries(payload)) {
|
|
574
666
|
if (key === "spend") continue;
|
|
575
667
|
if (key === "driver" && typeof value === "string") {
|
|
@@ -581,6 +673,7 @@ function assignTurnDetail(attrs, payload) {
|
|
|
581
673
|
attrs["llm.model_name"] = value;
|
|
582
674
|
continue;
|
|
583
675
|
}
|
|
676
|
+
if (!telemetry.includeMetadata) continue;
|
|
584
677
|
if (Array.isArray(value)) {
|
|
585
678
|
const scalars = value.filter((v) => typeof v === "string" || typeof v === "number");
|
|
586
679
|
if (scalars.length > 0) attrs[`tangle.supervise.turn.${key}`] = truncate(scalars.join(","));
|
|
@@ -590,7 +683,18 @@ function assignTurnDetail(attrs, payload) {
|
|
|
590
683
|
else if (typeof value === "number" || typeof value === "boolean") attrs[`tangle.supervise.turn.${key}`] = value;
|
|
591
684
|
}
|
|
592
685
|
}
|
|
593
|
-
function
|
|
686
|
+
function assignFailure(attrs, value) {
|
|
687
|
+
const failure = record(value);
|
|
688
|
+
for (const key of [
|
|
689
|
+
"owner",
|
|
690
|
+
"code",
|
|
691
|
+
"providerCode"
|
|
692
|
+
]) {
|
|
693
|
+
const field = str(failure[key]);
|
|
694
|
+
if (field) attrs[`tangle.failure.${key}`] = field;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
function rootAttrs(base, agentName, outcome, telemetry) {
|
|
594
698
|
const attrs = {
|
|
595
699
|
...base,
|
|
596
700
|
[ATTR.spanKind]: "AGENT",
|
|
@@ -607,7 +711,8 @@ function rootAttrs(base, agentName, outcome) {
|
|
|
607
711
|
attrs["tangle.supervise.down_count"] = result.downCount;
|
|
608
712
|
if (result.reason === "driver-failed") {
|
|
609
713
|
attrs["error.type"] = result.error.name;
|
|
610
|
-
attrs
|
|
714
|
+
assignFailure(attrs, result.error.failure);
|
|
715
|
+
if (telemetry.includeEvalDetails) attrs["error.message"] = truncate(result.error.message);
|
|
611
716
|
}
|
|
612
717
|
}
|
|
613
718
|
assignSpend(attrs, result.spentTotal);
|
|
@@ -616,7 +721,7 @@ function rootAttrs(base, agentName, outcome) {
|
|
|
616
721
|
attrs["tangle.supervise.result"] = "error";
|
|
617
722
|
const err = outcome.error;
|
|
618
723
|
attrs["error.type"] = err instanceof Error ? err.name : typeof err;
|
|
619
|
-
attrs["error.message"] = truncate(err instanceof Error ? err.message : String(err));
|
|
724
|
+
if (telemetry.includeEvalDetails) attrs["error.message"] = truncate(err instanceof Error ? err.message : String(err));
|
|
620
725
|
}
|
|
621
726
|
return attrs;
|
|
622
727
|
}
|
|
@@ -625,8 +730,8 @@ function rootStatus(outcome) {
|
|
|
625
730
|
if (!outcome?.result) return STATUS_UNSET;
|
|
626
731
|
return outcome.result.kind === "winner" ? STATUS_OK : STATUS_ERROR;
|
|
627
732
|
}
|
|
628
|
-
function rootMessage(outcome) {
|
|
629
|
-
if (outcome?.error !== void 0) return truncate(outcome.error instanceof Error ? outcome.error.message : String(outcome.error));
|
|
733
|
+
function rootMessage(outcome, telemetry) {
|
|
734
|
+
if (outcome?.error !== void 0 && telemetry.includeEvalDetails) return truncate(outcome.error instanceof Error ? outcome.error.message : String(outcome.error));
|
|
630
735
|
const result = outcome?.result;
|
|
631
736
|
return result && result.kind === "no-winner" ? result.reason : void 0;
|
|
632
737
|
}
|
|
@@ -660,8 +765,16 @@ function assignBudget(attrs, value) {
|
|
|
660
765
|
if (!isRecord(value)) return;
|
|
661
766
|
const budget = value;
|
|
662
767
|
if (typeof budget.maxTokens === "number") attrs["tangle.supervise.budget.max_tokens"] = budget.maxTokens;
|
|
663
|
-
if (typeof budget.maxIterations === "number")
|
|
768
|
+
if (typeof budget.maxIterations === "number") {
|
|
769
|
+
attrs["tangle.supervise.budget.max_iterations"] = budget.maxIterations;
|
|
770
|
+
attrs[ATTR.maxIterations] = budget.maxIterations;
|
|
771
|
+
}
|
|
664
772
|
if (typeof budget.maxUsd === "number") attrs["tangle.supervise.budget.max_usd"] = budget.maxUsd;
|
|
773
|
+
if (typeof budget.deadlineMs === "number") attrs["tangle.supervise.budget.deadline_ms"] = budget.deadlineMs;
|
|
774
|
+
for (const [name, cap] of Object.entries(budget.resources ?? {})) {
|
|
775
|
+
attrs[`tangle.supervise.budget.resource.${name}.limit`] = cap.limit;
|
|
776
|
+
attrs[`tangle.supervise.budget.resource.${name}.unit`] = cap.unit;
|
|
777
|
+
}
|
|
665
778
|
}
|
|
666
779
|
/** A trace id is 32 hex chars: a valid caller-supplied one is used verbatim, anything else (and the
|
|
667
780
|
* default) is DERIVED by content address from namespace and run id, so a resumed run rejoins its trace. */
|
|
@@ -677,6 +790,7 @@ function str(value) {
|
|
|
677
790
|
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
678
791
|
}
|
|
679
792
|
function truncate(value) {
|
|
793
|
+
value = String(defaultRedactor(value));
|
|
680
794
|
return value.length > MAX_DETAIL_CHARS ? `${value.slice(0, MAX_DETAIL_CHARS)}…` : value;
|
|
681
795
|
}
|
|
682
796
|
// src/runtime/anytime.ts
|
|
@@ -2406,7 +2520,7 @@ function ledgerAnalyzes(ledger, edge, outcome, bytes, reason, workerId) {
|
|
|
2406
2520
|
*/
|
|
2407
2521
|
function exhaustedBudgetChannels(budget, nowMs) {
|
|
2408
2522
|
const spent = [];
|
|
2409
|
-
if (budget.tokensLeft <= 0) spent.push(`tokens: ${-budget.tokensLeft} over the cap`);
|
|
2523
|
+
if (budget.tokensCapped !== false && budget.tokensLeft <= 0) spent.push(`tokens: ${-budget.tokensLeft} over the cap`);
|
|
2410
2524
|
if (budget.usdCapped && budget.usdLeft <= 0) spent.push(`usd: ${(-budget.usdLeft).toFixed(4)} over the cap`);
|
|
2411
2525
|
for (const [name, resource] of Object.entries(budget.resources ?? {})) if (!resource.known) spent.push(`resource ${name}: unmetered`);
|
|
2412
2526
|
else if (resource.remaining <= 0) spent.push(`resource ${name}: ${-resource.remaining} ${resource.unit} over the cap`);
|
|
@@ -2415,7 +2529,7 @@ function exhaustedBudgetChannels(budget, nowMs) {
|
|
|
2415
2529
|
}
|
|
2416
2530
|
/** A per-child budget may not exceed the conserved pool it is reserved from. */
|
|
2417
2531
|
function assertPerWorkerWithinPool(perWorker, pool) {
|
|
2418
|
-
const over = (child, total, field) => child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null;
|
|
2532
|
+
const over = (child, total, field) => child !== void 0 && total !== void 0 && child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null;
|
|
2419
2533
|
for (const [name, resource] of Object.entries(pool.resources ?? {})) {
|
|
2420
2534
|
const child = perWorker.resources?.[name];
|
|
2421
2535
|
if (!child) throw new ValidationError(`supervise perWorker resource ${name}: child must declare its limit`);
|
|
@@ -2438,6 +2552,7 @@ function assertPerWorkerWithinPool(perWorker, pool) {
|
|
|
2438
2552
|
* next turn is refused as budget-exhausted while the child still holds the rest. */
|
|
2439
2553
|
function assertPerWorkerAdmittable(perWorker, pool, ownerShare) {
|
|
2440
2554
|
const unreservable = (child, total, kept, field) => {
|
|
2555
|
+
if (child === void 0 || total === void 0) return null;
|
|
2441
2556
|
if (ownerShare > 0) {
|
|
2442
2557
|
const reservable = total - kept;
|
|
2443
2558
|
return child > reservable ? `supervise perWorker.${field} (${child}) exceeds the reservable budget.${field} (${reservable}: ${total} less the owner's kept ${kept} at reservationPolicy.ownerShare ${ownerShare})` : null;
|
|
@@ -2445,8 +2560,8 @@ function assertPerWorkerAdmittable(perWorker, pool, ownerShare) {
|
|
|
2445
2560
|
return child >= total ? `supervise perWorker.${field} (${child}) equals the whole budget.${field} (${total}), so the first child takes every unit and the owner has none left for its next turn` : null;
|
|
2446
2561
|
};
|
|
2447
2562
|
const unadmittable = [
|
|
2448
|
-
unreservable(perWorker.maxTokens, pool.maxTokens, Math.ceil(pool.maxTokens * ownerShare), "maxTokens"),
|
|
2449
|
-
unreservable(perWorker.maxIterations, pool.maxIterations, Math.ceil(pool.maxIterations * ownerShare), "maxIterations"),
|
|
2563
|
+
unreservable(perWorker.maxTokens, pool.maxTokens, Math.ceil((pool.maxTokens ?? 0) * ownerShare), "maxTokens"),
|
|
2564
|
+
unreservable(perWorker.maxIterations, pool.maxIterations, Math.ceil((pool.maxIterations ?? 0) * ownerShare), "maxIterations"),
|
|
2450
2565
|
perWorker.maxUsd !== void 0 && pool.maxUsd !== void 0 && ownerShare > 0 ? unreservable(perWorker.maxUsd, pool.maxUsd, pool.maxUsd * ownerShare, "maxUsd") : null
|
|
2451
2566
|
].filter((x) => x !== null);
|
|
2452
2567
|
if (unadmittable.length > 0) throw new ValidationError(`${unadmittable.join("; ")} — lower perWorker on that channel so a child fits beside the owner's own turns; otherwise children are refused or starve at spawn hours into the run.`);
|
|
@@ -2463,8 +2578,8 @@ function defaultPerWorker(budget, ownerShare, workerSlots) {
|
|
|
2463
2578
|
unit: value.unit,
|
|
2464
2579
|
limit: Math.floor(value.limit * share)
|
|
2465
2580
|
}])) },
|
|
2466
|
-
maxIterations: Math.max(
|
|
2467
|
-
maxTokens: Math.max(
|
|
2581
|
+
...budget.maxIterations === void 0 ? {} : { maxIterations: Math.max(0, Math.floor(budget.maxIterations * share)) },
|
|
2582
|
+
...budget.maxTokens === void 0 ? {} : { maxTokens: Math.max(0, Math.floor(budget.maxTokens * share)) },
|
|
2468
2583
|
...budget.maxUsd !== void 0 ? { maxUsd: budget.maxUsd * share } : {}
|
|
2469
2584
|
};
|
|
2470
2585
|
}
|
|
@@ -2720,17 +2835,13 @@ async function runBrainLoop(opts) {
|
|
|
2720
2835
|
};
|
|
2721
2836
|
}
|
|
2722
2837
|
// src/runtime/supervise/driver-budget-guards.ts
|
|
2723
|
-
/**
|
|
2724
|
-
*
|
|
2725
|
-
* in-loop budget guard the turn cap alone never provided). Checks BOTH conserved channels: tokens
|
|
2726
|
-
* (can't afford a worker) and usd (a usd-capped pool whose ceiling the driver's own metered
|
|
2727
|
-
* inference has drained — `meter` debits usd, so without this a huge-token/small-usd pool would
|
|
2728
|
-
* overspend usd up to the turn tripwire). */
|
|
2838
|
+
/** Stop when no work is in flight and a capped channel cannot fund another worker.
|
|
2839
|
+
* Owner inference also debits the pool, so a turn limit alone cannot enforce its ceilings. */
|
|
2729
2840
|
function poolStarved(scope, perWorker, preserveOwnerTurns = false) {
|
|
2730
2841
|
const b = scope.budget;
|
|
2731
2842
|
if (scope.view.inFlight > 0) return false;
|
|
2732
|
-
const tokenStarved =
|
|
2733
|
-
const iterationStarved = b.iterationsLeft <= 0;
|
|
2843
|
+
const tokenStarved = b.tokensCapped !== false && (b.tokensLeft <= 0 || !preserveOwnerTurns && perWorker.maxTokens !== void 0 && b.tokensLeft < perWorker.maxTokens);
|
|
2844
|
+
const iterationStarved = b.iterationsCapped !== false && b.iterationsLeft <= 0;
|
|
2734
2845
|
const usdStarved = b.usdCapped && (b.usdLeft <= 0 || !preserveOwnerTurns && perWorker.maxUsd !== void 0 && b.usdLeft < perWorker.maxUsd);
|
|
2735
2846
|
const resourceStarved = Object.entries(b.resources ?? {}).some(([name, value]) => !value.known || value.remaining <= 0 || !preserveOwnerTurns && value.remaining < (perWorker.resources?.[name]?.limit ?? 0));
|
|
2736
2847
|
return tokenStarved || iterationStarved || usdStarved || resourceStarved;
|
|
@@ -2751,7 +2862,7 @@ const UNMETERED_DRIVER_TURNS = 16;
|
|
|
2751
2862
|
function ownTurnsUnbounded(scope, poolTokens, turns, maxTurns) {
|
|
2752
2863
|
const b = scope.budget;
|
|
2753
2864
|
if (maxTurns !== void 0 || b.deadlineMs > 0) return false;
|
|
2754
|
-
if (poolTokens > 0 && b.tokensLeft < -poolTokens) return true;
|
|
2865
|
+
if (b.tokensCapped !== false && poolTokens > 0 && b.tokensLeft < -poolTokens) return true;
|
|
2755
2866
|
return !b.tokensKnown && !b.usdCapped && turns > UNMETERED_DRIVER_TURNS;
|
|
2756
2867
|
}
|
|
2757
2868
|
/** The absolute wall-clock deadline (when the root set one) has passed. */
|
|
@@ -4881,8 +4992,8 @@ function observeManagerControls(config, session) {
|
|
|
4881
4992
|
// src/runtime/supervise/driver-retry-loop.ts
|
|
4882
4993
|
const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3;
|
|
4883
4994
|
const DEFAULT_MAX_ATTEMPTS = Number.POSITIVE_INFINITY;
|
|
4884
|
-
const DEFAULT_INITIAL_BACKOFF_MS
|
|
4885
|
-
const DEFAULT_MAX_BACKOFF_MS
|
|
4995
|
+
const DEFAULT_INITIAL_BACKOFF_MS = 2e3;
|
|
4996
|
+
const DEFAULT_MAX_BACKOFF_MS = 3e4;
|
|
4886
4997
|
const DEFAULT_TRANSIENT_OUTAGE_MS = 9e5;
|
|
4887
4998
|
const BARREN_PAUSE_FIRST_MS = 3e5;
|
|
4888
4999
|
const BARREN_PAUSE_MAX_MS = 36e5;
|
|
@@ -4932,7 +5043,7 @@ function abortStop(signal) {
|
|
|
4932
5043
|
/** The budget's own verdict on whether another attempt may run at all. */
|
|
4933
5044
|
function budgetStop(budget, atMs) {
|
|
4934
5045
|
if (budget.deadlineMs > 0 && atMs >= budget.deadlineMs) return "deadline";
|
|
4935
|
-
return budget.tokensLeft <= 0 || budget.iterationsLeft <= 0 || budget.usdCapped && (budget.usdLeft <= 0 || budget.usdKnown === false) || Object.values(budget.resources ?? {}).some((r) => !r.known || r.remaining <= 0) ? "budget-exhausted" : void 0;
|
|
5046
|
+
return budget.tokensCapped !== false && budget.tokensLeft <= 0 || budget.iterationsCapped !== false && budget.iterationsLeft <= 0 || budget.usdCapped && (budget.usdLeft <= 0 || budget.usdKnown === false) || Object.values(budget.resources ?? {}).some((r) => !r.known || r.remaining <= 0) ? "budget-exhausted" : void 0;
|
|
4936
5047
|
}
|
|
4937
5048
|
/** The cause an admission or continuation refusal carries: it has no failure of its own. */
|
|
4938
5049
|
function admissionError(stop) {
|
|
@@ -4951,23 +5062,23 @@ function madeProgress(before, after) {
|
|
|
4951
5062
|
if (contractOf(after) === "unmet") return false;
|
|
4952
5063
|
return after.poolTokensSpent > before.poolTokensSpent || after.settledCount > before.settledCount;
|
|
4953
5064
|
}
|
|
4954
|
-
async function defaultSleep
|
|
5065
|
+
async function defaultSleep(ms, signal) {
|
|
4955
5066
|
if (ms <= 0 || signal.aborted) return;
|
|
4956
5067
|
await sleep(ms, signal, false);
|
|
4957
5068
|
}
|
|
4958
5069
|
function driverRetryLimits(run) {
|
|
4959
5070
|
const policy = run.policy ?? {};
|
|
4960
|
-
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS
|
|
5071
|
+
const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
|
|
4961
5072
|
return {
|
|
4962
5073
|
now: run.now ?? Date.now,
|
|
4963
|
-
sleep: run.sleep ?? defaultSleep
|
|
5074
|
+
sleep: run.sleep ?? defaultSleep,
|
|
4964
5075
|
random: run.random ?? Math.random,
|
|
4965
5076
|
policy,
|
|
4966
5077
|
retryEnabled: policy.enabled !== false,
|
|
4967
5078
|
maxConsecutive: Math.max(0, policy.maxConsecutiveFailures ?? DEFAULT_MAX_CONSECUTIVE_FAILURES),
|
|
4968
5079
|
maxAttempts: Math.max(1, policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS),
|
|
4969
5080
|
initialBackoff,
|
|
4970
|
-
maxBackoff: Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS
|
|
5081
|
+
maxBackoff: Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS),
|
|
4971
5082
|
transientOutage: Math.max(0, policy.transientOutageMs ?? DEFAULT_TRANSIENT_OUTAGE_MS)
|
|
4972
5083
|
};
|
|
4973
5084
|
}
|
|
@@ -4976,7 +5087,7 @@ function driverRetryLimits(run) {
|
|
|
4976
5087
|
function extend(streak, error, at) {
|
|
4977
5088
|
const chain = causeChain(error);
|
|
4978
5089
|
const root = chain.find((cause) => !(cause instanceof RetainedExecutionPendingError)) ?? error;
|
|
4979
|
-
const code = chain.find((cause) => cause instanceof HarnessTurnFailedError)?.errorCode;
|
|
5090
|
+
const code = turnFailureOf(error)?.code ?? chain.find((cause) => cause instanceof HarnessTurnFailedError)?.errorCode;
|
|
4980
5091
|
const text = (code ? errMessage(root).replace(` (${code})`, "") : errMessage(root)).replace(/\d{4}-\d\d-\d\dT[\d:.]+Z?/gu, "<time>").replace(/\b(?=[\da-f-]*\d)[\da-f]{8,}(?:-[\da-f]{4,})*\b/giu, "<id>").replace(/\d{6,}/gu, "<n>");
|
|
4981
5092
|
if (streak?.error !== text || (streak.code ?? code) !== (code ?? streak.code)) return {
|
|
4982
5093
|
error: text,
|
|
@@ -5130,10 +5241,12 @@ var DriverRetryLoop = class {
|
|
|
5130
5241
|
};
|
|
5131
5242
|
}
|
|
5132
5243
|
failureRecord(failure) {
|
|
5244
|
+
const producerFailure = turnFailureOf(failure.error);
|
|
5133
5245
|
return {
|
|
5134
5246
|
attempt: failure.attempt,
|
|
5135
5247
|
durationMs: failure.durationMs,
|
|
5136
5248
|
error: errMessage(failure.error),
|
|
5249
|
+
...producerFailure === void 0 ? {} : { failure: producerFailure },
|
|
5137
5250
|
classification: failure.classification,
|
|
5138
5251
|
...failure.classification === "transient" ? { failureKind: transientFailureKind(failure.error) } : {},
|
|
5139
5252
|
madeProgress: failure.progressed
|
|
@@ -5188,6 +5301,8 @@ var DriverRetryLoop = class {
|
|
|
5188
5301
|
async pause(failure) {
|
|
5189
5302
|
const { run, limits } = this;
|
|
5190
5303
|
const { error, progressed } = failure;
|
|
5304
|
+
const unavailableSignal = upstreamUnavailableSignal(error) ?? "unavailable";
|
|
5305
|
+
const rotatingSeat = run.seatChain === true && unavailableSignal === "subscription-usage-limit";
|
|
5191
5306
|
if (progressed) this.barrenReentries = 0;
|
|
5192
5307
|
const stop = (() => {
|
|
5193
5308
|
if (run.signal.aborted) return abortStop(run.signal);
|
|
@@ -5195,9 +5310,8 @@ var DriverRetryLoop = class {
|
|
|
5195
5310
|
const byBudget = budgetStop(run.budget(), limits.now());
|
|
5196
5311
|
if (byBudget) return byBudget;
|
|
5197
5312
|
const deadline = run.budget().deadlineMs;
|
|
5198
|
-
return deadline > 0 && (reportedResetAt(error) ?? 0) >= deadline ? "deadline" : void 0;
|
|
5313
|
+
return !rotatingSeat && deadline > 0 && (reportedResetAt(error, limits.now()) ?? 0) >= deadline ? "deadline" : void 0;
|
|
5199
5314
|
})();
|
|
5200
|
-
const unavailableSignal = upstreamUnavailableSignal(error) ?? "unavailable";
|
|
5201
5315
|
if (stop !== void 0) {
|
|
5202
5316
|
await this.stopOrPause(error, stop, {
|
|
5203
5317
|
...this.failureRecord(failure),
|
|
@@ -5208,9 +5322,9 @@ var DriverRetryLoop = class {
|
|
|
5208
5322
|
this.pauses += 1;
|
|
5209
5323
|
this.consecutivePauses = progressed ? 1 : this.consecutivePauses + 1;
|
|
5210
5324
|
const pausedAt = limits.now();
|
|
5211
|
-
const resetAt = unavailableResumeAt(reportedResetAt(error), pausedAt);
|
|
5325
|
+
const resetAt = rotatingSeat ? void 0 : unavailableResumeAt(reportedResetAt(error, limits.now()), pausedAt);
|
|
5212
5326
|
const deadlineMs = run.budget().deadlineMs;
|
|
5213
|
-
const pause = resetAt === void 0 ? jitteredMs(unavailablePauseMs(this.consecutivePauses, limits.policy), limits.random) : Math.max(1e3, Math.min(resetAt, deadlineMs > 0 ? deadlineMs : resetAt) - pausedAt);
|
|
5327
|
+
const pause = rotatingSeat ? 0 : resetAt === void 0 ? jitteredMs(unavailablePauseMs(this.consecutivePauses, limits.policy), limits.random) : Math.max(1e3, Math.min(resetAt, deadlineMs > 0 ? deadlineMs : resetAt) - pausedAt);
|
|
5214
5328
|
const exitCode = undispatchedCredentialExit(error);
|
|
5215
5329
|
const resumeAt = resetAt ?? (exitCode === void 0 ? void 0 : pausedAt + pause);
|
|
5216
5330
|
await this.emit({
|
|
@@ -5269,14 +5383,14 @@ var DriverRetryLoop = class {
|
|
|
5269
5383
|
failureStop(failure, failedAt) {
|
|
5270
5384
|
const { run, limits } = this;
|
|
5271
5385
|
const { error, classification, progressed, before, after } = failure;
|
|
5272
|
-
const
|
|
5386
|
+
const kind = classification === "transient" ? transientFailureKind(error) : void 0;
|
|
5387
|
+
const idle = kind === "infra" && !progressed && after.poolTokensSpent === before.poolTokensSpent && after.settledCount === before.settledCount;
|
|
5273
5388
|
if (!idle) this.outageSince = void 0;
|
|
5274
5389
|
else this.outageSince ??= failedAt;
|
|
5275
5390
|
const inOutage = idle && failedAt - (this.outageSince ?? failedAt) < limits.transientOutage;
|
|
5276
5391
|
if (!inOutage) this.chargedFailures += 1;
|
|
5277
5392
|
const sameFailure = failure.durationMs >= INSTANT_FAILURE_MS ? void 0 : extend(this.sameFailure, error, failedAt);
|
|
5278
5393
|
this.sameFailure = sameFailure;
|
|
5279
|
-
const kind = classification === "transient" ? transientFailureKind(error) : void 0;
|
|
5280
5394
|
if (progressed || kind === void 0 || isContentFilterRefusal(error)) this.sameIdle = void 0;
|
|
5281
5395
|
else if (failure.pending === void 0) this.sameIdle = extend(this.sameIdle, error, failedAt);
|
|
5282
5396
|
const sameIdle = this.sameIdle;
|
|
@@ -5374,10 +5488,10 @@ var DriverRetryLoop = class {
|
|
|
5374
5488
|
this.refusals = 0;
|
|
5375
5489
|
this.abandonments = 0;
|
|
5376
5490
|
const durationMs = this.limits.now() - startedAt;
|
|
5377
|
-
|
|
5491
|
+
let after = run.progress();
|
|
5378
5492
|
const progressed = madeProgress(before, after);
|
|
5379
|
-
|
|
5380
|
-
|
|
5493
|
+
let contract = contractOf(after);
|
|
5494
|
+
let completed = {
|
|
5381
5495
|
attempt,
|
|
5382
5496
|
durationMs,
|
|
5383
5497
|
madeProgress: progressed,
|
|
@@ -5388,7 +5502,18 @@ var DriverRetryLoop = class {
|
|
|
5388
5502
|
this.stopPauses = 0;
|
|
5389
5503
|
this.lastProgressAt = this.limits.now();
|
|
5390
5504
|
} else if (!this.beforeModelEvent && this.reentry !== void 0 && this.reentry.reason !== "wake" && contract === "unmet") this.barrenReentries += 1;
|
|
5391
|
-
if (run.wait !== void 0 && contract !== "met"
|
|
5505
|
+
if (run.wait !== void 0 && contract !== "met") {
|
|
5506
|
+
const waited = await this.wake(attempt, completed, after);
|
|
5507
|
+
if (waited.reentered) return true;
|
|
5508
|
+
after = waited.progress;
|
|
5509
|
+
contract = contractOf(after);
|
|
5510
|
+
completed = {
|
|
5511
|
+
attempt,
|
|
5512
|
+
durationMs,
|
|
5513
|
+
madeProgress: madeProgress(before, after),
|
|
5514
|
+
...contract === "none" ? {} : { contract }
|
|
5515
|
+
};
|
|
5516
|
+
}
|
|
5392
5517
|
if (contract === "unmet" && continuation !== void 0) {
|
|
5393
5518
|
let decision = await this.decideContinuation(continuation, attempt, after);
|
|
5394
5519
|
while ("refusedBy" in decision && decision.refusedBy === "budget-exhausted") {
|
|
@@ -5429,17 +5554,36 @@ var DriverRetryLoop = class {
|
|
|
5429
5554
|
});
|
|
5430
5555
|
return false;
|
|
5431
5556
|
}
|
|
5432
|
-
async wake(attempt, completed, held) {
|
|
5557
|
+
async wake(attempt, completed, beforeWait, held) {
|
|
5433
5558
|
const { run } = this;
|
|
5434
5559
|
const wait = run.wait;
|
|
5435
|
-
if (wait === void 0 || (run.continuation?.closed() ?? false) || run.signal.aborted || !wait.open()) return
|
|
5560
|
+
if (wait === void 0 || (run.continuation?.closed() ?? false) || run.signal.aborted || !wait.open()) return {
|
|
5561
|
+
reentered: false,
|
|
5562
|
+
progress: beforeWait
|
|
5563
|
+
};
|
|
5436
5564
|
const stop = budgetStop(run.budget(), this.limits.now());
|
|
5437
|
-
if (stop === "deadline" || stop !== void 0 && this.pausesStop(stop)) return
|
|
5565
|
+
if (stop === "deadline" || stop !== void 0 && this.pausesStop(stop)) return {
|
|
5566
|
+
reentered: false,
|
|
5567
|
+
progress: beforeWait
|
|
5568
|
+
};
|
|
5438
5569
|
const woken = await wait.wake(attempt + 1);
|
|
5439
|
-
|
|
5570
|
+
const afterWait = run.progress();
|
|
5571
|
+
if (madeProgress(beforeWait, afterWait)) {
|
|
5572
|
+
completed = {
|
|
5573
|
+
...completed,
|
|
5574
|
+
madeProgress: true
|
|
5575
|
+
};
|
|
5576
|
+
this.barrenReentries = 0;
|
|
5577
|
+
this.stopPauses = 0;
|
|
5578
|
+
this.lastProgressAt = this.limits.now();
|
|
5579
|
+
}
|
|
5580
|
+
if (woken === void 0 || run.signal.aborted) return {
|
|
5581
|
+
reentered: false,
|
|
5582
|
+
progress: afterWait
|
|
5583
|
+
};
|
|
5440
5584
|
const input = held ? `${held.input}\n\n${woken.input}` : woken.input;
|
|
5441
5585
|
const idleMs = woken.idleMs + (held?.idleMs ?? 0);
|
|
5442
|
-
if (budgetStop(run.budget(), this.limits.now()) !== void 0) return this.wake(attempt, completed, {
|
|
5586
|
+
if (budgetStop(run.budget(), this.limits.now()) !== void 0) return this.wake(attempt, completed, afterWait, {
|
|
5443
5587
|
input,
|
|
5444
5588
|
idleMs
|
|
5445
5589
|
});
|
|
@@ -5454,101 +5598,32 @@ var DriverRetryLoop = class {
|
|
|
5454
5598
|
input,
|
|
5455
5599
|
wake: this.wakes
|
|
5456
5600
|
};
|
|
5457
|
-
return
|
|
5601
|
+
return {
|
|
5602
|
+
reentered: true,
|
|
5603
|
+
progress: afterWait
|
|
5604
|
+
};
|
|
5458
5605
|
}
|
|
5459
5606
|
};
|
|
5460
5607
|
// src/runtime/supervise/driver-retry.ts
|
|
5461
5608
|
/**
|
|
5462
|
-
*
|
|
5463
|
-
*
|
|
5464
|
-
* One dropped connection, one SIGKILLed harness process or one upstream 5xx used to end a run of
|
|
5465
|
-
* arbitrary length with `reason: 'driver-failed'` and tear down every live child (#741). This
|
|
5466
|
-
* module runs the driver again on the SAME scope, coordination server and live children until the
|
|
5467
|
-
* budget or the deadline says stop. It restarts no children and replays no work: the bridge
|
|
5468
|
-
* backend reattaches the harness session because the execution id is bound durably per node.
|
|
5469
|
-
*
|
|
5470
|
-
* Failure classes (all conservative):
|
|
5471
|
-
* - TERMINAL: Runtime's own refusals (`ValidationError`/`ConfigError`, exhausted budget, the run's
|
|
5472
|
-
* own abort, a client-side transport status such as 401/404/422). Retrying re-runs a decision.
|
|
5473
|
-
* Fails at once.
|
|
5474
|
-
* - TRANSIENT: everything foreign (harness exit without a reason, a stream cut mid-turn, a 5xx, a
|
|
5475
|
-
* socket reset, an `AbortError` that neither the run's signal nor the manager's own stop
|
|
5476
|
-
* issued). A harness turn that RETURNED a failed outcome is the same accident reported as
|
|
5477
|
-
* a value, so the drive raises {@link HarnessTurnFailedError} and it is classified like any other.
|
|
5478
|
-
* - UNAVAILABLE: the upstream said "not now" (429, 503, 529, or the router's own codes: quota,
|
|
5479
|
-
* rate limit, overload). The driver PAUSES and re-enters; a pause consumes neither `maxAttempts`
|
|
5480
|
-
* nor the barren streak, so only the deadline, the budget and cancellation bound it. Measured
|
|
5481
|
-
* 2026-09-24 on play anomaly-referee-v3d: during the router's flash quota outage four of five
|
|
5482
|
-
* lead lanes ended `driver-failed` after 12 to 13 attempts with budget and an 8-hour deadline
|
|
5483
|
-
* almost untouched.
|
|
5484
|
-
*
|
|
5485
|
-
* A driver that dies INSTANTLY and repeatedly spends nothing, so the budget alone cannot bound it.
|
|
5486
|
-
* Progress is measured between attempts; a run of attempts that changes nothing stops at
|
|
5487
|
-
* `maxConsecutiveFailures` once it has outlasted the outage window below. A failure that made
|
|
5488
|
-
* progress resets that counter.
|
|
5489
|
-
*
|
|
5490
|
-
* WHAT COUNTS AS PROGRESS was got wrong first. Reading metered spend, settled children and an
|
|
5491
|
-
* accepted submission measured the filesystem and the meter, never the goal: across 1,422 settled
|
|
5492
|
-
* discovery-lab runs it retried the runs that produced NOTHING 629 of 827 times
|
|
5493
|
-
* (76.1%) and the runs that HAD left an artifact 21 of 399 times (5.3%). So when the caller
|
|
5494
|
-
* declares a completion check, spend and settlements alone are NOT progress while it is unmet; only
|
|
5495
|
-
* a delivery (an accepted submission, a child that passed the check, the contract turning met, or a
|
|
5496
|
-
* rise in the check's best composite) resets the barren counter. A caller that declares no check
|
|
5497
|
-
* reports `contract: 'none'` and keeps the historical reading.
|
|
5498
|
-
*
|
|
5499
|
-
* CONTINUATION. `budgetStop` used to be consulted only after a failure, so a driver that RETURNED
|
|
5500
|
-
* with the contract unmet ended the run silently (376 of 376 winning lab runs ended on this loop's
|
|
5501
|
-
* own `stop: 'completed'`). A completed drive whose contract is unmet is now a first-class moment:
|
|
5502
|
-
* the manager's `continuation` policy (`./continuation.ts`) re-enters the driver with Runtime's
|
|
5503
|
-
* continuation note, crossing the same budget, deadline and abort bounds. Successful
|
|
5504
|
-
* continuations do not consume the failure allowance. Progress paces the continuations: once
|
|
5505
|
-
* `continuation.maxBarren` re-entered drives in a row complete without progress, the loop PAUSES
|
|
5506
|
-
* before each further continuation, 5 minutes and doubling per barren drive up to 60, journals one
|
|
5507
|
-
* `barren-pause` row per pause for an operator alarm, and re-enters. A barren streak never ends the
|
|
5508
|
-
* run: the check, the deadline, the budget and cancellation do. Progress resets the streak and the
|
|
5509
|
-
* backoff. There is no count cap and no caller veto (the old prompt-owned loop logged 896 of 1,011
|
|
5510
|
-
* rounds with no progress). The barren bound
|
|
5511
|
-
* settled a 48-hour run `no-winner` after 14.3 hours, while its last two turns updated its pages
|
|
5512
|
-
* and waited on its own 2^20 prove.
|
|
5513
|
-
*
|
|
5514
|
-
* This loop decides WHETHER to re-enter, not WHERE the next drive runs: the drive harness owns
|
|
5515
|
-
* whether it continues the same session, environment or a replacement, and the caller composes the
|
|
5516
|
-
* re-entry task from it (`./reentry.ts`).
|
|
5609
|
+
* Re-enter a driver on the same scope, coordination server and live children. The harness owns
|
|
5610
|
+
* session/environment reuse; this loop owns retry, continuation, budget and deadline decisions.
|
|
5517
5611
|
*
|
|
5518
|
-
*
|
|
5519
|
-
*
|
|
5520
|
-
*
|
|
5521
|
-
*
|
|
5522
|
-
* magnetar-rust-20261019f: after a host reboot the root was driven with the bare original task and
|
|
5523
|
-
* no word of its three open workers, two barren continuations followed, and the barren bound of the
|
|
5524
|
-
* time settled the run `no-result-selected`.
|
|
5612
|
+
* Runtime refusals and cancellation stay terminal. Canonical producer failures distinguish caller,
|
|
5613
|
+
* capacity, infrastructure and work; legacy transport/harness evidence remains a fallback.
|
|
5614
|
+
* Capacity pauses do not consume failure allowances. A seat-bearing drive first lets its existing
|
|
5615
|
+
* chain rotate; the chain owns its exhausted-seat wait, while the run's bounds still apply.
|
|
5525
5616
|
*
|
|
5526
|
-
*
|
|
5527
|
-
*
|
|
5528
|
-
*
|
|
5529
|
-
* blanked) on every re-entry with no progress stop with `repeated-failure` inside the window: a
|
|
5530
|
-
* `PartialFailureError` box at the second, a `nested` manager's failed harness turn at the third,
|
|
5531
|
-
* and its other failures at the repeated-failure count and window.
|
|
5532
|
-
* A Platform restart answered
|
|
5533
|
-
* `HTTP 502 ... Platform key verification unavailable` from 03:04:34 to 03:09:41Z, and the
|
|
5534
|
-
* coordinator's twelve attempts ran out at 03:07:12Z with no token spent.
|
|
5617
|
+
* Only idle infrastructure failures earn an outage window. Repeated deterministic failures with
|
|
5618
|
+
* no progress stop inside that window; work failures consume the normal attempt/barren bounds.
|
|
5619
|
+
* With a declared check, progress means delivery or an improved verdict, not spend alone.
|
|
5535
5620
|
*
|
|
5536
|
-
*
|
|
5537
|
-
*
|
|
5621
|
+
* Completed drives with an unmet contract continue with Runtime's note. Barren continuations
|
|
5622
|
+
* wait with increasing backoff; progress resets their streak. A resumed run first receives its
|
|
5623
|
+
* open workers, and counts barren drives only after a model event in this process.
|
|
5538
5624
|
*
|
|
5539
|
-
*
|
|
5540
|
-
*
|
|
5541
|
-
* `no-progress`, `max-attempts`, `repeated-failure`, `pending-unresolved` and
|
|
5542
|
-
* `budget-exhausted` each emit one record with `replacesStop` and pause 5 minutes, doubling per
|
|
5543
|
-
* stop-pause in a row up to 60, never past a declared deadline and not jittered. An attempt that
|
|
5544
|
-
* made progress resets the streak. A cancel ends the pause and settles the run; a root steer ends
|
|
5545
|
-
* any pause at once, a budget pause included, because a steer is how an operator changes a live
|
|
5546
|
-
* run. Ending a pause admits nothing by itself: the next admission reads the pool again, a pool
|
|
5547
|
-
* still spent pauses again with its own row, and the steer stays queued until a turn reads it.
|
|
5548
|
-
* After the pause the per-window counters start over and the driver is re-entered: after
|
|
5549
|
-
* a failure as `driver-failure` with `pausedMs`, after an admission or continuation refusal as the
|
|
5550
|
-
* re-entry it was about to use. Runs must not fail early: over 259 runs, 8 of the 10 stops settled runs that
|
|
5551
|
-
* nobody solved or stopped, including 17 roots whose harness never streamed.
|
|
5625
|
+
* Opted-in root stop-pauses alarm and wait instead of settling recoverable stops. Cancellation
|
|
5626
|
+
* ends the wait; a steer wakes it but grants no admission. The next attempt rechecks the budget.
|
|
5552
5627
|
*/
|
|
5553
5628
|
function assertContinuationPolicy(continuation) {
|
|
5554
5629
|
if (continuation === void 0) return;
|
|
@@ -5948,6 +6023,7 @@ async function driveManagerTurns(inputs) {
|
|
|
5948
6023
|
sleep: (ms, signal) => sleepScopeWait(scope, ms, signal, "paused-root"),
|
|
5949
6024
|
...rootStopPause(config, session),
|
|
5950
6025
|
nested: (config.nodeContextSeed?.depth ?? 0) > 0,
|
|
6026
|
+
seatChain: config.driveHarness.seatChain === true && authoredSeatStages(config.profile) !== void 0,
|
|
5951
6027
|
resolvePending: (failure, failures) => abandonScopeRetainedOwnerInvocation(scope, failure, failures),
|
|
5952
6028
|
wait: {
|
|
5953
6029
|
open: () => !mcp.submittedResult() && !mcp.isStopped() && session.state.progressStopReason === void 0 && (session.controls.hasOpenWork() || session.state.nextTurnInbox.pendingAuthority() > 0),
|
|
@@ -6056,7 +6132,7 @@ function readManagerProgress(loop) {
|
|
|
6056
6132
|
const submitted = Boolean(mcp.submittedResult());
|
|
6057
6133
|
const composite = bestComposite(controls.checkReads());
|
|
6058
6134
|
return {
|
|
6059
|
-
poolTokensSpent: loop.baseTokensLeft - scope.budget.tokensLeft,
|
|
6135
|
+
poolTokensSpent: scope.budget.tokensCapped === false ? scope.budget.committedTokens ?? 0 : loop.baseTokensLeft - scope.budget.tokensLeft,
|
|
6060
6136
|
settledCount: settled.length,
|
|
6061
6137
|
submitted,
|
|
6062
6138
|
deliveredCount,
|
|
@@ -7287,7 +7363,7 @@ function leafWorker(env, seat, admitted, spawnContext) {
|
|
|
7287
7363
|
const { options, deliverable, runNamespace } = env.setup;
|
|
7288
7364
|
const { childDeliverable } = admitted;
|
|
7289
7365
|
if (childDeliverable !== deliverable && !options.backend) throw new ValidationError("supervise: resolveDeliverable selected a per-spawn deliverable but there is no backend to derive that leaf from; makeLeafAgent owns its own completion check");
|
|
7290
|
-
return withModelSubstitution((childDeliverable === deliverable ? env.makeLeaf :
|
|
7366
|
+
return withModelSubstitution((childDeliverable === deliverable ? env.makeLeaf : workerFromBackend(options.backend, childDeliverable))(admitted.authorized, Object.freeze({
|
|
7291
7367
|
...admitted.authorizedContext,
|
|
7292
7368
|
assignmentId: workerAssignmentNamespace(runNamespace, seat.parentOwnerId, spawnContext.assignmentId)
|
|
7293
7369
|
})), admitted.modelSubstitution);
|
|
@@ -7406,232 +7482,6 @@ function registerRecovery(env, makeRecursiveWorker) {
|
|
|
7406
7482
|
});
|
|
7407
7483
|
env.recoveryFactories.set(makeRecursiveWorker, factory);
|
|
7408
7484
|
}
|
|
7409
|
-
// src/runtime/supervise/worker-retry.ts
|
|
7410
|
-
/**
|
|
7411
|
-
* WORKER-SPAWN RETRY — the second chance `driverRetry` gives the root, given to a leaf.
|
|
7412
|
-
*
|
|
7413
|
-
* `driverRetry` guards the root and nothing else. A worker that dies is typed into a `down`
|
|
7414
|
-
* settlement, which is the right shape when the worker RAN. It is the wrong shape when the worker
|
|
7415
|
-
* never started, and one failure produces exactly that:
|
|
7416
|
-
*
|
|
7417
|
-
* bridgeExecutor: bridge stream error: host-executor: acquire timeout after 60000ms
|
|
7418
|
-
* (in_flight=4/4, queued=1)
|
|
7419
|
-
*
|
|
7420
|
-
* Two live cells held 2 leads and up to 8
|
|
7421
|
-
* workers on one bridge whose host executor caps live harness children at 4. A worker spawn that
|
|
7422
|
-
* queued past the bridge's single 60 s acquire deadline died there, and the run lost that worker
|
|
7423
|
-
* permanently even though a slot freed seconds later. The bridge retries nothing.
|
|
7424
|
-
*
|
|
7425
|
-
* Retrying execution is dangerous by default, so this seam is FAIL-CLOSED on two independent
|
|
7426
|
-
* proofs that the attempt did no work:
|
|
7427
|
-
*
|
|
7428
|
-
* 1. The error carries a PRE-SPAWN signature — a refusal the backend emits before the harness
|
|
7429
|
-
* child or the box exists ({@link isPreSpawnExecutorFailure}). No provider call started.
|
|
7430
|
-
* 2. The attempt yielded NO execution event. An attempt that streamed anything may have metered
|
|
7431
|
-
* usage, and a blind re-run would double-spend, so it stays fatal whatever its message says.
|
|
7432
|
-
*
|
|
7433
|
-
* Either proof missing means no retry. This is deliberately NARROWER than "is this an
|
|
7434
|
-
* infrastructure hiccup": `ECONNRESET` is a transient transport failure and is NOT retried here,
|
|
7435
|
-
* because it can arrive mid-stream after a metered call.
|
|
7436
|
-
*
|
|
7437
|
-
* Re-entry re-invokes `execute` on the SAME executor object. Nothing is rebuilt, so the executor's
|
|
7438
|
-
* runtime-owned materialization attestation, its bound session id, and its inbox are all the ones
|
|
7439
|
-
* the kernel admitted. For the bridge backend a second `execute` is a fresh session stream over the
|
|
7440
|
-
* same durable execution id, and a saturation refusal happens before that session's child spawned.
|
|
7441
|
-
*
|
|
7442
|
-
* @experimental
|
|
7443
|
-
*/
|
|
7444
|
-
/**
|
|
7445
|
-
* The backend refusals that are emitted BEFORE any child process, box, or provider request exists.
|
|
7446
|
-
*
|
|
7447
|
-
* Each is a queue admission deadline: the executor asked for a slot, waited, and was refused
|
|
7448
|
-
* without anything being started. The three names are the pools cli-bridge and the container host
|
|
7449
|
-
* admit through — the host executor, one scoped per-run host executor, and the container pool.
|
|
7450
|
-
*
|
|
7451
|
-
* The pattern is anchored to the whole phrase, including the `after <n>ms` deadline, so a message
|
|
7452
|
-
* that merely mentions a queue does not qualify.
|
|
7453
|
-
*/
|
|
7454
|
-
const PRE_SPAWN_SIGNATURES = Object.freeze([/\b(?:host-executor|scoped-host-executor|container-pool): acquire timeout after \d+ms\b/u]);
|
|
7455
|
-
const DEFAULT_MAX_TOTAL_MS = 9e5;
|
|
7456
|
-
const DEFAULT_INITIAL_BACKOFF_MS = 5e3;
|
|
7457
|
-
const DEFAULT_MAX_BACKOFF_MS = 6e4;
|
|
7458
|
-
/**
|
|
7459
|
-
* True when `error` is a backend refusal issued before anything ran.
|
|
7460
|
-
*
|
|
7461
|
-
* This answers "did this fail BEFORE any provider call", which is a strictly narrower question
|
|
7462
|
-
* than "is this an infrastructure hiccup". The broader question admits `ECONNRESET` and a bare
|
|
7463
|
-
* `fetch failed`, both of which can arrive after a metered call, and answering the broad question
|
|
7464
|
-
* where the narrow one is required is what makes a retry double-spend.
|
|
7465
|
-
*/
|
|
7466
|
-
function isPreSpawnExecutorFailure(error, additionalSignatures = []) {
|
|
7467
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
7468
|
-
for (const signature of PRE_SPAWN_SIGNATURES) if (signature.test(message)) return true;
|
|
7469
|
-
for (const signature of additionalSignatures) if (signature.test(message)) return true;
|
|
7470
|
-
return false;
|
|
7471
|
-
}
|
|
7472
|
-
/** Read the policy, refusing a number a run cannot act on rather than silently clamping it. */
|
|
7473
|
-
function resolveWorkerSpawnRetry(policy) {
|
|
7474
|
-
if (policy === void 0 || policy.enabled === false) return void 0;
|
|
7475
|
-
const maxTotalMs = policy.maxTotalMs ?? DEFAULT_MAX_TOTAL_MS;
|
|
7476
|
-
const initialBackoffMs = policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
|
|
7477
|
-
const maxBackoffMs = policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
|
|
7478
|
-
for (const [name, value] of [
|
|
7479
|
-
["maxTotalMs", maxTotalMs],
|
|
7480
|
-
["initialBackoffMs", initialBackoffMs],
|
|
7481
|
-
["maxBackoffMs", maxBackoffMs]
|
|
7482
|
-
]) if (!Number.isFinite(value) || value < 0) throw new ValidationError(`workerRetry.${name} must be a nonnegative finite number of milliseconds`);
|
|
7483
|
-
const signatures = policy.additionalPreSpawnSignatures ?? [];
|
|
7484
|
-
for (const signature of signatures) if (!(signature instanceof RegExp)) throw new ValidationError("workerRetry.additionalPreSpawnSignatures must contain RegExp values");
|
|
7485
|
-
return Object.freeze({
|
|
7486
|
-
maxTotalMs,
|
|
7487
|
-
initialBackoffMs,
|
|
7488
|
-
maxBackoffMs: Math.max(initialBackoffMs, maxBackoffMs),
|
|
7489
|
-
signatures: Object.freeze([...signatures])
|
|
7490
|
-
});
|
|
7491
|
-
}
|
|
7492
|
-
/**
|
|
7493
|
-
* Wrap a worker seam so a leaf whose spawn is refused before it runs is re-entered instead of
|
|
7494
|
-
* settling dead.
|
|
7495
|
-
*
|
|
7496
|
-
* The wrapper composes the agent's `executorFactory`, which is where a leaf executor is built with
|
|
7497
|
-
* the real child signal and node context, and it never replaces the executor object the factory
|
|
7498
|
-
* returned: the retry lives in one `execute` wrapper that carries the inner executor's attestation
|
|
7499
|
-
* forward and delegates every other surface to it.
|
|
7500
|
-
*/
|
|
7501
|
-
function withWorkerSpawnRetry(make, policy, hooks = {}) {
|
|
7502
|
-
const resolved = resolveWorkerSpawnRetry(policy);
|
|
7503
|
-
if (resolved === void 0) return make;
|
|
7504
|
-
return (profile, spawnContext) => {
|
|
7505
|
-
const agent = make(profile, spawnContext);
|
|
7506
|
-
const spec = agent.executorSpec;
|
|
7507
|
-
if (spec === void 0) return agent;
|
|
7508
|
-
const seamHooks = {
|
|
7509
|
-
...hooks,
|
|
7510
|
-
...typeof profile.name === "string" ? { worker: profile.name } : {}
|
|
7511
|
-
};
|
|
7512
|
-
if (spec.executorFactory !== void 0) {
|
|
7513
|
-
const innerFactory = spec.executorFactory;
|
|
7514
|
-
const executorFactory = (builtSpec, ctx) => retryPreSpawnRefusals(innerFactory(builtSpec, ctx), resolved, seamHooks);
|
|
7515
|
-
return {
|
|
7516
|
-
...agent,
|
|
7517
|
-
executorSpec: {
|
|
7518
|
-
...spec,
|
|
7519
|
-
executorFactory
|
|
7520
|
-
}
|
|
7521
|
-
};
|
|
7522
|
-
}
|
|
7523
|
-
if (spec.executor !== void 0) {
|
|
7524
|
-
const executor = retryPreSpawnRefusals(spec.executor, resolved, seamHooks);
|
|
7525
|
-
return {
|
|
7526
|
-
...agent,
|
|
7527
|
-
executorSpec: {
|
|
7528
|
-
...spec,
|
|
7529
|
-
executor
|
|
7530
|
-
}
|
|
7531
|
-
};
|
|
7532
|
-
}
|
|
7533
|
-
return agent;
|
|
7534
|
-
};
|
|
7535
|
-
}
|
|
7536
|
-
/**
|
|
7537
|
-
* Re-enter `execute` on one executor while a pre-spawn refusal keeps proving nothing ran.
|
|
7538
|
-
*
|
|
7539
|
-
* Exported so an owner of `makeLeafAgent` — which builds its own executors — gets the same seam
|
|
7540
|
-
* without reimplementing the two proofs.
|
|
7541
|
-
*/
|
|
7542
|
-
function retryPreSpawnRefusals(inner, policy, hooks = {}) {
|
|
7543
|
-
const now = hooks.now ?? Date.now;
|
|
7544
|
-
const sleep = hooks.sleep ?? defaultSleep;
|
|
7545
|
-
/** The wait before the next attempt, or `null` when this failure stays fatal. */
|
|
7546
|
-
const nextWait = (error, attempt, yielded, startedAt, signal) => {
|
|
7547
|
-
if (yielded) return null;
|
|
7548
|
-
if (signal?.aborted) return null;
|
|
7549
|
-
if (!isPreSpawnExecutorFailure(error, policy.signatures)) return null;
|
|
7550
|
-
const backoff = Math.min(policy.maxBackoffMs, policy.initialBackoffMs * 2 ** (attempt - 1));
|
|
7551
|
-
if (now() - startedAt + backoff >= policy.maxTotalMs) return null;
|
|
7552
|
-
return backoff;
|
|
7553
|
-
};
|
|
7554
|
-
const report = (error, attempt, waitMs) => {
|
|
7555
|
-
hooks.onRetry?.({
|
|
7556
|
-
attempt,
|
|
7557
|
-
waitMs,
|
|
7558
|
-
...hooks.worker === void 0 ? {} : { worker: hooks.worker },
|
|
7559
|
-
error: error instanceof Error ? error.message : String(error)
|
|
7560
|
-
});
|
|
7561
|
-
};
|
|
7562
|
-
const wrapped = {
|
|
7563
|
-
runtime: inner.runtime,
|
|
7564
|
-
...inner.budgetExempt !== void 0 ? { budgetExempt: inner.budgetExempt } : {},
|
|
7565
|
-
...inner.deliver ? { deliver: (message) => inner.deliver?.(message) } : {},
|
|
7566
|
-
...inner.progress ? { progress: () => inner.progress?.() } : {},
|
|
7567
|
-
...inner.traceSource ? { traceSource: () => inner.traceSource?.() } : {},
|
|
7568
|
-
...inner.metered ? { metered: () => inner.metered?.() } : {},
|
|
7569
|
-
...inner.accounting ? { accounting: () => inner.accounting?.() } : {},
|
|
7570
|
-
...inner.resultAccepted ? { resultAccepted: () => inner.resultAccepted?.() === true } : {},
|
|
7571
|
-
...inner.harnessTranscript ? { harnessTranscript: () => inner.harnessTranscript?.() } : {},
|
|
7572
|
-
...inner.harnessTranscriptSettled ? { harnessTranscriptSettled: () => inner.harnessTranscriptSettled?.() ?? Promise.resolve() } : {},
|
|
7573
|
-
...inner.interactive === void 0 ? {} : { interactive: () => interactiveOf(inner) },
|
|
7574
|
-
execute(task, signal) {
|
|
7575
|
-
const first = inner.execute(task, signal);
|
|
7576
|
-
const restart = () => inner.execute(task, signal);
|
|
7577
|
-
return isAsyncIterable(first) ? retryStream(first, restart, signal) : retryOneShot(first, restart, signal);
|
|
7578
|
-
},
|
|
7579
|
-
...inner.recover ? { recover: (task, signal) => inner.recover(task, signal) } : {},
|
|
7580
|
-
teardown: (grace) => inner.teardown(grace),
|
|
7581
|
-
...teardownSurfaces(inner),
|
|
7582
|
-
resultArtifact: () => inner.resultArtifact()
|
|
7583
|
-
};
|
|
7584
|
-
async function* retryStream(first, restart, signal) {
|
|
7585
|
-
const startedAt = now();
|
|
7586
|
-
let execution = first;
|
|
7587
|
-
for (let attempt = 1;; attempt += 1) {
|
|
7588
|
-
let yielded = false;
|
|
7589
|
-
try {
|
|
7590
|
-
for await (const event of execution) {
|
|
7591
|
-
yielded = true;
|
|
7592
|
-
yield event;
|
|
7593
|
-
}
|
|
7594
|
-
return;
|
|
7595
|
-
} catch (error) {
|
|
7596
|
-
const waitMs = nextWait(error, attempt, yielded, startedAt, signal);
|
|
7597
|
-
if (waitMs === null) throw error;
|
|
7598
|
-
report(error, attempt, waitMs);
|
|
7599
|
-
await sleep(waitMs, signal);
|
|
7600
|
-
const next = restart();
|
|
7601
|
-
if (!isAsyncIterable(next)) throw new ValidationError("retryPreSpawnRefusals: executor changed execution shape between attempts");
|
|
7602
|
-
execution = next;
|
|
7603
|
-
}
|
|
7604
|
-
}
|
|
7605
|
-
}
|
|
7606
|
-
async function retryOneShot(first, restart, signal) {
|
|
7607
|
-
const startedAt = now();
|
|
7608
|
-
let execution = first;
|
|
7609
|
-
for (let attempt = 1;; attempt += 1) try {
|
|
7610
|
-
return await execution;
|
|
7611
|
-
} catch (error) {
|
|
7612
|
-
const waitMs = nextWait(error, attempt, false, startedAt, signal);
|
|
7613
|
-
if (waitMs === null) throw error;
|
|
7614
|
-
report(error, attempt, waitMs);
|
|
7615
|
-
await sleep(waitMs, signal);
|
|
7616
|
-
const next = restart();
|
|
7617
|
-
if (isAsyncIterable(next)) throw new ValidationError("retryPreSpawnRefusals: executor changed execution shape between attempts");
|
|
7618
|
-
execution = next;
|
|
7619
|
-
}
|
|
7620
|
-
}
|
|
7621
|
-
return inheritRuntimeOwnedExecutorAttestation(inner, wrapped);
|
|
7622
|
-
}
|
|
7623
|
-
/** Forward the inner executor's interactive contract exactly. The method is present only when the
|
|
7624
|
-
* inner executor implements it, so the reachable call always has one to delegate to. */
|
|
7625
|
-
function interactiveOf(inner) {
|
|
7626
|
-
if (inner.interactive === void 0) throw new ValidationError("retryPreSpawnRefusals: inner executor exposes no interactive session");
|
|
7627
|
-
return inner.interactive();
|
|
7628
|
-
}
|
|
7629
|
-
/** The wait stays REF'd: a worker waiting out a saturated executor is live work, and an unref'd
|
|
7630
|
-
* timer lets the process exit mid-backoff when nothing else is pending. */
|
|
7631
|
-
function defaultSleep(ms, signal) {
|
|
7632
|
-
if (ms <= 0 || signal?.aborted) return Promise.resolve();
|
|
7633
|
-
return sleep(ms, signal);
|
|
7634
|
-
}
|
|
7635
7485
|
// src/runtime/supervise/supervise-recursive-workers.ts
|
|
7636
7486
|
function createSuperviseWorkers(setup, cancelDurableRun) {
|
|
7637
7487
|
const { options, canonicalProfile, rootExecution, guidanceComposer, deliverable, analysts, journal, runId, rootOwnerId, now } = setup;
|
|
@@ -7682,15 +7532,13 @@ function createSuperviseWorkers(setup, cancelDurableRun) {
|
|
|
7682
7532
|
let makeWorkerAgent = options.makeWorkerAgent;
|
|
7683
7533
|
if (!makeWorkerAgent) {
|
|
7684
7534
|
if (!options.backend && !options.makeLeafAgent) throw new ValidationError("supervise: provide opts.backend (where workers run), opts.makeLeafAgent, or opts.makeWorkerAgent");
|
|
7685
|
-
const withRetry = (make) => withWorkerSpawnRetry(make, options.workerRetry, { ...options.onWorkerRetry ? { onRetry: options.onWorkerRetry } : {} });
|
|
7686
7535
|
makeWorkerAgent = recursiveWorkerFactoryFor({
|
|
7687
7536
|
setup,
|
|
7688
7537
|
cancelDurableRun,
|
|
7689
7538
|
graphSubtasks,
|
|
7690
7539
|
graphOverlayFor,
|
|
7691
7540
|
recoveryFactories,
|
|
7692
|
-
|
|
7693
|
-
makeLeaf: withRetry(options.makeLeafAgent ?? workerFromBackend(options.backend, deliverable)),
|
|
7541
|
+
makeLeaf: options.makeLeafAgent ?? workerFromBackend(options.backend, deliverable),
|
|
7694
7542
|
securityPolicy: options.profileSecurity ?? DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY
|
|
7695
7543
|
}, {
|
|
7696
7544
|
parent: canonicalProfile,
|
|
@@ -8129,8 +7977,7 @@ var DriverExecutionAttempt = class {
|
|
|
8129
7977
|
sink.state.driveTurns += 1;
|
|
8130
7978
|
if (turnStop !== void 0 && turns >= sink.turnCap && !turnStop.signal.aborted) turnStop.abort(`supervise: maxTurns ${sink.turnCap} reached`);
|
|
8131
7979
|
} else if (event.kind === "progress") {
|
|
8132
|
-
sink.
|
|
8133
|
-
sink.noteDriveProgress(event.progress);
|
|
7980
|
+
sink.observeProgress(event.progress);
|
|
8134
7981
|
sink.onModelEvent?.();
|
|
8135
7982
|
} else this.pendingUsage.push(event);
|
|
8136
7983
|
await this.meterPending();
|
|
@@ -8402,13 +8249,18 @@ var ManagerDriveSession = class {
|
|
|
8402
8249
|
const paused = [...await seatSelection.journal.loadTree(seatJournalRoot) ?? []].reverse().find((event) => event.id === scope.view.root && (event.kind === "seat-paused" || event.kind === "seat-segment" && event.phase === "started"));
|
|
8403
8250
|
let pausedUntil = paused?.kind === "seat-paused" ? paused.resumeAt : void 0;
|
|
8404
8251
|
let afterRecheck = pausedUntil !== void 0 && Date.parse(pausedUntil) <= now();
|
|
8252
|
+
let resumeAdmission;
|
|
8405
8253
|
for (;;) try {
|
|
8406
8254
|
const seat = await seatChain.next(usageLimit && !afterRecheck, afterRecheck);
|
|
8255
|
+
await resumeAdmission?.();
|
|
8407
8256
|
this.activeSeat = seat;
|
|
8408
8257
|
if (afterRecheck) await seatSelection.onPause(scope.view.root, "", "resumed");
|
|
8409
8258
|
return seat;
|
|
8410
8259
|
} catch (error) {
|
|
8411
8260
|
if (!(error instanceof SeatPauseError)) throw error;
|
|
8261
|
+
resumeAdmission = await pauseScopeSeatAdmission(scope, error.resumeAt);
|
|
8262
|
+
await releaseScopeRetainedOwnerForSeatPause(scope, error.resumeAt).catch(() => void 0);
|
|
8263
|
+
resumeAdmission = await pauseScopeSeatAdmission(scope, error.resumeAt);
|
|
8412
8264
|
if (error.resumeAt !== pausedUntil) {
|
|
8413
8265
|
pausedUntil = error.resumeAt;
|
|
8414
8266
|
await seatSelection.journal.appendEvent(seatJournalRoot, {
|
|
@@ -8421,7 +8273,6 @@ var ManagerDriveSession = class {
|
|
|
8421
8273
|
});
|
|
8422
8274
|
await seatSelection.onPause(scope.view.root, error.resumeAt, "paused");
|
|
8423
8275
|
}
|
|
8424
|
-
await releaseScopeRetainedOwnerForSeatPause(scope, error.resumeAt).catch(() => void 0);
|
|
8425
8276
|
const wakeAt = Date.now() + Math.max(0, Date.parse(error.resumeAt) - now());
|
|
8426
8277
|
while (Date.now() < wakeAt) {
|
|
8427
8278
|
scope.signal.throwIfAborted();
|
|
@@ -8435,7 +8286,7 @@ var ManagerDriveSession = class {
|
|
|
8435
8286
|
async function executeDriverAttempt(input) {
|
|
8436
8287
|
const { state, scope, retainedOwner, turnCap } = input;
|
|
8437
8288
|
if (!(scope.view.inFlight > 0)) {
|
|
8438
|
-
const spent = exhaustedBudgetChannels(scope.budget, input.now()) ?? (scope.budget.iterationsLeft <= 0 ? "iterations: none left" : void 0);
|
|
8289
|
+
const spent = exhaustedBudgetChannels(scope.budget, input.now()) ?? (scope.budget.iterationsCapped !== false && scope.budget.iterationsLeft <= 0 ? "iterations: none left" : void 0);
|
|
8439
8290
|
if (spent) throw new ValidationError(`driveHarnessFromBackend: supervisor budget exhausted (${spent})`);
|
|
8440
8291
|
}
|
|
8441
8292
|
const identity = admitDriverProfiles(input);
|
|
@@ -8457,10 +8308,16 @@ async function executeDriverAttempt(input) {
|
|
|
8457
8308
|
...input.coordinationMcpHeaders ? { headers: input.coordinationMcpHeaders } : {},
|
|
8458
8309
|
metadata: { toolTimeoutMs: COORDINATION_CLIENT_TOOL_TIMEOUT_MS }
|
|
8459
8310
|
};
|
|
8460
|
-
const
|
|
8311
|
+
const factory = reusedAttemptFactory(input) ?? await providerAttemptFactory(input, identity.providerProfile.harness);
|
|
8312
|
+
const observation = scopeOwnerExecutionObservation(scope, (event) => {
|
|
8313
|
+
input.rootStream?.append(event);
|
|
8314
|
+
input.noteDriveProgress(event);
|
|
8315
|
+
});
|
|
8316
|
+
const executor = factory(spec, {
|
|
8461
8317
|
signal: scope.signal,
|
|
8462
8318
|
node: scopeOwnerExecutorNodeContext(scope),
|
|
8463
8319
|
seams: {
|
|
8320
|
+
[executorObservationSeamKey]: observation,
|
|
8464
8321
|
...retainedOwner ? { [retainedExecutorSeamKey]: {
|
|
8465
8322
|
...retainedOwner,
|
|
8466
8323
|
onResult: async (result) => attempt.commitOwnerResult(result)
|
|
@@ -8488,7 +8345,10 @@ async function executeDriverAttempt(input) {
|
|
|
8488
8345
|
if (retainedOwner?.admissions.length && !executor.recover) throw new ValidationError("retained owner executor does not support recovery");
|
|
8489
8346
|
await input.rootStream?.beginAttempt();
|
|
8490
8347
|
const run = retainedOwner?.admissions.length ? executor.recover(input.originalTask, scope.signal) : executor.execute(input.originalTask, scope.signal);
|
|
8491
|
-
if (isAsyncIterable(run)) await attempt.drainStream(run,
|
|
8348
|
+
if (isAsyncIterable(run)) await attempt.drainStream(run, {
|
|
8349
|
+
...input,
|
|
8350
|
+
observeProgress: observation.progress
|
|
8351
|
+
});
|
|
8492
8352
|
else await attempt.settleReturned(await run);
|
|
8493
8353
|
await attempt.publishAcknowledgement(pending);
|
|
8494
8354
|
attempt.completed = true;
|
|
@@ -8684,16 +8544,13 @@ function driveHarnessFromBackend(backend, executionId, now = Date.now, maxTurns,
|
|
|
8684
8544
|
const failures = (await Promise.allSettled([manager.closeSeatSegment(reason), ...[...state.coordinationSecrets?.values() ?? []].map((secret) => secret.release())])).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
|
|
8685
8545
|
if (failures.length) throw new AggregateError(failures, "manager settlement cleanup failed");
|
|
8686
8546
|
};
|
|
8687
|
-
return attestRuntimeOwnedScopeOwner(drive, manager.ownerRuntime);
|
|
8547
|
+
return attestRuntimeOwnedScopeOwner(Object.assign(drive, seatSelection ? { seatChain: true } : {}), manager.ownerRuntime);
|
|
8688
8548
|
}
|
|
8689
|
-
async function bindDriveRetainedOwner(manager, { scope
|
|
8690
|
-
const { state
|
|
8549
|
+
async function bindDriveRetainedOwner(manager, { scope }) {
|
|
8550
|
+
const { state } = manager;
|
|
8691
8551
|
const retainedOwner = state.boundBackend.backend === "provider" ? scopeRetainedOwnerContext(scope) : void 0;
|
|
8692
8552
|
if (retainedOwner && state.boundBackend.backend === "provider") {
|
|
8693
|
-
bindScopeRetainedOwnerProvider(scope, resolveAgentEnvironmentProvider(state.boundBackend.provider, state.boundBackend.registry)
|
|
8694
|
-
port: state.boundBackend.workspaceRetention,
|
|
8695
|
-
profile: activeSeat ? seatProviderProfile(authoredProfile, activeSeat) : profile
|
|
8696
|
-
});
|
|
8553
|
+
bindScopeRetainedOwnerProvider(scope, resolveAgentEnvironmentProvider(state.boundBackend.provider, state.boundBackend.registry));
|
|
8697
8554
|
bindScopeRetainedOwnerEnvironmentId(scope, () => {
|
|
8698
8555
|
if (state.activeExecutor === void 0) return void 0;
|
|
8699
8556
|
const execution = runtimeOwnedExecutorMaterialization(state.activeExecutor)?.execution;
|
|
@@ -8985,21 +8842,34 @@ async function assertSqlRunProvider(options) {
|
|
|
8985
8842
|
if (!capabilities.retainedControl || !capabilities.streaming.turnIdempotency || !capabilities.streaming.replay || !provider.get) throw new ValidationError("supervise: SQL runContext requires retainedControl, replay, turn idempotency, and provider.get");
|
|
8986
8843
|
}
|
|
8987
8844
|
// src/runtime/supervise/supervise-run-setup.ts
|
|
8988
|
-
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
assertValidBudget(options.budget, "supervise budget");
|
|
8845
|
+
/** Canonical root identity is needed for control operations before executable admission. */
|
|
8846
|
+
function prepareSuperviseIdentity(profile, task, opts, mode = "execution") {
|
|
8847
|
+
const options = captureSuperviseOptions(opts, mode);
|
|
8992
8848
|
const guidanceComposer = profileGuidanceComposer(options.profileGuidance);
|
|
8993
8849
|
const { applyTraceTools, composeSpawnProfile } = traceToolPolicy(options, guidanceComposer);
|
|
8994
|
-
const peerAuditFor = peerAuditRegistry(options);
|
|
8995
8850
|
const registeredProfile = applyTraceTools(superviseRootProfile(profile, options.profileGuidance));
|
|
8996
8851
|
if (options.segment !== void 0) admitSegment(options.segment);
|
|
8997
8852
|
const canonicalProfile = options.segment === void 0 ? registeredProfile : withRootSegment(registeredProfile, options.segment);
|
|
8998
|
-
if (!testBrain || !isRuntimeFlowBrain(testBrain)) assertExecutableAgentProfile(canonicalProfile, "supervise root");
|
|
8999
8853
|
const canonicalTask = freezeDetached(task);
|
|
8854
|
+
return {
|
|
8855
|
+
options,
|
|
8856
|
+
guidanceComposer,
|
|
8857
|
+
applyTraceTools,
|
|
8858
|
+
composeSpawnProfile,
|
|
8859
|
+
registeredProfile,
|
|
8860
|
+
canonicalProfile,
|
|
8861
|
+
canonicalTask,
|
|
8862
|
+
rootExecution: canonicalExecution(registeredProfile, canonicalTask, options.execution, "supervise root")
|
|
8863
|
+
};
|
|
8864
|
+
}
|
|
8865
|
+
function prepareSuperviseRun(profile, task, opts, testBrain) {
|
|
8866
|
+
const { options, guidanceComposer, applyTraceTools, composeSpawnProfile, registeredProfile, canonicalProfile, canonicalTask, rootExecution } = prepareSuperviseIdentity(profile, task, opts);
|
|
8867
|
+
assertSqlRunOptions(options);
|
|
8868
|
+
assertValidBudget(options.budget, "supervise budget");
|
|
8869
|
+
const peerAuditFor = peerAuditRegistry(options);
|
|
8870
|
+
if (!testBrain || !isRuntimeFlowBrain(testBrain)) assertExecutableAgentProfile(canonicalProfile, "supervise root");
|
|
9000
8871
|
assertWorkerSeams(options);
|
|
9001
8872
|
const authorizeDownFor = downMessageAuthorizer(options);
|
|
9002
|
-
const rootExecution = canonicalExecution(registeredProfile, canonicalTask, options.execution, "supervise root");
|
|
9003
8873
|
assertRunModelsAllowed(canonicalProfile, options);
|
|
9004
8874
|
const { deliverable, finalizer, analysts, profileTable } = resolveRunRegistry(options);
|
|
9005
8875
|
assertCoordinationBinding(options.coordination);
|
|
@@ -9129,7 +8999,6 @@ function peerAuditRegistry(options) {
|
|
|
9129
8999
|
function assertWorkerSeams(options) {
|
|
9130
9000
|
if (options.makeWorkerAgent && options.authorizeSpawn) throw new ValidationError("supervise: authorizeSpawn cannot be combined with caller-owned makeWorkerAgent; use makeLeafAgent to override leaf execution inside the authorized path, or use backend-derived workers");
|
|
9131
9001
|
if (options.makeWorkerAgent && options.makeLeafAgent) throw new ValidationError("supervise: makeWorkerAgent replaces the worker path and makeLeafAgent overrides a leaf inside it; pass one");
|
|
9132
|
-
if (options.makeWorkerAgent && (options.workerRetry || options.onWorkerRetry)) throw new ValidationError("supervise: workerRetry applies only to backend-derived workers; compose withWorkerSpawnRetry onto a caller-owned makeWorkerAgent explicitly");
|
|
9133
9002
|
if (options.makeWorkerAgent && options.resolveDeliverable) throw new ValidationError("supervise: resolveDeliverable applies only to backend-derived workers; wrap a caller-owned makeWorkerAgent with its completion checks explicitly");
|
|
9134
9003
|
}
|
|
9135
9004
|
function downMessageAuthorizer(options) {
|
|
@@ -9206,7 +9075,13 @@ function managerDriverPlan(options, canonicalProfile, externalRoot) {
|
|
|
9206
9075
|
if (authoredSeatStages(canonicalProfile) && !hasCustomDriveHarness && !options.selectSeat) throw new ValidationError("supervise: AgentProfile.seats requires selectSeat before execution");
|
|
9207
9076
|
const automaticDriver = managerBackend !== void 0 && automaticDriverBackendSupported(managerBackend, options.coordination);
|
|
9208
9077
|
const spawnResourceRoot = hasCustomDriveHarness ? void 0 : rootSpawnResourceRoot(managerBackend);
|
|
9209
|
-
const spawnPreflight = composeSpawnPreflights(
|
|
9078
|
+
const spawnPreflight = composeSpawnPreflights(async (profile, context) => {
|
|
9079
|
+
const external = isExternalSupervisor(profile);
|
|
9080
|
+
const manager = declaredRuntimeToolNames(profile).length > 0 || external && authoredSeatStages(profile) !== void 0;
|
|
9081
|
+
const custom = manager ? external && hasCustomDriveHarness : options.makeLeafAgent !== void 0;
|
|
9082
|
+
const backend = manager ? external ? managerBackend : void 0 : options.backend;
|
|
9083
|
+
return modelSettingsSpawnPreflight(options.makeWorkerAgent === void 0 && !custom, backend?.backend === "provider" ? "provider" : "router")?.(profile, context);
|
|
9084
|
+
}, profileToolSpawnPreflight(options.makeWorkerAgent === void 0, options.resolveSupervisorTools !== void 0), coordinationChannelSpawnPreflight(options.makeWorkerAgent === void 0, hasCustomDriveHarness, managerBackend, options.coordination), options.backend?.backend === "bridge" ? bridgeSpawnPreflight(options.backend) : void 0, options.backend?.backend === "provider" && typeof options.backend.provider === "object" ? providerSpawnPreflight(options.backend.provider) : void 0);
|
|
9210
9085
|
const driverMaterialization = hasCustomDriveHarness ? options.driveHarnessMaterialization ?? fullProfileMaterialization : managerBackend && automaticDriver ? backendProfileMaterialization(managerBackend) : void 0;
|
|
9211
9086
|
if (externalRoot && !hasCustomDriveHarness && !automaticDriver) throw new ValidationError(`supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge, a provider with authenticated coordination.publicUrl and runtime MCP attachments, or an explicit driveHarness with reachable coordination transport`);
|
|
9212
9087
|
return {
|
|
@@ -9237,6 +9112,8 @@ function spawnProfileComposer(options, driver, childrenCanSubmit, composeSpawnPr
|
|
|
9237
9112
|
}
|
|
9238
9113
|
// src/runtime/supervise/supervise-run-lifecycle.ts
|
|
9239
9114
|
function superviseInternal(profile, task, opts, testBrain) {
|
|
9115
|
+
const cancelled = pendingCancellationRun(profile, task, opts);
|
|
9116
|
+
if (cancelled !== void 0) return cancelled;
|
|
9240
9117
|
const setup = prepareSuperviseRun(profile, task, opts, testBrain);
|
|
9241
9118
|
const { options } = setup;
|
|
9242
9119
|
const workers = createSuperviseWorkers(setup, (reason, request) => cancelDurableRun(reason, request));
|
|
@@ -9254,6 +9131,82 @@ function superviseInternal(profile, task, opts, testBrain) {
|
|
|
9254
9131
|
rootContinuation: void 0
|
|
9255
9132
|
});
|
|
9256
9133
|
}
|
|
9134
|
+
/** The aborted kernel entry owns replay and resource custody without executable admission. */
|
|
9135
|
+
function pendingCancellationRun(profile, task, opts) {
|
|
9136
|
+
const runDir = opts.runDir;
|
|
9137
|
+
if (runDir === void 0 || opts.runContext?.durability === "sql") return void 0;
|
|
9138
|
+
const request = readRunCancelRequest(runDir);
|
|
9139
|
+
if (request === void 0) return void 0;
|
|
9140
|
+
const acknowledged = readRunCancellation(runDir, request.operationId);
|
|
9141
|
+
if (acknowledged !== void 0 && acknowledged.effect !== "cancel_requested") return void 0;
|
|
9142
|
+
const { options, registeredProfile, canonicalProfile, canonicalTask, rootExecution } = prepareSuperviseIdentity(profile, task, opts, "control");
|
|
9143
|
+
const now = options.now ?? Date.now;
|
|
9144
|
+
const controller = new AbortController();
|
|
9145
|
+
applyRunCancellation(runDir, (reason, request) => {
|
|
9146
|
+
controller.abort(request === void 0 ? new Error(reason) : new RunCancellationReason(request.source, reason, request.operationId, request.operator));
|
|
9147
|
+
}, () => new Date(now()).toISOString(), "observer");
|
|
9148
|
+
const context = options.runContext ?? createFileRunContext(runDir, { withDriver: true });
|
|
9149
|
+
return (async () => {
|
|
9150
|
+
if (!controller.signal.aborted) throw new Error("run cancellation changed before control entry");
|
|
9151
|
+
const runId = options.runId ?? context.runId ?? "supervise";
|
|
9152
|
+
const journal = options.journal ?? context.journal;
|
|
9153
|
+
const prior = await journal.loadTree(runId);
|
|
9154
|
+
const root = prior?.find((event) => event.kind === "spawned" && event.id === runId);
|
|
9155
|
+
const reopened = [...prior ?? []].reverse().find((event) => event.kind === "reopened" && event.id === runId && event.budget !== void 0);
|
|
9156
|
+
const budget = options.budget ?? (reopened?.kind === "reopened" ? reopened.budget : void 0) ?? (root?.kind === "spawned" ? root.budget : void 0);
|
|
9157
|
+
if (budget === void 0) throw new Error("cancelled run has no declared or recorded budget");
|
|
9158
|
+
assertValidBudget(budget, "supervise budget");
|
|
9159
|
+
const rootAgent = {
|
|
9160
|
+
name: canonicalProfile.name ?? "supervise",
|
|
9161
|
+
async act() {
|
|
9162
|
+
throw new Error("cancelled run cannot enter execution");
|
|
9163
|
+
}
|
|
9164
|
+
};
|
|
9165
|
+
const backend = options.driverBackend ?? (options.rootDriverFromBackend === false ? void 0 : options.backend);
|
|
9166
|
+
if (backend?.backend === "provider" && !options.driveHarness && !options.resolveDriveHarness) {
|
|
9167
|
+
const provider = typeof backend.provider === "string" ? backend.registry?.get(backend.provider) : backend.provider;
|
|
9168
|
+
const priorProviders = new Set((prior ?? []).flatMap((event) => event.id === runId && event.kind === "execution-admitted" ? [event.admission.phase === "dispatched" ? event.admission.controlRef.provider : event.admission.provider] : []));
|
|
9169
|
+
if (provider !== void 0 && priorProviders.size === 1 && priorProviders.has(provider.name)) attestRuntimeOwnedScopeOwner(rootAgent, root?.kind === "spawned" ? root.runtime : provider.name, { provider });
|
|
9170
|
+
}
|
|
9171
|
+
const result = await createSupervisor().run(rootAgent, canonicalTask, {
|
|
9172
|
+
budget,
|
|
9173
|
+
runId,
|
|
9174
|
+
journal,
|
|
9175
|
+
blobs: options.blobs ?? context.blobs,
|
|
9176
|
+
executors: context.executors,
|
|
9177
|
+
rootIdentity: rootExecution.identity,
|
|
9178
|
+
rootMaterialization: {
|
|
9179
|
+
runtime: root?.kind === "spawned" ? root.runtime : "inline",
|
|
9180
|
+
declaration: "deferred",
|
|
9181
|
+
authoredProfile: registeredProfile
|
|
9182
|
+
},
|
|
9183
|
+
...options.segment === void 0 ? {} : { rootSegment: {
|
|
9184
|
+
registered: rootSegmentOf(registeredProfile),
|
|
9185
|
+
current: {
|
|
9186
|
+
...rootSegmentOf(canonicalProfile),
|
|
9187
|
+
...options.segment.stack === void 0 ? {} : { stack: { ...options.segment.stack } }
|
|
9188
|
+
},
|
|
9189
|
+
reason: options.segment.reason
|
|
9190
|
+
} },
|
|
9191
|
+
...options.recoverExecutor === void 0 ? {} : { recoverExecutor: options.recoverExecutor },
|
|
9192
|
+
...options.reservationPolicy === void 0 ? {} : { reservationPolicy: options.reservationPolicy },
|
|
9193
|
+
...options.workerSlots === void 0 ? {} : { workerSlots: options.workerSlots },
|
|
9194
|
+
...options.teardownConfirmMs === void 0 ? {} : { teardownConfirmMs: options.teardownConfirmMs },
|
|
9195
|
+
...context.resume === true || options.resume === true ? { resume: true } : {},
|
|
9196
|
+
retainedAtSettlement: options.retainedAtSettlement ?? "release",
|
|
9197
|
+
hooks: options.hooks,
|
|
9198
|
+
now,
|
|
9199
|
+
signal: options.signal === void 0 ? controller.signal : AbortSignal.any([controller.signal, options.signal]),
|
|
9200
|
+
interactiveBindingDir: runDir
|
|
9201
|
+
});
|
|
9202
|
+
recordRunCancellationOutcome(runDir, result, now);
|
|
9203
|
+
return {
|
|
9204
|
+
...result,
|
|
9205
|
+
rootProviderModel: rootProviderModelEvidence([]),
|
|
9206
|
+
...rootExecution.ref === void 0 ? {} : { execution: rootExecution.ref }
|
|
9207
|
+
};
|
|
9208
|
+
})();
|
|
9209
|
+
}
|
|
9257
9210
|
async function startSupervisedRun(run) {
|
|
9258
9211
|
const { setup, runControl, cancelDurableRun } = run;
|
|
9259
9212
|
const { options, ctx, log, runId, rootOwnerId, canonicalTask } = setup;
|
|
@@ -9393,7 +9346,7 @@ function runHooks(options, recorder, workers) {
|
|
|
9393
9346
|
}
|
|
9394
9347
|
function rootRunOptions(run, runDir, hooks, recorder) {
|
|
9395
9348
|
const { setup, workers } = run;
|
|
9396
|
-
const { options, ctx, registeredProfile,
|
|
9349
|
+
const { options, ctx, registeredProfile, rootOwnerRuntime } = setup;
|
|
9397
9350
|
const recoverExecutor = options.recoverExecutor ?? workers.recoveryFactories.get(workers.workerFactory);
|
|
9398
9351
|
return {
|
|
9399
9352
|
budget: options.budget,
|
|
@@ -9420,7 +9373,6 @@ function rootRunOptions(run, runDir, hooks, recorder) {
|
|
|
9420
9373
|
...options.childSettleGraceMs !== void 0 ? { childSettleGraceMs: options.childSettleGraceMs } : {},
|
|
9421
9374
|
...options.teardownConfirmMs !== void 0 ? { teardownConfirmMs: options.teardownConfirmMs } : {},
|
|
9422
9375
|
...options.retainedAtSettlement !== void 0 ? { retainedAtSettlement: options.retainedAtSettlement } : {},
|
|
9423
|
-
ownerWorkspaceRetention: managerBackend?.backend === "provider" && managerBackend.workspaceRetention !== void 0,
|
|
9424
9376
|
...options.workerSlots !== void 0 ? { workerSlots: options.workerSlots } : {},
|
|
9425
9377
|
...options.reservationPolicy ? { reservationPolicy: options.reservationPolicy } : {},
|
|
9426
9378
|
...options.deadlineCheckIn === void 0 ? {} : { deadlineCheckIn: options.deadlineCheckIn },
|
|
@@ -9492,6 +9444,6 @@ function superviseWithContext(profile, task, opts, brain) {
|
|
|
9492
9444
|
...signal === void 0 ? {} : { signal }
|
|
9493
9445
|
}, brain));
|
|
9494
9446
|
}
|
|
9495
|
-
export {
|
|
9447
|
+
export { dumbContinuationPassPrompt as A, areaUnderCurve as B, defaultEdgeTraversalCap as C, createPromptRegistry as D, analyzesFindingsReportPrompt as E, allWorkersStalled as F, plateauLength as H, anyOf as I, noProgressFor as L, kernelPromptRegistry as M, naiveContinuationPrompt as N, delegatesWorkerBriefPrompt as O, promptHandle as P, plateau as R, validateFlowGraph as S, validateGraphTopology as T, renderAnytimeTable as U, bestSoFar as V, createSupervisorSpanRecorder as W, GraphRouteError as _, superviseRootProfile as a, journaledGateDecider as b, supervisorAgentWithTestBrain as c, composeReentryTask as d, serveCoordinationMcp as f, createGraphOverlay as g, runBrainLoop as h, profileGuidanceComposer as i, formatPromptHandle as j, dumbContinuationFailPrompt as k, runToolSpans as l, driverAgent as m, superviseFleetCapabilities as n, workerFromBackend as o, coordinationControlToolNames as p, superviseWithTestBrain as r, supervisorAgent as s, supervise as t, UNPROVEN_CONTINUITY as u, createFlowRun as v, graphRootProfile as w, routeFlowOf as x, flowRootProfile as y, anytimeReport as z };
|
|
9496
9448
|
|
|
9497
|
-
//# sourceMappingURL=supervise-
|
|
9449
|
+
//# sourceMappingURL=supervise-Cqhkb2Oj.js.map
|