@tangle-network/agent-runtime 0.299.2 → 0.300.0

package/dist/{supervisor-Ck9oX5A9.js → supervisor-CQSu5XGl.js}

@@ -11033,9 +11033,20 @@ const DEFAULT_MAX_UNAVAILABLE_PAUSE_MS = 3e5;
 * continues in its environment, so an outage costs both the same number of turns.
 */
 function unavailablePauseMs(consecutive, policy = {}) {
-	const first = Math.max(0, policy.unavailablePauseMs ?? DEFAULT_UNAVAILABLE_PAUSE_MS);
-	const ceiling = Math.max(first, policy.maxUnavailablePauseMs ?? DEFAULT_MAX_UNAVAILABLE_PAUSE_MS);
-	return Math.min(ceiling, first * 2 ** Math.max(0, consecutive - 1));
+	return cappedDoublingMs(consecutive, policy.unavailablePauseMs ?? DEFAULT_UNAVAILABLE_PAUSE_MS, policy.maxUnavailablePauseMs ?? DEFAULT_MAX_UNAVAILABLE_PAUSE_MS);
+}
+/** `first` doubled per earlier consecutive wait (`consecutive` is 1-based), up to `ceiling`. */
+function cappedDoublingMs(consecutive, first, ceiling) {
+	const start = Math.max(0, first);
+	return Math.min(Math.max(start, ceiling), start * 2 ** Math.max(0, consecutive - 1));
+}
+/**
+* Spread a wait over `[ms / 2, ms]`. Every driver a Platform restart cut off fails within the same
+* seconds; without jitter their doubling schedules stay aligned and they all re-enter together
+* when the service returns. `random` returns a value in `[0, 1)`; 0 keeps the full wait.
+*/
+function jitteredMs(ms, random) {
+	return ms - Math.floor(ms / 2 * Math.min(1, Math.max(0, random())));
 }
 //#endregion
 //#region src/runtime/supervise/workspace-checkpoint.ts
@@ -27662,8 +27673,9 @@ function createContinuationKeeper(input) {
 * The one loop the budget alone cannot bound is a driver that dies INSTANTLY and repeatedly — a
 * dead-on-arrival credential, a harness that refuses to start. Spending nothing, it would retry
 * until the deadline hours later. So progress is measured between attempts, and a run of attempts
-* that changes nothing stops at `maxConsecutiveFailures`. A failure that made progress resets that
-* counter: a long run may be rescued many times, a hopeless one gives up in seconds.
+* that changes nothing stops at `maxConsecutiveFailures` once it has outlasted the outage window
+* described below. A failure that made progress resets that counter: a long run may be rescued
+* many times, a hopeless one gives up after the window.
 *
 * WHAT COUNTS AS PROGRESS is the part this module got wrong first, and the correction is measured.
 * The original mark read metered spend, settled children, and an accepted submission — the
@@ -27705,11 +27717,26 @@ function createContinuationKeeper(input) {
 * five lead lanes ended `driver-failed` between minutes 101 and 118, after 12 to 13 attempts, while
 * the router answered some turns in between (one lead spent 1.7M and 3.3M input tokens on two of
 * them) and each run's budget and 8-hour deadline were almost untouched.
+*
+* A TRANSIENT failure that spent nothing, settled nothing, and made no progress is an
+* infrastructure outage until its streak has lasted `transientOutageMs`: nothing ran, so nothing
+* is being burned. Within that window it consumes neither `maxAttempts` nor the barren streak,
+* exactly like a pause, and only the deadline, the budget, and cancellation bound it. After the
+* window the failure bounds apply again, so a dead-on-arrival driver still stops, in minutes
+* rather than hours. A failure that spent tokens without delivering keeps the bounds above,
+* because that is the burn the progress rule exists to stop. Measured 2026-10-04 on Discovery run research-nqs-20261003system2: a Platform
+* restart answered `HTTP 502 ... Platform key verification unavailable` from 03:04:34 to 03:09:41Z,
+* and the coordinator's twelve attempts ran out at 03:07:12Z with no token spent, while a sibling
+* that kept retrying survived.
+*
+* Every wait, a failure backoff or a pause, is jittered over its upper half so drivers that one
+* outage cut off together do not re-enter together.
 */
 const DEFAULT_MAX_CONSECUTIVE_FAILURES = 3;
 const DEFAULT_MAX_ATTEMPTS = Number.POSITIVE_INFINITY;
 const DEFAULT_INITIAL_BACKOFF_MS = 2e3;
 const DEFAULT_MAX_BACKOFF_MS = 3e4;
+const DEFAULT_TRANSIENT_OUTAGE_MS = 9e5;
 /**
 * Bridge error classes the bridge itself never retries: a request that fails identically on
 * every attempt, mapped below 5xx on its HTTP path (`parse_error` 400, the other two 501). On the
@@ -27956,12 +27983,14 @@ async function defaultSleep(ms, signal) {
 async function runDriverWithRetry(run) {
 	const now = run.now ?? Date.now;
 	const sleep = run.sleep ?? defaultSleep;
+	const random = run.random ?? Math.random;
 	const policy = run.policy ?? {};
 	const retryEnabled = policy.enabled !== false;
 	const maxConsecutive = Math.max(0, policy.maxConsecutiveFailures ?? DEFAULT_MAX_CONSECUTIVE_FAILURES);
 	const maxAttempts = Math.max(1, policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
 	const initialBackoff = Math.max(0, policy.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS);
 	const maxBackoff = Math.max(initialBackoff, policy.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS);
+	const transientOutage = Math.max(0, policy.transientOutageMs ?? DEFAULT_TRANSIENT_OUTAGE_MS);
 	const continuation = run.continuation;
 	if (continuation !== void 0) {
 		if (!Number.isFinite(continuation.deadlineMs) || continuation.deadlineMs <= 0) throw new ValidationError$1("runDriverWithRetry: a continuation needs a finite positive deadline");
@@ -27971,6 +28000,8 @@ async function runDriverWithRetry(run) {
 	let consecutiveBarren = 0;
 	let barrenReentries = 0;
 	let failures = 0;
+	let chargedFailures = 0;
+	let outageSince;
 	let consecutivePauses = 0;
 	let pauses = 0;
 	let continuations = 0;
@@ -28027,7 +28058,8 @@ async function runDriverWithRetry(run) {
 		} catch (error) {
 			const durationMs = now() - startedAt;
 			const classification = classifyDriverFailure(error, run.signal);
-			const progressed = madeProgress(before, run.progress());
+			const after = run.progress();
+			const progressed = madeProgress(before, after);
 			if (classification === "unavailable") {
 				if (progressed) barrenReentries = 0;
 				const stop = (() => {
@@ -28050,7 +28082,7 @@ async function runDriverWithRetry(run) {
 				}
 				pauses += 1;
 				consecutivePauses = progressed ? 1 : consecutivePauses + 1;
-				const pause = unavailablePauseMs(consecutivePauses, policy);
+				const pause = jitteredMs(unavailablePauseMs(consecutivePauses, policy), random);
 				await emit({
 					attempt,
 					durationMs,
@@ -28073,13 +28105,20 @@ async function runDriverWithRetry(run) {
 			}
 			consecutivePauses = 0;
 			failures += 1;
+			const failedAt = now();
+			const idle = classification === "transient" && !progressed && after.poolTokensSpent === before.poolTokensSpent && after.settledCount === before.settledCount;
+			if (!idle) outageSince = void 0;
+			else outageSince ??= failedAt;
+			const inOutage = idle && failedAt - (outageSince ?? failedAt) < transientOutage;
+			if (!inOutage) chargedFailures += 1;
 			const stop = (() => {
 				if (classification === "terminal") return "terminal-error";
 				if (!retryEnabled) return "retry-disabled";
 				if (run.signal.aborted) return "aborted";
-				const byBudget = budgetStop(run.budget(), now());
+				const byBudget = budgetStop(run.budget(), failedAt);
 				if (byBudget) return byBudget;
-				if (failures >= maxAttempts) return "max-attempts";
+				if (inOutage) return void 0;
+				if (chargedFailures >= maxAttempts) return "max-attempts";
 				if (progressed) return void 0;
 				return consecutiveBarren + 1 >= maxConsecutive ? "no-progress" : void 0;
 			})();
@@ -28096,7 +28135,7 @@ async function runDriverWithRetry(run) {
 				throw new DriverAttemptsExhaustedError(error, attempts, stop);
 			}
 			consecutiveBarren = progressed ? 0 : consecutiveBarren + 1;
-			const backoff = Math.min(maxBackoff, initialBackoff * 2 ** Math.max(0, consecutiveBarren - 1));
+			const backoff = jitteredMs(cappedDoublingMs(consecutiveBarren, initialBackoff, maxBackoff), random);
 			await emit({
 				attempt,
 				durationMs,
@@ -28118,6 +28157,7 @@ async function runDriverWithRetry(run) {
 		}
 		consecutiveBarren = 0;
 		consecutivePauses = 0;
+		outageSince = void 0;
 		const durationMs = now() - startedAt;
 		const after = run.progress();
 		const progressed = madeProgress(before, after);
@@ -29277,4 +29317,4 @@ function isNonEmptySpend(s) {
 //#endregion
 export { bindScopeRetainedOwnerProvider as $, detachedFrozen as $i, parseWorkerToolTraceArtifact as $n, sanitizeRuntimeStreamEvent as $r, isPeerMailEnvelope as $t, driverExecutorFactory as A, addResourceSpend as Aa, prepareJsonlAppend as Ai, resolveAgentEnvironmentProvider as An, contentAddress as Ar, createExecutor as At, recordScopeOwnerDriverAttempt as B, promptFromAgentTurnInput as Ba, newExecutionAttemptId as Bi, InMemoryResultBlobStore as Bn, buildRuntimeEventOtelSpans as Br, workerTraceEnv as Bt, createContinuationKeeper as C, sleep as Ca, renderProfileMaterializationIssues as Ci, readPromptOptions as Cn, runAbortable as Cr, freeSlots as Ct, passedItems as D, unmeteredSpend as Da, validateProfileMaterialization as Di, createAgentEnvironmentProviderRegistry as Dn, readHarnessTranscript as Dr, captureReusableExecutorConfig as Dt, failedItems as E, throwIfAborted as Ea, unsupportedProfileDimensions as Ei, LEAF_CONTINUATION_TASK as En, persistHarnessTranscript as Er, bindReusableExecutorExecutionId as Et, beginScopeOwnerAttempt as F, awaitAbortable$1 as Fa, authoredProfileDigest as Fi, sandboxSessionTraceSource as Fn, defaultRedactor as Fr, createSteerableSandboxSession as Ft, settledToIteration as G, runtimeOwnedExecutorMaterialization as Gi, insideCursorNamespace as Gn, loopEventToOtelSpan as Gr, createActivityLog as Gt, recordScopeOwnerPause as H, providerMessageText as Ha, recordRuntimeOwnedDriveHarnessProviderEvidence as Hi, SpawnEventIndex as Hn, createOtelExporter as Hr, workerTraceSeamKey as Ht, createScope as I, DEFAULT_SUCCESSFUL_SHUTDOWN_MS as Ia, finalizeRuntimeOwnedPendingExecutor as Ii, registerRetainedExecutorPreparation as In, defaultRedactorIdentityMaterial as Ir, createRouterTranscript as It, pollFor as J, runtimeOwnedScopeOwnerRuntime as Ji, pendingWaits as Jn, toOtelAttributes as Jr, AUTHORITY_MARKERS as Jt, createWaitProbes as K, runtimeOwnedExecutorProviderEvidence as Ki, loadSpawnForest as Kn, padSpanId as Kr, readWorkerProgress as Kt, deriveNodeExecutionIdentity as L, armDeadlineTimer as La, inheritRuntimeOwnedExecutorAttestation as Li, retainedExecutorSeamKey as Ln, resolveRedactor as Lr, createInPlaceCliExecutor as Lt, withDriverExecutor as M, withBudgetResources as Ma, writeAllBytes as Mi, unavailableSignalOfFailure as Mn, mergeTraceEnv as Mr, snapshotExecutorConfig as Mt, SUBTREE_RESULT_LIMIT as N, decodeHarnessUsage as Na, attestRuntimeOwnedPendingExecutor as Ni, createPushTraceSource as Nn, readTraceContextFromEnv as Nr, createWorktreeCliExecutor as Nt, verdictFromJudgeScore as O, withTimeout as Oa, worktreeCliProfileMaterialization as Oi, providerAsExecutor as On, resolveHarnessTranscriptCapture as Or, cliInPlaceExecutor as Ot, assertRecursiveReservationPolicy as P, abortError$2 as Pa, attestRuntimeOwnedScopeOwner as Pi, decodeToolPart as Pn, traceContextToEnv as Pr, DEFAULT_SANDBOX_STEERING_MAX_TURNS as Pt, bindScopeRetainedOwnerEnvironmentId as Q, executableAgentSpecSnapshot as Qi, isTraceAnalysisStore as Qn, sanitizeKnowledgeReadinessReport as Qr, createPeerMailbox as Qt, meterRuntimeOwnedAccounting as R, teardownExecutor as Ra, knownExecutionBindingReceipt as Ri, FileResultBlobStore as Rn, buildLoopOtelSpans as Rr, WORKER_TRACE_PROPAGATION as Rt, composeContinuationNote as S, randomSuffix as Sa, promptResourceProfileMaterialization as Si, assertBoxlessPromptOptions as Sn, linkAbort as Sr, startRetainedInteractiveRun as St, expandQuestions as T, throwAbort as Ta, sandboxActProfileMaterialization as Ti, runBrainLoop as Tn, harnessTranscriptArtifact as Tr, rollingDispatch as Tt, restoreScopeOwnerAcceptedExecution as U, runtimeOwnedDriveHarnessProviderEvidence as Ui, assertContentAddress as Un, flatOtelSpan as Ur, taskToPrompt as Ut, recordScopeOwnerMaterialization as V, promptOptionsFromAgentTurnInput as Va, providerAttemptEvidence as Vi, InMemorySpawnJournal as Vn, createOpenInferenceFileExporter as Vr, workerTraceHeaders as Vt, scopeOwnerExecutorNodeContext as W, runtimeOwnedExecutorExecutionBinding as Wi, encodeResultBlob as Wn, generateSpanId as Wr, DEFAULT_STALL_AFTER_MS as Wt, validateWaitSpec as X, unknownMaterializationReceipt as Xi, WORKER_TOOL_TRACE_SCHEMA_VERSION as Xn, createRuntimeStreamEventCollector as Xr, PEER_MAIL_WIRE_KEY as Xt, timerAt as Y, unknownExecutionBindingReceipt as Yi, replaySpawnTree as Yn, createRuntimeEventCollector as Yr, DEFAULT_PEER_MAIL_LIMITS as Yt, waitUntil as Z, executableAgentProfileSnapshot as Zi, captureWorkerTraceEvidence as Zn, sanitizeAgentRuntimeEvent as Zr, claimsAuthority as Zt, CheckUnavailableError as _, cloneSpend as _a, fullProfileMaterialization as _i, createSandboxLineage as _n, environmentGone as _r, readWorkerInteractiveBinding as _t, collectDelivered as a, createSandboxToolPartState as aa, DEFAULT_LOCAL_HARNESS as ai, bridgeModelRouteRefusal as an, createBudgetPool as ar, scopeRetainedOwnerPriorSpend as at, bestComposite as b, isAbortError$2 as ba, promptModelProfileMaterialization as bi, SandboxEvidenceRetentionError as bn, RunCancellationReason as br, reconnectRetainedInteractiveRun as bt, runTree as c, isSandboxTerminalEvent as ca, harnessSupportsReasoningEffort as ci, harnessUsageIsEmpty as cn, spendFromUsageEvents as cr, scopeRetainedOwnerResult as ct, classifyDriverFailure as d, notifySandboxEventObserver as da, CodexExecutionDiagnosticError as di, bridgeStopSignalKey as dn, reconnectRetainedRun as dr, hostDirectoryReader as dt, detachedSnapshot as ea, runSettledCommand as ei, peerMailTools as en, workerTraceAnalysisStore as er, consumeScopeRetainedOwnerResult as et, runDriverWithRetry as f, sandboxEventServedBackend as fa, terminateProcessTreeAndConfirm as fi, TERMINAL_DECISIONS as fn, recoverRetainedRun as fr, resolveSpawnResourcePaths as ft, CONTINUATION_FACTS as g, chargedTokens as ga, defineProfileMaterializationContract as gi, runAgentRounds as gn, destroyInteractiveEnvironment as gr, attachWorker as gt, CONTINUATIONS_DIR as h, addSpend as ha, controlProfileMaterialization as hi, isTerminalDecision as hn, retainedCreateMaterial as hr, workerInteractiveAdmissionFile as ht, bestDelivered as i, canonicalStreamEventFromSandboxEvent as ia, removeWorktree as ii, bridgeAdmissionRefusal as in, assertValidBudget as ir, scopeRetainedOwnerContext as it, isDriverSpec as j, resourceTelemetry as ja, readCommittedJsonLines as ji, unavailablePauseMs as jn, createPropagatingTraceEmitter as jr, createExecutorRegistry as jt, driverChild as k, zeroSpend as ka, isNoEntError as ki, providerAsSandboxClient as kn, retainHarnessTranscript as kr, cliWorktreeExecutor as kt, DriverAttemptsExhaustedError as l, mapSandboxEvent as la, localHarnessExecutable as li, readCodexRolloutSession as ln, spendFromUsageTotals as lr, SPAWN_RESOURCE_PATH_MAX_BYTES as lt, upstreamUnavailableSignal as m, sumSandboxUsage as ma, assertProfileMaterialization as mi, defaultSelectWinner as mn, startRetainedRunInEnvironment as mr, readWorkerInteractiveAdmissions as mt, createRootHandle as n, readSandboxOutcome as na, captureWorktreeDiff as ni, isLiveNodeStatus as nn, BudgetReconcileFault as nr, prepareScopeRetainedOwnerTask as nt, pickBestDelivered as o, createSandboxUsageLedger as oa, LOCAL_HARNESSES as oi, addHarnessUsage as on, meterUsageEvent as or, scopeRetainedOwnerResourceReader as ot, summarizeDriverAttempts as p, sandboxProgressEvents as pa, AGENT_PROFILE_MATERIALIZATION_AXES as pi, createSandboxForSpec as pn, startRetainedRun as pr, interactiveAdmissionSeamKey as pt, isWaitOutcome as q, runtimeOwnedPendingExecutorMaterialization as qi, materializeTreeView as qn, padTraceId as qr, createInbox as qt, createSupervisor as r, assertSandboxServedModel as ra, createWorktree as ri, isTerminalNodeStatus as rn, ReservationWaitRefused as rr, reconcileScopeRetainedOwnerEnvironment as rt, runFinalizer as s, extractLlmCallEvent as sa, harnessInvocation as si, createCodexRolloutStoreReader as sn, newUsageTotals as sr, scopeRetainedOwnerRestorePoint as st, DEFAULT_MAX_DEPTH as t, projectSandboxOutcome as ta, runWorktreeHarness as ti, peerMailVerbNames as tn, executorFailure as tr, noteScopeRetainedOwnerCoordination as tt, HarnessTurnFailedError as u, mapSandboxToolEvent as ua, parseCodexTokenUsage as ui, bridgeRuntimeAttachmentsKey as un, errMessage as ur, environmentReader as ut, admitContinuationPolicy as v, deleteBoxSafe as va, profileMaterializationAxes$1 as vi, probeSandboxCapabilities as vn, createWorkerSlots as vr, workerInteractiveBindingFile as vt, distillFindings as w, stringifySafe as wa, renderUnsupported as wi, routerBrain as wn, captureHarnessTranscript as wr, queueOf as wt, checkVerdictOf as x, promptCacheTokenClasses as xa, promptOnlyProfileMaterialization as xi, captureBeforeDestroy as xn, abortError as xr, recoverRetainedInteractiveRun as xt, admitFinding as y, hasCompleteCacheBreakdown as ya, promptControlProfileMaterialization as yi, acquireSandbox as yn, openSlotGroup as yr, workerInteractiveBindingsDir as yt, meterRuntimeOwnedProviderAttempt as z, teardownSurfaces as za, knownMaterializationReceipt as zi, FileSpawnJournal as zn, buildLoopSpanNodes as zr, readWorkerTraceContext as zt };
 
-//# sourceMappingURL=supervisor-Ck9oX5A9.js.map
+//# sourceMappingURL=supervisor-CQSu5XGl.js.map