@zq-silk/yui 0.13.8 → 0.13.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/cli/commandCatalog.js +21 -2
- package/dist/cli.js +42 -13
- package/dist/commands/agentCommands.js +1 -1
- package/dist/commands/configCommands.js +1 -86
- package/dist/commands/executionAuditCommands.js +17 -16
- package/dist/commands/globalRoleCommands.js +4 -4
- package/dist/commands/sessionCommands.js +2 -6
- package/dist/commands/taskActor.js +1 -2
- package/dist/commands/taskCommands.js +92 -9
- package/dist/commands/taskRoleRuntimeStatus.js +3 -3
- package/dist/config/configCatalog.js +1 -6
- package/dist/config/yuiConfig.js +0 -79
- package/dist/controller/clientRuntime.js +40 -3
- package/dist/controller/controller.js +29 -52
- package/dist/controller/fileSchedulerStoreAdapter.js +305 -1056
- package/dist/controller/runtime.js +12 -5
- package/dist/controller/runtimeHookRunFence.js +3 -9
- package/dist/controller/runtimeLaunchCoordinator.js +44 -67
- package/dist/controller/structuredProviderObservation.js +18 -5
- package/dist/coordination/workMailbox.js +4 -4
- package/dist/execution/executionHealth.js +1 -1
- package/dist/executor/agentExecutor.js +92 -68
- package/dist/executor/executorRegistry.js +8 -20
- package/dist/executor/fileRoleLaunchPlanner.js +24 -90
- package/dist/executor/turnCompletion.js +5 -5
- package/dist/lifecycle/exactRunTerminalization.js +1 -1
- package/dist/observability/executionAudit.js +40 -94
- package/dist/operator/operatorSessionHistory.js +7 -5
- package/dist/role/role.js +1 -1
- package/dist/run/agentRun.js +4 -54
- package/dist/runtime/agentDriver.js +2 -0
- package/dist/runtime/agentError.js +114 -0
- package/dist/runtime/agentHost.js +55 -82
- package/dist/runtime/builtinAgentDrivers.js +21 -9
- package/dist/runtime/builtinAgentErrorMappers.js +150 -0
- package/dist/runtime/exactControlPlane.js +6 -12
- package/dist/runtime/index.js +0 -1
- package/dist/runtime/launchBroker.js +5 -19
- package/dist/runtime/lifecycleReservation.js +20 -4
- package/dist/runtime/providerRuntimeIdentity.js +3 -2
- package/dist/runtime/runtimeBinding.js +0 -27
- package/dist/runtime/runtimeObservation.js +7 -16
- package/dist/runtime/runtimeSessionCandidate.js +3 -10
- package/dist/runtime/sessionLaunchRequest.js +1 -2
- package/dist/runtime/sessionReconciliation.js +2 -2
- package/dist/runtime/structuredProviderHost.js +44 -79
- package/dist/runtime/taskRuntimeIsolation.js +0 -7
- package/dist/runtime/tmuxAdapters.js +6 -49
- package/dist/scheduler/activeRoleRunDelivery.js +218 -168
- package/dist/scheduler/leaderWakeupProcessor.js +126 -86
- package/dist/scheduler/roleRunLiveness.js +4 -1
- package/dist/scheduler/roleRunStall.js +7 -11
- package/dist/scheduler/wakeReason.js +4 -0
- package/dist/storage/migration/productionRegistry.js +332 -0
- package/dist/storage/sqliteSchema.js +54 -2
- package/dist/storage/sqliteStore.js +11 -44
- package/dist/storage/taskStore.js +7 -35
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +30 -8
- package/skills/yui-operator/SKILL.md +14 -6
- package/skills/yui-runtime/SKILL.md +6 -4
- package/dist/lifecycle/providerErrorClass.js +0 -152
- package/dist/run/providerRetry.js +0 -226
- package/dist/run/providerRetryConfig.js +0 -27
- package/dist/runtime/providerErrorCodes.js +0 -278
- package/dist/runtime/providerRecoveryDecision.js +0 -55
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { isDeepStrictEqual } from "node:util";
|
|
3
|
-
import { activeLiveRoleAgentSession, bindTaskRoleProviderRuntime, bindTaskRoleRun, clearTaskRoleRun, createRoleSessionSet, markTaskRoleRunDelivered, markTaskRoleRunPushed,
|
|
3
|
+
import { activeLiveRoleAgentSession, bindTaskRoleProviderRuntime, bindTaskRoleRun, clearTaskRoleRun, createRoleSessionSet, markTaskRoleRunDelivered, markTaskRoleRunPushed, recordRoleAgentSession, replaceTaskRoleAgentSession, recordTaskRoleTurnBoundary, rememberRoleAgentCompletedTurn, detachRoleAgentSessionHost, updateRoleAgentSessionStatus, updateTaskRoleProviderRuntime } from "../executor/agentExecutor.js";
|
|
4
4
|
import { acceptProviderTurn, beginProviderTurn, createProviderRuntimeBinding, endProviderActivation, currentProviderActivation, currentProviderConversation, markProviderTurnDeliveryUnknown, rejectProviderTurn, settleProviderTurnSubmission, settleProviderTurn, startProviderActivation, supersedeProviderConversation, updateProviderConversationRecoverability } from "../runtime/providerRuntimeIdentity.js";
|
|
5
|
-
import { decideProviderRecovery } from "../runtime/providerRecoveryDecision.js";
|
|
6
5
|
import { hasRecentTurnId } from "../executor/turnCompletion.js";
|
|
7
6
|
import { createTaskEvent } from "../event/taskEvent.js";
|
|
8
7
|
import { operationalTaskRecords } from "../task/taskRecordRetirement.js";
|
|
@@ -12,11 +11,8 @@ import { answerInputRequest } from "../input/inputRequest.js";
|
|
|
12
11
|
import { activeRoleAgentBinding } from "../role/role.js";
|
|
13
12
|
import { effectiveLaunchWithTaskMainWorkspace, effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskSession, resolveEffectiveLaunch, validateEffectiveLaunchSnapshot } from "../executor/effectiveLaunch.js";
|
|
14
13
|
import { SYSTEM_OPERATOR_ROLE } from "../role/systemRoles.js";
|
|
15
|
-
import {
|
|
14
|
+
import { markAgentRunDelivered, markAgentRunPushed, agentRunDeliveryReceiptId } from "../run/agentRun.js";
|
|
16
15
|
import { markWorkflowOutcomeRequestDispatched } from "../run/runControlRequest.js";
|
|
17
|
-
import { classifyProviderError, isRetryableProviderErrorClass } from "../lifecycle/providerErrorClass.js";
|
|
18
|
-
import { deferProviderRetry, markProviderRetryDispatched, providerRetryIsDue, scheduleProviderRetry } from "../run/providerRetry.js";
|
|
19
|
-
import { providerRetryAdapterEnabled, providerRetryConfig } from "../run/providerRetryConfig.js";
|
|
20
16
|
import { classifyRuntimeProcessExit, validateRuntimeProcessExitObservation } from "../runtime/processExitObservation.js";
|
|
21
17
|
import { terminalizeExactTaskRun } from "../lifecycle/exactRunTerminalization.js";
|
|
22
18
|
import { createCanonicalLifecycleEvent, foldCanonicalLifecycleEvent } from "../lifecycle/canonicalLifecycleEvent.js";
|
|
@@ -28,15 +24,14 @@ import { wakeReason } from "../scheduler/wakeReason.js";
|
|
|
28
24
|
import { foldRunProgressFacts, latestRunDurableProgressAt, latestRunEventTime, latestStallEvidenceKey, isRoleRunStalled, RUN_PROGRESS_EVENT, RUN_DIAGNOSTIC_FINISHED_EVENT, RUN_RECOVERED_EVENT, RUN_STALLED_EVENT } from "../scheduler/roleRunStall.js";
|
|
29
25
|
import { projectProviderContinuations } from "../runtime/runtimeContinuationProjection.js";
|
|
30
26
|
import { providerContinuationKey } from "../runtime/providerContinuation.js";
|
|
31
|
-
import {
|
|
32
|
-
import { recordExecutionLaneResult } from "../execution/executionGroup.js";
|
|
33
|
-
import { formatAgentRunReceiptId, formatTaskRecordReference } from "../task/taskRecordReference.js";
|
|
27
|
+
import { formatAgentRunReceiptId } from "../task/taskRecordReference.js";
|
|
34
28
|
import { bindExecution, claimPending, claimInputDelivery as claimMailboxInputDelivery, completeInputDelivery as completeMailboxInputDelivery, completeProcessing, mailboxHasPending, mailboxHasWork, pendingLane, markInputDeliveryPushed as markMailboxInputDeliveryPushed, markInputDeliveryUnknown as markMailboxInputDeliveryUnknown, releaseInputDelivery as releaseMailboxInputDelivery, resolveInputDeliveryNotAccepted as resolveMailboxInputDeliveryNotAccepted, releaseProcessing } from "../coordination/workMailbox.js";
|
|
35
29
|
import { enqueueWork } from "../coordination/workMailboxQueue.js";
|
|
36
|
-
import { RUNTIME_CLEANUP_REQUIRED_REASON, RUNTIME_LAUNCH_RESERVED_REASON, RUNTIME_LIFECYCLE_OWNER, RuntimeLifecycleBusyError, hasRuntimeCleanupObligation, hasRuntimeLifecycleWork, isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
30
|
+
import { RUNTIME_CLEANUP_REQUIRED_REASON, RUNTIME_HOST_DETACH_REQUIRED_REASON, RUNTIME_LAUNCH_RESERVED_REASON, RUNTIME_LIFECYCLE_OWNER, RuntimeLifecycleBusyError, hasRuntimeCleanupObligation, hasRuntimeLifecycleWork, isRuntimeCleanupReason, isRuntimeLaunchReservation, runtimeCleanupDisposition, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
37
31
|
import { nativeSessionIdForLaunch } from "../runtime/preallocatedNativeSession.js";
|
|
38
32
|
import { builtinAgentDriverRegistry } from "../runtime/builtinAgentDrivers.js";
|
|
39
|
-
import {
|
|
33
|
+
import { standardAgentError } from "../runtime/agentError.js";
|
|
34
|
+
import { RUNTIME_OBSERVATION_TASK_EVENT, createRuntimeObservation, isRuntimeTokenEvidence, runtimeObservationFenceMatches, runtimeObservationFromTaskEvent, runtimeObservationRunFenceMatches, runtimeObservationTaskEventPayload } from "../runtime/runtimeObservation.js";
|
|
40
35
|
import { projectRuntimeTaskEvents } from "../runtime/runtimeProjection.js";
|
|
41
36
|
import { contextSnapshotRef } from "../context/contextSnapshot.js";
|
|
42
37
|
import { contextSnapshotDeltaRefIds, freezeRunContextSnapshot } from "../context/runContextPack.js";
|
|
@@ -126,51 +121,10 @@ export class FileSchedulerStoreAdapter {
|
|
|
126
121
|
}
|
|
127
122
|
case "turn.failed": {
|
|
128
123
|
const failureEvidence = input.payload.failure;
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
}
|
|
133
|
-
const active = this.store.getActiveAgentRun(taskId, input.fence.roleName);
|
|
134
|
-
const recordedTurn = this.store.getTaskRoleSessionSet(taskId, input.fence.roleName)?.providerBinding?.turn;
|
|
135
|
-
if (input.fence.runId !== undefined
|
|
136
|
-
&& active?.id !== input.fence.runId
|
|
137
|
-
&& recordedTurn !== null
|
|
138
|
-
&& recordedTurn !== undefined
|
|
139
|
-
&& recordedTurn.runId === input.fence.runId
|
|
140
|
-
&& recordedTurn.turnId === input.fence.nativeTurnId) {
|
|
141
|
-
outcome = this.foldProviderTurnBoundary(input, adapterId, "failed", input.payload.summary ?? `Provider Turn failed: ${failureEvidence.code}.`, now);
|
|
142
|
-
break;
|
|
143
|
-
}
|
|
144
|
-
const failure = {
|
|
145
|
-
eventId: input.eventId,
|
|
146
|
-
eventType: input.kind,
|
|
147
|
-
taskId,
|
|
148
|
-
roleName: input.fence.roleName,
|
|
149
|
-
agentId: input.fence.agentId,
|
|
150
|
-
adapterId,
|
|
151
|
-
launchId: input.fence.launchId,
|
|
152
|
-
nativeSessionId: input.fence.nativeSessionId,
|
|
153
|
-
nativeTurnId: input.fence.nativeTurnId,
|
|
154
|
-
runId: input.fence.runId,
|
|
155
|
-
...(failureEvidence.errorCode === undefined
|
|
156
|
-
? {}
|
|
157
|
-
: { errorCode: failureEvidence.errorCode }),
|
|
158
|
-
error: failureEvidence.code,
|
|
159
|
-
...(failureEvidence.details === undefined
|
|
160
|
-
? {}
|
|
161
|
-
: { errorDetails: failureEvidence.details }),
|
|
162
|
-
...(failureEvidence.lastOutput === undefined
|
|
163
|
-
? {}
|
|
164
|
-
: { lastAssistantMessage: failureEvidence.lastOutput }),
|
|
165
|
-
...(failureEvidence.retryAfterMs === undefined
|
|
166
|
-
? {}
|
|
167
|
-
: { retryAfterMs: failureEvidence.retryAfterMs })
|
|
168
|
-
};
|
|
169
|
-
if (this.classifyRuntimeTurnFailed(failure) !== "apply")
|
|
170
|
-
return "obsolete";
|
|
171
|
-
outcome = this.observeRuntimeTurnFailed(failure, now).disposition === "applied"
|
|
172
|
-
? "applied"
|
|
173
|
-
: "obsolete";
|
|
124
|
+
// A Provider Turn is an activation boundary, not a Yui Run outcome.
|
|
125
|
+
// Keep the Run and Session identity available for Agent-directed
|
|
126
|
+
// restore + submit, even when a provider marks its Turn terminal.
|
|
127
|
+
outcome = this.foldProviderTurnBoundary(input, adapterId, "failed", input.payload.summary ?? `Agent Turn failed: ${failureEvidence.error.message}`, now);
|
|
174
128
|
break;
|
|
175
129
|
}
|
|
176
130
|
case "operation.started":
|
|
@@ -206,12 +160,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
206
160
|
default:
|
|
207
161
|
outcome = "obsolete";
|
|
208
162
|
}
|
|
209
|
-
if (outcome === "applied"
|
|
210
|
-
&& input.kind === "activity.observed"
|
|
211
|
-
&& input.payload.activityId !== undefined
|
|
212
|
-
&& input.payload.usage === undefined) {
|
|
213
|
-
this.clearProviderRetryForProgress(input, "correlated-activity", now);
|
|
214
|
-
}
|
|
215
163
|
if (outcome === "applied")
|
|
216
164
|
this.persistRuntimeObservation(input, now);
|
|
217
165
|
return outcome;
|
|
@@ -313,8 +261,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
313
261
|
recordCanonicalObservationObsolete(store, input, "runtime-session-not-current", now);
|
|
314
262
|
return "obsolete";
|
|
315
263
|
}
|
|
316
|
-
const status =
|
|
317
|
-
let updatedSessions = updateRoleAgentSessionStatus(sessions, input.fence.agentId, status, now);
|
|
264
|
+
const status = "ended";
|
|
265
|
+
let updatedSessions = updateRoleAgentSessionStatus(sessions, input.fence.agentId, status, now, input.kind === "session.failed" ? "failed" : "stopped");
|
|
318
266
|
if (updatedSessions.providerBinding !== null) {
|
|
319
267
|
const activationId = input.fence.activationId ?? input.fence.launchId;
|
|
320
268
|
updatedSessions = updateTaskRoleProviderRuntime(updatedSessions, endProviderActivation(updatedSessions.providerBinding, activationId, {
|
|
@@ -488,13 +436,52 @@ export class FileSchedulerStoreAdapter {
|
|
|
488
436
|
lane: "normal"
|
|
489
437
|
});
|
|
490
438
|
}
|
|
491
|
-
if (input.kind === "turn.failed" && input.payload.failure
|
|
492
|
-
|
|
439
|
+
if (input.kind === "turn.failed" && input.payload.failure !== undefined) {
|
|
440
|
+
const failure = input.payload.failure;
|
|
441
|
+
const error = failure.error;
|
|
442
|
+
const run = input.fence.runId === undefined
|
|
443
|
+
? null
|
|
444
|
+
: store.getAgentRun(taskId, input.fence.runId);
|
|
445
|
+
const errorEvent = createTaskEvent(store.nextEventId(taskId), taskId, "runtime.agent-error", {
|
|
446
|
+
sourceEventId: input.eventId,
|
|
447
|
+
observationEventId,
|
|
448
|
+
runId: input.fence.runId ?? "",
|
|
449
|
+
roleName: input.fence.roleName,
|
|
450
|
+
agentId: input.fence.agentId,
|
|
451
|
+
adapterId: run?.effective.adapterId ?? "unknown",
|
|
452
|
+
driverId: input.fence.driverId,
|
|
453
|
+
launchId: input.fence.launchId,
|
|
454
|
+
nativeSessionId: input.fence.nativeSessionId ?? "",
|
|
455
|
+
nativeTurnId: input.fence.nativeTurnId ?? "",
|
|
456
|
+
source: error.source,
|
|
457
|
+
phase: error.phase,
|
|
458
|
+
category: error.category,
|
|
459
|
+
code: error.code,
|
|
460
|
+
message: error.message,
|
|
461
|
+
raw: error.raw,
|
|
462
|
+
inputDisposition: error.inputDisposition,
|
|
463
|
+
sessionDisposition: error.sessionDisposition,
|
|
464
|
+
...(error.retryAfterMs === undefined
|
|
465
|
+
? {}
|
|
466
|
+
: { retryAfterMs: String(error.retryAfterMs) }),
|
|
467
|
+
...(failure.lastOutput === undefined ? {} : { lastOutput: failure.lastOutput })
|
|
468
|
+
}, now);
|
|
469
|
+
store.saveEvent(taskId, errorEvent);
|
|
470
|
+
enqueueWork(store, { kind: "role", taskId, roleName: "leader" }, wakeReason("agent-error", errorEvent.id), now, [{ type: "event", taskId, id: errorEvent.id }], {
|
|
493
471
|
source: input.fence.driverId,
|
|
494
|
-
dedupeKey: input.
|
|
472
|
+
dedupeKey: `agent-error:${taskId}:${input.eventId}`,
|
|
495
473
|
deliveryMode: "followup",
|
|
496
474
|
lane: "normal"
|
|
497
475
|
});
|
|
476
|
+
if (input.fence.roleName === "leader"
|
|
477
|
+
&& error.sessionDisposition === "unrecoverable") {
|
|
478
|
+
enqueueWork(store, { kind: "operator" }, "leader-session-unrecoverable", now, [{ type: "event", taskId, id: errorEvent.id }], {
|
|
479
|
+
source: input.fence.driverId,
|
|
480
|
+
dedupeKey: `leader-session-unrecoverable:${taskId}:${input.eventId}`,
|
|
481
|
+
deliveryMode: "followup",
|
|
482
|
+
lane: "normal"
|
|
483
|
+
});
|
|
484
|
+
}
|
|
498
485
|
}
|
|
499
486
|
if ((input.kind === "operation.completed" || input.kind === "operation.failed")
|
|
500
487
|
&& input.payload.operation === "subagent") {
|
|
@@ -503,7 +490,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
503
490
|
// A live provider Session owns delivery of its native child result.
|
|
504
491
|
// Route a Yui wake only when that parent is gone and another observer
|
|
505
492
|
// supplied the child's terminal fact.
|
|
506
|
-
if (role !== null &&
|
|
493
|
+
if (role !== null && session?.status === "ended") {
|
|
507
494
|
routeRoleEvent(store, observationEvent, role.name, "detached-native-subagent-terminal", now);
|
|
508
495
|
}
|
|
509
496
|
}
|
|
@@ -626,7 +613,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
626
613
|
return null;
|
|
627
614
|
const sessions = this.store.getGlobalRoleSessionSet(SYSTEM_OPERATOR_ROLE);
|
|
628
615
|
const effectiveSession = sessions?.sessions[sessions.activeAgentId];
|
|
629
|
-
if (effectiveSession?.status !== "
|
|
616
|
+
if (effectiveSession?.status !== "active")
|
|
630
617
|
return null;
|
|
631
618
|
return {
|
|
632
619
|
roleName: SYSTEM_OPERATOR_ROLE,
|
|
@@ -634,15 +621,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
634
621
|
};
|
|
635
622
|
}
|
|
636
623
|
markOperatorTurnStarted(now) {
|
|
637
|
-
|
|
638
|
-
const sessions = store.getGlobalRoleSessionSet(SYSTEM_OPERATOR_ROLE);
|
|
639
|
-
if (sessions === null)
|
|
640
|
-
return;
|
|
641
|
-
const active = sessions.sessions[sessions.activeAgentId];
|
|
642
|
-
if (active === undefined || active.status !== "ready")
|
|
643
|
-
return;
|
|
644
|
-
store.saveGlobalRoleSessionSet(updateRoleAgentSessionStatus(sessions, sessions.activeAgentId, "running", now));
|
|
645
|
-
});
|
|
624
|
+
void now;
|
|
646
625
|
}
|
|
647
626
|
resolveExpiredInputRecommendations(now, taskIds) {
|
|
648
627
|
return this.store.transaction((store) => {
|
|
@@ -888,6 +867,53 @@ export class FileSchedulerStoreAdapter {
|
|
|
888
867
|
resolvedAt: input.now.toISOString()
|
|
889
868
|
});
|
|
890
869
|
store.saveTaskRoleSessionSet(updateTaskRoleProviderRuntime(sessions, updated, input.now));
|
|
870
|
+
const run = store.getAgentRun(input.taskId, input.runId);
|
|
871
|
+
const session = sessions.sessions[sessions.activeAgentId];
|
|
872
|
+
const driver = run === null
|
|
873
|
+
? null
|
|
874
|
+
: this.drivers.findByAdapterId(run.effective.adapterId);
|
|
875
|
+
const error = standardAgentError({
|
|
876
|
+
source: "driver",
|
|
877
|
+
phase: "turn-submit",
|
|
878
|
+
classification: driver?.runtime.mapError({
|
|
879
|
+
message: input.reason,
|
|
880
|
+
raw: input.raw
|
|
881
|
+
}),
|
|
882
|
+
message: input.reason,
|
|
883
|
+
raw: input.raw,
|
|
884
|
+
inputDisposition: input.status === "rejected" ? "not-accepted" : "unknown"
|
|
885
|
+
});
|
|
886
|
+
const errorEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.agent-error", {
|
|
887
|
+
sourceEventId: input.attemptId,
|
|
888
|
+
runId: input.runId,
|
|
889
|
+
roleName: input.roleName,
|
|
890
|
+
agentId: run?.effective.agentId ?? sessions.activeAgentId,
|
|
891
|
+
adapterId: run?.effective.adapterId ?? session?.adapterId ?? "unknown",
|
|
892
|
+
driverId: driver?.id ?? "unknown",
|
|
893
|
+
launchId: session?.launchId ?? "",
|
|
894
|
+
nativeSessionId: session?.nativeSessionId ?? "",
|
|
895
|
+
nativeTurnId: "",
|
|
896
|
+
source: error.source,
|
|
897
|
+
phase: error.phase,
|
|
898
|
+
category: error.category,
|
|
899
|
+
code: error.code,
|
|
900
|
+
message: error.message,
|
|
901
|
+
raw: error.raw,
|
|
902
|
+
inputDisposition: error.inputDisposition,
|
|
903
|
+
sessionDisposition: error.sessionDisposition
|
|
904
|
+
}, input.now);
|
|
905
|
+
store.saveEvent(input.taskId, errorEvent);
|
|
906
|
+
const route = input.roleName === "leader"
|
|
907
|
+
? { kind: "operator" }
|
|
908
|
+
: { kind: "role", taskId: input.taskId, roleName: "leader" };
|
|
909
|
+
enqueueWork(store, route, input.roleName === "leader"
|
|
910
|
+
? "leader-turn-submission-error"
|
|
911
|
+
: wakeReason("agent-error", errorEvent.id), input.now, [{ type: "event", taskId: input.taskId, id: errorEvent.id }], {
|
|
912
|
+
source: driver?.id ?? "agent-host",
|
|
913
|
+
dedupeKey: `agent-error:${input.taskId}:${input.attemptId}`,
|
|
914
|
+
deliveryMode: "followup",
|
|
915
|
+
lane: "normal"
|
|
916
|
+
});
|
|
891
917
|
});
|
|
892
918
|
}
|
|
893
919
|
getProviderAuthorityFence(input) {
|
|
@@ -924,6 +950,80 @@ export class FileSchedulerStoreAdapter {
|
|
|
924
950
|
enqueueLeaderWakeup(taskId, reason, now) {
|
|
925
951
|
return this.store.transaction((store) => (queueLeaderWakeup(store, taskId, reason, now)));
|
|
926
952
|
}
|
|
953
|
+
recordAgentError(input, now) {
|
|
954
|
+
return this.store.transaction((store) => {
|
|
955
|
+
const duplicate = [...store.listEvents(input.taskId)].reverse().find((event) => (event.type === "runtime.agent-error"
|
|
956
|
+
&& event.payload.runId === input.runId
|
|
957
|
+
&& event.payload.phase === input.phase
|
|
958
|
+
&& event.payload.raw === input.raw));
|
|
959
|
+
if (duplicate !== undefined)
|
|
960
|
+
return duplicate.id;
|
|
961
|
+
const run = store.getAgentRun(input.taskId, input.runId);
|
|
962
|
+
const sessionSet = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
963
|
+
const sessionAgentId = run?.effective.agentId ?? sessionSet?.activeAgentId;
|
|
964
|
+
const session = sessionAgentId === undefined
|
|
965
|
+
? undefined
|
|
966
|
+
: sessionSet?.sessions[sessionAgentId];
|
|
967
|
+
const driver = run === null
|
|
968
|
+
? null
|
|
969
|
+
: this.drivers.findByAdapterId(run.effective.adapterId);
|
|
970
|
+
const error = standardAgentError({
|
|
971
|
+
source: input.source,
|
|
972
|
+
phase: input.phase,
|
|
973
|
+
classification: driver?.runtime.mapError({
|
|
974
|
+
message: input.message,
|
|
975
|
+
raw: input.raw
|
|
976
|
+
}),
|
|
977
|
+
message: input.message,
|
|
978
|
+
raw: input.raw,
|
|
979
|
+
...(input.inputDisposition === undefined
|
|
980
|
+
? {}
|
|
981
|
+
: { inputDisposition: input.inputDisposition }),
|
|
982
|
+
...(input.sessionDisposition === undefined
|
|
983
|
+
? {}
|
|
984
|
+
: { sessionDisposition: input.sessionDisposition })
|
|
985
|
+
});
|
|
986
|
+
const event = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.agent-error", {
|
|
987
|
+
sourceEventId: `${input.runId}:${input.phase}`,
|
|
988
|
+
runId: input.runId,
|
|
989
|
+
roleName: input.roleName,
|
|
990
|
+
agentId: run?.effective.agentId ?? session?.agentId ?? "unknown",
|
|
991
|
+
adapterId: run?.effective.adapterId ?? session?.adapterId ?? "unknown",
|
|
992
|
+
driverId: driver?.id ?? "unknown",
|
|
993
|
+
launchId: session?.launchId ?? "",
|
|
994
|
+
nativeSessionId: session?.nativeSessionId ?? "",
|
|
995
|
+
nativeTurnId: "",
|
|
996
|
+
source: error.source,
|
|
997
|
+
phase: error.phase,
|
|
998
|
+
category: error.category,
|
|
999
|
+
code: error.code,
|
|
1000
|
+
message: error.message,
|
|
1001
|
+
raw: error.raw,
|
|
1002
|
+
inputDisposition: error.inputDisposition,
|
|
1003
|
+
sessionDisposition: error.sessionDisposition
|
|
1004
|
+
}, now);
|
|
1005
|
+
store.saveEvent(input.taskId, event);
|
|
1006
|
+
const leaderCannotReceive = input.roleName === "leader"
|
|
1007
|
+
&& ["host-start", "session-start", "session-restore", "turn-submit"].includes(input.phase);
|
|
1008
|
+
if (!leaderCannotReceive) {
|
|
1009
|
+
enqueueWork(store, { kind: "role", taskId: input.taskId, roleName: "leader" }, wakeReason("agent-error", event.id), now, [{ type: "event", taskId: input.taskId, id: event.id }], {
|
|
1010
|
+
source: driver?.id ?? input.source,
|
|
1011
|
+
dedupeKey: `agent-error:${input.taskId}:${input.runId}:${input.phase}:${event.id}`,
|
|
1012
|
+
deliveryMode: "followup",
|
|
1013
|
+
lane: "normal"
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
1016
|
+
if (leaderCannotReceive) {
|
|
1017
|
+
enqueueWork(store, { kind: "operator" }, "leader-agent-error", now, [{ type: "event", taskId: input.taskId, id: event.id }], {
|
|
1018
|
+
source: driver?.id ?? input.source,
|
|
1019
|
+
dedupeKey: `leader-agent-error:${input.taskId}:${input.runId}:${event.id}`,
|
|
1020
|
+
deliveryMode: "followup",
|
|
1021
|
+
lane: "normal"
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
return event.id;
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
927
1027
|
/**
|
|
928
1028
|
* Records that a Leader wake was suppressed by scheduler single-flight
|
|
929
1029
|
* (the Role runtime lifecycle lane was busy). The wake stays durable and
|
|
@@ -1007,15 +1107,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1007
1107
|
const pending = pendingLane(mailbox, input.lane);
|
|
1008
1108
|
if (pending === null)
|
|
1009
1109
|
return { status: "empty" };
|
|
1010
|
-
const claimed = claimMailboxInputDelivery(mailbox, {
|
|
1011
|
-
attemptId: input.attemptId,
|
|
1012
|
-
lane: input.lane,
|
|
1013
|
-
mode: input.mode,
|
|
1014
|
-
owner: input.owner,
|
|
1015
|
-
startedAt: input.now.toISOString(),
|
|
1016
|
-
executionRef: input.executionRef,
|
|
1017
|
-
providerFence: input.providerFence
|
|
1018
|
-
});
|
|
1019
1110
|
if (input.target.kind === "role"
|
|
1020
1111
|
&& input.executionRef.type === "run"
|
|
1021
1112
|
&& input.executionRef.taskId === input.target.taskId) {
|
|
@@ -1026,6 +1117,11 @@ export class FileSchedulerStoreAdapter {
|
|
|
1026
1117
|
|| binding.authority.owner !== "controller") {
|
|
1027
1118
|
throw new Error("Controller does not own the Provider writer authority.");
|
|
1028
1119
|
}
|
|
1120
|
+
if (binding.turn !== null
|
|
1121
|
+
&& ["submitting", "accepted", "running", "delivery-unknown"]
|
|
1122
|
+
.includes(binding.turn.status)) {
|
|
1123
|
+
return { status: "busy" };
|
|
1124
|
+
}
|
|
1029
1125
|
const conversation = currentProviderConversation(binding);
|
|
1030
1126
|
const activation = currentProviderActivation(binding);
|
|
1031
1127
|
if (input.providerFence === undefined
|
|
@@ -1041,6 +1137,15 @@ export class FileSchedulerStoreAdapter {
|
|
|
1041
1137
|
});
|
|
1042
1138
|
store.saveTaskRoleSessionSet(updateTaskRoleProviderRuntime(sessions, next, input.now));
|
|
1043
1139
|
}
|
|
1140
|
+
const claimed = claimMailboxInputDelivery(mailbox, {
|
|
1141
|
+
attemptId: input.attemptId,
|
|
1142
|
+
lane: input.lane,
|
|
1143
|
+
mode: input.mode,
|
|
1144
|
+
owner: input.owner,
|
|
1145
|
+
startedAt: input.now.toISOString(),
|
|
1146
|
+
executionRef: input.executionRef,
|
|
1147
|
+
providerFence: input.providerFence
|
|
1148
|
+
});
|
|
1044
1149
|
store.saveWorkMailbox(claimed);
|
|
1045
1150
|
return { status: "claimed", delivery: claimed.inputDelivery };
|
|
1046
1151
|
});
|
|
@@ -1144,19 +1249,20 @@ export class FileSchedulerStoreAdapter {
|
|
|
1144
1249
|
completeRuntimeCleanup(target, now) {
|
|
1145
1250
|
return this.store.transaction((store) => {
|
|
1146
1251
|
let mailbox = store.getWorkMailbox(target);
|
|
1147
|
-
|
|
1252
|
+
const disposition = runtimeCleanupDisposition(mailbox);
|
|
1253
|
+
if (mailbox === null || disposition === null)
|
|
1148
1254
|
return false;
|
|
1149
1255
|
if (mailbox.processing !== null) {
|
|
1150
1256
|
if (!isRuntimeLaunchReservation(mailbox.processing)
|
|
1151
|
-
&& !mailbox.processing.batch.reasons.
|
|
1257
|
+
&& !mailbox.processing.batch.reasons.every(isRuntimeCleanupReason)) {
|
|
1152
1258
|
return false;
|
|
1153
1259
|
}
|
|
1154
1260
|
mailbox = completeProcessing(mailbox, mailbox.processing.batchId);
|
|
1155
1261
|
}
|
|
1156
1262
|
const pending = pendingLane(mailbox, "normal");
|
|
1157
1263
|
if (pending !== null) {
|
|
1158
|
-
if (pending.reasons.length
|
|
1159
|
-
|| pending.reasons
|
|
1264
|
+
if (pending.reasons.length === 0
|
|
1265
|
+
|| !pending.reasons.every(isRuntimeCleanupReason)) {
|
|
1160
1266
|
return false;
|
|
1161
1267
|
}
|
|
1162
1268
|
const batchId = `runtime-cleanup-complete:${pending.fromSequence}-${pending.toSequence}`;
|
|
@@ -1167,7 +1273,12 @@ export class FileSchedulerStoreAdapter {
|
|
|
1167
1273
|
}), batchId);
|
|
1168
1274
|
}
|
|
1169
1275
|
const owner = runtimeOwnerFromTarget(target);
|
|
1170
|
-
|
|
1276
|
+
if (disposition === "end-session") {
|
|
1277
|
+
endRuntimeOwnerSession(store, owner, now);
|
|
1278
|
+
}
|
|
1279
|
+
else {
|
|
1280
|
+
detachRuntimeOwnerHost(store, owner, now);
|
|
1281
|
+
}
|
|
1171
1282
|
saveRuntimeLifecycleMailbox(store, mailbox);
|
|
1172
1283
|
return true;
|
|
1173
1284
|
});
|
|
@@ -1178,7 +1289,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1178
1289
|
if (!isRuntimeLaunchReservation(mailbox?.processing, batchId)) {
|
|
1179
1290
|
return false;
|
|
1180
1291
|
}
|
|
1181
|
-
|
|
1292
|
+
endRuntimeOwnerSession(store, runtimeOwnerFromTarget(target), now);
|
|
1182
1293
|
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, batchId));
|
|
1183
1294
|
return true;
|
|
1184
1295
|
});
|
|
@@ -1203,14 +1314,22 @@ export class FileSchedulerStoreAdapter {
|
|
|
1203
1314
|
listRuntimeSessionCandidates(query = {}) {
|
|
1204
1315
|
return this.store.listRuntimeSessionCandidates(query);
|
|
1205
1316
|
}
|
|
1206
|
-
|
|
1317
|
+
enqueueRuntimeCleanup(owner, now = new Date(), expectedDormantCandidate) {
|
|
1207
1318
|
return this.store.transaction((store) => {
|
|
1208
|
-
if (
|
|
1209
|
-
|
|
1210
|
-
|
|
1319
|
+
if (expectedDormantCandidate !== undefined
|
|
1320
|
+
&& (!sameRuntimeOwner(owner, expectedDormantCandidate.owner)
|
|
1321
|
+
|| !dormantRuntimeCandidateIsCurrent(store, expectedDormantCandidate))) {
|
|
1322
|
+
return null;
|
|
1323
|
+
}
|
|
1324
|
+
if (owner.scope === "task" && store.getTask(owner.taskId) === null) {
|
|
1325
|
+
return null;
|
|
1326
|
+
}
|
|
1327
|
+
const target = runtimeLifecycleTarget(owner);
|
|
1328
|
+
enqueueWork(store, target, RUNTIME_CLEANUP_REQUIRED_REASON, now, owner.scope === "task" ? [{ type: "task", id: owner.taskId }] : []);
|
|
1329
|
+
return target;
|
|
1211
1330
|
});
|
|
1212
1331
|
}
|
|
1213
|
-
|
|
1332
|
+
enqueueRuntimeHostDetach(owner, now = new Date(), expectedDormantCandidate) {
|
|
1214
1333
|
return this.store.transaction((store) => {
|
|
1215
1334
|
if (expectedDormantCandidate !== undefined
|
|
1216
1335
|
&& (!sameRuntimeOwner(owner, expectedDormantCandidate.owner)
|
|
@@ -1221,7 +1340,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1221
1340
|
return null;
|
|
1222
1341
|
}
|
|
1223
1342
|
const target = runtimeLifecycleTarget(owner);
|
|
1224
|
-
enqueueWork(store, target,
|
|
1343
|
+
enqueueWork(store, target, RUNTIME_HOST_DETACH_REQUIRED_REASON, now, owner.scope === "task" ? [{ type: "task", id: owner.taskId }] : []);
|
|
1225
1344
|
return target;
|
|
1226
1345
|
});
|
|
1227
1346
|
}
|
|
@@ -1309,7 +1428,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1309
1428
|
saveTaskSession(store, role, {
|
|
1310
1429
|
...input.session,
|
|
1311
1430
|
nativeSessionId: input.session.nativeSessionId
|
|
1312
|
-
}, "
|
|
1431
|
+
}, "active", input.now);
|
|
1313
1432
|
}
|
|
1314
1433
|
store.clearLeaderFailure(input.task.id);
|
|
1315
1434
|
return "claimed";
|
|
@@ -1327,17 +1446,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1327
1446
|
throw new Error(`Active Agent run changed before delivery was persisted: ${input.run.id}.`);
|
|
1328
1447
|
}
|
|
1329
1448
|
let deliveryRun = active;
|
|
1330
|
-
if (active.providerRetry?.state === "dispatching") {
|
|
1331
|
-
deliveryRun = withProviderRetry(active, markProviderRetryDispatched(active.providerRetry));
|
|
1332
|
-
store.saveAgentRun(deliveryRun);
|
|
1333
|
-
store.saveEvent(task.id, createTaskEvent(store.nextEventId(task.id), task.id, "runtime.provider-retry-dispatched", {
|
|
1334
|
-
runId: active.id,
|
|
1335
|
-
roleName: active.roleName,
|
|
1336
|
-
episodeId: active.providerRetry.episodeId,
|
|
1337
|
-
receiptId: active.providerRetry.lastRetryReceiptId ?? "",
|
|
1338
|
-
dispatchedRetries: String(active.providerRetry.dispatchedRetries + 1)
|
|
1339
|
-
}, input.now));
|
|
1340
|
-
}
|
|
1341
1449
|
if (active.controlRequest?.state === "dispatching") {
|
|
1342
1450
|
deliveryRun = {
|
|
1343
1451
|
...active,
|
|
@@ -1370,18 +1478,16 @@ export class FileSchedulerStoreAdapter {
|
|
|
1370
1478
|
}
|
|
1371
1479
|
if (input.session !== null && input.session.nativeSessionId !== undefined) {
|
|
1372
1480
|
const existing = store.getRoleSession(input.task.id, input.role.name);
|
|
1373
|
-
const reservation = matchingPreparedRuntimeReservation(store, input);
|
|
1374
1481
|
if (existing?.agentId !== input.session.agentId
|
|
1375
1482
|
|| existing.adapterId !== input.session.adapterId
|
|
1376
1483
|
|| existing.nativeSessionId !== input.session.nativeSessionId
|
|
1377
1484
|
|| (input.launchId !== undefined && existing.launchId !== input.launchId)
|
|
1378
|
-
|| existing.status !== "
|
|
1485
|
+
|| existing.status !== "active") {
|
|
1379
1486
|
saveTaskSession(store, role, {
|
|
1380
1487
|
...input.session,
|
|
1381
1488
|
nativeSessionId: input.session.nativeSessionId
|
|
1382
|
-
}, "
|
|
1489
|
+
}, "active", input.now, input.launchId);
|
|
1383
1490
|
}
|
|
1384
|
-
completePreparedRuntimeReservation(store, reservation);
|
|
1385
1491
|
}
|
|
1386
1492
|
});
|
|
1387
1493
|
}
|
|
@@ -1398,7 +1504,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1398
1504
|
}
|
|
1399
1505
|
if (input.session !== null && input.session.nativeSessionId !== undefined) {
|
|
1400
1506
|
const existing = store.getRoleSession(input.task.id, input.role.name);
|
|
1401
|
-
matchingPreparedRuntimeReservation(store, input);
|
|
1402
1507
|
// A terminal Session is audit history, not the current execution
|
|
1403
1508
|
// identity. Preallocated providers must publish the replacement fence
|
|
1404
1509
|
// before the new Agent Host starts so its exact-runtime preflight sees
|
|
@@ -1406,16 +1511,15 @@ export class FileSchedulerStoreAdapter {
|
|
|
1406
1511
|
const defersConversationReplacement = active.mode === "new"
|
|
1407
1512
|
&& existing !== null
|
|
1408
1513
|
&& existing.nativeSessionId !== input.session.nativeSessionId
|
|
1409
|
-
&& existing.status
|
|
1410
|
-
&& existing.status !== "broken";
|
|
1514
|
+
&& existing.status === "active";
|
|
1411
1515
|
if (!defersConversationReplacement
|
|
1412
1516
|
&& (existing?.nativeSessionId !== input.session.nativeSessionId
|
|
1413
1517
|
|| (input.launchId !== undefined && existing.launchId !== input.launchId)
|
|
1414
|
-
|| existing.status !== "
|
|
1518
|
+
|| existing.status !== "active")) {
|
|
1415
1519
|
saveTaskSession(store, role, {
|
|
1416
1520
|
...input.session,
|
|
1417
1521
|
nativeSessionId: input.session.nativeSessionId
|
|
1418
|
-
}, "
|
|
1522
|
+
}, "active", input.now, input.launchId);
|
|
1419
1523
|
}
|
|
1420
1524
|
}
|
|
1421
1525
|
bindTaskRoleRunInFlight(store, role, input.run, input.now);
|
|
@@ -1433,11 +1537,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1433
1537
|
taskId: input.taskId,
|
|
1434
1538
|
roleName: input.roleName
|
|
1435
1539
|
});
|
|
1436
|
-
const runtimeMailbox = store.getWorkMailbox(runtimeLifecycleTarget({
|
|
1437
|
-
scope: "task",
|
|
1438
|
-
taskId: input.taskId,
|
|
1439
|
-
roleName: input.roleName
|
|
1440
|
-
}));
|
|
1441
1540
|
if (task === null
|
|
1442
1541
|
|| task.status !== "active"
|
|
1443
1542
|
|| task.executionGate.state !== "enabled"
|
|
@@ -1452,12 +1551,11 @@ export class FileSchedulerStoreAdapter {
|
|
|
1452
1551
|
|| mailbox.processing.executionRef?.type !== "run"
|
|
1453
1552
|
|| mailbox.processing.executionRef.taskId !== input.taskId
|
|
1454
1553
|
|| mailbox.processing.executionRef.id !== input.runId
|
|
1455
|
-
|| !preparedDeliveryFailureSessionFenceMatches(sessions, session, input, agentRunDeliveryReceiptId(active))
|
|
1456
|
-
|| !preparedReservationMatches(runtimeMailbox, input.taskId, input.runId, input.launchId)) {
|
|
1554
|
+
|| !preparedDeliveryFailureSessionFenceMatches(sessions, session, input, agentRunDeliveryReceiptId(active))) {
|
|
1457
1555
|
return "state-changed";
|
|
1458
1556
|
}
|
|
1459
1557
|
const summary = input.summary
|
|
1460
|
-
?? `Role delivery
|
|
1558
|
+
?? `Role delivery failed conclusively before exact Run input acceptance: ${input.runId}.`;
|
|
1461
1559
|
const result = terminalizeExactTaskRun(store, {
|
|
1462
1560
|
taskId: input.taskId,
|
|
1463
1561
|
roleName: input.roleName,
|
|
@@ -1478,7 +1576,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1478
1576
|
scope: "task",
|
|
1479
1577
|
taskId: input.taskId,
|
|
1480
1578
|
roleName: input.roleName
|
|
1481
|
-
}),
|
|
1579
|
+
}), RUNTIME_HOST_DETACH_REQUIRED_REASON, input.now, [{ type: "task", id: input.taskId }]);
|
|
1482
1580
|
}
|
|
1483
1581
|
const terminal = result.run;
|
|
1484
1582
|
const deliveryFailureEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.role-delivery-failed", {
|
|
@@ -1492,160 +1590,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1492
1590
|
}
|
|
1493
1591
|
else {
|
|
1494
1592
|
store.saveLeaderFailure(recordLeaderFailure(input.taskId, session?.nativeSessionId ?? "(unregistered)", summary, input.now, store.getLeaderFailure(input.taskId)));
|
|
1495
|
-
routeRoleEvent(store, deliveryFailureEvent, input.roleName, "leader-run-failed", input.now);
|
|
1496
|
-
}
|
|
1497
|
-
return "failed";
|
|
1498
|
-
});
|
|
1499
|
-
}
|
|
1500
|
-
saveLeaderDispatchFailure(input) {
|
|
1501
|
-
return this.store.transaction((store) => {
|
|
1502
|
-
const task = store.getTask(input.task.id);
|
|
1503
|
-
const role = store.getRole(input.task.id, input.role.name);
|
|
1504
|
-
const active = store.getActiveAgentRun(input.task.id, input.role.name);
|
|
1505
|
-
const target = {
|
|
1506
|
-
kind: "role",
|
|
1507
|
-
taskId: input.task.id,
|
|
1508
|
-
roleName: input.role.name
|
|
1509
|
-
};
|
|
1510
|
-
const mailbox = store.getWorkMailbox(target);
|
|
1511
|
-
if (task === null
|
|
1512
|
-
|| task.status !== "active"
|
|
1513
|
-
|| task.executionGate.state !== "enabled"
|
|
1514
|
-
|| role === null
|
|
1515
|
-
|| active === null
|
|
1516
|
-
|| active.id !== input.claimed.run.id
|
|
1517
|
-
|| active.status !== "active"
|
|
1518
|
-
|| mailbox?.processing?.executionRef?.type !== "run"
|
|
1519
|
-
|| mailbox.processing.executionRef.taskId !== input.task.id
|
|
1520
|
-
|| mailbox.processing.executionRef.id !== input.claimed.run.id
|
|
1521
|
-
|| !schedulerRoleSessionsMatch(store.getRoleSession(input.task.id, input.role.name), input.session)) {
|
|
1522
|
-
return "state-changed";
|
|
1523
|
-
}
|
|
1524
|
-
store.saveAgentRun(failAgentRun(active, input.failure.message, input.now));
|
|
1525
|
-
store.clearActiveAgentRun(input.task.id, input.role.name);
|
|
1526
|
-
clearTaskRoleRunInFlight(store, role, active, input.now);
|
|
1527
|
-
// Consume this failed delivery attempt. Releasing it would immediately
|
|
1528
|
-
// redispatch the same wake and manufacture another Leader Session.
|
|
1529
|
-
store.saveWorkMailbox(completeProcessing(mailbox, mailbox.processing.batchId));
|
|
1530
|
-
breakTaskSessionIfPresent(store, input.task.id, role.name, active.effective.agentId, input.now);
|
|
1531
|
-
const failure = recordLeaderFailure(input.task.id, input.failure.nativeSessionId, input.failure.message, input.now, store.getLeaderFailure(input.task.id));
|
|
1532
|
-
store.saveLeaderFailure(failure);
|
|
1533
|
-
recordLeaderAttentionRequired(store, {
|
|
1534
|
-
taskId: input.task.id,
|
|
1535
|
-
reason: "leader-recovery-failed",
|
|
1536
|
-
payload: {
|
|
1537
|
-
message: input.failure.message,
|
|
1538
|
-
runId: active.id
|
|
1539
|
-
},
|
|
1540
|
-
now: input.now
|
|
1541
|
-
});
|
|
1542
|
-
return "failed";
|
|
1543
|
-
});
|
|
1544
|
-
}
|
|
1545
|
-
saveExitedRoleRun(input) {
|
|
1546
|
-
return this.store.transaction((store) => {
|
|
1547
|
-
const task = store.getTask(input.task.id);
|
|
1548
|
-
const role = store.getRole(input.task.id, input.role.name);
|
|
1549
|
-
const currentRun = store.getActiveAgentRun(input.task.id, input.role.name);
|
|
1550
|
-
if (task === null
|
|
1551
|
-
|| task.status !== "active"
|
|
1552
|
-
|| task.executionGate.state !== "enabled"
|
|
1553
|
-
|| role === null
|
|
1554
|
-
|| currentRun === null
|
|
1555
|
-
|| currentRun.id !== input.run.id
|
|
1556
|
-
|| currentRun.status !== "active") {
|
|
1557
|
-
return "state-changed";
|
|
1558
|
-
}
|
|
1559
|
-
const sessions = store.getTaskRoleSessionSet(task.id, role.name);
|
|
1560
|
-
if (sessions?.inFlight !== null
|
|
1561
|
-
&& sessions !== null
|
|
1562
|
-
&& (sessions.inFlight.agentId !== currentRun.effective.agentId
|
|
1563
|
-
|| sessions.inFlight.runId !== currentRun.id
|
|
1564
|
-
|| sessions.inFlight.receiptId !== agentRunDeliveryReceiptId(currentRun))) {
|
|
1565
|
-
return "state-changed";
|
|
1566
|
-
}
|
|
1567
|
-
// Review launch failure uses the aggregate exact-terminalization path.
|
|
1568
|
-
// It owns mailbox settlement and ReviewRound convergence together, so a
|
|
1569
|
-
// pending pre-delivery dispatch cannot be consumed before its Run and
|
|
1570
|
-
// Round pass the same immutable identity fences.
|
|
1571
|
-
if (currentRun.purpose === "review") {
|
|
1572
|
-
if (input.run.taskId !== currentRun.taskId
|
|
1573
|
-
|| input.run.roleName !== currentRun.roleName
|
|
1574
|
-
|| input.run.purpose !== "review"
|
|
1575
|
-
|| input.run.reviewRoundId !== currentRun.reviewRoundId
|
|
1576
|
-
|| input.run.workItemId !== currentRun.workItemId
|
|
1577
|
-
|| input.run.effective.agentId !== currentRun.effective.agentId
|
|
1578
|
-
|| input.run.effective.adapterId !== currentRun.effective.adapterId) {
|
|
1579
|
-
return "state-changed";
|
|
1580
|
-
}
|
|
1581
|
-
const terminal = terminalizeExactTaskRun(store, {
|
|
1582
|
-
taskId: task.id,
|
|
1583
|
-
roleName: role.name,
|
|
1584
|
-
agentId: currentRun.effective.agentId,
|
|
1585
|
-
runId: currentRun.id,
|
|
1586
|
-
receiptId: agentRunDeliveryReceiptId(currentRun),
|
|
1587
|
-
...(input.session?.nativeSessionId === undefined
|
|
1588
|
-
? {}
|
|
1589
|
-
: { nativeSessionId: input.session.nativeSessionId }),
|
|
1590
|
-
outcome: { status: "failed", summary: input.summary }
|
|
1591
|
-
}, input.now);
|
|
1592
|
-
if (terminal.disposition !== "applied")
|
|
1593
|
-
return "state-changed";
|
|
1594
|
-
stopTaskSessionIfPresent(store, task.id, role.name, currentRun.effective.agentId, input.now);
|
|
1595
|
-
queueLeaderWakeup(store, task.id, wakeReason("review-failed"), input.now);
|
|
1596
|
-
return "failed";
|
|
1597
|
-
}
|
|
1598
|
-
const target = { kind: "role", taskId: task.id, roleName: role.name };
|
|
1599
|
-
const mailbox = store.getWorkMailbox(target);
|
|
1600
|
-
const processing = mailbox?.processing;
|
|
1601
|
-
if (mailbox !== null && processing !== null && processing !== undefined) {
|
|
1602
|
-
if (processing.executionRef === undefined
|
|
1603
|
-
|| processing.executionRef.type !== "run"
|
|
1604
|
-
|| processing.executionRef.taskId !== task.id
|
|
1605
|
-
|| processing.executionRef.id !== currentRun.id) {
|
|
1606
|
-
return "state-changed";
|
|
1607
|
-
}
|
|
1608
|
-
}
|
|
1609
|
-
store.saveAgentRun(failAgentRun(currentRun, input.summary, input.now));
|
|
1610
|
-
store.clearActiveAgentRun(task.id, role.name);
|
|
1611
|
-
clearTaskRoleRunInFlight(store, role, currentRun, input.now);
|
|
1612
|
-
if (mailbox !== null && processing !== null && processing !== undefined) {
|
|
1613
|
-
store.saveWorkMailbox(completeProcessing(mailbox, processing.batchId));
|
|
1614
|
-
}
|
|
1615
|
-
if (currentRun.purpose === "execution" && currentRun.workItemId !== undefined) {
|
|
1616
|
-
const workItem = store.getWorkItem(task.id, currentRun.workItemId);
|
|
1617
|
-
if (workItem !== null && !["completed", "failed", "retired"].includes(workItem.status)) {
|
|
1618
|
-
// Terminalize the bound execution lane in the same transaction that
|
|
1619
|
-
// fails the Run. Without this, a scheduler-driven Worker failure
|
|
1620
|
-
// leaves the lane "running" behind a "failed" WorkItem, and a later
|
|
1621
|
-
// `yui task run retry` is rejected because the lane is not terminal.
|
|
1622
|
-
// The lane result and the WorkItem status are two ordered single-step
|
|
1623
|
-
// record revisions, matching the aggregate terminalization path.
|
|
1624
|
-
if (currentRun.executionGroupId !== undefined
|
|
1625
|
-
&& currentRun.executionLaneId !== undefined
|
|
1626
|
-
&& currentWorkItemExecutionGroup(workItem) !== undefined) {
|
|
1627
|
-
const group = currentWorkItemExecutionGroup(workItem);
|
|
1628
|
-
store.saveWorkItem(task.id, updateWorkItemExecutionGroup(workItem, recordExecutionLaneResult(group, currentRun.executionLaneId, { summary: input.summary }, "failed", input.now), input.now));
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
const leaderFailed = role.name === "leader";
|
|
1633
|
-
stopTaskSessionIfPresent(store, task.id, role.name, currentRun.effective.agentId, input.now);
|
|
1634
|
-
if (leaderFailed) {
|
|
1635
|
-
const failure = recordLeaderFailure(task.id, input.session?.nativeSessionId ?? "(unregistered)", input.summary, input.now, store.getLeaderFailure(task.id));
|
|
1636
|
-
store.saveLeaderFailure(failure);
|
|
1637
|
-
recordLeaderAttentionRequired(store, {
|
|
1638
|
-
taskId: task.id,
|
|
1639
|
-
reason: "leader-recovery-failed",
|
|
1640
|
-
payload: {
|
|
1641
|
-
message: failure.message,
|
|
1642
|
-
runId: currentRun.id
|
|
1643
|
-
},
|
|
1644
|
-
now: input.now
|
|
1645
|
-
});
|
|
1646
|
-
return "failed";
|
|
1647
1593
|
}
|
|
1648
|
-
queueLeaderWakeup(store, task.id, wakeReason("role-run-failed"), input.now);
|
|
1649
1594
|
return "failed";
|
|
1650
1595
|
});
|
|
1651
1596
|
}
|
|
@@ -1662,47 +1607,32 @@ export class FileSchedulerStoreAdapter {
|
|
|
1662
1607
|
if (hasRuntimeCleanupObligation(existing)) {
|
|
1663
1608
|
throw new Error("Runtime cleanup is still pending.");
|
|
1664
1609
|
}
|
|
1665
|
-
const executionRef = existing.processing.executionRef;
|
|
1666
|
-
if (executionRef?.type === "run"
|
|
1667
|
-
&& (input.owner.scope !== "task"
|
|
1668
|
-
|| executionRef.taskId !== input.owner.taskId)) {
|
|
1669
|
-
throw new Error("Runtime launch reservation belongs to another Task Run.");
|
|
1670
|
-
}
|
|
1671
1610
|
return {
|
|
1672
1611
|
status: "existing",
|
|
1673
|
-
launchId: existing.processing.batchId
|
|
1674
|
-
...(executionRef?.type === "run"
|
|
1675
|
-
? { runId: executionRef.id }
|
|
1676
|
-
: {})
|
|
1612
|
+
launchId: existing.processing.batchId
|
|
1677
1613
|
};
|
|
1678
1614
|
}
|
|
1679
|
-
if (existing
|
|
1680
|
-
&& (existing.processing !== null || existing.pending !== null)) {
|
|
1615
|
+
if (hasRuntimeLifecycleWork(existing)) {
|
|
1681
1616
|
throw new RuntimeLifecycleBusyError("Runtime lifecycle work is already pending.");
|
|
1682
1617
|
}
|
|
1683
1618
|
enqueueWork(store, target, RUNTIME_LAUNCH_RESERVED_REASON, now, input.owner.scope === "task"
|
|
1684
1619
|
? [{ type: "task", id: input.owner.taskId }]
|
|
1685
1620
|
: []);
|
|
1686
1621
|
const queued = store.getWorkMailbox(target);
|
|
1687
|
-
if (queued === null
|
|
1622
|
+
if (queued === null
|
|
1623
|
+
|| pendingLane(queued, "normal") === null
|
|
1624
|
+
|| queued.processing !== null) {
|
|
1688
1625
|
throw new Error("Runtime launch reservation could not be queued.");
|
|
1689
1626
|
}
|
|
1690
|
-
|
|
1627
|
+
const claimed = claimPending(queued, {
|
|
1691
1628
|
batchId: input.launchId,
|
|
1692
1629
|
owner: RUNTIME_LIFECYCLE_OWNER,
|
|
1693
1630
|
startedAt: now.toISOString()
|
|
1694
1631
|
});
|
|
1695
|
-
if (input.runId !== undefined) {
|
|
1696
|
-
if (input.owner.scope !== "task") {
|
|
1697
|
-
throw new Error("A Run-bound runtime reservation requires a Task owner.");
|
|
1698
|
-
}
|
|
1699
|
-
claimed = bindExecution(claimed, input.launchId, { type: "run", taskId: input.owner.taskId, id: input.runId });
|
|
1700
|
-
}
|
|
1701
1632
|
store.saveWorkMailbox(claimed);
|
|
1702
1633
|
return {
|
|
1703
1634
|
status: "reserved",
|
|
1704
|
-
launchId: input.launchId
|
|
1705
|
-
...(input.runId === undefined ? {} : { runId: input.runId })
|
|
1635
|
+
launchId: input.launchId
|
|
1706
1636
|
};
|
|
1707
1637
|
});
|
|
1708
1638
|
}
|
|
@@ -1722,7 +1652,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1722
1652
|
// the already-settled reservation as a launch failure.
|
|
1723
1653
|
const sessions = runtimeOwnerSessionSet(store, input.owner);
|
|
1724
1654
|
const active = sessions?.sessions[sessions.activeAgentId];
|
|
1725
|
-
if (active?.status === "
|
|
1655
|
+
if (active?.status === "active"
|
|
1726
1656
|
&& active.launchId === input.launchId)
|
|
1727
1657
|
return "provider-bound";
|
|
1728
1658
|
throw new Error("Runtime launch reservation no longer matches the launch.");
|
|
@@ -1757,25 +1687,12 @@ export class FileSchedulerStoreAdapter {
|
|
|
1757
1687
|
return session;
|
|
1758
1688
|
});
|
|
1759
1689
|
}
|
|
1760
|
-
completeRuntimeLaunchReservation(owner, launchId,
|
|
1690
|
+
completeRuntimeLaunchReservation(owner, launchId, beforeComplete) {
|
|
1761
1691
|
return this.store.transaction((store) => {
|
|
1762
1692
|
const target = runtimeLifecycleTarget(owner);
|
|
1763
1693
|
const mailbox = store.getWorkMailbox(target);
|
|
1764
1694
|
if (!isRuntimeLaunchReservation(mailbox?.processing, launchId))
|
|
1765
1695
|
return false;
|
|
1766
|
-
if (expectedTerminalRunId !== undefined) {
|
|
1767
|
-
const expectedRun = owner.scope === "task"
|
|
1768
|
-
? store.getAgentRun(owner.taskId, expectedTerminalRunId)
|
|
1769
|
-
: null;
|
|
1770
|
-
if (owner.scope !== "task"
|
|
1771
|
-
|| mailbox?.processing?.executionRef?.type !== "run"
|
|
1772
|
-
|| mailbox.processing.executionRef.taskId !== owner.taskId
|
|
1773
|
-
|| mailbox.processing.executionRef.id !== expectedTerminalRunId
|
|
1774
|
-
|| expectedRun === null
|
|
1775
|
-
|| expectedRun.status === "active") {
|
|
1776
|
-
return false;
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
1696
|
beforeComplete?.();
|
|
1780
1697
|
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, launchId));
|
|
1781
1698
|
return true;
|
|
@@ -1797,7 +1714,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1797
1714
|
const sessions = runtimeOwnerSessionSet(store, input.owner);
|
|
1798
1715
|
const active = sessions?.sessions[sessions.activeAgentId];
|
|
1799
1716
|
if (runtimeSessionMatchesSettledLaunch(active, input)) {
|
|
1800
|
-
|
|
1717
|
+
endRuntimeOwnerSession(store, input.owner, now);
|
|
1801
1718
|
}
|
|
1802
1719
|
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, input.launchId));
|
|
1803
1720
|
return true;
|
|
@@ -1814,8 +1731,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
1814
1731
|
const active = sessions.sessions[sessions.activeAgentId];
|
|
1815
1732
|
if (!runtimeSessionMatchesSettledLaunch(active, input))
|
|
1816
1733
|
return false;
|
|
1817
|
-
if (active.status !== "
|
|
1818
|
-
|
|
1734
|
+
if (active.status !== "ended") {
|
|
1735
|
+
endRuntimeOwnerSession(store, input.owner, now);
|
|
1819
1736
|
}
|
|
1820
1737
|
return true;
|
|
1821
1738
|
});
|
|
@@ -1945,19 +1862,17 @@ export class FileSchedulerStoreAdapter {
|
|
|
1945
1862
|
: sessions.inFlight !== null && sessions.inFlight.pushedAt === undefined) {
|
|
1946
1863
|
throw new Error("Runtime turn completion requires an independently committed transport receipt.");
|
|
1947
1864
|
}
|
|
1948
|
-
const
|
|
1949
|
-
|| effectiveExisting?.status === "broken"
|
|
1950
|
-
? effectiveExisting.status
|
|
1951
|
-
: "ready";
|
|
1865
|
+
const sessionStatus = effectiveExisting?.status ?? "active";
|
|
1952
1866
|
sessions = recordRoleAgentSession(sessions, {
|
|
1953
1867
|
agentId: input.agentId,
|
|
1954
1868
|
adapterId: input.adapterId,
|
|
1955
1869
|
nativeSessionId: input.nativeSessionId,
|
|
1956
1870
|
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
1957
1871
|
policy: "fixed",
|
|
1958
|
-
status:
|
|
1959
|
-
|
|
1960
|
-
|
|
1872
|
+
status: sessionStatus,
|
|
1873
|
+
...(effectiveExisting?.endReason === undefined
|
|
1874
|
+
? {}
|
|
1875
|
+
: { endReason: effectiveExisting.endReason }),
|
|
1961
1876
|
effective
|
|
1962
1877
|
}, now);
|
|
1963
1878
|
sessions = settleStructuredProviderTurn(sessions, canonicalTurnId, input.providerStatus ?? "completed", now);
|
|
@@ -1965,18 +1880,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1965
1880
|
sessions = rememberRoleAgentCompletedTurn(sessions, input.agentId, input.nativeSessionId, canonicalTurnId, now);
|
|
1966
1881
|
store.saveTaskRoleSessionSet(sessions);
|
|
1967
1882
|
completeRuntimeHookReservation(store, owner, input.launchId);
|
|
1968
|
-
// The in-flight fence was cleared by an in-place provider retry
|
|
1969
|
-
// decision. If the active Run is still in its retry window and this
|
|
1970
|
-
// completion belongs to it, the recovered CLI finished the original
|
|
1971
|
-
// turn: clear the retry projection so the deadline does not re-push
|
|
1972
|
-
// the prompt. The Run stays active for the normal completion path.
|
|
1973
|
-
const activeRun = store.getActiveAgentRun(task.id, role.name);
|
|
1974
|
-
if (activeRun !== null
|
|
1975
|
-
&& activeRun.providerRetry !== undefined
|
|
1976
|
-
&& input.runId === activeRun.id) {
|
|
1977
|
-
const { providerRetry: _removed, ...rest } = activeRun;
|
|
1978
|
-
store.saveAgentRun({ ...rest, updatedAt: now.toISOString() });
|
|
1979
|
-
}
|
|
1980
1883
|
return { session: sessions.sessions[input.agentId], duplicate: false };
|
|
1981
1884
|
}
|
|
1982
1885
|
// A provider Turn is an activation boundary, not a Yui Run outcome.
|
|
@@ -1996,367 +1899,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1996
1899
|
};
|
|
1997
1900
|
});
|
|
1998
1901
|
}
|
|
1999
|
-
classifyRuntimeTurnFailed(input) {
|
|
2000
|
-
const task = this.store.getTask(input.taskId);
|
|
2001
|
-
if (task?.status !== "active" || task.executionGate.state !== "enabled")
|
|
2002
|
-
return "obsolete";
|
|
2003
|
-
const role = this.store.getRole(input.taskId, input.roleName);
|
|
2004
|
-
if (role === null)
|
|
2005
|
-
return "obsolete";
|
|
2006
|
-
const run = this.store.getAgentRun(input.taskId, input.runId);
|
|
2007
|
-
const active = this.store.getActiveAgentRun(input.taskId, input.roleName);
|
|
2008
|
-
if (run === null
|
|
2009
|
-
|| run.status !== "active"
|
|
2010
|
-
|| active?.id !== run.id
|
|
2011
|
-
|| run.roleName !== input.roleName
|
|
2012
|
-
|| run.effective.agentId !== input.agentId
|
|
2013
|
-
|| run.effective.adapterId !== input.adapterId)
|
|
2014
|
-
return "obsolete";
|
|
2015
|
-
const sessions = this.store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
2016
|
-
const session = sessions?.sessions[input.agentId];
|
|
2017
|
-
if (sessions === null
|
|
2018
|
-
|| session?.nativeSessionId !== input.nativeSessionId
|
|
2019
|
-
|| session.launchId !== input.launchId
|
|
2020
|
-
|| sessions.inFlight?.agentId !== input.agentId
|
|
2021
|
-
|| sessions.inFlight.runId !== input.runId
|
|
2022
|
-
|| sessions.inFlight.receiptId !== agentRunDeliveryReceiptId(run))
|
|
2023
|
-
return "obsolete";
|
|
2024
|
-
return "apply";
|
|
2025
|
-
}
|
|
2026
|
-
observeRuntimeTurnFailed(input, now = new Date()) {
|
|
2027
|
-
return this.store.transaction((store) => {
|
|
2028
|
-
const before = store.getAgentRun(input.taskId, input.runId);
|
|
2029
|
-
if (before === null) {
|
|
2030
|
-
recordObsoleteRuntimeEvent(store, input, "run-missing", now);
|
|
2031
|
-
return { disposition: "obsolete", runId: input.runId };
|
|
2032
|
-
}
|
|
2033
|
-
const sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
2034
|
-
if (sessions?.providerBinding?.turn?.runId === input.runId
|
|
2035
|
-
&& sessions.providerBinding.turn.turnId === input.nativeTurnId) {
|
|
2036
|
-
store.saveTaskRoleSessionSet(settleStructuredProviderTurn(sessions, input.nativeTurnId, "failed", now));
|
|
2037
|
-
}
|
|
2038
|
-
const summary = runtimeTurnFailureSummary(input);
|
|
2039
|
-
// Issue 04: a classified transient Provider failure keeps the exact Run
|
|
2040
|
-
// and Native Session and is retried in place instead of terminalizing.
|
|
2041
|
-
const retryDecision = this.providerRetryDecision(store, input, summary, now);
|
|
2042
|
-
if (retryDecision !== null)
|
|
2043
|
-
return retryDecision;
|
|
2044
|
-
const result = terminalizeExactTaskRun(store, {
|
|
2045
|
-
taskId: input.taskId,
|
|
2046
|
-
roleName: input.roleName,
|
|
2047
|
-
agentId: input.agentId,
|
|
2048
|
-
runId: input.runId,
|
|
2049
|
-
receiptId: agentRunDeliveryReceiptId(before),
|
|
2050
|
-
nativeSessionId: input.nativeSessionId,
|
|
2051
|
-
launchId: input.launchId,
|
|
2052
|
-
outcome: {
|
|
2053
|
-
status: "failed",
|
|
2054
|
-
summary
|
|
2055
|
-
}
|
|
2056
|
-
}, now);
|
|
2057
|
-
if (result.disposition === "obsolete" || result.run === null) {
|
|
2058
|
-
recordObsoleteRuntimeEvent(store, input, result.reason ?? "identity-mismatch-or-terminal", now);
|
|
2059
|
-
return { disposition: "obsolete", runId: input.runId };
|
|
2060
|
-
}
|
|
2061
|
-
enqueueWork(store, runtimeLifecycleTarget({
|
|
2062
|
-
scope: "task",
|
|
2063
|
-
taskId: input.taskId,
|
|
2064
|
-
roleName: input.roleName
|
|
2065
|
-
}), RUNTIME_CLEANUP_REQUIRED_REASON, now, [{ type: "task", id: input.taskId }]);
|
|
2066
|
-
const terminal = result.run;
|
|
2067
|
-
const failureEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.turn-failed", {
|
|
2068
|
-
eventId: input.eventId,
|
|
2069
|
-
runId: input.runId,
|
|
2070
|
-
roleName: input.roleName,
|
|
2071
|
-
outcome: terminal.status
|
|
2072
|
-
}, now);
|
|
2073
|
-
store.saveEvent(input.taskId, failureEvent);
|
|
2074
|
-
routeRoleEvent(store, failureEvent, input.roleName, input.roleName !== "leader" && before.purpose === "review"
|
|
2075
|
-
? "review-failed"
|
|
2076
|
-
: input.roleName === "leader" ? "leader-run-failed" : "role-run-failed", now);
|
|
2077
|
-
return { disposition: "applied", runId: input.runId };
|
|
2078
|
-
});
|
|
2079
|
-
}
|
|
2080
|
-
/**
|
|
2081
|
-
* Issue 04 in-place retry decision for one classified Provider failure.
|
|
2082
|
-
* Returns an observation when the failure was handled without terminalizing
|
|
2083
|
-
* the Run, or `null` to fall through to the existing terminalization path.
|
|
2084
|
-
*/
|
|
2085
|
-
providerRetryDecision(store, input, summary, now) {
|
|
2086
|
-
const config = providerRetryConfig(this.store.getConfig());
|
|
2087
|
-
if (config.mode === "off")
|
|
2088
|
-
return null;
|
|
2089
|
-
const classification = classifyProviderError({
|
|
2090
|
-
adapterId: input.adapterId,
|
|
2091
|
-
...(input.errorCode === undefined ? {} : { errorCode: input.errorCode }),
|
|
2092
|
-
error: input.error,
|
|
2093
|
-
errorDetails: input.errorDetails,
|
|
2094
|
-
summary: input.lastAssistantMessage
|
|
2095
|
-
});
|
|
2096
|
-
const adapterEnabled = providerRetryAdapterEnabled(config, input.adapterId);
|
|
2097
|
-
const retryable = adapterEnabled
|
|
2098
|
-
&& isRetryableProviderErrorClass(classification.errorClass);
|
|
2099
|
-
const shadow = config.mode === "shadow";
|
|
2100
|
-
// Shadow mode records the classification and "would retry" fact without
|
|
2101
|
-
// changing any behavior; the Run still terminalizes below.
|
|
2102
|
-
if (shadow || !retryable) {
|
|
2103
|
-
if (adapterEnabled) {
|
|
2104
|
-
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2105
|
-
wouldRetry: String(retryable),
|
|
2106
|
-
basis: classification.basis,
|
|
2107
|
-
shadow: String(shadow)
|
|
2108
|
-
}, now);
|
|
2109
|
-
}
|
|
2110
|
-
if (!shadow && classification.errorClass === "policy-denied") {
|
|
2111
|
-
const blocked = this.applyProviderPolicyBlock(store, input, summary, now);
|
|
2112
|
-
if (blocked !== null)
|
|
2113
|
-
return blocked;
|
|
2114
|
-
}
|
|
2115
|
-
if (!shadow && classification.errorClass === "context-capacity") {
|
|
2116
|
-
const blocked = this.applyContextCapacityBlock(store, input, summary, now);
|
|
2117
|
-
if (blocked !== null)
|
|
2118
|
-
return blocked;
|
|
2119
|
-
}
|
|
2120
|
-
if (!shadow && classification.errorClass === "session-dead") {
|
|
2121
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2122
|
-
if (run !== null && run.status === "active" && run.mode === "resume") {
|
|
2123
|
-
// A structured failure on the exact resume launch is the only Core
|
|
2124
|
-
// evidence that native continuation is unrecoverable. The normal
|
|
2125
|
-
// terminal path below may now request a fresh managed generation.
|
|
2126
|
-
store.saveEvent(input.taskId, createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.native-resume-unrecoverable", {
|
|
2127
|
-
eventId: input.eventId,
|
|
2128
|
-
runId: input.runId,
|
|
2129
|
-
roleName: input.roleName,
|
|
2130
|
-
launchId: input.launchId,
|
|
2131
|
-
nativeSessionId: input.nativeSessionId,
|
|
2132
|
-
...(run.assignment.contextSnapshotRef === undefined
|
|
2133
|
-
? {}
|
|
2134
|
-
: {
|
|
2135
|
-
contextSnapshotId: run.assignment.contextSnapshotRef.id,
|
|
2136
|
-
contextSnapshotDigest: run.assignment.contextSnapshotRef.digest
|
|
2137
|
-
})
|
|
2138
|
-
}, now));
|
|
2139
|
-
}
|
|
2140
|
-
else {
|
|
2141
|
-
const blocked = this.applyNativeSessionRecoverabilityBlock(store, input, summary, now);
|
|
2142
|
-
if (blocked !== null)
|
|
2143
|
-
return blocked;
|
|
2144
|
-
}
|
|
2145
|
-
}
|
|
2146
|
-
return null;
|
|
2147
|
-
}
|
|
2148
|
-
// transport-uncertain: the Provider may have accepted the turn. If a
|
|
2149
|
-
// native completion is already durable, the completion path owns the
|
|
2150
|
-
// yield; do not resend.
|
|
2151
|
-
if (classification.errorClass === "transport-uncertain") {
|
|
2152
|
-
const driver = this.drivers.requireByAdapterId(input.adapterId);
|
|
2153
|
-
if (driver.capabilities.lifecycle.deliveryDeduplication !== "exact") {
|
|
2154
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2155
|
-
if (run === null || run.status !== "active")
|
|
2156
|
-
return null;
|
|
2157
|
-
const blocked = scheduleProviderRetry(run.providerRetry, {
|
|
2158
|
-
failureEventId: input.eventId,
|
|
2159
|
-
errorClass: classification.errorClass,
|
|
2160
|
-
launchId: input.launchId,
|
|
2161
|
-
nativeSessionId: input.nativeSessionId,
|
|
2162
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2163
|
-
lastErrorSummary: summary,
|
|
2164
|
-
scheduleNextAttempt: false
|
|
2165
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2166
|
-
if (blocked.outcome === "exhausted")
|
|
2167
|
-
return null;
|
|
2168
|
-
store.saveAgentRun(withProviderRetry(run, blocked.retry));
|
|
2169
|
-
const classifiedEvent = this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2170
|
-
wouldRetry: "false",
|
|
2171
|
-
shadow: "false",
|
|
2172
|
-
note: "delivery-deduplication-unproven"
|
|
2173
|
-
}, now);
|
|
2174
|
-
routeRoleEvent(store, classifiedEvent, input.roleName, "provider-delivery-uncertain", now);
|
|
2175
|
-
return { disposition: "applied", runId: input.runId };
|
|
2176
|
-
}
|
|
2177
|
-
}
|
|
2178
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2179
|
-
if (run === null || run.status !== "active")
|
|
2180
|
-
return null;
|
|
2181
|
-
const retryDecision = scheduleProviderRetry(run.providerRetry, {
|
|
2182
|
-
failureEventId: input.eventId,
|
|
2183
|
-
errorClass: classification.errorClass,
|
|
2184
|
-
launchId: input.launchId,
|
|
2185
|
-
nativeSessionId: input.nativeSessionId,
|
|
2186
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2187
|
-
lastErrorSummary: summary,
|
|
2188
|
-
...(input.retryAfterMs === undefined ? {} : { retryAfterMs: input.retryAfterMs })
|
|
2189
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2190
|
-
if (retryDecision.outcome === "exhausted") {
|
|
2191
|
-
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2192
|
-
wouldRetry: "false",
|
|
2193
|
-
shadow: "false",
|
|
2194
|
-
note: `episode-exhausted-${retryDecision.reason}`,
|
|
2195
|
-
consecutiveFailures: String(run.providerRetry?.consecutiveFailures ?? 0),
|
|
2196
|
-
dispatchedRetries: String(run.providerRetry?.dispatchedRetries ?? 0)
|
|
2197
|
-
}, now);
|
|
2198
|
-
if (run.providerRetry === undefined)
|
|
2199
|
-
return null;
|
|
2200
|
-
const finalized = finalizeProviderRetryDeadline(store, run, retryDecision.reason === "attempts"
|
|
2201
|
-
? `Provider retry failed after all ${run.providerRetry.maxRetries} in-Session continuation attempts.`
|
|
2202
|
-
: retryDecision.reason === "retry-after-window"
|
|
2203
|
-
? `Provider Retry-After exceeded the bounded ${config.maxWindowMs / 1_000}-second episode window.`
|
|
2204
|
-
: `Provider retry did not recover within the bounded ${config.maxWindowMs / 1_000}-second episode window.`, retryDecision.reason === "attempts" ? "attempts-exhausted" : "episode-window-exhausted", now);
|
|
2205
|
-
return finalized ? { disposition: "applied", runId: input.runId } : null;
|
|
2206
|
-
}
|
|
2207
|
-
const retry = retryDecision.retry;
|
|
2208
|
-
store.saveAgentRun(withProviderRetry(run, retry));
|
|
2209
|
-
// The in-flight fence and Provider Conversation stay bound while the
|
|
2210
|
-
// retry waits. resolveDueProviderRetries advances only the delivery
|
|
2211
|
-
// receipt immediately before the exact redispatch.
|
|
2212
|
-
// Settle the delivery claim so the retry push can re-claim the mailbox.
|
|
2213
|
-
const target = runtimeLifecycleTarget({
|
|
2214
|
-
scope: "task",
|
|
2215
|
-
taskId: input.taskId,
|
|
2216
|
-
roleName: input.roleName
|
|
2217
|
-
});
|
|
2218
|
-
const mailbox = store.getWorkMailbox(target);
|
|
2219
|
-
if (mailbox !== null && mailbox.processing !== null) {
|
|
2220
|
-
const settled = completeProcessing(mailbox, mailbox.processing.batchId);
|
|
2221
|
-
if (settled.processing === null && settled.pending === null) {
|
|
2222
|
-
store.removeWorkMailbox(target);
|
|
2223
|
-
}
|
|
2224
|
-
else {
|
|
2225
|
-
store.saveWorkMailbox(settled);
|
|
2226
|
-
}
|
|
2227
|
-
}
|
|
2228
|
-
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2229
|
-
wouldRetry: "true",
|
|
2230
|
-
shadow: "false",
|
|
2231
|
-
consecutiveFailures: String(retry.consecutiveFailures),
|
|
2232
|
-
dispatchedRetries: String(retry.dispatchedRetries),
|
|
2233
|
-
nextAttemptAt: retry.nextAttemptAt
|
|
2234
|
-
}, now);
|
|
2235
|
-
return { disposition: "applied", runId: input.runId };
|
|
2236
|
-
}
|
|
2237
|
-
/**
|
|
2238
|
-
* Issue 04 policy-denied handling: the Run stays active on its original
|
|
2239
|
-
* Session, no retry is scheduled, and the Leader receives a bounded blocker.
|
|
2240
|
-
* Yui never works around a policy denial by switching Session or widening
|
|
2241
|
-
* permission.
|
|
2242
|
-
*/
|
|
2243
|
-
applyProviderPolicyBlock(store, input, summary, now) {
|
|
2244
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2245
|
-
if (run === null || run.status !== "active")
|
|
2246
|
-
return null;
|
|
2247
|
-
const retryDecision = scheduleProviderRetry(run.providerRetry, {
|
|
2248
|
-
failureEventId: input.eventId,
|
|
2249
|
-
errorClass: "policy-denied",
|
|
2250
|
-
launchId: input.launchId,
|
|
2251
|
-
nativeSessionId: input.nativeSessionId,
|
|
2252
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2253
|
-
lastErrorSummary: summary,
|
|
2254
|
-
scheduleNextAttempt: false
|
|
2255
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2256
|
-
if (retryDecision.outcome === "exhausted")
|
|
2257
|
-
return null;
|
|
2258
|
-
store.saveAgentRun(withProviderRetry(run, retryDecision.retry));
|
|
2259
|
-
const classifiedEvent = this.recordProviderRetryClassified(store, input, "policy-denied", {
|
|
2260
|
-
wouldRetry: "false",
|
|
2261
|
-
shadow: "false",
|
|
2262
|
-
note: "policy-denied-blocker"
|
|
2263
|
-
}, now);
|
|
2264
|
-
routeRoleEvent(store, classifiedEvent, input.roleName, input.roleName === "leader"
|
|
2265
|
-
? "leader-provider-policy-blocked"
|
|
2266
|
-
: "provider-policy-blocked", now);
|
|
2267
|
-
return { disposition: "applied", runId: input.runId };
|
|
2268
|
-
}
|
|
2269
|
-
/**
|
|
2270
|
-
* Exact Provider capacity failures never trigger a token-threshold rollover.
|
|
2271
|
-
* Keep the native lineage intact and surface the Driver's explicit native
|
|
2272
|
-
* compaction/resume capability; unsupported/unknown capabilities fail safe
|
|
2273
|
-
* for Leader/Operator action instead of forcing a fresh generation.
|
|
2274
|
-
*/
|
|
2275
|
-
applyContextCapacityBlock(store, input, summary, now) {
|
|
2276
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2277
|
-
if (run === null || run.status !== "active")
|
|
2278
|
-
return null;
|
|
2279
|
-
const decision = scheduleProviderRetry(run.providerRetry, {
|
|
2280
|
-
failureEventId: input.eventId,
|
|
2281
|
-
errorClass: "context-capacity",
|
|
2282
|
-
launchId: input.launchId,
|
|
2283
|
-
nativeSessionId: input.nativeSessionId,
|
|
2284
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2285
|
-
lastErrorSummary: summary,
|
|
2286
|
-
scheduleNextAttempt: false
|
|
2287
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2288
|
-
if (decision.outcome === "exhausted")
|
|
2289
|
-
return null;
|
|
2290
|
-
store.saveAgentRun(withProviderRetry(run, decision.retry));
|
|
2291
|
-
const driver = this.drivers.requireByAdapterId(input.adapterId);
|
|
2292
|
-
const capacityEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.context-capacity-failure", {
|
|
2293
|
-
eventId: input.eventId,
|
|
2294
|
-
runId: input.runId,
|
|
2295
|
-
roleName: input.roleName,
|
|
2296
|
-
nativeSessionId: input.nativeSessionId,
|
|
2297
|
-
launchId: input.launchId,
|
|
2298
|
-
compactionCapability: driver.capabilities.lifecycle.compaction,
|
|
2299
|
-
nativeResumeCapability: driver.capabilities.lifecycle.nativeConversationResume,
|
|
2300
|
-
action: driver.capabilities.lifecycle.compaction === "native-explicit"
|
|
2301
|
-
? "native-explicit-compaction-required"
|
|
2302
|
-
: "await-provider-native-recovery"
|
|
2303
|
-
}, now);
|
|
2304
|
-
store.saveEvent(input.taskId, capacityEvent);
|
|
2305
|
-
this.recordProviderRetryClassified(store, input, "context-capacity", {
|
|
2306
|
-
wouldRetry: "false",
|
|
2307
|
-
shadow: "false",
|
|
2308
|
-
note: "provider-native-capacity-recovery-only"
|
|
2309
|
-
}, now);
|
|
2310
|
-
routeRoleEvent(store, capacityEvent, input.roleName, "provider-context-capacity-recovery-required", now);
|
|
2311
|
-
return { disposition: "applied", runId: input.runId };
|
|
2312
|
-
}
|
|
2313
|
-
applyNativeSessionRecoverabilityBlock(store, input, summary, now) {
|
|
2314
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2315
|
-
if (run === null || run.status !== "active")
|
|
2316
|
-
return null;
|
|
2317
|
-
const decision = scheduleProviderRetry(run.providerRetry, {
|
|
2318
|
-
failureEventId: input.eventId,
|
|
2319
|
-
errorClass: "session-dead",
|
|
2320
|
-
launchId: input.launchId,
|
|
2321
|
-
nativeSessionId: input.nativeSessionId,
|
|
2322
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2323
|
-
lastErrorSummary: summary,
|
|
2324
|
-
scheduleNextAttempt: false
|
|
2325
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2326
|
-
if (decision.outcome === "exhausted")
|
|
2327
|
-
return null;
|
|
2328
|
-
store.saveAgentRun(withProviderRetry(run, decision.retry));
|
|
2329
|
-
const recoverabilityEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.native-session-recoverability-unproven", {
|
|
2330
|
-
eventId: input.eventId,
|
|
2331
|
-
runId: input.runId,
|
|
2332
|
-
roleName: input.roleName,
|
|
2333
|
-
launchId: input.launchId,
|
|
2334
|
-
nativeSessionId: input.nativeSessionId,
|
|
2335
|
-
requiredEvidence: "exact-native-resume-outcome"
|
|
2336
|
-
}, now);
|
|
2337
|
-
store.saveEvent(input.taskId, recoverabilityEvent);
|
|
2338
|
-
routeRoleEvent(store, recoverabilityEvent, input.roleName, "native-session-recoverability-unproven", now);
|
|
2339
|
-
return { disposition: "applied", runId: input.runId };
|
|
2340
|
-
}
|
|
2341
|
-
recordProviderRetryClassified(store, input, errorClass, extra, now) {
|
|
2342
|
-
const event = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.provider-retry-classified", {
|
|
2343
|
-
eventId: input.eventId,
|
|
2344
|
-
runId: input.runId,
|
|
2345
|
-
roleName: input.roleName,
|
|
2346
|
-
errorClass,
|
|
2347
|
-
...extra
|
|
2348
|
-
}, now);
|
|
2349
|
-
store.saveEvent(input.taskId, event);
|
|
2350
|
-
return event;
|
|
2351
|
-
}
|
|
2352
|
-
/**
|
|
2353
|
-
* Issue 04 durable retry timer: lists active Runs whose in-place retry is
|
|
2354
|
-
* due. The Controller arms its deadline timer from this projection, so a
|
|
2355
|
-
* Controller restart resumes the same attempt lineage.
|
|
2356
|
-
*/
|
|
2357
|
-
listPendingProviderRetries(taskIds) {
|
|
2358
|
-
return this.store.listPendingProviderRetries(taskIds);
|
|
2359
|
-
}
|
|
2360
1902
|
saveRoleHostExitObservation(input) {
|
|
2361
1903
|
this.store.transaction((store) => {
|
|
2362
1904
|
const identity = [
|
|
@@ -2411,159 +1953,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
2411
1953
|
* blocker; a live Session is reopened for the existing delivery path, which
|
|
2412
1954
|
* re-pushes the exact same input in the same pass.
|
|
2413
1955
|
*/
|
|
2414
|
-
resolveDueProviderRetries(now, taskIds) {
|
|
2415
|
-
const selectedTaskIds = taskIds === undefined
|
|
2416
|
-
? undefined
|
|
2417
|
-
: [...taskIds].sort((left, right) => (left.localeCompare(right, undefined, { numeric: true })));
|
|
2418
|
-
const due = this.listPendingProviderRetries(selectedTaskIds).filter((entry) => Date.parse(entry.dueAt) <= now.getTime());
|
|
2419
|
-
const reopened = [];
|
|
2420
|
-
for (const entry of due) {
|
|
2421
|
-
this.store.transaction((store) => {
|
|
2422
|
-
const run = store.getAgentRun(entry.taskId, entry.runId);
|
|
2423
|
-
if (run === null
|
|
2424
|
-
|| run.status !== "active"
|
|
2425
|
-
|| run.providerRetry === undefined) {
|
|
2426
|
-
return;
|
|
2427
|
-
}
|
|
2428
|
-
if (run.providerRetry.state === "dispatching"
|
|
2429
|
-
|| run.providerRetry.state === "awaiting-progress") {
|
|
2430
|
-
if (now.getTime() < Date.parse(run.providerRetry.episodeDeadlineAt))
|
|
2431
|
-
return;
|
|
2432
|
-
finalizeProviderRetryDeadline(store, run, run.providerRetry.state === "dispatching"
|
|
2433
|
-
? "Provider retry delivery outcome remained uncertain through the bounded episode deadline; no correlated Provider progress was observed."
|
|
2434
|
-
: "Provider retry produced no correlated Provider input/output progress before the bounded episode deadline.", run.providerRetry.state === "dispatching"
|
|
2435
|
-
? "retry-delivery-outcome-uncertain"
|
|
2436
|
-
: "provider-progress-timeout", now);
|
|
2437
|
-
return;
|
|
2438
|
-
}
|
|
2439
|
-
if (run.providerRetry.state !== "scheduled"
|
|
2440
|
-
|| run.providerRetry.nextAttemptAt === undefined
|
|
2441
|
-
|| !providerRetryIsDue(run.providerRetry, now)) {
|
|
2442
|
-
return;
|
|
2443
|
-
}
|
|
2444
|
-
const sessions = store.getTaskRoleSessionSet(entry.taskId, entry.roleName);
|
|
2445
|
-
const providerBinding = sessions?.providerBinding;
|
|
2446
|
-
if (providerBinding === null || providerBinding === undefined) {
|
|
2447
|
-
deferUnprovenProviderRecovery(store, run, entry.roleName, "Provider Conversation identity is missing; automatic recovery is fenced.", now);
|
|
2448
|
-
return;
|
|
2449
|
-
}
|
|
2450
|
-
const conversation = currentProviderConversation(providerBinding);
|
|
2451
|
-
const roleMailbox = store.getWorkMailbox({
|
|
2452
|
-
kind: "role",
|
|
2453
|
-
taskId: entry.taskId,
|
|
2454
|
-
roleName: entry.roleName
|
|
2455
|
-
});
|
|
2456
|
-
const activeTurnId = providerBinding.turn !== null
|
|
2457
|
-
&& ["accepted", "running"].includes(providerBinding.turn.status)
|
|
2458
|
-
? providerBinding.turn.turnId
|
|
2459
|
-
: undefined;
|
|
2460
|
-
const recovery = decideProviderRecovery({
|
|
2461
|
-
binding: providerBinding,
|
|
2462
|
-
probe: {
|
|
2463
|
-
state: conversation.recoverability === "recoverable"
|
|
2464
|
-
? "exists"
|
|
2465
|
-
: conversation.recoverability === "unrecoverable" ? "missing" : "unknown",
|
|
2466
|
-
conversationId: conversation.conversationId,
|
|
2467
|
-
...(activeTurnId === undefined ? {} : { activeTurnId })
|
|
2468
|
-
},
|
|
2469
|
-
unsettledInputDelivery: roleMailbox?.inputDelivery != null
|
|
2470
|
-
});
|
|
2471
|
-
if (recovery.action === "attention") {
|
|
2472
|
-
deferUnprovenProviderRecovery(store, run, entry.roleName, recovery.reason, now);
|
|
2473
|
-
return;
|
|
2474
|
-
}
|
|
2475
|
-
if (recovery.action === "observe-active-turn") {
|
|
2476
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
2477
|
-
if (deferred === null) {
|
|
2478
|
-
finalizeProviderRetryDeadline(store, run, `Provider Turn ${recovery.turnId} remained active through the bounded recovery deadline.`, "provider-progress-timeout", now);
|
|
2479
|
-
}
|
|
2480
|
-
else {
|
|
2481
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
2482
|
-
}
|
|
2483
|
-
return;
|
|
2484
|
-
}
|
|
2485
|
-
if (recovery.action === "restart-run") {
|
|
2486
|
-
const finalized = finalizeProviderRetryDeadline(store, run, "Provider Conversation is unavailable; retry the work with a fresh Run and Session.", "provider-conversation-unavailable", now);
|
|
2487
|
-
if (finalized) {
|
|
2488
|
-
const currentSessions = store.getTaskRoleSessionSet(entry.taskId, entry.roleName);
|
|
2489
|
-
const currentSession = currentSessions?.sessions[run.effective.agentId];
|
|
2490
|
-
if (currentSessions !== null && currentSessions !== undefined
|
|
2491
|
-
&& currentSession !== undefined
|
|
2492
|
-
&& currentSession.status !== "stopped"
|
|
2493
|
-
&& currentSession.status !== "broken") {
|
|
2494
|
-
store.saveTaskRoleSessionSet(updateRoleAgentSessionStatus(currentSessions, run.effective.agentId, "broken", now));
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
return;
|
|
2498
|
-
}
|
|
2499
|
-
const session = sessions?.sessions[run.effective.agentId];
|
|
2500
|
-
const identityMatches = session !== undefined
|
|
2501
|
-
&& session.adapterId === run.effective.adapterId
|
|
2502
|
-
&& (run.providerRetry.nativeSessionId === undefined
|
|
2503
|
-
|| session.nativeSessionId === run.providerRetry.nativeSessionId)
|
|
2504
|
-
&& (run.providerRetry.launchId === undefined
|
|
2505
|
-
|| session.launchId === run.providerRetry.launchId);
|
|
2506
|
-
if (!identityMatches || session?.nativeSessionId === undefined) {
|
|
2507
|
-
// Local Session projection loss is not native-session-dead evidence.
|
|
2508
|
-
// Keep the bounded retry episode; if native identity never returns,
|
|
2509
|
-
// the episode fails and a later Run can continue from Task progress.
|
|
2510
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
2511
|
-
if (deferred === null) {
|
|
2512
|
-
finalizeProviderRetryDeadline(store, run, "Provider retry safety could not be re-established before the bounded episode deadline; native Session recoverability remains unproven.", "native-resume-unproven", now);
|
|
2513
|
-
return;
|
|
2514
|
-
}
|
|
2515
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
2516
|
-
let attentionEvent = store.listEvents(entry.taskId).find((event) => (event.type === "runtime.provider-retry-native-resume-unproven"
|
|
2517
|
-
&& event.payload.runId === run.id));
|
|
2518
|
-
if (attentionEvent === undefined) {
|
|
2519
|
-
attentionEvent = createTaskEvent(store.nextEventId(entry.taskId), entry.taskId, "runtime.provider-retry-native-resume-unproven", {
|
|
2520
|
-
runId: run.id,
|
|
2521
|
-
consecutiveFailures: String(run.providerRetry.consecutiveFailures),
|
|
2522
|
-
dispatchedRetries: String(run.providerRetry.dispatchedRetries)
|
|
2523
|
-
}, now);
|
|
2524
|
-
store.saveEvent(entry.taskId, attentionEvent);
|
|
2525
|
-
}
|
|
2526
|
-
routeRoleEvent(store, attentionEvent, entry.roleName, "provider-retry-native-resume-unproven", now);
|
|
2527
|
-
return;
|
|
2528
|
-
}
|
|
2529
|
-
if (runHasActiveRuntimeOperations(store.listEvents(entry.taskId), {
|
|
2530
|
-
taskId: entry.taskId,
|
|
2531
|
-
roleName: entry.roleName,
|
|
2532
|
-
runId: run.id,
|
|
2533
|
-
agentId: run.effective.agentId
|
|
2534
|
-
})) {
|
|
2535
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
2536
|
-
if (deferred === null) {
|
|
2537
|
-
finalizeProviderRetryDeadline(store, run, "Provider retry stopped because tool/subagent outcome remained uncertain through the bounded episode deadline.", "operation-outcome-uncertain", now);
|
|
2538
|
-
return;
|
|
2539
|
-
}
|
|
2540
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
2541
|
-
return;
|
|
2542
|
-
}
|
|
2543
|
-
// Reopen the Run on its original Session: reset transport markers,
|
|
2544
|
-
// switch to resume, and mark the projection in-flight. The delivery
|
|
2545
|
-
// pass re-pushes the exact same input in this same pass.
|
|
2546
|
-
const reopenedRun = reopenRunForProviderRetry(run, `provider-retry-${run.providerRetry.episodeId}-${run.providerRetry.dispatchedRetries + 1}`, now, "resume");
|
|
2547
|
-
store.saveAgentRun(reopenedRun);
|
|
2548
|
-
if (sessions !== null && sessions.inFlight !== null) {
|
|
2549
|
-
store.saveTaskRoleSessionSet(prepareTaskRoleRunRedispatch(sessions, {
|
|
2550
|
-
agentId: run.effective.agentId,
|
|
2551
|
-
runId: run.id,
|
|
2552
|
-
receiptId: agentRunDeliveryReceiptId(reopenedRun)
|
|
2553
|
-
}, now));
|
|
2554
|
-
}
|
|
2555
|
-
// The retry decision settled the original delivery claim; re-queue the
|
|
2556
|
-
// Role mailbox so the delivery pass can claim and re-push the Run.
|
|
2557
|
-
enqueueWork(store, { kind: "role", taskId: entry.taskId, roleName: entry.roleName }, "provider-retry-repush", now, [{ type: "run", taskId: entry.taskId, id: run.id }]);
|
|
2558
|
-
reopened.push(formatTaskRecordReference(entry.taskId, run.id, "agentRun"));
|
|
2559
|
-
});
|
|
2560
|
-
}
|
|
2561
|
-
return reopened;
|
|
2562
|
-
}
|
|
2563
|
-
/**
|
|
2564
|
-
* Applies already-normalized Session observations. Driver-specific startup
|
|
2565
|
-
* names and source variants have terminated before this boundary.
|
|
2566
|
-
*/
|
|
2567
1956
|
observeProviderRuntimeIdentity(input, now) {
|
|
2568
1957
|
return this.store.transaction((store) => {
|
|
2569
1958
|
const taskId = input.fence.taskId;
|
|
@@ -2659,7 +2048,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2659
2048
|
&& existingNativeSessionId !== decision.outcome.nativeSessionId;
|
|
2660
2049
|
const replacementBasis = replacingNativeSession
|
|
2661
2050
|
&& run.mode === "new"
|
|
2662
|
-
&&
|
|
2051
|
+
&& existingSession?.status === "ended"
|
|
2663
2052
|
? "terminal-session"
|
|
2664
2053
|
: null;
|
|
2665
2054
|
if (replacingNativeSession && replacementBasis === null) {
|
|
@@ -2672,7 +2061,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2672
2061
|
nativeSessionId: decision.outcome.nativeSessionId,
|
|
2673
2062
|
launchId: input.fence.launchId,
|
|
2674
2063
|
policy: "fixed",
|
|
2675
|
-
status: "
|
|
2064
|
+
status: "active",
|
|
2676
2065
|
effective: run.effective
|
|
2677
2066
|
};
|
|
2678
2067
|
const bound = replacementBasis === null
|
|
@@ -2737,7 +2126,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2737
2126
|
recordCanonicalObservationObsolete(store, input, "human-turn-fence-mismatch", now);
|
|
2738
2127
|
return "obsolete";
|
|
2739
2128
|
}
|
|
2740
|
-
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(
|
|
2129
|
+
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(sessions, input, now));
|
|
2741
2130
|
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "run.human-input-delivered", {
|
|
2742
2131
|
attemptId: humanAttemptId,
|
|
2743
2132
|
runId: active.id,
|
|
@@ -2760,7 +2149,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2760
2149
|
recordCanonicalObservationObsolete(store, input, "continuation-fence-mismatch", now);
|
|
2761
2150
|
return "obsolete";
|
|
2762
2151
|
}
|
|
2763
|
-
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(
|
|
2152
|
+
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(sessions, input, now));
|
|
2764
2153
|
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "run.input-delivered", {
|
|
2765
2154
|
attemptId: inputDelivery.attemptId,
|
|
2766
2155
|
runId: active.id,
|
|
@@ -2771,9 +2160,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
2771
2160
|
: { nativeTurnId: input.fence.nativeTurnId })
|
|
2772
2161
|
}, now));
|
|
2773
2162
|
store.saveWorkMailbox(completeMailboxInputDelivery(mailbox, inputDelivery.attemptId, now));
|
|
2774
|
-
if (active.providerRetry !== undefined) {
|
|
2775
|
-
this.clearProviderRetryProjection(store, active, input, "provider-accepted", now);
|
|
2776
|
-
}
|
|
2777
2163
|
return "applied";
|
|
2778
2164
|
}
|
|
2779
2165
|
const event = createCanonicalLifecycleEvent({
|
|
@@ -2807,24 +2193,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
2807
2193
|
if (sessions !== null) {
|
|
2808
2194
|
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(sessions, input, now));
|
|
2809
2195
|
}
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
store.saveAgentRun(progressed);
|
|
2813
|
-
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "runtime.provider-retry-recovered", {
|
|
2814
|
-
runId: active.id,
|
|
2815
|
-
roleName: active.roleName,
|
|
2816
|
-
evidence: "provider-accepted",
|
|
2817
|
-
nativeSessionId: input.fence.nativeSessionId ?? "",
|
|
2818
|
-
episodeId: active.providerRetry?.episodeId ?? "",
|
|
2819
|
-
consecutiveFailures: String(active.providerRetry?.consecutiveFailures ?? 0),
|
|
2820
|
-
dispatchedRetries: String(active.providerRetry?.dispatchedRetries ?? 0),
|
|
2821
|
-
recoveryLatencyMs: String(active.providerRetry === undefined
|
|
2822
|
-
? 0
|
|
2823
|
-
: Math.max(0, now.getTime() - Date.parse(active.providerRetry.firstFailureAt)))
|
|
2824
|
-
}, now));
|
|
2825
|
-
}
|
|
2826
|
-
if (progressed.deliveredAt === undefined) {
|
|
2827
|
-
const delivered = markAgentRunDelivered(progressed, now);
|
|
2196
|
+
if (active.deliveredAt === undefined) {
|
|
2197
|
+
const delivered = markAgentRunDelivered(active, now);
|
|
2828
2198
|
store.saveAgentRun(delivered);
|
|
2829
2199
|
markTaskRoleRunDeliveredInFlight(store, role, delivered, now);
|
|
2830
2200
|
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "run.delivered", runLaunchEventPayload(delivered), now));
|
|
@@ -2839,35 +2209,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
2839
2209
|
return "applied";
|
|
2840
2210
|
});
|
|
2841
2211
|
}
|
|
2842
|
-
clearProviderRetryForProgress(input, evidence, now) {
|
|
2843
|
-
this.store.transaction((store) => {
|
|
2844
|
-
const taskId = input.fence.taskId;
|
|
2845
|
-
const runId = input.fence.runId;
|
|
2846
|
-
if (taskId === undefined || runId === undefined)
|
|
2847
|
-
return;
|
|
2848
|
-
const run = store.getAgentRun(taskId, runId);
|
|
2849
|
-
if (run === null || run.status !== "active" || run.providerRetry === undefined)
|
|
2850
|
-
return;
|
|
2851
|
-
this.clearProviderRetryProjection(store, run, input, evidence, now);
|
|
2852
|
-
});
|
|
2853
|
-
}
|
|
2854
|
-
clearProviderRetryProjection(store, run, input, evidence, now) {
|
|
2855
|
-
const retry = run.providerRetry;
|
|
2856
|
-
if (retry === undefined)
|
|
2857
|
-
return;
|
|
2858
|
-
store.saveAgentRun(clearProviderRetryOnProgress(run));
|
|
2859
|
-
store.saveEvent(run.taskId, createTaskEvent(store.nextEventId(run.taskId), run.taskId, "runtime.provider-retry-recovered", {
|
|
2860
|
-
runId: run.id,
|
|
2861
|
-
roleName: input.fence.roleName,
|
|
2862
|
-
evidence,
|
|
2863
|
-
episodeId: retry.episodeId,
|
|
2864
|
-
consecutiveFailures: String(retry.consecutiveFailures),
|
|
2865
|
-
dispatchedRetries: String(retry.dispatchedRetries),
|
|
2866
|
-
recoveryLatencyMs: String(Math.max(0, now.getTime() - Date.parse(retry.firstFailureAt))),
|
|
2867
|
-
nativeSessionId: input.fence.nativeSessionId ?? "",
|
|
2868
|
-
activityId: input.payload.activityId ?? ""
|
|
2869
|
-
}, now));
|
|
2870
|
-
}
|
|
2871
2212
|
foldRuntimeLifecycleEvent(store, event, observation) {
|
|
2872
2213
|
const expectation = this.projectRunExpectation(store, event.fence, observation.fence.runId);
|
|
2873
2214
|
if (expectation === null) {
|
|
@@ -2994,10 +2335,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2994
2335
|
if (effective.agentId !== input.agentId || effective.adapterId !== input.adapterId) {
|
|
2995
2336
|
throw new Error("Runtime turn completion does not match the effective global launch identity.");
|
|
2996
2337
|
}
|
|
2997
|
-
const completedStatus = effectiveExisting?.status
|
|
2998
|
-
|| effectiveExisting?.status === "broken"
|
|
2999
|
-
? effectiveExisting.status
|
|
3000
|
-
: "ready";
|
|
2338
|
+
const completedStatus = effectiveExisting?.status ?? "active";
|
|
3001
2339
|
current = recordRoleAgentSession(current, {
|
|
3002
2340
|
agentId: input.agentId,
|
|
3003
2341
|
adapterId: input.adapterId,
|
|
@@ -3007,6 +2345,9 @@ export class FileSchedulerStoreAdapter {
|
|
|
3007
2345
|
preview: effectiveExisting?.preview ?? (input.summary === undefined ? undefined : sessionPreview(input.summary)),
|
|
3008
2346
|
policy: "fixed",
|
|
3009
2347
|
status: completedStatus,
|
|
2348
|
+
...(effectiveExisting?.endReason === undefined
|
|
2349
|
+
? {}
|
|
2350
|
+
: { endReason: effectiveExisting.endReason }),
|
|
3010
2351
|
effective
|
|
3011
2352
|
}, now);
|
|
3012
2353
|
current = rememberRoleAgentCompletedTurn(current, input.agentId, nativeSessionId, input.turnId, now);
|
|
@@ -3014,13 +2355,13 @@ export class FileSchedulerStoreAdapter {
|
|
|
3014
2355
|
completeRuntimeHookReservation(store, owner, input.launchId);
|
|
3015
2356
|
if (input.roleName === SYSTEM_OPERATOR_ROLE
|
|
3016
2357
|
&& input.adapterId === "codex"
|
|
3017
|
-
&& completedStatus === "
|
|
2358
|
+
&& completedStatus === "active") {
|
|
3018
2359
|
const operatorMailbox = store.getWorkMailbox({ kind: "operator" });
|
|
3019
2360
|
// A completed foreground Codex Turn leaves its native TUI attached to
|
|
3020
2361
|
// the thread. Stop only that idle Role runtime so Desktop can become
|
|
3021
2362
|
// the writer; the durable nativeSessionId remains available to resume.
|
|
3022
2363
|
if (operatorMailbox === null || !mailboxHasWork(operatorMailbox)) {
|
|
3023
|
-
enqueueWork(store, runtimeLifecycleTarget(owner),
|
|
2364
|
+
enqueueWork(store, runtimeLifecycleTarget(owner), RUNTIME_HOST_DETACH_REQUIRED_REASON, now);
|
|
3024
2365
|
}
|
|
3025
2366
|
}
|
|
3026
2367
|
return current.sessions[input.agentId];
|
|
@@ -3054,18 +2395,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
3054
2395
|
return "apply";
|
|
3055
2396
|
}
|
|
3056
2397
|
}
|
|
3057
|
-
function runtimeTurnFailureSummary(input) {
|
|
3058
|
-
return [
|
|
3059
|
-
"Agent runtime turn failed.",
|
|
3060
|
-
`error: ${input.error}`,
|
|
3061
|
-
...(input.errorDetails === undefined
|
|
3062
|
-
? []
|
|
3063
|
-
: [`error_details: ${input.errorDetails}`]),
|
|
3064
|
-
...(input.lastAssistantMessage === undefined
|
|
3065
|
-
? []
|
|
3066
|
-
: [`last_assistant_message: ${input.lastAssistantMessage}`])
|
|
3067
|
-
].join("\n");
|
|
3068
|
-
}
|
|
3069
2398
|
function runtimeObservationLifecycleFence(input, adapterId) {
|
|
3070
2399
|
return {
|
|
3071
2400
|
taskId: input.fence.taskId,
|
|
@@ -3213,12 +2542,8 @@ function preallocatedRuntimeReadyAwaitingProjection(store, observation, drivers)
|
|
|
3213
2542
|
taskId,
|
|
3214
2543
|
roleName: observation.fence.roleName
|
|
3215
2544
|
}));
|
|
3216
|
-
const executionRef = mailbox?.processing?.executionRef;
|
|
3217
2545
|
return isRuntimeLaunchReservation(mailbox?.processing, observation.fence.launchId)
|
|
3218
2546
|
&& !hasRuntimeCleanupObligation(mailbox)
|
|
3219
|
-
&& executionRef?.type === "run"
|
|
3220
|
-
&& executionRef.taskId === taskId
|
|
3221
|
-
&& executionRef.id === runId
|
|
3222
2547
|
&& nativeSessionId === nativeSessionIdForLaunch(store.rootDirectory(), observation.fence.launchId, observation.fence.agentId, driver.adapterId);
|
|
3223
2548
|
}
|
|
3224
2549
|
function completeRuntimeHookReservation(store, owner, launchId) {
|
|
@@ -3267,14 +2592,14 @@ function dormantRuntimeCandidateIsCurrent(store, candidate) {
|
|
|
3267
2592
|
const sessions = runtimeOwnerSessionSet(store, owner);
|
|
3268
2593
|
const active = sessions?.sessions[sessions.activeAgentId];
|
|
3269
2594
|
return active !== undefined
|
|
3270
|
-
&& active.status
|
|
2595
|
+
&& active.status === "active"
|
|
3271
2596
|
&& active.agentId === candidate.agentId
|
|
3272
2597
|
&& active.adapterId === candidate.adapterId
|
|
3273
2598
|
&& active.nativeSessionId === candidate.nativeSessionId
|
|
3274
2599
|
&& active.launchId === candidate.launchId
|
|
3275
2600
|
&& active.updatedAt === candidate.sessionUpdatedAt;
|
|
3276
2601
|
}
|
|
3277
|
-
function
|
|
2602
|
+
function endRuntimeOwnerSession(store, owner, now) {
|
|
3278
2603
|
if (owner.scope === "task") {
|
|
3279
2604
|
const sessions = store.getTaskRoleSessionSet(owner.taskId, owner.roleName);
|
|
3280
2605
|
if (sessions === null)
|
|
@@ -3282,7 +2607,19 @@ function markRuntimeOwnerSessionStopped(store, owner, now) {
|
|
|
3282
2607
|
const active = sessions.sessions[sessions.activeAgentId];
|
|
3283
2608
|
if (active === undefined)
|
|
3284
2609
|
return false;
|
|
3285
|
-
|
|
2610
|
+
let stopped = active.status === "ended"
|
|
2611
|
+
? sessions
|
|
2612
|
+
: updateRoleAgentSessionStatus(sessions, sessions.activeAgentId, "ended", now, "stopped");
|
|
2613
|
+
const activation = stopped.providerBinding === null
|
|
2614
|
+
? null
|
|
2615
|
+
: currentProviderActivation(stopped.providerBinding);
|
|
2616
|
+
if (activation !== null) {
|
|
2617
|
+
stopped = updateTaskRoleProviderRuntime(stopped, endProviderActivation(stopped.providerBinding, activation.activationId, {
|
|
2618
|
+
status: "ended",
|
|
2619
|
+
endedAt: now.toISOString(),
|
|
2620
|
+
reason: "session-ended"
|
|
2621
|
+
}), now);
|
|
2622
|
+
}
|
|
3286
2623
|
if (stopped === sessions)
|
|
3287
2624
|
return false;
|
|
3288
2625
|
store.saveTaskRoleSessionSet(stopped);
|
|
@@ -3292,9 +2629,29 @@ function markRuntimeOwnerSessionStopped(store, owner, now) {
|
|
|
3292
2629
|
if (sessions === null)
|
|
3293
2630
|
return false;
|
|
3294
2631
|
const active = sessions.sessions[sessions.activeAgentId];
|
|
3295
|
-
if (active === undefined || active.status === "
|
|
2632
|
+
if (active === undefined || active.status === "ended")
|
|
2633
|
+
return false;
|
|
2634
|
+
store.saveGlobalRoleSessionSet(updateRoleAgentSessionStatus(sessions, sessions.activeAgentId, "ended", now, "stopped"));
|
|
2635
|
+
return true;
|
|
2636
|
+
}
|
|
2637
|
+
function detachRuntimeOwnerHost(store, owner, now) {
|
|
2638
|
+
if (owner.scope === "task") {
|
|
2639
|
+
const sessions = store.getTaskRoleSessionSet(owner.taskId, owner.roleName);
|
|
2640
|
+
if (sessions === null)
|
|
2641
|
+
return false;
|
|
2642
|
+
const detached = detachRoleAgentSessionHost(sessions, now);
|
|
2643
|
+
if (detached === sessions)
|
|
2644
|
+
return false;
|
|
2645
|
+
store.saveTaskRoleSessionSet(detached);
|
|
2646
|
+
return true;
|
|
2647
|
+
}
|
|
2648
|
+
const sessions = store.getGlobalRoleSessionSet(owner.roleName);
|
|
2649
|
+
if (sessions === null)
|
|
3296
2650
|
return false;
|
|
3297
|
-
|
|
2651
|
+
const detached = detachRoleAgentSessionHost(sessions, now);
|
|
2652
|
+
if (detached === sessions)
|
|
2653
|
+
return false;
|
|
2654
|
+
store.saveGlobalRoleSessionSet(detached);
|
|
3298
2655
|
return true;
|
|
3299
2656
|
}
|
|
3300
2657
|
function runtimeOwnerSessionSet(store, owner) {
|
|
@@ -3314,8 +2671,7 @@ function preparedSessionMatches(session, agentId, adapterId, nativeSessionId, la
|
|
|
3314
2671
|
return nativeSessionId === undefined;
|
|
3315
2672
|
if (session.agentId !== agentId
|
|
3316
2673
|
|| session.adapterId !== adapterId
|
|
3317
|
-
|| session.status === "
|
|
3318
|
-
|| session.status === "broken"
|
|
2674
|
+
|| session.status === "ended"
|
|
3319
2675
|
|| (nativeSessionId !== undefined
|
|
3320
2676
|
&& session.nativeSessionId !== nativeSessionId)) {
|
|
3321
2677
|
return false;
|
|
@@ -3324,45 +2680,13 @@ function preparedSessionMatches(session, agentId, adapterId, nativeSessionId, la
|
|
|
3324
2680
|
? session.launchId === undefined
|
|
3325
2681
|
: session.launchId === launchId;
|
|
3326
2682
|
}
|
|
3327
|
-
function finalizeProviderRetryDeadline(store, run, summary, reason, now) {
|
|
3328
|
-
const retry = run.providerRetry;
|
|
3329
|
-
if (retry === undefined)
|
|
3330
|
-
return false;
|
|
3331
|
-
const terminalization = terminalizeExactTaskRun(store, {
|
|
3332
|
-
taskId: run.taskId,
|
|
3333
|
-
roleName: run.roleName,
|
|
3334
|
-
agentId: run.effective.agentId,
|
|
3335
|
-
runId: run.id,
|
|
3336
|
-
receiptId: agentRunDeliveryReceiptId(run),
|
|
3337
|
-
outcome: { status: "failed", summary }
|
|
3338
|
-
}, now);
|
|
3339
|
-
if (terminalization.disposition !== "applied" || terminalization.run === null)
|
|
3340
|
-
return false;
|
|
3341
|
-
const terminal = terminalization.run;
|
|
3342
|
-
const exhaustionEvent = createTaskEvent(store.nextEventId(run.taskId), run.taskId, "runtime.provider-retry-exhausted", {
|
|
3343
|
-
runId: run.id,
|
|
3344
|
-
roleName: run.roleName,
|
|
3345
|
-
episodeId: retry.episodeId,
|
|
3346
|
-
reason,
|
|
3347
|
-
consecutiveFailures: String(retry.consecutiveFailures),
|
|
3348
|
-
dispatchedRetries: String(retry.dispatchedRetries),
|
|
3349
|
-
elapsedMs: String(Math.max(0, now.getTime() - Date.parse(retry.firstFailureAt)))
|
|
3350
|
-
}, now);
|
|
3351
|
-
store.saveEvent(run.taskId, exhaustionEvent);
|
|
3352
|
-
if (run.roleName === "leader") {
|
|
3353
|
-
store.saveLeaderFailure(recordLeaderFailure(run.taskId, retry.nativeSessionId ?? "(unproven)", summary, now, store.getLeaderFailure(run.taskId)));
|
|
3354
|
-
}
|
|
3355
|
-
routeRoleEvent(store, exhaustionEvent, run.roleName, "provider-retry-exhausted", now);
|
|
3356
|
-
return true;
|
|
3357
|
-
}
|
|
3358
2683
|
function preparedDeliveryFailureSessionFenceMatches(sessions, session, input, expectedReceiptId) {
|
|
3359
2684
|
if (sessions === null)
|
|
3360
2685
|
return input.nativeSessionId === undefined;
|
|
3361
2686
|
if (sessions.activeAgentId !== input.agentId)
|
|
3362
2687
|
return false;
|
|
3363
2688
|
if (sessions.inFlight === null) {
|
|
3364
|
-
return input.nativeSessionId
|
|
3365
|
-
&& session === undefined;
|
|
2689
|
+
return preparedSessionMatches(session, input.agentId, input.adapterId, input.nativeSessionId, input.launchId);
|
|
3366
2690
|
}
|
|
3367
2691
|
return sessions.inFlight.agentId === input.agentId
|
|
3368
2692
|
&& sessions.inFlight.runId === input.runId
|
|
@@ -3370,41 +2694,6 @@ function preparedDeliveryFailureSessionFenceMatches(sessions, session, input, ex
|
|
|
3370
2694
|
&& sessions.inFlight.pushedAt === undefined
|
|
3371
2695
|
&& preparedSessionMatches(session, input.agentId, input.adapterId, input.nativeSessionId, input.launchId);
|
|
3372
2696
|
}
|
|
3373
|
-
function preparedReservationMatches(mailbox, taskId, runId, launchId) {
|
|
3374
|
-
if (launchId === undefined) {
|
|
3375
|
-
return !isRuntimeLaunchReservation(mailbox?.processing);
|
|
3376
|
-
}
|
|
3377
|
-
const processing = mailbox?.processing;
|
|
3378
|
-
return isRuntimeLaunchReservation(processing, launchId)
|
|
3379
|
-
&& !hasRuntimeCleanupObligation(mailbox)
|
|
3380
|
-
&& processing?.executionRef?.type === "run"
|
|
3381
|
-
&& processing.executionRef.taskId === taskId
|
|
3382
|
-
&& processing.executionRef.id === runId;
|
|
3383
|
-
}
|
|
3384
|
-
function matchingPreparedRuntimeReservation(store, input) {
|
|
3385
|
-
if (input.launchId === undefined
|
|
3386
|
-
|| input.session?.nativeSessionId === undefined) {
|
|
3387
|
-
return null;
|
|
3388
|
-
}
|
|
3389
|
-
const owner = {
|
|
3390
|
-
scope: "task",
|
|
3391
|
-
taskId: input.task.id,
|
|
3392
|
-
roleName: input.role.name
|
|
3393
|
-
};
|
|
3394
|
-
const mailbox = store.getWorkMailbox(runtimeLifecycleTarget(owner));
|
|
3395
|
-
if (isRuntimeLaunchReservation(mailbox?.processing, input.launchId)
|
|
3396
|
-
&& mailbox?.processing?.executionRef?.type === "run"
|
|
3397
|
-
&& mailbox.processing.executionRef.id === input.run.id
|
|
3398
|
-
&& !hasRuntimeCleanupObligation(mailbox)) {
|
|
3399
|
-
return mailbox;
|
|
3400
|
-
}
|
|
3401
|
-
throw new Error("Prepared Role session does not match its launch generation.");
|
|
3402
|
-
}
|
|
3403
|
-
function completePreparedRuntimeReservation(store, mailbox) {
|
|
3404
|
-
if (mailbox === null || mailbox.processing === null)
|
|
3405
|
-
return;
|
|
3406
|
-
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, mailbox.processing.batchId));
|
|
3407
|
-
}
|
|
3408
2697
|
function recordTaskRuntimeNativeSession(store, input, now) {
|
|
3409
2698
|
const task = store.getTask(input.taskId);
|
|
3410
2699
|
if (task === null)
|
|
@@ -3420,7 +2709,7 @@ function recordTaskRuntimeNativeSession(store, input, now) {
|
|
|
3420
2709
|
?? createRoleSessionSet({ scope: "task", taskId: input.taskId, roleName: input.roleName }, input.agentId, now);
|
|
3421
2710
|
const existing = current.sessions[input.agentId];
|
|
3422
2711
|
const effectiveExisting = nativeTransitionExisting(store, { scope: "task", taskId: input.taskId, roleName: input.roleName }, existing, input.nativeSessionId, input.launchId, "Native session registration conflicts with the fixed Role session.");
|
|
3423
|
-
if (existing?.status === "
|
|
2712
|
+
if (existing?.status === "active"
|
|
3424
2713
|
&& (input.launchId === undefined || existing.launchId === input.launchId))
|
|
3425
2714
|
return existing;
|
|
3426
2715
|
const resolvedEffective = taskSessionEffective(store, input.taskId, input.roleName, input.agentId, effectiveExisting);
|
|
@@ -3439,7 +2728,7 @@ function recordTaskRuntimeNativeSession(store, input, now) {
|
|
|
3439
2728
|
nativeSessionId: input.nativeSessionId,
|
|
3440
2729
|
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
3441
2730
|
policy: "fixed",
|
|
3442
|
-
status: "
|
|
2731
|
+
status: "active",
|
|
3443
2732
|
effective: effectiveExisting?.effective ?? effective
|
|
3444
2733
|
}, now);
|
|
3445
2734
|
store.saveRoleSessionSet(updated);
|
|
@@ -3453,7 +2742,7 @@ function recordGlobalRuntimeNativeSession(store, input, now) {
|
|
|
3453
2742
|
?? createRoleSessionSet({ scope: "global", roleName: input.roleName }, input.agentId, now);
|
|
3454
2743
|
const existing = current.sessions[input.agentId];
|
|
3455
2744
|
const effectiveExisting = nativeTransitionExisting(store, { scope: "global", roleName: input.roleName }, existing, input.nativeSessionId, input.launchId, "Native session registration conflicts with the fixed global Role session.");
|
|
3456
|
-
if (existing?.status === "
|
|
2745
|
+
if (existing?.status === "active"
|
|
3457
2746
|
&& (input.launchId === undefined || existing.launchId === input.launchId))
|
|
3458
2747
|
return existing;
|
|
3459
2748
|
const resolvedEffective = globalSessionEffective(role, effectiveExisting);
|
|
@@ -3472,7 +2761,7 @@ function recordGlobalRuntimeNativeSession(store, input, now) {
|
|
|
3472
2761
|
nativeSessionId: input.nativeSessionId,
|
|
3473
2762
|
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
3474
2763
|
policy: "fixed",
|
|
3475
|
-
status: "
|
|
2764
|
+
status: "active",
|
|
3476
2765
|
effective
|
|
3477
2766
|
}, now);
|
|
3478
2767
|
store.saveGlobalRoleSessionSet(updated);
|
|
@@ -3482,7 +2771,7 @@ function nativeTransitionExisting(store, owner, existing, nativeSessionId, launc
|
|
|
3482
2771
|
if (existing === undefined || existing.nativeSessionId === nativeSessionId) {
|
|
3483
2772
|
return existing;
|
|
3484
2773
|
}
|
|
3485
|
-
if (
|
|
2774
|
+
if (existing.status === "ended"
|
|
3486
2775
|
&& runtimeHookMatchesReservation(store, owner, launchId)) {
|
|
3487
2776
|
return undefined;
|
|
3488
2777
|
}
|
|
@@ -3584,6 +2873,7 @@ function mapSession(session) {
|
|
|
3584
2873
|
...(session.launchId === undefined ? {} : { launchId: session.launchId }),
|
|
3585
2874
|
...(session.title === undefined ? {} : { title: session.title }),
|
|
3586
2875
|
status: session.status,
|
|
2876
|
+
...(session.endReason === undefined ? {} : { endReason: session.endReason }),
|
|
3587
2877
|
effective: session.effective,
|
|
3588
2878
|
updatedAt: session.updatedAt
|
|
3589
2879
|
};
|
|
@@ -3599,6 +2889,9 @@ function saveTaskSession(store, role, session, status, now, launchId) {
|
|
|
3599
2889
|
...(session.title === undefined ? {} : { title: session.title }),
|
|
3600
2890
|
policy: "fixed",
|
|
3601
2891
|
status,
|
|
2892
|
+
...(status !== "ended" || session.endReason === undefined
|
|
2893
|
+
? {}
|
|
2894
|
+
: { endReason: session.endReason }),
|
|
3602
2895
|
effective: session.effective
|
|
3603
2896
|
}, now);
|
|
3604
2897
|
store.saveRoleSessionSet(updated);
|
|
@@ -3612,8 +2905,7 @@ function bindTaskRoleRunInFlight(store, role, run, now) {
|
|
|
3612
2905
|
throw new Error("Task Role Session identity cannot change with an unsettled Run fence.");
|
|
3613
2906
|
}
|
|
3614
2907
|
const liveConflict = Object.values(current.sessions).find((session) => (session.agentId !== agentId
|
|
3615
|
-
&& session.status
|
|
3616
|
-
&& session.status !== "broken"));
|
|
2908
|
+
&& session.status === "active"));
|
|
3617
2909
|
if (liveConflict !== undefined) {
|
|
3618
2910
|
throw new Error(`Task Role Session identity cannot change while ${liveConflict.agentId} is live.`);
|
|
3619
2911
|
}
|
|
@@ -3667,27 +2959,6 @@ function clearTaskRoleRunInFlight(store, role, run, now) {
|
|
|
3667
2959
|
}, now);
|
|
3668
2960
|
store.saveRoleSessionSet(updated);
|
|
3669
2961
|
}
|
|
3670
|
-
function breakTaskSessionIfPresent(store, taskId, roleName, agentId, now) {
|
|
3671
|
-
updateTaskSessionStatusIfPresent(store, taskId, roleName, agentId, "broken", now);
|
|
3672
|
-
}
|
|
3673
|
-
function stopTaskSessionIfPresent(store, taskId, roleName, agentId, now) {
|
|
3674
|
-
updateTaskSessionStatusIfPresent(store, taskId, roleName, agentId, "stopped", now);
|
|
3675
|
-
}
|
|
3676
|
-
function updateTaskSessionStatusIfPresent(store, taskId, roleName, agentId, status, now) {
|
|
3677
|
-
const set = store.getRoleSessionSet(taskId, roleName);
|
|
3678
|
-
if (set === null || set.sessions[agentId] === undefined)
|
|
3679
|
-
return;
|
|
3680
|
-
store.saveRoleSessionSet(updateRoleAgentSessionStatus(set, agentId, status, now));
|
|
3681
|
-
}
|
|
3682
|
-
function schedulerRoleSessionsMatch(current, expected) {
|
|
3683
|
-
if (current === null || expected === null)
|
|
3684
|
-
return current === expected;
|
|
3685
|
-
return current.agentId === expected.agentId
|
|
3686
|
-
&& current.adapterId === expected.adapterId
|
|
3687
|
-
&& current.nativeSessionId === expected.nativeSessionId
|
|
3688
|
-
&& current.launchId === expected.launchId
|
|
3689
|
-
&& isDeepStrictEqual(current.effective, expected.effective);
|
|
3690
|
-
}
|
|
3691
2962
|
/**
|
|
3692
2963
|
* The durable Run/mailbox claim is the pre-Host delivery intent. The Host may
|
|
3693
2964
|
* create a ProviderTurn only while that exact reservation still owns the Run;
|
|
@@ -3910,24 +3181,6 @@ function rejectSubmittingProviderTurn(store, target, delivery, now, reason) {
|
|
|
3910
3181
|
reason
|
|
3911
3182
|
}), now));
|
|
3912
3183
|
}
|
|
3913
|
-
function deferUnprovenProviderRecovery(store, run, roleName, reason, now) {
|
|
3914
|
-
if (run.providerRetry === undefined)
|
|
3915
|
-
return;
|
|
3916
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
3917
|
-
if (deferred === null) {
|
|
3918
|
-
finalizeProviderRetryDeadline(store, run, reason, "native-resume-unproven", now);
|
|
3919
|
-
return;
|
|
3920
|
-
}
|
|
3921
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
3922
|
-
const attentionEvent = createTaskEvent(store.nextEventId(run.taskId), run.taskId, "runtime.provider-recovery-attention", {
|
|
3923
|
-
runId: run.id,
|
|
3924
|
-
roleName,
|
|
3925
|
-
reason,
|
|
3926
|
-
nextAttemptAt: deferred.nextAttemptAt ?? ""
|
|
3927
|
-
}, now);
|
|
3928
|
-
store.saveEvent(run.taskId, attentionEvent);
|
|
3929
|
-
routeRoleEvent(store, attentionEvent, roleName, "provider-recovery-attention", now);
|
|
3930
|
-
}
|
|
3931
3184
|
function markSubmittingProviderTurnUnknown(store, target, delivery, now, reason) {
|
|
3932
3185
|
if (target.kind !== "role"
|
|
3933
3186
|
|| delivery.executionRef.type !== "run"
|
|
@@ -3990,7 +3243,3 @@ function compareCanonicalObservationOrder(left, right) {
|
|
|
3990
3243
|
|| (left.ordinal ?? -1) - (right.ordinal ?? -1)
|
|
3991
3244
|
|| left.eventId.localeCompare(right.eventId);
|
|
3992
3245
|
}
|
|
3993
|
-
function configuredProviderRetryPolicy(store) {
|
|
3994
|
-
const config = providerRetryConfig(store.getConfig());
|
|
3995
|
-
return { delaysMs: config.delaysMs, maxWindowMs: config.maxWindowMs };
|
|
3996
|
-
}
|