@zq-silk/yui 0.13.7 → 0.13.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +40 -4
- package/README.md +23 -4
- package/dist/cli/commandCatalog.js +21 -2
- package/dist/cli/updatePorts.js +4 -4
- package/dist/cli.js +43 -14
- package/dist/commands/agentCommands.js +1 -1
- package/dist/commands/configCommands.js +1 -86
- package/dist/commands/executionAuditCommands.js +17 -16
- package/dist/commands/globalRoleCommands.js +4 -4
- package/dist/commands/sessionCommands.js +2 -6
- package/dist/commands/taskActor.js +1 -2
- package/dist/commands/taskCommands.js +93 -21
- package/dist/commands/taskContextCommand.js +1 -1
- package/dist/commands/taskExecutionCommands.js +0 -5
- package/dist/commands/taskOverviewCommand.js +5 -1
- package/dist/commands/taskRoleRuntimeStatus.js +3 -16
- package/dist/config/configCatalog.js +1 -6
- package/dist/config/yuiConfig.js +0 -79
- package/dist/context/sessionBootstrapManifest.js +26 -23
- package/dist/controller/clientRuntime.js +40 -3
- package/dist/controller/controller.js +29 -54
- package/dist/controller/fileSchedulerStoreAdapter.js +372 -1117
- package/dist/controller/runtime.js +12 -5
- package/dist/controller/runtimeHookRunFence.js +3 -9
- package/dist/controller/runtimeLaunchCoordinator.js +44 -67
- package/dist/controller/structuredProviderObservation.js +18 -5
- package/dist/coordination/workMailbox.js +4 -4
- package/dist/execution/executionHealth.js +1 -1
- package/dist/executor/agentExecutor.js +103 -93
- package/dist/executor/executorRegistry.js +8 -20
- package/dist/executor/fileRoleLaunchPlanner.js +26 -93
- package/dist/executor/turnCompletion.js +5 -5
- package/dist/lifecycle/exactRunTerminalization.js +2 -4
- package/dist/observability/executionAudit.js +40 -94
- package/dist/operator/operatorSessionHistory.js +7 -5
- package/dist/output/rolePresentation.js +0 -1
- package/dist/repository/taskWorkspacePreparer.js +0 -1
- package/dist/role/role.js +13 -21
- package/dist/run/agentRun.js +4 -54
- package/dist/runtime/agentDriver.js +2 -0
- package/dist/runtime/agentError.js +114 -0
- package/dist/runtime/agentHost.js +55 -82
- package/dist/runtime/builtinAgentDrivers.js +21 -9
- package/dist/runtime/builtinAgentErrorMappers.js +150 -0
- package/dist/runtime/exactControlPlane.js +6 -12
- package/dist/runtime/index.js +1 -2
- package/dist/runtime/launchBroker.js +5 -19
- package/dist/runtime/lifecycleReservation.js +20 -4
- package/dist/runtime/providerRuntimeIdentity.js +11 -19
- package/dist/runtime/runtimeBinding.js +0 -27
- package/dist/runtime/runtimeObservation.js +7 -16
- package/dist/runtime/runtimeSessionCandidate.js +3 -10
- package/dist/runtime/sessionLaunchRequest.js +1 -2
- package/dist/runtime/sessionReconciliation.js +2 -2
- package/dist/runtime/structuredProviderHost.js +44 -79
- package/dist/runtime/taskRuntimeIsolation.js +0 -7
- package/dist/runtime/tmuxAdapters.js +6 -49
- package/dist/scheduler/activeRoleRunDelivery.js +220 -178
- package/dist/scheduler/activeTaskProgress.js +1 -4
- package/dist/scheduler/leaderWakeupProcessor.js +123 -88
- package/dist/scheduler/roleRunLiveness.js +4 -1
- package/dist/scheduler/roleRunStall.js +10 -14
- package/dist/scheduler/wakeReason.js +4 -0
- package/dist/storage/migration/productionRegistry.js +475 -0
- package/dist/storage/sqliteSchema.js +54 -2
- package/dist/storage/sqliteStore.js +11 -44
- package/dist/storage/taskStore.js +8 -36
- package/dist/web/webSnapshot.js +3 -0
- package/i18n/README.zh-CN.md +11 -3
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +30 -8
- package/skills/yui-operator/SKILL.md +14 -6
- package/skills/yui-runtime/SKILL.md +11 -8
- package/dist/lifecycle/providerErrorClass.js +0 -152
- package/dist/run/providerRetry.js +0 -226
- package/dist/run/providerRetryConfig.js +0 -27
- package/dist/runtime/providerErrorCodes.js +0 -278
- package/dist/runtime/providerRecoveryDecision.js +0 -55
|
@@ -1,22 +1,18 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { isDeepStrictEqual } from "node:util";
|
|
3
|
-
import { activeLiveRoleAgentSession, bindTaskRoleProviderRuntime, bindTaskRoleRun, clearTaskRoleRun, createRoleSessionSet, markTaskRoleRunDelivered, markTaskRoleRunPushed,
|
|
3
|
+
import { activeLiveRoleAgentSession, bindTaskRoleProviderRuntime, bindTaskRoleRun, clearTaskRoleRun, createRoleSessionSet, markTaskRoleRunDelivered, markTaskRoleRunPushed, recordRoleAgentSession, replaceTaskRoleAgentSession, recordTaskRoleTurnBoundary, rememberRoleAgentCompletedTurn, detachRoleAgentSessionHost, updateRoleAgentSessionStatus, updateTaskRoleProviderRuntime } from "../executor/agentExecutor.js";
|
|
4
4
|
import { acceptProviderTurn, beginProviderTurn, createProviderRuntimeBinding, endProviderActivation, currentProviderActivation, currentProviderConversation, markProviderTurnDeliveryUnknown, rejectProviderTurn, settleProviderTurnSubmission, settleProviderTurn, startProviderActivation, supersedeProviderConversation, updateProviderConversationRecoverability } from "../runtime/providerRuntimeIdentity.js";
|
|
5
|
-
import { decideProviderRecovery } from "../runtime/providerRecoveryDecision.js";
|
|
6
5
|
import { hasRecentTurnId } from "../executor/turnCompletion.js";
|
|
7
6
|
import { createTaskEvent } from "../event/taskEvent.js";
|
|
8
7
|
import { operationalTaskRecords } from "../task/taskRecordRetirement.js";
|
|
9
8
|
import { buildTaskWakeEnvelope } from "../context/wakeNotification.js";
|
|
10
9
|
import { createTaskWake, fallbackWakeCursor, latestTaskWake } from "../scheduler/taskWake.js";
|
|
11
10
|
import { answerInputRequest } from "../input/inputRequest.js";
|
|
12
|
-
import { activeRoleAgentBinding
|
|
11
|
+
import { activeRoleAgentBinding } from "../role/role.js";
|
|
13
12
|
import { effectiveLaunchWithTaskMainWorkspace, effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskSession, resolveEffectiveLaunch, validateEffectiveLaunchSnapshot } from "../executor/effectiveLaunch.js";
|
|
14
13
|
import { SYSTEM_OPERATOR_ROLE } from "../role/systemRoles.js";
|
|
15
|
-
import {
|
|
14
|
+
import { markAgentRunDelivered, markAgentRunPushed, agentRunDeliveryReceiptId } from "../run/agentRun.js";
|
|
16
15
|
import { markWorkflowOutcomeRequestDispatched } from "../run/runControlRequest.js";
|
|
17
|
-
import { classifyProviderError, isRetryableProviderErrorClass } from "../lifecycle/providerErrorClass.js";
|
|
18
|
-
import { deferProviderRetry, markProviderRetryDispatched, providerRetryIsDue, scheduleProviderRetry } from "../run/providerRetry.js";
|
|
19
|
-
import { providerRetryAdapterEnabled, providerRetryConfig } from "../run/providerRetryConfig.js";
|
|
20
16
|
import { classifyRuntimeProcessExit, validateRuntimeProcessExitObservation } from "../runtime/processExitObservation.js";
|
|
21
17
|
import { terminalizeExactTaskRun } from "../lifecycle/exactRunTerminalization.js";
|
|
22
18
|
import { createCanonicalLifecycleEvent, foldCanonicalLifecycleEvent } from "../lifecycle/canonicalLifecycleEvent.js";
|
|
@@ -28,15 +24,14 @@ import { wakeReason } from "../scheduler/wakeReason.js";
|
|
|
28
24
|
import { foldRunProgressFacts, latestRunDurableProgressAt, latestRunEventTime, latestStallEvidenceKey, isRoleRunStalled, RUN_PROGRESS_EVENT, RUN_DIAGNOSTIC_FINISHED_EVENT, RUN_RECOVERED_EVENT, RUN_STALLED_EVENT } from "../scheduler/roleRunStall.js";
|
|
29
25
|
import { projectProviderContinuations } from "../runtime/runtimeContinuationProjection.js";
|
|
30
26
|
import { providerContinuationKey } from "../runtime/providerContinuation.js";
|
|
31
|
-
import {
|
|
32
|
-
import { recordExecutionLaneResult } from "../execution/executionGroup.js";
|
|
33
|
-
import { formatAgentRunReceiptId, formatTaskRecordReference } from "../task/taskRecordReference.js";
|
|
27
|
+
import { formatAgentRunReceiptId } from "../task/taskRecordReference.js";
|
|
34
28
|
import { bindExecution, claimPending, claimInputDelivery as claimMailboxInputDelivery, completeInputDelivery as completeMailboxInputDelivery, completeProcessing, mailboxHasPending, mailboxHasWork, pendingLane, markInputDeliveryPushed as markMailboxInputDeliveryPushed, markInputDeliveryUnknown as markMailboxInputDeliveryUnknown, releaseInputDelivery as releaseMailboxInputDelivery, resolveInputDeliveryNotAccepted as resolveMailboxInputDeliveryNotAccepted, releaseProcessing } from "../coordination/workMailbox.js";
|
|
35
29
|
import { enqueueWork } from "../coordination/workMailboxQueue.js";
|
|
36
|
-
import { RUNTIME_CLEANUP_REQUIRED_REASON, RUNTIME_LAUNCH_RESERVED_REASON, RUNTIME_LIFECYCLE_OWNER, RuntimeLifecycleBusyError, hasRuntimeCleanupObligation, hasRuntimeLifecycleWork, isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
30
|
+
import { RUNTIME_CLEANUP_REQUIRED_REASON, RUNTIME_HOST_DETACH_REQUIRED_REASON, RUNTIME_LAUNCH_RESERVED_REASON, RUNTIME_LIFECYCLE_OWNER, RuntimeLifecycleBusyError, hasRuntimeCleanupObligation, hasRuntimeLifecycleWork, isRuntimeCleanupReason, isRuntimeLaunchReservation, runtimeCleanupDisposition, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
37
31
|
import { nativeSessionIdForLaunch } from "../runtime/preallocatedNativeSession.js";
|
|
38
32
|
import { builtinAgentDriverRegistry } from "../runtime/builtinAgentDrivers.js";
|
|
39
|
-
import {
|
|
33
|
+
import { standardAgentError } from "../runtime/agentError.js";
|
|
34
|
+
import { RUNTIME_OBSERVATION_TASK_EVENT, createRuntimeObservation, isRuntimeTokenEvidence, runtimeObservationFenceMatches, runtimeObservationFromTaskEvent, runtimeObservationRunFenceMatches, runtimeObservationTaskEventPayload } from "../runtime/runtimeObservation.js";
|
|
40
35
|
import { projectRuntimeTaskEvents } from "../runtime/runtimeProjection.js";
|
|
41
36
|
import { contextSnapshotRef } from "../context/contextSnapshot.js";
|
|
42
37
|
import { contextSnapshotDeltaRefIds, freezeRunContextSnapshot } from "../context/runContextPack.js";
|
|
@@ -101,7 +96,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
101
96
|
outcome = this.observeRuntimePromptAccepted(input, adapterId, now);
|
|
102
97
|
break;
|
|
103
98
|
case "turn.completed": {
|
|
104
|
-
const
|
|
99
|
+
const classification = this.classifyRuntimeTurnCompleted({
|
|
105
100
|
taskId,
|
|
106
101
|
roleName: input.fence.roleName,
|
|
107
102
|
agentId: input.fence.agentId,
|
|
@@ -110,10 +105,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
110
105
|
nativeSessionId: input.fence.nativeSessionId,
|
|
111
106
|
turnId: input.fence.nativeTurnId,
|
|
112
107
|
attemptId: input.fence.receiptId,
|
|
113
|
-
runId: input.fence.runId
|
|
114
|
-
|
|
115
|
-
};
|
|
116
|
-
const classification = this.classifyRuntimeTurnCompleted(completed);
|
|
108
|
+
runId: input.fence.runId
|
|
109
|
+
});
|
|
117
110
|
if (classification !== "apply")
|
|
118
111
|
return classification;
|
|
119
112
|
if (this.runtimeTurnHasActiveNativeSubagents(input)) {
|
|
@@ -123,78 +116,21 @@ export class FileSchedulerStoreAdapter {
|
|
|
123
116
|
outcome = this.validateCanonicalRunObservation(input, now);
|
|
124
117
|
break;
|
|
125
118
|
}
|
|
126
|
-
|
|
127
|
-
outcome = result.disposition === "obsolete" ? "obsolete" : "applied";
|
|
119
|
+
outcome = this.foldProviderTurnBoundary(input, adapterId, "completed", input.payload.summary ?? "Agent turn completed without a workflow outcome.", now);
|
|
128
120
|
break;
|
|
129
121
|
}
|
|
130
122
|
case "turn.failed": {
|
|
131
123
|
const failureEvidence = input.payload.failure;
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
agentId: input.fence.agentId,
|
|
137
|
-
adapterId,
|
|
138
|
-
launchId: input.fence.launchId,
|
|
139
|
-
nativeSessionId: input.fence.nativeSessionId,
|
|
140
|
-
turnId: input.fence.nativeTurnId,
|
|
141
|
-
runId: input.fence.runId
|
|
142
|
-
});
|
|
143
|
-
if (classification !== "apply")
|
|
144
|
-
return classification;
|
|
145
|
-
const boundary = this.observeRuntimeTurnCompleted({
|
|
146
|
-
taskId,
|
|
147
|
-
roleName: input.fence.roleName,
|
|
148
|
-
agentId: input.fence.agentId,
|
|
149
|
-
adapterId,
|
|
150
|
-
launchId: input.fence.launchId,
|
|
151
|
-
nativeSessionId: input.fence.nativeSessionId,
|
|
152
|
-
turnId: input.fence.nativeTurnId,
|
|
153
|
-
attemptId: input.fence.receiptId,
|
|
154
|
-
runId: input.fence.runId,
|
|
155
|
-
summary: input.payload.summary ?? `Provider Turn failed: ${failureEvidence.code}.`,
|
|
156
|
-
providerStatus: "failed"
|
|
157
|
-
}, now);
|
|
158
|
-
outcome = boundary.disposition === "obsolete" ? "obsolete" : "applied";
|
|
159
|
-
break;
|
|
160
|
-
}
|
|
161
|
-
const failure = {
|
|
162
|
-
eventId: input.eventId,
|
|
163
|
-
eventType: input.kind,
|
|
164
|
-
taskId,
|
|
165
|
-
roleName: input.fence.roleName,
|
|
166
|
-
agentId: input.fence.agentId,
|
|
167
|
-
adapterId,
|
|
168
|
-
launchId: input.fence.launchId,
|
|
169
|
-
nativeSessionId: input.fence.nativeSessionId,
|
|
170
|
-
nativeTurnId: input.fence.nativeTurnId,
|
|
171
|
-
runId: input.fence.runId,
|
|
172
|
-
...(failureEvidence.errorCode === undefined
|
|
173
|
-
? {}
|
|
174
|
-
: { errorCode: failureEvidence.errorCode }),
|
|
175
|
-
error: failureEvidence.code,
|
|
176
|
-
...(failureEvidence.details === undefined
|
|
177
|
-
? {}
|
|
178
|
-
: { errorDetails: failureEvidence.details }),
|
|
179
|
-
...(failureEvidence.lastOutput === undefined
|
|
180
|
-
? {}
|
|
181
|
-
: { lastAssistantMessage: failureEvidence.lastOutput }),
|
|
182
|
-
...(failureEvidence.retryAfterMs === undefined
|
|
183
|
-
? {}
|
|
184
|
-
: { retryAfterMs: failureEvidence.retryAfterMs })
|
|
185
|
-
};
|
|
186
|
-
if (this.classifyRuntimeTurnFailed(failure) !== "apply")
|
|
187
|
-
return "obsolete";
|
|
188
|
-
outcome = this.observeRuntimeTurnFailed(failure, now).disposition === "applied"
|
|
189
|
-
? "applied"
|
|
190
|
-
: "obsolete";
|
|
124
|
+
// A Provider Turn is an activation boundary, not a Yui Run outcome.
|
|
125
|
+
// Keep the Run and Session identity available for Agent-directed
|
|
126
|
+
// restore + submit, even when a provider marks its Turn terminal.
|
|
127
|
+
outcome = this.foldProviderTurnBoundary(input, adapterId, "failed", input.payload.summary ?? `Agent Turn failed: ${failureEvidence.error.message}`, now);
|
|
191
128
|
break;
|
|
192
129
|
}
|
|
193
130
|
case "operation.started":
|
|
194
131
|
case "operation.completed":
|
|
195
132
|
case "operation.failed":
|
|
196
133
|
case "turn.waiting":
|
|
197
|
-
case "turn.cancelled":
|
|
198
134
|
case "activity.observed":
|
|
199
135
|
case "observer.health":
|
|
200
136
|
case "native-work.snapshot":
|
|
@@ -204,6 +140,10 @@ export class FileSchedulerStoreAdapter {
|
|
|
204
140
|
case "input.delivery-unknown":
|
|
205
141
|
outcome = this.validateCanonicalRunObservation(input, now);
|
|
206
142
|
break;
|
|
143
|
+
case "turn.cancelled": {
|
|
144
|
+
outcome = this.foldProviderTurnBoundary(input, adapterId, "cancelled", input.payload.summary ?? "Provider Turn was cancelled.", now);
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
207
147
|
case "conversation.observed":
|
|
208
148
|
case "activation.started":
|
|
209
149
|
case "activation.ended":
|
|
@@ -220,12 +160,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
220
160
|
default:
|
|
221
161
|
outcome = "obsolete";
|
|
222
162
|
}
|
|
223
|
-
if (outcome === "applied"
|
|
224
|
-
&& input.kind === "activity.observed"
|
|
225
|
-
&& input.payload.activityId !== undefined
|
|
226
|
-
&& input.payload.usage === undefined) {
|
|
227
|
-
this.clearProviderRetryForProgress(input, "correlated-activity", now);
|
|
228
|
-
}
|
|
229
163
|
if (outcome === "applied")
|
|
230
164
|
this.persistRuntimeObservation(input, now);
|
|
231
165
|
return outcome;
|
|
@@ -292,6 +226,26 @@ export class FileSchedulerStoreAdapter {
|
|
|
292
226
|
const projection = projectRuntimeTaskEvents(input.fence, run.createdAt, this.store.listEvents(taskId));
|
|
293
227
|
return Object.values(projection.operations).some(({ kind }) => kind === "subagent");
|
|
294
228
|
}
|
|
229
|
+
foldProviderTurnBoundary(input, adapterId, providerStatus, summary, now) {
|
|
230
|
+
const completed = {
|
|
231
|
+
taskId: input.fence.taskId,
|
|
232
|
+
roleName: input.fence.roleName,
|
|
233
|
+
agentId: input.fence.agentId,
|
|
234
|
+
adapterId,
|
|
235
|
+
launchId: input.fence.launchId,
|
|
236
|
+
nativeSessionId: input.fence.nativeSessionId,
|
|
237
|
+
turnId: input.fence.nativeTurnId,
|
|
238
|
+
attemptId: input.fence.receiptId,
|
|
239
|
+
runId: input.fence.runId,
|
|
240
|
+
summary,
|
|
241
|
+
providerStatus
|
|
242
|
+
};
|
|
243
|
+
const classification = this.classifyRuntimeTurnCompleted(completed);
|
|
244
|
+
if (classification !== "apply")
|
|
245
|
+
return classification;
|
|
246
|
+
const result = this.observeRuntimeTurnCompleted(completed, now);
|
|
247
|
+
return result.disposition === "obsolete" ? "obsolete" : "applied";
|
|
248
|
+
}
|
|
295
249
|
validateCanonicalSessionObservation(input, now) {
|
|
296
250
|
return this.store.transaction((store) => {
|
|
297
251
|
const run = store.getAgentRun(input.fence.taskId, input.fence.runId);
|
|
@@ -307,8 +261,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
307
261
|
recordCanonicalObservationObsolete(store, input, "runtime-session-not-current", now);
|
|
308
262
|
return "obsolete";
|
|
309
263
|
}
|
|
310
|
-
const status =
|
|
311
|
-
let updatedSessions = updateRoleAgentSessionStatus(sessions, input.fence.agentId, status, now);
|
|
264
|
+
const status = "ended";
|
|
265
|
+
let updatedSessions = updateRoleAgentSessionStatus(sessions, input.fence.agentId, status, now, input.kind === "session.failed" ? "failed" : "stopped");
|
|
312
266
|
if (updatedSessions.providerBinding !== null) {
|
|
313
267
|
const activationId = input.fence.activationId ?? input.fence.launchId;
|
|
314
268
|
updatedSessions = updateTaskRoleProviderRuntime(updatedSessions, endProviderActivation(updatedSessions.providerBinding, activationId, {
|
|
@@ -381,7 +335,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
381
335
|
&& active.status === "active"
|
|
382
336
|
&& active.deliveredAt !== undefined
|
|
383
337
|
&& role !== null) {
|
|
384
|
-
store.saveRole(input.fence.taskId, updateRoleStatus(role, "detached", now));
|
|
385
338
|
if (role.name === "leader") {
|
|
386
339
|
const message = [
|
|
387
340
|
`Leader native Session became unavailable while Run ${active.id} remains active.`,
|
|
@@ -483,13 +436,52 @@ export class FileSchedulerStoreAdapter {
|
|
|
483
436
|
lane: "normal"
|
|
484
437
|
});
|
|
485
438
|
}
|
|
486
|
-
if (input.kind === "turn.failed" && input.payload.failure
|
|
487
|
-
|
|
439
|
+
if (input.kind === "turn.failed" && input.payload.failure !== undefined) {
|
|
440
|
+
const failure = input.payload.failure;
|
|
441
|
+
const error = failure.error;
|
|
442
|
+
const run = input.fence.runId === undefined
|
|
443
|
+
? null
|
|
444
|
+
: store.getAgentRun(taskId, input.fence.runId);
|
|
445
|
+
const errorEvent = createTaskEvent(store.nextEventId(taskId), taskId, "runtime.agent-error", {
|
|
446
|
+
sourceEventId: input.eventId,
|
|
447
|
+
observationEventId,
|
|
448
|
+
runId: input.fence.runId ?? "",
|
|
449
|
+
roleName: input.fence.roleName,
|
|
450
|
+
agentId: input.fence.agentId,
|
|
451
|
+
adapterId: run?.effective.adapterId ?? "unknown",
|
|
452
|
+
driverId: input.fence.driverId,
|
|
453
|
+
launchId: input.fence.launchId,
|
|
454
|
+
nativeSessionId: input.fence.nativeSessionId ?? "",
|
|
455
|
+
nativeTurnId: input.fence.nativeTurnId ?? "",
|
|
456
|
+
source: error.source,
|
|
457
|
+
phase: error.phase,
|
|
458
|
+
category: error.category,
|
|
459
|
+
code: error.code,
|
|
460
|
+
message: error.message,
|
|
461
|
+
raw: error.raw,
|
|
462
|
+
inputDisposition: error.inputDisposition,
|
|
463
|
+
sessionDisposition: error.sessionDisposition,
|
|
464
|
+
...(error.retryAfterMs === undefined
|
|
465
|
+
? {}
|
|
466
|
+
: { retryAfterMs: String(error.retryAfterMs) }),
|
|
467
|
+
...(failure.lastOutput === undefined ? {} : { lastOutput: failure.lastOutput })
|
|
468
|
+
}, now);
|
|
469
|
+
store.saveEvent(taskId, errorEvent);
|
|
470
|
+
enqueueWork(store, { kind: "role", taskId, roleName: "leader" }, wakeReason("agent-error", errorEvent.id), now, [{ type: "event", taskId, id: errorEvent.id }], {
|
|
488
471
|
source: input.fence.driverId,
|
|
489
|
-
dedupeKey: input.
|
|
472
|
+
dedupeKey: `agent-error:${taskId}:${input.eventId}`,
|
|
490
473
|
deliveryMode: "followup",
|
|
491
474
|
lane: "normal"
|
|
492
475
|
});
|
|
476
|
+
if (input.fence.roleName === "leader"
|
|
477
|
+
&& error.sessionDisposition === "unrecoverable") {
|
|
478
|
+
enqueueWork(store, { kind: "operator" }, "leader-session-unrecoverable", now, [{ type: "event", taskId, id: errorEvent.id }], {
|
|
479
|
+
source: input.fence.driverId,
|
|
480
|
+
dedupeKey: `leader-session-unrecoverable:${taskId}:${input.eventId}`,
|
|
481
|
+
deliveryMode: "followup",
|
|
482
|
+
lane: "normal"
|
|
483
|
+
});
|
|
484
|
+
}
|
|
493
485
|
}
|
|
494
486
|
if ((input.kind === "operation.completed" || input.kind === "operation.failed")
|
|
495
487
|
&& input.payload.operation === "subagent") {
|
|
@@ -498,7 +490,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
498
490
|
// A live provider Session owns delivery of its native child result.
|
|
499
491
|
// Route a Yui wake only when that parent is gone and another observer
|
|
500
492
|
// supplied the child's terminal fact.
|
|
501
|
-
if (role !== null &&
|
|
493
|
+
if (role !== null && session?.status === "ended") {
|
|
502
494
|
routeRoleEvent(store, observationEvent, role.name, "detached-native-subagent-terminal", now);
|
|
503
495
|
}
|
|
504
496
|
}
|
|
@@ -621,7 +613,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
621
613
|
return null;
|
|
622
614
|
const sessions = this.store.getGlobalRoleSessionSet(SYSTEM_OPERATOR_ROLE);
|
|
623
615
|
const effectiveSession = sessions?.sessions[sessions.activeAgentId];
|
|
624
|
-
if (effectiveSession?.status !== "
|
|
616
|
+
if (effectiveSession?.status !== "active")
|
|
625
617
|
return null;
|
|
626
618
|
return {
|
|
627
619
|
roleName: SYSTEM_OPERATOR_ROLE,
|
|
@@ -629,15 +621,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
629
621
|
};
|
|
630
622
|
}
|
|
631
623
|
markOperatorTurnStarted(now) {
|
|
632
|
-
|
|
633
|
-
const sessions = store.getGlobalRoleSessionSet(SYSTEM_OPERATOR_ROLE);
|
|
634
|
-
if (sessions === null)
|
|
635
|
-
return;
|
|
636
|
-
const active = sessions.sessions[sessions.activeAgentId];
|
|
637
|
-
if (active === undefined || active.status !== "ready")
|
|
638
|
-
return;
|
|
639
|
-
store.saveGlobalRoleSessionSet(updateRoleAgentSessionStatus(sessions, sessions.activeAgentId, "running", now));
|
|
640
|
-
});
|
|
624
|
+
void now;
|
|
641
625
|
}
|
|
642
626
|
resolveExpiredInputRecommendations(now, taskIds) {
|
|
643
627
|
return this.store.transaction((store) => {
|
|
@@ -827,10 +811,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
827
811
|
return existing && !recovered ? "already-recorded" : "recorded";
|
|
828
812
|
});
|
|
829
813
|
}
|
|
830
|
-
hasInFlightTurn(taskId, roleName) {
|
|
831
|
-
const sessions = this.store.getTaskRoleSessionSet(taskId, roleName);
|
|
832
|
-
return sessions !== null && sessions.inFlight !== null;
|
|
833
|
-
}
|
|
834
814
|
beginAgentHostProviderTurn(input) {
|
|
835
815
|
this.store.transaction((store) => {
|
|
836
816
|
const sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
@@ -844,7 +824,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
844
824
|
if (sessions === null || sessions === undefined
|
|
845
825
|
|| binding === null || binding === undefined
|
|
846
826
|
|| sessions.inFlight?.runId !== input.runId
|
|
847
|
-
|| binding.runId !== input.runId
|
|
848
827
|
|| session?.launchId !== input.launchId
|
|
849
828
|
|| session.nativeSessionId !== input.nativeSessionId
|
|
850
829
|
|| currentProviderConversation(binding).conversationId !== input.nativeSessionId
|
|
@@ -854,7 +833,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
854
833
|
|| !processingOwnsExactInitialDelivery(mailbox, input.taskId, input.runId, input.attemptId)) {
|
|
855
834
|
throw new AgentHostProviderTurnFenceError("Agent Host Provider Turn carries a stale durable writer fence. Release and reacquire Provider authority before retrying input.");
|
|
856
835
|
}
|
|
857
|
-
const exactReplay = binding.turn?.
|
|
836
|
+
const exactReplay = binding.turn?.runId === input.runId
|
|
837
|
+
&& binding.turn.attemptId === input.attemptId
|
|
858
838
|
&& binding.turn.authorityEpoch === input.authorityEpoch
|
|
859
839
|
&& binding.turn.status === "submitting";
|
|
860
840
|
if (!exactReplay && binding.turn !== null
|
|
@@ -863,6 +843,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
863
843
|
throw new AgentHostProviderTurnFenceError("Provider Conversation already has an unsettled Turn.");
|
|
864
844
|
}
|
|
865
845
|
store.saveTaskRoleSessionSet(updateTaskRoleProviderRuntime(sessions, beginProviderTurn(binding, {
|
|
846
|
+
runId: input.runId,
|
|
866
847
|
attemptId: input.attemptId,
|
|
867
848
|
authorityEpoch: input.authorityEpoch,
|
|
868
849
|
submittedAt: input.now.toISOString()
|
|
@@ -875,7 +856,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
875
856
|
const binding = sessions?.providerBinding;
|
|
876
857
|
if (sessions === null || sessions === undefined
|
|
877
858
|
|| binding === null || binding === undefined
|
|
878
|
-
|| binding.runId !== input.runId
|
|
859
|
+
|| binding.turn?.runId !== input.runId
|
|
879
860
|
|| binding.turn?.attemptId !== input.attemptId) {
|
|
880
861
|
throw new Error("Agent Host Provider Turn submission is no longer current.");
|
|
881
862
|
}
|
|
@@ -886,6 +867,53 @@ export class FileSchedulerStoreAdapter {
|
|
|
886
867
|
resolvedAt: input.now.toISOString()
|
|
887
868
|
});
|
|
888
869
|
store.saveTaskRoleSessionSet(updateTaskRoleProviderRuntime(sessions, updated, input.now));
|
|
870
|
+
const run = store.getAgentRun(input.taskId, input.runId);
|
|
871
|
+
const session = sessions.sessions[sessions.activeAgentId];
|
|
872
|
+
const driver = run === null
|
|
873
|
+
? null
|
|
874
|
+
: this.drivers.findByAdapterId(run.effective.adapterId);
|
|
875
|
+
const error = standardAgentError({
|
|
876
|
+
source: "driver",
|
|
877
|
+
phase: "turn-submit",
|
|
878
|
+
classification: driver?.runtime.mapError({
|
|
879
|
+
message: input.reason,
|
|
880
|
+
raw: input.raw
|
|
881
|
+
}),
|
|
882
|
+
message: input.reason,
|
|
883
|
+
raw: input.raw,
|
|
884
|
+
inputDisposition: input.status === "rejected" ? "not-accepted" : "unknown"
|
|
885
|
+
});
|
|
886
|
+
const errorEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.agent-error", {
|
|
887
|
+
sourceEventId: input.attemptId,
|
|
888
|
+
runId: input.runId,
|
|
889
|
+
roleName: input.roleName,
|
|
890
|
+
agentId: run?.effective.agentId ?? sessions.activeAgentId,
|
|
891
|
+
adapterId: run?.effective.adapterId ?? session?.adapterId ?? "unknown",
|
|
892
|
+
driverId: driver?.id ?? "unknown",
|
|
893
|
+
launchId: session?.launchId ?? "",
|
|
894
|
+
nativeSessionId: session?.nativeSessionId ?? "",
|
|
895
|
+
nativeTurnId: "",
|
|
896
|
+
source: error.source,
|
|
897
|
+
phase: error.phase,
|
|
898
|
+
category: error.category,
|
|
899
|
+
code: error.code,
|
|
900
|
+
message: error.message,
|
|
901
|
+
raw: error.raw,
|
|
902
|
+
inputDisposition: error.inputDisposition,
|
|
903
|
+
sessionDisposition: error.sessionDisposition
|
|
904
|
+
}, input.now);
|
|
905
|
+
store.saveEvent(input.taskId, errorEvent);
|
|
906
|
+
const route = input.roleName === "leader"
|
|
907
|
+
? { kind: "operator" }
|
|
908
|
+
: { kind: "role", taskId: input.taskId, roleName: "leader" };
|
|
909
|
+
enqueueWork(store, route, input.roleName === "leader"
|
|
910
|
+
? "leader-turn-submission-error"
|
|
911
|
+
: wakeReason("agent-error", errorEvent.id), input.now, [{ type: "event", taskId: input.taskId, id: errorEvent.id }], {
|
|
912
|
+
source: driver?.id ?? "agent-host",
|
|
913
|
+
dedupeKey: `agent-error:${input.taskId}:${input.attemptId}`,
|
|
914
|
+
deliveryMode: "followup",
|
|
915
|
+
lane: "normal"
|
|
916
|
+
});
|
|
889
917
|
});
|
|
890
918
|
}
|
|
891
919
|
getProviderAuthorityFence(input) {
|
|
@@ -893,7 +921,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
893
921
|
const session = sessions?.sessions[input.agentId];
|
|
894
922
|
const binding = sessions?.providerBinding;
|
|
895
923
|
if (binding === null || binding === undefined
|
|
896
|
-
|| binding.runId !== input.runId
|
|
897
924
|
|| session?.launchId !== input.launchId
|
|
898
925
|
|| session.nativeSessionId !== input.nativeSessionId
|
|
899
926
|
|| currentProviderConversation(binding).conversationId !== input.nativeSessionId)
|
|
@@ -923,6 +950,80 @@ export class FileSchedulerStoreAdapter {
|
|
|
923
950
|
enqueueLeaderWakeup(taskId, reason, now) {
|
|
924
951
|
return this.store.transaction((store) => (queueLeaderWakeup(store, taskId, reason, now)));
|
|
925
952
|
}
|
|
953
|
+
recordAgentError(input, now) {
|
|
954
|
+
return this.store.transaction((store) => {
|
|
955
|
+
const duplicate = [...store.listEvents(input.taskId)].reverse().find((event) => (event.type === "runtime.agent-error"
|
|
956
|
+
&& event.payload.runId === input.runId
|
|
957
|
+
&& event.payload.phase === input.phase
|
|
958
|
+
&& event.payload.raw === input.raw));
|
|
959
|
+
if (duplicate !== undefined)
|
|
960
|
+
return duplicate.id;
|
|
961
|
+
const run = store.getAgentRun(input.taskId, input.runId);
|
|
962
|
+
const sessionSet = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
963
|
+
const sessionAgentId = run?.effective.agentId ?? sessionSet?.activeAgentId;
|
|
964
|
+
const session = sessionAgentId === undefined
|
|
965
|
+
? undefined
|
|
966
|
+
: sessionSet?.sessions[sessionAgentId];
|
|
967
|
+
const driver = run === null
|
|
968
|
+
? null
|
|
969
|
+
: this.drivers.findByAdapterId(run.effective.adapterId);
|
|
970
|
+
const error = standardAgentError({
|
|
971
|
+
source: input.source,
|
|
972
|
+
phase: input.phase,
|
|
973
|
+
classification: driver?.runtime.mapError({
|
|
974
|
+
message: input.message,
|
|
975
|
+
raw: input.raw
|
|
976
|
+
}),
|
|
977
|
+
message: input.message,
|
|
978
|
+
raw: input.raw,
|
|
979
|
+
...(input.inputDisposition === undefined
|
|
980
|
+
? {}
|
|
981
|
+
: { inputDisposition: input.inputDisposition }),
|
|
982
|
+
...(input.sessionDisposition === undefined
|
|
983
|
+
? {}
|
|
984
|
+
: { sessionDisposition: input.sessionDisposition })
|
|
985
|
+
});
|
|
986
|
+
const event = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.agent-error", {
|
|
987
|
+
sourceEventId: `${input.runId}:${input.phase}`,
|
|
988
|
+
runId: input.runId,
|
|
989
|
+
roleName: input.roleName,
|
|
990
|
+
agentId: run?.effective.agentId ?? session?.agentId ?? "unknown",
|
|
991
|
+
adapterId: run?.effective.adapterId ?? session?.adapterId ?? "unknown",
|
|
992
|
+
driverId: driver?.id ?? "unknown",
|
|
993
|
+
launchId: session?.launchId ?? "",
|
|
994
|
+
nativeSessionId: session?.nativeSessionId ?? "",
|
|
995
|
+
nativeTurnId: "",
|
|
996
|
+
source: error.source,
|
|
997
|
+
phase: error.phase,
|
|
998
|
+
category: error.category,
|
|
999
|
+
code: error.code,
|
|
1000
|
+
message: error.message,
|
|
1001
|
+
raw: error.raw,
|
|
1002
|
+
inputDisposition: error.inputDisposition,
|
|
1003
|
+
sessionDisposition: error.sessionDisposition
|
|
1004
|
+
}, now);
|
|
1005
|
+
store.saveEvent(input.taskId, event);
|
|
1006
|
+
const leaderCannotReceive = input.roleName === "leader"
|
|
1007
|
+
&& ["host-start", "session-start", "session-restore", "turn-submit"].includes(input.phase);
|
|
1008
|
+
if (!leaderCannotReceive) {
|
|
1009
|
+
enqueueWork(store, { kind: "role", taskId: input.taskId, roleName: "leader" }, wakeReason("agent-error", event.id), now, [{ type: "event", taskId: input.taskId, id: event.id }], {
|
|
1010
|
+
source: driver?.id ?? input.source,
|
|
1011
|
+
dedupeKey: `agent-error:${input.taskId}:${input.runId}:${input.phase}:${event.id}`,
|
|
1012
|
+
deliveryMode: "followup",
|
|
1013
|
+
lane: "normal"
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
1016
|
+
if (leaderCannotReceive) {
|
|
1017
|
+
enqueueWork(store, { kind: "operator" }, "leader-agent-error", now, [{ type: "event", taskId: input.taskId, id: event.id }], {
|
|
1018
|
+
source: driver?.id ?? input.source,
|
|
1019
|
+
dedupeKey: `leader-agent-error:${input.taskId}:${input.runId}:${event.id}`,
|
|
1020
|
+
deliveryMode: "followup",
|
|
1021
|
+
lane: "normal"
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
return event.id;
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
926
1027
|
/**
|
|
927
1028
|
* Records that a Leader wake was suppressed by scheduler single-flight
|
|
928
1029
|
* (the Role runtime lifecycle lane was busy). The wake stays durable and
|
|
@@ -1006,15 +1107,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1006
1107
|
const pending = pendingLane(mailbox, input.lane);
|
|
1007
1108
|
if (pending === null)
|
|
1008
1109
|
return { status: "empty" };
|
|
1009
|
-
const claimed = claimMailboxInputDelivery(mailbox, {
|
|
1010
|
-
attemptId: input.attemptId,
|
|
1011
|
-
lane: input.lane,
|
|
1012
|
-
mode: input.mode,
|
|
1013
|
-
owner: input.owner,
|
|
1014
|
-
startedAt: input.now.toISOString(),
|
|
1015
|
-
executionRef: input.executionRef,
|
|
1016
|
-
providerFence: input.providerFence
|
|
1017
|
-
});
|
|
1018
1110
|
if (input.target.kind === "role"
|
|
1019
1111
|
&& input.executionRef.type === "run"
|
|
1020
1112
|
&& input.executionRef.taskId === input.target.taskId) {
|
|
@@ -1022,10 +1114,14 @@ export class FileSchedulerStoreAdapter {
|
|
|
1022
1114
|
const binding = sessions?.providerBinding;
|
|
1023
1115
|
if (sessions === null || sessions === undefined
|
|
1024
1116
|
|| binding === null || binding === undefined
|
|
1025
|
-
|| binding.runId !== input.executionRef.id
|
|
1026
1117
|
|| binding.authority.owner !== "controller") {
|
|
1027
1118
|
throw new Error("Controller does not own the Provider writer authority.");
|
|
1028
1119
|
}
|
|
1120
|
+
if (binding.turn !== null
|
|
1121
|
+
&& ["submitting", "accepted", "running", "delivery-unknown"]
|
|
1122
|
+
.includes(binding.turn.status)) {
|
|
1123
|
+
return { status: "busy" };
|
|
1124
|
+
}
|
|
1029
1125
|
const conversation = currentProviderConversation(binding);
|
|
1030
1126
|
const activation = currentProviderActivation(binding);
|
|
1031
1127
|
if (input.providerFence === undefined
|
|
@@ -1034,12 +1130,22 @@ export class FileSchedulerStoreAdapter {
|
|
|
1034
1130
|
throw new Error("Provider input claim carries a stale Conversation/Activation fence.");
|
|
1035
1131
|
}
|
|
1036
1132
|
const next = beginProviderTurn(binding, {
|
|
1133
|
+
runId: input.executionRef.id,
|
|
1037
1134
|
attemptId: input.attemptId,
|
|
1038
1135
|
authorityEpoch: binding.authority.epoch,
|
|
1039
1136
|
submittedAt: input.now.toISOString()
|
|
1040
1137
|
});
|
|
1041
1138
|
store.saveTaskRoleSessionSet(updateTaskRoleProviderRuntime(sessions, next, input.now));
|
|
1042
1139
|
}
|
|
1140
|
+
const claimed = claimMailboxInputDelivery(mailbox, {
|
|
1141
|
+
attemptId: input.attemptId,
|
|
1142
|
+
lane: input.lane,
|
|
1143
|
+
mode: input.mode,
|
|
1144
|
+
owner: input.owner,
|
|
1145
|
+
startedAt: input.now.toISOString(),
|
|
1146
|
+
executionRef: input.executionRef,
|
|
1147
|
+
providerFence: input.providerFence
|
|
1148
|
+
});
|
|
1043
1149
|
store.saveWorkMailbox(claimed);
|
|
1044
1150
|
return { status: "claimed", delivery: claimed.inputDelivery };
|
|
1045
1151
|
});
|
|
@@ -1143,19 +1249,20 @@ export class FileSchedulerStoreAdapter {
|
|
|
1143
1249
|
completeRuntimeCleanup(target, now) {
|
|
1144
1250
|
return this.store.transaction((store) => {
|
|
1145
1251
|
let mailbox = store.getWorkMailbox(target);
|
|
1146
|
-
|
|
1252
|
+
const disposition = runtimeCleanupDisposition(mailbox);
|
|
1253
|
+
if (mailbox === null || disposition === null)
|
|
1147
1254
|
return false;
|
|
1148
1255
|
if (mailbox.processing !== null) {
|
|
1149
1256
|
if (!isRuntimeLaunchReservation(mailbox.processing)
|
|
1150
|
-
&& !mailbox.processing.batch.reasons.
|
|
1257
|
+
&& !mailbox.processing.batch.reasons.every(isRuntimeCleanupReason)) {
|
|
1151
1258
|
return false;
|
|
1152
1259
|
}
|
|
1153
1260
|
mailbox = completeProcessing(mailbox, mailbox.processing.batchId);
|
|
1154
1261
|
}
|
|
1155
1262
|
const pending = pendingLane(mailbox, "normal");
|
|
1156
1263
|
if (pending !== null) {
|
|
1157
|
-
if (pending.reasons.length
|
|
1158
|
-
|| pending.reasons
|
|
1264
|
+
if (pending.reasons.length === 0
|
|
1265
|
+
|| !pending.reasons.every(isRuntimeCleanupReason)) {
|
|
1159
1266
|
return false;
|
|
1160
1267
|
}
|
|
1161
1268
|
const batchId = `runtime-cleanup-complete:${pending.fromSequence}-${pending.toSequence}`;
|
|
@@ -1166,7 +1273,12 @@ export class FileSchedulerStoreAdapter {
|
|
|
1166
1273
|
}), batchId);
|
|
1167
1274
|
}
|
|
1168
1275
|
const owner = runtimeOwnerFromTarget(target);
|
|
1169
|
-
|
|
1276
|
+
if (disposition === "end-session") {
|
|
1277
|
+
endRuntimeOwnerSession(store, owner, now);
|
|
1278
|
+
}
|
|
1279
|
+
else {
|
|
1280
|
+
detachRuntimeOwnerHost(store, owner, now);
|
|
1281
|
+
}
|
|
1170
1282
|
saveRuntimeLifecycleMailbox(store, mailbox);
|
|
1171
1283
|
return true;
|
|
1172
1284
|
});
|
|
@@ -1177,7 +1289,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1177
1289
|
if (!isRuntimeLaunchReservation(mailbox?.processing, batchId)) {
|
|
1178
1290
|
return false;
|
|
1179
1291
|
}
|
|
1180
|
-
|
|
1292
|
+
endRuntimeOwnerSession(store, runtimeOwnerFromTarget(target), now);
|
|
1181
1293
|
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, batchId));
|
|
1182
1294
|
return true;
|
|
1183
1295
|
});
|
|
@@ -1202,14 +1314,22 @@ export class FileSchedulerStoreAdapter {
|
|
|
1202
1314
|
listRuntimeSessionCandidates(query = {}) {
|
|
1203
1315
|
return this.store.listRuntimeSessionCandidates(query);
|
|
1204
1316
|
}
|
|
1205
|
-
|
|
1317
|
+
enqueueRuntimeCleanup(owner, now = new Date(), expectedDormantCandidate) {
|
|
1206
1318
|
return this.store.transaction((store) => {
|
|
1207
|
-
if (
|
|
1208
|
-
|
|
1209
|
-
|
|
1319
|
+
if (expectedDormantCandidate !== undefined
|
|
1320
|
+
&& (!sameRuntimeOwner(owner, expectedDormantCandidate.owner)
|
|
1321
|
+
|| !dormantRuntimeCandidateIsCurrent(store, expectedDormantCandidate))) {
|
|
1322
|
+
return null;
|
|
1323
|
+
}
|
|
1324
|
+
if (owner.scope === "task" && store.getTask(owner.taskId) === null) {
|
|
1325
|
+
return null;
|
|
1326
|
+
}
|
|
1327
|
+
const target = runtimeLifecycleTarget(owner);
|
|
1328
|
+
enqueueWork(store, target, RUNTIME_CLEANUP_REQUIRED_REASON, now, owner.scope === "task" ? [{ type: "task", id: owner.taskId }] : []);
|
|
1329
|
+
return target;
|
|
1210
1330
|
});
|
|
1211
1331
|
}
|
|
1212
|
-
|
|
1332
|
+
enqueueRuntimeHostDetach(owner, now = new Date(), expectedDormantCandidate) {
|
|
1213
1333
|
return this.store.transaction((store) => {
|
|
1214
1334
|
if (expectedDormantCandidate !== undefined
|
|
1215
1335
|
&& (!sameRuntimeOwner(owner, expectedDormantCandidate.owner)
|
|
@@ -1220,7 +1340,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1220
1340
|
return null;
|
|
1221
1341
|
}
|
|
1222
1342
|
const target = runtimeLifecycleTarget(owner);
|
|
1223
|
-
enqueueWork(store, target,
|
|
1343
|
+
enqueueWork(store, target, RUNTIME_HOST_DETACH_REQUIRED_REASON, now, owner.scope === "task" ? [{ type: "task", id: owner.taskId }] : []);
|
|
1224
1344
|
return target;
|
|
1225
1345
|
});
|
|
1226
1346
|
}
|
|
@@ -1282,7 +1402,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1282
1402
|
store.saveAgentRun(input.run);
|
|
1283
1403
|
store.saveActiveAgentRun(input.run);
|
|
1284
1404
|
store.saveWorkMailbox(claimed);
|
|
1285
|
-
store.saveRole(input.task.id, updateRoleStatus(role, "running", input.now));
|
|
1286
1405
|
bindTaskRoleRunInFlight(store, role, input.run, input.now);
|
|
1287
1406
|
store.saveEvent(input.task.id, createTaskEvent(store.nextEventId(input.task.id), input.task.id, "run.dispatched", runLaunchEventPayload(input.run), input.now));
|
|
1288
1407
|
if (input.wakeId !== undefined && input.wakeFromCursor !== undefined) {
|
|
@@ -1309,7 +1428,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1309
1428
|
saveTaskSession(store, role, {
|
|
1310
1429
|
...input.session,
|
|
1311
1430
|
nativeSessionId: input.session.nativeSessionId
|
|
1312
|
-
}, "
|
|
1431
|
+
}, "active", input.now);
|
|
1313
1432
|
}
|
|
1314
1433
|
store.clearLeaderFailure(input.task.id);
|
|
1315
1434
|
return "claimed";
|
|
@@ -1327,17 +1446,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1327
1446
|
throw new Error(`Active Agent run changed before delivery was persisted: ${input.run.id}.`);
|
|
1328
1447
|
}
|
|
1329
1448
|
let deliveryRun = active;
|
|
1330
|
-
if (active.providerRetry?.state === "dispatching") {
|
|
1331
|
-
deliveryRun = withProviderRetry(active, markProviderRetryDispatched(active.providerRetry));
|
|
1332
|
-
store.saveAgentRun(deliveryRun);
|
|
1333
|
-
store.saveEvent(task.id, createTaskEvent(store.nextEventId(task.id), task.id, "runtime.provider-retry-dispatched", {
|
|
1334
|
-
runId: active.id,
|
|
1335
|
-
roleName: active.roleName,
|
|
1336
|
-
episodeId: active.providerRetry.episodeId,
|
|
1337
|
-
receiptId: active.providerRetry.lastRetryReceiptId ?? "",
|
|
1338
|
-
dispatchedRetries: String(active.providerRetry.dispatchedRetries + 1)
|
|
1339
|
-
}, input.now));
|
|
1340
|
-
}
|
|
1341
1449
|
if (active.controlRequest?.state === "dispatching") {
|
|
1342
1450
|
deliveryRun = {
|
|
1343
1451
|
...active,
|
|
@@ -1368,23 +1476,18 @@ export class FileSchedulerStoreAdapter {
|
|
|
1368
1476
|
markTaskRoleRunPushedInFlight(store, role, deliveryRun, input.now);
|
|
1369
1477
|
}
|
|
1370
1478
|
}
|
|
1371
|
-
if (role.status !== "running") {
|
|
1372
|
-
store.saveRole(input.task.id, updateRoleStatus(role, "running", input.now));
|
|
1373
|
-
}
|
|
1374
1479
|
if (input.session !== null && input.session.nativeSessionId !== undefined) {
|
|
1375
1480
|
const existing = store.getRoleSession(input.task.id, input.role.name);
|
|
1376
|
-
const reservation = matchingPreparedRuntimeReservation(store, input);
|
|
1377
1481
|
if (existing?.agentId !== input.session.agentId
|
|
1378
1482
|
|| existing.adapterId !== input.session.adapterId
|
|
1379
1483
|
|| existing.nativeSessionId !== input.session.nativeSessionId
|
|
1380
1484
|
|| (input.launchId !== undefined && existing.launchId !== input.launchId)
|
|
1381
|
-
|| existing.status !== "
|
|
1485
|
+
|| existing.status !== "active") {
|
|
1382
1486
|
saveTaskSession(store, role, {
|
|
1383
1487
|
...input.session,
|
|
1384
1488
|
nativeSessionId: input.session.nativeSessionId
|
|
1385
|
-
}, "
|
|
1489
|
+
}, "active", input.now, input.launchId);
|
|
1386
1490
|
}
|
|
1387
|
-
completePreparedRuntimeReservation(store, reservation);
|
|
1388
1491
|
}
|
|
1389
1492
|
});
|
|
1390
1493
|
}
|
|
@@ -1401,7 +1504,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1401
1504
|
}
|
|
1402
1505
|
if (input.session !== null && input.session.nativeSessionId !== undefined) {
|
|
1403
1506
|
const existing = store.getRoleSession(input.task.id, input.role.name);
|
|
1404
|
-
matchingPreparedRuntimeReservation(store, input);
|
|
1405
1507
|
// A terminal Session is audit history, not the current execution
|
|
1406
1508
|
// identity. Preallocated providers must publish the replacement fence
|
|
1407
1509
|
// before the new Agent Host starts so its exact-runtime preflight sees
|
|
@@ -1409,16 +1511,15 @@ export class FileSchedulerStoreAdapter {
|
|
|
1409
1511
|
const defersConversationReplacement = active.mode === "new"
|
|
1410
1512
|
&& existing !== null
|
|
1411
1513
|
&& existing.nativeSessionId !== input.session.nativeSessionId
|
|
1412
|
-
&& existing.status
|
|
1413
|
-
&& existing.status !== "broken";
|
|
1514
|
+
&& existing.status === "active";
|
|
1414
1515
|
if (!defersConversationReplacement
|
|
1415
1516
|
&& (existing?.nativeSessionId !== input.session.nativeSessionId
|
|
1416
1517
|
|| (input.launchId !== undefined && existing.launchId !== input.launchId)
|
|
1417
|
-
|| existing.status !== "
|
|
1518
|
+
|| existing.status !== "active")) {
|
|
1418
1519
|
saveTaskSession(store, role, {
|
|
1419
1520
|
...input.session,
|
|
1420
1521
|
nativeSessionId: input.session.nativeSessionId
|
|
1421
|
-
}, "
|
|
1522
|
+
}, "active", input.now, input.launchId);
|
|
1422
1523
|
}
|
|
1423
1524
|
}
|
|
1424
1525
|
bindTaskRoleRunInFlight(store, role, input.run, input.now);
|
|
@@ -1436,11 +1537,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1436
1537
|
taskId: input.taskId,
|
|
1437
1538
|
roleName: input.roleName
|
|
1438
1539
|
});
|
|
1439
|
-
const runtimeMailbox = store.getWorkMailbox(runtimeLifecycleTarget({
|
|
1440
|
-
scope: "task",
|
|
1441
|
-
taskId: input.taskId,
|
|
1442
|
-
roleName: input.roleName
|
|
1443
|
-
}));
|
|
1444
1540
|
if (task === null
|
|
1445
1541
|
|| task.status !== "active"
|
|
1446
1542
|
|| task.executionGate.state !== "enabled"
|
|
@@ -1455,12 +1551,11 @@ export class FileSchedulerStoreAdapter {
|
|
|
1455
1551
|
|| mailbox.processing.executionRef?.type !== "run"
|
|
1456
1552
|
|| mailbox.processing.executionRef.taskId !== input.taskId
|
|
1457
1553
|
|| mailbox.processing.executionRef.id !== input.runId
|
|
1458
|
-
|| !preparedDeliveryFailureSessionFenceMatches(sessions, session, input, agentRunDeliveryReceiptId(active))
|
|
1459
|
-
|| !preparedReservationMatches(runtimeMailbox, input.taskId, input.runId, input.launchId)) {
|
|
1554
|
+
|| !preparedDeliveryFailureSessionFenceMatches(sessions, session, input, agentRunDeliveryReceiptId(active))) {
|
|
1460
1555
|
return "state-changed";
|
|
1461
1556
|
}
|
|
1462
1557
|
const summary = input.summary
|
|
1463
|
-
?? `Role delivery
|
|
1558
|
+
?? `Role delivery failed conclusively before exact Run input acceptance: ${input.runId}.`;
|
|
1464
1559
|
const result = terminalizeExactTaskRun(store, {
|
|
1465
1560
|
taskId: input.taskId,
|
|
1466
1561
|
roleName: input.roleName,
|
|
@@ -1481,7 +1576,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1481
1576
|
scope: "task",
|
|
1482
1577
|
taskId: input.taskId,
|
|
1483
1578
|
roleName: input.roleName
|
|
1484
|
-
}),
|
|
1579
|
+
}), RUNTIME_HOST_DETACH_REQUIRED_REASON, input.now, [{ type: "task", id: input.taskId }]);
|
|
1485
1580
|
}
|
|
1486
1581
|
const terminal = result.run;
|
|
1487
1582
|
const deliveryFailureEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.role-delivery-failed", {
|
|
@@ -1494,168 +1589,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
1494
1589
|
routeRoleEvent(store, deliveryFailureEvent, input.roleName, terminal.purpose === "review" ? "review-failed" : "role-run-failed", input.now);
|
|
1495
1590
|
}
|
|
1496
1591
|
else {
|
|
1497
|
-
const failedRole = store.getRole(input.taskId, input.roleName);
|
|
1498
|
-
if (failedRole !== null) {
|
|
1499
|
-
store.saveRole(input.taskId, updateRoleStatus(failedRole, "failed", input.now));
|
|
1500
|
-
}
|
|
1501
1592
|
store.saveLeaderFailure(recordLeaderFailure(input.taskId, session?.nativeSessionId ?? "(unregistered)", summary, input.now, store.getLeaderFailure(input.taskId)));
|
|
1502
|
-
routeRoleEvent(store, deliveryFailureEvent, input.roleName, "leader-run-failed", input.now);
|
|
1503
|
-
}
|
|
1504
|
-
return "failed";
|
|
1505
|
-
});
|
|
1506
|
-
}
|
|
1507
|
-
saveLeaderDispatchFailure(input) {
|
|
1508
|
-
return this.store.transaction((store) => {
|
|
1509
|
-
const task = store.getTask(input.task.id);
|
|
1510
|
-
const role = store.getRole(input.task.id, input.role.name);
|
|
1511
|
-
const active = store.getActiveAgentRun(input.task.id, input.role.name);
|
|
1512
|
-
const target = {
|
|
1513
|
-
kind: "role",
|
|
1514
|
-
taskId: input.task.id,
|
|
1515
|
-
roleName: input.role.name
|
|
1516
|
-
};
|
|
1517
|
-
const mailbox = store.getWorkMailbox(target);
|
|
1518
|
-
if (task === null
|
|
1519
|
-
|| task.status !== "active"
|
|
1520
|
-
|| task.executionGate.state !== "enabled"
|
|
1521
|
-
|| role === null
|
|
1522
|
-
|| active === null
|
|
1523
|
-
|| active.id !== input.claimed.run.id
|
|
1524
|
-
|| active.status !== "active"
|
|
1525
|
-
|| mailbox?.processing?.executionRef?.type !== "run"
|
|
1526
|
-
|| mailbox.processing.executionRef.taskId !== input.task.id
|
|
1527
|
-
|| mailbox.processing.executionRef.id !== input.claimed.run.id
|
|
1528
|
-
|| !schedulerRoleSessionsMatch(store.getRoleSession(input.task.id, input.role.name), input.session)) {
|
|
1529
|
-
return "state-changed";
|
|
1530
|
-
}
|
|
1531
|
-
store.saveAgentRun(failAgentRun(active, input.failure.message, input.now));
|
|
1532
|
-
store.clearActiveAgentRun(input.task.id, input.role.name);
|
|
1533
|
-
clearTaskRoleRunInFlight(store, role, active, input.now);
|
|
1534
|
-
// Consume this failed delivery attempt. Releasing it would immediately
|
|
1535
|
-
// redispatch the same wake and manufacture another Leader Session.
|
|
1536
|
-
store.saveWorkMailbox(completeProcessing(mailbox, mailbox.processing.batchId));
|
|
1537
|
-
store.saveRole(input.task.id, updateRoleStatus(role, "failed", input.now));
|
|
1538
|
-
breakTaskSessionIfPresent(store, input.task.id, role.name, active.effective.agentId, input.now);
|
|
1539
|
-
const failure = recordLeaderFailure(input.task.id, input.failure.nativeSessionId, input.failure.message, input.now, store.getLeaderFailure(input.task.id));
|
|
1540
|
-
store.saveLeaderFailure(failure);
|
|
1541
|
-
recordLeaderAttentionRequired(store, {
|
|
1542
|
-
taskId: input.task.id,
|
|
1543
|
-
reason: "leader-recovery-failed",
|
|
1544
|
-
payload: {
|
|
1545
|
-
message: input.failure.message,
|
|
1546
|
-
runId: active.id
|
|
1547
|
-
},
|
|
1548
|
-
now: input.now
|
|
1549
|
-
});
|
|
1550
|
-
return "failed";
|
|
1551
|
-
});
|
|
1552
|
-
}
|
|
1553
|
-
saveExitedRoleRun(input) {
|
|
1554
|
-
return this.store.transaction((store) => {
|
|
1555
|
-
const task = store.getTask(input.task.id);
|
|
1556
|
-
const role = store.getRole(input.task.id, input.role.name);
|
|
1557
|
-
const currentRun = store.getActiveAgentRun(input.task.id, input.role.name);
|
|
1558
|
-
if (task === null
|
|
1559
|
-
|| task.status !== "active"
|
|
1560
|
-
|| task.executionGate.state !== "enabled"
|
|
1561
|
-
|| role === null
|
|
1562
|
-
|| currentRun === null
|
|
1563
|
-
|| currentRun.id !== input.run.id
|
|
1564
|
-
|| currentRun.status !== "active") {
|
|
1565
|
-
return "state-changed";
|
|
1566
|
-
}
|
|
1567
|
-
const sessions = store.getTaskRoleSessionSet(task.id, role.name);
|
|
1568
|
-
if (sessions?.inFlight !== null
|
|
1569
|
-
&& sessions !== null
|
|
1570
|
-
&& (sessions.inFlight.agentId !== currentRun.effective.agentId
|
|
1571
|
-
|| sessions.inFlight.runId !== currentRun.id
|
|
1572
|
-
|| sessions.inFlight.receiptId !== agentRunDeliveryReceiptId(currentRun))) {
|
|
1573
|
-
return "state-changed";
|
|
1574
|
-
}
|
|
1575
|
-
// Review launch failure uses the aggregate exact-terminalization path.
|
|
1576
|
-
// It owns mailbox settlement and ReviewRound convergence together, so a
|
|
1577
|
-
// pending pre-delivery dispatch cannot be consumed before its Run and
|
|
1578
|
-
// Round pass the same immutable identity fences.
|
|
1579
|
-
if (currentRun.purpose === "review") {
|
|
1580
|
-
if (input.run.taskId !== currentRun.taskId
|
|
1581
|
-
|| input.run.roleName !== currentRun.roleName
|
|
1582
|
-
|| input.run.purpose !== "review"
|
|
1583
|
-
|| input.run.reviewRoundId !== currentRun.reviewRoundId
|
|
1584
|
-
|| input.run.workItemId !== currentRun.workItemId
|
|
1585
|
-
|| input.run.effective.agentId !== currentRun.effective.agentId
|
|
1586
|
-
|| input.run.effective.adapterId !== currentRun.effective.adapterId) {
|
|
1587
|
-
return "state-changed";
|
|
1588
|
-
}
|
|
1589
|
-
const terminal = terminalizeExactTaskRun(store, {
|
|
1590
|
-
taskId: task.id,
|
|
1591
|
-
roleName: role.name,
|
|
1592
|
-
agentId: currentRun.effective.agentId,
|
|
1593
|
-
runId: currentRun.id,
|
|
1594
|
-
receiptId: agentRunDeliveryReceiptId(currentRun),
|
|
1595
|
-
...(input.session?.nativeSessionId === undefined
|
|
1596
|
-
? {}
|
|
1597
|
-
: { nativeSessionId: input.session.nativeSessionId }),
|
|
1598
|
-
outcome: { status: "failed", summary: input.summary }
|
|
1599
|
-
}, input.now);
|
|
1600
|
-
if (terminal.disposition !== "applied")
|
|
1601
|
-
return "state-changed";
|
|
1602
|
-
store.saveRole(task.id, updateRoleStatus(role, "exited", input.now));
|
|
1603
|
-
stopTaskSessionIfPresent(store, task.id, role.name, currentRun.effective.agentId, input.now);
|
|
1604
|
-
queueLeaderWakeup(store, task.id, wakeReason("review-failed"), input.now);
|
|
1605
|
-
return "failed";
|
|
1606
|
-
}
|
|
1607
|
-
const target = { kind: "role", taskId: task.id, roleName: role.name };
|
|
1608
|
-
const mailbox = store.getWorkMailbox(target);
|
|
1609
|
-
const processing = mailbox?.processing;
|
|
1610
|
-
if (mailbox !== null && processing !== null && processing !== undefined) {
|
|
1611
|
-
if (processing.executionRef === undefined
|
|
1612
|
-
|| processing.executionRef.type !== "run"
|
|
1613
|
-
|| processing.executionRef.taskId !== task.id
|
|
1614
|
-
|| processing.executionRef.id !== currentRun.id) {
|
|
1615
|
-
return "state-changed";
|
|
1616
|
-
}
|
|
1617
|
-
}
|
|
1618
|
-
store.saveAgentRun(failAgentRun(currentRun, input.summary, input.now));
|
|
1619
|
-
store.clearActiveAgentRun(task.id, role.name);
|
|
1620
|
-
clearTaskRoleRunInFlight(store, role, currentRun, input.now);
|
|
1621
|
-
if (mailbox !== null && processing !== null && processing !== undefined) {
|
|
1622
|
-
store.saveWorkMailbox(completeProcessing(mailbox, processing.batchId));
|
|
1623
|
-
}
|
|
1624
|
-
if (currentRun.purpose === "execution" && currentRun.workItemId !== undefined) {
|
|
1625
|
-
const workItem = store.getWorkItem(task.id, currentRun.workItemId);
|
|
1626
|
-
if (workItem !== null && !["completed", "failed", "retired"].includes(workItem.status)) {
|
|
1627
|
-
// Terminalize the bound execution lane in the same transaction that
|
|
1628
|
-
// fails the Run. Without this, a scheduler-driven Worker failure
|
|
1629
|
-
// leaves the lane "running" behind a "failed" WorkItem, and a later
|
|
1630
|
-
// `yui task run retry` is rejected because the lane is not terminal.
|
|
1631
|
-
// The lane result and the WorkItem status are two ordered single-step
|
|
1632
|
-
// record revisions, matching the aggregate terminalization path.
|
|
1633
|
-
if (currentRun.executionGroupId !== undefined
|
|
1634
|
-
&& currentRun.executionLaneId !== undefined
|
|
1635
|
-
&& currentWorkItemExecutionGroup(workItem) !== undefined) {
|
|
1636
|
-
const group = currentWorkItemExecutionGroup(workItem);
|
|
1637
|
-
store.saveWorkItem(task.id, updateWorkItemExecutionGroup(workItem, recordExecutionLaneResult(group, currentRun.executionLaneId, { summary: input.summary }, "failed", input.now), input.now));
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
1593
|
}
|
|
1641
|
-
const leaderFailed = role.name === "leader";
|
|
1642
|
-
store.saveRole(task.id, updateRoleStatus(role, leaderFailed ? "failed" : "exited", input.now));
|
|
1643
|
-
stopTaskSessionIfPresent(store, task.id, role.name, currentRun.effective.agentId, input.now);
|
|
1644
|
-
if (leaderFailed) {
|
|
1645
|
-
const failure = recordLeaderFailure(task.id, input.session?.nativeSessionId ?? "(unregistered)", input.summary, input.now, store.getLeaderFailure(task.id));
|
|
1646
|
-
store.saveLeaderFailure(failure);
|
|
1647
|
-
recordLeaderAttentionRequired(store, {
|
|
1648
|
-
taskId: task.id,
|
|
1649
|
-
reason: "leader-recovery-failed",
|
|
1650
|
-
payload: {
|
|
1651
|
-
message: failure.message,
|
|
1652
|
-
runId: currentRun.id
|
|
1653
|
-
},
|
|
1654
|
-
now: input.now
|
|
1655
|
-
});
|
|
1656
|
-
return "failed";
|
|
1657
|
-
}
|
|
1658
|
-
queueLeaderWakeup(store, task.id, wakeReason("role-run-failed"), input.now);
|
|
1659
1594
|
return "failed";
|
|
1660
1595
|
});
|
|
1661
1596
|
}
|
|
@@ -1672,47 +1607,32 @@ export class FileSchedulerStoreAdapter {
|
|
|
1672
1607
|
if (hasRuntimeCleanupObligation(existing)) {
|
|
1673
1608
|
throw new Error("Runtime cleanup is still pending.");
|
|
1674
1609
|
}
|
|
1675
|
-
const executionRef = existing.processing.executionRef;
|
|
1676
|
-
if (executionRef?.type === "run"
|
|
1677
|
-
&& (input.owner.scope !== "task"
|
|
1678
|
-
|| executionRef.taskId !== input.owner.taskId)) {
|
|
1679
|
-
throw new Error("Runtime launch reservation belongs to another Task Run.");
|
|
1680
|
-
}
|
|
1681
1610
|
return {
|
|
1682
1611
|
status: "existing",
|
|
1683
|
-
launchId: existing.processing.batchId
|
|
1684
|
-
...(executionRef?.type === "run"
|
|
1685
|
-
? { runId: executionRef.id }
|
|
1686
|
-
: {})
|
|
1612
|
+
launchId: existing.processing.batchId
|
|
1687
1613
|
};
|
|
1688
1614
|
}
|
|
1689
|
-
if (existing
|
|
1690
|
-
&& (existing.processing !== null || existing.pending !== null)) {
|
|
1615
|
+
if (hasRuntimeLifecycleWork(existing)) {
|
|
1691
1616
|
throw new RuntimeLifecycleBusyError("Runtime lifecycle work is already pending.");
|
|
1692
1617
|
}
|
|
1693
1618
|
enqueueWork(store, target, RUNTIME_LAUNCH_RESERVED_REASON, now, input.owner.scope === "task"
|
|
1694
1619
|
? [{ type: "task", id: input.owner.taskId }]
|
|
1695
1620
|
: []);
|
|
1696
1621
|
const queued = store.getWorkMailbox(target);
|
|
1697
|
-
if (queued === null
|
|
1622
|
+
if (queued === null
|
|
1623
|
+
|| pendingLane(queued, "normal") === null
|
|
1624
|
+
|| queued.processing !== null) {
|
|
1698
1625
|
throw new Error("Runtime launch reservation could not be queued.");
|
|
1699
1626
|
}
|
|
1700
|
-
|
|
1627
|
+
const claimed = claimPending(queued, {
|
|
1701
1628
|
batchId: input.launchId,
|
|
1702
1629
|
owner: RUNTIME_LIFECYCLE_OWNER,
|
|
1703
1630
|
startedAt: now.toISOString()
|
|
1704
1631
|
});
|
|
1705
|
-
if (input.runId !== undefined) {
|
|
1706
|
-
if (input.owner.scope !== "task") {
|
|
1707
|
-
throw new Error("A Run-bound runtime reservation requires a Task owner.");
|
|
1708
|
-
}
|
|
1709
|
-
claimed = bindExecution(claimed, input.launchId, { type: "run", taskId: input.owner.taskId, id: input.runId });
|
|
1710
|
-
}
|
|
1711
1632
|
store.saveWorkMailbox(claimed);
|
|
1712
1633
|
return {
|
|
1713
1634
|
status: "reserved",
|
|
1714
|
-
launchId: input.launchId
|
|
1715
|
-
...(input.runId === undefined ? {} : { runId: input.runId })
|
|
1635
|
+
launchId: input.launchId
|
|
1716
1636
|
};
|
|
1717
1637
|
});
|
|
1718
1638
|
}
|
|
@@ -1732,7 +1652,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1732
1652
|
// the already-settled reservation as a launch failure.
|
|
1733
1653
|
const sessions = runtimeOwnerSessionSet(store, input.owner);
|
|
1734
1654
|
const active = sessions?.sessions[sessions.activeAgentId];
|
|
1735
|
-
if (active?.status === "
|
|
1655
|
+
if (active?.status === "active"
|
|
1736
1656
|
&& active.launchId === input.launchId)
|
|
1737
1657
|
return "provider-bound";
|
|
1738
1658
|
throw new Error("Runtime launch reservation no longer matches the launch.");
|
|
@@ -1767,25 +1687,12 @@ export class FileSchedulerStoreAdapter {
|
|
|
1767
1687
|
return session;
|
|
1768
1688
|
});
|
|
1769
1689
|
}
|
|
1770
|
-
completeRuntimeLaunchReservation(owner, launchId,
|
|
1690
|
+
completeRuntimeLaunchReservation(owner, launchId, beforeComplete) {
|
|
1771
1691
|
return this.store.transaction((store) => {
|
|
1772
1692
|
const target = runtimeLifecycleTarget(owner);
|
|
1773
1693
|
const mailbox = store.getWorkMailbox(target);
|
|
1774
1694
|
if (!isRuntimeLaunchReservation(mailbox?.processing, launchId))
|
|
1775
1695
|
return false;
|
|
1776
|
-
if (expectedTerminalRunId !== undefined) {
|
|
1777
|
-
const expectedRun = owner.scope === "task"
|
|
1778
|
-
? store.getAgentRun(owner.taskId, expectedTerminalRunId)
|
|
1779
|
-
: null;
|
|
1780
|
-
if (owner.scope !== "task"
|
|
1781
|
-
|| mailbox?.processing?.executionRef?.type !== "run"
|
|
1782
|
-
|| mailbox.processing.executionRef.taskId !== owner.taskId
|
|
1783
|
-
|| mailbox.processing.executionRef.id !== expectedTerminalRunId
|
|
1784
|
-
|| expectedRun === null
|
|
1785
|
-
|| expectedRun.status === "active") {
|
|
1786
|
-
return false;
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
1696
|
beforeComplete?.();
|
|
1790
1697
|
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, launchId));
|
|
1791
1698
|
return true;
|
|
@@ -1807,7 +1714,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
1807
1714
|
const sessions = runtimeOwnerSessionSet(store, input.owner);
|
|
1808
1715
|
const active = sessions?.sessions[sessions.activeAgentId];
|
|
1809
1716
|
if (runtimeSessionMatchesSettledLaunch(active, input)) {
|
|
1810
|
-
|
|
1717
|
+
endRuntimeOwnerSession(store, input.owner, now);
|
|
1811
1718
|
}
|
|
1812
1719
|
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, input.launchId));
|
|
1813
1720
|
return true;
|
|
@@ -1824,8 +1731,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
1824
1731
|
const active = sessions.sessions[sessions.activeAgentId];
|
|
1825
1732
|
if (!runtimeSessionMatchesSettledLaunch(active, input))
|
|
1826
1733
|
return false;
|
|
1827
|
-
if (active.status !== "
|
|
1828
|
-
|
|
1734
|
+
if (active.status !== "ended") {
|
|
1735
|
+
endRuntimeOwnerSession(store, input.owner, now);
|
|
1829
1736
|
}
|
|
1830
1737
|
return true;
|
|
1831
1738
|
});
|
|
@@ -1860,6 +1767,19 @@ export class FileSchedulerStoreAdapter {
|
|
|
1860
1767
|
catch {
|
|
1861
1768
|
return "obsolete";
|
|
1862
1769
|
}
|
|
1770
|
+
const providerTurn = sessions?.providerBinding?.turn;
|
|
1771
|
+
const canonicalTurnId = sessions === null || sessions === undefined
|
|
1772
|
+
? input.turnId
|
|
1773
|
+
: canonicalStructuredProviderTurnId(sessions, input.turnId, input.attemptId);
|
|
1774
|
+
const recordedProviderTurn = input.runId !== undefined
|
|
1775
|
+
&& providerTurn?.runId === input.runId
|
|
1776
|
+
&& providerTurn.turnId === canonicalTurnId;
|
|
1777
|
+
if (recordedProviderTurn) {
|
|
1778
|
+
const run = this.store.getAgentRun(input.taskId, input.runId);
|
|
1779
|
+
return run === null
|
|
1780
|
+
? "obsolete"
|
|
1781
|
+
: run.pushedAt === undefined ? "deferred" : "apply";
|
|
1782
|
+
}
|
|
1863
1783
|
// A normal explicit CLI yield may precede its native Hook; that Hook still
|
|
1864
1784
|
// owns the turn fact. A forced cleanup boundary or stopped process instead
|
|
1865
1785
|
// makes the old generation obsolete.
|
|
@@ -1893,6 +1813,13 @@ export class FileSchedulerStoreAdapter {
|
|
|
1893
1813
|
let sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName)
|
|
1894
1814
|
?? createRoleSessionSet({ scope: "task", taskId: input.taskId, roleName: input.roleName }, input.agentId, now);
|
|
1895
1815
|
const canonicalTurnId = canonicalStructuredProviderTurnId(sessions, input.turnId, input.attemptId);
|
|
1816
|
+
const providerTurn = sessions.providerBinding?.turn;
|
|
1817
|
+
const recordedProviderTurn = input.runId !== undefined
|
|
1818
|
+
&& providerTurn?.runId === input.runId
|
|
1819
|
+
&& providerTurn.turnId === canonicalTurnId;
|
|
1820
|
+
const observedRun = input.runId === undefined
|
|
1821
|
+
? null
|
|
1822
|
+
: store.getAgentRun(input.taskId, input.runId);
|
|
1896
1823
|
const existing = sessions.sessions[input.agentId];
|
|
1897
1824
|
const owner = {
|
|
1898
1825
|
scope: "task",
|
|
@@ -1912,7 +1839,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
1912
1839
|
&& hasRecentTurnId(effectiveExisting.recentCompletedTurnIds, canonicalTurnId)) {
|
|
1913
1840
|
return { session: effectiveExisting, duplicate: true };
|
|
1914
1841
|
}
|
|
1915
|
-
if (
|
|
1842
|
+
if (!recordedProviderTurn
|
|
1843
|
+
&& sessions.inFlight === null
|
|
1916
1844
|
&& existing !== undefined
|
|
1917
1845
|
&& (hasRuntimeCleanupObligation(store.getWorkMailbox(runtimeLifecycleTarget(owner)))
|
|
1918
1846
|
|| isObsoleteTerminalRuntimeRun(store, input))) {
|
|
@@ -1920,7 +1848,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
1920
1848
|
}
|
|
1921
1849
|
// Classification is only an optimization. Revalidate the Run inside the
|
|
1922
1850
|
// authoritative transaction before a Hook may claim a native identity.
|
|
1923
|
-
if (
|
|
1851
|
+
if (!recordedProviderTurn
|
|
1852
|
+
&& sessions.inFlight !== null
|
|
1924
1853
|
&& (input.runId === undefined
|
|
1925
1854
|
|| sessions.inFlight.runId !== input.runId)) {
|
|
1926
1855
|
if (effectiveExisting === undefined) {
|
|
@@ -1928,22 +1857,22 @@ export class FileSchedulerStoreAdapter {
|
|
|
1928
1857
|
}
|
|
1929
1858
|
return { session: effectiveExisting, duplicate: false };
|
|
1930
1859
|
}
|
|
1931
|
-
if (
|
|
1860
|
+
if (recordedProviderTurn
|
|
1861
|
+
? observedRun?.pushedAt === undefined
|
|
1862
|
+
: sessions.inFlight !== null && sessions.inFlight.pushedAt === undefined) {
|
|
1932
1863
|
throw new Error("Runtime turn completion requires an independently committed transport receipt.");
|
|
1933
1864
|
}
|
|
1934
|
-
const
|
|
1935
|
-
|| effectiveExisting?.status === "broken"
|
|
1936
|
-
? effectiveExisting.status
|
|
1937
|
-
: "ready";
|
|
1865
|
+
const sessionStatus = effectiveExisting?.status ?? "active";
|
|
1938
1866
|
sessions = recordRoleAgentSession(sessions, {
|
|
1939
1867
|
agentId: input.agentId,
|
|
1940
1868
|
adapterId: input.adapterId,
|
|
1941
1869
|
nativeSessionId: input.nativeSessionId,
|
|
1942
1870
|
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
1943
1871
|
policy: "fixed",
|
|
1944
|
-
status:
|
|
1945
|
-
|
|
1946
|
-
|
|
1872
|
+
status: sessionStatus,
|
|
1873
|
+
...(effectiveExisting?.endReason === undefined
|
|
1874
|
+
? {}
|
|
1875
|
+
: { endReason: effectiveExisting.endReason }),
|
|
1947
1876
|
effective
|
|
1948
1877
|
}, now);
|
|
1949
1878
|
sessions = settleStructuredProviderTurn(sessions, canonicalTurnId, input.providerStatus ?? "completed", now);
|
|
@@ -1951,18 +1880,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1951
1880
|
sessions = rememberRoleAgentCompletedTurn(sessions, input.agentId, input.nativeSessionId, canonicalTurnId, now);
|
|
1952
1881
|
store.saveTaskRoleSessionSet(sessions);
|
|
1953
1882
|
completeRuntimeHookReservation(store, owner, input.launchId);
|
|
1954
|
-
// The in-flight fence was cleared by an in-place provider retry
|
|
1955
|
-
// decision. If the active Run is still in its retry window and this
|
|
1956
|
-
// completion belongs to it, the recovered CLI finished the original
|
|
1957
|
-
// turn: clear the retry projection so the deadline does not re-push
|
|
1958
|
-
// the prompt. The Run stays active for the normal completion path.
|
|
1959
|
-
const activeRun = store.getActiveAgentRun(task.id, role.name);
|
|
1960
|
-
if (activeRun !== null
|
|
1961
|
-
&& activeRun.providerRetry !== undefined
|
|
1962
|
-
&& input.runId === activeRun.id) {
|
|
1963
|
-
const { providerRetry: _removed, ...rest } = activeRun;
|
|
1964
|
-
store.saveAgentRun({ ...rest, updatedAt: now.toISOString() });
|
|
1965
|
-
}
|
|
1966
1883
|
return { session: sessions.sessions[input.agentId], duplicate: false };
|
|
1967
1884
|
}
|
|
1968
1885
|
// A provider Turn is an activation boundary, not a Yui Run outcome.
|
|
@@ -1982,366 +1899,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
1982
1899
|
};
|
|
1983
1900
|
});
|
|
1984
1901
|
}
|
|
1985
|
-
classifyRuntimeTurnFailed(input) {
|
|
1986
|
-
const task = this.store.getTask(input.taskId);
|
|
1987
|
-
if (task?.status !== "active" || task.executionGate.state !== "enabled")
|
|
1988
|
-
return "obsolete";
|
|
1989
|
-
const role = this.store.getRole(input.taskId, input.roleName);
|
|
1990
|
-
if (role === null)
|
|
1991
|
-
return "obsolete";
|
|
1992
|
-
const run = this.store.getAgentRun(input.taskId, input.runId);
|
|
1993
|
-
const active = this.store.getActiveAgentRun(input.taskId, input.roleName);
|
|
1994
|
-
if (run === null
|
|
1995
|
-
|| run.status !== "active"
|
|
1996
|
-
|| active?.id !== run.id
|
|
1997
|
-
|| run.roleName !== input.roleName
|
|
1998
|
-
|| run.effective.agentId !== input.agentId
|
|
1999
|
-
|| run.effective.adapterId !== input.adapterId)
|
|
2000
|
-
return "obsolete";
|
|
2001
|
-
const sessions = this.store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
2002
|
-
const session = sessions?.sessions[input.agentId];
|
|
2003
|
-
if (sessions === null
|
|
2004
|
-
|| session?.nativeSessionId !== input.nativeSessionId
|
|
2005
|
-
|| session.launchId !== input.launchId
|
|
2006
|
-
|| sessions.inFlight?.agentId !== input.agentId
|
|
2007
|
-
|| sessions.inFlight.runId !== input.runId
|
|
2008
|
-
|| sessions.inFlight.receiptId !== agentRunDeliveryReceiptId(run))
|
|
2009
|
-
return "obsolete";
|
|
2010
|
-
return "apply";
|
|
2011
|
-
}
|
|
2012
|
-
observeRuntimeTurnFailed(input, now = new Date()) {
|
|
2013
|
-
return this.store.transaction((store) => {
|
|
2014
|
-
const before = store.getAgentRun(input.taskId, input.runId);
|
|
2015
|
-
if (before === null) {
|
|
2016
|
-
recordObsoleteRuntimeEvent(store, input, "run-missing", now);
|
|
2017
|
-
return { disposition: "obsolete", runId: input.runId };
|
|
2018
|
-
}
|
|
2019
|
-
const summary = runtimeTurnFailureSummary(input);
|
|
2020
|
-
// Issue 04: a classified transient Provider failure keeps the exact Run
|
|
2021
|
-
// and Native Session and is retried in place instead of terminalizing.
|
|
2022
|
-
const retryDecision = this.providerRetryDecision(store, input, summary, now);
|
|
2023
|
-
if (retryDecision !== null)
|
|
2024
|
-
return retryDecision;
|
|
2025
|
-
const result = terminalizeExactTaskRun(store, {
|
|
2026
|
-
taskId: input.taskId,
|
|
2027
|
-
roleName: input.roleName,
|
|
2028
|
-
agentId: input.agentId,
|
|
2029
|
-
runId: input.runId,
|
|
2030
|
-
receiptId: agentRunDeliveryReceiptId(before),
|
|
2031
|
-
nativeSessionId: input.nativeSessionId,
|
|
2032
|
-
launchId: input.launchId,
|
|
2033
|
-
outcome: {
|
|
2034
|
-
status: "failed",
|
|
2035
|
-
summary
|
|
2036
|
-
}
|
|
2037
|
-
}, now);
|
|
2038
|
-
if (result.disposition === "obsolete" || result.run === null) {
|
|
2039
|
-
recordObsoleteRuntimeEvent(store, input, result.reason ?? "identity-mismatch-or-terminal", now);
|
|
2040
|
-
return { disposition: "obsolete", runId: input.runId };
|
|
2041
|
-
}
|
|
2042
|
-
enqueueWork(store, runtimeLifecycleTarget({
|
|
2043
|
-
scope: "task",
|
|
2044
|
-
taskId: input.taskId,
|
|
2045
|
-
roleName: input.roleName
|
|
2046
|
-
}), RUNTIME_CLEANUP_REQUIRED_REASON, now, [{ type: "task", id: input.taskId }]);
|
|
2047
|
-
const terminal = result.run;
|
|
2048
|
-
const role = store.getRole(input.taskId, input.roleName);
|
|
2049
|
-
if (role !== null) {
|
|
2050
|
-
store.saveRole(input.taskId, updateRoleStatus(role, input.roleName === "leader" ? "failed" : "idle", now));
|
|
2051
|
-
}
|
|
2052
|
-
const failureEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.turn-failed", {
|
|
2053
|
-
eventId: input.eventId,
|
|
2054
|
-
runId: input.runId,
|
|
2055
|
-
roleName: input.roleName,
|
|
2056
|
-
outcome: terminal.status
|
|
2057
|
-
}, now);
|
|
2058
|
-
store.saveEvent(input.taskId, failureEvent);
|
|
2059
|
-
routeRoleEvent(store, failureEvent, input.roleName, input.roleName !== "leader" && before.purpose === "review"
|
|
2060
|
-
? "review-failed"
|
|
2061
|
-
: input.roleName === "leader" ? "leader-run-failed" : "role-run-failed", now);
|
|
2062
|
-
return { disposition: "applied", runId: input.runId };
|
|
2063
|
-
});
|
|
2064
|
-
}
|
|
2065
|
-
/**
|
|
2066
|
-
* Issue 04 in-place retry decision for one classified Provider failure.
|
|
2067
|
-
* Returns an observation when the failure was handled without terminalizing
|
|
2068
|
-
* the Run, or `null` to fall through to the existing terminalization path.
|
|
2069
|
-
*/
|
|
2070
|
-
providerRetryDecision(store, input, summary, now) {
|
|
2071
|
-
const config = providerRetryConfig(this.store.getConfig());
|
|
2072
|
-
if (config.mode === "off")
|
|
2073
|
-
return null;
|
|
2074
|
-
const classification = classifyProviderError({
|
|
2075
|
-
adapterId: input.adapterId,
|
|
2076
|
-
...(input.errorCode === undefined ? {} : { errorCode: input.errorCode }),
|
|
2077
|
-
error: input.error,
|
|
2078
|
-
errorDetails: input.errorDetails,
|
|
2079
|
-
summary: input.lastAssistantMessage
|
|
2080
|
-
});
|
|
2081
|
-
const adapterEnabled = providerRetryAdapterEnabled(config, input.adapterId);
|
|
2082
|
-
const retryable = adapterEnabled
|
|
2083
|
-
&& isRetryableProviderErrorClass(classification.errorClass);
|
|
2084
|
-
const shadow = config.mode === "shadow";
|
|
2085
|
-
// Shadow mode records the classification and "would retry" fact without
|
|
2086
|
-
// changing any behavior; the Run still terminalizes below.
|
|
2087
|
-
if (shadow || !retryable) {
|
|
2088
|
-
if (adapterEnabled) {
|
|
2089
|
-
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2090
|
-
wouldRetry: String(retryable),
|
|
2091
|
-
basis: classification.basis,
|
|
2092
|
-
shadow: String(shadow)
|
|
2093
|
-
}, now);
|
|
2094
|
-
}
|
|
2095
|
-
if (!shadow && classification.errorClass === "policy-denied") {
|
|
2096
|
-
const blocked = this.applyProviderPolicyBlock(store, input, summary, now);
|
|
2097
|
-
if (blocked !== null)
|
|
2098
|
-
return blocked;
|
|
2099
|
-
}
|
|
2100
|
-
if (!shadow && classification.errorClass === "context-capacity") {
|
|
2101
|
-
const blocked = this.applyContextCapacityBlock(store, input, summary, now);
|
|
2102
|
-
if (blocked !== null)
|
|
2103
|
-
return blocked;
|
|
2104
|
-
}
|
|
2105
|
-
if (!shadow && classification.errorClass === "session-dead") {
|
|
2106
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2107
|
-
if (run !== null && run.status === "active" && run.mode === "resume") {
|
|
2108
|
-
// A structured failure on the exact resume launch is the only Core
|
|
2109
|
-
// evidence that native continuation is unrecoverable. The normal
|
|
2110
|
-
// terminal path below may now request a fresh managed generation.
|
|
2111
|
-
store.saveEvent(input.taskId, createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.native-resume-unrecoverable", {
|
|
2112
|
-
eventId: input.eventId,
|
|
2113
|
-
runId: input.runId,
|
|
2114
|
-
roleName: input.roleName,
|
|
2115
|
-
launchId: input.launchId,
|
|
2116
|
-
nativeSessionId: input.nativeSessionId,
|
|
2117
|
-
...(run.assignment.contextSnapshotRef === undefined
|
|
2118
|
-
? {}
|
|
2119
|
-
: {
|
|
2120
|
-
contextSnapshotId: run.assignment.contextSnapshotRef.id,
|
|
2121
|
-
contextSnapshotDigest: run.assignment.contextSnapshotRef.digest
|
|
2122
|
-
})
|
|
2123
|
-
}, now));
|
|
2124
|
-
}
|
|
2125
|
-
else {
|
|
2126
|
-
const blocked = this.applyNativeSessionRecoverabilityBlock(store, input, summary, now);
|
|
2127
|
-
if (blocked !== null)
|
|
2128
|
-
return blocked;
|
|
2129
|
-
}
|
|
2130
|
-
}
|
|
2131
|
-
return null;
|
|
2132
|
-
}
|
|
2133
|
-
// transport-uncertain: the Provider may have accepted the turn. If a
|
|
2134
|
-
// native completion is already durable, the completion path owns the
|
|
2135
|
-
// yield; do not resend.
|
|
2136
|
-
if (classification.errorClass === "transport-uncertain") {
|
|
2137
|
-
const driver = this.drivers.requireByAdapterId(input.adapterId);
|
|
2138
|
-
if (driver.capabilities.lifecycle.deliveryDeduplication !== "exact") {
|
|
2139
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2140
|
-
if (run === null || run.status !== "active")
|
|
2141
|
-
return null;
|
|
2142
|
-
const blocked = scheduleProviderRetry(run.providerRetry, {
|
|
2143
|
-
failureEventId: input.eventId,
|
|
2144
|
-
errorClass: classification.errorClass,
|
|
2145
|
-
launchId: input.launchId,
|
|
2146
|
-
nativeSessionId: input.nativeSessionId,
|
|
2147
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2148
|
-
lastErrorSummary: summary,
|
|
2149
|
-
scheduleNextAttempt: false
|
|
2150
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2151
|
-
if (blocked.outcome === "exhausted")
|
|
2152
|
-
return null;
|
|
2153
|
-
store.saveAgentRun(withProviderRetry(run, blocked.retry));
|
|
2154
|
-
const classifiedEvent = this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2155
|
-
wouldRetry: "false",
|
|
2156
|
-
shadow: "false",
|
|
2157
|
-
note: "delivery-deduplication-unproven"
|
|
2158
|
-
}, now);
|
|
2159
|
-
routeRoleEvent(store, classifiedEvent, input.roleName, "provider-delivery-uncertain", now);
|
|
2160
|
-
return { disposition: "applied", runId: input.runId };
|
|
2161
|
-
}
|
|
2162
|
-
}
|
|
2163
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2164
|
-
if (run === null || run.status !== "active")
|
|
2165
|
-
return null;
|
|
2166
|
-
const retryDecision = scheduleProviderRetry(run.providerRetry, {
|
|
2167
|
-
failureEventId: input.eventId,
|
|
2168
|
-
errorClass: classification.errorClass,
|
|
2169
|
-
launchId: input.launchId,
|
|
2170
|
-
nativeSessionId: input.nativeSessionId,
|
|
2171
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2172
|
-
lastErrorSummary: summary,
|
|
2173
|
-
...(input.retryAfterMs === undefined ? {} : { retryAfterMs: input.retryAfterMs })
|
|
2174
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2175
|
-
if (retryDecision.outcome === "exhausted") {
|
|
2176
|
-
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2177
|
-
wouldRetry: "false",
|
|
2178
|
-
shadow: "false",
|
|
2179
|
-
note: `episode-exhausted-${retryDecision.reason}`,
|
|
2180
|
-
consecutiveFailures: String(run.providerRetry?.consecutiveFailures ?? 0),
|
|
2181
|
-
dispatchedRetries: String(run.providerRetry?.dispatchedRetries ?? 0)
|
|
2182
|
-
}, now);
|
|
2183
|
-
if (run.providerRetry === undefined)
|
|
2184
|
-
return null;
|
|
2185
|
-
const finalized = finalizeProviderRetryDeadline(store, run, retryDecision.reason === "attempts"
|
|
2186
|
-
? `Provider retry failed after all ${run.providerRetry.maxRetries} in-Session continuation attempts.`
|
|
2187
|
-
: retryDecision.reason === "retry-after-window"
|
|
2188
|
-
? `Provider Retry-After exceeded the bounded ${config.maxWindowMs / 1_000}-second episode window.`
|
|
2189
|
-
: `Provider retry did not recover within the bounded ${config.maxWindowMs / 1_000}-second episode window.`, retryDecision.reason === "attempts" ? "attempts-exhausted" : "episode-window-exhausted", now);
|
|
2190
|
-
return finalized ? { disposition: "applied", runId: input.runId } : null;
|
|
2191
|
-
}
|
|
2192
|
-
const retry = retryDecision.retry;
|
|
2193
|
-
store.saveAgentRun(withProviderRetry(run, retry));
|
|
2194
|
-
// The in-flight fence and Provider Conversation stay bound while the
|
|
2195
|
-
// retry waits. resolveDueProviderRetries advances only the delivery
|
|
2196
|
-
// receipt immediately before the exact redispatch.
|
|
2197
|
-
// Settle the delivery claim so the retry push can re-claim the mailbox.
|
|
2198
|
-
const target = runtimeLifecycleTarget({
|
|
2199
|
-
scope: "task",
|
|
2200
|
-
taskId: input.taskId,
|
|
2201
|
-
roleName: input.roleName
|
|
2202
|
-
});
|
|
2203
|
-
const mailbox = store.getWorkMailbox(target);
|
|
2204
|
-
if (mailbox !== null && mailbox.processing !== null) {
|
|
2205
|
-
const settled = completeProcessing(mailbox, mailbox.processing.batchId);
|
|
2206
|
-
if (settled.processing === null && settled.pending === null) {
|
|
2207
|
-
store.removeWorkMailbox(target);
|
|
2208
|
-
}
|
|
2209
|
-
else {
|
|
2210
|
-
store.saveWorkMailbox(settled);
|
|
2211
|
-
}
|
|
2212
|
-
}
|
|
2213
|
-
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2214
|
-
wouldRetry: "true",
|
|
2215
|
-
shadow: "false",
|
|
2216
|
-
consecutiveFailures: String(retry.consecutiveFailures),
|
|
2217
|
-
dispatchedRetries: String(retry.dispatchedRetries),
|
|
2218
|
-
nextAttemptAt: retry.nextAttemptAt
|
|
2219
|
-
}, now);
|
|
2220
|
-
return { disposition: "applied", runId: input.runId };
|
|
2221
|
-
}
|
|
2222
|
-
/**
|
|
2223
|
-
* Issue 04 policy-denied handling: the Run stays active on its original
|
|
2224
|
-
* Session, no retry is scheduled, and the Leader receives a bounded blocker.
|
|
2225
|
-
* Yui never works around a policy denial by switching Session or widening
|
|
2226
|
-
* permission.
|
|
2227
|
-
*/
|
|
2228
|
-
applyProviderPolicyBlock(store, input, summary, now) {
|
|
2229
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2230
|
-
if (run === null || run.status !== "active")
|
|
2231
|
-
return null;
|
|
2232
|
-
const retryDecision = scheduleProviderRetry(run.providerRetry, {
|
|
2233
|
-
failureEventId: input.eventId,
|
|
2234
|
-
errorClass: "policy-denied",
|
|
2235
|
-
launchId: input.launchId,
|
|
2236
|
-
nativeSessionId: input.nativeSessionId,
|
|
2237
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2238
|
-
lastErrorSummary: summary,
|
|
2239
|
-
scheduleNextAttempt: false
|
|
2240
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2241
|
-
if (retryDecision.outcome === "exhausted")
|
|
2242
|
-
return null;
|
|
2243
|
-
store.saveAgentRun(withProviderRetry(run, retryDecision.retry));
|
|
2244
|
-
const classifiedEvent = this.recordProviderRetryClassified(store, input, "policy-denied", {
|
|
2245
|
-
wouldRetry: "false",
|
|
2246
|
-
shadow: "false",
|
|
2247
|
-
note: "policy-denied-blocker"
|
|
2248
|
-
}, now);
|
|
2249
|
-
routeRoleEvent(store, classifiedEvent, input.roleName, input.roleName === "leader"
|
|
2250
|
-
? "leader-provider-policy-blocked"
|
|
2251
|
-
: "provider-policy-blocked", now);
|
|
2252
|
-
return { disposition: "applied", runId: input.runId };
|
|
2253
|
-
}
|
|
2254
|
-
/**
|
|
2255
|
-
* Exact Provider capacity failures never trigger a token-threshold rollover.
|
|
2256
|
-
* Keep the native lineage intact and surface the Driver's explicit native
|
|
2257
|
-
* compaction/resume capability; unsupported/unknown capabilities fail safe
|
|
2258
|
-
* for Leader/Operator action instead of forcing a fresh generation.
|
|
2259
|
-
*/
|
|
2260
|
-
applyContextCapacityBlock(store, input, summary, now) {
|
|
2261
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2262
|
-
if (run === null || run.status !== "active")
|
|
2263
|
-
return null;
|
|
2264
|
-
const decision = scheduleProviderRetry(run.providerRetry, {
|
|
2265
|
-
failureEventId: input.eventId,
|
|
2266
|
-
errorClass: "context-capacity",
|
|
2267
|
-
launchId: input.launchId,
|
|
2268
|
-
nativeSessionId: input.nativeSessionId,
|
|
2269
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2270
|
-
lastErrorSummary: summary,
|
|
2271
|
-
scheduleNextAttempt: false
|
|
2272
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2273
|
-
if (decision.outcome === "exhausted")
|
|
2274
|
-
return null;
|
|
2275
|
-
store.saveAgentRun(withProviderRetry(run, decision.retry));
|
|
2276
|
-
const driver = this.drivers.requireByAdapterId(input.adapterId);
|
|
2277
|
-
const capacityEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.context-capacity-failure", {
|
|
2278
|
-
eventId: input.eventId,
|
|
2279
|
-
runId: input.runId,
|
|
2280
|
-
roleName: input.roleName,
|
|
2281
|
-
nativeSessionId: input.nativeSessionId,
|
|
2282
|
-
launchId: input.launchId,
|
|
2283
|
-
compactionCapability: driver.capabilities.lifecycle.compaction,
|
|
2284
|
-
nativeResumeCapability: driver.capabilities.lifecycle.nativeConversationResume,
|
|
2285
|
-
action: driver.capabilities.lifecycle.compaction === "native-explicit"
|
|
2286
|
-
? "native-explicit-compaction-required"
|
|
2287
|
-
: "await-provider-native-recovery"
|
|
2288
|
-
}, now);
|
|
2289
|
-
store.saveEvent(input.taskId, capacityEvent);
|
|
2290
|
-
this.recordProviderRetryClassified(store, input, "context-capacity", {
|
|
2291
|
-
wouldRetry: "false",
|
|
2292
|
-
shadow: "false",
|
|
2293
|
-
note: "provider-native-capacity-recovery-only"
|
|
2294
|
-
}, now);
|
|
2295
|
-
routeRoleEvent(store, capacityEvent, input.roleName, "provider-context-capacity-recovery-required", now);
|
|
2296
|
-
return { disposition: "applied", runId: input.runId };
|
|
2297
|
-
}
|
|
2298
|
-
applyNativeSessionRecoverabilityBlock(store, input, summary, now) {
|
|
2299
|
-
const run = store.getAgentRun(input.taskId, input.runId);
|
|
2300
|
-
if (run === null || run.status !== "active")
|
|
2301
|
-
return null;
|
|
2302
|
-
const decision = scheduleProviderRetry(run.providerRetry, {
|
|
2303
|
-
failureEventId: input.eventId,
|
|
2304
|
-
errorClass: "session-dead",
|
|
2305
|
-
launchId: input.launchId,
|
|
2306
|
-
nativeSessionId: input.nativeSessionId,
|
|
2307
|
-
failedNativeTurnId: input.nativeTurnId,
|
|
2308
|
-
lastErrorSummary: summary,
|
|
2309
|
-
scheduleNextAttempt: false
|
|
2310
|
-
}, now, configuredProviderRetryPolicy(store));
|
|
2311
|
-
if (decision.outcome === "exhausted")
|
|
2312
|
-
return null;
|
|
2313
|
-
store.saveAgentRun(withProviderRetry(run, decision.retry));
|
|
2314
|
-
const recoverabilityEvent = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.native-session-recoverability-unproven", {
|
|
2315
|
-
eventId: input.eventId,
|
|
2316
|
-
runId: input.runId,
|
|
2317
|
-
roleName: input.roleName,
|
|
2318
|
-
launchId: input.launchId,
|
|
2319
|
-
nativeSessionId: input.nativeSessionId,
|
|
2320
|
-
requiredEvidence: "exact-native-resume-outcome"
|
|
2321
|
-
}, now);
|
|
2322
|
-
store.saveEvent(input.taskId, recoverabilityEvent);
|
|
2323
|
-
routeRoleEvent(store, recoverabilityEvent, input.roleName, "native-session-recoverability-unproven", now);
|
|
2324
|
-
return { disposition: "applied", runId: input.runId };
|
|
2325
|
-
}
|
|
2326
|
-
recordProviderRetryClassified(store, input, errorClass, extra, now) {
|
|
2327
|
-
const event = createTaskEvent(store.nextEventId(input.taskId), input.taskId, "runtime.provider-retry-classified", {
|
|
2328
|
-
eventId: input.eventId,
|
|
2329
|
-
runId: input.runId,
|
|
2330
|
-
roleName: input.roleName,
|
|
2331
|
-
errorClass,
|
|
2332
|
-
...extra
|
|
2333
|
-
}, now);
|
|
2334
|
-
store.saveEvent(input.taskId, event);
|
|
2335
|
-
return event;
|
|
2336
|
-
}
|
|
2337
|
-
/**
|
|
2338
|
-
* Issue 04 durable retry timer: lists active Runs whose in-place retry is
|
|
2339
|
-
* due. The Controller arms its deadline timer from this projection, so a
|
|
2340
|
-
* Controller restart resumes the same attempt lineage.
|
|
2341
|
-
*/
|
|
2342
|
-
listPendingProviderRetries(taskIds) {
|
|
2343
|
-
return this.store.listPendingProviderRetries(taskIds);
|
|
2344
|
-
}
|
|
2345
1902
|
saveRoleHostExitObservation(input) {
|
|
2346
1903
|
this.store.transaction((store) => {
|
|
2347
1904
|
const identity = [
|
|
@@ -2396,166 +1953,12 @@ export class FileSchedulerStoreAdapter {
|
|
|
2396
1953
|
* blocker; a live Session is reopened for the existing delivery path, which
|
|
2397
1954
|
* re-pushes the exact same input in the same pass.
|
|
2398
1955
|
*/
|
|
2399
|
-
resolveDueProviderRetries(now, taskIds) {
|
|
2400
|
-
const selectedTaskIds = taskIds === undefined
|
|
2401
|
-
? undefined
|
|
2402
|
-
: [...taskIds].sort((left, right) => (left.localeCompare(right, undefined, { numeric: true })));
|
|
2403
|
-
const due = this.listPendingProviderRetries(selectedTaskIds).filter((entry) => Date.parse(entry.dueAt) <= now.getTime());
|
|
2404
|
-
const reopened = [];
|
|
2405
|
-
for (const entry of due) {
|
|
2406
|
-
this.store.transaction((store) => {
|
|
2407
|
-
const run = store.getAgentRun(entry.taskId, entry.runId);
|
|
2408
|
-
if (run === null
|
|
2409
|
-
|| run.status !== "active"
|
|
2410
|
-
|| run.providerRetry === undefined) {
|
|
2411
|
-
return;
|
|
2412
|
-
}
|
|
2413
|
-
if (run.providerRetry.state === "dispatching"
|
|
2414
|
-
|| run.providerRetry.state === "awaiting-progress") {
|
|
2415
|
-
if (now.getTime() < Date.parse(run.providerRetry.episodeDeadlineAt))
|
|
2416
|
-
return;
|
|
2417
|
-
finalizeProviderRetryDeadline(store, run, run.providerRetry.state === "dispatching"
|
|
2418
|
-
? "Provider retry delivery outcome remained uncertain through the bounded episode deadline; no correlated Provider progress was observed."
|
|
2419
|
-
: "Provider retry produced no correlated Provider input/output progress before the bounded episode deadline.", run.providerRetry.state === "dispatching"
|
|
2420
|
-
? "retry-delivery-outcome-uncertain"
|
|
2421
|
-
: "provider-progress-timeout", now);
|
|
2422
|
-
return;
|
|
2423
|
-
}
|
|
2424
|
-
if (run.providerRetry.state !== "scheduled"
|
|
2425
|
-
|| run.providerRetry.nextAttemptAt === undefined
|
|
2426
|
-
|| !providerRetryIsDue(run.providerRetry, now)) {
|
|
2427
|
-
return;
|
|
2428
|
-
}
|
|
2429
|
-
const sessions = store.getTaskRoleSessionSet(entry.taskId, entry.roleName);
|
|
2430
|
-
const providerBinding = sessions?.providerBinding;
|
|
2431
|
-
if (providerBinding === null || providerBinding === undefined) {
|
|
2432
|
-
deferUnprovenProviderRecovery(store, run, entry.roleName, "Provider Conversation identity is missing; automatic recovery is fenced.", now);
|
|
2433
|
-
return;
|
|
2434
|
-
}
|
|
2435
|
-
const conversation = currentProviderConversation(providerBinding);
|
|
2436
|
-
const roleMailbox = store.getWorkMailbox({
|
|
2437
|
-
kind: "role",
|
|
2438
|
-
taskId: entry.taskId,
|
|
2439
|
-
roleName: entry.roleName
|
|
2440
|
-
});
|
|
2441
|
-
const activeTurnId = providerBinding.turn !== null
|
|
2442
|
-
&& ["accepted", "running"].includes(providerBinding.turn.status)
|
|
2443
|
-
? providerBinding.turn.turnId
|
|
2444
|
-
: undefined;
|
|
2445
|
-
const recovery = decideProviderRecovery({
|
|
2446
|
-
binding: providerBinding,
|
|
2447
|
-
probe: {
|
|
2448
|
-
state: conversation.recoverability === "recoverable"
|
|
2449
|
-
? "exists"
|
|
2450
|
-
: conversation.recoverability === "unrecoverable" ? "missing" : "unknown",
|
|
2451
|
-
conversationId: conversation.conversationId,
|
|
2452
|
-
...(activeTurnId === undefined ? {} : { activeTurnId })
|
|
2453
|
-
},
|
|
2454
|
-
unsettledInputDelivery: roleMailbox?.inputDelivery != null
|
|
2455
|
-
});
|
|
2456
|
-
if (recovery.action === "attention") {
|
|
2457
|
-
deferUnprovenProviderRecovery(store, run, entry.roleName, recovery.reason, now);
|
|
2458
|
-
return;
|
|
2459
|
-
}
|
|
2460
|
-
if (recovery.action === "observe-active-turn") {
|
|
2461
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
2462
|
-
if (deferred === null) {
|
|
2463
|
-
finalizeProviderRetryDeadline(store, run, `Provider Turn ${recovery.turnId} remained active through the bounded recovery deadline.`, "provider-progress-timeout", now);
|
|
2464
|
-
}
|
|
2465
|
-
else {
|
|
2466
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
2467
|
-
}
|
|
2468
|
-
return;
|
|
2469
|
-
}
|
|
2470
|
-
if (recovery.action === "restart-run") {
|
|
2471
|
-
const finalized = finalizeProviderRetryDeadline(store, run, "Provider Conversation is unavailable; retry the work with a fresh Run and Session.", "provider-conversation-unavailable", now);
|
|
2472
|
-
if (finalized) {
|
|
2473
|
-
const currentSessions = store.getTaskRoleSessionSet(entry.taskId, entry.roleName);
|
|
2474
|
-
const currentSession = currentSessions?.sessions[run.effective.agentId];
|
|
2475
|
-
if (currentSessions !== null && currentSessions !== undefined
|
|
2476
|
-
&& currentSession !== undefined
|
|
2477
|
-
&& currentSession.status !== "stopped"
|
|
2478
|
-
&& currentSession.status !== "broken") {
|
|
2479
|
-
store.saveTaskRoleSessionSet(updateRoleAgentSessionStatus(currentSessions, run.effective.agentId, "broken", now));
|
|
2480
|
-
}
|
|
2481
|
-
}
|
|
2482
|
-
return;
|
|
2483
|
-
}
|
|
2484
|
-
const session = sessions?.sessions[run.effective.agentId];
|
|
2485
|
-
const identityMatches = session !== undefined
|
|
2486
|
-
&& session.adapterId === run.effective.adapterId
|
|
2487
|
-
&& (run.providerRetry.nativeSessionId === undefined
|
|
2488
|
-
|| session.nativeSessionId === run.providerRetry.nativeSessionId)
|
|
2489
|
-
&& (run.providerRetry.launchId === undefined
|
|
2490
|
-
|| session.launchId === run.providerRetry.launchId);
|
|
2491
|
-
if (!identityMatches || session?.nativeSessionId === undefined) {
|
|
2492
|
-
// Local Session projection loss is not native-session-dead evidence.
|
|
2493
|
-
// Keep the bounded retry episode; if native identity never returns,
|
|
2494
|
-
// the episode fails and a later Run can continue from Task progress.
|
|
2495
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
2496
|
-
if (deferred === null) {
|
|
2497
|
-
finalizeProviderRetryDeadline(store, run, "Provider retry safety could not be re-established before the bounded episode deadline; native Session recoverability remains unproven.", "native-resume-unproven", now);
|
|
2498
|
-
return;
|
|
2499
|
-
}
|
|
2500
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
2501
|
-
let attentionEvent = store.listEvents(entry.taskId).find((event) => (event.type === "runtime.provider-retry-native-resume-unproven"
|
|
2502
|
-
&& event.payload.runId === run.id));
|
|
2503
|
-
if (attentionEvent === undefined) {
|
|
2504
|
-
attentionEvent = createTaskEvent(store.nextEventId(entry.taskId), entry.taskId, "runtime.provider-retry-native-resume-unproven", {
|
|
2505
|
-
runId: run.id,
|
|
2506
|
-
consecutiveFailures: String(run.providerRetry.consecutiveFailures),
|
|
2507
|
-
dispatchedRetries: String(run.providerRetry.dispatchedRetries)
|
|
2508
|
-
}, now);
|
|
2509
|
-
store.saveEvent(entry.taskId, attentionEvent);
|
|
2510
|
-
}
|
|
2511
|
-
routeRoleEvent(store, attentionEvent, entry.roleName, "provider-retry-native-resume-unproven", now);
|
|
2512
|
-
return;
|
|
2513
|
-
}
|
|
2514
|
-
if (runHasActiveRuntimeOperations(store.listEvents(entry.taskId), {
|
|
2515
|
-
taskId: entry.taskId,
|
|
2516
|
-
roleName: entry.roleName,
|
|
2517
|
-
runId: run.id,
|
|
2518
|
-
agentId: run.effective.agentId
|
|
2519
|
-
})) {
|
|
2520
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
2521
|
-
if (deferred === null) {
|
|
2522
|
-
finalizeProviderRetryDeadline(store, run, "Provider retry stopped because tool/subagent outcome remained uncertain through the bounded episode deadline.", "operation-outcome-uncertain", now);
|
|
2523
|
-
return;
|
|
2524
|
-
}
|
|
2525
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
2526
|
-
return;
|
|
2527
|
-
}
|
|
2528
|
-
// Reopen the Run on its original Session: reset transport markers,
|
|
2529
|
-
// switch to resume, and mark the projection in-flight. The delivery
|
|
2530
|
-
// pass re-pushes the exact same input in this same pass.
|
|
2531
|
-
const reopenedRun = reopenRunForProviderRetry(run, `provider-retry-${run.providerRetry.episodeId}-${run.providerRetry.dispatchedRetries + 1}`, now, "resume");
|
|
2532
|
-
store.saveAgentRun(reopenedRun);
|
|
2533
|
-
if (sessions !== null && sessions.inFlight !== null) {
|
|
2534
|
-
store.saveTaskRoleSessionSet(prepareTaskRoleRunRedispatch(sessions, {
|
|
2535
|
-
agentId: run.effective.agentId,
|
|
2536
|
-
runId: run.id,
|
|
2537
|
-
receiptId: agentRunDeliveryReceiptId(reopenedRun)
|
|
2538
|
-
}, now));
|
|
2539
|
-
}
|
|
2540
|
-
// The retry decision settled the original delivery claim; re-queue the
|
|
2541
|
-
// Role mailbox so the delivery pass can claim and re-push the Run.
|
|
2542
|
-
enqueueWork(store, { kind: "role", taskId: entry.taskId, roleName: entry.roleName }, "provider-retry-repush", now, [{ type: "run", taskId: entry.taskId, id: run.id }]);
|
|
2543
|
-
reopened.push(formatTaskRecordReference(entry.taskId, run.id, "agentRun"));
|
|
2544
|
-
});
|
|
2545
|
-
}
|
|
2546
|
-
return reopened;
|
|
2547
|
-
}
|
|
2548
|
-
/**
|
|
2549
|
-
* Applies already-normalized Session observations. Driver-specific startup
|
|
2550
|
-
* names and source variants have terminated before this boundary.
|
|
2551
|
-
*/
|
|
2552
1956
|
observeProviderRuntimeIdentity(input, now) {
|
|
2553
1957
|
return this.store.transaction((store) => {
|
|
2554
1958
|
const taskId = input.fence.taskId;
|
|
2555
1959
|
const sessions = store.getTaskRoleSessionSet(taskId, input.fence.roleName);
|
|
2556
1960
|
const run = store.getAgentRun(taskId, input.fence.runId);
|
|
2557
1961
|
if (sessions === null || sessions.providerBinding === null || run === null
|
|
2558
|
-
|| sessions.providerBinding.runId !== input.fence.runId
|
|
2559
1962
|
|| input.fence.conversationId === undefined) {
|
|
2560
1963
|
recordCanonicalObservationObsolete(store, input, "provider-binding-missing", now);
|
|
2561
1964
|
return "obsolete";
|
|
@@ -2645,7 +2048,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2645
2048
|
&& existingNativeSessionId !== decision.outcome.nativeSessionId;
|
|
2646
2049
|
const replacementBasis = replacingNativeSession
|
|
2647
2050
|
&& run.mode === "new"
|
|
2648
|
-
&&
|
|
2051
|
+
&& existingSession?.status === "ended"
|
|
2649
2052
|
? "terminal-session"
|
|
2650
2053
|
: null;
|
|
2651
2054
|
if (replacingNativeSession && replacementBasis === null) {
|
|
@@ -2658,7 +2061,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2658
2061
|
nativeSessionId: decision.outcome.nativeSessionId,
|
|
2659
2062
|
launchId: input.fence.launchId,
|
|
2660
2063
|
policy: "fixed",
|
|
2661
|
-
status: "
|
|
2064
|
+
status: "active",
|
|
2662
2065
|
effective: run.effective
|
|
2663
2066
|
};
|
|
2664
2067
|
const bound = replacementBasis === null
|
|
@@ -2723,7 +2126,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2723
2126
|
recordCanonicalObservationObsolete(store, input, "human-turn-fence-mismatch", now);
|
|
2724
2127
|
return "obsolete";
|
|
2725
2128
|
}
|
|
2726
|
-
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(
|
|
2129
|
+
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(sessions, input, now));
|
|
2727
2130
|
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "run.human-input-delivered", {
|
|
2728
2131
|
attemptId: humanAttemptId,
|
|
2729
2132
|
runId: active.id,
|
|
@@ -2746,11 +2149,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2746
2149
|
recordCanonicalObservationObsolete(store, input, "continuation-fence-mismatch", now);
|
|
2747
2150
|
return "obsolete";
|
|
2748
2151
|
}
|
|
2749
|
-
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(
|
|
2750
|
-
const role = store.getRole(input.fence.taskId, input.fence.roleName);
|
|
2751
|
-
if (role !== null && role.status !== "running") {
|
|
2752
|
-
store.saveRole(input.fence.taskId, updateRoleStatus(role, "running", now));
|
|
2753
|
-
}
|
|
2152
|
+
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(sessions, input, now));
|
|
2754
2153
|
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "run.input-delivered", {
|
|
2755
2154
|
attemptId: inputDelivery.attemptId,
|
|
2756
2155
|
runId: active.id,
|
|
@@ -2761,9 +2160,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
2761
2160
|
: { nativeTurnId: input.fence.nativeTurnId })
|
|
2762
2161
|
}, now));
|
|
2763
2162
|
store.saveWorkMailbox(completeMailboxInputDelivery(mailbox, inputDelivery.attemptId, now));
|
|
2764
|
-
if (active.providerRetry !== undefined) {
|
|
2765
|
-
this.clearProviderRetryProjection(store, active, input, "provider-accepted", now);
|
|
2766
|
-
}
|
|
2767
2163
|
return "applied";
|
|
2768
2164
|
}
|
|
2769
2165
|
const event = createCanonicalLifecycleEvent({
|
|
@@ -2797,24 +2193,8 @@ export class FileSchedulerStoreAdapter {
|
|
|
2797
2193
|
if (sessions !== null) {
|
|
2798
2194
|
store.saveTaskRoleSessionSet(recordStructuredProviderAcceptance(sessions, input, now));
|
|
2799
2195
|
}
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
store.saveAgentRun(progressed);
|
|
2803
|
-
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "runtime.provider-retry-recovered", {
|
|
2804
|
-
runId: active.id,
|
|
2805
|
-
roleName: active.roleName,
|
|
2806
|
-
evidence: "provider-accepted",
|
|
2807
|
-
nativeSessionId: input.fence.nativeSessionId ?? "",
|
|
2808
|
-
episodeId: active.providerRetry?.episodeId ?? "",
|
|
2809
|
-
consecutiveFailures: String(active.providerRetry?.consecutiveFailures ?? 0),
|
|
2810
|
-
dispatchedRetries: String(active.providerRetry?.dispatchedRetries ?? 0),
|
|
2811
|
-
recoveryLatencyMs: String(active.providerRetry === undefined
|
|
2812
|
-
? 0
|
|
2813
|
-
: Math.max(0, now.getTime() - Date.parse(active.providerRetry.firstFailureAt)))
|
|
2814
|
-
}, now));
|
|
2815
|
-
}
|
|
2816
|
-
if (progressed.deliveredAt === undefined) {
|
|
2817
|
-
const delivered = markAgentRunDelivered(progressed, now);
|
|
2196
|
+
if (active.deliveredAt === undefined) {
|
|
2197
|
+
const delivered = markAgentRunDelivered(active, now);
|
|
2818
2198
|
store.saveAgentRun(delivered);
|
|
2819
2199
|
markTaskRoleRunDeliveredInFlight(store, role, delivered, now);
|
|
2820
2200
|
store.saveEvent(input.fence.taskId, createTaskEvent(store.nextEventId(input.fence.taskId), input.fence.taskId, "run.delivered", runLaunchEventPayload(delivered), now));
|
|
@@ -2829,35 +2209,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
2829
2209
|
return "applied";
|
|
2830
2210
|
});
|
|
2831
2211
|
}
|
|
2832
|
-
clearProviderRetryForProgress(input, evidence, now) {
|
|
2833
|
-
this.store.transaction((store) => {
|
|
2834
|
-
const taskId = input.fence.taskId;
|
|
2835
|
-
const runId = input.fence.runId;
|
|
2836
|
-
if (taskId === undefined || runId === undefined)
|
|
2837
|
-
return;
|
|
2838
|
-
const run = store.getAgentRun(taskId, runId);
|
|
2839
|
-
if (run === null || run.status !== "active" || run.providerRetry === undefined)
|
|
2840
|
-
return;
|
|
2841
|
-
this.clearProviderRetryProjection(store, run, input, evidence, now);
|
|
2842
|
-
});
|
|
2843
|
-
}
|
|
2844
|
-
clearProviderRetryProjection(store, run, input, evidence, now) {
|
|
2845
|
-
const retry = run.providerRetry;
|
|
2846
|
-
if (retry === undefined)
|
|
2847
|
-
return;
|
|
2848
|
-
store.saveAgentRun(clearProviderRetryOnProgress(run));
|
|
2849
|
-
store.saveEvent(run.taskId, createTaskEvent(store.nextEventId(run.taskId), run.taskId, "runtime.provider-retry-recovered", {
|
|
2850
|
-
runId: run.id,
|
|
2851
|
-
roleName: input.fence.roleName,
|
|
2852
|
-
evidence,
|
|
2853
|
-
episodeId: retry.episodeId,
|
|
2854
|
-
consecutiveFailures: String(retry.consecutiveFailures),
|
|
2855
|
-
dispatchedRetries: String(retry.dispatchedRetries),
|
|
2856
|
-
recoveryLatencyMs: String(Math.max(0, now.getTime() - Date.parse(retry.firstFailureAt))),
|
|
2857
|
-
nativeSessionId: input.fence.nativeSessionId ?? "",
|
|
2858
|
-
activityId: input.payload.activityId ?? ""
|
|
2859
|
-
}, now));
|
|
2860
|
-
}
|
|
2861
2212
|
foldRuntimeLifecycleEvent(store, event, observation) {
|
|
2862
2213
|
const expectation = this.projectRunExpectation(store, event.fence, observation.fence.runId);
|
|
2863
2214
|
if (expectation === null) {
|
|
@@ -2984,10 +2335,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2984
2335
|
if (effective.agentId !== input.agentId || effective.adapterId !== input.adapterId) {
|
|
2985
2336
|
throw new Error("Runtime turn completion does not match the effective global launch identity.");
|
|
2986
2337
|
}
|
|
2987
|
-
const completedStatus = effectiveExisting?.status
|
|
2988
|
-
|| effectiveExisting?.status === "broken"
|
|
2989
|
-
? effectiveExisting.status
|
|
2990
|
-
: "ready";
|
|
2338
|
+
const completedStatus = effectiveExisting?.status ?? "active";
|
|
2991
2339
|
current = recordRoleAgentSession(current, {
|
|
2992
2340
|
agentId: input.agentId,
|
|
2993
2341
|
adapterId: input.adapterId,
|
|
@@ -2997,6 +2345,9 @@ export class FileSchedulerStoreAdapter {
|
|
|
2997
2345
|
preview: effectiveExisting?.preview ?? (input.summary === undefined ? undefined : sessionPreview(input.summary)),
|
|
2998
2346
|
policy: "fixed",
|
|
2999
2347
|
status: completedStatus,
|
|
2348
|
+
...(effectiveExisting?.endReason === undefined
|
|
2349
|
+
? {}
|
|
2350
|
+
: { endReason: effectiveExisting.endReason }),
|
|
3000
2351
|
effective
|
|
3001
2352
|
}, now);
|
|
3002
2353
|
current = rememberRoleAgentCompletedTurn(current, input.agentId, nativeSessionId, input.turnId, now);
|
|
@@ -3004,13 +2355,13 @@ export class FileSchedulerStoreAdapter {
|
|
|
3004
2355
|
completeRuntimeHookReservation(store, owner, input.launchId);
|
|
3005
2356
|
if (input.roleName === SYSTEM_OPERATOR_ROLE
|
|
3006
2357
|
&& input.adapterId === "codex"
|
|
3007
|
-
&& completedStatus === "
|
|
2358
|
+
&& completedStatus === "active") {
|
|
3008
2359
|
const operatorMailbox = store.getWorkMailbox({ kind: "operator" });
|
|
3009
2360
|
// A completed foreground Codex Turn leaves its native TUI attached to
|
|
3010
2361
|
// the thread. Stop only that idle Role runtime so Desktop can become
|
|
3011
2362
|
// the writer; the durable nativeSessionId remains available to resume.
|
|
3012
2363
|
if (operatorMailbox === null || !mailboxHasWork(operatorMailbox)) {
|
|
3013
|
-
enqueueWork(store, runtimeLifecycleTarget(owner),
|
|
2364
|
+
enqueueWork(store, runtimeLifecycleTarget(owner), RUNTIME_HOST_DETACH_REQUIRED_REASON, now);
|
|
3014
2365
|
}
|
|
3015
2366
|
}
|
|
3016
2367
|
return current.sessions[input.agentId];
|
|
@@ -3044,18 +2395,6 @@ export class FileSchedulerStoreAdapter {
|
|
|
3044
2395
|
return "apply";
|
|
3045
2396
|
}
|
|
3046
2397
|
}
|
|
3047
|
-
function runtimeTurnFailureSummary(input) {
|
|
3048
|
-
return [
|
|
3049
|
-
"Agent runtime turn failed.",
|
|
3050
|
-
`error: ${input.error}`,
|
|
3051
|
-
...(input.errorDetails === undefined
|
|
3052
|
-
? []
|
|
3053
|
-
: [`error_details: ${input.errorDetails}`]),
|
|
3054
|
-
...(input.lastAssistantMessage === undefined
|
|
3055
|
-
? []
|
|
3056
|
-
: [`last_assistant_message: ${input.lastAssistantMessage}`])
|
|
3057
|
-
].join("\n");
|
|
3058
|
-
}
|
|
3059
2398
|
function runtimeObservationLifecycleFence(input, adapterId) {
|
|
3060
2399
|
return {
|
|
3061
2400
|
taskId: input.fence.taskId,
|
|
@@ -3203,12 +2542,8 @@ function preallocatedRuntimeReadyAwaitingProjection(store, observation, drivers)
|
|
|
3203
2542
|
taskId,
|
|
3204
2543
|
roleName: observation.fence.roleName
|
|
3205
2544
|
}));
|
|
3206
|
-
const executionRef = mailbox?.processing?.executionRef;
|
|
3207
2545
|
return isRuntimeLaunchReservation(mailbox?.processing, observation.fence.launchId)
|
|
3208
2546
|
&& !hasRuntimeCleanupObligation(mailbox)
|
|
3209
|
-
&& executionRef?.type === "run"
|
|
3210
|
-
&& executionRef.taskId === taskId
|
|
3211
|
-
&& executionRef.id === runId
|
|
3212
2547
|
&& nativeSessionId === nativeSessionIdForLaunch(store.rootDirectory(), observation.fence.launchId, observation.fence.agentId, driver.adapterId);
|
|
3213
2548
|
}
|
|
3214
2549
|
function completeRuntimeHookReservation(store, owner, launchId) {
|
|
@@ -3257,14 +2592,14 @@ function dormantRuntimeCandidateIsCurrent(store, candidate) {
|
|
|
3257
2592
|
const sessions = runtimeOwnerSessionSet(store, owner);
|
|
3258
2593
|
const active = sessions?.sessions[sessions.activeAgentId];
|
|
3259
2594
|
return active !== undefined
|
|
3260
|
-
&& active.status
|
|
2595
|
+
&& active.status === "active"
|
|
3261
2596
|
&& active.agentId === candidate.agentId
|
|
3262
2597
|
&& active.adapterId === candidate.adapterId
|
|
3263
2598
|
&& active.nativeSessionId === candidate.nativeSessionId
|
|
3264
2599
|
&& active.launchId === candidate.launchId
|
|
3265
2600
|
&& active.updatedAt === candidate.sessionUpdatedAt;
|
|
3266
2601
|
}
|
|
3267
|
-
function
|
|
2602
|
+
function endRuntimeOwnerSession(store, owner, now) {
|
|
3268
2603
|
if (owner.scope === "task") {
|
|
3269
2604
|
const sessions = store.getTaskRoleSessionSet(owner.taskId, owner.roleName);
|
|
3270
2605
|
if (sessions === null)
|
|
@@ -3272,7 +2607,19 @@ function markRuntimeOwnerSessionStopped(store, owner, now) {
|
|
|
3272
2607
|
const active = sessions.sessions[sessions.activeAgentId];
|
|
3273
2608
|
if (active === undefined)
|
|
3274
2609
|
return false;
|
|
3275
|
-
|
|
2610
|
+
let stopped = active.status === "ended"
|
|
2611
|
+
? sessions
|
|
2612
|
+
: updateRoleAgentSessionStatus(sessions, sessions.activeAgentId, "ended", now, "stopped");
|
|
2613
|
+
const activation = stopped.providerBinding === null
|
|
2614
|
+
? null
|
|
2615
|
+
: currentProviderActivation(stopped.providerBinding);
|
|
2616
|
+
if (activation !== null) {
|
|
2617
|
+
stopped = updateTaskRoleProviderRuntime(stopped, endProviderActivation(stopped.providerBinding, activation.activationId, {
|
|
2618
|
+
status: "ended",
|
|
2619
|
+
endedAt: now.toISOString(),
|
|
2620
|
+
reason: "session-ended"
|
|
2621
|
+
}), now);
|
|
2622
|
+
}
|
|
3276
2623
|
if (stopped === sessions)
|
|
3277
2624
|
return false;
|
|
3278
2625
|
store.saveTaskRoleSessionSet(stopped);
|
|
@@ -3282,9 +2629,29 @@ function markRuntimeOwnerSessionStopped(store, owner, now) {
|
|
|
3282
2629
|
if (sessions === null)
|
|
3283
2630
|
return false;
|
|
3284
2631
|
const active = sessions.sessions[sessions.activeAgentId];
|
|
3285
|
-
if (active === undefined || active.status === "
|
|
2632
|
+
if (active === undefined || active.status === "ended")
|
|
2633
|
+
return false;
|
|
2634
|
+
store.saveGlobalRoleSessionSet(updateRoleAgentSessionStatus(sessions, sessions.activeAgentId, "ended", now, "stopped"));
|
|
2635
|
+
return true;
|
|
2636
|
+
}
|
|
2637
|
+
function detachRuntimeOwnerHost(store, owner, now) {
|
|
2638
|
+
if (owner.scope === "task") {
|
|
2639
|
+
const sessions = store.getTaskRoleSessionSet(owner.taskId, owner.roleName);
|
|
2640
|
+
if (sessions === null)
|
|
2641
|
+
return false;
|
|
2642
|
+
const detached = detachRoleAgentSessionHost(sessions, now);
|
|
2643
|
+
if (detached === sessions)
|
|
2644
|
+
return false;
|
|
2645
|
+
store.saveTaskRoleSessionSet(detached);
|
|
2646
|
+
return true;
|
|
2647
|
+
}
|
|
2648
|
+
const sessions = store.getGlobalRoleSessionSet(owner.roleName);
|
|
2649
|
+
if (sessions === null)
|
|
2650
|
+
return false;
|
|
2651
|
+
const detached = detachRoleAgentSessionHost(sessions, now);
|
|
2652
|
+
if (detached === sessions)
|
|
3286
2653
|
return false;
|
|
3287
|
-
store.saveGlobalRoleSessionSet(
|
|
2654
|
+
store.saveGlobalRoleSessionSet(detached);
|
|
3288
2655
|
return true;
|
|
3289
2656
|
}
|
|
3290
2657
|
function runtimeOwnerSessionSet(store, owner) {
|
|
@@ -3304,8 +2671,7 @@ function preparedSessionMatches(session, agentId, adapterId, nativeSessionId, la
|
|
|
3304
2671
|
return nativeSessionId === undefined;
|
|
3305
2672
|
if (session.agentId !== agentId
|
|
3306
2673
|
|| session.adapterId !== adapterId
|
|
3307
|
-
|| session.status === "
|
|
3308
|
-
|| session.status === "broken"
|
|
2674
|
+
|| session.status === "ended"
|
|
3309
2675
|
|| (nativeSessionId !== undefined
|
|
3310
2676
|
&& session.nativeSessionId !== nativeSessionId)) {
|
|
3311
2677
|
return false;
|
|
@@ -3314,49 +2680,13 @@ function preparedSessionMatches(session, agentId, adapterId, nativeSessionId, la
|
|
|
3314
2680
|
? session.launchId === undefined
|
|
3315
2681
|
: session.launchId === launchId;
|
|
3316
2682
|
}
|
|
3317
|
-
function finalizeProviderRetryDeadline(store, run, summary, reason, now) {
|
|
3318
|
-
const retry = run.providerRetry;
|
|
3319
|
-
if (retry === undefined)
|
|
3320
|
-
return false;
|
|
3321
|
-
const terminalization = terminalizeExactTaskRun(store, {
|
|
3322
|
-
taskId: run.taskId,
|
|
3323
|
-
roleName: run.roleName,
|
|
3324
|
-
agentId: run.effective.agentId,
|
|
3325
|
-
runId: run.id,
|
|
3326
|
-
receiptId: agentRunDeliveryReceiptId(run),
|
|
3327
|
-
outcome: { status: "failed", summary }
|
|
3328
|
-
}, now);
|
|
3329
|
-
if (terminalization.disposition !== "applied" || terminalization.run === null)
|
|
3330
|
-
return false;
|
|
3331
|
-
const terminal = terminalization.run;
|
|
3332
|
-
const exhaustionEvent = createTaskEvent(store.nextEventId(run.taskId), run.taskId, "runtime.provider-retry-exhausted", {
|
|
3333
|
-
runId: run.id,
|
|
3334
|
-
roleName: run.roleName,
|
|
3335
|
-
episodeId: retry.episodeId,
|
|
3336
|
-
reason,
|
|
3337
|
-
consecutiveFailures: String(retry.consecutiveFailures),
|
|
3338
|
-
dispatchedRetries: String(retry.dispatchedRetries),
|
|
3339
|
-
elapsedMs: String(Math.max(0, now.getTime() - Date.parse(retry.firstFailureAt)))
|
|
3340
|
-
}, now);
|
|
3341
|
-
store.saveEvent(run.taskId, exhaustionEvent);
|
|
3342
|
-
const role = store.getRole(run.taskId, run.roleName);
|
|
3343
|
-
if (role !== null) {
|
|
3344
|
-
store.saveRole(run.taskId, updateRoleStatus(role, run.roleName === "leader" ? "failed" : "idle", now));
|
|
3345
|
-
}
|
|
3346
|
-
if (run.roleName === "leader") {
|
|
3347
|
-
store.saveLeaderFailure(recordLeaderFailure(run.taskId, retry.nativeSessionId ?? "(unproven)", summary, now, store.getLeaderFailure(run.taskId)));
|
|
3348
|
-
}
|
|
3349
|
-
routeRoleEvent(store, exhaustionEvent, run.roleName, "provider-retry-exhausted", now);
|
|
3350
|
-
return true;
|
|
3351
|
-
}
|
|
3352
2683
|
function preparedDeliveryFailureSessionFenceMatches(sessions, session, input, expectedReceiptId) {
|
|
3353
2684
|
if (sessions === null)
|
|
3354
2685
|
return input.nativeSessionId === undefined;
|
|
3355
2686
|
if (sessions.activeAgentId !== input.agentId)
|
|
3356
2687
|
return false;
|
|
3357
2688
|
if (sessions.inFlight === null) {
|
|
3358
|
-
return input.nativeSessionId
|
|
3359
|
-
&& session === undefined;
|
|
2689
|
+
return preparedSessionMatches(session, input.agentId, input.adapterId, input.nativeSessionId, input.launchId);
|
|
3360
2690
|
}
|
|
3361
2691
|
return sessions.inFlight.agentId === input.agentId
|
|
3362
2692
|
&& sessions.inFlight.runId === input.runId
|
|
@@ -3364,41 +2694,6 @@ function preparedDeliveryFailureSessionFenceMatches(sessions, session, input, ex
|
|
|
3364
2694
|
&& sessions.inFlight.pushedAt === undefined
|
|
3365
2695
|
&& preparedSessionMatches(session, input.agentId, input.adapterId, input.nativeSessionId, input.launchId);
|
|
3366
2696
|
}
|
|
3367
|
-
function preparedReservationMatches(mailbox, taskId, runId, launchId) {
|
|
3368
|
-
if (launchId === undefined) {
|
|
3369
|
-
return !isRuntimeLaunchReservation(mailbox?.processing);
|
|
3370
|
-
}
|
|
3371
|
-
const processing = mailbox?.processing;
|
|
3372
|
-
return isRuntimeLaunchReservation(processing, launchId)
|
|
3373
|
-
&& !hasRuntimeCleanupObligation(mailbox)
|
|
3374
|
-
&& processing?.executionRef?.type === "run"
|
|
3375
|
-
&& processing.executionRef.taskId === taskId
|
|
3376
|
-
&& processing.executionRef.id === runId;
|
|
3377
|
-
}
|
|
3378
|
-
function matchingPreparedRuntimeReservation(store, input) {
|
|
3379
|
-
if (input.launchId === undefined
|
|
3380
|
-
|| input.session?.nativeSessionId === undefined) {
|
|
3381
|
-
return null;
|
|
3382
|
-
}
|
|
3383
|
-
const owner = {
|
|
3384
|
-
scope: "task",
|
|
3385
|
-
taskId: input.task.id,
|
|
3386
|
-
roleName: input.role.name
|
|
3387
|
-
};
|
|
3388
|
-
const mailbox = store.getWorkMailbox(runtimeLifecycleTarget(owner));
|
|
3389
|
-
if (isRuntimeLaunchReservation(mailbox?.processing, input.launchId)
|
|
3390
|
-
&& mailbox?.processing?.executionRef?.type === "run"
|
|
3391
|
-
&& mailbox.processing.executionRef.id === input.run.id
|
|
3392
|
-
&& !hasRuntimeCleanupObligation(mailbox)) {
|
|
3393
|
-
return mailbox;
|
|
3394
|
-
}
|
|
3395
|
-
throw new Error("Prepared Role session does not match its launch generation.");
|
|
3396
|
-
}
|
|
3397
|
-
function completePreparedRuntimeReservation(store, mailbox) {
|
|
3398
|
-
if (mailbox === null || mailbox.processing === null)
|
|
3399
|
-
return;
|
|
3400
|
-
saveRuntimeLifecycleMailbox(store, completeProcessing(mailbox, mailbox.processing.batchId));
|
|
3401
|
-
}
|
|
3402
2697
|
function recordTaskRuntimeNativeSession(store, input, now) {
|
|
3403
2698
|
const task = store.getTask(input.taskId);
|
|
3404
2699
|
if (task === null)
|
|
@@ -3414,7 +2709,7 @@ function recordTaskRuntimeNativeSession(store, input, now) {
|
|
|
3414
2709
|
?? createRoleSessionSet({ scope: "task", taskId: input.taskId, roleName: input.roleName }, input.agentId, now);
|
|
3415
2710
|
const existing = current.sessions[input.agentId];
|
|
3416
2711
|
const effectiveExisting = nativeTransitionExisting(store, { scope: "task", taskId: input.taskId, roleName: input.roleName }, existing, input.nativeSessionId, input.launchId, "Native session registration conflicts with the fixed Role session.");
|
|
3417
|
-
if (existing?.status === "
|
|
2712
|
+
if (existing?.status === "active"
|
|
3418
2713
|
&& (input.launchId === undefined || existing.launchId === input.launchId))
|
|
3419
2714
|
return existing;
|
|
3420
2715
|
const resolvedEffective = taskSessionEffective(store, input.taskId, input.roleName, input.agentId, effectiveExisting);
|
|
@@ -3433,7 +2728,7 @@ function recordTaskRuntimeNativeSession(store, input, now) {
|
|
|
3433
2728
|
nativeSessionId: input.nativeSessionId,
|
|
3434
2729
|
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
3435
2730
|
policy: "fixed",
|
|
3436
|
-
status: "
|
|
2731
|
+
status: "active",
|
|
3437
2732
|
effective: effectiveExisting?.effective ?? effective
|
|
3438
2733
|
}, now);
|
|
3439
2734
|
store.saveRoleSessionSet(updated);
|
|
@@ -3447,7 +2742,7 @@ function recordGlobalRuntimeNativeSession(store, input, now) {
|
|
|
3447
2742
|
?? createRoleSessionSet({ scope: "global", roleName: input.roleName }, input.agentId, now);
|
|
3448
2743
|
const existing = current.sessions[input.agentId];
|
|
3449
2744
|
const effectiveExisting = nativeTransitionExisting(store, { scope: "global", roleName: input.roleName }, existing, input.nativeSessionId, input.launchId, "Native session registration conflicts with the fixed global Role session.");
|
|
3450
|
-
if (existing?.status === "
|
|
2745
|
+
if (existing?.status === "active"
|
|
3451
2746
|
&& (input.launchId === undefined || existing.launchId === input.launchId))
|
|
3452
2747
|
return existing;
|
|
3453
2748
|
const resolvedEffective = globalSessionEffective(role, effectiveExisting);
|
|
@@ -3466,7 +2761,7 @@ function recordGlobalRuntimeNativeSession(store, input, now) {
|
|
|
3466
2761
|
nativeSessionId: input.nativeSessionId,
|
|
3467
2762
|
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
3468
2763
|
policy: "fixed",
|
|
3469
|
-
status: "
|
|
2764
|
+
status: "active",
|
|
3470
2765
|
effective
|
|
3471
2766
|
}, now);
|
|
3472
2767
|
store.saveGlobalRoleSessionSet(updated);
|
|
@@ -3476,7 +2771,7 @@ function nativeTransitionExisting(store, owner, existing, nativeSessionId, launc
|
|
|
3476
2771
|
if (existing === undefined || existing.nativeSessionId === nativeSessionId) {
|
|
3477
2772
|
return existing;
|
|
3478
2773
|
}
|
|
3479
|
-
if (
|
|
2774
|
+
if (existing.status === "ended"
|
|
3480
2775
|
&& runtimeHookMatchesReservation(store, owner, launchId)) {
|
|
3481
2776
|
return undefined;
|
|
3482
2777
|
}
|
|
@@ -3527,8 +2822,7 @@ function mapRole(store, role) {
|
|
|
3527
2822
|
...(binding.config.effort === undefined ? {} : { effort: binding.config.effort }),
|
|
3528
2823
|
effective,
|
|
3529
2824
|
workspace: role.workspace,
|
|
3530
|
-
...(workspace === undefined ? {} : { managedWorkspace: workspace })
|
|
3531
|
-
status: role.status
|
|
2825
|
+
...(workspace === undefined ? {} : { managedWorkspace: workspace })
|
|
3532
2826
|
};
|
|
3533
2827
|
}
|
|
3534
2828
|
function taskSessionEffective(store, taskId, roleName, agentId, existing) {
|
|
@@ -3579,6 +2873,7 @@ function mapSession(session) {
|
|
|
3579
2873
|
...(session.launchId === undefined ? {} : { launchId: session.launchId }),
|
|
3580
2874
|
...(session.title === undefined ? {} : { title: session.title }),
|
|
3581
2875
|
status: session.status,
|
|
2876
|
+
...(session.endReason === undefined ? {} : { endReason: session.endReason }),
|
|
3582
2877
|
effective: session.effective,
|
|
3583
2878
|
updatedAt: session.updatedAt
|
|
3584
2879
|
};
|
|
@@ -3594,6 +2889,9 @@ function saveTaskSession(store, role, session, status, now, launchId) {
|
|
|
3594
2889
|
...(session.title === undefined ? {} : { title: session.title }),
|
|
3595
2890
|
policy: "fixed",
|
|
3596
2891
|
status,
|
|
2892
|
+
...(status !== "ended" || session.endReason === undefined
|
|
2893
|
+
? {}
|
|
2894
|
+
: { endReason: session.endReason }),
|
|
3597
2895
|
effective: session.effective
|
|
3598
2896
|
}, now);
|
|
3599
2897
|
store.saveRoleSessionSet(updated);
|
|
@@ -3607,8 +2905,7 @@ function bindTaskRoleRunInFlight(store, role, run, now) {
|
|
|
3607
2905
|
throw new Error("Task Role Session identity cannot change with an unsettled Run fence.");
|
|
3608
2906
|
}
|
|
3609
2907
|
const liveConflict = Object.values(current.sessions).find((session) => (session.agentId !== agentId
|
|
3610
|
-
&& session.status
|
|
3611
|
-
&& session.status !== "broken"));
|
|
2908
|
+
&& session.status === "active"));
|
|
3612
2909
|
if (liveConflict !== undefined) {
|
|
3613
2910
|
throw new Error(`Task Role Session identity cannot change while ${liveConflict.agentId} is live.`);
|
|
3614
2911
|
}
|
|
@@ -3662,27 +2959,6 @@ function clearTaskRoleRunInFlight(store, role, run, now) {
|
|
|
3662
2959
|
}, now);
|
|
3663
2960
|
store.saveRoleSessionSet(updated);
|
|
3664
2961
|
}
|
|
3665
|
-
function breakTaskSessionIfPresent(store, taskId, roleName, agentId, now) {
|
|
3666
|
-
updateTaskSessionStatusIfPresent(store, taskId, roleName, agentId, "broken", now);
|
|
3667
|
-
}
|
|
3668
|
-
function stopTaskSessionIfPresent(store, taskId, roleName, agentId, now) {
|
|
3669
|
-
updateTaskSessionStatusIfPresent(store, taskId, roleName, agentId, "stopped", now);
|
|
3670
|
-
}
|
|
3671
|
-
function updateTaskSessionStatusIfPresent(store, taskId, roleName, agentId, status, now) {
|
|
3672
|
-
const set = store.getRoleSessionSet(taskId, roleName);
|
|
3673
|
-
if (set === null || set.sessions[agentId] === undefined)
|
|
3674
|
-
return;
|
|
3675
|
-
store.saveRoleSessionSet(updateRoleAgentSessionStatus(set, agentId, status, now));
|
|
3676
|
-
}
|
|
3677
|
-
function schedulerRoleSessionsMatch(current, expected) {
|
|
3678
|
-
if (current === null || expected === null)
|
|
3679
|
-
return current === expected;
|
|
3680
|
-
return current.agentId === expected.agentId
|
|
3681
|
-
&& current.adapterId === expected.adapterId
|
|
3682
|
-
&& current.nativeSessionId === expected.nativeSessionId
|
|
3683
|
-
&& current.launchId === expected.launchId
|
|
3684
|
-
&& isDeepStrictEqual(current.effective, expected.effective);
|
|
3685
|
-
}
|
|
3686
2962
|
/**
|
|
3687
2963
|
* The durable Run/mailbox claim is the pre-Host delivery intent. The Host may
|
|
3688
2964
|
* create a ProviderTurn only while that exact reservation still owns the Run;
|
|
@@ -3822,7 +3098,9 @@ function recordStructuredProviderAcceptance(sessions, input, now) {
|
|
|
3822
3098
|
const turnId = input.fence.nativeTurnId;
|
|
3823
3099
|
if (current === null || attemptId === undefined || turnId === undefined)
|
|
3824
3100
|
return sessions;
|
|
3825
|
-
if (current.turn
|
|
3101
|
+
if (current.turn === null || current.turn.runId !== input.fence.runId)
|
|
3102
|
+
return sessions;
|
|
3103
|
+
if (current.turn.attemptId === attemptId
|
|
3826
3104
|
&& ["accepted", "running", "completed", "failed", "cancelled"].includes(current.turn.status))
|
|
3827
3105
|
return sessions;
|
|
3828
3106
|
const binding = acceptProviderTurn(current, {
|
|
@@ -3855,7 +3133,6 @@ function bindOrSupersedeProviderRuntime(sessions, input, now, replacementBasis)
|
|
|
3855
3133
|
return bindTaskRoleProviderRuntime(sessions, createProviderRuntimeBinding({
|
|
3856
3134
|
providerNamespace: input.fence.driverId,
|
|
3857
3135
|
accountScope: input.fence.agentId,
|
|
3858
|
-
runId: sessions.inFlight.runId,
|
|
3859
3136
|
conversationId,
|
|
3860
3137
|
activationId,
|
|
3861
3138
|
startedAt: sessions.inFlight.preparedAt
|
|
@@ -3893,7 +3170,7 @@ function rejectSubmittingProviderTurn(store, target, delivery, now, reason) {
|
|
|
3893
3170
|
const binding = sessions?.providerBinding;
|
|
3894
3171
|
if (sessions === null || sessions === undefined
|
|
3895
3172
|
|| binding === null || binding === undefined
|
|
3896
|
-
|| binding.runId !== delivery.executionRef.id
|
|
3173
|
+
|| binding.turn?.runId !== delivery.executionRef.id
|
|
3897
3174
|
|| binding.turn?.attemptId !== delivery.attemptId
|
|
3898
3175
|
|| (binding.turn.status !== "submitting"
|
|
3899
3176
|
&& binding.turn.status !== "delivery-unknown"))
|
|
@@ -3904,24 +3181,6 @@ function rejectSubmittingProviderTurn(store, target, delivery, now, reason) {
|
|
|
3904
3181
|
reason
|
|
3905
3182
|
}), now));
|
|
3906
3183
|
}
|
|
3907
|
-
function deferUnprovenProviderRecovery(store, run, roleName, reason, now) {
|
|
3908
|
-
if (run.providerRetry === undefined)
|
|
3909
|
-
return;
|
|
3910
|
-
const deferred = deferProviderRetry(run.providerRetry, now);
|
|
3911
|
-
if (deferred === null) {
|
|
3912
|
-
finalizeProviderRetryDeadline(store, run, reason, "native-resume-unproven", now);
|
|
3913
|
-
return;
|
|
3914
|
-
}
|
|
3915
|
-
store.saveAgentRun(withProviderRetry(run, deferred));
|
|
3916
|
-
const attentionEvent = createTaskEvent(store.nextEventId(run.taskId), run.taskId, "runtime.provider-recovery-attention", {
|
|
3917
|
-
runId: run.id,
|
|
3918
|
-
roleName,
|
|
3919
|
-
reason,
|
|
3920
|
-
nextAttemptAt: deferred.nextAttemptAt ?? ""
|
|
3921
|
-
}, now);
|
|
3922
|
-
store.saveEvent(run.taskId, attentionEvent);
|
|
3923
|
-
routeRoleEvent(store, attentionEvent, roleName, "provider-recovery-attention", now);
|
|
3924
|
-
}
|
|
3925
3184
|
function markSubmittingProviderTurnUnknown(store, target, delivery, now, reason) {
|
|
3926
3185
|
if (target.kind !== "role"
|
|
3927
3186
|
|| delivery.executionRef.type !== "run"
|
|
@@ -3931,7 +3190,7 @@ function markSubmittingProviderTurnUnknown(store, target, delivery, now, reason)
|
|
|
3931
3190
|
const binding = sessions?.providerBinding;
|
|
3932
3191
|
if (sessions === null || sessions === undefined
|
|
3933
3192
|
|| binding === null || binding === undefined
|
|
3934
|
-
|| binding.runId !== delivery.executionRef.id
|
|
3193
|
+
|| binding.turn?.runId !== delivery.executionRef.id
|
|
3935
3194
|
|| binding.turn?.attemptId !== delivery.attemptId
|
|
3936
3195
|
|| binding.turn.status !== "submitting")
|
|
3937
3196
|
return;
|
|
@@ -3984,7 +3243,3 @@ function compareCanonicalObservationOrder(left, right) {
|
|
|
3984
3243
|
|| (left.ordinal ?? -1) - (right.ordinal ?? -1)
|
|
3985
3244
|
|| left.eventId.localeCompare(right.eventId);
|
|
3986
3245
|
}
|
|
3987
|
-
function configuredProviderRetryPolicy(store) {
|
|
3988
|
-
const config = providerRetryConfig(store.getConfig());
|
|
3989
|
-
return { delaysMs: config.delaysMs, maxWindowMs: config.maxWindowMs };
|
|
3990
|
-
}
|