@zq-silk/yui 0.2.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/ARCHITECTURE.md +603 -133
  2. package/README.md +806 -31
  3. package/dist/agent/agent.js +2 -1
  4. package/dist/agent/argumentPolicy.js +3 -1
  5. package/dist/agent/launchEnvironment.js +106 -0
  6. package/dist/agent/managedRuntimeEnvironment.js +34 -0
  7. package/dist/brief/taskBrief.js +11 -1
  8. package/dist/cli/agentConfigurationPicker.js +287 -0
  9. package/dist/cli/commandCatalog.js +488 -60
  10. package/dist/cli/completion.js +146 -22
  11. package/dist/cli/helpRenderer.js +3 -1
  12. package/dist/cli/interactionCandidates.js +53 -15
  13. package/dist/cli/interactionPolicy.js +267 -30
  14. package/dist/cli/interactiveSelection.js +6 -2
  15. package/dist/cli/invocationRouter.js +5 -1
  16. package/dist/cli/operatorWizard.js +87 -0
  17. package/dist/cli/roleOptionCatalog.js +1 -0
  18. package/dist/cli/roleWizard.js +185 -21
  19. package/dist/cli/updateCommand.js +62 -19
  20. package/dist/cli/updateOrchestrator.js +539 -0
  21. package/dist/cli/updatePorts.js +1119 -0
  22. package/dist/cli/upgradeCommand.js +112 -0
  23. package/dist/cli.js +1420 -86
  24. package/dist/commands/agentCommands.js +146 -3
  25. package/dist/commands/configCommands.js +126 -0
  26. package/dist/commands/controllerCommands.js +365 -0
  27. package/dist/commands/globalRoleCommands.js +168 -126
  28. package/dist/commands/jobCommands.js +18 -8
  29. package/dist/commands/operatorCommands.js +159 -9
  30. package/dist/commands/profileCommands.js +203 -0
  31. package/dist/commands/projectCommands.js +650 -0
  32. package/dist/commands/roleConfiguration.js +85 -24
  33. package/dist/commands/roleRuntimeGuard.js +12 -0
  34. package/dist/commands/roleSkillValidation.js +47 -0
  35. package/dist/commands/taskActor.js +127 -0
  36. package/dist/commands/taskCommands.js +4201 -313
  37. package/dist/commands/taskCompletionGate.js +131 -0
  38. package/dist/commands/taskContextCommand.js +244 -30
  39. package/dist/commands/taskInputCommands.js +177 -59
  40. package/dist/commands/taskIntegrationCommands.js +303 -0
  41. package/dist/commands/taskOverviewCommand.js +363 -0
  42. package/dist/commands/taskRoleRuntimeStatus.js +125 -19
  43. package/dist/commands/textInput.js +15 -0
  44. package/dist/completion/completionInstaller.js +26 -22
  45. package/dist/config/yuiConfig.js +4 -3
  46. package/dist/context/dispatchContext.js +90 -38
  47. package/dist/context/roleSessionContext.js +119 -0
  48. package/dist/controller/claudeLifecycleHook.js +203 -0
  49. package/dist/controller/clientRuntime.js +408 -56
  50. package/dist/controller/codexLifecycleHook.js +108 -0
  51. package/dist/controller/controller.js +1089 -32
  52. package/dist/controller/domainIdentity.js +505 -0
  53. package/dist/controller/ephemeralResourceReaper.js +131 -0
  54. package/dist/controller/fileSchedulerStoreAdapter.js +2153 -103
  55. package/dist/controller/providerHookRunFence.js +127 -0
  56. package/dist/controller/resourceCleanupLinux.js +286 -0
  57. package/dist/controller/resourceInventory.js +531 -0
  58. package/dist/controller/resourceInventoryLinux.js +610 -0
  59. package/dist/controller/runtime.js +629 -10
  60. package/dist/controller/runtimeEventInbox.js +564 -0
  61. package/dist/controller/runtimeEventProcessor.js +248 -0
  62. package/dist/controller/runtimeLaunchCoordinator.js +477 -0
  63. package/dist/controller/sessionNotify.js +121 -78
  64. package/dist/coordination/deadlineScheduler.js +15 -0
  65. package/dist/coordination/mailboxScheduler.js +108 -0
  66. package/dist/coordination/workMailbox.js +329 -0
  67. package/dist/coordination/workMailboxQueue.js +86 -0
  68. package/dist/core/controllerClient.js +19 -5
  69. package/dist/core/controllerEndpoint.js +37 -0
  70. package/dist/core/controllerServer.js +218 -10
  71. package/dist/core/protocol.js +6 -2
  72. package/dist/decision/decision.js +2 -1
  73. package/dist/doctor/doctor.js +681 -32
  74. package/dist/domain/validation.js +53 -0
  75. package/dist/errors/cliError.js +5 -3
  76. package/dist/event/taskEvent.js +7 -3
  77. package/dist/execution/codexThreadNaming.js +160 -0
  78. package/dist/execution/executionGroup.js +579 -0
  79. package/dist/executor/agentAdapter.js +255 -40
  80. package/dist/executor/agentConfigurationCatalog.js +326 -0
  81. package/dist/executor/agentConfigurationProbe.js +506 -0
  82. package/dist/executor/agentExecutor.js +625 -10
  83. package/dist/executor/codexConfigConflict.js +290 -0
  84. package/dist/executor/effectiveLaunch.js +340 -0
  85. package/dist/executor/executorRegistry.js +238 -36
  86. package/dist/executor/fileRoleLaunchPlanner.js +550 -40
  87. package/dist/executor/turnCompletion.js +126 -0
  88. package/dist/input/inputRequest.js +30 -9
  89. package/dist/integration/changeSet.js +36 -0
  90. package/dist/integration/checkResult.js +24 -0
  91. package/dist/integration/gitIntegrationService.js +695 -0
  92. package/dist/integration/integrationAttempt.js +142 -0
  93. package/dist/interaction/operatorPresentation.js +96 -0
  94. package/dist/lifecycle/canonicalLifecycleEvent.js +342 -0
  95. package/dist/lifecycle/exactRunTerminalization.js +572 -0
  96. package/dist/lifecycle/providerLifecycleMapping.js +190 -0
  97. package/dist/lifecycle/taskRoleSessionReset.js +124 -0
  98. package/dist/message/message.js +23 -7
  99. package/dist/milestone/milestone.js +2 -1
  100. package/dist/operator/operatorSessionHistory.js +124 -0
  101. package/dist/output/agentConfigurationPresentation.js +43 -0
  102. package/dist/output/rolePresentation.js +34 -10
  103. package/dist/output/terminal.js +8 -0
  104. package/dist/output/timePresentation.js +55 -0
  105. package/dist/profile/agentProfile.js +128 -0
  106. package/dist/repository/gitWorkspace.js +578 -24
  107. package/dist/repository/project.js +213 -0
  108. package/dist/repository/taskWorkspaceCoordinator.js +392 -0
  109. package/dist/repository/taskWorkspacePreparer.js +1688 -191
  110. package/dist/review/reviewConfig.js +11 -0
  111. package/dist/review/reviewRound.js +399 -0
  112. package/dist/review/taskFinalReviewContract.js +90 -0
  113. package/dist/role/role.js +124 -23
  114. package/dist/run/agentRun.js +155 -12
  115. package/dist/run/runIdentity.js +82 -0
  116. package/dist/runtime/exactControlPlane.js +472 -0
  117. package/dist/runtime/index.js +8 -0
  118. package/dist/runtime/lifecycleReservation.js +38 -0
  119. package/dist/runtime/ports.js +11 -0
  120. package/dist/runtime/preallocatedNativeSession.js +13 -0
  121. package/dist/runtime/promptEnvelope.js +30 -0
  122. package/dist/runtime/runtimeBinding.js +31 -0
  123. package/dist/runtime/runtimeOwner.js +14 -0
  124. package/dist/runtime/sessionLaunchRequest.js +62 -0
  125. package/dist/runtime/sessionTitle.js +54 -0
  126. package/dist/runtime/taskRuntimeIsolation.js +643 -0
  127. package/dist/runtime/tmuxAdapters.js +315 -0
  128. package/dist/runtime/turnCompletion.js +3 -0
  129. package/dist/runtime/validation.js +23 -0
  130. package/dist/scheduler/activeRoleRunDelivery.js +342 -32
  131. package/dist/scheduler/activeTaskProgress.js +63 -0
  132. package/dist/scheduler/leaderFailure.js +2 -1
  133. package/dist/scheduler/leaderWakeupProcessor.js +307 -66
  134. package/dist/scheduler/operatorInputNotificationProcessor.js +109 -46
  135. package/dist/scheduler/operatorNotification.js +44 -2
  136. package/dist/scheduler/ports.js +28 -1
  137. package/dist/scheduler/roleRunLiveness.js +131 -25
  138. package/dist/scheduler/roleRunStall.js +951 -0
  139. package/dist/scheduler/taskExecutionProjection.js +544 -0
  140. package/dist/scheduler/wakeupQueue.js +3 -0
  141. package/dist/setup/setupCommand.js +302 -52
  142. package/dist/storage/compatibleTaskStore.js +102 -0
  143. package/dist/storage/migration/baseline.js +78 -0
  144. package/dist/storage/migration/classifier.js +51 -0
  145. package/dist/storage/migration/compatibleCodec.js +53 -0
  146. package/dist/storage/migration/engine.js +147 -0
  147. package/dist/storage/migration/index.js +33 -0
  148. package/dist/storage/migration/planner.js +154 -0
  149. package/dist/storage/migration/productionRegistry.js +486 -0
  150. package/dist/storage/migration/registry.js +169 -0
  151. package/dist/storage/migration/report.js +54 -0
  152. package/dist/storage/migration/types.js +31 -0
  153. package/dist/storage/storageSchema.js +147 -123
  154. package/dist/storage/storageVersions.js +11 -0
  155. package/dist/storage/taskStore.js +1793 -197
  156. package/dist/storage/upgrade/homeClassification.js +156 -0
  157. package/dist/storage/upgrade/homeMigrationTarget.js +595 -0
  158. package/dist/storage/upgrade/offlineUpgradeInventory.js +315 -0
  159. package/dist/storage/upgrade/productionMigrationRegistry.js +6 -0
  160. package/dist/storage/upgrade/recordVersionScan.js +176 -0
  161. package/dist/storage/upgrade/recordVersions.js +159 -0
  162. package/dist/storage/upgrade/switchProgress.js +80 -0
  163. package/dist/storage/upgrade/upgradeOrchestrator.js +948 -0
  164. package/dist/storage/upgrade/upgradeReceipt.js +161 -0
  165. package/dist/storage/upgradeCoordination.js +186 -0
  166. package/dist/storage/upgradeFence.js +366 -0
  167. package/dist/task/task.js +132 -26
  168. package/dist/task/taskRecordReference.js +66 -0
  169. package/dist/tmux/commandExecutor.js +75 -2
  170. package/dist/tmux/tmuxManager.js +747 -49
  171. package/dist/version.js +23 -0
  172. package/dist/web/assets/assetManifest.js +62 -0
  173. package/dist/web/assets/client/app.js +631 -0
  174. package/dist/web/assets/client/components.js +605 -0
  175. package/dist/web/assets/client/dom.js +14 -0
  176. package/dist/web/assets/client/format.js +28 -0
  177. package/dist/web/assets/client/i18n.js +494 -0
  178. package/dist/web/assets/client/markdown.js +114 -0
  179. package/dist/web/assets/client/theme.js +32 -0
  180. package/dist/web/assets/client/view.js +458 -0
  181. package/dist/web/assets/fontData.js +12 -0
  182. package/dist/web/assets/fonts.js +12 -0
  183. package/dist/web/assets/shell.js +114 -0
  184. package/dist/web/assets/styles/cards.js +135 -0
  185. package/dist/web/assets/styles/layout.js +47 -0
  186. package/dist/web/assets/styles/markdown.js +29 -0
  187. package/dist/web/assets/styles/responsive.js +39 -0
  188. package/dist/web/assets/styles/tokens.js +101 -0
  189. package/dist/web/assets/styles/widgets.js +147 -0
  190. package/dist/web/tmuxWebTerminal.js +158 -0
  191. package/dist/web/webServer.js +463 -0
  192. package/dist/web/webSnapshot.js +148 -0
  193. package/dist/workItem/workItem.js +642 -23
  194. package/dist/workspace/gitChangeSetCapture.js +86 -0
  195. package/dist/workspace/workItemChangeSetManager.js +445 -0
  196. package/dist/worktree/managedWorkspace.js +202 -0
  197. package/docs/task-local-identity.md +62 -0
  198. package/i18n/README.zh-CN.md +406 -31
  199. package/package.json +10 -2
  200. package/skills/yui-leader/SKILL.md +601 -39
  201. package/skills/yui-operator/SKILL.md +255 -34
  202. package/skills/yui-reviewer/SKILL.md +57 -0
  203. package/skills/yui-worker/SKILL.md +214 -17
  204. package/dist/commands/repositoryCommands.js +0 -86
  205. package/dist/operator/operatorContext.js +0 -66
  206. package/dist/repository/repository.js +0 -55
  207. package/dist/scheduler/archivedTaskRuntime.js +0 -12
  208. package/dist/worktree/roleWorkspace.js +0 -62
@@ -1,33 +1,535 @@
1
1
  import { reconciliationIntervalMilliseconds } from "../config/yuiConfig.js";
2
2
  import { processLeaderWakeups } from "../scheduler/leaderWakeupProcessor.js";
3
3
  import { processActiveRoleRunDeliveries } from "../scheduler/activeRoleRunDelivery.js";
4
- import { stopArchivedTaskRuntimes } from "../scheduler/archivedTaskRuntime.js";
4
+ import { selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
5
5
  import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
6
+ import { DEFAULT_STALL_WINDOW_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
7
+ import { repairOrphanedActiveTasks } from "../scheduler/activeTaskProgress.js";
6
8
  import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
7
9
  import { startControllerServer } from "../core/controllerServer.js";
10
+ import { MailboxScheduler } from "../coordination/mailboxScheduler.js";
11
+ import { nearestDeadlineBatch } from "../coordination/deadlineScheduler.js";
12
+ import { hasRuntimeCleanupObligation, isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
13
+ import { formatTaskRecordReference } from "../task/taskRecordReference.js";
8
14
  const DEFAULT_RECONCILIATION_INTERVAL_MS = reconciliationIntervalMilliseconds();
15
+ const DEFAULT_SIGNAL_WINDOW_MS = 100;
16
+ const DEFAULT_DELIVERY_RETRY_MS = 250;
17
+ const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
18
+ const RUNTIME_RESERVATION_RECOVERY_AGE_MS = 120_000;
19
+ const MAX_TIMER_DELAY_MS = 2_147_483_647;
20
+ class RuntimeEventApplyError extends AggregateError {
21
+ }
9
22
  /**
10
- * Runs one lean scheduler pass. Liveness always precedes wakeup processing so
11
- * an exited busy Leader is cleared and reconsidered in the same pass.
23
+ * Runs one lean scheduler pass. Due native Turn completions are folded before
24
+ * liveness, so a valid Hook boundary fences destructive process reconciliation.
12
25
  */
13
- export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer) {
14
- await workspacePreparer?.prepareActiveTaskWorkspaces();
15
- const stoppedArchivedTaskIds = await stopArchivedTaskRuntimes(store, delivery, now);
16
- await workspacePreparer?.cleanupArchivedTaskWorkspaces();
17
- const activeRunDeliveries = await processActiveRoleRunDeliveries(store, delivery, now);
18
- const failedRunIds = await reconcileExitedRoleRuns(store, delivery, now);
19
- const autoResolvedInputs = store.resolveExpiredInputRecommendations(now);
20
- const wakeups = await processLeaderWakeups(store, delivery, now);
21
- const inputNotifications = await processOperatorInputNotifications(store, delivery);
26
+ export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, resourceSuppressionKeys = new Set()) {
27
+ const compiledSelection = compileReconcileSelection(scope);
28
+ const selection = includeOperator
29
+ ? compiledSelection
30
+ : { ...compiledSelection, operator: false };
31
+ queueSelectedCompletedTaskRuntimeCleanups(store, selection, now);
32
+ const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes);
33
+ const roleSelection = selectionWithoutFailedCleanupRoles(store, selection, failedCleanupRoles);
34
+ const wakeupSelection = selectionWithoutFailedLeaderCleanupTasks(store, selection, failedCleanupRoles);
35
+ if (selection.full)
36
+ repairOrphanedActiveTasks(store, now, selection);
37
+ const claimedTaskMailboxes = claimSelectedTaskMailboxes(store, selection, now);
38
+ try {
39
+ const failedTaskMailboxes = await prepareActiveWorkspaces(store, workspacePreparer, selection);
40
+ const activeRunDeliveries = await processActiveRoleRunDeliveries(store, delivery, now, roleSelection);
41
+ const unsettledRunRefs = new Set(activeRunDeliveries.flatMap((result) => (result.reason === "delivery-uncertain" || result.terminalFailure !== undefined
42
+ ? [formatTaskRecordReference(result.taskId, result.runId, "agentRun")]
43
+ : [])));
44
+ resolveDueRuntimeTurnCompletions(store, delivery, selection, now);
45
+ const liveStatuses = new Map();
46
+ const resourceEvidence = new Map();
47
+ const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence);
48
+ await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, resourceSuppressionKeys);
49
+ await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now);
50
+ const autoResolvedInputs = selection.full
51
+ ? store.resolveExpiredInputRecommendations(now)
52
+ : selection.taskIds.size === 0
53
+ ? []
54
+ : store.resolveExpiredInputRecommendations(now, selection.taskIds);
55
+ const wakeups = await processLeaderWakeups(store, delivery, now, wakeupSelection);
56
+ const inputNotifications = includeOperator
57
+ ? await processOperatorInputNotifications(store, delivery, selection, now)
58
+ : [];
59
+ for (const claim of claimedTaskMailboxes) {
60
+ if (failedTaskMailboxes.has(claim.target.taskId)) {
61
+ store.releaseWorkMailbox(claim.target, claim.processing.batchId);
62
+ }
63
+ else {
64
+ store.completeWorkMailbox(claim.target, claim.processing.batchId);
65
+ }
66
+ }
67
+ return {
68
+ activeRunDeliveries,
69
+ failedRunRefs,
70
+ wakeups,
71
+ inputNotifications,
72
+ autoResolvedInputs
73
+ };
74
+ }
75
+ catch (error) {
76
+ for (const claim of claimedTaskMailboxes) {
77
+ store.releaseWorkMailbox(claim.target, claim.processing.batchId);
78
+ }
79
+ throw error;
80
+ }
81
+ }
82
+ function queueSelectedCompletedTaskRuntimeCleanups(store, selection, now) {
83
+ if (store.enqueueRuntimeCleanup === undefined)
84
+ return;
85
+ for (const task of selectedSchedulerTasks(store, selection)) {
86
+ if (task.status !== "completed")
87
+ continue;
88
+ for (const role of selectedSchedulerRoles(store, task.id, selection)) {
89
+ if (store.getActiveAgentRun(task.id, role.name) !== null)
90
+ continue;
91
+ const session = store.getRoleSession(task.id, role.name);
92
+ if (!schedulerSessionRequiresRuntimeCleanup(session))
93
+ continue;
94
+ const target = runtimeLifecycleTarget({
95
+ scope: "task",
96
+ taskId: task.id,
97
+ roleName: role.name
98
+ });
99
+ if (hasRuntimeCleanupObligation(store.getWorkMailbox(target)))
100
+ continue;
101
+ store.enqueueRuntimeCleanup({
102
+ scope: "task",
103
+ taskId: task.id,
104
+ roleName: role.name
105
+ }, now);
106
+ }
107
+ }
108
+ }
109
+ function schedulerSessionRequiresRuntimeCleanup(session) {
110
+ if (session === null || session.status === "stopped" || session.status === "broken") {
111
+ return false;
112
+ }
113
+ return session.status === "running" || session.launchId !== undefined;
114
+ }
115
+ async function processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, outcomes) {
116
+ const targets = selectedRuntimeLifecycleTargets(store, scope);
117
+ const failedRoles = new Set();
118
+ for (const target of targets) {
119
+ const mailbox = store.getWorkMailbox(target);
120
+ if (mailbox === null)
121
+ continue;
122
+ const owner = runtimeOwner(target);
123
+ const batchId = runtimeLifecycleBatchIdentity(target, mailbox);
124
+ if (hasRuntimeCleanupObligation(mailbox)) {
125
+ try {
126
+ if (lifecycleHost === undefined) {
127
+ throw new Error("Role runtime cleanup host is unavailable.");
128
+ }
129
+ if (!await lifecycleHost.stopOwner(owner)) {
130
+ throw new Error(`Role runtime cleanup could not confirm the host stopped: ${runtimeOwnerLabel(owner)}.`);
131
+ }
132
+ if (target.kind === "role-runtime") {
133
+ const session = store.getRoleSession(target.taskId, target.roleName);
134
+ const reservedLaunchId = isRuntimeLaunchReservation(mailbox.processing)
135
+ ? mailbox.processing.batchId
136
+ : undefined;
137
+ if (reservedLaunchId !== undefined
138
+ && session?.launchId !== undefined
139
+ && reservedLaunchId !== session.launchId) {
140
+ throw new Error(`Task runtime cleanup launch identity is ambiguous: ${target.taskId}.`);
141
+ }
142
+ const launchId = reservedLaunchId ?? session?.launchId;
143
+ if (launchId !== undefined
144
+ && lifecycleHost.cleanupTaskLaunch !== undefined) {
145
+ const task = store.getTask(target.taskId);
146
+ const reason = task?.status === "completed"
147
+ ? "completion"
148
+ : "interruption";
149
+ lifecycleHost.cleanupTaskLaunch({
150
+ taskId: target.taskId,
151
+ launchId,
152
+ reason
153
+ });
154
+ }
155
+ }
156
+ if (store.completeRuntimeCleanup === undefined
157
+ || !store.completeRuntimeCleanup(target, now)) {
158
+ throw new Error(`Role runtime cleanup mailbox changed: ${runtimeOwnerLabel(owner)}.`);
159
+ }
160
+ forgetPreparedRuntimeOwner(delivery, owner);
161
+ outcomes.push({ target, batchId, status: "completed" });
162
+ }
163
+ catch (error) {
164
+ markFailedRuntimeTarget(failedRoles, target);
165
+ outcomes.push({ target, batchId, status: "failed", error });
166
+ }
167
+ continue;
168
+ }
169
+ const reservation = mailbox.processing;
170
+ if (reservation === null
171
+ || !isRuntimeLaunchReservation(reservation)
172
+ || now.getTime() - Date.parse(reservation.startedAt)
173
+ < RUNTIME_RESERVATION_RECOVERY_AGE_MS) {
174
+ continue;
175
+ }
176
+ try {
177
+ if (lifecycleHost === undefined) {
178
+ throw new Error("Role runtime reservation inspection is unavailable.");
179
+ }
180
+ const inspection = await lifecycleHost.inspectOwner(owner);
181
+ if (inspection.state === "running" || inspection.state === "starting") {
182
+ continue;
183
+ }
184
+ if (inspection.state === "unavailable") {
185
+ throw new Error(`Role runtime reservation could not inspect the host: ${runtimeOwnerLabel(owner)}.`);
186
+ }
187
+ const completed = store.completeStoppedRuntimeReservation === undefined
188
+ ? store.completeWorkMailbox(target, reservation.batchId)
189
+ : store.completeStoppedRuntimeReservation(target, reservation.batchId, now);
190
+ if (!completed) {
191
+ throw new Error(`Role runtime reservation mailbox changed: ${runtimeOwnerLabel(owner)}.`);
192
+ }
193
+ forgetPreparedRuntimeOwner(delivery, owner);
194
+ outcomes.push({ target, batchId, status: "completed" });
195
+ }
196
+ catch (error) {
197
+ markFailedRuntimeTarget(failedRoles, target);
198
+ outcomes.push({ target, batchId, status: "failed", error });
199
+ }
200
+ }
201
+ return failedRoles;
202
+ }
203
+ async function reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now) {
204
+ if (scope.kind !== "full"
205
+ || lifecycleHost === undefined
206
+ || store.listDormantRuntimeOwners === undefined
207
+ || store.markRuntimeOwnerStopped === undefined) {
208
+ return;
209
+ }
210
+ const candidates = store.listDormantRuntimeOwners();
211
+ if (candidates.length === 0)
212
+ return;
213
+ const owners = candidates.map((candidate) => candidate.owner);
214
+ let inspections;
215
+ try {
216
+ inspections = lifecycleHost.inspectOwners === undefined
217
+ ? await Promise.all(owners.map(async (owner) => ({
218
+ owner,
219
+ inspection: await lifecycleHost.inspectOwner(owner)
220
+ })))
221
+ : await lifecycleHost.inspectOwners(owners);
222
+ }
223
+ catch {
224
+ // Host inventory is an advisory safety scan. Unknown state must never
225
+ // mutate persisted session facts; the next full pass will retry.
226
+ return;
227
+ }
228
+ const requested = new Set(owners.map(runtimeOwnerIdentity));
229
+ const byOwner = new Map();
230
+ for (const result of inspections) {
231
+ const identity = runtimeOwnerIdentity(result.owner);
232
+ if (!requested.has(identity)
233
+ || byOwner.has(identity)) {
234
+ return;
235
+ }
236
+ byOwner.set(identity, result.inspection);
237
+ }
238
+ if (byOwner.size !== requested.size)
239
+ return;
240
+ for (const candidate of candidates) {
241
+ if (byOwner.get(runtimeOwnerIdentity(candidate.owner))?.state
242
+ === "stopped") {
243
+ if (candidate.owner.scope === "task"
244
+ && candidate.launchId !== undefined) {
245
+ // The exact launch-owned resources must settle through the durable
246
+ // cleanup lane before its Session fact becomes stopped. The candidate
247
+ // is passed back as the CAS fence so a concurrent Hook/launch cannot
248
+ // redirect cleanup to a newer generation.
249
+ store.enqueueRuntimeCleanup?.(candidate.owner, now, candidate);
250
+ continue;
251
+ }
252
+ if (store.markRuntimeOwnerStopped(candidate, now)) {
253
+ forgetPreparedRuntimeOwner(delivery, candidate.owner);
254
+ }
255
+ }
256
+ }
257
+ }
258
+ function forgetPreparedRuntimeOwner(delivery, owner) {
259
+ if (owner.scope !== "task")
260
+ return;
261
+ delivery.forgetPrepared?.({
262
+ taskId: owner.taskId,
263
+ roleName: owner.roleName
264
+ });
265
+ }
266
+ function runtimeOwnerIdentity(owner) {
267
+ return owner.scope === "task"
268
+ ? `task\0${owner.taskId}\0${owner.roleName}`
269
+ : `global\0${owner.roleName}`;
270
+ }
271
+ function resolveDueRuntimeTurnCompletions(store, delivery, selection, now) {
272
+ if (typeof store.resolveDueRuntimeTurnCompletions !== "function")
273
+ return;
274
+ const selectedTaskIds = selection.full ? undefined : selection.taskIds;
275
+ if (selectedTaskIds?.size === 0)
276
+ return;
277
+ const candidates = store.listPendingRuntimeTurnCompletions().filter((completion) => (selectedTaskIds === undefined
278
+ || selectedTaskIds.has(completion.taskId)));
279
+ const finalized = new Set(store.resolveDueRuntimeTurnCompletions(now, selectedTaskIds));
280
+ if (finalized.size === 0)
281
+ return;
282
+ for (const completion of candidates) {
283
+ if (!finalized.has(formatTaskRecordReference(completion.taskId, completion.runId, "agentRun")))
284
+ continue;
285
+ delivery.forgetPrepared?.({
286
+ taskId: completion.taskId,
287
+ roleName: completion.roleName,
288
+ runId: completion.runId
289
+ });
290
+ }
291
+ }
292
+ function selectedRuntimeLifecycleTargets(store, scope) {
293
+ if (scope.kind === "full") {
294
+ return store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "role-runtime"
295
+ || mailbox.target.kind === "global-role-runtime"
296
+ ? [mailbox.target]
297
+ : []));
298
+ }
299
+ const targets = new Map();
300
+ for (const key of scope.keys) {
301
+ const parsed = parseMailboxKey(key);
302
+ if (parsed.kind === "task") {
303
+ for (const role of store.listRoles(parsed.taskId)) {
304
+ const target = {
305
+ kind: "role-runtime",
306
+ taskId: parsed.taskId,
307
+ roleName: role.name
308
+ };
309
+ targets.set(runtimeTargetIdentity(target), target);
310
+ }
311
+ }
312
+ else if (parsed.kind === "role") {
313
+ const target = {
314
+ kind: "role-runtime",
315
+ taskId: parsed.taskId,
316
+ roleName: parsed.roleName
317
+ };
318
+ targets.set(runtimeTargetIdentity(target), target);
319
+ }
320
+ else if (parsed.kind === "global-role") {
321
+ const target = {
322
+ kind: "global-role-runtime",
323
+ roleName: parsed.roleName
324
+ };
325
+ targets.set(runtimeTargetIdentity(target), target);
326
+ }
327
+ }
328
+ return [...targets.values()];
329
+ }
330
+ function runtimeOwner(target) {
331
+ return target.kind === "role-runtime"
332
+ ? {
333
+ scope: "task",
334
+ taskId: target.taskId,
335
+ roleName: target.roleName
336
+ }
337
+ : { scope: "global", roleName: target.roleName };
338
+ }
339
+ function runtimeOwnerLabel(owner) {
340
+ return owner.scope === "task"
341
+ ? `${owner.taskId}/${owner.roleName}`
342
+ : `global/${owner.roleName}`;
343
+ }
344
+ function runtimeTargetIdentity(target) {
345
+ return target.kind === "role-runtime"
346
+ ? `task\0${target.taskId}\0${target.roleName}`
347
+ : `global\0${target.roleName}`;
348
+ }
349
+ function runtimeCleanupMailboxKey(target) {
350
+ return target.kind === "role-runtime"
351
+ ? `role:${encodeURIComponent(target.taskId)}/${encodeURIComponent(target.roleName)}`
352
+ : `global-role:${encodeURIComponent(target.roleName)}`;
353
+ }
354
+ function runtimeLifecycleBatchIdentity(target, mailbox) {
355
+ if (mailbox.processing !== null)
356
+ return mailbox.processing.batchId;
357
+ const pending = mailbox.pending;
358
+ return pending === null
359
+ ? runtimeTargetIdentity(target)
360
+ : `${runtimeTargetIdentity(target)}:${pending.fromSequence}-${pending.toSequence}`;
361
+ }
362
+ function markFailedRuntimeTarget(failedRoles, target) {
363
+ if (target.kind === "role-runtime") {
364
+ failedRoles.add(roleIdentity(target.taskId, target.roleName));
365
+ }
366
+ }
367
+ function selectionWithoutFailedCleanupRoles(store, selection, failedRoles) {
368
+ if (failedRoles.size === 0)
369
+ return selection;
370
+ const taskIds = selection.full
371
+ ? new Set(store.listTasks().map((task) => task.id))
372
+ : new Set(selection.taskIds);
373
+ const rolesByTask = new Map();
374
+ for (const taskId of taskIds) {
375
+ const roleNames = selection.full || selection.allRoleTaskIds.has(taskId)
376
+ ? store.listRoles(taskId).map((role) => role.name)
377
+ : [...(selection.rolesByTask.get(taskId) ?? [])];
378
+ rolesByTask.set(taskId, new Set(roleNames.filter((roleName) => (!failedRoles.has(roleIdentity(taskId, roleName))))));
379
+ }
380
+ return {
381
+ full: false,
382
+ taskIds,
383
+ allRoleTaskIds: new Set(),
384
+ rolesByTask,
385
+ operator: selection.operator
386
+ };
387
+ }
388
+ function selectionWithoutFailedLeaderCleanupTasks(store, selection, failedRoles) {
389
+ if (![...failedRoles].some((identity) => identity.endsWith("\0leader"))) {
390
+ return selection;
391
+ }
392
+ const taskIds = selection.full
393
+ ? new Set(store.listTasks().map((task) => task.id))
394
+ : new Set(selection.taskIds);
395
+ for (const taskId of taskIds) {
396
+ if (failedRoles.has(roleIdentity(taskId, "leader")))
397
+ taskIds.delete(taskId);
398
+ }
399
+ return {
400
+ full: false,
401
+ taskIds,
402
+ allRoleTaskIds: new Set(),
403
+ rolesByTask: new Map(),
404
+ operator: selection.operator
405
+ };
406
+ }
407
+ function roleIdentity(taskId, roleName) {
408
+ return `${taskId}\0${roleName}`;
409
+ }
410
+ function claimSelectedTaskMailboxes(store, selection, now) {
411
+ const targets = selection.full
412
+ ? store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
413
+ : [...selection.allRoleTaskIds].map((taskId) => ({ kind: "task", taskId }));
414
+ const claims = [];
415
+ for (const target of targets) {
416
+ const mailbox = store.getWorkMailbox(target);
417
+ if (mailbox === null || (mailbox.processing === null && mailbox.pending === null))
418
+ continue;
419
+ const batchId = mailbox.processing?.batchId
420
+ ?? `task:${encodeURIComponent(target.taskId)}:${mailbox.pending.fromSequence}-${mailbox.pending.toSequence}`;
421
+ const claim = store.claimWorkMailbox({
422
+ target,
423
+ batchId,
424
+ owner: "controller",
425
+ now
426
+ });
427
+ if (claim.status !== "empty")
428
+ claims.push({ target, processing: claim.processing });
429
+ }
430
+ return claims;
431
+ }
432
+ export function compileReconcileSelection(scope) {
433
+ if (scope.kind === "full") {
434
+ return {
435
+ full: true,
436
+ taskIds: new Set(),
437
+ allRoleTaskIds: new Set(),
438
+ rolesByTask: new Map(),
439
+ operator: true
440
+ };
441
+ }
442
+ const taskIds = new Set();
443
+ const allRoleTaskIds = new Set();
444
+ const mutableRoles = new Map();
445
+ let operator = false;
446
+ for (const key of scope.keys) {
447
+ const target = parseMailboxKey(key);
448
+ if (target.kind === "operator") {
449
+ operator = true;
450
+ }
451
+ else if (target.kind === "task") {
452
+ taskIds.add(target.taskId);
453
+ allRoleTaskIds.add(target.taskId);
454
+ }
455
+ else if (target.kind === "role") {
456
+ taskIds.add(target.taskId);
457
+ const roles = mutableRoles.get(target.taskId) ?? new Set();
458
+ roles.add(target.roleName);
459
+ mutableRoles.set(target.taskId, roles);
460
+ }
461
+ }
22
462
  return {
23
- stoppedArchivedTaskIds,
24
- activeRunDeliveries,
25
- failedRunIds,
26
- wakeups,
27
- inputNotifications,
28
- autoResolvedInputs
463
+ full: false,
464
+ taskIds,
465
+ allRoleTaskIds,
466
+ rolesByTask: mutableRoles,
467
+ operator
29
468
  };
30
469
  }
470
+ async function prepareActiveWorkspaces(store, workspace, selection) {
471
+ if (workspace === undefined)
472
+ return new Set();
473
+ const taskIds = selection.full
474
+ ? new Set(store.listTasks()
475
+ .filter((task) => task.status === "active")
476
+ .map((task) => task.id))
477
+ : new Set([...selection.taskIds, ...selection.allRoleTaskIds]);
478
+ const failed = new Set();
479
+ for (const taskId of taskIds) {
480
+ if (store.getTask(taskId)?.status === "active") {
481
+ try {
482
+ const result = await workspace.prepareTaskWorkspace(taskId);
483
+ if (result.status === "failed")
484
+ failed.add(taskId);
485
+ }
486
+ catch {
487
+ failed.add(taskId);
488
+ }
489
+ }
490
+ }
491
+ return failed;
492
+ }
493
+ function parseMailboxKey(key) {
494
+ if (key === "operator")
495
+ return { kind: "operator" };
496
+ if (key.startsWith("task:")) {
497
+ return { kind: "task", taskId: mailboxPart(key.slice("task:".length), key) };
498
+ }
499
+ if (key.startsWith("global-role:")) {
500
+ return {
501
+ kind: "global-role",
502
+ roleName: mailboxPart(key.slice("global-role:".length), key)
503
+ };
504
+ }
505
+ if (key.startsWith("role:")) {
506
+ const value = key.slice("role:".length);
507
+ const separator = value.indexOf("/");
508
+ if (separator > 0 && separator < value.length - 1 && value.indexOf("/", separator + 1) < 0) {
509
+ return {
510
+ kind: "role",
511
+ taskId: mailboxPart(value.slice(0, separator), key),
512
+ roleName: mailboxPart(value.slice(separator + 1), key)
513
+ };
514
+ }
515
+ }
516
+ throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
517
+ }
518
+ function mailboxPart(value, key) {
519
+ if (value.length === 0 || value.trim() !== value || value.includes("\0")) {
520
+ throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
521
+ }
522
+ try {
523
+ const decoded = decodeURIComponent(value);
524
+ if (decoded.length === 0 || decoded.includes("\0") || encodeURIComponent(decoded) !== value) {
525
+ throw new Error("not canonical");
526
+ }
527
+ return decoded;
528
+ }
529
+ catch {
530
+ throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
531
+ }
532
+ }
31
533
  /**
32
534
  * Single-owner periodic runtime for FileTaskStore-backed scheduling. Concurrent
33
535
  * pump requests coalesce into one follow-up pass; scheduler effects never
@@ -40,9 +542,30 @@ export class FileTaskController {
40
542
  #now;
41
543
  #onError;
42
544
  #workspacePreparer;
545
+ #deliveryRetryMs;
546
+ #deliveryRetryLimit;
547
+ #stallWindowMs;
548
+ /** Narrow-port fallback; FileSchedulerStoreAdapter durably records these keys. */
549
+ #resourceSuppressionKeys = new Set();
550
+ #runtimeEventProcessor;
551
+ #lifecycleHost;
552
+ #deliveryRetryAttempts = new Map();
553
+ #deliveryRetryTimers = new Map();
554
+ #passRetryTimer;
555
+ #passRetryAttempt = 0;
43
556
  #timer;
557
+ #deadlineTimer;
558
+ #signalScheduler;
559
+ #operatorSignalScheduler;
560
+ #configuration;
561
+ #resourceReaper;
562
+ #onExpiredEphemeralDomain;
44
563
  #current;
45
- #rerunRequested = false;
564
+ #operatorCurrent;
565
+ #pendingFull = false;
566
+ #pendingKeys = new Set();
567
+ #operatorStartupRetryArmed = false;
568
+ #lastOperatorSignalIdentity;
46
569
  #stopped = false;
47
570
  constructor(store, delivery, options = {}) {
48
571
  this.store = store;
@@ -51,33 +574,151 @@ export class FileTaskController {
51
574
  this.#now = options.now ?? (() => new Date());
52
575
  this.#onError = options.onError ?? (() => { });
53
576
  this.#workspacePreparer = options.workspacePreparer;
577
+ this.#deliveryRetryMs = positiveInteger(options.deliveryRetryMs, DEFAULT_DELIVERY_RETRY_MS, "Controller delivery retry delay");
578
+ this.#deliveryRetryLimit = positiveInteger(options.deliveryRetryLimit, DEFAULT_DELIVERY_RETRY_LIMIT, "Controller delivery retry limit");
579
+ this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
580
+ this.#runtimeEventProcessor = options.runtimeEventProcessor;
581
+ this.#lifecycleHost = options.lifecycleHost;
582
+ this.#configuration = options.configuration;
583
+ this.#resourceReaper = options.resourceReaper;
584
+ this.#onExpiredEphemeralDomain = options.onExpiredEphemeralDomain;
585
+ this.#signalScheduler = new MailboxScheduler(async (keys) => { await this.#requestPass({ kind: "dirty", keys }); }, {
586
+ windowMs: options.signalWindowMs ?? DEFAULT_SIGNAL_WINDOW_MS,
587
+ onError: this.#onError,
588
+ setTimer: (callback, delayMs) => {
589
+ const timer = setTimeout(callback, delayMs);
590
+ timer.unref();
591
+ return timer;
592
+ }
593
+ });
594
+ this.#operatorSignalScheduler = new MailboxScheduler(async () => {
595
+ const running = this.#runOperatorPass();
596
+ this.#operatorCurrent = running;
597
+ try {
598
+ await running;
599
+ }
600
+ finally {
601
+ if (this.#operatorCurrent === running)
602
+ this.#operatorCurrent = undefined;
603
+ }
604
+ }, {
605
+ windowMs: options.signalWindowMs ?? DEFAULT_SIGNAL_WINDOW_MS,
606
+ onError: this.#onError,
607
+ setTimer: (callback, delayMs) => {
608
+ const timer = setTimeout(callback, delayMs);
609
+ timer.unref();
610
+ return timer;
611
+ }
612
+ });
54
613
  }
55
614
  get reconciliationIntervalMs() {
56
615
  return this.#intervalMs;
57
616
  }
617
+ updateReconciliationInterval(intervalMs) {
618
+ const next = positiveInteger(intervalMs, DEFAULT_RECONCILIATION_INTERVAL_MS, "Controller reconciliation interval");
619
+ if (this.#stopped)
620
+ throw new Error("Controller runtime is stopped.");
621
+ if (next === this.#intervalMs)
622
+ return;
623
+ this.#intervalMs = next;
624
+ if (this.#timer !== undefined) {
625
+ clearInterval(this.#timer);
626
+ this.#timer = setInterval(() => {
627
+ this.#requestBackgroundPump();
628
+ }, this.#intervalMs);
629
+ this.#timer.unref();
630
+ }
631
+ }
632
+ reloadReconciliationInterval() {
633
+ if (this.#configuration !== undefined) {
634
+ this.updateReconciliationInterval(this.#configuration.reconciliationIntervalMs());
635
+ }
636
+ return this.#intervalMs;
637
+ }
58
638
  start() {
59
639
  if (this.#timer !== undefined)
60
640
  return;
61
- this.#stopped = false;
62
- void this.pump().catch(this.#onError);
641
+ if (this.#stopped)
642
+ throw new Error("Controller runtime is stopped.");
643
+ this.#requestBackgroundPump();
644
+ // Arm the Operator lane once for work that was already pending when the
645
+ // Controller started. Subsequent main-lane passes signal it only when a
646
+ // new Operator batch is durably queued; an unchanged pending batch must
647
+ // not keep an unavailable Operator lane in a zero-delay drain loop.
648
+ this.#signalOperatorMailbox();
63
649
  this.#timer = setInterval(() => {
64
- void this.pump().catch(this.#onError);
650
+ this.#requestBackgroundPump();
65
651
  }, this.#intervalMs);
66
652
  this.#timer.unref();
67
653
  }
68
654
  stop() {
69
655
  this.#stopped = true;
656
+ this.#signalScheduler.stop();
657
+ this.#operatorSignalScheduler.stop();
658
+ if (this.#deadlineTimer !== undefined) {
659
+ clearTimeout(this.#deadlineTimer);
660
+ this.#deadlineTimer = undefined;
661
+ }
70
662
  if (this.#timer !== undefined) {
71
663
  clearInterval(this.#timer);
72
664
  this.#timer = undefined;
73
665
  }
666
+ for (const timer of this.#deliveryRetryTimers.values())
667
+ clearTimeout(timer);
668
+ this.#deliveryRetryTimers.clear();
669
+ this.#deliveryRetryAttempts.clear();
670
+ if (this.#passRetryTimer !== undefined) {
671
+ clearTimeout(this.#passRetryTimer);
672
+ this.#passRetryTimer = undefined;
673
+ }
674
+ }
675
+ async shutdownAndDrain() {
676
+ this.stop();
677
+ await Promise.allSettled([
678
+ this.#current ?? Promise.resolve(),
679
+ this.#operatorCurrent ?? Promise.resolve()
680
+ ]);
681
+ }
682
+ /** Adds one dirty Task key to the fixed-window wake queue and returns immediately. */
683
+ signal(key) {
684
+ if (this.#stopped)
685
+ throw new Error("Controller runtime is stopped.");
686
+ parseMailboxKey(key);
687
+ if (key === "operator") {
688
+ // Main lane owns durable Hook folding; the Operator lane is the sole
689
+ // consumer of the Operator delivery mailbox.
690
+ this.#signalScheduler.signal("operator");
691
+ this.#operatorSignalScheduler.signal("operator");
692
+ }
693
+ else
694
+ this.#signalScheduler.signal(key);
74
695
  }
75
696
  pump() {
697
+ return this.#requestPass({ kind: "full" });
698
+ }
699
+ armOperatorStartupRetry() {
700
+ if (this.#stopped)
701
+ return;
702
+ const mailbox = this.store.getWorkMailbox({ kind: "operator" });
703
+ const shouldArm = mailbox !== null
704
+ && (mailbox.pending !== null || mailbox.processing !== null);
705
+ if (shouldArm)
706
+ this.#clearDeliveryRetry("operator");
707
+ this.#operatorStartupRetryArmed = shouldArm;
708
+ }
709
+ #requestPass(scope) {
76
710
  if (this.#stopped) {
77
711
  return Promise.reject(new Error("Controller runtime is stopped."));
78
712
  }
713
+ if (scope.kind === "full") {
714
+ this.#pendingFull = true;
715
+ this.#pendingKeys.clear();
716
+ }
717
+ else if (!this.#pendingFull) {
718
+ for (const key of scope.keys)
719
+ this.#pendingKeys.add(key);
720
+ }
79
721
  if (this.#current !== undefined) {
80
- this.#rerunRequested = true;
81
722
  return this.#current;
82
723
  }
83
724
  const running = this.#runCoalesced();
@@ -88,43 +729,410 @@ export class FileTaskController {
88
729
  }).catch(() => { });
89
730
  return running;
90
731
  }
732
+ #requestBackgroundPump() {
733
+ void this.pump().catch(this.#onError);
734
+ }
91
735
  async #runCoalesced() {
92
736
  let result = {
93
- stoppedArchivedTaskIds: [],
94
737
  activeRunDeliveries: [],
95
- failedRunIds: [],
738
+ failedRunRefs: [],
96
739
  wakeups: [],
97
740
  inputNotifications: [],
98
741
  autoResolvedInputs: []
99
742
  };
100
- do {
101
- this.#rerunRequested = false;
102
- result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer);
103
- } while (this.#rerunRequested && !this.#stopped);
743
+ try {
744
+ while (this.#pendingFull || this.#pendingKeys.size > 0) {
745
+ const scope = this.#pendingFull
746
+ ? { kind: "full" }
747
+ : { kind: "dirty", keys: [...this.#pendingKeys] };
748
+ const runtimeCleanupOutcomes = [];
749
+ this.#pendingFull = false;
750
+ this.#pendingKeys.clear();
751
+ try {
752
+ if (scope.kind === "full")
753
+ this.reloadReconciliationInterval();
754
+ if (scope.kind === "full" && this.#resourceReaper !== undefined) {
755
+ const reap = await this.#resourceReaper();
756
+ for (const failure of reap.failed) {
757
+ this.#onError(new Error(`Ephemeral runtime reap failed for ${failure.id}: ${failure.message}`));
758
+ }
759
+ // The reaper owns the exact resource fences. Once a whole expired
760
+ // domain converges, let the detached Controller close itself on a
761
+ // later turn; never close while this reconciliation pass is still
762
+ // the in-flight server request.
763
+ if (reap.failed.length === 0 && (reap.expiredDomains?.length ?? 0) > 0) {
764
+ for (const domain of reap.expiredDomains ?? []) {
765
+ queueMicrotask(() => {
766
+ this.#onExpiredEphemeralDomain?.(domain);
767
+ });
768
+ }
769
+ }
770
+ }
771
+ const firstRuntimeDrain = this.#drainRuntimeEvents();
772
+ result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#resourceSuppressionKeys);
773
+ const secondRuntimeDrain = this.#drainRuntimeEvents();
774
+ this.#clearPassRetry();
775
+ this.#scheduleRuntimeCleanupRetries(runtimeCleanupOutcomes);
776
+ this.#scheduleDeliveryRetries(result);
777
+ this.#scheduleTaskMailboxRetries(scope);
778
+ if (scope.kind === "full"
779
+ || scope.keys.some((key) => key !== "operator")
780
+ || (firstRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0
781
+ || (secondRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0) {
782
+ this.#signalOperatorMailbox();
783
+ }
784
+ }
785
+ catch (error) {
786
+ if (scope.kind === "full") {
787
+ this.#pendingFull = true;
788
+ this.#pendingKeys.clear();
789
+ }
790
+ else if (!this.#pendingFull) {
791
+ for (const key of scope.keys)
792
+ this.#pendingKeys.add(key);
793
+ }
794
+ this.#schedulePassRetry();
795
+ throw error;
796
+ }
797
+ if (this.#stopped)
798
+ break;
799
+ }
800
+ return result;
801
+ }
802
+ finally {
803
+ this.#scheduleNextInputDeadline();
804
+ }
805
+ }
806
+ async #runOperatorPass() {
807
+ const result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), undefined, { kind: "dirty", keys: ["operator"] });
808
+ if (this.#operatorStartupRetryArmed && result.inputNotifications.length === 0) {
809
+ const mailbox = this.store.getWorkMailbox({ kind: "operator" });
810
+ if (mailbox === null
811
+ || (mailbox.pending === null && mailbox.processing === null)) {
812
+ this.#operatorStartupRetryArmed = false;
813
+ }
814
+ }
815
+ this.#scheduleDeliveryRetries(result);
816
+ this.#clearEmptyOperatorRetry();
817
+ this.#scheduleNextInputDeadline();
818
+ }
819
+ #signalOperatorMailbox() {
820
+ if (this.#stopped)
821
+ return;
822
+ const mailbox = this.store.getWorkMailbox({ kind: "operator" });
823
+ const identity = operatorMailboxBatchIdentity(mailbox);
824
+ if (identity === null) {
825
+ this.#lastOperatorSignalIdentity = undefined;
826
+ return;
827
+ }
828
+ if (identity !== this.#lastOperatorSignalIdentity) {
829
+ this.#operatorSignalScheduler.signal("operator");
830
+ this.#lastOperatorSignalIdentity = identity;
831
+ }
832
+ }
833
+ #drainRuntimeEvents() {
834
+ const result = this.#runtimeEventProcessor?.drain(this.#now());
835
+ if (result !== undefined && result.failed.length > 0) {
836
+ throw new RuntimeEventApplyError(result.failed.map((failure) => failure.error), "One or more native Turn events could not be applied.");
837
+ }
104
838
  return result;
105
839
  }
840
+ #schedulePassRetry() {
841
+ if (this.#stopped
842
+ || this.#passRetryTimer !== undefined
843
+ || this.#passRetryAttempt >= this.#deliveryRetryLimit)
844
+ return;
845
+ const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(this.#passRetryAttempt, 3)));
846
+ this.#passRetryAttempt += 1;
847
+ this.#passRetryTimer = setTimeout(() => {
848
+ this.#passRetryTimer = undefined;
849
+ if (this.#stopped)
850
+ return;
851
+ void this.#requestPass({ kind: "dirty", keys: [] }).catch(this.#onError);
852
+ }, delayMs);
853
+ this.#passRetryTimer.unref();
854
+ }
855
+ #clearPassRetry() {
856
+ if (this.#passRetryTimer !== undefined)
857
+ clearTimeout(this.#passRetryTimer);
858
+ this.#passRetryTimer = undefined;
859
+ this.#passRetryAttempt = 0;
860
+ }
861
+ #scheduleNextInputDeadline() {
862
+ if (this.#deadlineTimer !== undefined) {
863
+ clearTimeout(this.#deadlineTimer);
864
+ this.#deadlineTimer = undefined;
865
+ }
866
+ if (this.#stopped)
867
+ return;
868
+ const deadlines = [
869
+ ...this.store.listOpenInputRequests()
870
+ .flatMap((request) => request.policy.kind === "recommended"
871
+ ? [{
872
+ key: `task:${encodeURIComponent(request.taskId)}`,
873
+ at: Date.parse(request.policy.timeoutAt)
874
+ }]
875
+ : []),
876
+ ...(typeof this.store.listPendingRuntimeTurnCompletions === "function"
877
+ ? this.store.listPendingRuntimeTurnCompletions()
878
+ : []).map((completion) => ({
879
+ key: `role:${encodeURIComponent(completion.taskId)}/${encodeURIComponent(completion.roleName)}`,
880
+ at: Date.parse(completion.dueAt)
881
+ }))
882
+ ];
883
+ const nearest = nearestDeadlineBatch(deadlines);
884
+ if (nearest === null)
885
+ return;
886
+ const now = this.#now().getTime();
887
+ // Preserve an upcoming semantic deadline even while another pass backs
888
+ // off. Once that deadline has fired, the bounded pass retry owns failures
889
+ // so an overdue record cannot create a zero-delay hot loop.
890
+ if (nearest.at <= now && this.#passRetryAttempt > 0)
891
+ return;
892
+ const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(0, nearest.at - now));
893
+ this.#deadlineTimer = setTimeout(() => {
894
+ this.#deadlineTimer = undefined;
895
+ for (const key of nearest.keys)
896
+ this.#signalScheduler.signal(key);
897
+ void this.#signalScheduler.drain().catch(this.#onError);
898
+ }, delayMs);
899
+ this.#deadlineTimer.unref();
900
+ }
901
+ #scheduleDeliveryRetries(result) {
902
+ const retry = new Map();
903
+ const settled = new Set();
904
+ const resignal = new Set();
905
+ for (const delivery of result.activeRunDeliveries) {
906
+ const key = `role:${encodeURIComponent(delivery.taskId)}/${encodeURIComponent(delivery.roleName)}`;
907
+ if (delivery.terminalized === true) {
908
+ settled.add(key);
909
+ resignal.add(key);
910
+ }
911
+ else if (delivery.reason === "not-ready"
912
+ || delivery.reason === "runtime-unavailable"
913
+ || delivery.reason === "delivery-uncertain") {
914
+ retry.set(key, {
915
+ identity: delivery.runId,
916
+ ...(delivery.terminalFailure === undefined
917
+ ? {}
918
+ : { terminalFailure: delivery.terminalFailure })
919
+ });
920
+ }
921
+ else if (delivery.status === "delivered" || delivery.status === "already-delivered")
922
+ settled.add(key);
923
+ }
924
+ for (const wakeup of result.wakeups) {
925
+ const key = `role:${encodeURIComponent(wakeup.taskId)}/leader`;
926
+ if (wakeup.reason === "not-ready"
927
+ || wakeup.reason === "delivery-uncertain") {
928
+ retry.set(key, { identity: wakeup.runId ?? key });
929
+ }
930
+ else if (wakeup.status === "dispatched")
931
+ settled.add(key);
932
+ }
933
+ const operatorRetries = result.inputNotifications.filter((notification) => notification.reason === "operator-not-ready"
934
+ || (notification.reason === "operator-unavailable"
935
+ && this.#operatorStartupRetryArmed));
936
+ if (operatorRetries.length > 0) {
937
+ retry.set("operator", {
938
+ identity: operatorRetries.map((notification) => ("inputRequestId" in notification
939
+ ? `input:${notification.inputRequestId}`
940
+ : "recoveryTaskId" in notification
941
+ ? `recovery:${notification.recoveryTaskId}`
942
+ : "stallTaskId" in notification
943
+ ? `stall:${notification.stallTaskId}`
944
+ : `terminal:${notification.terminalTaskId}`)).join("|")
945
+ });
946
+ }
947
+ else if (result.inputNotifications.some((notification) => notification.status === "sent"
948
+ || notification.status === "already-sent")) {
949
+ this.#operatorStartupRetryArmed = false;
950
+ settled.add("operator");
951
+ }
952
+ for (const key of settled)
953
+ this.#clearDeliveryRetry(key);
954
+ for (const key of resignal)
955
+ this.signal(key);
956
+ for (const [key, candidate] of retry) {
957
+ this.#scheduleDeliveryRetry(key, candidate.identity, candidate.terminalFailure);
958
+ }
959
+ }
960
+ #scheduleRuntimeCleanupRetries(outcomes) {
961
+ for (const outcome of outcomes) {
962
+ const key = runtimeCleanupMailboxKey(outcome.target);
963
+ const identity = `runtime-cleanup:${outcome.batchId}`;
964
+ if (outcome.status === "failed") {
965
+ if (outcome.error !== undefined)
966
+ this.#onError(outcome.error);
967
+ this.#scheduleDeliveryRetry(key, identity);
968
+ continue;
969
+ }
970
+ if (this.#deliveryRetryAttempts.get(key)?.identity === identity) {
971
+ this.#clearDeliveryRetry(key);
972
+ }
973
+ }
974
+ }
975
+ #scheduleTaskMailboxRetries(scope) {
976
+ const selection = compileReconcileSelection(scope);
977
+ const targets = selection.full
978
+ ? this.store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
979
+ : [...selection.allRoleTaskIds].map((taskId) => ({ kind: "task", taskId }));
980
+ for (const target of targets) {
981
+ const key = `task:${encodeURIComponent(target.taskId)}`;
982
+ const mailbox = this.store.getWorkMailbox(target);
983
+ const batch = mailbox?.pending ?? mailbox?.processing?.batch;
984
+ if (batch === undefined || batch === null) {
985
+ this.#clearDeliveryRetry(key);
986
+ continue;
987
+ }
988
+ this.#scheduleDeliveryRetry(key, `${batch.fromSequence}-${batch.toSequence}`);
989
+ }
990
+ }
991
+ #scheduleDeliveryRetry(key, identity, terminalFailure) {
992
+ let previous = this.#deliveryRetryAttempts.get(key);
993
+ if (previous !== undefined && previous.identity !== identity) {
994
+ this.#clearDeliveryRetry(key);
995
+ previous = undefined;
996
+ }
997
+ const stableTerminalFailure = previous?.terminalFailure ?? terminalFailure;
998
+ if (this.#deliveryRetryTimers.has(key)) {
999
+ if (previous !== undefined && stableTerminalFailure !== previous.terminalFailure) {
1000
+ this.#deliveryRetryAttempts.set(key, {
1001
+ ...previous,
1002
+ terminalFailure: stableTerminalFailure
1003
+ });
1004
+ }
1005
+ return;
1006
+ }
1007
+ const attempts = previous?.attempts ?? 0;
1008
+ if (attempts >= this.#deliveryRetryLimit) {
1009
+ if (key === "operator")
1010
+ this.#operatorStartupRetryArmed = false;
1011
+ this.#terminalizePreparedAfterRetryExhaustion(key, identity, stableTerminalFailure);
1012
+ return;
1013
+ }
1014
+ this.#deliveryRetryAttempts.set(key, {
1015
+ identity,
1016
+ attempts: attempts + 1,
1017
+ ...(stableTerminalFailure === undefined
1018
+ ? {}
1019
+ : { terminalFailure: stableTerminalFailure })
1020
+ });
1021
+ const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
1022
+ const timer = setTimeout(() => {
1023
+ this.#deliveryRetryTimers.delete(key);
1024
+ if (!this.#stopped)
1025
+ this.signal(key);
1026
+ }, delayMs);
1027
+ timer.unref();
1028
+ this.#deliveryRetryTimers.set(key, timer);
1029
+ }
1030
+ #terminalizePreparedAfterRetryExhaustion(key, runId, failure) {
1031
+ if (!key.startsWith("role:")
1032
+ || runId.startsWith("runtime-cleanup:")
1033
+ || failure === undefined) {
1034
+ return;
1035
+ }
1036
+ const target = parseMailboxKey(key);
1037
+ if (target.kind !== "role")
1038
+ return;
1039
+ if (target.taskId !== failure.taskId
1040
+ || target.roleName !== failure.roleName
1041
+ || runId !== failure.runId) {
1042
+ throw new Error(`Role delivery retry identity changed: ${key}/${runId}.`);
1043
+ }
1044
+ const result = this.store.saveRoleRunDeliveryFailure({
1045
+ ...failure,
1046
+ now: this.#now()
1047
+ });
1048
+ this.#clearDeliveryRetry(key);
1049
+ if (result !== "failed")
1050
+ return;
1051
+ this.delivery.forgetPrepared?.({
1052
+ taskId: failure.taskId,
1053
+ roleName: failure.roleName,
1054
+ runId: failure.runId,
1055
+ ...(failure.launchId === undefined
1056
+ ? {}
1057
+ : { launchId: failure.launchId })
1058
+ });
1059
+ if (!this.#stopped)
1060
+ this.signal(key);
1061
+ }
1062
+ #clearEmptyOperatorRetry() {
1063
+ if (!this.#operatorStartupRetryArmed
1064
+ && !this.#deliveryRetryAttempts.has("operator"))
1065
+ return;
1066
+ const mailbox = this.store.getWorkMailbox({ kind: "operator" });
1067
+ if (mailbox === null
1068
+ || (mailbox.pending === null && mailbox.processing === null)) {
1069
+ this.#operatorStartupRetryArmed = false;
1070
+ this.#clearDeliveryRetry("operator");
1071
+ }
1072
+ }
1073
+ #clearDeliveryRetry(key) {
1074
+ const timer = this.#deliveryRetryTimers.get(key);
1075
+ if (timer !== undefined)
1076
+ clearTimeout(timer);
1077
+ this.#deliveryRetryTimers.delete(key);
1078
+ this.#deliveryRetryAttempts.delete(key);
1079
+ }
106
1080
  }
107
1081
  /**
108
1082
  * Starts the single private Unix-socket Controller for one YUI_HOME. The
109
1083
  * shared server owns status/stop and rejects a second live instance; this
110
- * layer adds only scheduler.scan plus an optional command dispatcher.
1084
+ * layer adds scheduler.signal/scheduler.scan plus an optional command dispatcher.
111
1085
  */
112
1086
  export async function startFileTaskController(home, store, delivery, dispatcher, options = {}) {
113
1087
  const runtime = new FileTaskController(store, delivery, options);
1088
+ let stopping = false;
1089
+ const lifecycleRequests = new Set();
114
1090
  const server = await startControllerServer(home, async (method, params) => {
1091
+ if (stopping) {
1092
+ throw controllerApplicationError("METHOD_NOT_FOUND", "Controller is stopping.");
1093
+ }
1094
+ if (method === "scheduler.signal") {
1095
+ runtime.signal(signalMailboxKey(params));
1096
+ return { accepted: true };
1097
+ }
115
1098
  if (method === "scheduler.scan") {
116
1099
  if (!isEmptyJsonObject(params)) {
117
1100
  throw controllerApplicationError("INVALID_PARAMS", "scheduler.scan params are invalid.");
118
1101
  }
119
1102
  return schedulerResultJson(await runtime.pump());
120
1103
  }
1104
+ if (method === "scheduler.configure") {
1105
+ requireEmptySchedulerConfigureParams(params);
1106
+ const intervalMs = runtime.reloadReconciliationInterval();
1107
+ return { configured: true, reconciliationIntervalMs: intervalMs };
1108
+ }
121
1109
  if (dispatcher === undefined) {
122
1110
  throw controllerApplicationError("METHOD_NOT_FOUND", "Controller method was not found.");
123
1111
  }
124
- return dispatcher(method, params);
1112
+ const request = Promise.resolve(dispatcher(method, params));
1113
+ lifecycleRequests.add(request);
1114
+ try {
1115
+ const result = await request;
1116
+ if (method === "runtime.ensure-role-session"
1117
+ && isGlobalOperatorSessionRequest(params)
1118
+ && isStartedRuntimeSessionResult(result)) {
1119
+ runtime.armOperatorStartupRetry();
1120
+ }
1121
+ return result;
1122
+ }
1123
+ finally {
1124
+ lifecycleRequests.delete(request);
1125
+ }
1126
+ }, async () => {
1127
+ stopping = true;
1128
+ runtime.stop();
1129
+ await Promise.allSettled([...lifecycleRequests]);
1130
+ await runtime.shutdownAndDrain();
1131
+ }, {
1132
+ domainIdentity: options.domainIdentity
125
1133
  });
126
1134
  runtime.start();
127
- const closed = server.closed.finally(() => runtime.stop());
1135
+ const closed = server.closed;
128
1136
  return {
129
1137
  runtime,
130
1138
  server,
@@ -132,9 +1140,34 @@ export async function startFileTaskController(home, store, delivery, dispatcher,
132
1140
  close: async () => {
133
1141
  runtime.stop();
134
1142
  await server.close();
1143
+ await runtime.shutdownAndDrain();
135
1144
  }
136
1145
  };
137
1146
  }
1147
+ function signalMailboxKey(value) {
1148
+ if (typeof value !== "object"
1149
+ || value === null
1150
+ || Array.isArray(value)) {
1151
+ throw controllerApplicationError("INVALID_PARAMS", "scheduler.signal params are invalid.");
1152
+ }
1153
+ const record = value;
1154
+ if (Object.keys(record).length !== 1
1155
+ || typeof record.key !== "string") {
1156
+ throw controllerApplicationError("INVALID_PARAMS", "scheduler.signal params are invalid.");
1157
+ }
1158
+ try {
1159
+ parseMailboxKey(record.key);
1160
+ return record.key;
1161
+ }
1162
+ catch {
1163
+ throw controllerApplicationError("INVALID_PARAMS", "scheduler.signal params are invalid.");
1164
+ }
1165
+ }
1166
+ function requireEmptySchedulerConfigureParams(value) {
1167
+ if (!isEmptyJsonObject(value)) {
1168
+ throw controllerApplicationError("INVALID_PARAMS", "scheduler.configure params are invalid.");
1169
+ }
1170
+ }
138
1171
  function positiveInteger(value, fallback, label) {
139
1172
  const resolved = value ?? fallback;
140
1173
  if (!Number.isSafeInteger(resolved) || resolved <= 0) {
@@ -142,6 +1175,17 @@ function positiveInteger(value, fallback, label) {
142
1175
  }
143
1176
  return resolved;
144
1177
  }
1178
+ function operatorMailboxBatchIdentity(mailbox) {
1179
+ const batch = mailbox?.pending ?? mailbox?.processing?.batch;
1180
+ if (batch === null || batch === undefined)
1181
+ return null;
1182
+ return [
1183
+ batch.fromSequence,
1184
+ batch.toSequence,
1185
+ batch.firstQueuedAt,
1186
+ batch.lastQueuedAt
1187
+ ].join(":");
1188
+ }
145
1189
  function schedulerResultJson(result) {
146
1190
  return JSON.parse(JSON.stringify(result));
147
1191
  }
@@ -151,6 +1195,19 @@ function isEmptyJsonObject(value) {
151
1195
  && !Array.isArray(value)
152
1196
  && Object.keys(value).length === 0;
153
1197
  }
1198
+ function isGlobalOperatorSessionRequest(value) {
1199
+ return typeof value === "object"
1200
+ && value !== null
1201
+ && !Array.isArray(value)
1202
+ && value.scope === "global"
1203
+ && value.roleName === "operator";
1204
+ }
1205
+ function isStartedRuntimeSessionResult(value) {
1206
+ return typeof value === "object"
1207
+ && value !== null
1208
+ && !Array.isArray(value)
1209
+ && value.sessionStarted === true;
1210
+ }
154
1211
  function controllerApplicationError(code, message) {
155
1212
  const error = Object.assign(new Error(message), { code });
156
1213
  error.name = "CoreApplicationError";