@zq-silk/yui 0.2.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +603 -133
- package/README.md +806 -31
- package/dist/agent/agent.js +2 -1
- package/dist/agent/argumentPolicy.js +3 -1
- package/dist/agent/launchEnvironment.js +106 -0
- package/dist/agent/managedRuntimeEnvironment.js +34 -0
- package/dist/brief/taskBrief.js +11 -1
- package/dist/cli/agentConfigurationPicker.js +287 -0
- package/dist/cli/commandCatalog.js +488 -60
- package/dist/cli/completion.js +146 -22
- package/dist/cli/helpRenderer.js +3 -1
- package/dist/cli/interactionCandidates.js +53 -15
- package/dist/cli/interactionPolicy.js +267 -30
- package/dist/cli/interactiveSelection.js +6 -2
- package/dist/cli/invocationRouter.js +5 -1
- package/dist/cli/operatorWizard.js +87 -0
- package/dist/cli/roleOptionCatalog.js +1 -0
- package/dist/cli/roleWizard.js +185 -21
- package/dist/cli/updateCommand.js +62 -19
- package/dist/cli/updateOrchestrator.js +539 -0
- package/dist/cli/updatePorts.js +1119 -0
- package/dist/cli/upgradeCommand.js +112 -0
- package/dist/cli.js +1420 -86
- package/dist/commands/agentCommands.js +146 -3
- package/dist/commands/configCommands.js +126 -0
- package/dist/commands/controllerCommands.js +365 -0
- package/dist/commands/globalRoleCommands.js +168 -126
- package/dist/commands/jobCommands.js +18 -8
- package/dist/commands/operatorCommands.js +159 -9
- package/dist/commands/profileCommands.js +203 -0
- package/dist/commands/projectCommands.js +650 -0
- package/dist/commands/roleConfiguration.js +85 -24
- package/dist/commands/roleRuntimeGuard.js +12 -0
- package/dist/commands/roleSkillValidation.js +47 -0
- package/dist/commands/taskActor.js +127 -0
- package/dist/commands/taskCommands.js +4201 -313
- package/dist/commands/taskCompletionGate.js +131 -0
- package/dist/commands/taskContextCommand.js +244 -30
- package/dist/commands/taskInputCommands.js +177 -59
- package/dist/commands/taskIntegrationCommands.js +303 -0
- package/dist/commands/taskOverviewCommand.js +363 -0
- package/dist/commands/taskRoleRuntimeStatus.js +125 -19
- package/dist/commands/textInput.js +15 -0
- package/dist/completion/completionInstaller.js +26 -22
- package/dist/config/yuiConfig.js +4 -3
- package/dist/context/dispatchContext.js +90 -38
- package/dist/context/roleSessionContext.js +119 -0
- package/dist/controller/claudeLifecycleHook.js +203 -0
- package/dist/controller/clientRuntime.js +408 -56
- package/dist/controller/codexLifecycleHook.js +108 -0
- package/dist/controller/controller.js +1089 -32
- package/dist/controller/domainIdentity.js +505 -0
- package/dist/controller/ephemeralResourceReaper.js +131 -0
- package/dist/controller/fileSchedulerStoreAdapter.js +2153 -103
- package/dist/controller/providerHookRunFence.js +127 -0
- package/dist/controller/resourceCleanupLinux.js +286 -0
- package/dist/controller/resourceInventory.js +531 -0
- package/dist/controller/resourceInventoryLinux.js +610 -0
- package/dist/controller/runtime.js +629 -10
- package/dist/controller/runtimeEventInbox.js +564 -0
- package/dist/controller/runtimeEventProcessor.js +248 -0
- package/dist/controller/runtimeLaunchCoordinator.js +477 -0
- package/dist/controller/sessionNotify.js +121 -78
- package/dist/coordination/deadlineScheduler.js +15 -0
- package/dist/coordination/mailboxScheduler.js +108 -0
- package/dist/coordination/workMailbox.js +329 -0
- package/dist/coordination/workMailboxQueue.js +86 -0
- package/dist/core/controllerClient.js +19 -5
- package/dist/core/controllerEndpoint.js +37 -0
- package/dist/core/controllerServer.js +218 -10
- package/dist/core/protocol.js +6 -2
- package/dist/decision/decision.js +2 -1
- package/dist/doctor/doctor.js +681 -32
- package/dist/domain/validation.js +53 -0
- package/dist/errors/cliError.js +5 -3
- package/dist/event/taskEvent.js +7 -3
- package/dist/execution/codexThreadNaming.js +160 -0
- package/dist/execution/executionGroup.js +579 -0
- package/dist/executor/agentAdapter.js +255 -40
- package/dist/executor/agentConfigurationCatalog.js +326 -0
- package/dist/executor/agentConfigurationProbe.js +506 -0
- package/dist/executor/agentExecutor.js +625 -10
- package/dist/executor/codexConfigConflict.js +290 -0
- package/dist/executor/effectiveLaunch.js +340 -0
- package/dist/executor/executorRegistry.js +238 -36
- package/dist/executor/fileRoleLaunchPlanner.js +550 -40
- package/dist/executor/turnCompletion.js +126 -0
- package/dist/input/inputRequest.js +30 -9
- package/dist/integration/changeSet.js +36 -0
- package/dist/integration/checkResult.js +24 -0
- package/dist/integration/gitIntegrationService.js +695 -0
- package/dist/integration/integrationAttempt.js +142 -0
- package/dist/interaction/operatorPresentation.js +96 -0
- package/dist/lifecycle/canonicalLifecycleEvent.js +342 -0
- package/dist/lifecycle/exactRunTerminalization.js +572 -0
- package/dist/lifecycle/providerLifecycleMapping.js +190 -0
- package/dist/lifecycle/taskRoleSessionReset.js +124 -0
- package/dist/message/message.js +23 -7
- package/dist/milestone/milestone.js +2 -1
- package/dist/operator/operatorSessionHistory.js +124 -0
- package/dist/output/agentConfigurationPresentation.js +43 -0
- package/dist/output/rolePresentation.js +34 -10
- package/dist/output/terminal.js +8 -0
- package/dist/output/timePresentation.js +55 -0
- package/dist/profile/agentProfile.js +128 -0
- package/dist/repository/gitWorkspace.js +578 -24
- package/dist/repository/project.js +213 -0
- package/dist/repository/taskWorkspaceCoordinator.js +392 -0
- package/dist/repository/taskWorkspacePreparer.js +1688 -191
- package/dist/review/reviewConfig.js +11 -0
- package/dist/review/reviewRound.js +399 -0
- package/dist/review/taskFinalReviewContract.js +90 -0
- package/dist/role/role.js +124 -23
- package/dist/run/agentRun.js +155 -12
- package/dist/run/runIdentity.js +82 -0
- package/dist/runtime/exactControlPlane.js +472 -0
- package/dist/runtime/index.js +8 -0
- package/dist/runtime/lifecycleReservation.js +38 -0
- package/dist/runtime/ports.js +11 -0
- package/dist/runtime/preallocatedNativeSession.js +13 -0
- package/dist/runtime/promptEnvelope.js +30 -0
- package/dist/runtime/runtimeBinding.js +31 -0
- package/dist/runtime/runtimeOwner.js +14 -0
- package/dist/runtime/sessionLaunchRequest.js +62 -0
- package/dist/runtime/sessionTitle.js +54 -0
- package/dist/runtime/taskRuntimeIsolation.js +643 -0
- package/dist/runtime/tmuxAdapters.js +315 -0
- package/dist/runtime/turnCompletion.js +3 -0
- package/dist/runtime/validation.js +23 -0
- package/dist/scheduler/activeRoleRunDelivery.js +342 -32
- package/dist/scheduler/activeTaskProgress.js +63 -0
- package/dist/scheduler/leaderFailure.js +2 -1
- package/dist/scheduler/leaderWakeupProcessor.js +307 -66
- package/dist/scheduler/operatorInputNotificationProcessor.js +109 -46
- package/dist/scheduler/operatorNotification.js +44 -2
- package/dist/scheduler/ports.js +28 -1
- package/dist/scheduler/roleRunLiveness.js +131 -25
- package/dist/scheduler/roleRunStall.js +951 -0
- package/dist/scheduler/taskExecutionProjection.js +544 -0
- package/dist/scheduler/wakeupQueue.js +3 -0
- package/dist/setup/setupCommand.js +302 -52
- package/dist/storage/compatibleTaskStore.js +102 -0
- package/dist/storage/migration/baseline.js +78 -0
- package/dist/storage/migration/classifier.js +51 -0
- package/dist/storage/migration/compatibleCodec.js +53 -0
- package/dist/storage/migration/engine.js +147 -0
- package/dist/storage/migration/index.js +33 -0
- package/dist/storage/migration/planner.js +154 -0
- package/dist/storage/migration/productionRegistry.js +486 -0
- package/dist/storage/migration/registry.js +169 -0
- package/dist/storage/migration/report.js +54 -0
- package/dist/storage/migration/types.js +31 -0
- package/dist/storage/storageSchema.js +147 -123
- package/dist/storage/storageVersions.js +11 -0
- package/dist/storage/taskStore.js +1793 -197
- package/dist/storage/upgrade/homeClassification.js +156 -0
- package/dist/storage/upgrade/homeMigrationTarget.js +595 -0
- package/dist/storage/upgrade/offlineUpgradeInventory.js +315 -0
- package/dist/storage/upgrade/productionMigrationRegistry.js +6 -0
- package/dist/storage/upgrade/recordVersionScan.js +176 -0
- package/dist/storage/upgrade/recordVersions.js +159 -0
- package/dist/storage/upgrade/switchProgress.js +80 -0
- package/dist/storage/upgrade/upgradeOrchestrator.js +948 -0
- package/dist/storage/upgrade/upgradeReceipt.js +161 -0
- package/dist/storage/upgradeCoordination.js +186 -0
- package/dist/storage/upgradeFence.js +366 -0
- package/dist/task/task.js +132 -26
- package/dist/task/taskRecordReference.js +66 -0
- package/dist/tmux/commandExecutor.js +75 -2
- package/dist/tmux/tmuxManager.js +747 -49
- package/dist/version.js +23 -0
- package/dist/web/assets/assetManifest.js +62 -0
- package/dist/web/assets/client/app.js +631 -0
- package/dist/web/assets/client/components.js +605 -0
- package/dist/web/assets/client/dom.js +14 -0
- package/dist/web/assets/client/format.js +28 -0
- package/dist/web/assets/client/i18n.js +494 -0
- package/dist/web/assets/client/markdown.js +114 -0
- package/dist/web/assets/client/theme.js +32 -0
- package/dist/web/assets/client/view.js +458 -0
- package/dist/web/assets/fontData.js +12 -0
- package/dist/web/assets/fonts.js +12 -0
- package/dist/web/assets/shell.js +114 -0
- package/dist/web/assets/styles/cards.js +135 -0
- package/dist/web/assets/styles/layout.js +47 -0
- package/dist/web/assets/styles/markdown.js +29 -0
- package/dist/web/assets/styles/responsive.js +39 -0
- package/dist/web/assets/styles/tokens.js +101 -0
- package/dist/web/assets/styles/widgets.js +147 -0
- package/dist/web/tmuxWebTerminal.js +158 -0
- package/dist/web/webServer.js +463 -0
- package/dist/web/webSnapshot.js +148 -0
- package/dist/workItem/workItem.js +642 -23
- package/dist/workspace/gitChangeSetCapture.js +86 -0
- package/dist/workspace/workItemChangeSetManager.js +445 -0
- package/dist/worktree/managedWorkspace.js +202 -0
- package/docs/task-local-identity.md +62 -0
- package/i18n/README.zh-CN.md +406 -31
- package/package.json +10 -2
- package/skills/yui-leader/SKILL.md +601 -39
- package/skills/yui-operator/SKILL.md +255 -34
- package/skills/yui-reviewer/SKILL.md +57 -0
- package/skills/yui-worker/SKILL.md +214 -17
- package/dist/commands/repositoryCommands.js +0 -86
- package/dist/operator/operatorContext.js +0 -66
- package/dist/repository/repository.js +0 -55
- package/dist/scheduler/archivedTaskRuntime.js +0 -12
- package/dist/worktree/roleWorkspace.js +0 -62
|
@@ -1,33 +1,535 @@
|
|
|
1
1
|
import { reconciliationIntervalMilliseconds } from "../config/yuiConfig.js";
|
|
2
2
|
import { processLeaderWakeups } from "../scheduler/leaderWakeupProcessor.js";
|
|
3
3
|
import { processActiveRoleRunDeliveries } from "../scheduler/activeRoleRunDelivery.js";
|
|
4
|
-
import {
|
|
4
|
+
import { selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
|
|
5
5
|
import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
|
|
6
|
+
import { DEFAULT_STALL_WINDOW_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
|
|
7
|
+
import { repairOrphanedActiveTasks } from "../scheduler/activeTaskProgress.js";
|
|
6
8
|
import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
|
|
7
9
|
import { startControllerServer } from "../core/controllerServer.js";
|
|
10
|
+
import { MailboxScheduler } from "../coordination/mailboxScheduler.js";
|
|
11
|
+
import { nearestDeadlineBatch } from "../coordination/deadlineScheduler.js";
|
|
12
|
+
import { hasRuntimeCleanupObligation, isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
13
|
+
import { formatTaskRecordReference } from "../task/taskRecordReference.js";
|
|
8
14
|
const DEFAULT_RECONCILIATION_INTERVAL_MS = reconciliationIntervalMilliseconds();
|
|
15
|
+
const DEFAULT_SIGNAL_WINDOW_MS = 100;
|
|
16
|
+
const DEFAULT_DELIVERY_RETRY_MS = 250;
|
|
17
|
+
const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
|
|
18
|
+
const RUNTIME_RESERVATION_RECOVERY_AGE_MS = 120_000;
|
|
19
|
+
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
20
|
+
class RuntimeEventApplyError extends AggregateError {
|
|
21
|
+
}
|
|
9
22
|
/**
|
|
10
|
-
* Runs one lean scheduler pass.
|
|
11
|
-
*
|
|
23
|
+
* Runs one lean scheduler pass. Due native Turn completions are folded before
|
|
24
|
+
* liveness, so a valid Hook boundary fences destructive process reconciliation.
|
|
12
25
|
*/
|
|
13
|
-
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer) {
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const
|
|
26
|
+
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, resourceSuppressionKeys = new Set()) {
|
|
27
|
+
const compiledSelection = compileReconcileSelection(scope);
|
|
28
|
+
const selection = includeOperator
|
|
29
|
+
? compiledSelection
|
|
30
|
+
: { ...compiledSelection, operator: false };
|
|
31
|
+
queueSelectedCompletedTaskRuntimeCleanups(store, selection, now);
|
|
32
|
+
const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes);
|
|
33
|
+
const roleSelection = selectionWithoutFailedCleanupRoles(store, selection, failedCleanupRoles);
|
|
34
|
+
const wakeupSelection = selectionWithoutFailedLeaderCleanupTasks(store, selection, failedCleanupRoles);
|
|
35
|
+
if (selection.full)
|
|
36
|
+
repairOrphanedActiveTasks(store, now, selection);
|
|
37
|
+
const claimedTaskMailboxes = claimSelectedTaskMailboxes(store, selection, now);
|
|
38
|
+
try {
|
|
39
|
+
const failedTaskMailboxes = await prepareActiveWorkspaces(store, workspacePreparer, selection);
|
|
40
|
+
const activeRunDeliveries = await processActiveRoleRunDeliveries(store, delivery, now, roleSelection);
|
|
41
|
+
const unsettledRunRefs = new Set(activeRunDeliveries.flatMap((result) => (result.reason === "delivery-uncertain" || result.terminalFailure !== undefined
|
|
42
|
+
? [formatTaskRecordReference(result.taskId, result.runId, "agentRun")]
|
|
43
|
+
: [])));
|
|
44
|
+
resolveDueRuntimeTurnCompletions(store, delivery, selection, now);
|
|
45
|
+
const liveStatuses = new Map();
|
|
46
|
+
const resourceEvidence = new Map();
|
|
47
|
+
const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence);
|
|
48
|
+
await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, resourceSuppressionKeys);
|
|
49
|
+
await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now);
|
|
50
|
+
const autoResolvedInputs = selection.full
|
|
51
|
+
? store.resolveExpiredInputRecommendations(now)
|
|
52
|
+
: selection.taskIds.size === 0
|
|
53
|
+
? []
|
|
54
|
+
: store.resolveExpiredInputRecommendations(now, selection.taskIds);
|
|
55
|
+
const wakeups = await processLeaderWakeups(store, delivery, now, wakeupSelection);
|
|
56
|
+
const inputNotifications = includeOperator
|
|
57
|
+
? await processOperatorInputNotifications(store, delivery, selection, now)
|
|
58
|
+
: [];
|
|
59
|
+
for (const claim of claimedTaskMailboxes) {
|
|
60
|
+
if (failedTaskMailboxes.has(claim.target.taskId)) {
|
|
61
|
+
store.releaseWorkMailbox(claim.target, claim.processing.batchId);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
store.completeWorkMailbox(claim.target, claim.processing.batchId);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
activeRunDeliveries,
|
|
69
|
+
failedRunRefs,
|
|
70
|
+
wakeups,
|
|
71
|
+
inputNotifications,
|
|
72
|
+
autoResolvedInputs
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
for (const claim of claimedTaskMailboxes) {
|
|
77
|
+
store.releaseWorkMailbox(claim.target, claim.processing.batchId);
|
|
78
|
+
}
|
|
79
|
+
throw error;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function queueSelectedCompletedTaskRuntimeCleanups(store, selection, now) {
|
|
83
|
+
if (store.enqueueRuntimeCleanup === undefined)
|
|
84
|
+
return;
|
|
85
|
+
for (const task of selectedSchedulerTasks(store, selection)) {
|
|
86
|
+
if (task.status !== "completed")
|
|
87
|
+
continue;
|
|
88
|
+
for (const role of selectedSchedulerRoles(store, task.id, selection)) {
|
|
89
|
+
if (store.getActiveAgentRun(task.id, role.name) !== null)
|
|
90
|
+
continue;
|
|
91
|
+
const session = store.getRoleSession(task.id, role.name);
|
|
92
|
+
if (!schedulerSessionRequiresRuntimeCleanup(session))
|
|
93
|
+
continue;
|
|
94
|
+
const target = runtimeLifecycleTarget({
|
|
95
|
+
scope: "task",
|
|
96
|
+
taskId: task.id,
|
|
97
|
+
roleName: role.name
|
|
98
|
+
});
|
|
99
|
+
if (hasRuntimeCleanupObligation(store.getWorkMailbox(target)))
|
|
100
|
+
continue;
|
|
101
|
+
store.enqueueRuntimeCleanup({
|
|
102
|
+
scope: "task",
|
|
103
|
+
taskId: task.id,
|
|
104
|
+
roleName: role.name
|
|
105
|
+
}, now);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
function schedulerSessionRequiresRuntimeCleanup(session) {
|
|
110
|
+
if (session === null || session.status === "stopped" || session.status === "broken") {
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
return session.status === "running" || session.launchId !== undefined;
|
|
114
|
+
}
|
|
115
|
+
async function processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, outcomes) {
|
|
116
|
+
const targets = selectedRuntimeLifecycleTargets(store, scope);
|
|
117
|
+
const failedRoles = new Set();
|
|
118
|
+
for (const target of targets) {
|
|
119
|
+
const mailbox = store.getWorkMailbox(target);
|
|
120
|
+
if (mailbox === null)
|
|
121
|
+
continue;
|
|
122
|
+
const owner = runtimeOwner(target);
|
|
123
|
+
const batchId = runtimeLifecycleBatchIdentity(target, mailbox);
|
|
124
|
+
if (hasRuntimeCleanupObligation(mailbox)) {
|
|
125
|
+
try {
|
|
126
|
+
if (lifecycleHost === undefined) {
|
|
127
|
+
throw new Error("Role runtime cleanup host is unavailable.");
|
|
128
|
+
}
|
|
129
|
+
if (!await lifecycleHost.stopOwner(owner)) {
|
|
130
|
+
throw new Error(`Role runtime cleanup could not confirm the host stopped: ${runtimeOwnerLabel(owner)}.`);
|
|
131
|
+
}
|
|
132
|
+
if (target.kind === "role-runtime") {
|
|
133
|
+
const session = store.getRoleSession(target.taskId, target.roleName);
|
|
134
|
+
const reservedLaunchId = isRuntimeLaunchReservation(mailbox.processing)
|
|
135
|
+
? mailbox.processing.batchId
|
|
136
|
+
: undefined;
|
|
137
|
+
if (reservedLaunchId !== undefined
|
|
138
|
+
&& session?.launchId !== undefined
|
|
139
|
+
&& reservedLaunchId !== session.launchId) {
|
|
140
|
+
throw new Error(`Task runtime cleanup launch identity is ambiguous: ${target.taskId}.`);
|
|
141
|
+
}
|
|
142
|
+
const launchId = reservedLaunchId ?? session?.launchId;
|
|
143
|
+
if (launchId !== undefined
|
|
144
|
+
&& lifecycleHost.cleanupTaskLaunch !== undefined) {
|
|
145
|
+
const task = store.getTask(target.taskId);
|
|
146
|
+
const reason = task?.status === "completed"
|
|
147
|
+
? "completion"
|
|
148
|
+
: "interruption";
|
|
149
|
+
lifecycleHost.cleanupTaskLaunch({
|
|
150
|
+
taskId: target.taskId,
|
|
151
|
+
launchId,
|
|
152
|
+
reason
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (store.completeRuntimeCleanup === undefined
|
|
157
|
+
|| !store.completeRuntimeCleanup(target, now)) {
|
|
158
|
+
throw new Error(`Role runtime cleanup mailbox changed: ${runtimeOwnerLabel(owner)}.`);
|
|
159
|
+
}
|
|
160
|
+
forgetPreparedRuntimeOwner(delivery, owner);
|
|
161
|
+
outcomes.push({ target, batchId, status: "completed" });
|
|
162
|
+
}
|
|
163
|
+
catch (error) {
|
|
164
|
+
markFailedRuntimeTarget(failedRoles, target);
|
|
165
|
+
outcomes.push({ target, batchId, status: "failed", error });
|
|
166
|
+
}
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
const reservation = mailbox.processing;
|
|
170
|
+
if (reservation === null
|
|
171
|
+
|| !isRuntimeLaunchReservation(reservation)
|
|
172
|
+
|| now.getTime() - Date.parse(reservation.startedAt)
|
|
173
|
+
< RUNTIME_RESERVATION_RECOVERY_AGE_MS) {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
try {
|
|
177
|
+
if (lifecycleHost === undefined) {
|
|
178
|
+
throw new Error("Role runtime reservation inspection is unavailable.");
|
|
179
|
+
}
|
|
180
|
+
const inspection = await lifecycleHost.inspectOwner(owner);
|
|
181
|
+
if (inspection.state === "running" || inspection.state === "starting") {
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
if (inspection.state === "unavailable") {
|
|
185
|
+
throw new Error(`Role runtime reservation could not inspect the host: ${runtimeOwnerLabel(owner)}.`);
|
|
186
|
+
}
|
|
187
|
+
const completed = store.completeStoppedRuntimeReservation === undefined
|
|
188
|
+
? store.completeWorkMailbox(target, reservation.batchId)
|
|
189
|
+
: store.completeStoppedRuntimeReservation(target, reservation.batchId, now);
|
|
190
|
+
if (!completed) {
|
|
191
|
+
throw new Error(`Role runtime reservation mailbox changed: ${runtimeOwnerLabel(owner)}.`);
|
|
192
|
+
}
|
|
193
|
+
forgetPreparedRuntimeOwner(delivery, owner);
|
|
194
|
+
outcomes.push({ target, batchId, status: "completed" });
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
markFailedRuntimeTarget(failedRoles, target);
|
|
198
|
+
outcomes.push({ target, batchId, status: "failed", error });
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return failedRoles;
|
|
202
|
+
}
|
|
203
|
+
async function reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now) {
|
|
204
|
+
if (scope.kind !== "full"
|
|
205
|
+
|| lifecycleHost === undefined
|
|
206
|
+
|| store.listDormantRuntimeOwners === undefined
|
|
207
|
+
|| store.markRuntimeOwnerStopped === undefined) {
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
const candidates = store.listDormantRuntimeOwners();
|
|
211
|
+
if (candidates.length === 0)
|
|
212
|
+
return;
|
|
213
|
+
const owners = candidates.map((candidate) => candidate.owner);
|
|
214
|
+
let inspections;
|
|
215
|
+
try {
|
|
216
|
+
inspections = lifecycleHost.inspectOwners === undefined
|
|
217
|
+
? await Promise.all(owners.map(async (owner) => ({
|
|
218
|
+
owner,
|
|
219
|
+
inspection: await lifecycleHost.inspectOwner(owner)
|
|
220
|
+
})))
|
|
221
|
+
: await lifecycleHost.inspectOwners(owners);
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
// Host inventory is an advisory safety scan. Unknown state must never
|
|
225
|
+
// mutate persisted session facts; the next full pass will retry.
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const requested = new Set(owners.map(runtimeOwnerIdentity));
|
|
229
|
+
const byOwner = new Map();
|
|
230
|
+
for (const result of inspections) {
|
|
231
|
+
const identity = runtimeOwnerIdentity(result.owner);
|
|
232
|
+
if (!requested.has(identity)
|
|
233
|
+
|| byOwner.has(identity)) {
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
byOwner.set(identity, result.inspection);
|
|
237
|
+
}
|
|
238
|
+
if (byOwner.size !== requested.size)
|
|
239
|
+
return;
|
|
240
|
+
for (const candidate of candidates) {
|
|
241
|
+
if (byOwner.get(runtimeOwnerIdentity(candidate.owner))?.state
|
|
242
|
+
=== "stopped") {
|
|
243
|
+
if (candidate.owner.scope === "task"
|
|
244
|
+
&& candidate.launchId !== undefined) {
|
|
245
|
+
// The exact launch-owned resources must settle through the durable
|
|
246
|
+
// cleanup lane before its Session fact becomes stopped. The candidate
|
|
247
|
+
// is passed back as the CAS fence so a concurrent Hook/launch cannot
|
|
248
|
+
// redirect cleanup to a newer generation.
|
|
249
|
+
store.enqueueRuntimeCleanup?.(candidate.owner, now, candidate);
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
if (store.markRuntimeOwnerStopped(candidate, now)) {
|
|
253
|
+
forgetPreparedRuntimeOwner(delivery, candidate.owner);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
function forgetPreparedRuntimeOwner(delivery, owner) {
|
|
259
|
+
if (owner.scope !== "task")
|
|
260
|
+
return;
|
|
261
|
+
delivery.forgetPrepared?.({
|
|
262
|
+
taskId: owner.taskId,
|
|
263
|
+
roleName: owner.roleName
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
function runtimeOwnerIdentity(owner) {
|
|
267
|
+
return owner.scope === "task"
|
|
268
|
+
? `task\0${owner.taskId}\0${owner.roleName}`
|
|
269
|
+
: `global\0${owner.roleName}`;
|
|
270
|
+
}
|
|
271
|
+
function resolveDueRuntimeTurnCompletions(store, delivery, selection, now) {
|
|
272
|
+
if (typeof store.resolveDueRuntimeTurnCompletions !== "function")
|
|
273
|
+
return;
|
|
274
|
+
const selectedTaskIds = selection.full ? undefined : selection.taskIds;
|
|
275
|
+
if (selectedTaskIds?.size === 0)
|
|
276
|
+
return;
|
|
277
|
+
const candidates = store.listPendingRuntimeTurnCompletions().filter((completion) => (selectedTaskIds === undefined
|
|
278
|
+
|| selectedTaskIds.has(completion.taskId)));
|
|
279
|
+
const finalized = new Set(store.resolveDueRuntimeTurnCompletions(now, selectedTaskIds));
|
|
280
|
+
if (finalized.size === 0)
|
|
281
|
+
return;
|
|
282
|
+
for (const completion of candidates) {
|
|
283
|
+
if (!finalized.has(formatTaskRecordReference(completion.taskId, completion.runId, "agentRun")))
|
|
284
|
+
continue;
|
|
285
|
+
delivery.forgetPrepared?.({
|
|
286
|
+
taskId: completion.taskId,
|
|
287
|
+
roleName: completion.roleName,
|
|
288
|
+
runId: completion.runId
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function selectedRuntimeLifecycleTargets(store, scope) {
|
|
293
|
+
if (scope.kind === "full") {
|
|
294
|
+
return store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "role-runtime"
|
|
295
|
+
|| mailbox.target.kind === "global-role-runtime"
|
|
296
|
+
? [mailbox.target]
|
|
297
|
+
: []));
|
|
298
|
+
}
|
|
299
|
+
const targets = new Map();
|
|
300
|
+
for (const key of scope.keys) {
|
|
301
|
+
const parsed = parseMailboxKey(key);
|
|
302
|
+
if (parsed.kind === "task") {
|
|
303
|
+
for (const role of store.listRoles(parsed.taskId)) {
|
|
304
|
+
const target = {
|
|
305
|
+
kind: "role-runtime",
|
|
306
|
+
taskId: parsed.taskId,
|
|
307
|
+
roleName: role.name
|
|
308
|
+
};
|
|
309
|
+
targets.set(runtimeTargetIdentity(target), target);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
else if (parsed.kind === "role") {
|
|
313
|
+
const target = {
|
|
314
|
+
kind: "role-runtime",
|
|
315
|
+
taskId: parsed.taskId,
|
|
316
|
+
roleName: parsed.roleName
|
|
317
|
+
};
|
|
318
|
+
targets.set(runtimeTargetIdentity(target), target);
|
|
319
|
+
}
|
|
320
|
+
else if (parsed.kind === "global-role") {
|
|
321
|
+
const target = {
|
|
322
|
+
kind: "global-role-runtime",
|
|
323
|
+
roleName: parsed.roleName
|
|
324
|
+
};
|
|
325
|
+
targets.set(runtimeTargetIdentity(target), target);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return [...targets.values()];
|
|
329
|
+
}
|
|
330
|
+
function runtimeOwner(target) {
|
|
331
|
+
return target.kind === "role-runtime"
|
|
332
|
+
? {
|
|
333
|
+
scope: "task",
|
|
334
|
+
taskId: target.taskId,
|
|
335
|
+
roleName: target.roleName
|
|
336
|
+
}
|
|
337
|
+
: { scope: "global", roleName: target.roleName };
|
|
338
|
+
}
|
|
339
|
+
function runtimeOwnerLabel(owner) {
|
|
340
|
+
return owner.scope === "task"
|
|
341
|
+
? `${owner.taskId}/${owner.roleName}`
|
|
342
|
+
: `global/${owner.roleName}`;
|
|
343
|
+
}
|
|
344
|
+
function runtimeTargetIdentity(target) {
|
|
345
|
+
return target.kind === "role-runtime"
|
|
346
|
+
? `task\0${target.taskId}\0${target.roleName}`
|
|
347
|
+
: `global\0${target.roleName}`;
|
|
348
|
+
}
|
|
349
|
+
function runtimeCleanupMailboxKey(target) {
|
|
350
|
+
return target.kind === "role-runtime"
|
|
351
|
+
? `role:${encodeURIComponent(target.taskId)}/${encodeURIComponent(target.roleName)}`
|
|
352
|
+
: `global-role:${encodeURIComponent(target.roleName)}`;
|
|
353
|
+
}
|
|
354
|
+
function runtimeLifecycleBatchIdentity(target, mailbox) {
|
|
355
|
+
if (mailbox.processing !== null)
|
|
356
|
+
return mailbox.processing.batchId;
|
|
357
|
+
const pending = mailbox.pending;
|
|
358
|
+
return pending === null
|
|
359
|
+
? runtimeTargetIdentity(target)
|
|
360
|
+
: `${runtimeTargetIdentity(target)}:${pending.fromSequence}-${pending.toSequence}`;
|
|
361
|
+
}
|
|
362
|
+
function markFailedRuntimeTarget(failedRoles, target) {
|
|
363
|
+
if (target.kind === "role-runtime") {
|
|
364
|
+
failedRoles.add(roleIdentity(target.taskId, target.roleName));
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
function selectionWithoutFailedCleanupRoles(store, selection, failedRoles) {
|
|
368
|
+
if (failedRoles.size === 0)
|
|
369
|
+
return selection;
|
|
370
|
+
const taskIds = selection.full
|
|
371
|
+
? new Set(store.listTasks().map((task) => task.id))
|
|
372
|
+
: new Set(selection.taskIds);
|
|
373
|
+
const rolesByTask = new Map();
|
|
374
|
+
for (const taskId of taskIds) {
|
|
375
|
+
const roleNames = selection.full || selection.allRoleTaskIds.has(taskId)
|
|
376
|
+
? store.listRoles(taskId).map((role) => role.name)
|
|
377
|
+
: [...(selection.rolesByTask.get(taskId) ?? [])];
|
|
378
|
+
rolesByTask.set(taskId, new Set(roleNames.filter((roleName) => (!failedRoles.has(roleIdentity(taskId, roleName))))));
|
|
379
|
+
}
|
|
380
|
+
return {
|
|
381
|
+
full: false,
|
|
382
|
+
taskIds,
|
|
383
|
+
allRoleTaskIds: new Set(),
|
|
384
|
+
rolesByTask,
|
|
385
|
+
operator: selection.operator
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
function selectionWithoutFailedLeaderCleanupTasks(store, selection, failedRoles) {
|
|
389
|
+
if (![...failedRoles].some((identity) => identity.endsWith("\0leader"))) {
|
|
390
|
+
return selection;
|
|
391
|
+
}
|
|
392
|
+
const taskIds = selection.full
|
|
393
|
+
? new Set(store.listTasks().map((task) => task.id))
|
|
394
|
+
: new Set(selection.taskIds);
|
|
395
|
+
for (const taskId of taskIds) {
|
|
396
|
+
if (failedRoles.has(roleIdentity(taskId, "leader")))
|
|
397
|
+
taskIds.delete(taskId);
|
|
398
|
+
}
|
|
399
|
+
return {
|
|
400
|
+
full: false,
|
|
401
|
+
taskIds,
|
|
402
|
+
allRoleTaskIds: new Set(),
|
|
403
|
+
rolesByTask: new Map(),
|
|
404
|
+
operator: selection.operator
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function roleIdentity(taskId, roleName) {
|
|
408
|
+
return `${taskId}\0${roleName}`;
|
|
409
|
+
}
|
|
410
|
+
function claimSelectedTaskMailboxes(store, selection, now) {
|
|
411
|
+
const targets = selection.full
|
|
412
|
+
? store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
|
|
413
|
+
: [...selection.allRoleTaskIds].map((taskId) => ({ kind: "task", taskId }));
|
|
414
|
+
const claims = [];
|
|
415
|
+
for (const target of targets) {
|
|
416
|
+
const mailbox = store.getWorkMailbox(target);
|
|
417
|
+
if (mailbox === null || (mailbox.processing === null && mailbox.pending === null))
|
|
418
|
+
continue;
|
|
419
|
+
const batchId = mailbox.processing?.batchId
|
|
420
|
+
?? `task:${encodeURIComponent(target.taskId)}:${mailbox.pending.fromSequence}-${mailbox.pending.toSequence}`;
|
|
421
|
+
const claim = store.claimWorkMailbox({
|
|
422
|
+
target,
|
|
423
|
+
batchId,
|
|
424
|
+
owner: "controller",
|
|
425
|
+
now
|
|
426
|
+
});
|
|
427
|
+
if (claim.status !== "empty")
|
|
428
|
+
claims.push({ target, processing: claim.processing });
|
|
429
|
+
}
|
|
430
|
+
return claims;
|
|
431
|
+
}
|
|
432
|
+
export function compileReconcileSelection(scope) {
|
|
433
|
+
if (scope.kind === "full") {
|
|
434
|
+
return {
|
|
435
|
+
full: true,
|
|
436
|
+
taskIds: new Set(),
|
|
437
|
+
allRoleTaskIds: new Set(),
|
|
438
|
+
rolesByTask: new Map(),
|
|
439
|
+
operator: true
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
const taskIds = new Set();
|
|
443
|
+
const allRoleTaskIds = new Set();
|
|
444
|
+
const mutableRoles = new Map();
|
|
445
|
+
let operator = false;
|
|
446
|
+
for (const key of scope.keys) {
|
|
447
|
+
const target = parseMailboxKey(key);
|
|
448
|
+
if (target.kind === "operator") {
|
|
449
|
+
operator = true;
|
|
450
|
+
}
|
|
451
|
+
else if (target.kind === "task") {
|
|
452
|
+
taskIds.add(target.taskId);
|
|
453
|
+
allRoleTaskIds.add(target.taskId);
|
|
454
|
+
}
|
|
455
|
+
else if (target.kind === "role") {
|
|
456
|
+
taskIds.add(target.taskId);
|
|
457
|
+
const roles = mutableRoles.get(target.taskId) ?? new Set();
|
|
458
|
+
roles.add(target.roleName);
|
|
459
|
+
mutableRoles.set(target.taskId, roles);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
22
462
|
return {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
autoResolvedInputs
|
|
463
|
+
full: false,
|
|
464
|
+
taskIds,
|
|
465
|
+
allRoleTaskIds,
|
|
466
|
+
rolesByTask: mutableRoles,
|
|
467
|
+
operator
|
|
29
468
|
};
|
|
30
469
|
}
|
|
470
|
+
async function prepareActiveWorkspaces(store, workspace, selection) {
|
|
471
|
+
if (workspace === undefined)
|
|
472
|
+
return new Set();
|
|
473
|
+
const taskIds = selection.full
|
|
474
|
+
? new Set(store.listTasks()
|
|
475
|
+
.filter((task) => task.status === "active")
|
|
476
|
+
.map((task) => task.id))
|
|
477
|
+
: new Set([...selection.taskIds, ...selection.allRoleTaskIds]);
|
|
478
|
+
const failed = new Set();
|
|
479
|
+
for (const taskId of taskIds) {
|
|
480
|
+
if (store.getTask(taskId)?.status === "active") {
|
|
481
|
+
try {
|
|
482
|
+
const result = await workspace.prepareTaskWorkspace(taskId);
|
|
483
|
+
if (result.status === "failed")
|
|
484
|
+
failed.add(taskId);
|
|
485
|
+
}
|
|
486
|
+
catch {
|
|
487
|
+
failed.add(taskId);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
return failed;
|
|
492
|
+
}
|
|
493
|
+
function parseMailboxKey(key) {
|
|
494
|
+
if (key === "operator")
|
|
495
|
+
return { kind: "operator" };
|
|
496
|
+
if (key.startsWith("task:")) {
|
|
497
|
+
return { kind: "task", taskId: mailboxPart(key.slice("task:".length), key) };
|
|
498
|
+
}
|
|
499
|
+
if (key.startsWith("global-role:")) {
|
|
500
|
+
return {
|
|
501
|
+
kind: "global-role",
|
|
502
|
+
roleName: mailboxPart(key.slice("global-role:".length), key)
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
if (key.startsWith("role:")) {
|
|
506
|
+
const value = key.slice("role:".length);
|
|
507
|
+
const separator = value.indexOf("/");
|
|
508
|
+
if (separator > 0 && separator < value.length - 1 && value.indexOf("/", separator + 1) < 0) {
|
|
509
|
+
return {
|
|
510
|
+
kind: "role",
|
|
511
|
+
taskId: mailboxPart(value.slice(0, separator), key),
|
|
512
|
+
roleName: mailboxPart(value.slice(separator + 1), key)
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
|
|
517
|
+
}
|
|
518
|
+
function mailboxPart(value, key) {
|
|
519
|
+
if (value.length === 0 || value.trim() !== value || value.includes("\0")) {
|
|
520
|
+
throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
|
|
521
|
+
}
|
|
522
|
+
try {
|
|
523
|
+
const decoded = decodeURIComponent(value);
|
|
524
|
+
if (decoded.length === 0 || decoded.includes("\0") || encodeURIComponent(decoded) !== value) {
|
|
525
|
+
throw new Error("not canonical");
|
|
526
|
+
}
|
|
527
|
+
return decoded;
|
|
528
|
+
}
|
|
529
|
+
catch {
|
|
530
|
+
throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
31
533
|
/**
|
|
32
534
|
* Single-owner periodic runtime for FileTaskStore-backed scheduling. Concurrent
|
|
33
535
|
* pump requests coalesce into one follow-up pass; scheduler effects never
|
|
@@ -40,9 +542,30 @@ export class FileTaskController {
|
|
|
40
542
|
#now;
|
|
41
543
|
#onError;
|
|
42
544
|
#workspacePreparer;
|
|
545
|
+
#deliveryRetryMs;
|
|
546
|
+
#deliveryRetryLimit;
|
|
547
|
+
#stallWindowMs;
|
|
548
|
+
/** Narrow-port fallback; FileSchedulerStoreAdapter durably records these keys. */
|
|
549
|
+
#resourceSuppressionKeys = new Set();
|
|
550
|
+
#runtimeEventProcessor;
|
|
551
|
+
#lifecycleHost;
|
|
552
|
+
#deliveryRetryAttempts = new Map();
|
|
553
|
+
#deliveryRetryTimers = new Map();
|
|
554
|
+
#passRetryTimer;
|
|
555
|
+
#passRetryAttempt = 0;
|
|
43
556
|
#timer;
|
|
557
|
+
#deadlineTimer;
|
|
558
|
+
#signalScheduler;
|
|
559
|
+
#operatorSignalScheduler;
|
|
560
|
+
#configuration;
|
|
561
|
+
#resourceReaper;
|
|
562
|
+
#onExpiredEphemeralDomain;
|
|
44
563
|
#current;
|
|
45
|
-
#
|
|
564
|
+
#operatorCurrent;
|
|
565
|
+
#pendingFull = false;
|
|
566
|
+
#pendingKeys = new Set();
|
|
567
|
+
#operatorStartupRetryArmed = false;
|
|
568
|
+
#lastOperatorSignalIdentity;
|
|
46
569
|
#stopped = false;
|
|
47
570
|
constructor(store, delivery, options = {}) {
|
|
48
571
|
this.store = store;
|
|
@@ -51,33 +574,151 @@ export class FileTaskController {
|
|
|
51
574
|
this.#now = options.now ?? (() => new Date());
|
|
52
575
|
this.#onError = options.onError ?? (() => { });
|
|
53
576
|
this.#workspacePreparer = options.workspacePreparer;
|
|
577
|
+
this.#deliveryRetryMs = positiveInteger(options.deliveryRetryMs, DEFAULT_DELIVERY_RETRY_MS, "Controller delivery retry delay");
|
|
578
|
+
this.#deliveryRetryLimit = positiveInteger(options.deliveryRetryLimit, DEFAULT_DELIVERY_RETRY_LIMIT, "Controller delivery retry limit");
|
|
579
|
+
this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
|
|
580
|
+
this.#runtimeEventProcessor = options.runtimeEventProcessor;
|
|
581
|
+
this.#lifecycleHost = options.lifecycleHost;
|
|
582
|
+
this.#configuration = options.configuration;
|
|
583
|
+
this.#resourceReaper = options.resourceReaper;
|
|
584
|
+
this.#onExpiredEphemeralDomain = options.onExpiredEphemeralDomain;
|
|
585
|
+
this.#signalScheduler = new MailboxScheduler(async (keys) => { await this.#requestPass({ kind: "dirty", keys }); }, {
|
|
586
|
+
windowMs: options.signalWindowMs ?? DEFAULT_SIGNAL_WINDOW_MS,
|
|
587
|
+
onError: this.#onError,
|
|
588
|
+
setTimer: (callback, delayMs) => {
|
|
589
|
+
const timer = setTimeout(callback, delayMs);
|
|
590
|
+
timer.unref();
|
|
591
|
+
return timer;
|
|
592
|
+
}
|
|
593
|
+
});
|
|
594
|
+
this.#operatorSignalScheduler = new MailboxScheduler(async () => {
|
|
595
|
+
const running = this.#runOperatorPass();
|
|
596
|
+
this.#operatorCurrent = running;
|
|
597
|
+
try {
|
|
598
|
+
await running;
|
|
599
|
+
}
|
|
600
|
+
finally {
|
|
601
|
+
if (this.#operatorCurrent === running)
|
|
602
|
+
this.#operatorCurrent = undefined;
|
|
603
|
+
}
|
|
604
|
+
}, {
|
|
605
|
+
windowMs: options.signalWindowMs ?? DEFAULT_SIGNAL_WINDOW_MS,
|
|
606
|
+
onError: this.#onError,
|
|
607
|
+
setTimer: (callback, delayMs) => {
|
|
608
|
+
const timer = setTimeout(callback, delayMs);
|
|
609
|
+
timer.unref();
|
|
610
|
+
return timer;
|
|
611
|
+
}
|
|
612
|
+
});
|
|
54
613
|
}
|
|
55
614
|
get reconciliationIntervalMs() {
|
|
56
615
|
return this.#intervalMs;
|
|
57
616
|
}
|
|
617
|
+
updateReconciliationInterval(intervalMs) {
|
|
618
|
+
const next = positiveInteger(intervalMs, DEFAULT_RECONCILIATION_INTERVAL_MS, "Controller reconciliation interval");
|
|
619
|
+
if (this.#stopped)
|
|
620
|
+
throw new Error("Controller runtime is stopped.");
|
|
621
|
+
if (next === this.#intervalMs)
|
|
622
|
+
return;
|
|
623
|
+
this.#intervalMs = next;
|
|
624
|
+
if (this.#timer !== undefined) {
|
|
625
|
+
clearInterval(this.#timer);
|
|
626
|
+
this.#timer = setInterval(() => {
|
|
627
|
+
this.#requestBackgroundPump();
|
|
628
|
+
}, this.#intervalMs);
|
|
629
|
+
this.#timer.unref();
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
reloadReconciliationInterval() {
|
|
633
|
+
if (this.#configuration !== undefined) {
|
|
634
|
+
this.updateReconciliationInterval(this.#configuration.reconciliationIntervalMs());
|
|
635
|
+
}
|
|
636
|
+
return this.#intervalMs;
|
|
637
|
+
}
|
|
58
638
|
start() {
|
|
59
639
|
if (this.#timer !== undefined)
|
|
60
640
|
return;
|
|
61
|
-
this.#stopped
|
|
62
|
-
|
|
641
|
+
if (this.#stopped)
|
|
642
|
+
throw new Error("Controller runtime is stopped.");
|
|
643
|
+
this.#requestBackgroundPump();
|
|
644
|
+
// Arm the Operator lane once for work that was already pending when the
|
|
645
|
+
// Controller started. Subsequent main-lane passes signal it only when a
|
|
646
|
+
// new Operator batch is durably queued; an unchanged pending batch must
|
|
647
|
+
// not keep an unavailable Operator lane in a zero-delay drain loop.
|
|
648
|
+
this.#signalOperatorMailbox();
|
|
63
649
|
this.#timer = setInterval(() => {
|
|
64
|
-
|
|
650
|
+
this.#requestBackgroundPump();
|
|
65
651
|
}, this.#intervalMs);
|
|
66
652
|
this.#timer.unref();
|
|
67
653
|
}
|
|
68
654
|
stop() {
|
|
69
655
|
this.#stopped = true;
|
|
656
|
+
this.#signalScheduler.stop();
|
|
657
|
+
this.#operatorSignalScheduler.stop();
|
|
658
|
+
if (this.#deadlineTimer !== undefined) {
|
|
659
|
+
clearTimeout(this.#deadlineTimer);
|
|
660
|
+
this.#deadlineTimer = undefined;
|
|
661
|
+
}
|
|
70
662
|
if (this.#timer !== undefined) {
|
|
71
663
|
clearInterval(this.#timer);
|
|
72
664
|
this.#timer = undefined;
|
|
73
665
|
}
|
|
666
|
+
for (const timer of this.#deliveryRetryTimers.values())
|
|
667
|
+
clearTimeout(timer);
|
|
668
|
+
this.#deliveryRetryTimers.clear();
|
|
669
|
+
this.#deliveryRetryAttempts.clear();
|
|
670
|
+
if (this.#passRetryTimer !== undefined) {
|
|
671
|
+
clearTimeout(this.#passRetryTimer);
|
|
672
|
+
this.#passRetryTimer = undefined;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
async shutdownAndDrain() {
|
|
676
|
+
this.stop();
|
|
677
|
+
await Promise.allSettled([
|
|
678
|
+
this.#current ?? Promise.resolve(),
|
|
679
|
+
this.#operatorCurrent ?? Promise.resolve()
|
|
680
|
+
]);
|
|
681
|
+
}
|
|
682
|
+
/** Adds one dirty Task key to the fixed-window wake queue and returns immediately. */
|
|
683
|
+
signal(key) {
|
|
684
|
+
if (this.#stopped)
|
|
685
|
+
throw new Error("Controller runtime is stopped.");
|
|
686
|
+
parseMailboxKey(key);
|
|
687
|
+
if (key === "operator") {
|
|
688
|
+
// Main lane owns durable Hook folding; the Operator lane is the sole
|
|
689
|
+
// consumer of the Operator delivery mailbox.
|
|
690
|
+
this.#signalScheduler.signal("operator");
|
|
691
|
+
this.#operatorSignalScheduler.signal("operator");
|
|
692
|
+
}
|
|
693
|
+
else
|
|
694
|
+
this.#signalScheduler.signal(key);
|
|
74
695
|
}
|
|
75
696
|
pump() {
|
|
697
|
+
return this.#requestPass({ kind: "full" });
|
|
698
|
+
}
|
|
699
|
+
armOperatorStartupRetry() {
|
|
700
|
+
if (this.#stopped)
|
|
701
|
+
return;
|
|
702
|
+
const mailbox = this.store.getWorkMailbox({ kind: "operator" });
|
|
703
|
+
const shouldArm = mailbox !== null
|
|
704
|
+
&& (mailbox.pending !== null || mailbox.processing !== null);
|
|
705
|
+
if (shouldArm)
|
|
706
|
+
this.#clearDeliveryRetry("operator");
|
|
707
|
+
this.#operatorStartupRetryArmed = shouldArm;
|
|
708
|
+
}
|
|
709
|
+
#requestPass(scope) {
|
|
76
710
|
if (this.#stopped) {
|
|
77
711
|
return Promise.reject(new Error("Controller runtime is stopped."));
|
|
78
712
|
}
|
|
713
|
+
if (scope.kind === "full") {
|
|
714
|
+
this.#pendingFull = true;
|
|
715
|
+
this.#pendingKeys.clear();
|
|
716
|
+
}
|
|
717
|
+
else if (!this.#pendingFull) {
|
|
718
|
+
for (const key of scope.keys)
|
|
719
|
+
this.#pendingKeys.add(key);
|
|
720
|
+
}
|
|
79
721
|
if (this.#current !== undefined) {
|
|
80
|
-
this.#rerunRequested = true;
|
|
81
722
|
return this.#current;
|
|
82
723
|
}
|
|
83
724
|
const running = this.#runCoalesced();
|
|
@@ -88,43 +729,410 @@ export class FileTaskController {
|
|
|
88
729
|
}).catch(() => { });
|
|
89
730
|
return running;
|
|
90
731
|
}
|
|
732
|
+
#requestBackgroundPump() {
|
|
733
|
+
void this.pump().catch(this.#onError);
|
|
734
|
+
}
|
|
91
735
|
async #runCoalesced() {
|
|
92
736
|
let result = {
|
|
93
|
-
stoppedArchivedTaskIds: [],
|
|
94
737
|
activeRunDeliveries: [],
|
|
95
|
-
|
|
738
|
+
failedRunRefs: [],
|
|
96
739
|
wakeups: [],
|
|
97
740
|
inputNotifications: [],
|
|
98
741
|
autoResolvedInputs: []
|
|
99
742
|
};
|
|
100
|
-
|
|
101
|
-
this.#
|
|
102
|
-
|
|
103
|
-
|
|
743
|
+
try {
|
|
744
|
+
while (this.#pendingFull || this.#pendingKeys.size > 0) {
|
|
745
|
+
const scope = this.#pendingFull
|
|
746
|
+
? { kind: "full" }
|
|
747
|
+
: { kind: "dirty", keys: [...this.#pendingKeys] };
|
|
748
|
+
const runtimeCleanupOutcomes = [];
|
|
749
|
+
this.#pendingFull = false;
|
|
750
|
+
this.#pendingKeys.clear();
|
|
751
|
+
try {
|
|
752
|
+
if (scope.kind === "full")
|
|
753
|
+
this.reloadReconciliationInterval();
|
|
754
|
+
if (scope.kind === "full" && this.#resourceReaper !== undefined) {
|
|
755
|
+
const reap = await this.#resourceReaper();
|
|
756
|
+
for (const failure of reap.failed) {
|
|
757
|
+
this.#onError(new Error(`Ephemeral runtime reap failed for ${failure.id}: ${failure.message}`));
|
|
758
|
+
}
|
|
759
|
+
// The reaper owns the exact resource fences. Once a whole expired
|
|
760
|
+
// domain converges, let the detached Controller close itself on a
|
|
761
|
+
// later turn; never close while this reconciliation pass is still
|
|
762
|
+
// the in-flight server request.
|
|
763
|
+
if (reap.failed.length === 0 && (reap.expiredDomains?.length ?? 0) > 0) {
|
|
764
|
+
for (const domain of reap.expiredDomains ?? []) {
|
|
765
|
+
queueMicrotask(() => {
|
|
766
|
+
this.#onExpiredEphemeralDomain?.(domain);
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
const firstRuntimeDrain = this.#drainRuntimeEvents();
|
|
772
|
+
result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#resourceSuppressionKeys);
|
|
773
|
+
const secondRuntimeDrain = this.#drainRuntimeEvents();
|
|
774
|
+
this.#clearPassRetry();
|
|
775
|
+
this.#scheduleRuntimeCleanupRetries(runtimeCleanupOutcomes);
|
|
776
|
+
this.#scheduleDeliveryRetries(result);
|
|
777
|
+
this.#scheduleTaskMailboxRetries(scope);
|
|
778
|
+
if (scope.kind === "full"
|
|
779
|
+
|| scope.keys.some((key) => key !== "operator")
|
|
780
|
+
|| (firstRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0
|
|
781
|
+
|| (secondRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0) {
|
|
782
|
+
this.#signalOperatorMailbox();
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
catch (error) {
|
|
786
|
+
if (scope.kind === "full") {
|
|
787
|
+
this.#pendingFull = true;
|
|
788
|
+
this.#pendingKeys.clear();
|
|
789
|
+
}
|
|
790
|
+
else if (!this.#pendingFull) {
|
|
791
|
+
for (const key of scope.keys)
|
|
792
|
+
this.#pendingKeys.add(key);
|
|
793
|
+
}
|
|
794
|
+
this.#schedulePassRetry();
|
|
795
|
+
throw error;
|
|
796
|
+
}
|
|
797
|
+
if (this.#stopped)
|
|
798
|
+
break;
|
|
799
|
+
}
|
|
800
|
+
return result;
|
|
801
|
+
}
|
|
802
|
+
finally {
|
|
803
|
+
this.#scheduleNextInputDeadline();
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
async #runOperatorPass() {
|
|
807
|
+
const result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), undefined, { kind: "dirty", keys: ["operator"] });
|
|
808
|
+
if (this.#operatorStartupRetryArmed && result.inputNotifications.length === 0) {
|
|
809
|
+
const mailbox = this.store.getWorkMailbox({ kind: "operator" });
|
|
810
|
+
if (mailbox === null
|
|
811
|
+
|| (mailbox.pending === null && mailbox.processing === null)) {
|
|
812
|
+
this.#operatorStartupRetryArmed = false;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
this.#scheduleDeliveryRetries(result);
|
|
816
|
+
this.#clearEmptyOperatorRetry();
|
|
817
|
+
this.#scheduleNextInputDeadline();
|
|
818
|
+
}
|
|
819
|
+
#signalOperatorMailbox() {
|
|
820
|
+
if (this.#stopped)
|
|
821
|
+
return;
|
|
822
|
+
const mailbox = this.store.getWorkMailbox({ kind: "operator" });
|
|
823
|
+
const identity = operatorMailboxBatchIdentity(mailbox);
|
|
824
|
+
if (identity === null) {
|
|
825
|
+
this.#lastOperatorSignalIdentity = undefined;
|
|
826
|
+
return;
|
|
827
|
+
}
|
|
828
|
+
if (identity !== this.#lastOperatorSignalIdentity) {
|
|
829
|
+
this.#operatorSignalScheduler.signal("operator");
|
|
830
|
+
this.#lastOperatorSignalIdentity = identity;
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
#drainRuntimeEvents() {
|
|
834
|
+
const result = this.#runtimeEventProcessor?.drain(this.#now());
|
|
835
|
+
if (result !== undefined && result.failed.length > 0) {
|
|
836
|
+
throw new RuntimeEventApplyError(result.failed.map((failure) => failure.error), "One or more native Turn events could not be applied.");
|
|
837
|
+
}
|
|
104
838
|
return result;
|
|
105
839
|
}
|
|
840
|
+
#schedulePassRetry() {
|
|
841
|
+
if (this.#stopped
|
|
842
|
+
|| this.#passRetryTimer !== undefined
|
|
843
|
+
|| this.#passRetryAttempt >= this.#deliveryRetryLimit)
|
|
844
|
+
return;
|
|
845
|
+
const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(this.#passRetryAttempt, 3)));
|
|
846
|
+
this.#passRetryAttempt += 1;
|
|
847
|
+
this.#passRetryTimer = setTimeout(() => {
|
|
848
|
+
this.#passRetryTimer = undefined;
|
|
849
|
+
if (this.#stopped)
|
|
850
|
+
return;
|
|
851
|
+
void this.#requestPass({ kind: "dirty", keys: [] }).catch(this.#onError);
|
|
852
|
+
}, delayMs);
|
|
853
|
+
this.#passRetryTimer.unref();
|
|
854
|
+
}
|
|
855
|
+
#clearPassRetry() {
|
|
856
|
+
if (this.#passRetryTimer !== undefined)
|
|
857
|
+
clearTimeout(this.#passRetryTimer);
|
|
858
|
+
this.#passRetryTimer = undefined;
|
|
859
|
+
this.#passRetryAttempt = 0;
|
|
860
|
+
}
|
|
861
|
+
#scheduleNextInputDeadline() {
|
|
862
|
+
if (this.#deadlineTimer !== undefined) {
|
|
863
|
+
clearTimeout(this.#deadlineTimer);
|
|
864
|
+
this.#deadlineTimer = undefined;
|
|
865
|
+
}
|
|
866
|
+
if (this.#stopped)
|
|
867
|
+
return;
|
|
868
|
+
const deadlines = [
|
|
869
|
+
...this.store.listOpenInputRequests()
|
|
870
|
+
.flatMap((request) => request.policy.kind === "recommended"
|
|
871
|
+
? [{
|
|
872
|
+
key: `task:${encodeURIComponent(request.taskId)}`,
|
|
873
|
+
at: Date.parse(request.policy.timeoutAt)
|
|
874
|
+
}]
|
|
875
|
+
: []),
|
|
876
|
+
...(typeof this.store.listPendingRuntimeTurnCompletions === "function"
|
|
877
|
+
? this.store.listPendingRuntimeTurnCompletions()
|
|
878
|
+
: []).map((completion) => ({
|
|
879
|
+
key: `role:${encodeURIComponent(completion.taskId)}/${encodeURIComponent(completion.roleName)}`,
|
|
880
|
+
at: Date.parse(completion.dueAt)
|
|
881
|
+
}))
|
|
882
|
+
];
|
|
883
|
+
const nearest = nearestDeadlineBatch(deadlines);
|
|
884
|
+
if (nearest === null)
|
|
885
|
+
return;
|
|
886
|
+
const now = this.#now().getTime();
|
|
887
|
+
// Preserve an upcoming semantic deadline even while another pass backs
|
|
888
|
+
// off. Once that deadline has fired, the bounded pass retry owns failures
|
|
889
|
+
// so an overdue record cannot create a zero-delay hot loop.
|
|
890
|
+
if (nearest.at <= now && this.#passRetryAttempt > 0)
|
|
891
|
+
return;
|
|
892
|
+
const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(0, nearest.at - now));
|
|
893
|
+
this.#deadlineTimer = setTimeout(() => {
|
|
894
|
+
this.#deadlineTimer = undefined;
|
|
895
|
+
for (const key of nearest.keys)
|
|
896
|
+
this.#signalScheduler.signal(key);
|
|
897
|
+
void this.#signalScheduler.drain().catch(this.#onError);
|
|
898
|
+
}, delayMs);
|
|
899
|
+
this.#deadlineTimer.unref();
|
|
900
|
+
}
|
|
901
|
+
#scheduleDeliveryRetries(result) {
|
|
902
|
+
const retry = new Map();
|
|
903
|
+
const settled = new Set();
|
|
904
|
+
const resignal = new Set();
|
|
905
|
+
for (const delivery of result.activeRunDeliveries) {
|
|
906
|
+
const key = `role:${encodeURIComponent(delivery.taskId)}/${encodeURIComponent(delivery.roleName)}`;
|
|
907
|
+
if (delivery.terminalized === true) {
|
|
908
|
+
settled.add(key);
|
|
909
|
+
resignal.add(key);
|
|
910
|
+
}
|
|
911
|
+
else if (delivery.reason === "not-ready"
|
|
912
|
+
|| delivery.reason === "runtime-unavailable"
|
|
913
|
+
|| delivery.reason === "delivery-uncertain") {
|
|
914
|
+
retry.set(key, {
|
|
915
|
+
identity: delivery.runId,
|
|
916
|
+
...(delivery.terminalFailure === undefined
|
|
917
|
+
? {}
|
|
918
|
+
: { terminalFailure: delivery.terminalFailure })
|
|
919
|
+
});
|
|
920
|
+
}
|
|
921
|
+
else if (delivery.status === "delivered" || delivery.status === "already-delivered")
|
|
922
|
+
settled.add(key);
|
|
923
|
+
}
|
|
924
|
+
for (const wakeup of result.wakeups) {
|
|
925
|
+
const key = `role:${encodeURIComponent(wakeup.taskId)}/leader`;
|
|
926
|
+
if (wakeup.reason === "not-ready"
|
|
927
|
+
|| wakeup.reason === "delivery-uncertain") {
|
|
928
|
+
retry.set(key, { identity: wakeup.runId ?? key });
|
|
929
|
+
}
|
|
930
|
+
else if (wakeup.status === "dispatched")
|
|
931
|
+
settled.add(key);
|
|
932
|
+
}
|
|
933
|
+
const operatorRetries = result.inputNotifications.filter((notification) => notification.reason === "operator-not-ready"
|
|
934
|
+
|| (notification.reason === "operator-unavailable"
|
|
935
|
+
&& this.#operatorStartupRetryArmed));
|
|
936
|
+
if (operatorRetries.length > 0) {
|
|
937
|
+
retry.set("operator", {
|
|
938
|
+
identity: operatorRetries.map((notification) => ("inputRequestId" in notification
|
|
939
|
+
? `input:${notification.inputRequestId}`
|
|
940
|
+
: "recoveryTaskId" in notification
|
|
941
|
+
? `recovery:${notification.recoveryTaskId}`
|
|
942
|
+
: "stallTaskId" in notification
|
|
943
|
+
? `stall:${notification.stallTaskId}`
|
|
944
|
+
: `terminal:${notification.terminalTaskId}`)).join("|")
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
else if (result.inputNotifications.some((notification) => notification.status === "sent"
|
|
948
|
+
|| notification.status === "already-sent")) {
|
|
949
|
+
this.#operatorStartupRetryArmed = false;
|
|
950
|
+
settled.add("operator");
|
|
951
|
+
}
|
|
952
|
+
for (const key of settled)
|
|
953
|
+
this.#clearDeliveryRetry(key);
|
|
954
|
+
for (const key of resignal)
|
|
955
|
+
this.signal(key);
|
|
956
|
+
for (const [key, candidate] of retry) {
|
|
957
|
+
this.#scheduleDeliveryRetry(key, candidate.identity, candidate.terminalFailure);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
#scheduleRuntimeCleanupRetries(outcomes) {
|
|
961
|
+
for (const outcome of outcomes) {
|
|
962
|
+
const key = runtimeCleanupMailboxKey(outcome.target);
|
|
963
|
+
const identity = `runtime-cleanup:${outcome.batchId}`;
|
|
964
|
+
if (outcome.status === "failed") {
|
|
965
|
+
if (outcome.error !== undefined)
|
|
966
|
+
this.#onError(outcome.error);
|
|
967
|
+
this.#scheduleDeliveryRetry(key, identity);
|
|
968
|
+
continue;
|
|
969
|
+
}
|
|
970
|
+
if (this.#deliveryRetryAttempts.get(key)?.identity === identity) {
|
|
971
|
+
this.#clearDeliveryRetry(key);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
#scheduleTaskMailboxRetries(scope) {
|
|
976
|
+
const selection = compileReconcileSelection(scope);
|
|
977
|
+
const targets = selection.full
|
|
978
|
+
? this.store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
|
|
979
|
+
: [...selection.allRoleTaskIds].map((taskId) => ({ kind: "task", taskId }));
|
|
980
|
+
for (const target of targets) {
|
|
981
|
+
const key = `task:${encodeURIComponent(target.taskId)}`;
|
|
982
|
+
const mailbox = this.store.getWorkMailbox(target);
|
|
983
|
+
const batch = mailbox?.pending ?? mailbox?.processing?.batch;
|
|
984
|
+
if (batch === undefined || batch === null) {
|
|
985
|
+
this.#clearDeliveryRetry(key);
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
this.#scheduleDeliveryRetry(key, `${batch.fromSequence}-${batch.toSequence}`);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
#scheduleDeliveryRetry(key, identity, terminalFailure) {
|
|
992
|
+
let previous = this.#deliveryRetryAttempts.get(key);
|
|
993
|
+
if (previous !== undefined && previous.identity !== identity) {
|
|
994
|
+
this.#clearDeliveryRetry(key);
|
|
995
|
+
previous = undefined;
|
|
996
|
+
}
|
|
997
|
+
const stableTerminalFailure = previous?.terminalFailure ?? terminalFailure;
|
|
998
|
+
if (this.#deliveryRetryTimers.has(key)) {
|
|
999
|
+
if (previous !== undefined && stableTerminalFailure !== previous.terminalFailure) {
|
|
1000
|
+
this.#deliveryRetryAttempts.set(key, {
|
|
1001
|
+
...previous,
|
|
1002
|
+
terminalFailure: stableTerminalFailure
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
const attempts = previous?.attempts ?? 0;
|
|
1008
|
+
if (attempts >= this.#deliveryRetryLimit) {
|
|
1009
|
+
if (key === "operator")
|
|
1010
|
+
this.#operatorStartupRetryArmed = false;
|
|
1011
|
+
this.#terminalizePreparedAfterRetryExhaustion(key, identity, stableTerminalFailure);
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
this.#deliveryRetryAttempts.set(key, {
|
|
1015
|
+
identity,
|
|
1016
|
+
attempts: attempts + 1,
|
|
1017
|
+
...(stableTerminalFailure === undefined
|
|
1018
|
+
? {}
|
|
1019
|
+
: { terminalFailure: stableTerminalFailure })
|
|
1020
|
+
});
|
|
1021
|
+
const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
|
|
1022
|
+
const timer = setTimeout(() => {
|
|
1023
|
+
this.#deliveryRetryTimers.delete(key);
|
|
1024
|
+
if (!this.#stopped)
|
|
1025
|
+
this.signal(key);
|
|
1026
|
+
}, delayMs);
|
|
1027
|
+
timer.unref();
|
|
1028
|
+
this.#deliveryRetryTimers.set(key, timer);
|
|
1029
|
+
}
|
|
1030
|
+
#terminalizePreparedAfterRetryExhaustion(key, runId, failure) {
|
|
1031
|
+
if (!key.startsWith("role:")
|
|
1032
|
+
|| runId.startsWith("runtime-cleanup:")
|
|
1033
|
+
|| failure === undefined) {
|
|
1034
|
+
return;
|
|
1035
|
+
}
|
|
1036
|
+
const target = parseMailboxKey(key);
|
|
1037
|
+
if (target.kind !== "role")
|
|
1038
|
+
return;
|
|
1039
|
+
if (target.taskId !== failure.taskId
|
|
1040
|
+
|| target.roleName !== failure.roleName
|
|
1041
|
+
|| runId !== failure.runId) {
|
|
1042
|
+
throw new Error(`Role delivery retry identity changed: ${key}/${runId}.`);
|
|
1043
|
+
}
|
|
1044
|
+
const result = this.store.saveRoleRunDeliveryFailure({
|
|
1045
|
+
...failure,
|
|
1046
|
+
now: this.#now()
|
|
1047
|
+
});
|
|
1048
|
+
this.#clearDeliveryRetry(key);
|
|
1049
|
+
if (result !== "failed")
|
|
1050
|
+
return;
|
|
1051
|
+
this.delivery.forgetPrepared?.({
|
|
1052
|
+
taskId: failure.taskId,
|
|
1053
|
+
roleName: failure.roleName,
|
|
1054
|
+
runId: failure.runId,
|
|
1055
|
+
...(failure.launchId === undefined
|
|
1056
|
+
? {}
|
|
1057
|
+
: { launchId: failure.launchId })
|
|
1058
|
+
});
|
|
1059
|
+
if (!this.#stopped)
|
|
1060
|
+
this.signal(key);
|
|
1061
|
+
}
|
|
1062
|
+
#clearEmptyOperatorRetry() {
|
|
1063
|
+
if (!this.#operatorStartupRetryArmed
|
|
1064
|
+
&& !this.#deliveryRetryAttempts.has("operator"))
|
|
1065
|
+
return;
|
|
1066
|
+
const mailbox = this.store.getWorkMailbox({ kind: "operator" });
|
|
1067
|
+
if (mailbox === null
|
|
1068
|
+
|| (mailbox.pending === null && mailbox.processing === null)) {
|
|
1069
|
+
this.#operatorStartupRetryArmed = false;
|
|
1070
|
+
this.#clearDeliveryRetry("operator");
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
#clearDeliveryRetry(key) {
|
|
1074
|
+
const timer = this.#deliveryRetryTimers.get(key);
|
|
1075
|
+
if (timer !== undefined)
|
|
1076
|
+
clearTimeout(timer);
|
|
1077
|
+
this.#deliveryRetryTimers.delete(key);
|
|
1078
|
+
this.#deliveryRetryAttempts.delete(key);
|
|
1079
|
+
}
|
|
106
1080
|
}
|
|
107
1081
|
/**
|
|
108
1082
|
* Starts the single private Unix-socket Controller for one YUI_HOME. The
|
|
109
1083
|
* shared server owns status/stop and rejects a second live instance; this
|
|
110
|
-
* layer adds
|
|
1084
|
+
* layer adds scheduler.signal/scheduler.scan plus an optional command dispatcher.
|
|
111
1085
|
*/
|
|
112
1086
|
export async function startFileTaskController(home, store, delivery, dispatcher, options = {}) {
|
|
113
1087
|
const runtime = new FileTaskController(store, delivery, options);
|
|
1088
|
+
let stopping = false;
|
|
1089
|
+
const lifecycleRequests = new Set();
|
|
114
1090
|
const server = await startControllerServer(home, async (method, params) => {
|
|
1091
|
+
if (stopping) {
|
|
1092
|
+
throw controllerApplicationError("METHOD_NOT_FOUND", "Controller is stopping.");
|
|
1093
|
+
}
|
|
1094
|
+
if (method === "scheduler.signal") {
|
|
1095
|
+
runtime.signal(signalMailboxKey(params));
|
|
1096
|
+
return { accepted: true };
|
|
1097
|
+
}
|
|
115
1098
|
if (method === "scheduler.scan") {
|
|
116
1099
|
if (!isEmptyJsonObject(params)) {
|
|
117
1100
|
throw controllerApplicationError("INVALID_PARAMS", "scheduler.scan params are invalid.");
|
|
118
1101
|
}
|
|
119
1102
|
return schedulerResultJson(await runtime.pump());
|
|
120
1103
|
}
|
|
1104
|
+
if (method === "scheduler.configure") {
|
|
1105
|
+
requireEmptySchedulerConfigureParams(params);
|
|
1106
|
+
const intervalMs = runtime.reloadReconciliationInterval();
|
|
1107
|
+
return { configured: true, reconciliationIntervalMs: intervalMs };
|
|
1108
|
+
}
|
|
121
1109
|
if (dispatcher === undefined) {
|
|
122
1110
|
throw controllerApplicationError("METHOD_NOT_FOUND", "Controller method was not found.");
|
|
123
1111
|
}
|
|
124
|
-
|
|
1112
|
+
const request = Promise.resolve(dispatcher(method, params));
|
|
1113
|
+
lifecycleRequests.add(request);
|
|
1114
|
+
try {
|
|
1115
|
+
const result = await request;
|
|
1116
|
+
if (method === "runtime.ensure-role-session"
|
|
1117
|
+
&& isGlobalOperatorSessionRequest(params)
|
|
1118
|
+
&& isStartedRuntimeSessionResult(result)) {
|
|
1119
|
+
runtime.armOperatorStartupRetry();
|
|
1120
|
+
}
|
|
1121
|
+
return result;
|
|
1122
|
+
}
|
|
1123
|
+
finally {
|
|
1124
|
+
lifecycleRequests.delete(request);
|
|
1125
|
+
}
|
|
1126
|
+
}, async () => {
|
|
1127
|
+
stopping = true;
|
|
1128
|
+
runtime.stop();
|
|
1129
|
+
await Promise.allSettled([...lifecycleRequests]);
|
|
1130
|
+
await runtime.shutdownAndDrain();
|
|
1131
|
+
}, {
|
|
1132
|
+
domainIdentity: options.domainIdentity
|
|
125
1133
|
});
|
|
126
1134
|
runtime.start();
|
|
127
|
-
const closed = server.closed
|
|
1135
|
+
const closed = server.closed;
|
|
128
1136
|
return {
|
|
129
1137
|
runtime,
|
|
130
1138
|
server,
|
|
@@ -132,9 +1140,34 @@ export async function startFileTaskController(home, store, delivery, dispatcher,
|
|
|
132
1140
|
close: async () => {
|
|
133
1141
|
runtime.stop();
|
|
134
1142
|
await server.close();
|
|
1143
|
+
await runtime.shutdownAndDrain();
|
|
135
1144
|
}
|
|
136
1145
|
};
|
|
137
1146
|
}
|
|
1147
|
+
function signalMailboxKey(value) {
|
|
1148
|
+
if (typeof value !== "object"
|
|
1149
|
+
|| value === null
|
|
1150
|
+
|| Array.isArray(value)) {
|
|
1151
|
+
throw controllerApplicationError("INVALID_PARAMS", "scheduler.signal params are invalid.");
|
|
1152
|
+
}
|
|
1153
|
+
const record = value;
|
|
1154
|
+
if (Object.keys(record).length !== 1
|
|
1155
|
+
|| typeof record.key !== "string") {
|
|
1156
|
+
throw controllerApplicationError("INVALID_PARAMS", "scheduler.signal params are invalid.");
|
|
1157
|
+
}
|
|
1158
|
+
try {
|
|
1159
|
+
parseMailboxKey(record.key);
|
|
1160
|
+
return record.key;
|
|
1161
|
+
}
|
|
1162
|
+
catch {
|
|
1163
|
+
throw controllerApplicationError("INVALID_PARAMS", "scheduler.signal params are invalid.");
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
function requireEmptySchedulerConfigureParams(value) {
|
|
1167
|
+
if (!isEmptyJsonObject(value)) {
|
|
1168
|
+
throw controllerApplicationError("INVALID_PARAMS", "scheduler.configure params are invalid.");
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
138
1171
|
function positiveInteger(value, fallback, label) {
|
|
139
1172
|
const resolved = value ?? fallback;
|
|
140
1173
|
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
@@ -142,6 +1175,17 @@ function positiveInteger(value, fallback, label) {
|
|
|
142
1175
|
}
|
|
143
1176
|
return resolved;
|
|
144
1177
|
}
|
|
1178
|
+
function operatorMailboxBatchIdentity(mailbox) {
|
|
1179
|
+
const batch = mailbox?.pending ?? mailbox?.processing?.batch;
|
|
1180
|
+
if (batch === null || batch === undefined)
|
|
1181
|
+
return null;
|
|
1182
|
+
return [
|
|
1183
|
+
batch.fromSequence,
|
|
1184
|
+
batch.toSequence,
|
|
1185
|
+
batch.firstQueuedAt,
|
|
1186
|
+
batch.lastQueuedAt
|
|
1187
|
+
].join(":");
|
|
1188
|
+
}
|
|
145
1189
|
function schedulerResultJson(result) {
|
|
146
1190
|
return JSON.parse(JSON.stringify(result));
|
|
147
1191
|
}
|
|
@@ -151,6 +1195,19 @@ function isEmptyJsonObject(value) {
|
|
|
151
1195
|
&& !Array.isArray(value)
|
|
152
1196
|
&& Object.keys(value).length === 0;
|
|
153
1197
|
}
|
|
1198
|
+
function isGlobalOperatorSessionRequest(value) {
|
|
1199
|
+
return typeof value === "object"
|
|
1200
|
+
&& value !== null
|
|
1201
|
+
&& !Array.isArray(value)
|
|
1202
|
+
&& value.scope === "global"
|
|
1203
|
+
&& value.roleName === "operator";
|
|
1204
|
+
}
|
|
1205
|
+
function isStartedRuntimeSessionResult(value) {
|
|
1206
|
+
return typeof value === "object"
|
|
1207
|
+
&& value !== null
|
|
1208
|
+
&& !Array.isArray(value)
|
|
1209
|
+
&& value.sessionStarted === true;
|
|
1210
|
+
}
|
|
154
1211
|
function controllerApplicationError(code, message) {
|
|
155
1212
|
const error = Object.assign(new Error(message), { code });
|
|
156
1213
|
error.name = "CoreApplicationError";
|