@zq-silk/yui 0.7.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +27 -28
- package/README.md +79 -71
- package/dist/cli/commandCatalog.js +283 -136
- package/dist/cli/completion.js +3 -3
- package/dist/cli/helpRenderer.js +3 -0
- package/dist/cli/interactionPolicy.js +48 -33
- package/dist/cli/interactiveSelection.js +1 -1
- package/dist/cli/invocationRouter.js +3 -2
- package/dist/cli/roleWizard.js +8 -8
- package/dist/cli.js +189 -93
- package/dist/commands/agentCommands.js +5 -5
- package/dist/commands/configCommands.js +351 -104
- package/dist/commands/configOverview.js +60 -0
- package/dist/commands/deliveryGuardPreflight.js +2 -2
- package/dist/commands/globalRoleCommands.js +9 -9
- package/dist/commands/profileCommands.js +8 -8
- package/dist/commands/resourcesCommands.js +6 -5
- package/dist/commands/taskCommands.js +111 -59
- package/dist/commands/taskRoleRuntimeStatus.js +3 -1
- package/dist/commands/telemetryCommands.js +11 -6
- package/dist/config/configCatalog.js +42 -0
- package/dist/config/yuiConfig.js +80 -35
- package/dist/context/sessionBootstrapManifest.js +1 -1
- package/dist/controller/clientRuntime.js +0 -2
- package/dist/controller/controller.js +21 -9
- package/dist/controller/fileSchedulerStoreAdapter.js +409 -79
- package/dist/controller/resourceInventory.js +9 -5
- package/dist/controller/runtime.js +112 -25
- package/dist/controller/runtimeLaunchCoordinator.js +18 -78
- package/dist/controller/structuredProviderObservation.js +273 -0
- package/dist/doctor/doctor.js +2 -2
- package/dist/executor/agentAdapter.js +40 -0
- package/dist/executor/agentExecutor.js +31 -7
- package/dist/executor/executorRegistry.js +11 -49
- package/dist/executor/fileRoleLaunchPlanner.js +115 -37
- package/dist/lifecycle/canonicalLifecycleEvent.js +5 -2
- package/dist/resources/autoResourceGc.js +3 -1
- package/dist/review/reviewConfig.js +0 -2
- package/dist/run/agentRun.js +2 -2
- package/dist/run/providerRetry.js +29 -16
- package/dist/run/providerRetryConfig.js +5 -3
- package/dist/runtime/agentHost.js +767 -158
- package/dist/runtime/builtinAgentDrivers.js +1 -5
- package/dist/runtime/codexAppServerRuntime.js +67 -60
- package/dist/runtime/exactControlPlane.js +7 -2
- package/dist/runtime/index.js +6 -2
- package/dist/runtime/launchBroker.js +30 -8
- package/dist/runtime/launchDiagnostics.js +1 -1
- package/dist/runtime/providerAuthorityFence.js +24 -0
- package/dist/runtime/providerControl.js +63 -0
- package/dist/runtime/providerRecoveryDecision.js +55 -0
- package/dist/runtime/providerRuntimeIdentity.js +269 -19
- package/dist/runtime/runtimeBinding.js +20 -11
- package/dist/runtime/structuredProviderHost.js +476 -0
- package/dist/runtime/tmuxAdapters.js +143 -42
- package/dist/scheduler/activeRoleRunDelivery.js +206 -120
- package/dist/scheduler/leaderWakeupProcessor.js +141 -16
- package/dist/scheduler/roleRunStall.js +12 -9
- package/dist/setup/setupCommand.js +153 -492
- package/dist/storage/compatibleTaskStore.js +9 -5
- package/dist/storage/migration/productionRegistry.js +169 -0
- package/dist/storage/taskStore.js +22 -3
- package/dist/telemetry/sqliteTelemetryStore.js +9 -1
- package/dist/telemetry/telemetryConfig.js +1 -18
- package/dist/telemetry/telemetryStore.js +2 -2
- package/dist/telemetry/telemetryWiring.js +6 -5
- package/dist/tmux/tmuxManager.js +1 -1
- package/dist/web/webSnapshot.js +5 -3
- package/i18n/README.zh-CN.md +48 -40
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +12 -5
- package/skills/yui-operator/SKILL.md +44 -6
- package/skills/yui-runtime/SKILL.md +1 -1
package/dist/config/yuiConfig.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { supportedAgentAdapterIds } from "../agent/adapterCatalog.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { DEFAULT_RUN_CAP, DEFAULT_TERMINAL_KEEP, MAX_RUN_CAP } from "../telemetry/telemetryConfig.js";
|
|
3
|
+
import { DEFAULT_RUNTIME_HEALTH_POLICY } from "../runtime/runtimeHealthPolicy.js";
|
|
4
4
|
export const DEFAULT_RECONCILIATION_INTERVAL_SECONDS = 120;
|
|
5
5
|
export const MIN_RECONCILIATION_INTERVAL_SECONDS = 5;
|
|
6
6
|
export const MAX_RECONCILIATION_INTERVAL_SECONDS = 300;
|
|
7
7
|
export const DEFAULT_RESOURCES_GC_MODE = "report";
|
|
8
|
+
export const DEFAULT_RESOURCES_QUARANTINE_TTL_HOURS = 24;
|
|
9
|
+
export const MIN_RESOURCES_QUARANTINE_TTL_HOURS = 1;
|
|
10
|
+
export const MAX_RESOURCES_QUARANTINE_TTL_HOURS = 720;
|
|
8
11
|
/**
|
|
9
12
|
* Resolves the Resource GC mode. `report` (default) only reports candidates;
|
|
10
13
|
* `quarantine` allows `yui resources gc --apply` to quarantine releasable
|
|
@@ -29,6 +32,9 @@ export function resolveResourcesGcAutoQuarantine(value) {
|
|
|
29
32
|
return value;
|
|
30
33
|
throw new TypeError("resourcesGcAutoQuarantine must be a boolean.");
|
|
31
34
|
}
|
|
35
|
+
export function resolveResourcesQuarantineTtlHours(value) {
|
|
36
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_RESOURCES_QUARANTINE_TTL_HOURS, MIN_RESOURCES_QUARANTINE_TTL_HOURS, MAX_RESOURCES_QUARANTINE_TTL_HOURS, "resourcesQuarantineTtlHours");
|
|
37
|
+
}
|
|
32
38
|
/**
|
|
33
39
|
* Resolves the durable Yui setting used for low-frequency recovery
|
|
34
40
|
* reconciliation. Normal durable state changes wake the Controller through
|
|
@@ -72,6 +78,9 @@ export function resolveLeaderNextActionMode(value) {
|
|
|
72
78
|
// ── Issue 01: Provider retry ──────────────────────────────────────────────
|
|
73
79
|
export const PROVIDER_RETRY_MODES = ["off", "shadow", "enforce"];
|
|
74
80
|
export const DEFAULT_PROVIDER_RETRY_MODE = "enforce";
|
|
81
|
+
export const DEFAULT_PROVIDER_RETRY_DELAYS_SECONDS = Object.freeze([2, 5, 15]);
|
|
82
|
+
export const DEFAULT_PROVIDER_RETRY_MAX_WINDOW_SECONDS = 600;
|
|
83
|
+
export const MAX_PROVIDER_RETRY_ATTEMPTS = 10;
|
|
75
84
|
export function resolveProviderRetryMode(value) {
|
|
76
85
|
if (value === undefined || value === null)
|
|
77
86
|
return DEFAULT_PROVIDER_RETRY_MODE;
|
|
@@ -122,21 +131,27 @@ export function resolveProviderRetryAdapters(value) {
|
|
|
122
131
|
}
|
|
123
132
|
return adapters;
|
|
124
133
|
}
|
|
125
|
-
export function
|
|
134
|
+
export function resolveProviderRetryDelaysSeconds(value) {
|
|
126
135
|
if (value === undefined || value === null)
|
|
127
|
-
return
|
|
128
|
-
if (
|
|
129
|
-
throw new TypeError(
|
|
136
|
+
return [...DEFAULT_PROVIDER_RETRY_DELAYS_SECONDS];
|
|
137
|
+
if (!Array.isArray(value) || value.length < 1 || value.length > MAX_PROVIDER_RETRY_ATTEMPTS) {
|
|
138
|
+
throw new TypeError(`providerRetryDelaysSeconds must contain 1-${MAX_PROVIDER_RETRY_ATTEMPTS} positive integers.`);
|
|
130
139
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
return
|
|
136
|
-
|
|
137
|
-
|
|
140
|
+
const delays = value.map((entry) => {
|
|
141
|
+
if (typeof entry !== "number" || !Number.isSafeInteger(entry) || entry < 1 || entry > 600) {
|
|
142
|
+
throw new TypeError("providerRetryDelaysSeconds entries must be integers from 1 to 600.");
|
|
143
|
+
}
|
|
144
|
+
return entry;
|
|
145
|
+
});
|
|
146
|
+
for (let index = 1; index < delays.length; index += 1) {
|
|
147
|
+
if (delays[index] < delays[index - 1]) {
|
|
148
|
+
throw new TypeError("providerRetryDelaysSeconds must be ordered from shortest to longest.");
|
|
149
|
+
}
|
|
138
150
|
}
|
|
139
|
-
return
|
|
151
|
+
return delays;
|
|
152
|
+
}
|
|
153
|
+
export function resolveProviderRetryMaxWindowSeconds(value) {
|
|
154
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_PROVIDER_RETRY_MAX_WINDOW_SECONDS, 1, Number.MAX_SAFE_INTEGER, "providerRetryMaxWindowSeconds");
|
|
140
155
|
}
|
|
141
156
|
// ── Executable paths ──────────────────────────────────────────────────────
|
|
142
157
|
export function resolveTmuxBin(value) {
|
|
@@ -147,29 +162,13 @@ export function resolveTmuxBin(value) {
|
|
|
147
162
|
}
|
|
148
163
|
return value.trim();
|
|
149
164
|
}
|
|
150
|
-
export function resolveGitBin(value) {
|
|
151
|
-
if (value === undefined || value === null)
|
|
152
|
-
return "git";
|
|
153
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
154
|
-
throw new TypeError("gitBin must be a non-empty string.");
|
|
155
|
-
}
|
|
156
|
-
return value.trim();
|
|
157
|
-
}
|
|
158
165
|
// ── Telemetry ─────────────────────────────────────────────────────────────
|
|
159
|
-
|
|
160
|
-
export function resolveTelemetryMode(value) {
|
|
166
|
+
export function resolveTelemetryEnabled(value) {
|
|
161
167
|
if (value === undefined || value === null)
|
|
162
|
-
return
|
|
163
|
-
if (typeof value !== "
|
|
164
|
-
throw new TypeError("
|
|
165
|
-
|
|
166
|
-
const normalized = value.trim().toLowerCase();
|
|
167
|
-
if (normalized.length === 0)
|
|
168
|
-
return DEFAULT_TELEMETRY_MODE;
|
|
169
|
-
if (!TELEMETRY_MODES.includes(normalized)) {
|
|
170
|
-
throw new TypeError(`telemetryMode must be one of ${TELEMETRY_MODES.join(", ")}; got ${JSON.stringify(normalized)}.`);
|
|
171
|
-
}
|
|
172
|
-
return normalized;
|
|
168
|
+
return false;
|
|
169
|
+
if (typeof value !== "boolean")
|
|
170
|
+
throw new TypeError("telemetryEnabled must be a boolean.");
|
|
171
|
+
return value;
|
|
173
172
|
}
|
|
174
173
|
export function resolveTelemetryTerminalKeep(value) {
|
|
175
174
|
if (value === undefined || value === null)
|
|
@@ -190,6 +189,43 @@ export function resolveTelemetryRunCap(value) {
|
|
|
190
189
|
}
|
|
191
190
|
return value;
|
|
192
191
|
}
|
|
192
|
+
// ── Runtime and workflow policy ───────────────────────────────────────────
|
|
193
|
+
export const DEFAULT_CONTROLLER_TASK_CONCURRENCY = 4;
|
|
194
|
+
export const MAX_CONTROLLER_TASK_CONCURRENCY = 32;
|
|
195
|
+
export const DEFAULT_AGENT_LAUNCH_INACTIVITY_TIMEOUT_SECONDS = 300;
|
|
196
|
+
export const DEFAULT_DELIVERY_TIMEOUT_SECONDS = 120;
|
|
197
|
+
export const DEFAULT_LEADER_SEMANTIC_BUDGET_TURNS = 3;
|
|
198
|
+
export const DEFAULT_TMUX_HISTORY_LIMIT = 100_000;
|
|
199
|
+
export function resolveRuntimeHealth(configured) {
|
|
200
|
+
if (configured === undefined || configured === null)
|
|
201
|
+
return DEFAULT_RUNTIME_HEALTH_POLICY;
|
|
202
|
+
if (typeof configured !== "object" || Array.isArray(configured)) {
|
|
203
|
+
throw new TypeError("runtimeHealth must be an object.");
|
|
204
|
+
}
|
|
205
|
+
const value = configured;
|
|
206
|
+
const quietAfterMs = resolveBoundedPositiveInteger(value.quietAfterSeconds, DEFAULT_RUNTIME_HEALTH_POLICY.quietAfterMs / 1_000, 30, 86_400, "runtimeHealth.quietAfterSeconds") * 1_000;
|
|
207
|
+
const diagnosticAfterMs = resolveBoundedPositiveInteger(value.diagnosticAfterSeconds, DEFAULT_RUNTIME_HEALTH_POLICY.diagnosticAfterMs / 1_000, 30, 86_400, "runtimeHealth.diagnosticAfterSeconds") * 1_000;
|
|
208
|
+
const stallWindowMs = resolveBoundedPositiveInteger(value.stallAfterSeconds, DEFAULT_RUNTIME_HEALTH_POLICY.stallWindowMs / 1_000, 60, 604_800, "runtimeHealth.stallAfterSeconds") * 1_000;
|
|
209
|
+
if (!(quietAfterMs < diagnosticAfterMs && diagnosticAfterMs < stallWindowMs)) {
|
|
210
|
+
throw new TypeError("runtimeHealth thresholds must be ordered quietAfterSeconds < diagnosticAfterSeconds < stallAfterSeconds.");
|
|
211
|
+
}
|
|
212
|
+
return Object.freeze({ quietAfterMs, diagnosticAfterMs, stallWindowMs });
|
|
213
|
+
}
|
|
214
|
+
export function resolveControllerTaskConcurrency(value) {
|
|
215
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_CONTROLLER_TASK_CONCURRENCY, 1, MAX_CONTROLLER_TASK_CONCURRENCY, "controllerTaskConcurrency");
|
|
216
|
+
}
|
|
217
|
+
export function resolveAgentLaunchInactivityTimeoutSeconds(value) {
|
|
218
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_AGENT_LAUNCH_INACTIVITY_TIMEOUT_SECONDS, 15, 3_600, "agentLaunchInactivityTimeoutSeconds");
|
|
219
|
+
}
|
|
220
|
+
export function resolveDeliveryTimeoutSeconds(value) {
|
|
221
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_DELIVERY_TIMEOUT_SECONDS, 5, 600, "deliveryTimeoutSeconds");
|
|
222
|
+
}
|
|
223
|
+
export function resolveLeaderSemanticBudgetTurns(value) {
|
|
224
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_LEADER_SEMANTIC_BUDGET_TURNS, 1, 20, "leaderSemanticBudgetTurns");
|
|
225
|
+
}
|
|
226
|
+
export function resolveTmuxHistoryLimit(value) {
|
|
227
|
+
return resolveBoundedPositiveInteger(value, DEFAULT_TMUX_HISTORY_LIMIT, 1_000, 1_000_000, "tmuxHistoryLimit");
|
|
228
|
+
}
|
|
193
229
|
/**
|
|
194
230
|
* Issue 04 (context token budget): thresholds for one native Session
|
|
195
231
|
* generation's observed per-request input peak, measured in tokens. When the
|
|
@@ -231,3 +267,12 @@ export function resolveContextBudget(configured) {
|
|
|
231
267
|
}
|
|
232
268
|
return { softTokens, hardTokens };
|
|
233
269
|
}
|
|
270
|
+
function resolveBoundedPositiveInteger(value, fallback, minimum, maximum, label) {
|
|
271
|
+
if (value === undefined || value === null)
|
|
272
|
+
return fallback;
|
|
273
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value)
|
|
274
|
+
|| value < minimum || value > maximum) {
|
|
275
|
+
throw new TypeError(`${label} must be an integer from ${minimum} to ${maximum}.`);
|
|
276
|
+
}
|
|
277
|
+
return value;
|
|
278
|
+
}
|
|
@@ -53,7 +53,7 @@ export function materializeSessionBootstrap(input) {
|
|
|
53
53
|
roleProfileRef: { digest: profileDigest, path: roleProfilePath },
|
|
54
54
|
contextProtocol: input.owner.scope === "global"
|
|
55
55
|
? {
|
|
56
|
-
loadCommand: `\"${sessionCliPath}\"
|
|
56
|
+
loadCommand: `\"${sessionCliPath}\" session context \"$YUI_ROLE\" --json`
|
|
57
57
|
}
|
|
58
58
|
: {
|
|
59
59
|
loadCommand: `\"${sessionCliPath}\" task run context \"$YUI_TASK_ID/<run-id>\" --json`,
|
|
@@ -29,8 +29,6 @@ const CONTROLLER_OPERATIONAL_ENVIRONMENT = [
|
|
|
29
29
|
// rr13/test: Forward the liveness seam so an integration test's Controller
|
|
30
30
|
// subprocess does not reap a saved active Leader Run without a real tmux role.
|
|
31
31
|
"YUI_TEST_ROLE_LIVENESS_PRESENT",
|
|
32
|
-
// Issue 02: agent-signal-driven launch monitoring is Controller-owned.
|
|
33
|
-
"YUI_LAUNCH_INACTIVITY_TIMEOUT_MS",
|
|
34
32
|
...EPHEMERAL_DOMAIN_ENVIRONMENT_NAMES
|
|
35
33
|
];
|
|
36
34
|
/**
|
|
@@ -4,7 +4,7 @@ import { pendingWakeupsMatch } from "../scheduler/pendingWakeup.js";
|
|
|
4
4
|
import { processActiveRoleRunDeliveries } from "../scheduler/activeRoleRunDelivery.js";
|
|
5
5
|
import { selectedActiveSchedulerTasks, selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
|
|
6
6
|
import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
|
|
7
|
-
import { DEFAULT_STALL_WINDOW_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
|
|
7
|
+
import { DEFAULT_STALL_WINDOW_MS, DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
|
|
8
8
|
import { repairOrphanedActiveTasks } from "../scheduler/activeTaskProgress.js";
|
|
9
9
|
import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
|
|
10
10
|
import { startControllerServer } from "../core/controllerServer.js";
|
|
@@ -22,6 +22,7 @@ const DEFAULT_SIGNAL_WINDOW_MS = 100;
|
|
|
22
22
|
const DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS = 1_000;
|
|
23
23
|
const DEFAULT_DELIVERY_RETRY_MS = 250;
|
|
24
24
|
const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
|
|
25
|
+
const DEFAULT_DELIVERY_TIMEOUT_MS = 120_000;
|
|
25
26
|
const DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT = 2;
|
|
26
27
|
const DEFAULT_TASK_CONCURRENCY = 4;
|
|
27
28
|
const MAX_TASK_CONCURRENCY = 32;
|
|
@@ -44,7 +45,7 @@ const ZERO_DRAIN_METRICS = Object.freeze({
|
|
|
44
45
|
* Runs one lean scheduler pass. Due native Turn completions are folded before
|
|
45
46
|
* liveness, so a valid Hook boundary fences destructive process reconciliation.
|
|
46
47
|
*/
|
|
47
|
-
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), inputDeliveryRecoveryCutoff) {
|
|
48
|
+
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), inputDeliveryRecoveryCutoff, diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS) {
|
|
48
49
|
const compiledSelection = compileReconcileSelection(scope);
|
|
49
50
|
const selection = includeOperator
|
|
50
51
|
? { ...compiledSelection, blockedTaskIds }
|
|
@@ -112,7 +113,7 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
|
|
|
112
113
|
const resourceEvidence = new Map();
|
|
113
114
|
const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty");
|
|
114
115
|
await controlEventLoopTurn();
|
|
115
|
-
await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence);
|
|
116
|
+
await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, diagnosticAfterMs);
|
|
116
117
|
await controlEventLoopTurn();
|
|
117
118
|
await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now, selection.blockedTaskIds);
|
|
118
119
|
const selectedInputTaskIds = selectedTaskIdsForBoundedPass(store, selection);
|
|
@@ -794,9 +795,12 @@ export class FileTaskController {
|
|
|
794
795
|
#workspacePreparer;
|
|
795
796
|
#deliveryRetryMs;
|
|
796
797
|
#deliveryRetryLimit;
|
|
798
|
+
#mailboxDeliveryRetryLimit;
|
|
799
|
+
#deliveryTimeoutMs;
|
|
797
800
|
#taskOrchestrationRetryLimit;
|
|
798
801
|
#taskConcurrency;
|
|
799
802
|
#stallWindowMs;
|
|
803
|
+
#diagnosticAfterMs;
|
|
800
804
|
#runtimeEventProcessor;
|
|
801
805
|
#runtimeObserver;
|
|
802
806
|
#runtimeObserverIntervalMs;
|
|
@@ -845,9 +849,14 @@ export class FileTaskController {
|
|
|
845
849
|
this.#workspacePreparer = options.workspacePreparer;
|
|
846
850
|
this.#deliveryRetryMs = positiveInteger(options.deliveryRetryMs, DEFAULT_DELIVERY_RETRY_MS, "Controller delivery retry delay");
|
|
847
851
|
this.#deliveryRetryLimit = positiveInteger(options.deliveryRetryLimit, DEFAULT_DELIVERY_RETRY_LIMIT, "Controller delivery retry limit");
|
|
852
|
+
this.#deliveryTimeoutMs = positiveInteger(options.deliveryTimeoutMs, DEFAULT_DELIVERY_TIMEOUT_MS, "Controller delivery timeout");
|
|
853
|
+
this.#mailboxDeliveryRetryLimit = options.deliveryRetryLimit === undefined
|
|
854
|
+
? Math.max(this.#deliveryRetryLimit, Math.ceil(this.#deliveryTimeoutMs / this.#deliveryRetryMs) + 2)
|
|
855
|
+
: this.#deliveryRetryLimit;
|
|
848
856
|
this.#taskOrchestrationRetryLimit = positiveInteger(options.taskOrchestrationRetryLimit, DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT, "Controller Task orchestration retry limit");
|
|
849
857
|
this.#taskConcurrency = boundedPositiveInteger(options.taskConcurrency, DEFAULT_TASK_CONCURRENCY, MAX_TASK_CONCURRENCY, "Controller Task concurrency");
|
|
850
858
|
this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
|
|
859
|
+
this.#diagnosticAfterMs = positiveInteger(options.diagnosticAfterMs, DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, "Controller Run diagnostic threshold");
|
|
851
860
|
this.#runtimeEventProcessor = options.runtimeEventProcessor;
|
|
852
861
|
this.#runtimeObserver = options.runtimeObserver;
|
|
853
862
|
this.#runtimeObserverIntervalMs = positiveInteger(options.runtimeObserverIntervalMs, DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS, "Controller runtime observer interval");
|
|
@@ -1109,7 +1118,7 @@ export class FileTaskController {
|
|
|
1109
1118
|
// processes Leader wakeups.
|
|
1110
1119
|
this.#jobSupervisor?.reconcile(this.#now());
|
|
1111
1120
|
if (scope.kind === "full") {
|
|
1112
|
-
result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#startedAt);
|
|
1121
|
+
result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#startedAt, this.#diagnosticAfterMs);
|
|
1113
1122
|
}
|
|
1114
1123
|
else {
|
|
1115
1124
|
const dirtyPass = await this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds);
|
|
@@ -1207,7 +1216,7 @@ export class FileTaskController {
|
|
|
1207
1216
|
const taskScopes = partition.taskScopes.filter((taskScope) => (!blockedTaskIds.has(taskScope.taskId)));
|
|
1208
1217
|
const orderedResults = [];
|
|
1209
1218
|
if (partition.globalKeys.length > 0) {
|
|
1210
|
-
orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt));
|
|
1219
|
+
orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt, this.#diagnosticAfterMs));
|
|
1211
1220
|
}
|
|
1212
1221
|
if (taskScopes.length === 0
|
|
1213
1222
|
|| this.#stopped
|
|
@@ -1240,7 +1249,7 @@ export class FileTaskController {
|
|
|
1240
1249
|
if (this.#stopped || this.#pendingFull)
|
|
1241
1250
|
continue;
|
|
1242
1251
|
try {
|
|
1243
|
-
taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt);
|
|
1252
|
+
taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt, this.#diagnosticAfterMs);
|
|
1244
1253
|
this.#clearTaskPassRetry(selected.taskScope.taskId);
|
|
1245
1254
|
}
|
|
1246
1255
|
catch (error) {
|
|
@@ -1576,10 +1585,12 @@ export class FileTaskController {
|
|
|
1576
1585
|
return;
|
|
1577
1586
|
}
|
|
1578
1587
|
const attempts = previous?.attempts ?? 0;
|
|
1588
|
+
const startedAtMs = previous?.startedAtMs ?? this.#now().getTime();
|
|
1579
1589
|
const retryLimit = key.startsWith("task:")
|
|
1580
1590
|
? this.#taskOrchestrationRetryLimit
|
|
1581
|
-
: this.#
|
|
1582
|
-
|
|
1591
|
+
: this.#mailboxDeliveryRetryLimit;
|
|
1592
|
+
const remainingMs = this.#deliveryTimeoutMs - (this.#now().getTime() - startedAtMs);
|
|
1593
|
+
if (attempts >= retryLimit || remainingMs <= 0) {
|
|
1583
1594
|
if (key === "operator")
|
|
1584
1595
|
this.#operatorStartupRetryArmed = false;
|
|
1585
1596
|
this.#terminalizePreparedAfterRetryExhaustion(key, identity, stableTerminalFailure);
|
|
@@ -1588,11 +1599,12 @@ export class FileTaskController {
|
|
|
1588
1599
|
this.#deliveryRetryAttempts.set(key, {
|
|
1589
1600
|
identity,
|
|
1590
1601
|
attempts: attempts + 1,
|
|
1602
|
+
startedAtMs,
|
|
1591
1603
|
...(stableTerminalFailure === undefined
|
|
1592
1604
|
? {}
|
|
1593
1605
|
: { terminalFailure: stableTerminalFailure })
|
|
1594
1606
|
});
|
|
1595
|
-
const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
|
|
1607
|
+
const delayMs = Math.min(remainingMs, 2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
|
|
1596
1608
|
const timer = setTimeout(() => {
|
|
1597
1609
|
this.#deliveryRetryTimers.delete(key);
|
|
1598
1610
|
if (!this.#stopped)
|