@zq-silk/yui 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -23
- package/dist/cli/commandCatalog.js +234 -122
- package/dist/cli/completion.js +3 -3
- package/dist/cli/helpRenderer.js +3 -0
- package/dist/cli/interactionPolicy.js +44 -23
- package/dist/cli/interactiveSelection.js +1 -1
- package/dist/cli/invocationRouter.js +1 -1
- package/dist/cli/roleWizard.js +8 -8
- package/dist/cli.js +116 -72
- package/dist/commands/agentCommands.js +5 -5
- package/dist/commands/configCommands.js +351 -104
- package/dist/commands/configOverview.js +60 -0
- package/dist/commands/deliveryGuardPreflight.js +2 -2
- package/dist/commands/globalRoleCommands.js +9 -9
- package/dist/commands/profileCommands.js +8 -8
- package/dist/commands/resourcesCommands.js +6 -5
- package/dist/commands/taskCommands.js +3 -6
- package/dist/commands/taskRoleRuntimeStatus.js +3 -1
- package/dist/commands/telemetryCommands.js +11 -6
- package/dist/config/configCatalog.js +42 -0
- package/dist/config/yuiConfig.js +80 -35
- package/dist/context/sessionBootstrapManifest.js +1 -1
- package/dist/controller/clientRuntime.js +0 -2
- package/dist/controller/controller.js +21 -9
- package/dist/controller/fileSchedulerStoreAdapter.js +20 -9
- package/dist/controller/runtime.js +32 -18
- package/dist/doctor/doctor.js +2 -2
- package/dist/resources/autoResourceGc.js +3 -1
- package/dist/review/reviewConfig.js +0 -2
- package/dist/run/providerRetry.js +29 -16
- package/dist/run/providerRetryConfig.js +5 -3
- package/dist/runtime/launchDiagnostics.js +1 -1
- package/dist/scheduler/roleRunStall.js +12 -9
- package/dist/setup/setupCommand.js +153 -492
- package/dist/storage/compatibleTaskStore.js +9 -5
- package/dist/storage/migration/productionRegistry.js +58 -0
- package/dist/storage/taskStore.js +21 -2
- package/dist/telemetry/sqliteTelemetryStore.js +9 -1
- package/dist/telemetry/telemetryConfig.js +1 -18
- package/dist/telemetry/telemetryStore.js +2 -2
- package/dist/telemetry/telemetryWiring.js +6 -5
- package/dist/web/webSnapshot.js +5 -3
- package/i18n/README.zh-CN.md +37 -32
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +12 -5
- package/skills/yui-operator/SKILL.md +44 -6
- package/skills/yui-runtime/SKILL.md +1 -1
|
@@ -4,7 +4,7 @@ import { pendingWakeupsMatch } from "../scheduler/pendingWakeup.js";
|
|
|
4
4
|
import { processActiveRoleRunDeliveries } from "../scheduler/activeRoleRunDelivery.js";
|
|
5
5
|
import { selectedActiveSchedulerTasks, selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
|
|
6
6
|
import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
|
|
7
|
-
import { DEFAULT_STALL_WINDOW_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
|
|
7
|
+
import { DEFAULT_STALL_WINDOW_MS, DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
|
|
8
8
|
import { repairOrphanedActiveTasks } from "../scheduler/activeTaskProgress.js";
|
|
9
9
|
import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
|
|
10
10
|
import { startControllerServer } from "../core/controllerServer.js";
|
|
@@ -22,6 +22,7 @@ const DEFAULT_SIGNAL_WINDOW_MS = 100;
|
|
|
22
22
|
const DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS = 1_000;
|
|
23
23
|
const DEFAULT_DELIVERY_RETRY_MS = 250;
|
|
24
24
|
const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
|
|
25
|
+
const DEFAULT_DELIVERY_TIMEOUT_MS = 120_000;
|
|
25
26
|
const DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT = 2;
|
|
26
27
|
const DEFAULT_TASK_CONCURRENCY = 4;
|
|
27
28
|
const MAX_TASK_CONCURRENCY = 32;
|
|
@@ -44,7 +45,7 @@ const ZERO_DRAIN_METRICS = Object.freeze({
|
|
|
44
45
|
* Runs one lean scheduler pass. Due native Turn completions are folded before
|
|
45
46
|
* liveness, so a valid Hook boundary fences destructive process reconciliation.
|
|
46
47
|
*/
|
|
47
|
-
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), inputDeliveryRecoveryCutoff) {
|
|
48
|
+
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), inputDeliveryRecoveryCutoff, diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS) {
|
|
48
49
|
const compiledSelection = compileReconcileSelection(scope);
|
|
49
50
|
const selection = includeOperator
|
|
50
51
|
? { ...compiledSelection, blockedTaskIds }
|
|
@@ -112,7 +113,7 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
|
|
|
112
113
|
const resourceEvidence = new Map();
|
|
113
114
|
const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty");
|
|
114
115
|
await controlEventLoopTurn();
|
|
115
|
-
await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence);
|
|
116
|
+
await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, diagnosticAfterMs);
|
|
116
117
|
await controlEventLoopTurn();
|
|
117
118
|
await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now, selection.blockedTaskIds);
|
|
118
119
|
const selectedInputTaskIds = selectedTaskIdsForBoundedPass(store, selection);
|
|
@@ -794,9 +795,12 @@ export class FileTaskController {
|
|
|
794
795
|
#workspacePreparer;
|
|
795
796
|
#deliveryRetryMs;
|
|
796
797
|
#deliveryRetryLimit;
|
|
798
|
+
#mailboxDeliveryRetryLimit;
|
|
799
|
+
#deliveryTimeoutMs;
|
|
797
800
|
#taskOrchestrationRetryLimit;
|
|
798
801
|
#taskConcurrency;
|
|
799
802
|
#stallWindowMs;
|
|
803
|
+
#diagnosticAfterMs;
|
|
800
804
|
#runtimeEventProcessor;
|
|
801
805
|
#runtimeObserver;
|
|
802
806
|
#runtimeObserverIntervalMs;
|
|
@@ -845,9 +849,14 @@ export class FileTaskController {
|
|
|
845
849
|
this.#workspacePreparer = options.workspacePreparer;
|
|
846
850
|
this.#deliveryRetryMs = positiveInteger(options.deliveryRetryMs, DEFAULT_DELIVERY_RETRY_MS, "Controller delivery retry delay");
|
|
847
851
|
this.#deliveryRetryLimit = positiveInteger(options.deliveryRetryLimit, DEFAULT_DELIVERY_RETRY_LIMIT, "Controller delivery retry limit");
|
|
852
|
+
this.#deliveryTimeoutMs = positiveInteger(options.deliveryTimeoutMs, DEFAULT_DELIVERY_TIMEOUT_MS, "Controller delivery timeout");
|
|
853
|
+
this.#mailboxDeliveryRetryLimit = options.deliveryRetryLimit === undefined
|
|
854
|
+
? Math.max(this.#deliveryRetryLimit, Math.ceil(this.#deliveryTimeoutMs / this.#deliveryRetryMs) + 2)
|
|
855
|
+
: this.#deliveryRetryLimit;
|
|
848
856
|
this.#taskOrchestrationRetryLimit = positiveInteger(options.taskOrchestrationRetryLimit, DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT, "Controller Task orchestration retry limit");
|
|
849
857
|
this.#taskConcurrency = boundedPositiveInteger(options.taskConcurrency, DEFAULT_TASK_CONCURRENCY, MAX_TASK_CONCURRENCY, "Controller Task concurrency");
|
|
850
858
|
this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
|
|
859
|
+
this.#diagnosticAfterMs = positiveInteger(options.diagnosticAfterMs, DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, "Controller Run diagnostic threshold");
|
|
851
860
|
this.#runtimeEventProcessor = options.runtimeEventProcessor;
|
|
852
861
|
this.#runtimeObserver = options.runtimeObserver;
|
|
853
862
|
this.#runtimeObserverIntervalMs = positiveInteger(options.runtimeObserverIntervalMs, DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS, "Controller runtime observer interval");
|
|
@@ -1109,7 +1118,7 @@ export class FileTaskController {
|
|
|
1109
1118
|
// processes Leader wakeups.
|
|
1110
1119
|
this.#jobSupervisor?.reconcile(this.#now());
|
|
1111
1120
|
if (scope.kind === "full") {
|
|
1112
|
-
result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#startedAt);
|
|
1121
|
+
result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#startedAt, this.#diagnosticAfterMs);
|
|
1113
1122
|
}
|
|
1114
1123
|
else {
|
|
1115
1124
|
const dirtyPass = await this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds);
|
|
@@ -1207,7 +1216,7 @@ export class FileTaskController {
|
|
|
1207
1216
|
const taskScopes = partition.taskScopes.filter((taskScope) => (!blockedTaskIds.has(taskScope.taskId)));
|
|
1208
1217
|
const orderedResults = [];
|
|
1209
1218
|
if (partition.globalKeys.length > 0) {
|
|
1210
|
-
orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt));
|
|
1219
|
+
orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt, this.#diagnosticAfterMs));
|
|
1211
1220
|
}
|
|
1212
1221
|
if (taskScopes.length === 0
|
|
1213
1222
|
|| this.#stopped
|
|
@@ -1240,7 +1249,7 @@ export class FileTaskController {
|
|
|
1240
1249
|
if (this.#stopped || this.#pendingFull)
|
|
1241
1250
|
continue;
|
|
1242
1251
|
try {
|
|
1243
|
-
taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt);
|
|
1252
|
+
taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#startedAt, this.#diagnosticAfterMs);
|
|
1244
1253
|
this.#clearTaskPassRetry(selected.taskScope.taskId);
|
|
1245
1254
|
}
|
|
1246
1255
|
catch (error) {
|
|
@@ -1576,10 +1585,12 @@ export class FileTaskController {
|
|
|
1576
1585
|
return;
|
|
1577
1586
|
}
|
|
1578
1587
|
const attempts = previous?.attempts ?? 0;
|
|
1588
|
+
const startedAtMs = previous?.startedAtMs ?? this.#now().getTime();
|
|
1579
1589
|
const retryLimit = key.startsWith("task:")
|
|
1580
1590
|
? this.#taskOrchestrationRetryLimit
|
|
1581
|
-
: this.#
|
|
1582
|
-
|
|
1591
|
+
: this.#mailboxDeliveryRetryLimit;
|
|
1592
|
+
const remainingMs = this.#deliveryTimeoutMs - (this.#now().getTime() - startedAtMs);
|
|
1593
|
+
if (attempts >= retryLimit || remainingMs <= 0) {
|
|
1583
1594
|
if (key === "operator")
|
|
1584
1595
|
this.#operatorStartupRetryArmed = false;
|
|
1585
1596
|
this.#terminalizePreparedAfterRetryExhaustion(key, identity, stableTerminalFailure);
|
|
@@ -1588,11 +1599,12 @@ export class FileTaskController {
|
|
|
1588
1599
|
this.#deliveryRetryAttempts.set(key, {
|
|
1589
1600
|
identity,
|
|
1590
1601
|
attempts: attempts + 1,
|
|
1602
|
+
startedAtMs,
|
|
1591
1603
|
...(stableTerminalFailure === undefined
|
|
1592
1604
|
? {}
|
|
1593
1605
|
: { terminalFailure: stableTerminalFailure })
|
|
1594
1606
|
});
|
|
1595
|
-
const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
|
|
1607
|
+
const delayMs = Math.min(remainingMs, 2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
|
|
1596
1608
|
const timer = setTimeout(() => {
|
|
1597
1609
|
this.#deliveryRetryTimers.delete(key);
|
|
1598
1610
|
if (!this.#stopped)
|
|
@@ -505,7 +505,14 @@ export class FileSchedulerStoreAdapter {
|
|
|
505
505
|
});
|
|
506
506
|
if (this.telemetry !== null && input.fence.runId !== undefined) {
|
|
507
507
|
try {
|
|
508
|
-
|
|
508
|
+
const entry = runtimeObservationTelemetryEntry(input);
|
|
509
|
+
this.telemetry.sink.observe(entry);
|
|
510
|
+
const run = this.store.getAgentRun(entry.taskId, input.fence.runId);
|
|
511
|
+
if (run !== null && run.status !== "active") {
|
|
512
|
+
void this.telemetry.retention.flush().then(() => {
|
|
513
|
+
this.telemetry?.retention.pruneGeneration(entry.taskId, entry.roleName, entry.runId, entry.generation);
|
|
514
|
+
}).catch(() => undefined);
|
|
515
|
+
}
|
|
509
516
|
}
|
|
510
517
|
catch {
|
|
511
518
|
// Runtime telemetry is diagnostic; the compact durable state snapshot
|
|
@@ -2084,7 +2091,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2084
2091
|
failedNativeTurnId: input.nativeTurnId,
|
|
2085
2092
|
lastErrorSummary: summary,
|
|
2086
2093
|
scheduleNextAttempt: false
|
|
2087
|
-
}, now);
|
|
2094
|
+
}, now, configuredProviderRetryPolicy(store));
|
|
2088
2095
|
if (blocked.outcome === "exhausted")
|
|
2089
2096
|
return null;
|
|
2090
2097
|
store.saveAgentRun(withProviderRetry(run, blocked.retry));
|
|
@@ -2110,7 +2117,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2110
2117
|
failedNativeTurnId: input.nativeTurnId,
|
|
2111
2118
|
lastErrorSummary: summary,
|
|
2112
2119
|
...(input.retryAfterMs === undefined ? {} : { retryAfterMs: input.retryAfterMs })
|
|
2113
|
-
}, now);
|
|
2120
|
+
}, now, configuredProviderRetryPolicy(store));
|
|
2114
2121
|
if (retryDecision.outcome === "exhausted") {
|
|
2115
2122
|
this.recordProviderRetryClassified(store, input, classification.errorClass, {
|
|
2116
2123
|
wouldRetry: "false",
|
|
@@ -2122,10 +2129,10 @@ export class FileSchedulerStoreAdapter {
|
|
|
2122
2129
|
if (run.providerRetry === undefined)
|
|
2123
2130
|
return null;
|
|
2124
2131
|
const finalized = finalizeProviderRetryDeadline(store, run, retryDecision.reason === "attempts"
|
|
2125
|
-
?
|
|
2132
|
+
? `Provider retry failed after all ${config.delaysMs.length} in-Session continuation attempts.`
|
|
2126
2133
|
: retryDecision.reason === "retry-after-window"
|
|
2127
|
-
?
|
|
2128
|
-
:
|
|
2134
|
+
? `Provider Retry-After exceeded the bounded ${config.maxWindowMs / 1_000}-second episode window.`
|
|
2135
|
+
: `Provider retry did not recover within the bounded ${config.maxWindowMs / 1_000}-second episode window.`, retryDecision.reason === "attempts" ? "attempts-exhausted" : "episode-window-exhausted", now);
|
|
2129
2136
|
return finalized ? { disposition: "applied", runId: input.runId } : null;
|
|
2130
2137
|
}
|
|
2131
2138
|
const retry = retryDecision.retry;
|
|
@@ -2183,7 +2190,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2183
2190
|
failedNativeTurnId: input.nativeTurnId,
|
|
2184
2191
|
lastErrorSummary: summary,
|
|
2185
2192
|
scheduleNextAttempt: false
|
|
2186
|
-
}, now);
|
|
2193
|
+
}, now, configuredProviderRetryPolicy(store));
|
|
2187
2194
|
if (retryDecision.outcome === "exhausted")
|
|
2188
2195
|
return null;
|
|
2189
2196
|
store.saveAgentRun(withProviderRetry(run, retryDecision.retry));
|
|
@@ -2223,7 +2230,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2223
2230
|
failedNativeTurnId: input.nativeTurnId,
|
|
2224
2231
|
lastErrorSummary: summary,
|
|
2225
2232
|
scheduleNextAttempt: false
|
|
2226
|
-
}, now);
|
|
2233
|
+
}, now, configuredProviderRetryPolicy(store));
|
|
2227
2234
|
if (decision.outcome === "exhausted")
|
|
2228
2235
|
return null;
|
|
2229
2236
|
store.saveAgentRun(withProviderRetry(run, decision.retry));
|
|
@@ -2262,7 +2269,7 @@ export class FileSchedulerStoreAdapter {
|
|
|
2262
2269
|
failedNativeTurnId: input.nativeTurnId,
|
|
2263
2270
|
lastErrorSummary: summary,
|
|
2264
2271
|
scheduleNextAttempt: false
|
|
2265
|
-
}, now);
|
|
2272
|
+
}, now, configuredProviderRetryPolicy(store));
|
|
2266
2273
|
if (decision.outcome === "exhausted")
|
|
2267
2274
|
return null;
|
|
2268
2275
|
store.saveAgentRun(withProviderRetry(run, decision.retry));
|
|
@@ -3695,3 +3702,7 @@ function compareCanonicalObservationOrder(left, right) {
|
|
|
3695
3702
|
|| (left.ordinal ?? -1) - (right.ordinal ?? -1)
|
|
3696
3703
|
|| left.eventId.localeCompare(right.eventId);
|
|
3697
3704
|
}
|
|
3705
|
+
function configuredProviderRetryPolicy(store) {
|
|
3706
|
+
const config = providerRetryConfig(store.getConfig());
|
|
3707
|
+
return { delaysMs: config.delaysMs, maxWindowMs: config.maxWindowMs };
|
|
3708
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { reconciliationIntervalMilliseconds, resolveTmuxBin } from "../config/yuiConfig.js";
|
|
1
|
+
import { reconciliationIntervalMilliseconds, resolveAgentLaunchInactivityTimeoutSeconds, resolveControllerTaskConcurrency, resolveDeliveryTimeoutSeconds, resolveRuntimeHealth, resolveTmuxBin, resolveTmuxHistoryLimit } from "../config/yuiConfig.js";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { resolve } from "node:path";
|
|
4
4
|
import { isDeepStrictEqual } from "node:util";
|
|
@@ -92,6 +92,7 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
92
92
|
? new SqliteTaskStore(home)
|
|
93
93
|
: openCompatibleFileTaskStore(home));
|
|
94
94
|
const homeId = store.getHomeIdentity().homeId;
|
|
95
|
+
const durableConfig = store.getConfig();
|
|
95
96
|
// When the worker backend is active, the db-touching observer folds run in
|
|
96
97
|
// the worker (off the main event loop). The client is closed on shutdown.
|
|
97
98
|
const asyncStoreClient = useWorker
|
|
@@ -113,8 +114,9 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
113
114
|
const planner = options.planner ?? new FileRoleLaunchPlanner(home, store, {
|
|
114
115
|
environment: options.environment
|
|
115
116
|
});
|
|
116
|
-
const tmux = options.tmux ?? new TmuxManager(resolveTmuxBin(
|
|
117
|
+
const tmux = options.tmux ?? new TmuxManager(resolveTmuxBin(durableConfig.tmuxBin), new NodeCommandExecutor(), {
|
|
117
118
|
yuiHome: home,
|
|
119
|
+
historyLimit: resolveTmuxHistoryLimit(durableConfig.tmuxHistoryLimit),
|
|
118
120
|
...(domainIdentity === undefined
|
|
119
121
|
? {}
|
|
120
122
|
: {
|
|
@@ -181,8 +183,7 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
181
183
|
}
|
|
182
184
|
throw new Error("Native session discovery was aborted.");
|
|
183
185
|
},
|
|
184
|
-
inactivityTimeoutMs:
|
|
185
|
-
?? process.env.YUI_LAUNCH_INACTIVITY_TIMEOUT_MS, 300_000),
|
|
186
|
+
inactivityTimeoutMs: resolveAgentLaunchInactivityTimeoutSeconds(durableConfig.agentLaunchInactivityTimeoutSeconds) * 1_000,
|
|
186
187
|
onHostCreated: ({ binding, pane }) => {
|
|
187
188
|
sessionOwners.recordHostOwner({
|
|
188
189
|
owner: binding.owner,
|
|
@@ -438,9 +439,16 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
438
439
|
intervalMs: options.intervalMs
|
|
439
440
|
?? reconciliationIntervalMilliseconds(store.getConfig().reconciliationIntervalSeconds),
|
|
440
441
|
signalWindowMs: options.signalWindowMs,
|
|
441
|
-
taskConcurrency: options.taskConcurrency
|
|
442
|
+
taskConcurrency: options.taskConcurrency
|
|
443
|
+
?? resolveControllerTaskConcurrency(durableConfig.controllerTaskConcurrency),
|
|
442
444
|
deliveryRetryMs: options.deliveryRetryMs,
|
|
443
445
|
deliveryRetryLimit: options.deliveryRetryLimit,
|
|
446
|
+
deliveryTimeoutMs: options.deliveryTimeoutMs
|
|
447
|
+
?? resolveDeliveryTimeoutSeconds(durableConfig.deliveryTimeoutSeconds) * 1_000,
|
|
448
|
+
stallWindowMs: options.stallWindowMs
|
|
449
|
+
?? resolveRuntimeHealth(durableConfig.runtimeHealth).stallWindowMs,
|
|
450
|
+
diagnosticAfterMs: options.diagnosticAfterMs
|
|
451
|
+
?? resolveRuntimeHealth(durableConfig.runtimeHealth).diagnosticAfterMs,
|
|
444
452
|
now: options.now,
|
|
445
453
|
onError: options.onError,
|
|
446
454
|
lifecycleHost,
|
|
@@ -497,13 +505,28 @@ export async function startFileTaskControllerRuntime(home, options = {}) {
|
|
|
497
505
|
catch (error) {
|
|
498
506
|
(options.onError ?? (() => undefined))(error);
|
|
499
507
|
}
|
|
508
|
+
let resourceClose;
|
|
509
|
+
const closeResources = () => {
|
|
510
|
+
resourceClose ??= Promise.all([
|
|
511
|
+
asyncStoreClient?.close() ?? Promise.resolve(),
|
|
512
|
+
inventoryClient?.close() ?? Promise.resolve()
|
|
513
|
+
]).then(() => undefined);
|
|
514
|
+
return resourceClose;
|
|
515
|
+
};
|
|
516
|
+
const closed = running.closed.then(closeResources);
|
|
500
517
|
return {
|
|
501
518
|
...running,
|
|
519
|
+
closed,
|
|
502
520
|
close: async () => {
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
521
|
+
try {
|
|
522
|
+
await running.close();
|
|
523
|
+
}
|
|
524
|
+
finally {
|
|
525
|
+
// RPC-driven Controller stops resolve `running.closed` without calling
|
|
526
|
+
// this wrapper. Share one cleanup promise so both lifecycle paths
|
|
527
|
+
// release the worker connections before the process can linger.
|
|
528
|
+
await closeResources();
|
|
529
|
+
}
|
|
507
530
|
},
|
|
508
531
|
store,
|
|
509
532
|
schedulerStore,
|
|
@@ -1026,15 +1049,6 @@ function abortableDelay(milliseconds, signal) {
|
|
|
1026
1049
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
1027
1050
|
});
|
|
1028
1051
|
}
|
|
1029
|
-
function positiveIntegerOption(value, fallback) {
|
|
1030
|
-
if (value === undefined || value.trim().length === 0)
|
|
1031
|
-
return fallback;
|
|
1032
|
-
const parsed = Number(value);
|
|
1033
|
-
if (!Number.isSafeInteger(parsed) || parsed <= 0) {
|
|
1034
|
-
throw new Error(`Invalid native session discovery timeout: ${value}`);
|
|
1035
|
-
}
|
|
1036
|
-
return parsed;
|
|
1037
|
-
}
|
|
1038
1052
|
function applicationError(code, message) {
|
|
1039
1053
|
const error = Object.assign(new Error(message), { code });
|
|
1040
1054
|
error.name = "CoreApplicationError";
|
package/dist/doctor/doctor.js
CHANGED
|
@@ -16,7 +16,7 @@ import { inspectStorageSchema } from "../storage/storageSchema.js";
|
|
|
16
16
|
import { resolveTaskStoreBackendForHome } from "../storage/sqliteStore.js";
|
|
17
17
|
import { resolveStoreWorkerEnabledForHome } from "../storage/storeRpc.js";
|
|
18
18
|
import { classifyHome } from "../storage/upgrade/homeClassification.js";
|
|
19
|
-
import {
|
|
19
|
+
import { resolveTmuxBin } from "../config/yuiConfig.js";
|
|
20
20
|
import { readMigrationReceipt } from "../storage/upgrade/migrationReceipt.js";
|
|
21
21
|
import { latestStorageVersionState } from "../storage/upgrade/recordVersions.js";
|
|
22
22
|
import { COMMITTED_DATABASE_FILENAME } from "../storage/upgrade/sqliteStateMigration.js";
|
|
@@ -985,7 +985,7 @@ function readDurableConfigSafely(home, storageOptions) {
|
|
|
985
985
|
const config = store.getConfig();
|
|
986
986
|
return {
|
|
987
987
|
tmuxBin: resolveTmuxBin(config.tmuxBin),
|
|
988
|
-
gitBin:
|
|
988
|
+
gitBin: "git"
|
|
989
989
|
};
|
|
990
990
|
}
|
|
991
991
|
catch {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* sources) applies identically.
|
|
12
12
|
*/
|
|
13
13
|
import { applyResourceGc, planResourceGc } from "./resourceGc.js";
|
|
14
|
-
import { resolveResourcesGcAutoQuarantine, resolveResourcesGcMode } from "../config/yuiConfig.js";
|
|
14
|
+
import { resolveResourcesGcAutoQuarantine, resolveResourcesGcMode, resolveResourcesQuarantineTtlHours } from "../config/yuiConfig.js";
|
|
15
15
|
/**
|
|
16
16
|
* Create the Controller's automatic Resource GC hook. The hook self-skips
|
|
17
17
|
* unless `resourcesGcMode=quarantine` and `resourcesGcAutoQuarantine=true`.
|
|
@@ -35,6 +35,7 @@ export function createResourceAutoGc(options) {
|
|
|
35
35
|
taskStatusById,
|
|
36
36
|
mode: "quarantine",
|
|
37
37
|
now,
|
|
38
|
+
quarantineTtlHours: resolveResourcesQuarantineTtlHours(config.resourcesQuarantineTtlHours),
|
|
38
39
|
environment,
|
|
39
40
|
activeWorkspaceOwnerPaths: collectActiveWorkspaceOwnerPaths(store)
|
|
40
41
|
};
|
|
@@ -75,6 +76,7 @@ export async function runAutoResourceGc(store, options = {}) {
|
|
|
75
76
|
taskStatusById,
|
|
76
77
|
mode: "quarantine",
|
|
77
78
|
now,
|
|
79
|
+
quarantineTtlHours: resolveResourcesQuarantineTtlHours(config.resourcesQuarantineTtlHours),
|
|
78
80
|
activeWorkspaceOwnerPaths: collectActiveWorkspaceOwnerPaths(store)
|
|
79
81
|
};
|
|
80
82
|
const plan = await planResourceGc(input);
|
|
@@ -11,8 +11,6 @@ export const REVIEW_DELTA_RECHECK_MODES = ["enabled", "disabled"];
|
|
|
11
11
|
/** Issue 07: conservative defaults for whether a delta attempt is allowed. */
|
|
12
12
|
export const DEFAULT_DELTA_RECHECK_MAX_CHANGED_LINES = 200;
|
|
13
13
|
export const DEFAULT_DELTA_RECHECK_MAX_CHANGED_FILES = 5;
|
|
14
|
-
/** The Reviewer Role seeded in a new Home by `yui setup`. */
|
|
15
|
-
export const DEFAULT_REVIEWER_ROLE = "reviewer";
|
|
16
14
|
export function validateReviewConfig(config) {
|
|
17
15
|
requireIdentity(config.roleName, "Review Role");
|
|
18
16
|
if (!REVIEW_TRIGGERS.includes(config.trigger)) {
|
|
@@ -1,24 +1,22 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { DEFAULT_PROVIDER_RETRY_DELAYS_SECONDS, DEFAULT_PROVIDER_RETRY_MAX_WINDOW_SECONDS, MAX_PROVIDER_RETRY_ATTEMPTS } from "../config/yuiConfig.js";
|
|
2
3
|
import { requireIdentity, requireText, requireTimestamp } from "../domain/validation.js";
|
|
3
|
-
export const PROVIDER_RETRY_DELAYS_MS = Object.freeze(
|
|
4
|
-
export const
|
|
5
|
-
export
|
|
6
|
-
/** Durable configuration keeps the historical name for compatibility. */
|
|
7
|
-
export const PROVIDER_RETRY_MAX_WINDOW_MS = PROVIDER_RETRY_EPISODE_WINDOW_MS;
|
|
8
|
-
export function nextProviderRetryDelayMs(retryIndex) {
|
|
4
|
+
export const PROVIDER_RETRY_DELAYS_MS = Object.freeze(DEFAULT_PROVIDER_RETRY_DELAYS_SECONDS.map((seconds) => seconds * 1_000));
|
|
5
|
+
export const PROVIDER_RETRY_EPISODE_WINDOW_MS = DEFAULT_PROVIDER_RETRY_MAX_WINDOW_SECONDS * 1_000;
|
|
6
|
+
export function nextProviderRetryDelayMs(retryIndex, delaysMs = PROVIDER_RETRY_DELAYS_MS) {
|
|
9
7
|
if (!Number.isSafeInteger(retryIndex)
|
|
10
8
|
|| retryIndex < 1
|
|
11
|
-
|| retryIndex >
|
|
9
|
+
|| retryIndex > delaysMs.length) {
|
|
12
10
|
throw new Error(`Provider retry index is out of range: ${String(retryIndex)}.`);
|
|
13
11
|
}
|
|
14
|
-
return
|
|
12
|
+
return delaysMs[retryIndex - 1];
|
|
15
13
|
}
|
|
16
14
|
/**
|
|
17
15
|
* True when the retry lineage has used its total wall-clock budget. The
|
|
18
16
|
* budget is measured from the first classified failure, so repeated failures
|
|
19
17
|
* never extend it.
|
|
20
18
|
*/
|
|
21
|
-
export function providerRetryBudgetExhausted(value, now, maxWindowMs =
|
|
19
|
+
export function providerRetryBudgetExhausted(value, now, maxWindowMs = PROVIDER_RETRY_EPISODE_WINDOW_MS) {
|
|
22
20
|
if (!Number.isSafeInteger(maxWindowMs) || maxWindowMs <= 0) {
|
|
23
21
|
throw new Error(`Provider retry max window must be a positive integer: ${String(maxWindowMs)}.`);
|
|
24
22
|
}
|
|
@@ -43,7 +41,9 @@ export function validateAgentRunProviderRetry(value) {
|
|
|
43
41
|
|| value.dispatchedRetries > value.maxRetries) {
|
|
44
42
|
throw new Error("Agent run providerRetry dispatchedRetries is invalid.");
|
|
45
43
|
}
|
|
46
|
-
if (value.maxRetries
|
|
44
|
+
if (!Number.isSafeInteger(value.maxRetries)
|
|
45
|
+
|| value.maxRetries < 1
|
|
46
|
+
|| value.maxRetries > MAX_PROVIDER_RETRY_ATTEMPTS) {
|
|
47
47
|
throw new Error("Agent run providerRetry maxRetries is invalid.");
|
|
48
48
|
}
|
|
49
49
|
requireTimestamp(value.firstFailureAt, "Agent run providerRetry firstFailureAt");
|
|
@@ -83,18 +83,22 @@ export function validateAgentRunProviderRetry(value) {
|
|
|
83
83
|
return value;
|
|
84
84
|
}
|
|
85
85
|
/** Advance one failure episode without ever changing the native Session. */
|
|
86
|
-
export function scheduleProviderRetry(previous, input, now
|
|
86
|
+
export function scheduleProviderRetry(previous, input, now, policy = {
|
|
87
|
+
delaysMs: PROVIDER_RETRY_DELAYS_MS,
|
|
88
|
+
maxWindowMs: PROVIDER_RETRY_EPISODE_WINDOW_MS
|
|
89
|
+
}) {
|
|
90
|
+
validateRetrySchedulePolicy(policy);
|
|
87
91
|
const at = now.toISOString();
|
|
88
92
|
const firstFailureAt = previous?.firstFailureAt ?? at;
|
|
89
93
|
const episodeDeadlineAt = previous?.episodeDeadlineAt
|
|
90
|
-
?? new Date(now.getTime() +
|
|
94
|
+
?? new Date(now.getTime() + policy.maxWindowMs).toISOString();
|
|
91
95
|
if (now.getTime() >= Date.parse(episodeDeadlineAt)) {
|
|
92
96
|
return Object.freeze({ outcome: "exhausted", reason: "window" });
|
|
93
97
|
}
|
|
94
98
|
const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + 1;
|
|
95
99
|
const dispatchedRetries = previous?.dispatchedRetries ?? 0;
|
|
96
100
|
const schedule = input.scheduleNextAttempt ?? true;
|
|
97
|
-
if (schedule && dispatchedRetries >=
|
|
101
|
+
if (schedule && dispatchedRetries >= policy.delaysMs.length) {
|
|
98
102
|
return Object.freeze({ outcome: "exhausted", reason: "attempts" });
|
|
99
103
|
}
|
|
100
104
|
const state = schedule ? "scheduled" : "blocked";
|
|
@@ -103,7 +107,7 @@ export function scheduleProviderRetry(previous, input, now) {
|
|
|
103
107
|
throw new Error("Provider retry Retry-After must be a positive safe integer.");
|
|
104
108
|
}
|
|
105
109
|
const delayMs = schedule
|
|
106
|
-
? Math.max(nextProviderRetryDelayMs(dispatchedRetries + 1), retryAfterMs ?? 0)
|
|
110
|
+
? Math.max(nextProviderRetryDelayMs(dispatchedRetries + 1, policy.delaysMs), retryAfterMs ?? 0)
|
|
107
111
|
: undefined;
|
|
108
112
|
const nextAttemptAt = delayMs === undefined
|
|
109
113
|
? undefined
|
|
@@ -125,7 +129,7 @@ export function scheduleProviderRetry(previous, input, now) {
|
|
|
125
129
|
errorClass: input.errorClass,
|
|
126
130
|
consecutiveFailures,
|
|
127
131
|
dispatchedRetries,
|
|
128
|
-
maxRetries:
|
|
132
|
+
maxRetries: policy.delaysMs.length,
|
|
129
133
|
firstFailureAt,
|
|
130
134
|
lastFailureAt: at,
|
|
131
135
|
episodeDeadlineAt,
|
|
@@ -139,6 +143,15 @@ export function scheduleProviderRetry(previous, input, now) {
|
|
|
139
143
|
});
|
|
140
144
|
return Object.freeze({ outcome: schedule ? "scheduled" : "blocked", retry });
|
|
141
145
|
}
|
|
146
|
+
function validateRetrySchedulePolicy(policy) {
|
|
147
|
+
if (!Number.isSafeInteger(policy.maxWindowMs) || policy.maxWindowMs < 1) {
|
|
148
|
+
throw new Error("Provider retry max window must be a positive safe integer.");
|
|
149
|
+
}
|
|
150
|
+
if (policy.delaysMs.length < 1 || policy.delaysMs.length > MAX_PROVIDER_RETRY_ATTEMPTS
|
|
151
|
+
|| policy.delaysMs.some((delay) => !Number.isSafeInteger(delay) || delay < 1)) {
|
|
152
|
+
throw new Error(`Provider retry delay schedule must contain 1-${MAX_PROVIDER_RETRY_ATTEMPTS} positive safe integers.`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
142
155
|
/** Mark that one short continuation request was dispatched and now awaits any correlated progress. */
|
|
143
156
|
export function prepareProviderRetryDispatch(value, receiptId, now) {
|
|
144
157
|
if (value.state !== "scheduled" || value.nextAttemptAt === undefined)
|
|
@@ -203,7 +216,7 @@ export function serializeProviderRetryEnvelope(input) {
|
|
|
203
216
|
return [
|
|
204
217
|
"Yui managed in-Session continuation retry.",
|
|
205
218
|
`task=${requireIdentity(input.taskId, "Provider retry task id")} run=${requireIdentity(input.runId, "Provider retry run id")} role=${requireIdentity(input.roleName, "Provider retry role")}`,
|
|
206
|
-
`episode=${input.retry.episodeId} retry=${retryOrdinal}/${
|
|
219
|
+
`episode=${input.retry.episodeId} retry=${retryOrdinal}/${input.retry.maxRetries} receipt=${input.retry.lastRetryReceiptId ?? "pending"}`,
|
|
207
220
|
`failureEvent=${input.retry.failureEventId}`,
|
|
208
221
|
...(input.retry.failedNativeTurnId === undefined
|
|
209
222
|
? []
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { resolveProviderRetryAdapters,
|
|
1
|
+
import { resolveProviderRetryAdapters, resolveProviderRetryDelaysSeconds, resolveProviderRetryMaxWindowSeconds, resolveProviderRetryMode } from "../config/yuiConfig.js";
|
|
2
2
|
/**
|
|
3
3
|
* Resolves the retry flags from the durable Yui config. Homes without the
|
|
4
4
|
* fields get the safe defaults: enforce mode, all supported adapters, receipt
|
|
@@ -10,8 +10,10 @@ export function providerRetryConfig(config) {
|
|
|
10
10
|
return {
|
|
11
11
|
mode: adapters.length === 0 ? "off" : mode,
|
|
12
12
|
adapters,
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
delaysMs: resolveProviderRetryDelaysSeconds(config.providerRetryDelaysSeconds)
|
|
14
|
+
.map((seconds) => seconds * 1_000),
|
|
15
|
+
maxWindowMs: resolveProviderRetryMaxWindowSeconds(config.providerRetryMaxWindowSeconds)
|
|
16
|
+
* 1_000
|
|
15
17
|
};
|
|
16
18
|
}
|
|
17
19
|
/** Whether the adapter has in-place retry enabled in the given mode. */
|
|
@@ -138,7 +138,7 @@ function launchHint(kind, agentId) {
|
|
|
138
138
|
case "config":
|
|
139
139
|
return agentId === undefined
|
|
140
140
|
? "Verify the Provider model and effort configuration."
|
|
141
|
-
: `Verify custom model/effort with yui agent capabilities ${agentId}.`;
|
|
141
|
+
: `Verify custom model/effort with yui config agent capabilities ${agentId}.`;
|
|
142
142
|
case "auth":
|
|
143
143
|
return "Verify Provider authentication and the Agent environment.";
|
|
144
144
|
case "executable":
|
|
@@ -51,6 +51,8 @@ export function clearMatchingLeaderStallAttention(store, taskId, runId) {
|
|
|
51
51
|
*/
|
|
52
52
|
export function projectRoleRunHealth(input) {
|
|
53
53
|
const windowMs = input.windowMs ?? DEFAULT_STALL_WINDOW_MS;
|
|
54
|
+
const diagnosticAfterMs = input.diagnosticAfterMs
|
|
55
|
+
?? DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS;
|
|
54
56
|
if (!Number.isFinite(windowMs) || windowMs <= 0) {
|
|
55
57
|
throw new Error("Role run stall window must be a positive number of milliseconds.");
|
|
56
58
|
}
|
|
@@ -65,7 +67,7 @@ export function projectRoleRunHealth(input) {
|
|
|
65
67
|
const candidate = Number.isFinite(candidateAge)
|
|
66
68
|
&& (input.deliveredAt === undefined
|
|
67
69
|
? candidateAge >= windowMs
|
|
68
|
-
: candidateAge >=
|
|
70
|
+
: candidateAge >= diagnosticAfterMs);
|
|
69
71
|
const providerAcceptance = input.providerAcceptance
|
|
70
72
|
?? (input.deliveredAt === undefined ? "ambiguous" : "accepted");
|
|
71
73
|
const hostLiveness = input.hostLiveness;
|
|
@@ -84,7 +86,7 @@ export function projectRoleRunHealth(input) {
|
|
|
84
86
|
: "matching";
|
|
85
87
|
const resourceActivity = hostLiveness === "present"
|
|
86
88
|
&& nativeSession === "matching"
|
|
87
|
-
&& resourceEvidenceIsFresh(input.resource, input.now, windowMs)
|
|
89
|
+
&& resourceEvidenceIsFresh(input.resource, input.now, windowMs, diagnosticAfterMs)
|
|
88
90
|
// Residency/RSS and an unchanged cumulative counter are not progress.
|
|
89
91
|
&& input.resource?.active === true
|
|
90
92
|
&& input.resource?.changed === true;
|
|
@@ -494,7 +496,7 @@ export function isRoleRunStalled(events, runId) {
|
|
|
494
496
|
* structured waiting/working facts. No branch sends terminal bytes, retries,
|
|
495
497
|
* replaces a Session, or changes Run status.
|
|
496
498
|
*/
|
|
497
|
-
export async function reconcileStalledRoleRuns(store, delivery, now, selection, windowMs = DEFAULT_STALL_WINDOW_MS, liveStatuses, resourceEvidence) {
|
|
499
|
+
export async function reconcileStalledRoleRuns(store, delivery, now, selection, windowMs = DEFAULT_STALL_WINDOW_MS, liveStatuses, resourceEvidence, diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS) {
|
|
498
500
|
// Dirty mailbox passes are intentionally not a second scheduler. Full
|
|
499
501
|
// reconcile owns the all-active-Run scan; dirty passes may still route the
|
|
500
502
|
// existing mailbox work without manufacturing another episode.
|
|
@@ -519,7 +521,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
519
521
|
session: store.getRoleSession(task.id, role.name, run.effective.agentId)
|
|
520
522
|
}];
|
|
521
523
|
})));
|
|
522
|
-
const stallCandidates = candidates.filter(({ run }) => isStallCandidate(run, now, windowMs));
|
|
524
|
+
const stallCandidates = candidates.filter(({ run }) => isStallCandidate(run, now, windowMs, diagnosticAfterMs));
|
|
523
525
|
if (stallCandidates.length === 0)
|
|
524
526
|
return [];
|
|
525
527
|
const stallCandidateKeys = new Set(stallCandidates.map(({ task, role }) => (`${task.id}\0${role.name}`)));
|
|
@@ -689,7 +691,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
689
691
|
&& sessionUsable
|
|
690
692
|
&& expectedResourceIdentity !== undefined
|
|
691
693
|
&& resourceEvidenceMatchesCurrentRun(resourceSnapshot, expectedResourceIdentity, progressAt)
|
|
692
|
-
&& resourceEvidenceIsFresh(resourceSnapshot, now, windowMs);
|
|
694
|
+
&& resourceEvidenceIsFresh(resourceSnapshot, now, windowMs, diagnosticAfterMs);
|
|
693
695
|
const resource = resourceIsCurrent ? resourceSnapshot : undefined;
|
|
694
696
|
const health = projectRoleRunHealth({
|
|
695
697
|
progressAt,
|
|
@@ -699,6 +701,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
699
701
|
: { deliveredAt: candidate.run.deliveredAt }),
|
|
700
702
|
now,
|
|
701
703
|
windowMs,
|
|
704
|
+
diagnosticAfterMs,
|
|
702
705
|
hostLiveness: live,
|
|
703
706
|
nativeSession: candidate.session,
|
|
704
707
|
providerAcceptance: candidate.run.deliveredAt === undefined
|
|
@@ -825,7 +828,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
825
828
|
}
|
|
826
829
|
return raised;
|
|
827
830
|
}
|
|
828
|
-
function isStallCandidate(run, now, windowMs) {
|
|
831
|
+
function isStallCandidate(run, now, windowMs, diagnosticAfterMs) {
|
|
829
832
|
const startAt = run.deliveredAt ?? run.createdAt;
|
|
830
833
|
const ageMs = now.getTime() - Date.parse(startAt);
|
|
831
834
|
if (!Number.isFinite(ageMs))
|
|
@@ -834,7 +837,7 @@ function isStallCandidate(run, now, windowMs) {
|
|
|
834
837
|
// window, but they never enter workflow-stall candidate filtering.
|
|
835
838
|
return run.deliveredAt === undefined
|
|
836
839
|
? ageMs >= windowMs
|
|
837
|
-
: ageMs >=
|
|
840
|
+
: ageMs >= diagnosticAfterMs;
|
|
838
841
|
}
|
|
839
842
|
function classifyLeaderStall(store, taskId, observed, now, windowMs) {
|
|
840
843
|
if (store.hasOpenInputRequest(taskId))
|
|
@@ -1020,7 +1023,7 @@ function resourceEvidenceMatchesCurrentRun(resource, expected, progressAt) {
|
|
|
1020
1023
|
function hasResourceIdentityText(value) {
|
|
1021
1024
|
return typeof value === "string" && value.trim().length > 0;
|
|
1022
1025
|
}
|
|
1023
|
-
function resourceEvidenceIsFresh(evidence, now, windowMs) {
|
|
1026
|
+
function resourceEvidenceIsFresh(evidence, now, windowMs, diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS) {
|
|
1024
1027
|
if (evidence === undefined)
|
|
1025
1028
|
return false;
|
|
1026
1029
|
const observedAt = Date.parse(evidence.observedAt);
|
|
@@ -1028,5 +1031,5 @@ function resourceEvidenceIsFresh(evidence, now, windowMs) {
|
|
|
1028
1031
|
return false;
|
|
1029
1032
|
// A sample from an earlier scheduler window is not a current health signal.
|
|
1030
1033
|
return observedAt <= now.getTime()
|
|
1031
|
-
&& now.getTime() - observedAt < Math.max(windowMs,
|
|
1034
|
+
&& now.getTime() - observedAt < Math.max(windowMs, diagnosticAfterMs);
|
|
1032
1035
|
}
|