@zq-silk/yui 2.16.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commandCatalog.js +19 -12
- package/dist/cli/interactionCandidates.js +20 -5
- package/dist/cli/interactionPolicy.js +5 -4
- package/dist/cli/roleWizard.js +7 -0
- package/dist/commands/controllerCommands.js +13 -4
- package/dist/commands/durableJobCommands.js +50 -3
- package/dist/commands/globalRoleCommands.js +31 -14
- package/dist/commands/inboxCommands.js +1 -0
- package/dist/commands/operatorCommands.js +12 -2
- package/dist/commands/profileCommands.js +19 -2
- package/dist/commands/roleConfiguration.js +3 -0
- package/dist/commands/taskCommands.js +16 -0
- package/dist/commands/taskRoleRuntimeStatus.js +3 -0
- package/dist/controlPlaneCli.js +21 -4
- package/dist/controller/agentJobControl.js +9 -1
- package/dist/controller/clientRuntime.js +34 -13
- package/dist/controller/controller.js +35 -22
- package/dist/controller/diagnosticJournal.js +214 -0
- package/dist/controller/fileSchedulerStoreAdapter.js +174 -105
- package/dist/controller/globalInputDelivery.js +19 -9
- package/dist/controller/globalRuntimeAttention.js +11 -3
- package/dist/controller/jobControl.js +1 -1
- package/dist/controller/jobDefinitionControl.js +24 -13
- package/dist/controller/jobSupervisor.js +4 -0
- package/dist/controller/providerRetryAdmission.js +9 -0
- package/dist/controller/providerRetryDelivery.js +16 -0
- package/dist/controller/runtime.js +80 -4
- package/dist/controller/runtimeLaunchCoordinator.js +3 -1
- package/dist/controller/workerSchedulerStore.js +24 -0
- package/dist/core/boundedRpc.js +4 -0
- package/dist/core/controllerServer.js +1 -1
- package/dist/core/controllerTelemetry.js +92 -19
- package/dist/executor/fileRoleLaunchPlanner.js +3 -0
- package/dist/job/globalShell.js +51 -0
- package/dist/job/jobBusinessResult.js +4 -3
- package/dist/job/jobDefinition.js +6 -1
- package/dist/job/jobRunner.js +9 -0
- package/dist/job/scheduleService.js +2 -0
- package/dist/message/globalInboxRecovery.js +30 -19
- package/dist/message/globalProviderRetry.js +7 -1
- package/dist/message/messageContinuation.js +4 -0
- package/dist/output/rolePresentation.js +3 -0
- package/dist/profile/agentProfile.js +8 -0
- package/dist/profile/agentProfileRuntime.js +16 -1
- package/dist/review/reviewDecision.js +6 -0
- package/dist/role/availability.js +28 -0
- package/dist/role/role.js +6 -0
- package/dist/runtime/builtinAgentErrorMappers.js +4 -0
- package/dist/runtime/codexNativeAccess.js +21 -4
- package/dist/runtime/globalSessionHealth.js +39 -0
- package/dist/runtime/tmuxAdapters.js +5 -0
- package/dist/scheduler/activeRoleRunDelivery.js +9 -0
- package/dist/scheduler/leaderWakeupProcessor.js +8 -1
- package/dist/scheduler/roleRunLiveness.js +3 -0
- package/dist/scheduler/roleRunStall.js +10 -10
- package/dist/storage/persistenceWorker.js +41 -19
- package/dist/storage/sqliteSchema.js +43 -9
- package/dist/storage/sqliteStore.js +86 -8
- package/dist/storage/storageVersions.js +1 -1
- package/dist/storage/storeRpc.js +25 -11
- package/dist/storage/taskReadRevisionSchema.js +39 -0
- package/dist/web/assets/client/app/catalog.js +0 -29
- package/dist/web/assets/client/app/jobs.js +115 -0
- package/dist/web/assets/client/app/main.js +32 -28
- package/dist/web/assets/client/app/manage.js +1 -0
- package/dist/web/assets/client/app/settings.js +1 -5
- package/dist/web/assets/client/app/taskView.js +7 -11
- package/dist/web/assets/client/i18n/en.js +265 -36
- package/dist/web/assets/client/i18n/zh-CN.js +264 -35
- package/dist/web/assets/client/index.js +4 -0
- package/dist/web/assets/client/layout/manageShell.js +61 -15
- package/dist/web/assets/client/layout/router.js +2 -1
- package/dist/web/assets/client/layout/shortcuts.js +2 -2
- package/dist/web/assets/client/layout/workspace.js +99 -112
- package/dist/web/assets/client/lib/prefs.js +1 -1
- package/dist/web/assets/client/ui/primitives.js +15 -2
- package/dist/web/assets/client/views/dock/conversation.js +421 -123
- package/dist/web/assets/client/views/dock/session.js +229 -0
- package/dist/web/assets/client/views/dock/terminal.js +48 -39
- package/dist/web/assets/client/views/jobs.js +1042 -160
- package/dist/web/assets/client/views/overview.js +2 -2
- package/dist/web/assets/client/views/sidebar.js +3 -0
- package/dist/web/assets/client/views/task/page.js +2 -12
- package/dist/web/assets/shared/manageNav.js +3 -1
- package/dist/web/assets/shell/appRail.js +2 -1
- package/dist/web/assets/shell/dock.js +9 -27
- package/dist/web/assets/shell/manage.js +7 -2
- package/dist/web/assets/shell/models.js +4 -3
- package/dist/web/assets/shell/settingsContent.js +0 -3
- package/dist/web/assets/shell/sidebar.js +0 -1
- package/dist/web/assets/styles/views/conversation.js +96 -17
- package/dist/web/assets/styles/views/dock.js +17 -5
- package/dist/web/assets/styles/views/jobs.js +301 -12
- package/dist/web/assets/styles/views/sidebar.js +1 -2
- package/dist/web/assets/styles/views/task.js +1 -2
- package/dist/web/assets/styles/views/terminal.js +12 -8
- package/dist/web/webConversation.js +1 -0
- package/dist/web/webServer.js +28 -1
- package/dist/web/webSettings.js +10 -2
- package/docs/global-shell-jobs.md +78 -0
- package/docs/job-runs.md +6 -3
- package/docs/managed-turn-and-session-runtime.md +84 -0
- package/docs/managed-turn-and-session-runtime.zh-CN.md +57 -0
- package/docs/release-workflow.md +67 -0
- package/docs/release-workflow.zh-CN.md +48 -0
- package/docs/roles-and-configuration.md +45 -1
- package/docs/roles-and-configuration.zh-CN.md +35 -0
- package/docs/storage-baseline.md +19 -1
- package/docs/storage-baseline.zh-CN.md +10 -1
- package/docs/testing/review-evaluation.md +35 -1
- package/docs/testing/review-evaluation.zh-CN.md +25 -1
- package/docs/testing/verification-levels.md +2 -2
- package/docs/testing/verification-levels.zh-CN.md +2 -2
- package/package.json +1 -1
- package/skills/yui-leader/VERSION +1 -1
- package/skills/yui-leader/references/execution.md +19 -8
- package/skills/yui-reviewer/SKILL.md +13 -2
- package/skills/yui-reviewer/VERSION +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { usageError } from "../errors/cliError.js";
|
|
2
|
+
import { parseEnabled } from "../role/availability.js";
|
|
2
3
|
import { defaultRoleAgentConfig, resolveAgentAdapter } from "../executor/agentAdapter.js";
|
|
3
4
|
const PROFILE_OPTIONS = [
|
|
4
5
|
["--description", "value"],
|
|
@@ -64,6 +65,7 @@ export function roleOptionSpecs(input) {
|
|
|
64
65
|
...(input.includeAgent === true ? [["--agent", "value"]] : []),
|
|
65
66
|
...(input.includeWorkspace === true ? [["--workspace", "value"]] : []),
|
|
66
67
|
...(input.update ? [["--yes", "flag"]] : []),
|
|
68
|
+
...(input.update ? [["--enabled", "value"]] : []),
|
|
67
69
|
...PROFILE_OPTIONS,
|
|
68
70
|
...agentValueOptions,
|
|
69
71
|
...agentRepeatableOptions,
|
|
@@ -123,6 +125,7 @@ export function roleProfilePatch(parsed) {
|
|
|
123
125
|
["--skill", "--clear-skills"]
|
|
124
126
|
]);
|
|
125
127
|
return {
|
|
128
|
+
...(parsed.has("--enabled") ? { enabled: parseEnabled(parsed.one("--enabled")) } : {}),
|
|
126
129
|
...(parsed.has("--description") ? { description: requiredText(parsed.one("--description")) } : {}),
|
|
127
130
|
...(parsed.has("--clear-description") ? { description: undefined } : {}),
|
|
128
131
|
...(parsed.has("--responsibility")
|
|
@@ -3,6 +3,7 @@ import { isDeepStrictEqual } from "node:util";
|
|
|
3
3
|
import { taskMessageInboxReceipt } from "../message/taskInboxOwnership.js";
|
|
4
4
|
import { agentExecutionComponentLabel } from "../agent/executionComponents.js";
|
|
5
5
|
import { assertLeaderArchiveAdmission } from "../task/leaderArchiveAuthority.js";
|
|
6
|
+
import { assertRoleEnabled, isEnabled } from "../role/availability.js";
|
|
6
7
|
import { createRun, runExecutionObservation, runPurposeAdmitsTaskState } from "../agentRun/agentRun.js";
|
|
7
8
|
import { isGitArtifactRefString, parseGitArtifactRef } from "../artifacts/gitArtifactRef.js";
|
|
8
9
|
import { contextContentDigest, contextSnapshotRef } from "../context/contextSnapshot.js";
|
|
@@ -2251,6 +2252,11 @@ function taskRoleCommand(args, store, options) {
|
|
|
2251
2252
|
return showTaskRole(rest, store);
|
|
2252
2253
|
if (command === "update")
|
|
2253
2254
|
return updateTaskRole(rest, store, options);
|
|
2255
|
+
if (command === "enable" || command === "disable") {
|
|
2256
|
+
if (rest.length !== 2)
|
|
2257
|
+
throw usageError(`Usage: yui task role ${command} <task> <role>`);
|
|
2258
|
+
return updateTaskRole([...rest, "--enabled", String(command === "enable")], store, options);
|
|
2259
|
+
}
|
|
2254
2260
|
if (command === "remove")
|
|
2255
2261
|
return output(removeTaskRole(rest, store, options));
|
|
2256
2262
|
if (command === "bind")
|
|
@@ -2751,6 +2757,7 @@ function listTaskRoles(args, store, options) {
|
|
|
2751
2757
|
const statuses = inspectTaskRoleRuntimeStatuses(task.id, roles, store, options.runtime?.inspectTaskRolePanes?.(task.id) ?? [], options.now?.() ?? new Date());
|
|
2752
2758
|
const data = recordPage(statuses.map(status => ({
|
|
2753
2759
|
roleName: status.roleName, agentId: status.agentId, recordedHealth: status.health,
|
|
2760
|
+
enabled: isEnabled(status.role),
|
|
2754
2761
|
hostObservation: "not-requested",
|
|
2755
2762
|
openInput: taskRoleOpenInputLabel(status), activeWork: taskRoleActiveWorkLabel(status),
|
|
2756
2763
|
lastRun: taskRoleLastRunLabel(status), nativeSession: taskRoleNativeSessionLabel(status),
|
|
@@ -2810,6 +2817,9 @@ function updateTaskRole(args, store, options) {
|
|
|
2810
2817
|
taskActor(tx, options, task.id);
|
|
2811
2818
|
const role = requireRole(tx, task.id, roleName);
|
|
2812
2819
|
const changesEnvironment = parsed.has("--environment") || parsed.has("--managed-environment");
|
|
2820
|
+
if (parsed.seen.size === 1 && parsed.has("--enabled")
|
|
2821
|
+
&& isEnabled(role) === roleProfilePatch(parsed).enabled)
|
|
2822
|
+
return role;
|
|
2813
2823
|
const changesLaunchContext = hasRoleLaunchContextOptions(parsed) || parsed.has("--profile")
|
|
2814
2824
|
|| changesEnvironment;
|
|
2815
2825
|
const changesAgentConfig = hasAgentConfigOptions(parsed);
|
|
@@ -3562,6 +3572,7 @@ function dispatchWork(args, store, options) {
|
|
|
3562
3572
|
}
|
|
3563
3573
|
const roles = lanePlan.roles.map((name) => requireRole(tx, task.id, name));
|
|
3564
3574
|
for (const role of roles) {
|
|
3575
|
+
assertRoleEnabled(role);
|
|
3565
3576
|
if (tx.getActiveRun(task.id, role.name) !== null) {
|
|
3566
3577
|
throw usageError(`${task.id}/${role.name} already has an active run.`);
|
|
3567
3578
|
}
|
|
@@ -3570,6 +3581,7 @@ function dispatchWork(args, store, options) {
|
|
|
3570
3581
|
let workItemForDispatch = prepareWorkItemDispatch(item, now);
|
|
3571
3582
|
if (lanePlan.roles.length === 0) {
|
|
3572
3583
|
const role = requireRole(tx, task.id, assignee);
|
|
3584
|
+
assertRoleEnabled(role);
|
|
3573
3585
|
if (tx.getActiveRun(task.id, role.name) !== null) {
|
|
3574
3586
|
throw usageError(`${task.id}/${role.name} already has an active run.`);
|
|
3575
3587
|
}
|
|
@@ -4383,6 +4395,9 @@ function assertTaskReviewRequestLane(store, taskId, reviewerRoleName, reusableRo
|
|
|
4383
4395
|
assertReviewerAvailable(store, taskId, reviewerRoleName, reusableRound);
|
|
4384
4396
|
}
|
|
4385
4397
|
function assertReviewerAvailable(store, taskId, reviewerRoleName, reusableRound) {
|
|
4398
|
+
const role = store.getRole(taskId, reviewerRoleName);
|
|
4399
|
+
if (role !== null)
|
|
4400
|
+
assertRoleEnabled(role);
|
|
4386
4401
|
const availability = projectReviewerAvailability(store, taskId, reviewerRoleName);
|
|
4387
4402
|
if (availability.kind === "available")
|
|
4388
4403
|
return;
|
|
@@ -5001,6 +5016,7 @@ function retryRunOperation(args, store, options) {
|
|
|
5001
5016
|
throw usageError(`Task does not admit ${previous.purpose} retry: ${task.id}/${task.status}.`);
|
|
5002
5017
|
}
|
|
5003
5018
|
const role = requireRole(tx, task.id, previous.roleName);
|
|
5019
|
+
assertRoleEnabled(role);
|
|
5004
5020
|
if (tx.getActiveRun(task.id, role.name) !== null) {
|
|
5005
5021
|
throw usageError(`${task.id}/${role.name} already has an active run.`);
|
|
5006
5022
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from "node:util";
|
|
2
2
|
import { runExecutionObservation } from "../agentRun/agentRun.js";
|
|
3
|
+
import { isEnabled, roleDisabledReason } from "../role/availability.js";
|
|
3
4
|
import { hasRuntimeCleanupObligation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
4
5
|
import { currentProviderConversation, managedProviderTurnId } from "../runtime/providerRuntimeIdentity.js";
|
|
5
6
|
import { isRoleRunStalled, latestStallProgressAt } from "../scheduler/roleRunStall.js";
|
|
@@ -118,6 +119,8 @@ export function renderTaskRoleRuntimeStatus(status) {
|
|
|
118
119
|
`Task Role status: ${status.taskId}/${status.roleName}`,
|
|
119
120
|
"",
|
|
120
121
|
` Health ${status.health}`,
|
|
122
|
+
` Enabled ${isEnabled(status.role)}`,
|
|
123
|
+
...(!isEnabled(status.role) ? [` Availability ${roleDisabledReason(status.role)}`] : []),
|
|
121
124
|
` Reason ${status.healthReason}`,
|
|
122
125
|
` Open inputs ${status.openInputRequestCount}`,
|
|
123
126
|
` Agent ${status.agentId}`,
|
package/dist/controlPlaneCli.js
CHANGED
|
@@ -532,8 +532,9 @@ export async function main() {
|
|
|
532
532
|
const result = await callController(home, "controller.status", {}, { timeoutMs: 2_000 });
|
|
533
533
|
runtime = parseControllerRuntimeSnapshot(result, droppedEvents);
|
|
534
534
|
}
|
|
535
|
-
catch {
|
|
536
|
-
runtime = { source: "unsupported", droppedEvents
|
|
535
|
+
catch (error) {
|
|
536
|
+
runtime = { source: "unsupported", droppedEvents,
|
|
537
|
+
error: describeCliFailure(error, failureContext()).message.slice(0, 1000) };
|
|
537
538
|
}
|
|
538
539
|
const mismatch = summarizeDurablePhysicalMismatch(snapshot);
|
|
539
540
|
const identitySection = renderRuntimeIdentitySection({
|
|
@@ -2611,6 +2612,7 @@ async function executeOperatorSessionControl(control, home, store, runtime, tmux
|
|
|
2611
2612
|
}
|
|
2612
2613
|
await prepareInteractiveCodexApp(control.targetAgentId, store, home);
|
|
2613
2614
|
await ensureFileTaskController(home, { environment: process.env });
|
|
2615
|
+
let stopDisposition = "not-required";
|
|
2614
2616
|
if (paneRunning
|
|
2615
2617
|
|| (active !== undefined
|
|
2616
2618
|
&& active.status === "active")
|
|
@@ -2618,9 +2620,24 @@ async function executeOperatorSessionControl(control, home, store, runtime, tmux
|
|
|
2618
2620
|
&& ["submitting", "accepted", "delivery-unknown"].includes(sessionSet.providerBinding.run.status))
|
|
2619
2621
|
|| (sessionSet?.providerBinding?.goal != null && sessionSet.providerBinding.goal.status !== "complete")) {
|
|
2620
2622
|
await runtime.stopGlobalRoleSession(role.name);
|
|
2623
|
+
stopDisposition = "completed";
|
|
2624
|
+
}
|
|
2625
|
+
try {
|
|
2626
|
+
applyOperatorSessionControl(control, store);
|
|
2627
|
+
}
|
|
2628
|
+
catch (error) {
|
|
2629
|
+
throw new Error(`Operator replacement selection failed; prior stop=${stopDisposition}; `
|
|
2630
|
+
+ "new Session startup has not begun. Inspect the retained Session state.", { cause: error });
|
|
2631
|
+
}
|
|
2632
|
+
try {
|
|
2633
|
+
await runtime.prepareGlobalRoleEnter(role.name);
|
|
2634
|
+
}
|
|
2635
|
+
catch (error) {
|
|
2636
|
+
throw new Error(`Operator replacement selection committed; prior stop=${stopDisposition}; its startup phase failed `
|
|
2637
|
+
+ "without readiness confirmation. Whether native startup began is unconfirmed. "
|
|
2638
|
+
+ "Inspect the exact current Session/Host before retrying; "
|
|
2639
|
+
+ "do not infer that startup had no effect.", { cause: error });
|
|
2621
2640
|
}
|
|
2622
|
-
applyOperatorSessionControl(control, store);
|
|
2623
|
-
await runtime.prepareGlobalRoleEnter(role.name);
|
|
2624
2641
|
tmux.attachRole("operator", role.name, "auto");
|
|
2625
2642
|
}
|
|
2626
2643
|
async function prepareInteractiveCodexApp(agentId, store, home) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { assertRoleEnabled } from "../role/availability.js";
|
|
2
3
|
import { mkdirSync } from "node:fs";
|
|
3
4
|
import { isDeepStrictEqual } from "node:util";
|
|
4
5
|
import { join } from "node:path";
|
|
@@ -63,7 +64,8 @@ function requireTaskTarget(store, target, runnable = true) {
|
|
|
63
64
|
const task = store.getTask(target.taskId);
|
|
64
65
|
if (!task || runnable && (task.status !== "active" || task.executionGate.state !== "enabled"))
|
|
65
66
|
throw new Error("Task Agent requires an active, execution-enabled Task.");
|
|
66
|
-
|
|
67
|
+
const role = store.getRole(target.taskId, target.roleName);
|
|
68
|
+
if (!role)
|
|
67
69
|
throw new Error("Task Agent Role is unavailable.");
|
|
68
70
|
const recipient = taskRecipient(store, target);
|
|
69
71
|
if (recipient.ownerRunId) {
|
|
@@ -117,6 +119,7 @@ function ownedRole(store, home, job, runId, now) {
|
|
|
117
119
|
const jobOwner = { scope: job.scope, jobId: job.id, ...(isolated ? { runId } : {}) };
|
|
118
120
|
const existing = store.getGlobalRole(name);
|
|
119
121
|
if (existing) {
|
|
122
|
+
assertRoleEnabled(existing);
|
|
120
123
|
if (!isDeepStrictEqual(existing.jobOwner, jobOwner) || existing.activeAgentId !== job.configuration.target.agentId) {
|
|
121
124
|
throw new Error("Job Role binding changed; inspect its Session/configuration before dispatch.");
|
|
122
125
|
}
|
|
@@ -155,6 +158,9 @@ function queueAttempt(store, home, job, runId, index, now) {
|
|
|
155
158
|
const inputControl = { action: "queue", requestId: `job-agent:${runId}:${index}` };
|
|
156
159
|
let messageId;
|
|
157
160
|
if (recipient.scope === "global") {
|
|
161
|
+
const role = store.getGlobalRole(recipient.roleName);
|
|
162
|
+
if (role !== null)
|
|
163
|
+
assertRoleEnabled(role);
|
|
158
164
|
messageId = store.nextGlobalRoleMessageId();
|
|
159
165
|
let message = createGlobalRoleMessage(messageId, recipient.roleName, body, "system", { type: "system" }, now, { inputControl });
|
|
160
166
|
const set = store.getGlobalRoleSessionSet(recipient.roleName);
|
|
@@ -165,6 +171,7 @@ function queueAttempt(store, home, job, runId, index, now) {
|
|
|
165
171
|
}
|
|
166
172
|
else {
|
|
167
173
|
requireTaskTarget(store, { ...recipient });
|
|
174
|
+
assertRoleEnabled(store.getRole(recipient.taskId, recipient.roleName));
|
|
168
175
|
const resolved = taskRecipient(store, recipient);
|
|
169
176
|
messageId = store.nextMessageId(recipient.taskId);
|
|
170
177
|
store.saveMessage(recipient.taskId, createTaskMessage(messageId, recipient.taskId, body, "system", { type: "system" }, now, { ...(resolved.ownerRunId ? { recipient: resolved } : {}), inputControl }));
|
|
@@ -256,6 +263,7 @@ export function controlAgentJob(store, home, job, input, now) {
|
|
|
256
263
|
if (op === "session-new") {
|
|
257
264
|
if (!role)
|
|
258
265
|
throw new Error("Create the Job Role first.");
|
|
266
|
+
assertRoleEnabled(role);
|
|
259
267
|
assertRoleRuntimeMutationAllowed(store, { scope: "global", roleName }, "Job Session replacement");
|
|
260
268
|
const set = store.getGlobalRoleSessionSet(roleName);
|
|
261
269
|
const old = set?.sessions[set.activeAgentId];
|
|
@@ -456,10 +456,7 @@ export class FileTaskWorkflowRuntime {
|
|
|
456
456
|
}
|
|
457
457
|
if (targets.length === 0)
|
|
458
458
|
return;
|
|
459
|
-
await
|
|
460
|
-
...this.clientOptions,
|
|
461
|
-
requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
|
|
462
|
-
});
|
|
459
|
+
await Promise.all(targets.map(target => this.#waitForRuntimeCleanup(target)));
|
|
463
460
|
for (const target of targets) {
|
|
464
461
|
if (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
|
|
465
462
|
throw new Error(`Role runtime did not stop: ${target.taskId}/${target.roleName}.`);
|
|
@@ -487,10 +484,7 @@ export class FileTaskWorkflowRuntime {
|
|
|
487
484
|
if (target === null) {
|
|
488
485
|
throw new Error(`Role Session changed before its exact stop was reserved: ${input.taskId}/${input.roleName}.`);
|
|
489
486
|
}
|
|
490
|
-
await
|
|
491
|
-
...this.clientOptions,
|
|
492
|
-
requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
|
|
493
|
-
});
|
|
487
|
+
await this.#waitForRuntimeCleanup(target);
|
|
494
488
|
if (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
|
|
495
489
|
throw new Error(`Role runtime did not stop: ${input.taskId}/${input.roleName}.`);
|
|
496
490
|
}
|
|
@@ -506,7 +500,7 @@ export class FileTaskWorkflowRuntime {
|
|
|
506
500
|
.filter((job) => job.taskId === taskId);
|
|
507
501
|
if (active.length === 0)
|
|
508
502
|
return;
|
|
509
|
-
await callFileTaskController(this.home, "scheduler.
|
|
503
|
+
await callFileTaskController(this.home, "scheduler.signal", { key: `task:${encodeURIComponent(taskId)}` }, {
|
|
510
504
|
...this.clientOptions,
|
|
511
505
|
requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
|
|
512
506
|
});
|
|
@@ -585,10 +579,7 @@ export class FileTaskWorkflowRuntime {
|
|
|
585
579
|
if (target?.kind !== "global-role-runtime") {
|
|
586
580
|
throw new Error(`Global Role runtime cleanup target is invalid: ${roleName}.`);
|
|
587
581
|
}
|
|
588
|
-
await
|
|
589
|
-
...this.clientOptions,
|
|
590
|
-
requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
|
|
591
|
-
});
|
|
582
|
+
await this.#waitForRuntimeCleanup(target);
|
|
592
583
|
if (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
|
|
593
584
|
throw new Error(`Global Role runtime did not stop: ${roleName}.`);
|
|
594
585
|
}
|
|
@@ -598,6 +589,36 @@ export class FileTaskWorkflowRuntime {
|
|
|
598
589
|
throw new Error(`Global Role runtime session is still active: ${roleName}.`);
|
|
599
590
|
}
|
|
600
591
|
}
|
|
592
|
+
/**
|
|
593
|
+
* The durable cleanup mailbox is completed only after stopOwner proves
|
|
594
|
+
* quiescence and commits the existing cleanup transaction. A scheduler hint
|
|
595
|
+
* is not that proof; nor is exhaustion of the Home-wide coalescing loop.
|
|
596
|
+
*/
|
|
597
|
+
async #waitForRuntimeCleanup(target) {
|
|
598
|
+
const timeoutMs = this.clientOptions.requestTimeoutMs ?? LIFECYCLE_REQUEST_TIMEOUT_MS;
|
|
599
|
+
const deadline = Date.now() + timeoutMs;
|
|
600
|
+
const label = controllerMailboxKey(target);
|
|
601
|
+
try {
|
|
602
|
+
await callFileTaskController(this.home, "scheduler.signal", { key: label }, {
|
|
603
|
+
...this.clientOptions, requestTimeoutMs: timeoutMs
|
|
604
|
+
});
|
|
605
|
+
while (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
|
|
606
|
+
if (Date.now() >= deadline) {
|
|
607
|
+
throw new Error("Timed out awaiting the target's durable cleanup transaction.");
|
|
608
|
+
}
|
|
609
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(this.clientOptions.pollIntervalMs ?? POLL_INTERVAL_MS, Math.max(1, deadline - Date.now()))));
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
catch (error) {
|
|
613
|
+
// A lost hint response must not hide cleanup that actually committed.
|
|
614
|
+
// No hint or native input is replayed here.
|
|
615
|
+
if (!hasRuntimeLifecycleWork(this.store.getWorkMailbox(target)))
|
|
616
|
+
return;
|
|
617
|
+
throw new Error(`Runtime stop remains unconfirmed for ${label}; cleanup obligation retained. `
|
|
618
|
+
+ "No replacement was started by this stop operation. Inspect the exact Session/Host "
|
|
619
|
+
+ "and cleanup failure before retrying; no physical exit is inferred.", { cause: error });
|
|
620
|
+
}
|
|
621
|
+
}
|
|
601
622
|
/** Drain pending runtime facts while an external maintenance fence is held. */
|
|
602
623
|
async drainController() {
|
|
603
624
|
await ensureFileTaskController(this.home, {
|
|
@@ -7,7 +7,7 @@ import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
|
|
|
7
7
|
import { DEFAULT_STALL_WINDOW_MS, DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
|
|
8
8
|
import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
|
|
9
9
|
import { startControllerServer } from "../core/controllerServer.js";
|
|
10
|
-
import { monotonicMilliseconds } from "../core/controllerTelemetry.js";
|
|
10
|
+
import { ControllerWorkObserver, monotonicMilliseconds } from "../core/controllerTelemetry.js";
|
|
11
11
|
import { isProjectMaintenanceFenced } from "../repository/projectMaintenanceLock.js";
|
|
12
12
|
import { isHandoverLockHeld } from "../release/runtimeRelease.js";
|
|
13
13
|
import { KeyedWorkQueue } from "../coordination/keyedWorkQueue.js";
|
|
@@ -44,7 +44,9 @@ const ZERO_DRAIN_METRICS = Object.freeze({
|
|
|
44
44
|
* Runs one lean scheduler pass. Due native Turn completions are folded before
|
|
45
45
|
* liveness, so a valid Hook boundary fences destructive process reconciliation.
|
|
46
46
|
*/
|
|
47
|
-
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, leaderWakeFence, onError) {
|
|
47
|
+
export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, leaderWakeFence, onError, observer) {
|
|
48
|
+
const target = scope.kind === "full" ? "home" : scope.keys.slice(0, 16).join(",");
|
|
49
|
+
const observe = (phase, action) => observer === undefined ? Promise.resolve().then(action) : observer.observe(phase, target, action);
|
|
48
50
|
const compiledSelection = compileReconcileSelection(scope);
|
|
49
51
|
const selection = includeOperator
|
|
50
52
|
? { ...compiledSelection, blockedTaskIds }
|
|
@@ -54,7 +56,7 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
|
|
|
54
56
|
// event-loop turn. Give already-written requests a poll boundary before the
|
|
55
57
|
// next durable phase; later phases retain their existing CAS fences.
|
|
56
58
|
await controlEventLoopRun();
|
|
57
|
-
const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes, blockedTaskIds);
|
|
59
|
+
const failedCleanupRoles = await observe("runtime-cleanup", () => processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes, blockedTaskIds));
|
|
58
60
|
const roleSelection = selectionWithoutFailedCleanupRoles(store, selection, failedCleanupRoles);
|
|
59
61
|
const availableWakeupSelection = () => (leaderWakeFence?.() === true
|
|
60
62
|
? exactTaskSelection(new Set())
|
|
@@ -64,19 +66,19 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
|
|
|
64
66
|
// Durable Leader work that already has a ready Task workspace belongs to
|
|
65
67
|
// the control path. Dispatch it before any unrelated Task workspace I/O;
|
|
66
68
|
// the processor itself retains the fail-closed workspace-ready guard.
|
|
67
|
-
const initialWakeups = selectedPendingWakeups(store, availableWakeupSelection());
|
|
68
|
-
const initialWakeupResults = await processLeaderWakeups(store, delivery, now, exactTaskSelection(new Set(initialWakeups.keys())));
|
|
69
|
+
const initialWakeups = await observe("wakeup-projection", () => selectedPendingWakeups(store, availableWakeupSelection()));
|
|
70
|
+
const initialWakeupResults = await observe("leader-delivery", () => processLeaderWakeups(store, delivery, now, exactTaskSelection(new Set(initialWakeups.keys()))));
|
|
69
71
|
// Preserve ready-Leader-first ordering, then bound the repeated state
|
|
70
72
|
// projections that follow it in this pass.
|
|
71
73
|
await controlEventLoopRun();
|
|
72
74
|
// A Task whose deferred activation was just released must become active
|
|
73
75
|
// before the workspace phase, so its first workspace preparation happens in
|
|
74
76
|
// this same pass rather than waiting for the next one.
|
|
75
|
-
await adoptReleasedTaskActivations(store, workspacePreparer, selection, onError);
|
|
77
|
+
await observe("activation", () => adoptReleasedTaskActivations(store, workspacePreparer, selection, onError));
|
|
76
78
|
await controlEventLoopRun();
|
|
77
|
-
const workspacePreparation = await prepareActiveWorkspaces(store, workspacePreparer, selection, maintenanceFence, onMaintenanceFenceDefer);
|
|
79
|
+
const workspacePreparation = await observe("workspace-preparation", () => prepareActiveWorkspaces(store, workspacePreparer, selection, maintenanceFence, onMaintenanceFenceDefer));
|
|
78
80
|
await controlEventLoopRun();
|
|
79
|
-
const activeRunDeliveries = await processActiveRoleRunDeliveries(store, delivery, now, roleSelection);
|
|
81
|
+
const activeRunDeliveries = await observe("run-delivery", () => processActiveRoleRunDeliveries(store, delivery, now, roleSelection));
|
|
80
82
|
await controlEventLoopRun();
|
|
81
83
|
const unsettledRunRefs = new Set(activeRunDeliveries.flatMap((result) => (result.reason === "delivery-uncertain"
|
|
82
84
|
? [formatTaskRecordReference(result.taskId, result.runId, "run")]
|
|
@@ -108,16 +110,16 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
|
|
|
108
110
|
await controlEventLoopRun();
|
|
109
111
|
const liveStatuses = new Map();
|
|
110
112
|
const resourceEvidence = new Map();
|
|
111
|
-
const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty");
|
|
113
|
+
const failedRunRefs = await observe("run-liveness", () => reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty"));
|
|
112
114
|
await controlEventLoopRun();
|
|
113
115
|
await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, diagnosticAfterMs);
|
|
114
116
|
await controlEventLoopRun();
|
|
115
117
|
const selectedInputTaskIds = selectedTaskIdsForBoundedPass(store, selection);
|
|
116
118
|
const autoResolvedInputs = selectedInputTaskIds === undefined
|
|
117
|
-
? store.resolveExpiredInputRecommendations(now)
|
|
119
|
+
? await store.resolveExpiredInputRecommendations(now)
|
|
118
120
|
: selectedInputTaskIds.size === 0
|
|
119
121
|
? []
|
|
120
|
-
: store.resolveExpiredInputRecommendations(now, selectedInputTaskIds);
|
|
122
|
+
: await store.resolveExpiredInputRecommendations(now, selectedInputTaskIds);
|
|
121
123
|
// Liveness and auto-resolution can durably queue new Leader work. Process
|
|
122
124
|
// only Tasks that were not part of phase one, except when a same-pass due
|
|
123
125
|
// completion made an initially busy Leader idle. Result order remains
|
|
@@ -694,7 +696,10 @@ export class FileTaskController {
|
|
|
694
696
|
#providerRetry;
|
|
695
697
|
#continuationReconciler;
|
|
696
698
|
#hostTerminalRecovery;
|
|
699
|
+
#persistenceMetrics;
|
|
700
|
+
#diagnosticMetrics;
|
|
697
701
|
#current;
|
|
702
|
+
#phases = new ControllerWorkObserver();
|
|
698
703
|
#operatorCurrent;
|
|
699
704
|
#runtimeObserverCurrent;
|
|
700
705
|
#pendingFull = false;
|
|
@@ -712,6 +717,8 @@ export class FileTaskController {
|
|
|
712
717
|
constructor(store, delivery, options = {}) {
|
|
713
718
|
this.store = store;
|
|
714
719
|
this.delivery = delivery;
|
|
720
|
+
this.#persistenceMetrics = options.persistenceMetrics;
|
|
721
|
+
this.#diagnosticMetrics = options.diagnosticMetrics;
|
|
715
722
|
this.#intervalMs = positiveInteger(options.intervalMs, DEFAULT_RECONCILIATION_INTERVAL_MS, "Controller reconciliation interval");
|
|
716
723
|
this.#now = options.now ?? (() => new Date());
|
|
717
724
|
this.#onError = options.onError ?? (() => { });
|
|
@@ -782,6 +789,12 @@ export class FileTaskController {
|
|
|
782
789
|
runtimeMetrics() {
|
|
783
790
|
const metrics = this.#lastRuntimeDrain?.metrics ?? ZERO_DRAIN_METRICS;
|
|
784
791
|
return {
|
|
792
|
+
diagnostics: this.#diagnosticMetrics?.() ?? null,
|
|
793
|
+
phases: this.#phases.snapshot(),
|
|
794
|
+
persistence: this.#persistenceMetrics?.() ?? null,
|
|
795
|
+
reads: this.store.readObservationMetrics?.() ?? null,
|
|
796
|
+
scheduler: { running: this.#current !== undefined, pendingFull: this.#pendingFull,
|
|
797
|
+
pendingKeys: this.#pendingKeys.size },
|
|
785
798
|
// Status is intentionally O(1): scanning the inbox while serving the
|
|
786
799
|
// control socket would recreate the starvation this metric diagnoses.
|
|
787
800
|
inbox: {
|
|
@@ -956,7 +969,7 @@ export class FileTaskController {
|
|
|
956
969
|
* without bypassing the scheduler's own ownership and delivery fences. */
|
|
957
970
|
async #observeMaintenance(phase, action) {
|
|
958
971
|
try {
|
|
959
|
-
return await action
|
|
972
|
+
return await this.#phases.observe(phase, "home", action);
|
|
960
973
|
}
|
|
961
974
|
catch (error) {
|
|
962
975
|
this.#onError(new Error(`${phase} failed: ${error instanceof Error ? error.message : String(error)}. `
|
|
@@ -998,12 +1011,12 @@ export class FileTaskController {
|
|
|
998
1011
|
}
|
|
999
1012
|
}
|
|
1000
1013
|
}
|
|
1001
|
-
const firstRuntimeDrain = await this.#drainRuntimeEvents();
|
|
1014
|
+
const firstRuntimeDrain = await this.#phases.observe("runtime-drain", "home", () => this.#drainRuntimeEvents());
|
|
1002
1015
|
if (scope.kind === "full" && this.#hostTerminalRecovery !== undefined) {
|
|
1003
1016
|
await this.#observeMaintenance("Host terminal recovery", this.#hostTerminalRecovery);
|
|
1004
1017
|
}
|
|
1005
|
-
await this.#providerRetry?.reconcile();
|
|
1006
|
-
await this.#globalInputDelivery?.({ reconcile: scope.kind === "full", now: this.#now() });
|
|
1018
|
+
await this.#phases.observe("provider-retry", "home", () => this.#providerRetry?.reconcile());
|
|
1019
|
+
await this.#phases.observe("global-input", "home", () => this.#globalInputDelivery?.({ reconcile: scope.kind === "full", now: this.#now() }));
|
|
1007
1020
|
for (const taskId of runtimeTaskFailureIds(firstRuntimeDrain)) {
|
|
1008
1021
|
runtimeFailedTaskIds.add(taskId);
|
|
1009
1022
|
}
|
|
@@ -1016,17 +1029,17 @@ export class FileTaskController {
|
|
|
1016
1029
|
// DurableJob reconciliation runs before the scheduler pass so a
|
|
1017
1030
|
// terminal job's Leader wakeup is enqueued in the same pass that
|
|
1018
1031
|
// processes Leader wakeups.
|
|
1019
|
-
this.#schedules?.reconcile(this.#now());
|
|
1032
|
+
await this.#schedules?.reconcile(this.#now());
|
|
1020
1033
|
this.#jobSupervisor?.reconcile(this.#now());
|
|
1021
1034
|
if (scope.kind === "full") {
|
|
1022
|
-
result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError);
|
|
1035
|
+
result = await this.#phases.observe("scheduler-full", "home", () => runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError, this.#phases));
|
|
1023
1036
|
}
|
|
1024
1037
|
else {
|
|
1025
|
-
const dirtyPass = await this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds);
|
|
1038
|
+
const dirtyPass = await this.#phases.observe("scheduler-dirty", scope.keys.slice(0, 16).join(","), () => this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds));
|
|
1026
1039
|
result = dirtyPass.result;
|
|
1027
1040
|
failedTaskScopes = dirtyPass.failedTaskScopes;
|
|
1028
1041
|
}
|
|
1029
|
-
const secondRuntimeDrain = await this.#drainRuntimeEvents();
|
|
1042
|
+
const secondRuntimeDrain = await this.#phases.observe("runtime-drain", "home", () => this.#drainRuntimeEvents());
|
|
1030
1043
|
for (const taskId of runtimeTaskFailureIds(secondRuntimeDrain)) {
|
|
1031
1044
|
runtimeFailedTaskIds.add(taskId);
|
|
1032
1045
|
}
|
|
@@ -1040,7 +1053,7 @@ export class FileTaskController {
|
|
|
1040
1053
|
// scheduler.
|
|
1041
1054
|
if (scope.kind === "full" && this.#resourceAutoGc !== undefined) {
|
|
1042
1055
|
try {
|
|
1043
|
-
const gc = await this.#resourceAutoGc
|
|
1056
|
+
const gc = await this.#phases.observe("resource-gc", "home", this.#resourceAutoGc);
|
|
1044
1057
|
if (!gc.skipped && gc.failed > 0) {
|
|
1045
1058
|
this.#onError(new Error(`Resource auto-GC failed to quarantine ${gc.failed} `
|
|
1046
1059
|
+ "resource(s); they stay in place and retry next pass."));
|
|
@@ -1120,7 +1133,7 @@ export class FileTaskController {
|
|
|
1120
1133
|
const taskScopes = partition.taskScopes.filter((taskScope) => (!blockedTaskIds.has(taskScope.taskId)));
|
|
1121
1134
|
const orderedResults = [];
|
|
1122
1135
|
if (partition.globalKeys.length > 0) {
|
|
1123
|
-
orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError));
|
|
1136
|
+
orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError, this.#phases));
|
|
1124
1137
|
}
|
|
1125
1138
|
if (taskScopes.length === 0
|
|
1126
1139
|
|| this.#stopped
|
|
@@ -1153,7 +1166,7 @@ export class FileTaskController {
|
|
|
1153
1166
|
if (this.#stopped || this.#pendingFull)
|
|
1154
1167
|
continue;
|
|
1155
1168
|
try {
|
|
1156
|
-
taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError);
|
|
1169
|
+
taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError, this.#phases);
|
|
1157
1170
|
this.#clearTaskPassRetry(selected.taskScope.taskId);
|
|
1158
1171
|
}
|
|
1159
1172
|
catch (error) {
|