@zq-silk/yui 2.16.0 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/dist/cli/commandCatalog.js +19 -12
  2. package/dist/cli/interactionCandidates.js +20 -5
  3. package/dist/cli/interactionPolicy.js +5 -4
  4. package/dist/cli/roleWizard.js +7 -0
  5. package/dist/commands/controllerCommands.js +13 -4
  6. package/dist/commands/durableJobCommands.js +50 -3
  7. package/dist/commands/globalRoleCommands.js +31 -14
  8. package/dist/commands/inboxCommands.js +1 -0
  9. package/dist/commands/operatorCommands.js +12 -2
  10. package/dist/commands/profileCommands.js +19 -2
  11. package/dist/commands/roleConfiguration.js +3 -0
  12. package/dist/commands/taskCommands.js +16 -0
  13. package/dist/commands/taskRoleRuntimeStatus.js +3 -0
  14. package/dist/controlPlaneCli.js +21 -4
  15. package/dist/controller/agentJobControl.js +9 -1
  16. package/dist/controller/clientRuntime.js +34 -13
  17. package/dist/controller/controller.js +35 -22
  18. package/dist/controller/diagnosticJournal.js +214 -0
  19. package/dist/controller/fileSchedulerStoreAdapter.js +174 -105
  20. package/dist/controller/globalInputDelivery.js +19 -9
  21. package/dist/controller/globalRuntimeAttention.js +11 -3
  22. package/dist/controller/jobControl.js +1 -1
  23. package/dist/controller/jobDefinitionControl.js +24 -13
  24. package/dist/controller/jobSupervisor.js +4 -0
  25. package/dist/controller/providerRetryAdmission.js +9 -0
  26. package/dist/controller/providerRetryDelivery.js +16 -0
  27. package/dist/controller/runtime.js +80 -4
  28. package/dist/controller/runtimeLaunchCoordinator.js +3 -1
  29. package/dist/controller/workerSchedulerStore.js +24 -0
  30. package/dist/core/boundedRpc.js +4 -0
  31. package/dist/core/controllerServer.js +1 -1
  32. package/dist/core/controllerTelemetry.js +92 -19
  33. package/dist/executor/fileRoleLaunchPlanner.js +3 -0
  34. package/dist/job/globalShell.js +51 -0
  35. package/dist/job/jobBusinessResult.js +4 -3
  36. package/dist/job/jobDefinition.js +6 -1
  37. package/dist/job/jobRunner.js +9 -0
  38. package/dist/job/scheduleService.js +2 -0
  39. package/dist/message/globalInboxRecovery.js +30 -19
  40. package/dist/message/globalProviderRetry.js +7 -1
  41. package/dist/message/messageContinuation.js +4 -0
  42. package/dist/output/rolePresentation.js +3 -0
  43. package/dist/profile/agentProfile.js +8 -0
  44. package/dist/profile/agentProfileRuntime.js +16 -1
  45. package/dist/review/reviewDecision.js +6 -0
  46. package/dist/role/availability.js +28 -0
  47. package/dist/role/role.js +6 -0
  48. package/dist/runtime/builtinAgentErrorMappers.js +4 -0
  49. package/dist/runtime/codexNativeAccess.js +21 -4
  50. package/dist/runtime/globalSessionHealth.js +39 -0
  51. package/dist/runtime/tmuxAdapters.js +5 -0
  52. package/dist/scheduler/activeRoleRunDelivery.js +9 -0
  53. package/dist/scheduler/leaderWakeupProcessor.js +8 -1
  54. package/dist/scheduler/roleRunLiveness.js +3 -0
  55. package/dist/scheduler/roleRunStall.js +10 -10
  56. package/dist/storage/persistenceWorker.js +41 -19
  57. package/dist/storage/sqliteSchema.js +43 -9
  58. package/dist/storage/sqliteStore.js +86 -8
  59. package/dist/storage/storageVersions.js +1 -1
  60. package/dist/storage/storeRpc.js +25 -11
  61. package/dist/storage/taskReadRevisionSchema.js +39 -0
  62. package/dist/web/assets/client/app/catalog.js +0 -29
  63. package/dist/web/assets/client/app/jobs.js +115 -0
  64. package/dist/web/assets/client/app/main.js +32 -28
  65. package/dist/web/assets/client/app/manage.js +1 -0
  66. package/dist/web/assets/client/app/settings.js +1 -5
  67. package/dist/web/assets/client/app/taskView.js +7 -11
  68. package/dist/web/assets/client/i18n/en.js +265 -36
  69. package/dist/web/assets/client/i18n/zh-CN.js +264 -35
  70. package/dist/web/assets/client/index.js +4 -0
  71. package/dist/web/assets/client/layout/manageShell.js +61 -15
  72. package/dist/web/assets/client/layout/router.js +2 -1
  73. package/dist/web/assets/client/layout/shortcuts.js +2 -2
  74. package/dist/web/assets/client/layout/workspace.js +99 -112
  75. package/dist/web/assets/client/lib/prefs.js +1 -1
  76. package/dist/web/assets/client/ui/primitives.js +15 -2
  77. package/dist/web/assets/client/views/dock/conversation.js +421 -123
  78. package/dist/web/assets/client/views/dock/session.js +229 -0
  79. package/dist/web/assets/client/views/dock/terminal.js +48 -39
  80. package/dist/web/assets/client/views/jobs.js +1042 -160
  81. package/dist/web/assets/client/views/overview.js +2 -2
  82. package/dist/web/assets/client/views/sidebar.js +3 -0
  83. package/dist/web/assets/client/views/task/page.js +2 -12
  84. package/dist/web/assets/shared/manageNav.js +3 -1
  85. package/dist/web/assets/shell/appRail.js +2 -1
  86. package/dist/web/assets/shell/dock.js +9 -27
  87. package/dist/web/assets/shell/manage.js +7 -2
  88. package/dist/web/assets/shell/models.js +4 -3
  89. package/dist/web/assets/shell/settingsContent.js +0 -3
  90. package/dist/web/assets/shell/sidebar.js +0 -1
  91. package/dist/web/assets/styles/views/conversation.js +96 -17
  92. package/dist/web/assets/styles/views/dock.js +17 -5
  93. package/dist/web/assets/styles/views/jobs.js +301 -12
  94. package/dist/web/assets/styles/views/sidebar.js +1 -2
  95. package/dist/web/assets/styles/views/task.js +1 -2
  96. package/dist/web/assets/styles/views/terminal.js +12 -8
  97. package/dist/web/webConversation.js +1 -0
  98. package/dist/web/webServer.js +28 -1
  99. package/dist/web/webSettings.js +10 -2
  100. package/docs/global-shell-jobs.md +78 -0
  101. package/docs/job-runs.md +6 -3
  102. package/docs/managed-turn-and-session-runtime.md +84 -0
  103. package/docs/managed-turn-and-session-runtime.zh-CN.md +57 -0
  104. package/docs/release-workflow.md +67 -0
  105. package/docs/release-workflow.zh-CN.md +48 -0
  106. package/docs/roles-and-configuration.md +45 -1
  107. package/docs/roles-and-configuration.zh-CN.md +35 -0
  108. package/docs/storage-baseline.md +19 -1
  109. package/docs/storage-baseline.zh-CN.md +10 -1
  110. package/docs/testing/review-evaluation.md +35 -1
  111. package/docs/testing/review-evaluation.zh-CN.md +25 -1
  112. package/docs/testing/verification-levels.md +2 -2
  113. package/docs/testing/verification-levels.zh-CN.md +2 -2
  114. package/package.json +1 -1
  115. package/skills/yui-leader/VERSION +1 -1
  116. package/skills/yui-leader/references/execution.md +19 -8
  117. package/skills/yui-reviewer/SKILL.md +13 -2
  118. package/skills/yui-reviewer/VERSION +1 -1
@@ -1,4 +1,5 @@
1
1
  import { usageError } from "../errors/cliError.js";
2
+ import { parseEnabled } from "../role/availability.js";
2
3
  import { defaultRoleAgentConfig, resolveAgentAdapter } from "../executor/agentAdapter.js";
3
4
  const PROFILE_OPTIONS = [
4
5
  ["--description", "value"],
@@ -64,6 +65,7 @@ export function roleOptionSpecs(input) {
64
65
  ...(input.includeAgent === true ? [["--agent", "value"]] : []),
65
66
  ...(input.includeWorkspace === true ? [["--workspace", "value"]] : []),
66
67
  ...(input.update ? [["--yes", "flag"]] : []),
68
+ ...(input.update ? [["--enabled", "value"]] : []),
67
69
  ...PROFILE_OPTIONS,
68
70
  ...agentValueOptions,
69
71
  ...agentRepeatableOptions,
@@ -123,6 +125,7 @@ export function roleProfilePatch(parsed) {
123
125
  ["--skill", "--clear-skills"]
124
126
  ]);
125
127
  return {
128
+ ...(parsed.has("--enabled") ? { enabled: parseEnabled(parsed.one("--enabled")) } : {}),
126
129
  ...(parsed.has("--description") ? { description: requiredText(parsed.one("--description")) } : {}),
127
130
  ...(parsed.has("--clear-description") ? { description: undefined } : {}),
128
131
  ...(parsed.has("--responsibility")
@@ -3,6 +3,7 @@ import { isDeepStrictEqual } from "node:util";
3
3
  import { taskMessageInboxReceipt } from "../message/taskInboxOwnership.js";
4
4
  import { agentExecutionComponentLabel } from "../agent/executionComponents.js";
5
5
  import { assertLeaderArchiveAdmission } from "../task/leaderArchiveAuthority.js";
6
+ import { assertRoleEnabled, isEnabled } from "../role/availability.js";
6
7
  import { createRun, runExecutionObservation, runPurposeAdmitsTaskState } from "../agentRun/agentRun.js";
7
8
  import { isGitArtifactRefString, parseGitArtifactRef } from "../artifacts/gitArtifactRef.js";
8
9
  import { contextContentDigest, contextSnapshotRef } from "../context/contextSnapshot.js";
@@ -2251,6 +2252,11 @@ function taskRoleCommand(args, store, options) {
2251
2252
  return showTaskRole(rest, store);
2252
2253
  if (command === "update")
2253
2254
  return updateTaskRole(rest, store, options);
2255
+ if (command === "enable" || command === "disable") {
2256
+ if (rest.length !== 2)
2257
+ throw usageError(`Usage: yui task role ${command} <task> <role>`);
2258
+ return updateTaskRole([...rest, "--enabled", String(command === "enable")], store, options);
2259
+ }
2254
2260
  if (command === "remove")
2255
2261
  return output(removeTaskRole(rest, store, options));
2256
2262
  if (command === "bind")
@@ -2751,6 +2757,7 @@ function listTaskRoles(args, store, options) {
2751
2757
  const statuses = inspectTaskRoleRuntimeStatuses(task.id, roles, store, options.runtime?.inspectTaskRolePanes?.(task.id) ?? [], options.now?.() ?? new Date());
2752
2758
  const data = recordPage(statuses.map(status => ({
2753
2759
  roleName: status.roleName, agentId: status.agentId, recordedHealth: status.health,
2760
+ enabled: isEnabled(status.role),
2754
2761
  hostObservation: "not-requested",
2755
2762
  openInput: taskRoleOpenInputLabel(status), activeWork: taskRoleActiveWorkLabel(status),
2756
2763
  lastRun: taskRoleLastRunLabel(status), nativeSession: taskRoleNativeSessionLabel(status),
@@ -2810,6 +2817,9 @@ function updateTaskRole(args, store, options) {
2810
2817
  taskActor(tx, options, task.id);
2811
2818
  const role = requireRole(tx, task.id, roleName);
2812
2819
  const changesEnvironment = parsed.has("--environment") || parsed.has("--managed-environment");
2820
+ if (parsed.seen.size === 1 && parsed.has("--enabled")
2821
+ && isEnabled(role) === roleProfilePatch(parsed).enabled)
2822
+ return role;
2813
2823
  const changesLaunchContext = hasRoleLaunchContextOptions(parsed) || parsed.has("--profile")
2814
2824
  || changesEnvironment;
2815
2825
  const changesAgentConfig = hasAgentConfigOptions(parsed);
@@ -3562,6 +3572,7 @@ function dispatchWork(args, store, options) {
3562
3572
  }
3563
3573
  const roles = lanePlan.roles.map((name) => requireRole(tx, task.id, name));
3564
3574
  for (const role of roles) {
3575
+ assertRoleEnabled(role);
3565
3576
  if (tx.getActiveRun(task.id, role.name) !== null) {
3566
3577
  throw usageError(`${task.id}/${role.name} already has an active run.`);
3567
3578
  }
@@ -3570,6 +3581,7 @@ function dispatchWork(args, store, options) {
3570
3581
  let workItemForDispatch = prepareWorkItemDispatch(item, now);
3571
3582
  if (lanePlan.roles.length === 0) {
3572
3583
  const role = requireRole(tx, task.id, assignee);
3584
+ assertRoleEnabled(role);
3573
3585
  if (tx.getActiveRun(task.id, role.name) !== null) {
3574
3586
  throw usageError(`${task.id}/${role.name} already has an active run.`);
3575
3587
  }
@@ -4383,6 +4395,9 @@ function assertTaskReviewRequestLane(store, taskId, reviewerRoleName, reusableRo
4383
4395
  assertReviewerAvailable(store, taskId, reviewerRoleName, reusableRound);
4384
4396
  }
4385
4397
  function assertReviewerAvailable(store, taskId, reviewerRoleName, reusableRound) {
4398
+ const role = store.getRole(taskId, reviewerRoleName);
4399
+ if (role !== null)
4400
+ assertRoleEnabled(role);
4386
4401
  const availability = projectReviewerAvailability(store, taskId, reviewerRoleName);
4387
4402
  if (availability.kind === "available")
4388
4403
  return;
@@ -5001,6 +5016,7 @@ function retryRunOperation(args, store, options) {
5001
5016
  throw usageError(`Task does not admit ${previous.purpose} retry: ${task.id}/${task.status}.`);
5002
5017
  }
5003
5018
  const role = requireRole(tx, task.id, previous.roleName);
5019
+ assertRoleEnabled(role);
5004
5020
  if (tx.getActiveRun(task.id, role.name) !== null) {
5005
5021
  throw usageError(`${task.id}/${role.name} already has an active run.`);
5006
5022
  }
@@ -1,5 +1,6 @@
1
1
  import { isDeepStrictEqual } from "node:util";
2
2
  import { runExecutionObservation } from "../agentRun/agentRun.js";
3
+ import { isEnabled, roleDisabledReason } from "../role/availability.js";
3
4
  import { hasRuntimeCleanupObligation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
4
5
  import { currentProviderConversation, managedProviderTurnId } from "../runtime/providerRuntimeIdentity.js";
5
6
  import { isRoleRunStalled, latestStallProgressAt } from "../scheduler/roleRunStall.js";
@@ -118,6 +119,8 @@ export function renderTaskRoleRuntimeStatus(status) {
118
119
  `Task Role status: ${status.taskId}/${status.roleName}`,
119
120
  "",
120
121
  ` Health ${status.health}`,
122
+ ` Enabled ${isEnabled(status.role)}`,
123
+ ...(!isEnabled(status.role) ? [` Availability ${roleDisabledReason(status.role)}`] : []),
121
124
  ` Reason ${status.healthReason}`,
122
125
  ` Open inputs ${status.openInputRequestCount}`,
123
126
  ` Agent ${status.agentId}`,
@@ -532,8 +532,9 @@ export async function main() {
532
532
  const result = await callController(home, "controller.status", {}, { timeoutMs: 2_000 });
533
533
  runtime = parseControllerRuntimeSnapshot(result, droppedEvents);
534
534
  }
535
- catch {
536
- runtime = { source: "unsupported", droppedEvents };
535
+ catch (error) {
536
+ runtime = { source: "unsupported", droppedEvents,
537
+ error: describeCliFailure(error, failureContext()).message.slice(0, 1000) };
537
538
  }
538
539
  const mismatch = summarizeDurablePhysicalMismatch(snapshot);
539
540
  const identitySection = renderRuntimeIdentitySection({
@@ -2611,6 +2612,7 @@ async function executeOperatorSessionControl(control, home, store, runtime, tmux
2611
2612
  }
2612
2613
  await prepareInteractiveCodexApp(control.targetAgentId, store, home);
2613
2614
  await ensureFileTaskController(home, { environment: process.env });
2615
+ let stopDisposition = "not-required";
2614
2616
  if (paneRunning
2615
2617
  || (active !== undefined
2616
2618
  && active.status === "active")
@@ -2618,9 +2620,24 @@ async function executeOperatorSessionControl(control, home, store, runtime, tmux
2618
2620
  && ["submitting", "accepted", "delivery-unknown"].includes(sessionSet.providerBinding.run.status))
2619
2621
  || (sessionSet?.providerBinding?.goal != null && sessionSet.providerBinding.goal.status !== "complete")) {
2620
2622
  await runtime.stopGlobalRoleSession(role.name);
2623
+ stopDisposition = "completed";
2624
+ }
2625
+ try {
2626
+ applyOperatorSessionControl(control, store);
2627
+ }
2628
+ catch (error) {
2629
+ throw new Error(`Operator replacement selection failed; prior stop=${stopDisposition}; `
2630
+ + "new Session startup has not begun. Inspect the retained Session state.", { cause: error });
2631
+ }
2632
+ try {
2633
+ await runtime.prepareGlobalRoleEnter(role.name);
2634
+ }
2635
+ catch (error) {
2636
+ throw new Error(`Operator replacement selection committed; prior stop=${stopDisposition}; its startup phase failed `
2637
+ + "without readiness confirmation. Whether native startup began is unconfirmed. "
2638
+ + "Inspect the exact current Session/Host before retrying; "
2639
+ + "do not infer that startup had no effect.", { cause: error });
2621
2640
  }
2622
- applyOperatorSessionControl(control, store);
2623
- await runtime.prepareGlobalRoleEnter(role.name);
2624
2641
  tmux.attachRole("operator", role.name, "auto");
2625
2642
  }
2626
2643
  async function prepareInteractiveCodexApp(agentId, store, home) {
@@ -1,4 +1,5 @@
1
1
  import { randomUUID } from "node:crypto";
2
+ import { assertRoleEnabled } from "../role/availability.js";
2
3
  import { mkdirSync } from "node:fs";
3
4
  import { isDeepStrictEqual } from "node:util";
4
5
  import { join } from "node:path";
@@ -63,7 +64,8 @@ function requireTaskTarget(store, target, runnable = true) {
63
64
  const task = store.getTask(target.taskId);
64
65
  if (!task || runnable && (task.status !== "active" || task.executionGate.state !== "enabled"))
65
66
  throw new Error("Task Agent requires an active, execution-enabled Task.");
66
- if (!store.getRole(target.taskId, target.roleName))
67
+ const role = store.getRole(target.taskId, target.roleName);
68
+ if (!role)
67
69
  throw new Error("Task Agent Role is unavailable.");
68
70
  const recipient = taskRecipient(store, target);
69
71
  if (recipient.ownerRunId) {
@@ -117,6 +119,7 @@ function ownedRole(store, home, job, runId, now) {
117
119
  const jobOwner = { scope: job.scope, jobId: job.id, ...(isolated ? { runId } : {}) };
118
120
  const existing = store.getGlobalRole(name);
119
121
  if (existing) {
122
+ assertRoleEnabled(existing);
120
123
  if (!isDeepStrictEqual(existing.jobOwner, jobOwner) || existing.activeAgentId !== job.configuration.target.agentId) {
121
124
  throw new Error("Job Role binding changed; inspect its Session/configuration before dispatch.");
122
125
  }
@@ -155,6 +158,9 @@ function queueAttempt(store, home, job, runId, index, now) {
155
158
  const inputControl = { action: "queue", requestId: `job-agent:${runId}:${index}` };
156
159
  let messageId;
157
160
  if (recipient.scope === "global") {
161
+ const role = store.getGlobalRole(recipient.roleName);
162
+ if (role !== null)
163
+ assertRoleEnabled(role);
158
164
  messageId = store.nextGlobalRoleMessageId();
159
165
  let message = createGlobalRoleMessage(messageId, recipient.roleName, body, "system", { type: "system" }, now, { inputControl });
160
166
  const set = store.getGlobalRoleSessionSet(recipient.roleName);
@@ -165,6 +171,7 @@ function queueAttempt(store, home, job, runId, index, now) {
165
171
  }
166
172
  else {
167
173
  requireTaskTarget(store, { ...recipient });
174
+ assertRoleEnabled(store.getRole(recipient.taskId, recipient.roleName));
168
175
  const resolved = taskRecipient(store, recipient);
169
176
  messageId = store.nextMessageId(recipient.taskId);
170
177
  store.saveMessage(recipient.taskId, createTaskMessage(messageId, recipient.taskId, body, "system", { type: "system" }, now, { ...(resolved.ownerRunId ? { recipient: resolved } : {}), inputControl }));
@@ -256,6 +263,7 @@ export function controlAgentJob(store, home, job, input, now) {
256
263
  if (op === "session-new") {
257
264
  if (!role)
258
265
  throw new Error("Create the Job Role first.");
266
+ assertRoleEnabled(role);
259
267
  assertRoleRuntimeMutationAllowed(store, { scope: "global", roleName }, "Job Session replacement");
260
268
  const set = store.getGlobalRoleSessionSet(roleName);
261
269
  const old = set?.sessions[set.activeAgentId];
@@ -456,10 +456,7 @@ export class FileTaskWorkflowRuntime {
456
456
  }
457
457
  if (targets.length === 0)
458
458
  return;
459
- await callFileTaskController(this.home, "scheduler.scan", {}, {
460
- ...this.clientOptions,
461
- requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
462
- });
459
+ await Promise.all(targets.map(target => this.#waitForRuntimeCleanup(target)));
463
460
  for (const target of targets) {
464
461
  if (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
465
462
  throw new Error(`Role runtime did not stop: ${target.taskId}/${target.roleName}.`);
@@ -487,10 +484,7 @@ export class FileTaskWorkflowRuntime {
487
484
  if (target === null) {
488
485
  throw new Error(`Role Session changed before its exact stop was reserved: ${input.taskId}/${input.roleName}.`);
489
486
  }
490
- await callFileTaskController(this.home, "scheduler.scan", {}, {
491
- ...this.clientOptions,
492
- requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
493
- });
487
+ await this.#waitForRuntimeCleanup(target);
494
488
  if (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
495
489
  throw new Error(`Role runtime did not stop: ${input.taskId}/${input.roleName}.`);
496
490
  }
@@ -506,7 +500,7 @@ export class FileTaskWorkflowRuntime {
506
500
  .filter((job) => job.taskId === taskId);
507
501
  if (active.length === 0)
508
502
  return;
509
- await callFileTaskController(this.home, "scheduler.scan", {}, {
503
+ await callFileTaskController(this.home, "scheduler.signal", { key: `task:${encodeURIComponent(taskId)}` }, {
510
504
  ...this.clientOptions,
511
505
  requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
512
506
  });
@@ -585,10 +579,7 @@ export class FileTaskWorkflowRuntime {
585
579
  if (target?.kind !== "global-role-runtime") {
586
580
  throw new Error(`Global Role runtime cleanup target is invalid: ${roleName}.`);
587
581
  }
588
- await callFileTaskController(this.home, "scheduler.scan", {}, {
589
- ...this.clientOptions,
590
- requestTimeoutMs: LIFECYCLE_REQUEST_TIMEOUT_MS
591
- });
582
+ await this.#waitForRuntimeCleanup(target);
592
583
  if (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
593
584
  throw new Error(`Global Role runtime did not stop: ${roleName}.`);
594
585
  }
@@ -598,6 +589,36 @@ export class FileTaskWorkflowRuntime {
598
589
  throw new Error(`Global Role runtime session is still active: ${roleName}.`);
599
590
  }
600
591
  }
592
+ /**
593
+ * The durable cleanup mailbox is completed only after stopOwner proves
594
+ * quiescence and commits the existing cleanup transaction. A scheduler hint
595
+ * is not that proof; nor is exhaustion of the Home-wide coalescing loop.
596
+ */
597
+ async #waitForRuntimeCleanup(target) {
598
+ const timeoutMs = this.clientOptions.requestTimeoutMs ?? LIFECYCLE_REQUEST_TIMEOUT_MS;
599
+ const deadline = Date.now() + timeoutMs;
600
+ const label = controllerMailboxKey(target);
601
+ try {
602
+ await callFileTaskController(this.home, "scheduler.signal", { key: label }, {
603
+ ...this.clientOptions, requestTimeoutMs: timeoutMs
604
+ });
605
+ while (hasRuntimeLifecycleWork(this.store.getWorkMailbox(target))) {
606
+ if (Date.now() >= deadline) {
607
+ throw new Error("Timed out awaiting the target's durable cleanup transaction.");
608
+ }
609
+ await new Promise(resolve => setTimeout(resolve, Math.min(this.clientOptions.pollIntervalMs ?? POLL_INTERVAL_MS, Math.max(1, deadline - Date.now()))));
610
+ }
611
+ }
612
+ catch (error) {
613
+ // A lost hint response must not hide cleanup that actually committed.
614
+ // No hint or native input is replayed here.
615
+ if (!hasRuntimeLifecycleWork(this.store.getWorkMailbox(target)))
616
+ return;
617
+ throw new Error(`Runtime stop remains unconfirmed for ${label}; cleanup obligation retained. `
618
+ + "No replacement was started by this stop operation. Inspect the exact Session/Host "
619
+ + "and cleanup failure before retrying; no physical exit is inferred.", { cause: error });
620
+ }
621
+ }
601
622
  /** Drain pending runtime facts while an external maintenance fence is held. */
602
623
  async drainController() {
603
624
  await ensureFileTaskController(this.home, {
@@ -7,7 +7,7 @@ import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
7
7
  import { DEFAULT_STALL_WINDOW_MS, DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
8
8
  import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
9
9
  import { startControllerServer } from "../core/controllerServer.js";
10
- import { monotonicMilliseconds } from "../core/controllerTelemetry.js";
10
+ import { ControllerWorkObserver, monotonicMilliseconds } from "../core/controllerTelemetry.js";
11
11
  import { isProjectMaintenanceFenced } from "../repository/projectMaintenanceLock.js";
12
12
  import { isHandoverLockHeld } from "../release/runtimeRelease.js";
13
13
  import { KeyedWorkQueue } from "../coordination/keyedWorkQueue.js";
@@ -44,7 +44,9 @@ const ZERO_DRAIN_METRICS = Object.freeze({
44
44
  * Runs one lean scheduler pass. Due native Turn completions are folded before
45
45
  * liveness, so a valid Hook boundary fences destructive process reconciliation.
46
46
  */
47
- export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, leaderWakeFence, onError) {
47
+ export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set(), diagnosticAfterMs = DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS, leaderWakeFence, onError, observer) {
48
+ const target = scope.kind === "full" ? "home" : scope.keys.slice(0, 16).join(",");
49
+ const observe = (phase, action) => observer === undefined ? Promise.resolve().then(action) : observer.observe(phase, target, action);
48
50
  const compiledSelection = compileReconcileSelection(scope);
49
51
  const selection = includeOperator
50
52
  ? { ...compiledSelection, blockedTaskIds }
@@ -54,7 +56,7 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
54
56
  // event-loop turn. Give already-written requests a poll boundary before the
55
57
  // next durable phase; later phases retain their existing CAS fences.
56
58
  await controlEventLoopRun();
57
- const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes, blockedTaskIds);
59
+ const failedCleanupRoles = await observe("runtime-cleanup", () => processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes, blockedTaskIds));
58
60
  const roleSelection = selectionWithoutFailedCleanupRoles(store, selection, failedCleanupRoles);
59
61
  const availableWakeupSelection = () => (leaderWakeFence?.() === true
60
62
  ? exactTaskSelection(new Set())
@@ -64,19 +66,19 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
64
66
  // Durable Leader work that already has a ready Task workspace belongs to
65
67
  // the control path. Dispatch it before any unrelated Task workspace I/O;
66
68
  // the processor itself retains the fail-closed workspace-ready guard.
67
- const initialWakeups = selectedPendingWakeups(store, availableWakeupSelection());
68
- const initialWakeupResults = await processLeaderWakeups(store, delivery, now, exactTaskSelection(new Set(initialWakeups.keys())));
69
+ const initialWakeups = await observe("wakeup-projection", () => selectedPendingWakeups(store, availableWakeupSelection()));
70
+ const initialWakeupResults = await observe("leader-delivery", () => processLeaderWakeups(store, delivery, now, exactTaskSelection(new Set(initialWakeups.keys()))));
69
71
  // Preserve ready-Leader-first ordering, then bound the repeated state
70
72
  // projections that follow it in this pass.
71
73
  await controlEventLoopRun();
72
74
  // A Task whose deferred activation was just released must become active
73
75
  // before the workspace phase, so its first workspace preparation happens in
74
76
  // this same pass rather than waiting for the next one.
75
- await adoptReleasedTaskActivations(store, workspacePreparer, selection, onError);
77
+ await observe("activation", () => adoptReleasedTaskActivations(store, workspacePreparer, selection, onError));
76
78
  await controlEventLoopRun();
77
- const workspacePreparation = await prepareActiveWorkspaces(store, workspacePreparer, selection, maintenanceFence, onMaintenanceFenceDefer);
79
+ const workspacePreparation = await observe("workspace-preparation", () => prepareActiveWorkspaces(store, workspacePreparer, selection, maintenanceFence, onMaintenanceFenceDefer));
78
80
  await controlEventLoopRun();
79
- const activeRunDeliveries = await processActiveRoleRunDeliveries(store, delivery, now, roleSelection);
81
+ const activeRunDeliveries = await observe("run-delivery", () => processActiveRoleRunDeliveries(store, delivery, now, roleSelection));
80
82
  await controlEventLoopRun();
81
83
  const unsettledRunRefs = new Set(activeRunDeliveries.flatMap((result) => (result.reason === "delivery-uncertain"
82
84
  ? [formatTaskRecordReference(result.taskId, result.runId, "run")]
@@ -108,16 +110,16 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
108
110
  await controlEventLoopRun();
109
111
  const liveStatuses = new Map();
110
112
  const resourceEvidence = new Map();
111
- const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty");
113
+ const failedRunRefs = await observe("run-liveness", () => reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty"));
112
114
  await controlEventLoopRun();
113
115
  await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, diagnosticAfterMs);
114
116
  await controlEventLoopRun();
115
117
  const selectedInputTaskIds = selectedTaskIdsForBoundedPass(store, selection);
116
118
  const autoResolvedInputs = selectedInputTaskIds === undefined
117
- ? store.resolveExpiredInputRecommendations(now)
119
+ ? await store.resolveExpiredInputRecommendations(now)
118
120
  : selectedInputTaskIds.size === 0
119
121
  ? []
120
- : store.resolveExpiredInputRecommendations(now, selectedInputTaskIds);
122
+ : await store.resolveExpiredInputRecommendations(now, selectedInputTaskIds);
121
123
  // Liveness and auto-resolution can durably queue new Leader work. Process
122
124
  // only Tasks that were not part of phase one, except when a same-pass due
123
125
  // completion made an initially busy Leader idle. Result order remains
@@ -694,7 +696,10 @@ export class FileTaskController {
694
696
  #providerRetry;
695
697
  #continuationReconciler;
696
698
  #hostTerminalRecovery;
699
+ #persistenceMetrics;
700
+ #diagnosticMetrics;
697
701
  #current;
702
+ #phases = new ControllerWorkObserver();
698
703
  #operatorCurrent;
699
704
  #runtimeObserverCurrent;
700
705
  #pendingFull = false;
@@ -712,6 +717,8 @@ export class FileTaskController {
712
717
  constructor(store, delivery, options = {}) {
713
718
  this.store = store;
714
719
  this.delivery = delivery;
720
+ this.#persistenceMetrics = options.persistenceMetrics;
721
+ this.#diagnosticMetrics = options.diagnosticMetrics;
715
722
  this.#intervalMs = positiveInteger(options.intervalMs, DEFAULT_RECONCILIATION_INTERVAL_MS, "Controller reconciliation interval");
716
723
  this.#now = options.now ?? (() => new Date());
717
724
  this.#onError = options.onError ?? (() => { });
@@ -782,6 +789,12 @@ export class FileTaskController {
782
789
  runtimeMetrics() {
783
790
  const metrics = this.#lastRuntimeDrain?.metrics ?? ZERO_DRAIN_METRICS;
784
791
  return {
792
+ diagnostics: this.#diagnosticMetrics?.() ?? null,
793
+ phases: this.#phases.snapshot(),
794
+ persistence: this.#persistenceMetrics?.() ?? null,
795
+ reads: this.store.readObservationMetrics?.() ?? null,
796
+ scheduler: { running: this.#current !== undefined, pendingFull: this.#pendingFull,
797
+ pendingKeys: this.#pendingKeys.size },
785
798
  // Status is intentionally O(1): scanning the inbox while serving the
786
799
  // control socket would recreate the starvation this metric diagnoses.
787
800
  inbox: {
@@ -956,7 +969,7 @@ export class FileTaskController {
956
969
  * without bypassing the scheduler's own ownership and delivery fences. */
957
970
  async #observeMaintenance(phase, action) {
958
971
  try {
959
- return await action();
972
+ return await this.#phases.observe(phase, "home", action);
960
973
  }
961
974
  catch (error) {
962
975
  this.#onError(new Error(`${phase} failed: ${error instanceof Error ? error.message : String(error)}. `
@@ -998,12 +1011,12 @@ export class FileTaskController {
998
1011
  }
999
1012
  }
1000
1013
  }
1001
- const firstRuntimeDrain = await this.#drainRuntimeEvents();
1014
+ const firstRuntimeDrain = await this.#phases.observe("runtime-drain", "home", () => this.#drainRuntimeEvents());
1002
1015
  if (scope.kind === "full" && this.#hostTerminalRecovery !== undefined) {
1003
1016
  await this.#observeMaintenance("Host terminal recovery", this.#hostTerminalRecovery);
1004
1017
  }
1005
- await this.#providerRetry?.reconcile();
1006
- await this.#globalInputDelivery?.({ reconcile: scope.kind === "full", now: this.#now() });
1018
+ await this.#phases.observe("provider-retry", "home", () => this.#providerRetry?.reconcile());
1019
+ await this.#phases.observe("global-input", "home", () => this.#globalInputDelivery?.({ reconcile: scope.kind === "full", now: this.#now() }));
1007
1020
  for (const taskId of runtimeTaskFailureIds(firstRuntimeDrain)) {
1008
1021
  runtimeFailedTaskIds.add(taskId);
1009
1022
  }
@@ -1016,17 +1029,17 @@ export class FileTaskController {
1016
1029
  // DurableJob reconciliation runs before the scheduler pass so a
1017
1030
  // terminal job's Leader wakeup is enqueued in the same pass that
1018
1031
  // processes Leader wakeups.
1019
- this.#schedules?.reconcile(this.#now());
1032
+ await this.#schedules?.reconcile(this.#now());
1020
1033
  this.#jobSupervisor?.reconcile(this.#now());
1021
1034
  if (scope.kind === "full") {
1022
- result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError);
1035
+ result = await this.#phases.observe("scheduler-full", "home", () => runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError, this.#phases));
1023
1036
  }
1024
1037
  else {
1025
- const dirtyPass = await this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds);
1038
+ const dirtyPass = await this.#phases.observe("scheduler-dirty", scope.keys.slice(0, 16).join(","), () => this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds));
1026
1039
  result = dirtyPass.result;
1027
1040
  failedTaskScopes = dirtyPass.failedTaskScopes;
1028
1041
  }
1029
- const secondRuntimeDrain = await this.#drainRuntimeEvents();
1042
+ const secondRuntimeDrain = await this.#phases.observe("runtime-drain", "home", () => this.#drainRuntimeEvents());
1030
1043
  for (const taskId of runtimeTaskFailureIds(secondRuntimeDrain)) {
1031
1044
  runtimeFailedTaskIds.add(taskId);
1032
1045
  }
@@ -1040,7 +1053,7 @@ export class FileTaskController {
1040
1053
  // scheduler.
1041
1054
  if (scope.kind === "full" && this.#resourceAutoGc !== undefined) {
1042
1055
  try {
1043
- const gc = await this.#resourceAutoGc();
1056
+ const gc = await this.#phases.observe("resource-gc", "home", this.#resourceAutoGc);
1044
1057
  if (!gc.skipped && gc.failed > 0) {
1045
1058
  this.#onError(new Error(`Resource auto-GC failed to quarantine ${gc.failed} `
1046
1059
  + "resource(s); they stay in place and retry next pass."));
@@ -1120,7 +1133,7 @@ export class FileTaskController {
1120
1133
  const taskScopes = partition.taskScopes.filter((taskScope) => (!blockedTaskIds.has(taskScope.taskId)));
1121
1134
  const orderedResults = [];
1122
1135
  if (partition.globalKeys.length > 0) {
1123
- orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError));
1136
+ orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError, this.#phases));
1124
1137
  }
1125
1138
  if (taskScopes.length === 0
1126
1139
  || this.#stopped
@@ -1153,7 +1166,7 @@ export class FileTaskController {
1153
1166
  if (this.#stopped || this.#pendingFull)
1154
1167
  continue;
1155
1168
  try {
1156
- taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError);
1169
+ taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds, this.#diagnosticAfterMs, this.#leaderWakeFence, this.#onError, this.#phases);
1157
1170
  this.#clearTaskPassRetry(selected.taskScope.taskId);
1158
1171
  }
1159
1172
  catch (error) {