agent-relay-server 0.121.0 → 0.121.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/docs/openapi.json +1 -1
  2. package/package.json +5 -4
  3. package/public/assets/{MessageBubble-BaPGFJxq.js → MessageBubble-CLqxCG9b.js} +2 -2
  4. package/public/assets/{MessageBubble-BaPGFJxq.js.map → MessageBubble-CLqxCG9b.js.map} +1 -1
  5. package/public/assets/{PlanGraphPanel-BXCVfmln.js → PlanGraphPanel-BkkToFEb.js} +2 -2
  6. package/public/assets/{PlanGraphPanel-BXCVfmln.js.map → PlanGraphPanel-BkkToFEb.js.map} +1 -1
  7. package/public/assets/{activity-D_shwAZt.js → activity-D-ocGQGs.js} +2 -2
  8. package/public/assets/{activity-D_shwAZt.js.map → activity-D-ocGQGs.js.map} +1 -1
  9. package/public/assets/{agent-profiles-BrfVz5IV.js → agent-profiles-CRPgYTyb.js} +2 -2
  10. package/public/assets/{agent-profiles-BrfVz5IV.js.map → agent-profiles-CRPgYTyb.js.map} +1 -1
  11. package/public/assets/{agents-B5w-Z9Ic.js → agents-Chw5E_2q.js} +2 -2
  12. package/public/assets/{agents-B5w-Z9Ic.js.map → agents-Chw5E_2q.js.map} +1 -1
  13. package/public/assets/{analytics-Bm3D12UN.js → analytics-BmNDG0hR.js} +2 -2
  14. package/public/assets/{analytics-Bm3D12UN.js.map → analytics-BmNDG0hR.js.map} +1 -1
  15. package/public/assets/{automation-DOTL8BFs.js → automation-D7g90kTv.js} +2 -2
  16. package/public/assets/{automation-DOTL8BFs.js.map → automation-D7g90kTv.js.map} +1 -1
  17. package/public/assets/{branch-state-badge-BlqeJZ5-.js → branch-state-badge-Bv7DhLBZ.js} +2 -2
  18. package/public/assets/{branch-state-badge-BlqeJZ5-.js.map → branch-state-badge-Bv7DhLBZ.js.map} +1 -1
  19. package/public/assets/{channels-BoiKfK6Q.js → channels-CLwPeEPi.js} +2 -2
  20. package/public/assets/{channels-BoiKfK6Q.js.map → channels-CLwPeEPi.js.map} +1 -1
  21. package/public/assets/{chat-PeU3iOaa.js → chat-Cgj7VKzc.js} +2 -2
  22. package/public/assets/{chat-PeU3iOaa.js.map → chat-Cgj7VKzc.js.map} +1 -1
  23. package/public/assets/{connectors-BvV8Hee2.js → connectors-Br6fiTny.js} +2 -2
  24. package/public/assets/{connectors-BvV8Hee2.js.map → connectors-Br6fiTny.js.map} +1 -1
  25. package/public/assets/{formatted-body-C1FgzlT1.js → formatted-body-CmkuD23M.js} +3 -3
  26. package/public/assets/{formatted-body-C1FgzlT1.js.map → formatted-body-CmkuD23M.js.map} +1 -1
  27. package/public/assets/{formatted-body-impl-BTpU1xH4.js → formatted-body-impl-DbdFmbwW.js} +2 -2
  28. package/public/assets/{formatted-body-impl-BTpU1xH4.js.map → formatted-body-impl-DbdFmbwW.js.map} +1 -1
  29. package/public/assets/{index-DZEahmyc.js → index-D9OogaB5.js} +6 -6
  30. package/public/assets/{index-DZEahmyc.js.map → index-D9OogaB5.js.map} +1 -1
  31. package/public/assets/{insights-BqmgSpQm.js → insights-qvaPqWCV.js} +2 -2
  32. package/public/assets/{insights-BqmgSpQm.js.map → insights-qvaPqWCV.js.map} +1 -1
  33. package/public/assets/{integrations-CuT92zQi.js → integrations-DYEGSqq_.js} +2 -2
  34. package/public/assets/{integrations-CuT92zQi.js.map → integrations-DYEGSqq_.js.map} +1 -1
  35. package/public/assets/{maintenance-lM7FeXVC.js → maintenance-C0HIEB-J.js} +2 -2
  36. package/public/assets/{maintenance-lM7FeXVC.js.map → maintenance-C0HIEB-J.js.map} +1 -1
  37. package/public/assets/{managed-agents-DaPNv3yY.js → managed-agents-DdS7aI38.js} +2 -2
  38. package/public/assets/{managed-agents-DaPNv3yY.js.map → managed-agents-DdS7aI38.js.map} +1 -1
  39. package/public/assets/{markdown-preview-impl-Wia3Ho1s.js → markdown-preview-impl-BnXMH1z1.js} +2 -2
  40. package/public/assets/{markdown-preview-impl-Wia3Ho1s.js.map → markdown-preview-impl-BnXMH1z1.js.map} +1 -1
  41. package/public/assets/{memory-B1v_zD-M.js → memory-CSwx7bz8.js} +2 -2
  42. package/public/assets/{memory-B1v_zD-M.js.map → memory-CSwx7bz8.js.map} +1 -1
  43. package/public/assets/{messages-BNOx3yK6.js → messages-Be9CmvDv.js} +2 -2
  44. package/public/assets/{messages-BNOx3yK6.js.map → messages-Be9CmvDv.js.map} +1 -1
  45. package/public/assets/{orchestrators-DWFy4asl.js → orchestrators-BgtvvHvo.js} +2 -2
  46. package/public/assets/{orchestrators-DWFy4asl.js.map → orchestrators-BgtvvHvo.js.map} +1 -1
  47. package/public/assets/{overview-C_12cFPM.js → overview-BvhbPWtF.js} +2 -2
  48. package/public/assets/{overview-C_12cFPM.js.map → overview-BvhbPWtF.js.map} +1 -1
  49. package/public/assets/{pairs-DIc7gX83.js → pairs-CpXSOMx0.js} +2 -2
  50. package/public/assets/{pairs-DIc7gX83.js.map → pairs-CpXSOMx0.js.map} +1 -1
  51. package/public/assets/{projects-DPjvJUzT.js → projects-DdPNPFJI.js} +2 -2
  52. package/public/assets/{projects-DPjvJUzT.js.map → projects-DdPNPFJI.js.map} +1 -1
  53. package/public/assets/{prompt-settings-CtumNgG8.js → prompt-settings-BpZse7-X.js} +2 -2
  54. package/public/assets/{prompt-settings-CtumNgG8.js.map → prompt-settings-BpZse7-X.js.map} +1 -1
  55. package/public/assets/{registry-DzUhgMq9.js → registry-pb1gTZEg.js} +2 -2
  56. package/public/assets/{registry-DzUhgMq9.js.map → registry-pb1gTZEg.js.map} +1 -1
  57. package/public/assets/{security-BJ3_u0uY.js → security-BgcX1n9D.js} +2 -2
  58. package/public/assets/{security-BJ3_u0uY.js.map → security-BgcX1n9D.js.map} +1 -1
  59. package/public/assets/{select-AWvlctyQ.js → select-DZPVpKPv.js} +2 -2
  60. package/public/assets/{select-AWvlctyQ.js.map → select-DZPVpKPv.js.map} +1 -1
  61. package/public/assets/{settings-5ofqi-bN.js → settings-hd5kzdRB.js} +2 -2
  62. package/public/assets/{settings-5ofqi-bN.js.map → settings-hd5kzdRB.js.map} +1 -1
  63. package/public/assets/store-DKTLubBT.js +9 -0
  64. package/public/assets/store-DKTLubBT.js.map +1 -0
  65. package/public/assets/{tasks-BEE9bBf7.js → tasks-C5d2LU5E.js} +2 -2
  66. package/public/assets/{tasks-BEE9bBf7.js.map → tasks-C5d2LU5E.js.map} +1 -1
  67. package/public/assets/{teams-C27H_Gwo.js → teams-B8f2pGJe.js} +2 -2
  68. package/public/assets/{teams-C27H_Gwo.js.map → teams-B8f2pGJe.js.map} +1 -1
  69. package/public/assets/{terminal-viewer-impl-WD2TboA7.js → terminal-viewer-impl-CxAscH0U.js} +2 -2
  70. package/public/assets/{terminal-viewer-impl-WD2TboA7.js.map → terminal-viewer-impl-CxAscH0U.js.map} +1 -1
  71. package/public/assets/types-uVOXgvMT.js.map +1 -1
  72. package/public/assets/{work-queue-CtmPK2hK.js → work-queue-9imc7aNK.js} +2 -2
  73. package/public/assets/{work-queue-CtmPK2hK.js.map → work-queue-9imc7aNK.js.map} +1 -1
  74. package/public/assets/{workspaces-B26EKv_H.js → workspaces-CLYc3yF3.js} +2 -2
  75. package/public/assets/{workspaces-B26EKv_H.js.map → workspaces-CLYc3yF3.js.map} +1 -1
  76. package/public/index.html +3 -3
  77. package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
  78. package/runner/plugins/claude/monitors/relay-monitor.provisioned.mjs +4 -0
  79. package/src/agent-issue-repo.ts +10 -3
  80. package/src/agent-lifecycle-events.ts +23 -1
  81. package/src/agent-ref.ts +18 -3
  82. package/src/bus-outbox.ts +44 -1
  83. package/src/bus.ts +28 -5
  84. package/src/channel-target.ts +4 -0
  85. package/src/cli/steward.ts +15 -14
  86. package/src/commands-db.ts +63 -9
  87. package/src/config.ts +50 -0
  88. package/src/db/claim-sql.ts +16 -0
  89. package/src/db/continuations.ts +44 -3
  90. package/src/db/message-reads.ts +34 -4
  91. package/src/db/messages.ts +128 -4
  92. package/src/db/pairs.ts +1 -0
  93. package/src/db/schema.ts +6 -0
  94. package/src/gate-resolver.ts +124 -0
  95. package/src/lifecycle-manager.ts +41 -10
  96. package/src/lifecycle-signal-registry.ts +87 -0
  97. package/src/maintenance/constants.ts +5 -1
  98. package/src/maintenance/jobs.ts +28 -2
  99. package/src/maintenance/workspaces.ts +75 -44
  100. package/src/managed-policy-escalation.ts +68 -3
  101. package/src/mcp/tools-spawn.ts +16 -6
  102. package/src/mcp-plan-tools.ts +2 -3
  103. package/src/notification-settings.ts +11 -0
  104. package/src/notification-types.ts +4 -0
  105. package/src/operator-alarm.ts +37 -0
  106. package/src/routes/agents.ts +4 -0
  107. package/src/routes/orchestrator-proxy.ts +13 -1
  108. package/src/routes/orchestrator.ts +4 -0
  109. package/src/security.ts +20 -0
  110. package/src/self-resume.ts +185 -21
  111. package/src/server.ts +2 -0
  112. package/src/services/plan-graph.ts +42 -12
  113. package/src/services/register-agent.ts +5 -1
  114. package/src/services/send-message.ts +36 -0
  115. package/src/services/spawn-agent.ts +21 -2
  116. package/src/services/task-semantic-events.ts +17 -1
  117. package/src/spawn-command.ts +3 -0
  118. package/src/spawn-targets.ts +26 -1
  119. package/src/sse.ts +13 -5
  120. package/src/workspace-auto-merge.ts +90 -34
  121. package/src/workspace-escalation.ts +52 -3
  122. package/src/workspace-human-escalation.ts +68 -0
  123. package/src/workspace-orphans.ts +11 -6
  124. package/src/workspace-phase.ts +22 -1
  125. package/public/assets/store-D5et8685.js +0 -9
  126. package/public/assets/store-D5et8685.js.map +0 -1
@@ -500,7 +500,25 @@ export function selectSpawnOrchestrator(
500
500
  orchestratorId?: string,
501
501
  cwd?: string,
502
502
  preferHost?: string,
503
+ /**
504
+ * #1034 — the caller token's `constraints.orchestrators` allow-set. When present, host
505
+ * SELECTION must stay INSIDE it: never silently fall back to a host the token isn't
506
+ * authorized for. Previously, when neither an explicit id nor a cwd nor a resolvable
507
+ * `preferHost` pinned the host (e.g. the caller's `machine` was transiently unset, or the
508
+ * caller's own host briefly flapped offline), selection grabbed `candidates[0]` — an
509
+ * arbitrary online host. If that host was outside the token's allow-set, the downstream
510
+ * `assertResourceAllowed` gate then rejected the spawn with a MISLEADING `-32001 component
511
+ * token cannot access this resource`, which cleared "seconds later" once the caller's host
512
+ * re-appeared (issue #1034). Filtering candidates to the allow-set makes selection
513
+ * authz-consistent: for the common single-authorized-host token it deterministically returns
514
+ * that host regardless of transient machine/cwd resolution, so concurrent spawns no longer
515
+ * spuriously reject; when the authorized host is genuinely offline, the error is an honest
516
+ * "authorized host offline", not a confusing authz denial. Undefined/empty = no filtering
517
+ * (admin `*` tokens, byte-identical to before).
518
+ */
519
+ allowedOrchestratorIds?: string[],
503
520
  ): Orchestrator {
521
+ const allowSet = allowedOrchestratorIds?.length ? new Set(allowedOrchestratorIds) : null;
504
522
  if (orchestratorId) {
505
523
  const orchestrator = getOrchestrator(orchestratorId);
506
524
  if (!orchestrator) throw new McpNotFoundError(`orchestrator ${orchestratorId} not found`);
@@ -515,7 +533,9 @@ export function selectSpawnOrchestrator(
515
533
  }
516
534
  return orchestrator;
517
535
  }
518
- const candidates = listOrchestrators().filter((item) => item.status === "online" && item.providers.includes(provider));
536
+ const candidates = listOrchestrators().filter(
537
+ (item) => item.status === "online" && item.providers.includes(provider) && (!allowSet || allowSet.has(item.id)),
538
+ );
519
539
  if (cwd) {
520
540
  const match = candidates.find((item) => isPathWithinBase(cwd, item.baseDir));
521
541
  if (match) return match;
@@ -530,6 +550,11 @@ export function selectSpawnOrchestrator(
530
550
  }
531
551
  const orchestrator = candidates[0];
532
552
  if (!orchestrator) {
553
+ // #1034 — when the token's host allow-set is what emptied the candidate list, say so plainly
554
+ // (the caller's authorized host is offline) instead of the generic "no orchestrator" miss.
555
+ if (allowSet) {
556
+ throw new McpNotFoundError(`no online orchestrator authorized for this token runs provider '${provider}'. Your token is scoped to host(s): ${[...allowSet].join(", ")} — check that one is online (call relay_spawn_targets for the live matrix).`);
557
+ }
533
558
  throw new McpNotFoundError(`no online orchestrator runs provider '${provider}'. Available: ${spawnAlternatives()} (call relay_spawn_targets for the live matrix).`);
534
559
  }
535
560
  return orchestrator;
package/src/sse.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { getAgent, getOrchestrator, matchingDeliveryAgents, onWorkspaceChange, sendMessageWithResult, setChannelSubscriberProbe } from "./db";
2
- import { getOldestOutboxCursor, getOutboxCursor, replayEvents, type BusEvent } from "./bus-outbox";
2
+ import { getOutboxCursor, hasReplayGap, replayEventsSlice, type BusEvent } from "./bus-outbox";
3
3
  import { projectAgentStatusData } from "./agent-card-projection";
4
4
  import { emitRelayEvent, subscribeRelayEvents, type RelayEvent } from "./events";
5
5
  import { messageMatchesAgent, targetMatchesAgent } from "./agent-ref";
@@ -348,8 +348,7 @@ interface ReplayPlan {
348
348
  function replayPlan(lastEventId: number): ReplayPlan {
349
349
  const since = Number.isSafeInteger(lastEventId) && lastEventId > 0 ? lastEventId : 0;
350
350
  if (since === 0) return { requested: false, since: 0, available: false, gap: false };
351
- const oldest = getOldestOutboxCursor();
352
- const gap = oldest > 0 && oldest > since + 1;
351
+ const gap = hasReplayGap(since);
353
352
  return { requested: true, since, available: !gap, gap };
354
353
  }
355
354
 
@@ -359,9 +358,18 @@ function replayEventsToConnection(conn: Connection | undefined, plan: ReplayPlan
359
358
  send(conn, "replay.gap", { since: plan.since }, getOutboxCursor());
360
359
  return;
361
360
  }
361
+ // Drain the whole backlog in pages. A single capped replay dropped everything past the first 500
362
+ // events between the client's cursor and the outbox head (#1022 F1) — the SSE stream then went
363
+ // live from the head with no signal, so the tail was lost. Loop until the page is not truncated.
362
364
  const agentCache = new Map<string, AgentCard | null>();
363
- for (const event of replayEvents(plan.since)) {
364
- sendReplayEvent(conn, outboxToRelayEvent(event), agentCache);
365
+ let since = plan.since;
366
+ for (;;) {
367
+ const slice = replayEventsSlice(since);
368
+ for (const event of slice.events) {
369
+ sendReplayEvent(conn, outboxToRelayEvent(event), agentCache);
370
+ }
371
+ if (!slice.truncated) break;
372
+ since = slice.lastSeq;
365
373
  }
366
374
  }
367
375
 
@@ -43,6 +43,7 @@ import { settleWorkspaceCommand } from "./workspace-command-settle";
43
43
  import { CLEAR_MERGE_NO_PROGRESS, MAX_NO_PROGRESS_MERGE_ATTEMPTS, mergeNoProgressCount, noteMergeNoProgress } from "./workspace-merge-progress";
44
44
  import { workspaceOwnerOrchestrator } from "./workspace-host";
45
45
  import { workspaceActiveClaim } from "./workspace-claim";
46
+ import { notifyWorkspaceHumanEscalation } from "./workspace-human-escalation";
46
47
  import { DIRTY_WORKTREE_LAND_SKIP_REASON, READY_TO_LAND_STATUSES } from "./workspace-phase";
47
48
  import { fetchHostMergePreview } from "./workspace-probe";
48
49
  import type { Command, WorkspaceAutoMergePolicy, WorkspaceLandingPolicy, WorkspaceMergePreview, WorkspaceMergeStrategy, WorkspaceRecord } from "./types";
@@ -119,20 +120,33 @@ function autoMergeRequestOptions(ws: WorkspaceRecord): AutoMergeRequestOptions {
119
120
  // auto-provision the policy on the workspace-owning host, then queue a `policy:` wake
120
121
  // message (which also spawns the on-demand agent now via onMessageForPolicy). Honors a
121
122
  // per-workspace cooldown so a persistent conflict/behind row isn't re-pinged every sweep.
122
- // Returns the steward policy name on a fresh wake, or null (disabled / no owner / cooled down).
123
- export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | null {
123
+ // The result of trying to wake the managed steward (#1021 A1). `ok` carries the policy name
124
+ // only when a steward is genuinely alive or was just spawned. On `!ok`, `cooldown` tells a
125
+ // fresh failure (nothing spawned — the caller should advance the loop-breaker) apart from a
126
+ // recent successful wake still inside its cooldown window (NOT a failure — leave it be).
127
+ export type StewardWakeOutcome =
128
+ | { ok: true; policy: string }
129
+ | { ok: false; cooldown: boolean; error?: string };
130
+
131
+ // Wake the managed per-host/repo steward for a workspace it should handle: auto-provision
132
+ // the policy on the workspace-owning host, queue a `policy:` wake message, and spawn the
133
+ // on-demand agent. Only reports `ok` when the spawn actually happened (or a steward was
134
+ // already live) — a queued message alone is NOT a wake (#1021 A1). Honors a per-workspace
135
+ // cooldown so a persistent conflict/behind row isn't re-pinged every sweep.
136
+ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): StewardWakeOutcome {
124
137
  const meta = ws.metadata as Record<string, unknown>;
125
138
  const lastWoke = typeof meta.stewardWokenAt === "number" ? meta.stewardWokenAt : 0;
126
- if (lastWoke && Date.now() - lastWoke < STEWARD_WAKE_COOLDOWN_MS) return null;
139
+ if (lastWoke && Date.now() - lastWoke < STEWARD_WAKE_COOLDOWN_MS) return { ok: false, cooldown: true };
127
140
  const ownerResult = stewardOwnerForWorkspace(ws);
128
141
  if ("error" in ownerResult) {
129
142
  patchWorkspaceMetadata(ws.id, { stewardWakeError: ownerResult.error, stewardWakeErrorAt: Date.now() });
130
- return null;
143
+ return { ok: false, cooldown: false, error: ownerResult.error };
131
144
  }
132
145
  const policyName = ensureRepoSteward(ws.repoRoot, ownerResult.owner);
133
146
  if (!policyName) {
134
- patchWorkspaceMetadata(ws.id, { stewardWakeError: "cannot spawn steward: steward policy could not be provisioned", stewardWakeErrorAt: Date.now() });
135
- return null;
147
+ const error = "cannot spawn steward: steward policy could not be provisioned";
148
+ patchWorkspaceMetadata(ws.id, { stewardWakeError: error, stewardWakeErrorAt: Date.now() });
149
+ return { ok: false, cooldown: false, error };
136
150
  }
137
151
  // #640 — resolve the identity triangle so the steward isn't blind to the people-graph:
138
152
  // the workspace's branch agent (owner) and that agent's coordinator (owner's `spawnedBy`).
@@ -163,7 +177,16 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
163
177
  payload: { kind: "workspace.steward-task", workspaceId: ws.id, repoRoot: ws.repoRoot, worktreePath: ws.worktreePath, branch: ws.branch, baseRef: ws.baseRef, status: ws.status, reason, ownerAgentId, coordinatorAgentId },
164
178
  },
165
179
  });
166
- getLifecycleManager().onMessageForPolicy(policyName);
180
+ // #1021 A1 — the durable `policy:` obligation is enqueued above (it survives until a
181
+ // steward claims it), but a wake is only real if a steward actually spawned or is
182
+ // already live. If nothing spawned (host offline / provider error), record the error and
183
+ // report failure — do NOT stamp stewardWokenAt (which arms the cooldown) or claim a
184
+ // hand-off that never happened.
185
+ const spawn = getLifecycleManager().onMessageForPolicy(policyName);
186
+ if (!spawn.woke) {
187
+ patchWorkspaceMetadata(ws.id, { stewardWakeError: spawn.reason, stewardWakeErrorAt: Date.now() });
188
+ return { ok: false, cooldown: false, error: spawn.reason };
189
+ }
167
190
  patchWorkspaceMetadata(ws.id, {
168
191
  stewardWokenAt: Date.now(),
169
192
  stewardPolicy: policyName,
@@ -173,10 +196,11 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
173
196
  });
174
197
  // #641/#642 — tell the owner which steward took over and proactively loop in its coordinator.
175
198
  notifyStewardHandoff({ workspace: ws, stewardId: policyName, coordinatorAgentId, reason });
176
- return policyName;
199
+ return { ok: true, policy: policyName };
177
200
  } catch (e) {
178
- patchWorkspaceMetadata(ws.id, { stewardWakeError: e instanceof Error ? e.message : "failed to wake steward", stewardWakeErrorAt: Date.now() });
179
- return null;
201
+ const error = e instanceof Error ? e.message : "failed to wake steward";
202
+ patchWorkspaceMetadata(ws.id, { stewardWakeError: error, stewardWakeErrorAt: Date.now() });
203
+ return { ok: false, cooldown: false, error };
180
204
  }
181
205
  }
182
206
 
@@ -227,25 +251,36 @@ function escalateNoProgressMerge(ws: WorkspaceRecord): void {
227
251
  const { coordinatorAgentId } = resolveStewardIdentity(ws);
228
252
  const fallback = stewardFallbackTarget();
229
253
  const recipients = [...new Set([coordinatorAgentId, fallback, ws.ownerAgentId].filter((r): r is string => Boolean(r)))];
254
+ const subject = "Workspace can't auto-land — needs attention";
255
+ const body = renderTemplate("steward.merge-stall", {
256
+ branch: `\`${ws.branch ?? ws.id}\``,
257
+ repoRoot: ws.repoRoot,
258
+ attempts: String(MAX_NO_PROGRESS_MERGE_ATTEMPTS),
259
+ mergeError,
260
+ baseRef: ws.baseRef ?? "base",
261
+ });
262
+ const payload = { kind: "workspace.auto-merge-stuck", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: ws.baseRef, attempts: MAX_NO_PROGRESS_MERGE_ATTEMPTS, reason: mergeError };
263
+ let escalated = false;
230
264
  if (recipients.length) {
231
265
  routeNotification({
232
266
  type: "workspace.stranded_escalation",
233
267
  scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ws.ownerAgentId, parent: coordinatorAgentId ?? undefined },
234
268
  recipients,
235
269
  message: {
236
- subject: "Workspace can't auto-land — needs attention",
237
- body: renderTemplate("steward.merge-stall", {
238
- branch: `\`${ws.branch ?? ws.id}\``,
239
- repoRoot: ws.repoRoot,
240
- attempts: String(MAX_NO_PROGRESS_MERGE_ATTEMPTS),
241
- mergeError,
242
- baseRef: ws.baseRef ?? "base",
243
- }),
244
- payload: { kind: "workspace.auto-merge-stuck", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: ws.baseRef, attempts: MAX_NO_PROGRESS_MERGE_ATTEMPTS, reason: mergeError },
270
+ subject,
271
+ body,
272
+ payload,
245
273
  },
246
274
  });
275
+ escalated = true;
276
+ } else {
277
+ const human = notifyWorkspaceHumanEscalation({ subject, body, payload, source: "workspace-auto-merge" });
278
+ escalated = human.accepted;
279
+ if (!human.accepted) {
280
+ console.warn(`[workspace-auto-merge] no-progress escalation for ${ws.id} had zero agent recipients and no human channel: ${human.reason ?? "unknown reason"}`);
281
+ }
247
282
  }
248
- patchWorkspaceMetadata(ws.id, { autoMergeBackedOffAt: Date.now() });
283
+ if (escalated) patchWorkspaceMetadata(ws.id, { autoMergeBackedOffAt: Date.now() });
249
284
  createActivityEvent({
250
285
  clientId: `workspace-auto-merge-backoff-${ws.id}-${Date.now()}`,
251
286
  kind: "state",
@@ -349,6 +384,22 @@ async function landCandidates(
349
384
  const preview = await fetchFirstUsableMergePreview(ws, orchestrators);
350
385
  if (!preview) {
351
386
  previewUnavailable.push(ws.id);
387
+ // #1025 A5 — a lane whose preview is persistently unusable (owner host offline, the
388
+ // worktree path is gone, or every probe returns error/missing) used to exit here with
389
+ // ZERO side effects: no no-progress, no wake, no notify — a ready branch would strand
390
+ // silently forever. Give it the SAME no-progress/strand path as the unknown-merge-state
391
+ // lane so no status is escalation-blind: advance the loop-breaker, and once it crosses
392
+ // the cap, escalateNoProgressMerge fires the owner `landed-failure` wake + steward/
393
+ // coordinator escalation. A transient blip self-clears when a usable preview returns.
394
+ const held = patchWorkspaceMetadata(ws.id, {
395
+ ...noteMergeNoProgress(ws.metadata),
396
+ mergeError: "no online host returned a usable merge preview (owner offline, worktree missing, or probe error)",
397
+ lastPreviewUnavailableAt: Date.now(),
398
+ }) ?? ws;
399
+ if (mergeNoProgressCount(held.metadata) >= MAX_NO_PROGRESS_MERGE_ATTEMPTS) {
400
+ backedOff.push(ws.id);
401
+ escalateNoProgressMerge(held);
402
+ }
352
403
  continue;
353
404
  }
354
405
  const p = preview as WorkspaceMergePreview;
@@ -401,21 +452,26 @@ async function landCandidates(
401
452
  }
402
453
  }
403
454
  leftForSteward.push(ws.id);
404
- let woke: string | null = null;
405
455
  if (stewardEnabled) {
406
- woke = wakeRepoSteward(handoffWorkspace, p.conflict === true ? "conflict" : "merge state unknown");
407
- if (woke) wokeStewards.push(woke);
408
- }
409
- if (stewardEnabled && !woke && p.conflict === true) {
410
- const failedHandoff = patchWorkspaceMetadata(ws.id, {
411
- ...noteMergeNoProgress(ws.metadata),
412
- mergeError: "cannot spawn steward on the workspace owning host",
413
- lastStewardWakeFailedAt: Date.now(),
414
- }) ?? ws;
415
- if (mergeNoProgressCount(failedHandoff.metadata) >= MAX_NO_PROGRESS_MERGE_ATTEMPTS) {
416
- backedOff.push(ws.id);
417
- escalateNoProgressMerge(failedHandoff);
418
- continue;
456
+ const wake = wakeRepoSteward(handoffWorkspace, p.conflict === true ? "conflict" : "merge state unknown");
457
+ if (wake.ok) {
458
+ wokeStewards.push(wake.policy);
459
+ } else if (!wake.cooldown && p.conflict === true) {
460
+ // #1021 A1+A2 — a GENUINE wake failure (nothing spawned: host offline / provider
461
+ // error) now correctly yields !ok, so the conflict lane's loop-breaker advances and
462
+ // escalateNoProgressMerge becomes reachable. A cooldown means a steward WAS woken
463
+ // recently and is presumably working — that is not a failure, so don't count it (and
464
+ // don't mislabel the cooldown as a spawn failure in mergeError). Carry the real error.
465
+ const failedHandoff = patchWorkspaceMetadata(ws.id, {
466
+ ...noteMergeNoProgress(ws.metadata),
467
+ mergeError: wake.error ?? "cannot spawn steward on the workspace owning host",
468
+ lastStewardWakeFailedAt: Date.now(),
469
+ }) ?? ws;
470
+ if (mergeNoProgressCount(failedHandoff.metadata) >= MAX_NO_PROGRESS_MERGE_ATTEMPTS) {
471
+ backedOff.push(ws.id);
472
+ escalateNoProgressMerge(failedHandoff);
473
+ continue;
474
+ }
419
475
  }
420
476
  }
421
477
  if (p.conflict === true) {
@@ -1,23 +1,43 @@
1
1
  import { stewardFallbackTarget } from "./config";
2
2
  import { routeNotification } from "./notification-router";
3
+ import { raiseOperatorAlarm } from "./operator-alarm";
3
4
  import { resolveStewardIdentity } from "./steward-identity";
5
+ import { notifyWorkspaceHumanEscalation } from "./workspace-human-escalation";
4
6
  import type { NotificationType } from "./notification-types";
5
7
  import type { WorkspaceRecord, WorkspaceStatus } from "./types";
6
8
 
7
9
  export const READY_TO_LAND_ESCALATION_STATUSES = new Set<WorkspaceStatus>(["ready", "review_requested"]);
8
10
 
9
- export function routeWorkspaceCustodyEscalation(input: {
11
+ export interface WorkspaceCustodyEscalationInput {
10
12
  workspace: Pick<WorkspaceRecord, "id" | "repoRoot" | "branch" | "status" | "ownerAgentId">;
11
13
  type?: NotificationType;
12
14
  subject: string;
13
15
  body: string;
14
16
  payload: Record<string, unknown>;
15
17
  idempotencyKey?: string;
16
- }): string[] {
18
+ }
19
+
20
+ export function routeWorkspaceCustodyEscalation(input: WorkspaceCustodyEscalationInput): string[] {
17
21
  const { ownerAgentId, coordinatorAgentId } = resolveStewardIdentity(input.workspace);
18
22
  const fallback = stewardFallbackTarget();
19
23
  const recipients = [...new Set([ownerAgentId, coordinatorAgentId, fallback].filter((value): value is string => Boolean(value)))];
20
- if (!recipients.length) return [];
24
+ if (!recipients.length) {
25
+ const payload = {
26
+ ...input.payload,
27
+ coordinatorAgentId,
28
+ ownerAgentId,
29
+ fallbackTarget: fallback || undefined,
30
+ };
31
+ const human = notifyWorkspaceHumanEscalation({
32
+ subject: input.subject,
33
+ body: input.body,
34
+ payload,
35
+ source: "workspace-custody",
36
+ });
37
+ if (human.accepted && human.channelId) return [human.channelId];
38
+ console.warn(`[workspace-escalation] zero agent recipients and no human channel for workspace ${input.workspace.id}: ${human.reason ?? "unknown reason"}`);
39
+ return [];
40
+ }
21
41
 
22
42
  routeNotification({
23
43
  type: input.type ?? "workspace.stranded_escalation",
@@ -43,3 +63,32 @@ export function routeWorkspaceCustodyEscalation(input: {
43
63
  });
44
64
  return recipients;
45
65
  }
66
+
67
+ // Route a custody escalation, and if it resolves to ZERO reachable recipients — owner
68
+ // exited, coordinator idle, no `AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK` configured —
69
+ // raise a louder operator-visible alarm instead of letting the caller silently record it
70
+ // as "escalated" (#1025 C1/C4, same class as #1021 A4). A disk-only orphan (ownerAgentId
71
+ // undefined) and most `review_requested` auto-abandons have no agent to page by default,
72
+ // so without this the notice is swallowed and the strand loops in the dark. Returns the
73
+ // (possibly empty) recipient list so the caller can still decide whether to stamp state.
74
+ export function routeWorkspaceCustodyEscalationOrAlarm(
75
+ input: WorkspaceCustodyEscalationInput & { source: string },
76
+ ): string[] {
77
+ const { source, ...escalation } = input;
78
+ const targets = routeWorkspaceCustodyEscalation(escalation);
79
+ if (!targets.length) {
80
+ raiseOperatorAlarm({
81
+ clientId: `${escalation.idempotencyKey ?? `${escalation.workspace.id}:${escalation.subject}`}:no-recipients`,
82
+ source,
83
+ subject: escalation.subject,
84
+ detail: `${escalation.body} — no owner, coordinator, or fallback configured to escalate to. Configure AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK or reclaim the worktree manually.`,
85
+ metadata: {
86
+ workspaceId: escalation.workspace.id,
87
+ repoRoot: escalation.workspace.repoRoot,
88
+ branch: escalation.workspace.branch,
89
+ status: escalation.workspace.status,
90
+ },
91
+ });
92
+ }
93
+ return targets;
94
+ }
@@ -0,0 +1,68 @@
1
+ import { errMessage } from "agent-relay-sdk";
2
+ import { stewardFallbackTarget } from "./config";
3
+ import { getChannel } from "./db";
4
+ import { relayEgressToChannel, relayNotifyNtfy } from "./mcp-egress-tools";
5
+ import { getResolvedNotificationsConfig } from "./notification-settings-store";
6
+
7
+ const DEFAULT_NTFY_CHANNEL_ID = "ntfy:default";
8
+ const WARN = "no human-reachable escalation target - stranded workspaces will page nobody. Set AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK or configure ntfy.";
9
+
10
+ export interface WorkspaceHumanEscalationInput {
11
+ subject: string;
12
+ body: string;
13
+ payload?: Record<string, unknown>;
14
+ priority?: "min" | "low" | "default" | "high" | "urgent";
15
+ tags?: string[];
16
+ source?: string;
17
+ }
18
+
19
+ export interface WorkspaceHumanEscalationResult {
20
+ accepted: boolean;
21
+ channelId?: string;
22
+ reason?: string;
23
+ }
24
+
25
+ export function workspaceHumanEscalationChannelId(): string {
26
+ return getResolvedNotificationsConfig().humanEscalationChannelId.trim();
27
+ }
28
+
29
+ export function hasHumanEscalationChannel(): boolean {
30
+ const channelId = workspaceHumanEscalationChannelId();
31
+ return Boolean(channelId && getChannel(channelId));
32
+ }
33
+
34
+ export function hasHumanReachableEscalationTarget(): boolean {
35
+ return Boolean(stewardFallbackTarget() || hasHumanEscalationChannel());
36
+ }
37
+
38
+ export function warnIfNoHumanReachableEscalationTarget(log: (message: string) => void = console.warn): boolean {
39
+ if (hasHumanReachableEscalationTarget()) return false;
40
+ log(WARN);
41
+ return true;
42
+ }
43
+
44
+ export function notifyWorkspaceHumanEscalation(input: WorkspaceHumanEscalationInput): WorkspaceHumanEscalationResult {
45
+ const channelId = workspaceHumanEscalationChannelId();
46
+ if (!channelId) return { accepted: false, reason: "no human escalation channel configured" };
47
+ if (!getChannel(channelId)) return { accepted: false, reason: `human escalation channel ${channelId} is not registered` };
48
+
49
+ const event: Record<string, unknown> = {
50
+ title: input.subject,
51
+ body: input.body,
52
+ priority: input.priority ?? "urgent",
53
+ tags: input.tags ?? ["agent-relay", "workspace"],
54
+ ...(input.payload ? { relay: input.payload } : {}),
55
+ ...(input.source ? { source: input.source } : {}),
56
+ };
57
+
58
+ try {
59
+ if (channelId === DEFAULT_NTFY_CHANNEL_ID) {
60
+ relayNotifyNtfy(event);
61
+ } else {
62
+ relayEgressToChannel({ channelId, event });
63
+ }
64
+ return { accepted: true, channelId };
65
+ } catch (error) {
66
+ return { accepted: false, channelId, reason: errMessage(error) };
67
+ }
68
+ }
@@ -30,7 +30,7 @@ import { isOwnerAlive, liveAgentCwds, worktreeInUseByLiveAgent } from "./workspa
30
30
  import { applyWorkspaceAction } from "./workspace-actions";
31
31
  import { workspaceOwnerOrchestrator } from "./workspace-host";
32
32
  import { UNLANDED_FLAG_COOLDOWN_MS, orphanGraceMs, orphanWorktreeReapEnabled, relayToken } from "./config";
33
- import { READY_TO_LAND_ESCALATION_STATUSES, routeWorkspaceCustodyEscalation } from "./workspace-escalation";
33
+ import { READY_TO_LAND_ESCALATION_STATUSES, routeWorkspaceCustodyEscalationOrAlarm } from "./workspace-escalation";
34
34
 
35
35
  // Don't re-flag the same un-landed orphan every sweep — surface it once, then
36
36
  // stay quiet for this window. In-memory (keyed by worktree path) like the
@@ -425,7 +425,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
425
425
  ? "uncommitted changes in the worktree"
426
426
  : `${orphan.unmergedAhead ?? orphan.ahead ?? "?"} un-landed commit(s)`;
427
427
  if (orphanHasWorkEvidence(orphan)) {
428
- routeWorkspaceCustodyEscalation({
428
+ routeWorkspaceCustodyEscalationOrAlarm({
429
+ source: "workspace-orphan-reaper",
429
430
  workspace: { id: `orphan:${orphan.worktreePath}`, repoRoot: orphan.repoRoot, branch: orphan.branch, status: "review_requested", ownerAgentId: undefined },
430
431
  subject: "Stranded worktree needs attention",
431
432
  body: `${orphan.branch ?? orphan.worktreePath} in ${orphan.repoRoot} is orphaned and holds work that has not landed - ${detail}. Recover the commits before removing it.`,
@@ -456,7 +457,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
456
457
  if (now - last < UNLANDED_FLAG_COOLDOWN_MS) continue;
457
458
  flaggedAt.set(key, now);
458
459
  if (workspaceNeedsCustodyEscalation(workspace, false)) {
459
- routeWorkspaceCustodyEscalation({
460
+ routeWorkspaceCustodyEscalationOrAlarm({
461
+ source: "workspace-orphan-reaper",
460
462
  workspace,
461
463
  subject: "Workspace row has no worktree on disk",
462
464
  body: `Workspace \`${workspace.branch ?? workspace.id}\` in ${workspace.repoRoot} is ${workspace.status}, but its worktree ${workspace.worktreePath} is missing on disk. This was in Relay custody; recover or explicitly abandon it.`,
@@ -506,7 +508,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
506
508
  autoAbandoned.push(workspace.id);
507
509
  flaggedAt.delete(key);
508
510
  if (workspaceNeedsCustodyEscalation(workspace, previewHasWorkEvidence(probe.kind === "preview" ? probe.preview : null))) {
509
- routeWorkspaceCustodyEscalation({
511
+ routeWorkspaceCustodyEscalationOrAlarm({
512
+ source: "workspace-orphan-reaper",
510
513
  workspace,
511
514
  type: "workspace.auto_abandoned",
512
515
  subject: "Workspace auto-abandoned",
@@ -547,7 +550,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
547
550
  ? `host could not confirm whether branch ${workspace.branch} has landed`
548
551
  : "host could not confirm whether recoverable branch work remains";
549
552
  if (workspaceNeedsCustodyEscalation(workspace, hasWorkEvidence)) {
550
- routeWorkspaceCustodyEscalation({
553
+ routeWorkspaceCustodyEscalationOrAlarm({
554
+ source: "workspace-orphan-reaper",
551
555
  workspace,
552
556
  subject: "Missing-worktree workspace needs attention",
553
557
  body: `${workspace.id} in ${workspace.repoRoot} has no worktree on disk and cannot be auto-abandoned safely - ${detail}. Recover it, then abandon or clean it up explicitly.`,
@@ -657,7 +661,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
657
661
  : `${preview?.unmergedAhead ?? preview?.ahead ?? "?"} un-landed commit(s)`;
658
662
  const hasWorkEvidence = previewHasWorkEvidence(preview);
659
663
  if (workspaceNeedsCustodyEscalation(workspace, hasWorkEvidence)) {
660
- routeWorkspaceCustodyEscalation({
664
+ routeWorkspaceCustodyEscalationOrAlarm({
665
+ source: "workspace-orphan-reaper",
661
666
  workspace,
662
667
  subject: "Dead-owner worktree needs attention",
663
668
  body: `${workspace.branch ?? workspace.id} in ${workspace.repoRoot} is owned by gone agent ${workspace.ownerAgentId ?? "?"} and holds work that has not landed (${detail}). Recover the commits, then clean it up explicitly.`,
@@ -200,7 +200,27 @@ export function describeWorkspacePhase(
200
200
  blockers: [],
201
201
  };
202
202
  }
203
- case "merge_planned":
203
+ case "merge_planned": {
204
+ // #1025 A6 — a merge is dispatched and normally settles in seconds; the in-flight
205
+ // watchdog recovers a genuinely-stuck one at ~20 min. But a lost merge command/result
206
+ // (target host gone) could leave the row here indefinitely while this view reported
207
+ // "healthy, wait" forever. Give it the same #242 stall clock as land-pending: past the
208
+ // bound, surface it as needs-attention with a real blocker instead of the anti-panic
209
+ // view. Clocked on `readyAt` (the stable hand-off time, immune to heartbeat bumps).
210
+ const now = opts.now ?? Date.now();
211
+ const stallMs = opts.stallMs ?? LAND_PENDING_STALL_MS;
212
+ const pendingMs = typeof workspace.readyAt === "number" ? now - workspace.readyAt : undefined;
213
+ if (pendingMs !== undefined && pendingMs > stallMs) {
214
+ const mins = Math.round(pendingMs / 60_000);
215
+ return {
216
+ phase: "landing",
217
+ headline: `Stalled — merge was dispatched ${mins} min ago but hasn't settled. A merge normally lands in seconds, so this is past the healthy window and likely stuck (the owning host went offline mid-merge, or the merge result was lost).`,
218
+ hint: "Do NOT merge, push, rebase, or touch the main checkout yourself. Flag this to a human or the repo steward — the merge dispatch isn't settling and needs attention.",
219
+ actionNeeded: true,
220
+ nextActions: [WAIT_ACTION],
221
+ blockers: [`merge dispatched ~${mins} min ago with no result — the land isn't settling`],
222
+ };
223
+ }
204
224
  return {
205
225
  phase: "landing",
206
226
  headline: renderTemplate("workspace.phase.merge-planned.headline"),
@@ -209,6 +229,7 @@ export function describeWorkspacePhase(
209
229
  nextActions: [WAIT_ACTION],
210
230
  blockers: [],
211
231
  };
232
+ }
212
233
  case "conflict":
213
234
  return {
214
235
  phase: "steward",