agent-relay-server 0.121.0 → 0.121.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +5 -4
- package/public/assets/{MessageBubble-BaPGFJxq.js → MessageBubble-CLqxCG9b.js} +2 -2
- package/public/assets/{MessageBubble-BaPGFJxq.js.map → MessageBubble-CLqxCG9b.js.map} +1 -1
- package/public/assets/{PlanGraphPanel-BXCVfmln.js → PlanGraphPanel-BkkToFEb.js} +2 -2
- package/public/assets/{PlanGraphPanel-BXCVfmln.js.map → PlanGraphPanel-BkkToFEb.js.map} +1 -1
- package/public/assets/{activity-D_shwAZt.js → activity-D-ocGQGs.js} +2 -2
- package/public/assets/{activity-D_shwAZt.js.map → activity-D-ocGQGs.js.map} +1 -1
- package/public/assets/{agent-profiles-BrfVz5IV.js → agent-profiles-CRPgYTyb.js} +2 -2
- package/public/assets/{agent-profiles-BrfVz5IV.js.map → agent-profiles-CRPgYTyb.js.map} +1 -1
- package/public/assets/{agents-B5w-Z9Ic.js → agents-Chw5E_2q.js} +2 -2
- package/public/assets/{agents-B5w-Z9Ic.js.map → agents-Chw5E_2q.js.map} +1 -1
- package/public/assets/{analytics-Bm3D12UN.js → analytics-BmNDG0hR.js} +2 -2
- package/public/assets/{analytics-Bm3D12UN.js.map → analytics-BmNDG0hR.js.map} +1 -1
- package/public/assets/{automation-DOTL8BFs.js → automation-D7g90kTv.js} +2 -2
- package/public/assets/{automation-DOTL8BFs.js.map → automation-D7g90kTv.js.map} +1 -1
- package/public/assets/{branch-state-badge-BlqeJZ5-.js → branch-state-badge-Bv7DhLBZ.js} +2 -2
- package/public/assets/{branch-state-badge-BlqeJZ5-.js.map → branch-state-badge-Bv7DhLBZ.js.map} +1 -1
- package/public/assets/{channels-BoiKfK6Q.js → channels-CLwPeEPi.js} +2 -2
- package/public/assets/{channels-BoiKfK6Q.js.map → channels-CLwPeEPi.js.map} +1 -1
- package/public/assets/{chat-PeU3iOaa.js → chat-Cgj7VKzc.js} +2 -2
- package/public/assets/{chat-PeU3iOaa.js.map → chat-Cgj7VKzc.js.map} +1 -1
- package/public/assets/{connectors-BvV8Hee2.js → connectors-Br6fiTny.js} +2 -2
- package/public/assets/{connectors-BvV8Hee2.js.map → connectors-Br6fiTny.js.map} +1 -1
- package/public/assets/{formatted-body-C1FgzlT1.js → formatted-body-CmkuD23M.js} +3 -3
- package/public/assets/{formatted-body-C1FgzlT1.js.map → formatted-body-CmkuD23M.js.map} +1 -1
- package/public/assets/{formatted-body-impl-BTpU1xH4.js → formatted-body-impl-DbdFmbwW.js} +2 -2
- package/public/assets/{formatted-body-impl-BTpU1xH4.js.map → formatted-body-impl-DbdFmbwW.js.map} +1 -1
- package/public/assets/{index-DZEahmyc.js → index-D9OogaB5.js} +6 -6
- package/public/assets/{index-DZEahmyc.js.map → index-D9OogaB5.js.map} +1 -1
- package/public/assets/{insights-BqmgSpQm.js → insights-qvaPqWCV.js} +2 -2
- package/public/assets/{insights-BqmgSpQm.js.map → insights-qvaPqWCV.js.map} +1 -1
- package/public/assets/{integrations-CuT92zQi.js → integrations-DYEGSqq_.js} +2 -2
- package/public/assets/{integrations-CuT92zQi.js.map → integrations-DYEGSqq_.js.map} +1 -1
- package/public/assets/{maintenance-lM7FeXVC.js → maintenance-C0HIEB-J.js} +2 -2
- package/public/assets/{maintenance-lM7FeXVC.js.map → maintenance-C0HIEB-J.js.map} +1 -1
- package/public/assets/{managed-agents-DaPNv3yY.js → managed-agents-DdS7aI38.js} +2 -2
- package/public/assets/{managed-agents-DaPNv3yY.js.map → managed-agents-DdS7aI38.js.map} +1 -1
- package/public/assets/{markdown-preview-impl-Wia3Ho1s.js → markdown-preview-impl-BnXMH1z1.js} +2 -2
- package/public/assets/{markdown-preview-impl-Wia3Ho1s.js.map → markdown-preview-impl-BnXMH1z1.js.map} +1 -1
- package/public/assets/{memory-B1v_zD-M.js → memory-CSwx7bz8.js} +2 -2
- package/public/assets/{memory-B1v_zD-M.js.map → memory-CSwx7bz8.js.map} +1 -1
- package/public/assets/{messages-BNOx3yK6.js → messages-Be9CmvDv.js} +2 -2
- package/public/assets/{messages-BNOx3yK6.js.map → messages-Be9CmvDv.js.map} +1 -1
- package/public/assets/{orchestrators-DWFy4asl.js → orchestrators-BgtvvHvo.js} +2 -2
- package/public/assets/{orchestrators-DWFy4asl.js.map → orchestrators-BgtvvHvo.js.map} +1 -1
- package/public/assets/{overview-C_12cFPM.js → overview-BvhbPWtF.js} +2 -2
- package/public/assets/{overview-C_12cFPM.js.map → overview-BvhbPWtF.js.map} +1 -1
- package/public/assets/{pairs-DIc7gX83.js → pairs-CpXSOMx0.js} +2 -2
- package/public/assets/{pairs-DIc7gX83.js.map → pairs-CpXSOMx0.js.map} +1 -1
- package/public/assets/{projects-DPjvJUzT.js → projects-DdPNPFJI.js} +2 -2
- package/public/assets/{projects-DPjvJUzT.js.map → projects-DdPNPFJI.js.map} +1 -1
- package/public/assets/{prompt-settings-CtumNgG8.js → prompt-settings-BpZse7-X.js} +2 -2
- package/public/assets/{prompt-settings-CtumNgG8.js.map → prompt-settings-BpZse7-X.js.map} +1 -1
- package/public/assets/{registry-DzUhgMq9.js → registry-pb1gTZEg.js} +2 -2
- package/public/assets/{registry-DzUhgMq9.js.map → registry-pb1gTZEg.js.map} +1 -1
- package/public/assets/{security-BJ3_u0uY.js → security-BgcX1n9D.js} +2 -2
- package/public/assets/{security-BJ3_u0uY.js.map → security-BgcX1n9D.js.map} +1 -1
- package/public/assets/{select-AWvlctyQ.js → select-DZPVpKPv.js} +2 -2
- package/public/assets/{select-AWvlctyQ.js.map → select-DZPVpKPv.js.map} +1 -1
- package/public/assets/{settings-5ofqi-bN.js → settings-hd5kzdRB.js} +2 -2
- package/public/assets/{settings-5ofqi-bN.js.map → settings-hd5kzdRB.js.map} +1 -1
- package/public/assets/store-DKTLubBT.js +9 -0
- package/public/assets/store-DKTLubBT.js.map +1 -0
- package/public/assets/{tasks-BEE9bBf7.js → tasks-C5d2LU5E.js} +2 -2
- package/public/assets/{tasks-BEE9bBf7.js.map → tasks-C5d2LU5E.js.map} +1 -1
- package/public/assets/{teams-C27H_Gwo.js → teams-B8f2pGJe.js} +2 -2
- package/public/assets/{teams-C27H_Gwo.js.map → teams-B8f2pGJe.js.map} +1 -1
- package/public/assets/{terminal-viewer-impl-WD2TboA7.js → terminal-viewer-impl-CxAscH0U.js} +2 -2
- package/public/assets/{terminal-viewer-impl-WD2TboA7.js.map → terminal-viewer-impl-CxAscH0U.js.map} +1 -1
- package/public/assets/types-uVOXgvMT.js.map +1 -1
- package/public/assets/{work-queue-CtmPK2hK.js → work-queue-9imc7aNK.js} +2 -2
- package/public/assets/{work-queue-CtmPK2hK.js.map → work-queue-9imc7aNK.js.map} +1 -1
- package/public/assets/{workspaces-B26EKv_H.js → workspaces-CLYc3yF3.js} +2 -2
- package/public/assets/{workspaces-B26EKv_H.js.map → workspaces-CLYc3yF3.js.map} +1 -1
- package/public/index.html +3 -3
- package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/runner/plugins/claude/monitors/relay-monitor.provisioned.mjs +4 -0
- package/src/agent-issue-repo.ts +10 -3
- package/src/agent-lifecycle-events.ts +23 -1
- package/src/agent-ref.ts +18 -3
- package/src/bus-outbox.ts +44 -1
- package/src/bus.ts +34 -7
- package/src/channel-target.ts +4 -0
- package/src/cli/steward.ts +15 -14
- package/src/commands-db.ts +63 -9
- package/src/config.ts +50 -0
- package/src/db/agents.ts +6 -2
- package/src/db/claim-sql.ts +16 -0
- package/src/db/continuations.ts +44 -3
- package/src/db/message-reads.ts +34 -4
- package/src/db/messages.ts +128 -4
- package/src/db/pairs.ts +1 -0
- package/src/db/schema.ts +6 -0
- package/src/event-loop-lag.ts +10 -1
- package/src/gate-resolver.ts +124 -0
- package/src/lifecycle-manager.ts +41 -10
- package/src/lifecycle-signal-registry.ts +87 -0
- package/src/maintenance/constants.ts +5 -1
- package/src/maintenance/jobs.ts +36 -4
- package/src/maintenance/workspaces.ts +75 -44
- package/src/managed-policy-escalation.ts +68 -3
- package/src/mcp/tools-spawn.ts +16 -6
- package/src/mcp-plan-tools.ts +2 -3
- package/src/notification-settings.ts +11 -0
- package/src/notification-types.ts +4 -0
- package/src/operator-alarm.ts +37 -0
- package/src/routes/agents.ts +4 -0
- package/src/routes/orchestrator-proxy.ts +13 -1
- package/src/routes/orchestrator.ts +4 -0
- package/src/routes/workspaces.ts +25 -7
- package/src/security.ts +35 -0
- package/src/self-resume.ts +185 -21
- package/src/server.ts +8 -0
- package/src/services/plan-graph.ts +42 -12
- package/src/services/register-agent.ts +5 -1
- package/src/services/send-message.ts +36 -0
- package/src/services/spawn-agent.ts +21 -2
- package/src/services/task-semantic-events.ts +17 -1
- package/src/spawn-command.ts +3 -0
- package/src/spawn-targets.ts +26 -1
- package/src/sse.ts +13 -5
- package/src/workspace-auto-merge.ts +90 -34
- package/src/workspace-escalation.ts +52 -3
- package/src/workspace-human-escalation.ts +68 -0
- package/src/workspace-merge.ts +16 -0
- package/src/workspace-orphans.ts +11 -6
- package/src/workspace-phase.ts +22 -1
- package/public/assets/store-D5et8685.js +0 -9
- package/public/assets/store-D5et8685.js.map +0 -1
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
// shared `buildSpawnCommand`/`buildManagedSpawnParams` payload home directly. Folding
|
|
24
24
|
// them through `spawnAgent(systemCtx)` is a clean belt-and-suspenders follow-up (#342),
|
|
25
25
|
// deliberately out of scope here — this slice converges the two external TRANSPORTS.
|
|
26
|
-
import type { SpawnProvider } from "agent-relay-sdk";
|
|
26
|
+
import type { ReportVerbosity, SpawnProvider } from "agent-relay-sdk";
|
|
27
27
|
import { DEFAULT_USER_ID, errMessage, isRecord, stringValue } from "agent-relay-sdk";
|
|
28
28
|
import { statSync } from "node:fs";
|
|
29
29
|
import { dirname, resolve } from "node:path";
|
|
@@ -211,6 +211,12 @@ export interface SpawnAgentInput {
|
|
|
211
211
|
* Explicit true is preserved in the command payload for compatibility only.
|
|
212
212
|
*/
|
|
213
213
|
expectReply?: boolean;
|
|
214
|
+
/**
|
|
215
|
+
* #1037 — per-spawn report-up verbosity (`quiet` default | `normal` | `verbose`). Persisted in
|
|
216
|
+
* the spawn command params and stamped onto the child's meta at registration, where the lineage
|
|
217
|
+
* report-up delivery gate reads it. Absent = `quiet`.
|
|
218
|
+
*/
|
|
219
|
+
reportVerbosity?: ReportVerbosity;
|
|
214
220
|
tags?: string[];
|
|
215
221
|
capabilities?: string[];
|
|
216
222
|
providerArgs?: string[];
|
|
@@ -249,6 +255,8 @@ export interface SpawnAgentResult {
|
|
|
249
255
|
agentId: string | null;
|
|
250
256
|
registered: boolean;
|
|
251
257
|
command: Command;
|
|
258
|
+
/** #1033 — the durable Task this spawn is bound to (explicit taskId or auto-scaffolded), when any. */
|
|
259
|
+
taskId?: number;
|
|
252
260
|
task?: { id: number; autoScaffolded: boolean; nudge: string };
|
|
253
261
|
}
|
|
254
262
|
|
|
@@ -332,7 +340,16 @@ export async function spawnAgent(input: SpawnAgentInput, ctx: AuthContext): Prom
|
|
|
332
340
|
|
|
333
341
|
// #255/#328 — bias the host to the caller's own machine when neither an explicit
|
|
334
342
|
// orchestratorId nor a cwd pins it, so an agent spawning a helper lands on its own host.
|
|
335
|
-
|
|
343
|
+
// #1034 — pass the token's orchestrator allow-set so host selection can NEVER land on a host
|
|
344
|
+
// the caller isn't authorized for (which would then trip assertResourceAllowed with a
|
|
345
|
+
// misleading `-32001 component token cannot access this resource` — the concurrent-spawn flap).
|
|
346
|
+
const orchestrator = selectSpawnOrchestrator(
|
|
347
|
+
input.provider,
|
|
348
|
+
input.orchestratorId,
|
|
349
|
+
requestedCwd,
|
|
350
|
+
caller?.machine ?? lineageAgent?.machine,
|
|
351
|
+
ctx.constraints?.orchestrators,
|
|
352
|
+
);
|
|
336
353
|
|
|
337
354
|
// #308 §3 — an explicit cwd must resolve within the TARGET host's base dir (a path valid on
|
|
338
355
|
// your host may not exist on a different orchestrator).
|
|
@@ -638,6 +655,7 @@ export async function spawnAgent(input: SpawnAgentInput, ctx: AuthContext): Prom
|
|
|
638
655
|
resumeWorkspace: input.resumeWorkspace,
|
|
639
656
|
acquisition,
|
|
640
657
|
...(expectReply ? { expectReply } : {}),
|
|
658
|
+
...(input.reportVerbosity ? { reportVerbosity: input.reportVerbosity } : {}),
|
|
641
659
|
policyName: input.policyName,
|
|
642
660
|
spawnRequestId,
|
|
643
661
|
env,
|
|
@@ -727,6 +745,7 @@ export async function spawnAgent(input: SpawnAgentInput, ctx: AuthContext): Prom
|
|
|
727
745
|
agentId,
|
|
728
746
|
registered: agentId !== null,
|
|
729
747
|
command,
|
|
748
|
+
...(input.taskId !== undefined ? { taskId: input.taskId } : {}),
|
|
730
749
|
...(autoScaffoldedTask ? { task: autoScaffoldedTask } : {}),
|
|
731
750
|
};
|
|
732
751
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { getAgent } from "../db";
|
|
2
2
|
import { addSubscription } from "../db/notification-subscriptions";
|
|
3
3
|
import { emitRelayEvent } from "../events";
|
|
4
|
+
import { hasRecentLifecycleSignal, taskSignalKey } from "../lifecycle-signal-registry";
|
|
4
5
|
import { notifySystemMessage } from "../notify";
|
|
5
6
|
import type { NotificationScope, NotificationType } from "../notification-types";
|
|
6
7
|
import { routeNotification } from "../notification-router";
|
|
@@ -216,7 +217,18 @@ function dedupeRecipients(recipients: Array<{ agentId: string; matchedRule: stri
|
|
|
216
217
|
return out;
|
|
217
218
|
}
|
|
218
219
|
|
|
220
|
+
// #1032b — a status pulse is redundant when a first-class lifecycle event (agent.ready, the lineage
|
|
221
|
+
// report-up, branch.landed/merge) already conveyed this task's state to this recipient. Suppress it
|
|
222
|
+
// rather than waking the spawner a second time for the same fact.
|
|
223
|
+
// #1038 — keyed by the pulse's own transition: a progress signal (report-up / ready) supersedes only
|
|
224
|
+
// a redundant `working` pulse, never a later DISTINCT-state transition (committed / landed / done)
|
|
225
|
+
// for the same task within the TTL.
|
|
226
|
+
function pulseSupersededByLifecycle(agentId: string, taskId: number, transition: TaskSemanticTransition): boolean {
|
|
227
|
+
return hasRecentLifecycleSignal(agentId, taskSignalKey(taskId, transition));
|
|
228
|
+
}
|
|
229
|
+
|
|
219
230
|
function enqueueStatusPulse(agentId: string, item: StatusPulseDigestItem): void {
|
|
231
|
+
if (pulseSupersededByLifecycle(agentId, item.taskId, item.transition)) return; // #1032b/#1038
|
|
220
232
|
const batch = statusPulseBatches.get(agentId) ?? { items: [] };
|
|
221
233
|
batch.items.push(item);
|
|
222
234
|
if (!batch.timer) {
|
|
@@ -233,7 +245,11 @@ export function flushTaskStatusPulses(agentId?: string): void {
|
|
|
233
245
|
if (batch.timer) clearTimeout(batch.timer);
|
|
234
246
|
statusPulseBatches.delete(target);
|
|
235
247
|
if (batch.items.length === 0) continue;
|
|
236
|
-
|
|
248
|
+
// #1032b — a lifecycle event that fired AFTER these items were queued (but within the batch
|
|
249
|
+
// window) also supersedes them; drop those before flushing so the digest carries only news.
|
|
250
|
+
const fresh = batch.items.filter((item) => !pulseSupersededByLifecycle(target, item.taskId, item.transition));
|
|
251
|
+
if (fresh.length === 0) continue;
|
|
252
|
+
const items = compactStatusPulseItems(fresh);
|
|
237
253
|
notifySystemMessage(target, {
|
|
238
254
|
subject: `Task status pulse (${items.length})`,
|
|
239
255
|
body: taskStatusPulseBody(items),
|
package/src/spawn-command.ts
CHANGED
|
@@ -109,6 +109,8 @@ interface BuildSpawnCommandOptions {
|
|
|
109
109
|
* registration from the persisted spawn command params.
|
|
110
110
|
*/
|
|
111
111
|
expectReply?: boolean;
|
|
112
|
+
/** #1037 — per-spawn report-up verbosity, persisted so registration can stamp it onto child meta. */
|
|
113
|
+
reportVerbosity?: string;
|
|
112
114
|
profile?: string;
|
|
113
115
|
agentProfile?: unknown;
|
|
114
116
|
headless?: boolean;
|
|
@@ -174,6 +176,7 @@ export function buildSpawnCommand(opts: BuildSpawnCommandOptions): Record<string
|
|
|
174
176
|
...def(opts.reviewer, "reviewer"),
|
|
175
177
|
...def(opts.taskId, "taskId"),
|
|
176
178
|
...def(opts.expectReply, "expectReply"),
|
|
179
|
+
...def(opts.reportVerbosity, "reportVerbosity"),
|
|
177
180
|
...def(opts.headless, "headless"),
|
|
178
181
|
...def(opts.policyName, "policyName"),
|
|
179
182
|
...def(opts.automationId, "automationId"),
|
package/src/spawn-targets.ts
CHANGED
|
@@ -500,7 +500,25 @@ export function selectSpawnOrchestrator(
|
|
|
500
500
|
orchestratorId?: string,
|
|
501
501
|
cwd?: string,
|
|
502
502
|
preferHost?: string,
|
|
503
|
+
/**
|
|
504
|
+
* #1034 — the caller token's `constraints.orchestrators` allow-set. When present, host
|
|
505
|
+
* SELECTION must stay INSIDE it: never silently fall back to a host the token isn't
|
|
506
|
+
* authorized for. Previously, when neither an explicit id nor a cwd nor a resolvable
|
|
507
|
+
* `preferHost` pinned the host (e.g. the caller's `machine` was transiently unset, or the
|
|
508
|
+
* caller's own host briefly flapped offline), selection grabbed `candidates[0]` — an
|
|
509
|
+
* arbitrary online host. If that host was outside the token's allow-set, the downstream
|
|
510
|
+
* `assertResourceAllowed` gate then rejected the spawn with a MISLEADING `-32001 component
|
|
511
|
+
* token cannot access this resource`, which cleared "seconds later" once the caller's host
|
|
512
|
+
* re-appeared (issue #1034). Filtering candidates to the allow-set makes selection
|
|
513
|
+
* authz-consistent: for the common single-authorized-host token it deterministically returns
|
|
514
|
+
* that host regardless of transient machine/cwd resolution, so concurrent spawns no longer
|
|
515
|
+
* spuriously reject; when the authorized host is genuinely offline, the error is an honest
|
|
516
|
+
* "authorized host offline", not a confusing authz denial. Undefined/empty = no filtering
|
|
517
|
+
* (admin `*` tokens, byte-identical to before).
|
|
518
|
+
*/
|
|
519
|
+
allowedOrchestratorIds?: string[],
|
|
503
520
|
): Orchestrator {
|
|
521
|
+
const allowSet = allowedOrchestratorIds?.length ? new Set(allowedOrchestratorIds) : null;
|
|
504
522
|
if (orchestratorId) {
|
|
505
523
|
const orchestrator = getOrchestrator(orchestratorId);
|
|
506
524
|
if (!orchestrator) throw new McpNotFoundError(`orchestrator ${orchestratorId} not found`);
|
|
@@ -515,7 +533,9 @@ export function selectSpawnOrchestrator(
|
|
|
515
533
|
}
|
|
516
534
|
return orchestrator;
|
|
517
535
|
}
|
|
518
|
-
const candidates = listOrchestrators().filter(
|
|
536
|
+
const candidates = listOrchestrators().filter(
|
|
537
|
+
(item) => item.status === "online" && item.providers.includes(provider) && (!allowSet || allowSet.has(item.id)),
|
|
538
|
+
);
|
|
519
539
|
if (cwd) {
|
|
520
540
|
const match = candidates.find((item) => isPathWithinBase(cwd, item.baseDir));
|
|
521
541
|
if (match) return match;
|
|
@@ -530,6 +550,11 @@ export function selectSpawnOrchestrator(
|
|
|
530
550
|
}
|
|
531
551
|
const orchestrator = candidates[0];
|
|
532
552
|
if (!orchestrator) {
|
|
553
|
+
// #1034 — when the token's host allow-set is what emptied the candidate list, say so plainly
|
|
554
|
+
// (the caller's authorized host is offline) instead of the generic "no orchestrator" miss.
|
|
555
|
+
if (allowSet) {
|
|
556
|
+
throw new McpNotFoundError(`no online orchestrator authorized for this token runs provider '${provider}'. Your token is scoped to host(s): ${[...allowSet].join(", ")} — check that one is online (call relay_spawn_targets for the live matrix).`);
|
|
557
|
+
}
|
|
533
558
|
throw new McpNotFoundError(`no online orchestrator runs provider '${provider}'. Available: ${spawnAlternatives()} (call relay_spawn_targets for the live matrix).`);
|
|
534
559
|
}
|
|
535
560
|
return orchestrator;
|
package/src/sse.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { getAgent, getOrchestrator, matchingDeliveryAgents, onWorkspaceChange, sendMessageWithResult, setChannelSubscriberProbe } from "./db";
|
|
2
|
-
import {
|
|
2
|
+
import { getOutboxCursor, hasReplayGap, replayEventsSlice, type BusEvent } from "./bus-outbox";
|
|
3
3
|
import { projectAgentStatusData } from "./agent-card-projection";
|
|
4
4
|
import { emitRelayEvent, subscribeRelayEvents, type RelayEvent } from "./events";
|
|
5
5
|
import { messageMatchesAgent, targetMatchesAgent } from "./agent-ref";
|
|
@@ -348,8 +348,7 @@ interface ReplayPlan {
|
|
|
348
348
|
function replayPlan(lastEventId: number): ReplayPlan {
|
|
349
349
|
const since = Number.isSafeInteger(lastEventId) && lastEventId > 0 ? lastEventId : 0;
|
|
350
350
|
if (since === 0) return { requested: false, since: 0, available: false, gap: false };
|
|
351
|
-
const
|
|
352
|
-
const gap = oldest > 0 && oldest > since + 1;
|
|
351
|
+
const gap = hasReplayGap(since);
|
|
353
352
|
return { requested: true, since, available: !gap, gap };
|
|
354
353
|
}
|
|
355
354
|
|
|
@@ -359,9 +358,18 @@ function replayEventsToConnection(conn: Connection | undefined, plan: ReplayPlan
|
|
|
359
358
|
send(conn, "replay.gap", { since: plan.since }, getOutboxCursor());
|
|
360
359
|
return;
|
|
361
360
|
}
|
|
361
|
+
// Drain the whole backlog in pages. A single capped replay dropped everything past the first 500
|
|
362
|
+
// events between the client's cursor and the outbox head (#1022 F1) — the SSE stream then went
|
|
363
|
+
// live from the head with no signal, so the tail was lost. Loop until the page is not truncated.
|
|
362
364
|
const agentCache = new Map<string, AgentCard | null>();
|
|
363
|
-
|
|
364
|
-
|
|
365
|
+
let since = plan.since;
|
|
366
|
+
for (;;) {
|
|
367
|
+
const slice = replayEventsSlice(since);
|
|
368
|
+
for (const event of slice.events) {
|
|
369
|
+
sendReplayEvent(conn, outboxToRelayEvent(event), agentCache);
|
|
370
|
+
}
|
|
371
|
+
if (!slice.truncated) break;
|
|
372
|
+
since = slice.lastSeq;
|
|
365
373
|
}
|
|
366
374
|
}
|
|
367
375
|
|
|
@@ -43,6 +43,7 @@ import { settleWorkspaceCommand } from "./workspace-command-settle";
|
|
|
43
43
|
import { CLEAR_MERGE_NO_PROGRESS, MAX_NO_PROGRESS_MERGE_ATTEMPTS, mergeNoProgressCount, noteMergeNoProgress } from "./workspace-merge-progress";
|
|
44
44
|
import { workspaceOwnerOrchestrator } from "./workspace-host";
|
|
45
45
|
import { workspaceActiveClaim } from "./workspace-claim";
|
|
46
|
+
import { notifyWorkspaceHumanEscalation } from "./workspace-human-escalation";
|
|
46
47
|
import { DIRTY_WORKTREE_LAND_SKIP_REASON, READY_TO_LAND_STATUSES } from "./workspace-phase";
|
|
47
48
|
import { fetchHostMergePreview } from "./workspace-probe";
|
|
48
49
|
import type { Command, WorkspaceAutoMergePolicy, WorkspaceLandingPolicy, WorkspaceMergePreview, WorkspaceMergeStrategy, WorkspaceRecord } from "./types";
|
|
@@ -119,20 +120,33 @@ function autoMergeRequestOptions(ws: WorkspaceRecord): AutoMergeRequestOptions {
|
|
|
119
120
|
// auto-provision the policy on the workspace-owning host, then queue a `policy:` wake
|
|
120
121
|
// message (which also spawns the on-demand agent now via onMessageForPolicy). Honors a
|
|
121
122
|
// per-workspace cooldown so a persistent conflict/behind row isn't re-pinged every sweep.
|
|
122
|
-
//
|
|
123
|
-
|
|
123
|
+
// The result of trying to wake the managed steward (#1021 A1). `ok` carries the policy name
|
|
124
|
+
// only when a steward is genuinely alive or was just spawned. On `!ok`, `cooldown` tells a
|
|
125
|
+
// fresh failure (nothing spawned — the caller should advance the loop-breaker) apart from a
|
|
126
|
+
// recent successful wake still inside its cooldown window (NOT a failure — leave it be).
|
|
127
|
+
export type StewardWakeOutcome =
|
|
128
|
+
| { ok: true; policy: string }
|
|
129
|
+
| { ok: false; cooldown: boolean; error?: string };
|
|
130
|
+
|
|
131
|
+
// Wake the managed per-host/repo steward for a workspace it should handle: auto-provision
|
|
132
|
+
// the policy on the workspace-owning host, queue a `policy:` wake message, and spawn the
|
|
133
|
+
// on-demand agent. Only reports `ok` when the spawn actually happened (or a steward was
|
|
134
|
+
// already live) — a queued message alone is NOT a wake (#1021 A1). Honors a per-workspace
|
|
135
|
+
// cooldown so a persistent conflict/behind row isn't re-pinged every sweep.
|
|
136
|
+
export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): StewardWakeOutcome {
|
|
124
137
|
const meta = ws.metadata as Record<string, unknown>;
|
|
125
138
|
const lastWoke = typeof meta.stewardWokenAt === "number" ? meta.stewardWokenAt : 0;
|
|
126
|
-
if (lastWoke && Date.now() - lastWoke < STEWARD_WAKE_COOLDOWN_MS) return
|
|
139
|
+
if (lastWoke && Date.now() - lastWoke < STEWARD_WAKE_COOLDOWN_MS) return { ok: false, cooldown: true };
|
|
127
140
|
const ownerResult = stewardOwnerForWorkspace(ws);
|
|
128
141
|
if ("error" in ownerResult) {
|
|
129
142
|
patchWorkspaceMetadata(ws.id, { stewardWakeError: ownerResult.error, stewardWakeErrorAt: Date.now() });
|
|
130
|
-
return
|
|
143
|
+
return { ok: false, cooldown: false, error: ownerResult.error };
|
|
131
144
|
}
|
|
132
145
|
const policyName = ensureRepoSteward(ws.repoRoot, ownerResult.owner);
|
|
133
146
|
if (!policyName) {
|
|
134
|
-
|
|
135
|
-
|
|
147
|
+
const error = "cannot spawn steward: steward policy could not be provisioned";
|
|
148
|
+
patchWorkspaceMetadata(ws.id, { stewardWakeError: error, stewardWakeErrorAt: Date.now() });
|
|
149
|
+
return { ok: false, cooldown: false, error };
|
|
136
150
|
}
|
|
137
151
|
// #640 — resolve the identity triangle so the steward isn't blind to the people-graph:
|
|
138
152
|
// the workspace's branch agent (owner) and that agent's coordinator (owner's `spawnedBy`).
|
|
@@ -163,7 +177,16 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
|
|
|
163
177
|
payload: { kind: "workspace.steward-task", workspaceId: ws.id, repoRoot: ws.repoRoot, worktreePath: ws.worktreePath, branch: ws.branch, baseRef: ws.baseRef, status: ws.status, reason, ownerAgentId, coordinatorAgentId },
|
|
164
178
|
},
|
|
165
179
|
});
|
|
166
|
-
|
|
180
|
+
// #1021 A1 — the durable `policy:` obligation is enqueued above (it survives until a
|
|
181
|
+
// steward claims it), but a wake is only real if a steward actually spawned or is
|
|
182
|
+
// already live. If nothing spawned (host offline / provider error), record the error and
|
|
183
|
+
// report failure — do NOT stamp stewardWokenAt (which arms the cooldown) or claim a
|
|
184
|
+
// hand-off that never happened.
|
|
185
|
+
const spawn = getLifecycleManager().onMessageForPolicy(policyName);
|
|
186
|
+
if (!spawn.woke) {
|
|
187
|
+
patchWorkspaceMetadata(ws.id, { stewardWakeError: spawn.reason, stewardWakeErrorAt: Date.now() });
|
|
188
|
+
return { ok: false, cooldown: false, error: spawn.reason };
|
|
189
|
+
}
|
|
167
190
|
patchWorkspaceMetadata(ws.id, {
|
|
168
191
|
stewardWokenAt: Date.now(),
|
|
169
192
|
stewardPolicy: policyName,
|
|
@@ -173,10 +196,11 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
|
|
|
173
196
|
});
|
|
174
197
|
// #641/#642 — tell the owner which steward took over and proactively loop in its coordinator.
|
|
175
198
|
notifyStewardHandoff({ workspace: ws, stewardId: policyName, coordinatorAgentId, reason });
|
|
176
|
-
return policyName;
|
|
199
|
+
return { ok: true, policy: policyName };
|
|
177
200
|
} catch (e) {
|
|
178
|
-
|
|
179
|
-
|
|
201
|
+
const error = e instanceof Error ? e.message : "failed to wake steward";
|
|
202
|
+
patchWorkspaceMetadata(ws.id, { stewardWakeError: error, stewardWakeErrorAt: Date.now() });
|
|
203
|
+
return { ok: false, cooldown: false, error };
|
|
180
204
|
}
|
|
181
205
|
}
|
|
182
206
|
|
|
@@ -227,25 +251,36 @@ function escalateNoProgressMerge(ws: WorkspaceRecord): void {
|
|
|
227
251
|
const { coordinatorAgentId } = resolveStewardIdentity(ws);
|
|
228
252
|
const fallback = stewardFallbackTarget();
|
|
229
253
|
const recipients = [...new Set([coordinatorAgentId, fallback, ws.ownerAgentId].filter((r): r is string => Boolean(r)))];
|
|
254
|
+
const subject = "Workspace can't auto-land — needs attention";
|
|
255
|
+
const body = renderTemplate("steward.merge-stall", {
|
|
256
|
+
branch: `\`${ws.branch ?? ws.id}\``,
|
|
257
|
+
repoRoot: ws.repoRoot,
|
|
258
|
+
attempts: String(MAX_NO_PROGRESS_MERGE_ATTEMPTS),
|
|
259
|
+
mergeError,
|
|
260
|
+
baseRef: ws.baseRef ?? "base",
|
|
261
|
+
});
|
|
262
|
+
const payload = { kind: "workspace.auto-merge-stuck", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: ws.baseRef, attempts: MAX_NO_PROGRESS_MERGE_ATTEMPTS, reason: mergeError };
|
|
263
|
+
let escalated = false;
|
|
230
264
|
if (recipients.length) {
|
|
231
265
|
routeNotification({
|
|
232
266
|
type: "workspace.stranded_escalation",
|
|
233
267
|
scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ws.ownerAgentId, parent: coordinatorAgentId ?? undefined },
|
|
234
268
|
recipients,
|
|
235
269
|
message: {
|
|
236
|
-
subject
|
|
237
|
-
body
|
|
238
|
-
|
|
239
|
-
repoRoot: ws.repoRoot,
|
|
240
|
-
attempts: String(MAX_NO_PROGRESS_MERGE_ATTEMPTS),
|
|
241
|
-
mergeError,
|
|
242
|
-
baseRef: ws.baseRef ?? "base",
|
|
243
|
-
}),
|
|
244
|
-
payload: { kind: "workspace.auto-merge-stuck", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: ws.baseRef, attempts: MAX_NO_PROGRESS_MERGE_ATTEMPTS, reason: mergeError },
|
|
270
|
+
subject,
|
|
271
|
+
body,
|
|
272
|
+
payload,
|
|
245
273
|
},
|
|
246
274
|
});
|
|
275
|
+
escalated = true;
|
|
276
|
+
} else {
|
|
277
|
+
const human = notifyWorkspaceHumanEscalation({ subject, body, payload, source: "workspace-auto-merge" });
|
|
278
|
+
escalated = human.accepted;
|
|
279
|
+
if (!human.accepted) {
|
|
280
|
+
console.warn(`[workspace-auto-merge] no-progress escalation for ${ws.id} had zero agent recipients and no human channel: ${human.reason ?? "unknown reason"}`);
|
|
281
|
+
}
|
|
247
282
|
}
|
|
248
|
-
patchWorkspaceMetadata(ws.id, { autoMergeBackedOffAt: Date.now() });
|
|
283
|
+
if (escalated) patchWorkspaceMetadata(ws.id, { autoMergeBackedOffAt: Date.now() });
|
|
249
284
|
createActivityEvent({
|
|
250
285
|
clientId: `workspace-auto-merge-backoff-${ws.id}-${Date.now()}`,
|
|
251
286
|
kind: "state",
|
|
@@ -349,6 +384,22 @@ async function landCandidates(
|
|
|
349
384
|
const preview = await fetchFirstUsableMergePreview(ws, orchestrators);
|
|
350
385
|
if (!preview) {
|
|
351
386
|
previewUnavailable.push(ws.id);
|
|
387
|
+
// #1025 A5 — a lane whose preview is persistently unusable (owner host offline, the
|
|
388
|
+
// worktree path is gone, or every probe returns error/missing) used to exit here with
|
|
389
|
+
// ZERO side effects: no no-progress, no wake, no notify — a ready branch would strand
|
|
390
|
+
// silently forever. Give it the SAME no-progress/strand path as the unknown-merge-state
|
|
391
|
+
// lane so no status is escalation-blind: advance the loop-breaker, and once it crosses
|
|
392
|
+
// the cap, escalateNoProgressMerge fires the owner `landed-failure` wake + steward/
|
|
393
|
+
// coordinator escalation. A transient blip self-clears when a usable preview returns.
|
|
394
|
+
const held = patchWorkspaceMetadata(ws.id, {
|
|
395
|
+
...noteMergeNoProgress(ws.metadata),
|
|
396
|
+
mergeError: "no online host returned a usable merge preview (owner offline, worktree missing, or probe error)",
|
|
397
|
+
lastPreviewUnavailableAt: Date.now(),
|
|
398
|
+
}) ?? ws;
|
|
399
|
+
if (mergeNoProgressCount(held.metadata) >= MAX_NO_PROGRESS_MERGE_ATTEMPTS) {
|
|
400
|
+
backedOff.push(ws.id);
|
|
401
|
+
escalateNoProgressMerge(held);
|
|
402
|
+
}
|
|
352
403
|
continue;
|
|
353
404
|
}
|
|
354
405
|
const p = preview as WorkspaceMergePreview;
|
|
@@ -401,21 +452,26 @@ async function landCandidates(
|
|
|
401
452
|
}
|
|
402
453
|
}
|
|
403
454
|
leftForSteward.push(ws.id);
|
|
404
|
-
let woke: string | null = null;
|
|
405
455
|
if (stewardEnabled) {
|
|
406
|
-
|
|
407
|
-
if (
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
456
|
+
const wake = wakeRepoSteward(handoffWorkspace, p.conflict === true ? "conflict" : "merge state unknown");
|
|
457
|
+
if (wake.ok) {
|
|
458
|
+
wokeStewards.push(wake.policy);
|
|
459
|
+
} else if (!wake.cooldown && p.conflict === true) {
|
|
460
|
+
// #1021 A1+A2 — a GENUINE wake failure (nothing spawned: host offline / provider
|
|
461
|
+
// error) now correctly yields !ok, so the conflict lane's loop-breaker advances and
|
|
462
|
+
// escalateNoProgressMerge becomes reachable. A cooldown means a steward WAS woken
|
|
463
|
+
// recently and is presumably working — that is not a failure, so don't count it (and
|
|
464
|
+
// don't mislabel the cooldown as a spawn failure in mergeError). Carry the real error.
|
|
465
|
+
const failedHandoff = patchWorkspaceMetadata(ws.id, {
|
|
466
|
+
...noteMergeNoProgress(ws.metadata),
|
|
467
|
+
mergeError: wake.error ?? "cannot spawn steward on the workspace owning host",
|
|
468
|
+
lastStewardWakeFailedAt: Date.now(),
|
|
469
|
+
}) ?? ws;
|
|
470
|
+
if (mergeNoProgressCount(failedHandoff.metadata) >= MAX_NO_PROGRESS_MERGE_ATTEMPTS) {
|
|
471
|
+
backedOff.push(ws.id);
|
|
472
|
+
escalateNoProgressMerge(failedHandoff);
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
419
475
|
}
|
|
420
476
|
}
|
|
421
477
|
if (p.conflict === true) {
|
|
@@ -1,23 +1,43 @@
|
|
|
1
1
|
import { stewardFallbackTarget } from "./config";
|
|
2
2
|
import { routeNotification } from "./notification-router";
|
|
3
|
+
import { raiseOperatorAlarm } from "./operator-alarm";
|
|
3
4
|
import { resolveStewardIdentity } from "./steward-identity";
|
|
5
|
+
import { notifyWorkspaceHumanEscalation } from "./workspace-human-escalation";
|
|
4
6
|
import type { NotificationType } from "./notification-types";
|
|
5
7
|
import type { WorkspaceRecord, WorkspaceStatus } from "./types";
|
|
6
8
|
|
|
7
9
|
export const READY_TO_LAND_ESCALATION_STATUSES = new Set<WorkspaceStatus>(["ready", "review_requested"]);
|
|
8
10
|
|
|
9
|
-
export
|
|
11
|
+
export interface WorkspaceCustodyEscalationInput {
|
|
10
12
|
workspace: Pick<WorkspaceRecord, "id" | "repoRoot" | "branch" | "status" | "ownerAgentId">;
|
|
11
13
|
type?: NotificationType;
|
|
12
14
|
subject: string;
|
|
13
15
|
body: string;
|
|
14
16
|
payload: Record<string, unknown>;
|
|
15
17
|
idempotencyKey?: string;
|
|
16
|
-
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function routeWorkspaceCustodyEscalation(input: WorkspaceCustodyEscalationInput): string[] {
|
|
17
21
|
const { ownerAgentId, coordinatorAgentId } = resolveStewardIdentity(input.workspace);
|
|
18
22
|
const fallback = stewardFallbackTarget();
|
|
19
23
|
const recipients = [...new Set([ownerAgentId, coordinatorAgentId, fallback].filter((value): value is string => Boolean(value)))];
|
|
20
|
-
if (!recipients.length)
|
|
24
|
+
if (!recipients.length) {
|
|
25
|
+
const payload = {
|
|
26
|
+
...input.payload,
|
|
27
|
+
coordinatorAgentId,
|
|
28
|
+
ownerAgentId,
|
|
29
|
+
fallbackTarget: fallback || undefined,
|
|
30
|
+
};
|
|
31
|
+
const human = notifyWorkspaceHumanEscalation({
|
|
32
|
+
subject: input.subject,
|
|
33
|
+
body: input.body,
|
|
34
|
+
payload,
|
|
35
|
+
source: "workspace-custody",
|
|
36
|
+
});
|
|
37
|
+
if (human.accepted && human.channelId) return [human.channelId];
|
|
38
|
+
console.warn(`[workspace-escalation] zero agent recipients and no human channel for workspace ${input.workspace.id}: ${human.reason ?? "unknown reason"}`);
|
|
39
|
+
return [];
|
|
40
|
+
}
|
|
21
41
|
|
|
22
42
|
routeNotification({
|
|
23
43
|
type: input.type ?? "workspace.stranded_escalation",
|
|
@@ -43,3 +63,32 @@ export function routeWorkspaceCustodyEscalation(input: {
|
|
|
43
63
|
});
|
|
44
64
|
return recipients;
|
|
45
65
|
}
|
|
66
|
+
|
|
67
|
+
// Route a custody escalation, and if it resolves to ZERO reachable recipients — owner
|
|
68
|
+
// exited, coordinator idle, no `AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK` configured —
|
|
69
|
+
// raise a louder operator-visible alarm instead of letting the caller silently record it
|
|
70
|
+
// as "escalated" (#1025 C1/C4, same class as #1021 A4). A disk-only orphan (ownerAgentId
|
|
71
|
+
// undefined) and most `review_requested` auto-abandons have no agent to page by default,
|
|
72
|
+
// so without this the notice is swallowed and the strand loops in the dark. Returns the
|
|
73
|
+
// (possibly empty) recipient list so the caller can still decide whether to stamp state.
|
|
74
|
+
export function routeWorkspaceCustodyEscalationOrAlarm(
|
|
75
|
+
input: WorkspaceCustodyEscalationInput & { source: string },
|
|
76
|
+
): string[] {
|
|
77
|
+
const { source, ...escalation } = input;
|
|
78
|
+
const targets = routeWorkspaceCustodyEscalation(escalation);
|
|
79
|
+
if (!targets.length) {
|
|
80
|
+
raiseOperatorAlarm({
|
|
81
|
+
clientId: `${escalation.idempotencyKey ?? `${escalation.workspace.id}:${escalation.subject}`}:no-recipients`,
|
|
82
|
+
source,
|
|
83
|
+
subject: escalation.subject,
|
|
84
|
+
detail: `${escalation.body} — no owner, coordinator, or fallback configured to escalate to. Configure AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK or reclaim the worktree manually.`,
|
|
85
|
+
metadata: {
|
|
86
|
+
workspaceId: escalation.workspace.id,
|
|
87
|
+
repoRoot: escalation.workspace.repoRoot,
|
|
88
|
+
branch: escalation.workspace.branch,
|
|
89
|
+
status: escalation.workspace.status,
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return targets;
|
|
94
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { errMessage } from "agent-relay-sdk";
|
|
2
|
+
import { stewardFallbackTarget } from "./config";
|
|
3
|
+
import { getChannel } from "./db";
|
|
4
|
+
import { relayEgressToChannel, relayNotifyNtfy } from "./mcp-egress-tools";
|
|
5
|
+
import { getResolvedNotificationsConfig } from "./notification-settings-store";
|
|
6
|
+
|
|
7
|
+
const DEFAULT_NTFY_CHANNEL_ID = "ntfy:default";
|
|
8
|
+
const WARN = "no human-reachable escalation target - stranded workspaces will page nobody. Set AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK or configure ntfy.";
|
|
9
|
+
|
|
10
|
+
export interface WorkspaceHumanEscalationInput {
|
|
11
|
+
subject: string;
|
|
12
|
+
body: string;
|
|
13
|
+
payload?: Record<string, unknown>;
|
|
14
|
+
priority?: "min" | "low" | "default" | "high" | "urgent";
|
|
15
|
+
tags?: string[];
|
|
16
|
+
source?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface WorkspaceHumanEscalationResult {
|
|
20
|
+
accepted: boolean;
|
|
21
|
+
channelId?: string;
|
|
22
|
+
reason?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function workspaceHumanEscalationChannelId(): string {
|
|
26
|
+
return getResolvedNotificationsConfig().humanEscalationChannelId.trim();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function hasHumanEscalationChannel(): boolean {
|
|
30
|
+
const channelId = workspaceHumanEscalationChannelId();
|
|
31
|
+
return Boolean(channelId && getChannel(channelId));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function hasHumanReachableEscalationTarget(): boolean {
|
|
35
|
+
return Boolean(stewardFallbackTarget() || hasHumanEscalationChannel());
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function warnIfNoHumanReachableEscalationTarget(log: (message: string) => void = console.warn): boolean {
|
|
39
|
+
if (hasHumanReachableEscalationTarget()) return false;
|
|
40
|
+
log(WARN);
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function notifyWorkspaceHumanEscalation(input: WorkspaceHumanEscalationInput): WorkspaceHumanEscalationResult {
|
|
45
|
+
const channelId = workspaceHumanEscalationChannelId();
|
|
46
|
+
if (!channelId) return { accepted: false, reason: "no human escalation channel configured" };
|
|
47
|
+
if (!getChannel(channelId)) return { accepted: false, reason: `human escalation channel ${channelId} is not registered` };
|
|
48
|
+
|
|
49
|
+
const event: Record<string, unknown> = {
|
|
50
|
+
title: input.subject,
|
|
51
|
+
body: input.body,
|
|
52
|
+
priority: input.priority ?? "urgent",
|
|
53
|
+
tags: input.tags ?? ["agent-relay", "workspace"],
|
|
54
|
+
...(input.payload ? { relay: input.payload } : {}),
|
|
55
|
+
...(input.source ? { source: input.source } : {}),
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
try {
|
|
59
|
+
if (channelId === DEFAULT_NTFY_CHANNEL_ID) {
|
|
60
|
+
relayNotifyNtfy(event);
|
|
61
|
+
} else {
|
|
62
|
+
relayEgressToChannel({ channelId, event });
|
|
63
|
+
}
|
|
64
|
+
return { accepted: true, channelId };
|
|
65
|
+
} catch (error) {
|
|
66
|
+
return { accepted: false, channelId, reason: errMessage(error) };
|
|
67
|
+
}
|
|
68
|
+
}
|
package/src/workspace-merge.ts
CHANGED
|
@@ -144,6 +144,22 @@ export function withOwnerOnline<T extends { ownerAgentId?: string }>(workspace:
|
|
|
144
144
|
return { ...workspace, ownerOnline: isOwnerAlive(workspace.ownerAgentId) };
|
|
145
145
|
}
|
|
146
146
|
|
|
147
|
+
// #1049 — process-level death, distinct from "offline". `offline` is set on a mere
|
|
148
|
+
// heartbeat lapse (exactly what a relay self-stall fakes for a batch of LIVE agents,
|
|
149
|
+
// the 2026-07-04 cascade). A genuine process exit is only known when the owning
|
|
150
|
+
// orchestrator REPORTS it: it stamps `meta.lastExit` (routes/orchestrator.ts). So
|
|
151
|
+
// require both offline AND a reported exit before treating the owner as gone — never
|
|
152
|
+
// nuke/land a worktree on a heartbeat gap alone. Conservative by design: an unmanaged
|
|
153
|
+
// agent with no orchestrator never gets `lastExit`, so it reads as maybe-alive and the
|
|
154
|
+
// orphan reaper's land-safety + grace window handle its eventual cleanup instead.
|
|
155
|
+
export function isOwnerProcessConfirmedGone(ownerAgentId: string | undefined): boolean {
|
|
156
|
+
if (!ownerAgentId) return false;
|
|
157
|
+
const agent = getAgent(ownerAgentId);
|
|
158
|
+
if (!agent || agent.status !== "offline") return false;
|
|
159
|
+
const meta = isRecord(agent.meta) ? agent.meta : undefined;
|
|
160
|
+
return Boolean(meta && meta.lastExit);
|
|
161
|
+
}
|
|
162
|
+
|
|
147
163
|
function workspaceOwnerHostKeys(workspace: WorkspaceRecord): Set<string> {
|
|
148
164
|
const keys = new Set<string>();
|
|
149
165
|
const owner = workspace.ownerAgentId ? getAgent(workspace.ownerAgentId) : undefined;
|
package/src/workspace-orphans.ts
CHANGED
|
@@ -30,7 +30,7 @@ import { isOwnerAlive, liveAgentCwds, worktreeInUseByLiveAgent } from "./workspa
|
|
|
30
30
|
import { applyWorkspaceAction } from "./workspace-actions";
|
|
31
31
|
import { workspaceOwnerOrchestrator } from "./workspace-host";
|
|
32
32
|
import { UNLANDED_FLAG_COOLDOWN_MS, orphanGraceMs, orphanWorktreeReapEnabled, relayToken } from "./config";
|
|
33
|
-
import { READY_TO_LAND_ESCALATION_STATUSES,
|
|
33
|
+
import { READY_TO_LAND_ESCALATION_STATUSES, routeWorkspaceCustodyEscalationOrAlarm } from "./workspace-escalation";
|
|
34
34
|
|
|
35
35
|
// Don't re-flag the same un-landed orphan every sweep — surface it once, then
|
|
36
36
|
// stay quiet for this window. In-memory (keyed by worktree path) like the
|
|
@@ -425,7 +425,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
|
|
|
425
425
|
? "uncommitted changes in the worktree"
|
|
426
426
|
: `${orphan.unmergedAhead ?? orphan.ahead ?? "?"} un-landed commit(s)`;
|
|
427
427
|
if (orphanHasWorkEvidence(orphan)) {
|
|
428
|
-
|
|
428
|
+
routeWorkspaceCustodyEscalationOrAlarm({
|
|
429
|
+
source: "workspace-orphan-reaper",
|
|
429
430
|
workspace: { id: `orphan:${orphan.worktreePath}`, repoRoot: orphan.repoRoot, branch: orphan.branch, status: "review_requested", ownerAgentId: undefined },
|
|
430
431
|
subject: "Stranded worktree needs attention",
|
|
431
432
|
body: `${orphan.branch ?? orphan.worktreePath} in ${orphan.repoRoot} is orphaned and holds work that has not landed - ${detail}. Recover the commits before removing it.`,
|
|
@@ -456,7 +457,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
|
|
|
456
457
|
if (now - last < UNLANDED_FLAG_COOLDOWN_MS) continue;
|
|
457
458
|
flaggedAt.set(key, now);
|
|
458
459
|
if (workspaceNeedsCustodyEscalation(workspace, false)) {
|
|
459
|
-
|
|
460
|
+
routeWorkspaceCustodyEscalationOrAlarm({
|
|
461
|
+
source: "workspace-orphan-reaper",
|
|
460
462
|
workspace,
|
|
461
463
|
subject: "Workspace row has no worktree on disk",
|
|
462
464
|
body: `Workspace \`${workspace.branch ?? workspace.id}\` in ${workspace.repoRoot} is ${workspace.status}, but its worktree ${workspace.worktreePath} is missing on disk. This was in Relay custody; recover or explicitly abandon it.`,
|
|
@@ -506,7 +508,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
|
|
|
506
508
|
autoAbandoned.push(workspace.id);
|
|
507
509
|
flaggedAt.delete(key);
|
|
508
510
|
if (workspaceNeedsCustodyEscalation(workspace, previewHasWorkEvidence(probe.kind === "preview" ? probe.preview : null))) {
|
|
509
|
-
|
|
511
|
+
routeWorkspaceCustodyEscalationOrAlarm({
|
|
512
|
+
source: "workspace-orphan-reaper",
|
|
510
513
|
workspace,
|
|
511
514
|
type: "workspace.auto_abandoned",
|
|
512
515
|
subject: "Workspace auto-abandoned",
|
|
@@ -547,7 +550,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
|
|
|
547
550
|
? `host could not confirm whether branch ${workspace.branch} has landed`
|
|
548
551
|
: "host could not confirm whether recoverable branch work remains";
|
|
549
552
|
if (workspaceNeedsCustodyEscalation(workspace, hasWorkEvidence)) {
|
|
550
|
-
|
|
553
|
+
routeWorkspaceCustodyEscalationOrAlarm({
|
|
554
|
+
source: "workspace-orphan-reaper",
|
|
551
555
|
workspace,
|
|
552
556
|
subject: "Missing-worktree workspace needs attention",
|
|
553
557
|
body: `${workspace.id} in ${workspace.repoRoot} has no worktree on disk and cannot be auto-abandoned safely - ${detail}. Recover it, then abandon or clean it up explicitly.`,
|
|
@@ -657,7 +661,8 @@ export async function reapOrphanedWorktrees(): Promise<Record<string, unknown>>
|
|
|
657
661
|
: `${preview?.unmergedAhead ?? preview?.ahead ?? "?"} un-landed commit(s)`;
|
|
658
662
|
const hasWorkEvidence = previewHasWorkEvidence(preview);
|
|
659
663
|
if (workspaceNeedsCustodyEscalation(workspace, hasWorkEvidence)) {
|
|
660
|
-
|
|
664
|
+
routeWorkspaceCustodyEscalationOrAlarm({
|
|
665
|
+
source: "workspace-orphan-reaper",
|
|
661
666
|
workspace,
|
|
662
667
|
subject: "Dead-owner worktree needs attention",
|
|
663
668
|
body: `${workspace.branch ?? workspace.id} in ${workspace.repoRoot} is owned by gone agent ${workspace.ownerAgentId ?? "?"} and holds work that has not landed (${detail}). Recover the commits, then clean it up explicitly.`,
|