agent-relay-server 0.120.4 → 0.120.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +3 -3
- package/public/assets/{MessageBubble-Cqi3XsZ7.js → MessageBubble-So86mU9-.js} +2 -2
- package/public/assets/{MessageBubble-Cqi3XsZ7.js.map → MessageBubble-So86mU9-.js.map} +1 -1
- package/public/assets/{PlanGraphPanel-D6Um05oW.js → PlanGraphPanel-DBZYAsww.js} +2 -2
- package/public/assets/{PlanGraphPanel-D6Um05oW.js.map → PlanGraphPanel-DBZYAsww.js.map} +1 -1
- package/public/assets/{activity-B0LsQHEz.js → activity-CZNwmXg1.js} +2 -2
- package/public/assets/{activity-B0LsQHEz.js.map → activity-CZNwmXg1.js.map} +1 -1
- package/public/assets/{agent-profiles-O--pe5Ic.js → agent-profiles-BhadoGwi.js} +2 -2
- package/public/assets/{agent-profiles-O--pe5Ic.js.map → agent-profiles-BhadoGwi.js.map} +1 -1
- package/public/assets/agents-DWn1xx9x.js +2 -0
- package/public/assets/{agents-Ii6XlVwR.js.map → agents-DWn1xx9x.js.map} +1 -1
- package/public/assets/{analytics-BCzdYWRT.js → analytics-DifGnk7q.js} +2 -2
- package/public/assets/{analytics-BCzdYWRT.js.map → analytics-DifGnk7q.js.map} +1 -1
- package/public/assets/{automation-Crr3Xzui.js → automation-BExrtKkb.js} +2 -2
- package/public/assets/{automation-Crr3Xzui.js.map → automation-BExrtKkb.js.map} +1 -1
- package/public/assets/{branch-state-badge-Bhw2rloy.js → branch-state-badge-CyC5Qrhc.js} +2 -2
- package/public/assets/{branch-state-badge-Bhw2rloy.js.map → branch-state-badge-CyC5Qrhc.js.map} +1 -1
- package/public/assets/{channels-CKs9rvsn.js → channels-BLkdt2XR.js} +2 -2
- package/public/assets/{channels-CKs9rvsn.js.map → channels-BLkdt2XR.js.map} +1 -1
- package/public/assets/chat-C7OhKtr6.js +2 -0
- package/public/assets/{chat-BCwEUj5R.js.map → chat-C7OhKtr6.js.map} +1 -1
- package/public/assets/{connectors-D_kQR_vw.js → connectors-C0n12Je8.js} +2 -2
- package/public/assets/{connectors-D_kQR_vw.js.map → connectors-C0n12Je8.js.map} +1 -1
- package/public/assets/{formatted-body-q_qzt1L-.js → formatted-body-BCQT2qSb.js} +3 -3
- package/public/assets/{formatted-body-q_qzt1L-.js.map → formatted-body-BCQT2qSb.js.map} +1 -1
- package/public/assets/{formatted-body-impl-CSRGuMcM.js → formatted-body-impl-Bk7wnESl.js} +2 -2
- package/public/assets/{formatted-body-impl-CSRGuMcM.js.map → formatted-body-impl-Bk7wnESl.js.map} +1 -1
- package/public/assets/index-CRraCeo_.js +25 -0
- package/public/assets/index-CRraCeo_.js.map +1 -0
- package/public/assets/{insights-DUhXqZTZ.js → insights-DEgES6jz.js} +2 -2
- package/public/assets/{insights-DUhXqZTZ.js.map → insights-DEgES6jz.js.map} +1 -1
- package/public/assets/{integrations-lbg5CjtE.js → integrations-v14ehskT.js} +2 -2
- package/public/assets/{integrations-lbg5CjtE.js.map → integrations-v14ehskT.js.map} +1 -1
- package/public/assets/{maintenance-CQgneSVf.js → maintenance-BgVB6_gO.js} +2 -2
- package/public/assets/{maintenance-CQgneSVf.js.map → maintenance-BgVB6_gO.js.map} +1 -1
- package/public/assets/{managed-agents-DsnYf6fm.js → managed-agents-8_NKSkhI.js} +3 -3
- package/public/assets/{managed-agents-DsnYf6fm.js.map → managed-agents-8_NKSkhI.js.map} +1 -1
- package/public/assets/{markdown-preview-impl-C1ZtGJp6.js → markdown-preview-impl-yxQkrd2M.js} +2 -2
- package/public/assets/{markdown-preview-impl-C1ZtGJp6.js.map → markdown-preview-impl-yxQkrd2M.js.map} +1 -1
- package/public/assets/{memory-SSLUTOI2.js → memory-CJD8eekg.js} +2 -2
- package/public/assets/{memory-SSLUTOI2.js.map → memory-CJD8eekg.js.map} +1 -1
- package/public/assets/{messages-BY26OP4K.js → messages-DLHMZbHZ.js} +2 -2
- package/public/assets/{messages-BY26OP4K.js.map → messages-DLHMZbHZ.js.map} +1 -1
- package/public/assets/{orchestrators-OY7yPzVI.js → orchestrators-DEOep7FX.js} +2 -2
- package/public/assets/{orchestrators-OY7yPzVI.js.map → orchestrators-DEOep7FX.js.map} +1 -1
- package/public/assets/{overview-Dj0u1tPG.js → overview-BgpfO6mt.js} +2 -2
- package/public/assets/{overview-Dj0u1tPG.js.map → overview-BgpfO6mt.js.map} +1 -1
- package/public/assets/{pairs-CrUR50ou.js → pairs-8NVKHh96.js} +2 -2
- package/public/assets/{pairs-CrUR50ou.js.map → pairs-8NVKHh96.js.map} +1 -1
- package/public/assets/{projects-B_AtxHtH.js → projects-CZJW35C7.js} +2 -2
- package/public/assets/{projects-B_AtxHtH.js.map → projects-CZJW35C7.js.map} +1 -1
- package/public/assets/{prompt-settings-CPtYSban.js → prompt-settings-fqyLwUa-.js} +2 -2
- package/public/assets/{prompt-settings-CPtYSban.js.map → prompt-settings-fqyLwUa-.js.map} +1 -1
- package/public/assets/{registry-BgUALWn5.js → registry-BKbLL4mO.js} +2 -2
- package/public/assets/{registry-BgUALWn5.js.map → registry-BKbLL4mO.js.map} +1 -1
- package/public/assets/{security-5lHfGk7g.js → security-BlTFTbq0.js} +2 -2
- package/public/assets/{security-5lHfGk7g.js.map → security-BlTFTbq0.js.map} +1 -1
- package/public/assets/{select-Cgx4QSLj.js → select-Zd_qh0uU.js} +2 -2
- package/public/assets/{select-Cgx4QSLj.js.map → select-Zd_qh0uU.js.map} +1 -1
- package/public/assets/{settings-DFGgxyun.js → settings-nHuSVRkL.js} +2 -2
- package/public/assets/{settings-DFGgxyun.js.map → settings-nHuSVRkL.js.map} +1 -1
- package/public/assets/store-nXGxgz2Q.js +9 -0
- package/public/assets/store-nXGxgz2Q.js.map +1 -0
- package/public/assets/{tasks-BvH5rluu.js → tasks-C3JhQ8vB.js} +2 -2
- package/public/assets/{tasks-BvH5rluu.js.map → tasks-C3JhQ8vB.js.map} +1 -1
- package/public/assets/{teams-VZOIOdoE.js → teams-C9E9Yt39.js} +2 -2
- package/public/assets/{teams-VZOIOdoE.js.map → teams-C9E9Yt39.js.map} +1 -1
- package/public/assets/{terminal-viewer-impl-DYY3Wmoe.js → terminal-viewer-impl-FP0JlsvS.js} +22 -22
- package/public/assets/{terminal-viewer-impl-DYY3Wmoe.js.map → terminal-viewer-impl-FP0JlsvS.js.map} +1 -1
- package/public/assets/types-uVOXgvMT.js.map +1 -1
- package/public/assets/{work-queue-B6Xk7QNe.js → work-queue-Ema7Hy3s.js} +2 -2
- package/public/assets/{work-queue-B6Xk7QNe.js.map → work-queue-Ema7Hy3s.js.map} +1 -1
- package/public/assets/{workspaces-CPIzAHAe.js → workspaces-DI317fMk.js} +2 -2
- package/public/assets/{workspaces-CPIzAHAe.js.map → workspaces-DI317fMk.js.map} +1 -1
- package/public/index.html +3 -3
- package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/src/auto-spawn-dispatch.ts +2 -2
- package/src/commands-db.ts +14 -0
- package/src/db/delivery.ts +85 -3
- package/src/db/index.ts +2 -0
- package/src/db/migrations.ts +2 -1
- package/src/db/provisioning-callmux.ts +180 -0
- package/src/db/provisioning-registry.ts +45 -374
- package/src/db/provisioning-resolve.ts +245 -0
- package/src/db/stats.ts +10 -8
- package/src/db/task-entity.ts +7 -8
- package/src/lifecycle-manager.ts +11 -12
- package/src/maintenance/jobs.ts +3 -1
- package/src/maintenance/workspaces.ts +28 -47
- package/src/managed-policy-escalation.ts +47 -0
- package/src/mcp-task-tools.ts +4 -5
- package/src/routes/commands.ts +8 -245
- package/src/routes/tasks.ts +3 -5
- package/src/routes/workspaces.ts +58 -22
- package/src/services/issue-lifecycle.ts +23 -8
- package/src/services/task-entity.ts +54 -2
- package/src/task-dispatcher.ts +12 -3
- package/src/workspace-auto-merge.ts +94 -34
- package/src/workspace-command-settle.ts +227 -0
- package/src/workspace-escalation.ts +45 -0
- package/src/workspace-merge-progress.ts +16 -0
- package/src/workspace-orphans.ts +96 -4
- package/public/assets/agents-Ii6XlVwR.js +0 -2
- package/public/assets/chat-BCwEUj5R.js +0 -2
- package/public/assets/index-Ceiwd02X.js +0 -25
- package/public/assets/index-Ceiwd02X.js.map +0 -1
- package/public/assets/store-BM4zM89B.js +0 -9
- package/public/assets/store-BM4zM89B.js.map +0 -1
|
@@ -21,10 +21,13 @@
|
|
|
21
21
|
import {
|
|
22
22
|
createActivityEvent,
|
|
23
23
|
getActiveMergePause,
|
|
24
|
+
getWorkspace,
|
|
24
25
|
listOrchestrators,
|
|
25
26
|
listWorkspaces,
|
|
26
27
|
patchWorkspaceMetadata,
|
|
28
|
+
updateWorkspaceStatus,
|
|
27
29
|
} from "./db";
|
|
30
|
+
import { findLatestCommandByCorrelation } from "./commands-db";
|
|
28
31
|
import { notifyBranchLandFailed, notifyStewardHandoff } from "./branch-landed";
|
|
29
32
|
import { resolveStewardIdentity } from "./steward-identity";
|
|
30
33
|
import { emitCommandEvent } from "./command-events";
|
|
@@ -36,17 +39,20 @@ import { isAwaitingReviewerApproval } from "./reviewer-pipeline";
|
|
|
36
39
|
import { emitWorkspaceTaskSemanticNotifications } from "./services/task-lifecycle";
|
|
37
40
|
import { ensureRepoSteward } from "./steward";
|
|
38
41
|
import { requestWorkspaceMerge, workspaceMergeOrchestrators } from "./workspace-merge";
|
|
42
|
+
import { settleWorkspaceCommand } from "./workspace-command-settle";
|
|
43
|
+
import { CLEAR_MERGE_NO_PROGRESS, MAX_NO_PROGRESS_MERGE_ATTEMPTS, mergeNoProgressCount, noteMergeNoProgress } from "./workspace-merge-progress";
|
|
39
44
|
import { workspaceOwnerOrchestrator } from "./workspace-host";
|
|
40
45
|
import { workspaceActiveClaim } from "./workspace-claim";
|
|
41
46
|
import { DIRTY_WORKTREE_LAND_SKIP_REASON, READY_TO_LAND_STATUSES } from "./workspace-phase";
|
|
42
47
|
import { fetchHostMergePreview } from "./workspace-probe";
|
|
43
|
-
import type { WorkspaceAutoMergePolicy, WorkspaceLandingPolicy, WorkspaceMergePreview, WorkspaceMergeStrategy, WorkspaceRecord } from "./types";
|
|
48
|
+
import type { Command, WorkspaceAutoMergePolicy, WorkspaceLandingPolicy, WorkspaceMergePreview, WorkspaceMergeStrategy, WorkspaceRecord } from "./types";
|
|
44
49
|
import { STEWARD_WAKE_COOLDOWN_MS, stewardFallbackTarget, workspaceAutoMergeEnabled } from "./config";
|
|
45
50
|
import { isPathWithinBase } from "./utils";
|
|
46
51
|
import type { Orchestrator } from "./types";
|
|
47
52
|
|
|
48
53
|
const WORKSPACE_LAND_STRATEGIES = ["direct", "pr"] as const;
|
|
49
54
|
const WORKSPACE_AUTO_MERGE_POLICIES = ["on-green", "on-approval", "manual"] as const;
|
|
55
|
+
export { CLEAR_MERGE_NO_PROGRESS, MAX_NO_PROGRESS_MERGE_ATTEMPTS, mergeNoProgressCount, noteMergeNoProgress } from "./workspace-merge-progress";
|
|
50
56
|
|
|
51
57
|
type AutoMergeRequestOptions = {
|
|
52
58
|
requestedBy: "auto-merge";
|
|
@@ -174,41 +180,13 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
|
|
|
174
180
|
}
|
|
175
181
|
}
|
|
176
182
|
|
|
177
|
-
// #638 — auto-merge loop breaker. A `workspace.merge` the orchestrator settles `failed`
|
|
178
|
-
// (no progress: origin moved ahead of local base, a replay conflict the clean-preview
|
|
179
|
-
// didn't predict, a push race) returns the workspace to a landable status
|
|
180
|
-
// (`review_requested`). The on-settle re-trigger (routes/commands.ts → scheduleRepoMerge)
|
|
181
|
-
// then re-dispatches it within a few seconds — forever. That's a silent merge-command
|
|
182
|
-
// storm that churns the per-repo lease and 409s every manual land (the live incident).
|
|
183
|
-
// Cap consecutive no-progress attempts per workspace; past the cap, stop dispatching and
|
|
184
|
-
// escalate. The count is incremented by the merge-result handler (server owns the
|
|
185
|
-
// transition) and cleared on genuine progress or a fresh ready/request-review hand-off.
|
|
186
|
-
export const MAX_NO_PROGRESS_MERGE_ATTEMPTS = 3;
|
|
187
|
-
|
|
188
183
|
// Don't re-escalate a backed-off lane more than once per this window.
|
|
189
184
|
const NO_PROGRESS_ESCALATION_COOLDOWN_MS = 30 * 60 * 1000;
|
|
190
185
|
export const AUTO_MERGE_ERROR_BACKOFF_MS = 3_000;
|
|
191
186
|
export const AUTO_MERGE_WATCHDOG_STALE_MS = 5 * 60 * 1000;
|
|
187
|
+
export const IN_FLIGHT_WORKSPACE_WATCHDOG_AGE_MS = 20 * 60 * 1000;
|
|
192
188
|
|
|
193
|
-
|
|
194
|
-
export function mergeNoProgressCount(metadata: Record<string, unknown>): number {
|
|
195
|
-
const n = metadata.autoMergeNoProgressCount;
|
|
196
|
-
return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0;
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
/** Metadata patch recording one more no-progress merge attempt (#638). Applied by the
|
|
200
|
-
* merge-result handler when a dispatched merge settles `failed` without landing. */
|
|
201
|
-
export function noteMergeNoProgress(metadata: Record<string, unknown>): Record<string, unknown> {
|
|
202
|
-
return { autoMergeNoProgressCount: mergeNoProgressCount(metadata) + 1, autoMergeNoProgressAt: Date.now() };
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
/** Metadata patch that clears the no-progress back-off — genuine progress (a land/recycle)
|
|
206
|
-
* or a fresh hand-off means the lane is worth attempting again (#638). */
|
|
207
|
-
export const CLEAR_MERGE_NO_PROGRESS: Record<string, unknown> = {
|
|
208
|
-
autoMergeNoProgressCount: 0,
|
|
209
|
-
autoMergeNoProgressAt: undefined,
|
|
210
|
-
autoMergeBackedOffAt: undefined,
|
|
211
|
-
};
|
|
189
|
+
const TERMINAL_COMMAND_STATUSES = new Set(["succeeded", "failed", "timed_out", "rejected", "canceled"]);
|
|
212
190
|
|
|
213
191
|
// Surface a lane auto-merge can't advance (#638): wake the repo steward (if enabled) and
|
|
214
192
|
// route a stranded-escalation notice to the configured fallback / workspace owner, so a
|
|
@@ -558,11 +536,93 @@ export function resetRepoMergeSchedulerForTests(): void {
|
|
|
558
536
|
autoMergeWatchdogStartedAt = Date.now();
|
|
559
537
|
}
|
|
560
538
|
|
|
539
|
+
function commandIsTerminal(command: Command | null): boolean {
|
|
540
|
+
return Boolean(command && TERMINAL_COMMAND_STATUSES.has(command.status));
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function latestWorkspaceCommand(ws: WorkspaceRecord): Command | null {
|
|
544
|
+
const type = ws.status === "merge_planned" ? "workspace.merge" : "workspace.cleanup";
|
|
545
|
+
return findLatestCommandByCorrelation(type, ws.id);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function recoverAgedInFlightWorkspaces(now: number): Record<string, string[]> {
|
|
549
|
+
const oldEnough = (ws: WorkspaceRecord) => now - ws.updatedAt >= IN_FLIGHT_WORKSPACE_WATCHDOG_AGE_MS;
|
|
550
|
+
const recoveredMergePlanned: string[] = [];
|
|
551
|
+
const recoveredCleanupRequested: string[] = [];
|
|
552
|
+
const settledTerminal: string[] = [];
|
|
553
|
+
const stillActive: string[] = [];
|
|
554
|
+
|
|
555
|
+
for (const ws of listWorkspaces().filter((row) =>
|
|
556
|
+
row.mode === "isolated"
|
|
557
|
+
&& Boolean(row.worktreePath)
|
|
558
|
+
&& (row.status === "merge_planned" || row.status === "cleanup_requested")
|
|
559
|
+
&& oldEnough(row),
|
|
560
|
+
)) {
|
|
561
|
+
const command = latestWorkspaceCommand(ws);
|
|
562
|
+
if (command && !commandIsTerminal(command)) {
|
|
563
|
+
stillActive.push(ws.id);
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
566
|
+
if (command) {
|
|
567
|
+
settleWorkspaceCommand(command, { onMergeSettled: scheduleRepoMerge });
|
|
568
|
+
settledTerminal.push(ws.id);
|
|
569
|
+
}
|
|
570
|
+
const fresh = getWorkspace(ws.id);
|
|
571
|
+
if (!fresh) continue;
|
|
572
|
+
if (fresh.status === "merge_planned") {
|
|
573
|
+
const reason = command
|
|
574
|
+
? `workspace.merge ${command.status} without settling workspace state`
|
|
575
|
+
: "workspace.merge command missing";
|
|
576
|
+
updateWorkspaceStatus(fresh.id, "review_requested", {
|
|
577
|
+
mergeCommandId: command?.id,
|
|
578
|
+
mergeError: reason,
|
|
579
|
+
inFlightWatchdogRecoveredAt: now,
|
|
580
|
+
...noteMergeNoProgress(fresh.metadata),
|
|
581
|
+
});
|
|
582
|
+
recoveredMergePlanned.push(fresh.id);
|
|
583
|
+
scheduleRepoMerge(fresh.repoRoot);
|
|
584
|
+
createActivityEvent({
|
|
585
|
+
clientId: `workspace-inflight-watchdog-merge-${fresh.id}-${now}`,
|
|
586
|
+
kind: "state",
|
|
587
|
+
title: "Recovered stuck workspace merge",
|
|
588
|
+
body: `${fresh.branch ?? fresh.id} was stuck in merge_planned; returned to review_requested (${reason})`,
|
|
589
|
+
meta: fresh.branch ?? fresh.id,
|
|
590
|
+
icon: "ti-refresh-alert",
|
|
591
|
+
view: "orchestrators",
|
|
592
|
+
metadata: { source: "server", maintenanceJobId: "workspace-auto-merge-watchdog", workspaceId: fresh.id, commandId: command?.id, commandStatus: command?.status },
|
|
593
|
+
});
|
|
594
|
+
} else if (fresh.status === "cleanup_requested") {
|
|
595
|
+
const reason = command
|
|
596
|
+
? `workspace.cleanup ${command.status} without cleaning workspace`
|
|
597
|
+
: "workspace.cleanup command missing";
|
|
598
|
+
updateWorkspaceStatus(fresh.id, "active", {
|
|
599
|
+
cleanupCommandId: command?.id,
|
|
600
|
+
cleanupError: reason,
|
|
601
|
+
inFlightWatchdogRecoveredAt: now,
|
|
602
|
+
});
|
|
603
|
+
recoveredCleanupRequested.push(fresh.id);
|
|
604
|
+
createActivityEvent({
|
|
605
|
+
clientId: `workspace-inflight-watchdog-cleanup-${fresh.id}-${now}`,
|
|
606
|
+
kind: "state",
|
|
607
|
+
title: "Recovered stuck workspace cleanup",
|
|
608
|
+
body: `${fresh.branch ?? fresh.id} was stuck in cleanup_requested; returned to active (${reason})`,
|
|
609
|
+
meta: fresh.branch ?? fresh.id,
|
|
610
|
+
icon: "ti-refresh-alert",
|
|
611
|
+
view: "orchestrators",
|
|
612
|
+
metadata: { source: "server", maintenanceJobId: "workspace-auto-merge-watchdog", workspaceId: fresh.id, commandId: command?.id, commandStatus: command?.status },
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
return { recoveredMergePlanned, recoveredCleanupRequested, settledTerminal, stillActive };
|
|
618
|
+
}
|
|
619
|
+
|
|
561
620
|
export function runAutoMergeWatchdog(
|
|
562
621
|
opts: { now?: number; staleMs?: number; log?: (message: string) => void } = {},
|
|
563
622
|
): Record<string, unknown> {
|
|
564
623
|
const now = opts.now ?? Date.now();
|
|
565
624
|
const staleMs = opts.staleMs ?? AUTO_MERGE_WATCHDOG_STALE_MS;
|
|
625
|
+
const inFlightRecovery = recoverAgedInFlightWorkspaces(now);
|
|
566
626
|
const ready = landableWorkspaces();
|
|
567
627
|
const repoRoots = [...new Set(ready.map((ws) => ws.repoRoot).filter(Boolean))];
|
|
568
628
|
const repoStaleForMs = Object.fromEntries(repoRoots.map((repoRoot) => {
|
|
@@ -571,10 +631,10 @@ export function runAutoMergeWatchdog(
|
|
|
571
631
|
return [repoRoot, now - lastProgressAt];
|
|
572
632
|
}));
|
|
573
633
|
const staleForMs = Object.values(repoStaleForMs).reduce((max, value) => Math.max(max, value), 0);
|
|
574
|
-
if (!ready.length) return { readyCount: 0, staleForMs, repoStaleForMs, restartedRepoRoots: [] };
|
|
634
|
+
if (!ready.length) return { readyCount: 0, staleForMs, repoStaleForMs, restartedRepoRoots: [], inFlightRecovery };
|
|
575
635
|
|
|
576
636
|
const restartedRepoRoots = repoRoots.filter((repoRoot) => repoStaleForMs[repoRoot]! >= staleMs);
|
|
577
|
-
if (!restartedRepoRoots.length) return { readyCount: ready.length, staleForMs, repoStaleForMs, restartedRepoRoots: [] };
|
|
637
|
+
if (!restartedRepoRoots.length) return { readyCount: ready.length, staleForMs, repoStaleForMs, restartedRepoRoots: [], inFlightRecovery };
|
|
578
638
|
|
|
579
639
|
const log = opts.log ?? console.warn;
|
|
580
640
|
log(`[workspace-auto-merge] Watchdog: no auto-merge cycle completed for ${staleForMs}ms while ${ready.length} ready workspace(s) exist; restarting ${restartedRepoRoots.length} repo merge loop(s)`);
|
|
@@ -583,7 +643,7 @@ export function runAutoMergeWatchdog(
|
|
|
583
643
|
if (existing) existing.rerun = true;
|
|
584
644
|
else scheduleRepoMerge(repoRoot);
|
|
585
645
|
}
|
|
586
|
-
return { readyCount: ready.length, staleForMs, repoStaleForMs, restartedRepoRoots };
|
|
646
|
+
return { readyCount: ready.length, staleForMs, repoStaleForMs, restartedRepoRoots, inFlightRecovery };
|
|
587
647
|
}
|
|
588
648
|
|
|
589
649
|
// Drain the landable rows for ONE repo (the event-driven fast path). Scoped to the
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import { isRecord, sanitizeFsName, type LandGateRunResult } from "agent-relay-sdk";
|
|
2
|
+
import { storeTextArtifact } from "./artifact-text";
|
|
3
|
+
import { notifyBaseWorktreeStrand, notifyBranchLandFailed, notifyBranchLanded, notifyLandGateFailed, notifyLandGateWarnings } from "./branch-landed";
|
|
4
|
+
import { getWorkspace, releaseMergeLease, setWorkspaceBranch, updateWorkspaceStatus } from "./db";
|
|
5
|
+
import { handleReviewerMergeResult } from "./reviewer-pipeline";
|
|
6
|
+
import { emitWorkspaceTaskSemanticNotifications } from "./services/task-lifecycle";
|
|
7
|
+
import { VALID_WORKSPACE_STATUSES, cleanString, optionalEnum } from "./validation";
|
|
8
|
+
import { CLEAR_MERGE_NO_PROGRESS, noteMergeNoProgress } from "./workspace-merge-progress";
|
|
9
|
+
import { claimMetadataPatch } from "./workspace-claim";
|
|
10
|
+
import { commandTargetsWorkspaceOwner } from "./workspace-host";
|
|
11
|
+
import type { Command, WorkspaceStatus } from "./types";
|
|
12
|
+
|
|
13
|
+
export interface WorkspaceCommandSettleResult {
|
|
14
|
+
settled: boolean;
|
|
15
|
+
ignoredWrongOwner?: boolean;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface WorkspaceCommandSettleOptions {
|
|
19
|
+
onMergeSettled?: (repoRoot: string | undefined) => void;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const WORKSPACE_MERGE_SETTLED_STATUSES = new Set(["succeeded", "failed", "timed_out", "rejected", "canceled"]);
|
|
23
|
+
const WORKSPACE_MERGE_FAILED_STATUSES = new Set(["failed", "timed_out", "rejected", "canceled"]);
|
|
24
|
+
const WORKSPACE_CLEANUP_FAILED_STATUSES = new Set(["failed", "timed_out", "rejected", "canceled"]);
|
|
25
|
+
|
|
26
|
+
function normalizeGateRunResult(value: unknown): LandGateRunResult | undefined {
|
|
27
|
+
if (!isRecord(value)) return undefined;
|
|
28
|
+
const name = typeof value.name === "string" ? value.name : undefined;
|
|
29
|
+
const command = typeof value.command === "string" ? value.command : "";
|
|
30
|
+
if (!name) return undefined;
|
|
31
|
+
return {
|
|
32
|
+
name,
|
|
33
|
+
command,
|
|
34
|
+
optional: value.optional === true,
|
|
35
|
+
passed: value.passed === true,
|
|
36
|
+
exitCode: typeof value.exitCode === "number" ? value.exitCode : null,
|
|
37
|
+
timedOut: value.timedOut === true,
|
|
38
|
+
durationMs: typeof value.durationMs === "number" ? value.durationMs : 0,
|
|
39
|
+
outputTail: typeof value.outputTail === "string" ? value.outputTail : "",
|
|
40
|
+
output: typeof value.output === "string" ? value.output : (typeof value.outputTail === "string" ? value.outputTail : ""),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function normalizeGateWarnings(value: unknown): LandGateRunResult[] {
|
|
45
|
+
if (!Array.isArray(value)) return [];
|
|
46
|
+
return value.map(normalizeGateRunResult).filter((g): g is LandGateRunResult => Boolean(g));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function uploadGateOutputArtifact(workspaceId: string, gate: LandGateRunResult): Promise<string | undefined> {
|
|
50
|
+
try {
|
|
51
|
+
const safeName = sanitizeFsName(gate.name, { replacement: "-", maxLen: 60, trimEdge: true, fallback: "gate" });
|
|
52
|
+
const stored = await storeTextArtifact({
|
|
53
|
+
text: gate.output || gate.outputTail || "(no output)",
|
|
54
|
+
filename: `land-gate-${safeName}-${workspaceId}.log`,
|
|
55
|
+
createdBy: "server",
|
|
56
|
+
metadata: { kind: "land-gate-output", workspaceId, gateName: gate.name, gateCommand: gate.command, exitCode: gate.exitCode, timedOut: gate.timedOut },
|
|
57
|
+
});
|
|
58
|
+
return stored.id;
|
|
59
|
+
} catch (error) {
|
|
60
|
+
console.warn(`land-gate artifact upload failed for ${workspaceId} gate "${gate.name}": ${String(error)}`);
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function commandFailureReason(command: Command, fallback: string): string {
|
|
66
|
+
if (command.error) return command.error;
|
|
67
|
+
if (command.status === "timed_out") return "command timed out";
|
|
68
|
+
if (command.status === "canceled") return "command canceled";
|
|
69
|
+
if (command.status === "rejected") return "command rejected";
|
|
70
|
+
return fallback;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function settleWorkspaceCleanupCommand(command: Command): WorkspaceCommandSettleResult {
|
|
74
|
+
if (command.type !== "workspace.cleanup") return { settled: false };
|
|
75
|
+
if (command.status === "succeeded" && isRecord(command.result)) {
|
|
76
|
+
const workspaceId = cleanString(command.result.workspaceId, "result.workspaceId", { max: 160 });
|
|
77
|
+
if (!workspaceId) return { settled: true };
|
|
78
|
+
const workspace = getWorkspace(workspaceId);
|
|
79
|
+
if (workspace && commandTargetsWorkspaceOwner(command, workspace)) {
|
|
80
|
+
updateWorkspaceStatus(workspaceId, "cleaned", {
|
|
81
|
+
cleanupResult: command.result,
|
|
82
|
+
cleanupCommandId: command.id,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return { settled: true };
|
|
86
|
+
}
|
|
87
|
+
if (!WORKSPACE_CLEANUP_FAILED_STATUSES.has(command.status)) return { settled: false };
|
|
88
|
+
const resultWorkspaceId = isRecord(command.result) ? cleanString(command.result.workspaceId, "result.workspaceId", { max: 160 }) : undefined;
|
|
89
|
+
const workspaceId = resultWorkspaceId ?? command.correlationId;
|
|
90
|
+
if (!workspaceId) return { settled: true };
|
|
91
|
+
const workspace = getWorkspace(workspaceId);
|
|
92
|
+
if (!workspace || workspace.status !== "cleanup_requested") return { settled: true };
|
|
93
|
+
if (!commandTargetsWorkspaceOwner(command, workspace)) return { settled: true, ignoredWrongOwner: true };
|
|
94
|
+
updateWorkspaceStatus(workspaceId, "active", {
|
|
95
|
+
cleanupCommandId: command.id,
|
|
96
|
+
cleanupError: commandFailureReason(command, "cleanup failed"),
|
|
97
|
+
cleanupFailedAt: Date.now(),
|
|
98
|
+
cleanupResult: isRecord(command.result) ? command.result : undefined,
|
|
99
|
+
});
|
|
100
|
+
return { settled: true };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function settleWorkspaceMergeCommand(command: Command, opts: WorkspaceCommandSettleOptions = {}): WorkspaceCommandSettleResult {
|
|
104
|
+
if (command.type !== "workspace.merge") return { settled: false };
|
|
105
|
+
if (WORKSPACE_MERGE_SETTLED_STATUSES.has(command.status)) {
|
|
106
|
+
releaseMergeLease({ commandId: command.id });
|
|
107
|
+
const settledRepoRoot = typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined;
|
|
108
|
+
opts.onMergeSettled?.(settledRepoRoot);
|
|
109
|
+
}
|
|
110
|
+
if (command.status === "succeeded" && isRecord(command.result)) {
|
|
111
|
+
if (handleReviewerMergeResult(command, command.result)) return { settled: true };
|
|
112
|
+
const workspaceId = cleanString(command.result.workspaceId, "result.workspaceId", { max: 160 });
|
|
113
|
+
const resultStatus = optionalEnum(command.result.status, "result.status", VALID_WORKSPACE_STATUSES) as WorkspaceStatus | undefined;
|
|
114
|
+
if (!workspaceId || !resultStatus) return { settled: true };
|
|
115
|
+
const landedWorkspace = getWorkspace(workspaceId);
|
|
116
|
+
if (landedWorkspace && !commandTargetsWorkspaceOwner(command, landedWorkspace)) return { settled: true, ignoredWrongOwner: true };
|
|
117
|
+
const newBranch = cleanString(command.result.newBranch, "result.newBranch", { max: 240 });
|
|
118
|
+
const normalizedStatus = command.result.merged === true
|
|
119
|
+
&& !newBranch
|
|
120
|
+
&& ["review_requested", "merge_planned", "conflict"].includes(resultStatus)
|
|
121
|
+
? "merged"
|
|
122
|
+
: resultStatus;
|
|
123
|
+
updateWorkspaceStatus(workspaceId, normalizedStatus, {
|
|
124
|
+
mergeResult: command.result,
|
|
125
|
+
mergeCommandId: command.id,
|
|
126
|
+
mergedAt: Date.now(),
|
|
127
|
+
...CLEAR_MERGE_NO_PROGRESS,
|
|
128
|
+
mergeError: undefined,
|
|
129
|
+
...(normalizedStatus !== resultStatus ? { mergeStatusNormalizedFrom: resultStatus } : {}),
|
|
130
|
+
...claimMetadataPatch(true),
|
|
131
|
+
});
|
|
132
|
+
const mergedSha = cleanString(command.result.mergedSha, "result.mergedSha", { max: 64 });
|
|
133
|
+
const baseSha = cleanString(command.result.baseSha, "result.baseSha", { max: 64 });
|
|
134
|
+
if (newBranch) setWorkspaceBranch(workspaceId, newBranch, baseSha ?? mergedSha);
|
|
135
|
+
const prLanded = isRecord(command.params.prLanded) ? command.params.prLanded : undefined;
|
|
136
|
+
const baseWorktreeSync = isRecord(command.result.baseWorktreeSync) ? command.result.baseWorktreeSync : undefined;
|
|
137
|
+
if (baseWorktreeSync && baseWorktreeSync.reconciled === false && landedWorkspace) {
|
|
138
|
+
notifyBaseWorktreeStrand({
|
|
139
|
+
workspace: landedWorkspace,
|
|
140
|
+
sync: {
|
|
141
|
+
reconciled: false,
|
|
142
|
+
unsyncedPaths: Array.isArray(baseWorktreeSync.unsyncedPaths) ? baseWorktreeSync.unsyncedPaths.filter((p): p is string => typeof p === "string") : undefined,
|
|
143
|
+
preservedWipPaths: Array.isArray(baseWorktreeSync.preservedWipPaths) ? baseWorktreeSync.preservedWipPaths.filter((p): p is string => typeof p === "string") : undefined,
|
|
144
|
+
message: typeof baseWorktreeSync.message === "string" ? baseWorktreeSync.message : undefined,
|
|
145
|
+
},
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
if ((command.result.merged === true || prLanded) && landedWorkspace) {
|
|
149
|
+
notifyBranchLanded({
|
|
150
|
+
workspace: landedWorkspace,
|
|
151
|
+
mergedSha: prLanded ? cleanString(prLanded.sha, "params.prLanded.sha", { max: 64 }) ?? mergedSha : mergedSha,
|
|
152
|
+
subject: prLanded
|
|
153
|
+
? cleanString(prLanded.subject, "params.prLanded.subject", { max: 200 })
|
|
154
|
+
: cleanString(command.result.subject, "result.subject", { max: 200 }),
|
|
155
|
+
prLand: Boolean(prLanded),
|
|
156
|
+
newBranch,
|
|
157
|
+
pushed: prLanded ? true : typeof command.result.pushed === "boolean" ? command.result.pushed : undefined,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
const gateWarnings = command.result.merged === true ? normalizeGateWarnings(command.result.gateWarnings) : [];
|
|
161
|
+
if (gateWarnings.length && landedWorkspace) {
|
|
162
|
+
const warnWorkspace = landedWorkspace;
|
|
163
|
+
void (async () => {
|
|
164
|
+
const combined = gateWarnings.map((g) => `### ${g.name} (${g.command}) - ${g.timedOut ? "timed out" : g.exitCode === null ? "could not run" : `exited ${g.exitCode}`}\n${g.output || g.outputTail || "(no output)"}`).join("\n\n");
|
|
165
|
+
let artifactId: string | undefined;
|
|
166
|
+
try {
|
|
167
|
+
artifactId = (await storeTextArtifact({ text: combined, filename: `land-gate-warnings-${warnWorkspace.id}.log`, createdBy: "server", metadata: { kind: "land-gate-warnings", workspaceId: warnWorkspace.id, gateNames: gateWarnings.map((g) => g.name) } })).id;
|
|
168
|
+
} catch (error) {
|
|
169
|
+
console.warn(`land-gate-warnings artifact upload failed for ${warnWorkspace.id}: ${String(error)}`);
|
|
170
|
+
}
|
|
171
|
+
try {
|
|
172
|
+
notifyLandGateWarnings({ workspace: warnWorkspace, warnings: gateWarnings, artifactId });
|
|
173
|
+
} catch (error) {
|
|
174
|
+
console.warn(`land-gate-warnings notify threw for ${warnWorkspace.id}: ${String(error)}`);
|
|
175
|
+
}
|
|
176
|
+
})();
|
|
177
|
+
}
|
|
178
|
+
return { settled: true };
|
|
179
|
+
}
|
|
180
|
+
if (WORKSPACE_MERGE_FAILED_STATUSES.has(command.status) && command.correlationId) {
|
|
181
|
+
const current = getWorkspace(command.correlationId);
|
|
182
|
+
if (!current || current.status !== "merge_planned") return { settled: true };
|
|
183
|
+
if (!commandTargetsWorkspaceOwner(command, current)) return { settled: true, ignoredWrongOwner: true };
|
|
184
|
+
const failedResultStatus = isRecord(command.result)
|
|
185
|
+
? optionalEnum(command.result.status, "result.status", VALID_WORKSPACE_STATUSES) as WorkspaceStatus | undefined
|
|
186
|
+
: undefined;
|
|
187
|
+
const nextStatus: WorkspaceStatus = failedResultStatus && failedResultStatus !== "merge_planned"
|
|
188
|
+
? failedResultStatus
|
|
189
|
+
: "review_requested";
|
|
190
|
+
const reason = commandFailureReason(command, "merge failed");
|
|
191
|
+
updateWorkspaceStatus(command.correlationId, nextStatus, {
|
|
192
|
+
mergeCommandId: command.id,
|
|
193
|
+
mergeError: reason,
|
|
194
|
+
...noteMergeNoProgress(current.metadata),
|
|
195
|
+
});
|
|
196
|
+
if (nextStatus === "conflict") {
|
|
197
|
+
emitWorkspaceTaskSemanticNotifications({
|
|
198
|
+
workspace: current,
|
|
199
|
+
transition: "land-conflict",
|
|
200
|
+
actor: "workspace-merge",
|
|
201
|
+
occurrenceKey: `workspace-merge-conflict:${current.id}:${command.id}`,
|
|
202
|
+
outcome: { reason, commandId: command.id, status: nextStatus },
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
const gateFailure = isRecord(command.result) ? normalizeGateRunResult(command.result.gateFailure) : undefined;
|
|
206
|
+
if (gateFailure) {
|
|
207
|
+
void (async () => {
|
|
208
|
+
const artifactId = await uploadGateOutputArtifact(current.id, gateFailure);
|
|
209
|
+
try {
|
|
210
|
+
notifyLandGateFailed({ workspace: current, gate: gateFailure, artifactId });
|
|
211
|
+
} catch (error) {
|
|
212
|
+
console.warn(`land-gate-failed notify threw for ${current.id}: ${String(error)}`);
|
|
213
|
+
}
|
|
214
|
+
})();
|
|
215
|
+
} else {
|
|
216
|
+
notifyBranchLandFailed({ workspace: current, commandId: command.id, reason });
|
|
217
|
+
}
|
|
218
|
+
return { settled: true };
|
|
219
|
+
}
|
|
220
|
+
return { settled: false };
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function settleWorkspaceCommand(command: Command, opts: WorkspaceCommandSettleOptions = {}): WorkspaceCommandSettleResult {
|
|
224
|
+
if (command.type === "workspace.merge") return settleWorkspaceMergeCommand(command, opts);
|
|
225
|
+
if (command.type === "workspace.cleanup") return settleWorkspaceCleanupCommand(command);
|
|
226
|
+
return { settled: false };
|
|
227
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { stewardFallbackTarget } from "./config";
|
|
2
|
+
import { routeNotification } from "./notification-router";
|
|
3
|
+
import { resolveStewardIdentity } from "./steward-identity";
|
|
4
|
+
import type { NotificationType } from "./notification-types";
|
|
5
|
+
import type { WorkspaceRecord, WorkspaceStatus } from "./types";
|
|
6
|
+
|
|
7
|
+
export const READY_TO_LAND_ESCALATION_STATUSES = new Set<WorkspaceStatus>(["ready", "review_requested"]);
|
|
8
|
+
|
|
9
|
+
export function routeWorkspaceCustodyEscalation(input: {
|
|
10
|
+
workspace: Pick<WorkspaceRecord, "id" | "repoRoot" | "branch" | "status" | "ownerAgentId">;
|
|
11
|
+
type?: NotificationType;
|
|
12
|
+
subject: string;
|
|
13
|
+
body: string;
|
|
14
|
+
payload: Record<string, unknown>;
|
|
15
|
+
idempotencyKey?: string;
|
|
16
|
+
}): string[] {
|
|
17
|
+
const { ownerAgentId, coordinatorAgentId } = resolveStewardIdentity(input.workspace);
|
|
18
|
+
const fallback = stewardFallbackTarget();
|
|
19
|
+
const recipients = [...new Set([ownerAgentId, coordinatorAgentId, fallback].filter((value): value is string => Boolean(value)))];
|
|
20
|
+
if (!recipients.length) return [];
|
|
21
|
+
|
|
22
|
+
routeNotification({
|
|
23
|
+
type: input.type ?? "workspace.stranded_escalation",
|
|
24
|
+
scope: {
|
|
25
|
+
workspaceId: input.workspace.id,
|
|
26
|
+
repoRoot: input.workspace.repoRoot,
|
|
27
|
+
author: ownerAgentId,
|
|
28
|
+
parent: coordinatorAgentId ?? undefined,
|
|
29
|
+
},
|
|
30
|
+
recipients,
|
|
31
|
+
message: {
|
|
32
|
+
subject: input.subject,
|
|
33
|
+
body: input.body,
|
|
34
|
+
payload: {
|
|
35
|
+
...input.payload,
|
|
36
|
+
coordinatorAgentId,
|
|
37
|
+
ownerAgentId,
|
|
38
|
+
fallbackTarget: fallback || undefined,
|
|
39
|
+
},
|
|
40
|
+
idempotencyKey: input.idempotencyKey,
|
|
41
|
+
},
|
|
42
|
+
ttlMs: null,
|
|
43
|
+
});
|
|
44
|
+
return recipients;
|
|
45
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export const MAX_NO_PROGRESS_MERGE_ATTEMPTS = 3;
|
|
2
|
+
|
|
3
|
+
export function mergeNoProgressCount(metadata: Record<string, unknown>): number {
|
|
4
|
+
const n = metadata.autoMergeNoProgressCount;
|
|
5
|
+
return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function noteMergeNoProgress(metadata: Record<string, unknown>): Record<string, unknown> {
|
|
9
|
+
return { autoMergeNoProgressCount: mergeNoProgressCount(metadata) + 1, autoMergeNoProgressAt: Date.now() };
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export const CLEAR_MERGE_NO_PROGRESS: Record<string, unknown> = {
|
|
13
|
+
autoMergeNoProgressCount: 0,
|
|
14
|
+
autoMergeNoProgressAt: undefined,
|
|
15
|
+
autoMergeBackedOffAt: undefined,
|
|
16
|
+
};
|