agent-relay-server 0.62.3 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +2 -2
- package/public/assets/{MessageBubble-DRrxfjyT.js → MessageBubble-Cl6ufPHw.js} +2 -2
- package/public/assets/MessageBubble-Cl6ufPHw.js.map +1 -0
- package/public/assets/{PlanGraphPanel-Dp2eYazz.js → PlanGraphPanel-BnfjmkQ8.js} +2 -2
- package/public/assets/{PlanGraphPanel-Dp2eYazz.js.map → PlanGraphPanel-BnfjmkQ8.js.map} +1 -1
- package/public/assets/{activity--t0U2yzq.js → activity-aFxvGLtd.js} +2 -2
- package/public/assets/{activity--t0U2yzq.js.map → activity-aFxvGLtd.js.map} +1 -1
- package/public/assets/{agent-profiles-BMtJYzxn.js → agent-profiles-B4mleUDz.js} +2 -2
- package/public/assets/{agent-profiles-BMtJYzxn.js.map → agent-profiles-B4mleUDz.js.map} +1 -1
- package/public/assets/{agents-Ck9AePlc.js → agents-DW5djToQ.js} +2 -2
- package/public/assets/{agents-Ck9AePlc.js.map → agents-DW5djToQ.js.map} +1 -1
- package/public/assets/{analytics-CzfK387L.js → analytics-CRi6thtg.js} +2 -2
- package/public/assets/{analytics-CzfK387L.js.map → analytics-CRi6thtg.js.map} +1 -1
- package/public/assets/{automation-Cgw8vtE8.js → automation-Do4OpauI.js} +2 -2
- package/public/assets/{automation-Cgw8vtE8.js.map → automation-Do4OpauI.js.map} +1 -1
- package/public/assets/{branch-state-badge-DnEEPtOP.js → branch-state-badge-D3W3w7zz.js} +2 -2
- package/public/assets/{branch-state-badge-DnEEPtOP.js.map → branch-state-badge-D3W3w7zz.js.map} +1 -1
- package/public/assets/{channels-Dc3L2MQs.js → channels-BytdR9NS.js} +2 -2
- package/public/assets/{channels-Dc3L2MQs.js.map → channels-BytdR9NS.js.map} +1 -1
- package/public/assets/{chat-Fc6iJ7sa.js → chat-CvaQ2czL.js} +2 -2
- package/public/assets/{chat-Fc6iJ7sa.js.map → chat-CvaQ2czL.js.map} +1 -1
- package/public/assets/{connectors-Dc2ttGWs.js → connectors-BLQDp8Bc.js} +2 -2
- package/public/assets/{connectors-Dc2ttGWs.js.map → connectors-BLQDp8Bc.js.map} +1 -1
- package/public/assets/{display-DzNvOu1j.js → display-B06I5sqa.js} +3 -3
- package/public/assets/display-B06I5sqa.js.map +1 -0
- package/public/assets/{formatted-body-impl-Bg-gx7rG.js → formatted-body-impl-0Lf8e1Ic.js} +2 -2
- package/public/assets/{formatted-body-impl-Bg-gx7rG.js.map → formatted-body-impl-0Lf8e1Ic.js.map} +1 -1
- package/public/assets/{index-BeDncjN4.js → index-C3su8ezq.js} +6 -6
- package/public/assets/{index-BeDncjN4.js.map → index-C3su8ezq.js.map} +1 -1
- package/public/assets/{insights-c08aW-7n.js → insights-64uExH1E.js} +2 -2
- package/public/assets/{insights-c08aW-7n.js.map → insights-64uExH1E.js.map} +1 -1
- package/public/assets/{integrations-BwpgcqnF.js → integrations-HyC1EaIp.js} +2 -2
- package/public/assets/{integrations-BwpgcqnF.js.map → integrations-HyC1EaIp.js.map} +1 -1
- package/public/assets/{maintenance-CTKP0kWB.js → maintenance-EVBpKH07.js} +2 -2
- package/public/assets/{maintenance-CTKP0kWB.js.map → maintenance-EVBpKH07.js.map} +1 -1
- package/public/assets/{managed-agents-BofCxo8d.js → managed-agents-BLZPxGuw.js} +2 -2
- package/public/assets/{managed-agents-BofCxo8d.js.map → managed-agents-BLZPxGuw.js.map} +1 -1
- package/public/assets/{markdown-preview-impl-CQ-6BIb8.js → markdown-preview-impl-Bg6rCWYN.js} +2 -2
- package/public/assets/{markdown-preview-impl-CQ-6BIb8.js.map → markdown-preview-impl-Bg6rCWYN.js.map} +1 -1
- package/public/assets/{memory-DvfXjMoi.js → memory-Bjr8bgjj.js} +2 -2
- package/public/assets/{memory-DvfXjMoi.js.map → memory-Bjr8bgjj.js.map} +1 -1
- package/public/assets/{messages-C9VsPzZ_.js → messages-CRYw1F4z.js} +2 -2
- package/public/assets/{messages-C9VsPzZ_.js.map → messages-CRYw1F4z.js.map} +1 -1
- package/public/assets/{orchestrators-MffVueeE.js → orchestrators-D-co1vdp.js} +2 -2
- package/public/assets/{orchestrators-MffVueeE.js.map → orchestrators-D-co1vdp.js.map} +1 -1
- package/public/assets/{overview-BSVajefM.js → overview-Ddy5czri.js} +2 -2
- package/public/assets/{overview-BSVajefM.js.map → overview-Ddy5czri.js.map} +1 -1
- package/public/assets/{pairs-B5711-Cs.js → pairs-CFVM8uHL.js} +2 -2
- package/public/assets/{pairs-B5711-Cs.js.map → pairs-CFVM8uHL.js.map} +1 -1
- package/public/assets/{security-DkUwK9Vj.js → security-Df4_76bh.js} +2 -2
- package/public/assets/{security-DkUwK9Vj.js.map → security-Df4_76bh.js.map} +1 -1
- package/public/assets/{select-DrM2RRiT.js → select-Cq3BBnL9.js} +2 -2
- package/public/assets/{select-DrM2RRiT.js.map → select-Cq3BBnL9.js.map} +1 -1
- package/public/assets/{settings-DdKCb-vy.js → settings-DOLEYXZL.js} +2 -2
- package/public/assets/{settings-DdKCb-vy.js.map → settings-DOLEYXZL.js.map} +1 -1
- package/public/assets/{store-iTiheuTD.js → store-C1E8jmNE.js} +4 -4
- package/public/assets/store-C1E8jmNE.js.map +1 -0
- package/public/assets/{tasks-e1y1PFfO.js → tasks-BcsmPi9_.js} +2 -2
- package/public/assets/{tasks-e1y1PFfO.js.map → tasks-BcsmPi9_.js.map} +1 -1
- package/public/assets/{teams-BUcxoGC_.js → teams-BQrV3eKI.js} +2 -2
- package/public/assets/{teams-BUcxoGC_.js.map → teams-BQrV3eKI.js.map} +1 -1
- package/public/assets/{terminal-viewer-impl-CmZjQOB2.js → terminal-viewer-impl-Dle04qpH.js} +2 -2
- package/public/assets/{terminal-viewer-impl-CmZjQOB2.js.map → terminal-viewer-impl-Dle04qpH.js.map} +1 -1
- package/public/assets/{work-queue-CXBQx43n.js → work-queue-BLhTLg4o.js} +2 -2
- package/public/assets/{work-queue-CXBQx43n.js.map → work-queue-BLhTLg4o.js.map} +1 -1
- package/public/assets/{workspaces-D2EfZh9a.js → workspaces-4NCNTmRQ.js} +3 -3
- package/public/assets/{workspaces-D2EfZh9a.js.map → workspaces-4NCNTmRQ.js.map} +1 -1
- package/public/index.html +3 -3
- package/src/agent-ref.ts +8 -6
- package/src/automations/constants.ts +12 -0
- package/src/automations/cron.ts +129 -0
- package/src/automations/crud.ts +84 -0
- package/src/automations/dispatch.ts +275 -0
- package/src/automations/index.ts +5 -0
- package/src/automations/reconcile.ts +245 -0
- package/src/automations/storage.ts +181 -0
- package/src/automations/types.ts +16 -0
- package/src/automations/validation.ts +140 -0
- package/src/automations.ts +1 -1045
- package/src/db/activity.ts +1 -1
- package/src/db/agent-predicates.ts +9 -0
- package/src/db/agent-search.ts +2 -2
- package/src/db/agents.ts +13 -41
- package/src/db/artifacts.ts +8 -8
- package/src/db/channel-identity.ts +38 -0
- package/src/db/channels.ts +37 -61
- package/src/db/continuation-archives.ts +4 -4
- package/src/db/delivery.ts +7 -12
- package/src/db/inbox.ts +1 -1
- package/src/db/integrations.ts +4 -4
- package/src/db/issue-lifecycle.ts +6 -6
- package/src/db/mappers.ts +1 -2
- package/src/db/merge-lease.ts +3 -3
- package/src/db/message-claims.ts +16 -0
- package/src/db/message-queries.ts +8 -0
- package/src/db/message-reads.ts +26 -9
- package/src/db/messages.ts +17 -41
- package/src/db/notification-subscriptions.ts +2 -2
- package/src/db/orchestrators.ts +4 -4
- package/src/db/pair-maintenance.ts +9 -0
- package/src/db/pairs.ts +10 -18
- package/src/db/plans.ts +1 -1
- package/src/db/policy-runtime.ts +14 -0
- package/src/db/project-entries.ts +1 -1
- package/src/db/projects.ts +2 -2
- package/src/db/shared.ts +7 -0
- package/src/db/stats.ts +2 -2
- package/src/db/task-core.ts +18 -0
- package/src/db/task-maintenance.ts +24 -0
- package/src/db/tasks.ts +5 -17
- package/src/db/teams.ts +3 -3
- package/src/db/workspaces.ts +6 -6
- package/src/issue-tracker/github.ts +3 -3
- package/src/maintenance/constants.ts +35 -0
- package/src/maintenance/events.ts +32 -0
- package/src/maintenance/index.ts +7 -0
- package/src/maintenance/jobs.ts +340 -0
- package/src/maintenance/orphaned-sessions.ts +86 -0
- package/src/maintenance/scheduler.ts +231 -0
- package/src/maintenance/teams.ts +65 -0
- package/src/maintenance/types.ts +31 -0
- package/src/maintenance/workspaces.ts +251 -0
- package/src/maintenance.ts +1 -1099
- package/src/mcp/identity.ts +131 -0
- package/src/mcp/index.ts +208 -0
- package/src/mcp/tools-messaging.ts +459 -0
- package/src/mcp/tools-spawn.ts +145 -0
- package/src/mcp/tools-workspace.ts +227 -0
- package/src/mcp/types.ts +24 -0
- package/src/mcp/validation.ts +104 -0
- package/src/mcp.ts +1 -1319
- package/src/notification-router.ts +2 -2
- package/src/notification-rules.ts +2 -2
- package/src/project-ingest.ts +3 -3
- package/src/project-instructions.ts +1 -1
- package/src/provider-config-store.ts +3 -3
- package/src/provider-state.ts +1 -1
- package/src/reviewer-pipeline.ts +1 -1
- package/src/self-resume.ts +3 -7
- package/src/services/agent-action.ts +2 -2
- package/src/services/issue-activity.ts +0 -4
- package/src/services/issue-associations.ts +2 -2
- package/src/services/issue-lifecycle.ts +7 -7
- package/src/services/managed-running.ts +1 -1
- package/src/services/parity-harness.ts +3 -3
- package/src/services/plan-graph.ts +3 -3
- package/src/services/send-message.ts +16 -12
- package/src/services/shutdown-agent.ts +2 -2
- package/src/services/team-templates.ts +2 -2
- package/src/services/team.ts +2 -2
- package/src/settings/modules.ts +1 -1
- package/src/settings/registry.ts +3 -3
- package/src/spawn-command.ts +1 -1
- package/src/sse.ts +0 -4
- package/src/workspace-actions.ts +1 -1
- package/src/workspace-probe.ts +3 -7
- package/public/assets/MessageBubble-DRrxfjyT.js.map +0 -1
- package/public/assets/display-DzNvOu1j.js.map +0 -1
- package/public/assets/store-iTiheuTD.js.map +0 -1
|
@@ -2,7 +2,7 @@ import type { IssueRef, IssueRefStatus } from "../types";
|
|
|
2
2
|
import type { IssueTrackerAdapter, IssueTrackerCloseOptions, IssueTrackerReopenOptions, IssueTrackerStatus } from "./adapter";
|
|
3
3
|
|
|
4
4
|
export const GITHUB_FIXED_LABEL = "fixed";
|
|
5
|
-
|
|
5
|
+
const GITHUB_CREDENTIAL_REQUIREMENT = "GitHub issue tracker adapter requires gh authenticated with a token that has issues:write scope";
|
|
6
6
|
|
|
7
7
|
type GitHubApiMethod = "GET" | "PATCH" | "POST";
|
|
8
8
|
|
|
@@ -12,7 +12,7 @@ export interface GitHubApiRequest {
|
|
|
12
12
|
body?: Record<string, unknown>;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
type GitHubApi = (request: GitHubApiRequest) => Promise<unknown>;
|
|
16
16
|
|
|
17
17
|
interface GitHubIssuePayload {
|
|
18
18
|
state?: unknown;
|
|
@@ -92,7 +92,7 @@ function uniqueLabels(labels: string[]): string[] {
|
|
|
92
92
|
return [...new Set(labels.map((label) => label.trim()).filter(Boolean))];
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
|
|
95
|
+
class GitHubIssueTrackerAdapter implements IssueTrackerAdapter {
|
|
96
96
|
constructor(private readonly api: GitHubApi = ghApi) {}
|
|
97
97
|
|
|
98
98
|
resolveLink(ref: IssueRef): string {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { DAY_MS, REAP_INTERVAL_MS } from "../config";
|
|
2
|
+
import { READY_TO_LAND_STATUSES, TERMINAL_WORKSPACE_STATUSES } from "../workspace-phase";
|
|
3
|
+
import type { WorkspaceStatus } from "../types";
|
|
4
|
+
|
|
5
|
+
export { DAY_MS, REAP_INTERVAL_MS, TERMINAL_WORKSPACE_STATUSES };
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_TIMEOUT_MS = 30_000;
|
|
8
|
+
export const SCHEDULER_TICK_MS = 10_000;
|
|
9
|
+
export const OUTBOX_RETENTION_MS = Number(process.env.AGENT_RELAY_OUTBOX_RETENTION_MS) || 60 * 60 * 1000;
|
|
10
|
+
export const TOKEN_RECORD_RETENTION_SECONDS = Number(process.env.AGENT_RELAY_TOKEN_RECORD_RETENTION_SECONDS) || 7 * 24 * 60 * 60;
|
|
11
|
+
export const CONFLICT_SCAN_INTERVAL_MS = Number(process.env.AGENT_RELAY_CONFLICT_SCAN_INTERVAL_MS) || 10 * 60 * 1000;
|
|
12
|
+
export const WORKSPACE_RETENTION_MS = Number(process.env.AGENT_RELAY_WORKSPACE_RETENTION_MS) || DAY_MS;
|
|
13
|
+
export const DB_MAINTENANCE_INTERVAL_MS = Number(process.env.AGENT_RELAY_DB_MAINTENANCE_INTERVAL_MS) || DAY_MS;
|
|
14
|
+
export const DB_VACUUM_EVERY = Number(process.env.AGENT_RELAY_DB_VACUUM_EVERY) || 7;
|
|
15
|
+
export const WORKSPACE_REVIEW_TTL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_REVIEW_TTL_MS) || 3 * DAY_MS;
|
|
16
|
+
export const WORKSPACE_GC_INTERVAL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_GC_INTERVAL_MS) || 60 * 60 * 1000;
|
|
17
|
+
export const WORKSPACE_ORPHAN_REAPER_INTERVAL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_ORPHAN_REAPER_INTERVAL_MS) || 30 * 60 * 1000;
|
|
18
|
+
export const WORKSPACE_AUTO_MERGE_INTERVAL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_AUTO_MERGE_INTERVAL_MS) || CONFLICT_SCAN_INTERVAL_MS;
|
|
19
|
+
export const ORPHAN_REAPER_INTERVAL_MS = Number(process.env.AGENT_RELAY_ORPHAN_REAPER_INTERVAL_MS) || 5 * 60 * 1000;
|
|
20
|
+
|
|
21
|
+
export const stewardEscalationMs = () => Number(process.env.AGENT_RELAY_WORKSPACE_STEWARD_ESCALATION_MS) || 60 * 60 * 1000;
|
|
22
|
+
export const stewardFallbackTarget = () => (process.env.AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK || "").trim();
|
|
23
|
+
export const STRANDABLE_STATUSES = new Set<WorkspaceStatus>([...READY_TO_LAND_STATUSES, "conflict"]);
|
|
24
|
+
export const CONFLICT_SCAN_STATUSES = new Set<WorkspaceStatus>(["active", "ready", "review_requested", "merge_planned", "conflict"]);
|
|
25
|
+
export const LANDED_RECONCILE_STATUSES = new Set<WorkspaceStatus>(["merge_planned", "review_requested", "conflict"]);
|
|
26
|
+
|
|
27
|
+
export const envMsOrDefault = (name: string, fallback: number): number => {
|
|
28
|
+
const v = Number(process.env[name]);
|
|
29
|
+
return Number.isFinite(v) && v >= 0 ? v : fallback;
|
|
30
|
+
};
|
|
31
|
+
export const orphanGraceMs = () => envMsOrDefault("AGENT_RELAY_ORPHAN_GRACE_MS", 30 * 60 * 1000);
|
|
32
|
+
export const orphanReapCooldownMs = () => envMsOrDefault("AGENT_RELAY_ORPHAN_REAP_COOLDOWN_MS", 5 * 60 * 1000);
|
|
33
|
+
export const teamReapGraceMs = () => envMsOrDefault("TEAM_REAP_GRACE_MS", 30 * 60 * 1000);
|
|
34
|
+
export const teamReapIntervalMs = () => envMsOrDefault("TEAM_REAP_INTERVAL_MS", 5 * 60 * 1000);
|
|
35
|
+
export const orphanReapEnabled = () => process.env.AGENT_RELAY_ORPHAN_REAP !== "0";
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { emitRelayEvent } from "../events";
|
|
2
|
+
import { routeNotification, type NotificationScope, type NotificationType } from "../notification-router";
|
|
3
|
+
import type { Command } from "../types";
|
|
4
|
+
|
|
5
|
+
export function emitCommand(command: Command): void {
|
|
6
|
+
const suffix = command.status === "timed_out" ? "timed_out" : command.status;
|
|
7
|
+
emitRelayEvent({
|
|
8
|
+
type: `command.${suffix}`,
|
|
9
|
+
source: command.source,
|
|
10
|
+
subject: command.id,
|
|
11
|
+
data: { command },
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// Route a maintenance notification, swallowing failures (a stale/missing/misconfigured
|
|
16
|
+
// target must never break the sweep). Returns the target on success, null otherwise.
|
|
17
|
+
export function notifyTarget(
|
|
18
|
+
type: NotificationType,
|
|
19
|
+
scope: NotificationScope,
|
|
20
|
+
target: string,
|
|
21
|
+
subject: string,
|
|
22
|
+
body: string,
|
|
23
|
+
payload: Record<string, unknown>,
|
|
24
|
+
): string | null {
|
|
25
|
+
if (!target) return null;
|
|
26
|
+
try {
|
|
27
|
+
routeNotification({ type, scope, recipients: [target], message: { subject, body, payload } });
|
|
28
|
+
return target;
|
|
29
|
+
} catch {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
import { getArtifactStorage } from "../artifact-storage";
|
|
2
|
+
import { expireStaleBusAgents } from "../bus";
|
|
3
|
+
import { pruneOutbox } from "../bus-outbox";
|
|
4
|
+
import { expireCommands } from "../commands-db";
|
|
5
|
+
import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
|
|
6
|
+
import {
|
|
7
|
+
createActivityEvent,
|
|
8
|
+
evaluatePoolBindings,
|
|
9
|
+
expireQueuedMessages,
|
|
10
|
+
pruneOrphanedSharedWorkspaces,
|
|
11
|
+
pruneOfflineAgents,
|
|
12
|
+
pruneOldMessages,
|
|
13
|
+
reapStaleAgents,
|
|
14
|
+
reapStaleOrchestrators,
|
|
15
|
+
releaseExpiredClaims,
|
|
16
|
+
releaseOrphanedTasks,
|
|
17
|
+
runDbMaintenance,
|
|
18
|
+
sweepArtifacts,
|
|
19
|
+
} from "../db";
|
|
20
|
+
import { notifyAgentOffline } from "../agent-lifecycle-events";
|
|
21
|
+
import { autoMergeCleanFastForwards } from "../workspace-auto-merge";
|
|
22
|
+
import { reapOrphanedWorktrees } from "../workspace-orphans";
|
|
23
|
+
import { getLifecycleManager } from "../lifecycle-manager";
|
|
24
|
+
import { applyCommandToRecipe } from "../recipe-runner";
|
|
25
|
+
import {
|
|
26
|
+
emitAgentRemoved,
|
|
27
|
+
emitAgentStatus,
|
|
28
|
+
emitMessageClaimReleased,
|
|
29
|
+
emitMessageExpired,
|
|
30
|
+
emitOrchestratorStatus,
|
|
31
|
+
emitPoolBindingChanged,
|
|
32
|
+
emitTaskChanged,
|
|
33
|
+
} from "../sse";
|
|
34
|
+
import { pruneExpiredTokenRecords } from "../token-db";
|
|
35
|
+
import { reapTransientAgents } from "../services/transient-agent-reaper";
|
|
36
|
+
import { resumeRateLimitedAgents } from "../services/rate-limit-resume";
|
|
37
|
+
import { MERGE_PAUSE_MAINTENANCE_JOB } from "../merge-pause-maintenance";
|
|
38
|
+
import {
|
|
39
|
+
DB_MAINTENANCE_INTERVAL_MS,
|
|
40
|
+
DB_VACUUM_EVERY,
|
|
41
|
+
DAY_MS,
|
|
42
|
+
ORPHAN_REAPER_INTERVAL_MS,
|
|
43
|
+
OUTBOX_RETENTION_MS,
|
|
44
|
+
TOKEN_RECORD_RETENTION_SECONDS,
|
|
45
|
+
WORKSPACE_AUTO_MERGE_INTERVAL_MS,
|
|
46
|
+
WORKSPACE_GC_INTERVAL_MS,
|
|
47
|
+
WORKSPACE_ORPHAN_REAPER_INTERVAL_MS,
|
|
48
|
+
CONFLICT_SCAN_INTERVAL_MS,
|
|
49
|
+
teamReapIntervalMs,
|
|
50
|
+
} from "./constants";
|
|
51
|
+
import { emitCommand } from "./events";
|
|
52
|
+
import { reapOrphanedSessions } from "./orphaned-sessions";
|
|
53
|
+
import { reapZeroMemberTeams } from "./teams";
|
|
54
|
+
import { scanWorkspaceConflicts, workspaceGC } from "./workspaces";
|
|
55
|
+
import type { MaintenanceJobDefinition } from "./types";
|
|
56
|
+
|
|
57
|
+
let dbMaintenanceRuns = 0;
|
|
58
|
+
|
|
59
|
+
export const LEGACY_REAPER_JOB_IDS = [
|
|
60
|
+
"claim-cleanup",
|
|
61
|
+
"task-orphan-release",
|
|
62
|
+
"queued-message-expiration",
|
|
63
|
+
"command-expiration",
|
|
64
|
+
"artifact-sweep",
|
|
65
|
+
"stale-agent-reaper",
|
|
66
|
+
"orchestrator-reaper",
|
|
67
|
+
"offline-agent-prune",
|
|
68
|
+
"pool-binding-reconcile",
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
export const definitions: MaintenanceJobDefinition[] = [
|
|
72
|
+
{
|
|
73
|
+
id: "claim-cleanup",
|
|
74
|
+
title: "Claim cleanup",
|
|
75
|
+
description: "Release expired message and task claims.",
|
|
76
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
77
|
+
runOnStart: true,
|
|
78
|
+
handler() {
|
|
79
|
+
const released = releaseExpiredClaims();
|
|
80
|
+
for (const id of released.messageIds) emitMessageClaimReleased(id);
|
|
81
|
+
for (const task of released.tasks) emitTaskChanged(task, "task.updated");
|
|
82
|
+
return {
|
|
83
|
+
releasedMessageIds: released.messageIds,
|
|
84
|
+
releasedTaskIds: released.tasks.map((task) => task.id),
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
MERGE_PAUSE_MAINTENANCE_JOB,
|
|
89
|
+
{
|
|
90
|
+
id: "task-orphan-release",
|
|
91
|
+
title: "Orphaned task release",
|
|
92
|
+
description: "Release active tasks owned by agents that are no longer online.",
|
|
93
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
94
|
+
runOnStart: true,
|
|
95
|
+
handler() {
|
|
96
|
+
const tasks = releaseOrphanedTasks();
|
|
97
|
+
for (const task of tasks) emitTaskChanged(task, "task.updated");
|
|
98
|
+
return { releasedOrphanedTaskIds: tasks.map((task) => task.id) };
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
id: "queued-message-expiration",
|
|
103
|
+
title: "Queued message expiration",
|
|
104
|
+
description: "Expire queued messages that exceeded their max age.",
|
|
105
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
106
|
+
runOnStart: true,
|
|
107
|
+
handler() {
|
|
108
|
+
const messages = expireQueuedMessages();
|
|
109
|
+
for (const message of messages) emitMessageExpired(message);
|
|
110
|
+
return { expiredQueuedMessageIds: messages.map((message) => message.id) };
|
|
111
|
+
},
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
id: "command-expiration",
|
|
115
|
+
title: "Command expiration",
|
|
116
|
+
description: "Timeout active commands past their lease.",
|
|
117
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
118
|
+
runOnStart: true,
|
|
119
|
+
handler() {
|
|
120
|
+
const commands = expireCommands();
|
|
121
|
+
for (const command of commands) {
|
|
122
|
+
applyCommandToRecipe(command);
|
|
123
|
+
emitCommand(command);
|
|
124
|
+
}
|
|
125
|
+
return { expiredCommandIds: commands.map((command) => command.id) };
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
id: "bus-agent-expiration",
|
|
130
|
+
title: "Bus agent expiration",
|
|
131
|
+
description: "Expire stale live bus attachments and release associated orphan work.",
|
|
132
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
133
|
+
runOnStart: true,
|
|
134
|
+
handler() {
|
|
135
|
+
const expired = expireStaleBusAgents();
|
|
136
|
+
for (const task of expired.orphanedTasks) emitTaskChanged(task, "task.updated");
|
|
137
|
+
return {
|
|
138
|
+
expiredBusAgentIds: expired.agentIds,
|
|
139
|
+
releasedOrphanedTaskIds: expired.orphanedTasks.map((task) => task.id),
|
|
140
|
+
};
|
|
141
|
+
},
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
id: "stale-agent-reaper",
|
|
145
|
+
title: "Stale agent reaper",
|
|
146
|
+
description: "Mark agents offline when heartbeats are too old.",
|
|
147
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
148
|
+
runOnStart: true,
|
|
149
|
+
handler() {
|
|
150
|
+
const reapedAgentIds = reapStaleAgents(STALE_TTL_MS);
|
|
151
|
+
for (const id of reapedAgentIds) {
|
|
152
|
+
emitAgentStatus(id);
|
|
153
|
+
getLifecycleManager().onAgentDisappeared(id);
|
|
154
|
+
notifyAgentOffline(id, "heartbeat lost");
|
|
155
|
+
createActivityEvent({
|
|
156
|
+
clientId: "server-agent-" + id + "-heartbeat-lost-" + Date.now(),
|
|
157
|
+
kind: "state",
|
|
158
|
+
title: "Heartbeat lost",
|
|
159
|
+
body: "Agent marked offline",
|
|
160
|
+
meta: id,
|
|
161
|
+
icon: "ti-heart-off",
|
|
162
|
+
view: "chat",
|
|
163
|
+
agentId: id,
|
|
164
|
+
metadata: { source: "server", maintenanceJobId: "stale-agent-reaper" },
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
return { reapedAgentIds };
|
|
168
|
+
},
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
id: "offline-agent-prune",
|
|
172
|
+
title: "Offline agent prune",
|
|
173
|
+
description: "Delete long-offline provider agents and release their claims.",
|
|
174
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
175
|
+
runOnStart: false,
|
|
176
|
+
handler() {
|
|
177
|
+
const prunedAgentIds = pruneOfflineAgents(OFFLINE_PRUNE_MS);
|
|
178
|
+
for (const id of prunedAgentIds) emitAgentRemoved(id);
|
|
179
|
+
return { prunedAgentIds };
|
|
180
|
+
},
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
id: "orphaned-session-reaper",
|
|
184
|
+
title: "Orphaned session reaper",
|
|
185
|
+
description: "Stop spawned sessions whose relay agent is offline/gone but whose process the orchestrator still reports running, after a grace period for self-heal.",
|
|
186
|
+
intervalMs: ORPHAN_REAPER_INTERVAL_MS,
|
|
187
|
+
runOnStart: false,
|
|
188
|
+
handler: reapOrphanedSessions,
|
|
189
|
+
},
|
|
190
|
+
{ id: "transient-agent-reaper", title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: reapTransientAgents },
|
|
191
|
+
{ id: "rate-limit-resume", title: "Rate-limit resume", description: "Auto-resume agents held on a provider usage/rate-limit stall once their reset window has passed (#286).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: resumeRateLimitedAgents },
|
|
192
|
+
{
|
|
193
|
+
id: "team-reap",
|
|
194
|
+
title: "Team lifecycle close-out",
|
|
195
|
+
description: "Dissolve active teams that have had zero live members for the configured grace window.",
|
|
196
|
+
intervalMs: teamReapIntervalMs(),
|
|
197
|
+
runOnStart: false,
|
|
198
|
+
handler: reapZeroMemberTeams,
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
id: "orchestrator-reaper",
|
|
202
|
+
title: "Orchestrator reaper",
|
|
203
|
+
description: "Mark stale orchestrator registrations offline.",
|
|
204
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
205
|
+
runOnStart: true,
|
|
206
|
+
handler() {
|
|
207
|
+
const reapedOrchestratorIds = reapStaleOrchestrators();
|
|
208
|
+
for (const id of reapedOrchestratorIds) emitOrchestratorStatus(id);
|
|
209
|
+
return { reapedOrchestratorIds };
|
|
210
|
+
},
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
id: "pool-binding-reconcile",
|
|
214
|
+
title: "Pool binding reconcile",
|
|
215
|
+
description: "Reassign pool channel bindings when current assignees are unavailable.",
|
|
216
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
217
|
+
runOnStart: true,
|
|
218
|
+
handler() {
|
|
219
|
+
const changes = evaluatePoolBindings();
|
|
220
|
+
for (const change of changes) emitPoolBindingChanged(change.bindingId, change.channelId, change.previousAgentId, change.newAgentId);
|
|
221
|
+
return { poolBindingChanges: changes };
|
|
222
|
+
},
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
id: "old-message-prune",
|
|
226
|
+
title: "Old message prune",
|
|
227
|
+
description: "Delete messages outside the configured retention window.",
|
|
228
|
+
intervalMs: DAY_MS,
|
|
229
|
+
runOnStart: false,
|
|
230
|
+
handler() {
|
|
231
|
+
const retentionDays = Number(process.env.RETENTION_DAYS) || 30;
|
|
232
|
+
return { prunedMessages: pruneOldMessages(retentionDays * DAY_MS) };
|
|
233
|
+
},
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
id: "db-maintenance",
|
|
237
|
+
title: "Database maintenance",
|
|
238
|
+
description: "Refresh planner stats (ANALYZE + optimize), truncate the WAL, and periodically VACUUM to reclaim space.",
|
|
239
|
+
intervalMs: DB_MAINTENANCE_INTERVAL_MS,
|
|
240
|
+
runOnStart: false,
|
|
241
|
+
timeoutMs: 5 * 60 * 1000,
|
|
242
|
+
handler() {
|
|
243
|
+
dbMaintenanceRuns += 1;
|
|
244
|
+
const vacuum = DB_VACUUM_EVERY > 0 && dbMaintenanceRuns % DB_VACUUM_EVERY === 0;
|
|
245
|
+
return runDbMaintenance({ vacuum });
|
|
246
|
+
},
|
|
247
|
+
},
|
|
248
|
+
{
|
|
249
|
+
id: "bus-outbox-prune",
|
|
250
|
+
title: "Bus outbox prune",
|
|
251
|
+
description: "Prune retained bus outbox events after replay retention.",
|
|
252
|
+
intervalMs: 5 * 60 * 1000,
|
|
253
|
+
runOnStart: false,
|
|
254
|
+
handler() {
|
|
255
|
+
return { prunedOutboxEvents: pruneOutbox(OUTBOX_RETENTION_MS) };
|
|
256
|
+
},
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
id: "artifact-sweep",
|
|
260
|
+
title: "Artifact sweep",
|
|
261
|
+
description: "Delete expired or unlinked artifacts and their local blobs.",
|
|
262
|
+
intervalMs: 60 * 60 * 1000,
|
|
263
|
+
runOnStart: false,
|
|
264
|
+
timeoutMs: 2 * 60 * 1000,
|
|
265
|
+
async handler() {
|
|
266
|
+
const swept = sweepArtifacts();
|
|
267
|
+
const failedBlobDeletes: string[] = [];
|
|
268
|
+
for (const blob of swept.blobs) {
|
|
269
|
+
try {
|
|
270
|
+
await getArtifactStorage().delete(blob.storageUri);
|
|
271
|
+
} catch {
|
|
272
|
+
failedBlobDeletes.push(blob.digest);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
expiredArtifactIds: swept.artifactIds,
|
|
277
|
+
sweptBlobDigests: swept.blobs.map((blob) => blob.digest),
|
|
278
|
+
failedBlobDeletes,
|
|
279
|
+
};
|
|
280
|
+
},
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
id: "token-record-prune",
|
|
284
|
+
title: "Token record prune",
|
|
285
|
+
description: "Delete expired or revoked token records after retention.",
|
|
286
|
+
intervalMs: 60 * 60 * 1000,
|
|
287
|
+
runOnStart: false,
|
|
288
|
+
handler() {
|
|
289
|
+
const prunedTokenJtis = pruneExpiredTokenRecords({ retentionSeconds: TOKEN_RECORD_RETENTION_SECONDS });
|
|
290
|
+
return { prunedTokenJtis };
|
|
291
|
+
},
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
id: "workspace-orphan-sweep",
|
|
295
|
+
title: "Workspace orphan sweep",
|
|
296
|
+
description: "Delete shared-mode workspace occupancy rows whose owner agent is offline or gone.",
|
|
297
|
+
intervalMs: REAP_INTERVAL_MS,
|
|
298
|
+
runOnStart: true,
|
|
299
|
+
handler() {
|
|
300
|
+
const prunedSharedWorkspaceIds = pruneOrphanedSharedWorkspaces();
|
|
301
|
+
return { prunedSharedWorkspaceIds };
|
|
302
|
+
},
|
|
303
|
+
},
|
|
304
|
+
{
|
|
305
|
+
id: "workspace-conflict-scan",
|
|
306
|
+
title: "Workspace conflict scan",
|
|
307
|
+
description: "Self-healing backstop (#237): probe worktrees for divergence/freshness and auto-flag conflicts. The turn-end stop-hook drives the common case; this catches dropped events, restarts, and direct human commits.",
|
|
308
|
+
intervalMs: CONFLICT_SCAN_INTERVAL_MS,
|
|
309
|
+
runOnStart: true,
|
|
310
|
+
timeoutMs: 60 * 1000,
|
|
311
|
+
handler: scanWorkspaceConflicts,
|
|
312
|
+
},
|
|
313
|
+
{
|
|
314
|
+
id: "workspace-auto-merge",
|
|
315
|
+
title: "Workspace auto-merge",
|
|
316
|
+
description: "Self-healing backstop (#237): land any non-conflicting ready/review_requested worktree under the per-repo lease (rebasing when base moved on); only real/unknown conflicts go to the steward. The on-ready/on-settle event chain drives the common case; runOnStart reclaims rows stranded across a relay restart.",
|
|
317
|
+
intervalMs: WORKSPACE_AUTO_MERGE_INTERVAL_MS,
|
|
318
|
+
runOnStart: true,
|
|
319
|
+
timeoutMs: 60 * 1000,
|
|
320
|
+
handler: autoMergeCleanFastForwards,
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
id: "workspace-gc",
|
|
324
|
+
title: "Workspace GC",
|
|
325
|
+
description: "Prune terminal workspace rows past retention, auto-abandon stale review_requested worktrees, and trigger git worktree prune on orchestrators.",
|
|
326
|
+
intervalMs: WORKSPACE_GC_INTERVAL_MS,
|
|
327
|
+
runOnStart: false,
|
|
328
|
+
timeoutMs: 60 * 1000,
|
|
329
|
+
handler: workspaceGC,
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
id: "workspace-orphan-reaper",
|
|
333
|
+
title: "Workspace orphan reaper",
|
|
334
|
+
description: "Reconcile disk<->DB: reap orphaned worktrees whose work has landed (or is empty), flag orphans holding un-landed work as needs-attention instead of deleting, and report rows whose worktree vanished. git worktree prune can't do this - it no-ops while the directory still exists.",
|
|
335
|
+
intervalMs: WORKSPACE_ORPHAN_REAPER_INTERVAL_MS,
|
|
336
|
+
runOnStart: true,
|
|
337
|
+
timeoutMs: 2 * 60 * 1000,
|
|
338
|
+
handler: reapOrphanedWorktrees,
|
|
339
|
+
},
|
|
340
|
+
];
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { createCommand } from "../commands-db";
|
|
2
|
+
import { createActivityEvent, getAgent, listOrchestrators } from "../db";
|
|
3
|
+
import { emitRelayEvent } from "../events";
|
|
4
|
+
import { orphanGraceMs, orphanReapCooldownMs, orphanReapEnabled } from "./constants";
|
|
5
|
+
|
|
6
|
+
const orphanTracker = new Map<string, { firstOrphanedAt: number; lastReapAt?: number }>();
|
|
7
|
+
|
|
8
|
+
export function resetOrphanSessionTrackerForTests(): void {
|
|
9
|
+
orphanTracker.clear();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// Stop orphaned spawned sessions: process reported alive by the orchestrator, but
|
|
13
|
+
// the relay agent is offline/pruned and self-heal has had its chance.
|
|
14
|
+
export function reapOrphanedSessions(): Record<string, unknown> {
|
|
15
|
+
const now = Date.now();
|
|
16
|
+
const grace = orphanGraceMs();
|
|
17
|
+
const cooldown = orphanReapCooldownMs();
|
|
18
|
+
const reapEnabled = orphanReapEnabled();
|
|
19
|
+
const seen = new Set<string>();
|
|
20
|
+
const reaped: string[] = [];
|
|
21
|
+
let orphaned = 0;
|
|
22
|
+
|
|
23
|
+
for (const orch of listOrchestrators()) {
|
|
24
|
+
if (orch.status !== "online") continue;
|
|
25
|
+
for (const agent of orch.managedAgents) {
|
|
26
|
+
const sessionId = agent.spawnRequestId || agent.tmuxSession || agent.sessionName || agent.agentId;
|
|
27
|
+
if (!sessionId) continue;
|
|
28
|
+
const key = `${orch.id}:${sessionId}`;
|
|
29
|
+
const relayAgent = agent.agentId ? getAgent(agent.agentId) : null;
|
|
30
|
+
const isOrphan = !relayAgent || relayAgent.status === "offline";
|
|
31
|
+
if (!isOrphan) { orphanTracker.delete(key); continue; }
|
|
32
|
+
seen.add(key);
|
|
33
|
+
orphaned++;
|
|
34
|
+
const entry = orphanTracker.get(key) ?? { firstOrphanedAt: now };
|
|
35
|
+
orphanTracker.set(key, entry);
|
|
36
|
+
if (now - entry.firstOrphanedAt < grace) continue;
|
|
37
|
+
if (!reapEnabled) continue;
|
|
38
|
+
if (entry.lastReapAt && now - entry.lastReapAt < cooldown) continue;
|
|
39
|
+
entry.lastReapAt = now;
|
|
40
|
+
const command = createCommand({
|
|
41
|
+
type: "agent.shutdown",
|
|
42
|
+
source: "system",
|
|
43
|
+
target: orch.agentId,
|
|
44
|
+
correlationId: agent.spawnRequestId,
|
|
45
|
+
params: {
|
|
46
|
+
action: "shutdown",
|
|
47
|
+
agentId: agent.agentId,
|
|
48
|
+
spawnRequestId: agent.spawnRequestId,
|
|
49
|
+
sessionName: agent.sessionName,
|
|
50
|
+
tmuxSession: agent.tmuxSession,
|
|
51
|
+
policyName: agent.policyName,
|
|
52
|
+
graceful: false,
|
|
53
|
+
timeoutMs: 10_000,
|
|
54
|
+
reason: "orphaned-session-reaper",
|
|
55
|
+
requestedBy: "orphaned-session-reaper",
|
|
56
|
+
requestedAt: now,
|
|
57
|
+
orchestratorId: orch.id,
|
|
58
|
+
},
|
|
59
|
+
});
|
|
60
|
+
emitRelayEvent({ type: "command.requested", source: "system", subject: command.id, data: { command } });
|
|
61
|
+
createActivityEvent({
|
|
62
|
+
clientId: `orphaned-session-reaper-${key}-${now}`,
|
|
63
|
+
kind: "state",
|
|
64
|
+
title: "Orphaned session reaped",
|
|
65
|
+
body: `${agent.label ?? agent.agentId ?? sessionId}: process still running on ${orch.id}, but its relay agent has been offline > ${Math.round(grace / 60000)}m and did not self-heal - stopping it`,
|
|
66
|
+
meta: agent.label ?? agent.agentId ?? sessionId,
|
|
67
|
+
icon: "ti-ghost",
|
|
68
|
+
view: "orchestrators",
|
|
69
|
+
agentId: agent.agentId || undefined,
|
|
70
|
+
metadata: {
|
|
71
|
+
source: "server",
|
|
72
|
+
maintenanceJobId: "orphaned-session-reaper",
|
|
73
|
+
orchestratorId: orch.id,
|
|
74
|
+
agentId: agent.agentId,
|
|
75
|
+
spawnRequestId: agent.spawnRequestId,
|
|
76
|
+
tmuxSession: agent.tmuxSession,
|
|
77
|
+
commandId: command.id,
|
|
78
|
+
orphanAgeMs: now - entry.firstOrphanedAt,
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
reaped.push(key);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
for (const key of orphanTracker.keys()) if (!seen.has(key)) orphanTracker.delete(key);
|
|
85
|
+
return { orphaned, reaped, tracked: orphanTracker.size, reapEnabled };
|
|
86
|
+
}
|