agent-relay-server 0.62.2 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +2 -2
- package/public/assets/{MessageBubble-DRrxfjyT.js → MessageBubble-Cl6ufPHw.js} +2 -2
- package/public/assets/MessageBubble-Cl6ufPHw.js.map +1 -0
- package/public/assets/{PlanGraphPanel-Dp2eYazz.js → PlanGraphPanel-BnfjmkQ8.js} +2 -2
- package/public/assets/{PlanGraphPanel-Dp2eYazz.js.map → PlanGraphPanel-BnfjmkQ8.js.map} +1 -1
- package/public/assets/{activity--t0U2yzq.js → activity-aFxvGLtd.js} +2 -2
- package/public/assets/{activity--t0U2yzq.js.map → activity-aFxvGLtd.js.map} +1 -1
- package/public/assets/{agent-profiles-BMtJYzxn.js → agent-profiles-B4mleUDz.js} +2 -2
- package/public/assets/{agent-profiles-BMtJYzxn.js.map → agent-profiles-B4mleUDz.js.map} +1 -1
- package/public/assets/{agents-Ck9AePlc.js → agents-DW5djToQ.js} +2 -2
- package/public/assets/{agents-Ck9AePlc.js.map → agents-DW5djToQ.js.map} +1 -1
- package/public/assets/{analytics-CzfK387L.js → analytics-CRi6thtg.js} +2 -2
- package/public/assets/{analytics-CzfK387L.js.map → analytics-CRi6thtg.js.map} +1 -1
- package/public/assets/{automation-Cgw8vtE8.js → automation-Do4OpauI.js} +2 -2
- package/public/assets/{automation-Cgw8vtE8.js.map → automation-Do4OpauI.js.map} +1 -1
- package/public/assets/{branch-state-badge-DnEEPtOP.js → branch-state-badge-D3W3w7zz.js} +2 -2
- package/public/assets/{branch-state-badge-DnEEPtOP.js.map → branch-state-badge-D3W3w7zz.js.map} +1 -1
- package/public/assets/{channels-Dc3L2MQs.js → channels-BytdR9NS.js} +2 -2
- package/public/assets/{channels-Dc3L2MQs.js.map → channels-BytdR9NS.js.map} +1 -1
- package/public/assets/{chat-Fc6iJ7sa.js → chat-CvaQ2czL.js} +2 -2
- package/public/assets/{chat-Fc6iJ7sa.js.map → chat-CvaQ2czL.js.map} +1 -1
- package/public/assets/{connectors-Dc2ttGWs.js → connectors-BLQDp8Bc.js} +2 -2
- package/public/assets/{connectors-Dc2ttGWs.js.map → connectors-BLQDp8Bc.js.map} +1 -1
- package/public/assets/{display-DzNvOu1j.js → display-B06I5sqa.js} +3 -3
- package/public/assets/display-B06I5sqa.js.map +1 -0
- package/public/assets/{formatted-body-impl-Bg-gx7rG.js → formatted-body-impl-0Lf8e1Ic.js} +2 -2
- package/public/assets/{formatted-body-impl-Bg-gx7rG.js.map → formatted-body-impl-0Lf8e1Ic.js.map} +1 -1
- package/public/assets/{index-BeDncjN4.js → index-C3su8ezq.js} +6 -6
- package/public/assets/{index-BeDncjN4.js.map → index-C3su8ezq.js.map} +1 -1
- package/public/assets/{insights-c08aW-7n.js → insights-64uExH1E.js} +2 -2
- package/public/assets/{insights-c08aW-7n.js.map → insights-64uExH1E.js.map} +1 -1
- package/public/assets/{integrations-BwpgcqnF.js → integrations-HyC1EaIp.js} +2 -2
- package/public/assets/{integrations-BwpgcqnF.js.map → integrations-HyC1EaIp.js.map} +1 -1
- package/public/assets/{maintenance-CTKP0kWB.js → maintenance-EVBpKH07.js} +2 -2
- package/public/assets/{maintenance-CTKP0kWB.js.map → maintenance-EVBpKH07.js.map} +1 -1
- package/public/assets/{managed-agents-BofCxo8d.js → managed-agents-BLZPxGuw.js} +2 -2
- package/public/assets/{managed-agents-BofCxo8d.js.map → managed-agents-BLZPxGuw.js.map} +1 -1
- package/public/assets/{markdown-preview-impl-CQ-6BIb8.js → markdown-preview-impl-Bg6rCWYN.js} +2 -2
- package/public/assets/{markdown-preview-impl-CQ-6BIb8.js.map → markdown-preview-impl-Bg6rCWYN.js.map} +1 -1
- package/public/assets/{memory-DvfXjMoi.js → memory-Bjr8bgjj.js} +2 -2
- package/public/assets/{memory-DvfXjMoi.js.map → memory-Bjr8bgjj.js.map} +1 -1
- package/public/assets/{messages-C9VsPzZ_.js → messages-CRYw1F4z.js} +2 -2
- package/public/assets/{messages-C9VsPzZ_.js.map → messages-CRYw1F4z.js.map} +1 -1
- package/public/assets/{orchestrators-MffVueeE.js → orchestrators-D-co1vdp.js} +2 -2
- package/public/assets/{orchestrators-MffVueeE.js.map → orchestrators-D-co1vdp.js.map} +1 -1
- package/public/assets/{overview-BSVajefM.js → overview-Ddy5czri.js} +2 -2
- package/public/assets/{overview-BSVajefM.js.map → overview-Ddy5czri.js.map} +1 -1
- package/public/assets/{pairs-B5711-Cs.js → pairs-CFVM8uHL.js} +2 -2
- package/public/assets/{pairs-B5711-Cs.js.map → pairs-CFVM8uHL.js.map} +1 -1
- package/public/assets/{security-DkUwK9Vj.js → security-Df4_76bh.js} +2 -2
- package/public/assets/{security-DkUwK9Vj.js.map → security-Df4_76bh.js.map} +1 -1
- package/public/assets/{select-DrM2RRiT.js → select-Cq3BBnL9.js} +2 -2
- package/public/assets/{select-DrM2RRiT.js.map → select-Cq3BBnL9.js.map} +1 -1
- package/public/assets/{settings-DdKCb-vy.js → settings-DOLEYXZL.js} +2 -2
- package/public/assets/{settings-DdKCb-vy.js.map → settings-DOLEYXZL.js.map} +1 -1
- package/public/assets/{store-iTiheuTD.js → store-C1E8jmNE.js} +4 -4
- package/public/assets/store-C1E8jmNE.js.map +1 -0
- package/public/assets/{tasks-e1y1PFfO.js → tasks-BcsmPi9_.js} +2 -2
- package/public/assets/{tasks-e1y1PFfO.js.map → tasks-BcsmPi9_.js.map} +1 -1
- package/public/assets/{teams-BUcxoGC_.js → teams-BQrV3eKI.js} +2 -2
- package/public/assets/{teams-BUcxoGC_.js.map → teams-BQrV3eKI.js.map} +1 -1
- package/public/assets/{terminal-viewer-impl-CmZjQOB2.js → terminal-viewer-impl-Dle04qpH.js} +2 -2
- package/public/assets/{terminal-viewer-impl-CmZjQOB2.js.map → terminal-viewer-impl-Dle04qpH.js.map} +1 -1
- package/public/assets/{work-queue-CXBQx43n.js → work-queue-BLhTLg4o.js} +2 -2
- package/public/assets/{work-queue-CXBQx43n.js.map → work-queue-BLhTLg4o.js.map} +1 -1
- package/public/assets/{workspaces-D2EfZh9a.js → workspaces-4NCNTmRQ.js} +3 -3
- package/public/assets/{workspaces-D2EfZh9a.js.map → workspaces-4NCNTmRQ.js.map} +1 -1
- package/public/index.html +3 -3
- package/scripts/orchestrator-spawn-smoke.ts +87 -67
- package/src/agent-ref.ts +8 -6
- package/src/automations/constants.ts +12 -0
- package/src/automations/cron.ts +129 -0
- package/src/automations/crud.ts +84 -0
- package/src/automations/dispatch.ts +275 -0
- package/src/automations/index.ts +5 -0
- package/src/automations/reconcile.ts +245 -0
- package/src/automations/storage.ts +181 -0
- package/src/automations/types.ts +16 -0
- package/src/automations/validation.ts +140 -0
- package/src/automations.ts +1 -1045
- package/src/db/activity.ts +1 -1
- package/src/db/agent-predicates.ts +9 -0
- package/src/db/agent-search.ts +2 -2
- package/src/db/agents.ts +13 -41
- package/src/db/artifacts.ts +8 -8
- package/src/db/channel-identity.ts +38 -0
- package/src/db/channels.ts +37 -61
- package/src/db/continuation-archives.ts +4 -4
- package/src/db/delivery.ts +75 -30
- package/src/db/inbox.ts +1 -1
- package/src/db/integrations.ts +4 -4
- package/src/db/issue-lifecycle.ts +6 -6
- package/src/db/mappers.ts +1 -2
- package/src/db/merge-lease.ts +3 -3
- package/src/db/message-claims.ts +16 -0
- package/src/db/message-queries.ts +8 -0
- package/src/db/message-reads.ts +157 -18
- package/src/db/messages.ts +23 -48
- package/src/db/notification-subscriptions.ts +2 -2
- package/src/db/orchestrators.ts +4 -4
- package/src/db/pair-maintenance.ts +9 -0
- package/src/db/pairs.ts +10 -18
- package/src/db/plans.ts +1 -1
- package/src/db/policy-runtime.ts +14 -0
- package/src/db/project-entries.ts +1 -1
- package/src/db/projects.ts +2 -2
- package/src/db/shared.ts +7 -0
- package/src/db/stats.ts +2 -2
- package/src/db/task-core.ts +18 -0
- package/src/db/task-maintenance.ts +24 -0
- package/src/db/tasks.ts +5 -17
- package/src/db/teams.ts +3 -3
- package/src/db/workspaces.ts +6 -6
- package/src/issue-tracker/github.ts +3 -3
- package/src/maintenance/constants.ts +35 -0
- package/src/maintenance/events.ts +32 -0
- package/src/maintenance/index.ts +7 -0
- package/src/maintenance/jobs.ts +340 -0
- package/src/maintenance/orphaned-sessions.ts +86 -0
- package/src/maintenance/scheduler.ts +231 -0
- package/src/maintenance/teams.ts +65 -0
- package/src/maintenance/types.ts +31 -0
- package/src/maintenance/workspaces.ts +251 -0
- package/src/maintenance.ts +1 -1099
- package/src/mcp/identity.ts +131 -0
- package/src/mcp/index.ts +208 -0
- package/src/mcp/tools-messaging.ts +459 -0
- package/src/mcp/tools-spawn.ts +145 -0
- package/src/mcp/tools-workspace.ts +227 -0
- package/src/mcp/types.ts +24 -0
- package/src/mcp/validation.ts +104 -0
- package/src/mcp.ts +1 -1319
- package/src/notification-router.ts +2 -2
- package/src/notification-rules.ts +2 -2
- package/src/project-ingest.ts +3 -3
- package/src/project-instructions.ts +1 -1
- package/src/provider-config-store.ts +3 -3
- package/src/provider-state.ts +1 -1
- package/src/reviewer-pipeline.ts +1 -1
- package/src/self-resume.ts +3 -7
- package/src/services/agent-action.ts +2 -2
- package/src/services/issue-activity.ts +0 -4
- package/src/services/issue-associations.ts +2 -2
- package/src/services/issue-lifecycle.ts +7 -7
- package/src/services/managed-running.ts +1 -1
- package/src/services/parity-harness.ts +3 -3
- package/src/services/plan-graph.ts +3 -3
- package/src/services/send-message.ts +16 -12
- package/src/services/shutdown-agent.ts +2 -2
- package/src/services/team-templates.ts +2 -2
- package/src/services/team.ts +2 -2
- package/src/settings/modules.ts +1 -1
- package/src/settings/registry.ts +3 -3
- package/src/spawn-command.ts +1 -1
- package/src/sse.ts +0 -4
- package/src/workspace-actions.ts +1 -1
- package/src/workspace-probe.ts +3 -7
- package/public/assets/MessageBubble-DRrxfjyT.js.map +0 -1
- package/public/assets/display-DzNvOu1j.js.map +0 -1
- package/public/assets/store-iTiheuTD.js.map +0 -1
package/src/maintenance.ts
CHANGED
|
@@ -1,1099 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { getArtifactStorage } from "./artifact-storage";
|
|
3
|
-
import { expireStaleBusAgents } from "./bus";
|
|
4
|
-
import { pruneOutbox } from "./bus-outbox";
|
|
5
|
-
import { createCommand, expireCommands } from "./commands-db";
|
|
6
|
-
import { DAY_MS, OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "./config";
|
|
7
|
-
import { resolve } from "node:path";
|
|
8
|
-
import {
|
|
9
|
-
createActivityEvent,
|
|
10
|
-
clearTeamZeroMembersSince,
|
|
11
|
-
countLiveTeamMembers,
|
|
12
|
-
evaluatePoolBindings,
|
|
13
|
-
expireQueuedMessages,
|
|
14
|
-
getAgent,
|
|
15
|
-
getDb,
|
|
16
|
-
getRepoSteward,
|
|
17
|
-
getWorkspace,
|
|
18
|
-
listActiveTeams,
|
|
19
|
-
listOrchestrators,
|
|
20
|
-
listWorkspaces,
|
|
21
|
-
patchWorkspaceMetadata,
|
|
22
|
-
pruneOfflineAgents,
|
|
23
|
-
pruneOldMessages,
|
|
24
|
-
deleteWorkspace,
|
|
25
|
-
pruneOrphanedSharedWorkspaces,
|
|
26
|
-
reapStaleAgents,
|
|
27
|
-
reapStaleOrchestrators,
|
|
28
|
-
reelectRepoSteward,
|
|
29
|
-
releaseExpiredClaims,
|
|
30
|
-
releaseExpiredMergeLeases,
|
|
31
|
-
releaseOrphanedTasks,
|
|
32
|
-
runDbMaintenance,
|
|
33
|
-
setTeamZeroMembersSince,
|
|
34
|
-
sweepArtifacts,
|
|
35
|
-
updateWorkspaceStatus,
|
|
36
|
-
} from "./db";
|
|
37
|
-
import type { WorkspaceMergePreview, WorkspaceRecord, WorkspaceStatus } from "./types";
|
|
38
|
-
import { reconcileLandedWorkspace } from "./branch-landed";
|
|
39
|
-
import { preparePrCompletionScan, reconcileLandedPr } from "./workspace-pr-completion";
|
|
40
|
-
import { notifyAgentOffline } from "./agent-lifecycle-events";
|
|
41
|
-
import { reapOrphanedWorktrees } from "./workspace-orphans";
|
|
42
|
-
import { deriveBranchState, READY_TO_LAND_STATUSES, TERMINAL_WORKSPACE_STATUSES } from "./workspace-phase";
|
|
43
|
-
import { applyBranchFreshness, fetchHostMergePreview, workspacePathWithinBase } from "./workspace-probe";
|
|
44
|
-
import { autoMergeCleanFastForwards, wakeRepoSteward } from "./workspace-auto-merge";
|
|
45
|
-
import { errMessage } from "agent-relay-sdk";
|
|
46
|
-
import { getStewardConfig } from "./config-store";
|
|
47
|
-
import { emitRelayEvent } from "./events";
|
|
48
|
-
import { getLifecycleManager } from "./lifecycle-manager";
|
|
49
|
-
import { applyCommandToRecipe } from "./recipe-runner";
|
|
50
|
-
import {
|
|
51
|
-
emitAgentRemoved,
|
|
52
|
-
emitAgentStatus,
|
|
53
|
-
emitMessageClaimReleased,
|
|
54
|
-
emitMessageExpired,
|
|
55
|
-
emitOrchestratorStatus,
|
|
56
|
-
emitPoolBindingChanged,
|
|
57
|
-
emitTaskChanged,
|
|
58
|
-
} from "./sse";
|
|
59
|
-
import { routeNotification, type NotificationType, type NotificationScope } from "./notification-router";
|
|
60
|
-
import { pruneExpiredTokenRecords } from "./token-db";
|
|
61
|
-
import { dissolveTeamById } from "./services/team";
|
|
62
|
-
import { reapTransientAgents, resetTransientTrackerForTests } from "./services/transient-agent-reaper";
|
|
63
|
-
import { resumeRateLimitedAgents, resetRateLimitResumeTrackerForTests } from "./services/rate-limit-resume";
|
|
64
|
-
import { MERGE_PAUSE_MAINTENANCE_JOB } from "./merge-pause-maintenance";
|
|
65
|
-
import type { Command, MaintenanceJob, MaintenanceJobRun } from "./types";
|
|
66
|
-
|
|
67
|
-
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
68
|
-
const SCHEDULER_TICK_MS = 10_000;
|
|
69
|
-
const OUTBOX_RETENTION_MS = Number(process.env.AGENT_RELAY_OUTBOX_RETENTION_MS) || 60 * 60 * 1000;
|
|
70
|
-
const TOKEN_RECORD_RETENTION_SECONDS = Number(process.env.AGENT_RELAY_TOKEN_RECORD_RETENTION_SECONDS) || 7 * 24 * 60 * 60;
|
|
71
|
-
// #237: demoted from the 2-min PRIMARY merge-back driver to a slow self-healing
|
|
72
|
-
// backstop now that the work is event-driven (stop-hook freshness + on-ready/on-settle
|
|
73
|
-
// land). ~10 min + runOnStart on both the conflict scan and the auto-merge job: they
|
|
74
|
-
// almost never have work because events did it in seconds; they exist to catch dropped
|
|
75
|
-
// events, relay restarts, and direct human commits with no agent turn.
|
|
76
|
-
const CONFLICT_SCAN_INTERVAL_MS = Number(process.env.AGENT_RELAY_CONFLICT_SCAN_INTERVAL_MS) || 10 * 60 * 1000;
|
|
77
|
-
const WORKSPACE_RETENTION_MS = Number(process.env.AGENT_RELAY_WORKSPACE_RETENTION_MS) || DAY_MS;
|
|
78
|
-
const DB_MAINTENANCE_INTERVAL_MS = Number(process.env.AGENT_RELAY_DB_MAINTENANCE_INTERVAL_MS) || DAY_MS;
|
|
79
|
-
// VACUUM rewrites the whole file (auto_vacuum is off), so run it sparingly.
|
|
80
|
-
// Default: every 7th db-maintenance pass (~weekly at the default daily interval).
|
|
81
|
-
const DB_VACUUM_EVERY = Number(process.env.AGENT_RELAY_DB_VACUUM_EVERY) || 7;
|
|
82
|
-
let dbMaintenanceRuns = 0;
|
|
83
|
-
const WORKSPACE_REVIEW_TTL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_REVIEW_TTL_MS) || 3 * DAY_MS;
|
|
84
|
-
const WORKSPACE_GC_INTERVAL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_GC_INTERVAL_MS) || 60 * 60 * 1000;
|
|
85
|
-
// Disk⇄DB orphan reconcile cadence (#244). Runs on start for a boot-time pass,
|
|
86
|
-
// then periodically — orphans accrue slowly (one per crashed/killed session), so
|
|
87
|
-
// a 30-min sweep is plenty without hammering the hosts with probes.
|
|
88
|
-
const WORKSPACE_ORPHAN_REAPER_INTERVAL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_ORPHAN_REAPER_INTERVAL_MS) || 30 * 60 * 1000;
|
|
89
|
-
// Deterministic auto-land (Layer 0): merge clean fast-forwards with no human in
|
|
90
|
-
// the loop. Default on for the seamless workflow; set AGENT_RELAY_WORKSPACE_AUTO_MERGE=0
|
|
91
|
-
// to require a manual or steward merge per repo. Read at call-time so operators can
|
|
92
|
-
// toggle it without a restart.
|
|
93
|
-
const WORKSPACE_AUTO_MERGE_INTERVAL_MS = Number(process.env.AGENT_RELAY_WORKSPACE_AUTO_MERGE_INTERVAL_MS) || CONFLICT_SCAN_INTERVAL_MS;
|
|
94
|
-
// How long a stranded review_requested/conflict worktree (no online steward) may
|
|
95
|
-
// sit before escalating to the configured fallback target, and the durable
|
|
96
|
-
// escalation target itself (`policy:<name>`, `label:<name>`, `cap:<name>`, an
|
|
97
|
-
// agent id, or `broadcast`). Read at call-time so config changes take effect
|
|
98
|
-
// without a restart (issue #157).
|
|
99
|
-
const stewardEscalationMs = () => Number(process.env.AGENT_RELAY_WORKSPACE_STEWARD_ESCALATION_MS) || 60 * 60 * 1000;
|
|
100
|
-
const stewardFallbackTarget = () => (process.env.AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK || "").trim();
|
|
101
|
-
// Statuses that need an owner — a stranded one of these is what escalation rescues.
|
|
102
|
-
// Derived from the shared ready-to-land set (#242) plus `conflict`, so a stranded
|
|
103
|
-
// `ready` worktree (no online steward) escalates to the fallback target instead of
|
|
104
|
-
// rotting silently — same gap that left the original #242 branch parked.
|
|
105
|
-
const STRANDABLE_STATUSES = new Set<WorkspaceStatus>([...READY_TO_LAND_STATUSES, "conflict"]);
|
|
106
|
-
// Live statuses worth scanning. Terminal (cleaned/merged/abandoned) and
|
|
107
|
-
// in-flight (cleanup_requested) states are skipped.
|
|
108
|
-
const CONFLICT_SCAN_STATUSES = new Set<WorkspaceStatus>(["active", "ready", "review_requested", "merge_planned", "conflict"]);
|
|
109
|
-
// In-flight merge statuses that should reconcile to `merged` once the host
|
|
110
|
-
// reports the branch's work has landed in base (squash/cherry-pick, or a merged
|
|
111
|
-
// PR). Excludes active/ready: an agent still working may have landed an early
|
|
112
|
-
// commit while more work is in flight — don't yank its workspace out from under it.
|
|
113
|
-
const LANDED_RECONCILE_STATUSES = new Set<WorkspaceStatus>(["merge_planned", "review_requested", "conflict"]);
|
|
114
|
-
|
|
115
|
-
// Orphaned-session reaper. A spawned agent's process can outlive its relay
|
|
116
|
-
// presence: the relay agent goes offline/pruned but the orchestrator still
|
|
117
|
-
// reports the session's process running, so it lingers forever (visible under the
|
|
118
|
-
// orchestrator, gone from the Agents panel). Runtime-token self-heal recovers the
|
|
119
|
-
// recoverable ones; this is the backstop that stops the genuinely stuck ones.
|
|
120
|
-
// Conservative by design — a session must be observed continuously orphaned by
|
|
121
|
-
// THIS relay for the grace window before it is reaped, and the tracker is in-memory
|
|
122
|
-
// so a relay restart restarts the clock (giving self-heal first crack every time).
|
|
123
|
-
const ORPHAN_REAPER_INTERVAL_MS = Number(process.env.AGENT_RELAY_ORPHAN_REAPER_INTERVAL_MS) || 5 * 60 * 1000;
|
|
124
|
-
// Read at call-time so changes take effect without a restart (and so tests can tune
|
|
125
|
-
// them). Parsed to allow an explicit 0 (immediate) — `|| default` would reject it.
|
|
126
|
-
const envMsOrDefault = (name: string, fallback: number): number => {
|
|
127
|
-
const v = Number(process.env[name]);
|
|
128
|
-
return Number.isFinite(v) && v >= 0 ? v : fallback;
|
|
129
|
-
};
|
|
130
|
-
const orphanGraceMs = () => envMsOrDefault("AGENT_RELAY_ORPHAN_GRACE_MS", 30 * 60 * 1000);
|
|
131
|
-
const orphanReapCooldownMs = () => envMsOrDefault("AGENT_RELAY_ORPHAN_REAP_COOLDOWN_MS", 5 * 60 * 1000);
|
|
132
|
-
const teamReapGraceMs = () => envMsOrDefault("TEAM_REAP_GRACE_MS", 30 * 60 * 1000);
|
|
133
|
-
const teamReapIntervalMs = () => envMsOrDefault("TEAM_REAP_INTERVAL_MS", 5 * 60 * 1000);
|
|
134
|
-
// Set AGENT_RELAY_ORPHAN_REAP=0 to detect + log orphans but never stop them.
|
|
135
|
-
const orphanReapEnabled = () => process.env.AGENT_RELAY_ORPHAN_REAP !== "0";
|
|
136
|
-
// orchestratorId + session identity -> when we first saw it orphaned (and last reaped).
|
|
137
|
-
const orphanTracker = new Map<string, { firstOrphanedAt: number; lastReapAt?: number }>();
|
|
138
|
-
|
|
139
|
-
export function resetOrphanTrackerForTests(): void { orphanTracker.clear(); resetTransientTrackerForTests(); resetRateLimitResumeTrackerForTests(); }
|
|
140
|
-
|
|
141
|
-
interface MaintenanceJobDefinition {
|
|
142
|
-
id: string;
|
|
143
|
-
title: string;
|
|
144
|
-
description: string;
|
|
145
|
-
intervalMs: number;
|
|
146
|
-
timeoutMs?: number;
|
|
147
|
-
runOnStart?: boolean;
|
|
148
|
-
handler: () => Promise<Record<string, unknown>> | Record<string, unknown>;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
interface MaintenanceJobRow {
|
|
152
|
-
id: string;
|
|
153
|
-
title: string;
|
|
154
|
-
description: string | null;
|
|
155
|
-
interval_ms: number;
|
|
156
|
-
timeout_ms: number;
|
|
157
|
-
enabled: number;
|
|
158
|
-
run_on_start: number;
|
|
159
|
-
last_run_at: number | null;
|
|
160
|
-
next_run_at: number | null;
|
|
161
|
-
last_duration_ms: number | null;
|
|
162
|
-
last_status: string;
|
|
163
|
-
last_error: string | null;
|
|
164
|
-
last_result: string | null;
|
|
165
|
-
consecutive_failures: number;
|
|
166
|
-
lease_owner: string | null;
|
|
167
|
-
lease_until: number | null;
|
|
168
|
-
updated_at: number;
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
const LEGACY_REAPER_JOB_IDS = [
|
|
172
|
-
"claim-cleanup",
|
|
173
|
-
"task-orphan-release",
|
|
174
|
-
"queued-message-expiration",
|
|
175
|
-
"command-expiration",
|
|
176
|
-
"artifact-sweep",
|
|
177
|
-
"stale-agent-reaper",
|
|
178
|
-
"orchestrator-reaper",
|
|
179
|
-
"offline-agent-prune",
|
|
180
|
-
"pool-binding-reconcile",
|
|
181
|
-
];
|
|
182
|
-
|
|
183
|
-
const definitions: MaintenanceJobDefinition[] = [
|
|
184
|
-
{
|
|
185
|
-
id: "claim-cleanup",
|
|
186
|
-
title: "Claim cleanup",
|
|
187
|
-
description: "Release expired message and task claims.",
|
|
188
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
189
|
-
runOnStart: true,
|
|
190
|
-
handler() {
|
|
191
|
-
const released = releaseExpiredClaims();
|
|
192
|
-
for (const id of released.messageIds) emitMessageClaimReleased(id);
|
|
193
|
-
for (const task of released.tasks) emitTaskChanged(task, "task.updated");
|
|
194
|
-
return {
|
|
195
|
-
releasedMessageIds: released.messageIds,
|
|
196
|
-
releasedTaskIds: released.tasks.map((task) => task.id),
|
|
197
|
-
};
|
|
198
|
-
},
|
|
199
|
-
},
|
|
200
|
-
MERGE_PAUSE_MAINTENANCE_JOB,
|
|
201
|
-
{
|
|
202
|
-
id: "task-orphan-release",
|
|
203
|
-
title: "Orphaned task release",
|
|
204
|
-
description: "Release active tasks owned by agents that are no longer online.",
|
|
205
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
206
|
-
runOnStart: true,
|
|
207
|
-
handler() {
|
|
208
|
-
const tasks = releaseOrphanedTasks();
|
|
209
|
-
for (const task of tasks) emitTaskChanged(task, "task.updated");
|
|
210
|
-
return { releasedOrphanedTaskIds: tasks.map((task) => task.id) };
|
|
211
|
-
},
|
|
212
|
-
},
|
|
213
|
-
{
|
|
214
|
-
id: "queued-message-expiration",
|
|
215
|
-
title: "Queued message expiration",
|
|
216
|
-
description: "Expire queued messages that exceeded their max age.",
|
|
217
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
218
|
-
runOnStart: true,
|
|
219
|
-
handler() {
|
|
220
|
-
const messages = expireQueuedMessages();
|
|
221
|
-
for (const message of messages) emitMessageExpired(message);
|
|
222
|
-
return { expiredQueuedMessageIds: messages.map((message) => message.id) };
|
|
223
|
-
},
|
|
224
|
-
},
|
|
225
|
-
{
|
|
226
|
-
id: "command-expiration",
|
|
227
|
-
title: "Command expiration",
|
|
228
|
-
description: "Timeout active commands past their lease.",
|
|
229
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
230
|
-
runOnStart: true,
|
|
231
|
-
handler() {
|
|
232
|
-
const commands = expireCommands();
|
|
233
|
-
for (const command of commands) {
|
|
234
|
-
applyCommandToRecipe(command);
|
|
235
|
-
emitCommand(command);
|
|
236
|
-
}
|
|
237
|
-
return { expiredCommandIds: commands.map((command) => command.id) };
|
|
238
|
-
},
|
|
239
|
-
},
|
|
240
|
-
{
|
|
241
|
-
id: "bus-agent-expiration",
|
|
242
|
-
title: "Bus agent expiration",
|
|
243
|
-
description: "Expire stale live bus attachments and release associated orphan work.",
|
|
244
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
245
|
-
runOnStart: true,
|
|
246
|
-
handler() {
|
|
247
|
-
const expired = expireStaleBusAgents();
|
|
248
|
-
for (const task of expired.orphanedTasks) emitTaskChanged(task, "task.updated");
|
|
249
|
-
return {
|
|
250
|
-
expiredBusAgentIds: expired.agentIds,
|
|
251
|
-
releasedOrphanedTaskIds: expired.orphanedTasks.map((task) => task.id),
|
|
252
|
-
};
|
|
253
|
-
},
|
|
254
|
-
},
|
|
255
|
-
{
|
|
256
|
-
id: "stale-agent-reaper",
|
|
257
|
-
title: "Stale agent reaper",
|
|
258
|
-
description: "Mark agents offline when heartbeats are too old.",
|
|
259
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
260
|
-
runOnStart: true,
|
|
261
|
-
handler() {
|
|
262
|
-
const reapedAgentIds = reapStaleAgents(STALE_TTL_MS);
|
|
263
|
-
for (const id of reapedAgentIds) {
|
|
264
|
-
emitAgentStatus(id);
|
|
265
|
-
getLifecycleManager().onAgentDisappeared(id);
|
|
266
|
-
// #308 — if a reaped agent was a spawned child, wake its parent: exited (it had become
|
|
267
|
-
// ready) vs spawn_failed (it never did — crash-loop / onboarding gate). No-op otherwise.
|
|
268
|
-
notifyAgentOffline(id, "heartbeat lost");
|
|
269
|
-
createActivityEvent({
|
|
270
|
-
clientId: "server-agent-" + id + "-heartbeat-lost-" + Date.now(),
|
|
271
|
-
kind: "state",
|
|
272
|
-
title: "Heartbeat lost",
|
|
273
|
-
body: "Agent marked offline",
|
|
274
|
-
meta: id,
|
|
275
|
-
icon: "ti-heart-off",
|
|
276
|
-
view: "chat",
|
|
277
|
-
agentId: id,
|
|
278
|
-
metadata: { source: "server", maintenanceJobId: "stale-agent-reaper" },
|
|
279
|
-
});
|
|
280
|
-
}
|
|
281
|
-
return { reapedAgentIds };
|
|
282
|
-
},
|
|
283
|
-
},
|
|
284
|
-
{
|
|
285
|
-
id: "offline-agent-prune",
|
|
286
|
-
title: "Offline agent prune",
|
|
287
|
-
description: "Delete long-offline provider agents and release their claims.",
|
|
288
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
289
|
-
runOnStart: false,
|
|
290
|
-
handler() {
|
|
291
|
-
const prunedAgentIds = pruneOfflineAgents(OFFLINE_PRUNE_MS);
|
|
292
|
-
for (const id of prunedAgentIds) emitAgentRemoved(id);
|
|
293
|
-
return { prunedAgentIds };
|
|
294
|
-
},
|
|
295
|
-
},
|
|
296
|
-
{
|
|
297
|
-
id: "orphaned-session-reaper",
|
|
298
|
-
title: "Orphaned session reaper",
|
|
299
|
-
description: "Stop spawned sessions whose relay agent is offline/gone but whose process the orchestrator still reports running, after a grace period for self-heal.",
|
|
300
|
-
intervalMs: ORPHAN_REAPER_INTERVAL_MS,
|
|
301
|
-
runOnStart: false,
|
|
302
|
-
handler: reapOrphanedSessions,
|
|
303
|
-
},
|
|
304
|
-
{ id: "transient-agent-reaper", title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: reapTransientAgents },
|
|
305
|
-
{ id: "rate-limit-resume", title: "Rate-limit resume", description: "Auto-resume agents held on a provider usage/rate-limit stall once their reset window has passed (#286).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: resumeRateLimitedAgents },
|
|
306
|
-
{
|
|
307
|
-
id: "team-reap",
|
|
308
|
-
title: "Team lifecycle close-out",
|
|
309
|
-
description: "Dissolve active teams that have had zero live members for the configured grace window.",
|
|
310
|
-
intervalMs: teamReapIntervalMs(),
|
|
311
|
-
runOnStart: false,
|
|
312
|
-
handler: reapZeroMemberTeams,
|
|
313
|
-
},
|
|
314
|
-
{
|
|
315
|
-
id: "orchestrator-reaper",
|
|
316
|
-
title: "Orchestrator reaper",
|
|
317
|
-
description: "Mark stale orchestrator registrations offline.",
|
|
318
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
319
|
-
runOnStart: true,
|
|
320
|
-
handler() {
|
|
321
|
-
const reapedOrchestratorIds = reapStaleOrchestrators();
|
|
322
|
-
for (const id of reapedOrchestratorIds) emitOrchestratorStatus(id);
|
|
323
|
-
return { reapedOrchestratorIds };
|
|
324
|
-
},
|
|
325
|
-
},
|
|
326
|
-
{
|
|
327
|
-
id: "pool-binding-reconcile",
|
|
328
|
-
title: "Pool binding reconcile",
|
|
329
|
-
description: "Reassign pool channel bindings when current assignees are unavailable.",
|
|
330
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
331
|
-
runOnStart: true,
|
|
332
|
-
handler() {
|
|
333
|
-
const changes = evaluatePoolBindings();
|
|
334
|
-
for (const change of changes) emitPoolBindingChanged(change.bindingId, change.channelId, change.previousAgentId, change.newAgentId);
|
|
335
|
-
return { poolBindingChanges: changes };
|
|
336
|
-
},
|
|
337
|
-
},
|
|
338
|
-
{
|
|
339
|
-
id: "old-message-prune",
|
|
340
|
-
title: "Old message prune",
|
|
341
|
-
description: "Delete messages outside the configured retention window.",
|
|
342
|
-
intervalMs: DAY_MS,
|
|
343
|
-
runOnStart: false,
|
|
344
|
-
handler() {
|
|
345
|
-
const retentionDays = Number(process.env.RETENTION_DAYS) || 30;
|
|
346
|
-
return { prunedMessages: pruneOldMessages(retentionDays * DAY_MS) };
|
|
347
|
-
},
|
|
348
|
-
},
|
|
349
|
-
{
|
|
350
|
-
id: "db-maintenance",
|
|
351
|
-
title: "Database maintenance",
|
|
352
|
-
description: "Refresh planner stats (ANALYZE + optimize), truncate the WAL, and periodically VACUUM to reclaim space.",
|
|
353
|
-
intervalMs: DB_MAINTENANCE_INTERVAL_MS,
|
|
354
|
-
runOnStart: false,
|
|
355
|
-
timeoutMs: 5 * 60 * 1000,
|
|
356
|
-
handler() {
|
|
357
|
-
dbMaintenanceRuns += 1;
|
|
358
|
-
const vacuum = DB_VACUUM_EVERY > 0 && dbMaintenanceRuns % DB_VACUUM_EVERY === 0;
|
|
359
|
-
return runDbMaintenance({ vacuum });
|
|
360
|
-
},
|
|
361
|
-
},
|
|
362
|
-
{
|
|
363
|
-
id: "bus-outbox-prune",
|
|
364
|
-
title: "Bus outbox prune",
|
|
365
|
-
description: "Prune retained bus outbox events after replay retention.",
|
|
366
|
-
intervalMs: 5 * 60 * 1000,
|
|
367
|
-
runOnStart: false,
|
|
368
|
-
handler() {
|
|
369
|
-
return { prunedOutboxEvents: pruneOutbox(OUTBOX_RETENTION_MS) };
|
|
370
|
-
},
|
|
371
|
-
},
|
|
372
|
-
{
|
|
373
|
-
id: "artifact-sweep",
|
|
374
|
-
title: "Artifact sweep",
|
|
375
|
-
description: "Delete expired or unlinked artifacts and their local blobs.",
|
|
376
|
-
intervalMs: 60 * 60 * 1000,
|
|
377
|
-
runOnStart: false,
|
|
378
|
-
timeoutMs: 2 * 60 * 1000,
|
|
379
|
-
async handler() {
|
|
380
|
-
const swept = sweepArtifacts();
|
|
381
|
-
const failedBlobDeletes: string[] = [];
|
|
382
|
-
for (const blob of swept.blobs) {
|
|
383
|
-
try {
|
|
384
|
-
await getArtifactStorage().delete(blob.storageUri);
|
|
385
|
-
} catch {
|
|
386
|
-
failedBlobDeletes.push(blob.digest);
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
return {
|
|
390
|
-
expiredArtifactIds: swept.artifactIds,
|
|
391
|
-
sweptBlobDigests: swept.blobs.map((blob) => blob.digest),
|
|
392
|
-
failedBlobDeletes,
|
|
393
|
-
};
|
|
394
|
-
},
|
|
395
|
-
},
|
|
396
|
-
{
|
|
397
|
-
id: "token-record-prune",
|
|
398
|
-
title: "Token record prune",
|
|
399
|
-
description: "Delete expired or revoked token records after retention.",
|
|
400
|
-
intervalMs: 60 * 60 * 1000,
|
|
401
|
-
runOnStart: false,
|
|
402
|
-
handler() {
|
|
403
|
-
const prunedTokenJtis = pruneExpiredTokenRecords({ retentionSeconds: TOKEN_RECORD_RETENTION_SECONDS });
|
|
404
|
-
return { prunedTokenJtis };
|
|
405
|
-
},
|
|
406
|
-
},
|
|
407
|
-
{
|
|
408
|
-
id: "workspace-orphan-sweep",
|
|
409
|
-
title: "Workspace orphan sweep",
|
|
410
|
-
description: "Delete shared-mode workspace occupancy rows whose owner agent is offline or gone.",
|
|
411
|
-
intervalMs: REAP_INTERVAL_MS,
|
|
412
|
-
runOnStart: true,
|
|
413
|
-
handler() {
|
|
414
|
-
const prunedSharedWorkspaceIds = pruneOrphanedSharedWorkspaces();
|
|
415
|
-
return { prunedSharedWorkspaceIds };
|
|
416
|
-
},
|
|
417
|
-
},
|
|
418
|
-
{
|
|
419
|
-
id: "workspace-conflict-scan",
|
|
420
|
-
title: "Workspace conflict scan",
|
|
421
|
-
description: "Self-healing backstop (#237): probe worktrees for divergence/freshness and auto-flag conflicts. The turn-end stop-hook drives the common case; this catches dropped events, restarts, and direct human commits.",
|
|
422
|
-
intervalMs: CONFLICT_SCAN_INTERVAL_MS,
|
|
423
|
-
runOnStart: true,
|
|
424
|
-
timeoutMs: 60 * 1000,
|
|
425
|
-
handler: scanWorkspaceConflicts,
|
|
426
|
-
},
|
|
427
|
-
{
|
|
428
|
-
id: "workspace-auto-merge",
|
|
429
|
-
title: "Workspace auto-merge",
|
|
430
|
-
description: "Self-healing backstop (#237): land any non-conflicting ready/review_requested worktree under the per-repo lease (rebasing when base moved on); only real/unknown conflicts go to the steward. The on-ready/on-settle event chain drives the common case; runOnStart reclaims rows stranded across a relay restart.",
|
|
431
|
-
intervalMs: WORKSPACE_AUTO_MERGE_INTERVAL_MS,
|
|
432
|
-
runOnStart: true,
|
|
433
|
-
timeoutMs: 60 * 1000,
|
|
434
|
-
handler: autoMergeCleanFastForwards,
|
|
435
|
-
},
|
|
436
|
-
{
|
|
437
|
-
id: "workspace-gc",
|
|
438
|
-
title: "Workspace GC",
|
|
439
|
-
description: "Prune terminal workspace rows past retention, auto-abandon stale review_requested worktrees, and trigger git worktree prune on orchestrators.",
|
|
440
|
-
intervalMs: WORKSPACE_GC_INTERVAL_MS,
|
|
441
|
-
runOnStart: false,
|
|
442
|
-
timeoutMs: 60 * 1000,
|
|
443
|
-
handler: workspaceGC,
|
|
444
|
-
},
|
|
445
|
-
{
|
|
446
|
-
id: "workspace-orphan-reaper",
|
|
447
|
-
title: "Workspace orphan reaper",
|
|
448
|
-
description: "Reconcile disk⇄DB: reap orphaned worktrees whose work has landed (or is empty), flag orphans holding un-landed work as needs-attention instead of deleting, and report rows whose worktree vanished. git worktree prune can't do this — it no-ops while the directory still exists.",
|
|
449
|
-
intervalMs: WORKSPACE_ORPHAN_REAPER_INTERVAL_MS,
|
|
450
|
-
runOnStart: true,
|
|
451
|
-
timeoutMs: 2 * 60 * 1000,
|
|
452
|
-
handler: reapOrphanedWorktrees,
|
|
453
|
-
},
|
|
454
|
-
];
|
|
455
|
-
|
|
456
|
-
// Iterate active worktrees and ask the owning host whether each can still merge
|
|
457
|
-
// cleanly. Auto-flag `conflict` when a clean merge is no longer possible, and
|
|
458
|
-
// auto-clear conflicts we set ourselves once they resolve (restoring the prior
|
|
459
|
-
// status). Human-set conflicts are never cleared.
|
|
460
|
-
// Stop orphaned spawned sessions: process reported alive by the orchestrator, but
|
|
461
|
-
// the relay agent is offline/pruned and self-heal has had its chance. See the
|
|
462
|
-
// ORPHAN_* notes above. Covers both policy-managed and dashboard/ad-hoc spawns by
|
|
463
|
-
// iterating the orchestrators' reported managedAgents directly.
|
|
464
|
-
function reapOrphanedSessions(): Record<string, unknown> {
|
|
465
|
-
const now = Date.now();
|
|
466
|
-
const grace = orphanGraceMs();
|
|
467
|
-
const cooldown = orphanReapCooldownMs();
|
|
468
|
-
const reapEnabled = orphanReapEnabled();
|
|
469
|
-
const seen = new Set<string>();
|
|
470
|
-
const reaped: string[] = [];
|
|
471
|
-
let orphaned = 0;
|
|
472
|
-
|
|
473
|
-
for (const orch of listOrchestrators()) {
|
|
474
|
-
if (orch.status !== "online") continue; // can't trust the report or deliver the stop
|
|
475
|
-
for (const agent of orch.managedAgents) {
|
|
476
|
-
const sessionId = agent.spawnRequestId || agent.tmuxSession || agent.sessionName || agent.agentId;
|
|
477
|
-
if (!sessionId) continue;
|
|
478
|
-
const key = `${orch.id}:${sessionId}`;
|
|
479
|
-
const relayAgent = agent.agentId ? getAgent(agent.agentId) : null;
|
|
480
|
-
// Orphan = orchestrator reports the process running, but no live relay agent.
|
|
481
|
-
// "stale" is a recent/borderline disconnect — treat as alive and give it time;
|
|
482
|
-
// it will either recover or progress to "offline" and be caught next pass.
|
|
483
|
-
const isOrphan = !relayAgent || relayAgent.status === "offline";
|
|
484
|
-
if (!isOrphan) { orphanTracker.delete(key); continue; }
|
|
485
|
-
seen.add(key);
|
|
486
|
-
orphaned++;
|
|
487
|
-
const entry = orphanTracker.get(key) ?? { firstOrphanedAt: now };
|
|
488
|
-
orphanTracker.set(key, entry);
|
|
489
|
-
if (now - entry.firstOrphanedAt < grace) continue; // let self-heal recover it first
|
|
490
|
-
if (!reapEnabled) continue; // detect-only mode
|
|
491
|
-
if (entry.lastReapAt && now - entry.lastReapAt < cooldown) continue; // don't spam shutdowns
|
|
492
|
-
entry.lastReapAt = now;
|
|
493
|
-
const command = createCommand({
|
|
494
|
-
type: "agent.shutdown",
|
|
495
|
-
source: "system",
|
|
496
|
-
target: orch.agentId,
|
|
497
|
-
correlationId: agent.spawnRequestId,
|
|
498
|
-
params: {
|
|
499
|
-
action: "shutdown",
|
|
500
|
-
agentId: agent.agentId,
|
|
501
|
-
spawnRequestId: agent.spawnRequestId,
|
|
502
|
-
sessionName: agent.sessionName,
|
|
503
|
-
tmuxSession: agent.tmuxSession,
|
|
504
|
-
policyName: agent.policyName,
|
|
505
|
-
graceful: false,
|
|
506
|
-
timeoutMs: 10_000,
|
|
507
|
-
reason: "orphaned-session-reaper",
|
|
508
|
-
requestedBy: "orphaned-session-reaper",
|
|
509
|
-
requestedAt: now,
|
|
510
|
-
orchestratorId: orch.id,
|
|
511
|
-
},
|
|
512
|
-
});
|
|
513
|
-
emitRelayEvent({ type: "command.requested", source: "system", subject: command.id, data: { command } });
|
|
514
|
-
createActivityEvent({
|
|
515
|
-
clientId: `orphaned-session-reaper-${key}-${now}`,
|
|
516
|
-
kind: "state",
|
|
517
|
-
title: "Orphaned session reaped",
|
|
518
|
-
body: `${agent.label ?? agent.agentId ?? sessionId}: process still running on ${orch.id}, but its relay agent has been offline > ${Math.round(grace / 60000)}m and did not self-heal — stopping it`,
|
|
519
|
-
meta: agent.label ?? agent.agentId ?? sessionId,
|
|
520
|
-
icon: "ti-ghost",
|
|
521
|
-
view: "orchestrators",
|
|
522
|
-
agentId: agent.agentId || undefined,
|
|
523
|
-
metadata: {
|
|
524
|
-
source: "server",
|
|
525
|
-
maintenanceJobId: "orphaned-session-reaper",
|
|
526
|
-
orchestratorId: orch.id,
|
|
527
|
-
agentId: agent.agentId,
|
|
528
|
-
spawnRequestId: agent.spawnRequestId,
|
|
529
|
-
tmuxSession: agent.tmuxSession,
|
|
530
|
-
commandId: command.id,
|
|
531
|
-
orphanAgeMs: now - entry.firstOrphanedAt,
|
|
532
|
-
},
|
|
533
|
-
});
|
|
534
|
-
reaped.push(key);
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
// Forget sessions that recovered or are no longer reported, so a future orphaning
|
|
538
|
-
// of the same session starts a fresh grace window.
|
|
539
|
-
for (const key of orphanTracker.keys()) if (!seen.has(key)) orphanTracker.delete(key);
|
|
540
|
-
return { orphaned, reaped, tracked: orphanTracker.size, reapEnabled };
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
function reapZeroMemberTeams(): Record<string, unknown> {
|
|
544
|
-
const now = Date.now();
|
|
545
|
-
const cutoff = now - STALE_TTL_MS;
|
|
546
|
-
const grace = teamReapGraceMs();
|
|
547
|
-
const reapedTeamIds: string[] = [];
|
|
548
|
-
const trackedTeamIds: string[] = [];
|
|
549
|
-
const clearedTeamIds: string[] = [];
|
|
550
|
-
let activeTeams = 0;
|
|
551
|
-
let zeroMemberTeams = 0;
|
|
552
|
-
|
|
553
|
-
for (const team of listActiveTeams()) {
|
|
554
|
-
activeTeams++;
|
|
555
|
-
const live = countLiveTeamMembers(team.id, cutoff);
|
|
556
|
-
if (live > 0) {
|
|
557
|
-
if (team.zeroMembersSince !== undefined) {
|
|
558
|
-
clearTeamZeroMembersSince(team.id);
|
|
559
|
-
clearedTeamIds.push(team.id);
|
|
560
|
-
}
|
|
561
|
-
continue;
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
zeroMemberTeams++;
|
|
565
|
-
if (team.zeroMembersSince === undefined) {
|
|
566
|
-
setTeamZeroMembersSince(team.id, now);
|
|
567
|
-
trackedTeamIds.push(team.id);
|
|
568
|
-
continue;
|
|
569
|
-
}
|
|
570
|
-
if (now - team.zeroMembersSince < grace) continue;
|
|
571
|
-
|
|
572
|
-
const outcomeSummary = `auto-reaped: zero live members for ${grace}ms`;
|
|
573
|
-
// Leave outcomeStatus null (unknown): an abandoned team is NOT a failed one — its
|
|
574
|
-
// members went away without an explicit dissolve, which says nothing about success.
|
|
575
|
-
// Stamping "failed" would skew the #358 outcome analytics. Reason lives in the summary.
|
|
576
|
-
const dissolved = dissolveTeamById(team.id, { outcomeSummary });
|
|
577
|
-
createActivityEvent({
|
|
578
|
-
clientId: `team-reap-${team.id}-${now}`,
|
|
579
|
-
kind: "state",
|
|
580
|
-
title: "Team auto-reaped",
|
|
581
|
-
body: outcomeSummary,
|
|
582
|
-
meta: team.id,
|
|
583
|
-
icon: "ti-users-minus",
|
|
584
|
-
view: "teams",
|
|
585
|
-
agentId: dissolved.coordinatorId,
|
|
586
|
-
metadata: {
|
|
587
|
-
source: "server",
|
|
588
|
-
maintenanceJobId: "team-reap",
|
|
589
|
-
teamId: team.id,
|
|
590
|
-
zeroMembersSince: team.zeroMembersSince,
|
|
591
|
-
graceMs: grace,
|
|
592
|
-
},
|
|
593
|
-
});
|
|
594
|
-
reapedTeamIds.push(team.id);
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
return { activeTeams, zeroMemberTeams, trackedTeamIds, clearedTeamIds, reapedTeamIds, graceMs: grace };
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
async function scanWorkspaceConflicts(): Promise<Record<string, unknown>> {
|
|
602
|
-
const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.apiUrl);
|
|
603
|
-
if (!orchestrators.length) return { scanned: 0, skipped: "no online orchestrators" };
|
|
604
|
-
|
|
605
|
-
const candidates = listWorkspaces().filter(
|
|
606
|
-
(ws) => ws.mode === "isolated" && Boolean(ws.worktreePath) && CONFLICT_SCAN_STATUSES.has(ws.status),
|
|
607
|
-
);
|
|
608
|
-
const flagged: string[] = [], cleared: string[] = [], merged: string[] = [], notifiedStewards: string[] = [], prRecycled: string[] = [];
|
|
609
|
-
const reviewerArmed: string[] = [], reviewerChangesRequested: string[] = [], relayPrMerged: string[] = [], relayPrWaiting: string[] = [];
|
|
610
|
-
|
|
611
|
-
for (const ws of candidates) {
|
|
612
|
-
const orch = orchestrators.find((candidate) => workspacePathWithinBase(ws.sourceCwd, candidate.baseDir));
|
|
613
|
-
if (!orch?.apiUrl) continue;
|
|
614
|
-
// Spend the `gh` PR-state round-trip only on reconcile-eligible rows; active/ready
|
|
615
|
-
// are excluded from reconcile, so their scan stays git-only (#304).
|
|
616
|
-
const preview = await fetchHostMergePreview(orch.apiUrl, ws, { checkPr: LANDED_RECONCILE_STATUSES.has(ws.status) });
|
|
617
|
-
if (!preview || (preview as { available?: false }).available === false) continue;
|
|
618
|
-
const p = preview as WorkspaceMergePreview;
|
|
619
|
-
if (p.error || p.missing) continue;
|
|
620
|
-
|
|
621
|
-
const meta = ws.metadata as Record<string, unknown>;
|
|
622
|
-
|
|
623
|
-
// #236: stash the host's latest ahead/dirty counts so deriveBranchState can tell
|
|
624
|
-
// idle (⚪) from changes (🟡). This periodic pass is the self-healing backstop now
|
|
625
|
-
// that the turn-end stop-hook (`probeWorkspaceState`) flips the badge instantly
|
|
626
|
-
// (#237); both share `applyBranchFreshness` so the rule can't drift. Status-changing
|
|
627
|
-
// branches below own their own badge refresh via updateWorkspaceStatus.
|
|
628
|
-
applyBranchFreshness(ws, p);
|
|
629
|
-
|
|
630
|
-
// Landing wins over conflict checks. A merged PR can look like a no-op or conflict
|
|
631
|
-
// locally, so reconcile before the conflict-undefined skip and keep finalization in
|
|
632
|
-
// branch-landed.ts (#304).
|
|
633
|
-
const landed = p.landed === true || p.prMerged === true;
|
|
634
|
-
if (landed && LANDED_RECONCILE_STATUSES.has(ws.status)) {
|
|
635
|
-
// A PR-land merges asynchronously on the remote, outside the relay's land path. For a
|
|
636
|
-
// LIVE owner that means recycle (land-and-continue, #423); for a gone owner, finalize
|
|
637
|
-
// terminal `merged` (#304). `reconcileLandedPr` owns that split — see workspace-pr-completion.
|
|
638
|
-
const outcome = reconcileLandedPr(ws, p);
|
|
639
|
-
if (outcome.kind === "recycle") { emitCommand(outcome.command); prRecycled.push(ws.id); continue; }
|
|
640
|
-
if (outcome.kind === "wait") { relayPrWaiting.push(`${ws.id}:${outcome.reason}`); continue; }
|
|
641
|
-
reconcileLandedWorkspace(ws, p);
|
|
642
|
-
merged.push(ws.id);
|
|
643
|
-
continue;
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
const prAction = preparePrCompletionScan(ws, p, orch.agentId);
|
|
647
|
-
if (prAction.armCommand) { emitCommand(prAction.armCommand); reviewerArmed.push(ws.id); continue; }
|
|
648
|
-
if (prAction.notifiedOwner) { reviewerChangesRequested.push(ws.id); continue; }
|
|
649
|
-
if (prAction.refreshCommand) { emitCommand(prAction.refreshCommand); continue; }
|
|
650
|
-
if (prAction.mergeCommand) { emitCommand(prAction.mergeCommand); relayPrMerged.push(ws.id); continue; }
|
|
651
|
-
if (prAction.waitReason) relayPrWaiting.push(`${ws.id}:${prAction.waitReason}`);
|
|
652
|
-
if (prAction.stop) continue;
|
|
653
|
-
|
|
654
|
-
// Past here we act on the conflict signal — skip when the host couldn't assess it
|
|
655
|
-
// (undefined): never flag/clear a conflict on incomplete data.
|
|
656
|
-
if (p.conflict === undefined) continue;
|
|
657
|
-
|
|
658
|
-
if (p.conflict === true && ws.status !== "conflict") {
|
|
659
|
-
updateWorkspaceStatus(ws.id, "conflict", {
|
|
660
|
-
autoConflict: true,
|
|
661
|
-
preConflictStatus: ws.status,
|
|
662
|
-
conflictAhead: p.ahead,
|
|
663
|
-
conflictBehind: p.behind,
|
|
664
|
-
conflictBaseRef: p.baseRef,
|
|
665
|
-
conflictDetectedAt: Date.now(),
|
|
666
|
-
});
|
|
667
|
-
flagged.push(ws.id);
|
|
668
|
-
createActivityEvent({
|
|
669
|
-
clientId: "server-workspace-" + ws.id + "-conflict-" + Date.now(),
|
|
670
|
-
kind: "state",
|
|
671
|
-
title: "Merge conflict detected",
|
|
672
|
-
body: `${ws.branch ?? ws.id} can no longer merge cleanly into ${p.baseRef ?? "base"}`,
|
|
673
|
-
meta: ws.branch ?? ws.id,
|
|
674
|
-
icon: "ti-alert-triangle",
|
|
675
|
-
view: "orchestrators",
|
|
676
|
-
metadata: { source: "server", maintenanceJobId: "workspace-conflict-scan", workspaceId: ws.id, ahead: p.ahead, behind: p.behind },
|
|
677
|
-
});
|
|
678
|
-
// Hand the conflict to a steward so it gets resolved instead of rotting
|
|
679
|
-
// until merge time. Once-per-onset (we only enter this branch on the
|
|
680
|
-
// active→conflict transition). When managed stewards are enabled, wake the
|
|
681
|
-
// auto-provisioned per-repo steward agent (#167); otherwise fall back to the
|
|
682
|
-
// legacy direct ping of the elected steward agent.
|
|
683
|
-
if (getStewardConfig().enabled) {
|
|
684
|
-
const woke = wakeRepoSteward(getWorkspace(ws.id) ?? ws, "conflict");
|
|
685
|
-
if (woke) notifiedStewards.push(woke);
|
|
686
|
-
} else if (ws.stewardAgentId) {
|
|
687
|
-
try {
|
|
688
|
-
// Obligation (ttl null by type default): a conflict handoff must reach the steward.
|
|
689
|
-
routeNotification({
|
|
690
|
-
type: "workspace.conflict",
|
|
691
|
-
scope: { workspaceId: ws.id, repoRoot: ws.repoRoot },
|
|
692
|
-
recipients: [ws.stewardAgentId],
|
|
693
|
-
message: {
|
|
694
|
-
subject: "Workspace merge conflict",
|
|
695
|
-
body: `Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} can no longer merge cleanly into ${p.baseRef ?? "base"} (${p.ahead ?? "?"} ahead, ${p.behind ?? "?"} behind). As repo steward, please coordinate resolution.`,
|
|
696
|
-
payload: { kind: "workspace.conflict", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: p.baseRef, ahead: p.ahead, behind: p.behind },
|
|
697
|
-
},
|
|
698
|
-
});
|
|
699
|
-
notifiedStewards.push(ws.stewardAgentId);
|
|
700
|
-
} catch {
|
|
701
|
-
// Steward unregistered/stale — the activity event still records it.
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
} else if (p.conflict === false && ws.status === "conflict" && meta.autoConflict === true) {
|
|
705
|
-
const prior = typeof meta.preConflictStatus === "string" && meta.preConflictStatus !== "conflict"
|
|
706
|
-
? meta.preConflictStatus as WorkspaceStatus
|
|
707
|
-
: "active";
|
|
708
|
-
updateWorkspaceStatus(ws.id, prior, { autoConflict: false, conflictClearedAt: Date.now() });
|
|
709
|
-
cleared.push(ws.id);
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
return { scanned: candidates.length, flagged, cleared, merged, prRecycled, notifiedStewards, reviewerArmed, reviewerChangesRequested, relayPrMerged, relayPrWaiting };
|
|
714
|
-
}
|
|
715
|
-
|
|
716
|
-
// Route a GC notification, swallowing failures (a stale/missing/misconfigured target must never
|
|
717
|
-
// break the GC sweep). Returns the target on success, null otherwise. These are all obligation-style
|
|
718
|
-
// workspace handoffs (null TTL by type default) — they must survive until the recipient acts.
|
|
719
|
-
function notifyTarget(type: NotificationType, scope: NotificationScope, target: string, subject: string, body: string, payload: Record<string, unknown>): string | null {
|
|
720
|
-
if (!target) return null;
|
|
721
|
-
try {
|
|
722
|
-
routeNotification({ type, scope, recipients: [target], message: { subject, body, payload } });
|
|
723
|
-
return target;
|
|
724
|
-
} catch {
|
|
725
|
-
return null;
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
async function workspaceGC(): Promise<Record<string, unknown>> {
|
|
730
|
-
const now = Date.now();
|
|
731
|
-
const cutoff = now - WORKSPACE_RETENTION_MS;
|
|
732
|
-
const reviewCutoff = now - WORKSPACE_REVIEW_TTL_MS;
|
|
733
|
-
|
|
734
|
-
// 0. Free any merge leases whose holder never reported back (orchestrator died
|
|
735
|
-
// mid-merge). The lease TTL is the safety net; this just reclaims them eagerly.
|
|
736
|
-
const releasedLeaseRepos = releaseExpiredMergeLeases(now);
|
|
737
|
-
|
|
738
|
-
// 1. Prune terminal rows past retention
|
|
739
|
-
const all = listWorkspaces();
|
|
740
|
-
const terminalIds: string[] = [];
|
|
741
|
-
for (const ws of all) {
|
|
742
|
-
if (TERMINAL_WORKSPACE_STATUSES.has(ws.status) && ws.updatedAt < cutoff) {
|
|
743
|
-
deleteWorkspace(ws.id);
|
|
744
|
-
terminalIds.push(ws.id);
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
// 2. Rescue stranded review_requested/conflict worktrees (issue #157). A
|
|
749
|
-
// worktree is "stranded" when its steward is gone (all repo agents offline).
|
|
750
|
-
// Re-elect first — an agent may have rejoined — and hand off to the new
|
|
751
|
-
// steward; if none can be elected past the TTL, escalate to the fallback
|
|
752
|
-
// target so it never rots in silence. Bookkeeping uses patchWorkspaceMetadata
|
|
753
|
-
// (no updated_at bump) so the auto-abandon clock below keeps ticking.
|
|
754
|
-
const escalatedIds: string[] = [];
|
|
755
|
-
const reassignedIds: string[] = [];
|
|
756
|
-
const escalationTargets: string[] = [];
|
|
757
|
-
const escalationMs = stewardEscalationMs();
|
|
758
|
-
const fallbackTarget = stewardFallbackTarget();
|
|
759
|
-
for (const ws of all) {
|
|
760
|
-
if (!STRANDABLE_STATUSES.has(ws.status) || ws.mode !== "isolated" || !ws.worktreePath) continue;
|
|
761
|
-
reelectRepoSteward(ws.repoRoot);
|
|
762
|
-
const fresh = getWorkspace(ws.id);
|
|
763
|
-
if (!fresh || !STRANDABLE_STATUSES.has(fresh.status)) continue;
|
|
764
|
-
const meta = fresh.metadata as Record<string, unknown>;
|
|
765
|
-
const steward = fresh.stewardAgentId;
|
|
766
|
-
const stewardOnline = Boolean(steward && getAgent(steward) && getAgent(steward)!.status !== "offline");
|
|
767
|
-
const strandedAt = typeof meta.strandedAt === "number" ? meta.strandedAt : undefined;
|
|
768
|
-
|
|
769
|
-
if (stewardOnline) {
|
|
770
|
-
// An online steward owns it. If it was previously stranded and this
|
|
771
|
-
// steward hasn't been told, hand it off explicitly, then clear markers.
|
|
772
|
-
if (strandedAt !== undefined && meta.strandedNotifiedSteward !== steward) {
|
|
773
|
-
const sent = notifyTarget(
|
|
774
|
-
"workspace.steward_reassigned",
|
|
775
|
-
{ workspaceId: fresh.id, repoRoot: fresh.repoRoot },
|
|
776
|
-
steward!,
|
|
777
|
-
"Workspace stewardship reassigned",
|
|
778
|
-
`You are now steward for ${fresh.repoRoot}. Workspace \`${fresh.branch ?? fresh.id}\` is ${fresh.status} and was stranded without an online steward — please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"}.`,
|
|
779
|
-
{ kind: "workspace.steward-reassigned", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status },
|
|
780
|
-
);
|
|
781
|
-
if (sent) reassignedIds.push(fresh.id);
|
|
782
|
-
}
|
|
783
|
-
patchWorkspaceMetadata(fresh.id, { strandedAt: undefined, escalatedAt: undefined, strandedNotifiedSteward: steward });
|
|
784
|
-
continue;
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
// Stranded: no online steward could be elected.
|
|
788
|
-
if (strandedAt === undefined) { patchWorkspaceMetadata(fresh.id, { strandedAt: now }); continue; }
|
|
789
|
-
if (now - strandedAt < escalationMs || meta.escalatedAt) continue;
|
|
790
|
-
const sent = notifyTarget(
|
|
791
|
-
"workspace.stranded_escalation",
|
|
792
|
-
{ workspaceId: fresh.id, repoRoot: fresh.repoRoot },
|
|
793
|
-
fallbackTarget,
|
|
794
|
-
"Stranded workspace needs an owner",
|
|
795
|
-
`Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} is ${fresh.status} with no online steward (all repo agents offline) for ${Math.round((now - strandedAt) / (60 * 60 * 1000))}h. Please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"} or clean up the worktree.`,
|
|
796
|
-
{ kind: "workspace.stranded-escalation", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt },
|
|
797
|
-
);
|
|
798
|
-
if (sent) escalationTargets.push(sent);
|
|
799
|
-
patchWorkspaceMetadata(fresh.id, { escalatedAt: now });
|
|
800
|
-
escalatedIds.push(fresh.id);
|
|
801
|
-
createActivityEvent({
|
|
802
|
-
clientId: `workspace-gc-escalate-${fresh.id}-${now}`,
|
|
803
|
-
kind: "state",
|
|
804
|
-
title: "Workspace escalated",
|
|
805
|
-
body: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} — stranded ${fresh.status} escalated${fallbackTarget ? ` to ${fallbackTarget}` : " (no fallback configured)"}`,
|
|
806
|
-
meta: fresh.branch ?? fresh.id,
|
|
807
|
-
icon: "ti-alert-octagon",
|
|
808
|
-
view: "orchestrators",
|
|
809
|
-
metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: fresh.id, fallback: fallbackTarget || null },
|
|
810
|
-
});
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
// 3. Auto-abandon stale review_requested worktrees
|
|
814
|
-
const abandonedIds: string[] = [];
|
|
815
|
-
const notifiedStewards: string[] = [];
|
|
816
|
-
for (const ws of all) {
|
|
817
|
-
if (ws.status === "review_requested" && ws.updatedAt < reviewCutoff) {
|
|
818
|
-
updateWorkspaceStatus(ws.id, "abandoned", { autoAbandoned: true, abandonedReason: "review_requested TTL exceeded", abandonedAt: now });
|
|
819
|
-
abandonedIds.push(ws.id);
|
|
820
|
-
// Notify the steward if one exists, else the configured fallback so a
|
|
821
|
-
// stranded abandon isn't silent (issue #157).
|
|
822
|
-
const target = ws.stewardAgentId ?? fallbackTarget;
|
|
823
|
-
const sent = notifyTarget(
|
|
824
|
-
"workspace.auto_abandoned",
|
|
825
|
-
{ workspaceId: ws.id, repoRoot: ws.repoRoot },
|
|
826
|
-
target,
|
|
827
|
-
"Workspace auto-abandoned",
|
|
828
|
-
`Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} was auto-abandoned after ${Math.round(WORKSPACE_REVIEW_TTL_MS / DAY_MS)}d without steward action. Run workspace cleanup to reclaim the worktree.`,
|
|
829
|
-
{ kind: "workspace.auto-abandoned", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch },
|
|
830
|
-
);
|
|
831
|
-
if (sent) notifiedStewards.push(sent);
|
|
832
|
-
createActivityEvent({
|
|
833
|
-
clientId: `workspace-gc-abandon-${ws.id}-${now}`,
|
|
834
|
-
kind: "state",
|
|
835
|
-
title: "Workspace auto-abandoned",
|
|
836
|
-
body: `${ws.branch ?? ws.id} in ${ws.repoRoot} — review_requested for ${Math.round((now - ws.updatedAt) / DAY_MS)}d`,
|
|
837
|
-
meta: ws.branch ?? ws.id,
|
|
838
|
-
icon: "ti-clock-x",
|
|
839
|
-
view: "orchestrators",
|
|
840
|
-
metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: ws.id },
|
|
841
|
-
});
|
|
842
|
-
}
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
// 3. Trigger git worktree prune on orchestrators that own isolated workspaces
|
|
846
|
-
const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.agentId);
|
|
847
|
-
const reposWithWorkspaces = new Set(
|
|
848
|
-
all.filter((ws) => ws.mode === "isolated" && Boolean(ws.worktreePath)).map((ws) => ws.repoRoot),
|
|
849
|
-
);
|
|
850
|
-
const pruneCommands: string[] = [];
|
|
851
|
-
for (const repoRoot of reposWithWorkspaces) {
|
|
852
|
-
const orch = orchestrators.find((candidate) => workspacePathWithinBase(repoRoot, candidate.baseDir));
|
|
853
|
-
if (!orch?.agentId) continue;
|
|
854
|
-
const command = createCommand({
|
|
855
|
-
type: "workspace.prune",
|
|
856
|
-
source: "system",
|
|
857
|
-
target: orch.agentId,
|
|
858
|
-
params: { repoRoot, requestedBy: "workspace-gc", requestedAt: now },
|
|
859
|
-
});
|
|
860
|
-
emitCommand(command);
|
|
861
|
-
pruneCommands.push(command.id);
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
return {
|
|
865
|
-
prunedTerminal: terminalIds,
|
|
866
|
-
autoAbandoned: abandonedIds,
|
|
867
|
-
notifiedStewards,
|
|
868
|
-
pruneCommands,
|
|
869
|
-
releasedLeaseRepos,
|
|
870
|
-
escalated: escalatedIds,
|
|
871
|
-
reassigned: reassignedIds,
|
|
872
|
-
escalationTargets,
|
|
873
|
-
};
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
let timer: Timer | null = null;
|
|
877
|
-
|
|
878
|
-
export function startMaintenanceScheduler(): void {
|
|
879
|
-
ensureMaintenanceJobs();
|
|
880
|
-
void runDueMaintenanceJobs();
|
|
881
|
-
if (timer) return;
|
|
882
|
-
timer = setInterval(() => void runDueMaintenanceJobs(), SCHEDULER_TICK_MS);
|
|
883
|
-
}
|
|
884
|
-
|
|
885
|
-
export function listMaintenanceJobs(): MaintenanceJob[] {
|
|
886
|
-
ensureMaintenanceJobs();
|
|
887
|
-
const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
|
|
888
|
-
const now = Date.now();
|
|
889
|
-
return rows.map((row) => rowToJob(row, now));
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
export async function runMaintenanceJobNow(id: string): Promise<MaintenanceJobRun> {
|
|
893
|
-
ensureMaintenanceJobs();
|
|
894
|
-
const definition = definitions.find((job) => job.id === id);
|
|
895
|
-
if (!definition) throw new Error("maintenance job not found");
|
|
896
|
-
return runJob(definition, { force: true });
|
|
897
|
-
}
|
|
898
|
-
|
|
899
|
-
export async function runLegacyMaintenanceReaper(): Promise<Record<string, unknown>> {
|
|
900
|
-
const runs = await runMaintenanceJobsNow(LEGACY_REAPER_JOB_IDS);
|
|
901
|
-
const result = aggregateRunResults(runs);
|
|
902
|
-
auditMaintenanceReaperRun(result);
|
|
903
|
-
return result;
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
async function runMaintenanceJobsNow(ids: string[]): Promise<MaintenanceJobRun[]> {
|
|
907
|
-
const runs: MaintenanceJobRun[] = [];
|
|
908
|
-
for (const id of ids) runs.push(await runMaintenanceJobNow(id));
|
|
909
|
-
return runs;
|
|
910
|
-
}
|
|
911
|
-
|
|
912
|
-
async function runDueMaintenanceJobs(): Promise<void> {
|
|
913
|
-
ensureMaintenanceJobs();
|
|
914
|
-
const now = Date.now();
|
|
915
|
-
const rows = getDb().query(`
|
|
916
|
-
SELECT id FROM maintenance_jobs
|
|
917
|
-
WHERE enabled = 1
|
|
918
|
-
AND next_run_at IS NOT NULL
|
|
919
|
-
AND next_run_at <= ?
|
|
920
|
-
AND (lease_until IS NULL OR lease_until <= ?)
|
|
921
|
-
ORDER BY next_run_at ASC
|
|
922
|
-
`).all(now, now) as Array<{ id: string }>;
|
|
923
|
-
for (const row of rows) {
|
|
924
|
-
const definition = definitions.find((job) => job.id === row.id);
|
|
925
|
-
if (definition) void runJob(definition).catch((error) => {
|
|
926
|
-
console.warn(`maintenance job ${definition.id} failed: ${errMessage(error)}`);
|
|
927
|
-
});
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
function ensureMaintenanceJobs(): void {
|
|
932
|
-
const db = getDb();
|
|
933
|
-
const now = Date.now();
|
|
934
|
-
const stmt = db.query(`
|
|
935
|
-
INSERT INTO maintenance_jobs (id, title, description, interval_ms, timeout_ms, enabled, run_on_start, next_run_at, updated_at)
|
|
936
|
-
VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?)
|
|
937
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
938
|
-
title = excluded.title,
|
|
939
|
-
description = excluded.description,
|
|
940
|
-
interval_ms = excluded.interval_ms,
|
|
941
|
-
timeout_ms = excluded.timeout_ms,
|
|
942
|
-
run_on_start = excluded.run_on_start,
|
|
943
|
-
next_run_at = COALESCE(maintenance_jobs.next_run_at, excluded.next_run_at),
|
|
944
|
-
updated_at = excluded.updated_at
|
|
945
|
-
`);
|
|
946
|
-
for (const definition of definitions) {
|
|
947
|
-
stmt.run(
|
|
948
|
-
definition.id,
|
|
949
|
-
definition.title,
|
|
950
|
-
definition.description,
|
|
951
|
-
definition.intervalMs,
|
|
952
|
-
definition.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
953
|
-
definition.runOnStart ? 1 : 0,
|
|
954
|
-
definition.runOnStart ? now : now + definition.intervalMs,
|
|
955
|
-
now,
|
|
956
|
-
);
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
async function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean } = {}): Promise<MaintenanceJobRun> {
|
|
961
|
-
const db = getDb();
|
|
962
|
-
const startedAt = Date.now();
|
|
963
|
-
const owner = `${process.pid}-${randomUUID()}`;
|
|
964
|
-
const timeoutMs = definition.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
965
|
-
const leaseUntil = startedAt + timeoutMs + 5_000;
|
|
966
|
-
const claim = db.query(`
|
|
967
|
-
UPDATE maintenance_jobs
|
|
968
|
-
SET lease_owner = ?, lease_until = ?, last_status = 'running', updated_at = ?
|
|
969
|
-
WHERE id = ?
|
|
970
|
-
AND enabled = 1
|
|
971
|
-
AND (lease_until IS NULL OR lease_until <= ?)
|
|
972
|
-
${options.force ? "" : "AND (next_run_at IS NULL OR next_run_at <= ?)"}
|
|
973
|
-
`).run(owner, leaseUntil, startedAt, definition.id, startedAt, ...(options.force ? [] : [startedAt]));
|
|
974
|
-
if (claim.changes === 0) {
|
|
975
|
-
const state = listMaintenanceJobs().find((job) => job.id === definition.id);
|
|
976
|
-
return { id: definition.id, status: "skipped", startedAt, finishedAt: Date.now(), durationMs: 0, result: { reason: "not due or already running", state } };
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
try {
|
|
980
|
-
const result = await withTimeout(Promise.resolve(definition.handler()), timeoutMs);
|
|
981
|
-
const finishedAt = Date.now();
|
|
982
|
-
const durationMs = finishedAt - startedAt;
|
|
983
|
-
db.query(`
|
|
984
|
-
UPDATE maintenance_jobs
|
|
985
|
-
SET last_run_at = ?, next_run_at = ?, last_duration_ms = ?, last_status = 'succeeded',
|
|
986
|
-
last_error = NULL, last_result = ?, consecutive_failures = 0,
|
|
987
|
-
lease_owner = NULL, lease_until = NULL, updated_at = ?
|
|
988
|
-
WHERE id = ?
|
|
989
|
-
`).run(finishedAt, finishedAt + definition.intervalMs, durationMs, JSON.stringify(result), finishedAt, definition.id);
|
|
990
|
-
return { id: definition.id, status: "succeeded", startedAt, finishedAt, durationMs, result };
|
|
991
|
-
} catch (error) {
|
|
992
|
-
const finishedAt = Date.now();
|
|
993
|
-
const durationMs = finishedAt - startedAt;
|
|
994
|
-
const message = errMessage(error);
|
|
995
|
-
db.query(`
|
|
996
|
-
UPDATE maintenance_jobs
|
|
997
|
-
SET last_run_at = ?, next_run_at = ?, last_duration_ms = ?, last_status = 'failed',
|
|
998
|
-
last_error = ?, consecutive_failures = consecutive_failures + 1,
|
|
999
|
-
lease_owner = NULL, lease_until = NULL, updated_at = ?
|
|
1000
|
-
WHERE id = ?
|
|
1001
|
-
`).run(finishedAt, finishedAt + definition.intervalMs, durationMs, message, finishedAt, definition.id);
|
|
1002
|
-
return { id: definition.id, status: "failed", startedAt, finishedAt, durationMs, error: message };
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
1005
|
-
|
|
1006
|
-
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
1007
|
-
let timeout: Timer | undefined;
|
|
1008
|
-
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
1009
|
-
timeout = setTimeout(() => reject(new Error(`maintenance job timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
1010
|
-
});
|
|
1011
|
-
try {
|
|
1012
|
-
return await Promise.race([promise, timeoutPromise]);
|
|
1013
|
-
} finally {
|
|
1014
|
-
if (timeout) clearTimeout(timeout);
|
|
1015
|
-
}
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
function rowToJob(row: MaintenanceJobRow, now: number): MaintenanceJob {
|
|
1019
|
-
const leaseUntil = row.lease_until ?? undefined;
|
|
1020
|
-
return {
|
|
1021
|
-
id: row.id,
|
|
1022
|
-
title: row.title,
|
|
1023
|
-
description: row.description ?? undefined,
|
|
1024
|
-
intervalMs: row.interval_ms,
|
|
1025
|
-
timeoutMs: row.timeout_ms,
|
|
1026
|
-
enabled: row.enabled === 1,
|
|
1027
|
-
runOnStart: row.run_on_start === 1,
|
|
1028
|
-
lastRunAt: row.last_run_at ?? undefined,
|
|
1029
|
-
nextRunAt: row.next_run_at ?? undefined,
|
|
1030
|
-
lastDurationMs: row.last_duration_ms ?? undefined,
|
|
1031
|
-
lastStatus: leaseUntil && leaseUntil > now ? "running" : row.last_status as MaintenanceJob["lastStatus"],
|
|
1032
|
-
lastError: row.last_error ?? undefined,
|
|
1033
|
-
lastResult: parseJsonObject(row.last_result),
|
|
1034
|
-
consecutiveFailures: row.consecutive_failures,
|
|
1035
|
-
running: Boolean(leaseUntil && leaseUntil > now),
|
|
1036
|
-
leaseUntil,
|
|
1037
|
-
updatedAt: row.updated_at,
|
|
1038
|
-
};
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
function aggregateRunResults(runs: MaintenanceJobRun[]): Record<string, unknown> {
|
|
1042
|
-
const aggregate: Record<string, unknown> = { ok: true, runs };
|
|
1043
|
-
for (const run of runs) {
|
|
1044
|
-
if (!run.result) continue;
|
|
1045
|
-
for (const [key, value] of Object.entries(run.result)) {
|
|
1046
|
-
if (Array.isArray(value)) {
|
|
1047
|
-
const current = Array.isArray(aggregate[key]) ? aggregate[key] as unknown[] : [];
|
|
1048
|
-
aggregate[key] = [...current, ...value];
|
|
1049
|
-
} else if (typeof value === "number") {
|
|
1050
|
-
aggregate[key] = Number(aggregate[key] ?? 0) + value;
|
|
1051
|
-
} else if (value !== undefined) {
|
|
1052
|
-
aggregate[key] = value;
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
return aggregate;
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
function auditMaintenanceReaperRun(result: Record<string, unknown>): void {
|
|
1060
|
-
const reapedAgentIds = Array.isArray(result.reapedAgentIds) ? result.reapedAgentIds : [];
|
|
1061
|
-
const reapedOrchestratorIds = Array.isArray(result.reapedOrchestratorIds) ? result.reapedOrchestratorIds : [];
|
|
1062
|
-
const releasedMessageIds = Array.isArray(result.releasedMessageIds) ? result.releasedMessageIds : [];
|
|
1063
|
-
const releasedTaskIds = Array.isArray(result.releasedTaskIds) ? result.releasedTaskIds : [];
|
|
1064
|
-
const releasedOrphanedTaskIds = Array.isArray(result.releasedOrphanedTaskIds) ? result.releasedOrphanedTaskIds : [];
|
|
1065
|
-
const expiredQueuedMessageIds = Array.isArray(result.expiredQueuedMessageIds) ? result.expiredQueuedMessageIds : [];
|
|
1066
|
-
const expiredCommandIds = Array.isArray(result.expiredCommandIds) ? result.expiredCommandIds : [];
|
|
1067
|
-
const expiredArtifactIds = Array.isArray(result.expiredArtifactIds) ? result.expiredArtifactIds : [];
|
|
1068
|
-
try {
|
|
1069
|
-
createActivityEvent({
|
|
1070
|
-
clientId: "server-maintenance-reaper-" + Date.now(),
|
|
1071
|
-
kind: "state",
|
|
1072
|
-
title: "Maintenance reaper run",
|
|
1073
|
-
body: `${reapedAgentIds.length} stale agent(s), ${reapedOrchestratorIds.length} stale orchestrator(s), ${releasedMessageIds.length} message claim(s), ${releasedTaskIds.length} task claim(s), ${releasedOrphanedTaskIds.length} orphaned task(s), ${expiredQueuedMessageIds.length} queued message(s), ${expiredCommandIds.length} command(s), ${expiredArtifactIds.length} artifact(s)`,
|
|
1074
|
-
icon: "ti-broom",
|
|
1075
|
-
view: "activity",
|
|
1076
|
-
metadata: { source: "server", result },
|
|
1077
|
-
});
|
|
1078
|
-
} catch {}
|
|
1079
|
-
}
|
|
1080
|
-
|
|
1081
|
-
function emitCommand(command: Command): void {
|
|
1082
|
-
const suffix = command.status === "timed_out" ? "timed_out" : command.status;
|
|
1083
|
-
emitRelayEvent({
|
|
1084
|
-
type: `command.${suffix}`,
|
|
1085
|
-
source: command.source,
|
|
1086
|
-
subject: command.id,
|
|
1087
|
-
data: { command },
|
|
1088
|
-
});
|
|
1089
|
-
}
|
|
1090
|
-
|
|
1091
|
-
function parseJsonObject(raw: string | null): Record<string, unknown> | undefined {
|
|
1092
|
-
if (!raw) return undefined;
|
|
1093
|
-
try {
|
|
1094
|
-
const value = JSON.parse(raw);
|
|
1095
|
-
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
|
|
1096
|
-
} catch {
|
|
1097
|
-
return undefined;
|
|
1098
|
-
}
|
|
1099
|
-
}
|
|
1
|
+
export * from "./maintenance/index";
|