agent-relay-server 0.62.2 → 0.63.0

package/src/maintenance/jobs.ts

@@ -0,0 +1,340 @@
+import { getArtifactStorage } from "../artifact-storage";
+import { expireStaleBusAgents } from "../bus";
+import { pruneOutbox } from "../bus-outbox";
+import { expireCommands } from "../commands-db";
+import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
+import {
+  createActivityEvent,
+  evaluatePoolBindings,
+  expireQueuedMessages,
+  pruneOrphanedSharedWorkspaces,
+  pruneOfflineAgents,
+  pruneOldMessages,
+  reapStaleAgents,
+  reapStaleOrchestrators,
+  releaseExpiredClaims,
+  releaseOrphanedTasks,
+  runDbMaintenance,
+  sweepArtifacts,
+} from "../db";
+import { notifyAgentOffline } from "../agent-lifecycle-events";
+import { autoMergeCleanFastForwards } from "../workspace-auto-merge";
+import { reapOrphanedWorktrees } from "../workspace-orphans";
+import { getLifecycleManager } from "../lifecycle-manager";
+import { applyCommandToRecipe } from "../recipe-runner";
+import {
+  emitAgentRemoved,
+  emitAgentStatus,
+  emitMessageClaimReleased,
+  emitMessageExpired,
+  emitOrchestratorStatus,
+  emitPoolBindingChanged,
+  emitTaskChanged,
+} from "../sse";
+import { pruneExpiredTokenRecords } from "../token-db";
+import { reapTransientAgents } from "../services/transient-agent-reaper";
+import { resumeRateLimitedAgents } from "../services/rate-limit-resume";
+import { MERGE_PAUSE_MAINTENANCE_JOB } from "../merge-pause-maintenance";
+import {
+  DB_MAINTENANCE_INTERVAL_MS,
+  DB_VACUUM_EVERY,
+  DAY_MS,
+  ORPHAN_REAPER_INTERVAL_MS,
+  OUTBOX_RETENTION_MS,
+  TOKEN_RECORD_RETENTION_SECONDS,
+  WORKSPACE_AUTO_MERGE_INTERVAL_MS,
+  WORKSPACE_GC_INTERVAL_MS,
+  WORKSPACE_ORPHAN_REAPER_INTERVAL_MS,
+  CONFLICT_SCAN_INTERVAL_MS,
+  teamReapIntervalMs,
+} from "./constants";
+import { emitCommand } from "./events";
+import { reapOrphanedSessions } from "./orphaned-sessions";
+import { reapZeroMemberTeams } from "./teams";
+import { scanWorkspaceConflicts, workspaceGC } from "./workspaces";
+import type { MaintenanceJobDefinition } from "./types";
+
+let dbMaintenanceRuns = 0;
+
+export const LEGACY_REAPER_JOB_IDS = [
+  "claim-cleanup",
+  "task-orphan-release",
+  "queued-message-expiration",
+  "command-expiration",
+  "artifact-sweep",
+  "stale-agent-reaper",
+  "orchestrator-reaper",
+  "offline-agent-prune",
+  "pool-binding-reconcile",
+];
+
+export const definitions: MaintenanceJobDefinition[] = [
+  {
+    id: "claim-cleanup",
+    title: "Claim cleanup",
+    description: "Release expired message and task claims.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const released = releaseExpiredClaims();
+      for (const id of released.messageIds) emitMessageClaimReleased(id);
+      for (const task of released.tasks) emitTaskChanged(task, "task.updated");
+      return {
+        releasedMessageIds: released.messageIds,
+        releasedTaskIds: released.tasks.map((task) => task.id),
+      };
+    },
+  },
+  MERGE_PAUSE_MAINTENANCE_JOB,
+  {
+    id: "task-orphan-release",
+    title: "Orphaned task release",
+    description: "Release active tasks owned by agents that are no longer online.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const tasks = releaseOrphanedTasks();
+      for (const task of tasks) emitTaskChanged(task, "task.updated");
+      return { releasedOrphanedTaskIds: tasks.map((task) => task.id) };
+    },
+  },
+  {
+    id: "queued-message-expiration",
+    title: "Queued message expiration",
+    description: "Expire queued messages that exceeded their max age.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const messages = expireQueuedMessages();
+      for (const message of messages) emitMessageExpired(message);
+      return { expiredQueuedMessageIds: messages.map((message) => message.id) };
+    },
+  },
+  {
+    id: "command-expiration",
+    title: "Command expiration",
+    description: "Timeout active commands past their lease.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const commands = expireCommands();
+      for (const command of commands) {
+        applyCommandToRecipe(command);
+        emitCommand(command);
+      }
+      return { expiredCommandIds: commands.map((command) => command.id) };
+    },
+  },
+  {
+    id: "bus-agent-expiration",
+    title: "Bus agent expiration",
+    description: "Expire stale live bus attachments and release associated orphan work.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const expired = expireStaleBusAgents();
+      for (const task of expired.orphanedTasks) emitTaskChanged(task, "task.updated");
+      return {
+        expiredBusAgentIds: expired.agentIds,
+        releasedOrphanedTaskIds: expired.orphanedTasks.map((task) => task.id),
+      };
+    },
+  },
+  {
+    id: "stale-agent-reaper",
+    title: "Stale agent reaper",
+    description: "Mark agents offline when heartbeats are too old.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const reapedAgentIds = reapStaleAgents(STALE_TTL_MS);
+      for (const id of reapedAgentIds) {
+        emitAgentStatus(id);
+        getLifecycleManager().onAgentDisappeared(id);
+        notifyAgentOffline(id, "heartbeat lost");
+        createActivityEvent({
+          clientId: "server-agent-" + id + "-heartbeat-lost-" + Date.now(),
+          kind: "state",
+          title: "Heartbeat lost",
+          body: "Agent marked offline",
+          meta: id,
+          icon: "ti-heart-off",
+          view: "chat",
+          agentId: id,
+          metadata: { source: "server", maintenanceJobId: "stale-agent-reaper" },
+        });
+      }
+      return { reapedAgentIds };
+    },
+  },
+  {
+    id: "offline-agent-prune",
+    title: "Offline agent prune",
+    description: "Delete long-offline provider agents and release their claims.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: false,
+    handler() {
+      const prunedAgentIds = pruneOfflineAgents(OFFLINE_PRUNE_MS);
+      for (const id of prunedAgentIds) emitAgentRemoved(id);
+      return { prunedAgentIds };
+    },
+  },
+  {
+    id: "orphaned-session-reaper",
+    title: "Orphaned session reaper",
+    description: "Stop spawned sessions whose relay agent is offline/gone but whose process the orchestrator still reports running, after a grace period for self-heal.",
+    intervalMs: ORPHAN_REAPER_INTERVAL_MS,
+    runOnStart: false,
+    handler: reapOrphanedSessions,
+  },
+  { id: "transient-agent-reaper", title: "Transient agent reaper", description: "Cleanly shut down transient spawned agents once they are idle and any isolated branch work has landed.", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: reapTransientAgents },
+  { id: "rate-limit-resume", title: "Rate-limit resume", description: "Auto-resume agents held on a provider usage/rate-limit stall once their reset window has passed (#286).", intervalMs: REAP_INTERVAL_MS, runOnStart: false, handler: resumeRateLimitedAgents },
+  {
+    id: "team-reap",
+    title: "Team lifecycle close-out",
+    description: "Dissolve active teams that have had zero live members for the configured grace window.",
+    intervalMs: teamReapIntervalMs(),
+    runOnStart: false,
+    handler: reapZeroMemberTeams,
+  },
+  {
+    id: "orchestrator-reaper",
+    title: "Orchestrator reaper",
+    description: "Mark stale orchestrator registrations offline.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const reapedOrchestratorIds = reapStaleOrchestrators();
+      for (const id of reapedOrchestratorIds) emitOrchestratorStatus(id);
+      return { reapedOrchestratorIds };
+    },
+  },
+  {
+    id: "pool-binding-reconcile",
+    title: "Pool binding reconcile",
+    description: "Reassign pool channel bindings when current assignees are unavailable.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const changes = evaluatePoolBindings();
+      for (const change of changes) emitPoolBindingChanged(change.bindingId, change.channelId, change.previousAgentId, change.newAgentId);
+      return { poolBindingChanges: changes };
+    },
+  },
+  {
+    id: "old-message-prune",
+    title: "Old message prune",
+    description: "Delete messages outside the configured retention window.",
+    intervalMs: DAY_MS,
+    runOnStart: false,
+    handler() {
+      const retentionDays = Number(process.env.RETENTION_DAYS) || 30;
+      return { prunedMessages: pruneOldMessages(retentionDays * DAY_MS) };
+    },
+  },
+  {
+    id: "db-maintenance",
+    title: "Database maintenance",
+    description: "Refresh planner stats (ANALYZE + optimize), truncate the WAL, and periodically VACUUM to reclaim space.",
+    intervalMs: DB_MAINTENANCE_INTERVAL_MS,
+    runOnStart: false,
+    timeoutMs: 5 * 60 * 1000,
+    handler() {
+      dbMaintenanceRuns += 1;
+      const vacuum = DB_VACUUM_EVERY > 0 && dbMaintenanceRuns % DB_VACUUM_EVERY === 0;
+      return runDbMaintenance({ vacuum });
+    },
+  },
+  {
+    id: "bus-outbox-prune",
+    title: "Bus outbox prune",
+    description: "Prune retained bus outbox events after replay retention.",
+    intervalMs: 5 * 60 * 1000,
+    runOnStart: false,
+    handler() {
+      return { prunedOutboxEvents: pruneOutbox(OUTBOX_RETENTION_MS) };
+    },
+  },
+  {
+    id: "artifact-sweep",
+    title: "Artifact sweep",
+    description: "Delete expired or unlinked artifacts and their local blobs.",
+    intervalMs: 60 * 60 * 1000,
+    runOnStart: false,
+    timeoutMs: 2 * 60 * 1000,
+    async handler() {
+      const swept = sweepArtifacts();
+      const failedBlobDeletes: string[] = [];
+      for (const blob of swept.blobs) {
+        try {
+          await getArtifactStorage().delete(blob.storageUri);
+        } catch {
+          failedBlobDeletes.push(blob.digest);
+        }
+      }
+      return {
+        expiredArtifactIds: swept.artifactIds,
+        sweptBlobDigests: swept.blobs.map((blob) => blob.digest),
+        failedBlobDeletes,
+      };
+    },
+  },
+  {
+    id: "token-record-prune",
+    title: "Token record prune",
+    description: "Delete expired or revoked token records after retention.",
+    intervalMs: 60 * 60 * 1000,
+    runOnStart: false,
+    handler() {
+      const prunedTokenJtis = pruneExpiredTokenRecords({ retentionSeconds: TOKEN_RECORD_RETENTION_SECONDS });
+      return { prunedTokenJtis };
+    },
+  },
+  {
+    id: "workspace-orphan-sweep",
+    title: "Workspace orphan sweep",
+    description: "Delete shared-mode workspace occupancy rows whose owner agent is offline or gone.",
+    intervalMs: REAP_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      const prunedSharedWorkspaceIds = pruneOrphanedSharedWorkspaces();
+      return { prunedSharedWorkspaceIds };
+    },
+  },
+  {
+    id: "workspace-conflict-scan",
+    title: "Workspace conflict scan",
+    description: "Self-healing backstop (#237): probe worktrees for divergence/freshness and auto-flag conflicts. The turn-end stop-hook drives the common case; this catches dropped events, restarts, and direct human commits.",
+    intervalMs: CONFLICT_SCAN_INTERVAL_MS,
+    runOnStart: true,
+    timeoutMs: 60 * 1000,
+    handler: scanWorkspaceConflicts,
+  },
+  {
+    id: "workspace-auto-merge",
+    title: "Workspace auto-merge",
+    description: "Self-healing backstop (#237): land any non-conflicting ready/review_requested worktree under the per-repo lease (rebasing when base moved on); only real/unknown conflicts go to the steward. The on-ready/on-settle event chain drives the common case; runOnStart reclaims rows stranded across a relay restart.",
+    intervalMs: WORKSPACE_AUTO_MERGE_INTERVAL_MS,
+    runOnStart: true,
+    timeoutMs: 60 * 1000,
+    handler: autoMergeCleanFastForwards,
+  },
+  {
+    id: "workspace-gc",
+    title: "Workspace GC",
+    description: "Prune terminal workspace rows past retention, auto-abandon stale review_requested worktrees, and trigger git worktree prune on orchestrators.",
+    intervalMs: WORKSPACE_GC_INTERVAL_MS,
+    runOnStart: false,
+    timeoutMs: 60 * 1000,
+    handler: workspaceGC,
+  },
+  {
+    id: "workspace-orphan-reaper",
+    title: "Workspace orphan reaper",
+    description: "Reconcile disk<->DB: reap orphaned worktrees whose work has landed (or is empty), flag orphans holding un-landed work as needs-attention instead of deleting, and report rows whose worktree vanished. git worktree prune can't do this - it no-ops while the directory still exists.",
+    intervalMs: WORKSPACE_ORPHAN_REAPER_INTERVAL_MS,
+    runOnStart: true,
+    timeoutMs: 2 * 60 * 1000,
+    handler: reapOrphanedWorktrees,
+  },
+];

package/src/maintenance/orphaned-sessions.ts

@@ -0,0 +1,86 @@
+import { createCommand } from "../commands-db";
+import { createActivityEvent, getAgent, listOrchestrators } from "../db";
+import { emitRelayEvent } from "../events";
+import { orphanGraceMs, orphanReapCooldownMs, orphanReapEnabled } from "./constants";
+
+const orphanTracker = new Map<string, { firstOrphanedAt: number; lastReapAt?: number }>();
+
+export function resetOrphanSessionTrackerForTests(): void {
+  orphanTracker.clear();
+}
+
+// Stop orphaned spawned sessions: process reported alive by the orchestrator, but
+// the relay agent is offline/pruned and self-heal has had its chance.
+export function reapOrphanedSessions(): Record<string, unknown> {
+  const now = Date.now();
+  const grace = orphanGraceMs();
+  const cooldown = orphanReapCooldownMs();
+  const reapEnabled = orphanReapEnabled();
+  const seen = new Set<string>();
+  const reaped: string[] = [];
+  let orphaned = 0;
+
+  for (const orch of listOrchestrators()) {
+    if (orch.status !== "online") continue;
+    for (const agent of orch.managedAgents) {
+      const sessionId = agent.spawnRequestId || agent.tmuxSession || agent.sessionName || agent.agentId;
+      if (!sessionId) continue;
+      const key = `${orch.id}:${sessionId}`;
+      const relayAgent = agent.agentId ? getAgent(agent.agentId) : null;
+      const isOrphan = !relayAgent || relayAgent.status === "offline";
+      if (!isOrphan) { orphanTracker.delete(key); continue; }
+      seen.add(key);
+      orphaned++;
+      const entry = orphanTracker.get(key) ?? { firstOrphanedAt: now };
+      orphanTracker.set(key, entry);
+      if (now - entry.firstOrphanedAt < grace) continue;
+      if (!reapEnabled) continue;
+      if (entry.lastReapAt && now - entry.lastReapAt < cooldown) continue;
+      entry.lastReapAt = now;
+      const command = createCommand({
+        type: "agent.shutdown",
+        source: "system",
+        target: orch.agentId,
+        correlationId: agent.spawnRequestId,
+        params: {
+          action: "shutdown",
+          agentId: agent.agentId,
+          spawnRequestId: agent.spawnRequestId,
+          sessionName: agent.sessionName,
+          tmuxSession: agent.tmuxSession,
+          policyName: agent.policyName,
+          graceful: false,
+          timeoutMs: 10_000,
+          reason: "orphaned-session-reaper",
+          requestedBy: "orphaned-session-reaper",
+          requestedAt: now,
+          orchestratorId: orch.id,
+        },
+      });
+      emitRelayEvent({ type: "command.requested", source: "system", subject: command.id, data: { command } });
+      createActivityEvent({
+        clientId: `orphaned-session-reaper-${key}-${now}`,
+        kind: "state",
+        title: "Orphaned session reaped",
+        body: `${agent.label ?? agent.agentId ?? sessionId}: process still running on ${orch.id}, but its relay agent has been offline > ${Math.round(grace / 60000)}m and did not self-heal - stopping it`,
+        meta: agent.label ?? agent.agentId ?? sessionId,
+        icon: "ti-ghost",
+        view: "orchestrators",
+        agentId: agent.agentId || undefined,
+        metadata: {
+          source: "server",
+          maintenanceJobId: "orphaned-session-reaper",
+          orchestratorId: orch.id,
+          agentId: agent.agentId,
+          spawnRequestId: agent.spawnRequestId,
+          tmuxSession: agent.tmuxSession,
+          commandId: command.id,
+          orphanAgeMs: now - entry.firstOrphanedAt,
+        },
+      });
+      reaped.push(key);
+    }
+  }
+  for (const key of orphanTracker.keys()) if (!seen.has(key)) orphanTracker.delete(key);
+  return { orphaned, reaped, tracked: orphanTracker.size, reapEnabled };
+}

package/src/maintenance/scheduler.ts

@@ -0,0 +1,231 @@
+import { randomUUID } from "node:crypto";
+import { errMessage } from "agent-relay-sdk";
+import { createActivityEvent, getDb } from "../db";
+import { resetRateLimitResumeTrackerForTests } from "../services/rate-limit-resume";
+import { resetTransientTrackerForTests } from "../services/transient-agent-reaper";
+import { DEFAULT_TIMEOUT_MS, SCHEDULER_TICK_MS } from "./constants";
+import { LEGACY_REAPER_JOB_IDS, definitions } from "./jobs";
+import { resetOrphanSessionTrackerForTests } from "./orphaned-sessions";
+import type { MaintenanceJob, MaintenanceJobRun } from "../types";
+import type { MaintenanceJobDefinition, MaintenanceJobRow } from "./types";
+
+let timer: Timer | null = null;
+
+export function resetOrphanTrackerForTests(): void {
+  resetOrphanSessionTrackerForTests();
+  resetTransientTrackerForTests();
+  resetRateLimitResumeTrackerForTests();
+}
+
+export function startMaintenanceScheduler(): void {
+  ensureMaintenanceJobs();
+  void runDueMaintenanceJobs();
+  if (timer) return;
+  timer = setInterval(() => void runDueMaintenanceJobs(), SCHEDULER_TICK_MS);
+}
+
+export function listMaintenanceJobs(): MaintenanceJob[] {
+  ensureMaintenanceJobs();
+  const rows = getDb().query("SELECT * FROM maintenance_jobs ORDER BY title COLLATE NOCASE ASC").all() as MaintenanceJobRow[];
+  const now = Date.now();
+  return rows.map((row) => rowToJob(row, now));
+}
+
+export async function runMaintenanceJobNow(id: string): Promise<MaintenanceJobRun> {
+  ensureMaintenanceJobs();
+  const definition = definitions.find((job) => job.id === id);
+  if (!definition) throw new Error("maintenance job not found");
+  return runJob(definition, { force: true });
+}
+
+export async function runLegacyMaintenanceReaper(): Promise<Record<string, unknown>> {
+  const runs = await runMaintenanceJobsNow(LEGACY_REAPER_JOB_IDS);
+  const result = aggregateRunResults(runs);
+  auditMaintenanceReaperRun(result);
+  return result;
+}
+
+async function runMaintenanceJobsNow(ids: string[]): Promise<MaintenanceJobRun[]> {
+  const runs: MaintenanceJobRun[] = [];
+  for (const id of ids) runs.push(await runMaintenanceJobNow(id));
+  return runs;
+}
+
+async function runDueMaintenanceJobs(): Promise<void> {
+  ensureMaintenanceJobs();
+  const now = Date.now();
+  const rows = getDb().query(`
+    SELECT id FROM maintenance_jobs
+    WHERE enabled = 1
+      AND next_run_at IS NOT NULL
+      AND next_run_at <= ?
+      AND (lease_until IS NULL OR lease_until <= ?)
+    ORDER BY next_run_at ASC
+  `).all(now, now) as Array<{ id: string }>;
+  for (const row of rows) {
+    const definition = definitions.find((job) => job.id === row.id);
+    if (definition) void runJob(definition).catch((error) => {
+      console.warn(`maintenance job ${definition.id} failed: ${errMessage(error)}`);
+    });
+  }
+}
+
+function ensureMaintenanceJobs(): void {
+  const db = getDb();
+  const now = Date.now();
+  const stmt = db.query(`
+    INSERT INTO maintenance_jobs (id, title, description, interval_ms, timeout_ms, enabled, run_on_start, next_run_at, updated_at)
+    VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      title = excluded.title,
+      description = excluded.description,
+      interval_ms = excluded.interval_ms,
+      timeout_ms = excluded.timeout_ms,
+      run_on_start = excluded.run_on_start,
+      next_run_at = COALESCE(maintenance_jobs.next_run_at, excluded.next_run_at),
+      updated_at = excluded.updated_at
+  `);
+  for (const definition of definitions) {
+    stmt.run(
+      definition.id,
+      definition.title,
+      definition.description,
+      definition.intervalMs,
+      definition.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+      definition.runOnStart ? 1 : 0,
+      definition.runOnStart ? now : now + definition.intervalMs,
+      now,
+    );
+  }
+}
+
+async function runJob(definition: MaintenanceJobDefinition, options: { force?: boolean } = {}): Promise<MaintenanceJobRun> {
+  const db = getDb();
+  const startedAt = Date.now();
+  const owner = `${process.pid}-${randomUUID()}`;
+  const timeoutMs = definition.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const leaseUntil = startedAt + timeoutMs + 5_000;
+  const claim = db.query(`
+    UPDATE maintenance_jobs
+    SET lease_owner = ?, lease_until = ?, last_status = 'running', updated_at = ?
+    WHERE id = ?
+      AND enabled = 1
+      AND (lease_until IS NULL OR lease_until <= ?)
+      ${options.force ? "" : "AND (next_run_at IS NULL OR next_run_at <= ?)"}
+  `).run(owner, leaseUntil, startedAt, definition.id, startedAt, ...(options.force ? [] : [startedAt]));
+  if (claim.changes === 0) {
+    const state = listMaintenanceJobs().find((job) => job.id === definition.id);
+    return { id: definition.id, status: "skipped", startedAt, finishedAt: Date.now(), durationMs: 0, result: { reason: "not due or already running", state } };
+  }
+
+  try {
+    const result = await withTimeout(Promise.resolve(definition.handler()), timeoutMs);
+    const finishedAt = Date.now();
+    const durationMs = finishedAt - startedAt;
+    db.query(`
+      UPDATE maintenance_jobs
+      SET last_run_at = ?, next_run_at = ?, last_duration_ms = ?, last_status = 'succeeded',
+          last_error = NULL, last_result = ?, consecutive_failures = 0,
+          lease_owner = NULL, lease_until = NULL, updated_at = ?
+      WHERE id = ?
+    `).run(finishedAt, finishedAt + definition.intervalMs, durationMs, JSON.stringify(result), finishedAt, definition.id);
+    return { id: definition.id, status: "succeeded", startedAt, finishedAt, durationMs, result };
+  } catch (error) {
+    const finishedAt = Date.now();
+    const durationMs = finishedAt - startedAt;
+    const message = errMessage(error);
+    db.query(`
+      UPDATE maintenance_jobs
+      SET last_run_at = ?, next_run_at = ?, last_duration_ms = ?, last_status = 'failed',
+          last_error = ?, consecutive_failures = consecutive_failures + 1,
+          lease_owner = NULL, lease_until = NULL, updated_at = ?
+      WHERE id = ?
+    `).run(finishedAt, finishedAt + definition.intervalMs, durationMs, message, finishedAt, definition.id);
+    return { id: definition.id, status: "failed", startedAt, finishedAt, durationMs, error: message };
+  }
+}
+
+async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  let timeout: Timer | undefined;
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => reject(new Error(`maintenance job timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
+
+function rowToJob(row: MaintenanceJobRow, now: number): MaintenanceJob {
+  const leaseUntil = row.lease_until ?? undefined;
+  return {
+    id: row.id,
+    title: row.title,
+    description: row.description ?? undefined,
+    intervalMs: row.interval_ms,
+    timeoutMs: row.timeout_ms,
+    enabled: row.enabled === 1,
+    runOnStart: row.run_on_start === 1,
+    lastRunAt: row.last_run_at ?? undefined,
+    nextRunAt: row.next_run_at ?? undefined,
+    lastDurationMs: row.last_duration_ms ?? undefined,
+    lastStatus: leaseUntil && leaseUntil > now ? "running" : row.last_status as MaintenanceJob["lastStatus"],
+    lastError: row.last_error ?? undefined,
+    lastResult: parseJsonObject(row.last_result),
+    consecutiveFailures: row.consecutive_failures,
+    running: Boolean(leaseUntil && leaseUntil > now),
+    leaseUntil,
+    updatedAt: row.updated_at,
+  };
+}
+
+function aggregateRunResults(runs: MaintenanceJobRun[]): Record<string, unknown> {
+  const aggregate: Record<string, unknown> = { ok: true, runs };
+  for (const run of runs) {
+    if (!run.result) continue;
+    for (const [key, value] of Object.entries(run.result)) {
+      if (Array.isArray(value)) {
+        const current = Array.isArray(aggregate[key]) ? aggregate[key] as unknown[] : [];
+        aggregate[key] = [...current, ...value];
+      } else if (typeof value === "number") {
+        aggregate[key] = Number(aggregate[key] ?? 0) + value;
+      } else if (value !== undefined) {
+        aggregate[key] = value;
+      }
+    }
+  }
+  return aggregate;
+}
+
+function auditMaintenanceReaperRun(result: Record<string, unknown>): void {
+  const reapedAgentIds = Array.isArray(result.reapedAgentIds) ? result.reapedAgentIds : [];
+  const reapedOrchestratorIds = Array.isArray(result.reapedOrchestratorIds) ? result.reapedOrchestratorIds : [];
+  const releasedMessageIds = Array.isArray(result.releasedMessageIds) ? result.releasedMessageIds : [];
+  const releasedTaskIds = Array.isArray(result.releasedTaskIds) ? result.releasedTaskIds : [];
+  const releasedOrphanedTaskIds = Array.isArray(result.releasedOrphanedTaskIds) ? result.releasedOrphanedTaskIds : [];
+  const expiredQueuedMessageIds = Array.isArray(result.expiredQueuedMessageIds) ? result.expiredQueuedMessageIds : [];
+  const expiredCommandIds = Array.isArray(result.expiredCommandIds) ? result.expiredCommandIds : [];
+  const expiredArtifactIds = Array.isArray(result.expiredArtifactIds) ? result.expiredArtifactIds : [];
+  try {
+    createActivityEvent({
+      clientId: "server-maintenance-reaper-" + Date.now(),
+      kind: "state",
+      title: "Maintenance reaper run",
+      body: `${reapedAgentIds.length} stale agent(s), ${reapedOrchestratorIds.length} stale orchestrator(s), ${releasedMessageIds.length} message claim(s), ${releasedTaskIds.length} task claim(s), ${releasedOrphanedTaskIds.length} orphaned task(s), ${expiredQueuedMessageIds.length} queued message(s), ${expiredCommandIds.length} command(s), ${expiredArtifactIds.length} artifact(s)`,
+      icon: "ti-broom",
+      view: "activity",
+      metadata: { source: "server", result },
+    });
+  } catch {}
+}
+
+function parseJsonObject(raw: string | null): Record<string, unknown> | undefined {
+  if (!raw) return undefined;
+  try {
+    const value = JSON.parse(raw);
+    return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+  } catch {
+    return undefined;
+  }
+}