agent-relay-server 0.62.2 → 0.63.0

package/src/maintenance/teams.ts

@@ -0,0 +1,65 @@
+import {
+  clearTeamZeroMembersSince,
+  countLiveTeamMembers,
+  createActivityEvent,
+  listActiveTeams,
+  setTeamZeroMembersSince,
+} from "../db";
+import { STALE_TTL_MS } from "../config";
+import { dissolveTeamById } from "../services/team";
+import { teamReapGraceMs } from "./constants";
+
+export function reapZeroMemberTeams(): Record<string, unknown> {
+  const now = Date.now();
+  const cutoff = now - STALE_TTL_MS;
+  const grace = teamReapGraceMs();
+  const reapedTeamIds: string[] = [];
+  const trackedTeamIds: string[] = [];
+  const clearedTeamIds: string[] = [];
+  let activeTeams = 0;
+  let zeroMemberTeams = 0;
+
+  for (const team of listActiveTeams()) {
+    activeTeams++;
+    const live = countLiveTeamMembers(team.id, cutoff);
+    if (live > 0) {
+      if (team.zeroMembersSince !== undefined) {
+        clearTeamZeroMembersSince(team.id);
+        clearedTeamIds.push(team.id);
+      }
+      continue;
+    }
+
+    zeroMemberTeams++;
+    if (team.zeroMembersSince === undefined) {
+      setTeamZeroMembersSince(team.id, now);
+      trackedTeamIds.push(team.id);
+      continue;
+    }
+    if (now - team.zeroMembersSince < grace) continue;
+
+    const outcomeSummary = `auto-reaped: zero live members for ${grace}ms`;
+    // Leave outcomeStatus null (unknown): an abandoned team is NOT a failed one.
+    const dissolved = dissolveTeamById(team.id, { outcomeSummary });
+    createActivityEvent({
+      clientId: `team-reap-${team.id}-${now}`,
+      kind: "state",
+      title: "Team auto-reaped",
+      body: outcomeSummary,
+      meta: team.id,
+      icon: "ti-users-minus",
+      view: "teams",
+      agentId: dissolved.coordinatorId,
+      metadata: {
+        source: "server",
+        maintenanceJobId: "team-reap",
+        teamId: team.id,
+        zeroMembersSince: team.zeroMembersSince,
+        graceMs: grace,
+      },
+    });
+    reapedTeamIds.push(team.id);
+  }
+
+  return { activeTeams, zeroMemberTeams, trackedTeamIds, clearedTeamIds, reapedTeamIds, graceMs: grace };
+}

package/src/maintenance/types.ts

@@ -0,0 +1,31 @@
+import type { MaintenanceJob } from "../types";
+
+export interface MaintenanceJobDefinition {
+  id: string;
+  title: string;
+  description: string;
+  intervalMs: number;
+  timeoutMs?: number;
+  runOnStart?: boolean;
+  handler: () => Promise<Record<string, unknown>> | Record<string, unknown>;
+}
+
+export interface MaintenanceJobRow {
+  id: string;
+  title: string;
+  description: string | null;
+  interval_ms: number;
+  timeout_ms: number;
+  enabled: number;
+  run_on_start: number;
+  last_run_at: number | null;
+  next_run_at: number | null;
+  last_duration_ms: number | null;
+  last_status: string;
+  last_error: string | null;
+  last_result: string | null;
+  consecutive_failures: number;
+  lease_owner: string | null;
+  lease_until: number | null;
+  updated_at: number;
+}

package/src/maintenance/workspaces.ts

@@ -0,0 +1,251 @@
+import { createCommand } from "../commands-db";
+import {
+  createActivityEvent,
+  deleteWorkspace,
+  getAgent,
+  getRepoSteward,
+  getWorkspace,
+  listOrchestrators,
+  listWorkspaces,
+  patchWorkspaceMetadata,
+  reelectRepoSteward,
+  releaseExpiredMergeLeases,
+  updateWorkspaceStatus,
+} from "../db";
+import type { WorkspaceMergePreview, WorkspaceStatus } from "../types";
+import { reconcileLandedWorkspace } from "../branch-landed";
+import { getStewardConfig } from "../config-store";
+import { preparePrCompletionScan, reconcileLandedPr } from "../workspace-pr-completion";
+import { applyBranchFreshness, fetchHostMergePreview, workspacePathWithinBase } from "../workspace-probe";
+import { wakeRepoSteward } from "../workspace-auto-merge";
+import {
+  CONFLICT_SCAN_STATUSES,
+  DAY_MS,
+  LANDED_RECONCILE_STATUSES,
+  STRANDABLE_STATUSES,
+  TERMINAL_WORKSPACE_STATUSES,
+  WORKSPACE_RETENTION_MS,
+  WORKSPACE_REVIEW_TTL_MS,
+  stewardEscalationMs,
+  stewardFallbackTarget,
+} from "./constants";
+import { emitCommand, notifyTarget } from "./events";
+
+export async function scanWorkspaceConflicts(): Promise<Record<string, unknown>> {
+  const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.apiUrl);
+  if (!orchestrators.length) return { scanned: 0, skipped: "no online orchestrators" };
+
+  const candidates = listWorkspaces().filter(
+    (ws) => ws.mode === "isolated" && Boolean(ws.worktreePath) && CONFLICT_SCAN_STATUSES.has(ws.status),
+  );
+  const flagged: string[] = [], cleared: string[] = [], merged: string[] = [], notifiedStewards: string[] = [], prRecycled: string[] = [];
+  const reviewerArmed: string[] = [], reviewerChangesRequested: string[] = [], relayPrMerged: string[] = [], relayPrWaiting: string[] = [];
+
+  for (const ws of candidates) {
+    const orch = orchestrators.find((candidate) => workspacePathWithinBase(ws.sourceCwd, candidate.baseDir));
+    if (!orch?.apiUrl) continue;
+    const preview = await fetchHostMergePreview(orch.apiUrl, ws, { checkPr: LANDED_RECONCILE_STATUSES.has(ws.status) });
+    if (!preview || (preview as { available?: false }).available === false) continue;
+    const p = preview as WorkspaceMergePreview;
+    if (p.error || p.missing) continue;
+
+    const meta = ws.metadata as Record<string, unknown>;
+    applyBranchFreshness(ws, p);
+
+    const landed = p.landed === true || p.prMerged === true;
+    if (landed && LANDED_RECONCILE_STATUSES.has(ws.status)) {
+      const outcome = reconcileLandedPr(ws, p);
+      if (outcome.kind === "recycle") { emitCommand(outcome.command); prRecycled.push(ws.id); continue; }
+      if (outcome.kind === "wait") { relayPrWaiting.push(`${ws.id}:${outcome.reason}`); continue; }
+      reconcileLandedWorkspace(ws, p);
+      merged.push(ws.id);
+      continue;
+    }
+
+    const prAction = preparePrCompletionScan(ws, p, orch.agentId);
+    if (prAction.armCommand) { emitCommand(prAction.armCommand); reviewerArmed.push(ws.id); continue; }
+    if (prAction.notifiedOwner) { reviewerChangesRequested.push(ws.id); continue; }
+    if (prAction.refreshCommand) { emitCommand(prAction.refreshCommand); continue; }
+    if (prAction.mergeCommand) { emitCommand(prAction.mergeCommand); relayPrMerged.push(ws.id); continue; }
+    if (prAction.waitReason) relayPrWaiting.push(`${ws.id}:${prAction.waitReason}`);
+    if (prAction.stop) continue;
+
+    if (p.conflict === undefined) continue;
+
+    if (p.conflict === true && ws.status !== "conflict") {
+      updateWorkspaceStatus(ws.id, "conflict", {
+        autoConflict: true,
+        preConflictStatus: ws.status,
+        conflictAhead: p.ahead,
+        conflictBehind: p.behind,
+        conflictBaseRef: p.baseRef,
+        conflictDetectedAt: Date.now(),
+      });
+      flagged.push(ws.id);
+      createActivityEvent({
+        clientId: "server-workspace-" + ws.id + "-conflict-" + Date.now(),
+        kind: "state",
+        title: "Merge conflict detected",
+        body: `${ws.branch ?? ws.id} can no longer merge cleanly into ${p.baseRef ?? "base"}`,
+        meta: ws.branch ?? ws.id,
+        icon: "ti-alert-triangle",
+        view: "orchestrators",
+        metadata: { source: "server", maintenanceJobId: "workspace-conflict-scan", workspaceId: ws.id, ahead: p.ahead, behind: p.behind },
+      });
+      if (getStewardConfig().enabled) {
+        const woke = wakeRepoSteward(getWorkspace(ws.id) ?? ws, "conflict");
+        if (woke) notifiedStewards.push(woke);
+      } else if (ws.stewardAgentId) {
+        try {
+          notifyTarget(
+            "workspace.conflict",
+            { workspaceId: ws.id, repoRoot: ws.repoRoot },
+            ws.stewardAgentId,
+            "Workspace merge conflict",
+            `Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} can no longer merge cleanly into ${p.baseRef ?? "base"} (${p.ahead ?? "?"} ahead, ${p.behind ?? "?"} behind). As repo steward, please coordinate resolution.`,
+            { kind: "workspace.conflict", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: p.baseRef, ahead: p.ahead, behind: p.behind },
+          );
+          notifiedStewards.push(ws.stewardAgentId);
+        } catch {}
+      }
+    } else if (p.conflict === false && ws.status === "conflict" && meta.autoConflict === true) {
+      const prior = typeof meta.preConflictStatus === "string" && meta.preConflictStatus !== "conflict"
+        ? meta.preConflictStatus as WorkspaceStatus
+        : "active";
+      updateWorkspaceStatus(ws.id, prior, { autoConflict: false, conflictClearedAt: Date.now() });
+      cleared.push(ws.id);
+    }
+  }
+
+  return { scanned: candidates.length, flagged, cleared, merged, prRecycled, notifiedStewards, reviewerArmed, reviewerChangesRequested, relayPrMerged, relayPrWaiting };
+}
+
+export async function workspaceGC(): Promise<Record<string, unknown>> {
+  const now = Date.now();
+  const cutoff = now - WORKSPACE_RETENTION_MS;
+  const reviewCutoff = now - WORKSPACE_REVIEW_TTL_MS;
+  const releasedLeaseRepos = releaseExpiredMergeLeases(now);
+
+  const all = listWorkspaces();
+  const terminalIds: string[] = [];
+  for (const ws of all) {
+    if (TERMINAL_WORKSPACE_STATUSES.has(ws.status) && ws.updatedAt < cutoff) {
+      deleteWorkspace(ws.id);
+      terminalIds.push(ws.id);
+    }
+  }
+
+  const escalatedIds: string[] = [];
+  const reassignedIds: string[] = [];
+  const escalationTargets: string[] = [];
+  const escalationMs = stewardEscalationMs();
+  const fallbackTarget = stewardFallbackTarget();
+  for (const ws of all) {
+    if (!STRANDABLE_STATUSES.has(ws.status) || ws.mode !== "isolated" || !ws.worktreePath) continue;
+    reelectRepoSteward(ws.repoRoot);
+    const fresh = getWorkspace(ws.id);
+    if (!fresh || !STRANDABLE_STATUSES.has(fresh.status)) continue;
+    const meta = fresh.metadata as Record<string, unknown>;
+    const steward = fresh.stewardAgentId;
+    const stewardOnline = Boolean(steward && getAgent(steward) && getAgent(steward)!.status !== "offline");
+    const strandedAt = typeof meta.strandedAt === "number" ? meta.strandedAt : undefined;
+
+    if (stewardOnline) {
+      if (strandedAt !== undefined && meta.strandedNotifiedSteward !== steward) {
+        const sent = notifyTarget(
+          "workspace.steward_reassigned",
+          { workspaceId: fresh.id, repoRoot: fresh.repoRoot },
+          steward!,
+          "Workspace stewardship reassigned",
+          `You are now steward for ${fresh.repoRoot}. Workspace \`${fresh.branch ?? fresh.id}\` is ${fresh.status} and was stranded without an online steward - please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"}.`,
+          { kind: "workspace.steward-reassigned", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status },
+        );
+        if (sent) reassignedIds.push(fresh.id);
+      }
+      patchWorkspaceMetadata(fresh.id, { strandedAt: undefined, escalatedAt: undefined, strandedNotifiedSteward: steward });
+      continue;
+    }
+
+    if (strandedAt === undefined) { patchWorkspaceMetadata(fresh.id, { strandedAt: now }); continue; }
+    if (now - strandedAt < escalationMs || meta.escalatedAt) continue;
+    const sent = notifyTarget(
+      "workspace.stranded_escalation",
+      { workspaceId: fresh.id, repoRoot: fresh.repoRoot },
+      fallbackTarget,
+      "Stranded workspace needs an owner",
+      `Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} is ${fresh.status} with no online steward (all repo agents offline) for ${Math.round((now - strandedAt) / (60 * 60 * 1000))}h. Please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"} or clean up the worktree.`,
+      { kind: "workspace.stranded-escalation", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt },
+    );
+    if (sent) escalationTargets.push(sent);
+    patchWorkspaceMetadata(fresh.id, { escalatedAt: now });
+    escalatedIds.push(fresh.id);
+    createActivityEvent({
+      clientId: `workspace-gc-escalate-${fresh.id}-${now}`,
+      kind: "state",
+      title: "Workspace escalated",
+      body: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} - stranded ${fresh.status} escalated${fallbackTarget ? ` to ${fallbackTarget}` : " (no fallback configured)"}`,
+      meta: fresh.branch ?? fresh.id,
+      icon: "ti-alert-octagon",
+      view: "orchestrators",
+      metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: fresh.id, fallback: fallbackTarget || null },
+    });
+  }
+
+  const abandonedIds: string[] = [];
+  const notifiedStewards: string[] = [];
+  for (const ws of all) {
+    if (ws.status === "review_requested" && ws.updatedAt < reviewCutoff) {
+      updateWorkspaceStatus(ws.id, "abandoned", { autoAbandoned: true, abandonedReason: "review_requested TTL exceeded", abandonedAt: now });
+      abandonedIds.push(ws.id);
+      const target = ws.stewardAgentId ?? fallbackTarget;
+      const sent = notifyTarget(
+        "workspace.auto_abandoned",
+        { workspaceId: ws.id, repoRoot: ws.repoRoot },
+        target,
+        "Workspace auto-abandoned",
+        `Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} was auto-abandoned after ${Math.round(WORKSPACE_REVIEW_TTL_MS / DAY_MS)}d without steward action. Run workspace cleanup to reclaim the worktree.`,
+        { kind: "workspace.auto-abandoned", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch },
+      );
+      if (sent) notifiedStewards.push(sent);
+      createActivityEvent({
+        clientId: `workspace-gc-abandon-${ws.id}-${now}`,
+        kind: "state",
+        title: "Workspace auto-abandoned",
+        body: `${ws.branch ?? ws.id} in ${ws.repoRoot} - review_requested for ${Math.round((now - ws.updatedAt) / DAY_MS)}d`,
+        meta: ws.branch ?? ws.id,
+        icon: "ti-clock-x",
+        view: "orchestrators",
+        metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: ws.id },
+      });
+    }
+  }
+
+  const orchestrators = listOrchestrators().filter((orch) => orch.status === "online" && orch.agentId);
+  const reposWithWorkspaces = new Set(
+    all.filter((ws) => ws.mode === "isolated" && Boolean(ws.worktreePath)).map((ws) => ws.repoRoot),
+  );
+  const pruneCommands: string[] = [];
+  for (const repoRoot of reposWithWorkspaces) {
+    const orch = orchestrators.find((candidate) => workspacePathWithinBase(repoRoot, candidate.baseDir));
+    if (!orch?.agentId) continue;
+    const command = createCommand({
+      type: "workspace.prune",
+      source: "system",
+      target: orch.agentId,
+      params: { repoRoot, requestedBy: "workspace-gc", requestedAt: now },
+    });
+    emitCommand(command);
+    pruneCommands.push(command.id);
+  }
+
+  return {
+    prunedTerminal: terminalIds,
+    autoAbandoned: abandonedIds,
+    notifiedStewards,
+    pruneCommands,
+    releasedLeaseRepos,
+    escalated: escalatedIds,
+    reassigned: reassignedIds,
+    escalationTargets,
+  };
+}