agent-relay-server 0.120.5 → 0.121.0

package/src/db/retention.ts

@@ -0,0 +1,130 @@
+import { getDb, getDbPath, runDbMaintenanceWorker } from "./connection.ts";
+
+interface RetentionPruneInput {
+  messageCutoff: number;
+  activityCutoff: number;
+  activityMaxRows: number;
+  spawnCommandCutoff: number;
+  batchSize: number;
+}
+
+interface RetentionPruneResult {
+  prunedMessages: number;
+  prunedActivityEvents: number;
+  prunedSpawnCommands: number;
+  offThread: boolean;
+}
+
+function boundedBatchSize(batchSize: number): number {
+  return Math.min(Math.max(Math.trunc(batchSize), 1), 50_000);
+}
+
+function pruneMessagesBefore(cutoff: number, batchSize: number): number {
+  return getDb().transaction(() => {
+    getDb().query(`
+      UPDATE messages
+      SET reply_to = NULL
+      WHERE reply_to IN (
+        SELECT id FROM messages
+        WHERE created_at < ?
+        ORDER BY created_at ASC, id ASC
+        LIMIT ?
+      )
+    `).run(cutoff, batchSize);
+    return getDb().query(`
+      DELETE FROM messages
+      WHERE id IN (
+        SELECT id FROM messages
+        WHERE created_at < ?
+        ORDER BY created_at ASC, id ASC
+        LIMIT ?
+      )
+    `).run(cutoff, batchSize).changes;
+  })();
+}
+
+function pruneActivityEvents(input: RetentionPruneInput): number {
+  return getDb().query(`
+    DELETE FROM activity_events
+    WHERE id IN (
+      SELECT ae.id
+      FROM activity_events ae
+      WHERE (
+          ae.created_at < ?
+          OR ae.id NOT IN (
+            SELECT id
+            FROM activity_events
+            ORDER BY created_at DESC, id DESC
+            LIMIT ?
+          )
+        )
+        AND NOT (
+          ae.kind = 'state'
+          AND ae.title = 'Agent registered'
+          AND EXISTS (
+            SELECT 1
+            FROM commands c
+            WHERE c.type = 'agent.spawn'
+              AND c.status = 'succeeded'
+              AND coalesce(json_extract(c.params, '$.registrationConfirmed'), 0) != 1
+              AND (
+                c.correlation_id = json_extract(ae.metadata, '$.spawnRequestId')
+                OR json_extract(c.params, '$.spawnRequestId') = json_extract(ae.metadata, '$.spawnRequestId')
+              )
+          )
+        )
+      ORDER BY ae.created_at ASC, ae.id ASC
+      LIMIT ?
+    )
+  `).run(input.activityCutoff, input.activityMaxRows, input.batchSize).changes;
+}
+
+function pruneTerminalSpawnCommands(cutoff: number, batchSize: number): number {
+  return getDb().query(`
+    DELETE FROM commands
+    WHERE id IN (
+      SELECT c.id
+      FROM commands c
+      WHERE c.type = 'agent.spawn'
+        AND c.status IN ('succeeded', 'failed', 'canceled', 'timed_out')
+        AND c.updated_at < ?
+        AND NOT (
+          c.status = 'succeeded'
+          AND coalesce(json_extract(c.params, '$.registrationConfirmed'), 0) != 1
+        )
+        AND NOT EXISTS (
+          SELECT 1
+          FROM managed_agent_state mas
+          WHERE mas.spawn_request_id IS NOT NULL
+            AND (
+              mas.spawn_request_id = c.correlation_id
+              OR mas.spawn_request_id = json_extract(c.params, '$.spawnRequestId')
+            )
+        )
+      ORDER BY c.updated_at ASC, c.created_at ASC
+      LIMIT ?
+    )
+  `).run(cutoff, batchSize).changes;
+}
+
+export function pruneRetentionBatch(input: RetentionPruneInput): Omit<RetentionPruneResult, "offThread"> {
+  const batchSize = boundedBatchSize(input.batchSize);
+  return getDb().transaction(() => ({
+    prunedMessages: pruneMessagesBefore(input.messageCutoff, batchSize),
+    prunedActivityEvents: pruneActivityEvents({ ...input, batchSize }),
+    prunedSpawnCommands: pruneTerminalSpawnCommands(input.spawnCommandCutoff, batchSize),
+  }))();
+}
+
+export async function pruneRetentionBatchAsync(input: RetentionPruneInput, options: { signal?: AbortSignal } = {}): Promise<RetentionPruneResult> {
+  const normalized = { ...input, batchSize: boundedBatchSize(input.batchSize) };
+  if (!getDbPath() || getDbPath() === ":memory:") {
+    return { ...pruneRetentionBatch(normalized), offThread: false };
+  }
+  const result = await runDbMaintenanceWorker<Omit<RetentionPruneResult, "offThread">>(
+    "retention-prune",
+    [JSON.stringify(normalized)],
+    options.signal,
+  );
+  return { ...result, offThread: true };
+}

package/src/db/schema.ts

@@ -114,6 +114,7 @@ export function initDb(path: string = "agent-relay.db"): Database {
       last_seen INTEGER NOT NULL,
       created_at INTEGER NOT NULL
     );
+    CREATE INDEX IF NOT EXISTS idx_agents_spawn_request ON agents(json_extract(meta, '$.spawnRequestId'));
     CREATE TABLE IF NOT EXISTS notification_subscriptions (
       id TEXT PRIMARY KEY,
       agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
@@ -865,6 +866,9 @@ export function initDb(path: string = "agent-relay.db"): Database {
     CREATE INDEX IF NOT EXISTS idx_activity_operator ON activity_events(operator_id, created_at);
     CREATE INDEX IF NOT EXISTS idx_activity_created ON activity_events(created_at);
     CREATE INDEX IF NOT EXISTS idx_activity_agent ON activity_events(agent_id, created_at); CREATE INDEX IF NOT EXISTS idx_activity_policy_created ON activity_events(json_extract(metadata, '$.policyName'), created_at DESC, id DESC);
+    CREATE INDEX IF NOT EXISTS idx_activity_registered_spawn_request
+      ON activity_events(json_extract(metadata, '$.spawnRequestId'))
+      WHERE kind = 'state' AND title = 'Agent registered';
     -- issue_ref_id/issue_action indexes are created in applyMigrations() AFTER the ALTER — never inline here (#483: inline index over a not-yet-migrated column crashes initDb on existing DBs).
 
     CREATE TABLE IF NOT EXISTS channels (
@@ -942,6 +946,7 @@ export function initDb(path: string = "agent-relay.db"): Database {
     CREATE INDEX IF NOT EXISTS idx_commands_status ON commands(status);
     CREATE INDEX IF NOT EXISTS idx_commands_created ON commands(created_at);
     CREATE INDEX IF NOT EXISTS idx_commands_correlation ON commands(correlation_id);
+    CREATE INDEX IF NOT EXISTS idx_commands_spawn_watchdog ON commands(type, status, updated_at);
 
     CREATE TABLE IF NOT EXISTS recipe_instances (
       id TEXT PRIMARY KEY,

package/src/db/stats.ts

@@ -16,10 +16,11 @@ import { STALE_TTL_MS, DAY_MS, CLAIM_LEASE_MS, POOL_CLAIM_LEASE_MS, WORKSPACE_ME
 import { matchAgents } from "../agent-ref";
 import { listAgents } from "./agents.ts";
 import { listChannels } from "./channels.ts";
-import { countExpiredUndeliveredMessages } from "./delivery.ts";
+import { summarizeExpiredUndeliveredMessages } from "./delivery.ts";
 import { messageTaskClosed } from "./claim-sql.ts";
 import { claimTaskWhere } from "./task-core.ts";
 import { getDb } from "./connection.ts";
+import { getEventLoopLagSnapshot } from "../event-loop-lag";
 import type {
   AgentCard,
   ActivityEvent,
@@ -408,6 +409,27 @@ export function getHealth(now: number = Date.now()): HealthReport {
     checks.push({ name: "database", status: "error", detail: e instanceof Error ? e.message : "database check failed" });
   }
 
+  const eventLoopLag = getEventLoopLagSnapshot();
+  const eventLoopLagHealthMaxMs = Math.max(eventLoopLag.maxMs, eventLoopLag.maxSinceLastReadMs);
+  const eventLoopLagWarn = eventLoopLag.warnThresholdMs > 0 && eventLoopLag.sampleCount > 0 && eventLoopLagHealthMaxMs >= eventLoopLag.warnThresholdMs;
+  checks.push({
+    name: "event-loop-lag",
+    status: eventLoopLagWarn ? "warn" : "ok",
+    detail: eventLoopLagWarn
+      ? `event loop lag max ${Math.round(eventLoopLagHealthMaxMs)}ms over ${eventLoopLag.sampleCount}/${eventLoopLag.windowSamples} sample(s)`
+      : undefined,
+    metrics: {
+      currentMs: eventLoopLag.currentMs,
+      maxMs: eventLoopLag.maxMs,
+      maxSinceLastReadMs: eventLoopLag.maxSinceLastReadMs,
+      sampleCount: eventLoopLag.sampleCount,
+      windowSamples: eventLoopLag.windowSamples,
+      sampleIntervalMs: eventLoopLag.sampleIntervalMs,
+      warnThresholdMs: eventLoopLag.warnThresholdMs,
+      ...(eventLoopLag.lastSampleAt === undefined ? {} : { lastSampleAt: eventLoopLag.lastSampleAt }),
+    },
+  });
+
   const staleLiveAgents = (getDb()
     .query("SELECT COUNT(*) as c FROM agents WHERE status != 'offline' AND last_seen <= ? AND id NOT IN ('user', 'system')")
     .get(now - STALE_TTL_MS) as any).c as number;
@@ -489,14 +511,17 @@ export function getHealth(now: number = Date.now()): HealthReport {
       : undefined,
   });
 
-  const expiredUndelivered = countExpiredUndeliveredMessages(now);
+  const expiredUndelivered = summarizeExpiredUndeliveredMessages(now);
   checks.push({
     name: "expired-undelivered-messages",
-    status: expiredUndelivered > 0 ? "warn" : "ok",
-    count: expiredUndelivered,
-    detail: expiredUndelivered > 0
-      ? `${expiredUndelivered} queued message(s) expired undelivered at max_age in the last 24h (silent data loss)`
-      : undefined,
+    status: expiredUndelivered.unexplainedCount > 0 ? "warn" : "ok",
+    count: expiredUndelivered.unexplainedCount,
+    detail: expiredUndelivered.unexplainedCount > 0
+      ? `${expiredUndelivered.unexplainedCount} queued message(s) expired undelivered at max_age in the last 24h (silent data loss)${expiredUndelivered.explainedManagedPolicyCount > 0 ? `; ${expiredUndelivered.explainedManagedPolicyCount} managed-policy expiry(s) explained by policy health` : ""}`
+      : expiredUndelivered.explainedManagedPolicyCount > 0
+        ? `${expiredUndelivered.explainedManagedPolicyCount} queued managed-policy message(s) expired while the policy target was unavailable; tracked by spawn-policy health, not silent data loss`
+        : undefined,
+    subjects: expiredUndelivered.subjects.length > 0 ? expiredUndelivered.subjects : undefined,
   });
 
   const rootTokenRuntimeAgents = listAgents().filter((agent) => {
@@ -539,4 +564,3 @@ export function getHealth(now: number = Date.now()): HealthReport {
       : "ok";
   return { status, version: VERSION, generatedAt: now, checks };
 }
-

package/src/db/task-entity.ts

@@ -152,7 +152,7 @@ export interface CreateDeliverableTaskInput {
   intent: string;
   complexity?: TaskComplexity;
   context?: string;
-  doneWhen?: string;
+  doneWhen?: string; branch?: string;
   teamId?: string;
   projectId?: string;
   repo?: string;
@@ -171,25 +171,24 @@ export function createDeliverableTask(input: CreateDeliverableTaskInput): TaskDe
   const dispatch = cleanDispatch(input.dispatch);
   const complexity = cleanTaskComplexity(input.complexity ?? dispatch?.complexity, "complexity") ?? DEFAULT_TASK_COMPLEXITY;
   const context = optionalText(input.context, "context", 1_000_000);
+  const branch = optionalText(input.branch, "branch", 500);
   const taskProject = cleanTaskProject(input);
   // Legacy NOT-NULL columns have no DB defaults (#834 finding 6) — set them explicitly.
   const result = getDb().query(`
     INSERT INTO tasks (
       source, title, body, severity, status, target, occurrence_count, metadata,
       created_at, updated_at, last_seen_at,
-      kind, intent, complexity, context, latest_steer, deliverable_stage, done_when, team_id, project_id, repo, created_by, dispatch
-    ) VALUES (?, ?, ?, 'info', 'open', ?, 1, '{}', ?, ?, ?, '${DELIVERABLE_TASK_KIND}', ?, ?, ?, ?, 'none', ?, ?, ?, ?, ?, ?)
+      kind, intent, complexity, context, latest_steer, deliverable_stage, done_when, branch, team_id, project_id, repo, created_by, dispatch
+    ) VALUES (?, ?, ?, 'info', 'open', ?, 1, '{}', ?, ?, ?, '${DELIVERABLE_TASK_KIND}', ?, ?, ?, ?, 'none', ?, ?, ?, ?, ?, ?, ?)
   `).run(
     optionalText(input.source, "source", 200) ?? "coordinator",
     title,
     intent,
     optionalText(input.target, "target", 200) ?? createdBy,
-    now, now, now,
-    intent,
-    complexity,
-    context ?? null,
+    now, now, now, intent, complexity, context ?? null,
     optionalText(input.latestSteer, "latestSteer", 1_000_000) ?? null,
     optionalText(input.doneWhen, "doneWhen", 1_000_000) ?? null,
+    branch ?? null,
     optionalText(input.teamId, "teamId", 200) ?? null,
     taskProject.projectId ?? null,
     taskProject.repo ?? null,
@@ -203,7 +202,7 @@ export function createDeliverableTask(input: CreateDeliverableTaskInput): TaskDe
     severity: "info",
     title: `Task created: ${title}`,
     body: intent,
-    metadata: { createdBy, complexity, ...(context ? { context } : {}), ...(taskProject.projectId ? { projectId: taskProject.projectId } : {}), ...(taskProject.repo ? { repo: taskProject.repo } : {}), ...(dispatch ? { dispatch } : {}) },
+    metadata: { createdBy, complexity, ...(context ? { context } : {}), ...(branch ? { branch } : {}), ...(taskProject.projectId ? { projectId: taskProject.projectId } : {}), ...(taskProject.repo ? { repo: taskProject.repo } : {}), ...(dispatch ? { dispatch } : {}) },
   }, now);
   return getTaskDetail(id)!;
 }

package/src/event-loop-lag.ts

@@ -0,0 +1,90 @@
+import {
+  EVENT_LOOP_LAG_LOG_MS,
+  EVENT_LOOP_LAG_SAMPLE_MS,
+  EVENT_LOOP_LAG_WARN_MS,
+  EVENT_LOOP_LAG_WINDOW_SAMPLES,
+} from "./config";
+
+export interface EventLoopLagSnapshot {
+  currentMs: number;
+  maxMs: number;
+  maxSinceLastReadMs: number;
+  sampleCount: number;
+  windowSamples: number;
+  sampleIntervalMs: number;
+  warnThresholdMs: number;
+  lastSampleAt?: number;
+}
+
+const samples = new Array<number>(EVENT_LOOP_LAG_WINDOW_SAMPLES).fill(0);
+let sampleIndex = 0;
+let sampleCount = 0;
+let currentMs = 0;
+let maxSinceLastReadMs = 0;
+let lastSampleAt: number | undefined;
+let timer: ReturnType<typeof setInterval> | undefined;
+let lastTickAt = 0;
+
+export function computeEventLoopLagMs(elapsedMs: number, expectedMs: number): number {
+  if (!Number.isFinite(elapsedMs) || !Number.isFinite(expectedMs)) return 0;
+  return Math.max(0, elapsedMs - expectedMs);
+}
+
+export function recordEventLoopLagSample(lagMs: number, sampledAt: number = Date.now(), logSpike = false): void {
+  const normalized = Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0;
+  currentMs = normalized;
+  maxSinceLastReadMs = Math.max(maxSinceLastReadMs, normalized);
+  lastSampleAt = sampledAt;
+  samples[sampleIndex] = normalized;
+  sampleIndex = (sampleIndex + 1) % samples.length;
+  sampleCount = Math.min(sampleCount + 1, samples.length);
+  if (logSpike && EVENT_LOOP_LAG_LOG_MS > 0 && normalized >= EVENT_LOOP_LAG_LOG_MS) {
+    console.warn(`[health] event-loop lag ${Math.round(normalized)}ms`);
+  }
+}
+
+export function getEventLoopLagSnapshot(): EventLoopLagSnapshot {
+  let maxMs = 0;
+  for (let i = 0; i < sampleCount; i += 1) maxMs = Math.max(maxMs, samples[i] ?? 0);
+  const maxSinceLastRead = maxSinceLastReadMs;
+  maxSinceLastReadMs = 0;
+  return {
+    currentMs,
+    maxMs,
+    maxSinceLastReadMs: maxSinceLastRead,
+    sampleCount,
+    windowSamples: EVENT_LOOP_LAG_WINDOW_SAMPLES,
+    sampleIntervalMs: EVENT_LOOP_LAG_SAMPLE_MS,
+    warnThresholdMs: EVENT_LOOP_LAG_WARN_MS,
+    lastSampleAt,
+  };
+}
+
+export function startEventLoopLagSampler(): void {
+  if (timer) return;
+  lastTickAt = performance.now();
+  timer = setInterval(() => {
+    const now = performance.now();
+    const lagMs = computeEventLoopLagMs(now - lastTickAt, EVENT_LOOP_LAG_SAMPLE_MS);
+    lastTickAt = now;
+    recordEventLoopLagSample(lagMs, Date.now(), true);
+  }, EVENT_LOOP_LAG_SAMPLE_MS);
+  timer.unref?.();
+}
+
+export function stopEventLoopLagSampler(): void {
+  if (!timer) return;
+  clearInterval(timer);
+  timer = undefined;
+}
+
+export function resetEventLoopLagForTests(): void {
+  stopEventLoopLagSampler();
+  samples.fill(0);
+  sampleIndex = 0;
+  sampleCount = 0;
+  currentMs = 0;
+  maxSinceLastReadMs = 0;
+  lastSampleAt = undefined;
+  lastTickAt = 0;
+}

package/src/lifecycle-manager.ts

@@ -2,14 +2,10 @@ import { createCommand } from "./commands-db";
 import { workspaceOwnerOrchestrator } from "./workspace-host";
 import { agentProviderState } from "./provider-state";
 import { createActivityEvent, deleteWorkspace, getAgent, getDb, getOrchestrator, listOrchestrators, listWorkspaces } from "./db";
-import {
-  getManagedAgentState,
-  listSpawnPolicies,
-  updateManagedAgentState,
-  upsertManagedAgentState,
-} from "./config-store";
+import { getManagedAgentState, listSpawnPolicies, updateManagedAgentState, upsertManagedAgentState } from "./config-store";
 import { emitRelayEvent } from "./events";
 import { alwaysReloadTags, reconcileLifecycleContextPressure, requestAgentCompact } from "./lifecycle-context-pressure";
+import { escalateOfflineOrchestrator } from "./managed-policy-escalation";
 import { markManagedAgentRunning, emitManagedState } from "./services/managed-running";
 import { authContextFromSystem } from "./services/auth-context";
 import { emitCommandEvent } from "./command-events";
@@ -189,15 +185,18 @@ export class LifecycleManager {
   spawnAgent(policy: SpawnPolicy, reason = "reconcile"): Command | null {
     const orch = getOrchestrator(policy.orchestratorId);
     if (!orch || orch.status !== "online") {
-      const state = upsertManagedAgentState({
+      const state = getManagedAgentState(policy.name) ?? {
         policyName: policy.name,
-        status: "backoff",
+        status: "stopped" as const,
         orchestratorId: policy.orchestratorId,
         provider: policy.provider,
-        backoffUntil: this.now() + this.backoffDelay(policy, getManagedAgentState(policy.name)) * 1000,
-        lastError: "orchestrator offline",
-      });
-      this.emitState(state);
+        restartCount: 0,
+        consecutiveFailures: 0,
+        updatedAt: this.now(),
+      };
+      this.markBackoff(policy, state, "orchestrator offline");
+      const next = getManagedAgentState(policy.name);
+      if (next) escalateOfflineOrchestrator(policy, next, orch);
       return null;
     }
     const spawnRequestId = generateSpawnRequestId();

package/src/maintenance/jobs.ts

@@ -3,6 +3,7 @@ import { deriveAgentBusyHealth } from "../agent-busy-health";
 import { expireStaleBusAgents } from "../bus";
 import { pruneOutbox } from "../bus-outbox";
 import { expireCommands } from "../commands-db";
+import { getRetentionConfig } from "../retention-config-store";
 import { OFFLINE_PRUNE_MS, REAP_INTERVAL_MS, STALE_TTL_MS } from "../config";
 import {
   createActivityEvent,
@@ -13,7 +14,7 @@ import {
   listStaleQueuedDirectTargets,
   pruneOrphanedSharedWorkspaces,
   pruneOfflineAgents,
-  pruneOldMessagesAsync,
+  pruneRetentionBatchAsync,
   pruneProviderQuotaHistory,
   reapStaleAgents,
   reapStaleOrchestrators,
@@ -27,10 +28,11 @@ import { isAgentUnavailable } from "../db/agent-predicates";
 import { flushQueuedDirectMessages } from "../services/managed-running";
 import { notifyAgentOffline } from "../agent-lifecycle-events";
 import { diagnoseAgentDeath } from "../agent-death-diagnosis";
-import { autoMergeCleanFastForwards, runAutoMergeWatchdog } from "../workspace-auto-merge";
+import { autoMergeCleanFastForwards, runAutoMergeWatchdog, scheduleRepoMerge } from "../workspace-auto-merge";
 import { reapOrphanedWorktrees } from "../workspace-orphans";
 import { getLifecycleManager } from "../lifecycle-manager";
 import { applyCommandToRecipe } from "../recipe-runner";
+import { settleWorkspaceCommand } from "../workspace-command-settle";
 import {
   emitAgentRemoved,
   emitAgentStatus,
@@ -237,6 +239,7 @@ export const definitions: MaintenanceJobDefinition[] = [
     handler() {
       const commands = expireCommands();
       for (const command of commands) {
+        settleWorkspaceCommand(command, { onMergeSettled: scheduleRepoMerge });
         applyCommandToRecipe(command);
         emitCommand(command);
       }
@@ -373,13 +376,26 @@ export const definitions: MaintenanceJobDefinition[] = [
   },
   {
     id: "old-message-prune",
-    title: "Old message prune",
-    description: "Delete messages outside the configured retention window.",
+    title: "Data retention prune",
+    description: "Delete old messages, activity events, and terminal spawn commands in bounded batches.",
     intervalMs: DAY_MS,
     runOnStart: false,
-    handler(context) {
-      const retentionDays = Number(process.env.RETENTION_DAYS) || 30;
-      return pruneOldMessagesAsync(retentionDays * DAY_MS, { signal: context?.signal });
+    async handler(context) {
+      const retention = getRetentionConfig();
+      const now = Date.now();
+      const result = await pruneRetentionBatchAsync({
+        messageCutoff: now - retention.messageRetentionDays * DAY_MS,
+        activityCutoff: now - retention.activityEventRetentionDays * DAY_MS,
+        activityMaxRows: retention.activityEventMaxRows,
+        spawnCommandCutoff: now - retention.spawnCommandTerminalRetentionDays * DAY_MS,
+        batchSize: retention.pruneBatchSize,
+      }, { signal: context?.signal });
+      return {
+        prunedMessages: result.prunedMessages,
+        prunedActivityEvents: result.prunedActivityEvents,
+        prunedSpawnCommands: result.prunedSpawnCommands,
+        offThread: result.offThread,
+      };
     },
   },
   {

package/src/maintenance/workspaces.ts

@@ -21,6 +21,7 @@ import { preparePrCompletionScan, reconcileLandedPr } from "../workspace-pr-comp
 import { applyBranchFreshness, fetchHostMergePreview, workspacePathWithinBase } from "../workspace-probe";
 import { wakeRepoSteward } from "../workspace-auto-merge";
 import { workspaceOwnerOrchestrator } from "../workspace-host";
+import { routeWorkspaceCustodyEscalation } from "../workspace-escalation";
 import {
   CONFLICT_SCAN_STATUSES,
   DAY_MS,
@@ -175,74 +176,53 @@ export async function workspaceGC(): Promise<Record<string, unknown>> {
   const escalationTargets: string[] = [];
   const reconciledLandedIds: string[] = [];
   const escalationMs = stewardEscalationMs();
-  const fallbackTarget = stewardFallbackTarget();
   for (const ws of all) {
     if (!STRANDABLE_STATUSES.has(ws.status) || ws.mode !== "isolated" || !ws.worktreePath) continue;
     reelectRepoSteward(ws.repoRoot);
     const fresh = getWorkspace(ws.id);
     if (!fresh || !STRANDABLE_STATUSES.has(fresh.status)) continue;
     const meta = fresh.metadata as Record<string, unknown>;
-    const steward = fresh.stewardAgentId;
-    const stewardOnline = Boolean(steward && getAgent(steward) && getAgent(steward)!.status !== "offline");
     const strandedAt = typeof meta.strandedAt === "number" ? meta.strandedAt : undefined;
 
-    if (!stewardOnline) {
-      const orch = workspaceOwnerOrchestrator(fresh, listOrchestrators().filter((o) => o.status === "online" && o.apiUrl), { requireApiUrl: true });
-      const preview = orch?.apiUrl ? await fetchHostMergePreview(orch.apiUrl, fresh, { checkPr: true }) : null;
-      if (preview && !("available" in preview) && !preview.error && !preview.missing) {
-        const landed = previewProvesLanded(preview);
-        if (landed) {
-          reconcileLandedWorkspace(fresh, preview);
-          patchWorkspaceMetadata(fresh.id, { strandedAt: undefined, escalatedAt: undefined, strandedNotifiedSteward: undefined });
-          reconciledLandedIds.push(fresh.id);
-          continue;
-        }
+    const orch = workspaceOwnerOrchestrator(fresh, listOrchestrators().filter((o) => o.status === "online" && o.apiUrl), { requireApiUrl: true });
+    const preview = orch?.apiUrl ? await fetchHostMergePreview(orch.apiUrl, fresh, { checkPr: true }) : null;
+    if (preview && !("available" in preview) && !preview.error && !preview.missing) {
+      const landed = previewProvesLanded(preview);
+      if (landed) {
+        reconcileLandedWorkspace(fresh, preview);
+        patchWorkspaceMetadata(fresh.id, { strandedAt: undefined, escalatedAt: undefined, strandedNotifiedSteward: undefined });
+        reconciledLandedIds.push(fresh.id);
+        continue;
       }
     }
 
-    if (stewardOnline) {
-      if (strandedAt !== undefined && meta.strandedNotifiedSteward !== steward) {
-        const sent = notifyTarget(
-          "workspace.steward_reassigned",
-          { workspaceId: fresh.id, repoRoot: fresh.repoRoot },
-          steward!,
-          "Workspace stewardship reassigned",
-          `You are now steward for ${fresh.repoRoot}. Workspace \`${fresh.branch ?? fresh.id}\` is ${fresh.status} and was stranded without an online steward - please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"}.`,
-          { kind: "workspace.steward-reassigned", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status },
-        );
-        if (sent) reassignedIds.push(fresh.id);
-      }
-      patchWorkspaceMetadata(fresh.id, { strandedAt: undefined, escalatedAt: undefined, strandedNotifiedSteward: steward });
-      continue;
-    }
-
     if (strandedAt === undefined) { patchWorkspaceMetadata(fresh.id, { strandedAt: now }); continue; }
-    if (now - strandedAt < escalationMs || meta.escalatedAt) continue;
-    // #642 — escalate to the branch agent's coordinator (owner's `spawnedBy`), falling back to
-    // the configured `stewardFallbackTarget` only when no coordinator resolves. The bare
-    // fallback was an unset `system` dead-end in the live incident.
-    const { coordinatorAgentId } = resolveStewardIdentity(fresh);
-    const escalationTarget = coordinatorAgentId ?? fallbackTarget;
-    const sent = notifyTarget(
-      "workspace.stranded_escalation",
-      { workspaceId: fresh.id, repoRoot: fresh.repoRoot, author: fresh.ownerAgentId, parent: coordinatorAgentId ?? undefined },
-      escalationTarget,
-      "Stranded workspace needs an owner",
-      `Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} is ${fresh.status} with no online steward (all repo agents offline) for ${Math.round((now - strandedAt) / (60 * 60 * 1000))}h.${coordinatorAgentId ? ` You are the branch agent \`${fresh.ownerAgentId ?? "?"}\`'s coordinator.` : ""} Please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"} or clean up the worktree.`,
-      { kind: "workspace.stranded-escalation", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt, coordinatorAgentId },
-    );
-    if (sent) escalationTargets.push(sent);
+    const escalatedAt = typeof meta.escalatedAt === "number" ? meta.escalatedAt : 0;
+    if (now - strandedAt < escalationMs) continue;
+    if (escalatedAt && now - escalatedAt < escalationMs) continue;
+
+    if (getStewardConfig().enabled) {
+      wakeRepoSteward(fresh, `workspace ${fresh.status} stranded for ${Math.round((now - strandedAt) / (60 * 60 * 1000))}h`);
+    }
+    const targets = routeWorkspaceCustodyEscalation({
+      workspace: fresh,
+      subject: "Stranded workspace needs an owner",
+      body: `Workspace \`${fresh.branch ?? fresh.id}\` in ${fresh.repoRoot} is ${fresh.status} and has been stranded for ${Math.round((now - strandedAt) / (60 * 60 * 1000))}h. Please coordinate ${fresh.status === "conflict" ? "conflict resolution" : "review/merge"} or clean up the worktree.`,
+      payload: { kind: "workspace.stranded-escalation", workspaceId: fresh.id, repoRoot: fresh.repoRoot, branch: fresh.branch, status: fresh.status, strandedAt },
+      idempotencyKey: `workspace-stranded:${fresh.id}:${strandedAt}:${Math.floor(now / escalationMs)}`,
+    });
+    escalationTargets.push(...targets);
     patchWorkspaceMetadata(fresh.id, { escalatedAt: now });
     escalatedIds.push(fresh.id);
     createActivityEvent({
       clientId: `workspace-gc-escalate-${fresh.id}-${now}`,
       kind: "state",
       title: "Workspace escalated",
-      body: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} - stranded ${fresh.status} escalated${escalationTarget ? ` to ${escalationTarget}${coordinatorAgentId ? " (coordinator)" : ""}` : " (no coordinator or fallback configured)"}`,
+      body: `${fresh.branch ?? fresh.id} in ${fresh.repoRoot} - stranded ${fresh.status} escalated${targets.length ? ` to ${targets.join(", ")}` : " (no owner, coordinator, or fallback configured)"}`,
       meta: fresh.branch ?? fresh.id,
       icon: "ti-alert-octagon",
       view: "orchestrators",
-      metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: fresh.id, escalationTarget: escalationTarget || null, coordinatorAgentId: coordinatorAgentId ?? null },
+      metadata: { source: "server", maintenanceJobId: "workspace-gc", workspaceId: fresh.id, escalationTargets: targets },
     });
   }
 
@@ -250,6 +230,7 @@ export async function workspaceGC(): Promise<Record<string, unknown>> {
   const autoAbandonDeferred: string[] = [];
   const notifiedStewards: string[] = [];
   const reviewOrchestrators = listOrchestrators();
+  const fallbackTarget = stewardFallbackTarget();
   for (const ws of all) {
     if (ws.status === "review_requested" && ws.updatedAt < reviewCutoff) {
       const fresh = getWorkspace(ws.id);

package/src/managed-policy-escalation.ts

@@ -0,0 +1,47 @@
+import { getAgent, getDb } from "./db";
+import { getSpawnPolicy } from "./config-store";
+import { stewardFallbackTarget } from "./config";
+import { routeNotification } from "./notification-router";
+import type { ManagedAgentState, Orchestrator, SpawnPolicy } from "./types";
+
+const ORCHESTRATOR_OFFLINE_ESCALATION_FAILURES = 3;
+
+export function escalateOfflineOrchestrator(policy: SpawnPolicy, state: ManagedAgentState, orch: Orchestrator | null | undefined): void {
+  if (state.consecutiveFailures < ORCHESTRATOR_OFFLINE_ESCALATION_FAILURES) return;
+  const updatedBy = getSpawnPolicy(policy.name)?.updatedBy?.trim();
+  const previousParent = state.agentId ? getAgent(state.agentId)?.spawnedBy?.trim() : undefined;
+  const fallback = stewardFallbackTarget();
+  const recipients = [...new Set([updatedBy, previousParent, fallback].filter((item): item is string => Boolean(item && item !== "system")))];
+  if (!recipients.length) return;
+  const host = orch?.hostname ? `${orch.hostname} (${policy.orchestratorId})` : policy.orchestratorId;
+  const queuedMessages = queuedPolicyMessageCount(policy.name);
+  const nextRetry = state.backoffUntil ? new Date(state.backoffUntil).toISOString() : "not scheduled";
+  routeNotification({
+    type: "agent.spawn_failed",
+    scope: { target: `policy:${policy.name}` },
+    recipients,
+    message: {
+      subject: "Managed policy pinned host offline",
+      body: [
+        `Managed policy \`${policy.name}\` cannot spawn because pinned orchestrator \`${host}\` is offline.`,
+        `Queued messages: ${queuedMessages}. Consecutive failures: ${state.consecutiveFailures}. Next retry: ${nextRetry}.`,
+      ].join("\n"),
+      payload: {
+        kind: "managed-policy.orchestrator-offline",
+        policyName: policy.name,
+        orchestratorId: policy.orchestratorId,
+        orchestratorHostname: orch?.hostname,
+        queuedMessages,
+        consecutiveFailures: state.consecutiveFailures,
+        backoffUntil: state.backoffUntil,
+      },
+      idempotencyKey: `managed-policy-orchestrator-offline:${policy.name}:${policy.orchestratorId}`,
+      replyExpected: false,
+    },
+  });
+}
+
+function queuedPolicyMessageCount(policyName: string): number {
+  const row = getDb().query("SELECT COUNT(*) AS count FROM messages WHERE to_target = ? AND delivery_status = 'queued'").get(`policy:${policyName}`) as { count?: number } | undefined;
+  return row?.count ?? 0;
+}