@moretea-labs/forge 1.5.1 → 1.7.0

package/src/runtime/control-plane/generated-cache-retention.ts

@@ -0,0 +1,271 @@
+import { execFileSync } from 'child_process';
+import {
+  existsSync,
+  lstatSync,
+  readdirSync,
+  realpathSync,
+  rmSync,
+} from 'fs';
+import { homedir } from 'os';
+import { basename, join, relative, resolve, sep } from 'path';
+
+const DEFAULT_GENERATED_CACHE_GRACE_MS = 6 * 60 * 60_000;
+const DEFAULT_SCAN_BUDGET = 1_000;
+const DEFAULT_REMOVAL_BUDGET = 8;
+
+export interface GeneratedCacheRetentionOptions {
+  nowMs?: number;
+  graceMs?: number;
+  maxEntries?: number;
+  maxRemovals?: number;
+  processCommands?: string[];
+}
+
+export interface GeneratedCacheRetentionReport {
+  inspected: number;
+  eligible: number;
+  removedPaths: string[];
+  retainedPaths: string[];
+  skippedByReason: Record<string, number>;
+  errors: string[];
+  budgetExhausted: boolean;
+}
+
+export interface XCTestDeviceCleanupReport {
+  attempted: boolean;
+  removedDevices: number;
+  skippedReason?: string;
+  error?: string;
+}
+
+interface Candidate {
+  path: string;
+  kind: 'forge_ios_derived_data' | 'harness_derived_data' | 'repo_harness_derived_data' | 'tmp_derived_data';
+}
+
+function increment(counts: Record<string, number>, reason: string): void {
+  counts[reason] = (counts[reason] ?? 0) + 1;
+}
+
+function canonical(path: string): string {
+  try { return realpathSync(path); } catch { return resolve(path); }
+}
+
+function pathInside(root: string, path: string): boolean {
+  const base = canonical(root);
+  const target = canonical(path);
+  const rel = relative(base, target);
+  return rel !== '' && rel !== '..' && !rel.startsWith(`..${sep}`);
+}
+
+function collectProcessCommands(): string[] {
+  try {
+    return execFileSync('ps', ['-axo', 'command='], {
+      encoding: 'utf8',
+      timeout: 2_000,
+      maxBuffer: 2 * 1024 * 1024,
+    }).split('\n').map((line) => line.trim()).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+function commandReferencesPath(commands: readonly string[], repoRoot: string, path: string): boolean {
+  const absolute = canonical(path);
+  const rel = relative(resolve(repoRoot), absolute).replace(/\\/g, '/');
+  return commands.some((command) => command.includes(absolute) || (rel && command.includes(rel)));
+}
+
+function containsTrackedFiles(repoRoot: string, path: string): boolean {
+  const rel = relative(resolve(repoRoot), resolve(path));
+  if (!rel || rel === '..' || rel.startsWith(`..${sep}`)) return true;
+  try {
+    const output = execFileSync('git', ['-C', repoRoot, 'ls-files', '--', rel], {
+      encoding: 'utf8',
+      timeout: 2_000,
+      maxBuffer: 256 * 1024,
+    });
+    return output.trim().length > 0;
+  } catch {
+    // If repository identity cannot be proven, fail closed.
+    return true;
+  }
+}
+
+function directChildDirectories(root: string, predicate: (name: string) => boolean, budget: { remaining: number }): string[] {
+  if (!existsSync(root) || budget.remaining <= 0) return [];
+  const found: string[] = [];
+  let entries;
+  try { entries = readdirSync(root, { withFileTypes: true }); } catch { return found; }
+  for (const entry of entries) {
+    if (budget.remaining <= 0) break;
+    budget.remaining -= 1;
+    const name = String(entry.name);
+    if (entry.isDirectory() && predicate(name)) found.push(join(root, name));
+  }
+  return found;
+}
+
+function nestedDerivedDataDirectories(root: string, maxDepth: number, budget: { remaining: number }): string[] {
+  if (!existsSync(root) || budget.remaining <= 0) return [];
+  const found: string[] = [];
+  const visit = (dir: string, depth: number): void => {
+    if (depth > maxDepth || budget.remaining <= 0) return;
+    let entries;
+    try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      if (budget.remaining <= 0) return;
+      budget.remaining -= 1;
+      if (!entry.isDirectory()) continue;
+      const path = join(dir, String(entry.name));
+      if (String(entry.name) === 'DerivedData') {
+        found.push(path);
+        continue;
+      }
+      visit(path, depth + 1);
+    }
+  };
+  visit(root, 0);
+  return found;
+}
+
+function discoverCandidates(repoRoot: string, maxEntries: number): { candidates: Candidate[]; inspected: number; truncated: boolean } {
+  const root = resolve(repoRoot);
+  const budget = { remaining: Math.max(1, maxEntries) };
+  const initial = budget.remaining;
+  const candidates: Candidate[] = [];
+
+  for (const path of directChildDirectories(join(root, '.forge', 'ios', 'DerivedData'), () => true, budget)) {
+    candidates.push({ path, kind: 'forge_ios_derived_data' });
+  }
+  for (const path of directChildDirectories(join(root, '.ai', 'harness'), (name) => name.startsWith('deriveddata-'), budget)) {
+    candidates.push({ path, kind: 'harness_derived_data' });
+  }
+  for (const path of nestedDerivedDataDirectories(join(root, '.repo-harness'), 5, budget)) {
+    candidates.push({ path, kind: 'repo_harness_derived_data' });
+  }
+  for (const path of nestedDerivedDataDirectories(join(root, '.tmp'), 6, budget)) {
+    candidates.push({ path, kind: 'tmp_derived_data' });
+  }
+
+  return {
+    candidates: candidates.filter((candidate, index, values) => values.findIndex((item) => resolve(item.path) === resolve(candidate.path)) === index),
+    inspected: initial - budget.remaining,
+    truncated: budget.remaining <= 0,
+  };
+}
+
+/**
+ * Reclaim only rebuildable Xcode DerivedData under Forge/repo-harness-owned
+ * namespaces. Result bundles, logs, edit-session metadata, and all tracked files
+ * are deliberately outside this cleanup surface.
+ */
+export function cleanupGeneratedRepositoryCaches(
+  repoRoot: string,
+  options: GeneratedCacheRetentionOptions = {},
+): GeneratedCacheRetentionReport {
+  const nowMs = options.nowMs ?? Date.now();
+  const graceMs = Math.max(60_000, Math.floor(options.graceMs ?? DEFAULT_GENERATED_CACHE_GRACE_MS));
+  const maxEntries = Math.max(1, Math.floor(options.maxEntries ?? DEFAULT_SCAN_BUDGET));
+  let remainingRemovals = Math.max(1, Math.floor(options.maxRemovals ?? DEFAULT_REMOVAL_BUDGET));
+  const commands = options.processCommands ?? collectProcessCommands();
+  const discovered = discoverCandidates(repoRoot, maxEntries);
+  const report: GeneratedCacheRetentionReport = {
+    inspected: discovered.inspected,
+    eligible: 0,
+    removedPaths: [],
+    retainedPaths: [],
+    skippedByReason: {},
+    errors: [],
+    budgetExhausted: discovered.truncated,
+  };
+
+  for (const candidate of discovered.candidates) {
+    const ownedRoot = candidate.kind === 'forge_ios_derived_data'
+      ? join(repoRoot, '.forge', 'ios', 'DerivedData')
+      : candidate.kind === 'harness_derived_data'
+        ? join(repoRoot, '.ai', 'harness')
+        : candidate.kind === 'repo_harness_derived_data'
+          ? join(repoRoot, '.repo-harness')
+          : join(repoRoot, '.tmp');
+    try {
+      const stat = lstatSync(candidate.path);
+      if (!stat.isDirectory() || stat.isSymbolicLink() || !pathInside(ownedRoot, candidate.path)) {
+        increment(report.skippedByReason, 'ownership_unproven');
+        report.retainedPaths.push(relative(repoRoot, candidate.path));
+        continue;
+      }
+      if (nowMs - stat.mtimeMs < graceMs) {
+        increment(report.skippedByReason, 'retention_grace');
+        continue;
+      }
+      if (commandReferencesPath(commands, repoRoot, candidate.path)) {
+        increment(report.skippedByReason, 'active_process');
+        report.retainedPaths.push(relative(repoRoot, candidate.path));
+        continue;
+      }
+      if (containsTrackedFiles(repoRoot, candidate.path)) {
+        increment(report.skippedByReason, 'tracked_content');
+        report.retainedPaths.push(relative(repoRoot, candidate.path));
+        continue;
+      }
+      report.eligible += 1;
+      if (remainingRemovals <= 0) {
+        report.budgetExhausted = true;
+        increment(report.skippedByReason, 'cleanup_budget_exhausted');
+        continue;
+      }
+      remainingRemovals -= 1;
+      rmSync(candidate.path, { recursive: true, force: true });
+      report.removedPaths.push(relative(repoRoot, candidate.path).replace(/\\/g, '/'));
+    } catch (error) {
+      report.errors.push(`${candidate.kind}:${relative(repoRoot, candidate.path)}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  return report;
+}
+
+/**
+ * XCTest's private device set is separate from the user's normal CoreSimulator
+ * devices. When Forge is idle, delete only that dedicated set through simctl so
+ * XCTest clones cannot accumulate indefinitely. Never rm the device set.
+ */
+export function cleanupIdleXCTestDevices(controllerHome: string): XCTestDeviceCleanupReport {
+  if (process.platform !== 'darwin') return { attempted: false, removedDevices: 0, skippedReason: 'not_darwin' };
+  const normalizedControllerHome = resolve(controllerHome).replace(/\\/g, '/');
+  if (!normalizedControllerHome.includes('/.forge/controller')) {
+    return { attempted: false, removedDevices: 0, skippedReason: 'nonstandard_controller_home' };
+  }
+  const deviceSet = join(homedir(), 'Library', 'Developer', 'XCTestDevices');
+  if (!existsSync(deviceSet)) return { attempted: false, removedDevices: 0, skippedReason: 'device_set_missing' };
+  let deviceDirectories = 0;
+  try {
+    deviceDirectories = readdirSync(deviceSet, { withFileTypes: true }).filter((entry) => entry.isDirectory()).length;
+  } catch (error) {
+    return { attempted: false, removedDevices: 0, error: error instanceof Error ? error.message : String(error) };
+  }
+  if (deviceDirectories <= 4) return { attempted: false, removedDevices: 0, skippedReason: 'below_cleanup_threshold' };
+
+  const commands = collectProcessCommands();
+  if (commands.some((command) => /(?:^|\/)(?:xcodebuild|xctest|swiftc)(?:\s|$)/.test(command))) {
+    return { attempted: false, removedDevices: 0, skippedReason: 'active_xcode_process' };
+  }
+  try {
+    const listed = execFileSync('xcrun', ['simctl', '--set', deviceSet, 'list', 'devices', '--json'], {
+      encoding: 'utf8',
+      timeout: 10_000,
+      maxBuffer: 2 * 1024 * 1024,
+    });
+    const parsed = JSON.parse(listed) as { devices?: Record<string, Array<{ state?: string }>> };
+    const devices = Object.values(parsed.devices ?? {}).flat();
+    if (devices.some((device) => device.state === 'Booted')) {
+      return { attempted: false, removedDevices: 0, skippedReason: 'booted_xctest_device' };
+    }
+    execFileSync('xcrun', ['simctl', '--set', deviceSet, 'shutdown', 'all'], { timeout: 15_000, stdio: 'ignore' });
+    execFileSync('xcrun', ['simctl', '--set', deviceSet, 'delete', 'all'], { timeout: 30_000, stdio: 'ignore' });
+    return { attempted: true, removedDevices: devices.length };
+  } catch (error) {
+    return { attempted: true, removedDevices: 0, error: error instanceof Error ? error.message : String(error) };
+  }
+}

package/src/runtime/control-plane/global-scheduler/config.ts

@@ -0,0 +1,44 @@
+export interface SchedulerConfig {
+  maxWorkers: number;
+  maxConcurrentRepositories: number;
+  pollIntervalMs: number;
+  idleBackoffMaxMs: number;
+  heartbeatIntervalMs: number;
+  heartbeatTimeoutMs: number;
+  maxHeavyChecks: number;
+  maxAgentProcesses: number;
+  maxCodexProcesses: number;
+  maxClaudeProcesses: number;
+  maxGitHubProcesses: number;
+  minFreeMemoryMb: number;
+  maxLoadPerCpu: number;
+}
+
+export function normalizeSchedulerConfig(
+  config: Partial<SchedulerConfig> = {},
+  env: NodeJS.ProcessEnv = process.env,
+): SchedulerConfig {
+  const pollIntervalMs = Math.max(50, config.pollIntervalMs ?? 250);
+  const idleBackoffMaxMs = Math.max(250, config.idleBackoffMaxMs ?? Number(env.FORGE_IDLE_BACKOFF_MAX_MS ?? 2_000));
+  const heartbeatIntervalMs = Math.max(25, config.heartbeatIntervalMs ?? Number(env.FORGE_SCHEDULER_HEARTBEAT_INTERVAL_MS ?? 1_000));
+  const heartbeatTimeoutMs = Math.max(
+    heartbeatIntervalMs * 4,
+    idleBackoffMaxMs * 4,
+    config.heartbeatTimeoutMs ?? Number(env.FORGE_SCHEDULER_HEARTBEAT_TIMEOUT_MS ?? 60_000),
+  );
+  return {
+    maxWorkers: Math.max(1, config.maxWorkers ?? Number(env.FORGE_MAX_WORKERS ?? 4)),
+    maxConcurrentRepositories: Math.max(1, config.maxConcurrentRepositories ?? Number(env.FORGE_MAX_ACTIVE_REPOS ?? 4)),
+    pollIntervalMs,
+    idleBackoffMaxMs,
+    heartbeatIntervalMs,
+    heartbeatTimeoutMs,
+    maxHeavyChecks: Math.max(1, config.maxHeavyChecks ?? Number(env.FORGE_MAX_HEAVY_CHECKS ?? 2)),
+    maxAgentProcesses: Math.max(1, config.maxAgentProcesses ?? Number(env.FORGE_MAX_AGENT_PROCESSES ?? 4)),
+    maxCodexProcesses: Math.max(1, config.maxCodexProcesses ?? Number(env.FORGE_MAX_CODEX_PROCESSES ?? 3)),
+    maxClaudeProcesses: Math.max(1, config.maxClaudeProcesses ?? Number(env.FORGE_MAX_CLAUDE_PROCESSES ?? 2)),
+    maxGitHubProcesses: Math.max(1, config.maxGitHubProcesses ?? Number(env.FORGE_MAX_GITHUB_PROCESSES ?? 2)),
+    minFreeMemoryMb: Math.max(64, config.minFreeMemoryMb ?? Number(env.FORGE_MIN_FREE_MEMORY_MB ?? 512)),
+    maxLoadPerCpu: Math.max(0.25, config.maxLoadPerCpu ?? Number(env.FORGE_MAX_LOAD_PER_CPU ?? 1.5)),
+  };
+}

package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts

@@ -0,0 +1,80 @@
+import type { ExecutionJob } from '../../execution/jobs/types';
+
+export type SchedulerAgentProvider = 'codex' | 'claude' | 'github-copilot';
+
+export interface SchedulerDispatchLimits {
+  maxWorkers: number;
+  maxHeavyChecks: number;
+  maxAgentProcesses: number;
+  maxCodexProcesses: number;
+  maxClaudeProcesses: number;
+  maxGitHubProcesses: number;
+}
+
+export interface SchedulerDispatchCapacity {
+  reservedJobs: ExecutionJob[];
+  workers: number;
+  heavyChecks: number;
+  agents: number;
+  providers: Map<SchedulerAgentProvider, number>;
+}
+
+export function schedulerAgentProvider(
+  job: Pick<ExecutionJob, 'payload'>,
+): SchedulerAgentProvider {
+  const agent = job.payload.arguments?.agent;
+  if (agent === 'claude' || agent === 'github-copilot') return agent;
+  return 'codex';
+}
+
+export function createSchedulerDispatchCapacity(
+  activeJobs: readonly ExecutionJob[],
+  limits: SchedulerDispatchLimits,
+  pressured: boolean,
+): SchedulerDispatchCapacity {
+  const reservedJobs = activeJobs.filter((job) => job.status === 'running' || job.status === 'dispatched');
+  const reservedAgents = reservedJobs.filter((job) => job.type === 'agent-run' || job.type === 'dispatch-task');
+  let workers = limits.maxWorkers - reservedJobs.length;
+  let heavyChecks = limits.maxHeavyChecks
+    - reservedJobs.filter((job) => job.type === 'check' || job.type === 'verify-edit').length;
+  let agents = limits.maxAgentProcesses - reservedAgents.length;
+  const providers = new Map<SchedulerAgentProvider, number>([
+    ['codex', limits.maxCodexProcesses - reservedAgents.filter((job) => schedulerAgentProvider(job) === 'codex').length],
+    ['claude', limits.maxClaudeProcesses - reservedAgents.filter((job) => schedulerAgentProvider(job) === 'claude').length],
+    ['github-copilot', limits.maxGitHubProcesses - reservedAgents.filter((job) => schedulerAgentProvider(job) === 'github-copilot').length],
+  ]);
+  if (pressured) {
+    workers = Math.min(workers, 1);
+    heavyChecks = Math.min(heavyChecks, 1);
+    agents = 0;
+    providers.set('codex', 0);
+    providers.set('claude', 0);
+    providers.set('github-copilot', 0);
+  }
+  return { reservedJobs, workers, heavyChecks, agents, providers };
+}
+
+export function schedulerDispatchCapacityAllows(
+  capacity: SchedulerDispatchCapacity,
+  job: Pick<ExecutionJob, 'type' | 'payload'>,
+): boolean {
+  if ((job.type === 'check' || job.type === 'verify-edit') && capacity.heavyChecks <= 0) return false;
+  if (job.type === 'agent-run' || job.type === 'dispatch-task') {
+    if (capacity.agents <= 0) return false;
+    if ((capacity.providers.get(schedulerAgentProvider(job)) ?? 0) <= 0) return false;
+  }
+  return true;
+}
+
+export function consumeSchedulerDispatchCapacity(
+  capacity: SchedulerDispatchCapacity,
+  job: Pick<ExecutionJob, 'type' | 'payload'>,
+): void {
+  capacity.workers -= 1;
+  if (job.type === 'check' || job.type === 'verify-edit') capacity.heavyChecks -= 1;
+  if (job.type === 'agent-run' || job.type === 'dispatch-task') {
+    capacity.agents -= 1;
+    const provider = schedulerAgentProvider(job);
+    capacity.providers.set(provider, (capacity.providers.get(provider) ?? 0) - 1);
+  }
+}

package/src/runtime/control-plane/global-scheduler/durable-admission.ts

@@ -0,0 +1,46 @@
+import {
+  listActiveExecutionJobs,
+  markExecutionJobSchedulerObserved,
+} from '../../execution/jobs/store';
+import { tickSchedules } from '../../workflow/schedules/engine';
+
+const SCHEDULE_TICK_INTERVAL_MS = 30_000;
+
+export interface SchedulerDurableAdmissionDependencies {
+  listActiveJobs: typeof listActiveExecutionJobs;
+  markSchedulerObserved: typeof markExecutionJobSchedulerObserved;
+  tickSchedules: typeof tickSchedules;
+}
+
+const DEFAULT_DEPENDENCIES: SchedulerDurableAdmissionDependencies = {
+  listActiveJobs: listActiveExecutionJobs,
+  markSchedulerObserved: markExecutionJobSchedulerObserved,
+  tickSchedules,
+};
+
+export async function runSchedulerDurableAdmission(input: {
+  controllerHome: string;
+  repositoryIds: readonly string[];
+  nowMs: number;
+  lastScheduleTickAt: number;
+}, dependencies: SchedulerDurableAdmissionDependencies = DEFAULT_DEPENDENCIES): Promise<{
+  scheduleTicked: boolean;
+}> {
+  // Observation closes the admission phase for durable Jobs and starts their
+  // independent queue budget. Each Job owns its own atomic state transition,
+  // so this intentionally remains outside the global dispatch reservation lock.
+  for (const job of dependencies.listActiveJobs(input.controllerHome)) {
+    if (job.status === 'running' || job.timings?.schedulerObservedAt) continue;
+    try {
+      dependencies.markSchedulerObserved(input.controllerHome, job.repoId, job.jobId);
+    } catch {
+      // Another Scheduler instance or a terminal transition won the Job-local race.
+    }
+  }
+
+  if (input.nowMs - input.lastScheduleTickAt < SCHEDULE_TICK_INTERVAL_MS) {
+    return { scheduleTicked: false };
+  }
+  await dependencies.tickSchedules(input.controllerHome, [...input.repositoryIds]);
+  return { scheduleTicked: true };
+}

package/src/runtime/control-plane/global-scheduler/maintenance.ts

@@ -0,0 +1,145 @@
+import type { RepositoryRecord } from '../../../cli/repositories/types';
+import { cleanupGeneratedRepositoryCaches, cleanupIdleXCTestDevices } from '../generated-cache-retention';
+import type { cleanupControllerRuntimeState } from '../runtime-cleanup';
+import type { reconcileTerminalWorkCleanups } from '../execution/work-terminal-cleanup';
+import type { gcTerminalProcesses } from '../../execution/process-runtime/gc';
+import type { reconcilePendingWorkValidations } from '../execution/work-validation-reconciler';
+import type { reconcilePendingEditValidations } from '../execution/edit-validation-coordinator';
+import {
+  buildControllerRoundRelayPrompt,
+  claimStalledControllerRoundRelays,
+  finishControllerRoundRelayDispatch,
+} from '../facade/controller-round-relay';
+import { assertAutomatedOperationAllowed } from '../governance/external-effects';
+import { runStandaloneChatgptPrompt } from '../launcher/chatgpt-work-continuation';
+
+export async function runSchedulerPeriodicCleanup(input: {
+  controllerHome: string;
+  controllerPid: number;
+  nowMs: number;
+  cleanupIntervalMs: number;
+  repositories: readonly RepositoryRecord[];
+  runtimeCleanup: typeof cleanupControllerRuntimeState;
+  terminalWorkCleanup: typeof reconcileTerminalWorkCleanups;
+  processGc: typeof gcTerminalProcesses;
+}): Promise<void> {
+  try {
+    input.runtimeCleanup(input.controllerHome, {
+      reason: 'periodic',
+      nowMs: input.nowMs,
+      protectedControllerPid: input.controllerPid,
+    });
+  } catch (error) {
+    console.error('[forge cleanup] periodic cleanup failed:', error);
+  }
+  try {
+    await input.terminalWorkCleanup(input.controllerHome, { nowMs: input.nowMs });
+  } catch (error) {
+    console.error('[forge cleanup] terminal Work cleanup failed:', error);
+  }
+  try {
+    const xctestCleanup = cleanupIdleXCTestDevices(input.controllerHome);
+    if (xctestCleanup.error) console.error('[forge cleanup] XCTest device cleanup failed:', xctestCleanup.error);
+  } catch (error) {
+    console.error('[forge cleanup] XCTest device cleanup failed:', error);
+  }
+  if (input.repositories.length === 0) return;
+  const slot = Math.floor(input.nowMs / input.cleanupIntervalMs) % input.repositories.length;
+  const repo = input.repositories[slot]!;
+  try {
+    const generated = cleanupGeneratedRepositoryCaches(repo.canonicalRoot, { nowMs: input.nowMs });
+    if (generated.errors.length > 0) {
+      console.error(`[forge cleanup] generated-cache retention reported ${generated.errors.length} error(s) for ${repo.repoId}`);
+    }
+  } catch (error) {
+    console.error(`[forge cleanup] generated-cache retention failed for ${repo.repoId}:`, error);
+  }
+  const result = input.processGc({ controllerHome: input.controllerHome, repoId: repo.repoId });
+  if (!result.ok) console.error('[forge cleanup] Process GC failed:', result.error ?? 'unknown error');
+}
+
+export async function runSchedulerValidationReconciliation(input: {
+  controllerHome: string;
+  repositories: readonly RepositoryRecord[];
+  workValidationReconcile: typeof reconcilePendingWorkValidations;
+  editValidationReconcile: typeof reconcilePendingEditValidations;
+}): Promise<void> {
+  for (const repository of input.repositories) {
+    const validation = input.workValidationReconcile(input.controllerHome, repository.repoId, 500);
+    if (validation.errors.length > 0) {
+      console.error(
+        `[forge validation] background Work reconciliation reported ${validation.errors.length} error(s) for ${repository.repoId}`,
+      );
+    }
+    const editValidation = await input.editValidationReconcile(input.controllerHome, repository, 200);
+    if (editValidation.errors.length > 0) {
+      console.error(
+        `[forge validation] background EditSession reconciliation reported ${editValidation.errors.length} error(s) for ${repository.repoId}`,
+      );
+    }
+  }
+}
+
+export async function runSchedulerControllerRoundRecovery(input: {
+  controllerHome: string;
+  nowMs: number;
+  repositories: readonly Pick<RepositoryRecord, 'repoId'>[];
+  graceMs?: number;
+  maxRecoveries?: number;
+  dispatchPrompt?: typeof runStandaloneChatgptPrompt;
+  authorizeWake?: typeof assertAutomatedOperationAllowed;
+}): Promise<{ claimed: number; dispatched: number; failed: number }> {
+  const dispatchPrompt = input.dispatchPrompt ?? runStandaloneChatgptPrompt;
+  const authorizeWake = input.authorizeWake ?? assertAutomatedOperationAllowed;
+  const maxRecoveries = Math.max(1, Math.min(Math.trunc(input.maxRecoveries ?? 2), 8));
+  let claimed = 0;
+  let dispatched = 0;
+  let failed = 0;
+
+  for (const repository of input.repositories) {
+    if (claimed >= maxRecoveries) break;
+    const store = { controllerHome: input.controllerHome, repoId: repository.repoId };
+    const records = claimStalledControllerRoundRelays(store, {
+      nowMs: input.nowMs,
+      graceMs: input.graceMs,
+      limit: maxRecoveries - claimed,
+    });
+    claimed += records.length;
+    for (const record of records) {
+      try {
+        authorizeWake('external_controller_wake', {
+          work_id: record.originWorkId,
+          controller_type: 'chatgpt',
+          relay_scope_id: record.relayScopeId,
+          recovery_reason: 'unclosed_dispatched_round',
+        });
+        const result = await dispatchPrompt({
+          controllerHome: input.controllerHome,
+          repoId: repository.repoId,
+          scopeId: `controller-relay:${record.relayScopeId}`,
+          prompt: buildControllerRoundRelayPrompt(store, record),
+          browserSessionId: record.browserSessionId,
+          conversationUrl: record.conversationUrl,
+          model: 'gpt-5.6',
+          reasoning: 'high',
+          tabPolicy: 'auto',
+          timeoutMs: 30_000,
+        });
+        if (result.status === 'failed') throw new Error(result.error?.message ?? 'CHATGPT_CONTROLLER_RELAY_RECOVERY_FAILED');
+        finishControllerRoundRelayDispatch(store, {
+          workId: record.originWorkId,
+          ok: true,
+          browserSessionId: result.browserSessionId,
+          conversationUrl: result.conversationUrl,
+        });
+        dispatched += 1;
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        finishControllerRoundRelayDispatch(store, { workId: record.originWorkId, ok: false, error: reason });
+        failed += 1;
+        console.error(`[forge controller relay] stalled round recovery failed for ${record.relayScopeId}:`, reason);
+      }
+    }
+  }
+  return { claimed, dispatched, failed };
+}

package/src/runtime/control-plane/global-scheduler/projection-refresh.ts

@@ -0,0 +1,84 @@
+import type { RepositoryRecord } from '../../../cli/repositories/types';
+import { getRuntimeWriteClaim } from '../../root/write-fence';
+import { rebuildRepositoryProjection, refreshRepositoryProjectionForRepository } from '../../projections/materialized-view';
+import { readRepositoryProjectionDirty } from '../../projections/invalidation';
+import { readRepositoryGitStatusSample } from '../../projections/git-status-sampler';
+
+export interface SchedulerProjectionRefreshTargets {
+  repositories: RepositoryRecord[];
+  rebuildRepoIds: string[];
+}
+
+export function selectSchedulerDirtyProjectionRepositories(
+  controllerHome: string,
+  repositories: readonly RepositoryRecord[],
+): RepositoryRecord[] {
+  return repositories.filter((repository) => readRepositoryProjectionDirty(controllerHome, repository.repoId) !== undefined);
+}
+
+export function selectSchedulerProjectionRefreshTargets(
+  repositories: readonly RepositoryRecord[],
+  sourceScanRepositories: readonly RepositoryRecord[],
+  projectionRefreshRepoIds: Iterable<string>,
+): SchedulerProjectionRefreshTargets {
+  const candidates = new Map(sourceScanRepositories.map((repository) => [repository.repoId, repository]));
+  const repositoryById = new Map(repositories.map((repository) => [repository.repoId, repository]));
+  const rebuildRepoIds: string[] = [];
+  const rebuildSeen = new Set<string>();
+  for (const repoId of projectionRefreshRepoIds) {
+    const repository = repositoryById.get(repoId);
+    if (repository) {
+      candidates.set(repoId, repository);
+      continue;
+    }
+    if (!rebuildSeen.has(repoId)) {
+      rebuildSeen.add(repoId);
+      rebuildRepoIds.push(repoId);
+    }
+  }
+  return {
+    repositories: [...candidates.values()],
+    rebuildRepoIds,
+  };
+}
+
+export function refreshSchedulerRepositoryProjections(input: {
+  controllerHome: string;
+  repositories: readonly RepositoryRecord[];
+  sourceScanRepositories: readonly RepositoryRecord[];
+  projectionRefreshRepoIds: Iterable<string>;
+  controllerPid: number;
+}): void {
+  const targets = selectSchedulerProjectionRefreshTargets(
+    input.repositories,
+    input.sourceScanRepositories,
+    input.projectionRefreshRepoIds,
+  );
+  for (const repoId of targets.rebuildRepoIds) {
+    try {
+      rebuildRepositoryProjection(input.controllerHome, repoId);
+    } catch (error) {
+      console.error('[forge scheduler] projection refresh failed:', error);
+    }
+  }
+  for (const repository of targets.repositories) {
+    try {
+      const sample = readRepositoryGitStatusSample(
+        input.controllerHome,
+        repository.repoId,
+        repository.activeCheckoutId,
+      );
+      const runtimeInstanceId = getRuntimeWriteClaim()?.runtimeInstanceId;
+      refreshRepositoryProjectionForRepository(input.controllerHome, repository, {
+        sourceRevision: sample?.head ?? undefined,
+        reason: 'scheduler-source-scan',
+        owner: {
+          pid: input.controllerPid,
+          ...(runtimeInstanceId ? { runtimeInstanceId } : {}),
+        },
+      });
+    } catch (error) {
+      console.error('[forge scheduler] projection refresh failed:', error);
+    }
+  }
+}