@moretea-labs/forge 1.6.0 → 1.7.0

package/src/runtime/control-plane/global-scheduler/config.ts

@@ -0,0 +1,44 @@
+export interface SchedulerConfig {
+  maxWorkers: number;
+  maxConcurrentRepositories: number;
+  pollIntervalMs: number;
+  idleBackoffMaxMs: number;
+  heartbeatIntervalMs: number;
+  heartbeatTimeoutMs: number;
+  maxHeavyChecks: number;
+  maxAgentProcesses: number;
+  maxCodexProcesses: number;
+  maxClaudeProcesses: number;
+  maxGitHubProcesses: number;
+  minFreeMemoryMb: number;
+  maxLoadPerCpu: number;
+}
+
+export function normalizeSchedulerConfig(
+  config: Partial<SchedulerConfig> = {},
+  env: NodeJS.ProcessEnv = process.env,
+): SchedulerConfig {
+  const pollIntervalMs = Math.max(50, config.pollIntervalMs ?? 250);
+  const idleBackoffMaxMs = Math.max(250, config.idleBackoffMaxMs ?? Number(env.FORGE_IDLE_BACKOFF_MAX_MS ?? 2_000));
+  const heartbeatIntervalMs = Math.max(25, config.heartbeatIntervalMs ?? Number(env.FORGE_SCHEDULER_HEARTBEAT_INTERVAL_MS ?? 1_000));
+  const heartbeatTimeoutMs = Math.max(
+    heartbeatIntervalMs * 4,
+    idleBackoffMaxMs * 4,
+    config.heartbeatTimeoutMs ?? Number(env.FORGE_SCHEDULER_HEARTBEAT_TIMEOUT_MS ?? 60_000),
+  );
+  return {
+    maxWorkers: Math.max(1, config.maxWorkers ?? Number(env.FORGE_MAX_WORKERS ?? 4)),
+    maxConcurrentRepositories: Math.max(1, config.maxConcurrentRepositories ?? Number(env.FORGE_MAX_ACTIVE_REPOS ?? 4)),
+    pollIntervalMs,
+    idleBackoffMaxMs,
+    heartbeatIntervalMs,
+    heartbeatTimeoutMs,
+    maxHeavyChecks: Math.max(1, config.maxHeavyChecks ?? Number(env.FORGE_MAX_HEAVY_CHECKS ?? 2)),
+    maxAgentProcesses: Math.max(1, config.maxAgentProcesses ?? Number(env.FORGE_MAX_AGENT_PROCESSES ?? 4)),
+    maxCodexProcesses: Math.max(1, config.maxCodexProcesses ?? Number(env.FORGE_MAX_CODEX_PROCESSES ?? 3)),
+    maxClaudeProcesses: Math.max(1, config.maxClaudeProcesses ?? Number(env.FORGE_MAX_CLAUDE_PROCESSES ?? 2)),
+    maxGitHubProcesses: Math.max(1, config.maxGitHubProcesses ?? Number(env.FORGE_MAX_GITHUB_PROCESSES ?? 2)),
+    minFreeMemoryMb: Math.max(64, config.minFreeMemoryMb ?? Number(env.FORGE_MIN_FREE_MEMORY_MB ?? 512)),
+    maxLoadPerCpu: Math.max(0.25, config.maxLoadPerCpu ?? Number(env.FORGE_MAX_LOAD_PER_CPU ?? 1.5)),
+  };
+}

package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts

@@ -0,0 +1,80 @@
+import type { ExecutionJob } from '../../execution/jobs/types';
+
+export type SchedulerAgentProvider = 'codex' | 'claude' | 'github-copilot';
+
+export interface SchedulerDispatchLimits {
+  maxWorkers: number;
+  maxHeavyChecks: number;
+  maxAgentProcesses: number;
+  maxCodexProcesses: number;
+  maxClaudeProcesses: number;
+  maxGitHubProcesses: number;
+}
+
+export interface SchedulerDispatchCapacity {
+  reservedJobs: ExecutionJob[];
+  workers: number;
+  heavyChecks: number;
+  agents: number;
+  providers: Map<SchedulerAgentProvider, number>;
+}
+
+export function schedulerAgentProvider(
+  job: Pick<ExecutionJob, 'payload'>,
+): SchedulerAgentProvider {
+  const agent = job.payload.arguments?.agent;
+  if (agent === 'claude' || agent === 'github-copilot') return agent;
+  return 'codex';
+}
+
+export function createSchedulerDispatchCapacity(
+  activeJobs: readonly ExecutionJob[],
+  limits: SchedulerDispatchLimits,
+  pressured: boolean,
+): SchedulerDispatchCapacity {
+  const reservedJobs = activeJobs.filter((job) => job.status === 'running' || job.status === 'dispatched');
+  const reservedAgents = reservedJobs.filter((job) => job.type === 'agent-run' || job.type === 'dispatch-task');
+  let workers = limits.maxWorkers - reservedJobs.length;
+  let heavyChecks = limits.maxHeavyChecks
+    - reservedJobs.filter((job) => job.type === 'check' || job.type === 'verify-edit').length;
+  let agents = limits.maxAgentProcesses - reservedAgents.length;
+  const providers = new Map<SchedulerAgentProvider, number>([
+    ['codex', limits.maxCodexProcesses - reservedAgents.filter((job) => schedulerAgentProvider(job) === 'codex').length],
+    ['claude', limits.maxClaudeProcesses - reservedAgents.filter((job) => schedulerAgentProvider(job) === 'claude').length],
+    ['github-copilot', limits.maxGitHubProcesses - reservedAgents.filter((job) => schedulerAgentProvider(job) === 'github-copilot').length],
+  ]);
+  if (pressured) {
+    workers = Math.min(workers, 1);
+    heavyChecks = Math.min(heavyChecks, 1);
+    agents = 0;
+    providers.set('codex', 0);
+    providers.set('claude', 0);
+    providers.set('github-copilot', 0);
+  }
+  return { reservedJobs, workers, heavyChecks, agents, providers };
+}
+
+export function schedulerDispatchCapacityAllows(
+  capacity: SchedulerDispatchCapacity,
+  job: Pick<ExecutionJob, 'type' | 'payload'>,
+): boolean {
+  if ((job.type === 'check' || job.type === 'verify-edit') && capacity.heavyChecks <= 0) return false;
+  if (job.type === 'agent-run' || job.type === 'dispatch-task') {
+    if (capacity.agents <= 0) return false;
+    if ((capacity.providers.get(schedulerAgentProvider(job)) ?? 0) <= 0) return false;
+  }
+  return true;
+}
+
+export function consumeSchedulerDispatchCapacity(
+  capacity: SchedulerDispatchCapacity,
+  job: Pick<ExecutionJob, 'type' | 'payload'>,
+): void {
+  capacity.workers -= 1;
+  if (job.type === 'check' || job.type === 'verify-edit') capacity.heavyChecks -= 1;
+  if (job.type === 'agent-run' || job.type === 'dispatch-task') {
+    capacity.agents -= 1;
+    const provider = schedulerAgentProvider(job);
+    capacity.providers.set(provider, (capacity.providers.get(provider) ?? 0) - 1);
+  }
+}

package/src/runtime/control-plane/global-scheduler/durable-admission.ts

@@ -0,0 +1,46 @@
+import {
+  listActiveExecutionJobs,
+  markExecutionJobSchedulerObserved,
+} from '../../execution/jobs/store';
+import { tickSchedules } from '../../workflow/schedules/engine';
+
+const SCHEDULE_TICK_INTERVAL_MS = 30_000;
+
+export interface SchedulerDurableAdmissionDependencies {
+  listActiveJobs: typeof listActiveExecutionJobs;
+  markSchedulerObserved: typeof markExecutionJobSchedulerObserved;
+  tickSchedules: typeof tickSchedules;
+}
+
+const DEFAULT_DEPENDENCIES: SchedulerDurableAdmissionDependencies = {
+  listActiveJobs: listActiveExecutionJobs,
+  markSchedulerObserved: markExecutionJobSchedulerObserved,
+  tickSchedules,
+};
+
+export async function runSchedulerDurableAdmission(input: {
+  controllerHome: string;
+  repositoryIds: readonly string[];
+  nowMs: number;
+  lastScheduleTickAt: number;
+}, dependencies: SchedulerDurableAdmissionDependencies = DEFAULT_DEPENDENCIES): Promise<{
+  scheduleTicked: boolean;
+}> {
+  // Observation closes the admission phase for durable Jobs and starts their
+  // independent queue budget. Each Job owns its own atomic state transition,
+  // so this intentionally remains outside the global dispatch reservation lock.
+  for (const job of dependencies.listActiveJobs(input.controllerHome)) {
+    if (job.status === 'running' || job.timings?.schedulerObservedAt) continue;
+    try {
+      dependencies.markSchedulerObserved(input.controllerHome, job.repoId, job.jobId);
+    } catch {
+      // Another Scheduler instance or a terminal transition won the Job-local race.
+    }
+  }
+
+  if (input.nowMs - input.lastScheduleTickAt < SCHEDULE_TICK_INTERVAL_MS) {
+    return { scheduleTicked: false };
+  }
+  await dependencies.tickSchedules(input.controllerHome, [...input.repositoryIds]);
+  return { scheduleTicked: true };
+}

package/src/runtime/control-plane/global-scheduler/maintenance.ts

@@ -0,0 +1,145 @@
+import type { RepositoryRecord } from '../../../cli/repositories/types';
+import { cleanupGeneratedRepositoryCaches, cleanupIdleXCTestDevices } from '../generated-cache-retention';
+import type { cleanupControllerRuntimeState } from '../runtime-cleanup';
+import type { reconcileTerminalWorkCleanups } from '../execution/work-terminal-cleanup';
+import type { gcTerminalProcesses } from '../../execution/process-runtime/gc';
+import type { reconcilePendingWorkValidations } from '../execution/work-validation-reconciler';
+import type { reconcilePendingEditValidations } from '../execution/edit-validation-coordinator';
+import {
+  buildControllerRoundRelayPrompt,
+  claimStalledControllerRoundRelays,
+  finishControllerRoundRelayDispatch,
+} from '../facade/controller-round-relay';
+import { assertAutomatedOperationAllowed } from '../governance/external-effects';
+import { runStandaloneChatgptPrompt } from '../launcher/chatgpt-work-continuation';
+
+export async function runSchedulerPeriodicCleanup(input: {
+  controllerHome: string;
+  controllerPid: number;
+  nowMs: number;
+  cleanupIntervalMs: number;
+  repositories: readonly RepositoryRecord[];
+  runtimeCleanup: typeof cleanupControllerRuntimeState;
+  terminalWorkCleanup: typeof reconcileTerminalWorkCleanups;
+  processGc: typeof gcTerminalProcesses;
+}): Promise<void> {
+  try {
+    input.runtimeCleanup(input.controllerHome, {
+      reason: 'periodic',
+      nowMs: input.nowMs,
+      protectedControllerPid: input.controllerPid,
+    });
+  } catch (error) {
+    console.error('[forge cleanup] periodic cleanup failed:', error);
+  }
+  try {
+    await input.terminalWorkCleanup(input.controllerHome, { nowMs: input.nowMs });
+  } catch (error) {
+    console.error('[forge cleanup] terminal Work cleanup failed:', error);
+  }
+  try {
+    const xctestCleanup = cleanupIdleXCTestDevices(input.controllerHome);
+    if (xctestCleanup.error) console.error('[forge cleanup] XCTest device cleanup failed:', xctestCleanup.error);
+  } catch (error) {
+    console.error('[forge cleanup] XCTest device cleanup failed:', error);
+  }
+  if (input.repositories.length === 0) return;
+  const slot = Math.floor(input.nowMs / input.cleanupIntervalMs) % input.repositories.length;
+  const repo = input.repositories[slot]!;
+  try {
+    const generated = cleanupGeneratedRepositoryCaches(repo.canonicalRoot, { nowMs: input.nowMs });
+    if (generated.errors.length > 0) {
+      console.error(`[forge cleanup] generated-cache retention reported ${generated.errors.length} error(s) for ${repo.repoId}`);
+    }
+  } catch (error) {
+    console.error(`[forge cleanup] generated-cache retention failed for ${repo.repoId}:`, error);
+  }
+  const result = input.processGc({ controllerHome: input.controllerHome, repoId: repo.repoId });
+  if (!result.ok) console.error('[forge cleanup] Process GC failed:', result.error ?? 'unknown error');
+}
+
+export async function runSchedulerValidationReconciliation(input: {
+  controllerHome: string;
+  repositories: readonly RepositoryRecord[];
+  workValidationReconcile: typeof reconcilePendingWorkValidations;
+  editValidationReconcile: typeof reconcilePendingEditValidations;
+}): Promise<void> {
+  for (const repository of input.repositories) {
+    const validation = input.workValidationReconcile(input.controllerHome, repository.repoId, 500);
+    if (validation.errors.length > 0) {
+      console.error(
+        `[forge validation] background Work reconciliation reported ${validation.errors.length} error(s) for ${repository.repoId}`,
+      );
+    }
+    const editValidation = await input.editValidationReconcile(input.controllerHome, repository, 200);
+    if (editValidation.errors.length > 0) {
+      console.error(
+        `[forge validation] background EditSession reconciliation reported ${editValidation.errors.length} error(s) for ${repository.repoId}`,
+      );
+    }
+  }
+}
+
+export async function runSchedulerControllerRoundRecovery(input: {
+  controllerHome: string;
+  nowMs: number;
+  repositories: readonly Pick<RepositoryRecord, 'repoId'>[];
+  graceMs?: number;
+  maxRecoveries?: number;
+  dispatchPrompt?: typeof runStandaloneChatgptPrompt;
+  authorizeWake?: typeof assertAutomatedOperationAllowed;
+}): Promise<{ claimed: number; dispatched: number; failed: number }> {
+  const dispatchPrompt = input.dispatchPrompt ?? runStandaloneChatgptPrompt;
+  const authorizeWake = input.authorizeWake ?? assertAutomatedOperationAllowed;
+  const maxRecoveries = Math.max(1, Math.min(Math.trunc(input.maxRecoveries ?? 2), 8));
+  let claimed = 0;
+  let dispatched = 0;
+  let failed = 0;
+
+  for (const repository of input.repositories) {
+    if (claimed >= maxRecoveries) break;
+    const store = { controllerHome: input.controllerHome, repoId: repository.repoId };
+    const records = claimStalledControllerRoundRelays(store, {
+      nowMs: input.nowMs,
+      graceMs: input.graceMs,
+      limit: maxRecoveries - claimed,
+    });
+    claimed += records.length;
+    for (const record of records) {
+      try {
+        authorizeWake('external_controller_wake', {
+          work_id: record.originWorkId,
+          controller_type: 'chatgpt',
+          relay_scope_id: record.relayScopeId,
+          recovery_reason: 'unclosed_dispatched_round',
+        });
+        const result = await dispatchPrompt({
+          controllerHome: input.controllerHome,
+          repoId: repository.repoId,
+          scopeId: `controller-relay:${record.relayScopeId}`,
+          prompt: buildControllerRoundRelayPrompt(store, record),
+          browserSessionId: record.browserSessionId,
+          conversationUrl: record.conversationUrl,
+          model: 'gpt-5.6',
+          reasoning: 'high',
+          tabPolicy: 'auto',
+          timeoutMs: 30_000,
+        });
+        if (result.status === 'failed') throw new Error(result.error?.message ?? 'CHATGPT_CONTROLLER_RELAY_RECOVERY_FAILED');
+        finishControllerRoundRelayDispatch(store, {
+          workId: record.originWorkId,
+          ok: true,
+          browserSessionId: result.browserSessionId,
+          conversationUrl: result.conversationUrl,
+        });
+        dispatched += 1;
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        finishControllerRoundRelayDispatch(store, { workId: record.originWorkId, ok: false, error: reason });
+        failed += 1;
+        console.error(`[forge controller relay] stalled round recovery failed for ${record.relayScopeId}:`, reason);
+      }
+    }
+  }
+  return { claimed, dispatched, failed };
+}

package/src/runtime/control-plane/global-scheduler/projection-refresh.ts

@@ -0,0 +1,84 @@
+import type { RepositoryRecord } from '../../../cli/repositories/types';
+import { getRuntimeWriteClaim } from '../../root/write-fence';
+import { rebuildRepositoryProjection, refreshRepositoryProjectionForRepository } from '../../projections/materialized-view';
+import { readRepositoryProjectionDirty } from '../../projections/invalidation';
+import { readRepositoryGitStatusSample } from '../../projections/git-status-sampler';
+
+export interface SchedulerProjectionRefreshTargets {
+  repositories: RepositoryRecord[];
+  rebuildRepoIds: string[];
+}
+
+export function selectSchedulerDirtyProjectionRepositories(
+  controllerHome: string,
+  repositories: readonly RepositoryRecord[],
+): RepositoryRecord[] {
+  return repositories.filter((repository) => readRepositoryProjectionDirty(controllerHome, repository.repoId) !== undefined);
+}
+
+export function selectSchedulerProjectionRefreshTargets(
+  repositories: readonly RepositoryRecord[],
+  sourceScanRepositories: readonly RepositoryRecord[],
+  projectionRefreshRepoIds: Iterable<string>,
+): SchedulerProjectionRefreshTargets {
+  const candidates = new Map(sourceScanRepositories.map((repository) => [repository.repoId, repository]));
+  const repositoryById = new Map(repositories.map((repository) => [repository.repoId, repository]));
+  const rebuildRepoIds: string[] = [];
+  const rebuildSeen = new Set<string>();
+  for (const repoId of projectionRefreshRepoIds) {
+    const repository = repositoryById.get(repoId);
+    if (repository) {
+      candidates.set(repoId, repository);
+      continue;
+    }
+    if (!rebuildSeen.has(repoId)) {
+      rebuildSeen.add(repoId);
+      rebuildRepoIds.push(repoId);
+    }
+  }
+  return {
+    repositories: [...candidates.values()],
+    rebuildRepoIds,
+  };
+}
+
+export function refreshSchedulerRepositoryProjections(input: {
+  controllerHome: string;
+  repositories: readonly RepositoryRecord[];
+  sourceScanRepositories: readonly RepositoryRecord[];
+  projectionRefreshRepoIds: Iterable<string>;
+  controllerPid: number;
+}): void {
+  const targets = selectSchedulerProjectionRefreshTargets(
+    input.repositories,
+    input.sourceScanRepositories,
+    input.projectionRefreshRepoIds,
+  );
+  for (const repoId of targets.rebuildRepoIds) {
+    try {
+      rebuildRepositoryProjection(input.controllerHome, repoId);
+    } catch (error) {
+      console.error('[forge scheduler] projection refresh failed:', error);
+    }
+  }
+  for (const repository of targets.repositories) {
+    try {
+      const sample = readRepositoryGitStatusSample(
+        input.controllerHome,
+        repository.repoId,
+        repository.activeCheckoutId,
+      );
+      const runtimeInstanceId = getRuntimeWriteClaim()?.runtimeInstanceId;
+      refreshRepositoryProjectionForRepository(input.controllerHome, repository, {
+        sourceRevision: sample?.head ?? undefined,
+        reason: 'scheduler-source-scan',
+        owner: {
+          pid: input.controllerPid,
+          ...(runtimeInstanceId ? { runtimeInstanceId } : {}),
+        },
+      });
+    } catch (error) {
+      console.error('[forge scheduler] projection refresh failed:', error);
+    }
+  }
+}

package/src/runtime/control-plane/global-scheduler/reconciliation.ts

@@ -1,4 +1,4 @@
-import { listActiveExecutionJobs, transitionExecutionJob } from '../../execution/jobs/store';
+import { reconcileActiveExecutionJobIndex, transitionExecutionJob } from '../../execution/jobs/store';
 import type { ExecutionJob } from '../../execution/jobs/types';
 import {
   executionTimeoutDecision,
@@ -142,7 +142,7 @@ function reconcileExecutionJobsWith(
   terminateWorker: (pid: number | undefined) => ProcessTreeTerminationResult,
   repoId?: string,
 ): ReconcileSummary {
-  const jobs = listActiveExecutionJobs(controllerHome, repoId);
+  const jobs = reconcileActiveExecutionJobIndex(controllerHome, repoId);
   let requeued = 0;
   let terminal = 0;
   let recovered = 0;
@@ -200,7 +200,7 @@ async function reconcileExecutionJobsAsyncWith(
   terminateWorker: (pid: number | undefined) => Promise<ProcessTreeTerminationResult>,
   repoId?: string,
 ): Promise<ReconcileSummary> {
-  const jobs = listActiveExecutionJobs(controllerHome, repoId);
+  const jobs = reconcileActiveExecutionJobIndex(controllerHome, repoId);
   let requeued = 0;
   let terminal = 0;
   let recovered = 0;