@moretea-labs/forge 1.6.0 → 1.7.1

package/src/runtime/control-plane/global-scheduler/durable-admission.ts

@@ -0,0 +1,46 @@
+import {
+  listActiveExecutionJobs,
+  markExecutionJobSchedulerObserved,
+} from '../../execution/jobs/store';
+import { tickSchedules } from '../../workflow/schedules/engine';
+
+const SCHEDULE_TICK_INTERVAL_MS = 30_000;
+
+export interface SchedulerDurableAdmissionDependencies {
+  listActiveJobs: typeof listActiveExecutionJobs;
+  markSchedulerObserved: typeof markExecutionJobSchedulerObserved;
+  tickSchedules: typeof tickSchedules;
+}
+
+const DEFAULT_DEPENDENCIES: SchedulerDurableAdmissionDependencies = {
+  listActiveJobs: listActiveExecutionJobs,
+  markSchedulerObserved: markExecutionJobSchedulerObserved,
+  tickSchedules,
+};
+
+export async function runSchedulerDurableAdmission(input: {
+  controllerHome: string;
+  repositoryIds: readonly string[];
+  nowMs: number;
+  lastScheduleTickAt: number;
+}, dependencies: SchedulerDurableAdmissionDependencies = DEFAULT_DEPENDENCIES): Promise<{
+  scheduleTicked: boolean;
+}> {
+  // Observation closes the admission phase for durable Jobs and starts their
+  // independent queue budget. Each Job owns its own atomic state transition,
+  // so this intentionally remains outside the global dispatch reservation lock.
+  for (const job of dependencies.listActiveJobs(input.controllerHome)) {
+    if (job.status === 'running' || job.timings?.schedulerObservedAt) continue;
+    try {
+      dependencies.markSchedulerObserved(input.controllerHome, job.repoId, job.jobId);
+    } catch {
+      // Another Scheduler instance or a terminal transition won the Job-local race.
+    }
+  }
+
+  if (input.nowMs - input.lastScheduleTickAt < SCHEDULE_TICK_INTERVAL_MS) {
+    return { scheduleTicked: false };
+  }
+  await dependencies.tickSchedules(input.controllerHome, [...input.repositoryIds]);
+  return { scheduleTicked: true };
+}

package/src/runtime/control-plane/global-scheduler/maintenance.ts

@@ -0,0 +1,152 @@
+import type { RepositoryRecord } from '../../../cli/repositories/types';
+import { cleanupGeneratedRepositoryCaches, cleanupIdleXCTestDevices } from '../generated-cache-retention';
+import type { cleanupControllerRuntimeState } from '../runtime-cleanup';
+import type { reconcileTerminalWorkCleanups } from '../execution/work-terminal-cleanup';
+import type { gcTerminalProcesses } from '../../execution/process-runtime/gc';
+import type { reconcilePendingWorkValidations } from '../execution/work-validation-reconciler';
+import type { reconcilePendingEditValidations } from '../execution/edit-validation-coordinator';
+import {
+  buildControllerRoundRelayPrompt,
+  claimStalledControllerRoundRelays,
+  finishControllerRoundRelayDispatch,
+} from '../facade/controller-round-relay';
+import { assertAutomatedOperationAllowed } from '../governance/external-effects';
+import { runWorkChatgptContinuation } from '../launcher/chatgpt-work-continuation';
+
+export async function runSchedulerPeriodicCleanup(input: {
+  controllerHome: string;
+  controllerPid: number;
+  nowMs: number;
+  cleanupIntervalMs: number;
+  repositories: readonly RepositoryRecord[];
+  runtimeCleanup: typeof cleanupControllerRuntimeState;
+  terminalWorkCleanup: typeof reconcileTerminalWorkCleanups;
+  processGc: typeof gcTerminalProcesses;
+}): Promise<void> {
+  try {
+    input.runtimeCleanup(input.controllerHome, {
+      reason: 'periodic',
+      nowMs: input.nowMs,
+      protectedControllerPid: input.controllerPid,
+    });
+  } catch (error) {
+    console.error('[forge cleanup] periodic cleanup failed:', error);
+  }
+  try {
+    await input.terminalWorkCleanup(input.controllerHome, { nowMs: input.nowMs });
+  } catch (error) {
+    console.error('[forge cleanup] terminal Work cleanup failed:', error);
+  }
+  try {
+    const xctestCleanup = cleanupIdleXCTestDevices(input.controllerHome);
+    if (xctestCleanup.error) console.error('[forge cleanup] XCTest device cleanup failed:', xctestCleanup.error);
+  } catch (error) {
+    console.error('[forge cleanup] XCTest device cleanup failed:', error);
+  }
+  if (input.repositories.length === 0) return;
+  const slot = Math.floor(input.nowMs / input.cleanupIntervalMs) % input.repositories.length;
+  const repo = input.repositories[slot]!;
+  try {
+    const generated = cleanupGeneratedRepositoryCaches(repo.canonicalRoot, { nowMs: input.nowMs });
+    if (generated.errors.length > 0) {
+      console.error(`[forge cleanup] generated-cache retention reported ${generated.errors.length} error(s) for ${repo.repoId}`);
+    }
+  } catch (error) {
+    console.error(`[forge cleanup] generated-cache retention failed for ${repo.repoId}:`, error);
+  }
+  const result = input.processGc({ controllerHome: input.controllerHome, repoId: repo.repoId });
+  if (!result.ok) console.error('[forge cleanup] Process GC failed:', result.error ?? 'unknown error');
+}
+
+export async function runSchedulerValidationReconciliation(input: {
+  controllerHome: string;
+  repositories: readonly RepositoryRecord[];
+  workValidationReconcile: typeof reconcilePendingWorkValidations;
+  editValidationReconcile: typeof reconcilePendingEditValidations;
+}): Promise<void> {
+  for (const repository of input.repositories) {
+    const validation = input.workValidationReconcile(input.controllerHome, repository.repoId, 500);
+    if (validation.errors.length > 0) {
+      console.error(
+        `[forge validation] background Work reconciliation reported ${validation.errors.length} error(s) for ${repository.repoId}`,
+      );
+    }
+    const editValidation = await input.editValidationReconcile(input.controllerHome, repository, 200);
+    if (editValidation.errors.length > 0) {
+      console.error(
+        `[forge validation] background EditSession reconciliation reported ${editValidation.errors.length} error(s) for ${repository.repoId}`,
+      );
+    }
+  }
+}
+
+export async function runSchedulerControllerRoundRecovery(input: {
+  controllerHome: string;
+  nowMs: number;
+  repositories: readonly Pick<RepositoryRecord, 'repoId' | 'canonicalRoot' | 'localRoot'>[];
+  graceMs?: number;
+  maxRecoveries?: number;
+  dispatchPrompt?: typeof runWorkChatgptContinuation;
+  authorizeWake?: typeof assertAutomatedOperationAllowed;
+}): Promise<{ claimed: number; dispatched: number; failed: number }> {
+  const dispatchPrompt = input.dispatchPrompt ?? runWorkChatgptContinuation;
+  const authorizeWake = input.authorizeWake ?? assertAutomatedOperationAllowed;
+  const maxRecoveries = Math.max(1, Math.min(Math.trunc(input.maxRecoveries ?? 2), 8));
+  let claimed = 0;
+  let dispatched = 0;
+  let failed = 0;
+
+  for (const repository of input.repositories) {
+    if (claimed >= maxRecoveries) break;
+    const store = { controllerHome: input.controllerHome, repoId: repository.repoId };
+    const records = claimStalledControllerRoundRelays(store, {
+      nowMs: input.nowMs,
+      graceMs: input.graceMs,
+      limit: maxRecoveries - claimed,
+    });
+    claimed += records.length;
+    for (const record of records) {
+      try {
+        authorizeWake('external_controller_wake', {
+          work_id: record.originWorkId,
+          controller_type: 'chatgpt',
+          relay_scope_id: record.relayScopeId,
+          recovery_reason: 'unclosed_dispatched_round',
+        });
+        const result = await dispatchPrompt({
+          controllerHome: input.controllerHome,
+          repoId: repository.repoId,
+          repoRoot: repository.canonicalRoot ?? repository.localRoot,
+          workId: record.originWorkId,
+          prompt: buildControllerRoundRelayPrompt(store, record, { exactOriginWork: true }),
+          browserSessionId: record.browserSessionId,
+          conversationUrl: record.conversationUrl,
+          model: 'gpt-5.6',
+          reasoning: 'high',
+          tabPolicy: 'auto',
+          timeoutMs: 30_000,
+        });
+        if (result.status === 'failed') throw new Error(result.error?.message ?? 'CHATGPT_CONTROLLER_RELAY_RECOVERY_FAILED');
+        finishControllerRoundRelayDispatch(store, {
+          workId: record.originWorkId,
+          ok: true,
+          browserSessionId: result.browserSessionId,
+          conversationUrl: result.conversationUrl,
+        });
+        dispatched += 1;
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        finishControllerRoundRelayDispatch(store, {
+          workId: record.originWorkId,
+          ok: false,
+          error: reason,
+          recovery: true,
+          nowMs: input.nowMs,
+        });
+        failed += 1;
+        console.error(`[forge controller relay] stalled round recovery failed for ${record.relayScopeId}:`, reason);
+      }
+    }
+  }
+  return { claimed, dispatched, failed };
+}

package/src/runtime/control-plane/global-scheduler/projection-refresh.ts

@@ -0,0 +1,84 @@
+import type { RepositoryRecord } from '../../../cli/repositories/types';
+import { getRuntimeWriteClaim } from '../../root/write-fence';
+import { rebuildRepositoryProjection, refreshRepositoryProjectionForRepository } from '../../projections/materialized-view';
+import { readRepositoryProjectionDirty } from '../../projections/invalidation';
+import { readRepositoryGitStatusSample } from '../../projections/git-status-sampler';
+
+export interface SchedulerProjectionRefreshTargets {
+  repositories: RepositoryRecord[];
+  rebuildRepoIds: string[];
+}
+
+export function selectSchedulerDirtyProjectionRepositories(
+  controllerHome: string,
+  repositories: readonly RepositoryRecord[],
+): RepositoryRecord[] {
+  return repositories.filter((repository) => readRepositoryProjectionDirty(controllerHome, repository.repoId) !== undefined);
+}
+
+export function selectSchedulerProjectionRefreshTargets(
+  repositories: readonly RepositoryRecord[],
+  sourceScanRepositories: readonly RepositoryRecord[],
+  projectionRefreshRepoIds: Iterable<string>,
+): SchedulerProjectionRefreshTargets {
+  const candidates = new Map(sourceScanRepositories.map((repository) => [repository.repoId, repository]));
+  const repositoryById = new Map(repositories.map((repository) => [repository.repoId, repository]));
+  const rebuildRepoIds: string[] = [];
+  const rebuildSeen = new Set<string>();
+  for (const repoId of projectionRefreshRepoIds) {
+    const repository = repositoryById.get(repoId);
+    if (repository) {
+      candidates.set(repoId, repository);
+      continue;
+    }
+    if (!rebuildSeen.has(repoId)) {
+      rebuildSeen.add(repoId);
+      rebuildRepoIds.push(repoId);
+    }
+  }
+  return {
+    repositories: [...candidates.values()],
+    rebuildRepoIds,
+  };
+}
+
+export function refreshSchedulerRepositoryProjections(input: {
+  controllerHome: string;
+  repositories: readonly RepositoryRecord[];
+  sourceScanRepositories: readonly RepositoryRecord[];
+  projectionRefreshRepoIds: Iterable<string>;
+  controllerPid: number;
+}): void {
+  const targets = selectSchedulerProjectionRefreshTargets(
+    input.repositories,
+    input.sourceScanRepositories,
+    input.projectionRefreshRepoIds,
+  );
+  for (const repoId of targets.rebuildRepoIds) {
+    try {
+      rebuildRepositoryProjection(input.controllerHome, repoId);
+    } catch (error) {
+      console.error('[forge scheduler] projection refresh failed:', error);
+    }
+  }
+  for (const repository of targets.repositories) {
+    try {
+      const sample = readRepositoryGitStatusSample(
+        input.controllerHome,
+        repository.repoId,
+        repository.activeCheckoutId,
+      );
+      const runtimeInstanceId = getRuntimeWriteClaim()?.runtimeInstanceId;
+      refreshRepositoryProjectionForRepository(input.controllerHome, repository, {
+        sourceRevision: sample?.head ?? undefined,
+        reason: 'scheduler-source-scan',
+        owner: {
+          pid: input.controllerPid,
+          ...(runtimeInstanceId ? { runtimeInstanceId } : {}),
+        },
+      });
+    } catch (error) {
+      console.error('[forge scheduler] projection refresh failed:', error);
+    }
+  }
+}

package/src/runtime/control-plane/global-scheduler/reconciliation.ts

@@ -1,4 +1,4 @@
-import { listActiveExecutionJobs, transitionExecutionJob } from '../../execution/jobs/store';
+import { reconcileActiveExecutionJobIndex, transitionExecutionJob } from '../../execution/jobs/store';
 import type { ExecutionJob } from '../../execution/jobs/types';
 import {
   executionTimeoutDecision,
@@ -142,7 +142,7 @@ function reconcileExecutionJobsWith(
   terminateWorker: (pid: number | undefined) => ProcessTreeTerminationResult,
   repoId?: string,
 ): ReconcileSummary {
-  const jobs = listActiveExecutionJobs(controllerHome, repoId);
+  const jobs = reconcileActiveExecutionJobIndex(controllerHome, repoId);
   let requeued = 0;
   let terminal = 0;
   let recovered = 0;
@@ -200,7 +200,7 @@ async function reconcileExecutionJobsAsyncWith(
   terminateWorker: (pid: number | undefined) => Promise<ProcessTreeTerminationResult>,
   repoId?: string,
 ): Promise<ReconcileSummary> {
-  const jobs = listActiveExecutionJobs(controllerHome, repoId);
+  const jobs = reconcileActiveExecutionJobIndex(controllerHome, repoId);
   let requeued = 0;
   let terminal = 0;
   let recovered = 0;