@moretea-labs/forge 1.7.1 → 1.8.0

package/src/runtime/control-plane/global-scheduler/scheduler.ts

@@ -1,5 +1,5 @@
 import { execFile, type ChildProcess } from 'child_process';
-import { resolve } from 'path';
+import { dirname, resolve } from 'path';
 import { cpus, freemem, loadavg } from 'os';
 import { listRepositories } from '../../../cli/repositories/registry';
 import { writeAgentExecutableReadinessSnapshot } from '../../../cli/agent-jobs/executable-resolver';
@@ -30,7 +30,10 @@ import {
   runSchedulerValidationReconciliation,
 } from './maintenance';
 import { planSchedulerSourceSampling } from './source-scan';
-import { runSchedulerDurableAdmission } from './durable-admission';
+import {
+  runSchedulerDurableAdmission,
+  schedulerDurableAdmissionRequiresPolicy,
+} from './durable-admission';
 import { sampleRepositoryGitStatusForRepositories } from '../../projections/git-status-sampler';
 import { selectExecutionJobDispatchRepositories } from '../dispatch-priority';
 import {
@@ -49,6 +52,7 @@ import {
 } from './projection-refresh';
 import { createSchedulerWorkerStderrCapture } from './worker-stderr';
 import { persistSchedulerWorkerAttachment } from './worker-attachment';
+import { reconcileWorkExecutionConcurrencyWaits } from '../concurrency/work-execution-concurrency';
 import {
   cleanupSchedulerWorkerProcesses,
   registerSchedulerWorkerProcess,
@@ -57,6 +61,10 @@ import {
 } from './worker-process';
 import { reconcileSchedulerWorkerExit } from './worker-exit-reconciler';
 import { persistSchedulerSpawnedWorkerLifecycle } from './worker-lifecycle-store';
+import {
+  spawnSchedulerPeriodicCleanup,
+  type SchedulerPeriodicCleanupSpawnResult,
+} from './periodic-cleanup-process';
 import {
   buildSchedulerWorkerExitedLifecycle,
   buildSchedulerWorkerSpawnFailureLifecycle,
@@ -155,6 +163,10 @@ export interface SchedulerRuntimeBinding {
   controllerPid?: number;
   runtimeSourceRoot?: string;
   workerEntrypoint?: string;
+  workerExecutable?: string;
+  periodicCleanupExecutable?: string;
+  /** Canonical in-process Runtime isolates periodic cleanup from the public event loop. */
+  isolatePeriodicCleanup?: boolean;
   /** Canonical Runtime treats a tick failure as a whole-Runtime failure. */
   fatalOnTickError?: boolean;
 }
@@ -163,10 +175,14 @@ export class GlobalScheduler {
   private readonly controllerHome: string;
   private readonly actors: RepoActorRegistry;
   private readonly children = new Map<string, ChildProcess>();
+  private readonly maintenanceChildren = new Map<string, ChildProcess>();
   private readonly config: SchedulerConfig;
   private readonly controllerPid: number;
   private readonly runtimeSourceRoot?: string;
   private readonly workerEntrypoint?: string;
+  private readonly workerExecutable?: string;
+  private readonly periodicCleanupExecutable?: string;
+  private readonly isolatePeriodicCleanup: boolean;
   private readonly fatalOnTickError: boolean;
   private lastScheduleTick = 0;
   private lastReconcile = 0;
@@ -185,6 +201,8 @@ export class GlobalScheduler {
   private runtimeCleanup = cleanupControllerRuntimeState;
   private terminalWorkCleanup = reconcileTerminalWorkCleanups;
   private processGc = gcTerminalProcesses;
+  private periodicCleanupSpawn = spawnSchedulerPeriodicCleanup;
+  private workExecutionConcurrencyReconcile = reconcileWorkExecutionConcurrencyWaits;
   private workValidationReconcile = reconcilePendingWorkValidations;
   private editValidationReconcile = reconcilePendingEditValidations;
   private repositoryList = listRepositories;
@@ -207,6 +225,9 @@ export class GlobalScheduler {
     this.actors = new RepoActorRegistry(controllerHome, { maxConcurrentWorkers: this.config.maxWorkers });
     this.runtimeSourceRoot = runtime.runtimeSourceRoot ? resolve(runtime.runtimeSourceRoot) : undefined;
     this.workerEntrypoint = runtime.workerEntrypoint ? resolve(runtime.workerEntrypoint) : undefined;
+    this.workerExecutable = runtime.workerExecutable ? resolve(runtime.workerExecutable) : undefined;
+    this.periodicCleanupExecutable = runtime.periodicCleanupExecutable ? resolve(runtime.periodicCleanupExecutable) : undefined;
+    this.isolatePeriodicCleanup = runtime.isolatePeriodicCleanup === true;
     this.fatalOnTickError = runtime.fatalOnTickError === true;
     const restoredState = restoreSchedulerState(readSchedulerHealthSnapshot(controllerHome));
     this.lastSourceScanAt = restoredState.lastSourceScanAt;
@@ -288,12 +309,13 @@ export class GlobalScheduler {
         return resolveSchedulerWorkerCommand({
           runtimeSourceRoot: this.runtimeSourceRoot,
           workerEntrypoint: this.workerEntrypoint,
+          standaloneExecutable: this.workerExecutable,
         });
       } catch (error) {
         const message = error instanceof Error ? error.message : String(error);
         const lifecycle = buildSchedulerWorkerSpawnFailureLifecycle({
-          executable: process.execPath,
-          cwd: this.runtimeSourceRoot ?? process.cwd(),
+          executable: this.workerExecutable ?? process.execPath,
+          cwd: this.workerExecutable ? dirname(this.workerExecutable) : (this.runtimeSourceRoot ?? process.cwd()),
           environment: selectSchedulerWorkerEnvironment(process.env),
           ownerPid: this.controllerPid,
           attempt: current.attempt,
@@ -319,7 +341,7 @@ export class GlobalScheduler {
       repoId,
       jobId,
       controllerPid: this.controllerPid,
-      runtimeSourceRoot: this.runtimeSourceRoot,
+      runtimeSourceRoot: this.workerExecutable ? undefined : this.runtimeSourceRoot,
       writeClaimEnvironment: writeClaim ? runtimeWriteClaimEnvironment(writeClaim) : {},
     });
     const stderrCapture = createSchedulerWorkerStderrCapture({
@@ -439,6 +461,49 @@ export class GlobalScheduler {
     };
   }
 
+  private launchPeriodicCleanup(nowMs: number): boolean {
+    const key = 'periodic-runtime-cleanup';
+    const tracked = this.maintenanceChildren.get(key);
+    if (tracked?.pid && this.pidAlive(tracked.pid)) return false;
+    if (tracked) this.maintenanceChildren.delete(key);
+
+    const fence = assertRuntimeMayWrite('cleanup', this.controllerHome);
+    if (!fence.allowed) return false;
+    const writeClaim = getRuntimeWriteClaim();
+    const spawned: SchedulerPeriodicCleanupSpawnResult = this.periodicCleanupSpawn({
+      controllerHome: this.controllerHome,
+      controllerPid: this.controllerPid,
+      nowMs,
+      cleanupIntervalMs: RUNTIME_CLEANUP_INTERVAL_MS,
+      runtimeSourceRoot: this.periodicCleanupExecutable ? undefined : this.runtimeSourceRoot,
+      cleanupExecutable: this.periodicCleanupExecutable,
+      writeClaimEnvironment: writeClaim ? runtimeWriteClaimEnvironment(writeClaim) : {},
+    });
+    if (!spawned.ok) {
+      console.error('[forge cleanup] failed to launch isolated periodic cleanup:', spawned.startupError);
+      return false;
+    }
+    const child = spawned.child;
+    this.maintenanceChildren.set(key, child);
+    const clear = () => {
+      if (this.maintenanceChildren.get(key) === child) this.maintenanceChildren.delete(key);
+    };
+    child.once('error', (error) => {
+      clear();
+      console.error('[forge cleanup] isolated periodic cleanup process failed:', error.message);
+    });
+    child.once('close', (code, signal) => {
+      clear();
+      if (code !== 0) {
+        console.error(
+          '[forge cleanup] isolated periodic cleanup exited unsuccessfully:',
+          `code=${code ?? 'null'} signal=${signal ?? 'none'}`,
+        );
+      }
+    });
+    return true;
+  }
+
   async tick(): Promise<{ activeJobs: number }> {
     const now = Date.now();
     this.lastHeartbeatAt = new Date(now).toISOString();
@@ -448,22 +513,36 @@ export class GlobalScheduler {
     let periodicCleanupRan = false;
     if (now - this.lastCleanupAt >= RUNTIME_CLEANUP_INTERVAL_MS) {
       // Advance the interval before cleanup so a failing pass cannot create a
-      // tight retry loop on every scheduler tick.
+      // tight retry loop on every scheduler tick. Canonical Runtime launches the
+      // synchronous cleanup scanner in a release-fenced child so recursive FS,
+      // process-table and legacy reconciliation work cannot block MCP traffic.
       this.lastCleanupAt = now;
-      await runSchedulerPeriodicCleanup({
-        controllerHome: this.controllerHome,
-        controllerPid: this.controllerPid,
-        nowMs: now,
-        cleanupIntervalMs: RUNTIME_CLEANUP_INTERVAL_MS,
-        repositories,
-        runtimeCleanup: this.runtimeCleanup,
-        terminalWorkCleanup: this.terminalWorkCleanup,
-        processGc: this.processGc,
-      });
+      if (this.isolatePeriodicCleanup) {
+        this.launchPeriodicCleanup(now);
+      } else {
+        await runSchedulerPeriodicCleanup({
+          controllerHome: this.controllerHome,
+          controllerPid: this.controllerPid,
+          nowMs: now,
+          cleanupIntervalMs: RUNTIME_CLEANUP_INTERVAL_MS,
+          repositories,
+          runtimeCleanup: this.runtimeCleanup,
+          terminalWorkCleanup: this.terminalWorkCleanup,
+          processGc: this.processGc,
+        });
+      }
       periodicCleanupRan = true;
     }
     if (now - this.lastReconcile >= 5_000) {
       await reconcileExecutionJobsAsync(this.controllerHome);
+      for (const repository of repositories) {
+        try {
+          this.workExecutionConcurrencyReconcile({ controllerHome: this.controllerHome, repoId: repository.repoId });
+        } catch (error) {
+          const reason = error instanceof Error ? error.message : String(error);
+          console.error(`[forge concurrency] Work wait reconciliation failed for ${repository.repoId}:`, reason);
+        }
+      }
       await runSchedulerValidationReconciliation({
         controllerHome: this.controllerHome,
         repositories,
@@ -490,7 +569,7 @@ export class GlobalScheduler {
       this.lastGitStatusSampleAt = now;
       this.lastSourceScanAt = now;
       this.lastSourceScanRepoCount = sourceScanRepositories.length;
-      sampleRepositoryGitStatusForRepositories(this.controllerHome, sourceScanRepositories);
+      await sampleRepositoryGitStatusForRepositories(this.controllerHome, sourceScanRepositories);
     }
     // Dirty projection markers are authoritative maintenance requests. Do not
     // make them wait behind the idle one-repository-per-minute safety scan: only
@@ -501,17 +580,24 @@ export class GlobalScheduler {
     const dirtyRepositoriesToSample = dirtyProjectionRepositories
       .filter((repository) => !alreadySampledRepoIds.has(repository.repoId));
     if (dirtyRepositoriesToSample.length > 0) {
-      sampleRepositoryGitStatusForRepositories(this.controllerHome, dirtyRepositoriesToSample);
+      await sampleRepositoryGitStatusForRepositories(this.controllerHome, dirtyRepositoriesToSample);
     }
     const projectionMaintenanceRepositories = Array.from(new Map(
       [...sourceScanRepositories, ...dirtyProjectionRepositories]
         .map((repository) => [repository.repoId, repository] as const),
     ).values());
     this.sourceScansAvoided += sourceSampling.avoidedRepositoryCount;
-    // Phase 0 reuses one durable Work admission policy. Cleanup, stale-state
-    // reconciliation, read-only source sampling, and projection maintenance remain
-    // available, but ordinary schedule/workflow advancement and Worker dispatch stop here.
-    if (!schedulerDispatchAllowed(this.controllerHome)) {
+    // Phase 0 reuses one durable Work admission policy. Exclusive-Work mode stops
+    // ordinary schedule/workflow advancement and Worker dispatch while preserving
+    // cleanup and projections. Convergence mode remains dispatchable because new
+    // Work creation is independently fenced at the Work contract authority, so
+    // existing Work continuations can keep draining the backlog.
+    const admissionPolicyRequired = periodicCleanupRan || schedulerDurableAdmissionRequiresPolicy({
+      activeJobs: activeJobSnapshot,
+      nowMs: now,
+      lastScheduleTickAt: this.lastScheduleTick,
+    });
+    if (admissionPolicyRequired && !schedulerDispatchAllowed(this.controllerHome)) {
       refreshSchedulerRepositoryProjections({
         controllerHome: this.controllerHome,
         repositories,
@@ -535,71 +621,74 @@ export class GlobalScheduler {
       repositoryIds: repositories.map((repo) => repo.repoId),
       nowMs: now,
       lastScheduleTickAt: this.lastScheduleTick,
+      activeJobs: activeJobSnapshot,
     });
     if (durableAdmission.scheduleTicked) this.lastScheduleTick = now;
     let activeJobs = 0;
     const pendingSpawns: Array<{ repoId: string; jobId: string }> = [];
     const projectionRefreshRepos = new Set<string>();
-    const pressure = this.resourcePressure();
-    try {
-      activeJobs = withControllerLock(
-        this.controllerHome,
-        {
-          scope: 'task',
-          repoId: '__controller__',
-          taskId: 'global-scheduler-dispatch',
-        },
-        `global-scheduler:${this.controllerPid}`,
-        () => {
-          const active = listActiveExecutionJobs(this.controllerHome);
-          const capacity = createSchedulerDispatchCapacity(active, this.config, pressure.pressured);
-          if (capacity.workers <= 0) return active.length;
+    if (activeJobSnapshot.length > 0) {
+      const pressure = this.resourcePressure();
+      try {
+        activeJobs = withControllerLock(
+          this.controllerHome,
+          {
+            scope: 'task',
+            repoId: '__controller__',
+            taskId: 'global-scheduler-dispatch',
+          },
+          `global-scheduler:${this.controllerPid}`,
+          () => {
+            const active = listActiveExecutionJobs(this.controllerHome);
+            const capacity = createSchedulerDispatchCapacity(active, this.config, pressure.pressured);
+            if (capacity.workers <= 0) return active.length;
 
-          const scheduleNow = Date.now();
-          const repoIds = selectExecutionJobDispatchRepositories(active, scheduleNow, this.lastRepoDispatch);
-          const reservedRepos = new Set(capacity.reservedJobs.map((job) => job.repoId));
-          let dispatchStateChanged = false;
-          const canDispatch = (job: (typeof active)[number]): boolean => schedulerDispatchCapacityAllows(capacity, job);
-          for (const repoId of repoIds) {
-            if (capacity.workers <= 0) break;
-            if (!reservedRepos.has(repoId) && reservedRepos.size >= this.config.maxConcurrentRepositories) continue;
-            const actor = this.actors.get(repoId);
-            let dispatch: ReturnType<typeof actor.tryClaimNext>;
-            try {
-              dispatch = actor.tryClaimNext({
-                scheduleNow,
-                canDispatch,
-                refreshProjection: false,
-                lockWaitMs: 0,
-              });
-              projectionRefreshRepos.add(repoId);
-            } catch (error) {
-              if (error instanceof Error && error.message.startsWith('LOCK_HELD:')) continue;
-              throw error;
-            }
-            if (!dispatch) continue;
+            const scheduleNow = Date.now();
+            const repoIds = selectExecutionJobDispatchRepositories(active, scheduleNow, this.lastRepoDispatch);
+            const reservedRepos = new Set(capacity.reservedJobs.map((job) => job.repoId));
+            let dispatchStateChanged = false;
+            const canDispatch = (job: (typeof active)[number]): boolean => schedulerDispatchCapacityAllows(capacity, job);
+            for (const repoId of repoIds) {
+              if (capacity.workers <= 0) break;
+              if (!reservedRepos.has(repoId) && reservedRepos.size >= this.config.maxConcurrentRepositories) continue;
+              const actor = this.actors.get(repoId);
+              let dispatch: ReturnType<typeof actor.tryClaimNext>;
+              try {
+                dispatch = actor.tryClaimNext({
+                  scheduleNow,
+                  canDispatch,
+                  refreshProjection: false,
+                  lockWaitMs: 0,
+                });
+                projectionRefreshRepos.add(repoId);
+              } catch (error) {
+                if (error instanceof Error && error.message.startsWith('LOCK_HELD:')) continue;
+                throw error;
+              }
+              if (!dispatch) continue;
 
-            // A successful claim is the capacity reservation. Count it immediately,
-            // before the worker PID is spawned or attached, so concurrent schedulers
-            // cannot over-dispatch through the dispatched -> running window.
-            consumeSchedulerDispatchCapacity(capacity, dispatch.job);
-            reservedRepos.add(repoId);
-            const dispatchedAt = Date.now();
-            this.lastRepoDispatch.set(repoId, dispatchedAt);
-            this.lastDispatchAt = new Date(dispatchedAt).toISOString();
-            dispatchStateChanged = true;
-            pendingSpawns.push({ repoId, jobId: dispatch.job.jobId });
-          }
-          if (dispatchStateChanged) this.persistState(true);
-          return active.length;
-        },
-        5_000,
-      );
-    } catch (error) {
-      if (!(error instanceof Error) || !error.message.startsWith('LOCK_HELD:')) throw error;
-      // Another scheduler owns the global dispatch reservation. Fail closed and
-      // leave all jobs queued for the next wake/tick rather than risking overrun.
-      activeJobs = listActiveExecutionJobs(this.controllerHome).length;
+              // A successful claim is the capacity reservation. Count it immediately,
+              // before the worker PID is spawned or attached, so concurrent schedulers
+              // cannot over-dispatch through the dispatched -> running window.
+              consumeSchedulerDispatchCapacity(capacity, dispatch.job);
+              reservedRepos.add(repoId);
+              const dispatchedAt = Date.now();
+              this.lastRepoDispatch.set(repoId, dispatchedAt);
+              this.lastDispatchAt = new Date(dispatchedAt).toISOString();
+              dispatchStateChanged = true;
+              pendingSpawns.push({ repoId, jobId: dispatch.job.jobId });
+            }
+            if (dispatchStateChanged) this.persistState(true);
+            return active.length;
+          },
+          5_000,
+        );
+      } catch (error) {
+        if (!(error instanceof Error) || !error.message.startsWith('LOCK_HELD:')) throw error;
+        // Another scheduler owns the global dispatch reservation. The pre-lock snapshot
+        // is sufficient for backoff accounting; the next wake/tick observes the winner's state.
+        activeJobs = activeJobSnapshot.length;
+      }
     }
     // Repo Actor mutations leave projection dirty markers. Refresh those repos
     // immediately; the independent source safety scan contributes only its bounded
@@ -671,7 +760,10 @@ export class GlobalScheduler {
       }
     } finally {
       clearInterval(heartbeatTimer);
-      await cleanupSchedulerWorkerProcesses(this.children);
+      await Promise.all([
+        cleanupSchedulerWorkerProcesses(this.children),
+        cleanupSchedulerWorkerProcesses(this.maintenanceChildren),
+      ]);
     }
   }
 }

package/src/runtime/control-plane/global-scheduler/worker-launch.ts

@@ -1,6 +1,6 @@
 import { existsSync } from 'fs';
 import { fileURLToPath } from 'url';
-import { join, resolve } from 'path';
+import { dirname, join, resolve } from 'path';
 import { resolveBunExecutable } from '../../shared/process-environment';
 
 const WORKER_ENVIRONMENT_KEYS = [
@@ -31,15 +31,24 @@ export interface SchedulerWorkerCommand {
   entry: string;
   loader: string;
   cwd: string;
+  /** Manifest-owned compiled artifact. Spawn directly without Bun/Node/source loaders. */
+  directExecutable?: true;
 }
 
 export function resolveSchedulerWorkerCommand(input: {
   runtimeSourceRoot?: string;
   workerEntrypoint?: string;
+  standaloneExecutable?: string;
   isBun?: boolean;
   cwd?: string;
   pathExists?: (path: string) => boolean;
 } = {}): SchedulerWorkerCommand {
+  const pathExists = input.pathExists ?? existsSync;
+  if (input.standaloneExecutable) {
+    const entry = resolve(input.standaloneExecutable);
+    if (!pathExists(entry)) throw new Error(`WORKER_ENTRYPOINT_MISSING: ${entry}`);
+    return { entry, loader: '', cwd: input.cwd ?? dirname(entry), directExecutable: true };
+  }
   const runtimeSourceRoot = input.runtimeSourceRoot ? resolve(input.runtimeSourceRoot) : undefined;
   const sourceEntry = runtimeSourceRoot
     ? join(runtimeSourceRoot, 'src', 'runtime', 'execution', 'workers', 'worker-entry.ts')
@@ -49,7 +58,6 @@ export function resolveSchedulerWorkerCommand(input: {
     : fileURLToPath(new URL('../../shared/node-ts-loader.mjs', import.meta.url));
   const entry = input.workerEntrypoint ? resolve(input.workerEntrypoint) : sourceEntry;
   const cwd = runtimeSourceRoot ?? input.cwd ?? process.cwd();
-  const pathExists = input.pathExists ?? existsSync;
   const isBun = input.isBun ?? Boolean(process.versions.bun);
   if (!pathExists(entry)) throw new Error(`WORKER_ENTRYPOINT_MISSING: ${entry}`);
   if (!isBun && !pathExists(loader)) throw new Error(`WORKER_LOADER_MISSING: ${loader}`);
@@ -84,16 +92,20 @@ export function buildSchedulerWorkerLaunchDescriptor(input: {
 }): SchedulerWorkerLaunchDescriptor {
   const isBun = input.isBun ?? Boolean(process.versions.bun);
   const environmentSource = input.environment ?? process.env;
-  const executable = resolveSchedulerWorkerExecutable(isBun, input.execPath ?? process.execPath, environmentSource);
+  const executable = input.command.directExecutable
+    ? input.command.entry
+    : resolveSchedulerWorkerExecutable(isBun, input.execPath ?? process.execPath, environmentSource);
   const workerArgs = [
     '--controller-home', input.controllerHome,
     '--repo-id', input.repoId,
     '--job-id', input.jobId,
     '--controller-pid', String(input.controllerPid),
   ];
-  const args = isBun
-    ? [input.command.entry, ...workerArgs]
-    : ['--loader', input.command.loader, input.command.entry, ...workerArgs];
+  const args = input.command.directExecutable
+    ? workerArgs
+    : isBun
+      ? [input.command.entry, ...workerArgs]
+      : ['--loader', input.command.loader, input.command.entry, ...workerArgs];
   const environment: NodeJS.ProcessEnv = {
     ...environmentSource,
     FORGE_EXECUTION_WORKER: '1',