@moretea-labs/forge 1.5.1 → 1.7.0

package/src/runtime/control-plane/global-scheduler/source-scan.ts

@@ -0,0 +1,59 @@
+const GIT_STATUS_SAMPLE_INTERVAL_MS = Math.max(
+  1_000,
+  Number(process.env.FORGE_GIT_STATUS_SAMPLE_INTERVAL_MS ?? 5_000),
+);
+const IDLE_REPOSITORY_SCAN_INTERVAL_MS = Math.max(
+  GIT_STATUS_SAMPLE_INTERVAL_MS,
+  Number(process.env.FORGE_IDLE_REPOSITORY_SCAN_INTERVAL_MS ?? 60_000),
+);
+
+export { GIT_STATUS_SAMPLE_INTERVAL_MS, IDLE_REPOSITORY_SCAN_INTERVAL_MS };
+
+export function selectSchedulerSourceScanRepositories<T extends { repoId: string }>(
+  repositories: readonly T[],
+  activeRepoIds: ReadonlySet<string>,
+  nowMs: number,
+  lastSourceScanAt: number,
+): T[] {
+  const active = repositories.filter((repository) => activeRepoIds.has(repository.repoId));
+  if (active.length > 0) return active;
+  if (repositories.length === 0 || nowMs - lastSourceScanAt < IDLE_REPOSITORY_SCAN_INTERVAL_MS) return [];
+  // fs.watch/dirty markers drive prompt refresh for active mutations. The idle
+  // scan is only a safety net, so spread it across minutes instead of blocking
+  // the Runtime event loop on every registered repository at once.
+  const slot = Math.floor(nowMs / IDLE_REPOSITORY_SCAN_INTERVAL_MS) % repositories.length;
+  return [repositories[slot]!];
+}
+
+export interface SchedulerSourceSamplingPlan<T extends { repoId: string }> {
+  sourceScanRepositories: T[];
+  shouldSample: boolean;
+  avoidedRepositoryCount: number;
+}
+
+export function planSchedulerSourceSampling<T extends { repoId: string }>(input: {
+  repositories: readonly T[];
+  activeRepoIds: ReadonlySet<string>;
+  nowMs: number;
+  lastSourceScanAt: number;
+  lastGitStatusSampleAt: number;
+}): SchedulerSourceSamplingPlan<T> {
+  const sourceScanRepositories = selectSchedulerSourceScanRepositories(
+    input.repositories,
+    input.activeRepoIds,
+    input.nowMs,
+    input.lastSourceScanAt,
+  );
+  const sourceScanDue = sourceScanRepositories.length > 0;
+  const shouldSample = sourceScanDue
+    && input.nowMs - input.lastGitStatusSampleAt >= GIT_STATUS_SAMPLE_INTERVAL_MS;
+  return {
+    sourceScanRepositories,
+    shouldSample,
+    avoidedRepositoryCount: shouldSample
+      ? Math.max(0, input.repositories.length - sourceScanRepositories.length)
+      : sourceScanDue
+        ? 0
+        : input.repositories.length,
+  };
+}

package/src/runtime/control-plane/global-scheduler/state.ts

@@ -0,0 +1,88 @@
+import { join } from 'path';
+import { resolveControllerHome } from '../../../cli/repositories/controller-home';
+import { readJsonFile, writeJsonAtomic } from '../../shared/json-files';
+
+function schedulerStatePath(controllerHome: string): string {
+  return join(resolveControllerHome(controllerHome), 'scheduler', 'state.json');
+}
+
+export interface SchedulerHealthSnapshot {
+  schemaVersion: 1;
+  updatedAt: string;
+  loopStartedAt?: string;
+  lastHeartbeatAt?: string;
+  heartbeatTimeoutMs?: number;
+  lastTickAt?: string;
+  lastDispatchAt?: string;
+  lastReconcileAt?: string;
+  lastSourceScanAt?: string;
+  lastSourceScanRepoCount?: number;
+  sourceScansAvoided?: number;
+  lastRepoDispatch: Record<string, number>;
+}
+
+export function readSchedulerHealthSnapshot(controllerHome: string): SchedulerHealthSnapshot {
+  return readJsonFile<SchedulerHealthSnapshot>(schedulerStatePath(controllerHome), {
+    schemaVersion: 1,
+    updatedAt: new Date().toISOString(),
+    lastRepoDispatch: {},
+  });
+}
+
+export interface SchedulerRestoredState {
+  lastSourceScanAt: number;
+  lastSourceScanRepoCount: number;
+  sourceScansAvoided: number;
+  lastRepoDispatch: Array<[string, number]>;
+}
+
+export function restoreSchedulerState(state: SchedulerHealthSnapshot): SchedulerRestoredState {
+  return {
+    lastSourceScanAt: state.lastSourceScanAt ? Date.parse(state.lastSourceScanAt) || 0 : 0,
+    lastSourceScanRepoCount: state.lastSourceScanRepoCount ?? 0,
+    sourceScansAvoided: state.sourceScansAvoided ?? 0,
+    lastRepoDispatch: Object.entries(state.lastRepoDispatch)
+      .filter(([, timestamp]) => Number.isFinite(timestamp)),
+  };
+}
+
+export interface SchedulerStateSnapshotInput {
+  loopStartedAt: string;
+  lastHeartbeatAt: string;
+  heartbeatTimeoutMs: number;
+  lastTickAt: string;
+  lastDispatchAt?: string;
+  lastReconcileAt?: string;
+  lastSourceScanAt: number;
+  lastSourceScanRepoCount: number;
+  sourceScansAvoided: number;
+  lastRepoDispatch: ReadonlyMap<string, number>;
+}
+
+export function buildSchedulerHealthSnapshot(
+  input: SchedulerStateSnapshotInput,
+  nowMs = Date.now(),
+): SchedulerHealthSnapshot {
+  return {
+    schemaVersion: 1,
+    updatedAt: new Date(nowMs).toISOString(),
+    loopStartedAt: input.loopStartedAt,
+    lastHeartbeatAt: input.lastHeartbeatAt,
+    heartbeatTimeoutMs: input.heartbeatTimeoutMs,
+    lastTickAt: input.lastTickAt,
+    lastDispatchAt: input.lastDispatchAt,
+    lastReconcileAt: input.lastReconcileAt,
+    lastSourceScanAt: input.lastSourceScanAt ? new Date(input.lastSourceScanAt).toISOString() : undefined,
+    lastSourceScanRepoCount: input.lastSourceScanRepoCount,
+    sourceScansAvoided: input.sourceScansAvoided,
+    lastRepoDispatch: Object.fromEntries(input.lastRepoDispatch),
+  };
+}
+
+export function writeSchedulerHealthSnapshot(
+  controllerHome: string,
+  input: SchedulerStateSnapshotInput,
+  nowMs = Date.now(),
+): void {
+  writeJsonAtomic(schedulerStatePath(controllerHome), buildSchedulerHealthSnapshot(input, nowMs));
+}

package/src/runtime/control-plane/global-scheduler/worker-attachment.ts

@@ -0,0 +1,57 @@
+import { attachExecutionWorker, updateExecutionJob } from '../../execution/jobs/store';
+import type { ExecutionJob, ExecutionWorkerLifecycle } from '../../execution/jobs/types';
+import { buildSchedulerWorkerRegisteredLifecycle } from './worker-lifecycle';
+
+export interface SchedulerWorkerAttachmentDependencies {
+  attachWorker(
+    controllerHome: string,
+    repoId: string,
+    jobId: string,
+    workerPid: number,
+  ): ExecutionJob | undefined;
+  updateJob(
+    controllerHome: string,
+    repoId: string,
+    jobId: string,
+    updater: (current: ExecutionJob) => ExecutionJob,
+  ): ExecutionJob;
+}
+
+const DEFAULT_DEPENDENCIES: SchedulerWorkerAttachmentDependencies = {
+  attachWorker: attachExecutionWorker,
+  updateJob: (controllerHome, repoId, jobId, updater) => updateExecutionJob(
+    controllerHome,
+    repoId,
+    jobId,
+    updater,
+  ),
+};
+
+export function persistSchedulerWorkerAttachment(input: {
+  controllerHome: string;
+  repoId: string;
+  jobId: string;
+  workerPid: number;
+  lifecycle: ExecutionWorkerLifecycle;
+}, dependencies: SchedulerWorkerAttachmentDependencies = DEFAULT_DEPENDENCIES): boolean {
+  const attached = dependencies.attachWorker(
+    input.controllerHome,
+    input.repoId,
+    input.jobId,
+    input.workerPid,
+  );
+  if (!attached) return false;
+  try {
+    dependencies.updateJob(input.controllerHome, input.repoId, input.jobId, (latest) => ({
+      ...latest,
+      workerLifecycle: buildSchedulerWorkerRegisteredLifecycle({
+        lifecycle: input.lifecycle,
+        currentLifecycle: latest.workerLifecycle,
+        workerPid: input.workerPid,
+      }),
+    }));
+  } catch {
+    // close/reconciliation may have finalized the Job after attachment.
+  }
+  return true;
+}

package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts

@@ -0,0 +1,38 @@
+import {
+  TERMINAL_JOB_STATUSES,
+  type ExecutionJob,
+  type ExecutionWorkerLifecycle,
+} from '../../execution/jobs/types';
+
+export type SchedulerWorkerExitDecision =
+  | { kind: 'ignore'; reason: 'attempt_mismatch' | 'pid_mismatch' }
+  | { kind: 'terminal'; job: ExecutionJob; lifecycle: ExecutionWorkerLifecycle }
+  | { kind: 'active'; job: ExecutionJob; lifecycle: ExecutionWorkerLifecycle };
+
+export function evaluateSchedulerWorkerExitCandidate(input: {
+  job: ExecutionJob;
+  attempt: number;
+  childPid?: number;
+  lifecycle: ExecutionWorkerLifecycle;
+  diagnosticLifecycle: ExecutionWorkerLifecycle;
+}): SchedulerWorkerExitDecision {
+  if (input.job.attempt !== input.attempt) {
+    return { kind: 'ignore', reason: 'attempt_mismatch' };
+  }
+  if (
+    input.childPid !== undefined
+    && input.job.workerPid !== undefined
+    && input.job.workerPid !== input.childPid
+  ) {
+    return { kind: 'ignore', reason: 'pid_mismatch' };
+  }
+
+  const lifecycle = {
+    ...(input.job.workerLifecycle ?? input.lifecycle),
+    ...input.diagnosticLifecycle,
+  };
+  if (TERMINAL_JOB_STATUSES.has(input.job.status)) {
+    return { kind: 'terminal', job: input.job, lifecycle };
+  }
+  return { kind: 'active', job: input.job, lifecycle };
+}

package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts

@@ -0,0 +1,175 @@
+import {
+  getExecutionJob,
+  transitionExecutionJob,
+  updateExecutionJob,
+} from '../../execution/jobs/store';
+import { tryRecoverJobFromWorkerReceipt } from '../../execution/jobs/receipt-recovery';
+import { releaseExecutionLeases } from '../../resources/leases/store';
+import { rebuildRepositoryProjection } from '../../projections/materialized-view';
+import type {
+  ExecutionJob,
+  ExecutionJobStatus,
+  ExecutionWorkerLifecycle,
+} from '../../execution/jobs/types';
+import { evaluateSchedulerWorkerExitCandidate } from './worker-exit-decision';
+import { buildSchedulerWorkerExitFailure } from './worker-lifecycle';
+import { persistSchedulerTerminalWorkerLifecycle } from './worker-lifecycle-store';
+
+export interface SchedulerWorkerExitReconcilerDependencies {
+  getJob(controllerHome: string, repoId: string, jobId: string): ExecutionJob;
+  recoverReceipt(controllerHome: string, job: ExecutionJob): ExecutionJob | undefined;
+  persistTerminalLifecycle(input: {
+    controllerHome: string;
+    repoId: string;
+    jobId: string;
+    status: ExecutionJobStatus;
+    lifecycle: ExecutionWorkerLifecycle;
+  }): boolean;
+  updateJob(
+    controllerHome: string,
+    repoId: string,
+    jobId: string,
+    updater: (current: ExecutionJob) => ExecutionJob,
+  ): ExecutionJob;
+  releaseLeases(
+    controllerHome: string,
+    repoId: string,
+    jobId: string,
+    leaseRefs: ExecutionJob['leaseRefs'],
+  ): unknown;
+  transitionJob(
+    controllerHome: string,
+    repoId: string,
+    jobId: string,
+    status: ExecutionJobStatus,
+    patch: Partial<ExecutionJob>,
+  ): ExecutionJob;
+  rebuildProjection(controllerHome: string, repoId: string): unknown;
+}
+
+const DEFAULT_DEPENDENCIES: SchedulerWorkerExitReconcilerDependencies = {
+  getJob: getExecutionJob,
+  recoverReceipt: tryRecoverJobFromWorkerReceipt,
+  persistTerminalLifecycle: persistSchedulerTerminalWorkerLifecycle,
+  updateJob: (controllerHome, repoId, jobId, updater) => updateExecutionJob(
+    controllerHome,
+    repoId,
+    jobId,
+    updater,
+  ),
+  releaseLeases: releaseExecutionLeases,
+  transitionJob: (controllerHome, repoId, jobId, status, patch) => transitionExecutionJob(
+    controllerHome,
+    repoId,
+    jobId,
+    status,
+    patch,
+  ),
+  rebuildProjection: rebuildRepositoryProjection,
+};
+
+export function reconcileSchedulerWorkerExit(input: {
+  controllerHome: string;
+  repoId: string;
+  jobId: string;
+  attempt: number;
+  childPid?: number;
+  lifecycle: ExecutionWorkerLifecycle;
+  diagnosticLifecycle: ExecutionWorkerLifecycle;
+  exitCode: number | null;
+  signal: string | null;
+  stderr: string;
+  stderrTruncated: boolean;
+  startupError?: string;
+}, dependencies: SchedulerWorkerExitReconcilerDependencies = DEFAULT_DEPENDENCIES): void {
+  try {
+    const current = evaluateSchedulerWorkerExitCandidate({
+      job: dependencies.getJob(input.controllerHome, input.repoId, input.jobId),
+      attempt: input.attempt,
+      childPid: input.childPid,
+      lifecycle: input.lifecycle,
+      diagnosticLifecycle: input.diagnosticLifecycle,
+    });
+    if (current.kind === 'ignore') return;
+    if (current.kind === 'terminal') {
+      dependencies.persistTerminalLifecycle({
+        controllerHome: input.controllerHome,
+        repoId: input.repoId,
+        jobId: input.jobId,
+        status: current.job.status,
+        lifecycle: current.lifecycle,
+      });
+      return;
+    }
+
+    const recovered = dependencies.recoverReceipt(input.controllerHome, current.job);
+    if (recovered) {
+      dependencies.updateJob(input.controllerHome, input.repoId, input.jobId, (latest) => ({
+        ...latest,
+        workerLifecycle: current.lifecycle,
+      }));
+      try {
+        dependencies.rebuildProjection(input.controllerHome, input.repoId);
+      } catch {
+        // The next scheduler/status read can retry projection materialization.
+      }
+      return;
+    }
+
+    const rechecked = evaluateSchedulerWorkerExitCandidate({
+      job: dependencies.getJob(input.controllerHome, input.repoId, input.jobId),
+      attempt: input.attempt,
+      childPid: input.childPid,
+      lifecycle: input.lifecycle,
+      diagnosticLifecycle: input.diagnosticLifecycle,
+    });
+    if (rechecked.kind === 'ignore') return;
+    if (rechecked.kind === 'terminal') {
+      dependencies.persistTerminalLifecycle({
+        controllerHome: input.controllerHome,
+        repoId: input.repoId,
+        jobId: input.jobId,
+        status: rechecked.job.status,
+        lifecycle: rechecked.lifecycle,
+      });
+      return;
+    }
+
+    const failure = buildSchedulerWorkerExitFailure({
+      lifecycle: rechecked.lifecycle,
+      attempt: rechecked.job.attempt,
+      maxAttempts: rechecked.job.maxAttempts,
+      exitCode: input.exitCode,
+      signal: input.signal,
+      stderr: input.stderr,
+      stderrTruncated: input.stderrTruncated,
+      startupError: input.startupError,
+    });
+    dependencies.releaseLeases(
+      input.controllerHome,
+      input.repoId,
+      input.jobId,
+      rechecked.job.leaseRefs,
+    );
+    dependencies.transitionJob(
+      input.controllerHome,
+      input.repoId,
+      input.jobId,
+      failure.retryable ? 'queued' : 'failed',
+      {
+        workerPid: undefined,
+        heartbeatAt: undefined,
+        leaseRefs: [],
+        workerLifecycle: rechecked.lifecycle,
+        error: failure.error,
+      },
+    );
+    try {
+      dependencies.rebuildProjection(input.controllerHome, input.repoId);
+    } catch {
+      // The next scheduler/status read can retry projection materialization.
+    }
+  } catch {
+    // The Job may have been finalized by the Worker or reconciliation.
+  }
+}

package/src/runtime/control-plane/global-scheduler/worker-launch.ts

@@ -0,0 +1,111 @@
+import { existsSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { join, resolve } from 'path';
+import { resolveBunExecutable } from '../../shared/process-environment';
+
+const WORKER_ENVIRONMENT_KEYS = [
+  'PATH',
+  'HOME',
+  'BUN_INSTALL',
+  'NODE_OPTIONS',
+  'FORGE_CONTROLLER_HOME',
+  'FORGE_CONTROLLER_RUNTIME_SOURCE_ROOT',
+  'FORGE_EXECUTION_WORKER',
+  'FORGE_RUNTIME_INSTANCE_ID',
+  'FORGE_RUNTIME_OWNER_PID',
+  'FORGE_RELEASE_AUTHORITY_REVISION',
+  'FORGE_RELEASE_ID',
+  'FORGE_ARTIFACT_IDENTITY',
+  'FORGE_WORKER_PROTOCOL_VERSION',
+] as const;
+
+export function resolveSchedulerWorkerExecutable(
+  isBun: boolean = Boolean(process.versions.bun),
+  execPath: string = process.execPath,
+  env: NodeJS.ProcessEnv = process.env,
+): string {
+  return isBun ? resolveBunExecutable(execPath, env) : execPath;
+}
+
+export interface SchedulerWorkerCommand {
+  entry: string;
+  loader: string;
+  cwd: string;
+}
+
+export function resolveSchedulerWorkerCommand(input: {
+  runtimeSourceRoot?: string;
+  workerEntrypoint?: string;
+  isBun?: boolean;
+  cwd?: string;
+  pathExists?: (path: string) => boolean;
+} = {}): SchedulerWorkerCommand {
+  const runtimeSourceRoot = input.runtimeSourceRoot ? resolve(input.runtimeSourceRoot) : undefined;
+  const sourceEntry = runtimeSourceRoot
+    ? join(runtimeSourceRoot, 'src', 'runtime', 'execution', 'workers', 'worker-entry.ts')
+    : fileURLToPath(new URL('../../execution/workers/worker-entry.ts', import.meta.url));
+  const loader = runtimeSourceRoot
+    ? join(runtimeSourceRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs')
+    : fileURLToPath(new URL('../../shared/node-ts-loader.mjs', import.meta.url));
+  const entry = input.workerEntrypoint ? resolve(input.workerEntrypoint) : sourceEntry;
+  const cwd = runtimeSourceRoot ?? input.cwd ?? process.cwd();
+  const pathExists = input.pathExists ?? existsSync;
+  const isBun = input.isBun ?? Boolean(process.versions.bun);
+  if (!pathExists(entry)) throw new Error(`WORKER_ENTRYPOINT_MISSING: ${entry}`);
+  if (!isBun && !pathExists(loader)) throw new Error(`WORKER_LOADER_MISSING: ${loader}`);
+  return { entry, loader, cwd };
+}
+
+export function selectSchedulerWorkerEnvironment(
+  env: NodeJS.ProcessEnv = process.env,
+): Record<string, string | undefined> {
+  return Object.fromEntries(WORKER_ENVIRONMENT_KEYS.map((key) => [key, env[key]]));
+}
+
+export interface SchedulerWorkerLaunchDescriptor {
+  executable: string;
+  args: string[];
+  cwd: string;
+  environment: NodeJS.ProcessEnv;
+  lifecycleEnvironment: Record<string, string | undefined>;
+}
+
+export function buildSchedulerWorkerLaunchDescriptor(input: {
+  command: SchedulerWorkerCommand;
+  controllerHome: string;
+  repoId: string;
+  jobId: string;
+  controllerPid: number;
+  runtimeSourceRoot?: string;
+  isBun?: boolean;
+  execPath?: string;
+  environment?: NodeJS.ProcessEnv;
+  writeClaimEnvironment?: NodeJS.ProcessEnv;
+}): SchedulerWorkerLaunchDescriptor {
+  const isBun = input.isBun ?? Boolean(process.versions.bun);
+  const environmentSource = input.environment ?? process.env;
+  const executable = resolveSchedulerWorkerExecutable(isBun, input.execPath ?? process.execPath, environmentSource);
+  const workerArgs = [
+    '--controller-home', input.controllerHome,
+    '--repo-id', input.repoId,
+    '--job-id', input.jobId,
+    '--controller-pid', String(input.controllerPid),
+  ];
+  const args = isBun
+    ? [input.command.entry, ...workerArgs]
+    : ['--loader', input.command.loader, input.command.entry, ...workerArgs];
+  const environment: NodeJS.ProcessEnv = {
+    ...environmentSource,
+    FORGE_EXECUTION_WORKER: '1',
+    FORGE_CONTROLLER_HOME: input.controllerHome,
+    ...(input.runtimeSourceRoot ? { FORGE_CONTROLLER_RUNTIME_SOURCE_ROOT: input.runtimeSourceRoot } : {}),
+    ...(input.writeClaimEnvironment ?? {}),
+  };
+  return {
+    executable,
+    args,
+    cwd: input.command.cwd,
+    environment,
+    lifecycleEnvironment: selectSchedulerWorkerEnvironment(environment),
+  };
+}

package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts

@@ -0,0 +1,64 @@
+import { rebuildRepositoryProjection } from '../../projections/materialized-view';
+import { updateExecutionJob } from '../../execution/jobs/store';
+import {
+  TERMINAL_JOB_STATUSES,
+  type ExecutionJob,
+  type ExecutionJobStatus,
+  type ExecutionWorkerLifecycle,
+} from '../../execution/jobs/types';
+
+export interface SchedulerWorkerLifecycleStoreDependencies {
+  updateJob(
+    controllerHome: string,
+    repoId: string,
+    jobId: string,
+    updater: (current: ExecutionJob) => ExecutionJob,
+  ): ExecutionJob;
+  rebuildProjection(controllerHome: string, repoId: string): unknown;
+}
+
+const DEFAULT_DEPENDENCIES: SchedulerWorkerLifecycleStoreDependencies = {
+  updateJob: (controllerHome, repoId, jobId, updater) => updateExecutionJob(
+    controllerHome,
+    repoId,
+    jobId,
+    updater,
+  ),
+  rebuildProjection: rebuildRepositoryProjection,
+};
+
+export function persistSchedulerSpawnedWorkerLifecycle(input: {
+  controllerHome: string;
+  repoId: string;
+  jobId: string;
+  lifecycle: ExecutionWorkerLifecycle;
+}, dependencies: SchedulerWorkerLifecycleStoreDependencies = DEFAULT_DEPENDENCIES): void {
+  try {
+    dependencies.updateJob(input.controllerHome, input.repoId, input.jobId, (current) => {
+      if (!['dispatched', 'running'].includes(current.status) || current.workerPid !== undefined) return current;
+      return { ...current, workerLifecycle: input.lifecycle };
+    });
+  } catch {
+    // The Job may have been superseded or made terminal.
+  }
+}
+
+export function persistSchedulerTerminalWorkerLifecycle(input: {
+  controllerHome: string;
+  repoId: string;
+  jobId: string;
+  status: ExecutionJobStatus;
+  lifecycle: ExecutionWorkerLifecycle;
+}, dependencies: SchedulerWorkerLifecycleStoreDependencies = DEFAULT_DEPENDENCIES): boolean {
+  if (!TERMINAL_JOB_STATUSES.has(input.status)) return false;
+  dependencies.updateJob(input.controllerHome, input.repoId, input.jobId, (latest) => ({
+    ...latest,
+    workerLifecycle: input.lifecycle,
+  }));
+  try {
+    dependencies.rebuildProjection(input.controllerHome, input.repoId);
+  } catch {
+    // The next scheduler/status read can retry projection materialization.
+  }
+  return true;
+}