@moretea-labs/forge 1.5.0-rc.1

package/src/runtime/control-plane/facade/work-task-projection.ts

@@ -0,0 +1,66 @@
+import type { ControllerTask, TaskStatus } from '../../../cli/controller/types';
+import { isRepositoryCompletionReceipt, type WorkContract, type WorkRisk } from './types';
+
+/**
+ * Legacy Task statuses are a read-model vocabulary. They are intentionally
+ * derived from Work and never fed back into Work or Requirement state.
+ */
+export function taskStatusFromWork(work: Pick<WorkContract, 'status' | 'phase'>): TaskStatus {
+  if (work.status === 'completed') return 'done';
+  if (work.status === 'cancelled') return 'cancelled';
+  if (work.status === 'failed') return 'changes_requested';
+  if (work.status === 'blocked') return 'blocked';
+  if (work.status === 'running') {
+    return work.phase === 'verification' ? 'verifying' : work.phase === 'delivery' ? 'integrating' : work.phase === 'cleanup' ? 'cleanup_pending' : 'running';
+  }
+  if (work.status === 'ready') return 'ready_to_integrate';
+  return 'planned';
+}
+
+function workRiskToTaskRisk(risk: WorkRisk): ControllerTask['risk'] {
+  return risk;
+}
+
+/**
+ * Project one Work into the old ControllerTask shape for compatibility UIs and
+ * readers. The source Task object is not mutated and its duplicated contract
+ * fields are never consulted by this projection.
+ */
+export function projectControllerTaskFromWork(
+  task: ControllerTask,
+  work: WorkContract,
+): ControllerTask {
+  const projected: ControllerTask = {
+    ...task,
+    workId: work.workId,
+    objective: work.objective,
+    allowedPaths: [...work.allowedPaths],
+    forbiddenPaths: [...work.forbiddenPaths],
+    checks: [...work.checks],
+    acceptanceCriteria: [...work.acceptanceCriteria],
+    risk: workRiskToTaskRisk(work.risk),
+    // A reviewed legacy outcome is user-facing evidence. Historical failed or
+    // cancelled execution facts cannot move it backwards.
+    status: task.status === 'done' && work.status !== 'completed' ? 'done' : taskStatusFromWork(work),
+  };
+  if (work.completionReceipt && projected.verification && isRepositoryCompletionReceipt(work.completionReceipt)) {
+    projected.verification = {
+      ...projected.verification,
+      completionReceipt: work.completionReceipt,
+    };
+  }
+  return projected;
+}
+
+export function taskExecutionContractFromWork(work: WorkContract): Pick<ControllerTask, 'workId' | 'objective' | 'allowedPaths' | 'forbiddenPaths' | 'checks' | 'acceptanceCriteria' | 'risk' | 'status'> {
+  return {
+    workId: work.workId,
+    objective: work.objective,
+    allowedPaths: [...work.allowedPaths],
+    forbiddenPaths: [...work.forbiddenPaths],
+    checks: [...work.checks],
+    acceptanceCriteria: [...work.acceptanceCriteria],
+    risk: workRiskToTaskRisk(work.risk),
+    status: taskStatusFromWork(work),
+  };
+}

package/src/runtime/control-plane/global-scheduler/reconciliation.ts

@@ -0,0 +1,365 @@
+import { listActiveExecutionJobs, transitionExecutionJob } from '../../execution/jobs/store';
+import type { ExecutionJob } from '../../execution/jobs/types';
+import {
+  executionTimeoutDecision,
+  type ExecutionTimeoutDecision,
+} from '../../execution/jobs/timeouts';
+import {
+  childReferenceFromJob,
+  childReferenceFromReceipt,
+  hasDurableChildReference,
+  isAgentDelegationOperation,
+  mergeChildReferences,
+  buildDelegatedExecutionResult,
+} from '../../execution/jobs/child-reference';
+import { recoverCompletedReceipt } from '../../execution/jobs/receipt-recovery';
+import { operationReceiptMatchesJobOwnership, readOperationReceipt } from '../../execution/jobs/receipt-store';
+import { releaseExecutionLeases, renewExecutionLeases } from '../../resources/leases/store';
+import { isProcessAlive, terminateProcessTree, terminateProcessTreeSync, type ProcessTreeTerminationResult } from '../../shared/process-tree';
+import { settleScheduledExecution } from '../../workflow/schedules/settlement';
+
+const WORKER_HEARTBEAT_STALE_MS = 45_000;
+
+function hasPotentialSideEffects(job: ExecutionJob): boolean {
+  return job.resourceClaims.some((claim) => claim.mode !== 'read');
+}
+
+function canAutomaticallyReplay(job: ExecutionJob): boolean {
+  if (job.operationMetadata) {
+    return job.operationMetadata.replayable && job.operationMetadata.retryPolicy !== 'none';
+  }
+  // Legacy Jobs predate explicit operation metadata. Preserve the old safe
+  // fallback: read-only claims may replay, writes fail closed.
+  return !hasPotentialSideEffects(job);
+}
+
+function isAgentDelegationJob(job: ExecutionJob): boolean {
+  return isAgentDelegationOperation(job.payload.operation)
+    || job.type === 'agent-run'
+    || job.type === 'dispatch-task'
+    || job.payload.arguments?.agentDelegation === true
+    || (job.payload.operation === 'legacy-local-job'
+      && typeof job.payload.arguments?.localAction === 'string'
+      && ['launch-task', 'quick-agent-session'].includes(String(job.payload.arguments.localAction)));
+}
+
+function ambiguousStartedOperation(controllerHome: string, job: ExecutionJob): boolean {
+  const receipt = readOperationReceipt(controllerHome, job.repoId, job.jobId);
+  if (!receipt || !operationReceiptMatchesJobOwnership(receipt, job)) return false;
+  // Delegated or completed receipts are never ambiguous.
+  if (receipt.state === 'delegated' || receipt.state === 'completed') return false;
+  if (hasDurableChildReference(childReferenceFromReceipt(receipt)) || hasDurableChildReference(childReferenceFromJob(job))) {
+    return false;
+  }
+  // Agent-delegation parents are requestId-idempotent. A started receipt without a
+  // durable child pointer means the Worker died before accept; safe retry is allowed.
+  if (isAgentDelegationJob(job)) return false;
+  return receipt.state === 'started';
+}
+
+type ReconcileSummary = { inspected: number; requeued: number; terminal: number; recovered: number; isolated: number };
+
+function recordIsolatedReconciliationFailure(job: ExecutionJob, error: unknown): void {
+  const message = error instanceof Error ? error.message : String(error);
+  // Losing the canonical write fence is a Runtime-wide invariant violation,
+  // not corrupt state owned by one historical Job. Preserve the Scheduler's
+  // fatal path so Runtime Root can stop the complete Runtime.
+  if (message.startsWith('WRITER_FENCED:')) throw error;
+  process.stderr.write(`${JSON.stringify({
+    event: 'forge_scheduler_execution_job_reconciliation_isolated',
+    repoId: job.repoId,
+    jobId: job.jobId,
+    code: message.includes(':') ? message.slice(0, message.indexOf(':')) : 'EXECUTION_JOB_RECONCILIATION_FAILED',
+    message: message.slice(0, 1_000),
+    observedAt: new Date().toISOString(),
+  })}\n`);
+}
+
+function heartbeatAgeMs(heartbeatAt: string | undefined): number {
+  if (!heartbeatAt) return Number.POSITIVE_INFINITY;
+  const parsed = Date.parse(heartbeatAt);
+  if (!Number.isFinite(parsed)) return Number.POSITIVE_INFINITY;
+  return Date.now() - parsed;
+}
+
+function workerLossContext(job: ExecutionJob): {
+  workerLost: boolean;
+  heartbeatAgeMs: number;
+  reason: 'missing_worker' | 'stale_heartbeat' | 'process_missing';
+} {
+  const ageMs = heartbeatAgeMs(job.heartbeatAt);
+  if (job.workerPid === undefined) {
+    return { workerLost: true, heartbeatAgeMs: ageMs, reason: 'missing_worker' };
+  }
+  if (!isProcessAlive(job.workerPid)) {
+    return { workerLost: true, heartbeatAgeMs: ageMs, reason: 'process_missing' };
+  }
+  if (ageMs >= WORKER_HEARTBEAT_STALE_MS) {
+    // Some bounded tool implementations (notably xcodebuild via spawnSync)
+    // legitimately block the Worker's JavaScript timer while the OS process is
+    // still alive. A stale heartbeat is therefore degraded observability, not
+    // proof of worker loss. The durable Job deadline remains the hard bound.
+    return { workerLost: false, heartbeatAgeMs: ageMs, reason: 'stale_heartbeat' };
+  }
+  return { workerLost: false, heartbeatAgeMs: ageMs, reason: 'process_missing' };
+}
+
+type WorkerLossReason = 'missing_worker' | 'stale_heartbeat' | 'process_missing' | 'lease_lost';
+
+function workerLostMessage(
+  reason: WorkerLossReason,
+  heartbeatAge: number,
+  terminal: boolean,
+): string {
+  const staleHeartbeat = Number.isFinite(heartbeatAge)
+    ? `Worker heartbeat became stale after ${Math.max(0, Math.trunc(heartbeatAge))}ms.`
+    : 'Worker heartbeat was missing.';
+  const processMissing = reason === 'missing_worker'
+    ? 'Worker PID was missing.'
+    : reason === 'lease_lost'
+      ? 'Worker no longer owned the expected fenced lease set.'
+      : 'Worker process was no longer alive.';
+  if (reason === 'stale_heartbeat') {
+    return terminal
+      ? `${staleHeartbeat} The job reached its final retry and was marked failed.`
+      : `${staleHeartbeat} The job was safely requeued before its outcome became ambiguous.`;
+  }
+  return terminal
+    ? `${processMissing} The job reached its final retry and was marked failed.`
+    : `${processMissing} The job was safely requeued before its outcome became ambiguous.`;
+}
+
+export function reconcileExecutionJobs(controllerHome: string, repoId?: string): ReconcileSummary {
+  return reconcileExecutionJobsWith(controllerHome, (pid) => terminateProcessTreeSync(pid), repoId);
+}
+
+export async function reconcileExecutionJobsAsync(controllerHome: string, repoId?: string): Promise<ReconcileSummary> {
+  return reconcileExecutionJobsAsyncWith(controllerHome, (pid) => terminateProcessTree(pid), repoId);
+}
+
+function reconcileExecutionJobsWith(
+  controllerHome: string,
+  terminateWorker: (pid: number | undefined) => ProcessTreeTerminationResult,
+  repoId?: string,
+): ReconcileSummary {
+  const jobs = listActiveExecutionJobs(controllerHome, repoId);
+  let requeued = 0;
+  let terminal = 0;
+  let recovered = 0;
+  let isolated = 0;
+  for (const job of jobs) {
+    try {
+      if (job.status !== 'running') {
+        const timeout = executionTimeoutDecision(job);
+        if (timeout) {
+          releaseExecutionLeases(controllerHome, job.repoId, job.jobId, job.leaseRefs);
+          const terminalJob = transitionExecutionJob(controllerHome, job.repoId, job.jobId, 'timed_out', {
+            error: {
+              code: timeout.code,
+              message: timeout.message,
+              retryable: false,
+              details: { timeoutPhase: timeout.phase, deadlineAt: timeout.deadlineAt },
+            },
+            leaseRefs: [],
+          });
+          settleScheduledExecution(controllerHome, terminalJob, 'failed', `Scheduled operation exceeded its ${timeout.phase} timeout.`);
+          terminal += 1;
+        }
+        continue;
+      }
+
+      const timeout = executionTimeoutDecision(job);
+      const loss = workerLossContext(job);
+      let lossReason: WorkerLossReason = loss.reason;
+      if (!timeout && !loss.workerLost) {
+        try {
+          if (job.leaseRefs.length > 0) {
+            renewExecutionLeases(controllerHome, job.repoId, job.jobId, 30_000, job.leaseRefs);
+          }
+          continue;
+        } catch {
+          lossReason = 'lease_lost';
+        }
+      }
+
+      const termination = terminateWorker(job.workerPid);
+      const outcome = finalizeRunningJob(controllerHome, job, timeout, termination, lossReason, loss.heartbeatAgeMs);
+      requeued += outcome.requeued;
+      terminal += outcome.terminal;
+      recovered += outcome.recovered;
+    } catch (error) {
+      recordIsolatedReconciliationFailure(job, error);
+      isolated += 1;
+    }
+  }
+  return { inspected: jobs.length, requeued, terminal, recovered, isolated };
+}
+
+async function reconcileExecutionJobsAsyncWith(
+  controllerHome: string,
+  terminateWorker: (pid: number | undefined) => Promise<ProcessTreeTerminationResult>,
+  repoId?: string,
+): Promise<ReconcileSummary> {
+  const jobs = listActiveExecutionJobs(controllerHome, repoId);
+  let requeued = 0;
+  let terminal = 0;
+  let recovered = 0;
+  let isolated = 0;
+  for (const job of jobs) {
+    try {
+      if (job.status !== 'running') {
+        const timeout = executionTimeoutDecision(job);
+        if (timeout) {
+          releaseExecutionLeases(controllerHome, job.repoId, job.jobId, job.leaseRefs);
+          const terminalJob = transitionExecutionJob(controllerHome, job.repoId, job.jobId, 'timed_out', {
+            error: {
+              code: timeout.code,
+              message: timeout.message,
+              retryable: false,
+              details: { timeoutPhase: timeout.phase, deadlineAt: timeout.deadlineAt },
+            },
+            leaseRefs: [],
+          });
+          settleScheduledExecution(controllerHome, terminalJob, 'failed', `Scheduled operation exceeded its ${timeout.phase} timeout.`);
+          terminal += 1;
+        }
+        continue;
+      }
+
+      const timeout = executionTimeoutDecision(job);
+      const loss = workerLossContext(job);
+      let lossReason: WorkerLossReason = loss.reason;
+      if (!timeout && !loss.workerLost) {
+        try {
+          if (job.leaseRefs.length > 0) {
+            renewExecutionLeases(controllerHome, job.repoId, job.jobId, 30_000, job.leaseRefs);
+          }
+          continue;
+        } catch {
+          lossReason = 'lease_lost';
+        }
+      }
+
+      const termination = await terminateWorker(job.workerPid);
+      const outcome = finalizeRunningJob(controllerHome, job, timeout, termination, lossReason, loss.heartbeatAgeMs);
+      requeued += outcome.requeued;
+      terminal += outcome.terminal;
+      recovered += outcome.recovered;
+    } catch (error) {
+      recordIsolatedReconciliationFailure(job, error);
+      isolated += 1;
+    }
+  }
+  return { inspected: jobs.length, requeued, terminal, recovered, isolated };
+}
+
+function finalizeRunningJob(
+  controllerHome: string,
+  job: ExecutionJob,
+  timeout: ExecutionTimeoutDecision | undefined,
+  termination: ProcessTreeTerminationResult,
+  workerLostReason: WorkerLossReason,
+  heartbeatAge: number,
+): { requeued: number; terminal: number; recovered: number } {
+  const deadlineElapsed = Boolean(timeout);
+  const receiptRecovery = recoverCompletedReceipt(controllerHome, job);
+  if (receiptRecovery) {
+    return { requeued: 0, terminal: 1, recovered: 1 };
+  }
+  // recoverCompletedReceipt already matched ownership; fall through only when
+  // no completed/delegated receipt exists for this attempt.
+
+  releaseExecutionLeases(controllerHome, job.repoId, job.jobId, job.leaseRefs);
+  // Prefer durable child recovery even when the Worker exited uncleanly.
+  const liveChild = mergeChildReferences(
+    childReferenceFromReceipt(readOperationReceipt(controllerHome, job.repoId, job.jobId)),
+    childReferenceFromJob(job),
+  );
+  if (isAgentDelegationJob(job) && hasDurableChildReference(liveChild) && liveChild) {
+    const recovered = transitionExecutionJob(controllerHome, job.repoId, job.jobId, 'succeeded', {
+      result: buildDelegatedExecutionResult({ childReference: liveChild }),
+      error: undefined,
+      workerPid: undefined,
+      leaseRefs: [],
+    }, { recoveredDelegatedChild: true });
+    settleScheduledExecution(
+      controllerHome,
+      recovered,
+      'succeeded',
+      'Agent-delegation parent recovered from durable child reference after Worker loss.',
+    );
+    return { requeued: 0, terminal: 1, recovered: 1 };
+  }
+
+  const ambiguousOutcome = ambiguousStartedOperation(controllerHome, job) && !canAutomaticallyReplay(job);
+  if (ambiguousOutcome) {
+    const terminalJob = transitionExecutionJob(controllerHome, job.repoId, job.jobId, 'human_attention_required', {
+      workerPid: undefined,
+      leaseRefs: [],
+      error: {
+        code: 'OPERATION_OUTCOME_AMBIGUOUS',
+        message: deadlineElapsed
+          ? 'A non-replayable Worker exceeded its deadline after execution started. It was stopped and will not be replayed automatically.'
+          : 'A non-replayable Worker disappeared after execution started. Automatic replay is blocked to prevent duplicate side effects.',
+        retryable: false,
+      },
+    });
+    settleScheduledExecution(controllerHome, terminalJob, 'failed', 'Scheduled non-replayable operation ended with an ambiguous outcome and requires human review.');
+    return { requeued: 0, terminal: 1, recovered: 0 };
+  }
+
+  if (!termination.exited) {
+    const terminalJob = transitionExecutionJob(controllerHome, job.repoId, job.jobId, 'orphaned', {
+      workerPid: undefined,
+      leaseRefs: [],
+      error: {
+        code: 'WORKER_TERMINATION_INCOMPLETE',
+        message: `The previous Worker process tree is still alive after termination was requested: ${termination.remainingPids.join(', ')}`,
+        retryable: false,
+      },
+    });
+    settleScheduledExecution(controllerHome, terminalJob, 'failed', 'Scheduled worker did not exit after termination was requested.');
+    return { requeued: 0, terminal: 1, recovered: 0 };
+  }
+
+  if (!deadlineElapsed && job.attempt < job.maxAttempts) {
+    transitionExecutionJob(controllerHome, job.repoId, job.jobId, 'queued', {
+      workerPid: undefined,
+      heartbeatAt: undefined,
+      leaseRefs: [],
+      error: {
+        code: 'WORKER_LOST',
+        message: workerLostMessage(workerLostReason, heartbeatAge, false),
+        retryable: true,
+        details: { workerLostReason, heartbeatAgeMs: heartbeatAge, attempt: job.attempt, maxAttempts: job.maxAttempts },
+      },
+    });
+    return { requeued: 1, terminal: 0, recovered: 0 };
+  }
+  const status = deadlineElapsed ? 'timed_out' : 'failed';
+  const terminalJob = transitionExecutionJob(controllerHome, job.repoId, job.jobId, status, {
+    workerPid: undefined,
+    heartbeatAt: undefined,
+    leaseRefs: [],
+    error: timeout
+      ? {
+          code: timeout.code,
+          message: timeout.message,
+          retryable: false,
+          details: { timeoutPhase: timeout.phase, deadlineAt: timeout.deadlineAt },
+        }
+      : {
+          code: 'WORKER_LOST',
+          message: workerLostMessage(workerLostReason, heartbeatAge, true),
+          retryable: false,
+          details: { workerLostReason, heartbeatAgeMs: heartbeatAge, attempt: job.attempt, maxAttempts: job.maxAttempts },
+        },
+  });
+  settleScheduledExecution(
+    controllerHome,
+    terminalJob,
+    'failed',
+    timeout ? `Scheduled operation exceeded its ${timeout.phase} timeout.` : 'Scheduled worker was lost after retries were exhausted.',
+  );
+  return { requeued: 0, terminal: 1, recovered: 0 };
+}