@moretea-labs/forge 1.5.0-rc.1

package/src/runtime/diagnostics/mcp-timing.ts

@@ -0,0 +1,64 @@
+import { appendFileSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { ensureControllerHome } from '../../cli/repositories/controller-home';
+
+export interface McpTimingTrace {
+  tool: string;
+  traceId?: string;
+  requestId?: string;
+  rpcId?: string | number;
+  outcome?: 'ok' | 'error' | 'exception';
+  errorCode?: string;
+  sessionResolutionMs?: number;
+  authenticationAuthorizationMs?: number;
+  repositoryResolutionMs?: number;
+  workHandleValidationMs?: number;
+  controllerQueueWaitMs?: number;
+  commandExecutionMs?: number;
+  resultSerializationMs?: number;
+  resultPersistenceMs?: number;
+  totalToolDurationMs: number;
+  sessionId?: string;
+  repoId?: string;
+  workId?: string;
+}
+
+export interface McpIncident {
+  traceId: string;
+  requestId: string;
+  rpcId?: string | number;
+  tool: string;
+  kind: 'tool_error' | 'exception' | 'supervisor_probe';
+  code: string;
+  message: string;
+  repoId?: string;
+  sessionId?: string;
+  workId?: string;
+  details?: Record<string, unknown>;
+}
+
+function appendDiagnostic(controllerHome: string, fileName: string, value: Record<string, unknown>): void {
+  const root = join(ensureControllerHome(controllerHome), 'audit');
+  mkdirSync(root, { recursive: true, mode: 0o700 });
+  appendFileSync(join(root, fileName), `${JSON.stringify(value)}\n`, 'utf-8');
+}
+
+export function recordMcpTiming(controllerHome: string, trace: McpTimingTrace): void {
+  try {
+    appendDiagnostic(controllerHome, 'mcp-timings.jsonl', { schemaVersion: 1, at: new Date().toISOString(), ...trace });
+  } catch {
+    // Timing is diagnostic evidence; it must never change the tool result.
+  }
+}
+
+export function recordMcpIncident(controllerHome: string, incident: McpIncident): void {
+  try {
+    appendDiagnostic(controllerHome, 'mcp-incidents.jsonl', {
+      schemaVersion: 1,
+      at: new Date().toISOString(),
+      ...incident,
+    });
+  } catch {
+    // Incident recording is diagnostic evidence; it must never change the tool result.
+  }
+}

package/src/runtime/diagnostics/performance.ts

@@ -0,0 +1,379 @@
+import { execFileSync } from 'child_process';
+import { existsSync, lstatSync, readdirSync, rmSync } from 'fs';
+import { tmpdir } from 'os';
+import { basename, dirname, join, resolve } from 'path';
+
+export interface RuntimeProcessSample {
+  pid: number;
+  ppid: number;
+  pgid: number;
+  elapsed: string;
+  cpu: number;
+  mem: number;
+  command: string;
+  kind: 'runtime' | 'worker' | 'local-controller' | 'mcp-server' | 'mcp-keepalive' | 'tunnel' | 'forge' | 'other';
+  repoRoot?: string;
+  jobId?: string;
+  orphan: boolean;
+  highCpu: boolean;
+}
+
+export interface RuntimeTempEntry {
+  path: string;
+  mtime: string;
+  ageMinutes: number;
+  occupiedByPid?: number;
+  symbolicLink?: boolean;
+  cleanupCandidate: boolean;
+}
+
+export interface RuntimePerformanceDiagnostics {
+  schemaVersion: 1;
+  repoId: string;
+  repoRoot: string;
+  generatedAt: string;
+  status: 'normal' | 'warning' | 'critical';
+  summary: string;
+  controller: { queueDepth: number; runningWorkers: number; activeLeases: number; activeJobs: string[] };
+  processSummary: {
+    totalForgeProcesses: number;
+    workerProcesses: number;
+    orphanWorkers: number;
+    highCpuProcesses: number;
+    totalForgeCpu: number;
+    localControllerRunning: boolean;
+    localControllerPid?: number;
+  };
+  processes: RuntimeProcessSample[];
+  temp: { scannedRoots: string[]; totalEntries: number; cleanupCandidates: number; entries: RuntimeTempEntry[] };
+  cleanupPreview?: { safeToTerminate: RuntimeProcessSample[]; safeToRemoveTempEntries: RuntimeTempEntry[]; requiresConfirmation: true };
+  truncated: {
+    summary: boolean;
+    processes: boolean;
+    tempEntries: boolean;
+    cleanupPreview: boolean;
+    recommendations: boolean;
+  };
+  findings: Array<{ severity: 'info' | 'warning' | 'critical'; code: string; message: string }>;
+  recommendations: string[];
+}
+
+interface DiagnosticsInput {
+  repoId: string;
+  repoRoot: string;
+  queueDepth?: number;
+  runningWorkers?: number;
+  activeLeases?: number;
+  activeJobIds?: string[];
+  includeProcesses?: boolean;
+  includeTempDirs?: boolean;
+  cleanupPreview?: boolean;
+  localControllerRunning?: boolean;
+  localControllerPid?: number;
+  localControllerEndpoint?: string;
+}
+
+const MAX_DIAGNOSTIC_SUMMARY_CHARS = 320;
+const MAX_DIAGNOSTIC_RECOMMENDATIONS = 5;
+const MAX_DIAGNOSTIC_PROCESSES = 80;
+const MAX_DIAGNOSTIC_TEMP_ENTRIES = 80;
+const MAX_DIAGNOSTIC_CLEANUP_PREVIEW = 50;
+export const RUNTIME_TEMP_RETENTION_MINUTES = 24 * 60;
+const RUNTIME_TEMP_NAME = /^forge(?:[-_.]|$)/;
+
+function parseNumber(value: string | undefined): number {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function boundedText(value: string, maxChars: number): { text: string; truncated: boolean } {
+  if (value.length <= maxChars) return { text: value, truncated: false };
+  return { text: `${value.slice(0, Math.max(0, maxChars - 3))}...`, truncated: true };
+}
+
+export function buildRuntimeDiagnosticSummary(
+  status: RuntimePerformanceDiagnostics['status'],
+  findings: RuntimePerformanceDiagnostics['findings'],
+): { summary: string; truncated: boolean } {
+  const message = status === 'normal'
+    ? 'No orphan workers or abnormal forge CPU pattern detected.'
+    : findings.map((finding) => finding.message.trim()).filter(Boolean).join(' ');
+  const bounded = boundedText(message, MAX_DIAGNOSTIC_SUMMARY_CHARS);
+  return { summary: bounded.text, truncated: bounded.truncated };
+}
+
+function extractFlag(command: string, flag: string): string | undefined {
+  const escaped = flag.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+  return command.match(new RegExp(escaped + '\\s+([^\\s]+)'))?.[1];
+}
+
+function normalizePath(value: string | undefined): string | undefined {
+  return value ? value.replace(/\\/g, '/') : undefined;
+}
+
+function normalizeRootForCompare(value: string | undefined): string | undefined {
+  let normalized = normalizePath(value);
+  while (normalized !== undefined && normalized.endsWith(String.fromCharCode(47))) normalized = normalized.slice(0, -1);
+  return normalized;
+}
+
+export function isHighCpuPeerMcpProcess(sample: RuntimeProcessSample, repoRoot: string): boolean {
+  if (!sample.highCpu) return false;
+  if (sample.kind !== 'mcp-server' && sample.kind !== 'mcp-keepalive') return false;
+  const currentRoot = normalizeRootForCompare(repoRoot);
+  const sampleRoot = normalizeRootForCompare(sample.repoRoot);
+  return currentRoot !== undefined && sampleRoot !== undefined && sampleRoot !== currentRoot;
+}
+
+export function isStaleCanonicalRuntimeProcess(sample: RuntimeProcessSample, repoRoot: string): boolean {
+  if (sample.kind !== 'runtime') return false;
+  if (sample.ppid !== 1) return false;
+  const controllerHome = normalizeRootForCompare(extractFlag(sample.command, '--controller-home'));
+  const currentRoot = normalizeRootForCompare(repoRoot);
+  if (!controllerHome || !currentRoot) return false;
+  const currentControllerHome = currentRoot + '/_ops/controller-home';
+  if (controllerHome === currentControllerHome || controllerHome.startsWith(currentControllerHome + '/')) return false;
+  return controllerHome.includes('/.ai/local/controller-home') || controllerHome.includes('/forge-controller-home-');
+}
+
+function extractRepoRoot(command: string): string | undefined {
+  const explicitRepo = extractFlag(command, '--repo');
+  if (explicitRepo) return normalizePath(explicitRepo);
+  const sourceMatch = command.match(/(\/[^\s]*forge[^\s]*)\/src\//);
+  if (sourceMatch) return normalizePath(sourceMatch[1]);
+  const worktreeMatch = command.match(/(\/[^\s]*forge[^\s]*)\/\.ai\//);
+  if (worktreeMatch) return normalizePath(worktreeMatch[1]);
+  return undefined;
+}
+
+function classify(command: string): RuntimeProcessSample['kind'] {
+  if (command.includes('forge-runtime') || command.includes('/runtime/root/entry.ts')) return 'runtime';
+  if (command.includes('worker-entry.ts') || command.includes('job-worker.ts') || command.includes('/runtime/execution/workers/')) return 'worker';
+  if (command.includes(' controller ui ') || command.includes('src/cli/index.ts controller ui')) return 'local-controller';
+  if (command.includes(' mcp serve ') || command.includes('src/cli/index.ts mcp serve')) return 'mcp-server';
+  if (command.includes(' mcp keepalive ') || command.includes('src/cli/index.ts mcp keepalive')) return 'mcp-keepalive';
+  if (command.includes('ngrok') || command.includes('controller-ngrok-rotation')) return 'tunnel';
+  if (command.includes('forge')) return 'forge';
+  return 'other';
+}
+
+/** Full-table `ps` is ~50–100ms. Cache the base list for readiness/status storms. */
+const PROCESS_LIST_CACHE_TTL_MS = 2_500;
+
+type RuntimeProcessBase = Omit<RuntimeProcessSample, 'orphan'>;
+
+interface CachedProcessList {
+  at: number;
+  samples: RuntimeProcessBase[];
+}
+
+let processListCache: CachedProcessList | undefined;
+
+export function clearRuntimeProcessListCacheForTest(): void {
+  processListCache = undefined;
+}
+
+function scanRuntimeProcessList(): RuntimeProcessBase[] {
+  let output = '';
+  try {
+    output = execFileSync('ps', ['-axo', 'pid=,ppid=,pgid=,etime=,%cpu=,%mem=,command='], { encoding: 'utf8', maxBuffer: 1024 * 1024 });
+  } catch (_error) {
+    return [];
+  }
+  const samples: RuntimeProcessBase[] = [];
+  for (const raw of output.split('\n')) {
+    const line = raw.trim();
+    if (!line) continue;
+    const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+([\d.]+)\s+([\d.]+)\s+(.+)$/);
+    if (!match) continue;
+    const command = match[7];
+    const kind = classify(command);
+    const relevant = kind !== 'other' || /\b(bun|node|tsx)\b/.test(command);
+    if (!relevant) continue;
+    const jobId = extractFlag(command, '--job-id');
+    const ppid = Number(match[2]);
+    const cpu = parseNumber(match[5]);
+    samples.push({
+      pid: Number(match[1]),
+      ppid,
+      pgid: Number(match[3]),
+      elapsed: match[4],
+      cpu,
+      mem: parseNumber(match[6]),
+      command,
+      kind,
+      repoRoot: extractRepoRoot(command),
+      jobId,
+      highCpu: cpu >= 20,
+    });
+  }
+  return samples.sort((a, b) => b.cpu - a.cpu);
+}
+
+export function collectRuntimeProcesses(
+  activeJobIds: string[] = [],
+  options: { forceRefresh?: boolean } = {},
+): RuntimeProcessSample[] {
+  const now = Date.now();
+  if (
+    options.forceRefresh
+    || !processListCache
+    || now - processListCache.at > PROCESS_LIST_CACHE_TTL_MS
+  ) {
+    processListCache = { at: now, samples: scanRuntimeProcessList() };
+  }
+  const active = new Set(activeJobIds);
+  return processListCache.samples.map((sample) => ({
+    ...sample,
+    orphan: sample.kind === 'worker' && (
+      sample.ppid === 1 || (sample.jobId ? !active.has(sample.jobId) : false)
+    ),
+  }));
+}
+
+export function inferLocalControllerProcess(repoRoot: string): { running: boolean; pid?: number; endpoint?: string; source: 'process-scan' } | undefined {
+  const normalizedRoot = normalizePath(repoRoot);
+  const sample = collectRuntimeProcesses().find((process) => process.kind === 'local-controller' && (normalizePath(process.repoRoot) === normalizedRoot || process.command.includes(repoRoot)));
+  if (!sample) return undefined;
+  const host = extractFlag(sample.command, '--host') ?? '127.0.0.1';
+  const port = extractFlag(sample.command, '--port') ?? '8766';
+  return { running: true, pid: sample.pid, endpoint: 'http://' + host + ':' + port + '/', source: 'process-scan' };
+}
+
+function normalizedRuntimeTempRoots(roots?: string[]): string[] {
+  return Array.from(new Set((roots ?? [tmpdir(), '/private/tmp'])
+    .filter((root) => existsSync(root))
+    .map((root) => resolve(root))));
+}
+
+export function scanRuntimeTempEntries(
+  processes: RuntimeProcessSample[],
+  options: { roots?: string[]; minAgeMinutes?: number; maxEntries?: number } = {},
+): { roots: string[]; entries: RuntimeTempEntry[] } {
+  const roots = normalizedRuntimeTempRoots(options.roots);
+  const minAgeMinutes = options.minAgeMinutes ?? RUNTIME_TEMP_RETENTION_MINUTES;
+  const maxEntries = Math.max(1, Math.min(options.maxEntries ?? 500, 2_000));
+  const now = Date.now();
+  const entries: RuntimeTempEntry[] = [];
+  for (const root of roots) {
+    let names: string[] = [];
+    try { names = readdirSync(root).filter((name) => RUNTIME_TEMP_NAME.test(name)); } catch (_error) { continue; }
+    for (const name of names.slice(0, maxEntries)) {
+      const path = join(root, name);
+      try {
+        const stat = lstatSync(path);
+        const symbolicLink = stat.isSymbolicLink();
+        const occupied = processes.find((process) => process.command.includes(path));
+        const ageMinutes = Math.max(0, Math.round((now - stat.mtimeMs) / 60000));
+        entries.push({
+          path,
+          mtime: stat.mtime.toISOString(),
+          ageMinutes,
+          occupiedByPid: occupied?.pid,
+          symbolicLink,
+          cleanupCandidate: !symbolicLink && !occupied && ageMinutes >= minAgeMinutes,
+        });
+      } catch (_error) {}
+    }
+  }
+  return { roots, entries: entries.sort((a, b) => b.ageMinutes - a.ageMinutes) };
+}
+
+export function removeRuntimeTempEntry(
+  path: string,
+  options: { roots?: string[]; minAgeMinutes?: number; processes?: RuntimeProcessSample[] } = {},
+): string {
+  const roots = normalizedRuntimeTempRoots(options.roots);
+  const resolvedPath = resolve(path);
+  const root = roots.find((candidate) => dirname(resolvedPath) === candidate);
+  if (!root) throw new Error('RUNTIME_TEMP_PATH_OUTSIDE_ROOT');
+  if (!RUNTIME_TEMP_NAME.test(basename(resolvedPath))) throw new Error('RUNTIME_TEMP_NAME_INVALID');
+  const stat = lstatSync(resolvedPath);
+  if (stat.isSymbolicLink()) throw new Error('RUNTIME_TEMP_SYMLINK_REFUSED');
+  const minAgeMinutes = options.minAgeMinutes ?? RUNTIME_TEMP_RETENTION_MINUTES;
+  const ageMinutes = Math.max(0, Math.round((Date.now() - stat.mtimeMs) / 60000));
+  if (ageMinutes < minAgeMinutes) throw new Error(`RUNTIME_TEMP_TOO_RECENT: ${ageMinutes}m < ${minAgeMinutes}m`);
+  const processes = options.processes ?? collectRuntimeProcesses();
+  if (processes.some((process) => process.command.includes(resolvedPath) || process.command.includes(path))) {
+    throw new Error('RUNTIME_TEMP_ENTRY_OCCUPIED');
+  }
+  rmSync(resolvedPath, { recursive: stat.isDirectory(), force: true });
+  return resolvedPath;
+}
+
+export function collectRuntimePerformanceDiagnostics(input: DiagnosticsInput): RuntimePerformanceDiagnostics {
+  const activeJobIds = input.activeJobIds ?? [];
+  const processes = input.includeProcesses === false ? [] : collectRuntimeProcesses(activeJobIds);
+  const repoRoot = normalizePath(input.repoRoot) ?? input.repoRoot;
+  const peerMcpProcesses = processes.filter((process) => isHighCpuPeerMcpProcess(process, repoRoot));
+  const staleForgeRuntimes = processes.filter((process) => isStaleCanonicalRuntimeProcess(process, repoRoot));
+  const repoProcesses = processes.filter((process) =>
+    normalizePath(process.repoRoot) === repoRoot
+    || process.command.includes(input.repoRoot)
+    || peerMcpProcesses.some((peer) => peer.pid === process.pid)
+    || staleForgeRuntimes.some((daemon) => daemon.pid === process.pid),
+  );
+  const workers = repoProcesses.filter((process) => process.kind === 'worker');
+  const orphanWorkers = workers.filter((process) => process.orphan);
+  const highCpu = repoProcesses.filter((process) => process.highCpu);
+  const highCpuPeerMcp = peerMcpProcesses;
+  const localController = repoProcesses.find((process) => process.kind === 'local-controller');
+  const localControllerRunning = localController !== undefined || input.localControllerRunning === true;
+  const temp = input.includeTempDirs === false
+    ? { roots: [] as string[], entries: [] as RuntimeTempEntry[] }
+    : scanRuntimeTempEntries(processes);
+  const findings: RuntimePerformanceDiagnostics['findings'] = [];
+  if (orphanWorkers.length > 0) findings.push({ severity: orphanWorkers.length >= 3 ? 'critical' : 'warning', code: 'ORPHAN_WORKERS', message: 'Detected ' + orphanWorkers.length + ' orphan worker process(es).' });
+  const totalCpu = Math.round(repoProcesses.reduce((sum, process) => sum + process.cpu, 0) * 10) / 10;
+  if (totalCpu >= 100 || highCpu.length >= 3) findings.push({ severity: 'warning', code: 'HIGH_FORGE_CPU', message: 'Forge-related CPU is ' + totalCpu + '%.' });
+  if (highCpuPeerMcp.length > 0) findings.push({ severity: 'warning', code: 'HIGH_CPU_PEER_MCP', message: 'Detected ' + highCpuPeerMcp.length + ' high-CPU MCP process(es) owned by another repository.' });
+  if (staleForgeRuntimes.length > 0) findings.push({ severity: 'warning', code: 'STALE_FORGE_RUNTIMES', message: 'Detected ' + staleForgeRuntimes.length + ' detached stale Forge Runtime process(es).' });
+  if ((input.queueDepth ?? 0) === 0 && (input.runningWorkers ?? 0) === 0 && orphanWorkers.length > 0) findings.push({ severity: 'critical', code: 'CONTROL_PLANE_IDLE_HOST_BUSY', message: 'Controller queue is idle but host still has orphan worker process(es).' });
+  const tempCleanupCandidates = temp.entries.filter((entry) => entry.cleanupCandidate);
+  if (tempCleanupCandidates.length >= 100) findings.push({ severity: 'warning', code: 'TEMP_DIR_ACCUMULATION', message: 'Detected ' + tempCleanupCandidates.length + ' stale forge temp entries.' });
+  if (!localController && input.localControllerRunning !== true) findings.push({ severity: 'info', code: 'LOCAL_CONTROLLER_NOT_IN_PROCESS_LIST', message: 'No Local Controller process was found for this repository.' });
+  const cleanupProcessCandidates = [...orphanWorkers, ...highCpuPeerMcp, ...staleForgeRuntimes]
+    .filter((sample, index, samples) => samples.findIndex((candidate) => candidate.pid === sample.pid) === index);
+  const rawCleanupProcesses = cleanupProcessCandidates.slice(0, MAX_DIAGNOSTIC_CLEANUP_PREVIEW);
+  const status: RuntimePerformanceDiagnostics['status'] = findings.some((finding) => finding.severity === 'critical') ? 'critical' : findings.some((finding) => finding.severity === 'warning') ? 'warning' : 'normal';
+  const summary = buildRuntimeDiagnosticSummary(status, findings);
+  const recommendations = status === 'normal' ? ['No restart is recommended from this diagnostic snapshot.'] : [
+    ...(orphanWorkers.length ? ['Terminate orphan workers only after reviewing cleanupPreview.safeToTerminate.'] : []),
+    ...(tempCleanupCandidates.length ? ['Remove stale temp entries through runtime_maintenance_apply after reviewing cleanupPreview.safeToRemoveTempEntries.'] : []),
+    ...(highCpu.length ? ['Inspect highCpuProcesses before restarting the controller stack.'] : []),
+  ];
+  return {
+    schemaVersion: 1,
+    repoId: input.repoId,
+    repoRoot: input.repoRoot,
+    generatedAt: new Date().toISOString(),
+    status,
+    summary: summary.summary,
+    controller: { queueDepth: input.queueDepth ?? 0, runningWorkers: input.runningWorkers ?? 0, activeLeases: input.activeLeases ?? 0, activeJobs: activeJobIds },
+    processSummary: { totalForgeProcesses: repoProcesses.length, workerProcesses: workers.length, orphanWorkers: orphanWorkers.length, highCpuProcesses: highCpu.length, totalForgeCpu: totalCpu, localControllerRunning, localControllerPid: localController?.pid ?? input.localControllerPid },
+    processes: repoProcesses.slice(0, MAX_DIAGNOSTIC_PROCESSES),
+    temp: {
+      scannedRoots: temp.roots,
+      totalEntries: temp.entries.length,
+      cleanupCandidates: tempCleanupCandidates.length,
+      entries: temp.entries.slice(0, MAX_DIAGNOSTIC_TEMP_ENTRIES),
+    },
+    cleanupPreview: input.cleanupPreview === true
+      ? {
+        safeToTerminate: rawCleanupProcesses,
+        safeToRemoveTempEntries: tempCleanupCandidates.slice(0, MAX_DIAGNOSTIC_CLEANUP_PREVIEW),
+        requiresConfirmation: true,
+      }
+      : undefined,
+    truncated: {
+      summary: summary.truncated,
+      processes: repoProcesses.length > MAX_DIAGNOSTIC_PROCESSES,
+      tempEntries: temp.entries.length > MAX_DIAGNOSTIC_TEMP_ENTRIES,
+      cleanupPreview: cleanupProcessCandidates.length > MAX_DIAGNOSTIC_CLEANUP_PREVIEW
+        || tempCleanupCandidates.length > MAX_DIAGNOSTIC_CLEANUP_PREVIEW,
+      recommendations: recommendations.length > MAX_DIAGNOSTIC_RECOMMENDATIONS,
+    },
+    findings,
+    recommendations: recommendations.slice(0, MAX_DIAGNOSTIC_RECOMMENDATIONS),
+  };
+}