@moretea-labs/forge 1.8.0 → 1.8.1

package/scripts/benchmark-background-check-overlap.ts

@@ -1,228 +0,0 @@
-#!/usr/bin/env bun
-
-/**
- * Executable goal-drift guard for the managed-check controller path.
- *
- * It deliberately performs useful rh_context work after run_check returns and
- * before its one final dependency join. A future change that turns this into
- * periodic process_wait polling, re-executes the check, or blocks the initial
- * handle return fails the benchmark rather than only a local Process test.
- */
-
-import { execFileSync } from 'child_process';
-import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { performance } from 'perf_hooks';
-import { createMcpToolContext } from '../src/cli/mcp/server';
-import { registerRepository } from '../src/cli/repositories/registry';
-import { callRuntimeTool } from '../src/runtime/gateway/mcp/runtime-tools';
-import { callProcessTool } from '../src/runtime/gateway/mcp/process-tools';
-import { routeDurableMcpCall } from '../src/runtime/gateway/mcp/router';
-import { getProcessRecord } from '../src/runtime/execution/process-runtime/store';
-import {
-  clearGitIdentityCacheForTest,
-  clearGitSnapshotCacheForTest,
-  gitSnapshotPerformanceSnapshot,
-} from '../src/cli/repository/inspector';
-
-const CHECK_DURATION_MS = 1_200;
-const MAX_INITIAL_HANDLE_LATENCY_MS = 700;
-const MAX_TERMINAL_OBSERVATION_LATENCY_MS = 1_000;
-const root = mkdtempSync(join(tmpdir(), 'forge-background-check-overlap-'));
-const controllerHome = join(root, 'controller');
-const repoRoot = join(root, 'repo');
-
-const behavioralContract = {
-  originalUserFacingGoal: 'A long focused check must run in the background while Forge continues useful independent work, then synchronize once at the true dependency boundary.',
-  invariants: {
-    backgroundCheck: 'long_check_returns_a_managed_handle_promptly',
-    noPollingWhileUsefulWorkExists: 'zero_process_wait_calls_before_independent_work_finishes',
-    dependencyDrivenJoin: 'one_final_process_wait_after_the_dependency_boundary',
-    exactlyOnce: 'one_physical_check_execution',
-    resultCorrectness: 'terminal_result_matches_the_returned_process_id',
-    noAdditionalOrchestration: 'no_execution_job_created_for_the_fast_check_route',
-    sourceRetrievalStaysFocused: 'ordinary_rh_context_search_skips_execution_preflight',
-    summaryStaysCheap: 'rh_status_summary_avoids_detail_diagnostics',
-  },
-  antiGoals: [
-    'periodic_process_wait_polling',
-    'synchronous_wait_before_independent_work',
-    'duplicate_check_execution',
-    'source_retrieval_building_unrequested_check_readiness',
-    'summary_status_triggering_detail_diagnostics',
-  ],
-  validation: 'Runs a real managed check, performs rh_context work before the sole join, and separately asserts the rh_status summary hot path.',
-};
-
-function percentile(values: number[], ratio: number): number {
-  const sorted = [...values].sort((left, right) => left - right);
-  return Math.round((sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * ratio))] ?? 0) * 100) / 100;
-}
-
-try {
-  mkdirSync(join(repoRoot, 'src'), { recursive: true });
-  mkdirSync(join(repoRoot, '.forge'), { recursive: true });
-  execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: repoRoot });
-  execFileSync('git', ['config', 'user.email', 'benchmark@example.test'], { cwd: repoRoot });
-  execFileSync('git', ['config', 'user.name', 'Forge Benchmark'], { cwd: repoRoot });
-  writeFileSync(join(repoRoot, 'src', 'fixture.ts'), 'export const backgroundFixture = true;\n');
-  const executionMarker = join(root, 'physical-check-executions.txt');
-  writeFileSync(join(repoRoot, '.forge', 'checks.json'), JSON.stringify({
-    version: 1,
-    checks: {
-      background: {
-        description: 'Synthetic long check for controller overlap behavior.',
-        command: [
-          process.execPath,
-          '-e',
-          [
-            `require('fs').appendFileSync(${JSON.stringify(executionMarker)}, '1');`,
-            `setTimeout(() => { process.stdout.write('BACKGROUND_CHECK_DONE\\n'); process.exit(0); }, ${CHECK_DURATION_MS});`,
-          ].join(' '),
-        ],
-        timeoutMs: 10_000,
-      },
-    },
-  }, null, 2));
-  execFileSync('git', ['add', '.'], { cwd: repoRoot });
-  execFileSync('git', ['commit', '-qm', 'fixture'], { cwd: repoRoot });
-
-  const repository = registerRepository({ path: repoRoot, controllerHome, displayName: 'background-check-overlap' });
-  const ctx = createMcpToolContext({ controllerHome, profile: 'controller', repo: repoRoot });
-
-  const startedAt = performance.now();
-  const started = await routeDurableMcpCall(ctx, 'run_check', {
-    repo_id: repository.repoId,
-    check_id: 'background',
-    interactive_wait_ms: 0,
-  });
-  const initialHandleLatencyMs = performance.now() - startedAt;
-  const startPayload = started?.structuredContent as Record<string, unknown> | undefined;
-  const processId = typeof startPayload?.processId === 'string' ? startPayload.processId : '';
-
-  const independentStartedAt = performance.now();
-  const independent = await callRuntimeTool(ctx, 'rh_context', {
-    repo_id: repository.repoId,
-    operation: 'search',
-    query: 'backgroundFixture',
-    known_paths: ['src/fixture.ts'],
-    structural_context: 'off',
-  });
-  const independentOperationLatencyMs = performance.now() - independentStartedAt;
-  const independentPayload = independent?.structuredContent as { data?: Record<string, unknown> } | undefined;
-  const independentData = independentPayload?.data;
-  const independentRetrievalPolicy = independentData?.retrievalPolicy as Record<string, unknown> | undefined;
-  const sourceRetrievalSkippedExecutionPreflight = independentData?.executionReadiness === undefined
-    && independentData?.registeredChecks === undefined
-    && independentRetrievalPolicy?.executionReadiness === 'requested_check_ids_only';
-  const processWaitCallsBeforeDependencyBoundary = 0;
-
-  const joinStartedAtMs = Date.now();
-  const joined = processId
-    ? await callProcessTool(ctx, 'process_wait', {
-      repo_id: repository.repoId,
-      process_id: processId,
-      timeout_ms: 10_000,
-    })
-    : undefined;
-  const resultObservedAtMs = Date.now();
-  const finalSynchronizationCount = processId ? 1 : 0;
-  const joinedPayload = joined?.structuredContent as { process?: { processId?: string; completed?: boolean; ok?: boolean } } | undefined;
-  const record = processId ? getProcessRecord(controllerHome, repository.repoId, processId) : undefined;
-  const finishedAtMs = record?.finishedAt ? Date.parse(record.finishedAt) : NaN;
-  const completionToObservationLatencyMs = Number.isFinite(finishedAtMs)
-    ? Math.max(0, resultObservedAtMs - finishedAtMs)
-    : undefined;
-  const physicalExecutionCount = readFileSync(executionMarker, 'utf8').length;
-
-  // Summary is a separate hot path: it must answer readiness without falling
-  // into the full Git status/diff-stat or maintenance/detail projection.
-  clearGitIdentityCacheForTest();
-  clearGitSnapshotCacheForTest();
-  const statusSummaryDurations: number[] = [];
-  let summaryPayload: Record<string, unknown> | undefined;
-  for (let iteration = 0; iteration < 7; iteration += 1) {
-    const statusStartedAt = performance.now();
-    const status = await callRuntimeTool(ctx, 'rh_status', { repo_id: repository.repoId, detail_level: 'summary' });
-    statusSummaryDurations.push(performance.now() - statusStartedAt);
-    summaryPayload = status?.structuredContent as Record<string, unknown> | undefined;
-  }
-  // Snapshot this before the intentional detail call below: detail is allowed
-  // to build the full Git projection, while summary is not.
-  const summaryGitSnapshot = gitSnapshotPerformanceSnapshot();
-  const detailStartedAt = performance.now();
-  const detail = await callRuntimeTool(ctx, 'rh_status', { repo_id: repository.repoId, detail_level: 'detail' });
-  const detailElapsedMs = performance.now() - detailStartedAt;
-  const detailPayload = detail?.structuredContent as Record<string, unknown> | undefined;
-  const detailMeta = detailPayload?.responseMeta as Record<string, unknown> | undefined;
-  const summaryData = summaryPayload?.data as Record<string, unknown> | undefined;
-  const summaryRepositoryState = summaryData?.repositoryState as Record<string, unknown> | undefined;
-  const summaryUsesCompactPayload = Boolean(summaryData)
-    && summaryData?.access === undefined
-    && summaryData?.pendingHandoffCount === undefined
-    && summaryData?.activeWorkCount === undefined
-    && summaryData?.activeProcessCount === undefined
-    && summaryRepositoryState?.status === undefined
-    && summaryRepositoryState?.diffStat === undefined;
-
-  const assertions = {
-    initialManagedHandle: Boolean(processId) && startPayload?.path === 'process_managed' && initialHandleLatencyMs <= MAX_INITIAL_HANDLE_LATENCY_MS,
-    independentUsefulOperation: independent?.isError !== true,
-    sourceRetrievalSkipsExecutionPreflight: sourceRetrievalSkippedExecutionPreflight,
-    noPollingBeforeDependencyBoundary: processWaitCallsBeforeDependencyBoundary === 0,
-    oneDependencyJoin: finalSynchronizationCount === 1,
-    exactlyOnce: physicalExecutionCount === 1,
-    terminalResultCorrect: joinedPayload?.process?.processId === processId && joinedPayload?.process?.completed === true && joinedPayload?.process?.ok === true,
-    promptResultObservation: completionToObservationLatencyMs === undefined || completionToObservationLatencyMs <= MAX_TERMINAL_OBSERVATION_LATENCY_MS,
-    statusSummaryUsesCompactProjection: Boolean(summaryPayload?.data)
-      && summaryGitSnapshot.refreshes === 0
-      && summaryGitSnapshot.subprocesses === 0
-      && summaryUsesCompactPayload,
-  };
-  const output = {
-    schemaVersion: 1,
-    behavioralContract,
-    check: {
-      configuredDurationMs: CHECK_DURATION_MS,
-      initialManagedHandleLatencyMs: Number(initialHandleLatencyMs.toFixed(2)),
-      processWaitCallsBeforeDependencyBoundary,
-      independentOperation: {
-        tool: 'rh_context.search',
-        latencyMs: Number(independentOperationLatencyMs.toFixed(2)),
-        completed: independent?.isError !== true,
-        executionPreflight: sourceRetrievalSkippedExecutionPreflight ? 'not_requested' : 'unexpected',
-      },
-      finalSynchronizationCount,
-      finalJoinDurationMs: resultObservedAtMs - joinStartedAtMs,
-      physicalExecutionCount,
-      terminalCompletionToObservationLatencyMs: completionToObservationLatencyMs,
-      processId,
-    },
-    status: {
-      summaryP50Ms: percentile(statusSummaryDurations, 0.5),
-      summaryP95Ms: percentile(statusSummaryDurations, 0.95),
-      detailElapsedMs: Number(detailElapsedMs.toFixed(2)),
-      detailServerDurationMs: detailMeta?.serverDurationMs,
-      detailPhaseTimingsMs: detailMeta?.phaseTimingsMs,
-      fullGitSnapshot: summaryGitSnapshot,
-      summaryOmissions: {
-        maintenance: true,
-        plugins: true,
-        workState: true,
-        accessPolicy: true,
-        gitStatusAndDiffStat: true,
-      },
-    },
-    thresholds: {
-      initialManagedHandleLatencyMs: MAX_INITIAL_HANDLE_LATENCY_MS,
-      terminalCompletionToObservationLatencyMs: MAX_TERMINAL_OBSERVATION_LATENCY_MS,
-    },
-    assertions,
-    passed: Object.values(assertions).every(Boolean),
-  };
-  process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
-  if (!output.passed) process.exitCode = 1;
-} finally {
-  rmSync(root, { recursive: true, force: true });
-}

package/scripts/benchmark-computer-hotpath.ts

@@ -1,255 +0,0 @@
-#!/usr/bin/env bun
-
-/**
- * Repeatable Computer semantic hot-path benchmark.
- *
- * Uses an isolated controller home and a local Unix JSONL provider fixture. It
- * measures the real Computer product dispatch path without touching the live
- * Desktop Operator or the repository checkout.
- */
-import { randomUUID } from 'crypto';
-import { mkdtempSync, rmSync } from 'fs';
-import { createServer, type Server } from 'net';
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { computerPluginAdapter } from '../src/runtime/plugins/computer-registration';
-import { createDesktopOperatorRegistrationInput } from '../src/runtime/plugins/desktop-operator-registration';
-import { installExternalPluginRegistration } from '../src/runtime/plugins/external-registration';
-import { disposeRuntimeComputerComposition } from '../src/runtime/root/computer-composition';
-import { setComputerPlatformForTest } from '../src/runtime/platform/computer-platform';
-import type { AssistantPluginActionExecutionInput } from '../src/runtime/plugins/types';
-
-const WARMUP = 5;
-const SAMPLES = Math.max(10, Number(process.env.FORGE_COMPUTER_BENCH_SAMPLES ?? 50));
-
-interface Counters {
-  connections: number;
-  handshakes: number;
-  manifests: number;
-  statuses: number;
-  sessionOpens: number;
-  observes: number;
-}
-
-function percentile(values: number[], quantile: number): number {
-  const sorted = [...values].sort((a, b) => a - b);
-  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * quantile))] ?? 0;
-}
-
-function rounded(value: number): number {
-  return Math.round(value * 1000) / 1000;
-}
-
-function actionInput(
-  controllerHome: string,
-  actionId: string,
-  args: Record<string, unknown>,
-  requestId: string,
-): AssistantPluginActionExecutionInput {
-  return {
-    controllerHome,
-    repoId: '__controller__',
-    repoRoot: controllerHome,
-    pluginId: 'computer',
-    actionId,
-    requestId,
-    args,
-    origin: { surface: 'system', actor: 'computer-hotpath-benchmark' },
-  };
-}
-
-function counterDelta(after: Counters, before: Counters): Counters {
-  return {
-    connections: after.connections - before.connections,
-    handshakes: after.handshakes - before.handshakes,
-    manifests: after.manifests - before.manifests,
-    statuses: after.statuses - before.statuses,
-    sessionOpens: after.sessionOpens - before.sessionOpens,
-    observes: after.observes - before.observes,
-  };
-}
-
-async function closeServer(server: Server): Promise<void> {
-  if (!server.listening) return;
-  await new Promise<void>((resolve) => server.close(() => resolve()));
-}
-
-async function main(): Promise<void> {
-  const controllerHome = mkdtempSync(join(tmpdir(), 'forge-computer-hotpath-'));
-  const socketPath = process.platform === 'win32'
-    ? `\\\\.\\pipe\\forge-computer-hotpath-${randomUUID()}`
-    : join(controllerHome, 'desktop.sock');
-  // Benchmark the macOS provider contract with a local transport fixture on
-  // every host. This is not a production platform fallback: it keeps the
-  // measured path identical while Windows still fails native calls without a
-  // registered Windows provider.
-  setComputerPlatformForTest('darwin');
-  const registration = installExternalPluginRegistration(controllerHome, createDesktopOperatorRegistrationInput({
-    socketPath,
-    pluginVersion: '0.3.2',
-    protocolVersion: '1.0',
-  }));
-  const sessions = new Map<string, Record<string, unknown>>();
-  const counters: Counters = {
-    connections: 0,
-    handshakes: 0,
-    manifests: 0,
-    statuses: 0,
-    sessionOpens: 0,
-    observes: 0,
-  };
-
-  const server = createServer((socket) => {
-    counters.connections += 1;
-    let buffer = '';
-    socket.on('data', (chunk) => {
-      buffer += chunk.toString('utf8');
-      while (true) {
-        const newline = buffer.indexOf('\n');
-        if (newline < 0) return;
-        const raw = buffer.slice(0, newline);
-        buffer = buffer.slice(newline + 1);
-        const request = JSON.parse(raw) as { id: string; method: string; params?: Record<string, unknown> };
-        const envelopeParams = request.params ?? {};
-        const actionId = request.method === 'execute' && typeof envelopeParams.action === 'string'
-          ? envelopeParams.action
-          : request.method;
-        const params = request.method === 'execute' && envelopeParams.arguments && typeof envelopeParams.arguments === 'object'
-          ? envelopeParams.arguments as Record<string, unknown>
-          : envelopeParams;
-        let result: Record<string, unknown>;
-        if (actionId === 'handshake') {
-          counters.handshakes += 1;
-          result = {
-            pluginId: registration.providerPluginId,
-            protocolVersion: registration.protocolVersion,
-            processId: 9001,
-            startedAt: '2026-09-09T00:00:00.000Z',
-            computerCapabilities: [],
-          };
-        } else if (actionId === 'manifest') {
-          counters.manifests += 1;
-          result = {
-            id: registration.providerPluginId,
-            name: registration.displayName,
-            version: registration.pluginVersion,
-            protocolVersion: registration.protocolVersion,
-            mode: 'external',
-            scope: registration.scope,
-            provider: registration.provider,
-            capabilities: registration.capabilities.map((capability) => capability.capabilityId),
-            actions: registration.actions.map((action) => action.actionId),
-          };
-        } else if (actionId === 'health') {
-          result = { state: 'ready', warnings: [] };
-        } else if (actionId === 'desktop_status') {
-          counters.statuses += 1;
-          result = { sessions: [...sessions.values()] };
-        } else if (actionId === 'desktop_session_open') {
-          counters.sessionOpens += 1;
-          const interactionId = `bench_session_${counters.sessionOpens}`;
-          const session = {
-            interactionId,
-            bundleIdentifier: typeof params.bundle_id === 'string' ? params.bundle_id : 'com.example.Benchmark',
-            appName: 'Benchmark',
-          };
-          sessions.set(interactionId, session);
-          result = session;
-        } else if (actionId === 'desktop_observe') {
-          if (typeof params.interaction_id !== 'string' || !sessions.has(params.interaction_id)) {
-            socket.write(`${JSON.stringify({ id: request.id, ok: false, error: { code: 'SESSION_NOT_FOUND', message: 'missing', retryable: false, domain: 'session' } })}\n`);
-            continue;
-          }
-          counters.observes += 1;
-          result = { observed: true, interactionId: params.interaction_id };
-        } else {
-          result = { ok: true };
-        }
-        socket.write(`${JSON.stringify({ id: request.id, ok: true, result })}\n`);
-      }
-    });
-  });
-
-  try {
-    await new Promise<void>((resolve, reject) => {
-      server.once('error', reject);
-      server.listen(socketPath, () => {
-        server.off('error', reject);
-        resolve();
-      });
-    });
-
-    const opened = await computerPluginAdapter.executeAction(actionInput(
-      controllerHome,
-      'desktop_target_open',
-      { bundle_id: 'com.example.Benchmark', launch: false, activate: false },
-      'benchmark-target-open',
-    ));
-    const targetId = String(opened.targetId);
-
-    const coldBefore = { ...counters };
-    const coldStartedAt = performance.now();
-    await computerPluginAdapter.executeAction(actionInput(
-      controllerHome,
-      'desktop_observe',
-      { target_id: targetId, max_depth: 1, max_nodes: 5 },
-      'benchmark-cold-observe',
-    ));
-    const coldMs = performance.now() - coldStartedAt;
-    const coldDelta = counterDelta(counters, coldBefore);
-
-    for (let index = 0; index < WARMUP; index += 1) {
-      await computerPluginAdapter.executeAction(actionInput(
-        controllerHome,
-        'desktop_observe',
-        { target_id: targetId, max_depth: 1, max_nodes: 5 },
-        `benchmark-warmup-${index}`,
-      ));
-    }
-
-    const warmBefore = { ...counters };
-    const samples: number[] = [];
-    for (let index = 0; index < SAMPLES; index += 1) {
-      const startedAt = performance.now();
-      await computerPluginAdapter.executeAction(actionInput(
-        controllerHome,
-        'desktop_observe',
-        { target_id: targetId, max_depth: 1, max_nodes: 5 },
-        `benchmark-sample-${index}`,
-      ));
-      samples.push(performance.now() - startedAt);
-    }
-    const warmDelta = counterDelta(counters, warmBefore);
-
-    console.log(JSON.stringify({
-      benchmark: 'computer-hotpath',
-      samples: SAMPLES,
-      warmup: WARMUP,
-      cold: {
-        ms: rounded(coldMs),
-        provider: coldDelta,
-      },
-      warm: {
-        p50Ms: rounded(percentile(samples, 0.50)),
-        p95Ms: rounded(percentile(samples, 0.95)),
-        maxMs: rounded(Math.max(...samples)),
-        provider: warmDelta,
-        providerRpcsPerAction: rounded(warmDelta.observes / SAMPLES),
-      },
-      contract: {
-        warmHandshakePerAction: 0,
-        warmManifestPreflightPerAction: 0,
-        warmStatusPreflightPerAction: 0,
-        retainedConnectionsDuringSamples: 0,
-        removedLegacyStatusPreflightRpcsPerWarmAction: 1,
-      },
-    }, null, 2));
-  } finally {
-    disposeRuntimeComputerComposition();
-    setComputerPlatformForTest(undefined);
-    await closeServer(server);
-    rmSync(controllerHome, { recursive: true, force: true });
-  }
-}
-
-await main();