@moretea-labs/forge 1.8.0 → 1.8.2

package/scripts/benchmark-mcp-transport.ts

@@ -1,169 +0,0 @@
-#!/usr/bin/env bun
-import { readFileSync } from 'fs';
-import { performance } from 'perf_hooks';
-import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
-import { readMcpServiceBearerToken } from '../adapters/mcp/auth';
-import { resolveRepoPreferredControllerHome } from '../src/cli/repositories/controller-home';
-import {
-  aggregateMcpTransportLatencySamples,
-  extractForgeServerDuration,
-  normalizeMcpTransportLatencySample,
-  type McpTransportLatencySample,
-} from '../src/runtime/diagnostics/mcp-transport-benchmark';
-
-function option(name: string): string | undefined {
-  const index = process.argv.indexOf(name);
-  return index >= 0 ? process.argv[index + 1] : undefined;
-}
-
-function integerAtLeast(name: string, fallback: number, minimum: number): number {
-  const raw = option(name);
-  if (!raw) return fallback;
-  const value = Number(raw);
-  if (!Number.isInteger(value) || value < minimum) {
-    const requirement = minimum === 0 ? 'a non-negative integer' : `an integer >= ${minimum}`;
-    throw new Error(`${name} must be ${requirement}`);
-  }
-  return value;
-}
-
-function positiveInteger(name: string, fallback: number): number {
-  return integerAtLeast(name, fallback, 1);
-}
-
-function readSamples(path: string): McpTransportLatencySample[] {
-  const text = readFileSync(path, 'utf8').trim();
-  if (!text) return [];
-  if (text.startsWith('[')) {
-    const values = JSON.parse(text);
-    if (!Array.isArray(values)) throw new Error('--input JSON must be an array or JSONL');
-    return values.map(normalizeMcpTransportLatencySample);
-  }
-  return text.split(/\r?\n/).filter(Boolean).map((line) => normalizeMcpTransportLatencySample(JSON.parse(line)));
-}
-
-export function timingLedgerServerDuration(
-  path: string | undefined,
-  tool: string,
-  meta: Record<string, unknown>,
-): number | undefined {
-  if (!path) return undefined;
-  const traceId = typeof meta.traceId === 'string' ? meta.traceId : undefined;
-  const requestId = typeof meta.requestId === 'string' ? meta.requestId : undefined;
-  if (!traceId && !requestId) return undefined;
-  try {
-    const text = readFileSync(path, 'utf8').trim();
-    if (!text) return undefined;
-    const lines = text.split(/\r?\n/).filter(Boolean);
-    for (let index = lines.length - 1; index >= 0; index -= 1) {
-      let value: Record<string, unknown>;
-      try {
-        value = JSON.parse(lines[index]!) as Record<string, unknown>;
-      } catch {
-        continue;
-      }
-      if (value.tool !== tool) continue;
-      if (traceId && value.traceId !== traceId) continue;
-      if (requestId && value.requestId !== requestId) continue;
-      const duration = value.totalToolDurationMs;
-      if (typeof duration === 'number' && Number.isFinite(duration) && duration >= 0) return duration;
-    }
-  } catch {
-    return undefined;
-  }
-  return undefined;
-}
-
-async function probe(): Promise<McpTransportLatencySample[]> {
-  const endpointRaw = option('--endpoint');
-  if (!endpointRaw) throw new Error('--endpoint is required unless --input is used');
-  const endpoint = new URL(endpointRaw);
-  const label = option('--label')?.trim() || endpoint.hostname;
-  const tool = option('--tool')?.trim() || 'rh_status';
-  const iterations = positiveInteger('--iterations', 30);
-  const warmup = integerAtLeast('--warmup', 3, 0);
-  const timeoutMs = positiveInteger('--timeout-ms', 120_000);
-  const timingLog = option('--timing-log')?.trim();
-  const includeConnect = process.argv.includes('--include-connect');
-  const timingScope = includeConnect ? 'connect_and_tool_call' as const : 'tool_call' as const;
-  const tokenEnv = option('--token-env')?.trim() || 'FORGE_MCP_BENCH_TOKEN';
-  const localServiceAuth = process.argv.includes('--local-service-auth');
-  const token = process.env[tokenEnv]?.trim()
-    || (localServiceAuth
-      ? readMcpServiceBearerToken(resolveRepoPreferredControllerHome(process.cwd()))?.trim()
-      : undefined);
-  if (!token) {
-    throw new Error(localServiceAuth
-      ? 'MCP_BENCH_LOCAL_SERVICE_TOKEN_UNAVAILABLE: no local Forge service bearer credential was found'
-      : `${tokenEnv} is not set; use --local-service-auth on the Forge host or keep benchmark bearer credentials in the environment, never CLI args`);
-  }
-  const argumentsRaw = option('--arguments');
-  const args = argumentsRaw ? JSON.parse(argumentsRaw) as Record<string, unknown> : {};
-  const repoId = option('--repo-id')?.trim();
-  if (repoId && args.repo_id === undefined) args.repo_id = repoId;
-
-  const createClient = () => {
-    const transport = new StreamableHTTPClientTransport(endpoint, {
-      requestInit: { headers: { Authorization: `Bearer ${token}` } },
-    });
-    const client = new Client({ name: 'forge-mcp-transport-benchmark', version: '1.0.0' });
-    return { client, transport };
-  };
-  const steadyState = includeConnect ? undefined : createClient();
-  const call = async (): Promise<{ response: unknown; totalMs: number }> => {
-    const connection = steadyState ?? createClient();
-    const started = performance.now();
-    try {
-      if (includeConnect) await connection.client.connect(connection.transport);
-      const response = await connection.client.callTool({ name: tool, arguments: args }, { timeout: timeoutMs });
-      return { response, totalMs: performance.now() - started };
-    } finally {
-      if (includeConnect) await connection.client.close().catch(() => undefined);
-    }
-  };
-  const samples: McpTransportLatencySample[] = [];
-  try {
-    if (steadyState) await steadyState.client.connect(steadyState.transport);
-    for (let index = 0; index < warmup; index += 1) await call();
-    for (let index = 0; index < iterations; index += 1) {
-      const { response, totalMs } = await call();
-      const structured = response && typeof response === 'object' && !Array.isArray(response)
-        ? (response as Record<string, unknown>).structuredContent
-        : undefined;
-      const responseMeta = structured && typeof structured === 'object' && !Array.isArray(structured)
-        ? (structured as Record<string, unknown>).responseMeta
-        : undefined;
-      const meta = responseMeta && typeof responseMeta === 'object' && !Array.isArray(responseMeta)
-        ? responseMeta as Record<string, unknown>
-        : {};
-      const serverDurationMs = extractForgeServerDuration(response)
-        ?? timingLedgerServerDuration(timingLog, tool, meta);
-      if (serverDurationMs === undefined) {
-        throw new Error('MCP_SERVER_DURATION_MISSING: target must return Forge responseMeta.serverDurationMs or expose a matching --timing-log ledger entry');
-      }
-      samples.push({
-        label,
-        tool,
-        timingScope,
-        clientTotalMs: totalMs,
-        serverDurationMs,
-        ...(typeof meta.traceId === 'string' ? { traceId: meta.traceId } : {}),
-        ...(typeof meta.requestId === 'string' ? { requestId: meta.requestId } : {}),
-        observedAt: new Date().toISOString(),
-      });
-    }
-  } finally {
-    if (steadyState) await steadyState.client.close().catch(() => undefined);
-  }
-  return samples;
-}
-
-if (import.meta.main) {
-  const input = option('--input');
-  const samples = input ? readSamples(input) : await probe();
-  const report = aggregateMcpTransportLatencySamples(samples);
-  console.log(JSON.stringify({
-    generatedAt: new Date().toISOString(),
-    ...report,
-  }, null, 2));
-}

package/scripts/benchmark-operational-memory-activation.ts

@@ -1,235 +0,0 @@
-#!/usr/bin/env bun
-import { execFileSync } from 'node:child_process';
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { performance } from 'node:perf_hooks';
-import {
-  evaluateOperationalMemoryActivation,
-  freezeOperationalMemoryActivationProtocol,
-  type OperationalMemoryActivationMeasurement,
-  type OperationalMemoryActivationPair,
-} from '../evaluation/lib/operational-memory-activation';
-import { controllerCheckEnvironmentFingerprint, listControllerChecks } from '../src/cli/controller/check-runner';
-import {
-  OPERATIONAL_MEMORY_NAMESPACE,
-  ingestCheckCompletionGraceProcess,
-  resolveCheckCompletionGraceWaitMs,
-} from '../src/runtime/control-plane/persistence/operational-prior-store';
-import { listControlPlaneRecords } from '../src/runtime/control-plane/persistence/sqlite-store';
-import { createProcessRecord } from '../src/runtime/execution/process-runtime/store';
-import { startLightweightControllerCheck, waitForLightweightProcess } from '../src/runtime/execution/process-runtime/lightweight-managed';
-import type { ManagedProcessRecord, ProcessHandle } from '../src/runtime/execution/process-runtime/types';
-
-interface Scenario {
-  id: string;
-  kind: 'eligible' | 'guard';
-  delayMs: number;
-  historicalDurationMs: number;
-}
-
-const scenarios: readonly Scenario[] = [
-  { id: 'check-grace-eligible-150ms', kind: 'eligible', delayMs: 150, historicalDurationMs: 150 },
-  { id: 'check-grace-eligible-180ms', kind: 'eligible', delayMs: 180, historicalDurationMs: 180 },
-  { id: 'check-grace-guard-400ms', kind: 'guard', delayMs: 400, historicalDurationMs: 400 },
-];
-const SAMPLES_PER_ARM = 3;
-const CHECK_ID = 'package:check:fixture';
-const CANONICAL_WAIT_MS = 100;
-
-function bytes(value: unknown): number {
-  return Buffer.byteLength(JSON.stringify(value), 'utf8');
-}
-
-function median(values: number[]): number {
-  const sorted = [...values].sort((a, b) => a - b);
-  return sorted[Math.floor(sorted.length / 2)]!;
-}
-
-function fixture(delayMs: number, label: string): { root: string; controllerHome: string; repoId: string } {
-  const root = mkdtempSync(join(tmpdir(), `forge-stage7f-${label}-repo-`));
-  const controllerHome = mkdtempSync(join(tmpdir(), `forge-stage7f-${label}-home-`));
-  writeFileSync(join(root, 'package.json'), JSON.stringify({ scripts: { 'check:fixture': `sleep ${delayMs / 1000}` } }, null, 2));
-  execFileSync('git', ['init', '-q'], { cwd: root });
-  execFileSync('git', ['config', 'user.email', 'stage7f@example.invalid'], { cwd: root });
-  execFileSync('git', ['config', 'user.name', 'Stage7F Fixture'], { cwd: root });
-  execFileSync('git', ['add', '.'], { cwd: root });
-  execFileSync('git', ['commit', '-qm', 'fixture'], { cwd: root });
-  return { root, controllerHome, repoId: `repo-stage7f-${label}` };
-}
-
-function historicalProcess(input: {
-  controllerHome: string;
-  repoId: string;
-  root: string;
-  processId: string;
-  environmentFingerprint: string;
-  durationMs: number;
-  index: number;
-}): ManagedProcessRecord {
-  const finishMs = Date.now() - (10_000 - input.index * 1_000);
-  const startMs = finishMs - input.durationMs;
-  return {
-    schemaVersion: 1,
-    processId: input.processId,
-    repoId: input.repoId,
-    checkoutId: 'historical-checkout',
-    commandId: `historical-command-${input.index}`,
-    controllerHome: input.controllerHome,
-    status: 'succeeded',
-    route: 'direct',
-    command: { kind: 'argv', executable: 'sleep', args: [String(input.durationMs / 1000)], cwd: input.root },
-    resourceClaims: [],
-    leaseRefs: [],
-    leasesReleased: true,
-    interactiveWaitMs: 0,
-    timeoutMs: 5_000,
-    maxOutputBytes: 4_096,
-    startedAt: new Date(startMs).toISOString(),
-    updatedAt: new Date(finishMs).toISOString(),
-    finishedAt: new Date(finishMs).toISOString(),
-    exitCode: 0,
-    timedOut: false,
-    cancelled: false,
-    terminalFenceToken: input.index + 1,
-    terminalWritten: true,
-    checkExecution: {
-      schemaVersion: 1,
-      checkId: CHECK_ID,
-      cacheKey: `historical-cache-${input.index}`,
-      revision: 'historical-revision',
-      definitionDigest: 'historical-definition',
-      environmentFingerprint: input.environmentFingerprint,
-      timeoutMs: 5_000,
-      reuseScope: 'repository',
-      scopeKey: 'repository',
-    },
-    origin: {
-      surface: 'check',
-      checkId: CHECK_ID,
-      requestId: `historical-request-${input.index}`,
-    },
-  };
-}
-
-function seedOperationalMemory(input: {
-  controllerHome: string;
-  repoId: string;
-  root: string;
-  historicalDurationMs: number;
-}): { learnedWaitMs?: number; lookupMs: number; recordBytes: number } {
-  const check = listControllerChecks(input.root).find((entry) => entry.id === CHECK_ID);
-  if (!check) throw new Error(`STAGE7F_BENCHMARK_CHECK_MISSING:${CHECK_ID}`);
-  const environmentFingerprint = controllerCheckEnvironmentFingerprint(check);
-  for (let index = 0; index < 3; index += 1) {
-    const processId = `proc_stage7f_history_${index}`;
-    createProcessRecord(historicalProcess({ ...input, processId, environmentFingerprint, durationMs: input.historicalDurationMs, index }));
-    const ingested = ingestCheckCompletionGraceProcess({ controllerHome: input.controllerHome, repoId: input.repoId, processId });
-    if (!ingested.stored) throw new Error(`STAGE7F_BENCHMARK_INGEST_FAILED:${processId}`);
-  }
-  const lookupStart = performance.now();
-  const learnedWaitMs = resolveCheckCompletionGraceWaitMs({
-    controllerHome: input.controllerHome,
-    repoId: input.repoId,
-    checkId: CHECK_ID,
-    environmentFingerprint,
-  });
-  const lookupMs = performance.now() - lookupStart;
-  const records = listControlPlaneRecords(input.controllerHome, { namespace: OPERATIONAL_MEMORY_NAMESPACE, scope: input.repoId, limit: 16 });
-  return { learnedWaitMs, lookupMs, recordBytes: bytes(records) };
-}
-
-async function runOne(input: {
-  scenario: Scenario;
-  active: boolean;
-  sample: number;
-}): Promise<OperationalMemoryActivationMeasurement> {
-  const label = `${input.scenario.id}-${input.active ? 'active' : 'cold'}-${input.sample}`;
-  const fx = fixture(input.scenario.delayMs, label);
-  try {
-    const memory = input.active
-      ? seedOperationalMemory({ controllerHome: fx.controllerHome, repoId: fx.repoId, root: fx.root, historicalDurationMs: input.scenario.historicalDurationMs })
-      : { learnedWaitMs: undefined, lookupMs: 0, recordBytes: 0 };
-    const appliedWaitMs = memory.learnedWaitMs ?? CANONICAL_WAIT_MS;
-    const started = performance.now();
-    const first = await startLightweightControllerCheck({
-      controllerHome: fx.controllerHome,
-      repoId: fx.repoId,
-      repoRoot: fx.root,
-      checkId: CHECK_ID,
-      interactiveWaitMs: appliedWaitMs,
-      timeoutMs: 5_000,
-      commandId: label,
-    });
-    const firstReturnMs = performance.now() - started;
-    const firstHandle: ProcessHandle = first.handle;
-    let finalHandle = firstHandle;
-    let toolRoundTrips = 1;
-    if (!finalHandle.completed) {
-      toolRoundTrips += 1;
-      finalHandle = await waitForLightweightProcess(fx.controllerHome, fx.repoId, finalHandle.processId, { timeoutMs: 5_000 });
-    }
-    const terminalLatencyMs = performance.now() - started + memory.lookupMs;
-    return {
-      toolRoundTrips,
-      firstReturnMs,
-      terminalLatencyMs,
-      completedInOrigin: firstHandle.completed === true,
-      controllerVisibleBytes: bytes(firstHandle) + (toolRoundTrips > 1 ? bytes(finalHandle) : 0),
-      controllerMemoryPayloadBytes: 0,
-      memoryLookupMs: memory.lookupMs,
-      memoryRecordBytes: memory.recordBytes,
-      appliedWaitMs,
-      ...(memory.learnedWaitMs === undefined ? {} : { learnedWaitMs: memory.learnedWaitMs }),
-      correctnessPassed: finalHandle.completed === true && finalHandle.ok === true,
-    };
-  } finally {
-    rmSync(fx.root, { recursive: true, force: true });
-    rmSync(fx.controllerHome, { recursive: true, force: true });
-  }
-}
-
-function aggregate(samples: OperationalMemoryActivationMeasurement[]): OperationalMemoryActivationMeasurement {
-  const learned = samples.map((sample) => sample.learnedWaitMs).filter((value): value is number => value !== undefined);
-  return {
-    toolRoundTrips: median(samples.map((sample) => sample.toolRoundTrips)),
-    firstReturnMs: median(samples.map((sample) => sample.firstReturnMs)),
-    terminalLatencyMs: median(samples.map((sample) => sample.terminalLatencyMs)),
-    completedInOrigin: samples.filter((sample) => sample.completedInOrigin).length >= Math.ceil(samples.length / 2),
-    controllerVisibleBytes: median(samples.map((sample) => sample.controllerVisibleBytes)),
-    controllerMemoryPayloadBytes: median(samples.map((sample) => sample.controllerMemoryPayloadBytes)),
-    memoryLookupMs: median(samples.map((sample) => sample.memoryLookupMs)),
-    memoryRecordBytes: median(samples.map((sample) => sample.memoryRecordBytes)),
-    appliedWaitMs: median(samples.map((sample) => sample.appliedWaitMs)),
-    ...(learned.length === 0 ? {} : { learnedWaitMs: median(learned) }),
-    correctnessPassed: samples.every((sample) => sample.correctnessPassed),
-  };
-}
-
-export async function runOperationalMemoryActivationBenchmark(candidateIdentity: string) {
-  const protocol = freezeOperationalMemoryActivationProtocol({ candidateIdentity });
-  const pairs: OperationalMemoryActivationPair[] = [];
-  for (const scenario of scenarios) {
-    const cold: OperationalMemoryActivationMeasurement[] = [];
-    const active: OperationalMemoryActivationMeasurement[] = [];
-    for (let sample = 0; sample < SAMPLES_PER_ARM; sample += 1) {
-      cold.push(await runOne({ scenario, active: false, sample }));
-      active.push(await runOne({ scenario, active: true, sample }));
-    }
-    pairs.push({ scenarioId: scenario.id, kind: scenario.kind, candidateIdentity, cold: aggregate(cold), active: aggregate(active) });
-  }
-  return { schemaVersion: 1 as const, protocol, pairs, report: evaluateOperationalMemoryActivation(protocol, pairs) };
-}
-
-function candidateArg(): string | undefined {
-  const index = process.argv.indexOf('--candidate');
-  return index >= 0 ? process.argv[index + 1] : undefined;
-}
-
-if (import.meta.main) {
-  const candidate = candidateArg();
-  if (!candidate) throw new Error('Usage: bun scripts/benchmark-operational-memory-activation.ts --candidate <frozen-candidate-identity>');
-  const result = await runOperationalMemoryActivationBenchmark(candidate);
-  console.log(JSON.stringify(result, null, 2));
-  if (!result.report.passed) process.exitCode = 1;
-}

package/scripts/benchmark-operational-shadow.ts

@@ -1,28 +0,0 @@
-#!/usr/bin/env bun
-import { performance } from 'node:perf_hooks';
-import { rankOperationalShadowPriors, reduceOperationalObservations, type OperationalObservation, type OperationalShadowPrior } from '../packages/kernel/memory/api/index';
-import { evaluateOperationalShadowPairs, freezeOperationalShadowProtocol, type OperationalShadowPair } from '../evaluation/lib/shadow-operational-prior';
-
-interface Fixture { id:string; domainKind:'engineering'|'non_engineering'; operation:string; environment:string; expectedAction:'candidate' }
-const fixtures:readonly Fixture[]=[
-  {id:'engineering-check-ordering',domainKind:'engineering',operation:'check_ordering',environment:'portable-fixture-v1',expectedAction:'candidate'},
-  {id:'research-source-ordering',domainKind:'non_engineering',operation:'source_ordering',environment:'portable-fixture-v1',expectedAction:'candidate'},
-];
-const asOf='2026-09-04T00:00:00.000Z',retainedUntil='2026-09-20T00:00:00.000Z';
-function obs(f:Fixture,action:'baseline'|'candidate',index:number,value:number,outcome:'executed_success'|'contention_wait'='executed_success'):OperationalObservation{
- const requestId=`${f.id}:${action}:${index}`,observedAt=`2026-09-03T00:${String(index).padStart(2,'0')}:00.000Z`;
- return{schemaVersion:1,observationId:`obs:${requestId}:${outcome}`,signalId:'execution_job.terminal_mechanical',targetId:'mechanical_candidate_ordering',actionId:action,metricId:'latency_ms',scope:{schemaVersion:1,kind:'project',id:`fixture:${f.id}`},source:{schema:'execution_job/v1',sourceId:`source:${requestId}`,eventKind:outcome==='contention_wait'?'waiting_for_workspace':'succeeded',evidenceRef:`evidence:${requestId}`,requestId,observedAt,retainedUntil},attribution:{kind:'exact',consumerId:'shadow_evaluation',requestId,sourceId:`source:${requestId}`},outcomeClass:outcome,value,compatibility:{operation:f.operation,environment:f.environment}};
-}
-function fixtureObservations(f:Fixture){return[obs(f,'baseline',1,900),obs(f,'baseline',2,880),obs(f,'baseline',3,920),obs(f,'candidate',4,520),obs(f,'candidate',5,500),obs(f,'candidate',6,510),obs(f,'candidate',7,0,'contention_wait')]}
-function retained(priors:readonly OperationalShadowPrior[]){return new Set(priors.flatMap(p=>p.supportEvidenceRefs))}
-function select(priors:readonly OperationalShadowPrior[],f:Fixture){return rankOperationalShadowPriors({priors,compatibility:{operation:f.operation,environment:f.environment},retainedEvidenceRefs:retained(priors),asOf})[0]?.prior.actionId}
-function benchmark(fn:()=>unknown,iterations=1200){for(let i=0;i<60;i++)fn();const start=performance.now();for(let i=0;i<iterations;i++)fn();return Math.max((performance.now()-start)/iterations,0.000001)}
-function bytes(value:unknown){return Buffer.byteLength(JSON.stringify(value),'utf8')}
-function pair(f:Fixture,candidateRevision:string):OperationalShadowPair{
- const observations=fixtureObservations(f);const shadowReduction=reduceOperationalObservations({observations,asOf});const priors=shadowReduction.priors;const coldSelect=()=>{const reduction=reduceOperationalObservations({observations,asOf});return select(reduction.priors,f)};const shadowSelect=()=>select(priors,f);const coldChoice=coldSelect(),shadowChoice=shadowSelect();
- const context={operation:f.operation,candidates:['baseline','candidate'],scope:`fixture:${f.id}`};const contextBytes=bytes(context),rawEvidenceBytes=bytes(observations),shadowBytes=bytes(priors);
- return{scenarioId:f.id,domainKind:f.domainKind,candidateRevision,cold:{controllerVisibleBytes:contextBytes+rawEvidenceBytes,contextBytes,shadowBytes:0,rhContextEvidenceBytes:rawEvidenceBytes,mechanicalRereads:observations.length,toolRoundTrips:observations.length,latencyMs:benchmark(coldSelect),staleNoValueRetrievals:shadowReduction.rejected.length,correctnessPassed:coldChoice===f.expectedAction},shadow:{controllerVisibleBytes:contextBytes+shadowBytes,contextBytes,shadowBytes,rhContextEvidenceBytes:0,mechanicalRereads:0,toolRoundTrips:1,latencyMs:benchmark(shadowSelect),staleNoValueRetrievals:0,correctnessPassed:shadowChoice===f.expectedAction}};
-}
-export function runOperationalShadowBenchmark(candidateRevision:string){const protocol=freezeOperationalShadowProtocol({candidateRevision,heldOutScenarioIds:fixtures.map(f=>f.id)});const pairs=fixtures.map(f=>pair(f,candidateRevision));const report=evaluateOperationalShadowPairs(protocol,pairs);return{schemaVersion:1 as const,protocol,pairs,report}}
-function candidateArg(){const i=process.argv.indexOf('--candidate');return i>=0?process.argv[i+1]:undefined}
-if(import.meta.main){const candidate=candidateArg();if(!candidate)throw new Error('Usage: bun scripts/benchmark-operational-shadow.ts --candidate <exact-revision>');const result=runOperationalShadowBenchmark(candidate);console.log(JSON.stringify(result,null,2));if(!result.report.passed)process.exitCode=1}

package/scripts/benchmark-quality-harness.ts

@@ -1,146 +0,0 @@
-#!/usr/bin/env bun
-
-import { execFileSync, spawn } from 'child_process';
-import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { performance } from 'perf_hooks';
-import { buildControllerContextPackAsync } from '../src/cli/controller/context-pack';
-import { getMcpPolicy } from '../src/cli/mcp/policy';
-import { closeCodeGraphReadProviderSessions } from '../src/runtime/context/codegraph-read-provider';
-import { clearAllSessionCachesForTest } from '../src/cli/repository/session-cache';
-import { startLightweightRepositoryCommand } from '../src/runtime/execution/process-runtime/lightweight-managed';
-import { registerRepository } from '../src/cli/repositories/registry';
-
-function percentile(values: number[], ratio: number): number {
-  const sorted = [...values].sort((a, b) => a - b);
-  return Math.round((sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * ratio))] ?? 0) * 100) / 100;
-}
-async function rawGitStatus(root: string): Promise<number> {
-  const started = performance.now();
-  await new Promise<void>((resolve, reject) => {
-    const child = spawn('git', ['status', '--short'], { cwd: root, stdio: 'ignore' });
-    child.on('error', reject); child.on('close', () => resolve());
-  });
-  return performance.now() - started;
-}
-
-const iterationsIndex = process.argv.indexOf('--iterations');
-const iterations = Math.max(2, Number(iterationsIndex >= 0 ? process.argv[iterationsIndex + 1] : 7) || 7);
-const structuralIndex = process.argv.indexOf('--structural');
-const structuralContext = process.argv[structuralIndex + 1] === 'off' ? 'off' as const : 'auto' as const;
-const sandboxRoot = mkdtempSync(join(tmpdir(), 'forge-quality-harness-benchmark-'));
-const root = join(sandboxRoot, 'repo');
-const controllerHome = join(sandboxRoot, 'controller');
-try {
-  mkdirSync(root, { recursive: true });
-  execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: root });
-  execFileSync('git', ['config', 'user.email', 'benchmark@example.test'], { cwd: root });
-  execFileSync('git', ['config', 'user.name', 'Forge Benchmark'], { cwd: root });
-  mkdirSync(join(root, 'src'), { recursive: true });
-  for (let index = 0; index < 120; index += 1) {
-    writeFileSync(join(root, 'src', `module-${index}.ts`), `export function symbol${index}(value: number) {\n  return value + ${index};\n}\n`);
-  }
-  writeFileSync(join(root, 'src', 'target.ts'), 'export function benchmarkTarget(value: number) {\n  const next = value + 1;\n  return next;\n}\n');
-  writeFileSync(join(root, 'src', 'consumer.ts'), "import { benchmarkTarget } from './target';\nexport function consumeBenchmarkTarget(value: number) {\n  return benchmarkTarget(value) * 2;\n}\n");
-  execFileSync('git', ['add', '.'], { cwd: root }); execFileSync('git', ['commit', '-qm', 'fixture'], { cwd: root });
-  const repository = registerRepository({ path: root, controllerHome, displayName: 'quality-harness-benchmark' });
-
-  const contextCold: number[] = [];
-  const contextHot: number[] = [];
-  const structuralProcess: number[] = [];
-  const structuralOpen: number[] = [];
-  const structuralQuery: number[] = [];
-  let hotCacheEvidence: unknown;
-  let contextAccuracyEvidence: { selected: string[]; required: string[]; recall: number; precision: number } | undefined;
-  const requiredContextPaths = ['src/target.ts', 'src/consumer.ts'];
-  for (let index = 0; index < iterations; index += 1) {
-    clearAllSessionCachesForTest();
-    const session = { sessionId: `benchmark-${index}`, repoId: repository.repoId, checkoutId: repository.activeCheckoutId };
-    let started = performance.now();
-    const cold = await buildControllerContextPackAsync(root, getMcpPolicy('controller'), { description: 'benchmarkTarget implementation and impact', searchTerms: ['benchmarkTarget'], knownPaths: ['src/target.ts'], structuralContext, session });
-    contextCold.push(performance.now() - started);
-    const structuralTimings = cold.structuralContext?.timingsMs;
-    if (typeof structuralTimings?.process === 'number') structuralProcess.push(structuralTimings.process);
-    if (typeof structuralTimings?.open === 'number') structuralOpen.push(structuralTimings.open);
-    if (typeof structuralTimings?.query === 'number') structuralQuery.push(structuralTimings.query);
-    started = performance.now();
-    const hot = await buildControllerContextPackAsync(root, getMcpPolicy('controller'), { description: 'benchmarkTarget implementation and impact', searchTerms: ['benchmarkTarget'], knownPaths: ['src/target.ts'], structuralContext, session });
-    contextHot.push(performance.now() - started);
-    hotCacheEvidence = hot.cache;
-    const selected = [...new Set(hot.files.map((file) => file.path))].sort();
-    const requiredHits = requiredContextPaths.filter((path) => selected.includes(path)).length;
-    contextAccuracyEvidence = {
-      selected,
-      required: requiredContextPaths,
-      recall: requiredHits / requiredContextPaths.length,
-      precision: selected.length === 0 ? 0 : requiredHits / selected.length,
-    };
-  }
-
-  const rawChild: number[] = [];
-  const harnessTotal: number[] = [];
-  const preSpawnHarness: number[] = [];
-  for (let index = 0; index < iterations; index += 1) {
-    rawChild.push(await rawGitStatus(root));
-    const started = performance.now();
-    const run = await startLightweightRepositoryCommand({
-      controllerHome, repository, interactiveWaitMs: 5_000, timeoutMs: 30_000,
-      execution: {
-        command: ['git', 'status', '--short'],
-        authorizationDecision: { decision: 'allow', source: 'full_access', reason: 'benchmark fixture' },
-      },
-    });
-    harnessTotal.push(performance.now() - started);
-    if (run.metrics.preSpawnHarnessMs !== undefined) preSpawnHarness.push(run.metrics.preSpawnHarnessMs);
-  }
-
-  const steadyContextCold = contextCold.slice(1);
-  const output = {
-    schemaVersion: 1,
-    iterations,
-    context: {
-      structuralContext,
-      coldP50Ms: percentile(contextCold, 0.5), coldP95Ms: percentile(contextCold, 0.95),
-      startupColdMs: Math.round((contextCold[0] ?? 0) * 100) / 100,
-      steadyColdP50Ms: percentile(steadyContextCold, 0.5),
-      steadyColdP95Ms: percentile(steadyContextCold, 0.95),
-      coldSamplesMs: contextCold.map((value) => Math.round(value * 100) / 100),
-      structuralProcessP50Ms: percentile(structuralProcess, 0.5),
-      structuralProcessP95Ms: percentile(structuralProcess, 0.95),
-      structuralOpenP50Ms: percentile(structuralOpen, 0.5),
-      structuralQueryP50Ms: percentile(structuralQuery, 0.5),
-      hotP50Ms: percentile(contextHot, 0.5), hotP95Ms: percentile(contextHot, 0.95),
-      hotToColdRatio: Math.round((percentile(contextHot, 0.5) / Math.max(0.01, percentile(contextCold, 0.5))) * 1_000) / 1_000,
-      cacheEvidence: hotCacheEvidence,
-      accuracy: contextAccuracyEvidence,
-    },
-    command: {
-      rawChildP50Ms: percentile(rawChild, 0.5), lightweightTotalP50Ms: percentile(harnessTotal, 0.5),
-      preSpawnHarnessP50Ms: percentile(preSpawnHarness, 0.5),
-      preSpawnHarnessP95Ms: percentile(preSpawnHarness, 0.95),
-      durableProcessWrites: 0, leaseOperations: 0,
-    },
-    thresholds: {
-      hotContextToColdP50MaxRatio: 0.9,
-      contextRecallMin: 1,
-      contextPrecisionMin: 1,
-      lightweightPreSpawnHarnessP95MaxMs: 750,
-      durableProcessWritesMax: 0,
-      leaseOperationsMax: 0,
-    },
-  };
-  const assertions = {
-    hotContextReuse: output.context.hotToColdRatio <= output.thresholds.hotContextToColdP50MaxRatio,
-    completeRequiredContext: (output.context.accuracy?.recall ?? 0) >= output.thresholds.contextRecallMin,
-    boundedContextNoise: (output.context.accuracy?.precision ?? 0) >= output.thresholds.contextPrecisionMin,
-    boundedPreSpawnHarness: output.command.preSpawnHarnessP95Ms <= output.thresholds.lightweightPreSpawnHarnessP95MaxMs,
-    zeroDurableWrites: output.command.durableProcessWrites <= output.thresholds.durableProcessWritesMax,
-    zeroLeaseOperations: output.command.leaseOperations <= output.thresholds.leaseOperationsMax,
-  };
-  process.stdout.write(`${JSON.stringify({ ...output, assertions, passed: Object.values(assertions).every(Boolean) }, null, 2)}\n`);
-  if (!Object.values(assertions).every(Boolean)) process.exitCode = 1;
-} finally {
-  await closeCodeGraphReadProviderSessions();
-  rmSync(sandboxRoot, { recursive: true, force: true });
-}