@moretea-labs/forge 1.7.2 → 1.8.1

package/scripts/benchmark-controller-hotpath.ts

@@ -1,212 +0,0 @@
-#!/usr/bin/env bun
-
-/**
- * Repeatable controller_context hot-path benchmark.
- *
- * This intentionally uses isolated temporary repositories so it does not
- * mutate the checkout being developed. Output is JSON for CI/local comparison.
- */
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { spawnSync } from 'child_process';
-import { getMcpPolicy } from '../src/cli/mcp/policy';
-import { createMcpToolContext } from '../src/cli/mcp/multi-repository';
-import { exposedControllerToolDefinitions } from '../src/cli/mcp/toolset';
-import {
-  cachedGitIdentity,
-  clearGitIdentityCacheForTest,
-  clearGitSnapshotCacheForTest,
-  gitIdentityPerformanceSnapshot,
-  gitSnapshot,
-  gitSnapshotPerformanceSnapshot,
-} from '../src/cli/repository/inspector';
-import { registerRepository } from '../src/cli/repositories/registry';
-import { callRuntimeTool } from '../src/runtime/gateway/mcp/runtime-tools';
-import type { MultiRepositoryMcpToolContext } from '../src/cli/mcp/multi-repository';
-
-const HOT_READS = 30;
-const COLD_GIT_READS = 20;
-const SCALES = [0, 25, 100, 200];
-
-function gitInit(repoRoot: string): void {
-  spawnSync('git', ['init', '-b', 'main'], { cwd: repoRoot, stdio: 'ignore' });
-  spawnSync('git', ['config', 'user.name', 'hotpath-benchmark'], { cwd: repoRoot, stdio: 'ignore' });
-  spawnSync('git', ['config', 'user.email', 'hotpath@example.invalid'], { cwd: repoRoot, stdio: 'ignore' });
-  writeFileSync(join(repoRoot, 'README.md'), 'controller hotpath benchmark\n');
-  spawnSync('git', ['add', 'README.md'], { cwd: repoRoot, stdio: 'ignore' });
-  spawnSync('git', ['commit', '-m', 'benchmark fixture'], { cwd: repoRoot, stdio: 'ignore' });
-}
-
-function percentile(values: number[], quantile: number): number {
-  const sorted = [...values].sort((a, b) => a - b);
-  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * quantile))] ?? 0;
-}
-
-function timed<T>(fn: () => T): { value: T; ms: number } {
-  const startedAt = performance.now();
-  const value = fn();
-  return { value, ms: Math.round((performance.now() - startedAt) * 100) / 100 };
-}
-
-function writeIssueFixture(repoRoot: string, index: number): void {
-  const id = `ISS-BENCH-${String(index).padStart(4, '0')}`;
-  const slug = `fixture-${index}`;
-  writeFileSync(join(repoRoot, 'tasks', 'issues', `${id}-${slug}.issue.json`), `${JSON.stringify({
-    schemaVersion: 5,
-    id,
-    slug,
-    kind: 'feature',
-    status: 'planned',
-    title: `Benchmark Issue ${index}`,
-    summary: 'Synthetic controller hotpath fixture.',
-    goals: [],
-    nonGoals: [],
-    acceptanceCriteria: [],
-    tasks: [],
-    createdAt: new Date(0).toISOString(),
-    updatedAt: new Date(0).toISOString(),
-    repoId: 'benchmark',
-  }, null, 2)}\n`);
-}
-
-function duration(result: { structuredContent?: unknown; content?: Array<{ text?: string }> } | undefined): { ms: number; bytes: number; cacheHit: boolean; largestFields: Array<{ key: string; bytes: number }> } {
-  const value = (result?.structuredContent
-    ?? JSON.parse(result?.content?.[0]?.text ?? '{}')) as Record<string, unknown>;
-  const responseMeta = (value.responseMeta ?? {}) as Record<string, unknown>;
-  const projection = (value.contextProjection ?? {}) as Record<string, unknown>;
-  return {
-    ms: typeof responseMeta.serverDurationMs === 'number' ? responseMeta.serverDurationMs : 0,
-    bytes: Buffer.byteLength(JSON.stringify(value), 'utf8'),
-    cacheHit: responseMeta.cacheHit === true || projection.cacheHit === true,
-    largestFields: Object.entries(value)
-      .map(([key, field]) => ({ key, bytes: Buffer.byteLength(JSON.stringify(field) ?? 'null', 'utf8') }))
-      .sort((a, b) => b.bytes - a.bytes)
-      .slice(0, 4),
-  };
-}
-
-async function benchmarkScale(scale: number): Promise<Record<string, unknown>> {
-  const workspace = mkdtempSync(join(tmpdir(), 'forge-hotpath-'));
-  try {
-    const controllerHome = join(workspace, 'controller-home');
-    const repoRoot = join(workspace, 'repo');
-    mkdirSync(repoRoot, { recursive: true });
-    mkdirSync(join(repoRoot, 'tasks', 'issues'), { recursive: true });
-    gitInit(repoRoot);
-    for (let index = 0; index < scale; index += 1) writeIssueFixture(repoRoot, index);
-    const repository = registerRepository({ path: repoRoot, controllerHome, displayName: `scale-${scale}` });
-    const ctx = createMcpToolContext({
-      repo: repoRoot,
-      profile: 'controller',
-      toolset: 'advanced',
-      controllerHome,
-    }) as unknown as MultiRepositoryMcpToolContext;
-
-    const coldSummary = duration(await callRuntimeTool(ctx, 'controller_context', { repo_id: repository.repoId }));
-    const coldResult = await callRuntimeTool(ctx, 'controller_context', { repo_id: repository.repoId, detail_level: 'detail' });
-    const cold = duration(coldResult!);
-    const samples: number[] = [];
-    let cacheHits = 0;
-    let maxSummaryBytes = 0;
-    let lastHotSample = cold;
-    for (let index = 0; index < HOT_READS; index += 1) {
-      const sample = duration(await callRuntimeTool(ctx, 'controller_context', { repo_id: repository.repoId }));
-      lastHotSample = sample;
-      samples.push(sample.ms);
-      cacheHits += sample.cacheHit ? 1 : 0;
-      maxSummaryBytes = Math.max(maxSummaryBytes, sample.bytes);
-    }
-    const sorted = [...samples].sort((a, b) => a - b);
-    return {
-      scale,
-      coldSummaryMs: coldSummary.ms,
-      coldSummaryResponseBytes: coldSummary.bytes,
-      coldMs: cold.ms,
-      coldResponseBytes: cold.bytes,
-      hotP50Ms: sorted[Math.floor(sorted.length * 0.5)] ?? 0,
-      hotP95Ms: sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))] ?? 0,
-      hotMaxMs: Math.max(...samples, 0),
-      cacheHits,
-      cacheHitRate: cacheHits / HOT_READS,
-      maxSummaryBytes,
-      largestHotFields: lastHotSample.largestFields,
-    };
-  } finally {
-    rmSync(workspace, { recursive: true, force: true });
-  }
-}
-
-function benchmarkGitSampling(): Record<string, unknown> {
-  const workspace = mkdtempSync(join(tmpdir(), 'forge-git-sampling-'));
-  try {
-    gitInit(workspace);
-    const coldSamples: number[] = [];
-    for (let index = 0; index < COLD_GIT_READS; index += 1) {
-      clearGitIdentityCacheForTest();
-      coldSamples.push(timed(() => cachedGitIdentity(workspace)).ms);
-    }
-
-    clearGitIdentityCacheForTest();
-    cachedGitIdentity(workspace);
-    const warmSamples: number[] = [];
-    for (let index = 0; index < HOT_READS; index += 1) {
-      warmSamples.push(timed(() => cachedGitIdentity(workspace)).ms);
-    }
-    const identityCounters = gitIdentityPerformanceSnapshot();
-
-    clearGitIdentityCacheForTest();
-    clearGitSnapshotCacheForTest();
-    const snapshot = timed(() => gitSnapshot(workspace));
-    return {
-      coldReads: COLD_GIT_READS,
-      coldP50Ms: percentile(coldSamples, 0.5),
-      coldP95Ms: percentile(coldSamples, 0.95),
-      warmReads: HOT_READS,
-      warmP50Ms: percentile(warmSamples, 0.5),
-      warmP95Ms: percentile(warmSamples, 0.95),
-      identityCounters,
-      snapshotMs: snapshot.ms,
-      snapshotCounters: gitSnapshotPerformanceSnapshot(),
-      snapshotIdentityCounters: gitIdentityPerformanceSnapshot(),
-    };
-  } finally {
-    rmSync(workspace, { recursive: true, force: true });
-  }
-}
-
-async function main(): Promise<void> {
-  const scaleResults = [];
-  for (const scale of SCALES) scaleResults.push(await benchmarkScale(scale));
-  const gitSampling = benchmarkGitSampling();
-  const profileRepo = mkdtempSync(join(tmpdir(), 'forge-tool-profile-'));
-  try {
-    const controllerHome = join(profileRepo, 'controller-home');
-    const repoRoot = join(profileRepo, 'repo');
-    mkdirSync(repoRoot, { recursive: true });
-    gitInit(repoRoot);
-    const counts: Record<string, number> = {};
-    for (const toolset of ['core', 'advanced', 'full'] as const) {
-      const ctx = createMcpToolContext({ repo: repoRoot, profile: 'controller', toolset, controllerHome }) as unknown as MultiRepositoryMcpToolContext;
-      counts[toolset] = exposedControllerToolDefinitions(ctx).length;
-    }
-    console.log(JSON.stringify({
-      benchmark: 'controller-hotpath',
-      hotReads: HOT_READS,
-      scales: scaleResults,
-      gitSampling,
-      toolsetCounts: counts,
-      acceptance: {
-        warmP95Ms: 100,
-        maxSummaryBytes: 32 * 1024,
-        coreMaxTools: 20,
-        identitySubprocessesPerSample: 1,
-        snapshotUniqueSubprocesses: 2,
-      },
-    }, null, 2));
-  } finally {
-    rmSync(profileRepo, { recursive: true, force: true });
-  }
-}
-
-await main();

package/scripts/benchmark-controller-round2.ts

@@ -1,139 +0,0 @@
-import { execFileSync } from 'child_process';
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { createMcpToolContext } from '../src/cli/mcp/multi-repository';
-import { callRuntimeTool } from '../src/runtime/gateway/mcp/runtime-tools';
-import { exposedControllerToolDefinitions } from '../src/cli/mcp/toolset';
-import { registerRepository } from '../src/cli/repositories/registry';
-
-type ToolCall = { value: Record<string, any>; elapsedMs: number };
-
-function arg(name: string): string | undefined {
-  const index = process.argv.indexOf(name);
-  return index >= 0 ? process.argv[index + 1] : undefined;
-}
-
-function revision(repoRoot: string): string | undefined {
-  try { return execFileSync('git', ['-C', repoRoot, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { return undefined; }
-}
-
-function bytes(value: unknown): number {
-  return Buffer.byteLength(JSON.stringify(value), 'utf8');
-}
-
-async function call(ctx: ReturnType<typeof createMcpToolContext>, name: string, args: Record<string, unknown> = {}): Promise<ToolCall> {
-  const startedAt = performance.now();
-  const response = await callRuntimeTool(ctx, name, args);
-  if (!response) throw new Error(`benchmark tool returned no result: ${name}`);
-  const raw = response.content?.[0]?.text ?? '{}';
-  const value = (response.structuredContent ?? JSON.parse(raw)) as Record<string, any>;
-  return { value, elapsedMs: Math.round((performance.now() - startedAt) * 100) / 100 };
-}
-
-function percentile(values: number[], p: number): number {
-  if (values.length === 0) return 0;
-  const sorted = [...values].sort((a, b) => a - b);
-  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] ?? 0;
-}
-
-async function main(): Promise<void> {
-  const repoRoot = arg('--repo') ?? process.cwd();
-  const suppliedHome = arg('--controller-home');
-  const generatedHome = suppliedHome ? undefined : mkdtempSync(join(tmpdir(), 'forge-benchmark-round2-'));
-  const controllerHome = suppliedHome ?? generatedHome!;
-  mkdirSync(controllerHome, { recursive: true });
-  const repository = registerRepository({ path: repoRoot, controllerHome });
-  const base = {
-    schemaVersion: 2,
-    revision: revision(repoRoot),
-    repoId: repository.repoId,
-    checkoutId: repository.activeCheckoutId,
-    repositoryRoot: repoRoot,
-    controllerHome,
-    generatedAt: new Date().toISOString(),
-    environment: {
-      platform: process.platform,
-      arch: process.arch,
-      bun: Bun.version,
-      node: process.version,
-    },
-  };
-
-  try {
-    const context = createMcpToolContext({
-      repo: repoRoot,
-      controllerHome,
-      profile: 'controller',
-      toolset: 'advanced',
-      sessionId: `benchmark-round2-${Date.now()}`,
-    });
-    const cold = await call(context, 'controller_context', { repo_id: repository.repoId });
-    const fresh = await call(context, 'controller_context', { repo_id: repository.repoId });
-    writeFileSync(join(repoRoot, '.forge-benchmark-round2-touch'), `${Date.now()}\n`);
-    const stale = await call(context, 'controller_context', { repo_id: repository.repoId });
-    rmSync(join(repoRoot, '.forge-benchmark-round2-touch'), { force: true });
-    const detail = await call(context, 'controller_context', { repo_id: repository.repoId, detail_level: 'detail' });
-    const concurrent = await Promise.all(Array.from({ length: 30 }, () => call(context, 'controller_context', { repo_id: repository.repoId })));
-    const timings = concurrent.map((item) => item.elapsedMs);
-    const readiness = await call(context, 'controller_ready', { repo_id: repository.repoId });
-    const performance = await call(context, 'runtime_performance_diagnostics', {
-      repo_id: repository.repoId,
-      include_processes: false,
-      include_temp_dirs: false,
-    });
-
-    const toolSurface: Record<string, unknown> = {};
-    for (const toolset of ['core', 'advanced', 'full'] as const) {
-      const toolContext = createMcpToolContext({ repo: repoRoot, controllerHome, profile: 'controller', toolset });
-      const definitions = exposedControllerToolDefinitions(toolContext);
-      toolSurface[toolset] = {
-        toolCount: definitions.length,
-        schemaBytes: definitions.reduce((total, tool) => total + bytes(tool.inputSchema), 0),
-        responseBytes: bytes(definitions),
-        p50Ms: undefined,
-        p95Ms: undefined,
-      };
-    }
-    const surfaceRecord = toolSurface.advanced as Record<string, unknown>;
-    surfaceRecord.p50Ms = percentile(timings, 0.5);
-    surfaceRecord.p95Ms = percentile(timings, 0.95);
-
-    const responseMeta = (value: Record<string, any>) => value.responseMeta ?? {};
-    const result = {
-      ...base,
-      scenarios: {
-        contextProjection: {
-          cold: { elapsedMs: cold.elapsedMs, bytes: bytes(cold.value), responseMeta: responseMeta(cold.value) },
-          fresh: { elapsedMs: fresh.elapsedMs, bytes: bytes(fresh.value), responseMeta: responseMeta(fresh.value) },
-          stale: { elapsedMs: stale.elapsedMs, bytes: bytes(stale.value), responseMeta: responseMeta(stale.value) },
-          detail: { elapsedMs: detail.elapsedMs, bytes: bytes(detail.value), responseMeta: responseMeta(detail.value) },
-          concurrentSummary: {
-            count: concurrent.length,
-            p50Ms: percentile(timings, 0.5),
-            p95Ms: percentile(timings, 0.95),
-            maxMs: Math.max(...timings),
-            bytes: concurrent.map((item) => bytes(item.value)),
-          },
-        },
-        readiness: { elapsedMs: readiness.elapsedMs, bytes: bytes(readiness.value) },
-      },
-      transportPhases: {
-        local: { status: 'measured', path: 'direct multi-repository runtime tool' },
-        external: { status: 'unavailable', reason: 'No external endpoint was supplied to this local benchmark.' },
-      },
-      resourceCost: performance.value.resourceCost ?? {},
-      toolSurface,
-      budgets: {
-        controllerContextSummaryBytes: 32 * 1024,
-        summaryBytes: bytes(fresh.value),
-        summaryWithinBudget: bytes(fresh.value) <= 32 * 1024,
-      },
-    };
-    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
-  } finally {
-    if (generatedHome) rmSync(generatedHome, { recursive: true, force: true });
-  }
-}
-
-await main();