@moretea-labs/forge 1.5.0-rc.1

package/scripts/benchmark-controller-hotpath.ts

@@ -0,0 +1,212 @@
+#!/usr/bin/env bun
+
+/**
+ * Repeatable controller_context hot-path benchmark.
+ *
+ * This intentionally uses isolated temporary repositories so it does not
+ * mutate the checkout being developed. Output is JSON for CI/local comparison.
+ */
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { spawnSync } from 'child_process';
+import { getMcpPolicy } from '../src/cli/mcp/policy';
+import { createMcpToolContext } from '../src/cli/mcp/multi-repository';
+import { exposedControllerToolDefinitions } from '../src/cli/mcp/toolset';
+import {
+  cachedGitIdentity,
+  clearGitIdentityCacheForTest,
+  clearGitSnapshotCacheForTest,
+  gitIdentityPerformanceSnapshot,
+  gitSnapshot,
+  gitSnapshotPerformanceSnapshot,
+} from '../src/cli/repository/inspector';
+import { registerRepository } from '../src/cli/repositories/registry';
+import { callRuntimeTool } from '../src/runtime/gateway/mcp/runtime-tools';
+import type { MultiRepositoryMcpToolContext } from '../src/cli/mcp/multi-repository';
+
+const HOT_READS = 30;
+const COLD_GIT_READS = 20;
+const SCALES = [0, 25, 100, 200];
+
+function gitInit(repoRoot: string): void {
+  spawnSync('git', ['init', '-b', 'main'], { cwd: repoRoot, stdio: 'ignore' });
+  spawnSync('git', ['config', 'user.name', 'hotpath-benchmark'], { cwd: repoRoot, stdio: 'ignore' });
+  spawnSync('git', ['config', 'user.email', 'hotpath@example.invalid'], { cwd: repoRoot, stdio: 'ignore' });
+  writeFileSync(join(repoRoot, 'README.md'), 'controller hotpath benchmark\n');
+  spawnSync('git', ['add', 'README.md'], { cwd: repoRoot, stdio: 'ignore' });
+  spawnSync('git', ['commit', '-m', 'benchmark fixture'], { cwd: repoRoot, stdio: 'ignore' });
+}
+
+function percentile(values: number[], quantile: number): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * quantile))] ?? 0;
+}
+
+function timed<T>(fn: () => T): { value: T; ms: number } {
+  const startedAt = performance.now();
+  const value = fn();
+  return { value, ms: Math.round((performance.now() - startedAt) * 100) / 100 };
+}
+
+function writeIssueFixture(repoRoot: string, index: number): void {
+  const id = `ISS-BENCH-${String(index).padStart(4, '0')}`;
+  const slug = `fixture-${index}`;
+  writeFileSync(join(repoRoot, 'tasks', 'issues', `${id}-${slug}.issue.json`), `${JSON.stringify({
+    schemaVersion: 5,
+    id,
+    slug,
+    kind: 'feature',
+    status: 'planned',
+    title: `Benchmark Issue ${index}`,
+    summary: 'Synthetic controller hotpath fixture.',
+    goals: [],
+    nonGoals: [],
+    acceptanceCriteria: [],
+    tasks: [],
+    createdAt: new Date(0).toISOString(),
+    updatedAt: new Date(0).toISOString(),
+    repoId: 'benchmark',
+  }, null, 2)}\n`);
+}
+
+function duration(result: { structuredContent?: unknown; content?: Array<{ text?: string }> } | undefined): { ms: number; bytes: number; cacheHit: boolean; largestFields: Array<{ key: string; bytes: number }> } {
+  const value = (result?.structuredContent
+    ?? JSON.parse(result?.content?.[0]?.text ?? '{}')) as Record<string, unknown>;
+  const responseMeta = (value.responseMeta ?? {}) as Record<string, unknown>;
+  const projection = (value.contextProjection ?? {}) as Record<string, unknown>;
+  return {
+    ms: typeof responseMeta.serverDurationMs === 'number' ? responseMeta.serverDurationMs : 0,
+    bytes: Buffer.byteLength(JSON.stringify(value), 'utf8'),
+    cacheHit: responseMeta.cacheHit === true || projection.cacheHit === true,
+    largestFields: Object.entries(value)
+      .map(([key, field]) => ({ key, bytes: Buffer.byteLength(JSON.stringify(field) ?? 'null', 'utf8') }))
+      .sort((a, b) => b.bytes - a.bytes)
+      .slice(0, 4),
+  };
+}
+
+async function benchmarkScale(scale: number): Promise<Record<string, unknown>> {
+  const workspace = mkdtempSync(join(tmpdir(), 'forge-hotpath-'));
+  try {
+    const controllerHome = join(workspace, 'controller-home');
+    const repoRoot = join(workspace, 'repo');
+    mkdirSync(repoRoot, { recursive: true });
+    mkdirSync(join(repoRoot, 'tasks', 'issues'), { recursive: true });
+    gitInit(repoRoot);
+    for (let index = 0; index < scale; index += 1) writeIssueFixture(repoRoot, index);
+    const repository = registerRepository({ path: repoRoot, controllerHome, displayName: `scale-${scale}` });
+    const ctx = createMcpToolContext({
+      repo: repoRoot,
+      profile: 'controller',
+      toolset: 'advanced',
+      controllerHome,
+    }) as unknown as MultiRepositoryMcpToolContext;
+
+    const coldSummary = duration(await callRuntimeTool(ctx, 'controller_context', { repo_id: repository.repoId }));
+    const coldResult = await callRuntimeTool(ctx, 'controller_context', { repo_id: repository.repoId, detail_level: 'detail' });
+    const cold = duration(coldResult!);
+    const samples: number[] = [];
+    let cacheHits = 0;
+    let maxSummaryBytes = 0;
+    let lastHotSample = cold;
+    for (let index = 0; index < HOT_READS; index += 1) {
+      const sample = duration(await callRuntimeTool(ctx, 'controller_context', { repo_id: repository.repoId }));
+      lastHotSample = sample;
+      samples.push(sample.ms);
+      cacheHits += sample.cacheHit ? 1 : 0;
+      maxSummaryBytes = Math.max(maxSummaryBytes, sample.bytes);
+    }
+    const sorted = [...samples].sort((a, b) => a - b);
+    return {
+      scale,
+      coldSummaryMs: coldSummary.ms,
+      coldSummaryResponseBytes: coldSummary.bytes,
+      coldMs: cold.ms,
+      coldResponseBytes: cold.bytes,
+      hotP50Ms: sorted[Math.floor(sorted.length * 0.5)] ?? 0,
+      hotP95Ms: sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))] ?? 0,
+      hotMaxMs: Math.max(...samples, 0),
+      cacheHits,
+      cacheHitRate: cacheHits / HOT_READS,
+      maxSummaryBytes,
+      largestHotFields: lastHotSample.largestFields,
+    };
+  } finally {
+    rmSync(workspace, { recursive: true, force: true });
+  }
+}
+
+function benchmarkGitSampling(): Record<string, unknown> {
+  const workspace = mkdtempSync(join(tmpdir(), 'forge-git-sampling-'));
+  try {
+    gitInit(workspace);
+    const coldSamples: number[] = [];
+    for (let index = 0; index < COLD_GIT_READS; index += 1) {
+      clearGitIdentityCacheForTest();
+      coldSamples.push(timed(() => cachedGitIdentity(workspace)).ms);
+    }
+
+    clearGitIdentityCacheForTest();
+    cachedGitIdentity(workspace);
+    const warmSamples: number[] = [];
+    for (let index = 0; index < HOT_READS; index += 1) {
+      warmSamples.push(timed(() => cachedGitIdentity(workspace)).ms);
+    }
+    const identityCounters = gitIdentityPerformanceSnapshot();
+
+    clearGitIdentityCacheForTest();
+    clearGitSnapshotCacheForTest();
+    const snapshot = timed(() => gitSnapshot(workspace));
+    return {
+      coldReads: COLD_GIT_READS,
+      coldP50Ms: percentile(coldSamples, 0.5),
+      coldP95Ms: percentile(coldSamples, 0.95),
+      warmReads: HOT_READS,
+      warmP50Ms: percentile(warmSamples, 0.5),
+      warmP95Ms: percentile(warmSamples, 0.95),
+      identityCounters,
+      snapshotMs: snapshot.ms,
+      snapshotCounters: gitSnapshotPerformanceSnapshot(),
+      snapshotIdentityCounters: gitIdentityPerformanceSnapshot(),
+    };
+  } finally {
+    rmSync(workspace, { recursive: true, force: true });
+  }
+}
+
+async function main(): Promise<void> {
+  const scaleResults = [];
+  for (const scale of SCALES) scaleResults.push(await benchmarkScale(scale));
+  const gitSampling = benchmarkGitSampling();
+  const profileRepo = mkdtempSync(join(tmpdir(), 'forge-tool-profile-'));
+  try {
+    const controllerHome = join(profileRepo, 'controller-home');
+    const repoRoot = join(profileRepo, 'repo');
+    mkdirSync(repoRoot, { recursive: true });
+    gitInit(repoRoot);
+    const counts: Record<string, number> = {};
+    for (const toolset of ['core', 'advanced', 'full'] as const) {
+      const ctx = createMcpToolContext({ repo: repoRoot, profile: 'controller', toolset, controllerHome }) as unknown as MultiRepositoryMcpToolContext;
+      counts[toolset] = exposedControllerToolDefinitions(ctx).length;
+    }
+    console.log(JSON.stringify({
+      benchmark: 'controller-hotpath',
+      hotReads: HOT_READS,
+      scales: scaleResults,
+      gitSampling,
+      toolsetCounts: counts,
+      acceptance: {
+        warmP95Ms: 100,
+        maxSummaryBytes: 32 * 1024,
+        coreMaxTools: 20,
+        identitySubprocessesPerSample: 1,
+        snapshotUniqueSubprocesses: 2,
+      },
+    }, null, 2));
+  } finally {
+    rmSync(profileRepo, { recursive: true, force: true });
+  }
+}
+
+await main();

package/scripts/benchmark-controller-round2.ts

@@ -0,0 +1,139 @@
+import { execFileSync } from 'child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { createMcpToolContext } from '../src/cli/mcp/multi-repository';
+import { callRuntimeTool } from '../src/runtime/gateway/mcp/runtime-tools';
+import { exposedControllerToolDefinitions } from '../src/cli/mcp/toolset';
+import { registerRepository } from '../src/cli/repositories/registry';
+
+type ToolCall = { value: Record<string, any>; elapsedMs: number };
+
+function arg(name: string): string | undefined {
+  const index = process.argv.indexOf(name);
+  return index >= 0 ? process.argv[index + 1] : undefined;
+}
+
+function revision(repoRoot: string): string | undefined {
+  try { return execFileSync('git', ['-C', repoRoot, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { return undefined; }
+}
+
+function bytes(value: unknown): number {
+  return Buffer.byteLength(JSON.stringify(value), 'utf8');
+}
+
+async function call(ctx: ReturnType<typeof createMcpToolContext>, name: string, args: Record<string, unknown> = {}): Promise<ToolCall> {
+  const startedAt = performance.now();
+  const response = await callRuntimeTool(ctx, name, args);
+  if (!response) throw new Error(`benchmark tool returned no result: ${name}`);
+  const raw = response.content?.[0]?.text ?? '{}';
+  const value = (response.structuredContent ?? JSON.parse(raw)) as Record<string, any>;
+  return { value, elapsedMs: Math.round((performance.now() - startedAt) * 100) / 100 };
+}
+
+function percentile(values: number[], p: number): number {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] ?? 0;
+}
+
+async function main(): Promise<void> {
+  const repoRoot = arg('--repo') ?? process.cwd();
+  const suppliedHome = arg('--controller-home');
+  const generatedHome = suppliedHome ? undefined : mkdtempSync(join(tmpdir(), 'forge-benchmark-round2-'));
+  const controllerHome = suppliedHome ?? generatedHome!;
+  mkdirSync(controllerHome, { recursive: true });
+  const repository = registerRepository({ path: repoRoot, controllerHome });
+  const base = {
+    schemaVersion: 2,
+    revision: revision(repoRoot),
+    repoId: repository.repoId,
+    checkoutId: repository.activeCheckoutId,
+    repositoryRoot: repoRoot,
+    controllerHome,
+    generatedAt: new Date().toISOString(),
+    environment: {
+      platform: process.platform,
+      arch: process.arch,
+      bun: Bun.version,
+      node: process.version,
+    },
+  };
+
+  try {
+    const context = createMcpToolContext({
+      repo: repoRoot,
+      controllerHome,
+      profile: 'controller',
+      toolset: 'advanced',
+      sessionId: `benchmark-round2-${Date.now()}`,
+    });
+    const cold = await call(context, 'controller_context', { repo_id: repository.repoId });
+    const fresh = await call(context, 'controller_context', { repo_id: repository.repoId });
+    writeFileSync(join(repoRoot, '.forge-benchmark-round2-touch'), `${Date.now()}\n`);
+    const stale = await call(context, 'controller_context', { repo_id: repository.repoId });
+    rmSync(join(repoRoot, '.forge-benchmark-round2-touch'), { force: true });
+    const detail = await call(context, 'controller_context', { repo_id: repository.repoId, detail_level: 'detail' });
+    const concurrent = await Promise.all(Array.from({ length: 30 }, () => call(context, 'controller_context', { repo_id: repository.repoId })));
+    const timings = concurrent.map((item) => item.elapsedMs);
+    const readiness = await call(context, 'controller_ready', { repo_id: repository.repoId });
+    const performance = await call(context, 'runtime_performance_diagnostics', {
+      repo_id: repository.repoId,
+      include_processes: false,
+      include_temp_dirs: false,
+    });
+
+    const toolSurface: Record<string, unknown> = {};
+    for (const toolset of ['core', 'advanced', 'full'] as const) {
+      const toolContext = createMcpToolContext({ repo: repoRoot, controllerHome, profile: 'controller', toolset });
+      const definitions = exposedControllerToolDefinitions(toolContext);
+      toolSurface[toolset] = {
+        toolCount: definitions.length,
+        schemaBytes: definitions.reduce((total, tool) => total + bytes(tool.inputSchema), 0),
+        responseBytes: bytes(definitions),
+        p50Ms: undefined,
+        p95Ms: undefined,
+      };
+    }
+    const surfaceRecord = toolSurface.advanced as Record<string, unknown>;
+    surfaceRecord.p50Ms = percentile(timings, 0.5);
+    surfaceRecord.p95Ms = percentile(timings, 0.95);
+
+    const responseMeta = (value: Record<string, any>) => value.responseMeta ?? {};
+    const result = {
+      ...base,
+      scenarios: {
+        contextProjection: {
+          cold: { elapsedMs: cold.elapsedMs, bytes: bytes(cold.value), responseMeta: responseMeta(cold.value) },
+          fresh: { elapsedMs: fresh.elapsedMs, bytes: bytes(fresh.value), responseMeta: responseMeta(fresh.value) },
+          stale: { elapsedMs: stale.elapsedMs, bytes: bytes(stale.value), responseMeta: responseMeta(stale.value) },
+          detail: { elapsedMs: detail.elapsedMs, bytes: bytes(detail.value), responseMeta: responseMeta(detail.value) },
+          concurrentSummary: {
+            count: concurrent.length,
+            p50Ms: percentile(timings, 0.5),
+            p95Ms: percentile(timings, 0.95),
+            maxMs: Math.max(...timings),
+            bytes: concurrent.map((item) => bytes(item.value)),
+          },
+        },
+        readiness: { elapsedMs: readiness.elapsedMs, bytes: bytes(readiness.value) },
+      },
+      transportPhases: {
+        local: { status: 'measured', path: 'direct multi-repository runtime tool' },
+        external: { status: 'unavailable', reason: 'No external endpoint was supplied to this local benchmark.' },
+      },
+      resourceCost: performance.value.resourceCost ?? {},
+      toolSurface,
+      budgets: {
+        controllerContextSummaryBytes: 32 * 1024,
+        summaryBytes: bytes(fresh.value),
+        summaryWithinBudget: bytes(fresh.value) <= 32 * 1024,
+      },
+    };
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+  } finally {
+    if (generatedHome) rmSync(generatedHome, { recursive: true, force: true });
+  }
+}
+
+await main();