@cassiomc1/forgeloop 1.7.0 → 1.8.0

package/scripts/report-tail-interpretation.mjs

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import {
+  aggregateBenchmarkRuns,
+  BENCHMARK_VERSION,
+} from "../src/core/execution-profile-benchmarks.js";
+import { readBenchmarkRunSets, readBenchmarkScenarios } from "./lib/execution-profile-benchmark-io.mjs";
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const defaultResultsDirectory = path.join(repositoryRoot, "benchmarks", "execution-profiles", "results");
+
+function parseArgs(argv) {
+  const options = { results: defaultResultsDirectory, runSetId: null, json: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === "--json") options.json = true;
+    else if (["--results", "--run-set"].includes(argument)) {
+      const value = argv[++index];
+      if (!value || value.startsWith("--")) throw new Error(`${argument} requires a value`);
+      if (argument === "--results") options.results = value;
+      else options.runSetId = value;
+    } else if (argument === "--help" || argument === "-h") options.help = true;
+    else throw new Error(`unknown option: ${argument}`);
+  }
+  return options;
+}
+
+function helpText() {
+  return "Usage: npm run benchmark:profiles:tail-analysis -- [--results <directory>] [--run-set <id>] [--json]";
+}
+
+function buildScenarioReport(aggregate) {
+  const directAgg = aggregate.modeAggregates?.direct;
+  const adaptiveComp = aggregate.comparisons?.forgeloopAdaptive;
+  const adaptiveAgg = aggregate.modeAggregates?.forgeloopAdaptive;
+
+  const directTokens = directAgg?.totalTokens ?? {};
+  const adaptiveTokens = adaptiveAgg?.totalTokens ?? {};
+
+  const paired = adaptiveComp?.pairedOverheadPercent ?? {};
+  const dist = adaptiveComp?.distributionDeltaPercent ?? {};
+  const tail = adaptiveComp?.tail ?? {};
+  const diag = adaptiveComp?.pairedRatioDiagnostics ?? {};
+
+  const hasComparableRuns = (adaptiveComp?.pairedRuns?.length ?? 0) > 0;
+  let compDirectP50 = diag.baselineP50 ?? directTokens.p50 ?? null;
+  let compDirectP95 = directTokens.p95 ?? null;
+  let compAdaptiveP50 = adaptiveTokens.p50 ?? null;
+  let compAdaptiveP95 = adaptiveTokens.p95 ?? null;
+
+  if (hasComparableRuns) {
+    const dTokens = adaptiveComp.pairedRuns.map((r) => r.directTokens).sort((a, b) => a - b);
+    const aTokens = adaptiveComp.pairedRuns.map((r) => r.candidateTokens).sort((a, b) => a - b);
+    const pos95 = (dTokens.length - 1) * 0.95;
+    const lower = Math.floor(pos95);
+    const upper = Math.ceil(pos95);
+    const calcP95 = (arr) => (lower === upper ? arr[lower] : arr[lower] + ((arr[upper] - arr[lower]) * (pos95 - lower)));
+    const pos50 = (dTokens.length - 1) * 0.5;
+    const lower50 = Math.floor(pos50);
+    const upper50 = Math.ceil(pos50);
+    const calcP50 = (arr) => (lower50 === upper50 ? arr[lower50] : arr[lower50] + ((arr[upper50] - arr[lower50]) * (pos50 - lower50)));
+
+    compDirectP50 = Number(calcP50(dTokens).toFixed(4));
+    compDirectP95 = Number(calcP95(dTokens).toFixed(4));
+    compAdaptiveP50 = Number(calcP50(aTokens).toFixed(4));
+    compAdaptiveP95 = Number(calcP95(aTokens).toFixed(4));
+  }
+
+  return {
+    scenarioId: aggregate.scenarioId,
+    expectedProfile: aggregate.expectedProfile,
+    totalRunsPerMode: directAgg?.runCount ?? 0,
+    verificationSuccessRate: adaptiveAgg?.verificationSuccessRate ?? 1,
+    directP50: compDirectP50,
+    adaptiveP50: compAdaptiveP50,
+    distributionP50DeltaPercent: dist.p50 ?? null,
+    directP95: compDirectP95,
+    adaptiveP95: compAdaptiveP95,
+    distributionP95DeltaPercent: dist.p95 ?? null,
+    pairedOverheadP50Percent: paired.p50 ?? null,
+    pairedOverheadP95Percent: paired.p95 ?? null,
+    lowBaselinePairCount: diag.lowBaselinePairCount ?? 0,
+    comparablePairCount: adaptiveComp?.tokenComparablePairs ?? 0,
+    pairedStatus: tail.pairedStatus ?? "NOT_ENOUGH_SAMPLES",
+    distributionStatus: tail.distributionStatus ?? "NOT_ENOUGH_SAMPLES",
+    combinedInterpretation: tail.combinedInterpretation ?? "TAIL_UNRESOLVED",
+    pairedRuns: adaptiveComp?.pairedRuns ?? [],
+  };
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    console.log(helpText());
+    return;
+  }
+  const scenarios = await readBenchmarkScenarios(repositoryRoot);
+  const runSets = await readBenchmarkRunSets(path.resolve(options.results), options.runSetId);
+  const reports = [];
+
+  for (const runSet of runSets) {
+    const scenarioReports = [];
+    for (const scenario of scenarios) {
+      const runs = runSet.runs.filter((run) => run.scenarioId === scenario.scenarioId);
+      if (runs.length === 0) continue;
+      const aggregate = aggregateBenchmarkRuns({ scenario, runs });
+      scenarioReports.push(buildScenarioReport(aggregate));
+    }
+    reports.push({
+      runSetId: runSet.runSetId,
+      runCount: runSet.runs.length,
+      scenarios: scenarioReports,
+    });
+  }
+
+  const output = {
+    schemaVersion: 1,
+    benchmarkVersion: BENCHMARK_VERSION,
+    status: reports.length === 0 ? "NOT_MEASURED" : "MEASURED",
+    runSets: reports,
+  };
+
+  if (options.json) {
+    console.log(JSON.stringify(output, null, 2));
+  } else {
+    for (const runSet of reports) {
+      console.log(`\n======================================================`);
+      console.log(`Run Set: ${runSet.runSetId} (${runSet.runCount} runs)`);
+      console.log(`======================================================`);
+      for (const sc of runSet.scenarios) {
+        console.log(`\nScenario: ${sc.scenarioId} (expectedProfile: ${sc.expectedProfile})`);
+        console.log(`  Direct P50:                   ${sc.directP50 ?? "N/A"}`);
+        console.log(`  Adaptive P50:                 ${sc.adaptiveP50 ?? "N/A"}`);
+        console.log(`  Distribution P50 Delta:       ${sc.distributionP50DeltaPercent !== null ? `${sc.distributionP50DeltaPercent > 0 ? "+" : ""}${sc.distributionP50DeltaPercent}%` : "N/A"}`);
+        console.log(`  Direct P95:                   ${sc.directP95 ?? "N/A"}`);
+        console.log(`  Adaptive P95:                 ${sc.adaptiveP95 ?? "N/A"}`);
+        console.log(`  Distribution P95 Delta:       ${sc.distributionP95DeltaPercent !== null ? `${sc.distributionP95DeltaPercent > 0 ? "+" : ""}${sc.distributionP95DeltaPercent}%` : "N/A"}`);
+        console.log(`  Paired Overhead P50:          ${sc.pairedOverheadP50Percent !== null ? `${sc.pairedOverheadP50Percent > 0 ? "+" : ""}${sc.pairedOverheadP50Percent}%` : "N/A"}`);
+        console.log(`  Paired Overhead P95:          ${sc.pairedOverheadP95Percent !== null ? `${sc.pairedOverheadP95Percent > 0 ? "+" : ""}${sc.pairedOverheadP95Percent}%` : "N/A"}`);
+        console.log(`  Low-Baseline Pairs:           ${sc.lowBaselinePairCount}`);
+        console.log(`  Comparable Pairs:             ${sc.comparablePairCount}`);
+        console.log(`  Paired Tail Status:           ${sc.pairedStatus}`);
+        console.log(`  Distribution Tail Status:     ${sc.distributionStatus}`);
+        console.log(`  Combined Interpretation:      ${sc.combinedInterpretation}`);
+      }
+    }
+  }
+}
+
+main().catch((error) => {
+  console.error(`ForgeLoop tail interpretation report: ${error.message}`);
+  process.exitCode = 1;
+});

package/scripts/run-execution-profile-benchmarks.mjs

@@ -12,6 +12,7 @@ import { assertSchema, readSchema } from "../src/core/schema-validation.js";
 import {
   aggregateBenchmarkRuns,
   BENCHMARK_MODES,
+  BENCHMARK_TIERS,
   BENCHMARK_VERSION,
   assertBenchmarkScenario,
   assertRequiredBenchmarkScenarios,
@@ -32,7 +33,8 @@ function parseArgs(argv) {
   const options = {
     target: process.cwd(),
     adapter: null,
-    runs: 5,
+    runs: null,
+    tier: null,
     runSetId: null,
     output: defaultOutputDirectory,
     json: false,
@@ -47,6 +49,7 @@ function parseArgs(argv) {
     if (argument === "--target") options.target = value();
     else if (argument === "--adapter") options.adapter = value();
     else if (argument === "--runs") options.runs = Number(value());
+    else if (argument === "--tier") options.tier = value();
     else if (argument === "--run-set") options.runSetId = value();
     else if (argument === "--output") options.output = value();
     else if (argument === "--json") options.json = true;
@@ -56,9 +59,19 @@ function parseArgs(argv) {
   }
   if (options.help) return options;
   if (!options.adapter) throw usageError("--adapter is required; ForgeLoop never invents provider or host measurements");
+  if (options.tier !== null && !Object.prototype.hasOwnProperty.call(BENCHMARK_TIERS, options.tier)) {
+    throw usageError(`--tier must be one of ${Object.keys(BENCHMARK_TIERS).join(", ")}`);
+  }
+  const tier = options.tier === null ? null : BENCHMARK_TIERS[options.tier];
+  if (options.runs === null) {
+    options.runs = tier === null ? 5 : tier.defaultRuns;
+  }
   if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 100) {
     throw usageError("--runs must be an integer from 1 through 100");
   }
+  if (tier !== null && (options.runs < tier.minimumRuns || options.runs > tier.maximumRuns)) {
+    throw usageError(`--runs must be between ${tier.minimumRuns} and ${tier.maximumRuns} for the ${options.tier} tier`);
+  }
   if (options.runSetId !== null && !/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/u.test(options.runSetId)) {
     throw usageError("--run-set must contain only portable identifier characters");
   }
@@ -70,7 +83,9 @@ function helpText() {
     "Usage: npm run benchmark:profiles -- --adapter <module> [options]",
     "",
     "The adapter must execute each scenario and return actual provider/host usage, verification, comparable steps, and optional host-reported contextUsage.",
-    "Options: --target <path> --runs <1..100> --run-set <id> --output <directory> --json",
+    "An optional finalizeBenchmark hook may attach independently evaluated quality by runner-owned runId after all host timings finish.",
+    "Tiers are benchmark sample-size policy: smoke (1-3 runs), evidence (5-10 runs), tail (20-30 runs).",
+    "Options: --target <path> --runs <1..100> --tier <smoke|evidence|tail> --run-set <id> --output <directory> --json",
   ].join("\n");
 }
 
@@ -93,11 +108,18 @@ async function loadScenarios() {
 async function loadAdapter(adapterSpecifier) {
   const adapterPath = path.resolve(process.cwd(), adapterSpecifier);
   const adapterModule = await import(pathToFileURL(adapterPath).href);
-  const adapter = adapterModule.runBenchmark ?? adapterModule.default?.runBenchmark ?? adapterModule.default;
-  if (typeof adapter !== "function") {
+  const defaultExport = adapterModule.default;
+  const runBenchmark = adapterModule.runBenchmark
+    ?? defaultExport?.runBenchmark
+    ?? (typeof defaultExport === "function" ? defaultExport : null);
+  if (typeof runBenchmark !== "function") {
     throw usageError("benchmark adapter must export runBenchmark(input) or be a function default export");
   }
-  return adapter;
+  return {
+    runBenchmark,
+    finalizeBenchmark: adapterModule.finalizeBenchmark ?? defaultExport?.finalizeBenchmark ?? null,
+    cleanup: adapterModule.cleanup ?? defaultExport?.cleanup ?? null,
+  };
 }
 
 function generatedRunSetId() {
@@ -139,14 +161,37 @@ function normalizedAdapterMetadata(response, target, repository, scenario, mode)
   };
 }
 
+function normalizeFinalizationResult(value, records) {
+  if (value === undefined || value === null) {
+    return { qualityByRunId: {}, summary: { status: "NOT_CONFIGURED" } };
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw usageError("benchmark finalizer must return an object");
+  }
+  const qualityByRunId = value.qualityByRunId ?? {};
+  if (!qualityByRunId || typeof qualityByRunId !== "object" || Array.isArray(qualityByRunId)) {
+    throw usageError("benchmark finalizer qualityByRunId must be an object");
+  }
+  const runIds = new Set(records.map((record) => record.runId));
+  for (const runId of Object.keys(qualityByRunId)) {
+    if (!runIds.has(runId)) throw usageError(`benchmark finalizer returned an unknown runId: ${runId}`);
+  }
+  return {
+    qualityByRunId,
+    summary: value.summary && typeof value.summary === "object" && !Array.isArray(value.summary)
+      ? value.summary
+      : { status: "MEASURED" },
+  };
+}
+
 async function executeRuns({ adapter, scenarios, target, runSetId, runs }) {
   const repository = await currentRepositoryFingerprint(target);
-  const allRuns = [];
+  const records = [];
   for (const scenario of scenarios) {
     for (const mode of BENCHMARK_MODES) {
       for (let runIndex = 1; runIndex <= runs; runIndex += 1) {
         const started = performance.now();
-        const response = await adapter({
+        const response = await adapter.runBenchmark({
           benchmarkVersion: BENCHMARK_VERSION,
           scenario: structuredClone(scenario),
           mode,
@@ -159,29 +204,50 @@ async function executeRuns({ adapter, scenarios, target, runSetId, runs }) {
         if (!response || typeof response !== "object" || Array.isArray(response)) {
           throw usageError(`${scenario.scenarioId}/${mode}/${runIndex}: adapter must return an object`);
         }
-        const run = createBenchmarkRun({
+        records.push({
           runSetId,
           runId: `run-${scenario.scenarioId}-${mode}-${String(runIndex).padStart(3, "0")}`,
           runIndex,
           scenario,
           mode,
-          usage: response.usage ?? {},
+          response,
           wallClockMs,
-          verification: response.verification ?? "NOT_AVAILABLE",
-          verificationCycles: response.verificationCycles ?? null,
-          comparableSteps: response.comparableSteps ?? null,
-          contextUsage: response.contextUsage,
-          quality: response.quality,
-          metadata: normalizedAdapterMetadata(response, target, repository, scenario, mode),
         });
-        allRuns.push(run);
       }
     }
   }
-  return allRuns;
+  const finalization = typeof adapter.finalizeBenchmark === "function"
+    ? normalizeFinalizationResult(await adapter.finalizeBenchmark({
+      benchmarkVersion: BENCHMARK_VERSION,
+      records,
+      scenarios,
+      target,
+      runSetId,
+    }), records)
+    : normalizeFinalizationResult(null, records);
+  const allRuns = records.map((record) => {
+    const { scenario, mode, response } = record;
+    return createBenchmarkRun({
+      runSetId: record.runSetId,
+      runId: record.runId,
+      runIndex: record.runIndex,
+      scenario,
+      mode,
+      usage: response.usage ?? {},
+      wallClockMs: record.wallClockMs,
+      verification: response.verification ?? "NOT_AVAILABLE",
+      verificationCycles: response.verificationCycles ?? null,
+      comparableSteps: response.comparableSteps ?? null,
+      diagnostics: response.diagnostics,
+      contextUsage: response.contextUsage,
+      quality: finalization.qualityByRunId[record.runId] ?? response.quality,
+      metadata: normalizedAdapterMetadata(response, target, repository, scenario, mode),
+    });
+  });
+  return { runs: allRuns, finalization: finalization.summary };
 }
 
-async function writeResults({ outputDirectory, runSetId, scenarios, runs }) {
+async function writeResults({ outputDirectory, runSetId, scenarios, runs, tier }) {
   const rawDirectory = path.join(outputDirectory, "raw", runSetId);
   const aggregateDirectory = path.join(outputDirectory, "aggregate", runSetId);
   try {
@@ -214,6 +280,7 @@ async function writeResults({ outputDirectory, runSetId, scenarios, runs }) {
     schemaVersion: 1,
     benchmarkVersion: BENCHMARK_VERSION,
     runSetId,
+    tier,
     scenarioCount: scenarios.length,
     runCount: runs.length,
     claimsAllowed: aggregates.some((aggregate) => aggregate.claimsAllowed),
@@ -223,6 +290,7 @@ async function writeResults({ outputDirectory, runSetId, scenarios, runs }) {
       claimsAllowed: aggregate.claimsAllowed,
       lightObjectives: aggregate.lightObjectives,
       contextInflation: aggregate.contextInflation ?? null,
+      outlierAnalysis: aggregate.outlierAnalysis ?? null,
     })),
   };
   await writeFile(path.join(aggregateDirectory, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
@@ -239,24 +307,32 @@ async function main() {
   const outputDirectory = path.resolve(options.output);
   const runSetId = options.runSetId ?? generatedRunSetId();
   const adapter = await loadAdapter(options.adapter);
-  const scenarios = await loadScenarios();
-  const runs = await executeRuns({ adapter, scenarios, target, runSetId, runs: options.runs });
-  const result = await writeResults({ outputDirectory, runSetId, scenarios, runs });
-  const output = {
-    status: "MEASURED",
-    runSetId,
-    scenarioCount: scenarios.length,
-    runCount: runs.length,
-    outputDirectory,
-    claimsAllowed: result.summary.claimsAllowed,
-  };
-  console.log(options.json ? JSON.stringify(output) : [
-    `Benchmark run set: ${runSetId}`,
-    `Scenarios: ${scenarios.length}`,
-    `Runs: ${runs.length}`,
-    `Claims allowed: ${result.summary.claimsAllowed ? "yes (observational only)" : "no"}`,
-    `Results: ${outputDirectory}`,
-  ].join("\n"));
+  try {
+    const scenarios = await loadScenarios();
+    const execution = await executeRuns({ adapter, scenarios, target, runSetId, runs: options.runs });
+    const result = await writeResults({ outputDirectory, runSetId, scenarios, runs: execution.runs, tier: options.tier });
+    const output = {
+      status: "MEASURED",
+      runSetId,
+      tier: options.tier,
+      scenarioCount: scenarios.length,
+      runCount: execution.runs.length,
+      outputDirectory,
+      claimsAllowed: result.summary.claimsAllowed,
+      qualityFinalization: execution.finalization,
+    };
+    console.log(options.json ? JSON.stringify(output) : [
+      `Benchmark run set: ${runSetId}`,
+      `Tier: ${options.tier ?? "custom"}`,
+      `Scenarios: ${scenarios.length}`,
+      `Runs: ${execution.runs.length}`,
+      `Claims allowed: ${result.summary.claimsAllowed ? "yes (observational only)" : "no"}`,
+      `Quality finalization: ${execution.finalization.status ?? "MEASURED"}`,
+      `Results: ${outputDirectory}`,
+    ].join("\n"));
+  } finally {
+    if (typeof adapter.cleanup === "function") await adapter.cleanup();
+  }
 }
 
 main().catch((error) => {