@cassiomc1/forgeloop 1.6.4 → 1.7.0

package/scripts/lib/execution-profile-benchmark-io.mjs

@@ -0,0 +1,67 @@
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+
+import { assertSchema, readSchema } from "../../src/core/schema-validation.js";
+import { getPackageRoot } from "../../src/core/templates.js";
+import {
+  assertBenchmarkScenario,
+  assertBenchmarkRun,
+  assertRequiredBenchmarkScenarios,
+} from "../../src/core/execution-profile-benchmarks.js";
+
+export async function readBenchmarkScenarios(repositoryRoot) {
+  const directory = path.join(repositoryRoot, "benchmarks", "execution-profiles");
+  const schema = await readSchema("execution-profile-benchmark-scenario", getPackageRoot());
+  const names = (await readdir(directory)).filter((name) => /^[a-z0-9-]+\.json$/u.test(name)).sort();
+  const scenarios = [];
+  for (const name of names) {
+    const scenario = JSON.parse(await readFile(path.join(directory, name), "utf8"));
+    assertSchema(scenario, schema, name);
+    scenarios.push(assertBenchmarkScenario(scenario));
+  }
+  return assertRequiredBenchmarkScenarios(scenarios);
+}
+
+async function readJsonFiles(directory) {
+  let entries;
+  try {
+    entries = await readdir(directory, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  }
+  const files = [];
+  for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+    const entryPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) files.push(...await readJsonFiles(entryPath));
+    else if (entry.isFile() && entry.name.endsWith(".json")) files.push(entryPath);
+  }
+  return files;
+}
+
+export async function readBenchmarkRunSets(resultsDirectory, runSetId = null) {
+  const rawDirectory = path.join(resultsDirectory, "raw");
+  let names;
+  try {
+    names = (await readdir(rawDirectory, { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory() && (runSetId === null || entry.name === runSetId))
+      .map((entry) => entry.name)
+      .sort();
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  }
+  const schema = await readSchema("execution-profile-benchmark-run", getPackageRoot());
+  const runSets = [];
+  for (const name of names) {
+    const files = await readJsonFiles(path.join(rawDirectory, name));
+    const runs = [];
+    for (const filename of files) {
+      const value = JSON.parse(await readFile(filename, "utf8"));
+      assertSchema(value, schema, filename);
+      runs.push(assertBenchmarkRun(value));
+    }
+    runSets.push({ runSetId: name, runs });
+  }
+  return runSets;
+}

package/scripts/run-execution-profile-benchmarks.mjs

@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+
+import { performance } from "node:perf_hooks";
+import { access, readFile, readdir, mkdir, writeFile } from "node:fs/promises";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import path from "node:path";
+
+import { evaluateRoute } from "../src/core/router.js";
+import { currentRepositoryFingerprint } from "../src/core/repository.js";
+import { getPackageRoot } from "../src/core/templates.js";
+import { assertSchema, readSchema } from "../src/core/schema-validation.js";
+import {
+  aggregateBenchmarkRuns,
+  BENCHMARK_MODES,
+  BENCHMARK_VERSION,
+  assertBenchmarkScenario,
+  assertRequiredBenchmarkScenarios,
+  createBenchmarkRun,
+} from "../src/core/execution-profile-benchmarks.js";
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const scenarioDirectory = path.join(repositoryRoot, "benchmarks", "execution-profiles");
+const defaultOutputDirectory = path.join(scenarioDirectory, "results");
+
+function usageError(message) {
+  const error = new Error(message);
+  error.code = "E_BENCHMARK_USAGE";
+  return error;
+}
+
+function parseArgs(argv) {
+  const options = {
+    target: process.cwd(),
+    adapter: null,
+    runs: 5,
+    runSetId: null,
+    output: defaultOutputDirectory,
+    json: false,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    const value = () => {
+      const next = argv[++index];
+      if (!next || next.startsWith("--")) throw usageError(`${argument} requires a value`);
+      return next;
+    };
+    if (argument === "--target") options.target = value();
+    else if (argument === "--adapter") options.adapter = value();
+    else if (argument === "--runs") options.runs = Number(value());
+    else if (argument === "--run-set") options.runSetId = value();
+    else if (argument === "--output") options.output = value();
+    else if (argument === "--json") options.json = true;
+    else if (argument === "--help" || argument === "-h") {
+      options.help = true;
+    } else throw usageError(`unknown option: ${argument}`);
+  }
+  if (options.help) return options;
+  if (!options.adapter) throw usageError("--adapter is required; ForgeLoop never invents provider or host measurements");
+  if (!Number.isInteger(options.runs) || options.runs < 1 || options.runs > 100) {
+    throw usageError("--runs must be an integer from 1 through 100");
+  }
+  if (options.runSetId !== null && !/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/u.test(options.runSetId)) {
+    throw usageError("--run-set must contain only portable identifier characters");
+  }
+  return options;
+}
+
+function helpText() {
+  return [
+    "Usage: npm run benchmark:profiles -- --adapter <module> [options]",
+    "",
+    "The adapter must execute each scenario and return actual provider/host usage, verification, comparable steps, and optional host-reported contextUsage.",
+    "Options: --target <path> --runs <1..100> --run-set <id> --output <directory> --json",
+  ].join("\n");
+}
+
+async function loadScenarios() {
+  const scenarioSchema = await readSchema("execution-profile-benchmark-scenario", getPackageRoot());
+  const names = (await readdir(scenarioDirectory)).filter((name) => /^[a-z0-9-]+\.json$/u.test(name)).sort();
+  const scenarios = [];
+  for (const name of names) {
+    const scenario = JSON.parse(await readFile(path.join(scenarioDirectory, name), "utf8"));
+    assertSchema(scenario, scenarioSchema, name);
+    scenarios.push(assertBenchmarkScenario(scenario));
+  }
+  try {
+    return assertRequiredBenchmarkScenarios(scenarios);
+  } catch (error) {
+    throw usageError(error.message);
+  }
+}
+
+async function loadAdapter(adapterSpecifier) {
+  const adapterPath = path.resolve(process.cwd(), adapterSpecifier);
+  const adapterModule = await import(pathToFileURL(adapterPath).href);
+  const adapter = adapterModule.runBenchmark ?? adapterModule.default?.runBenchmark ?? adapterModule.default;
+  if (typeof adapter !== "function") {
+    throw usageError("benchmark adapter must export runBenchmark(input) or be a function default export");
+  }
+  return adapter;
+}
+
+function generatedRunSetId() {
+  return new Date().toISOString().replace(/[-:.]/gu, "");
+}
+
+function environmentMetadata(adapterMetadata = {}) {
+  return {
+    environmentClass: adapterMetadata.environmentClass ?? `${process.platform}-node${process.versions.node.split(".")[0]}`,
+    nodeVersion: adapterMetadata.nodeVersion ?? process.versions.node,
+    os: adapterMetadata.os ?? process.platform,
+    arch: adapterMetadata.arch ?? process.arch,
+  };
+}
+
+function profileForMode(scenario, mode) {
+  if (mode === "direct") return { requestedProfile: null, resolvedProfile: null };
+  const requestedProfile = mode === "forgeloopBalanced" ? "balanced" : "auto";
+  const route = evaluateRoute(scenario.input, { requestedProfile });
+  return { requestedProfile, resolvedProfile: route.executionProfile.resolved };
+}
+
+function normalizedAdapterMetadata(response, target, repository, scenario, mode) {
+  const adapterMetadata = response.metadata && typeof response.metadata === "object"
+    ? response.metadata
+    : {};
+  const profile = profileForMode(scenario, mode);
+  const environment = environmentMetadata(adapterMetadata);
+  return {
+    ...environment,
+    model: response.usage?.model ?? adapterMetadata.model ?? null,
+    provider: response.usage?.provider ?? adapterMetadata.provider ?? null,
+    promptSpecFingerprint: response.promptSpecFingerprint ?? adapterMetadata.promptSpecFingerprint ?? null,
+    // A Git revision is derived from the target rather than trusted from the adapter.
+    projectRevision: repository.head ?? adapterMetadata.projectRevision ?? null,
+    requestedProfile: profile.requestedProfile,
+    resolvedProfile: profile.resolvedProfile,
+    targetPath: target,
+  };
+}
+
+async function executeRuns({ adapter, scenarios, target, runSetId, runs }) {
+  const repository = await currentRepositoryFingerprint(target);
+  const allRuns = [];
+  for (const scenario of scenarios) {
+    for (const mode of BENCHMARK_MODES) {
+      for (let runIndex = 1; runIndex <= runs; runIndex += 1) {
+        const started = performance.now();
+        const response = await adapter({
+          benchmarkVersion: BENCHMARK_VERSION,
+          scenario: structuredClone(scenario),
+          mode,
+          runIndex,
+          target,
+          projectRevision: repository.head ?? null,
+          // The adapter owns execution; the runner owns elapsed-time measurement.
+        });
+        const wallClockMs = Number((performance.now() - started).toFixed(4));
+        if (!response || typeof response !== "object" || Array.isArray(response)) {
+          throw usageError(`${scenario.scenarioId}/${mode}/${runIndex}: adapter must return an object`);
+        }
+        const run = createBenchmarkRun({
+          runSetId,
+          runId: `run-${scenario.scenarioId}-${mode}-${String(runIndex).padStart(3, "0")}`,
+          runIndex,
+          scenario,
+          mode,
+          usage: response.usage ?? {},
+          wallClockMs,
+          verification: response.verification ?? "NOT_AVAILABLE",
+          verificationCycles: response.verificationCycles ?? null,
+          comparableSteps: response.comparableSteps ?? null,
+          contextUsage: response.contextUsage,
+          quality: response.quality,
+          metadata: normalizedAdapterMetadata(response, target, repository, scenario, mode),
+        });
+        allRuns.push(run);
+      }
+    }
+  }
+  return allRuns;
+}
+
+async function writeResults({ outputDirectory, runSetId, scenarios, runs }) {
+  const rawDirectory = path.join(outputDirectory, "raw", runSetId);
+  const aggregateDirectory = path.join(outputDirectory, "aggregate", runSetId);
+  try {
+    await access(rawDirectory);
+    throw usageError(`run set already exists and will not be overwritten: ${runSetId}`);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
+  try {
+    await access(aggregateDirectory);
+    throw usageError(`aggregate run set already exists and will not be overwritten: ${runSetId}`);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
+  await mkdir(rawDirectory, { recursive: true });
+  await mkdir(aggregateDirectory, { recursive: true });
+  const aggregates = [];
+  for (const scenario of scenarios) {
+    const scenarioRuns = runs.filter((run) => run.scenarioId === scenario.scenarioId);
+    const aggregate = aggregateBenchmarkRuns({ scenario, runs: scenarioRuns });
+    aggregates.push(aggregate);
+    for (const run of scenarioRuns) {
+      const directory = path.join(rawDirectory, scenario.scenarioId, run.mode);
+      await mkdir(directory, { recursive: true });
+      await writeFile(path.join(directory, `run-${String(run.runIndex).padStart(3, "0")}.json`), `${JSON.stringify(run, null, 2)}\n`, "utf8");
+    }
+    await writeFile(path.join(aggregateDirectory, `${scenario.scenarioId}.json`), `${JSON.stringify(aggregate, null, 2)}\n`, "utf8");
+  }
+  const summary = {
+    schemaVersion: 1,
+    benchmarkVersion: BENCHMARK_VERSION,
+    runSetId,
+    scenarioCount: scenarios.length,
+    runCount: runs.length,
+    claimsAllowed: aggregates.some((aggregate) => aggregate.claimsAllowed),
+    scenarios: aggregates.map((aggregate) => ({
+      scenarioId: aggregate.scenarioId,
+      expectedProfile: aggregate.expectedProfile,
+      claimsAllowed: aggregate.claimsAllowed,
+      lightObjectives: aggregate.lightObjectives,
+      contextInflation: aggregate.contextInflation ?? null,
+    })),
+  };
+  await writeFile(path.join(aggregateDirectory, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+  return { rawDirectory, aggregateDirectory, summary };
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    console.log(helpText());
+    return;
+  }
+  const target = path.resolve(options.target);
+  const outputDirectory = path.resolve(options.output);
+  const runSetId = options.runSetId ?? generatedRunSetId();
+  const adapter = await loadAdapter(options.adapter);
+  const scenarios = await loadScenarios();
+  const runs = await executeRuns({ adapter, scenarios, target, runSetId, runs: options.runs });
+  const result = await writeResults({ outputDirectory, runSetId, scenarios, runs });
+  const output = {
+    status: "MEASURED",
+    runSetId,
+    scenarioCount: scenarios.length,
+    runCount: runs.length,
+    outputDirectory,
+    claimsAllowed: result.summary.claimsAllowed,
+  };
+  console.log(options.json ? JSON.stringify(output) : [
+    `Benchmark run set: ${runSetId}`,
+    `Scenarios: ${scenarios.length}`,
+    `Runs: ${runs.length}`,
+    `Claims allowed: ${result.summary.claimsAllowed ? "yes (observational only)" : "no"}`,
+    `Results: ${outputDirectory}`,
+  ].join("\n"));
+}
+
+main().catch((error) => {
+  console.error(`ForgeLoop benchmark runner: ${error.message}`);
+  process.exitCode = 1;
+});

package/scripts/summarize-execution-profile-benchmarks.mjs

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { aggregateBenchmarkRuns, BENCHMARK_VERSION } from "../src/core/execution-profile-benchmarks.js";
+import { readBenchmarkRunSets, readBenchmarkScenarios } from "./lib/execution-profile-benchmark-io.mjs";
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const defaultResultsDirectory = path.join(repositoryRoot, "benchmarks", "execution-profiles", "results");
+
+function parseArgs(argv) {
+  const options = { results: defaultResultsDirectory, runSetId: null, json: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === "--json") options.json = true;
+    else if (["--results", "--run-set"].includes(argument)) {
+      const value = argv[++index];
+      if (!value || value.startsWith("--")) throw new Error(`${argument} requires a value`);
+      if (argument === "--results") options.results = value;
+      else options.runSetId = value;
+    } else if (argument === "--help" || argument === "-h") options.help = true;
+    else throw new Error(`unknown option: ${argument}`);
+  }
+  return options;
+}
+
+function helpText() {
+  return "Usage: npm run benchmark:profiles:summary -- [--results <directory>] [--run-set <id>] [--json]";
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    console.log(helpText());
+    return;
+  }
+  const scenarios = await readBenchmarkScenarios(repositoryRoot);
+  const runSets = await readBenchmarkRunSets(path.resolve(options.results), options.runSetId);
+  const summaries = [];
+  for (const runSet of runSets) {
+    const aggregates = scenarios
+      .map((scenario) => {
+        const runs = runSet.runs.filter((run) => run.scenarioId === scenario.scenarioId);
+        return runs.length > 0 ? aggregateBenchmarkRuns({ scenario, runs }) : null;
+      });
+    summaries.push({
+      runSetId: runSet.runSetId,
+      runCount: runSet.runs.length,
+      scenarioCount: aggregates.filter(Boolean).length,
+      claimsAllowed: aggregates.some((aggregate) => aggregate?.claimsAllowed === true),
+      scenarios: aggregates.filter(Boolean).map((aggregate) => ({
+        scenarioId: aggregate.scenarioId,
+        expectedProfile: aggregate.expectedProfile,
+        claimsAllowed: aggregate.claimsAllowed,
+        comparisons: aggregate.comparisons,
+        lightObjectives: aggregate.lightObjectives,
+        modeAggregates: aggregate.modeAggregates,
+        contextInflation: aggregate.contextInflation ?? null,
+      })),
+    });
+  }
+  const output = {
+    schemaVersion: 1,
+    benchmarkVersion: BENCHMARK_VERSION,
+    status: summaries.length === 0 ? "NOT_MEASURED" : "MEASURED",
+    claimsAllowed: summaries.some((summary) => summary.claimsAllowed),
+    scenarioCount: scenarios.length,
+    runSets: summaries,
+    claimPolicy: "No efficiency claim is allowed when trusted measurements are unavailable or non-comparable.",
+  };
+  if (options.json) console.log(JSON.stringify(output));
+  else {
+    console.log(`Benchmark status: ${output.status}`);
+    console.log(`Run sets: ${summaries.length}`);
+    console.log(`Claims allowed: ${output.claimsAllowed ? "yes (observational only)" : "no"}`);
+    if (summaries.length === 0) console.log("No provider or host benchmark measurements are present.");
+  }
+}
+
+main().catch((error) => {
+  console.error(`ForgeLoop benchmark summary: ${error.message}`);
+  process.exitCode = 1;
+});

package/scripts/validate-execution-profile-benchmarks.mjs

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { assertSchema, readSchema } from "../src/core/schema-validation.js";
+import { getPackageRoot } from "../src/core/templates.js";
+import { aggregateBenchmarkRuns, BENCHMARK_MODES } from "../src/core/execution-profile-benchmarks.js";
+import { readBenchmarkRunSets, readBenchmarkScenarios } from "./lib/execution-profile-benchmark-io.mjs";
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const defaultResultsDirectory = path.join(repositoryRoot, "benchmarks", "execution-profiles", "results");
+
+function parseArgs(argv) {
+  const options = { results: defaultResultsDirectory, json: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === "--json") options.json = true;
+    else if (argument === "--results") {
+      const value = argv[++index];
+      if (!value || value.startsWith("--")) throw new Error("--results requires a value");
+      options.results = value;
+    } else if (argument === "--help" || argument === "-h") options.help = true;
+    else throw new Error(`unknown option: ${argument}`);
+  }
+  return options;
+}
+
+function assertCompleteRunSet(scenario, runs) {
+  if (runs.length === 0) throw new Error(`${scenario.scenarioId}: no raw runs found`);
+  const byMode = Object.fromEntries(BENCHMARK_MODES.map((mode) => [mode, runs.filter((run) => run.mode === mode)]));
+  const counts = new Set(BENCHMARK_MODES.map((mode) => byMode[mode].length));
+  if (counts.size !== 1 || counts.has(0)) throw new Error(`${scenario.scenarioId}: every mode must have the same non-zero run count`);
+  const expectedIndexes = byMode.direct.map((run) => run.runIndex).sort((a, b) => a - b);
+  for (const mode of BENCHMARK_MODES) {
+    const indexes = byMode[mode].map((run) => run.runIndex).sort((a, b) => a - b);
+    if (JSON.stringify(indexes) !== JSON.stringify(expectedIndexes)) {
+      throw new Error(`${scenario.scenarioId}: ${mode} run indexes do not match direct baseline`);
+    }
+  }
+  return aggregateBenchmarkRuns({ scenario, runs });
+}
+
+async function validateStoredAggregates(resultsDirectory, runSetId, scenarios, expectedAggregates) {
+  const schema = await readSchema("execution-profile-benchmark-aggregate", getPackageRoot());
+  const aggregateDirectory = path.join(resultsDirectory, "aggregate", runSetId);
+  const aggregateEntries = await readdir(aggregateDirectory, { withFileTypes: true });
+  const expectedNames = new Set(scenarios.map((scenario) => `${scenario.scenarioId}.json`));
+  for (const entry of aggregateEntries) {
+    if (entry.isFile() && entry.name.endsWith(".json") && entry.name !== "summary.json" && !expectedNames.has(entry.name)) {
+      throw new Error(`${runSetId}: aggregate has an unexpected scenario file: ${entry.name}`);
+    }
+  }
+  for (const aggregate of expectedAggregates) {
+    const filename = path.join(resultsDirectory, "aggregate", runSetId, `${aggregate.scenarioId}.json`);
+    const stored = JSON.parse(await readFile(filename, "utf8"));
+    assertSchema(stored, schema, filename);
+    if (JSON.stringify(stored) !== JSON.stringify(aggregate)) {
+      throw new Error(`${aggregate.scenarioId}: stored aggregate does not reproduce raw measurements`);
+    }
+  }
+  const summaryPath = path.join(resultsDirectory, "aggregate", runSetId, "summary.json");
+  const summary = JSON.parse(await readFile(summaryPath, "utf8"));
+  if (summary.runSetId !== runSetId
+    || summary.scenarioCount !== scenarios.length
+    || summary.runCount !== expectedAggregates.reduce((sum, aggregate) => sum + aggregate.generatedFromRunCount, 0)
+    || summary.claimsAllowed !== expectedAggregates.some((aggregate) => aggregate.claimsAllowed)) {
+    throw new Error(`${runSetId}: aggregate summary identity is inconsistent`);
+  }
+}
+
+async function readAggregateRunSetNames(resultsDirectory) {
+  try {
+    return (await readdir(path.join(resultsDirectory, "aggregate"), { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+      .sort();
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  }
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    console.log("Usage: npm run benchmark:profiles:check -- [--results <directory>] [--json]");
+    return;
+  }
+  const scenarios = await readBenchmarkScenarios(repositoryRoot);
+  const resultsDirectory = path.resolve(options.results);
+  const runSets = await readBenchmarkRunSets(resultsDirectory);
+  const aggregateRunSetNames = await readAggregateRunSetNames(resultsDirectory);
+  const rawRunSetNames = new Set(runSets.map((runSet) => runSet.runSetId));
+  for (const name of aggregateRunSetNames) {
+    if (!rawRunSetNames.has(name)) throw new Error(`${name}: aggregate exists without raw benchmark history`);
+  }
+  if (runSets.length === 0) {
+    const output = { status: "VALID", benchmarkStatus: "NOT_MEASURED", scenarioCount: scenarios.length, runSets: 0 };
+    console.log(options.json ? JSON.stringify(output) : "Benchmark schemas valid; no measured run sets are present.");
+    return;
+  }
+  const results = [];
+  for (const runSet of runSets) {
+    const aggregates = scenarios.map((scenario) => assertCompleteRunSet(
+      scenario,
+      runSet.runs.filter((run) => run.scenarioId === scenario.scenarioId),
+    ));
+    await validateStoredAggregates(resultsDirectory, runSet.runSetId, scenarios, aggregates);
+    results.push({ runSetId: runSet.runSetId, runCount: runSet.runs.length, claimsAllowed: aggregates.some((aggregate) => aggregate.claimsAllowed) });
+  }
+  const output = { status: "VALID", benchmarkStatus: "MEASURED", scenarioCount: scenarios.length, runSets: results };
+  console.log(options.json ? JSON.stringify(output) : `Benchmark schemas and aggregates valid for ${results.length} run set(s).`);
+}
+
+main().catch((error) => {
+  console.error(`ForgeLoop benchmark validation: ${error.message}`);
+  process.exitCode = 1;
+});

package/src/cli.js

@@ -35,6 +35,8 @@ import { formatActionVerifyResult } from "./commands/action-verify.js";
 import { formatActionShowResult } from "./commands/action-show.js";
 import { formatActionReconcileResult } from "./commands/action-reconcile.js";
 import { formatMetricsResult } from "./commands/metrics.js";
+import { formatUsageRecordResult } from "./commands/usage-record.js";
+import { formatEfficiencyResult } from "./commands/efficiency.js";
 import { formatEvalResult } from "./commands/eval.js";
 import { formatApprovalRequestResult } from "./commands/approval-request.js";
 import { formatApprovalResolveResult } from "./commands/approval-resolve.js";
@@ -48,14 +50,14 @@ import { formatTraceResult } from "./commands/trace.js";
 import { formatReflectResult } from "./commands/reflect.js";
 import { formatProgressResult } from "./commands/progress.js";
 import { formatRecordDecisionCriterionResult } from "./commands/record-decision-criterion.js";
-import { formatNextActionResult } from "./commands/next.js";
+import { formatNextActionResult, formatCompactNextActionResult } from "./commands/next.js";
 import { formatContinuityResult } from "./commands/continuity.js";
 import { formatRecordContinuityResult } from "./commands/record-continuity.js";
 import { formatReconcileContinuityResult } from "./commands/reconcile-continuity.js";
 import { formatClearContinuityResult } from "./commands/clear-continuity.js";
 import { formatTaskCreateResult } from "./commands/task-create.js";
 import { formatTaskListResult } from "./commands/task-list.js";
-import { formatTaskShowResult } from "./commands/task-show.js";
+import { formatTaskShowResult, formatCompactTaskShowResult } from "./commands/task-show.js";
 import { formatTaskScopeResult } from "./commands/task-scope.js";
 import { formatTaskMigrateResult } from "./commands/task-migrate.js";
 import { formatMigrateProtocolResult } from "./commands/migrate-protocol.js";
@@ -384,7 +386,8 @@ export const COMMAND_HANDLERS = Object.freeze({
   },
   next: async ({ target, packageRoot, options }) => {
     const { result } = await COMMAND_EXECUTORS.next({ target, packageRoot, options });
-    renderJsonOr(options, result, formatNextActionResult);
+    if (options.compact) console.log(formatCompactNextActionResult(result));
+    else renderJsonOr(options, result, formatNextActionResult);
     return 0;
   },
   continuity: async ({ target, packageRoot, options }) => {
@@ -510,6 +513,14 @@ export const COMMAND_HANDLERS = Object.freeze({
     const { result } = await COMMAND_EXECUTORS.metrics({ target, packageRoot, options });
     renderJsonOr(options, result, formatMetricsResult); return 0;
   },
+  "usage-record": async ({ target, packageRoot, options }) => {
+    const { result } = await COMMAND_EXECUTORS["usage-record"]({ target, packageRoot, options });
+    renderJsonOr(options, result, formatUsageRecordResult); return 0;
+  },
+  efficiency: async ({ target, packageRoot, options }) => {
+    const { result } = await COMMAND_EXECUTORS.efficiency({ target, packageRoot, options });
+    renderJsonOr(options, result, formatEfficiencyResult); return 0;
+  },
   eval: async ({ target, packageRoot, options }) => {
     const { result, exitCode } = await COMMAND_EXECUTORS.eval({ target, packageRoot, options });
     renderJsonOr(options, result, formatEvalResult); return exitCode;
@@ -677,7 +688,8 @@ export const COMMAND_HANDLERS = Object.freeze({
   },
   "task-show": async ({ target, packageRoot, options }) => {
     const { result } = await COMMAND_EXECUTORS["task-show"]({ target, packageRoot, options });
-    renderJsonOr(options, result, formatTaskShowResult);
+    if (options.compact) console.log(formatCompactTaskShowResult(result));
+    else renderJsonOr(options, result, formatTaskShowResult);
     return 0;
   },
   "task-lock-status": async ({ target, packageRoot, options }) => {

package/src/commands/efficiency.js

@@ -0,0 +1,12 @@
+import { buildEfficiencyReport } from "../core/efficiency.js";
+import { withResolvedTask } from "../core/task-command.js";
+
+export async function runEfficiency({ target, packageRoot, taskId, baselinePath = null, runtimeContext = null }) {
+  return withResolvedTask(target, { taskId, packageRoot, explicitRequired: true }, (ctx) =>
+    buildEfficiencyReport({ target, packageRoot, taskId: ctx.taskId, baselinePath, runtimeContext }));
+}
+
+export function formatEfficiencyResult(result) {
+  return `${JSON.stringify(result, null, 2)}\n`;
+}
+

package/src/commands/eval.js

@@ -1,6 +1,12 @@
 import { evaluateTrajectory } from "../core/trajectory-evaluation.js";
 import { withResolvedTask } from "../core/task-command.js";
-export async function runEval({ target, packageRoot, taskId, scenarioPath }) {
-  return withResolvedTask(target, { taskId, packageRoot }, (ctx) => evaluateTrajectory({ target, packageRoot, taskId: ctx.taskId, scenarioPath }));
+export async function runEval({ target, packageRoot, taskId, scenarioPath, runtimeContext = null }) {
+  return withResolvedTask(target, { taskId, packageRoot }, (ctx) => evaluateTrajectory({
+    target,
+    packageRoot,
+    taskId: ctx.taskId,
+    scenarioPath,
+    runtimeContext,
+  }));
 }
 export function formatEvalResult(result) { return `${JSON.stringify(result, null, 2)}\n`; }

package/src/commands/metrics.js

@@ -1,7 +1,7 @@
 import { buildTrajectoryMetrics } from "../core/trajectory-metrics.js";
 import { withResolvedTask } from "../core/task-command.js";
-export async function runMetrics({ target, packageRoot, taskId }) {
+export async function runMetrics({ target, packageRoot, taskId, runtimeContext = null }) {
   return withResolvedTask(target, { taskId, packageRoot }, (ctx) =>
-    buildTrajectoryMetrics({ target, packageRoot, taskId: ctx.taskId }));
+    buildTrajectoryMetrics({ target, packageRoot, taskId: ctx.taskId, runtimeContext }));
 }
 export function formatMetricsResult(result) { return `${JSON.stringify(result, null, 2)}\n`; }