@cassiomc1/forgeloop 1.6.5 → 1.8.0

package/schemas/execution-profile-benchmark-aggregate.schema.json

@@ -0,0 +1,93 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "forgeloop://schemas/execution-profile-benchmark-aggregate.schema.json",
+  "title": "ForgeLoop execution-profile benchmark aggregate",
+  "type": "object",
+  "required": ["schemaVersion", "benchmarkVersion", "runSetId", "scenarioId", "expectedProfile", "modeAggregates", "comparisons", "lightObjectives", "sourcePolicy", "claimsAllowed", "generatedFromRunCount"],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "benchmarkVersion": { "enum": ["1", "2"] },
+    "runSetId": { "type": "string", "minLength": 1 },
+    "scenarioId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,127}$" },
+    "expectedProfile": { "enum": ["light", "balanced", "full"] },
+    "outlierAnalysis": { "$ref": "#/$defs/outlierAnalysis" },
+    "modeAggregates": {
+      "type": "object",
+      "required": ["direct", "forgeloopBalanced", "forgeloopAdaptive"],
+      "additionalProperties": { "type": "object" }
+    },
+    "comparisons": {
+      "type": "object",
+      "required": ["direct", "forgeloopBalanced", "forgeloopAdaptive"],
+      "additionalProperties": true
+    },
+    "lightObjectives": { "type": ["object", "null"] },
+    "contextInflation": {
+      "type": "object",
+      "required": ["status", "comparablePairs", "inflatedPairs", "lightContextTokens", "balancedContextTokens", "blocking", "reason"],
+      "properties": {
+        "status": { "enum": ["CONTEXT_INFLATION", "NOT_DETECTED", "NOT_COMPARABLE"] },
+        "comparablePairs": { "type": "integer", "minimum": 0 },
+        "inflatedPairs": { "type": "integer", "minimum": 0 },
+        "lightContextTokens": { "type": "object" },
+        "balancedContextTokens": { "type": "object" },
+        "blocking": { "const": false },
+        "reason": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    },
+    "sourcePolicy": { "const": "PROVIDER_REPORTED_OR_HOST_REPORTED_ONLY" },
+    "claimsAllowed": { "type": "boolean" },
+    "generatedFromRunCount": { "type": "integer", "minimum": 1 }
+  },
+  "$defs": {
+    "outlierAnalysis": {
+      "type": "object",
+      "required": ["policy", "minimumSamples", "modes"],
+      "properties": {
+        "policy": { "const": "TOKEN_IQR_1_5" },
+        "minimumSamples": { "type": "integer", "minimum": 1 },
+        "modes": {
+          "type": "object",
+          "required": ["direct", "forgeloopBalanced", "forgeloopAdaptive"],
+          "additionalProperties": { "$ref": "#/$defs/modeOutlierAnalysis" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "modeOutlierAnalysis": {
+      "type": "object",
+      "required": ["sampleCount", "measuredCount", "status", "q1", "q3", "iqr", "median", "upperFence", "outliers"],
+      "properties": {
+        "sampleCount": { "type": "integer", "minimum": 0 },
+        "measuredCount": { "type": "integer", "minimum": 0 },
+        "status": { "enum": ["MEASURED", "NOT_ENOUGH_SAMPLES"] },
+        "q1": { "type": ["number", "null"] },
+        "q3": { "type": ["number", "null"] },
+        "iqr": { "type": ["number", "null"] },
+        "median": { "type": ["number", "null"] },
+        "upperFence": { "type": ["number", "null"] },
+        "outliers": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/outlierRecord" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "outlierRecord": {
+      "type": "object",
+      "required": ["runId", "runIndex", "totalTokens", "scenarioMedianTokens", "ratioToMedian", "reasons", "diagnosticSignals"],
+      "properties": {
+        "runId": { "type": "string", "pattern": "^run-[A-Za-z0-9][A-Za-z0-9_-]{0,127}$" },
+        "runIndex": { "type": "integer", "minimum": 1 },
+        "totalTokens": { "type": "integer", "minimum": 0 },
+        "scenarioMedianTokens": { "type": ["number", "null"] },
+        "ratioToMedian": { "type": ["number", "null"] },
+        "reasons": { "type": "array", "items": { "const": "TOKEN_IQR_OUTLIER" } },
+        "diagnosticSignals": { "type": "array", "items": { "type": "string", "minLength": 1 } }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/execution-profile-benchmark-run.schema.json

@@ -0,0 +1,144 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "forgeloop://schemas/execution-profile-benchmark-run.schema.json",
+  "title": "ForgeLoop execution-profile benchmark run",
+  "type": "object",
+  "required": ["schemaVersion", "benchmarkVersion", "runSetId", "runId", "scenarioId", "mode", "runIndex", "recordedAt", "usage", "wallClockMs", "verification", "verificationCycles", "comparableSteps", "metadata"],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "benchmarkVersion": { "enum": ["1", "2"] },
+    "runSetId": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" },
+    "runId": { "type": "string", "pattern": "^run-[A-Za-z0-9][A-Za-z0-9_-]{0,127}$" },
+    "scenarioId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,127}$" },
+    "mode": { "enum": ["direct", "forgeloopBalanced", "forgeloopAdaptive"] },
+    "runIndex": { "type": "integer", "minimum": 1 },
+    "recordedAt": { "type": "string", "minLength": 1 },
+    "usage": {
+      "type": "object",
+      "required": ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "totalTokens", "costUsd", "model", "provider", "source"],
+      "properties": {
+        "inputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "cacheReadTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "cacheWriteTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "totalTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "costUsd": { "type": ["number", "null"], "minimum": 0 },
+        "model": { "type": ["string", "null"] },
+        "provider": { "type": ["string", "null"] },
+        "source": { "enum": ["PROVIDER_REPORTED", "HOST_REPORTED", "UNKNOWN"] }
+      },
+      "additionalProperties": false
+    },
+    "wallClockMs": { "type": ["number", "null"], "minimum": 0 },
+    "verification": { "enum": ["PASS", "FAIL", "NOT_AVAILABLE"] },
+    "verificationCycles": { "type": ["integer", "null"], "minimum": 0 },
+    "comparableSteps": { "type": ["integer", "null"], "minimum": 0 },
+    "diagnostics": { "$ref": "#/$defs/diagnostics" },
+    "contextUsage": { "$ref": "#/$defs/contextUsage" },
+    "quality": { "$ref": "#/$defs/quality" },
+    "metadata": {
+      "type": "object",
+      "required": ["scenarioId", "mode", "model", "provider", "promptSpecFingerprint", "projectRevision", "benchmarkVersion", "environmentClass", "requestedProfile", "resolvedProfile", "verificationCycles", "comparableSteps"],
+      "properties": {
+        "scenarioId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,127}$" },
+        "mode": { "enum": ["direct", "forgeloopBalanced", "forgeloopAdaptive"] },
+        "model": { "type": ["string", "null"] },
+        "provider": { "type": ["string", "null"] },
+        "promptSpecFingerprint": { "type": "string", "minLength": 1 },
+        "projectRevision": { "type": ["string", "null"] },
+        "benchmarkVersion": { "enum": ["1", "2"] },
+        "environmentClass": { "type": "string", "minLength": 1 },
+        "requestedProfile": { "type": ["string", "null"] },
+        "resolvedProfile": { "type": ["string", "null"] },
+        "verificationCycles": { "type": ["integer", "null"], "minimum": 0 },
+        "comparableSteps": { "type": ["integer", "null"], "minimum": 0 },
+        "nodeVersion": { "type": "string", "minLength": 1 },
+        "os": { "type": "string", "minLength": 1 },
+        "arch": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    }
+  },
+  "$defs": {
+    "diagnostics": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "executionProfile": { "enum": ["light", "balanced", "full", null] },
+        "verificationCycles": { "type": ["integer", "null"], "minimum": 0 },
+        "modelTurns": { "type": ["integer", "null"], "minimum": 0 },
+        "toolCalls": { "type": ["integer", "null"], "minimum": 0 },
+        "retries": { "type": ["integer", "null"], "minimum": 0 },
+        "correctionCycles": { "type": ["integer", "null"], "minimum": 0 },
+        "filesRead": { "type": ["integer", "null"], "minimum": 0 },
+        "filesWritten": { "type": ["integer", "null"], "minimum": 0 },
+        "contextRefreshes": { "type": ["integer", "null"], "minimum": 0 },
+        "guideCount": { "type": ["integer", "null"], "minimum": 0 },
+        "guideIds": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "hostWarnings": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "terminationReason": { "type": ["string", "null"], "minLength": 1 },
+        "flags": {
+          "type": "array",
+          "items": {
+            "enum": [
+              "EXCESSIVE_MODEL_TURNS",
+              "EXCESSIVE_TOOL_CALLS",
+              "EXCESSIVE_VERIFICATION_CYCLES",
+              "REPEATED_CONTEXT_REFRESH",
+              "REPEATED_FILE_READ",
+              "REPEATED_GUIDE_LOAD",
+              "UNEXPECTED_RETRY",
+              "CORRECTION_LOOP",
+              "HOST_RETRY",
+              "MODEL_STALL",
+              "UNKNOWN_TOKEN_SPIKE"
+            ]
+          }
+        }
+      }
+    },
+    "contextUsage": {
+      "type": "object",
+      "required": ["source", "profile", "items"],
+      "properties": {
+        "source": { "enum": ["HOST_REPORTED", "UNKNOWN"] },
+        "profile": { "enum": ["light", "balanced", "full", null] },
+        "items": {
+          "type": "object",
+          "required": ["taskContext", "guides", "history", "protocolInstructions", "repositoryContext", "other"],
+          "properties": {
+            "taskContext": { "type": ["integer", "null"], "minimum": 0 },
+            "guides": { "type": ["integer", "null"], "minimum": 0 },
+            "history": { "type": ["integer", "null"], "minimum": 0 },
+            "protocolInstructions": { "type": ["integer", "null"], "minimum": 0 },
+            "repositoryContext": { "type": ["integer", "null"], "minimum": 0 },
+            "other": { "type": ["integer", "null"], "minimum": 0 }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "quality": {
+      "type": "object",
+      "required": ["source", "scores"],
+      "properties": {
+        "source": { "enum": ["EXTERNAL_REPORTED", "HOST_REPORTED", "UNKNOWN"] },
+        "scores": {
+          "type": "object",
+          "required": ["visualQuality", "responsiveQuality", "accessibility", "interactionPolish", "requirementsCompleteness"],
+          "properties": {
+            "visualQuality": { "type": ["number", "null"], "minimum": 0, "maximum": 5 },
+            "responsiveQuality": { "type": ["number", "null"], "minimum": 0, "maximum": 5 },
+            "accessibility": { "type": ["number", "null"], "minimum": 0, "maximum": 5 },
+            "interactionPolish": { "type": ["number", "null"], "minimum": 0, "maximum": 5 },
+            "requirementsCompleteness": { "type": ["number", "null"], "minimum": 0, "maximum": 5 }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/execution-profile-benchmark-scenario.schema.json

@@ -0,0 +1,66 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "forgeloop://schemas/execution-profile-benchmark-scenario.schema.json",
+  "title": "ForgeLoop execution-profile benchmark scenario",
+  "type": "object",
+  "required": ["schemaVersion", "benchmarkVersion", "scenarioId", "description", "input", "expectedProfile", "measurements"],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "benchmarkVersion": { "type": "string", "minLength": 1 },
+    "scenarioId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,127}$" },
+    "description": { "type": "string", "minLength": 1 },
+    "input": {
+      "type": "object",
+      "required": ["workType", "surfaces", "risks", "platforms"],
+      "properties": {
+        "workType": { "type": "string", "minLength": 1 },
+        "surfaces": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "risks": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "platforms": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "behaviorChange": { "type": "boolean" },
+        "executableChange": { "type": "boolean" }
+      },
+      "additionalProperties": false
+    },
+    "expectedProfile": { "enum": ["light", "balanced", "full"] },
+    "referenceTask": { "$ref": "#/$defs/referenceTask" },
+    "measurements": {
+      "type": "object",
+      "required": ["direct", "forgeloopBalanced", "forgeloopAdaptive"],
+      "properties": {
+        "direct": { "$ref": "#/$defs/measurement" },
+        "forgeloopBalanced": { "$ref": "#/$defs/measurement" },
+        "forgeloopAdaptive": { "$ref": "#/$defs/measurement" }
+      },
+      "additionalProperties": false
+    }
+  },
+  "$defs": {
+    "referenceTask": {
+      "type": "object",
+      "required": ["name", "requirements", "exclusions", "artifactMode"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "requirements": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1 },
+        "exclusions": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1 },
+        "artifactMode": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    },
+    "measurement": {
+      "type": "object",
+      "required": ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "totalTokens", "wallClockMs", "verification"],
+      "properties": {
+        "inputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "cacheReadTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "cacheWriteTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "totalTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "wallClockMs": { "type": ["number", "null"], "minimum": 0 },
+        "verification": { "enum": ["PASS", "FAIL", "NOT_AVAILABLE"] }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/routing-result.schema.json

@@ -8,6 +8,18 @@
     "schemaVersion": { "const": 1 },
     "protocolVersion": { "const": 1 },
     "contractFingerprint": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "executionProfile": {
+      "type": "object",
+      "required": ["requested", "floor", "resolved", "reasons", "escalated"],
+      "additionalProperties": false,
+      "properties": {
+        "requested": { "enum": ["auto", "light", "balanced", "full"] },
+        "floor": { "enum": ["light", "balanced", "full"] },
+        "resolved": { "enum": ["light", "balanced", "full"] },
+        "reasons": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } },
+        "escalated": { "type": "boolean" }
+      }
+    },
     "input": { "type": "object" },
     "primary": { "oneOf": [{ "type": "string" }, { "type": "null" }] },
     "guides": { "type": "array", "items": { "type": "string" } },

package/schemas/usage.schema.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "forgeloop://schemas/usage.schema.json",
+  "title": "ForgeLoop task usage telemetry",
+  "type": "object",
+  "required": ["schemaVersion", "protocolVersion", "taskId", "recordedAt", "usage"],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "protocolVersion": { "const": 1 },
+    "taskId": { "type": "string", "minLength": 1 },
+    "recordedAt": { "type": "string", "minLength": 1 },
+    "usage": {
+      "type": "object",
+      "required": ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "totalTokens", "costUsd", "model", "provider", "source"],
+      "additionalProperties": false,
+      "properties": {
+        "inputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "cacheReadTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "cacheWriteTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "totalTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "costUsd": { "type": ["number", "null"], "minimum": 0 },
+        "model": { "type": ["string", "null"] },
+        "provider": { "type": ["string", "null"] },
+        "source": { "enum": ["PROVIDER_REPORTED", "HOST_REPORTED", "ACTOR_REPORTED", "UNKNOWN"] }
+      }
+    }
+  },
+  "additionalProperties": false
+}

package/scripts/check-efficiency-regression.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { aggregateBenchmarkRuns } from "../src/core/execution-profile-benchmarks.js";
+import { readBenchmarkRunSets, readBenchmarkScenarios } from "./lib/execution-profile-benchmark-io.mjs";
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const defaultResultsDirectory = path.join(repositoryRoot, "benchmarks", "execution-profiles", "results");
+
+function parseArgs(argv) {
+  const options = { results: defaultResultsDirectory, json: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === "--json") options.json = true;
+    else if (argument === "--results") {
+      const value = argv[++index];
+      if (!value || value.startsWith("--")) throw new Error("--results requires a value");
+      options.results = value;
+    } else if (argument === "--help" || argument === "-h") options.help = true;
+    else throw new Error(`unknown option: ${argument}`);
+  }
+  return options;
+}
+
+function statusForAggregate(aggregate, runs) {
+  const hasMeasuredUsage = runs.some((run) => (
+    ["PROVIDER_REPORTED", "HOST_REPORTED"].includes(run.usage.source)
+    && run.usage.totalTokens !== null
+  ));
+  if (!hasMeasuredUsage) return "NOT_MEASURED";
+  if (!aggregate.claimsAllowed) return "NOT_COMPARABLE";
+  if (aggregate.lightObjectives
+    && (aggregate.lightObjectives.p50Pass === false || aggregate.lightObjectives.p95Pass === false)) {
+    return "EFFICIENCY_REGRESSION";
+  }
+  return "OK";
+}
+
+function overallStatus(statuses) {
+  if (statuses.includes("EFFICIENCY_REGRESSION")) return "EFFICIENCY_REGRESSION";
+  if (statuses.includes("NOT_COMPARABLE")) return "NOT_COMPARABLE";
+  if (statuses.length === 0 || statuses.every((status) => status === "NOT_MEASURED")) return "NOT_MEASURED";
+  return "OK";
+}
+
+function helpText() {
+  return "Usage: npm run benchmark:profiles:regression -- [--results <directory>] [--json]";
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    console.log(helpText());
+    return;
+  }
+  const scenarios = await readBenchmarkScenarios(repositoryRoot);
+  const runSets = await readBenchmarkRunSets(path.resolve(options.results));
+  const reports = [];
+  for (const runSet of runSets) {
+    for (const scenario of scenarios) {
+      const runs = runSet.runs.filter((run) => run.scenarioId === scenario.scenarioId);
+      if (runs.length === 0) {
+        reports.push({ runSetId: runSet.runSetId, scenarioId: scenario.scenarioId, status: "NOT_COMPARABLE" });
+        continue;
+      }
+      const aggregate = aggregateBenchmarkRuns({ scenario, runs });
+      reports.push({
+        runSetId: runSet.runSetId,
+        scenarioId: scenario.scenarioId,
+        expectedProfile: scenario.expectedProfile,
+        status: statusForAggregate(aggregate, runs),
+        claimsAllowed: aggregate.claimsAllowed,
+        contextInflation: aggregate.contextInflation?.status ?? null,
+      });
+    }
+  }
+  const status = overallStatus(reports.map((report) => report.status));
+  const output = {
+    status,
+    blocking: false,
+    runSetCount: runSets.length,
+    scenarioCount: scenarios.length,
+    reports,
+    policy: "EFFICIENCY_REGRESSION is an observational release-quality warning and never a ForgeLoop lifecycle failure.",
+  };
+  if (options.json) console.log(JSON.stringify(output));
+  else {
+    console.log(`Efficiency regression status: ${status}`);
+    console.log(`Run sets: ${runSets.length}`);
+    console.log("This report is observational and does not change lifecycle completion.");
+  }
+}
+
+main().catch((error) => {
+  console.error(`ForgeLoop efficiency regression check: ${error.message}`);
+  process.exitCode = 1;
+});

package/scripts/generate-agent-protocol-summary.mjs

@@ -79,6 +79,41 @@ Package version: ${packageJson.version}
 6. Run forgeloop complete; accept completion only when the validator returns VALID.
 7. Run forgeloop next again and follow the returned lifecycle action to a terminal state or an explicit blocker.
 
+## Adaptive execution profiles
+
+\`complianceMode\` controls how strongly project policy is enforced. The
+orthogonal \`executionProfile\` controls process and context depth: \`auto\`
+resolves deterministically to \`light\`, \`balanced\`, or \`full\` from route,
+contract, and task metadata. CLI requests take precedence over project
+configuration, but a safety floor always wins. Profiles never remove required
+contracts, gates, verification, provenance, lifecycle phases, or validated
+completion. Protocol-v1 routes without the field project to \`balanced\` for
+compatibility without rewriting historical artifacts.
+
+For \`light\` tasks, hosts should use \`next --compact\` or \`task-show --compact\`,
+load only relevant guide sections, keep plans concise, and avoid optional
+reflection, trajectory evaluation, handoff, attestation, and continuity
+artifacts unless requested, required, or needed for recovery. The lifecycle
+chronology remains unchanged.
+
+The read-only \`task/context\` integration resource provides the canonical
+profile-aware projection: objective, deliverables, constraints, selected guide
+IDs, phase, next action, verification requirements, and context policy. A
+profile changes presentation and optional context only; it never permits a
+phase or required gate to be skipped.
+
+Usage telemetry is provider or host reported when available, actor-reported
+only through the explicit \`usage-record\` fallback, and \`UNKNOWN\` otherwise.
+ForgeLoop never estimates tokens or treats usage as verification evidence.
+\`efficiency --task\` is read-only and returns \`NOT_COMPARABLE\` unless a
+project-local baseline has matching metadata.
+
+Measured execution-profile benchmarks are observational. The reproducible
+runner accepts provider or host usage, records actual timing, requires PASS
+verification and matching metadata for comparisons, and reports \`NOT_MEASURED\`
+or \`NOT_COMPARABLE\` when evidence is absent or incompatible. See
+\`docs/EXECUTION_PROFILE_BENCHMARKS.md\` for the runner and schemas.
+
 ## Authority boundaries
 
 - Protocol-derived facts outrank actor-provided labels, free-form summaries, and guessed identities.

package/scripts/lib/execution-profile-benchmark-io.mjs

@@ -0,0 +1,67 @@
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+
+import { assertSchema, readSchema } from "../../src/core/schema-validation.js";
+import { getPackageRoot } from "../../src/core/templates.js";
+import {
+  assertBenchmarkScenario,
+  assertBenchmarkRun,
+  assertRequiredBenchmarkScenarios,
+} from "../../src/core/execution-profile-benchmarks.js";
+
+export async function readBenchmarkScenarios(repositoryRoot) {
+  const directory = path.join(repositoryRoot, "benchmarks", "execution-profiles");
+  const schema = await readSchema("execution-profile-benchmark-scenario", getPackageRoot());
+  const names = (await readdir(directory)).filter((name) => /^[a-z0-9-]+\.json$/u.test(name)).sort();
+  const scenarios = [];
+  for (const name of names) {
+    const scenario = JSON.parse(await readFile(path.join(directory, name), "utf8"));
+    assertSchema(scenario, schema, name);
+    scenarios.push(assertBenchmarkScenario(scenario));
+  }
+  return assertRequiredBenchmarkScenarios(scenarios);
+}
+
+async function readJsonFiles(directory) {
+  let entries;
+  try {
+    entries = await readdir(directory, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  }
+  const files = [];
+  for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+    const entryPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) files.push(...await readJsonFiles(entryPath));
+    else if (entry.isFile() && entry.name.endsWith(".json")) files.push(entryPath);
+  }
+  return files;
+}
+
+export async function readBenchmarkRunSets(resultsDirectory, runSetId = null) {
+  const rawDirectory = path.join(resultsDirectory, "raw");
+  let names;
+  try {
+    names = (await readdir(rawDirectory, { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory() && (runSetId === null || entry.name === runSetId))
+      .map((entry) => entry.name)
+      .sort();
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  }
+  const schema = await readSchema("execution-profile-benchmark-run", getPackageRoot());
+  const runSets = [];
+  for (const name of names) {
+    const files = await readJsonFiles(path.join(rawDirectory, name));
+    const runs = [];
+    for (const filename of files) {
+      const value = JSON.parse(await readFile(filename, "utf8"));
+      assertSchema(value, schema, filename);
+      runs.push(assertBenchmarkRun(value));
+    }
+    runSets.push({ runSetId: name, runs });
+  }
+  return runSets;
+}