@cassiomc1/forgeloop 1.7.0 → 1.8.1

package/DOCS_INDEX.md

@@ -30,7 +30,7 @@ integration and guide context. Use this map before editing documentation.
 | Platform adapters | [`docs/PLATFORM_ADAPTERS.md`](./docs/PLATFORM_ADAPTERS.md) | Generic CI boundary and platform mapping guidance |
 | Universal integration API | [`docs/UNIVERSAL_INTEGRATION.md`](./docs/UNIVERSAL_INTEGRATION.md) | Programmatic integration subpath, envelope semantics, and consumer map |
 | Local-first MCP adapter | [`docs/MCP.md`](./docs/MCP.md) | stdio default, optional strict loopback HTTP; server modes/capabilities and canonical resources |
-| Adaptive execution-profile benchmarks | [`docs/EXECUTION_PROFILE_BENCHMARKS.md`](./docs/EXECUTION_PROFILE_BENCHMARKS.md) | Measured provider/host runs, comparability, aggregates, and profile-aware host context |
+| Adaptive execution-profile benchmarks | [`docs/EXECUTION_PROFILE_BENCHMARKS.md`](./docs/EXECUTION_PROFILE_BENCHMARKS.md) | Measured provider/host runs, robust statistics, paired/distribution deltas, tail status, outliers, and profile-aware host context |
 | Documentation guide | [`docs/DOCUMENTATION_GUIDE.md`](./docs/DOCUMENTATION_GUIDE.md) | Rules and checklist for modifying documentation |
 | Current release checklist | [`docs/RELEASE_CHECKLIST.md`](./docs/RELEASE_CHECKLIST.md) | Package, protocol, attestation, integration, and publication gates |
 | ForgeLoop 1.6.1 release checklist (historical) | [`docs/RELEASE_CHECKLIST_1_6_1.md`](./docs/RELEASE_CHECKLIST_1_6_1.md) | Verification adapter boundary, isolation invariants, and publication gates |

package/benchmarks/execution-profiles/README.md

@@ -42,3 +42,21 @@ Initial non-blocking efficiency goals for LIGHT scenarios are P50 token
 overhead of at most +35% and P95 token overhead of at most +60% against a
 comparable direct baseline. These are benchmark objectives, not completion
 rules, and quality or verification coverage may not be traded for them.
+
+Raw runs may carry an optional `diagnostics` object with host-observed
+execution detail (model turns, tool calls, retries, correction cycles, files
+written, guide selections, termination reason, runaway-signal flags). Every
+field is nullable; never estimate a value the host does not actually report.
+
+Benchmark methodology v2 (`benchmarkVersion: "2"`) adds robust variability
+statistics (median, quartiles, IQR, MAD), per-comparison tail stability
+status, and deterministic token-outlier classification under the
+`TOKEN_IQR_1_5` policy. Runner repetitions are bounded by explicit tiers:
+`smoke` (1–3), `evidence` (5–10), and `tail` (20–30). Tail conclusions
+require at least 20 comparable pairs.
+
+Methodology versioning protects history: readers accept benchmark versions
+`1` and `2`, historical v1 run sets are never rewritten, and their stored
+aggregates reproduce byte-for-byte under the frozen v1 methodology. All of
+these diagnostics are observational and never change lifecycle truth,
+required gates, verification validity, or completion authority.

package/benchmarks/execution-profiles/api-feature.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "api-feature",
   "description": "Ordinary API behavior change without authentication or publication authority.",
   "input": {

package/benchmarks/execution-profiles/authentication-change.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "authentication-change",
   "description": "Authentication behavior change crossing a trust boundary.",
   "input": {

package/benchmarks/execution-profiles/documentation-correction.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "documentation-correction",
   "description": "Narrow documentation correction with no executable behavior or sensitive content.",
   "input": {

package/benchmarks/execution-profiles/infrastructure-release.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "infrastructure-release",
   "description": "Infrastructure or release/publication work with external and potentially irreversible effects.",
   "input": {

package/benchmarks/execution-profiles/novatask-saas-landing-page.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "novatask-saas-landing-page",
   "description": "NovaTask reference workload: a local static SaaS landing page with no external or sensitive boundary.",
   "input": {

package/benchmarks/execution-profiles/small-bug-fix.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "small-bug-fix",
   "description": "Small application bug fix with behavior implications and focused regression checks.",
   "input": {

package/benchmarks/execution-profiles/static-landing-page.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "benchmarkVersion": "1",
+  "benchmarkVersion": "2",
   "scenarioId": "static-landing-page",
   "description": "Small local HTML, CSS, and JavaScript landing page without external services or sensitive boundaries.",
   "input": {

package/docs/AGENT_PROTOCOL_SUMMARY.md

@@ -7,7 +7,7 @@
 ForgeLoop is a portable protocol and support CLI for verifiable engineering workflows. It records and validates task state, contracts, routing, checks, evidence, continuity, and optional code attestations. It does not become an agent scheduler, delegation service, source-control authority, or secret manager.
 
 Protocol version: 1
-Package version: 1.7.0
+Package version: 1.8.1
 
 ## Canonical loop
 

package/docs/EXECUTION_PROFILE_BENCHMARKS.md

@@ -56,6 +56,20 @@ npm run benchmark:profiles -- \
   --json
 ```
 
+Repetition counts are bounded by an explicit tier so the statistical purpose
+of a run set is visible in the command itself:
+
+| Tier | Runs | Purpose |
+| --- | --- | --- |
+| `smoke` | 1–3 | Fast adapter and pipeline checks; never efficiency evidence. |
+| `evidence` | 5–10 | Central-tendency comparisons (default when no tier is given: 5 runs). |
+| `tail` | 20–30 | Tail-risk and P95 stability analysis. |
+
+`--tier smoke|evidence|tail` enforces these bounds and selects the tier
+default when `--runs` is omitted. Runs outside a tier's bounds are rejected.
+Tail-stability conclusions additionally require the sample minimum described
+under tail analysis below.
+
 The adapter exports `runBenchmark(input)` and must execute the supplied
 scenario. Its result must include actual usage and verification data:
 
@@ -78,6 +92,22 @@ export async function runBenchmark({ scenario, mode, runIndex, target }) {
     verification: "PASS",
     verificationCycles: 1,
     comparableSteps: 4,
+    diagnostics: {
+      executionProfile: "light",
+      verificationCycles: 1,
+      modelTurns: 3,
+      toolCalls: 5,
+      retries: null,
+      correctionCycles: null,
+      filesRead: null,
+      filesWritten: 2,
+      contextRefreshes: null,
+      guideCount: 1,
+      guideIds: ["test"],
+      hostWarnings: [],
+      terminationReason: "COMPLETED",
+      flags: [],
+    },
   };
 }
 ```
@@ -88,6 +118,14 @@ measurement and derives the Git revision from the target checkout. Results
 are written under a unique run-set directory and existing history is never
 overwritten.
 
+The optional `diagnostics` object records host-observed execution detail for
+tail-risk attribution. Every field is nullable: a host must leave a value
+`null` (or omit `diagnostics` entirely) whenever it does not actually observe
+that telemetry, and must never estimate it from prompt size, elapsed time, or
+file size. `flags` may carry host-reported runaway signals from the fixed
+signal vocabulary; the aggregate additionally derives deterministic signals
+where recorded diagnostics support them.
+
 An adapter may also return host-observed context usage. ForgeLoop does not
 tokenize provider prompts. The optional shape is:
 
@@ -136,13 +174,76 @@ host observation is available. Implementer self-ratings should not be used as
 independent quality evidence; for NovaTask, review direct and ForgeLoop
 outputs blind to execution mode before revealing the labels.
 
+## Blind UI quality finalization
+
+The runner supports an optional post-run finalization hook for adapters that
+can collect independent UI evidence:
+
+```js
+export async function finalizeBenchmark({ benchmarkVersion, records, scenarios, target, runSetId }) {
+  // Render and evaluate retained UI workspaces after all host timings finish.
+  return {
+    qualityByRunId: {
+      // Keys are the runner-owned run IDs; values use the quality shape above.
+    },
+    summary: { status: "MEASURED" },
+  };
+}
+```
+
+The runner calls this hook only after every `runBenchmark` call has returned.
+The adapter's `wallClockMs` therefore excludes browser rendering and evaluator
+latency. The finalizer may attach quality only to known runner run IDs; usage,
+timing, verification, comparable steps, context telemetry, and metadata are
+not rewritten by finalization. A finalizer must leave quality `UNKNOWN` when a
+candidate cannot be rendered or the evaluator cannot return schema-valid
+scores.
+
+The maintained Codex visual evaluator is intentionally outside ForgeLoop
+Core. It requires an explicitly supplied, already-installed Playwright runtime
+and Chromium executable, renders desktop (1440×900) and mobile (390×844)
+screenshots, blocks HTTP(S) requests, and gives the separate evaluator only
+requirements, neutral browser observations, and anonymized candidate labels.
+The evaluator receives no execution-mode labels, run IDs, source files, or
+implementer self-ratings. Its validated scores are recorded as
+`EXTERNAL_REPORTED`; this is independent observational evidence, not a human
+acceptance decision and not lifecycle authority. If the evaluator is not
+configured or fails, quality remains `UNKNOWN` and no quality-preservation
+claim is permitted.
+
 ## Inspecting and validating results
 
 ```bash
 npm run benchmark:profiles:summary -- --json
 npm run benchmark:profiles:check -- --json
+npm run benchmark:profiles:outliers -- --json
 ```
 
+To enable the optional blind UI finalizer, provide paths to the host's
+existing Playwright and Chromium installations and a separate evaluator
+model. The benchmark does not install these resources:
+
+```bash
+FORGELOOP_BENCHMARK_MODEL=gpt-5.4-mini \
+FORGELOOP_BENCHMARK_REASONING=low \
+FORGELOOP_BENCHMARK_QUALITY_EVALUATOR=./benchmarks/evaluators/codex-cli-blind-visual.mjs \
+FORGELOOP_BENCHMARK_PLAYWRIGHT_ROOT=/absolute/path/to/playwright \
+FORGELOOP_BENCHMARK_BROWSER=/absolute/path/to/Google\ Chrome\ for\ Testing \
+FORGELOOP_BENCHMARK_EVALUATOR_MODEL=gpt-5.4 \
+npm run benchmark:profiles -- \
+  --adapter ./benchmarks/adapters/codex-cli-real-host.mjs \
+  --runs 5 \
+  --run-set codex-quality-repeat5-YYYYMMDD \
+  --output /tmp/forgeloop-benchmark-results-quality \
+  --json
+```
+
+The two evaluator processes have separate responsibilities: the
+implementation host reports actual usage and deterministic verification, and
+the read-only evaluator reports blind screenshot scores. Neither process may
+invent context-item counts, token counts, costs, cache values, or lifecycle
+state.
+
 Before a host run exists, the summary is intentionally:
 
 ```json
@@ -154,9 +255,12 @@ Before a host run exists, the summary is intentionally:
 
 Raw runs are stored below `benchmarks/execution-profiles/results/raw/` and
 recomputed aggregates below `benchmarks/execution-profiles/results/aggregate/`.
-The validator checks all scenario, raw-run, and aggregate schemas and
-recomputes aggregates from raw measurements. A changed overhead target is
-observable in the report; it is not a blocking lifecycle gate.
+Those recorded measurements are repository evidence and are intentionally not
+included in the core npm package. The package ships the deterministic scenario
+definitions and benchmark schemas needed to run a new measurement; the
+validator checks all scenario, raw-run, and aggregate schemas and recomputes
+aggregates from raw measurements. A changed overhead target is observable in
+the report; it is not a blocking lifecycle gate.
 
 CI can run `npm run benchmark:profiles:regression -- --json`. It reports
 `OK`, `EFFICIENCY_REGRESSION`, `NOT_MEASURED`, or `NOT_COMPARABLE`; a regression
@@ -167,6 +271,117 @@ overhead no greater than +35% and P95 token overhead no greater than +60%
 against a comparable direct baseline. These objectives never trade away
 verification quality, requirement coverage, or protocol safety.
 
+## Robust statistics and tail stability
+
+Benchmark methodology v2 reports robust variability for skewed token
+distributions. Methodology-v2 mode aggregates include, for every measured
+quantity, `count`, `average`, `minimum`, `p25`, `p50`, `p75`, `p90`, `p95`,
+`maximum`, `iqr`, `mad`, and `outlierCount`. Percentiles use linear
+interpolation; `iqr` is the interquartile range and `mad` the median absolute
+deviation around the median. Methodology-v1 aggregates keep their frozen
+historical shape.
+
+Every non-direct comparison in a v2 aggregate gains a `tail` object, `pairedRatioDiagnostics`, `pairedRuns`, and separate paired and distribution statistics:
+
+```json
+{
+  "pairedOverheadPercent": {
+    "p50": 3.5685,
+    "p95": 174.1502
+  },
+  "distributionDeltaPercent": {
+    "p50": 3.4708,
+    "p95": 13.1906
+  },
+  "pairedRatioDiagnostics": {
+    "pairCount": 20,
+    "baselineMinimum": 63298,
+    "baselineP25": 65742.25,
+    "baselineP50": 140632,
+    "lowBaselineThreshold": 84379.2,
+    "lowBaselinePairCount": 7
+  },
+  "tail": {
+    "sampleMinimum": 20,
+    "sampleCount": 20,
+    "p95TokenOverheadPercent": 174.1502,
+    "pairedOverheadP95Percent": 174.1502,
+    "distributionP95DeltaPercent": 13.1906,
+    "outlierCount": 0,
+    "status": "TAIL_REGRESSION",
+    "pairedStatus": "TAIL_REGRESSION",
+    "distributionStatus": "TAIL_ACCEPTABLE",
+    "combinedInterpretation": "TAIL_PAIRED_RATIO_SENSITIVE"
+  }
+}
+```
+
+### Paired overhead vs. distribution tail delta
+
+`pairedOverheadP95Percent` is the P95 of run-level relative overhead values (`((adaptive_i - direct_i) / direct_i) * 100`) across matched run indices. Because execution runs are independent stochastic trials, paired ratios can become very large when an individual direct baseline run finishes in an unusually low token regime (`LOW_BASELINE_TOKEN_REGIME`, defined as `directTokens < directP50 * 0.60`).
+
+`distributionP95DeltaPercent` compares the P95 values of the candidate and baseline token distributions directly (`((adaptiveP95 - directP95) / directP95) * 100`).
+
+These are distinct statistics and must not be presented as interchangeable metrics:
+
+- When a candidate distribution is genuinely heavy-tailed, `distributionP95DeltaPercent` itself exceeds the objective (`TAIL_DISTRIBUTION_REGRESSION`).
+- When the candidate distribution P95 is acceptable but paired overhead is inflated by low-baseline runs, the combined interpretation is `TAIL_PAIRED_RATIO_SENSITIVE`.
+
+```bash
+npm run benchmark:profiles:tail-analysis -- [--results <dir>] [--run-set <id>] [--json]
+```
+
+Tail status is observational and is classified as:
+
+- `NOT_ENOUGH_SAMPLES` when comparable pairs are below the sample minimum (20) or P95 telemetry is unavailable;
+- `TAIL_REGRESSION` when the P95 token metric exceeds the +60% objective;
+- `TAIL_WARNING` when the sample is sufficient but IQR outliers are present;
+- `TAIL_STABLE` / `TAIL_ACCEPTABLE` otherwise.
+
+Combined interpretation states:
+
+- `TAIL_CONSISTENT`: paired and distribution views agree;
+- `TAIL_PAIRED_RATIO_SENSITIVE`: paired P95 fails while distribution P95 passes and low-baseline pairs are present;
+- `TAIL_DISTRIBUTION_REGRESSION`: absolute distribution P95 delta exceeds the objective;
+- `TAIL_UNRESOLVED`: sample count is insufficient or evidence is incomplete.
+
+Tail status never gates lifecycle completion; it tells operators when a run
+set is large enough to trust its tail.
+
+## Outlier classification and runaway signals
+
+Token outliers are classified deterministically per scenario and mode with
+the `TOKEN_IQR_1_5` policy: a measured run whose total tokens exceed
+`Q3 + 1.5 × IQR` of that mode's measured distribution is an outlier, and at
+least 4 measured samples are required before classification. Each outlier
+record keeps the run identity, token values, `ratioToMedian`, and the
+diagnostic signals that explain it.
+
+```bash
+npm run benchmark:profiles:outliers -- [--results <dir>] [--run-set <id>] [--json]
+```
+
+The report is read-only: it never rewrites raw runs or aggregates and it
+never changes lifecycle truth. Signals come from two sources: host-reported
+`diagnostics.flags` and deterministic derivations from recorded diagnostics
+(model turns or tool calls more than double the mode median, repeated
+verification or correction cycles, retries, repeated context refreshes). An
+outlier with no recorded explanation is reported as `UNKNOWN_TOKEN_SPIKE`,
+which is an admission of missing telemetry, not an estimate. Historical
+methodology-v1 runs carry no diagnostics, so their spikes legitimately remain
+`UNKNOWN_TOKEN_SPIKE`.
+
+## Methodology versioning and historical compatibility
+
+Scenarios, runs, and aggregates carry a `benchmarkVersion`. Methodology v2
+bumps the current version to `2`; readers accept versions `1` and `2`.
+Historical v1 run sets are immutable evidence: their raw runs and stored
+aggregates are never rewritten, and recomputation reproduces the frozen v1
+aggregate shape exactly. Version checks therefore never invalidate committed
+history; a run set aggregates under the methodology its runs were measured
+with. Only new run sets recorded under methodology v2 receive robust
+statistics, tail status, outlier analysis, and diagnostics.
+
 ## Host context contract
 
 The universal integration API exposes `task/context`, and the MCP adapter

package/docs/MCP.md

@@ -119,7 +119,7 @@ forgeloop-mcp-http --project /repo --mode safe          # 127.0.0.1:3333
 
 | Component | Current contract |
 | --- | --- |
-| ForgeLoop core package | `>=1.5.0 <2` dependency range; current repository generation `1.6.x` |
+| ForgeLoop core package | `>=1.5.0 <2` dependency range; current repository generation `1.8.x` |
 | ForgeLoop protocol | `1` |
 | Integration API | `1` |
 | MCP package | `0.1.x` initial package |

package/docs/RELEASE_CHECKLIST.md

@@ -10,6 +10,9 @@ preparation and verification checklist; it does not authorize publication.
 - [ ] `PROTOCOL_VERSION` and the integration API version remain compatible.
 - [ ] `npm run release:identity` passes for the candidate version.
 - [ ] No release tag or registry version collision exists.
+- [ ] `npm pack --dry-run` contains the required scenario definitions but no
+      raw or aggregate benchmark results, tests, local state, or repository
+      metadata.
 
 ## Protocol and attestation
 
@@ -20,6 +23,13 @@ preparation and verification checklist; it does not authorize publication.
       is reported as `NOT_MEASURED`, never as zero or a passing efficiency claim.
 - [ ] `npm run benchmark:profiles:regression` reports the observed status;
       `EFFICIENCY_REGRESSION` remains a non-blocking warning.
+- [ ] `npm run benchmark:profiles:outliers` and
+      `npm run benchmark:profiles:tail-analysis` report the observed
+      `TOKEN_IQR_1_5`, paired-ratio, and distribution-tail results without
+      conflating `pairedOverheadPercent` with `distributionDeltaPercent`.
+- [ ] Historical v1 run sets remain byte-for-byte immutable while v2 run sets
+      carry robust statistics, low-baseline diagnostics, and explicit tail
+      interpretations.
 - [ ] `npm run coverage` passes the configured global and critical-module gates.
 - [ ] `npm run docs:check`, `npm run docs:generated:check`,
       `npm run docs:conformance`, and `npm run docs:examples:check` pass.
@@ -51,6 +61,8 @@ preparation and verification checklist; it does not authorize publication.
 - [ ] The exact validated commit is the release source.
 - [ ] GitHub Actions remain immutably pinned and use least-privilege permissions.
 - [ ] The npm workflow retains trusted OIDC publishing and explicit provenance.
+- [ ] The publication workflow fails closed when the candidate npm version
+      already exists or the package inspection finds forbidden paths.
 - [ ] Publication is performed only by the authorized release workflow.
 - [ ] Post-publication registry, tag, checksum, and release identity checks pass.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cassiomc1/forgeloop",
-  "version": "1.7.0",
+  "version": "1.8.1",
   "description": "Portable, verifiable engineering protocol for AI coding environments and developer workflows",
   "repository": {
     "type": "git",
@@ -22,7 +22,8 @@
     "src",
     "ENG",
     "schemas",
-    "benchmarks/execution-profiles",
+    "benchmarks/execution-profiles/*.json",
+    "benchmarks/execution-profiles/README.md",
     ".forgeloop/forgeloop.gitignore",
     "AGENTS.md",
     "CLAUDE.md",
@@ -72,6 +73,8 @@
     "scripts/run-execution-profile-benchmarks.mjs",
     "scripts/summarize-execution-profile-benchmarks.mjs",
     "scripts/validate-execution-profile-benchmarks.mjs",
+    "scripts/report-execution-profile-outliers.mjs",
+    "scripts/report-tail-interpretation.mjs",
     "scripts/check-efficiency-regression.mjs",
     "scripts/lib/execution-profile-benchmark-io.mjs",
     "scripts/benchmark-cli-startup.mjs",
@@ -115,6 +118,8 @@
     "benchmark:profiles": "node scripts/run-execution-profile-benchmarks.mjs",
     "benchmark:profiles:summary": "node scripts/summarize-execution-profile-benchmarks.mjs",
     "benchmark:profiles:check": "node scripts/validate-execution-profile-benchmarks.mjs",
+    "benchmark:profiles:outliers": "node scripts/report-execution-profile-outliers.mjs",
+    "benchmark:profiles:tail-analysis": "node scripts/report-tail-interpretation.mjs",
     "benchmark:profiles:regression": "node scripts/check-efficiency-regression.mjs"
   },
   "devDependencies": {

package/schemas/execution-profile-benchmark-aggregate.schema.json

@@ -6,10 +6,11 @@
   "required": ["schemaVersion", "benchmarkVersion", "runSetId", "scenarioId", "expectedProfile", "modeAggregates", "comparisons", "lightObjectives", "sourcePolicy", "claimsAllowed", "generatedFromRunCount"],
   "properties": {
     "schemaVersion": { "const": 1 },
-    "benchmarkVersion": { "const": "1" },
+    "benchmarkVersion": { "enum": ["1", "2"] },
     "runSetId": { "type": "string", "minLength": 1 },
     "scenarioId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,127}$" },
     "expectedProfile": { "enum": ["light", "balanced", "full"] },
+    "outlierAnalysis": { "$ref": "#/$defs/outlierAnalysis" },
     "modeAggregates": {
       "type": "object",
       "required": ["direct", "forgeloopBalanced", "forgeloopAdaptive"],
@@ -39,5 +40,54 @@
     "claimsAllowed": { "type": "boolean" },
     "generatedFromRunCount": { "type": "integer", "minimum": 1 }
   },
+  "$defs": {
+    "outlierAnalysis": {
+      "type": "object",
+      "required": ["policy", "minimumSamples", "modes"],
+      "properties": {
+        "policy": { "const": "TOKEN_IQR_1_5" },
+        "minimumSamples": { "type": "integer", "minimum": 1 },
+        "modes": {
+          "type": "object",
+          "required": ["direct", "forgeloopBalanced", "forgeloopAdaptive"],
+          "additionalProperties": { "$ref": "#/$defs/modeOutlierAnalysis" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "modeOutlierAnalysis": {
+      "type": "object",
+      "required": ["sampleCount", "measuredCount", "status", "q1", "q3", "iqr", "median", "upperFence", "outliers"],
+      "properties": {
+        "sampleCount": { "type": "integer", "minimum": 0 },
+        "measuredCount": { "type": "integer", "minimum": 0 },
+        "status": { "enum": ["MEASURED", "NOT_ENOUGH_SAMPLES"] },
+        "q1": { "type": ["number", "null"] },
+        "q3": { "type": ["number", "null"] },
+        "iqr": { "type": ["number", "null"] },
+        "median": { "type": ["number", "null"] },
+        "upperFence": { "type": ["number", "null"] },
+        "outliers": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/outlierRecord" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "outlierRecord": {
+      "type": "object",
+      "required": ["runId", "runIndex", "totalTokens", "scenarioMedianTokens", "ratioToMedian", "reasons", "diagnosticSignals"],
+      "properties": {
+        "runId": { "type": "string", "pattern": "^run-[A-Za-z0-9][A-Za-z0-9_-]{0,127}$" },
+        "runIndex": { "type": "integer", "minimum": 1 },
+        "totalTokens": { "type": "integer", "minimum": 0 },
+        "scenarioMedianTokens": { "type": ["number", "null"] },
+        "ratioToMedian": { "type": ["number", "null"] },
+        "reasons": { "type": "array", "items": { "const": "TOKEN_IQR_OUTLIER" } },
+        "diagnosticSignals": { "type": "array", "items": { "type": "string", "minLength": 1 } }
+      },
+      "additionalProperties": false
+    }
+  },
   "additionalProperties": false
 }

package/schemas/execution-profile-benchmark-run.schema.json

@@ -6,7 +6,7 @@
   "required": ["schemaVersion", "benchmarkVersion", "runSetId", "runId", "scenarioId", "mode", "runIndex", "recordedAt", "usage", "wallClockMs", "verification", "verificationCycles", "comparableSteps", "metadata"],
   "properties": {
     "schemaVersion": { "const": 1 },
-    "benchmarkVersion": { "const": "1" },
+    "benchmarkVersion": { "enum": ["1", "2"] },
     "runSetId": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" },
     "runId": { "type": "string", "pattern": "^run-[A-Za-z0-9][A-Za-z0-9_-]{0,127}$" },
     "scenarioId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,127}$" },
@@ -33,6 +33,7 @@
     "verification": { "enum": ["PASS", "FAIL", "NOT_AVAILABLE"] },
     "verificationCycles": { "type": ["integer", "null"], "minimum": 0 },
     "comparableSteps": { "type": ["integer", "null"], "minimum": 0 },
+    "diagnostics": { "$ref": "#/$defs/diagnostics" },
     "contextUsage": { "$ref": "#/$defs/contextUsage" },
     "quality": { "$ref": "#/$defs/quality" },
     "metadata": {
@@ -45,7 +46,7 @@
         "provider": { "type": ["string", "null"] },
         "promptSpecFingerprint": { "type": "string", "minLength": 1 },
         "projectRevision": { "type": ["string", "null"] },
-        "benchmarkVersion": { "const": "1" },
+        "benchmarkVersion": { "enum": ["1", "2"] },
         "environmentClass": { "type": "string", "minLength": 1 },
         "requestedProfile": { "type": ["string", "null"] },
         "resolvedProfile": { "type": ["string", "null"] },
@@ -59,6 +60,43 @@
     }
   },
   "$defs": {
+    "diagnostics": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "executionProfile": { "enum": ["light", "balanced", "full", null] },
+        "verificationCycles": { "type": ["integer", "null"], "minimum": 0 },
+        "modelTurns": { "type": ["integer", "null"], "minimum": 0 },
+        "toolCalls": { "type": ["integer", "null"], "minimum": 0 },
+        "retries": { "type": ["integer", "null"], "minimum": 0 },
+        "correctionCycles": { "type": ["integer", "null"], "minimum": 0 },
+        "filesRead": { "type": ["integer", "null"], "minimum": 0 },
+        "filesWritten": { "type": ["integer", "null"], "minimum": 0 },
+        "contextRefreshes": { "type": ["integer", "null"], "minimum": 0 },
+        "guideCount": { "type": ["integer", "null"], "minimum": 0 },
+        "guideIds": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "hostWarnings": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+        "terminationReason": { "type": ["string", "null"], "minLength": 1 },
+        "flags": {
+          "type": "array",
+          "items": {
+            "enum": [
+              "EXCESSIVE_MODEL_TURNS",
+              "EXCESSIVE_TOOL_CALLS",
+              "EXCESSIVE_VERIFICATION_CYCLES",
+              "REPEATED_CONTEXT_REFRESH",
+              "REPEATED_FILE_READ",
+              "REPEATED_GUIDE_LOAD",
+              "UNEXPECTED_RETRY",
+              "CORRECTION_LOOP",
+              "HOST_RETRY",
+              "MODEL_STALL",
+              "UNKNOWN_TOKEN_SPIKE"
+            ]
+          }
+        }
+      }
+    },
     "contextUsage": {
       "type": "object",
       "required": ["source", "profile", "items"],