@openwop/openwop-conformance 2.46.25 → 2.46.26

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [2.46.26] — 2026-10-25 — RFC 0244 model-usage and cost legs
+
+- **New leg `0244.efficiency-model-usage` in `v2-eval-trajectory-trials`**, on the existing trajectory and trials runs. Per-task `modelCallCount`, `inputTokens` and `outputTokens`, times the trial count, must sum to the agent's `provider.usage` inside its `agent.invocation.*` brackets (judge and delegate calls excluded); without brackets they must not exceed the run's usage.
+- **New leg `0244.cost-every-trial`**, on the trials run: `totalCostUsd` must be at least the agent's priced usage over all trials. Judges in `lib/eval-efficiency-witness.ts`.
+- **Host impact:** a run with no model usage, no brackets or unpriced usage records `inapplicable`; one that counts judge calls as the agent's, or prices one trial, fails.
+
 ## [2.46.25] — 2026-10-25 — RFC 0244 §C efficiency leg
 
 - **New leg `0244.efficiency-tool-calls` in `v2-eval-trajectory-trials`**, on the existing `conformance-eval-trajectory` run (and the `conformance-eval-trials` run when served), so no extra run. Each task's `efficiency.toolCallCount` must equal its script length, and on the one-trial run the counts must sum to the log's `agent.tool-called` events. Judge in `lib/eval-efficiency-witness.ts`.

package/dist/spec-artifacts.lock.json

@@ -1,5 +1,5 @@
 {
   "package": "@openwop/spec-artifacts",
-  "version": "2.46.25",
-  "stampSha256": "d50d0faa638afab3defdf1ab0771218528dfddbcf4867d112cb2c3cc68dfb120"
+  "version": "2.46.26",
+  "stampSha256": "e53058ead4546c46dc23498974219d081a05193fdb5dec16db93a221d023e0d9"
 }

package/fixtures.md

@@ -1054,7 +1054,7 @@ Every task targets **the scripted agent `core.conformance.agents.scripted`**. It
 | Fixture | Facet | What the suite asserts |
 | --- | --- | --- |
 | `conformance-eval-trajectory` | `agents.evalSuite.trajectory` | each task's `trajectory` counts equal `lib/eval-trajectory.ts`'s grade of the script; an unmatched trajectory fails; a `kind: "trajectory"` task scores 1 or 0; the log carries one `agent.tool-called` per scripted call |
-| `conformance-eval-trials` | `agents.evalSuite.trials` | `eval.started.trialCount: 3`; three `eval.scored` per task with `trialIndex` 0..2; the per-task `trials` block and `trialStats` equal the §B formulas. Without the facet: `400 validation_error` at `createRun` |
+| `conformance-eval-trials` | `agents.evalSuite.trials` | `eval.started.trialCount: 3`; three `eval.scored` per task with `trialIndex` 0..2; the per-task `trials` block and `trialStats` equal the §B formulas; `totalCostUsd`, when served with bracketed priced usage, is at least the agent's cost over all three trials. Without the facet: `400 validation_error` at `createRun` |
 | `conformance-eval-fixture-miss` | `agents.evalSuite.fixtureMatching` | `missed` reports `fixtureMissCount: 1`, `passed: false` and `score: 0` (summary and every `eval.scored`); `served` passes with no miss. That no live tool ran is shown by the counter fixtures below (RFC 0244 G2) |
 | `conformance-eval-trajectory-gate` | `agents.evalSuite.trajectory` | RFC 0246: both tasks keep `score` 1 and `aggregateScore` 1; `path-matched` carries `gatedScore` 1 and `path-wrong` `gatedScore` 0; `gatedAggregateScore` 0.5; suite `passed: false` although `aggregateScore` clears `passScore` 0.6 |
 | `conformance-eval-trajectory-gate-mean` | `agents.evalSuite.trajectory` + `agents.evalSuite.trials` | RFC 0246: two `eval.scored` with `score` 1 and `passed: false`; `trials.meanScore` 1, `gatedScore` 0, task `passed: false`; the summary carries `gatedAggregateScore` |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openwop/openwop-conformance",
-  "version": "2.46.25",
+  "version": "2.46.26",
   "description": "Black-box conformance suite for OpenWOP hosts (v2 current; v1 scenarios retained for the frozen v1 tree).",
   "repository": {
     "type": "git",
@@ -56,6 +56,6 @@
     "@openwop/spec-artifacts": "file:../spec-artifacts"
   },
   "peerDependencies": {
-    "@openwop/spec-artifacts": "2.46.25"
+    "@openwop/spec-artifacts": "2.46.26"
   }
 }

package/requirements.json

@@ -3,10 +3,10 @@
   "generatedFrom": "src/scenarios/*.test.ts",
   "counts": {
     "files": 749,
-    "tests": 2844,
-    "withStableId": 2844,
+    "tests": 2846,
+    "withStableId": 2846,
     "interpolatedTitles": 0,
-    "explicitIds": 2717
+    "explicitIds": 2719
   },
   "records": [
     {
@@ -31092,7 +31092,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-suite-summary-and-event-fields-validate-and-the-negatives-are-refus",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 185,
+      "line": 196,
       "title": "the RFC 0244 suite, summary and event fields validate and the negatives are refused",
       "explicitId": "openwop.requirement.0244.wire-shape",
       "citations": [
@@ -31131,7 +31131,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0244-additions-are-content-free-eval-summary-no-content-leak",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 208,
+      "line": 219,
       "title": "the RFC 0244 additions are content-free (eval-summary-no-content-leak)",
       "explicitId": "openwop.requirement.0244.content-free",
       "citations": [
@@ -31148,7 +31148,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-eval-suite-fixtures-validate-against-the-v2-suite-schema",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 215,
+      "line": 226,
       "title": "the eval-suite fixtures validate against the v2 suite schema",
       "explicitId": "openwop.requirement.0244.fixtures-valid",
       "citations": [
@@ -31162,7 +31162,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-rfc-0246-gating-fields-validate-and-stay-in-0-1",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 222,
+      "line": 233,
       "title": "the RFC 0246 gating fields validate and stay in 0–1",
       "explicitId": "openwop.requirement.0246.gated-fields-shape",
       "citations": [
@@ -31184,7 +31184,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-may-be-marked-iserror-and-only-as-a-boolean-rfc-0245-a",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 230,
+      "line": 241,
       "title": "a fixture entry may be marked isError, and only as a boolean (RFC 0245 §A)",
       "explicitId": "openwop.requirement.0245.fixture-error-shape",
       "citations": [
@@ -31206,7 +31206,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-trajectory-suite-is-graded-per-the-reference-grader",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 240,
+      "line": 251,
       "title": "a trajectory suite is graded per the reference grader",
       "explicitId": "openwop.requirement.0244.trajectory-graded",
       "citations": [
@@ -31240,7 +31240,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.an-eval-run-brackets-its-scores-with-one-eval-started-and-one-eval-completed-rfc",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 274,
+      "line": 285,
       "title": "an eval run brackets its scores with one eval.started and one eval.completed (RFC 0081 §C)",
       "explicitId": "openwop.requirement.0081.eval-envelope",
       "citations": [
@@ -31254,7 +31254,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-served-evalsummary-validates-and-its-counts-match-its-rows-rfc-0081-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 282,
+      "line": 293,
       "title": "the served EvalSummary validates and its counts match its rows (RFC 0081 §C)",
       "explicitId": "openwop.requirement.0081.summary-served",
       "citations": [
@@ -31268,7 +31268,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.repeated-trials-emit-one-eval-scored-per-task-per-trial",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 303,
+      "line": 314,
       "title": "repeated trials emit one eval.scored per task per trial",
       "explicitId": "openwop.requirement.0244.trials-per-task",
       "citations": [
@@ -31286,7 +31286,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.the-per-task-trials-block-and-trialstats-follow-the-b-formulas",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 314,
+      "line": 325,
       "title": "the per-task trials block and trialStats follow the §B formulas",
       "explicitId": "openwop.requirement.0244.trial-stats-formula",
       "citations": [
@@ -31325,7 +31325,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.efficiency-toolcallcount-counts-the-scripted-agent-tool-called-events-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 337,
+      "line": 348,
       "title": "efficiency.toolCallCount counts the scripted agent.tool-called events (§C)",
       "explicitId": "openwop.requirement.0244.efficiency-tool-calls",
       "citations": [
@@ -31337,9 +31337,37 @@
       ]
     },
     {
-      "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-live-reaches-the-live-counter-and-is-counted-d",
+      "id": "openwop.it.v2-eval-trajectory-trials.efficiency-model-and-token-counts-match-the-agent-s-provider-usage-c",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 360,
+      "title": "efficiency model and token counts match the agent's provider.usage (§C)",
+      "explicitId": "openwop.requirement.0244.efficiency-model-usage",
+      "citations": [
+        {
+          "section": "eval-summary.schema.json efficiency (RFC 0244 §C)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.totalcostusd-counts-every-trial-b",
       "file": "v2-eval-trajectory-trials.test.ts",
       "line": 371,
+      "title": "totalCostUsd counts every trial (§B)",
+      "explicitId": "openwop.requirement.0244.cost-every-trial",
+      "citations": [
+        {
+          "section": "agent-eval-suite.schema.json thresholds.maxCostUsd, eval-summary.schema.json totalCostUsd (RFC 0244 §B)",
+          "requirement": null,
+          "interpolated": true
+        }
+      ]
+    },
+    {
+      "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-live-reaches-the-live-counter-and-is-counted-d",
+      "file": "v2-eval-trajectory-trials.test.ts",
+      "line": 401,
       "title": "a miss under onMiss \"live\" reaches the live counter and is counted (§D)",
       "explicitId": "openwop.requirement.0244.live-miss-counted",
       "citations": [
@@ -31353,7 +31381,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.repeated-calls-are-served-the-first-unconsumed-fixture-entry-once-each-d",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 379,
+      "line": 409,
       "title": "repeated calls are served the first unconsumed fixture entry, once each (§D)",
       "explicitId": "openwop.requirement.0244.first-unconsumed",
       "citations": [
@@ -31367,7 +31395,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-miss-under-onmiss-fail-never-reaches-the-live-counter-d",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 387,
+      "line": 417,
       "title": "a miss under onMiss \"fail\" never reaches the live counter (§D)",
       "explicitId": "openwop.requirement.0244.miss-no-live-call",
       "citations": [
@@ -31381,7 +31409,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-miss-under-onmiss-fail-fails-the-trial-and-is-counted",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 395,
+      "line": 425,
       "title": "a fixture miss under onMiss \"fail\" fails the trial and is counted",
       "explicitId": "openwop.requirement.0244.fixture-miss-fails",
       "citations": [
@@ -31418,7 +31446,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-an-unadvertised-rfc-0244-feature-is-refused-at-createrun",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 415,
+      "line": 445,
       "title": "a suite using an unadvertised RFC 0244 feature is refused at createRun",
       "explicitId": "openwop.requirement.0244.unadvertised-feature-refused",
       "citations": [
@@ -31436,7 +31464,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.under-passrule-mean-a-task-whose-trials-failed-their-trajectory-does-not-pass-rf",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 431,
+      "line": 461,
       "title": "under passRule \"mean\" a task whose trials failed their trajectory does not pass (RFC 0246)",
       "explicitId": "openwop.requirement.0246.mean-gated",
       "citations": [
@@ -31450,7 +31478,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-whose-tasks-failed-their-trajectory-does-not-pass-on-the-output-scores-r",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 444,
+      "line": 474,
       "title": "a suite whose tasks failed their trajectory does not pass on the output scores (RFC 0246)",
       "explicitId": "openwop.requirement.0246.suite-gated",
       "citations": [
@@ -31464,7 +31492,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-fixture-entry-marked-iserror-is-replayed-as-a-tool-error-not-a-success-rfc-024",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 455,
+      "line": 485,
       "title": "a fixture entry marked isError is replayed as a tool error, not a success (RFC 0245)",
       "explicitId": "openwop.requirement.0245.fixture-error-replayed",
       "citations": [
@@ -31478,7 +31506,7 @@
     {
       "id": "openwop.it.v2-eval-trajectory-trials.a-suite-using-iserror-is-refused-at-createrun-without-fixtureerrors-rfc-0245-c",
       "file": "v2-eval-trajectory-trials.test.ts",
-      "line": 469,
+      "line": 499,
       "title": "a suite using isError is refused at createRun without fixtureErrors (RFC 0245 §C)",
       "explicitId": "openwop.requirement.0245.unadvertised-refused",
       "citations": [

package/schemas/CORPUS-STAMP.json

@@ -1,17 +1,17 @@
 {
   "_comment": "Provenance of @openwop/spec-artifacts (RFC 0168 §D.2). files: SHA-256 per file; the conformance suite compares the installed peer against dist/spec-artifacts.lock.json at start.",
   "package": "@openwop/spec-artifacts",
-  "version": "2.46.25",
-  "corpusTag": "v2.46.25",
+  "version": "2.46.26",
+  "corpusTag": "v2.46.26",
   "files": {
     "api/.redocly.lint-ignore.yaml": "bf5a8350b88a72fa43f59605ed8d903ed24b6cfccda5e45509c9f6ed9ee4e712",
     "api/asyncapi.yaml": "d5ecb9ee6114582be3b1f662c84bfac9ae96dae7bacb853e461168f70a8e1c7d",
     "api/grpc/openwop.proto": "c3e72bb17cba514ee98feb6434e6c9b6ea6795bfd086489ec69fd882dd1ad977",
     "api/openapi.yaml": "ac6de5303872664b78ff419c035915593e01599e02835e777a0123de1baeb398",
     "api/redocly.yaml": "b0604c89b2ca6d5076ec25725c539dad44a741a811fe524439ee6daef8baa09f",
-    "api/seams-v2.yaml": "8fd581fc6b17706f32f6793d5cf00fd6de9c1d4791645d37efe90df2d430d9d5",
-    "api/v2/asyncapi.yaml": "f1a40cc22ed7c3c7069e49ff50da59b893f8cac43474fed8f48b4dbed4bdf5a2",
-    "api/v2/openapi.yaml": "8287abaade3983fc4546cb2a7c7d5e4e50595377b4fdf53ca65738d7284ce1a4",
+    "api/seams-v2.yaml": "fc5aaff321913531e04b13522de92a9a8fa3f61be836696e9ab83fbc563caa87",
+    "api/v2/asyncapi.yaml": "0ea85f3cb0af276cd7ed82e14810c1093917f0d1daa80bc0226b78e127c4e17d",
+    "api/v2/openapi.yaml": "7c7cd37cc0cabcd37285502594977f6ddc5a59f30938d7aede36f69e05322244",
     "api/v2/redocly.yaml": "1e66b60e6118ad11a823bb620678be464d99dfe50a40e3e6f93ec9429b88b34c",
     "schemas/README.md": "276fc1d51eb08e14e9127b9fd27761d7046ee33c12000f3a64a801322db8bf58",
     "schemas/a2a-task-state.schema.json": "75d5049dea8bd873ff0e7546f1c60c8d36c219bec7084264360be54a8be30ae1",
@@ -212,7 +212,7 @@
     "schemas/workspace-file.schema.json": "464de85c2a068243084ee9c1d969bc7cd5d8f7948574e58450d6493c38a0e1e4",
     "spec/v1/alias-detectors.json": "22444c22bb88c568339ef512553f9fc38146e22b566d11ab8608f655d688d19c",
     "spec/v1/capability-declaration-classes.json": "e7729aed5c4b4e1dd02abab0530f14cc95f5d4070fe51fb139e7f5cccefa00c6",
-    "spec/v1/core-standard-manifest.json": "c3705885e787498f604ffd256c4d6e26afcc929a79a548b7ecd46bf87e9e8a0f",
+    "spec/v1/core-standard-manifest.json": "580b853bb37383d5acea488fba6747193d1db4e6e63025255433dababf008d60",
     "spec/v1/deprecations.json": "e723bc11d7272c71789ad1303bb7f21809a8a024abf57594f9d5c35dc199ece9",
     "spec/v1/deprecations.schema.json": "3e393c405d2a41b467d8c5e3c468549078df1ce6a6d2588fc488097b95d9b55b",
     "spec/v1/end-of-support.json": "35f0bf3b70fe78b16e2a2fd81a153fc27e703f6c16376c6fe7cdf1f4f2c4e6d4",
@@ -318,9 +318,9 @@
     "spec/v2/path-manifest.json": "20f598f56df460ad3ed50aba3d6bc2933ea16a3c96ce125da1c5dd6989d4155c",
     "spec/v2/peer-dependency-aliases.json": "d10299280abee08258502925bc327293ee413e0108cd6e6ec75ff6110653308d",
     "spec/v2/profiles.json": "b382b261c7844798575f7c1b37c934f0da19846016a2337d883d4f5ef6e05479",
-    "spec/v2/release.json": "0d42cf69cb062a05c6b7d7d09dea7bbc44e918ffb056112303459883ddd032f0",
+    "spec/v2/release.json": "e2a9a7d13015eb3ca85bdbcd6ec422b6e91780fb9019b0e75e2855b826728b1d",
     "spec/v2/retention-floors.json": "eaf3722d95c79947af1d4269ef85117e126518c588cfcf1a2b21b97269f51624",
-    "spec/v2/surface-baseline.json": "585d61f9dae285fdb9e302568fefc2d91faaa166bd6a8c20e3bac1a82940f8af"
+    "spec/v2/surface-baseline.json": "4a1e5ab2349349eeeb9fa173e392627d49c2bee77ac8a42897aeacf2c3a4d784"
   },
-  "corpusCommit": "77aeb0cacb91551dab835cf0e985be8423e06542"
+  "corpusCommit": "e5e8306a4f9c8c41394cfbe63fb58a08a74fe5b9"
 }

package/src/lib/eval-efficiency-witness.ts

@@ -56,3 +56,84 @@ export function judgeToolCallCount(summary: Rec, tasks: readonly ScriptedTask[],
   }
   return { applicable: true, findings };
 }
+
+/**
+ * The `provider.usage` payloads of the agent under test, in log order.
+ * `provider.usage` carries no `agentId`, so attribution is positional: a usage
+ * event counts while an `agent.invocation.started` of `agentId` is open and no
+ * other agent's invocation (a judge, a delegate) is open inside it. Brackets
+ * are emitted only by an `agents.liveRuntime` host; `bracketed: false` means
+ * the log has none, and the usage list is then every `provider.usage` of the run.
+ */
+export function agentUsage(events: readonly { readonly type: string; readonly payload: Rec }[], agentId: string): { bracketed: boolean; usage: Rec[] } {
+  const open = new Map<string, boolean>();
+  let mine = 0;
+  let other = 0;
+  let bracketed = false;
+  const inside: Rec[] = [];
+  const all: Rec[] = [];
+  for (const e of events) {
+    const id = e.payload['invocationId'];
+    if (e.type === 'agent.invocation.started' && typeof id === 'string') {
+      bracketed = true;
+      const isMine = e.payload['agentId'] === agentId;
+      open.set(id, isMine);
+      if (isMine) mine += 1; else other += 1;
+    } else if (e.type === 'agent.invocation.completed' && typeof id === 'string' && open.has(id)) {
+      if (open.get(id)) mine -= 1; else other -= 1;
+      open.delete(id);
+    } else if (e.type === 'provider.usage') {
+      all.push(e.payload);
+      if (mine > 0 && other === 0) inside.push(e.payload);
+    }
+  }
+  return bracketed ? { bracketed, usage: inside } : { bracketed, usage: all };
+}
+
+const FIELDS = [['modelCallCount', () => 1], ['inputTokens', (u: Rec) => u['inputTokens']], ['outputTokens', (u: Rec) => u['outputTokens']]] as const;
+const close = (a: number, b: number): boolean => Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(b));
+
+/**
+ * §C `modelCallCount`, `inputTokens` and `outputTokens`. Each is a mean over
+ * `trials` identical trials, so the per-task values times `trials` sum to the
+ * agent's usage in the log. With brackets the sums MUST be equal; without them
+ * the run's usage also holds any host-internal call, so the sums MUST NOT
+ * exceed it. A field is judged only when every task reports it. A run with no
+ * usage and zero counts witnesses nothing (`applicable: false`).
+ */
+export function judgeModelUsage(summary: Rec, taskIds: readonly string[], log: { bracketed: boolean; usage: readonly Rec[] }, trials: number): EfficiencyVerdict {
+  const entries = (Array.isArray(summary['tasks']) ? (summary['tasks'] as unknown[]) : []).filter(isRec);
+  const effs = taskIds.map((id) => entries.find((e) => e['taskId'] === id)?.['efficiency']);
+  const findings: EfficiencyFinding[] = [];
+  let witnessed = false;
+  for (const [field, of] of FIELDS) {
+    const vals = effs.map((x) => (isRec(x) ? x[field] : undefined));
+    if (!vals.every((v) => typeof v === 'number')) continue;
+    const reported = (vals as number[]).reduce((a, b) => a + b, 0) * trials;
+    const inLog = log.usage.reduce((a, u) => a + (typeof of(u) === 'number' ? (of(u) as number) : 0), 0);
+    if (reported === 0 && inLog === 0) continue;
+    witnessed = true;
+    findings.push(log.bracketed
+      ? { ok: close(reported, inLog), message: `the per-task efficiency.${field} values times ${trials} trial(s) MUST sum to the ${inLog} the agent's provider.usage events inside its invocation brackets carry (got ${reported})` }
+      : { ok: reported <= inLog + 1e-9 * Math.max(1, inLog), message: `the per-task efficiency.${field} values times ${trials} trial(s) MUST NOT exceed the ${inLog} the run's provider.usage events carry (got ${reported})` });
+  }
+  return witnessed ? { applicable: true, findings } : { applicable: false, findings: [] };
+}
+
+/**
+ * §B "cost counts every trial": on a multi-trial run the summary's
+ * `totalCostUsd` holds the `costEstimateUsd` of the agent's usage in every
+ * trial, so a host that priced only one trial reads short. Only that floor is
+ * judged: whether a judge's or the host's own calls are priced in is not
+ * pinned, so a higher total is not a failure. Judged only
+ * with brackets (otherwise the agent's usage cannot be told from the host's
+ * own) and only when every attributed usage event carries a cost.
+ */
+export function judgeCostEveryTrial(summary: Rec, log: { bracketed: boolean; usage: readonly Rec[] }): EfficiencyVerdict {
+  const total = summary['totalCostUsd'];
+  if (typeof total !== 'number' || !log.bracketed || log.usage.length === 0) return { applicable: false, findings: [] };
+  const costs = log.usage.map((u) => u['costEstimateUsd']);
+  if (!costs.every((c) => typeof c === 'number')) return { applicable: false, findings: [] };
+  const sum = (costs as number[]).reduce((a, b) => a + b, 0);
+  return { applicable: true, findings: [{ ok: total >= sum || close(total, sum), message: `totalCostUsd MUST count every trial: it MUST be at least the ${sum} costEstimateUsd of the agent's ${log.usage.length} provider.usage events across all trials (got ${total})` }] };
+}

package/src/scenarios/v2-eval-trajectory-trials.test.ts

@@ -38,6 +38,15 @@
  *      optional, so a host that reports none records `inapplicable`. Judge in
  *      `lib/eval-efficiency-witness.ts`. The never-gated rule is not judged:
  *      no fixture makes efficiency the only difference between two verdicts.
+ *   6c. §C model usage and §B cost (2.46.26), on the same runs. The per-task
+ *      `modelCallCount`, `inputTokens` and `outputTokens`, times the trial
+ *      count, sum to the agent's `provider.usage` inside its
+ *      `agent.invocation.*` brackets (judge and delegate calls excluded), or do
+ *      not exceed the run's usage when the host emits no brackets
+ *      (`0244.efficiency-model-usage`). On the trials run, `totalCostUsd` is
+ *      at least the agent's priced usage over all three trials
+ *      (`0244.cost-every-trial`). The scripted agent need not call a model, so
+ *      a run with no usage, no brackets or no prices records `inapplicable`.
  *   6b. §D through the counter tool `conformance.eval:counter` (2.46.25, G2),
  *      gated on `fixtureMatching` and both counter fixtures in discovery.
  *      Three sequential runs: `conformance-eval-counter-live` (A), then
@@ -100,7 +109,7 @@ import { judgeEvalEnvelope, judgeSummaryServed } from '../lib/eval-envelope-witn
 import { gradeTrajectory, passAtK, passHatK, scoreStderr, withinTolerance, type TrajectorySpec } from '../lib/eval-trajectory.js';
 import { judgeMeanGated, judgeSuiteGated } from '../lib/eval-gate-witness.js';
 import { judgeFixtureErrorReplayed } from '../lib/eval-fixture-error-witness.js';
-import { judgeToolCallCount } from '../lib/eval-efficiency-witness.js';
+import { agentUsage, judgeCostEveryTrial, judgeModelUsage, judgeToolCallCount } from '../lib/eval-efficiency-witness.js';
 import { counterCalls, judgeFirstUnconsumed, judgeLiveMiss, judgeMissNoLiveCall, type CounterCall } from '../lib/eval-counter-witness.js';
 
 const PROFILE = majorProfile(2);
@@ -117,6 +126,8 @@ const ID_TRIAL_STATS = 'openwop.requirement.0244.trial-stats-formula';
 const ID_MISS = 'openwop.requirement.0244.fixture-miss-fails';
 const ID_REFUSED = 'openwop.requirement.0244.unadvertised-feature-refused';
 const ID_EFFICIENCY = 'openwop.requirement.0244.efficiency-tool-calls';
+const ID_MODEL_USAGE = 'openwop.requirement.0244.efficiency-model-usage';
+const ID_COST_TRIALS = 'openwop.requirement.0244.cost-every-trial';
 const ID_LIVE_MISS = 'openwop.requirement.0244.live-miss-counted';
 const ID_FIRST_UNCONSUMED = 'openwop.requirement.0244.first-unconsumed';
 const ID_NO_LIVE_CALL = 'openwop.requirement.0244.miss-no-live-call';
@@ -346,6 +357,25 @@ describe('v2 RFC 0244 — host behaviour', () => {
     for (const v of verdicts) for (const x of v.findings) expect(x.ok, req(ID_EFFICIENCY, 'eval-summary.schema.json efficiency (RFC 0244 §C)', x.message)).toBe(true);
   });
 
+  it('efficiency model and token counts match the agent\'s provider.usage (§C)', async () => {
+    const r = await trajectoryRun();
+    if ('skip' in r) return softSkip(...r.skip);
+    const ids = (s: Suite) => s.tasks.map((t) => t.taskId);
+    const verdicts = [judgeModelUsage(r.o.summary, ids(fixture('conformance-eval-trajectory')), agentUsage(r.o.events, AGENT), 1)];
+    const t = await trialsRun();
+    if (!('skip' in t)) verdicts.push(judgeModelUsage(t.o.summary, ids(t.suite), agentUsage(t.o.events, AGENT), t.n));
+    if (!verdicts.some((v) => v.applicable)) return softSkip('inapplicable', 'no run carries both a reported model count and provider.usage to check it against (the scripted agent need not call a model; RFC 0244 §C fields are optional)');
+    for (const v of verdicts) for (const x of v.findings) expect(x.ok, req(ID_MODEL_USAGE, 'eval-summary.schema.json efficiency (RFC 0244 §C)', x.message)).toBe(true);
+  });
+
+  it('totalCostUsd counts every trial (§B)', async () => {
+    const t = await trialsRun();
+    if ('skip' in t) return softSkip(...t.skip);
+    const v = judgeCostEveryTrial(t.o.summary, agentUsage(t.o.events, AGENT));
+    if (!v.applicable) return softSkip('inapplicable', 'the trials run carries no totalCostUsd, no agent.invocation.* brackets, or an unpriced provider.usage, so the per-trial cost is not observable');
+    for (const x of v.findings) expect(x.ok, req(ID_COST_TRIALS, 'agent-eval-suite.schema.json thresholds.maxCostUsd, eval-summary.schema.json totalCostUsd (RFC 0244 §B)', x.message)).toBe(true);
+  });
+
   type CounterRuns = { skip: [Parameters<typeof softSkip>[0], string] } | { a: CounterCall[]; b: CounterCall[]; c: CounterCall[]; aMisses: unknown };
   let counterSeq: Promise<CounterRuns> | undefined;
   /** A → B → C, strictly in order, once per worker: the three counter legs read the same runs. */