driftproof 0.11.2 → 0.12.0

package/spec/receipt.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://driftproofhq.com/spec/receipt.schema.json",
   "title": "driftproof receipt",
-  "description": "A hash-verified, dated record of running one agent skill eval suite with and without the skill on one model version, with the GENERATION sampled n times per arm and the judge sampled k times inside each draw. Receipt spec v0.7: over v0.6, the fields are unchanged and the verdict a reader derives gains UNDERPOWERED: a case the band rule does not separate whose spreads and measured draws cannot resolve a shift of the effect floor (spec 035, spec/RECEIPT.md § What v0.7 adds). Over v0.5, as v0.6 said, the receipt says what answered it: `run.answered_by` (kind model|stub|external, whether the surface echoed a model id and which, and which spawn path produced it); `run.surface` and `run.judge.surface` may read `stub` (nothing answered; the text was canned) and a TESTED receipt requires answered_by.kind model; `run.judge.model_id` and `run.judge.prompt_template_hash` say which judge ran; every draw records its `stop_reason` and whether it was `truncated` (a truncated draw is unmeasured, never judged) and the case counts `n_truncated`; `case_status` gains `failed_unmeasured` (every draw of the arm was unmeasured for a reason that is not a timeout), recorded without fabricated samples or hashes exactly as a timeout is; `results.aggregates.band_rule` states the formula the aggregate band is derived by; an aggregate `stddev` is null only when its arm has fewer than two cases and `comparison.delta_uncertainty` is null only beside `delta_uncertainty_unavailable`. Every prior receipt still validates against its own frozen schema (v0.1, v0.2, v0.3, v0.3.1, v0.4, v0.5, v0.6).",
+  "description": "Receipt spec v0.9: over v0.8, imported receipts say what produced them (spec 049, spec/RECEIPT.md § What v0.9 adds). Every field it adds is optional: `run.harness` (the agent harness that produced the answers), `run.import` (the imported document: tool, format, format version, sha256, the import time, any sidecar file read, and a notice for each thing the source did not establish), `skill.unit` (a skill, or a plugin measured whole), and per case `activation` (whether the skill fired, never a score), `excluded_draws` (runs kept out of the band, with their reasons) and `label` (the source's display name). `economics.basis` adds `source-list-price-estimate` and `source-reported`, an arm may carry `mean_total_tokens`, `run.provider` may be `unknown`, and `run.date_utc` may be null on an external receipt whose source records no date. v0.8 is frozen as receipt.v0.8.schema.json. Over v0.7, v0.8 said: counts and clocks (spec 043, spec/RECEIPT.md § What v0.8 adds). `run.judge.samples` is optional, so an absent count stays absent; generation and judge counts are separate optional fields, `generations_per_arm` and `judge_samples_per_generation`, held in a `counts` object at the narrowest honest scope (`run.counts`, `run.arms.<mode>.counts`, `results.cases[].counts`); `run.arms.<mode>` may name an archived arm's own `model_id` and `generated_at`; `run.generated_at` is when generation began, and `run.judged_at` with `run.grader_revision` record a re-judge and appear together or not at all; a case may be `no_observations`. v0.7 is frozen as receipt.v0.7.schema.json. A hash-verified, dated record of running one agent skill eval suite with and without the skill on one model version, with the GENERATION sampled n times per arm and the judge sampled k times inside each draw. Over v0.6, v0.7 said: the fields are unchanged and the verdict a reader derives gains UNDERPOWERED: a case the band rule does not separate whose spreads and measured draws cannot resolve a shift of the effect floor (spec 035, spec/RECEIPT.md § What v0.7 adds). Over v0.5, as v0.6 said, the receipt says what answered it: `run.answered_by` (kind model|stub|external, whether the surface echoed a model id and which, and which spawn path produced it); `run.surface` and `run.judge.surface` may read `stub` (nothing answered; the text was canned) and a TESTED receipt requires answered_by.kind model; `run.judge.model_id` and `run.judge.prompt_template_hash` say which judge ran; every draw records its `stop_reason` and whether it was `truncated` (a truncated draw is unmeasured, never judged) and the case counts `n_truncated`; `case_status` gains `failed_unmeasured` (every draw of the arm was unmeasured for a reason that is not a timeout), recorded without fabricated samples or hashes exactly as a timeout is; `results.aggregates.band_rule` states the formula the aggregate band is derived by; an aggregate `stddev` is null only when its arm has fewer than two cases and `comparison.delta_uncertainty` is null only beside `delta_uncertainty_unavailable`. Every prior receipt still validates against its own frozen schema (v0.1, v0.2, v0.3, v0.3.1, v0.4, v0.5, v0.6).",
   "type": "object",
   "additionalProperties": false,
   "required": [
@@ -274,7 +274,7 @@
   "properties": {
     "schema_version": {
       "type": "string",
-      "const": "0.7"
+      "const": "0.9"
     },
     "skill": {
       "type": "object",
@@ -305,6 +305,14 @@
           "type": "integer",
           "minimum": 0,
           "description": "v0.3.1 (optional). Estimated token size of the skill's SKILL.md (a coarse chars/4 proxy, not a model tokenizer), used for the value-per-token axis (delta per 1k skill tokens). See docs/methodology.html."
+        },
+        "unit": {
+          "type": "string",
+          "enum": [
+            "skill",
+            "plugin"
+          ],
+          "description": "v0.9 (optional). What was measured: one skill, or a plugin loaded whole with every skill it carries (a claude plugin eval import). Absent means a skill."
         }
       }
     },
@@ -370,10 +378,11 @@
         },
         "provider": {
           "type": "string",
-          "description": "v0.3.1. The two-axis provider the target model ran on (registry `provider`, else inferred from the id).",
+          "description": "v0.3.1. The two-axis provider the target model ran on (registry `provider`, else inferred from the id). v0.9: `unknown` on an imported receipt whose source records no model.",
           "enum": [
             "anthropic",
-            "openai"
+            "openai",
+            "unknown"
           ]
         },
         "surface": {
@@ -415,8 +424,11 @@
           "minLength": 1
         },
         "date_utc": {
-          "type": "string",
-          "description": "ISO 8601 UTC timestamp of when the run finished.",
+          "type": [
+            "string",
+            "null"
+          ],
+          "description": "ISO 8601 UTC. As lib/run.js writes it: the time the run's calls finished when the CLI stamps it, or the stamp a caller passes, which for a batch is one stamp shared by every receipt in the batch. For when generation began, read run.generated_at (v0.8). Receipts before v0.8 described this field as when the run finished; that was true of the CLI and not of a batch. v0.9: null only on an external (imported) receipt whose source records no date; an import never writes the time it ran here (that is run.import.imported_at).",
           "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}"
         },
         "registry": {
@@ -441,7 +453,6 @@
           "additionalProperties": false,
           "description": "Judge sampling settings for this run.",
           "required": [
-            "samples",
             "temperature",
             "sampling",
             "model_id",
@@ -451,7 +462,7 @@
             "samples": {
               "type": "integer",
               "minimum": 1,
-              "description": "Judge samples taken per case."
+              "description": "Judge samples taken per generation. v0.8: optional; absent means the count is unknown, never 1. When present it equals run.counts.judge_samples_per_generation if that is present (the loader refuses a receipt where they differ)."
             },
             "temperature": {
               "type": [
@@ -587,8 +598,214 @@
               ]
             }
           }
+        },
+        "generated_at": {
+          "type": "string",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}",
+          "description": "v0.8 (optional). When generation began for this receipt, stamped by the runner before its first generation call. An archived arm's own generation time is in run.arms.<mode>.generated_at."
+        },
+        "judged_at": {
+          "type": "string",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}",
+          "description": "v0.8 (optional). Present only on a re-judge: when frozen outputs were graded again. Requires grader_revision."
+        },
+        "grader_revision": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "prompt_template_hash",
+            "rubric_hashes"
+          ],
+          "description": "v0.8 (optional). Present only on a re-judge, with judged_at: the grading definition that re-judge applied.",
+          "properties": {
+            "prompt_template_hash": {
+              "type": [
+                "string",
+                "null"
+              ]
+            },
+            "rubric_hashes": {
+              "type": "array",
+              "items": {
+                "type": [
+                  "string",
+                  "null"
+                ]
+              }
+            }
+          }
+        },
+        "counts": {
+          "$ref": "#/$defs/counts"
+        },
+        "arms": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "with_skill": {
+              "type": "object",
+              "additionalProperties": false,
+              "description": "v0.8 (optional). A per-arm override. An arm carrying its own generated_at is archived: it was generated in an earlier run, and it never inherits run.counts; an absent model_id or generated_at inherits the run's.",
+              "properties": {
+                "model_id": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "generated_at": {
+                  "type": "string",
+                  "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}"
+                },
+                "counts": {
+                  "$ref": "#/$defs/counts"
+                }
+              }
+            },
+            "baseline": {
+              "type": "object",
+              "additionalProperties": false,
+              "description": "v0.8 (optional). A per-arm override. An arm carrying its own generated_at is archived: it was generated in an earlier run, and it never inherits run.counts; an absent model_id or generated_at inherits the run's.",
+              "properties": {
+                "model_id": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "generated_at": {
+                  "type": "string",
+                  "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}"
+                },
+                "counts": {
+                  "$ref": "#/$defs/counts"
+                }
+              }
+            }
+          }
+        },
+        "harness": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "name",
+            "version"
+          ],
+          "description": "v0.9 (optional). The agent harness that produced the answers, as the source names it (e.g. claude-code and its version for a claude plugin eval import). version is null when the source does not record it.",
+          "properties": {
+            "name": {
+              "type": "string",
+              "minLength": 1
+            },
+            "version": {
+              "type": [
+                "string",
+                "null"
+              ]
+            }
+          }
+        },
+        "import": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "tool",
+            "format",
+            "format_version",
+            "source_sha256",
+            "imported_at"
+          ],
+          "description": "v0.9 (optional). The document an imported receipt was converted from. imported_at is the only place the import time appears.",
+          "properties": {
+            "tool": {
+              "type": "string",
+              "minLength": 1,
+              "description": "The --from value, e.g. claude-plugin-eval or skill-creator."
+            },
+            "format": {
+              "type": "string",
+              "minLength": 1,
+              "description": "The document read, e.g. aggregate-result.json or benchmark.json."
+            },
+            "format_version": {
+              "type": [
+                "string",
+                "integer",
+                "null"
+              ],
+              "description": "The document's own version field as written (claude plugin eval's schemaVersion), or null when the format carries none."
+            },
+            "source_sha256": {
+              "type": "string",
+              "pattern": "^[a-f0-9]{64}$",
+              "description": "sha256 of the document's bytes."
+            },
+            "imported_at": {
+              "type": "string",
+              "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}",
+              "description": "ISO 8601 UTC: when the import ran. Never the run date."
+            },
+            "sidecars": {
+              "type": "array",
+              "description": "Other files the import read beside the document (skill-up's result.json), by name and sha256.",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "file",
+                  "sha256"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "sha256": {
+                    "type": "string",
+                    "pattern": "^[a-f0-9]{64}$"
+                  }
+                }
+              }
+            },
+            "notices": {
+              "type": "array",
+              "description": "One sentence per thing the source did not establish, or per choice the import made that a reader must know (an unknown model, a date read from a directory name, errored runs counted).",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            }
+          }
         }
-      }
+      },
+      "dependentRequired": {
+        "judged_at": [
+          "grader_revision"
+        ],
+        "grader_revision": [
+          "judged_at"
+        ]
+      },
+      "allOf": [
+        {
+          "description": "v0.9: run.date_utc may be null only on an external (imported) receipt.",
+          "if": {
+            "not": {
+              "properties": {
+                "surface": {
+                  "const": "external"
+                }
+              },
+              "required": [
+                "surface"
+              ]
+            }
+          },
+          "then": {
+            "properties": {
+              "date_utc": {
+                "type": "string"
+              }
+            }
+          }
+        }
+      ]
     },
     "results": {
       "type": "object",
@@ -636,7 +853,12 @@
                     "stddev",
                     "samples",
                     "judge"
-                  ]
+                  ],
+                  "properties": {
+                    "samples": {
+                      "minItems": 1
+                    }
+                  }
                 },
                 "else": {
                   "required": [
@@ -661,11 +883,12 @@
               },
               "case_status": {
                 "type": "string",
-                "description": "v0.3.1 (optional; default 'ok'). 'failed_timeout' = the case's model/judge call persistently timed out after retries. v0.6: 'failed_unmeasured' = every draw of the arm was unmeasured for a reason that is not a timeout (an empty generation, a judge output with no score, a truncated draw). Either way the case is recorded but EXCLUDED from aggregates/verdicts — no samples/hashes are fabricated for it.",
+                "description": "v0.3.1 (optional; default 'ok'). 'failed_timeout' = the case's model/judge call persistently timed out after retries. v0.6: 'failed_unmeasured' = every draw of the arm was unmeasured for a reason that is not a timeout (an empty generation, a judge output with no score, a truncated draw). Either way the case is recorded but EXCLUDED from aggregates/verdicts; no samples/hashes are fabricated for it. v0.8: 'no_observations' = the source supplied no observation for the case (an imported task whose trials list is empty, samples [], or absent, no samples); recorded, excluded from aggregates, named in excluded_cases.",
                 "enum": [
                   "ok",
                   "failed_timeout",
-                  "failed_unmeasured"
+                  "failed_unmeasured",
+                  "no_observations"
                 ]
               },
               "outcome": {
@@ -701,7 +924,7 @@
                   "minimum": 0,
                   "maximum": 1
                 },
-                "minItems": 1
+                "minItems": 0
               },
               "generation_hash": {
                 "type": "string",
@@ -1108,6 +1331,65 @@
                     }
                   }
                 ]
+              },
+              "counts": {
+                "$ref": "#/$defs/counts"
+              },
+              "label": {
+                "type": "string",
+                "minLength": 1,
+                "description": "v0.9 (optional). The source's display name for the case (benchmark.json eval_name). Cases are identified by id, never by label."
+              },
+              "activation": {
+                "type": "array",
+                "minItems": 1,
+                "description": "v0.9 (optional). Whether the skill fired, per unscored skill-invocation indicator the source ran (a claude plugin eval tool_used: Skill grader with scored: false): fired of runs, over the measured draws of this case and arm. Never part of any score, band or comparison.",
+                "items": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "indicator",
+                    "fired",
+                    "runs"
+                  ],
+                  "properties": {
+                    "indicator": {
+                      "type": "string",
+                      "minLength": 1
+                    },
+                    "fired": {
+                      "type": "integer",
+                      "minimum": 0
+                    },
+                    "runs": {
+                      "type": "integer",
+                      "minimum": 1
+                    }
+                  }
+                }
+              },
+              "excluded_draws": {
+                "type": "array",
+                "minItems": 1,
+                "description": "v0.9 (optional). Runs the source recorded for this case and arm that are kept out of its samples, each with its 1-based position in the source's list (or its run number) and the reason.",
+                "items": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "draw_index",
+                    "reason"
+                  ],
+                  "properties": {
+                    "draw_index": {
+                      "type": "integer",
+                      "minimum": 1
+                    },
+                    "reason": {
+                      "type": "string",
+                      "minLength": 1
+                    }
+                  }
+                }
               }
             }
           }
@@ -1295,9 +1577,11 @@
         "basis": {
           "enum": [
             "metered",
-            "metered-equivalent"
+            "metered-equivalent",
+            "source-list-price-estimate",
+            "source-reported"
           ],
-          "description": "metered = real spend on an api surface; metered-equivalent = what the same tokens would have cost on the metered API (subscription CLI surfaces, where actual spend is $0)."
+          "description": "metered = real spend on an api surface; metered-equivalent = what the same tokens would have cost on the metered API (subscription CLI surfaces, where actual spend is $0). v0.9, imported receipts only: source-list-price-estimate = the source tool's own cost estimate at list price, not a Driftproof figure; source-reported = tokens and time as the source reports them, with no cost."
         },
         "surface": {
           "type": "string"
@@ -1351,6 +1635,13 @@
                 "number",
                 "null"
               ]
+            },
+            "mean_total_tokens": {
+              "type": [
+                "number",
+                "null"
+              ],
+              "description": "v0.9 (optional). Mean tokens per call where the source reports one total (benchmark.json result.tokens), not input and output apart."
             }
           }
         },
@@ -1403,6 +1694,13 @@
                 "number",
                 "null"
               ]
+            },
+            "mean_total_tokens": {
+              "type": [
+                "number",
+                "null"
+              ],
+              "description": "v0.9 (optional). Mean tokens per call where the source reports one total (benchmark.json result.tokens), not input and output apart."
             }
           }
         },
@@ -1558,6 +1856,23 @@
           }
         }
       ]
+    },
+    "counts": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "v0.8. How the data came to be, at the scope it holds: every case of every arm generated in this run (run.counts), one arm (run.arms.<mode>.counts), or one case and arm (results.cases[].counts). A reader takes the narrowest scope present. An absent field is unknown, never 1.",
+      "properties": {
+        "generations_per_arm": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Independent candidate generations. 0 only where the source establishes zero."
+        },
+        "judge_samples_per_generation": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Judge samples taken of each generation."
+        }
+      }
     }
   }
 }