champollion 0.3.3

package/shared/schemas/metric-registry.schema.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://champollion.dev/schemas/metric-registry.schema.json",
+  "title": "Champollion Metric Identity Registry (SSOT)",
+  "description": "Declarative registry of evaluation-metric identity — the one place that maps each canonical metric id to its Python plugin name, language-card evalMetrics key, and denormalized run_cards DB column. Shared by the Python arena harness and the JS CLI/website. See shared/metric-registry.json and cli/website/docs/network/specifications/scoring.md §2.",
+  "type": "object",
+  "required": ["version", "entries"],
+  "additionalProperties": true,
+  "properties": {
+    "_comment": { "type": "string" },
+    "version": { "type": "integer", "minimum": 1 },
+    "entries": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": { "$ref": "#/$defs/entry" },
+      "propertyNames": {
+        "pattern": "^[a-z0-9][a-z0-9_]*$",
+        "description": "canonical_id: the run-card scores key (snake_case). Also the primary lookup key across all runtimes."
+      }
+    },
+    "_proposed_renames": {
+      "type": "array",
+      "description": "Rename proposals the FOUNDER decides (published run cards embed these names). Advisory — not consumed by code.",
+      "items": {
+        "type": "object",
+        "required": ["current", "proposal", "impact", "decision"],
+        "additionalProperties": false,
+        "properties": {
+          "current": { "type": "string" },
+          "proposal": { "type": "string" },
+          "impact": { "type": "string" },
+          "decision": { "type": "string", "enum": ["founder-pending", "accepted", "rejected"] }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "entry": {
+      "type": "object",
+      "required": [
+        "category", "status", "display_name", "plugin_name", "card_key",
+        "db_column", "scale", "direction", "level", "in_composite",
+        "verifier_reproducible"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "category": {
+          "type": "string",
+          "enum": ["surface", "structural", "semantic", "neural", "behavioral", "compliance", "comparator", "composite", "efficiency"],
+          "description": "The scoring-spec §2 category (plus composite/efficiency for §4/§6/§7 derived figures)."
+        },
+        "status": {
+          "type": "string",
+          "enum": ["implemented", "partial", "planned", "proposed"],
+          "description": "Scoring spec §3 status tier: implemented (values in run cards today), partial (language-specific proxy only), planned (specified, not built), proposed (under discussion)."
+        },
+        "display_name": { "type": "string", "minLength": 1 },
+        "plugin_name": {
+          "type": ["string", "null"],
+          "description": "The Python MetricPlugin `name` attribute that computes this metric, or null for harness-core metrics with no plugin (e.g. exact_match_rate, chrf_plus_plus) and derived figures."
+        },
+        "card_key": {
+          "type": ["string", "null"],
+          "description": "The language-card evalMetrics key that declares the plugin (e.g. 'lyss-eq'), or null when the metric is not card-declared (harness-core or always-loaded behavioral plugins)."
+        },
+        "db_column": {
+          "type": ["string", "null"],
+          "description": "The denormalized run_cards column (arena/DATABASE_SCHEMA.md), or null when the metric lives only in the run_card JSONB / plugin_metrics."
+        },
+        "scale": {
+          "type": "string",
+          "description": "Human-readable value range (e.g. '0.0-1.0', '0-100', '0-25', '0-inf', 'USD', or a label enumeration)."
+        },
+        "direction": {
+          "type": "string",
+          "enum": ["higher", "lower", "neutral"],
+          "description": "Whether higher or lower is better (neutral = diagnostic/informational, e.g. length_ratio, coverage, quality_tier label)."
+        },
+        "level": {
+          "type": "string",
+          "enum": ["entry", "corpus", "both"],
+          "description": "Whether the metric is computed per-entry, corpus-level, or both."
+        },
+        "in_composite": {
+          "type": "boolean",
+          "description": "True iff the metric currently carries weight in at least one active profile in scoring.PROFILE_REGISTRY when available (declared-but-INACTIVE metrics like orthographic_accuracy are false)."
+        },
+        "verifier_reproducible": {
+          "type": "boolean",
+          "description": "True iff the verifier can deterministically re-derive it from the sha-pinned corpus + stored entries (+ card-pinned FST / pinned neural model under the fail-closed contract). Speed/cost/style metrics are false."
+        },
+        "notes": { "type": "string" }
+      }
+    }
+  }
+}

package/shared/schemas/metric-reliability.schema.json

@@ -0,0 +1,178 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://champollion.dev/schemas/metric-reliability.schema.json",
+  "title": "Metric Reliability Index (WMT meta-evaluation, machine-derived)",
+  "description": "Correlations between automatic MT metrics and WMT Metrics-task human judgments (DA/MQM/ESA, wmt19-wmt25), per (test set, language pair, human lane, level, metric), rolled up per TARGET-language family (master plan B3). Every value is champollion-derived from pinned upstream data (google-research/mt-metrics-eval + google/wmt-mqm-human-evaluation); no upstream text, judgment or score is redistributed. RELIABILITY evidence, not quality scores. Non-commercial hold until the founder clears the upstream data license (license_lane). Regenerated by arena/scripts/import_wmt_metaeval.py; tracked in the monorepo, NOT bundled into the npm package or PyPI wheel.",
+  "type": "object",
+  "required": ["version", "provider", "sources", "provenance", "license_lane", "score_orientation", "correlation_definitions", "human_lanes", "metrics", "languages", "families", "cells", "excluded", "counts"],
+  "additionalProperties": false,
+  "properties": {
+    "_comment": { "type": "string" },
+    "version": { "type": "integer", "minimum": 1 },
+    "provider": { "enum": ["wmt-metrics-metaeval"] },
+    "sources": {
+      "type": "array",
+      "minItems": 2,
+      "items": {
+        "type": "object",
+        "required": ["name", "repo", "commit", "retrieved", "license", "credit"],
+        "additionalProperties": true,
+        "properties": {
+          "name": { "type": "string" },
+          "repo": { "type": "string", "format": "uri" },
+          "commit": { "type": "string", "pattern": "^[0-9a-f]{40}$", "description": "Upstream commit SHA pinned at probe time." },
+          "data_url": { "type": "string", "format": "uri" },
+          "data_sha256": { "type": ["string", "null"], "pattern": "^[0-9a-f]{64}$", "description": "sha256 of the downloaded data tarball — REQUIRED for the mt-metrics-eval source because the tarball is not immutable." },
+          "data_etag": { "type": ["string", "null"] },
+          "data_last_modified": { "type": ["string", "null"] },
+          "data_bytes": { "type": ["integer", "null"] },
+          "retrieved": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" },
+          "license": { "type": "string", "minLength": 5 },
+          "credit": { "type": "string", "minLength": 10, "description": "Human credit for the upstream project and annotators (always-cite-work principle)." },
+          "note": { "type": "string" }
+        }
+      }
+    },
+    "provenance": { "type": "string", "pattern": "champollion-derived", "description": "Must carry champollion-derived naming the upstreams: the correlations are our computation; the judgments and scores are theirs." },
+    "license_lane": {
+      "type": "object",
+      "required": ["commercial_ok", "note"],
+      "properties": {
+        "commercial_ok": { "enum": [false], "description": "Pinned false until the founder license review clears the upstream data — the bundled human-judgment/test-set text has no explicit upstream license statement." },
+        "note": { "type": "string", "minLength": 20 }
+      }
+    },
+    "score_orientation": { "type": "string", "minLength": 20 },
+    "correlation_definitions": {
+      "type": "object",
+      "required": ["sys", "seg"],
+      "properties": {
+        "sys": { "type": "string", "minLength": 20 },
+        "seg": { "type": "string", "minLength": 20 }
+      }
+    },
+    "human_lanes": {
+      "type": "object",
+      "required": ["preference", "note"],
+      "properties": {
+        "preference": { "type": "array", "minItems": 1, "items": { "type": "string" } },
+        "note": { "type": "string" }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "description": "Keyed by shared/metric-registry.json canonical id. Every cells[].metric MUST appear here (cross-checked by cli/test/metric-reliability.test.js).",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["registry_id", "upstream_candidates", "upstream_names_used", "note"],
+        "properties": {
+          "registry_id": { "type": "string" },
+          "upstream_candidates": { "type": "array", "items": { "type": "string" } },
+          "upstream_names_used": { "type": "array", "items": { "type": "string" } },
+          "note": { "type": "string" }
+        }
+      }
+    },
+    "languages": {
+      "type": "object",
+      "description": "Every src/tgt code used in cells -> iso639-3 + family/genus from cli/data/champollion.db.",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["iso639_3", "family"],
+        "properties": {
+          "iso639_3": { "type": "string", "minLength": 2 },
+          "family": { "type": ["string", "null"] },
+          "genus": { "type": ["string", "null"] }
+        }
+      }
+    },
+    "family_rollup_note": { "type": "string" },
+    "families": {
+      "type": "object",
+      "description": "TARGET-language family -> per-metric per-level weighted-mean correlations over preferred cells only.",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["n_pairs", "metrics"],
+        "properties": {
+          "n_pairs": { "type": "integer", "minimum": 1 },
+          "metrics": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "object",
+                "required": ["n_cells", "n_pairs", "weight", "pairs"],
+                "properties": {
+                  "n_cells": { "type": "integer", "minimum": 1 },
+                  "n_pairs": { "type": "integer", "minimum": 1 },
+                  "weight": { "type": "integer", "minimum": 1 },
+                  "pairs": { "type": "array", "items": { "type": "string" } },
+                  "pearson_weighted_mean": { "type": "number", "minimum": -1, "maximum": 1 },
+                  "spearman_weighted_mean": { "type": "number", "minimum": -1, "maximum": 1 },
+                  "kendall_tau_b_weighted_mean": { "type": "number", "minimum": -1, "maximum": 1 },
+                  "pairwise_accuracy_weighted_mean": { "type": "number", "minimum": 0, "maximum": 1 }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "cells": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["testset", "pair", "pair_verbatim", "src", "tgt", "human", "level", "preferred", "metric", "upstream_metric", "upstream_ref", "computed", "n_sys"],
+        "additionalProperties": false,
+        "properties": {
+          "testset": { "type": "string" },
+          "pair": { "type": "string", "pattern": "^[a-z]{2,3}-[a-z]{2,3}$" },
+          "pair_verbatim": { "type": "string" },
+          "src": { "type": "string" },
+          "tgt": { "type": "string" },
+          "human": { "type": "string" },
+          "level": { "enum": ["sys", "seg"] },
+          "preferred": { "type": "boolean" },
+          "metric": { "type": "string" },
+          "upstream_metric": { "type": "string" },
+          "upstream_ref": { "type": "string" },
+          "computed": { "type": "boolean", "description": "true = Champollion computed the metric scores (sacrebleu chrF++ lane); false = scores verbatim from the WMT submission/baseline files." },
+          "n_sys": { "type": "integer", "minimum": 2 },
+          "n_points": { "type": "integer", "minimum": 10 },
+          "pearson": { "type": ["number", "null"], "minimum": -1, "maximum": 1 },
+          "spearman": { "type": ["number", "null"], "minimum": -1, "maximum": 1 },
+          "kendall_tau_b": { "type": ["number", "null"], "minimum": -1, "maximum": 1 },
+          "pairwise_accuracy": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }
+        }
+      }
+    },
+    "excluded": {
+      "type": "array",
+      "description": "Fail-honest ledger: everything skipped, with a reason (label-honesty rule A4).",
+      "items": {
+        "type": "object",
+        "required": ["reason"],
+        "properties": {
+          "testset": { "type": "string" },
+          "pair": { "type": "string" },
+          "human": { "type": "string" },
+          "level": { "type": "string" },
+          "metric": { "type": "string" },
+          "reason": { "type": "string", "minLength": 10 }
+        }
+      }
+    },
+    "counts": {
+      "type": "object",
+      "required": ["cells", "preferred_cells", "testsets", "pairs", "families", "excluded"],
+      "properties": {
+        "cells": { "type": "integer", "minimum": 0 },
+        "preferred_cells": { "type": "integer", "minimum": 0 },
+        "testsets": { "type": "integer", "minimum": 0 },
+        "pairs": { "type": "integer", "minimum": 0 },
+        "families": { "type": "integer", "minimum": 0 },
+        "excluded": { "type": "integer", "minimum": 0 }
+      }
+    }
+  }
+}

package/shared/schemas/model-aliases.schema.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://champollion.dev/schemas/model-aliases.schema.json",
+  "title": "Champollion model aliases (shared/model-aliases.json)",
+  "description": "Cross-runtime SSOT mapping short convenience aliases (e.g. 'gemini-flash') to full provider-qualified model ids (e.g. 'google/gemini-3.5-flash'). Keys other than the underscore-prefixed metadata fields are alias names; their values are model ids. The bundled mini-schema validator does not support additionalProperties-as-schema, so per-entry validation iterates the entries against $defs.aliasName / $defs.modelId (see cli/test/shared-ssot-schemas.test.js).",
+  "type": "object",
+  "properties": {
+    "_comment": {
+      "type": "string",
+      "description": "Human note; not an alias."
+    }
+  },
+  "$defs": {
+    "aliasName": {
+      "type": "string",
+      "description": "Short alias key: lowercase alphanumerics and hyphens.",
+      "pattern": "^[a-z0-9][a-z0-9-]*$",
+      "minLength": 2
+    },
+    "modelId": {
+      "type": "string",
+      "description": "Full OpenRouter-style model id: <provider>/<model>.",
+      "pattern": "^[a-z0-9-]+/[A-Za-z0-9._-]+$",
+      "minLength": 3
+    }
+  }
+}

package/shared/schemas/source-snapshot.schema.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://champollion.dev/schemas/source-snapshot.schema.json",
+  "title": "Source snapshot",
+  "description": "The provenance record a fetcher writes into cli/data/<source>/SNAPSHOT.json. It is what makes 'regenerate the cards from source' a checkable claim rather than an aspiration: it names the upstream, pins ONE immutable release, and checksums the bytes taken from it. A source with no snapshot cannot be rebuilt, and facts derived from it carry no release.",
+  "type": "object",
+  "required": ["source", "upstream", "files", "verified", "fetchedBy", "fetchedAt"],
+  "additionalProperties": false,
+  "properties": {
+    "$schema": { "type": "string" },
+    "source": {
+      "type": "string",
+      "description": "The source name used in facts.source. Must match exactly, or the pin attaches to nothing.",
+      "minLength": 1
+    },
+    "upstream": {
+      "type": "string",
+      "description": "The URL the fetcher read. A landing page for a versioned deposit is fine; an undated 'latest' file URL is not a pin and must be reflected by pin:null."
+    },
+    "license": {
+      "type": ["string", "null"],
+      "description": "The licence the UPSTREAM declares, verbatim (SPDX id where the upstream gives one). Never our reading of it — that lives in the licence register, which this evidences."
+    },
+    "licenseUrl": { "type": ["string", "null"] },
+    "citation": {
+      "type": ["string", "null"],
+      "description": "How the upstream asks to be cited."
+    },
+    "pin": {
+      "type": ["object", "null"],
+      "description": "The immutable release identifier. NULL means this source could not be pinned — which is a real state and must not be faked. Downstream, no pin means no source_release row, which means every fact from it is flagged not-yet-regenerable.",
+      "required": ["kind", "value"],
+      "additionalProperties": false,
+      "properties": {
+        "kind": {
+          "enum": ["doi", "commit", "release", "version"],
+          "description": "doi = a versioned DOI (strongest). commit = a git SHA. release = a publisher's dated/named release. version = a declared version string with nothing stronger behind it."
+        },
+        "value": { "type": "string", "minLength": 1 },
+        "doi": { "type": ["string", "null"] },
+        "date": {
+          "type": ["string", "null"],
+          "description": "When the PUBLISHER released it. Not when we fetched it — that is fetchedAt, and conflating the two misdates our own provenance."
+        }
+      }
+    },
+    "files": {
+      "type": "array",
+      "minItems": 1,
+      "description": "Every file this snapshot vouches for. Anything in the directory not listed here is unaccounted for.",
+      "items": {
+        "type": "object",
+        "required": ["path", "bytes", "sha256"],
+        "additionalProperties": false,
+        "properties": {
+          "path": { "type": "string", "description": "Relative to the source directory." },
+          "bytes": { "type": "integer", "minimum": 0 },
+          "sha256": {
+            "type": "string",
+            "pattern": "^[0-9a-f]{64}$",
+            "description": "Computed by us on retrieval. Proves the file has not changed since; does NOT by itself prove it is what the publisher shipped."
+          },
+          "url": { "type": ["string", "null"] },
+          "upstreamChecksum": {
+            "type": ["string", "null"],
+            "description": "The publisher's own checksum, '<algo>:<hex>'. Null where the publisher provides none (SIL, for instance)."
+          },
+          "upstreamVerified": {
+            "type": "boolean",
+            "description": "True ONLY when upstreamChecksum was present AND matched the bytes on disk. This is the difference between provenance and assumption: a matching publisher checksum is proof the local file IS the named release; a plausible filename is not."
+          },
+          "derivedFrom": {
+            "type": ["string", "null"],
+            "description": "For a file extracted from a verified archive: the archive it came out of. This is how a CSV inherits a DOI's authority legibly — the CSV's own hash is ours, but it came from bytes the publisher vouched for. Without this the chain would have to be assumed."
+          }
+        }
+      }
+    },
+    "verified": {
+      "type": "boolean",
+      "description": "Whether this snapshot's pin is trustworthy. Must be false whenever pin is null — a snapshot cannot be verified against no particular release."
+    },
+    "fetchedBy": { "type": "string", "description": "Path of the fetcher that wrote this." },
+    "fetchedAt": { "type": "string", "description": "ISO 8601. When WE retrieved it." },
+    "notes": { "type": ["string", "null"] }
+  },
+  "allOf": [
+    {
+      "if": { "properties": { "verified": { "const": true } }, "required": ["verified"] },
+      "then": {
+        "properties": { "pin": { "type": "object" } },
+        "required": ["pin"]
+      }
+    }
+  ]
+}