eval-quality 2.0.0 → 3.0.0

package/README.md

@@ -179,7 +179,7 @@ import spec from 'eval-quality/schemas/eval-contract.schema.json' with { type: '
 
 The import attribute is required: ESM on Node 22 and 24 both throw `ERR_IMPORT_ATTRIBUTE_MISSING` without it. The development corpus ships the same way, at `eval-quality/corpus/dev/`, so you can read twenty-four real contracts and one compiled-and-sealed pair without cloning this repository.
 
-Version 1.0 is out and the published surface is stable: a breaking change to a command, an export, or a schema is a major version bump. `compile` refuses a contract whose `schemaVersion` differs from the one this build reads, so check the stamp on anything you did not author against this version. `CHANGELOG.md` records what each release breaks.
+Version 1.0 is out and the published surface is stable: a breaking change to a command, an export, or a schema is a major version bump. `compile` refuses a contract whose `schemaVersion` differs from the one this build reads, and `preflight` and `score` refuse a probe the same way, so check the stamp on anything you did not author against this version. `CHANGELOG.md` records what each release breaks.
 
 ## Relationship with BMad and TEA
 

package/dist/core/compile/compile.js

@@ -55,7 +55,12 @@ export function compile(contract, options) {
     // First, and before any check reads a declaration. AD-11 makes an unequal
     // stamp a rejection rather than a degraded read, and every check below is
     // written against this version's field shapes.
-    checkSchemaVersion(contract.schemaVersion, EVAL_CONTRACT_SCHEMA_VERSION, 'EvalContract.schemaVersion');
+    checkSchemaVersion({
+        stamped: contract.schemaVersion,
+        accepted: EVAL_CONTRACT_SCHEMA_VERSION,
+        artifactPath: 'EvalContract.schemaVersion',
+        consequence: 'since its stale version would travel into the scoring version',
+    });
     checkRequirementLinkage(contract);
     checkObservableSuccessCriterion(contract);
     // Ahead of reachability, because a pointer naming an artifact nothing

package/dist/core/compile/schema-version.d.ts

@@ -1,2 +1,15 @@
-/** `schema-version-mismatch`: the stamp is not the version this build reads. */
-export declare function checkSchemaVersion(stamped: number, accepted: number, artifactPath: string): void;
+/**
+ * `schema-version-mismatch`: the stamp is not the version this build reads.
+ *
+ * An options object rather than four positional arguments. `artifactPath` and
+ * `consequence` are both free-form strings, and transposed they typecheck: the
+ * fault's `artifactPath`, which callers read programmatically, would then hold
+ * a sentence.
+ */
+export declare function checkSchemaVersion(options: {
+    readonly stamped: number;
+    readonly accepted: number;
+    readonly artifactPath: string;
+    /** Why an unequal stamp is a rejection for this artifact, as a clause. */
+    readonly consequence: string;
+}): void;

package/dist/core/compile/schema-version.js

@@ -17,9 +17,17 @@
  * happened to notice first.
  */
 import { RuntimeFault } from '../schemas/faults.js';
-/** `schema-version-mismatch`: the stamp is not the version this build reads. */
-export function checkSchemaVersion(stamped, accepted, artifactPath) {
+/**
+ * `schema-version-mismatch`: the stamp is not the version this build reads.
+ *
+ * An options object rather than four positional arguments. `artifactPath` and
+ * `consequence` are both free-form strings, and transposed they typecheck: the
+ * fault's `artifactPath`, which callers read programmatically, would then hold
+ * a sentence.
+ */
+export function checkSchemaVersion(options) {
+    const { stamped, accepted, artifactPath, consequence } = options;
     if (stamped === accepted)
         return;
-    throw new RuntimeFault('schema-version-mismatch', artifactPath, `carries "schemaVersion" ${stamped} where this build reads ${accepted}; a contract written for another version is not read leniently, since its stale version would travel into the scoring version (AD-11)`);
+    throw new RuntimeFault('schema-version-mismatch', artifactPath, `carries "schemaVersion" ${stamped} where this build reads ${accepted}; an artifact written for another version is not read leniently, ${consequence} (AD-11)`);
 }

package/dist/core/preflight/plan.js

@@ -9,11 +9,13 @@
  * identifier up in at reduce time.
  */
 import { isSupportedInterfaceKind, SUPPORTED_KINDS_CLAUSE, } from '../compile/interface-inventory.js';
+import { checkSchemaVersion } from '../compile/schema-version.js';
 import { checkInputsAgainstShape, isApiWitnessInputs, isMcpWitnessInputs, } from '../compile/sensitivity-witness.js';
 import { declaresNoRequiredKeys, isCommandOperation, isMcpOperation, } from '../declared-inputs.js';
 import { referenceSetKeysOf } from '../evaluate/evidence-resolution.js';
 import { StructuralFailure } from '../failure-codes.js';
 import { operationsOf } from '../schemas/interface.js';
+import { PROBE_SCHEMA_VERSION, PROBE_SCHEMA_VERSION_CONSEQUENCE, } from '../schemas/probe.js';
 import { referenceSetMembers } from './witness-evidence.js';
 const requestOf = (legId, interfaceId, operation, inputs) => {
     // NFR9's correlation by identifier: the port echoes these back, and they
@@ -202,6 +204,19 @@ const selectControl = (contract, permittedInterfaces) => {
  */
 export const planPreflight = (input) => {
     const { contract, probes, runId } = input;
+    // First, and before any check reads a probe. AD-11 makes an unequal stamp a
+    // rejection rather than a degraded read, and every rule below is written
+    // against this version's field shapes. `compile` performs the same check for
+    // the contract; this is the probe's reader, since a probe reaches no other
+    // core stage on the pre-flight path.
+    for (const probe of probes) {
+        checkSchemaVersion({
+            stamped: probe.schemaVersion,
+            accepted: PROBE_SCHEMA_VERSION,
+            artifactPath: `Probe[probeId=${probe.probeId}].schemaVersion`,
+            consequence: PROBE_SCHEMA_VERSION_CONSEQUENCE,
+        });
+    }
     for (const iface of contract.permittedInterfaces) {
         // Already thrown at compile; asserted again because the plan is
         // reachable from a caller who assembled a contract by hand. Reads the

package/dist/core/schemas/probe.d.ts

@@ -71,6 +71,47 @@ export declare const Defect: z.ZodObject<{
         relation: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
     }, z.core.$strict>>;
 }, z.core.$strict>;
+/**
+ * The probe's current schema version. `EVAL_CONTRACT_SCHEMA_VERSION` is the
+ * same thing for the eval contract.
+ *
+ * It exists for the reason that one does: `lineage.ts` keeps the field a plain
+ * integer so a stale artifact fails as AD-28's `schema-version-mismatch` rather
+ * than as an anonymous parse error, which puts the comparison on the reader.
+ * `compile` is that reader for a contract. A probe has two in this pipeline and
+ * both compare against this constant: `planPreflight` before it plans a leg,
+ * and `score` before it seals the probe. An unequal stamp leaves by the fault
+ * path rather than being read leniently.
+ *
+ * A third reader exists for a caller outside this package.
+ * `validateLineageChain` takes an `acceptedSchemaVersion` and raises the same
+ * code over a presented chain, and `Probe` carries lineage, so a caller who
+ * presents one gets the comparison there. It words the fault its own way, which
+ * is why the two spellings of `schema-version-mismatch` in this tree are not a
+ * drift.
+ *
+ * It is also the single place the number is written. The committed
+ * worked-example chains build their probes from it and `check:doc-claims` reads
+ * it to hold the published sentence that names it, where the number was a
+ * literal in three places that could disagree in silence.
+ */
+export declare const PROBE_SCHEMA_VERSION = 5;
+/**
+ * Why a stale probe stamp is a rejection, in the words the fault carries.
+ *
+ * One string for both readers. Two copies of a sentence this long disagree in
+ * silence exactly as the number did before `PROBE_SCHEMA_VERSION` existed, and
+ * it lives here because this is where a version bump is one edit.
+ *
+ * It splits the shapes by branch, because a clean control carries neither a
+ * `defectSignature` nor a manifestation witness: the signature is declared on
+ * the seeded branch below and a witness hangs off a `Defect`, which that branch
+ * bounds at zero. What every probe carries is the qualification record, whose
+ * arrival on both branches is what made the 1 to 2 bump breaking. A clean
+ * control is the probe a reader is most likely to meet this message with, so
+ * naming a field it does not have would be the wrong half to lead with.
+ */
+export declare const PROBE_SCHEMA_VERSION_CONSEQUENCE: string;
 /**
  * The prior art's `expectedClean` conditional, re-expressed as a discriminated
  * union per AD-13 (a boolean literal discriminator parses on this pin,

package/dist/core/schemas/probe.js

@@ -48,6 +48,49 @@ const probeCommonFields = {
     rationale: z.string().min(1),
     qualification: ProbeQualification.describe('AD-9\'s qualification record: which of the five routes earned this probe its ground truth, and the evidence that route demands. Required on every branch and on every class, canaries included, because AD-9 closes with "an unqualified probe cannot enter a sealed set" and spells a route for all five kinds. That the route is compatible with this probe\'s class and `expectedClean` flag is a cross-field rule the export cannot carry; the corpus qualification gate enforces it and returns a reason code. Required, not optional, which with `defectSignature` below makes the probe\'s `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that "adding an optional field is a `schemaVersion` bump recorded in the field\'s own description; removing or retyping is breaking". This field is on both branches, so it alone is what stops every version-1 probe from parsing.'),
 };
+/**
+ * The probe's current schema version. `EVAL_CONTRACT_SCHEMA_VERSION` is the
+ * same thing for the eval contract.
+ *
+ * It exists for the reason that one does: `lineage.ts` keeps the field a plain
+ * integer so a stale artifact fails as AD-28's `schema-version-mismatch` rather
+ * than as an anonymous parse error, which puts the comparison on the reader.
+ * `compile` is that reader for a contract. A probe has two in this pipeline and
+ * both compare against this constant: `planPreflight` before it plans a leg,
+ * and `score` before it seals the probe. An unequal stamp leaves by the fault
+ * path rather than being read leniently.
+ *
+ * A third reader exists for a caller outside this package.
+ * `validateLineageChain` takes an `acceptedSchemaVersion` and raises the same
+ * code over a presented chain, and `Probe` carries lineage, so a caller who
+ * presents one gets the comparison there. It words the fault its own way, which
+ * is why the two spellings of `schema-version-mismatch` in this tree are not a
+ * drift.
+ *
+ * It is also the single place the number is written. The committed
+ * worked-example chains build their probes from it and `check:doc-claims` reads
+ * it to hold the published sentence that names it, where the number was a
+ * literal in three places that could disagree in silence.
+ */
+export const PROBE_SCHEMA_VERSION = 5;
+/**
+ * Why a stale probe stamp is a rejection, in the words the fault carries.
+ *
+ * One string for both readers. Two copies of a sentence this long disagree in
+ * silence exactly as the number did before `PROBE_SCHEMA_VERSION` existed, and
+ * it lives here because this is where a version bump is one edit.
+ *
+ * It splits the shapes by branch, because a clean control carries neither a
+ * `defectSignature` nor a manifestation witness: the signature is declared on
+ * the seeded branch below and a witness hangs off a `Defect`, which that branch
+ * bounds at zero. What every probe carries is the qualification record, whose
+ * arrival on both branches is what made the 1 to 2 bump breaking. A clean
+ * control is the probe a reader is most likely to meet this message with, so
+ * naming a field it does not have would be the wrong half to lead with.
+ */
+export const PROBE_SCHEMA_VERSION_CONSEQUENCE = 'since the stamp says which shapes the probe was authored against: the ' +
+    'qualification record on every probe, and the witness legs and the defect ' +
+    'signature grammar on a seeded one';
 /**
  * The prior art's `expectedClean` conditional, re-expressed as a discriminated
  * union per AD-13 (a boolean literal discriminator parses on this pin,

package/dist/core/score/score.d.ts

@@ -31,7 +31,7 @@
 import type { ValidatedObservations } from '../ingest/ingest.ts';
 import { type EvalContract } from '../schemas/eval-contract.ts';
 import type { Outcome } from '../schemas/evidence-artifact.ts';
-import type { Probe } from '../schemas/probe.ts';
+import { type Probe } from '../schemas/probe.ts';
 import type { ScoringPolicy } from '../schemas/scoring-policy.ts';
 import type { ScoreStage } from '../stage-contracts.ts';
 import type { ContractAssessment, LadderResolution, ProductionAssessment } from './ladder.ts';

package/dist/core/score/score.js

@@ -29,10 +29,12 @@
  * rather than a new one.
  */
 import { walkExpression } from '../compile/expression-legality.js';
+import { checkSchemaVersion } from '../compile/schema-version.js';
 import { evaluateCoverage } from '../coverage/coverage.js';
 import { makePointerDenotesCollection, makeResolveOperand, referenceSetKeysOf, } from '../evaluate/evidence-resolution.js';
 import { resolveCheck } from '../evaluate/resolution.js';
 import { SEVERITY_LEVELS, } from '../schemas/eval-contract.js';
+import { PROBE_SCHEMA_VERSION, PROBE_SCHEMA_VERSION_CONSEQUENCE, } from '../schemas/probe.js';
 import { buildPlanIndex } from '../seal/plan-index.js';
 import { resolveCapturedBindings, selectWithBindings } from './bindings.js';
 import { resolveContractVerdict, resolveProductionVerdict } from './ladder.js';
@@ -236,6 +238,27 @@ const INVALIDATING_OUTCOME_STATES = new Set(TRIAL_VOTE_STATES.invalidating);
  * each arrives named and explicit rather than a hardcoded literal.
  */
 export const score = (contract, trials, probe, preflightVerdict, policy, waiver, evaluationFault) => {
+    // First, and before the probe is sealed. AD-11 makes an unequal stamp a
+    // rejection rather than a degraded read. `compile` performs the same check
+    // for the contract.
+    //
+    // A throw rather than a reported outcome, against this stage's usual rule
+    // that a rejected probe is a domain result the ladder carries. Two reasons,
+    // and the second is the one that decides it. `qualifyProbe` returns a code
+    // from a closed vocabulary keyed to this version's field shapes, so a
+    // foreign probe's reason code is uninterpretable rather than merely
+    // unreliable. And the reporting path drops the fact: an
+    // `unqualified-probe-in-sealed-set` condition lands the run on the Invalid
+    // rung, where `RunScoreResult.artifact` is null, so there is no field for
+    // the reason to travel in. The fault carries an artifact path naming the
+    // probe and both numbers, which is the only path that tells a caller which
+    // probe to restamp.
+    checkSchemaVersion({
+        stamped: probe.schemaVersion,
+        accepted: PROBE_SCHEMA_VERSION,
+        artifactPath: `Probe[probeId=${probe.probeId}].schemaVersion`,
+        consequence: PROBE_SCHEMA_VERSION_CONSEQUENCE,
+    });
     // Probe sealing: once per run, never per trial, since qualification reads
     // the probe and the contract's operation inventory alone. `probeQualified`
     // reads whichever bucket the probe actually lands in -- never a throw on

package/dist/index.d.ts

@@ -12,4 +12,4 @@ export type { ScoringPolicy } from './core/schemas/scoring-policy.ts';
 export type { SealedEvaluatorBrief } from './core/schemas/sealed-evaluator-brief.ts';
 export type { SealedRunRecord } from './core/schemas/sealed-run-record.ts';
 export type { FixtureReset, ManifestationWitness, SensitivityWitness, SensitivityWitnessLeg, WitnessChannel, WitnessInputs, } from './core/schemas/sensitivity-witness.ts';
-export declare const VERSION = "2.0.0";
+export declare const VERSION = "3.0.0";

package/dist/index.js

@@ -19,4 +19,4 @@
 // subpath, where AD-37 puts the conformance definition an adapter author
 // reads; the reference adapters stay at `eval-quality/adapters`.
 export * from './application/index.js';
-export const VERSION = '2.0.0';
+export const VERSION = '3.0.0';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eval-quality",
-  "version": "2.0.0",
+  "version": "3.0.0",
   "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
   "author": "Murat Ozcan",
   "license": "Apache-2.0",