eval-quality 0.1.0 → 0.2.0

package/dist/core/score/selection.js

@@ -0,0 +1,98 @@
+/**
+ * Owed item 2's fix: a step selects observations by `operationId`, ordered by
+ * the record's monotonic `sequence` (ADR-006 forbids array position as
+ * ordering). `selectObservations` reports a several-match ambiguity as data;
+ * no first-match or last-match convention picks a winner. Pure and total:
+ * every input produces a defined result, nothing throws, and no AD-6 outcome
+ * state is assigned. Assigning outcome state is AD-33's reference decision
+ * procedure's job.
+ *
+ * `resolveTemporalAnchor` is a separate, narrower resolution built on top:
+ * picking a concrete point in time for an `any`-cardinality temporal anchor
+ * with several matches, by lowest sequence. The policy above still holds
+ * here: a single-valued cardinality's several-match ambiguity still comes
+ * back unresolved from `resolveTemporalAnchor` too.
+ *
+ * No stage claims this module yet; `stage-table.ts`'s `score` row stays
+ * `module: null` throughout.
+ */
+/**
+ * Matches `step.operationId` against every observation's `operationId`,
+ * ordered by `sequence` ascending. Reads no other field of `step`: whether
+ * `several` is the named ambiguity condition or a legitimate `any`-cardinality
+ * match is a fact about the step's declared cardinality, decided by whoever
+ * reads this result, not by this function.
+ *
+ * Sorts a copy; the input `observations` array is never mutated, and its own
+ * order is never read as meaning anything (NFR9: a permutation of the same
+ * observations yields byte-identical `matchedObservationIds`). The schema
+ * layer enforces per-record `sequence` uniqueness, so a real `SealedRunRecord`
+ * never presents a tie. `observationId` is the secondary sort key regardless:
+ * this function's own permutation invariance holds even against a hand-built
+ * or malformed `observations` array carrying a duplicate or non-finite
+ * `sequence`, decided by this comparator alone.
+ *
+ * Matching on `operationId` alone, ignoring `step.inputBinding`, is a
+ * deliberate scope boundary that stays: candidate-tuple resolution lives in
+ * `selectWithBindings` (`score/bindings.ts`), which wraps this function and
+ * filters its matches against the step's own resolved bindings. Splitting them
+ * keeps this function's permutation guarantee provable on its own.
+ *
+ * `tests/seal/fixtures.ts`'s `irreducibleCollisionPair` was once cited here as
+ * a case "distinguishable only by input binding". It is not: its two steps
+ * bind nothing in any channel, so a filter over zero bindings separates
+ * nothing and both stay `several` even with `selectWithBindings`. The pair
+ * that separates is `literalCollisionPair`, whose two steps bind one key to
+ * two different literals.
+ */
+export function selectObservations(step, observations) {
+    const matched = observations
+        .filter((observation) => observation.operationId === step.operationId)
+        .sort((a, b) => a.sequence - b.sequence || (a.observationId < b.observationId ? -1 : 1));
+    const matchedObservationIds = matched.map((observation) => observation.observationId);
+    const result = matchedObservationIds.length === 0
+        ? 'none'
+        : matchedObservationIds.length === 1
+            ? 'one'
+            : 'several';
+    return { result, matchedObservationIds };
+}
+/**
+ * Resolves an `after` temporal clause to the single observation it denotes,
+ * built directly from `selectObservations`'s own result over the anchor.
+ *
+ * Takes the anchor step itself, already resolved from the dependent step's
+ * `after` identifier (a one-line lookup against whichever declared plan the
+ * caller holds). A step whose clause is `null`, or whose clause names a step
+ * the plan does not declare (AD-39's permissive dangling reference), names no
+ * anchor and never reaches this function.
+ */
+export function resolveTemporalAnchor(anchorStep, observations) {
+    const selection = selectObservations(anchorStep, observations);
+    if (selection.result === 'one') {
+        return {
+            resolved: true,
+            // `result === 'one'` guarantees exactly one member.
+            observationId: selection
+                .matchedObservationIds[0],
+            matchedObservationIds: selection.matchedObservationIds,
+        };
+    }
+    if (selection.result === 'several' && anchorStep.cardinality === 'any') {
+        // Ascending-`sequence` order already holds the lowest-sequence match first.
+        return {
+            resolved: true,
+            observationId: selection
+                .matchedObservationIds[0],
+            matchedObservationIds: selection.matchedObservationIds,
+        };
+    }
+    // `none`, or `several` under a single-valued cardinality: no single
+    // observation resolves. Reported as data; routing this to a verdict rung
+    // is later work.
+    return {
+        resolved: false,
+        result: selection.result,
+        matchedObservationIds: selection.matchedObservationIds,
+    };
+}

package/dist/core/score/strength.d.ts

@@ -0,0 +1,49 @@
+/**
+ * AD-7's rate vector and its four-valued dominance relation.
+ *
+ * `buildStrengthVector` is a pure aggregation over a qualified probe set and
+ * the trial-set reducer's per-probe results: unweighted, per probe class,
+ * unique qualified probe identifiers over unique qualified probe identifiers
+ * exercised, with canary probes and clean controls excluded regardless of
+ * class or trial outcome. `compareDominance` takes two already-computed
+ * results and never re-derives a vector, reads a port, a corpus, or a clock;
+ * comparability is checked first, and the severity-floor override can only
+ * push the relation toward `incomparable`.
+ */
+import { type Severity } from '../schemas/eval-contract.ts';
+import type { Outcome, Strength, StrengthVector } from '../schemas/evidence-artifact.ts';
+import type { QualifiedProbe } from './qualification.ts';
+import type { TrialSetResult } from './reduce-trials.ts';
+export declare const DOMINANCE_RELATIONS: readonly ['a-dominates-b', 'b-dominates-a', 'equivalent', 'incomparable'];
+export type DominanceRelationValue = (typeof DOMINANCE_RELATIONS)[number];
+/**
+ * The slice the dominance comparator reads: the aggregate `Strength` plus the
+ * per-probe `outcomes` array the severity-floor override needs, since that
+ * identity is lost once probes are aggregated into `ClassStrength` counts,
+ * and the `comparabilityKey` the comparator checks before comparing anything
+ * else. Every field already lives on `EvidenceArtifact`; this is the read
+ * projection the comparator needs from it, not a new artifact shape.
+ */
+export type ComparableResult = {
+    readonly outcomes: readonly Outcome[];
+    readonly strength: Strength;
+    readonly comparabilityKey: string;
+};
+/**
+ * AD-7's rate vector: per probe class, the catch rate over unique qualified
+ * probe identifiers, with raw counts alongside. `admitted` carries each
+ * probe's identifier once by construction, so grouping by class needs no
+ * deduplication of its own.
+ */
+export declare function buildStrengthVector(admitted: readonly QualifiedProbe[], results: ReadonlyMap<string, TrialSetResult>): StrengthVector;
+/**
+ * AD-7's four-valued dominance relation. `comparabilityKey` and each side's
+ * own `strength.comparable` are checked before any component-wise comparison
+ * runs: a key mismatch means the two runs are not measuring a shared probe
+ * set, and `comparable: false` means AD-21 already marked that one side's own
+ * vector as thinner than the policy's declared minimum, so a `caught`/`rate`
+ * on it is not fit to decide a comparison either way. The severity-floor
+ * override runs only against the side the raw comparison favoured, and only
+ * ever downgrades that result to `incomparable`.
+ */
+export declare function compareDominance(a: ComparableResult, b: ComparableResult, severityFloor: Severity): DominanceRelationValue;

package/dist/core/score/strength.js

@@ -0,0 +1,193 @@
+/**
+ * AD-7's rate vector and its four-valued dominance relation.
+ *
+ * `buildStrengthVector` is a pure aggregation over a qualified probe set and
+ * the trial-set reducer's per-probe results: unweighted, per probe class,
+ * unique qualified probe identifiers over unique qualified probe identifiers
+ * exercised, with canary probes and clean controls excluded regardless of
+ * class or trial outcome. `compareDominance` takes two already-computed
+ * results and never re-derives a vector, reads a port, a corpus, or a clock;
+ * comparability is checked first, and the severity-floor override can only
+ * push the relation toward `incomparable`.
+ */
+import { SEVERITY_LEVELS } from '../schemas/eval-contract.js';
+export const DOMINANCE_RELATIONS = [
+    'a-dominates-b',
+    'b-dominates-a',
+    'equivalent',
+    'incomparable',
+];
+const STRENGTH_VECTOR_CLASSES = [
+    'defect',
+    'gameability',
+    'zero-action',
+];
+/**
+ * `admitted`, after excluding `canary` and every `expectedClean: true` probe:
+ * AD-7's "canary probes and clean controls never enter the vector" applies
+ * regardless of class or trial outcome, and a canary carries `expectedClean:
+ * false` on its own schema branch, so both conditions are checked.
+ */
+const vectorEligible = (admitted) => admitted.filter(({ probe }) => probe.probeClass !== 'canary' && !probe.expectedClean);
+/**
+ * One class's aggregate, or `null` when the eligible set admits no probe of
+ * that class. A probe with no `TrialSetResult`, or one that is `exercised:
+ * false`, contributes to neither `caught` nor `exercised`, matching the
+ * reducer's own "zero valid trials excludes a probe entirely" rule. A class
+ * with admitted probes but zero exercised ones is still a present
+ * `ClassStrength` of `{ caught: 0, exercised: 0, rate: null }`, never a
+ * `null` class: `rate`'s nullability exists specifically for that case, and
+ * collapsing the whole class to `null` would make it unobservable.
+ */
+const classStrengthOf = (probesInClass, results) => {
+    if (probesInClass.length === 0)
+        return null;
+    let exercised = 0;
+    let caught = 0;
+    for (const { probe } of probesInClass) {
+        const result = results.get(probe.probeId);
+        if (result === undefined || !result.exercised)
+            continue;
+        exercised += 1;
+        if (result.caught)
+            caught += 1;
+    }
+    return {
+        exercised,
+        caught,
+        rate: exercised === 0 ? null : caught / exercised,
+    };
+};
+/**
+ * AD-7's rate vector: per probe class, the catch rate over unique qualified
+ * probe identifiers, with raw counts alongside. `admitted` carries each
+ * probe's identifier once by construction, so grouping by class needs no
+ * deduplication of its own.
+ */
+export function buildStrengthVector(admitted, results) {
+    const eligible = vectorEligible(admitted);
+    const byClass = Object.fromEntries(STRENGTH_VECTOR_CLASSES.map((probeClass) => [
+        probeClass,
+        classStrengthOf(eligible.filter(({ probe }) => probe.probeClass === probeClass), results),
+    ]));
+    return {
+        defect: byClass.defect,
+        gameability: byClass.gameability,
+        'zero-action': byClass['zero-action'],
+    };
+}
+/**
+ * A class contributes to the comparison only when it is a non-null
+ * `ClassStrength` on both sides and both sides' `rate` is also non-null; a
+ * class absent on either side, or present with a `null` rate on either side,
+ * carries no comparative evidence and is skipped exactly alike. `equivalent`
+ * compares `caught` and `exercised` rather than the derived `rate`, avoiding
+ * a floating-point equality check.
+ *
+ * A class whose two sides tie on `rate` while disagreeing on `caught` or
+ * `exercised` contributes to the comparison, blocks `equivalent` (the counts
+ * are not equal), and hands neither side a win (neither `rate` is strictly
+ * greater). That third possibility has no named outcome of its own in AD-7's
+ * three stated cases, and `incomparable` is where a tied vector with no
+ * winner on either side belongs: the same value the "no class contributes at
+ * all" case already returns.
+ */
+function componentComparison(a, b) {
+    let aWinsAClass = false;
+    let bWinsAClass = false;
+    let everyContributingClassEqual = true;
+    let contributingClasses = 0;
+    for (const key of STRENGTH_VECTOR_CLASSES) {
+        const left = a[key];
+        const right = b[key];
+        if (left === null || right === null)
+            continue;
+        if (left.rate === null || right.rate === null)
+            continue;
+        contributingClasses += 1;
+        if (left.caught !== right.caught || left.exercised !== right.exercised) {
+            everyContributingClassEqual = false;
+        }
+        if (left.rate > right.rate)
+            aWinsAClass = true;
+        if (right.rate > left.rate)
+            bWinsAClass = true;
+    }
+    if (contributingClasses === 0)
+        return 'incomparable';
+    if (everyContributingClassEqual)
+        return 'equivalent';
+    if (aWinsAClass && !bWinsAClass)
+        return 'a-dominates-b';
+    if (bWinsAClass && !aWinsAClass)
+        return 'b-dominates-a';
+    return 'incomparable';
+}
+const atOrAboveFloor = (severity, floor) => SEVERITY_LEVELS.indexOf(severity) >= SEVERITY_LEVELS.indexOf(floor);
+/**
+ * Keyed by the first outcome carrying each `probeId`, not the last: two
+ * `Outcome` entries sharing one `probeId` is itself a defect somewhere
+ * upstream (this map has no way to tell which entry is the real one), and a
+ * silent last-write-wins overwrite would drop the earlier entry from the
+ * severity-floor scan below with no trace it was ever there.
+ */
+const outcomesByProbeId = (outcomes) => {
+    const byProbeId = new Map();
+    for (const outcome of outcomes) {
+        if (outcome.probeId === null)
+            continue;
+        if (byProbeId.has(outcome.probeId))
+            continue;
+        byProbeId.set(outcome.probeId, outcome);
+    }
+    return byProbeId;
+};
+/**
+ * Whether `favored` failed to catch a probe that `other` caught at or above
+ * `severityFloor`: the condition that disqualifies `favored` from dominating,
+ * per AD-7's "a contract that missed a behaviour at or above the scoring
+ * policy's severity floor never dominates one that caught it, regardless of
+ * the rest of the vector". An outcome with `probeId: null` is not tied to any
+ * probe and is outside the vector entirely, so both sides skip it.
+ */
+function favoredMissesWhatOtherCaught(favored, other, severityFloor) {
+    const favoredByProbeId = outcomesByProbeId(favored.outcomes);
+    const otherByProbeId = outcomesByProbeId(other.outcomes);
+    for (const [probeId, otherOutcome] of otherByProbeId) {
+        if (otherOutcome.state !== 'caught')
+            continue;
+        if (!atOrAboveFloor(otherOutcome.severity, severityFloor))
+            continue;
+        const favoredOutcome = favoredByProbeId.get(probeId);
+        if (favoredOutcome === undefined || favoredOutcome.state !== 'caught') {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * AD-7's four-valued dominance relation. `comparabilityKey` and each side's
+ * own `strength.comparable` are checked before any component-wise comparison
+ * runs: a key mismatch means the two runs are not measuring a shared probe
+ * set, and `comparable: false` means AD-21 already marked that one side's own
+ * vector as thinner than the policy's declared minimum, so a `caught`/`rate`
+ * on it is not fit to decide a comparison either way. The severity-floor
+ * override runs only against the side the raw comparison favoured, and only
+ * ever downgrades that result to `incomparable`.
+ */
+export function compareDominance(a, b, severityFloor) {
+    if (a.comparabilityKey !== b.comparabilityKey)
+        return 'incomparable';
+    if (!a.strength.comparable || !b.strength.comparable)
+        return 'incomparable';
+    const raw = componentComparison(a.strength.vector, b.strength.vector);
+    if (raw === 'a-dominates-b' &&
+        favoredMissesWhatOtherCaught(a, b, severityFloor)) {
+        return 'incomparable';
+    }
+    if (raw === 'b-dominates-a' &&
+        favoredMissesWhatOtherCaught(b, a, severityFloor)) {
+        return 'incomparable';
+    }
+    return raw;
+}

package/dist/core/score/witness.d.ts

@@ -0,0 +1,117 @@
+import type { DefectSignature } from '../schemas/defect-signature.ts';
+import type { PermittedInterface } from '../schemas/interface.ts';
+import type { Probe } from '../schemas/probe.ts';
+import type { SealedRunRecord } from '../schemas/sealed-run-record.ts';
+/**
+ * A module constant, following pre-flight's precedent for the same problem:
+ * `resolveCheck`'s budget parameter is a bare required number whose declared
+ * home is the scoring policy, and a probe is not scored under a policy at match
+ * time. Two implementations choosing two budgets would disagree about when the
+ * evaluator reports an exhausted budget, which is the disagreement one shared
+ * constant removes.
+ */
+export declare const PROBE_REGEX_MATCH_STEP_BUDGET = 1000000;
+/**
+ * The six results, declared in the evaluation order the match applies, so a
+ * reader cannot mistake this constant for a different ordering.
+ *
+ * - `unexercised`: the evaluator never invoked the home operation.
+ * - `unwitnessed-claim`: a defect finding cited this probe and cited home-operation
+ *   observations, none of which satisfies the condition. An AD-32
+ *   declared-versus-observed inconsistency, and it outranks a competing
+ *   `matched` so a broken reporter cannot mask it.
+ * - `matched`: a defect finding cited an observation the condition satisfies.
+ * - `manifested-unclaimed`: the condition was satisfied and no finding claimed
+ *   it. This is what makes non-detection reachable.
+ * - `not-triggered`: the system was examined and did not manifest the seeded
+ *   defect. AD-6 calls this the common case on any defect probe. Named for what
+ *   it is rather than "not manifested", because AD-40 already uses that phrase
+ *   for the vacuous case and the two route to opposite verdicts.
+ * - `vacuous`: every candidate resolved insufficient-evidence, so the corpus
+ *   presented no defect to detect. A fact about the instrument.
+ */
+export declare const PROBE_WITNESS_RESULTS: readonly ['unexercised', 'unwitnessed-claim', 'matched', 'manifested-unclaimed', 'not-triggered', 'vacuous'];
+export type ProbeWitnessResultValue = (typeof PROBE_WITNESS_RESULTS)[number];
+/** A probe on the seeding branch whose signature is present. */
+export type SignedProbe = Extract<Probe, {
+    expectedClean: false;
+}> & {
+    readonly defectSignature: DefectSignature;
+};
+export type WitnessPartition = {
+    /** the candidates the condition resolved `true` over. */
+    readonly satisfying: readonly string[];
+    /** the candidates it resolved `false` over: the system examined and behaving. */
+    readonly refuting: readonly string[];
+    /** the candidates it could not examine. Never detection, and never manifestation. */
+    readonly inconclusive: readonly string[];
+};
+export type ProbeWitnessMatch = {
+    readonly result: ProbeWitnessResultValue;
+    /**
+     * AD-40 forbids pooling a quotation-reconstructed detection with a measured
+     * catch rate, so every result says which it is. The witness match resolves
+     * over cited identifiers and is always `measured`.
+     */
+    readonly basis: 'measured';
+    readonly homeOperationResolved: boolean;
+    readonly exercised: boolean;
+    /** every candidate identifier the match read, ascending by `sequence`. */
+    readonly observationIds: readonly string[];
+    readonly partition: WitnessPartition;
+    readonly partitionSizes: {
+        readonly satisfying: number;
+        readonly refuting: number;
+        readonly inconclusive: number;
+    };
+    /** the satisfying observations a defect finding actually cited. */
+    readonly witnessObservationIds: readonly string[];
+    /** the defect findings whose home-operation citations witness nothing. */
+    readonly unwitnessedFindingIds: readonly string[];
+};
+/**
+ * AD-40's deterministic witness match over one probe and one sealed run record.
+ *
+ * The candidates are partitioned by AD-4 resolution into satisfying, refuting,
+ * and inconclusive, and the result is read off that partition rather than off a
+ * precedence list. A partition is exhaustive and totally ordered over the whole
+ * input space; a precedence list left mixed refuting/inconclusive with no home
+ * and let an empty candidate set satisfy two rows that route to opposite
+ * verdicts.
+ *
+ * The verdict reads cited identifiers alone and never quotation. Quoted evidence
+ * that appears in no cited observation is a separate audit, and its invalidation
+ * belongs to ingest.
+ *
+ * No output reads an array's position. Observation identifiers follow the
+ * record's `sequence`; finding identifiers are sorted by identifier, because
+ * `findings` carries no ordering field of its own.
+ */
+export declare function matchProbeWitness(probe: SignedProbe, interfaces: readonly PermittedInterface[], record: Pick<SealedRunRecord, 'observations' | 'findings'>): ProbeWitnessMatch;
+export type MappedFinding = {
+    readonly findingId: string;
+    readonly probeId: string;
+};
+export type FindingMap = {
+    /** cited to a signed probe, and touching that signature's home operation. */
+    readonly mapped: readonly MappedFinding[];
+    /**
+     * cited to a signed probe whose home operation the cited observations never
+     * touch. An unexpected real defect under AD-23, never a catch.
+     */
+    readonly unmapped: readonly MappedFinding[];
+    /** cited to a probe identifier the set does not declare: an AD-32 cross-artifact dangling reference. */
+    readonly dangling: readonly MappedFinding[];
+    /** cited to a canary or a clean control, which carry no signature to map against. */
+    readonly signatureless: readonly MappedFinding[];
+};
+/**
+ * Sorts every defect finding in the record into the four buckets AD-40 and
+ * AD-23 between them require, so no finding is silently dropped and none is
+ * counted as a catch it did not earn.
+ *
+ * Defect findings only: they are the only type that enters a detection measure.
+ * The uncited case proper, a finding naming no oracle, is a different rule and
+ * a different record.
+ */
+export declare function mapFindings(probes: readonly Probe[], interfaces: readonly PermittedInterface[], record: Pick<SealedRunRecord, 'observations' | 'findings'>): FindingMap;