eval-quality 0.1.0

package/dist/core/lineage/chain.js

@@ -0,0 +1,210 @@
+/**
+ * AD-12's presented-chain reader and AD-29's one constructor. Nothing here
+ * holds state between calls, and no lineage is tracked across invocations
+ * (AD-12). `checks` is AD-12's three named booleans, the shape `emit`
+ * serializes onto every Evidence Artifact; `findings` says which of the three
+ * failed and why.
+ *
+ * Chain defects are returned. AD-29 commands no code in either registry and
+ * AD-28's audit rule makes minting one a spine amendment, so the nine codes
+ * below are a module-local vocabulary in neither registry. Two faults are
+ * still thrown, both already registered and both commanded by another AD:
+ * `schema-version-mismatch` (AD-11) and, from `digestArtifact`,
+ * `non-canonicalizable-value` (AD-36).
+ *
+ * Each member must be a whole artifact already parsed against one published
+ * schema, the precondition every `core/` function has. A member narrowed to
+ * the three lineage fields digests differently and breaks its own chain.
+ */
+import { digestArtifact } from '../canonical/digest.js';
+import { RuntimeFault } from '../schemas/faults.js';
+import { freezeArtifact } from './freeze.js';
+export const LINEAGE_DEFECT_CODES = [
+    'lineage-root-invalid',
+    'lineage-duplicate-artifact',
+    'lineage-no-root',
+    'lineage-multiple-roots',
+    'lineage-parent-absent',
+    'lineage-revision-not-successor',
+    'lineage-revision-conflict',
+    'lineage-length-inconsistent',
+    'lineage-remediation-cap-exceeded',
+];
+/**
+ * Which of AD-12's three booleans each code clears. A `Record` over the code
+ * union, so a tenth code is a compile error here before it can reach a report
+ * with no boolean to affect. The cap maps to null; AD-12 records it in
+ * `Remediation.cap`, beside `lineageChain`.
+ */
+export const CHECK_PROJECTION = {
+    'lineage-root-invalid': 'noGap',
+    'lineage-duplicate-artifact': 'noRepeatedDigest',
+    'lineage-no-root': 'noGap',
+    'lineage-multiple-roots': 'noGap',
+    'lineage-parent-absent': 'noGap',
+    'lineage-revision-not-successor': 'noGap',
+    'lineage-revision-conflict': 'noGap',
+    'lineage-length-inconsistent': 'lengthConsistent',
+    'lineage-remediation-cap-exceeded': null,
+};
+const codeRank = (code) => LINEAGE_DEFECT_CODES.indexOf(code);
+function groupBy(items, key) {
+    const groups = new Map();
+    for (const item of items) {
+        const bucket = groups.get(key(item));
+        if (bucket === undefined)
+            groups.set(key(item), [item]);
+        else
+            bucket.push(item);
+    }
+    return groups;
+}
+function readMembers(artifacts, options) {
+    return artifacts.map((artifact) => {
+        if (artifact.schemaVersion !== options.acceptedSchemaVersion) {
+            throw new RuntimeFault('schema-version-mismatch', options.artifactPath, `chain member carries "schemaVersion" ${artifact.schemaVersion} where the reader accepts ${options.acceptedSchemaVersion} (AD-11)`);
+        }
+        return {
+            digest: digestArtifact(artifact, options.artifactPath),
+            parentDigest: artifact.parentDigest,
+            revisionCount: artifact.revisionCount,
+        };
+    });
+}
+function assertCount(value, label, path) {
+    if (Number.isSafeInteger(value) && value >= 0)
+        return;
+    throw new TypeError(`validateLineageChain() received a "${label}" that is not a non-negative safe integer: ${path}`);
+}
+/**
+ * Validates one presented chain of revisions of a single artifact type
+ * (AD-12, AD-29). Every defect is collected, so a chain with three problems
+ * reports three findings.
+ */
+export function validateLineageChain(artifacts, options) {
+    const { artifactPath } = options;
+    assertCount(options.declaredRevisionCount, 'declaredRevisionCount', artifactPath);
+    if (options.remediationCap !== null) {
+        assertCount(options.remediationCap, 'remediationCap', artifactPath);
+    }
+    const members = readMembers(artifacts, options);
+    const at = (digest) => `${artifactPath}[digest=${digest}]`;
+    const findings = [];
+    const add = (code, path, detail) => {
+        findings.push({ code, artifactPath: path, detail });
+    };
+    // The biconditional `lineage.ts` states and the constraint ledger records
+    // as not expressible in the published schema.
+    for (const member of members) {
+        const isRoot = member.revisionCount === 0;
+        if (isRoot === (member.parentDigest === null))
+            continue;
+        add('lineage-root-invalid', at(member.digest), isRoot
+            ? '"revisionCount" is 0 with a non-null "parentDigest" (AD-29)'
+            : `"revisionCount" is ${member.revisionCount} with a null "parentDigest" (AD-29)`);
+    }
+    for (const [digest, group] of groupBy(members, (member) => member.digest)) {
+        if (group.length < 2)
+            continue;
+        add('lineage-duplicate-artifact', at(digest), `${group.length} members share this digest, so one artifact is presented more than once (AD-12)`);
+    }
+    const roots = members.filter((member) => member.revisionCount === 0);
+    if (roots.length === 0) {
+        add('lineage-no-root', artifactPath, `no member carries "revisionCount" 0, so the chain of ${members.length} has no origin (AD-29)`);
+    }
+    else if (roots.length > 1) {
+        add('lineage-multiple-roots', artifactPath, `${roots.length} members carry "revisionCount" 0, so the presented set is more than one lineage (AD-29)`);
+    }
+    const byDigest = new Map(members.map((member) => [member.digest, member]));
+    for (const member of members) {
+        if (member.parentDigest === null)
+            continue;
+        const parent = byDigest.get(member.parentDigest);
+        if (parent === undefined) {
+            add('lineage-parent-absent', at(member.digest), `names parent "${member.parentDigest}", which no presented member digests to (AD-12)`);
+            continue;
+        }
+        if (member.revisionCount === parent.revisionCount + 1)
+            continue;
+        add('lineage-revision-not-successor', at(member.digest), `carries "revisionCount" ${member.revisionCount} where its parent carries ${parent.revisionCount} (AD-29)`);
+    }
+    // AD-29's conflict is same parent, same count, differing content. A group of
+    // byte-identical members is reported as a duplicate above.
+    const revisions = members.filter((member) => member.parentDigest !== null);
+    for (const [, group] of groupBy(revisions, (member) => `${member.parentDigest} ${member.revisionCount}`)) {
+        const distinct = [...new Set(group.map((member) => member.digest))].sort();
+        if (distinct.length < 2)
+            continue;
+        const first = group[0];
+        if (first === undefined)
+            continue;
+        add('lineage-revision-conflict', `${artifactPath}[parent=${first.parentDigest},revision=${first.revisionCount}]`, `${distinct.length} differing members claim this parent and revision: ${distinct.join(', ')} (AD-29)`);
+    }
+    if (members.length !== options.declaredRevisionCount + 1) {
+        add('lineage-length-inconsistent', artifactPath, `presents ${members.length} members against a declared revision count of ${options.declaredRevisionCount} (AD-12)`);
+    }
+    if (options.remediationCap !== null &&
+        options.declaredRevisionCount > options.remediationCap) {
+        add('lineage-remediation-cap-exceeded', artifactPath, `declares ${options.declaredRevisionCount} revisions against a caller-attested cap of ${options.remediationCap} (AD-12)`);
+    }
+    // Addresses are content-derived, so this order holds for any permutation of
+    // the input. Two byte-identical members produce two findings at one address
+    // with identical details, so an equal key is a genuine tie.
+    findings.sort((left, right) => {
+        const byCode = codeRank(left.code) - codeRank(right.code);
+        if (byCode !== 0)
+            return byCode;
+        if (left.artifactPath === right.artifactPath)
+            return 0;
+        return left.artifactPath < right.artifactPath ? -1 : 1;
+    });
+    const checks = {
+        lengthConsistent: true,
+        noRepeatedDigest: true,
+        noGap: true,
+    };
+    for (const finding of findings) {
+        const bucket = CHECK_PROJECTION[finding.code];
+        if (bucket !== null)
+            checks[bucket] = false;
+    }
+    return freezeArtifact({
+        artifactPath,
+        findings,
+        checks,
+        passed: findings.length === 0,
+    });
+}
+/**
+ * Mints the next revision of `parent`: a new artifact carrying the parent's
+ * digest and a revision count one greater (AD-29). The clone is load-bearing.
+ * `{ ...body }` shares its subtrees with the caller and `freezeArtifact`
+ * freezes in place, so without it this call would freeze the caller's `body`.
+ */
+export function reviseArtifact(parent, body, artifactPath) {
+    if ((parent.revisionCount === 0) !== (parent.parentDigest === null)) {
+        throw new TypeError(`reviseArtifact() received a parent whose parentDigest and revisionCount disagree: ${artifactPath}`);
+    }
+    const next = parent.revisionCount + 1;
+    if (!Number.isSafeInteger(parent.revisionCount) || parent.revisionCount < 0) {
+        throw new TypeError(`reviseArtifact() received a parent whose revisionCount is not a non-negative safe integer: ${artifactPath}`);
+    }
+    if (!Number.isSafeInteger(next)) {
+        throw new TypeError(`reviseArtifact() would mint a revisionCount outside the safe integer range: ${artifactPath}`);
+    }
+    if (body.schemaVersion !== parent.schemaVersion) {
+        throw new TypeError(`reviseArtifact() received a body at schemaVersion ${body.schemaVersion} for a parent at ${parent.schemaVersion}: ${artifactPath}`);
+    }
+    // The type forbids these keys and a cast defeats the type, so the guard is
+    // what stops a stale count winning or losing on spread order.
+    for (const field of ['parentDigest', 'revisionCount']) {
+        if (!Object.hasOwn(body, field))
+            continue;
+        throw new TypeError(`reviseArtifact() received a body carrying "${field}", which the producing stage owns: ${artifactPath}`);
+    }
+    return freezeArtifact(structuredClone({
+        ...body,
+        parentDigest: digestArtifact(parent, artifactPath),
+        revisionCount: next,
+    }));
+}

package/dist/core/lineage/freeze.d.ts

@@ -0,0 +1,12 @@
+/**
+ * AD-29's "created once and never edited in place", made mechanical. A stage
+ * freezes the artifact it owns; `application/` freezes what crosses the
+ * package boundary.
+ */
+/**
+ * Deep-freezes a parsed artifact in place and returns it. Call it on a value
+ * you own: every wired site hands it a Zod clone or a fresh literal. An
+ * already-frozen node stops the walk, which is what terminates a cycle and
+ * what leaves a hand-frozen subtree's descendants alone.
+ */
+export declare function freezeArtifact<T>(value: T): T;

package/dist/core/lineage/freeze.js

@@ -0,0 +1,35 @@
+/**
+ * AD-29's "created once and never edited in place", made mechanical. A stage
+ * freezes the artifact it owns; `application/` freezes what crosses the
+ * package boundary.
+ */
+/**
+ * Arrays and plain objects, every node an AD-27 JSON artifact has. Freezing a
+ * `Date`, `Map`, or `Set` protects nothing they hold, and `Object.freeze` on a
+ * non-empty typed array throws.
+ */
+function isJsonContainer(value) {
+    if (value === null || typeof value !== 'object')
+        return false;
+    if (Array.isArray(value))
+        return true;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+/**
+ * Deep-freezes a parsed artifact in place and returns it. Call it on a value
+ * you own: every wired site hands it a Zod clone or a fresh literal. An
+ * already-frozen node stops the walk, which is what terminates a cycle and
+ * what leaves a hand-frozen subtree's descendants alone.
+ */
+export function freezeArtifact(value) {
+    if (!isJsonContainer(value))
+        return value;
+    if (Object.isFrozen(value))
+        return value;
+    Object.freeze(value);
+    // An array's indices are its enumerable own keys, so this covers both.
+    for (const member of Object.values(value))
+        freezeArtifact(member);
+    return value;
+}

package/dist/core/lineage/stage-table.d.ts

@@ -0,0 +1,43 @@
+/**
+ * AD-24's stage-signature table and AD-29's producer map, as data: six stages
+ * with their inputs, their one owned output, and the lineage edge each writes;
+ * twelve interchange artifacts with one producer apiece. The `check:lineage`
+ * scanner derives its allowlist from `module`, so the registry import stays
+ * type-only to keep zod off that gate's load path.
+ */
+import type { InterchangeArtifactKey } from '../schemas/artifact.ts';
+/** AD-24's six stages, in the order the Consistency Conventions list them. */
+export declare const PIPELINE_STAGES: readonly ['compile', 'seal', 'ingest', 'preflight', 'score', 'emit'];
+export type PipelineStage = (typeof PIPELINE_STAGES)[number];
+/**
+ * Typed stage products no schema publishes (AD-24). `probe-plan` comes from
+ * pre-flight's plan half and `probe-observations` from the probe port, so
+ * neither is a stage's owned output.
+ */
+export declare const INTERNAL_PRODUCTS: readonly ['probe-plan', 'probe-observations', 'validated-observations', 'scored-outcomes-and-verdict'];
+export type InternalProduct = (typeof INTERNAL_PRODUCTS)[number];
+/**
+ * What a stage does to the two AD-29 fields on the artifact it owns. `mints`
+ * covers a root and a revision alike, since both write the fields.
+ */
+export type LineageEdge = 'mints' | 'carries-through' | 'none';
+/** Who produces an interchange artifact. */
+export type ArtifactProducer = PipelineStage | 'caller' | 'embedded';
+export type StageSignature = {
+    readonly inputs: readonly (InterchangeArtifactKey | InternalProduct)[];
+    readonly owns: InterchangeArtifactKey | InternalProduct;
+    /** the owned output's registry key; null when the output is internal. */
+    readonly ownsInterchange: InterchangeArtifactKey | null;
+    readonly lineage: LineageEdge;
+    /** the module that writes the artifact; null while the stage is unbuilt. */
+    readonly module: string | null;
+};
+export declare const STAGE_SIGNATURES: Record<PipelineStage, StageSignature>;
+export declare const ARTIFACT_PRODUCERS: Record<InterchangeArtifactKey, ArtifactProducer>;
+/**
+ * The modules permitted to write `parentDigest` or `revisionCount`, derived
+ * from the table it is handed. A function, because a module-level constant is
+ * evaluated once at import and a test mutating the table would see nothing.
+ */
+export declare function deriveLineageWriterModules(signatures: Record<PipelineStage, StageSignature>): readonly string[];
+export declare const LINEAGE_WRITER_MODULES: readonly string[];

package/dist/core/lineage/stage-table.js

@@ -0,0 +1,106 @@
+/** AD-24's six stages, in the order the Consistency Conventions list them. */
+export const PIPELINE_STAGES = [
+    'compile',
+    'seal',
+    'ingest',
+    'preflight',
+    'score',
+    'emit',
+];
+/**
+ * Typed stage products no schema publishes (AD-24). `probe-plan` comes from
+ * pre-flight's plan half and `probe-observations` from the probe port, so
+ * neither is a stage's owned output.
+ */
+export const INTERNAL_PRODUCTS = [
+    'probe-plan',
+    'probe-observations',
+    'validated-observations',
+    'scored-outcomes-and-verdict',
+];
+export const STAGE_SIGNATURES = {
+    compile: {
+        inputs: ['eval-contract'],
+        owns: 'eval-contract',
+        ownsInterchange: 'eval-contract',
+        // The caller authors the contract and its lineage; `compile` validates
+        // and returns it. The Seed's behaviour-input transformation is unbuilt,
+        // so the row is an identity.
+        lineage: 'carries-through',
+        module: 'src/core/compile/compile.ts',
+    },
+    seal: {
+        inputs: ['eval-contract'],
+        owns: 'sealed-evaluator-brief',
+        ownsInterchange: 'sealed-evaluator-brief',
+        lineage: 'mints',
+        module: 'src/core/seal/seal.ts',
+    },
+    ingest: {
+        inputs: [
+            'sealed-run-record',
+            'isolation-manifest',
+            'evaluator-configuration',
+        ],
+        owns: 'validated-observations',
+        ownsInterchange: null,
+        lineage: 'none',
+        module: null,
+    },
+    preflight: {
+        // AD-34 splits the stage into `plan` and `reduce`; both halves' inputs
+        // are the stage's inputs, and only `reduce` returns an artifact.
+        inputs: ['eval-contract', 'probe', 'probe-plan', 'probe-observations'],
+        owns: 'preflight-verdict',
+        ownsInterchange: 'preflight-verdict',
+        lineage: 'mints',
+        module: 'src/core/preflight/reduce.ts',
+    },
+    score: {
+        inputs: [
+            'eval-contract',
+            'validated-observations',
+            'probe',
+            'preflight-verdict',
+            'scoring-policy',
+        ],
+        // AD-24: "score produces the outcome and verdict values emit
+        // serializes". Owed item 6 records that type as unnamed; this names it.
+        owns: 'scored-outcomes-and-verdict',
+        ownsInterchange: null,
+        lineage: 'none',
+        module: null,
+    },
+    emit: {
+        inputs: ['scored-outcomes-and-verdict'],
+        owns: 'evidence-artifact',
+        ownsInterchange: 'evidence-artifact',
+        lineage: 'mints',
+        module: null,
+    },
+};
+export const ARTIFACT_PRODUCERS = {
+    'eval-contract': 'compile',
+    rubric: 'caller',
+    'sealed-evaluator-brief': 'seal',
+    'sealed-run-record': 'caller',
+    'isolation-manifest': 'caller',
+    'evaluator-configuration': 'caller',
+    probe: 'caller',
+    'artifact-reference': 'embedded',
+    'private-artifact-manifest': 'caller',
+    'preflight-verdict': 'preflight',
+    'scoring-policy': 'caller',
+    'evidence-artifact': 'emit',
+};
+/**
+ * The modules permitted to write `parentDigest` or `revisionCount`, derived
+ * from the table it is handed. A function, because a module-level constant is
+ * evaluated once at import and a test mutating the table would see nothing.
+ */
+export function deriveLineageWriterModules(signatures) {
+    return PIPELINE_STAGES.map((stage) => signatures[stage]).flatMap((signature) => signature.lineage === 'mints' && signature.module !== null
+        ? [signature.module]
+        : []);
+}
+export const LINEAGE_WRITER_MODULES = deriveLineageWriterModules(STAGE_SIGNATURES);

package/dist/core/preflight/plan.d.ts

@@ -0,0 +1,67 @@
+import type { EvalContract } from '../schemas/eval-contract.ts';
+import type { Operation } from '../schemas/interface.ts';
+import type { ProbeRequest } from '../schemas/port-messages.ts';
+import type { JsonValue } from '../schemas/primitives.ts';
+import type { Probe } from '../schemas/probe.ts';
+import type { ManifestationWitness, SensitivityWitness, WitnessInputs } from '../schemas/sensitivity-witness.ts';
+import type { PlanStage } from '../stage-contracts.ts';
+export type PreflightPlanInput = {
+    readonly contract: EvalContract;
+    readonly probes: readonly Probe[];
+    readonly runId: string;
+};
+export type PlannedLegPurpose = 'sensitivity' | 'control-observe' | 'control-mutate' | 'control-reset' | 'seeded-fault';
+export type PlannedLeg = {
+    readonly legId: string;
+    readonly purpose: PlannedLegPurpose;
+    readonly request: ProbeRequest;
+    readonly operation: Operation;
+    readonly inputs: WitnessInputs;
+};
+export type PlannedCheck = {
+    readonly kind: 'interface-present';
+    readonly interfaceId: string;
+    readonly operationId: string;
+    readonly legIds: readonly string[];
+} | {
+    readonly kind: 'input-sensitivity';
+    readonly interfaceId: string;
+    readonly operationId: string;
+    readonly witness: SensitivityWitness | null;
+    readonly operation: Operation;
+} | {
+    readonly kind: 'state-reset';
+    readonly legIds: readonly [string, string];
+} | {
+    readonly kind: 'clean-control';
+    readonly legIds: readonly string[];
+} | {
+    readonly kind: 'seeded-faults-scoped';
+    readonly defectId: string;
+    readonly witness: ManifestationWitness;
+    readonly operation: Operation;
+    readonly cleanLegIds: readonly string[];
+} | {
+    readonly kind: 'seeded-fault-fired';
+    readonly defectId: string;
+    readonly witness: ManifestationWitness | null;
+    readonly operation: Operation | null;
+};
+export type PreflightPlan = {
+    readonly runId: string;
+    readonly legs: readonly PlannedLeg[];
+    readonly checks: readonly PlannedCheck[];
+    readonly referenceSets: Readonly<Record<string, JsonValue[]>>;
+};
+/**
+ * AD-10's plan, derived from the interfaces the contract's probes exercise.
+ *
+ * Three constructions worth knowing. An `interface-present` check is emitted
+ * only for an operation that has at least one leg, because a check over no legs
+ * is satisfied vacuously and asserts nothing. The `state-reset` and
+ * `clean-control` checks are emitted only when control legs could be planned at
+ * all, for the same reason. And the fixture reset's own leg is one more leg
+ * through the same port, so its identifier shares the namespace every other leg
+ * identifier sits in.
+ */
+export declare const planPreflight: PlanStage<PreflightPlanInput, PreflightPlan>;