eval-quality 0.1.0

package/dist/cli/render.js

@@ -0,0 +1,53 @@
+/**
+ * The four shapes the binary writes, and the exit-code table two documents
+ * share. Every line the CLI emits is produced here, so a format change is one
+ * file.
+ */
+import { RuntimeFault, StructuralFailure, serializeArtifact, } from '../application/index.js';
+const PREFIX = 'eval-quality';
+/**
+ * Delegates to `serializeArtifact`; the canonical bytes are not re-derived
+ * here, so the text written to stdout is the text `digestArtifact` hashes.
+ */
+export function renderArtifact(artifact, artifactPath) {
+    return serializeArtifact(artifact, artifactPath);
+}
+/** `eval-quality: <stage>: <runId>: <message>` */
+export function renderDiagnostic(diagnostic) {
+    return `${PREFIX}: ${diagnostic.stage}: ${diagnostic.runId}: ${diagnostic.message}`;
+}
+/**
+ * `eval-quality: <code>: <artifactPath>: <detail>` for either error class.
+ * Anything else falls back to `String(error)`, which is what a defect in our
+ * own code looks like from outside.
+ */
+export function renderError(error) {
+    if (error instanceof StructuralFailure || error instanceof RuntimeFault) {
+        const prefix = `${error.code} in ${error.artifactPath}: `;
+        const detail = error.message.startsWith(prefix)
+            ? error.message.slice(prefix.length)
+            : error.message;
+        return `${PREFIX}: ${error.code}: ${error.artifactPath}: ${detail}`;
+    }
+    return `${PREFIX}: ${String(error)}`;
+}
+/**
+ * AD-21's seven exit codes, one line each. The `--help` output and the README
+ * table are this text, so the two cannot drift.
+ */
+export const EXIT_CODE_TABLE = `Exit codes (AD-21):
+  0   success, and every verdict other than FAIL or a promoted CONCERNS
+  1   CONCERNS promoted by --strict
+  2   FAIL
+  3   invalid: a pre-flight verdict that did not pass
+  4   structural failure
+  5   runtime fault
+  64  usage error
+
+  1 and 2 report a scored verdict. Scoring ships in a later release, so no
+  command here reaches either yet, and --strict changes no code this binary
+  produces.`;
+/** `eval-quality: usage: <message>` */
+export function renderUsage(message) {
+    return `${PREFIX}: usage: ${message}`;
+}

package/dist/cli/run.d.ts

@@ -0,0 +1,41 @@
+/**
+ * The three commands. Each one reads its inputs, makes exactly one call into
+ * `application/`, serializes what came back, and returns the outcome the exit
+ * code is derived from. Every effect is a member of `RunEnvironment`, so this
+ * module touches no stream, no `process`, and no Node builtin, and its tests
+ * run in memory.
+ */
+import { compile, preflightFromObservations, seal } from '../application/index.ts';
+import type { Command, ParsedInvocation } from './arguments.ts';
+import type { CommandOutcome } from './exit-codes.ts';
+export type RunEnvironment = {
+    readonly readInput: (source: string | null) => Promise<string>;
+    readonly writeArtifact: (path: string, body: string) => Promise<void>;
+    readonly writeOut: (body: string) => void;
+    readonly writeDiagnostic: (line: string) => void;
+    readonly resolvePath: (path: string) => string;
+    readonly joinPath: (directory: string, name: string) => string;
+    /**
+     * Whether two paths name one file on disk. String normalization cannot
+     * settle it: a symlink and a case-insensitive filesystem both alias two
+     * spellings that no amount of folding brings together.
+     */
+    readonly sameFile: (left: string, right: string) => Promise<boolean>;
+    readonly version: string;
+};
+/**
+ * The three orchestration calls, behind one object so a test can count them.
+ * An input/output count says nothing about how many calls into `application/`
+ * happened, which is the property AD-14 constrains.
+ */
+export type ApplicationFacade = {
+    readonly compile: typeof compile;
+    readonly seal: typeof seal;
+    readonly preflightFromObservations: typeof preflightFromObservations;
+};
+export declare const APPLICATION: ApplicationFacade;
+export type RunResult = {
+    readonly outcome: CommandOutcome;
+};
+export declare function helpText(command: Command | null): string;
+export declare function run(invocation: ParsedInvocation, environment: RunEnvironment, application?: ApplicationFacade): Promise<RunResult>;

package/dist/cli/run.js

@@ -0,0 +1,203 @@
+/**
+ * The three commands. Each one reads its inputs, makes exactly one call into
+ * `application/`, serializes what came back, and returns the outcome the exit
+ * code is derived from. Every effect is a member of `RunEnvironment`, so this
+ * module touches no stream, no `process`, and no Node builtin, and its tests
+ * run in memory.
+ */
+import { compile, preflightFromObservations, RuntimeFault, StructuralFailure, seal, } from '../application/index.js';
+import { EXIT_CODE_TABLE, renderArtifact, renderDiagnostic, renderError, renderUsage, } from './render.js';
+export const APPLICATION = {
+    compile,
+    seal,
+    preflightFromObservations,
+};
+/** The artifact each command emits: its schema name and its file name. */
+const EMITTED = {
+    compile: { artifactPath: 'EvalContract', kind: 'eval-contract' },
+    seal: {
+        artifactPath: 'SealedEvaluatorBrief',
+        kind: 'sealed-evaluator-brief',
+    },
+    preflight: { artifactPath: 'PreflightVerdict', kind: 'preflight-verdict' },
+};
+/** The schema an input key deserializes into, for the parse fault's path. */
+const INPUT_ARTIFACT_PATH = {
+    in: 'EvalContract',
+    contract: 'EvalContract',
+    probes: 'Probe',
+    observations: 'ProbeObservation',
+};
+const USAGE = `Usage:
+  eval-quality compile          [--in <path>] [--out <target>]
+                                [--strict-inputs | --no-strict-inputs] [--strict]
+  eval-quality seal             [--in <path>] [--out <target>]
+                                [--strict-inputs | --no-strict-inputs] [--strict]
+  eval-quality preflight         --contract <path> --probes <path> --observations <path>
+                                 --run-id <id> [--out <target>] [--strict]
+  eval-quality --help | -h | help [<command>]
+  eval-quality --version | -V`;
+const COMMAND_USAGE = {
+    compile: `Usage:
+  eval-quality compile          [--in <path>] [--out <target>]
+                                [--strict-inputs | --no-strict-inputs] [--strict]
+
+  --in <path>              the contract to compile; stdin when absent or "-"
+  --out <target>           a .json file path, or a directory taking eval-contract.json
+  --strict-inputs          reject undeclared inputs (default)
+  --no-strict-inputs       allow undeclared inputs
+  --strict                 promote CONCERNS to exit 1`,
+    seal: `Usage:
+  eval-quality seal             [--in <path>] [--out <target>]
+                                [--strict-inputs | --no-strict-inputs] [--strict]
+
+  --in <path>              the contract to compile and seal; stdin when absent or "-"
+  --out <target>           a .json file path, or a directory taking sealed-evaluator-brief.json
+  --strict-inputs          reject undeclared inputs (default)
+  --no-strict-inputs       allow undeclared inputs
+  --strict                 promote CONCERNS to exit 1`,
+    preflight: `Usage:
+  eval-quality preflight         --contract <path> --probes <path> --observations <path>
+                                 --run-id <id> [--out <target>] [--strict]
+
+  --contract <path>        the compiled contract the plan is built from
+  --probes <path>          the probe list the plan is built from
+  --observations <path>    the observations to reduce over
+  --run-id <id>            the run identifier the verdict is minted for
+  --out <target>           a .json file path, or a directory taking preflight-verdict.json
+  --strict                 promote CONCERNS to exit 1`,
+};
+const IO_RULES = `Inputs and outputs:
+  An input flag left out reads stdin, and "-" names stdin explicitly; at most
+  one input may be "-". Without --out the artifact goes to stdout. An --out
+  ending in .json is a file path; anything else is a directory taking
+  <target>/<kind>.json. Diagnostics and errors go to stderr.`;
+export function helpText(command) {
+    if (command === null) {
+        return `${USAGE}\n\n${IO_RULES}\n\n${EXIT_CODE_TABLE}`;
+    }
+    return `${COMMAND_USAGE[command]}\n\n${EXIT_CODE_TABLE}`;
+}
+/** `undefined` and `"-"` both mean stdin, which `readInput` takes as `null`. */
+const sourceOf = (value) => value === undefined || value === '-' ? null : value;
+async function readJson(environment, key, value) {
+    const text = await environment.readInput(sourceOf(value));
+    try {
+        return JSON.parse(text);
+    }
+    catch (error) {
+        // AD-28's `schema-parse-failure` covers an artifact that does not parse,
+        // and the CLI is the boundary that deserializes.
+        throw new RuntimeFault('schema-parse-failure', INPUT_ARTIFACT_PATH[key], `--${key} is not JSON: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+    }
+}
+/** The file `--out` names, or `null` when the artifact goes to stdout. */
+function outputPath(environment, out, kind) {
+    if (out === null)
+        return null;
+    // The suffix is the whole classifier: the CLI never stats to decide this.
+    // Matched case-insensitively, because a case-insensitive filesystem accepts
+    // `run.JSON` for the same file and would otherwise be read as a directory.
+    return out.toLowerCase().endsWith('.json')
+        ? out
+        : environment.joinPath(out, `${kind}.json`);
+}
+/**
+ * An input is never mutated in place. Two checks, because neither alone is
+ * enough: comparing resolved paths catches `.`, `..`, and a doubled separator,
+ * and `sameFile` catches a symlink and a case-insensitive filesystem, which no
+ * string normalization can fold together.
+ */
+async function collides(environment, target, inputs) {
+    const resolvedTarget = environment.resolvePath(target);
+    for (const [key, value] of Object.entries(inputs)) {
+        if (value === undefined || value === '-')
+            continue;
+        const resolvedInput = environment.resolvePath(value);
+        if (resolvedInput === resolvedTarget ||
+            (await environment.sameFile(resolvedInput, resolvedTarget))) {
+            return `--out resolves to "${resolvedTarget}", which is also --${key} "${value}"`;
+        }
+    }
+    return null;
+}
+async function emitArtifact(environment, artifact, command, target) {
+    const body = renderArtifact(artifact, EMITTED[command].artifactPath);
+    if (target === null) {
+        environment.writeOut(body);
+        return;
+    }
+    await environment.writeArtifact(target, body);
+}
+export async function run(invocation, environment, application = APPLICATION) {
+    switch (invocation.kind) {
+        case 'help':
+            environment.writeOut(`${helpText(invocation.command)}\n`);
+            return { outcome: { kind: 'artifact' } };
+        case 'version':
+            environment.writeOut(`${environment.version}\n`);
+            return { outcome: { kind: 'artifact' } };
+        case 'usage-error':
+            environment.writeDiagnostic(renderUsage(invocation.message));
+            return { outcome: { kind: 'usage-error' } };
+        case 'run':
+            return await runCommand(invocation, environment, application);
+    }
+}
+async function runCommand(invocation, environment, application) {
+    const { command, inputs, out, strictInputs } = invocation;
+    const target = outputPath(environment, out, EMITTED[command].kind);
+    if (target !== null) {
+        const collision = await collides(environment, target, inputs);
+        if (collision !== null) {
+            environment.writeDiagnostic(renderUsage(collision));
+            return { outcome: { kind: 'usage-error' } };
+        }
+    }
+    try {
+        if (command === 'preflight') {
+            const verdict = await runPreflightCommand(invocation, environment, application, target);
+            return { outcome: { kind: 'preflight', passed: verdict.passed } };
+        }
+        const input = await readJson(environment, 'in', inputs.in);
+        const options = { strict: strictInputs };
+        const artifact = command === 'compile'
+            ? application.compile(input, options)
+            : application.seal(input, options);
+        // No diagnostic on success: neither command carries a run identifier,
+        // so any line either wrote would name no run and no stage.
+        await emitArtifact(environment, artifact, command, target);
+        return { outcome: { kind: 'artifact' } };
+    }
+    catch (error) {
+        if (error instanceof StructuralFailure) {
+            environment.writeDiagnostic(renderError(error));
+            return { outcome: { kind: 'structural-failure' } };
+        }
+        if (error instanceof RuntimeFault) {
+            environment.writeDiagnostic(renderError(error));
+            return { outcome: { kind: 'fault' } };
+        }
+        // A defect in our own code surfaces as a stack, never as exit 5.
+        throw error;
+    }
+}
+async function runPreflightCommand(invocation, environment, application, target) {
+    const { inputs, runId } = invocation;
+    const contract = (await readJson(environment, 'contract', inputs.contract));
+    const probes = (await readJson(environment, 'probes', inputs.probes));
+    const observations = (await readJson(environment, 'observations', inputs.observations));
+    const sink = (diagnostic) => {
+        environment.writeDiagnostic(renderDiagnostic(diagnostic));
+    };
+    const verdict = application.preflightFromObservations({
+        contract,
+        probes,
+        // The parser requires `--run-id` on this command, so it is never null.
+        runId: runId ?? '',
+        observations,
+        sink,
+    });
+    await emitArtifact(environment, verdict, 'preflight', target);
+    return verdict;
+}

package/dist/core/canonical/canonicalize.d.ts

@@ -0,0 +1 @@
+export declare function canonicalize(value: unknown, artifactPath: string): Uint8Array;

package/dist/core/canonical/canonicalize.js

@@ -0,0 +1,113 @@
+/** JSON to bytes */
+import { RuntimeFault } from '../schemas/faults.js';
+import { assertDomainNumber, assertDomainString, MAX_NESTING_DEPTH, } from './value-domain.js';
+// RFC 8785 (JCS) canonical serialization, written in-house per AD-27; native
+// JSON.stringify on individual numbers already implements JCS's number
+// algorithm, so no hand-rolled Ryū is needed.
+//
+// Validation is fused into serialization as one traversal over a single
+// ownKeys/descriptor snapshot, so exactly what was validated is what gets
+// emitted. A second traversal would reopen the TOCTOU channel: an accessor or
+// lying Proxy could answer the emit-time read differently than the
+// validation read.
+export function canonicalize(value, artifactPath) {
+    return new TextEncoder().encode(serialize(value, artifactPath, '$', new Set(), 0));
+}
+const fault = (artifactPath, detail) => {
+    throw new RuntimeFault('non-canonicalizable-value', artifactPath, detail);
+};
+function serialize(value, artifactPath, location, ancestors, depth) {
+    if (value === null)
+        return 'null';
+    switch (typeof value) {
+        case 'boolean':
+            return value ? 'true' : 'false';
+        case 'number':
+            assertDomainNumber(value, artifactPath, location);
+            // Whole-value JSON.stringify would drop undefined properties, honour
+            // toJSON, and skip key sorting, so this runs per scalar instead.
+            return JSON.stringify(value);
+        case 'string':
+            assertDomainString(value, artifactPath, location);
+            return JSON.stringify(value);
+        case 'object':
+            break;
+        default:
+            // undefined, function, bigint, symbol: unrepresentable, never coerced silently.
+            fault(artifactPath, `${typeof value} is not a hashed-artifact value at ${location}`);
+    }
+    const object = value;
+    if (depth >= MAX_NESTING_DEPTH) {
+        fault(artifactPath, `nesting depth exceeds ${MAX_NESTING_DEPTH} at ${location}`);
+    }
+    if (ancestors.has(object)) {
+        fault(artifactPath, `cyclic reference at ${location}`);
+    }
+    ancestors.add(object);
+    let rendered;
+    if (Array.isArray(object)) {
+        if (Object.getPrototypeOf(object) !== Array.prototype) {
+            fault(artifactPath, `non-plain array at ${location}`);
+        }
+        // One property-table read; elements come from descriptor values, never a
+        // second [[Get]]. A toJSON carrier surfaces as a non-index key (own);
+        // the prototype check rules out an inherited one.
+        const descriptors = Object.getOwnPropertyDescriptors(object);
+        if (Object.getOwnPropertySymbols(descriptors).length > 0) {
+            fault(artifactPath, `symbol-keyed property at ${location}`);
+        }
+        const elements = [];
+        for (const [key, descriptor] of Object.entries(descriptors)) {
+            if (key === 'length')
+                continue;
+            if (!('value' in descriptor)) {
+                fault(artifactPath, `accessor property ${JSON.stringify(key)} at ${location}`);
+            }
+            if (!descriptor.enumerable) {
+                fault(artifactPath, `non-enumerable property ${JSON.stringify(key)} at ${location}`);
+            }
+            if (key !== String(elements.length)) {
+                fault(artifactPath, `array with holes or non-index properties at ${location}`);
+            }
+            elements.push(descriptor.value);
+        }
+        if (elements.length !== object.length) {
+            fault(artifactPath, `array with holes or non-index properties at ${location}`);
+        }
+        rendered = `[${elements
+            .map((element, index) => serialize(element, artifactPath, `${location}[${index}]`, ancestors, depth + 1))
+            .join(',')}]`;
+    }
+    else {
+        const prototype = Object.getPrototypeOf(object);
+        if (prototype !== Object.prototype && prototype !== null) {
+            fault(artifactPath, `non-plain object at ${location}`);
+        }
+        const descriptors = Object.getOwnPropertyDescriptors(object);
+        if (Object.getOwnPropertySymbols(descriptors).length > 0) {
+            fault(artifactPath, `symbol-keyed property at ${location}`);
+        }
+        const properties = [];
+        for (const [key, descriptor] of Object.entries(descriptors)) {
+            if (!('value' in descriptor)) {
+                fault(artifactPath, `accessor property ${JSON.stringify(key)} at ${location}`);
+            }
+            if (!descriptor.enumerable) {
+                fault(artifactPath, `non-enumerable property ${JSON.stringify(key)} at ${location}`);
+            }
+            // Keys are validated on the same snapshot they are emitted from.
+            assertDomainString(key, artifactPath, `${location} (object key)`);
+            // A callable toJSON is rejected wherever it sits: as an own data
+            // property its function value faults in the recursion below.
+            properties.push([key, descriptor.value]);
+        }
+        // Plain < on JS strings compares UTF-16 code units, the required order:
+        // never localeCompare, never Intl, never .normalize().
+        properties.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+        rendered = `{${properties
+            .map(([key, entry]) => `${JSON.stringify(key)}:${serialize(entry, artifactPath, `${location}.${key}`, ancestors, depth + 1)}`)
+            .join(',')}}`;
+    }
+    ancestors.delete(object);
+    return rendered;
+}

package/dist/core/canonical/digest.d.ts

@@ -0,0 +1,7 @@
+export declare const COMPOSITE_PROTOCOL_TAG = "eval-quality/composite/v1";
+export declare const DIRECTORY_PROTOCOL_TAG = "eval-quality/directory/v1";
+export declare function digestArtifact(value: unknown, artifactPath: string): string;
+export declare function digestBytes(bytes: Uint8Array): string;
+export declare function digestJson(input: Uint8Array | string, artifactPath: string): string;
+export declare function digestComposite(fields: Record<string, unknown>, artifactPath: string): string;
+export declare function digestDirectory(members: Record<string, string>, artifactPath: string): string;

package/dist/core/canonical/digest.js

@@ -0,0 +1,81 @@
+/** the five digest functions. */
+import { createHash } from 'node:crypto';
+import { DIGEST_FORM } from '../schemas/primitives.js';
+import { canonicalize } from './canonicalize.js';
+import { scanJson } from './scan-json.js';
+// AD-27 digest computation. node:crypto is the one permitted builtin in core/
+// (AD-1: digesting is deterministic; there is deliberately no digest port).
+// No artifact carries its own digest: digests live only in referring
+// artifacts, so there is no self-exclusion rule here.
+export const COMPOSITE_PROTOCOL_TAG = 'eval-quality/composite/v1';
+export const DIRECTORY_PROTOCOL_TAG = 'eval-quality/directory/v1';
+const render = (bytes) => `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
+export function digestArtifact(value, artifactPath) {
+    return render(canonicalize(value, artifactPath));
+}
+export function digestBytes(bytes) {
+    return render(bytes);
+}
+// Entry point for raw hashed-artifact text or bytes: scans lexically first
+// (see scan-json.ts for why), then digests. Callers holding raw input use
+// this, never digestArtifact(JSON.parse(text)).
+export function digestJson(input, artifactPath) {
+    return digestArtifact(scanJson(input, artifactPath), artifactPath);
+}
+// A composite digest is a digest over a domain-separated tagged object with
+// named fields, never a concatenation of member strings.
+export function digestComposite(fields, artifactPath) {
+    if (Object.keys(fields).length === 0) {
+        // Empty composites are rejected: relaxing this later can't break frozen
+        // digests, but legalizing it now would be irreversible.
+        throw new TypeError('composite requires at least one field');
+    }
+    if (Object.hasOwn(fields, 'protocol')) {
+        throw new TypeError('composite fields must not carry a "protocol" member');
+    }
+    // The spread snapshots each field once into a fresh plain object, so the
+    // canonicalizer digests exactly what was read here.
+    return digestArtifact({ protocol: COMPOSITE_PROTOCOL_TAG, ...fields }, artifactPath);
+}
+// Member paths must arrive in one canonical spelling: relative, forward-slash,
+// no empty/dot/dot-dot segments. Two producers naming the same file "a" and
+// "./a" would otherwise mint different digests with no fault raised. Unicode
+// normalization is deliberately not applied (the canonicalizer is never
+// normalization-aware); NFC vs NFD spelling is the producer's responsibility
+// per the fixtures README.
+function assertMemberPath(path) {
+    if (path === '') {
+        throw new TypeError('directory member path must not be empty');
+    }
+    if (path.includes('\\')) {
+        throw new TypeError(`directory member path must use forward slashes: ${JSON.stringify(path)}`);
+    }
+    if (path.startsWith('/')) {
+        throw new TypeError(`directory member path must be relative: ${JSON.stringify(path)}`);
+    }
+    for (const segment of path.split('/')) {
+        if (segment === '' || segment === '.' || segment === '..') {
+            throw new TypeError(`directory member path not in canonical form: ${JSON.stringify(path)}`);
+        }
+    }
+}
+// A directory digest is a composite over its members ordered by path. Members
+// nest under a fixed "members" field so a path can never collide with the
+// protocol tag; canonical key sorting orders them by path automatically:
+// UTF-16 code-unit order, not the UTF-8 byte order git produces.
+export function digestDirectory(members, artifactPath) {
+    const entries = Object.entries(members);
+    if (entries.length === 0) {
+        throw new TypeError('directory requires at least one member');
+    }
+    for (const [path, digest] of entries) {
+        assertMemberPath(path);
+        if (!DIGEST_FORM.test(digest)) {
+            throw new TypeError(`directory member ${JSON.stringify(path)} is not a sha256: digest`);
+        }
+    }
+    // Digest the snapshot read above, not the caller's object: a getter or
+    // Proxy could otherwise answer the canonicalizer's read with a value the
+    // DIGEST_FORM check never saw.
+    return digestArtifact({ protocol: DIRECTORY_PROTOCOL_TAG, members: Object.fromEntries(entries) }, artifactPath);
+}

package/dist/core/canonical/scan-json.d.ts

@@ -0,0 +1 @@
+export declare function scanJson(input: Uint8Array | string, artifactPath: string): unknown;