eval-quality 0.1.0 → 0.3.0

package/dist/core/stage-contracts.d.ts

@@ -1,7 +1,9 @@
 /**
  * AD-34's stage-shape vocabulary. Two conformance types pin the exact shapes
- * of `compile` and `seal`; two generic types record the conditional
- * plan/reduce pair, which `preflight` implements.
+ * of `compile` and `seal`; four generic types cover the rest -- the
+ * conditional plan/reduce pair, which `preflight` implements, `IngestStage`,
+ * generic over the one product it owns, and `ScoreStage`, generic over both
+ * the trial set's element type and the product it owns.
  *
  * AD-24's six-stage input/output/owner/lineage table lives in
  * `core/lineage/stage-table.ts`. This file holds the stage *shapes* TypeScript
@@ -9,7 +11,15 @@
  * Neither imports the other.
  */
 import type { EvalContract } from './schemas/eval-contract.ts';
+import type { EvaluatorConfiguration } from './schemas/evaluator-configuration.ts';
+import type { EvidenceArtifact } from './schemas/evidence-artifact.ts';
+import type { IsolationManifest } from './schemas/isolation-manifest.ts';
+import type { PreflightVerdict } from './schemas/preflight-verdict.ts';
+import type { Probe } from './schemas/probe.ts';
+import type { ScoringPolicy } from './schemas/scoring-policy.ts';
 import type { SealedEvaluatorBrief } from './schemas/sealed-evaluator-brief.ts';
+import type { SealedRunRecord } from './schemas/sealed-run-record.ts';
+import type { WaiverStateValue } from './score/outcome.ts';
 /** The core compile stage's one runtime option. Core behavior never depends on an implicit configuration source (AD-1), so this is required; `application/compile.ts` is the only caller that supplies a default. */
 export type CompileOptions = {
     readonly strict: boolean;
@@ -28,3 +38,53 @@ export type SealStage = (contract: EvalContract) => SealedEvaluatorBrief;
 export type PlanStage<InputArtifact, RequestDescription> = (input: InputArtifact) => RequestDescription;
 /** A stage's pure reducing half: the plan plus the observations a port produced, reduced to the next artifact. */
 export type ReduceStage<RequestDescription, Observation, OutputArtifact> = (plan: RequestDescription, observations: Observation) => OutputArtifact;
+/**
+ * The ingest stage: the three artifacts AD-24's table declares as its inputs,
+ * validated into the internal product it owns. `core/ingest/ingest.ts`
+ * implements it.
+ *
+ * Three parameters and no fourth. AD-24 names `mode` as a value input and the
+ * sealed run record is where mode is read from, so restating it as an argument
+ * would create a disagreement case the design has no rule for.
+ *
+ * The manifest and the evaluator configuration both admit `null` because AD-16
+ * and AD-24 each make an absent artifact a condition this stage records rather
+ * than a shape a caller may not present. Their other shared clause, incomplete,
+ * is a schema rejection raised before this stage sees anything.
+ *
+ * Generic over the product for the same reason `ReduceStage` is: a concrete
+ * return type would import from `core/ingest/`, which imports this file back.
+ */
+export type IngestStage<Product> = (record: SealedRunRecord, manifest: IsolationManifest | null, configuration: EvaluatorConfiguration | null) => Product;
+/**
+ * The score stage: `STAGE_SIGNATURES.score`'s five declared artifact inputs,
+ * plus the two caller-supplied value parameters neither the trial set nor
+ * any declared input carries a source for (`score.ts`'s own Boundaries:
+ * `waiver` mirrors `outcome.ts:58-61`'s AD-5 expiry citation, and
+ * `evaluationFault` has no citation anywhere and is recorded as a genuine
+ * gap this parameter closes).
+ *
+ * Generic over both the trial set's element type and the owned product, for
+ * the same reason `IngestStage` is generic over its product: a concrete
+ * `ValidatedObservations` here would import `core/ingest/ingest.ts`, and a
+ * concrete `ScoredOutcomesAndVerdict` would import `core/score/score.ts`,
+ * either of which imports this file back to type itself.
+ */
+export type ScoreStage<Trials, Product> = (contract: EvalContract, trials: readonly Trials[], probe: Probe, preflightVerdict: PreflightVerdict, policy: ScoringPolicy, waiver: WaiverStateValue, evaluationFault: boolean) => Product;
+/**
+ * The emit stage: `STAGE_SIGNATURES.emit`'s one declared artifact input plus
+ * the three caller-attested AD-11 digests, minting the `EvidenceArtifact`
+ * neither `score` nor any earlier stage can construct alone. One generic
+ * parameter, not two: `EvidenceArtifact` is imported concretely here, the
+ * same way this file already imports several other concrete schema types
+ * above, so importing one more from the same directory opens no cycle;
+ * only the input (`ScoredOutcomesAndVerdict`, defined in
+ * `core/score/score.ts`) needs genericity, for the same import-cycle reason
+ * `IngestStage`/`ScoreStage` are generic over theirs.
+ *
+ * The three digests are plain `string` rather than a `Digest` type: no
+ * `core/schemas` module exports a bare TypeScript `Digest` type, only the
+ * Zod schema, and `EvidenceArtifact`'s own parse is where their AD-27 shape
+ * is actually enforced.
+ */
+export type EmitStage<Input> = (scored: Input, corpusDigest: string, fixtureDigest: string, evaluatorConfigurationDigest: string) => EvidenceArtifact;

package/dist/index.d.ts

@@ -12,4 +12,4 @@ export type { ScoringPolicy } from './core/schemas/scoring-policy.ts';
 export type { SealedEvaluatorBrief } from './core/schemas/sealed-evaluator-brief.ts';
 export type { SealedRunRecord } from './core/schemas/sealed-run-record.ts';
 export type { FixtureReset, ManifestationWitness, SensitivityWitness, SensitivityWitnessLeg, WitnessChannel, WitnessInputs, } from './core/schemas/sensitivity-witness.ts';
-export declare const VERSION = "0.1.0";
+export declare const VERSION = "0.3.0";

package/dist/index.js

@@ -19,4 +19,4 @@
 // subpath, where AD-37 puts the conformance definition an adapter author
 // reads; the reference adapters stay at `eval-quality/adapters`.
 export * from './application/index.js';
-export const VERSION = '0.1.0';
+export const VERSION = '0.3.0';

package/dist/ports/environment-probe-port.d.ts

@@ -31,10 +31,11 @@ export type EnvironmentProbePort = {
 };
 /** the boundary parsers `application/` and the conformance suite validate with. */
 export declare const probeParsers: {
-    readonly request: import("zod").ZodObject<{
+    readonly request: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
         probeId: import("zod").ZodString;
         interfaceId: import("zod").ZodString;
         operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"api">;
         method: import("zod").ZodEnum<{
             DELETE: "DELETE";
             GET: "GET";
@@ -56,11 +57,33 @@ export declare const probeParsers: {
                 kind: import("zod").ZodLiteral<"absent">;
             }, import("zod/v4/core").$strict>], "kind">;
         }, import("zod/v4/core").$strict>;
-    }, import("zod/v4/core").$strict>;
-    readonly response: import("zod").ZodObject<{
+    }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
         probeId: import("zod").ZodString;
         interfaceId: import("zod").ZodString;
         operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"cli">;
+        executable: import("zod").ZodString;
+        subcommandPath: import("zod").ZodArray<import("zod").ZodString>;
+        channels: import("zod").ZodObject<{
+            argument: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
+            option: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
+            environment: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
+            stdin: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+                kind: import("zod").ZodLiteral<"json">;
+                value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+            }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                kind: import("zod").ZodLiteral<"text">;
+                value: import("zod").ZodString;
+            }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                kind: import("zod").ZodLiteral<"absent">;
+            }, import("zod/v4/core").$strict>], "kind">;
+        }, import("zod/v4/core").$strict>;
+    }, import("zod/v4/core").$strict>], "kind">;
+    readonly response: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+        probeId: import("zod").ZodString;
+        interfaceId: import("zod").ZodString;
+        operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"api">;
         status: import("zod").ZodInt;
         headers: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
         body: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
@@ -72,5 +95,38 @@ export declare const probeParsers: {
         }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
             kind: import("zod").ZodLiteral<"absent">;
         }, import("zod/v4/core").$strict>], "kind">;
-    }, import("zod/v4/core").$strict>;
+    }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+        probeId: import("zod").ZodString;
+        interfaceId: import("zod").ZodString;
+        operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"cli">;
+        exitCode: import("zod").ZodInt;
+        stdout: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"json">;
+            value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"text">;
+            value: import("zod").ZodString;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"absent">;
+        }, import("zod/v4/core").$strict>], "kind">;
+        stderr: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"json">;
+            value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"text">;
+            value: import("zod").ZodString;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"absent">;
+        }, import("zod/v4/core").$strict>], "kind">;
+        artifacts: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"json">;
+            value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"text">;
+            value: import("zod").ZodString;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"absent">;
+        }, import("zod/v4/core").$strict>], "kind">>;
+    }, import("zod/v4/core").$strict>], "kind">;
 };

package/dist/testing/index.d.ts

@@ -10,6 +10,8 @@
  * Type re-exports use `export type`; `biome.json` sets
  * `style.useExportType: "error"`.
  */
+export type { RuntimeFaultCode } from '../core/schemas/faults.ts';
+export { RUNTIME_FAULT_CODES, RuntimeFault } from '../core/schemas/faults.ts';
 export type { ClockReadRequest, ClockReadResponse, CorpusResolveRequest, CorpusResolveResponse, FileReadRequest, FileReadResponse, FileWriteRequest, FileWriteResponse, ProbeObservation, ProbeObservedBody, ProbeRequest, ProbeRequestBody, } from '../core/schemas/port-messages.ts';
 export type { ProbeTargetAuthorization, ProbeTargetPolicy, } from '../core/schemas/probe-policy.ts';
 export type { ClockPort } from '../ports/clock-port.ts';

package/dist/testing/index.js

@@ -10,6 +10,7 @@
  * Type re-exports use `export type`; `biome.json` sets
  * `style.useExportType: "error"`.
  */
+export { RUNTIME_FAULT_CODES, RuntimeFault } from '../core/schemas/faults.js';
 export { clockReadParsers } from '../ports/clock-port.js';
 export { corpusResolveParsers } from '../ports/corpus-port.js';
 export { probeParsers } from '../ports/environment-probe-port.js';

package/dist/testing/probe-conformance.d.ts

@@ -6,6 +6,35 @@
  * Each of the thirteen builds its own `'resolves'` subject and reads
  * `underlyingCalls()` from a counter starting at zero, so every count below is
  * absolute.
+ *
+ * **This suite certifies an adapter for the `api` mechanism and says nothing
+ * about the `cli` one.** Every scenario is HTTP: redirects, methods, schemes,
+ * an anomalous status. `ProbeSubject` requires an unauthorized-method request,
+ * a redirecting request and a not-found request, none of which a command
+ * adapter has. So an adapter that runs commands cannot present a subject here,
+ * and an adapter that passes all nineteen has been shown nothing about whether
+ * it can run one safely: it may deny an unmapped executable or execute a shell
+ * string, and this suite cannot tell the difference.
+ *
+ * That is a real gap in the published surface and it is wider than this suite.
+ * `ProbeTargetPolicy` has one authorization shape and every field in it is
+ * HTTP: scheme, host, port, resolved addresses, methods, safe methods,
+ * redirects, request and response byte caps. **A command interface cannot be
+ * authorized at all**, so AD-35's "an adapter denies by default and permits
+ * only what that mapping names" has nothing to name for the mechanism this
+ * release adds, and a conformance arm built on top of a policy that cannot
+ * express a command authorization would certify against nothing.
+ *
+ * Closing it is a declaration first and a suite second: a command
+ * authorization naming a permitted executable, the subcommand paths and
+ * environment keys it may carry, and its own elapsed and output-byte caps;
+ * then the denials to certify — an executable no mapping names, a refusal to
+ * accept a pre-built argument vector, a non-zero exit as an observation rather
+ * than a fault, each cap enforced. That is a design addition to the published
+ * surface rather than a repair, and it is not made here. Until it exists an
+ * adopter writing a command adapter has no policy to declare and no harness to
+ * run, and this comment is here so nobody concludes from a green run that they
+ * have either.
  */
 import type { ProbeRequest } from '../core/schemas/port-messages.ts';
 import type { ProbeTargetPolicy } from '../core/schemas/probe-policy.ts';

package/dist/testing/probe-conformance.js

@@ -23,9 +23,9 @@ const PROBE_ASSERTIONS = [
         request: (subject) => subject.faultingRequest,
         expectation: {
             kind: 'resolves',
-            check: (observation) => observation.status === 500
+            check: (observation) => observation.kind === 'api' && observation.status === 500
                 ? undefined
-                : `observed status ${observation.status}, expected 500`,
+                : `observed ${observation.kind === 'api' ? `status ${observation.status}` : 'a command observation'}, expected status 500`,
         },
     },
     {
@@ -114,14 +114,42 @@ const PROBE_ASSERTIONS = [
         expectation: { kind: 'rejects', code: CAPPED },
     },
 ];
-function checkResolved(assertion, expectation, value) {
+function checkResolved(assertion, expectation, value, request) {
     const parsed = probeParsers.response.safeParse(value);
     if (!parsed.success) {
         return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
     }
+    // Schema validity is not correlation. Both messages are unions now, so an
+    // adapter can answer a command request with a schema-valid HTTP observation
+    // and satisfy every assertion below without running a command or consulting
+    // a policy. The four echoed fields are what tie one answer to one question,
+    // and `ProbeRequest.probeId`'s own description already says the port returns
+    // them unchanged.
+    const mismatch = echoMismatch(request, parsed.data);
+    if (mismatch !== undefined) {
+        return titledOutcome(assertion.id, assertion.title, false, mismatch);
+    }
     const complaint = expectation.check?.(parsed.data);
     return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
 }
+/**
+ * Which of the four echoed fields the observation failed to return unchanged,
+ * or `undefined` when it answered the question it was asked.
+ */
+function echoMismatch(request, observation) {
+    const echoed = [
+        ['kind', request.kind, observation.kind],
+        ['probeId', request.probeId, observation.probeId],
+        ['interfaceId', request.interfaceId, observation.interfaceId],
+        ['operationId', request.operationId, observation.operationId],
+    ];
+    for (const [field, asked, answered] of echoed) {
+        if (asked === answered)
+            continue;
+        return `observed ${field} "${answered}" for a request carrying "${asked}", so the answer does not correlate with the question`;
+    }
+    return undefined;
+}
 function checkRejected(assertion, expectedCode, error) {
     const view = faultView(error);
     if (view === undefined) {
@@ -153,12 +181,13 @@ async function runProbeAssertion(assertion, subject) {
     if (!run.ok) {
         return titledOutcome(assertion.id, assertion.title, false, run.detail);
     }
-    const settled = await settle(run.built.port(assertion.request(subject), new AbortController().signal));
+    const request = assertion.request(subject);
+    const settled = await settle(run.built.port(request, new AbortController().signal));
     let result;
     if (assertion.expectation.kind === 'resolves') {
         result =
             settled.kind === 'resolved'
-                ? checkResolved(assertion, assertion.expectation, settled.value)
+                ? checkResolved(assertion, assertion.expectation, settled.value, request)
                 : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
     }
     else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eval-quality",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
   "author": "Murat Ozcan",
   "license": "Apache-2.0",
@@ -87,11 +87,17 @@
     "check:ad28-registry": "node scripts/check-ad28-registry.ts",
     "generate:ad31-table": "node scripts/generate-ad31-table.ts",
     "check:ad31-table": "node scripts/check-ad31-table.ts",
+    "generate:ad33-table": "node scripts/generate-ad33-table.ts",
+    "check:ad33-table": "node scripts/check-ad33-table.ts",
+    "generate:ad21-table": "node scripts/generate-ad21-table.ts",
+    "check:ad21-table": "node scripts/check-ad21-table.ts",
     "check:layers": "node scripts/check-dependency-direction.ts",
     "check:lineage": "node scripts/check-lineage-ownership.ts",
     "check:boundary": "node scripts/check-package-boundary.ts",
     "generate:dev-corpus": "node scripts/generate-dev-corpus.ts",
     "check:corpus": "node scripts/check-dev-corpus.ts",
+    "generate:worked-example": "node scripts/generate-worked-example.ts",
+    "check:worked-example": "node scripts/check-worked-example.ts",
     "check:website-deps": "node scripts/audit-lockfile-age.mjs --lockfile website/package-lock.json && node scripts/check-licenses.mjs --lockfile website/package-lock.json",
     "check:shareable": "node scripts/check-shareable.mjs",
     "bench:digest": "node scripts/bench-digest.ts",
@@ -99,9 +105,12 @@
     "docs:build": "node tools/build-docs.mjs",
     "docs:preview": "npm --prefix website run preview",
     "docs:validate-links": "node tools/validate-doc-links.js",
+    "release:patch": "gh workflow run publish.yml --ref main -f bump=patch",
+    "release:minor": "gh workflow run publish.yml --ref main -f bump=minor",
+    "release:major": "gh workflow run publish.yml --ref main -f bump=major",
     "release:prepare": "node scripts/release-prepare.mjs",
-    "release:publish": "gh workflow run publish.yml --ref main",
-    "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:website-deps && npm run test:coverage",
+    "release:publish": "gh workflow run publish.yml --ref main -f bump=none",
+    "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:worked-example && npm run check:website-deps && npm run test:coverage",
     "prepack": "npm run clean && npm run build",
     "prepublishOnly": "node scripts/assert-publish-authorized.mjs"
   },