eval-quality 0.1.0 → 0.3.0

package/dist/core/schemas/port-messages.js

@@ -2,7 +2,7 @@
 import { z } from 'zod';
 import { HttpMethod, PathTemplate } from './interface.js';
 import { Identifier, JsonValue, KeyName, Rfc3339Utc } from './primitives.js';
-import { ProbeObservedBody, ProbeRequestBody } from './probe-body.js';
+import { ProbeObservedBody, ProbeRequestBody, ProbeRequestStdin, } from './probe-body.js';
 export { ProbeObservedBody, ProbeRequestBody } from './probe-body.js';
 // AD-8: the corpus port resolves an opaque reference to bytes from a
 // caller-owned location. It does not check the digest: AD-8 puts digest
@@ -52,10 +52,20 @@ export const FileWriteResponse = z.strictObject({
  * authorization material is the adapter's, supplied by the same mapping that
  * authorizes the target.
  */
-export const ProbeRequest = z.strictObject({
+const probeCorrelation = {
     probeId: Identifier.describe('An opaque correlation label minted by the pre-flight plan and echoed unchanged on the observation. Two witnesses of one operation, and the three observations a state-reset differential needs, are otherwise distinguishable only by array position, which NFR9 forbids any stage from reading.'),
     interfaceId: Identifier,
     operationId: Identifier,
+};
+/**
+ * A request to an interface that speaks HTTP. Tagged on `kind` so an adapter
+ * knows what it is being asked to do before it reads anything else, and so the
+ * two shapes cannot be confused for one another by a field that happens to be
+ * absent.
+ */
+export const ApiProbeRequest = z.strictObject({
+    ...probeCorrelation,
+    kind: z.literal('api'),
     method: HttpMethod,
     pathTemplate: PathTemplate,
     channels: z.strictObject({
@@ -67,6 +77,38 @@ export const ProbeRequest = z.strictObject({
         body: ProbeRequestBody,
     }),
 });
+/**
+ * A request to run a command.
+ *
+ * AD-35's rule is that a contract names a logical identifier and the caller
+ * maps it, and the analogue for a command is exactly the same shape: the
+ * `executable` is the logical name the contract declared, and the adapter maps
+ * it to something runnable from configuration outside the contract. No
+ * filesystem path, no interpreter, no shell string. The adapter builds the
+ * argument vector; it is never handed one to execute.
+ *
+ * `environment` is string-valued for the reason `header` is, and carries no
+ * credential for the reason AD-18 gives: authorization material is the
+ * adapter's, supplied by the same mapping that authorizes the target.
+ */
+export const CommandProbeRequest = z.strictObject({
+    ...probeCorrelation,
+    kind: z.literal('cli'),
+    executable: Identifier.describe('A logical executable name, never a filesystem path, a URL, a host, or a port (AD-35). The adapter maps it to an authorized target through configuration outside the contract.'),
+    subcommandPath: z.array(Identifier),
+    channels: z.strictObject({
+        argument: z.record(KeyName, JsonValue),
+        option: z.record(KeyName, JsonValue),
+        environment: z
+            .record(KeyName, z.string())
+            .describe('String-valued because an environment variable is a string to the process; the other channels carry the declared JSON value.'),
+        stdin: ProbeRequestStdin,
+    }),
+});
+export const ProbeRequest = z.discriminatedUnion('kind', [
+    ApiProbeRequest,
+    CommandProbeRequest,
+]);
 /**
  * What the adapter observed. Deliberately response content only: no elapsed
  * time, no redirect count, no retry count. AD-35's caps are safety limits, so
@@ -78,13 +120,38 @@ export const ProbeRequest = z.strictObject({
  * policy denial, a cap, an abort, or a transport failure throws; a 500 is the
  * payload AD-10's seeded-fault check reads, never an error.
  */
-export const ProbeObservation = z.strictObject({
-    probeId: Identifier,
-    interfaceId: Identifier,
-    operationId: Identifier,
+export const ApiProbeObservation = z.strictObject({
+    ...probeCorrelation,
+    kind: z.literal('api'),
     status: z.int().min(100).max(599),
     headers: z
         .record(KeyName, z.string())
         .describe('Repeated headers are joined with ", " per RFC 9110 before they reach this shape. `set-cookie` is the one header that rule is wrong for, and it is dropped rather than mangled: nothing in AD-10 reads it, and a joined `set-cookie` is a value no consumer can split back.'),
     body: ProbeObservedBody,
 });
+/**
+ * What the adapter observed of a command run: its two streams, its exit code,
+ * and the files it wrote.
+ *
+ * `exitCode` is signed, unlike an HTTP status, because a process terminated by
+ * a signal is conventionally reported as a negative code and this field records
+ * what happened rather than what is tidy. A non-zero exit is an observation
+ * exactly as a 500 is: only a policy denial, a cap, an abort, or a failure to
+ * start the process throws.
+ *
+ * `artifacts` is keyed by the identifier the operation declared, not by a
+ * filesystem path. Which path each identifier names is the adapter's mapping,
+ * the same disclosure boundary AD-35 draws around the executable itself.
+ */
+export const CommandProbeObservation = z.strictObject({
+    ...probeCorrelation,
+    kind: z.literal('cli'),
+    exitCode: z.int(),
+    stdout: ProbeObservedBody,
+    stderr: ProbeObservedBody,
+    artifacts: z.record(Identifier, ProbeObservedBody),
+});
+export const ProbeObservation = z.discriminatedUnion('kind', [
+    ApiProbeObservation,
+    CommandProbeObservation,
+]);

package/dist/core/schemas/probe-body.d.ts

@@ -21,5 +21,23 @@ export declare const ProbeObservedBody: z.ZodDiscriminatedUnion<[z.ZodObject<{
 }, z.core.$strict>, z.ZodObject<{
     kind: z.ZodLiteral<"absent">;
 }, z.core.$strict>], "kind">;
+/**
+ * What one leg writes to a command's standard input. Three arms, like the
+ * observed body rather than the request body: standard input is a byte stream,
+ * so a command that reads a prompt receives text that was never JSON, and
+ * routing it through the `json` arm as a JSON string would make the leg claim
+ * a serialisation the command never sees. `absent` is the leg that writes
+ * nothing and closes the stream.
+ */
+export declare const ProbeRequestStdin: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"json">;
+    value: z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"text">;
+    value: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"absent">;
+}, z.core.$strict>], "kind">;
 export type ProbeRequestBody = z.infer<typeof ProbeRequestBody>;
 export type ProbeObservedBody = z.infer<typeof ProbeObservedBody>;
+export type ProbeRequestStdin = z.infer<typeof ProbeRequestStdin>;

package/dist/core/schemas/probe-body.js

@@ -21,3 +21,16 @@ export const ProbeObservedBody = z.discriminatedUnion('kind', [
     z.strictObject({ kind: z.literal('text'), value: z.string() }),
     z.strictObject({ kind: z.literal('absent') }),
 ]);
+/**
+ * What one leg writes to a command's standard input. Three arms, like the
+ * observed body rather than the request body: standard input is a byte stream,
+ * so a command that reads a prompt receives text that was never JSON, and
+ * routing it through the `json` arm as a JSON string would make the leg claim
+ * a serialisation the command never sees. `absent` is the leg that writes
+ * nothing and closes the stream.
+ */
+export const ProbeRequestStdin = z.discriminatedUnion('kind', [
+    z.strictObject({ kind: z.literal('json'), value: JsonValue }),
+    z.strictObject({ kind: z.literal('text'), value: z.string() }),
+    z.strictObject({ kind: z.literal('absent') }),
+]);

package/dist/core/schemas/probe-qualification.d.ts

@@ -0,0 +1,168 @@
+/** AD-9's five qualification routes, one per way ground truth is earned. */
+import { z } from 'zod';
+/**
+ * AD-9 spells a route for each of the five ways a probe earns ground truth. The
+ * order is AD-9's own: historical fix boundary, controlled mutation,
+ * gameability demonstration, canary, clean control.
+ */
+export declare const QUALIFICATION_ROUTES: readonly ['historical', 'controlled-mutation', 'gameability', 'canary', 'clean-control'];
+export type QualificationRouteValue = (typeof QUALIFICATION_ROUTES)[number];
+export declare const QualificationRoute: z.ZodEnum<{
+    canary: "canary";
+    "clean-control": "clean-control";
+    "controlled-mutation": "controlled-mutation";
+    gameability: "gameability";
+    historical: "historical";
+}>;
+/**
+ * AD-9's qualification record, as a tagged union on `route`.
+ *
+ * A union rather than one wide object with nullable fields: the five routes
+ * demand disjoint evidence, and a flat shape would make every route's evidence
+ * optional on every other route, which is the shape AD-9 was written against:
+ * "a mined fix commit being trusted when its own test already passed at the
+ * parent commit". Each branch names what its route must record and nothing
+ * else.
+ *
+ * Required on every probe, canaries included. AD-9 spells a route for all five
+ * kinds and closes with "an unqualified probe cannot enter a sealed set" with
+ * no exception, so exempting the canary would leave the route AD-9 writes for
+ * it with nothing to write it in. The exemption AD-9 does grant attaches to the
+ * defect signature, which is AD-40's field.
+ *
+ * Which route a probe may take is fixed by its class and its `expectedClean`
+ * flag, and that pairing is enforced by the corpus qualification gate rather
+ * than by a schema refinement: all combinations parse here, so an incompatible
+ * pairing stays a reason code carrying an artifact path rather than an
+ * anonymous parse failure.
+ *
+ * Inlined rather than given a `.meta({ id })`. Two `$defs` sharing an internal
+ * path are indistinguishable from a schema path alone, and every branch here
+ * carries an `ArtifactReference`, so a `$defs` entry would share def-relative
+ * suffixes with `Expression` and `ArtifactReference` and make the published
+ * keyword walk's stated no-collision premise false.
+ */
+export declare const ProbeQualification: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    route: z.ZodLiteral<"historical">;
+    failBeforeEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    passAfterEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    fixCommitDigest: z.ZodString;
+    oracleStableAcrossRevisions: z.ZodBoolean;
+}, z.core.$strict>, z.ZodObject<{
+    route: z.ZodLiteral<"controlled-mutation">;
+    mutationSource: z.ZodString;
+    mutationOperator: z.ZodString;
+    targetArtifact: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    expectedObservableFailure: z.ZodString;
+    baselinePassEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    mutatedFailEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    rollbackVerified: z.ZodBoolean;
+}, z.core.$strict>, z.ZodObject<{
+    route: z.ZodLiteral<"gameability">;
+    degenerateResponse: z.ZodString;
+    naiveOracleSatisfiedEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    disciplinedOracleRejectedEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+}, z.core.$strict>, z.ZodObject<{
+    route: z.ZodLiteral<"canary">;
+    indicts: z.ZodEnum<{
+        corpus: "corpus";
+        fixture: "fixture";
+    }>;
+    nonDetectionEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+}, z.core.$strict>, z.ZodObject<{
+    route: z.ZodLiteral<"clean-control">;
+    baselinePassEvidence: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        storage: z.ZodLiteral<"public">;
+        path: z.ZodString;
+        privateRef: z.ZodNull;
+        digest: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        storage: z.ZodLiteral<"private">;
+        path: z.ZodNull;
+        privateRef: z.ZodString;
+        digest: z.ZodString;
+    }, z.core.$strict>], "storage">;
+    revisionCommitDigest: z.ZodString;
+    noKnownDefectStatement: z.ZodString;
+}, z.core.$strict>], "route">;
+export type ProbeQualification = z.infer<typeof ProbeQualification>;

package/dist/core/schemas/probe-qualification.js

@@ -0,0 +1,106 @@
+/** AD-9's five qualification routes, one per way ground truth is earned. */
+import { z } from 'zod';
+import { ArtifactReference } from './artifact-reference.js';
+import { Digest } from './primitives.js';
+/**
+ * AD-9 spells a route for each of the five ways a probe earns ground truth. The
+ * order is AD-9's own: historical fix boundary, controlled mutation,
+ * gameability demonstration, canary, clean control.
+ */
+export const QUALIFICATION_ROUTES = [
+    'historical',
+    'controlled-mutation',
+    'gameability',
+    'canary',
+    'clean-control',
+];
+export const QualificationRoute = z.enum(QUALIFICATION_ROUTES);
+/**
+ * AD-9's qualification record, as a tagged union on `route`.
+ *
+ * A union rather than one wide object with nullable fields: the five routes
+ * demand disjoint evidence, and a flat shape would make every route's evidence
+ * optional on every other route, which is the shape AD-9 was written against:
+ * "a mined fix commit being trusted when its own test already passed at the
+ * parent commit". Each branch names what its route must record and nothing
+ * else.
+ *
+ * Required on every probe, canaries included. AD-9 spells a route for all five
+ * kinds and closes with "an unqualified probe cannot enter a sealed set" with
+ * no exception, so exempting the canary would leave the route AD-9 writes for
+ * it with nothing to write it in. The exemption AD-9 does grant attaches to the
+ * defect signature, which is AD-40's field.
+ *
+ * Which route a probe may take is fixed by its class and its `expectedClean`
+ * flag, and that pairing is enforced by the corpus qualification gate rather
+ * than by a schema refinement: all combinations parse here, so an incompatible
+ * pairing stays a reason code carrying an artifact path rather than an
+ * anonymous parse failure.
+ *
+ * Inlined rather than given a `.meta({ id })`. Two `$defs` sharing an internal
+ * path are indistinguishable from a schema path alone, and every branch here
+ * carries an `ArtifactReference`, so a `$defs` entry would share def-relative
+ * suffixes with `Expression` and `ArtifactReference` and make the published
+ * keyword walk's stated no-collision premise false.
+ */
+export const ProbeQualification = z.discriminatedUnion('route', [
+    z
+        .strictObject({
+        route: z.literal('historical'),
+        failBeforeEvidence: ArtifactReference.describe('The recorded failure at the parent revision. AD-9 exists because 2 of 18 mined fix commits had a test that already passed at the parent, so this is the half that cannot be assumed.'),
+        passAfterEvidence: ArtifactReference,
+        fixCommitDigest: Digest.describe('The causally isolated fix boundary AD-9 requires the two evidence records to straddle.'),
+        oracleStableAcrossRevisions: z
+            .boolean()
+            .describe("AD-9 requires the oracle to be stable across both revisions. `false` parses and fails the gate: a route that cannot record its own unmet precondition cannot be audited, and the schema is not where AD-9's admission decision is made."),
+    })
+        .describe('A historical probe: a defect mined from a real fix boundary, seeded by reverting it.'),
+    z
+        .strictObject({
+        route: z.literal('controlled-mutation'),
+        mutationSource: z
+            .string()
+            .min(1)
+            .describe("Where the mutation came from, in the corpus author's own terms. An opaque caller string, unrelated to `Defect.source`'s two-member enum, which records whether the defect is natural or introduced."),
+        mutationOperator: z.string().min(1),
+        targetArtifact: ArtifactReference,
+        expectedObservableFailure: z.string().min(1),
+        baselinePassEvidence: ArtifactReference,
+        mutatedFailEvidence: ArtifactReference,
+        rollbackVerified: z
+            .boolean()
+            .describe("AD-9's verified rollback or cleanup. `false` parses and fails the gate, for the same reason `oracleStableAcrossRevisions` does."),
+    })
+        .describe('A controlled mutation: a defect introduced by a named operator against a named artifact.'),
+    z
+        .strictObject({
+        route: z.literal('gameability'),
+        degenerateResponse: z
+            .string()
+            .min(1)
+            .describe('The compliant-looking degenerate response, described rather than embedded: AD-8 forbids sealed-case content from entering any artifact.'),
+        naiveOracleSatisfiedEvidence: ArtifactReference,
+        disciplinedOracleRejectedEvidence: ArtifactReference,
+    })
+        .describe('A gameability probe: the demonstration that a degenerate response satisfies a naive oracle and is rejected by a disciplined one.'),
+    z
+        .strictObject({
+        route: z.literal('canary'),
+        indicts: z
+            .enum(['corpus', 'fixture'])
+            .describe("AD-9's canary qualifies by demonstrating that non-detection indicts one of these two rather than the contract. Which one it indicts is the demonstration's own result and is recorded rather than left to the reader."),
+        nonDetectionEvidence: ArtifactReference,
+    })
+        .describe('A canary: a probe whose non-detection is evidence about the instrument, never about the contract.'),
+    z
+        .strictObject({
+        route: z.literal('clean-control'),
+        baselinePassEvidence: ArtifactReference,
+        revisionCommitDigest: Digest,
+        noKnownDefectStatement: z
+            .string()
+            .min(1)
+            .describe('AD-9 qualifies a clean control at "a revision with no known defect in the probed interface". No field can prove an absence, so the statement is recorded as an attestation under AD-32 and read as trust rather than as a computed fact.'),
+    })
+        .describe('A known-clean control: recorded baseline-pass evidence at a revision with no known defect in the probed interface.'),
+]);