eval-quality 0.2.0 → 0.3.0

package/dist/ports/environment-probe-port.d.ts

@@ -31,10 +31,11 @@ export type EnvironmentProbePort = {
 };
 /** the boundary parsers `application/` and the conformance suite validate with. */
 export declare const probeParsers: {
-    readonly request: import("zod").ZodObject<{
+    readonly request: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
         probeId: import("zod").ZodString;
         interfaceId: import("zod").ZodString;
         operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"api">;
         method: import("zod").ZodEnum<{
             DELETE: "DELETE";
             GET: "GET";
@@ -56,11 +57,33 @@ export declare const probeParsers: {
                 kind: import("zod").ZodLiteral<"absent">;
             }, import("zod/v4/core").$strict>], "kind">;
         }, import("zod/v4/core").$strict>;
-    }, import("zod/v4/core").$strict>;
-    readonly response: import("zod").ZodObject<{
+    }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
         probeId: import("zod").ZodString;
         interfaceId: import("zod").ZodString;
         operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"cli">;
+        executable: import("zod").ZodString;
+        subcommandPath: import("zod").ZodArray<import("zod").ZodString>;
+        channels: import("zod").ZodObject<{
+            argument: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
+            option: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
+            environment: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
+            stdin: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+                kind: import("zod").ZodLiteral<"json">;
+                value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+            }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                kind: import("zod").ZodLiteral<"text">;
+                value: import("zod").ZodString;
+            }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                kind: import("zod").ZodLiteral<"absent">;
+            }, import("zod/v4/core").$strict>], "kind">;
+        }, import("zod/v4/core").$strict>;
+    }, import("zod/v4/core").$strict>], "kind">;
+    readonly response: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+        probeId: import("zod").ZodString;
+        interfaceId: import("zod").ZodString;
+        operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"api">;
         status: import("zod").ZodInt;
         headers: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
         body: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
@@ -72,5 +95,38 @@ export declare const probeParsers: {
         }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
             kind: import("zod").ZodLiteral<"absent">;
         }, import("zod/v4/core").$strict>], "kind">;
-    }, import("zod/v4/core").$strict>;
+    }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+        probeId: import("zod").ZodString;
+        interfaceId: import("zod").ZodString;
+        operationId: import("zod").ZodString;
+        kind: import("zod").ZodLiteral<"cli">;
+        exitCode: import("zod").ZodInt;
+        stdout: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"json">;
+            value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"text">;
+            value: import("zod").ZodString;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"absent">;
+        }, import("zod/v4/core").$strict>], "kind">;
+        stderr: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"json">;
+            value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"text">;
+            value: import("zod").ZodString;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"absent">;
+        }, import("zod/v4/core").$strict>], "kind">;
+        artifacts: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"json">;
+            value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"text">;
+            value: import("zod").ZodString;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            kind: import("zod").ZodLiteral<"absent">;
+        }, import("zod/v4/core").$strict>], "kind">>;
+    }, import("zod/v4/core").$strict>], "kind">;
 };

package/dist/testing/probe-conformance.d.ts

@@ -6,6 +6,35 @@
  * Each of the thirteen builds its own `'resolves'` subject and reads
  * `underlyingCalls()` from a counter starting at zero, so every count below is
  * absolute.
+ *
+ * **This suite certifies an adapter for the `api` mechanism and says nothing
+ * about the `cli` one.** Every scenario is HTTP: redirects, methods, schemes,
+ * an anomalous status. `ProbeSubject` requires an unauthorized-method request,
+ * a redirecting request and a not-found request, none of which a command
+ * adapter has. So an adapter that runs commands cannot present a subject here,
+ * and an adapter that passes all nineteen has been shown nothing about whether
+ * it can run one safely: it may deny an unmapped executable or execute a shell
+ * string, and this suite cannot tell the difference.
+ *
+ * That is a real gap in the published surface and it is wider than this suite.
+ * `ProbeTargetPolicy` has one authorization shape and every field in it is
+ * HTTP: scheme, host, port, resolved addresses, methods, safe methods,
+ * redirects, request and response byte caps. **A command interface cannot be
+ * authorized at all**, so AD-35's "an adapter denies by default and permits
+ * only what that mapping names" has nothing to name for the mechanism this
+ * release adds, and a conformance arm built on top of a policy that cannot
+ * express a command authorization would certify against nothing.
+ *
+ * Closing it is a declaration first and a suite second: a command
+ * authorization naming a permitted executable, the subcommand paths and
+ * environment keys it may carry, and its own elapsed and output-byte caps;
+ * then the denials to certify — an executable no mapping names, a refusal to
+ * accept a pre-built argument vector, a non-zero exit as an observation rather
+ * than a fault, each cap enforced. That is a design addition to the published
+ * surface rather than a repair, and it is not made here. Until it exists an
+ * adopter writing a command adapter has no policy to declare and no harness to
+ * run, and this comment is here so nobody concludes from a green run that they
+ * have either.
  */
 import type { ProbeRequest } from '../core/schemas/port-messages.ts';
 import type { ProbeTargetPolicy } from '../core/schemas/probe-policy.ts';

package/dist/testing/probe-conformance.js

@@ -23,9 +23,9 @@ const PROBE_ASSERTIONS = [
         request: (subject) => subject.faultingRequest,
         expectation: {
             kind: 'resolves',
-            check: (observation) => observation.status === 500
+            check: (observation) => observation.kind === 'api' && observation.status === 500
                 ? undefined
-                : `observed status ${observation.status}, expected 500`,
+                : `observed ${observation.kind === 'api' ? `status ${observation.status}` : 'a command observation'}, expected status 500`,
         },
     },
     {
@@ -114,14 +114,42 @@ const PROBE_ASSERTIONS = [
         expectation: { kind: 'rejects', code: CAPPED },
     },
 ];
-function checkResolved(assertion, expectation, value) {
+function checkResolved(assertion, expectation, value, request) {
     const parsed = probeParsers.response.safeParse(value);
     if (!parsed.success) {
         return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
     }
+    // Schema validity is not correlation. Both messages are unions now, so an
+    // adapter can answer a command request with a schema-valid HTTP observation
+    // and satisfy every assertion below without running a command or consulting
+    // a policy. The four echoed fields are what tie one answer to one question,
+    // and `ProbeRequest.probeId`'s own description already says the port returns
+    // them unchanged.
+    const mismatch = echoMismatch(request, parsed.data);
+    if (mismatch !== undefined) {
+        return titledOutcome(assertion.id, assertion.title, false, mismatch);
+    }
     const complaint = expectation.check?.(parsed.data);
     return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
 }
+/**
+ * Which of the four echoed fields the observation failed to return unchanged,
+ * or `undefined` when it answered the question it was asked.
+ */
+function echoMismatch(request, observation) {
+    const echoed = [
+        ['kind', request.kind, observation.kind],
+        ['probeId', request.probeId, observation.probeId],
+        ['interfaceId', request.interfaceId, observation.interfaceId],
+        ['operationId', request.operationId, observation.operationId],
+    ];
+    for (const [field, asked, answered] of echoed) {
+        if (asked === answered)
+            continue;
+        return `observed ${field} "${answered}" for a request carrying "${asked}", so the answer does not correlate with the question`;
+    }
+    return undefined;
+}
 function checkRejected(assertion, expectedCode, error) {
     const view = faultView(error);
     if (view === undefined) {
@@ -153,12 +181,13 @@ async function runProbeAssertion(assertion, subject) {
     if (!run.ok) {
         return titledOutcome(assertion.id, assertion.title, false, run.detail);
     }
-    const settled = await settle(run.built.port(assertion.request(subject), new AbortController().signal));
+    const request = assertion.request(subject);
+    const settled = await settle(run.built.port(request, new AbortController().signal));
     let result;
     if (assertion.expectation.kind === 'resolves') {
         result =
             settled.kind === 'resolved'
-                ? checkResolved(assertion, assertion.expectation, settled.value)
+                ? checkResolved(assertion, assertion.expectation, settled.value, request)
                 : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
     }
     else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eval-quality",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
   "author": "Murat Ozcan",
   "license": "Apache-2.0",