eval-quality 0.3.0 → 0.5.0

package/README.md

@@ -267,7 +267,7 @@ import spec from 'eval-quality/schemas/eval-contract.schema.json' with { type: '
 The import attribute is required: ESM on Node 22 and 24 both throw `ERR_IMPORT_ATTRIBUTE_MISSING`
 without it. The development corpus ships the same way, at `eval-quality/corpus/dev/`, so an adopter
 can read real compiled contracts and one compiled-and-sealed pair without cloning this repository.
-`eval-quality/adapters` is one of the five published subpaths, holding the three reference adapters
+`eval-quality/adapters` is one of the five published subpaths, holding the four reference adapters
 the conformance suite runs against.
 
 Eleven of the twelve published schemas carry a `schemaVersion`. `artifact-reference` is exempt: it
@@ -378,10 +378,10 @@ names as that artifact's producer, which today are `src/core/seal/seal.ts`,
 The `eval-quality/conformance` subpath publishes the port boundary: the four port types, the message
 shapes they carry, the AD-28 `RUNTIME_FAULT_CODES` registry and `RuntimeFaultCode` type a conforming
 adapter throws against, and an executable conformance suite. An adapter is conforming when
-`runCorpusPortConformance`, `runClockPortConformance`, `runFileSystemPortConformance`, or
-`runEnvironmentProbePortConformance` returns a report whose `passed` is true, which is the definition;
-each returns a report, so the suite carries no test framework and runs under whichever one you
-already use.
+`runCorpusPortConformance`, `runClockPortConformance`, `runFileSystemPortConformance`,
+`runEnvironmentProbePortConformance`, or `runCommandLineProbeConformance` returns a report whose
+`passed` is true, which is the definition; each returns a report, so the suite carries no test
+framework and runs under whichever one you already use.
 
 ```ts
 import { runCorpusPortConformance, type CorpusPort } from 'eval-quality/conformance'
@@ -390,10 +390,15 @@ import { runCorpusPortConformance, type CorpusPort } from 'eval-quality/conforma
 The suite drives a subject through four scenarios and checks six assertions per port method: a
 mechanism failure is a typed fault, exactly one underlying call happens on success and on failure, an
 aborted signal rejects promptly, an in-band error value is thrown as a fault, and a
-successful call returns a response the published schema accepts. The environment-probe port adds
-thirteen more from AD-35's default-deny target policy. `npm run test:conformance` runs the suite
-against the three adapters this package ships and against an in-repository probe subject that exists
-only as the suite's own subject.
+successful call returns a response the published schema accepts. `EnvironmentProbePort` adds a second
+arm per mechanism: `runEnvironmentProbePortConformance` (`api`) adds thirteen more from AD-35's
+default-deny target policy, and `runCommandLineProbeConformance` (`cli`) adds nine, once
+`CommandTargetPolicy` gave that mechanism something to authorize — an unmapped interface, an
+unmapped executable, an unauthorized subcommand path, a non-zero exit read as an observation, a
+shell-metacharacter argument proven to reach the process as one literal token, a declared artifact
+captured, and both caps enforced. `npm run test:conformance` runs the suite against the four
+adapters this package ships and against two in-repository probe subjects, one per mechanism, that
+exist only as the suite's own subjects.
 
 `docs/ad21-verdict-decision.generated.md` holds AD-21's two published verdict ladders, production and
 contract-scoring, emitted from the rule tables in `src/core/score/ladder.ts` together with the

package/dist/adapters/command-line-adapter.d.ts

@@ -0,0 +1,44 @@
+import type { CommandProbeRequest } from '../core/schemas/port-messages.ts';
+import type { CommandTargetPolicy } from '../core/schemas/probe-policy.ts';
+import type { EnvironmentProbePort } from '../ports/environment-probe-port.ts';
+/** One process run, already reduced to what an observation needs. No truncation flag: exceeding `maxOutputBytes` rejects with `budget-exhausted` rather than resolving with a partial stream, so a resolved run's output is always the whole thing. */
+export type CommandRunResult = {
+    readonly exitCode: number;
+    readonly stdout: string;
+    readonly stderr: string;
+};
+export type CommandRunRequest = {
+    readonly target: string;
+    readonly subcommandPath: readonly string[];
+    readonly argv: readonly string[];
+    readonly env: Readonly<Record<string, string>>;
+    readonly stdin: {
+        readonly kind: 'json' | 'text' | 'absent';
+        readonly value?: unknown;
+    };
+    readonly cwd: string;
+    readonly maxElapsedMs: number;
+    readonly maxOutputBytes: number;
+};
+export type ArtifactRead = {
+    readonly present: boolean;
+    readonly text: string;
+    readonly truncated: boolean;
+};
+/** Swappable so the conformance subject can spy on calls and script every scenario, matching the other three shipped adapters. */
+export type CommandMechanism = {
+    readonly run: (request: CommandRunRequest, signal: AbortSignal) => Promise<CommandRunResult>;
+    readonly readArtifact: (path: string, maxBytes: number) => Promise<ArtifactRead>;
+};
+/** Options first as `--{key}`, positionals after, both in the record's own key order. Exported for its own unit tests: this is the one place shell-injection safety is decided. */
+export declare function buildArgv(channels: CommandProbeRequest['channels']): string[];
+/**
+ * The real mechanism: an actual child process, an actual file read. Exported,
+ * unlike the other three shipped adapters' defaults, because AD-37's own
+ * conformance subject for this port has to exercise real process behaviour
+ * (a real timeout, a real output cap, a real argv) the way the `api` arm's
+ * subject exercises a real loopback server; a synthetic mechanism would prove
+ * nothing about the one thing this adapter exists to get right.
+ */
+export declare const nodeCommandMechanism: CommandMechanism;
+export declare function createCommandLineAdapter(policy: CommandTargetPolicy, mechanism?: CommandMechanism): EnvironmentProbePort;

package/dist/adapters/command-line-adapter.js

@@ -0,0 +1,284 @@
+/**
+ * The environment-probe port over a real child process, for the `cli`
+ * mechanism, which `probe-conformance.ts` had documented as unauthorizable
+ * and unshipped. `evaluateCommandTarget` closes the authorization half; this
+ * file closes the execution half.
+ *
+ * Every rule below is what "what's a command allowed to do" resolves to, a
+ * decision settled here in the implementation rather than as an architecture
+ * revision:
+ *
+ * 1. `target` is spawned directly with an argv array (`shell: false`,
+ *    Node's own default, stated anyway since it is the one thing this
+ *    adapter must never turn off). No channel value is ever concatenated
+ *    into a shell string; a value containing `;`, `$(...)`, or a quote
+ *    reaches the child as one literal argv element, not a metacharacter.
+ * 2. `argument` and `option` build the argv the same way a well-behaved CLI
+ *    parser reads one: options first as `--{key}` (a boolean `true` is a
+ *    bare flag, `false` is omitted, anything else gets one value token),
+ *    positionals after in the record's own key order. `environment` passes
+ *    through as declared, plus the host's own `PATH` so a `target` naming a
+ *    bare command still resolves; a declared `PATH` key wins over that
+ *    default. `stdin` is written and the stream is closed; `absent` closes
+ *    it with nothing written.
+ * 3. `maxElapsedMs` and `maxOutputBytes` are enforced by this adapter, not
+ *    borrowed from `AbortSignal`: exceeding either kills the process with
+ *    `SIGKILL` and throws `budget-exhausted`, exactly as an HTTP cap does.
+ *    `maxOutputBytes` applies independently to stdout, to stderr, and to
+ *    each artifact file read back after exit.
+ * 4. A non-zero exit is an observation, never a fault, matching AD-10's rule
+ *    for an HTTP status. Only a policy denial, a cap, an abort, or a failure
+ *    to start the process throws. `exitCode` is negative when a signal
+ *    ended the process, the same convention `CommandProbeObservation`
+ *    documents for itself.
+ */
+import { spawn } from 'node:child_process';
+import { open } from 'node:fs/promises';
+import { constants as osConstants } from 'node:os';
+import { isAbsolute, resolve as resolvePath } from 'node:path';
+import { RuntimeFault } from '../core/schemas/faults.js';
+import { probeParsers } from '../ports/environment-probe-port.js';
+import { evaluateCommandTarget } from './command-target-policy.js';
+import { runPortMethod } from './port-boundary.js';
+function stringifyScalar(value) {
+    if (typeof value === 'string')
+        return value;
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return String(value);
+    return JSON.stringify(value);
+}
+/** Options first as `--{key}`, positionals after, both in the record's own key order. Exported for its own unit tests: this is the one place shell-injection safety is decided. */
+export function buildArgv(channels) {
+    const optionTokens = [];
+    for (const [key, value] of Object.entries(channels.option)) {
+        if (value === false)
+            continue;
+        optionTokens.push(`--${key}`);
+        if (value !== true)
+            optionTokens.push(stringifyScalar(value));
+    }
+    const argumentTokens = Object.values(channels.argument).map(stringifyScalar);
+    return [...optionTokens, ...argumentTokens];
+}
+function buildEnv(declared) {
+    const base = {};
+    if (process.env.PATH !== undefined)
+        base.PATH = process.env.PATH;
+    return { ...base, ...declared };
+}
+function buildStdin(stdin) {
+    if (stdin.kind === 'absent')
+        return { kind: 'absent' };
+    if (stdin.kind === 'json')
+        return { kind: 'json', value: stdin.value };
+    return { kind: 'text', value: stdin.value };
+}
+/** A heuristic, not a declared content type: the port carries none for a process stream. JSON-shaped text reads as `json`; anything else as `text`. */
+function bodyFromText(text) {
+    if (text.length === 0)
+        return { kind: 'text', value: text };
+    try {
+        return { kind: 'json', value: JSON.parse(text) };
+    }
+    catch {
+        return { kind: 'text', value: text };
+    }
+}
+function capped(detail) {
+    return new RuntimeFault('budget-exhausted', 'CommandProbeRequest', detail);
+}
+function forbidden(detail) {
+    return new RuntimeFault('forbidden-target', 'ProbeRequest', detail);
+}
+function writeStdin(child, stdin) {
+    const stream = child.stdin;
+    if (stream === null)
+        return;
+    // A child that exits before reading stdin closes its end of the pipe, and
+    // the pending `end()` write below then raises EPIPE on this stream, not on
+    // `child` itself. With no listener that is an unhandled 'error' and crashes
+    // the host process; the run itself is unaffected, since `close` still fires
+    // and carries the exit code this adapter already reads it from.
+    stream.on('error', () => { });
+    if (stdin.kind === 'absent') {
+        stream.end();
+        return;
+    }
+    stream.end(stdin.kind === 'json' ? JSON.stringify(stdin.value) : String(stdin.value));
+}
+/** `-signalNumber` when a signal ended the process, matching `CommandProbeObservation.exitCode`'s own documented convention. */
+function exitCodeOf(code, signalName) {
+    if (code !== null)
+        return code;
+    if (signalName === null)
+        return -1;
+    const numeric = osConstants.signals[signalName];
+    return numeric === undefined ? -1 : -numeric;
+}
+async function runChildProcess(request, signal) {
+    return new Promise((settlePromise, rejectPromise) => {
+        const child = spawn(request.target, [...request.subcommandPath, ...request.argv], {
+            cwd: request.cwd,
+            env: request.env,
+            shell: false,
+            signal,
+        });
+        let stdout = '';
+        let stderr = '';
+        let settled = false;
+        let timedOut = false;
+        let overCap = false;
+        const finish = (action) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            action();
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            child.kill('SIGKILL');
+        }, request.maxElapsedMs);
+        const capture = (chunk, append, currentLength) => {
+            if (currentLength() > request.maxOutputBytes)
+                return;
+            append(chunk.toString('utf8'));
+            if (currentLength() > request.maxOutputBytes) {
+                overCap = true;
+                child.kill('SIGKILL');
+            }
+        };
+        child.stdout?.on('data', (chunk) => capture(chunk, (next) => {
+            stdout += next;
+        }, () => Buffer.byteLength(stdout, 'utf8')));
+        child.stderr?.on('data', (chunk) => capture(chunk, (next) => {
+            stderr += next;
+        }, () => Buffer.byteLength(stderr, 'utf8')));
+        child.once('error', (error) => {
+            finish(() => rejectPromise(error));
+        });
+        child.once('close', (code, signalName) => {
+            finish(() => {
+                if (timedOut) {
+                    rejectPromise(capped(`exceeded maxElapsedMs (${request.maxElapsedMs}ms) and was killed`));
+                    return;
+                }
+                if (overCap) {
+                    rejectPromise(capped(`stdout or stderr exceeded maxOutputBytes (${request.maxOutputBytes}) and the process was killed`));
+                    return;
+                }
+                settlePromise({
+                    exitCode: exitCodeOf(code, signalName),
+                    stdout,
+                    stderr,
+                });
+            });
+        });
+        writeStdin(child, request.stdin);
+    });
+}
+/** Reads at most `maxBytes + 1` bytes rather than the whole file: an oversize artifact is capped before its bytes are loaded, not after, so a run that wrote past the limit cannot make this adapter allocate the excess first. */
+async function readArtifactFile(path, maxBytes) {
+    let handle;
+    try {
+        handle = await open(path, 'r');
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return { present: false, text: '', truncated: false };
+        }
+        throw error;
+    }
+    try {
+        const buffer = Buffer.alloc(maxBytes + 1);
+        const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, 0);
+        const truncated = bytesRead > maxBytes;
+        const text = buffer
+            .subarray(0, Math.min(bytesRead, maxBytes))
+            .toString('utf8');
+        return { present: true, text, truncated };
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * The real mechanism: an actual child process, an actual file read. Exported,
+ * unlike the other three shipped adapters' defaults, because AD-37's own
+ * conformance subject for this port has to exercise real process behaviour
+ * (a real timeout, a real output cap, a real argv) the way the `api` arm's
+ * subject exercises a real loopback server; a synthetic mechanism would prove
+ * nothing about the one thing this adapter exists to get right.
+ */
+export const nodeCommandMechanism = {
+    run: runChildProcess,
+    readArtifact: readArtifactFile,
+};
+export function createCommandLineAdapter(policy, mechanism = nodeCommandMechanism) {
+    return {
+        probe: (request, signal) => runPortMethod({
+            request,
+            requestParser: probeParsers.request,
+            responseParser: probeParsers.response,
+            requestPath: 'ProbeRequest',
+            responsePath: 'ProbeObservation',
+            signal,
+            mechanism: async (parsed, innerSignal) => {
+                if (parsed.kind !== 'cli') {
+                    // This adapter authorizes no `api` target, so an `api` request
+                    // meets the same "the mapping names nothing" denial an
+                    // unmapped interfaceId would, before any process spawns.
+                    throw forbidden('this adapter runs cli requests only; no api target is ever authorized');
+                }
+                const decision = evaluateCommandTarget(policy, {
+                    interfaceId: parsed.interfaceId,
+                    executable: parsed.executable,
+                    subcommandPath: parsed.subcommandPath,
+                });
+                if (!decision.allowed)
+                    throw forbidden(decision.detail);
+                const { authorization } = decision;
+                const runResult = await mechanism.run({
+                    target: authorization.target,
+                    subcommandPath: parsed.subcommandPath,
+                    argv: buildArgv(parsed.channels),
+                    env: buildEnv(parsed.channels.environment),
+                    stdin: buildStdin(parsed.channels.stdin),
+                    cwd: authorization.cwd,
+                    maxElapsedMs: authorization.maxElapsedMs,
+                    maxOutputBytes: authorization.maxOutputBytes,
+                }, innerSignal);
+                const artifactEntries = await Promise.all(Object.entries(authorization.artifacts).map(async ([artifactId, relativePath]) => {
+                    const absolute = isAbsolute(relativePath)
+                        ? relativePath
+                        : resolvePath(authorization.cwd, relativePath);
+                    const read = await mechanism.readArtifact(absolute, authorization.maxOutputBytes);
+                    if (read.truncated) {
+                        throw capped(`artifact "${artifactId}" exceeded maxOutputBytes (${authorization.maxOutputBytes})`);
+                    }
+                    return [artifactId, read];
+                }));
+                return { parsed, runResult, artifactEntries };
+            },
+            assemble: (raw) => {
+                const { parsed, runResult, artifactEntries } = raw;
+                const observation = {
+                    kind: 'cli',
+                    probeId: parsed.probeId,
+                    interfaceId: parsed.interfaceId,
+                    operationId: parsed.operationId,
+                    exitCode: runResult.exitCode,
+                    stdout: bodyFromText(runResult.stdout),
+                    stderr: bodyFromText(runResult.stderr),
+                    artifacts: Object.fromEntries(artifactEntries.map(([id, read]) => [
+                        id,
+                        read.present
+                            ? bodyFromText(read.text)
+                            : { kind: 'absent' },
+                    ])),
+                };
+                return observation;
+            },
+        }),
+    };
+}

package/dist/adapters/command-target-policy.d.ts

@@ -0,0 +1,36 @@
+/**
+ * AD-35's default-deny decision for the `cli` mechanism, as a pure function.
+ *
+ * Lives under `adapters/` rather than beside `core/probe/target-policy.ts`
+ * (the `api` evaluator), even though the two are otherwise siblings: the
+ * repository's mechanically-enforced dependency direction permits
+ * `adapters/` to import `core/schemas` but not `core/`, so a
+ * target-authorization decision an adapter calls has to be adapter-owned
+ * infrastructure, not a `core/` module. `target-policy.ts` itself is never
+ * imported from `src/adapters/` for the same reason — no shipped `api`
+ * adapter exists to call it, which is the gap this file closes for `cli`.
+ */
+import type { CommandTargetAuthorization, CommandTargetPolicy } from '../core/schemas/probe-policy.ts';
+/** Why a command target was denied. Thrown as the single AD-28 `forbidden-target` fault, same as the HTTP reasons. */
+export declare const COMMAND_DENIAL_REASONS: readonly ['interface-not-authorized', 'executable-not-authorized', 'subcommand-not-authorized'];
+export type CommandDenialReason = (typeof COMMAND_DENIAL_REASONS)[number];
+export type CommandResolvedTarget = {
+    readonly interfaceId: string;
+    readonly executable: string;
+    readonly subcommandPath: readonly string[];
+};
+export type CommandPolicyDecision = {
+    readonly allowed: true;
+    readonly authorization: CommandTargetAuthorization;
+} | {
+    readonly allowed: false;
+    readonly reason: CommandDenialReason;
+    readonly detail: string;
+};
+/**
+ * The interface-and-executable check runs first, so an unmapped pair never
+ * reaches the subcommand comparison. Where several authorizations name one
+ * pair, each is tried in declaration order and the first that allows wins;
+ * mirrors `evaluateTarget`'s own ordering rule for the same reason.
+ */
+export declare function evaluateCommandTarget(policy: CommandTargetPolicy, target: CommandResolvedTarget): CommandPolicyDecision;

package/dist/adapters/command-target-policy.js

@@ -0,0 +1,33 @@
+/** Why a command target was denied. Thrown as the single AD-28 `forbidden-target` fault, same as the HTTP reasons. */
+export const COMMAND_DENIAL_REASONS = [
+    'interface-not-authorized',
+    'executable-not-authorized',
+    'subcommand-not-authorized',
+];
+function subcommandPathsMatch(a, b) {
+    return (a.length === b.length && a.every((segment, index) => segment === b[index]));
+}
+function deny(reason, detail) {
+    return { allowed: false, reason, detail };
+}
+/**
+ * The interface-and-executable check runs first, so an unmapped pair never
+ * reaches the subcommand comparison. Where several authorizations name one
+ * pair, each is tried in declaration order and the first that allows wins;
+ * mirrors `evaluateTarget`'s own ordering rule for the same reason.
+ */
+export function evaluateCommandTarget(policy, target) {
+    const matchingPair = policy.authorizations.filter((authorization) => authorization.interfaceId === target.interfaceId &&
+        authorization.executable === target.executable);
+    if (matchingPair.length === 0) {
+        return deny(policy.authorizations.some((authorization) => authorization.interfaceId === target.interfaceId)
+            ? 'executable-not-authorized'
+            : 'interface-not-authorized', `no authorization names interface "${target.interfaceId}" and executable "${target.executable}" together`);
+    }
+    for (const authorization of matchingPair) {
+        if (authorization.permittedSubcommandPaths.some((permitted) => subcommandPathsMatch(permitted, target.subcommandPath))) {
+            return { allowed: true, authorization };
+        }
+    }
+    return deny('subcommand-not-authorized', `subcommand path [${target.subcommandPath.join(', ')}] is not among the authorized paths for interface "${target.interfaceId}" and executable "${target.executable}"`);
+}

package/dist/adapters/index.d.ts

@@ -4,6 +4,8 @@
  * subpath and the root barrel keeps the `root -> adapters` edge the matrix
  * does not grant.
  */
+export type { CommandMechanism, CommandRunRequest, CommandRunResult, } from './command-line-adapter.ts';
+export { createCommandLineAdapter, nodeCommandMechanism, } from './command-line-adapter.ts';
 export type { CorpusMechanism } from './local-corpus-adapter.ts';
 export { createLocalCorpusAdapter } from './local-corpus-adapter.ts';
 export type { FileSystemMechanism } from './node-file-system-adapter.ts';

package/dist/adapters/index.js

@@ -1,3 +1,4 @@
+export { createCommandLineAdapter, nodeCommandMechanism, } from './command-line-adapter.js';
 export { createLocalCorpusAdapter } from './local-corpus-adapter.js';
 export { createNodeFileSystemAdapter } from './node-file-system-adapter.js';
 export { createSystemClockAdapter } from './system-clock-adapter.js';

package/dist/core/schemas/probe-policy.d.ts

@@ -74,3 +74,39 @@ export declare const ProbeTargetPolicy: z.ZodObject<{
 }, z.core.$strict>;
 export type ProbeTargetAuthorization = z.infer<typeof ProbeTargetAuthorization>;
 export type ProbeTargetPolicy = z.infer<typeof ProbeTargetPolicy>;
+/**
+ * One authorized command target, AD-35's mapping for the `cli` mechanism.
+ * `probe-conformance.ts`'s own published note had recorded that this shape
+ * did not exist: `ProbeTargetPolicy` above is entirely HTTP-shaped, and
+ * "a command interface cannot be authorized at all" without one.
+ *
+ * Keyed by `(interfaceId, executable)` rather than `interfaceId` alone,
+ * because `CommandInvocation` is declared per operation (AD-19), not per
+ * interface: two operations under one CLI interface may name two different
+ * logical executables. Everything else about the pair is shared the way host
+ * and port are shared across an HTTP authorization's methods.
+ */
+export declare const CommandTargetAuthorization: z.ZodObject<{
+    interfaceId: z.ZodString;
+    executable: z.ZodString;
+    target: z.ZodString;
+    permittedSubcommandPaths: z.ZodArray<z.ZodArray<z.ZodString>>;
+    cwd: z.ZodString;
+    artifacts: z.ZodRecord<z.ZodString, z.ZodString>;
+    maxElapsedMs: z.ZodInt;
+    maxOutputBytes: z.ZodInt;
+}, z.core.$strict>;
+export declare const CommandTargetPolicy: z.ZodObject<{
+    authorizations: z.ZodArray<z.ZodObject<{
+        interfaceId: z.ZodString;
+        executable: z.ZodString;
+        target: z.ZodString;
+        permittedSubcommandPaths: z.ZodArray<z.ZodArray<z.ZodString>>;
+        cwd: z.ZodString;
+        artifacts: z.ZodRecord<z.ZodString, z.ZodString>;
+        maxElapsedMs: z.ZodInt;
+        maxOutputBytes: z.ZodInt;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type CommandTargetAuthorization = z.infer<typeof CommandTargetAuthorization>;
+export type CommandTargetPolicy = z.infer<typeof CommandTargetPolicy>;

package/dist/core/schemas/probe-policy.js

@@ -31,3 +31,47 @@ export const ProbeTargetPolicy = z.strictObject({
         .array(ProbeTargetAuthorization)
         .describe('An empty array is legal and authorizes nothing, which is the default-deny base case and must stay representable.'),
 });
+/**
+ * One authorized command target, AD-35's mapping for the `cli` mechanism.
+ * `probe-conformance.ts`'s own published note had recorded that this shape
+ * did not exist: `ProbeTargetPolicy` above is entirely HTTP-shaped, and
+ * "a command interface cannot be authorized at all" without one.
+ *
+ * Keyed by `(interfaceId, executable)` rather than `interfaceId` alone,
+ * because `CommandInvocation` is declared per operation (AD-19), not per
+ * interface: two operations under one CLI interface may name two different
+ * logical executables. Everything else about the pair is shared the way host
+ * and port are shared across an HTTP authorization's methods.
+ */
+export const CommandTargetAuthorization = z.strictObject({
+    interfaceId: Identifier.describe('The logical interface identifier the contract names.'),
+    executable: Identifier.describe('The logical executable name a CommandInvocation declares. Paired with interfaceId since one interface may declare more than one.'),
+    target: z
+        .string()
+        .min(1)
+        .describe('The real, spawnable command: an absolute path, or a name the adapter resolves through its own PATH. Never taken from the contract, which never carries one (AD-35).'),
+    permittedSubcommandPaths: z
+        .array(z.array(Identifier))
+        .min(1)
+        .describe('The exact subcommand paths this authorization allows, compared literally the way AD-40 compares them. An empty inner array authorizes invoking target with no subcommand. A path the request declares that matches none of these is denied before target is ever spawned, the same role methods plays on the HTTP side.'),
+    cwd: z
+        .string()
+        .min(1)
+        .describe('The working directory every invocation runs from. Declared rather than inherited from the adapter process, so a relative artifact path below resolves against a target the mapping names rather than wherever the host process happened to start.'),
+    artifacts: z
+        .record(Identifier, z.string().min(1))
+        .describe("Where each declared artifact identifier reads from on disk, relative to cwd unless absolute. `CommandProbeObservation.artifacts` is keyed by these identifiers; one this run did not write resolves `absent` rather than missing the key, since the record itself is mandatory. An identifier absent from this map can never appear in an observation, which is the same disclosure boundary AD-35 draws around target itself: an operation may declare an artifact the mapping has no path for, and that operation's pre-flight runs with that artifact permanently absent until the mapping is extended."),
+    maxElapsedMs: z
+        .int()
+        .min(1)
+        .describe('Wall-clock budget for one invocation. Exceeding it kills the process and throws budget-exhausted, the same fault an HTTP cap throws.'),
+    maxOutputBytes: z
+        .int()
+        .min(1)
+        .describe('Applies independently to stdout, to stderr, and to each artifact file read back. The first channel to cross it kills the process (for stdout/stderr, mid-run) or fails the read (for an artifact, after exit) with budget-exhausted.'),
+});
+export const CommandTargetPolicy = z.strictObject({
+    authorizations: z
+        .array(CommandTargetAuthorization)
+        .describe('An empty array is legal and authorizes nothing, the same default-deny base case as ProbeTargetPolicy.'),
+});

package/dist/index.d.ts

@@ -12,4 +12,4 @@ export type { ScoringPolicy } from './core/schemas/scoring-policy.ts';
 export type { SealedEvaluatorBrief } from './core/schemas/sealed-evaluator-brief.ts';
 export type { SealedRunRecord } from './core/schemas/sealed-run-record.ts';
 export type { FixtureReset, ManifestationWitness, SensitivityWitness, SensitivityWitnessLeg, WitnessChannel, WitnessInputs, } from './core/schemas/sensitivity-witness.ts';
-export declare const VERSION = "0.3.0";
+export declare const VERSION = "0.5.0";

package/dist/index.js

@@ -19,4 +19,4 @@
 // subpath, where AD-37 puts the conformance definition an adapter author
 // reads; the reference adapters stay at `eval-quality/adapters`.
 export * from './application/index.js';
-export const VERSION = '0.3.0';
+export const VERSION = '0.5.0';

package/dist/testing/conformance.d.ts

@@ -7,13 +7,14 @@ export type ConformanceOutcome = {
     readonly passed: boolean;
     readonly detail: string;
 };
-export type ConformancePort = 'corpus' | 'clock' | 'file-system' | 'environment-probe';
+export type ConformancePort = 'corpus' | 'clock' | 'file-system' | 'environment-probe' | 'command-probe';
 /** how many outcomes a complete run of each port produces. Asserted as literals by fixture 58. */
 export declare const CONFORMANCE_OUTCOME_COUNTS: {
     readonly corpus: 6;
     readonly clock: 6;
     readonly 'file-system': 12;
     readonly 'environment-probe': 19;
+    readonly 'command-probe': 15;
 };
 export type ConformanceReport = {
     readonly subject: string;

package/dist/testing/conformance.js

@@ -19,6 +19,7 @@ export const CONFORMANCE_OUTCOME_COUNTS = {
     clock: 6,
     'file-system': 12,
     'environment-probe': 19,
+    'command-probe': 15,
 };
 /** "Promptly" with no bound is unfalsifiable. One second clears any real adapter's abort latency and sits well inside a default test timeout. */
 export const DEFAULT_ABORT_BUDGET_MS = 1000;

package/dist/testing/index.d.ts

@@ -13,7 +13,7 @@
 export type { RuntimeFaultCode } from '../core/schemas/faults.ts';
 export { RUNTIME_FAULT_CODES, RuntimeFault } from '../core/schemas/faults.ts';
 export type { ClockReadRequest, ClockReadResponse, CorpusResolveRequest, CorpusResolveResponse, FileReadRequest, FileReadResponse, FileWriteRequest, FileWriteResponse, ProbeObservation, ProbeObservedBody, ProbeRequest, ProbeRequestBody, } from '../core/schemas/port-messages.ts';
-export type { ProbeTargetAuthorization, ProbeTargetPolicy, } from '../core/schemas/probe-policy.ts';
+export type { CommandTargetAuthorization, CommandTargetPolicy, ProbeTargetAuthorization, ProbeTargetPolicy, } from '../core/schemas/probe-policy.ts';
 export type { ClockPort } from '../ports/clock-port.ts';
 export { clockReadParsers } from '../ports/clock-port.ts';
 export type { CorpusPort } from '../ports/corpus-port.ts';
@@ -24,5 +24,5 @@ export type { FileSystemPort } from '../ports/file-system-port.ts';
 export { fileReadParsers, fileWriteParsers, } from '../ports/file-system-port.ts';
 export type { BuiltSubject, ConformanceOutcome, ConformancePort, ConformanceReport, PortSubject, ScenarioKind, } from './conformance.ts';
 export { CONFORMANCE_OUTCOME_COUNTS, formatConformanceReport, runClockPortConformance, runCorpusPortConformance, runFileSystemPortConformance, } from './conformance.ts';
-export type { ProbeSubject } from './probe-conformance.ts';
-export { runEnvironmentProbePortConformance } from './probe-conformance.ts';
+export type { CommandProbeSubject, ProbeSubject } from './probe-conformance.ts';
+export { runCommandLineProbeConformance, runEnvironmentProbePortConformance, } from './probe-conformance.ts';

package/dist/testing/index.js

@@ -16,4 +16,4 @@ export { corpusResolveParsers } from '../ports/corpus-port.js';
 export { probeParsers } from '../ports/environment-probe-port.js';
 export { fileReadParsers, fileWriteParsers, } from '../ports/file-system-port.js';
 export { CONFORMANCE_OUTCOME_COUNTS, formatConformanceReport, runClockPortConformance, runCorpusPortConformance, runFileSystemPortConformance, } from './conformance.js';
-export { runEnvironmentProbePortConformance } from './probe-conformance.js';
+export { runCommandLineProbeConformance, runEnvironmentProbePortConformance, } from './probe-conformance.js';

package/dist/testing/probe-conformance.d.ts

@@ -1,43 +1,27 @@
 /**
- * AD-35's thirteen extra assertions, for the environment-probe port only. The
- * subject supplies its policy and one request per denial, since only it knows
- * how its own interface-to-target mapping is wired.
+ * AD-35's extra assertions for the environment-probe port, one arm per
+ * mechanism. The subject supplies its policy and one request per denial,
+ * since only it knows how its own interface-to-target mapping is wired.
  *
- * Each of the thirteen builds its own `'resolves'` subject and reads
+ * `runEnvironmentProbePortConformance` is the `api` arm: thirteen assertions,
+ * every scenario HTTP (redirects, methods, schemes, an anomalous status).
+ * `runCommandLineProbeConformance` is the `cli` arm, added once
+ * `CommandTargetPolicy` gave the mechanism something to authorize: an
+ * unmapped interface, an unmapped executable, an unauthorized subcommand path,
+ * a non-zero exit read as an observation, a shell-metacharacter argument
+ * proven to reach the process as one literal token rather than a shell
+ * expansion, a declared artifact captured, and both caps enforced. The two
+ * arms are separate functions rather than one, because their subjects need
+ * disjoint fixtures (an HTTP redirect chain has no command analogue, and a
+ * subcommand allowlist has no HTTP one) and a subject presenting for one
+ * mechanism is not asked to fake the other's scenarios.
+ *
+ * Each assertion in either arm builds its own `'resolves'` subject and reads
  * `underlyingCalls()` from a counter starting at zero, so every count below is
  * absolute.
- *
- * **This suite certifies an adapter for the `api` mechanism and says nothing
- * about the `cli` one.** Every scenario is HTTP: redirects, methods, schemes,
- * an anomalous status. `ProbeSubject` requires an unauthorized-method request,
- * a redirecting request and a not-found request, none of which a command
- * adapter has. So an adapter that runs commands cannot present a subject here,
- * and an adapter that passes all nineteen has been shown nothing about whether
- * it can run one safely: it may deny an unmapped executable or execute a shell
- * string, and this suite cannot tell the difference.
- *
- * That is a real gap in the published surface and it is wider than this suite.
- * `ProbeTargetPolicy` has one authorization shape and every field in it is
- * HTTP: scheme, host, port, resolved addresses, methods, safe methods,
- * redirects, request and response byte caps. **A command interface cannot be
- * authorized at all**, so AD-35's "an adapter denies by default and permits
- * only what that mapping names" has nothing to name for the mechanism this
- * release adds, and a conformance arm built on top of a policy that cannot
- * express a command authorization would certify against nothing.
- *
- * Closing it is a declaration first and a suite second: a command
- * authorization naming a permitted executable, the subcommand paths and
- * environment keys it may carry, and its own elapsed and output-byte caps;
- * then the denials to certify — an executable no mapping names, a refusal to
- * accept a pre-built argument vector, a non-zero exit as an observation rather
- * than a fault, each cap enforced. That is a design addition to the published
- * surface rather than a repair, and it is not made here. Until it exists an
- * adopter writing a command adapter has no policy to declare and no harness to
- * run, and this comment is here so nobody concludes from a green run that they
- * have either.
  */
 import type { ProbeRequest } from '../core/schemas/port-messages.ts';
-import type { ProbeTargetPolicy } from '../core/schemas/probe-policy.ts';
+import type { CommandTargetPolicy, ProbeTargetPolicy } from '../core/schemas/probe-policy.ts';
 import type { ConformanceReport, PortSubject } from './conformance.ts';
 export type ProbeSubject = PortSubject<ProbeRequest> & {
     readonly policy: ProbeTargetPolicy;
@@ -72,3 +56,41 @@ export type ProbeSubject = PortSubject<ProbeRequest> & {
  * stays declared and adapter-enforced.
  */
 export declare function runEnvironmentProbePortConformance(subject: ProbeSubject): Promise<ConformanceReport>;
+/**
+ * The `cli` arm's subject. Every field is a request `evaluateCommandTarget`
+ * decides one specific way against the subject's own `policy`, the same
+ * division `ProbeSubject` draws for the `api` arm: the subject knows how its
+ * mapping is wired, the suite only knows what each request should produce.
+ */
+export type CommandProbeSubject = PortSubject<ProbeRequest> & {
+    readonly policy: CommandTargetPolicy;
+    /** a request the policy ALLOWS, run against the subject's own fixture executable. */
+    readonly authorizedRequest: ProbeRequest;
+    /** an interfaceId no authorization names. */
+    readonly unmappedInterfaceRequest: ProbeRequest;
+    /** an interfaceId the policy names, paired with an executable it never names. */
+    readonly unmappedExecutableRequest: ProbeRequest;
+    /** an interface-executable pair the policy names, with a subcommandPath no authorization for it permits. */
+    readonly unauthorizedSubcommandRequest: ProbeRequest;
+    /** authorized, and answered by the fixture exiting non-zero. */
+    readonly nonZeroExitRequest: ProbeRequest;
+    /** authorized, carrying a shell-metacharacter value on one argument channel key. */
+    readonly injectionRequest: ProbeRequest;
+    /** the exact literal value `injectionRequest` carries, so the assertion can confirm the fixture received it byte for byte. */
+    readonly injectionArgumentValue: string;
+    /** authorized, and its authorization declares one artifact identifier the fixture writes when run. */
+    readonly artifactRequest: ProbeRequest;
+    readonly artifactId: string;
+    readonly artifactExpectedText: string;
+    /** authorized against a maxElapsedMs the fixture is made to sleep past. */
+    readonly overElapsedRequest: ProbeRequest;
+    /** authorized against a maxOutputBytes the fixture is made to write past. */
+    readonly overOutputRequest: ProbeRequest;
+};
+/**
+ * Fifteen outcomes: the six shared assertions plus the nine above. Unlike the
+ * `api` arm, every cap here has an assertion: a command subject's fixture
+ * script can be told to overrun a byte cap directly, with no oversize-request
+ * problem to work around.
+ */
+export declare function runCommandLineProbeConformance(subject: CommandProbeSubject): Promise<ConformanceReport>;

package/dist/testing/probe-conformance.js

@@ -150,6 +150,7 @@ function echoMismatch(request, observation) {
     }
     return undefined;
 }
+/** Reads only `id` and `title`, so both arms' assertion shapes satisfy it structurally. */
 function checkRejected(assertion, expectedCode, error) {
     const view = faultView(error);
     if (view === undefined) {
@@ -213,3 +214,164 @@ export async function runEnvironmentProbePortConformance(subject) {
     }
     return reportOf(subject.name, 'environment-probe', [...shared, ...additional]);
 }
+const COMMAND_ASSERTIONS = [
+    {
+        id: 'command/allow-authorized-invocation',
+        title: 'an explicitly authorized command runs and is observed',
+        request: (subject) => subject.authorizedRequest,
+        expectation: { kind: 'resolves' },
+    },
+    {
+        id: 'command/observe-nonzero-exit',
+        title: 'a non-zero exit from an authorized command is an observation, not a fault',
+        request: (subject) => subject.nonZeroExitRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation) => observation.kind === 'cli' && observation.exitCode !== 0
+                ? undefined
+                : `observed ${observation.kind === 'cli' ? `exit code ${observation.exitCode}` : 'an api observation'}, expected a non-zero exit`,
+        },
+    },
+    {
+        id: 'command/deny-unmapped-interface',
+        title: 'an interface no authorization names is refused before a process spawns',
+        request: (subject) => subject.unmappedInterfaceRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'command/deny-unmapped-executable',
+        title: 'an executable the interface is never paired with is refused before a process spawns',
+        request: (subject) => subject.unmappedExecutableRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'command/deny-unauthorized-subcommand',
+        title: 'a subcommand path outside the authorized set is refused before a process spawns',
+        request: (subject) => subject.unauthorizedSubcommandRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        // The strongest available proof that no shell ever reads a channel value:
+        // an argument built to look like a command substitution, checked to reach
+        // the process as the one, unmodified literal token it was declared as.
+        id: 'command/argument-passed-literally',
+        title: 'a shell-metacharacter argument reaches the process as one unmodified literal token, never a shell expansion',
+        request: (subject) => subject.injectionRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation, subject) => {
+                if (observation.kind !== 'cli' || observation.stdout.kind !== 'json') {
+                    return 'the observation carries no parsed stdout to check the received argument against';
+                }
+                const argv = observation.stdout.value.argv;
+                const received = Array.isArray(argv) ? argv : [];
+                return received.length === 1 &&
+                    received[0] === subject.injectionArgumentValue
+                    ? undefined
+                    : `the process observed argv ${JSON.stringify(received)}, expected exactly one element equal to the declared literal`;
+            },
+        },
+    },
+    {
+        id: 'command/capture-declared-artifact',
+        title: "a declared artifact identifier reads the file the run wrote, keyed by the operation's own identifier",
+        request: (subject) => subject.artifactRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation, subject) => {
+                if (observation.kind !== 'cli') {
+                    return 'the observation is not a command observation';
+                }
+                const artifact = observation.artifacts[subject.artifactId];
+                return artifact?.kind === 'text' &&
+                    artifact.value === subject.artifactExpectedText
+                    ? undefined
+                    : `artifacts["${subject.artifactId}"] was ${JSON.stringify(artifact)}, expected the fixture's declared text`;
+            },
+        },
+    },
+    {
+        id: 'command/cap-elapsed',
+        title: 'a process past maxElapsedMs is capped, not left to finish',
+        request: (subject) => subject.overElapsedRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+    },
+    {
+        id: 'command/cap-output-bytes',
+        title: 'output past maxOutputBytes is capped, not silently truncated and returned',
+        request: (subject) => subject.overOutputRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+    },
+];
+function checkCommandResolved(assertion, expectation, value, request, subject) {
+    const parsed = probeParsers.response.safeParse(value);
+    if (!parsed.success) {
+        return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
+    }
+    const mismatch = echoMismatch(request, parsed.data);
+    if (mismatch !== undefined) {
+        return titledOutcome(assertion.id, assertion.title, false, mismatch);
+    }
+    const complaint = expectation.check?.(parsed.data, subject);
+    return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
+}
+/** `checkCalls`'s own logic, typed against `CommandAssertion`/`CommandProbeSubject` rather than the `api` arm's pair: a function parameter type is checked contravariantly, so the two assertion shapes cannot share one checker. */
+function checkCommandCalls(assertion, subject, built, base) {
+    if (assertion.expectedCalls === undefined)
+        return base;
+    const expected = assertion.expectedCalls(subject);
+    if (typeof expected === 'string') {
+        return { ...base, passed: false, detail: expected };
+    }
+    const actual = countCalls(built);
+    if (typeof actual === 'string') {
+        return { ...base, passed: false, detail: actual };
+    }
+    if (actual === expected)
+        return base;
+    return {
+        ...base,
+        passed: false,
+        detail: `${base.passed ? '' : `${base.detail}; `}underlyingCalls() was ${actual}, expected ${expected}`,
+    };
+}
+async function runCommandAssertion(assertion, subject) {
+    const run = await buildScenario(subject, 'resolves');
+    if (!run.ok) {
+        return titledOutcome(assertion.id, assertion.title, false, run.detail);
+    }
+    const request = assertion.request(subject);
+    const settled = await settle(run.built.port(request, new AbortController().signal));
+    let result;
+    if (assertion.expectation.kind === 'resolves') {
+        result =
+            settled.kind === 'resolved'
+                ? checkCommandResolved(assertion, assertion.expectation, settled.value, request, subject)
+                : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
+    }
+    else {
+        result =
+            settled.kind === 'rejected'
+                ? checkRejected(assertion, assertion.expectation.code, settled.error)
+                : titledOutcome(assertion.id, assertion.title, false, `resolved instead of rejecting with "${assertion.expectation.code}"`);
+    }
+    result = checkCommandCalls(assertion, subject, run.built, result);
+    return withDispose(result, await disposeScenario(run.built));
+}
+/**
+ * Fifteen outcomes: the six shared assertions plus the nine above. Unlike the
+ * `api` arm, every cap here has an assertion: a command subject's fixture
+ * script can be told to overrun a byte cap directly, with no oversize-request
+ * problem to work around.
+ */
+export async function runCommandLineProbeConformance(subject) {
+    const shared = await runSharedAssertions('probe', subject, probeParsers.response);
+    const additional = [];
+    for (const assertion of COMMAND_ASSERTIONS) {
+        additional.push(await runCommandAssertion(assertion, subject));
+    }
+    return reportOf(subject.name, 'command-probe', [...shared, ...additional]);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eval-quality",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
   "author": "Murat Ozcan",
   "license": "Apache-2.0",