eval-quality 0.2.0 → 0.5.0

package/dist/adapters/command-line-adapter.js

@@ -0,0 +1,284 @@
+/**
+ * The environment-probe port over a real child process, for the `cli`
+ * mechanism, which `probe-conformance.ts` had documented as unauthorizable
+ * and unshipped. `evaluateCommandTarget` closes the authorization half; this
+ * file closes the execution half.
+ *
+ * Every rule below is what "what's a command allowed to do" resolves to, a
+ * decision settled here in the implementation rather than as an architecture
+ * revision:
+ *
+ * 1. `target` is spawned directly with an argv array (`shell: false`,
+ *    Node's own default, stated anyway since it is the one thing this
+ *    adapter must never turn off). No channel value is ever concatenated
+ *    into a shell string; a value containing `;`, `$(...)`, or a quote
+ *    reaches the child as one literal argv element, not a metacharacter.
+ * 2. `argument` and `option` build the argv the same way a well-behaved CLI
+ *    parser reads one: options first as `--{key}` (a boolean `true` is a
+ *    bare flag, `false` is omitted, anything else gets one value token),
+ *    positionals after in the record's own key order. `environment` passes
+ *    through as declared, plus the host's own `PATH` so a `target` naming a
+ *    bare command still resolves; a declared `PATH` key wins over that
+ *    default. `stdin` is written and the stream is closed; `absent` closes
+ *    it with nothing written.
+ * 3. `maxElapsedMs` and `maxOutputBytes` are enforced by this adapter, not
+ *    borrowed from `AbortSignal`: exceeding either kills the process with
+ *    `SIGKILL` and throws `budget-exhausted`, exactly as an HTTP cap does.
+ *    `maxOutputBytes` applies independently to stdout, to stderr, and to
+ *    each artifact file read back after exit.
+ * 4. A non-zero exit is an observation, never a fault, matching AD-10's rule
+ *    for an HTTP status. Only a policy denial, a cap, an abort, or a failure
+ *    to start the process throws. `exitCode` is negative when a signal
+ *    ended the process, the same convention `CommandProbeObservation`
+ *    documents for itself.
+ */
+import { spawn } from 'node:child_process';
+import { open } from 'node:fs/promises';
+import { constants as osConstants } from 'node:os';
+import { isAbsolute, resolve as resolvePath } from 'node:path';
+import { RuntimeFault } from '../core/schemas/faults.js';
+import { probeParsers } from '../ports/environment-probe-port.js';
+import { evaluateCommandTarget } from './command-target-policy.js';
+import { runPortMethod } from './port-boundary.js';
+function stringifyScalar(value) {
+    if (typeof value === 'string')
+        return value;
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return String(value);
+    return JSON.stringify(value);
+}
+/** Options first as `--{key}`, positionals after, both in the record's own key order. Exported for its own unit tests: this is the one place shell-injection safety is decided. */
+export function buildArgv(channels) {
+    const optionTokens = [];
+    for (const [key, value] of Object.entries(channels.option)) {
+        if (value === false)
+            continue;
+        optionTokens.push(`--${key}`);
+        if (value !== true)
+            optionTokens.push(stringifyScalar(value));
+    }
+    const argumentTokens = Object.values(channels.argument).map(stringifyScalar);
+    return [...optionTokens, ...argumentTokens];
+}
+function buildEnv(declared) {
+    const base = {};
+    if (process.env.PATH !== undefined)
+        base.PATH = process.env.PATH;
+    return { ...base, ...declared };
+}
+function buildStdin(stdin) {
+    if (stdin.kind === 'absent')
+        return { kind: 'absent' };
+    if (stdin.kind === 'json')
+        return { kind: 'json', value: stdin.value };
+    return { kind: 'text', value: stdin.value };
+}
+/** A heuristic, not a declared content type: the port carries none for a process stream. JSON-shaped text reads as `json`; anything else as `text`. */
+function bodyFromText(text) {
+    if (text.length === 0)
+        return { kind: 'text', value: text };
+    try {
+        return { kind: 'json', value: JSON.parse(text) };
+    }
+    catch {
+        return { kind: 'text', value: text };
+    }
+}
+function capped(detail) {
+    return new RuntimeFault('budget-exhausted', 'CommandProbeRequest', detail);
+}
+function forbidden(detail) {
+    return new RuntimeFault('forbidden-target', 'ProbeRequest', detail);
+}
+function writeStdin(child, stdin) {
+    const stream = child.stdin;
+    if (stream === null)
+        return;
+    // A child that exits before reading stdin closes its end of the pipe, and
+    // the pending `end()` write below then raises EPIPE on this stream, not on
+    // `child` itself. With no listener that is an unhandled 'error' and crashes
+    // the host process; the run itself is unaffected, since `close` still fires
+    // and carries the exit code this adapter already reads it from.
+    stream.on('error', () => { });
+    if (stdin.kind === 'absent') {
+        stream.end();
+        return;
+    }
+    stream.end(stdin.kind === 'json' ? JSON.stringify(stdin.value) : String(stdin.value));
+}
+/** `-signalNumber` when a signal ended the process, matching `CommandProbeObservation.exitCode`'s own documented convention. */
+function exitCodeOf(code, signalName) {
+    if (code !== null)
+        return code;
+    if (signalName === null)
+        return -1;
+    const numeric = osConstants.signals[signalName];
+    return numeric === undefined ? -1 : -numeric;
+}
+async function runChildProcess(request, signal) {
+    return new Promise((settlePromise, rejectPromise) => {
+        const child = spawn(request.target, [...request.subcommandPath, ...request.argv], {
+            cwd: request.cwd,
+            env: request.env,
+            shell: false,
+            signal,
+        });
+        let stdout = '';
+        let stderr = '';
+        let settled = false;
+        let timedOut = false;
+        let overCap = false;
+        const finish = (action) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            action();
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            child.kill('SIGKILL');
+        }, request.maxElapsedMs);
+        const capture = (chunk, append, currentLength) => {
+            if (currentLength() > request.maxOutputBytes)
+                return;
+            append(chunk.toString('utf8'));
+            if (currentLength() > request.maxOutputBytes) {
+                overCap = true;
+                child.kill('SIGKILL');
+            }
+        };
+        child.stdout?.on('data', (chunk) => capture(chunk, (next) => {
+            stdout += next;
+        }, () => Buffer.byteLength(stdout, 'utf8')));
+        child.stderr?.on('data', (chunk) => capture(chunk, (next) => {
+            stderr += next;
+        }, () => Buffer.byteLength(stderr, 'utf8')));
+        child.once('error', (error) => {
+            finish(() => rejectPromise(error));
+        });
+        child.once('close', (code, signalName) => {
+            finish(() => {
+                if (timedOut) {
+                    rejectPromise(capped(`exceeded maxElapsedMs (${request.maxElapsedMs}ms) and was killed`));
+                    return;
+                }
+                if (overCap) {
+                    rejectPromise(capped(`stdout or stderr exceeded maxOutputBytes (${request.maxOutputBytes}) and the process was killed`));
+                    return;
+                }
+                settlePromise({
+                    exitCode: exitCodeOf(code, signalName),
+                    stdout,
+                    stderr,
+                });
+            });
+        });
+        writeStdin(child, request.stdin);
+    });
+}
+/** Reads at most `maxBytes + 1` bytes rather than the whole file: an oversize artifact is capped before its bytes are loaded, not after, so a run that wrote past the limit cannot make this adapter allocate the excess first. */
+async function readArtifactFile(path, maxBytes) {
+    let handle;
+    try {
+        handle = await open(path, 'r');
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return { present: false, text: '', truncated: false };
+        }
+        throw error;
+    }
+    try {
+        const buffer = Buffer.alloc(maxBytes + 1);
+        const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, 0);
+        const truncated = bytesRead > maxBytes;
+        const text = buffer
+            .subarray(0, Math.min(bytesRead, maxBytes))
+            .toString('utf8');
+        return { present: true, text, truncated };
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * The real mechanism: an actual child process, an actual file read. Exported,
+ * unlike the other three shipped adapters' defaults, because AD-37's own
+ * conformance subject for this port has to exercise real process behaviour
+ * (a real timeout, a real output cap, a real argv) the way the `api` arm's
+ * subject exercises a real loopback server; a synthetic mechanism would prove
+ * nothing about the one thing this adapter exists to get right.
+ */
+export const nodeCommandMechanism = {
+    run: runChildProcess,
+    readArtifact: readArtifactFile,
+};
+export function createCommandLineAdapter(policy, mechanism = nodeCommandMechanism) {
+    return {
+        probe: (request, signal) => runPortMethod({
+            request,
+            requestParser: probeParsers.request,
+            responseParser: probeParsers.response,
+            requestPath: 'ProbeRequest',
+            responsePath: 'ProbeObservation',
+            signal,
+            mechanism: async (parsed, innerSignal) => {
+                if (parsed.kind !== 'cli') {
+                    // This adapter authorizes no `api` target, so an `api` request
+                    // meets the same "the mapping names nothing" denial an
+                    // unmapped interfaceId would, before any process spawns.
+                    throw forbidden('this adapter runs cli requests only; no api target is ever authorized');
+                }
+                const decision = evaluateCommandTarget(policy, {
+                    interfaceId: parsed.interfaceId,
+                    executable: parsed.executable,
+                    subcommandPath: parsed.subcommandPath,
+                });
+                if (!decision.allowed)
+                    throw forbidden(decision.detail);
+                const { authorization } = decision;
+                const runResult = await mechanism.run({
+                    target: authorization.target,
+                    subcommandPath: parsed.subcommandPath,
+                    argv: buildArgv(parsed.channels),
+                    env: buildEnv(parsed.channels.environment),
+                    stdin: buildStdin(parsed.channels.stdin),
+                    cwd: authorization.cwd,
+                    maxElapsedMs: authorization.maxElapsedMs,
+                    maxOutputBytes: authorization.maxOutputBytes,
+                }, innerSignal);
+                const artifactEntries = await Promise.all(Object.entries(authorization.artifacts).map(async ([artifactId, relativePath]) => {
+                    const absolute = isAbsolute(relativePath)
+                        ? relativePath
+                        : resolvePath(authorization.cwd, relativePath);
+                    const read = await mechanism.readArtifact(absolute, authorization.maxOutputBytes);
+                    if (read.truncated) {
+                        throw capped(`artifact "${artifactId}" exceeded maxOutputBytes (${authorization.maxOutputBytes})`);
+                    }
+                    return [artifactId, read];
+                }));
+                return { parsed, runResult, artifactEntries };
+            },
+            assemble: (raw) => {
+                const { parsed, runResult, artifactEntries } = raw;
+                const observation = {
+                    kind: 'cli',
+                    probeId: parsed.probeId,
+                    interfaceId: parsed.interfaceId,
+                    operationId: parsed.operationId,
+                    exitCode: runResult.exitCode,
+                    stdout: bodyFromText(runResult.stdout),
+                    stderr: bodyFromText(runResult.stderr),
+                    artifacts: Object.fromEntries(artifactEntries.map(([id, read]) => [
+                        id,
+                        read.present
+                            ? bodyFromText(read.text)
+                            : { kind: 'absent' },
+                    ])),
+                };
+                return observation;
+            },
+        }),
+    };
+}

package/dist/adapters/command-target-policy.d.ts

@@ -0,0 +1,36 @@
+/**
+ * AD-35's default-deny decision for the `cli` mechanism, as a pure function.
+ *
+ * Lives under `adapters/` rather than beside `core/probe/target-policy.ts`
+ * (the `api` evaluator), even though the two are otherwise siblings: the
+ * repository's mechanically-enforced dependency direction permits
+ * `adapters/` to import `core/schemas` but not `core/`, so a
+ * target-authorization decision an adapter calls has to be adapter-owned
+ * infrastructure, not a `core/` module. `target-policy.ts` itself is never
+ * imported from `src/adapters/` for the same reason — no shipped `api`
+ * adapter exists to call it, which is the gap this file closes for `cli`.
+ */
+import type { CommandTargetAuthorization, CommandTargetPolicy } from '../core/schemas/probe-policy.ts';
+/** Why a command target was denied. Thrown as the single AD-28 `forbidden-target` fault, same as the HTTP reasons. */
+export declare const COMMAND_DENIAL_REASONS: readonly ['interface-not-authorized', 'executable-not-authorized', 'subcommand-not-authorized'];
+export type CommandDenialReason = (typeof COMMAND_DENIAL_REASONS)[number];
+export type CommandResolvedTarget = {
+    readonly interfaceId: string;
+    readonly executable: string;
+    readonly subcommandPath: readonly string[];
+};
+export type CommandPolicyDecision = {
+    readonly allowed: true;
+    readonly authorization: CommandTargetAuthorization;
+} | {
+    readonly allowed: false;
+    readonly reason: CommandDenialReason;
+    readonly detail: string;
+};
+/**
+ * The interface-and-executable check runs first, so an unmapped pair never
+ * reaches the subcommand comparison. Where several authorizations name one
+ * pair, each is tried in declaration order and the first that allows wins;
+ * mirrors `evaluateTarget`'s own ordering rule for the same reason.
+ */
+export declare function evaluateCommandTarget(policy: CommandTargetPolicy, target: CommandResolvedTarget): CommandPolicyDecision;

package/dist/adapters/command-target-policy.js

@@ -0,0 +1,33 @@
+/** Why a command target was denied. Thrown as the single AD-28 `forbidden-target` fault, same as the HTTP reasons. */
+export const COMMAND_DENIAL_REASONS = [
+    'interface-not-authorized',
+    'executable-not-authorized',
+    'subcommand-not-authorized',
+];
+function subcommandPathsMatch(a, b) {
+    return (a.length === b.length && a.every((segment, index) => segment === b[index]));
+}
+function deny(reason, detail) {
+    return { allowed: false, reason, detail };
+}
+/**
+ * The interface-and-executable check runs first, so an unmapped pair never
+ * reaches the subcommand comparison. Where several authorizations name one
+ * pair, each is tried in declaration order and the first that allows wins;
+ * mirrors `evaluateTarget`'s own ordering rule for the same reason.
+ */
+export function evaluateCommandTarget(policy, target) {
+    const matchingPair = policy.authorizations.filter((authorization) => authorization.interfaceId === target.interfaceId &&
+        authorization.executable === target.executable);
+    if (matchingPair.length === 0) {
+        return deny(policy.authorizations.some((authorization) => authorization.interfaceId === target.interfaceId)
+            ? 'executable-not-authorized'
+            : 'interface-not-authorized', `no authorization names interface "${target.interfaceId}" and executable "${target.executable}" together`);
+    }
+    for (const authorization of matchingPair) {
+        if (authorization.permittedSubcommandPaths.some((permitted) => subcommandPathsMatch(permitted, target.subcommandPath))) {
+            return { allowed: true, authorization };
+        }
+    }
+    return deny('subcommand-not-authorized', `subcommand path [${target.subcommandPath.join(', ')}] is not among the authorized paths for interface "${target.interfaceId}" and executable "${target.executable}"`);
+}

package/dist/adapters/index.d.ts

@@ -4,6 +4,8 @@
  * subpath and the root barrel keeps the `root -> adapters` edge the matrix
  * does not grant.
  */
+export type { CommandMechanism, CommandRunRequest, CommandRunResult, } from './command-line-adapter.ts';
+export { createCommandLineAdapter, nodeCommandMechanism, } from './command-line-adapter.ts';
 export type { CorpusMechanism } from './local-corpus-adapter.ts';
 export { createLocalCorpusAdapter } from './local-corpus-adapter.ts';
 export type { FileSystemMechanism } from './node-file-system-adapter.ts';

package/dist/adapters/index.js

@@ -1,3 +1,4 @@
+export { createCommandLineAdapter, nodeCommandMechanism, } from './command-line-adapter.js';
 export { createLocalCorpusAdapter } from './local-corpus-adapter.js';
 export { createNodeFileSystemAdapter } from './node-file-system-adapter.js';
 export { createSystemClockAdapter } from './system-clock-adapter.js';

package/dist/cli/render.d.ts

@@ -1,8 +1,3 @@
-/**
- * The four shapes the binary writes, and the exit-code table two documents
- * share. Every line the CLI emits is produced here, so a format change is one
- * file.
- */
 import { type Diagnostic } from '../application/index.ts';
 /**
  * Delegates to `serializeArtifact`; the canonical bytes are not re-derived
@@ -13,6 +8,8 @@ export declare function renderArtifact(artifact: unknown, artifactPath: string):
 export declare function renderDiagnostic(diagnostic: Diagnostic): string;
 /**
  * `eval-quality: <code>: <artifactPath>: <detail>` for either error class.
+ * A fault carrying a Zod error as its cause prints that error's issues under
+ * the first line, one indented `<location>: <message>` per issue.
  * Anything else falls back to `String(error)`, which is what a defect in our
  * own code looks like from outside.
  */

package/dist/cli/render.js

@@ -3,8 +3,55 @@
  * share. Every line the CLI emits is produced here, so a format change is one
  * file.
  */
+import { z } from 'zod';
 import { RuntimeFault, StructuralFailure, serializeArtifact, } from '../application/index.js';
 const PREFIX = 'eval-quality';
+/**
+ * How many parse issues the renderer prints before it stops and counts the
+ * rest.
+ * An author fixing a contract reads the first few and edits; a wall of four
+ * hundred lines from one missing required key teaches nothing and buries the
+ * first line, which is the one that names the code.
+ */
+const ISSUE_LIMIT = 20;
+/**
+ * An RFC 6901 pointer over the failing value, so the location the renderer
+ * prints is the same spelling the contract itself uses to address evidence.
+ * A numeric segment is an array index and is printed as one.
+ */
+function issueLocation(path) {
+    if (path.length === 0)
+        return '(root)';
+    return path
+        .map((segment) => String(segment).replaceAll('~', '~0').replaceAll('/', '~1'))
+        .map((segment) => `/${segment}`)
+        .join('');
+}
+/**
+ * The issue list a failed parse already carries, which the first line of the
+ * error alone does not expose.
+ * A contract author outside this repository has the published schema and this
+ * message and nothing else, so a parse failure that names no field costs a
+ * bisect over the whole document.
+ * Issues are sorted by location so two runs over the same input print the same
+ * bytes.
+ * A Zod issue message names the expectation and never echoes the value that
+ * failed, which matters because this goes to stderr and a contract declares an
+ * environment channel; the one thing a message does quote is an unrecognized
+ * key name, which is a field name.
+ */
+function renderParseIssues(cause) {
+    if (!(cause instanceof z.ZodError))
+        return '';
+    const lines = cause.issues
+        .map((issue) => `  ${issueLocation(issue.path)}: ${issue.message}`)
+        .sort();
+    const shown = lines.slice(0, ISSUE_LIMIT);
+    const hidden = lines.length - shown.length;
+    if (hidden > 0)
+        shown.push(`  ... and ${hidden} more`);
+    return shown.length === 0 ? '' : `\n${shown.join('\n')}`;
+}
 /**
  * Delegates to `serializeArtifact`; the canonical bytes are not re-derived
  * here, so the text written to stdout is the text `digestArtifact` hashes.
@@ -18,6 +65,8 @@ export function renderDiagnostic(diagnostic) {
 }
 /**
  * `eval-quality: <code>: <artifactPath>: <detail>` for either error class.
+ * A fault carrying a Zod error as its cause prints that error's issues under
+ * the first line, one indented `<location>: <message>` per issue.
  * Anything else falls back to `String(error)`, which is what a defect in our
  * own code looks like from outside.
  */
@@ -27,7 +76,8 @@ export function renderError(error) {
         const detail = error.message.startsWith(prefix)
             ? error.message.slice(prefix.length)
             : error.message;
-        return `${PREFIX}: ${error.code}: ${error.artifactPath}: ${detail}`;
+        const issues = renderParseIssues(error.cause);
+        return `${PREFIX}: ${error.code}: ${error.artifactPath}: ${detail}${issues}`;
     }
     return `${PREFIX}: ${String(error)}`;
 }

package/dist/core/compile/bindings.d.ts

@@ -1,10 +1,23 @@
 import type { EvalContract } from '../schemas/eval-contract.ts';
 import type { InteractionStep } from '../schemas/plan.ts';
-import { type TransportChannelName } from '../schemas/pointer.ts';
+import type { InputChannelName } from '../schemas/pointer.ts';
 import { type EvidenceTarget } from '../seal/plan-index.ts';
+/**
+ * The one channel a captured pointer may name: whichever channel the
+ * referenced operation's own response descriptor describes.
+ *
+ * `ResponseDescriptor` declares `requiredKeys`, `permittedKeys`, `types`,
+ * `successIndicator`, `channelRoles`, and `collectionLocations`, and every one
+ * of them is about the channel the operation nominates. Off an interface that
+ * speaks HTTP that is the response body; off a command it is the stream the
+ * operation names. `response-headers` and `response-status` are never it,
+ * because `Observation.responseHeaders` admits objects, arrays, numbers, and
+ * `null`, so a header capture compiled as a `string` could resolve to an
+ * object at score time.
+ */
 /** One `{ captured }` binding, resolved to the pointer target it addresses. */
 export type CapturedBinding = {
-    readonly transportChannel: TransportChannelName;
+    readonly transportChannel: InputChannelName;
     readonly key: string;
     readonly pointer: string;
     readonly target: EvidenceTarget;
@@ -45,11 +58,17 @@ export declare function capturedBindings(step: InteractionStep): readonly Captur
 export declare function checkBindingCycle(contract: EvalContract): void;
 /**
  * `captured-channel-undeclared`: a captured pointer naming any AD-26 channel
- * but `response-body`. `call-inputs` addresses a step's own request,
- * `stdout`, `stderr`, and `exit-code` are process channels no operation
- * surviving `unsupported-interface-kind` produces, and `response-headers` and
- * `response-status` have no declared structure to give a captured value a
- * type.
+ * but the one the referenced operation's response descriptor describes. That
+ * is `response-body` off an interface that speaks HTTP and whichever channel
+ * `descriptorChannel` nominates off one that runs behind a command, which is
+ * the same rule read against the declaration rather than assumed.
+ *
+ * Every other channel is refused for a reason that does not depend on the
+ * kind: `call-inputs` addresses a step's own request, and `response-headers`,
+ * `response-status`, and `exit-code` have no declared structure to give a
+ * captured value a type. On the `artifact` channel the identifier is compared
+ * too, so a capture from a file the operation writes but does not describe is
+ * refused alongside one from a file it never writes.
  */
 export declare function checkCapturedChannel(contract: EvalContract): void;
 /**