eval-quality 0.1.0 → 0.3.0

package/dist/core/declared-inputs.js

@@ -1,18 +1,115 @@
-import { TRANSPORT_CHANNELS } from './schemas/pointer.js';
+import { COMMAND_CHANNELS, INPUT_CHANNELS, TRANSPORT_CHANNELS, } from './schemas/pointer.js';
+/**
+ * Which output channel the operation's one response descriptor describes.
+ *
+ * AD-19 gives every operation exactly one descriptor, and the channel that
+ * descriptor describes is what makes its keys addressable. For an interface
+ * that speaks HTTP that channel is the response body; a command operation
+ * declares its own. One rule, three roots: every consumer that used to hard-code
+ * `response-body` asks this instead.
+ */
+export const descriptorChannelOf = (operation) => {
+    if (!isCommandOperation(operation))
+        return 'response-body';
+    const { descriptorChannel } = operation;
+    return descriptorChannel.kind === 'stream'
+        ? descriptorChannel.channel
+        : 'artifact';
+};
+/**
+ * Which artifact the descriptor describes, or `null` when it describes a
+ * stream. An artifact pointer descends through the descriptor only when it
+ * names this one; every other declared artifact is known to exist and declares
+ * no structure.
+ */
+export const descriptorArtifactOf = (operation) => {
+    if (!isCommandOperation(operation))
+        return null;
+    const { descriptorChannel } = operation;
+    return descriptorChannel.kind === 'artifact'
+        ? descriptorChannel.artifactId
+        : null;
+};
+/**
+ * Whether an evidence target addresses the channel this operation's response
+ * descriptor describes.
+ *
+ * The channel name alone is not the answer on the artifact channel: an
+ * operation may declare several files while its one descriptor describes one of
+ * them, so a pointer at a different file names a channel with no declared
+ * structure. Every consumer asks this rather than comparing the channel itself,
+ * because comparing only the channel was a defect at three separate sites and a
+ * fourth site would have made the same mistake for the same reason.
+ */
+export const targetsDescribedChannel = (operation, target) => {
+    if (target.channel !== descriptorChannelOf(operation))
+        return false;
+    if (target.channel !== 'artifact')
+        return true;
+    return target.artifactId === descriptorArtifactOf(operation);
+};
+/** Every artifact identifier the operation declares it writes. */
+export const declaredArtifactsOf = (operation) => isCommandOperation(operation) ? operation.artifacts : [];
+/**
+ * Which operation shape this is. Reads `invocation` rather than a kind field,
+ * because an operation does not carry its interface's kind; the two shapes are
+ * distinguished by a required field only one of them declares.
+ */
+export const isCommandOperation = (operation) => 'invocation' in operation;
+/** The same question the other way round, for the callers that filter. */
+export const isApiOperation = (operation) => !isCommandOperation(operation);
+/**
+ * The operation's input channels paired with the shapes they declare.
+ *
+ * Callers take the pairs rather than a channel list they then index the
+ * request shape with. Indexing is what breaks under the operation union:
+ * TypeScript cannot prove that a channel name drawn from one kind's tuple is a
+ * key of the other kind's shape, and both ways around that are casts. Pairing
+ * dereferences the union once, here.
+ */
+export function requestChannelsOf(operation) {
+    if (isCommandOperation(operation)) {
+        return COMMAND_CHANNELS.map((channel) => ({
+            channel,
+            shape: operation.requestShape[channel],
+        }));
+    }
+    return TRANSPORT_CHANNELS.map((channel) => ({
+        channel,
+        shape: operation.requestShape[channel],
+    }));
+}
+/** The shape one named channel declares, or `undefined` off this kind. */
+export function requestShapeOf(operation, channel) {
+    return requestChannelsOf(operation).find((entry) => entry.channel === channel)
+        ?.shape;
+}
+/** The channel names an operation of this kind may declare inputs on. */
+export const inputChannelsOf = (operation) => isCommandOperation(operation) ? COMMAND_CHANNELS : TRANSPORT_CHANNELS;
+/**
+ * A step's bound channels paired with what each binds, for the same reason
+ * `requestChannelsOf` pairs: `InputBinding` is a union and a channel name
+ * drawn from one branch is not a key of the other.
+ *
+ * Ordered by `INPUT_CHANNELS` rather than by the parsed object's own key
+ * order, so which binding a check reports never depends on how the document
+ * happened to be written.
+ */
+export function boundChannelsOf(binding) {
+    const present = binding;
+    return INPUT_CHANNELS.filter((channel) => channel in binding).map((channel) => ({ channel, bound: present[channel] ?? null }));
+}
 /**
  * AD-10's exemption predicate. Any of a channel's three lists naming a key
  * counts: a permitted-only or types-only channel is still a surface a witness
  * can vary.
  */
 export function declaresNoRequestKeys(operation) {
-    return TRANSPORT_CHANNELS.every((channel) => {
-        const shape = operation.requestShape[channel];
-        return (shape.requiredKeys.length === 0 &&
-            shape.permittedKeys.length === 0 &&
-            Object.keys(shape.types).length === 0);
-    });
+    return requestChannelsOf(operation).every(({ shape }) => shape.requiredKeys.length === 0 &&
+        shape.permittedKeys.length === 0 &&
+        Object.keys(shape.types).length === 0);
 }
 /** Whether no channel declares a required key. */
 export function declaresNoRequiredKeys(operation) {
-    return TRANSPORT_CHANNELS.every((channel) => operation.requestShape[channel].requiredKeys.length === 0);
+    return requestChannelsOf(operation).every(({ shape }) => shape.requiredKeys.length === 0);
 }

package/dist/core/emit/emit.d.ts

@@ -0,0 +1,9 @@
+import type { ScoredOutcomesAndVerdict } from '../score/score.ts';
+import type { EmitStage } from '../stage-contracts.ts';
+/**
+ * The stage. Builds `scoringVersionInputs`/`scoringVersion`/
+ * `comparabilityKey`/the strength block/the remediation block once, shared
+ * between the two mode branches, then the mode-discriminated artifact
+ * literal, then calls `checkModeAgreement` immediately before returning.
+ */
+export declare const emit: EmitStage<ScoredOutcomesAndVerdict>;

package/dist/core/emit/emit.js

@@ -0,0 +1,214 @@
+/**
+ * AD-24's emit stage: `score`'s widened product plus the three caller-
+ * attested digests AD-11 fixes, minting the `EvidenceArtifact` this package
+ * has never built until now. Pure per AD-1/AD-2, and total the way every
+ * other stage in this package is: nothing here throws for a domain input.
+ *
+ * `comparabilityKey`, `strength`, and `remediation` are lifted verbatim from
+ * `scripts/worked-example-target.ts`'s own hand-assembly (its former
+ * lines 1432-1487), the field-by-field precedent this module generalises
+ * from one caller-assembled record to `score`'s own widened product.
+ */
+import { digestArtifact } from '../canonical/digest.js';
+import { freezeArtifact } from '../lineage/freeze.js';
+import { EvidenceArtifact, } from '../schemas/evidence-artifact.js';
+import { checkModeAgreement } from '../score/mode-agreement.js';
+import { buildStrengthVector } from '../score/strength.js';
+const SCORING_POLICY_ARTIFACT_PATH = 'ScoringPolicy';
+const SCORING_VERSION_INPUTS_ARTIFACT_PATH = 'ScoringVersionInputs';
+const COMPARABILITY_KEY_ARTIFACT_PATH = 'ComparabilityKey';
+/**
+ * `mode` joins AD-11's three caller-attested digest names as a fourth
+ * member: like them, it has no artifact source anywhere in this pipeline and
+ * can only ever be caller-supplied, never re-derived by `score`.
+ */
+const CALLER_ATTESTED_INPUTS = ['corpusDigest', 'fixtureDigest', 'evaluatorConfigurationDigest', 'mode'];
+/**
+ * The stage. Builds `scoringVersionInputs`/`scoringVersion`/
+ * `comparabilityKey`/the strength block/the remediation block once, shared
+ * between the two mode branches, then the mode-discriminated artifact
+ * literal, then calls `checkModeAgreement` immediately before returning.
+ */
+export const emit = (scored, corpusDigest, fixtureDigest, evaluatorConfigurationDigest) => {
+    const scoringPolicyDigest = digestArtifact(scored.policy, SCORING_POLICY_ARTIFACT_PATH);
+    // AD-7's declared key: the scoring policy digest plus the corpus digest
+    // restricted to the probes both results cover -- here, the admitted probe
+    // identifiers, sorted so array order never enters the digest.
+    const comparabilityKey = digestArtifact({
+        scoringPolicyDigest,
+        probeIds: scored.sealedProbes.admitted
+            .map((entry) => entry.probe.probeId)
+            .sort(),
+    }, COMPARABILITY_KEY_ARTIFACT_PATH);
+    const excludedProbeIds = scored.sealedProbes.rejected.map((entry) => entry.probe.probeId);
+    const { trials, outcomes: scoredOutcomes } = scored.assessment.outcomeState;
+    const trialCount = `${trials.completed} completed trial${trials.completed === 1 ? '' : 's'}`;
+    const unreachedOracles = scoredOutcomes.filter((outcome) => outcome.resolution.state === 'unreached');
+    // AD-21: a run that resolved `unreached` or completed fewer trials than
+    // the policy's declared minimum marks the vector non-comparable rather
+    // than silently comparing it.
+    const comparable = trials.completed >= trials.declaredMinimum && unreachedOracles.length === 0;
+    const strengthNote = [
+        `${scored.sealedProbes.admitted.length} admitted probe over ${trialCount}.`,
+        trials.completed < trials.declaredMinimum
+            ? `Below the declared minimum of ${trials.declaredMinimum}.`
+            : null,
+        unreachedOracles.length > 0
+            ? `${unreachedOracles.map((outcome) => outcome.oracleId).join(', ')} resolved unreached.`
+            : null,
+        comparable ? null : 'The vector is reported and marked non-comparable.',
+    ]
+        .filter((part) => part !== null)
+        .join(' ');
+    const vector = buildStrengthVector(scored.sealedProbes.admitted, new Map([[scored.probe.probeId, scored.trialSetResult]]));
+    // `scored.ladder.verdict` is `Verdict | null`: `null` is AD-21's Invalid
+    // rung, which "never becomes a contract verdict" -- a run reaching that
+    // rung is the caller's own signal to stop before minting an artifact at
+    // all, exactly as `scripts/worked-example-target.ts`'s own
+    // `if (ladder.verdict === null) fail(...)` guard, immediately before its
+    // call into this stage, already enforces. `emit` throws nothing for a
+    // domain input (the one throw this stage carries is the mode-agreement
+    // check below), so a verdict-less ladder reaching this point is a
+    // precondition violation the same way `seal()`'s own null-direction
+    // guard is one call earlier in the pipeline: trusted rather than
+    // re-checked a second time.
+    const verdict = scored.ladder.verdict;
+    const commonFields = {
+        schemaVersion: 3,
+        parentDigest: null,
+        // v0 mints no revision path for an evidence artifact: every `emit`
+        // call is a lineage root, matching `seal.ts`'s own root-artifact
+        // precedent for the one other `lineage: 'mints'` stage with a built
+        // module.
+        revisionCount: 0,
+        runId: scored.runId,
+        comparabilityKey,
+        excludedProbeIds,
+        exitCode: scored.ladder.exitCode,
+        verdictBasis: [...scored.ladder.basis],
+        callerAttestedInputs: [...CALLER_ATTESTED_INPUTS],
+        trials,
+        outcomes: [...scored.outcomes],
+        uncitedFindings: [...scored.uncitedFindings],
+        coverageGaps: [...scored.assessment.coverageGaps],
+        strength: {
+            denominator: `unique qualified probe identifiers exercised per class, across ${trialCount}`,
+            // `matchProbeWitness` is the only reachable producer of a
+            // `Strength.basis` value and it always returns `'measured'`;
+            // `'reconstructed'`'s one producer is dead code in v0. No live
+            // second value to select between, so this is a constant.
+            basis: 'measured',
+            vector,
+            comparable,
+            note: strengthNote,
+        },
+        remediation: {
+            // AD-12's REMEDIATION count, distinct from this artifact's own root
+            // `revisionCount` above; the two are named identically only because
+            // they nest at different depths.
+            revisionCount: scored.contract.revisionCount,
+            cap: scored.policy.remediationCap,
+            // AD-12 has the package only validate the cap; nothing enforces it,
+            // so there is no second legal value.
+            capSource: 'caller-attested',
+            // `remediationState` is already declared `LineageChain` (`ladder.ts`):
+            // reused from the assessment, never recomputed here.
+            lineageChain: scored.assessment.remediationState,
+        },
+    };
+    /**
+     * Runs both of this stage's checks, then freezes and returns. AD-29: a
+     * stage freezes the artifact it owns, matching `seal.ts`/
+     * `preflight/reduce.ts`'s own precedent for the other two minting stages.
+     * Both checks fire only on a precondition violation, never a domain
+     * input: neither is reachable through any path this module's own
+     * construction takes.
+     */
+    const finalize = (artifact) => {
+        const agreement = checkModeAgreement({ mode: scored.assessment.mode }, { mode: artifact.mode });
+        if (!agreement.agrees) {
+            // Reachable only through a type-system bypass: `artifact.mode` is
+            // stamped from `scored.assessment.mode` two lines above in every
+            // branch this function builds, so the two can disagree only if a
+            // caller assembled `scored` from two independently-sourced values
+            // -- the future caller this check exists for, mirroring
+            // `reduceTrialSet`'s own `TypeError`-for-bypass-only-input
+            // precedent.
+            throw new TypeError(`emit(): assembled an artifact whose mode ("${agreement.artifactMode}") disagrees with the assessment mode ("${agreement.recordMode}") it was built from`);
+        }
+        return freezeArtifact(validateAssembledArtifact(artifact));
+    };
+    if (scored.assessment.mode === 'production') {
+        const scoringVersionInputs = {
+            contractSchemaVersion: scored.contract.schemaVersion,
+            corpusDigest,
+            fixtureDigest,
+            evaluatorConfigurationDigest,
+            scoringPolicyDigest,
+            mode: 'production',
+        };
+        const artifact = {
+            ...commonFields,
+            mode: 'production',
+            scoringVersionInputs,
+            scoringVersion: digestArtifact(scoringVersionInputs, SCORING_VERSION_INPUTS_ARTIFACT_PATH),
+            productionVerdict: verdict,
+        };
+        return finalize(artifact);
+    }
+    const scoringVersionInputs = {
+        contractSchemaVersion: scored.contract.schemaVersion,
+        corpusDigest,
+        fixtureDigest,
+        evaluatorConfigurationDigest,
+        scoringPolicyDigest,
+        mode: 'contract-scoring',
+    };
+    const artifact = {
+        ...commonFields,
+        mode: 'contract-scoring',
+        scoringVersionInputs,
+        scoringVersion: digestArtifact(scoringVersionInputs, SCORING_VERSION_INPUTS_ARTIFACT_PATH),
+        contractVerdict: verdict,
+        uncitedFindingGaps: [...scored.assessment.uncitedDefectFindings],
+        systemRecommendationRecorded: scored.assessment.systemRecommendationRecorded,
+        systemRecommendationNote: scored.assessment.systemRecommendationNote,
+    };
+    return finalize(artifact);
+};
+/**
+ * The runtime backstop `finalize()` calls before freezing: TypeScript
+ * guarantees this module's own object literals match `EvidenceArtifact`'s
+ * shape at compile time, except at the one place that casts past it
+ * (`scored.ladder.verdict as Verdict`, above). Mirrors `seal.ts`'s own
+ * `validateAssembledBrief` -- same `safeParse`-then-`TypeError` shape, same
+ * first-issue diagnosis -- so a verdict that reached this function `null`
+ * despite the caller's own precondition fails loudly here instead of
+ * shipping a schema-invalid artifact silently.
+ */
+function validateAssembledArtifact(artifact) {
+    const result = EvidenceArtifact.safeParse(artifact);
+    if (result.success)
+        return result.data;
+    const issueCount = result.error.issues.length;
+    const firstIssue = result.error.issues[0];
+    const firstPath = firstIssue ? dotPath(firstIssue.path) : '';
+    throw new TypeError(`emit(): assembled an artifact that failed EvidenceArtifact validation: ${issueCount} issue${issueCount === 1 ? '' : 's'}, first at "${firstPath === '' ? '(root)' : firstPath}"`, { cause: result.error });
+}
+/** `seal.ts`'s own formatter, duplicated rather than shared: both are small, stage-local, and neither exports it today. */
+function dotPath(path) {
+    const segments = [];
+    for (const segment of path) {
+        if (typeof segment === 'number')
+            segments.push(`[${segment}]`);
+        else if (typeof segment === 'symbol' || /[^\w$]/.test(segment)) {
+            segments.push(`[${JSON.stringify(String(segment))}]`);
+        }
+        else {
+            if (segments.length > 0)
+                segments.push('.');
+            segments.push(segment);
+        }
+    }
+    return segments.join('');
+}

package/dist/core/emit/private-artifact-digest.d.ts

@@ -0,0 +1,11 @@
+import type { PrivateArtifactManifest } from '../schemas/private-artifact-manifest.ts';
+/**
+ * Throws `RuntimeFault('digest-mismatch', ...)` on the first
+ * `entries[i]` whose declared `digest` disagrees with
+ * `resolvedDigests.get(entry.privateRef)`. An entry with no resolved digest
+ * at all -- `resolvedDigests` carries no key for its `privateRef` -- is the
+ * same disagreement: there is no byte the manifest's declared digest can be
+ * checked against, which this comparator treats no differently from a
+ * genuine mismatch.
+ */
+export declare function checkPrivateArtifactManifestDigests(manifest: PrivateArtifactManifest, resolvedDigests: ReadonlyMap<string, string>): void;

package/dist/core/emit/private-artifact-digest.js

@@ -0,0 +1,31 @@
+/**
+ * The pure half of `private-artifact-manifest.ts`'s own "a mismatch is an
+ * AD-28 `digest-mismatch` fault": given a manifest and each entry's already-
+ * resolved digest, throws on the first disagreement. AD-8 requires the
+ * digest recomputed from the resolved bytes; resolving those bytes needs
+ * `CorpusPort.resolve`, an async port method, and AD-34 makes awaiting a
+ * port `application/`'s job, never `core/`'s. This module is the comparator
+ * alone and has no caller yet: whoever first awaits `CorpusPort.resolve` to
+ * build `resolvedDigests` is this function's caller, and that await belongs
+ * in `application/`, not here.
+ */
+import { RuntimeFault } from '../schemas/faults.js';
+/**
+ * Throws `RuntimeFault('digest-mismatch', ...)` on the first
+ * `entries[i]` whose declared `digest` disagrees with
+ * `resolvedDigests.get(entry.privateRef)`. An entry with no resolved digest
+ * at all -- `resolvedDigests` carries no key for its `privateRef` -- is the
+ * same disagreement: there is no byte the manifest's declared digest can be
+ * checked against, which this comparator treats no differently from a
+ * genuine mismatch.
+ */
+export function checkPrivateArtifactManifestDigests(manifest, resolvedDigests) {
+    manifest.entries.forEach((entry, index) => {
+        const resolved = resolvedDigests.get(entry.privateRef);
+        if (resolved === entry.digest)
+            return;
+        throw new RuntimeFault('digest-mismatch', `PrivateArtifactManifest.entries[${index}]`, `entry privateRef "${entry.privateRef}" declares digest "${entry.digest}", but the resolved bytes digest to ${resolved === undefined
+            ? 'nothing (no resolved digest supplied for this privateRef)'
+            : `"${resolved}"`}`);
+    });
+}

package/dist/core/evaluate/evidence-resolution.d.ts

@@ -8,8 +8,8 @@
 import type { EvalContract } from '../schemas/eval-contract.ts';
 import type { JsonValue } from '../schemas/primitives.ts';
 import type { Observation } from '../schemas/sealed-run-record.ts';
-import { type PlanIndex } from '../seal/plan-index.ts';
-import type { PointerDenotesCollection, ResolveOperand } from './resolution.ts';
+import { type EvidenceTarget, type PlanIndex } from '../seal/plan-index.ts';
+import type { PointerDenotesCollection, ReferenceSetKeys, ResolveOperand } from './resolution.ts';
 import { type ResolvedValue } from './resolved-value.ts';
 /**
  * A canonical RFC 6901 array-index token: no leading zero except "0" itself,
@@ -26,7 +26,7 @@ export declare const ARRAY_INDEX_PATTERN: RegExp;
  * falls through to `Object.prototype`. Any miss, type mismatch, or tail
  * running past a scalar collapses to `ABSENT` uniformly (AD-26).
  */
-export declare function walkTail(root: JsonValue, tail: readonly string[]): ResolvedValue;
+export declare function walkTail(root: ResolvedValue, tail: readonly string[]): ResolvedValue;
 /**
  * Decodes `BoundElementPointer`'s tail. "@/" is the shortest legal pointer
  * form and denotes the element itself. `decodeTail` special-cases only a
@@ -34,6 +34,7 @@ export declare function walkTail(root: JsonValue, tail: readonly string[]): Reso
  * calling it.
  */
 export declare function decodeBoundElementTail(pointer: string): readonly string[];
+export declare function channelRoot(observation: Observation, target: EvidenceTarget): ResolvedValue;
 /**
  * The `ResolveOperand`. `stepObservations` holds one already-selected
  * `Observation` per interaction step; `referenceSets` mirrors the contract's
@@ -42,6 +43,12 @@ export declare function decodeBoundElementTail(pointer: string): readonly string
  * prototype-chain gotcha `walkTail` guards against.
  */
 export declare function makeResolveOperand(stepObservations: Readonly<Record<string, Observation>>, referenceSets: Readonly<Record<string, JsonValue[]>>): ResolveOperand;
+/**
+ * The `keys` a contract's reference sets declare. Built from the same
+ * `contract.referenceSets` the members map handed to `makeResolveOperand` is
+ * built from, so a set that resolves to members always has its keys here.
+ */
+export declare function referenceSetKeysOf(contract: EvalContract): ReferenceSetKeys;
 /**
  * Only `response-body` can ever answer `true` (AD-19: `collectionLocations`
  * is the only declared-collection surface, scoped to the body alone). The

package/dist/core/evaluate/evidence-resolution.js

@@ -5,7 +5,8 @@
  * bound-element `@/` form, and walk it into an already-selected
  * `Observation`.
  */
-import { buildPlanIndex, decodeTail, parseEvidenceTarget, } from '../seal/plan-index.js';
+import { targetsDescribedChannel } from '../declared-inputs.js';
+import { anyOperationOf, buildPlanIndex, decodeTail, parseEvidenceTarget, } from '../seal/plan-index.js';
 import { ABSENT } from './resolved-value.js';
 /**
  * A canonical RFC 6901 array-index token: no leading zero except "0" itself,
@@ -23,6 +24,12 @@ export const ARRAY_INDEX_PATTERN = /^(?:0|[1-9][0-9]*)$/;
  * running past a scalar collapses to `ABSENT` uniformly (AD-26).
  */
 export function walkTail(root, tail) {
+    // A root that did not resolve stays unresolved however short the tail is.
+    // The bare-pointer case is the one that matters: without this, an absent
+    // channel with no tail returned its own root and a caller could not tell it
+    // from a resolved value.
+    if (root === ABSENT)
+        return ABSENT;
     let current = root;
     for (const token of tail) {
         if (current === null || typeof current !== 'object')
@@ -53,12 +60,30 @@ export function decodeBoundElementTail(pointer) {
     return tailSource === '/' ? [] : decodeTail(tailSource);
 }
 /**
- * Selects the channel `target` names off one `Observation`. `stdout`/`stderr`
- * can carry a tail even though they're bare strings; `walkTail` already
- * resolves any non-empty tail against a string to `ABSENT`, so no special
- * case is needed here.
+ * Selects the channel `target` names off one `Observation`.
+ *
+ * `stdout`, `stderr`, and each written artifact are tagged, so a channel a
+ * harness captured as JSON resolves to the value and a tail walks into it,
+ * while one captured as text resolves to the string and a tail over it
+ * resolves `ABSENT`. That is what makes a command operation's nominated output
+ * channel resolvable at score time on the same terms as a response body.
+ *
+ * Exported so `core/score/bindings.ts` reads a captured pointer's channel off
+ * an observation through this one spelling, keeping the switch in one place.
  */
-function channelRoot(observation, target) {
+/**
+ * The value a tagged observed channel resolves to.
+ *
+ * A JSON body resolves to the value itself, so a tail walks into it; text
+ * resolves to the string, so a tail over it resolves `ABSENT` through
+ * `walkTail`, which is the truthful answer for output nothing parsed. An absent
+ * channel resolves to `ABSENT` and never to `null`: the tag exists precisely to
+ * tell "nothing was observed here" from "a value that was JSON null", and
+ * collapsing the two would make `existence` answer true for a file the run
+ * never wrote, since AD-26 counts `null` as present.
+ */
+const observedValue = (body) => body.kind === 'absent' ? ABSENT : body.value;
+export function channelRoot(observation, target) {
     switch (target.channel) {
         case 'response-body':
             return observation.responseBody;
@@ -67,9 +92,9 @@ function channelRoot(observation, target) {
         case 'response-status':
             return observation.responseStatus;
         case 'stdout':
-            return observation.stdout;
+            return observedValue(observation.stdout);
         case 'stderr':
-            return observation.stderr;
+            return observedValue(observation.stderr);
         case 'exit-code':
             return observation.exitCode;
         case 'call-inputs': {
@@ -81,6 +106,27 @@ function channelRoot(observation, target) {
             }
             return observation.callInputs[transportChannel];
         }
+        case 'artifact': {
+            const { artifactId } = target;
+            if (artifactId === null) {
+                // parseEvidenceTarget sets artifactId exactly when the channel
+                // is 'artifact', so this throw should never fire.
+                throw new TypeError('artifact evidence target names no artifact');
+            }
+            // `Object.hasOwn` for the reason this module's header gives for
+            // `stepObservations` and `referenceSets`: `Identifier` admits
+            // `constructor`, `toString` and `valueOf`, so a bare index on an
+            // unwritten file of one of those names resolves to an inherited
+            // function rather than to nothing.
+            //
+            // A file the run did not write is absent evidence, not a `null`
+            // value. `null` would read as present under AD-26 and invert every
+            // oracle asserting the file exists or does not.
+            if (!Object.hasOwn(observation.artifacts, artifactId))
+                return ABSENT;
+            const written = observation.artifacts[artifactId];
+            return written === undefined ? ABSENT : observedValue(written);
+        }
     }
 }
 /**
@@ -115,6 +161,17 @@ export function makeResolveOperand(stepObservations, referenceSets) {
         return walkTail(channelRoot(observation, target), target.tail);
     };
 }
+/**
+ * The `keys` a contract's reference sets declare. Built from the same
+ * `contract.referenceSets` the members map handed to `makeResolveOperand` is
+ * built from, so a set that resolves to members always has its keys here.
+ */
+export function referenceSetKeysOf(contract) {
+    return Object.fromEntries(Object.entries(contract.referenceSets ?? {}).map(([id, set]) => [
+        id,
+        set.keys,
+    ]));
+}
 function tokensEqual(a, b) {
     return a.length === b.length && a.every((token, index) => token === b[index]);
 }
@@ -137,14 +194,20 @@ export function makePointerDenotesCollection(contract, providedIndex) {
         if (pointer.startsWith('@'))
             return false;
         const target = parseEvidenceTarget(pointer);
-        if (target.channel !== 'response-body')
-            return false;
         const step = getIndex().stepOf(target.stepId);
         if (step === undefined)
             return false;
-        const operation = getIndex().operationOf(step.operationId);
+        const operation = anyOperationOf(getIndex(), step.operationId);
         if (operation === undefined)
             return false;
+        // The channel is tested against the operation's own descriptor rather
+        // than against `response-body`. AD-4's empty-collection resolution
+        // applies to whichever channel an operation says carries its declared
+        // collections, and hard-coding the body left it inapplicable to every
+        // command contract: a quantifier over an empty declared collection
+        // resolved `false` instead of `insufficient-evidence`.
+        if (!targetsDescribedChannel(operation, target))
+            return false;
         const { collectionLocations } = operation.responseDescriptor;
         if (collectionLocations === null)
             return false;

package/dist/core/evaluate/operators.d.ts

@@ -31,6 +31,18 @@ export declare function setMembership(value: ResolvedValue, set: JsonValue[], ar
  * single element to search for: this function receives resolved values only,
  * so it cannot tell a `{ literal }` array from a resolved `{ referenceSet }`.
  * Known, accepted limitation.
+ *
+ * A separate property, and a separate limitation: a `{ referenceSet }`
+ * candidate matches **whole declared members** by structural equality. A
+ * container element carrying any field the reference set does not declare
+ * therefore never matches, which is the ordinary case for rows read off a
+ * response body. `set-membership`'s set position projects a reference set to
+ * its single declared key and this position does not, because projecting only
+ * where a container happens to hold objects would make the operand's meaning
+ * depend on the evidence's runtime shape. An author reconciling rows against a
+ * declared set wants `covers-by-key`, which compares on named keys on both
+ * sides, or `for-all(rows, set-membership(@/key, { referenceSet }))`, which
+ * reads the declared key.
  */
 export declare function containment(container: ResolvedValue, candidate: ResolvedValue | JsonValue[], artifactPath: string): boolean;
 /**
@@ -62,6 +74,16 @@ export declare function countTolerance(collection: ResolvedValue, expected: numb
  * that is required and not permitted reaches here and fails every value.
  */
 export declare function shape(value: ResolvedValue, descriptor: KeyedShapeDescriptor, _artifactPath: string): boolean;
+/**
+ * Own-property lookup only, so a key like `__proto__` reads as missing
+ * rather than inherited. Returns `ABSENT`, never throws (AD-4: a missing key
+ * resolves `false`, not an error).
+ *
+ * Exported so `resolution.ts`'s `set-membership` set-operand projection reads
+ * a declared member's key through this one spelling, keeping the own-property
+ * guard in one place.
+ */
+export declare function keyValueOf(element: JsonValue, key: string): JsonValue | typeof ABSENT;
 /**
  * AD-4's bijection: equal cardinality and a distinct `actual` match per
  * `expected` element on the named keys. `ABSENT` on either side resolves

package/dist/core/evaluate/operators.js

@@ -90,6 +90,18 @@ export function setMembership(value, set, artifactPath) {
  * single element to search for: this function receives resolved values only,
  * so it cannot tell a `{ literal }` array from a resolved `{ referenceSet }`.
  * Known, accepted limitation.
+ *
+ * A separate property, and a separate limitation: a `{ referenceSet }`
+ * candidate matches **whole declared members** by structural equality. A
+ * container element carrying any field the reference set does not declare
+ * therefore never matches, which is the ordinary case for rows read off a
+ * response body. `set-membership`'s set position projects a reference set to
+ * its single declared key and this position does not, because projecting only
+ * where a container happens to hold objects would make the operand's meaning
+ * depend on the evidence's runtime shape. An author reconciling rows against a
+ * declared set wants `covers-by-key`, which compares on named keys on both
+ * sides, or `for-all(rows, set-membership(@/key, { referenceSet }))`, which
+ * reads the declared key.
  */
 export function containment(container, candidate, artifactPath) {
     if (container === ABSENT)
@@ -287,8 +299,12 @@ export function shape(value, descriptor, _artifactPath) {
  * Own-property lookup only, so a key like `__proto__` reads as missing
  * rather than inherited. Returns `ABSENT`, never throws (AD-4: a missing key
  * resolves `false`, not an error).
+ *
+ * Exported so `resolution.ts`'s `set-membership` set-operand projection reads
+ * a declared member's key through this one spelling, keeping the own-property
+ * guard in one place.
  */
-function keyValueOf(element, key) {
+export function keyValueOf(element, key) {
     if (!isPlainObject(element) || !Object.hasOwn(element, key))
         return ABSENT;
     // `Object.hasOwn` above already proves the key is present; `noUncheckedIndexedAccess`