@adcp/sdk 14.0.0-rc.38 → 14.0.0-rc.39

package/dist/lib/testing/storyboard/types.d.mts

@@ -29,7 +29,7 @@ export interface Storyboard {
     version: string;
     /**
      * AdCP compliance cache version this storyboard was loaded from, e.g.
-     * "3.0.12" or "3.1.0-beta.7". Injected by the local cache loader; not
+     * "3.1.18" or "3.2.0-rc.3". Injected by the local cache loader; not
      * authored in storyboard YAML.
      */
     adcp_version?: string;
@@ -897,7 +897,22 @@ export type StoryboardValidationCheck = 'response_schema' | 'field_present' | 'f
  * envelope fields such as `adcp_major_version` should use
  * `envelope_field_value`.
  */
- | 'envelope_field_pattern' | 'field_contains' | 'status_code' | 'error_code' | 'http_status' | 'http_status_in' | 'on_401_require_header' | 'resource_equals_agent_url' | 'oauth_metadata_graph' | 'any_of' | 'a2a_submitted_artifact' | 'a2a_context_continuity' | 'refs_resolve'
+ | 'envelope_field_pattern' | 'field_contains' | 'status_code' | 'error_code' | 'http_status' | 'http_status_in' | 'on_401_require_header'
+/**
+ * Assert that the step's probe reported no grading error, without asserting
+ * anything about an HTTP exchange. For probes whose verdict is decided
+ * in-library rather than on the wire — the `jwks_override` request-signing
+ * negatives, which mutate a JWK the agent under test never publishes and so
+ * are graded against the SDK verifier at `http_status: 0` — this is the only
+ * honest assertion: synthesizing a 401 from a library boolean would claim a
+ * wire exchange that never happened (RFC 9421 §3; adcp-client#2955).
+ *
+ * Passes when the probe result carries no `error`; fails with the probe's
+ * own diagnostic otherwise. Like the `http_status` family it requires a
+ * probe result, so authoring it on an ordinary task step fails the step
+ * rather than passing vacuously.
+ */
+ | 'probe_passed' | 'resource_equals_agent_url' | 'oauth_metadata_graph' | 'any_of' | 'a2a_submitted_artifact' | 'a2a_context_continuity' | 'refs_resolve'
 /**
  * Assert a numeric field in the current step's response is strictly less than
  * a comparand. The comparand is either a context-captured runtime value
@@ -1362,6 +1377,44 @@ export type WebhookAssertionErrorCode = 'no_webhook_received' | 'schema_violatio
  * (tests, tooling) can reproduce the grader's exact validation.
  */
 export declare const WEBHOOK_IDEMPOTENCY_KEY_PATTERN: RegExp;
+/**
+ * Runner-native sentinel task for the MCP session auth probe
+ * (adcp-client#2940).
+ *
+ * `$test_kit.auth.probe_task` normally resolves to an advertised entry of
+ * `PROBE_TASK_ALLOWLIST`. When MCP discovery succeeds but the agent advertises
+ * none of them, `selectProbeTask` resolves to this sentinel instead of
+ * `undefined` — but only on an explicit `protocol: 'mcp'`. The runner then
+ * drives a complete MCP session lifecycle: `initialize` →
+ * `notifications/initialized` → `tools/call` against a canonical AdCP
+ * protected read the agent advertises, then `DELETE` to terminate the session.
+ *
+ * The graded call is `tools/call`, not MCP discovery: `tools/list` is not an
+ * AdCP protected task (`get_adcp_capabilities` is mandatory-public), so it can
+ * never be evidence about this agent's authentication. Grading a real
+ * protected operation — rather than stopping at the handshake — is also what
+ * makes the verdict independent of whether the agent enforces auth at the
+ * session boundary or per operation.
+ *
+ * The target is drawn from the canonical AdCP tool registries, excludes the
+ * public tier and mutating tasks, and must declare no required request field.
+ * The runner still cannot guarantee an agent accepts an empty-argument call:
+ * the next candidate is tried, and if every one refuses the shape the step
+ * reports *inconclusive* on its response rather than an auth result. When the
+ * agent advertises no eligible target at all, the step is skipped
+ * `session_probe_ungradable` instead.
+ *
+ * What the probe can establish is bounded: that the endpoint accepts a
+ * credential the *runner was configured with* and refuses the states it was
+ * told to refuse, **at the tool it graded**. It is evidence that the auth
+ * mechanism is enforced, not proof that every tool enforces it. It does not
+ * verify any cryptographic property of that credential, and does not prove
+ * which issuer minted it.
+ *
+ * Deliberately **not** a member of `PROBE_TASK_ALLOWLIST`: operators cannot
+ * select it via `test_kit.auth.probe_task`, only the runner can resolve to it.
+ */
+export declare const MCP_SESSION_PROBE_TASK = "mcp_session_probe";
 /**
  * Raw HTTP probe result for tasks like `protected_resource_metadata` that
  * bypass the MCP transport. Carried through the runner alongside
@@ -1377,6 +1430,18 @@ export interface HttpProbeResult {
     body: unknown;
     /** Optional error — set when the fetch failed (network, SSRF guard, etc.). */
     error?: string;
+    /**
+     * The probe could not complete for a reason the runner owns — a transport
+     * or precondition failure (MCP initialize, DNS, an unreadable fixture
+     * cache), as opposed to a verdict the probe reached.
+     *
+     * Needed because both shapes report `status: 0` with `error` set: an
+     * in-library request-signing grade that failed is a real verdict about the
+     * agent's expected behavior, while a failed MCP handshake says only that
+     * the run never got far enough to ask. Coverage reporting has to tell an
+     * operator which of the two happened, and the remedies are opposites.
+     */
+    probe_error?: boolean;
     /**
      * Probe was intentionally skipped (e.g. operator opted out of a vector,
      * capability profile mismatch, or test-kit contract not in scope). When
@@ -1493,8 +1558,8 @@ export interface StoryboardRunOptions extends TestOptions {
      *
      * Reuse a profile only for the same agent URL, authentication, AdCP version,
      * and route that produced it. Route-specific profiles are not interchangeable.
-     * In an `agents` run this value supplies only the run-level/default-agent
-     * gating context; each routed agent is still discovered independently.
+     * Ignored in an `agents` run: each routed agent is discovered independently,
+     * and its profile governs the steps selected for that agent.
      * Profile reuse does not retain or reuse a client or transport connection.
      *
      * `AgentProfile` does not carry the server's exact wire version. Pass
@@ -1530,7 +1595,11 @@ export interface StoryboardRunOptions extends TestOptions {
     contributions?: string[];
     /** Override the step's sample_request with a custom request */
     request?: Record<string, unknown>;
-    /** Agent's available tools for storyboard/step-level tool gates. */
+    /**
+     * Agent's available tools for storyboard/step-level tool gates. Ignored when
+     * `agents` is supplied: routed discovery determines applicability and each
+     * selected agent's tool list controls its execution gates.
+     */
     agentTools?: string[];
     /**
      * Allow plain-http agent URLs during compliance runs and permit guarded
@@ -1587,15 +1656,35 @@ export interface StoryboardRunOptions extends TestOptions {
          *     operation AdCP endpoint (e.g. `<baseUrl>/create_media_buy`).
          *     Works for agents that expose AdCP tools as discrete HTTP
          *     operations.
-         *   - `mcp` (default) — wraps each vector body in a JSON-RPC `tools/call`
+         *   - `mcp` — wraps each vector body in a JSON-RPC `tools/call`
          *     envelope and POSTs to the agent's single `/mcp` mount. Required
          *     for MCP-only agents that don't expose per-operation endpoints.
          *     The operation name is derived from the last path segment of the
          *     vector's target URL.
          *
-         * Matches the `adcp grade request-signing --transport <mode>` CLI flag.
-         * Agents that only speak MCP JSON-RPC can't grade under `raw`; use
-         * `mcp` to let the runner round-trip every vector through `tools/call`.
+         * Defaults to the shape the run's `protocol` can actually grade: `mcp`
+         * on an MCP run, and neither shape on an A2A run — this runner
+         * implements only those two dispatches, and neither is an A2A request.
+         * Vectors that need a wire exchange then skip as
+         * `signing_transport_unavailable` — and because no vector reached the
+         * agent, the storyboard cannot report a pass and the track grades
+         * `partial` rather than the run being scored against a transport error
+         * (adcp-client#2954). Only the
+         * in-library `jwks_override` negatives still run there: the grader
+         * decides them against the library verifier with no wire exchange at
+         * all, so the run's protocol is irrelevant to them. Every other probed
+         * vector reports the coverage gap, the protocol-method negatives
+         * included — the official `@a2a-js/sdk` client can issue those methods,
+         * but this runner does not yet sign and dispatch a vector through it,
+         * and writing the fixture bytes to the endpoint directly would be a
+         * hand-rolled A2A dispatch. Setting this field explicitly overrides the
+         * inference on any protocol; on an A2A run, do that only when the
+         * agent's MCP or REST binding answers at the same URL.
+         *
+         * Matches the `adcp grade request-signing --transport <mode>` CLI flag,
+         * and `adcp storyboard run --signing-transport <mode>`. Agents that only
+         * speak MCP JSON-RPC can't grade under `raw`; use `mcp` to let the runner
+         * round-trip every vector through `tools/call`.
          */
         transport?: 'raw' | 'mcp';
         /**
@@ -1640,6 +1729,12 @@ export interface StoryboardRunOptions extends TestOptions {
      * (`StoryboardStep.task`) is resolved to the agent that claims its
      * specialism via `TASK_FEATURE_MAP` × per-agent `get_adcp_capabilities`.
      * An optional explicit `StoryboardStep.agent` override takes precedence.
+     * Discovered tools across the map determine storyboard-level any-of
+     * applicability. Each dispatched step uses its selected agent's tools
+     * and profile; run-level `agentTools` cannot override routed discovery.
+     * Per-entry authentication and transport override caller-supplied run
+     * defaults. Other run options, including headers and request signing,
+     * remain shared defaults across the map.
      *
      * Mutually exclusive with `multi_instance_strategy` (which is replica
      * round-robin, a different concept) and with the legacy `_client`
@@ -1663,7 +1758,7 @@ export interface StoryboardRunOptions extends TestOptions {
     agents?: Record<string, AgentEntry>;
     /**
      * Fallback agent key (must be present in `agents`) for tasks with no entry
-     * in `TASK_FEATURE_MAP` — e.g., `comply_test_controller`, future tasks
+     * in `TASK_FEATURE_MAP` — e.g., `sync_creatives`, future tasks
      * shipped before the SDK adds them to the map. When omitted, unmapped
      * tasks fail-fast with `unroutable_task`.
      */
@@ -1930,6 +2025,26 @@ export type RunnerDetailedSkipReason = 'probe_skipped' | 'rate_abuse_opt_out' |
  | 'capability_profile_mismatch'
 /** Request-signing vector cannot be graded faithfully by the selected transport. */
  | 'transport_ungradable'
+/**
+ * The run's protocol has no shape the request-signing vectors can be
+ * framed in, so the runner could not grade them at all — a runner-owned
+ * coverage gap, not a property of the agent.
+ *
+ * Canonicalizes to `not_applicable` with `skip.detail` set to this exact
+ * token, the shape `runner-output-contract.yaml` defines for a registered
+ * `canonical_detail_sub_reasons.not_applicable.*` sub-reason (see
+ * `rate_limit_not_triggered`, the shipped precedent). It deliberately does
+ * NOT use `fixture_unavailable`: that reason's contract text mandates a
+ * `creative_asset_fixture_unavailable:` detail prefix and says the
+ * storyboard grades `not_applicable` with no verdict movement, which is
+ * the opposite of what this gap must do.
+ *
+ * The gap is kept visible by coverage, not by the reason: `signingCoverage`
+ * in the runner decides the storyboard verdict, the track rollup, the
+ * report block and the CLI exit from whether any probe reached the agent.
+ * See that function for why a skip reason cannot carry that weight.
+ */
+ | 'signing_transport_unavailable'
 /** Request-signing grader's MCP-transport mode collapses URL-edge vectors (#617). */
  | 'mcp_mode_flattens_url_edges'
 /** RFC 9728 protected-resource metadata returned 404 → agent is not advertising OAuth, cascade-skip oauth_discovery (#677). */
@@ -1955,6 +2070,13 @@ export type RunnerDetailedSkipReason = 'probe_skipped' | 'rate_abuse_opt_out' |
  | 'fixture_seed_unsupported'
 /** A valid fixture strategy ladder exhausted without finding a binding. */
  | 'fixture_unsatisfied'
+/**
+ * The MCP session auth probe (`mcp_session_probe`) cannot grade this step:
+ * its authored validations assert an AdCP task response body that no MCP
+ * protocol operation produces. Distinct from the generic `probe_skipped` so
+ * human and JUnit output can surface the actionable detail (adcp-client#2940).
+ */
+ | 'session_probe_ungradable'
 /**
  * A root capability predicate on the storyboard evaluated to false —
  * the agent explicitly declared it does not support the capability this

package/dist/lib/testing/storyboard/types.d.ts

@@ -29,7 +29,7 @@ export interface Storyboard {
     version: string;
     /**
      * AdCP compliance cache version this storyboard was loaded from, e.g.
-     * "3.0.12" or "3.1.0-beta.7". Injected by the local cache loader; not
+     * "3.1.18" or "3.2.0-rc.3". Injected by the local cache loader; not
      * authored in storyboard YAML.
      */
     adcp_version?: string;
@@ -897,7 +897,22 @@ export type StoryboardValidationCheck = 'response_schema' | 'field_present' | 'f
  * envelope fields such as `adcp_major_version` should use
  * `envelope_field_value`.
  */
- | 'envelope_field_pattern' | 'field_contains' | 'status_code' | 'error_code' | 'http_status' | 'http_status_in' | 'on_401_require_header' | 'resource_equals_agent_url' | 'oauth_metadata_graph' | 'any_of' | 'a2a_submitted_artifact' | 'a2a_context_continuity' | 'refs_resolve'
+ | 'envelope_field_pattern' | 'field_contains' | 'status_code' | 'error_code' | 'http_status' | 'http_status_in' | 'on_401_require_header'
+/**
+ * Assert that the step's probe reported no grading error, without asserting
+ * anything about an HTTP exchange. For probes whose verdict is decided
+ * in-library rather than on the wire — the `jwks_override` request-signing
+ * negatives, which mutate a JWK the agent under test never publishes and so
+ * are graded against the SDK verifier at `http_status: 0` — this is the only
+ * honest assertion: synthesizing a 401 from a library boolean would claim a
+ * wire exchange that never happened (RFC 9421 §3; adcp-client#2955).
+ *
+ * Passes when the probe result carries no `error`; fails with the probe's
+ * own diagnostic otherwise. Like the `http_status` family it requires a
+ * probe result, so authoring it on an ordinary task step fails the step
+ * rather than passing vacuously.
+ */
+ | 'probe_passed' | 'resource_equals_agent_url' | 'oauth_metadata_graph' | 'any_of' | 'a2a_submitted_artifact' | 'a2a_context_continuity' | 'refs_resolve'
 /**
  * Assert a numeric field in the current step's response is strictly less than
  * a comparand. The comparand is either a context-captured runtime value
@@ -1362,6 +1377,44 @@ export type WebhookAssertionErrorCode = 'no_webhook_received' | 'schema_violatio
  * (tests, tooling) can reproduce the grader's exact validation.
  */
 export declare const WEBHOOK_IDEMPOTENCY_KEY_PATTERN: RegExp;
+/**
+ * Runner-native sentinel task for the MCP session auth probe
+ * (adcp-client#2940).
+ *
+ * `$test_kit.auth.probe_task` normally resolves to an advertised entry of
+ * `PROBE_TASK_ALLOWLIST`. When MCP discovery succeeds but the agent advertises
+ * none of them, `selectProbeTask` resolves to this sentinel instead of
+ * `undefined` — but only on an explicit `protocol: 'mcp'`. The runner then
+ * drives a complete MCP session lifecycle: `initialize` →
+ * `notifications/initialized` → `tools/call` against a canonical AdCP
+ * protected read the agent advertises, then `DELETE` to terminate the session.
+ *
+ * The graded call is `tools/call`, not MCP discovery: `tools/list` is not an
+ * AdCP protected task (`get_adcp_capabilities` is mandatory-public), so it can
+ * never be evidence about this agent's authentication. Grading a real
+ * protected operation — rather than stopping at the handshake — is also what
+ * makes the verdict independent of whether the agent enforces auth at the
+ * session boundary or per operation.
+ *
+ * The target is drawn from the canonical AdCP tool registries, excludes the
+ * public tier and mutating tasks, and must declare no required request field.
+ * The runner still cannot guarantee an agent accepts an empty-argument call:
+ * the next candidate is tried, and if every one refuses the shape the step
+ * reports *inconclusive* on its response rather than an auth result. When the
+ * agent advertises no eligible target at all, the step is skipped
+ * `session_probe_ungradable` instead.
+ *
+ * What the probe can establish is bounded: that the endpoint accepts a
+ * credential the *runner was configured with* and refuses the states it was
+ * told to refuse, **at the tool it graded**. It is evidence that the auth
+ * mechanism is enforced, not proof that every tool enforces it. It does not
+ * verify any cryptographic property of that credential, and does not prove
+ * which issuer minted it.
+ *
+ * Deliberately **not** a member of `PROBE_TASK_ALLOWLIST`: operators cannot
+ * select it via `test_kit.auth.probe_task`, only the runner can resolve to it.
+ */
+export declare const MCP_SESSION_PROBE_TASK = "mcp_session_probe";
 /**
  * Raw HTTP probe result for tasks like `protected_resource_metadata` that
  * bypass the MCP transport. Carried through the runner alongside
@@ -1377,6 +1430,18 @@ export interface HttpProbeResult {
     body: unknown;
     /** Optional error — set when the fetch failed (network, SSRF guard, etc.). */
     error?: string;
+    /**
+     * The probe could not complete for a reason the runner owns — a transport
+     * or precondition failure (MCP initialize, DNS, an unreadable fixture
+     * cache), as opposed to a verdict the probe reached.
+     *
+     * Needed because both shapes report `status: 0` with `error` set: an
+     * in-library request-signing grade that failed is a real verdict about the
+     * agent's expected behavior, while a failed MCP handshake says only that
+     * the run never got far enough to ask. Coverage reporting has to tell an
+     * operator which of the two happened, and the remedies are opposites.
+     */
+    probe_error?: boolean;
     /**
      * Probe was intentionally skipped (e.g. operator opted out of a vector,
      * capability profile mismatch, or test-kit contract not in scope). When
@@ -1493,8 +1558,8 @@ export interface StoryboardRunOptions extends TestOptions {
      *
      * Reuse a profile only for the same agent URL, authentication, AdCP version,
      * and route that produced it. Route-specific profiles are not interchangeable.
-     * In an `agents` run this value supplies only the run-level/default-agent
-     * gating context; each routed agent is still discovered independently.
+     * Ignored in an `agents` run: each routed agent is discovered independently,
+     * and its profile governs the steps selected for that agent.
      * Profile reuse does not retain or reuse a client or transport connection.
      *
      * `AgentProfile` does not carry the server's exact wire version. Pass
@@ -1530,7 +1595,11 @@ export interface StoryboardRunOptions extends TestOptions {
     contributions?: string[];
     /** Override the step's sample_request with a custom request */
     request?: Record<string, unknown>;
-    /** Agent's available tools for storyboard/step-level tool gates. */
+    /**
+     * Agent's available tools for storyboard/step-level tool gates. Ignored when
+     * `agents` is supplied: routed discovery determines applicability and each
+     * selected agent's tool list controls its execution gates.
+     */
     agentTools?: string[];
     /**
      * Allow plain-http agent URLs during compliance runs and permit guarded
@@ -1587,15 +1656,35 @@ export interface StoryboardRunOptions extends TestOptions {
          *     operation AdCP endpoint (e.g. `<baseUrl>/create_media_buy`).
          *     Works for agents that expose AdCP tools as discrete HTTP
          *     operations.
-         *   - `mcp` (default) — wraps each vector body in a JSON-RPC `tools/call`
+         *   - `mcp` — wraps each vector body in a JSON-RPC `tools/call`
          *     envelope and POSTs to the agent's single `/mcp` mount. Required
          *     for MCP-only agents that don't expose per-operation endpoints.
          *     The operation name is derived from the last path segment of the
          *     vector's target URL.
          *
-         * Matches the `adcp grade request-signing --transport <mode>` CLI flag.
-         * Agents that only speak MCP JSON-RPC can't grade under `raw`; use
-         * `mcp` to let the runner round-trip every vector through `tools/call`.
+         * Defaults to the shape the run's `protocol` can actually grade: `mcp`
+         * on an MCP run, and neither shape on an A2A run — this runner
+         * implements only those two dispatches, and neither is an A2A request.
+         * Vectors that need a wire exchange then skip as
+         * `signing_transport_unavailable` — and because no vector reached the
+         * agent, the storyboard cannot report a pass and the track grades
+         * `partial` rather than the run being scored against a transport error
+         * (adcp-client#2954). Only the
+         * in-library `jwks_override` negatives still run there: the grader
+         * decides them against the library verifier with no wire exchange at
+         * all, so the run's protocol is irrelevant to them. Every other probed
+         * vector reports the coverage gap, the protocol-method negatives
+         * included — the official `@a2a-js/sdk` client can issue those methods,
+         * but this runner does not yet sign and dispatch a vector through it,
+         * and writing the fixture bytes to the endpoint directly would be a
+         * hand-rolled A2A dispatch. Setting this field explicitly overrides the
+         * inference on any protocol; on an A2A run, do that only when the
+         * agent's MCP or REST binding answers at the same URL.
+         *
+         * Matches the `adcp grade request-signing --transport <mode>` CLI flag,
+         * and `adcp storyboard run --signing-transport <mode>`. Agents that only
+         * speak MCP JSON-RPC can't grade under `raw`; use `mcp` to let the runner
+         * round-trip every vector through `tools/call`.
          */
         transport?: 'raw' | 'mcp';
         /**
@@ -1640,6 +1729,12 @@ export interface StoryboardRunOptions extends TestOptions {
      * (`StoryboardStep.task`) is resolved to the agent that claims its
      * specialism via `TASK_FEATURE_MAP` × per-agent `get_adcp_capabilities`.
      * An optional explicit `StoryboardStep.agent` override takes precedence.
+     * Discovered tools across the map determine storyboard-level any-of
+     * applicability. Each dispatched step uses its selected agent's tools
+     * and profile; run-level `agentTools` cannot override routed discovery.
+     * Per-entry authentication and transport override caller-supplied run
+     * defaults. Other run options, including headers and request signing,
+     * remain shared defaults across the map.
      *
      * Mutually exclusive with `multi_instance_strategy` (which is replica
      * round-robin, a different concept) and with the legacy `_client`
@@ -1663,7 +1758,7 @@ export interface StoryboardRunOptions extends TestOptions {
     agents?: Record<string, AgentEntry>;
     /**
      * Fallback agent key (must be present in `agents`) for tasks with no entry
-     * in `TASK_FEATURE_MAP` — e.g., `comply_test_controller`, future tasks
+     * in `TASK_FEATURE_MAP` — e.g., `sync_creatives`, future tasks
      * shipped before the SDK adds them to the map. When omitted, unmapped
      * tasks fail-fast with `unroutable_task`.
      */
@@ -1930,6 +2025,26 @@ export type RunnerDetailedSkipReason = 'probe_skipped' | 'rate_abuse_opt_out' |
  | 'capability_profile_mismatch'
 /** Request-signing vector cannot be graded faithfully by the selected transport. */
  | 'transport_ungradable'
+/**
+ * The run's protocol has no shape the request-signing vectors can be
+ * framed in, so the runner could not grade them at all — a runner-owned
+ * coverage gap, not a property of the agent.
+ *
+ * Canonicalizes to `not_applicable` with `skip.detail` set to this exact
+ * token, the shape `runner-output-contract.yaml` defines for a registered
+ * `canonical_detail_sub_reasons.not_applicable.*` sub-reason (see
+ * `rate_limit_not_triggered`, the shipped precedent). It deliberately does
+ * NOT use `fixture_unavailable`: that reason's contract text mandates a
+ * `creative_asset_fixture_unavailable:` detail prefix and says the
+ * storyboard grades `not_applicable` with no verdict movement, which is
+ * the opposite of what this gap must do.
+ *
+ * The gap is kept visible by coverage, not by the reason: `signingCoverage`
+ * in the runner decides the storyboard verdict, the track rollup, the
+ * report block and the CLI exit from whether any probe reached the agent.
+ * See that function for why a skip reason cannot carry that weight.
+ */
+ | 'signing_transport_unavailable'
 /** Request-signing grader's MCP-transport mode collapses URL-edge vectors (#617). */
  | 'mcp_mode_flattens_url_edges'
 /** RFC 9728 protected-resource metadata returned 404 → agent is not advertising OAuth, cascade-skip oauth_discovery (#677). */
@@ -1955,6 +2070,13 @@ export type RunnerDetailedSkipReason = 'probe_skipped' | 'rate_abuse_opt_out' |
  | 'fixture_seed_unsupported'
 /** A valid fixture strategy ladder exhausted without finding a binding. */
  | 'fixture_unsatisfied'
+/**
+ * The MCP session auth probe (`mcp_session_probe`) cannot grade this step:
+ * its authored validations assert an AdCP task response body that no MCP
+ * protocol operation produces. Distinct from the generic `probe_skipped` so
+ * human and JUnit output can surface the actionable detail (adcp-client#2940).
+ */
+ | 'session_probe_ungradable'
 /**
  * A root capability predicate on the storyboard evaluated to false —
  * the agent explicitly declared it does not support the capability this

package/dist/lib/testing/storyboard/types.js

@@ -20,22 +20,38 @@ var types_exports = {};
 __export(types_exports, {
   DETAILED_SKIP_TO_CANONICAL: () => DETAILED_SKIP_TO_CANONICAL,
   KNOWN_REQUIREMENTS: () => KNOWN_REQUIREMENTS,
+  MCP_SESSION_PROBE_TASK: () => MCP_SESSION_PROBE_TASK,
   WEBHOOK_IDEMPOTENCY_KEY_PATTERN: () => WEBHOOK_IDEMPOTENCY_KEY_PATTERN
 });
 module.exports = __toCommonJS(types_exports);
 const WEBHOOK_IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9_.:-]{16,255}$/;
+const MCP_SESSION_PROBE_TASK = "mcp_session_probe";
 const DETAILED_SKIP_TO_CANONICAL = {
   probe_skipped: "not_applicable",
   not_in_only_vectors: "not_applicable",
   grader_skipped: "not_applicable",
   capability_profile_mismatch: "not_applicable",
+  // Scope note (adcp-client#2954): `transport_ungradable` marks vectors the
+  // grader's own TRANSPORT_UNGRADABLE table carves out on every run because
+  // no HTTP client can put those bytes on the wire — `026-non-ascii-host`
+  // for any storyboard-synthesized run, plus a profile-3.2 entry the
+  // storyboard never synthesizes. It stays `not_applicable`: those vectors
+  // are inapplicable to every binding, and the other ~38 still verify the
+  // agent.
+  //
+  // `signing_transport_unavailable` is also `not_applicable` at the canonical
+  // layer, but for the contract-shape reason documented on the enum member
+  // above — not because the gap is tolerable. Its weight is carried by
+  // `signingCoverage`, which asks whether any vector reached the agent.
   transport_ungradable: "not_applicable",
+  signing_transport_unavailable: "not_applicable",
   mcp_mode_flattens_url_edges: "not_applicable",
   oauth_not_advertised: "not_applicable",
   rate_limit_not_triggered: "not_applicable",
   force_scenario_unsupported: "not_applicable",
   fixture_seed_unsupported: "not_applicable",
   fixture_unsatisfied: "not_applicable",
+  session_probe_ungradable: "not_applicable",
   capability_unsupported: "not_applicable",
   capability_prerequisite_unavailable: "not_applicable",
   rate_abuse_opt_out: "unsatisfied_contract",
@@ -59,5 +75,6 @@ const KNOWN_REQUIREMENTS = /* @__PURE__ */ new Set([
 0 && (module.exports = {
   DETAILED_SKIP_TO_CANONICAL,
   KNOWN_REQUIREMENTS,
+  MCP_SESSION_PROBE_TASK,
   WEBHOOK_IDEMPOTENCY_KEY_PATTERN
 });

package/dist/lib/testing/storyboard/types.mjs

@@ -1,16 +1,31 @@
 const WEBHOOK_IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9_.:-]{16,255}$/;
+const MCP_SESSION_PROBE_TASK = "mcp_session_probe";
 const DETAILED_SKIP_TO_CANONICAL = {
   probe_skipped: "not_applicable",
   not_in_only_vectors: "not_applicable",
   grader_skipped: "not_applicable",
   capability_profile_mismatch: "not_applicable",
+  // Scope note (adcp-client#2954): `transport_ungradable` marks vectors the
+  // grader's own TRANSPORT_UNGRADABLE table carves out on every run because
+  // no HTTP client can put those bytes on the wire — `026-non-ascii-host`
+  // for any storyboard-synthesized run, plus a profile-3.2 entry the
+  // storyboard never synthesizes. It stays `not_applicable`: those vectors
+  // are inapplicable to every binding, and the other ~38 still verify the
+  // agent.
+  //
+  // `signing_transport_unavailable` is also `not_applicable` at the canonical
+  // layer, but for the contract-shape reason documented on the enum member
+  // above — not because the gap is tolerable. Its weight is carried by
+  // `signingCoverage`, which asks whether any vector reached the agent.
   transport_ungradable: "not_applicable",
+  signing_transport_unavailable: "not_applicable",
   mcp_mode_flattens_url_edges: "not_applicable",
   oauth_not_advertised: "not_applicable",
   rate_limit_not_triggered: "not_applicable",
   force_scenario_unsupported: "not_applicable",
   fixture_seed_unsupported: "not_applicable",
   fixture_unsatisfied: "not_applicable",
+  session_probe_ungradable: "not_applicable",
   capability_unsupported: "not_applicable",
   capability_prerequisite_unavailable: "not_applicable",
   rate_abuse_opt_out: "unsatisfied_contract",
@@ -33,5 +48,6 @@ const KNOWN_REQUIREMENTS = /* @__PURE__ */ new Set([
 export {
   DETAILED_SKIP_TO_CANONICAL,
   KNOWN_REQUIREMENTS,
+  MCP_SESSION_PROBE_TASK,
   WEBHOOK_IDEMPOTENCY_KEY_PATTERN
 };

package/dist/lib/testing/storyboard/validations.d.mts

@@ -197,7 +197,7 @@ export interface UpstreamTrafficQueryResult {
  */
 export declare function runValidations(validations: StoryboardValidation[], context: ValidationContext): ValidationResult[];
 /** Capability semver emitted in run summaries and used for advisory expiry. */
-export declare const RUNNER_CAPABILITY_VERSION = "14.0.0-rc.38";
+export declare const RUNNER_CAPABILITY_VERSION = "14.0.0-rc.39";
 /** True when a failed validation contributes to the owning step's grade. */
 export declare function validationFailsStep(result: ValidationResult): boolean;
 /**

package/dist/lib/testing/storyboard/validations.d.ts

@@ -197,7 +197,7 @@ export interface UpstreamTrafficQueryResult {
  */
 export declare function runValidations(validations: StoryboardValidation[], context: ValidationContext): ValidationResult[];
 /** Capability semver emitted in run summaries and used for advisory expiry. */
-export declare const RUNNER_CAPABILITY_VERSION = "14.0.0-rc.38";
+export declare const RUNNER_CAPABILITY_VERSION = "14.0.0-rc.39";
 /** True when a failed validation contributes to the owning step's grade. */
 export declare function validationFailsStep(result: ValidationResult): boolean;
 /**

package/dist/lib/testing/storyboard/validations.js

@@ -107,6 +107,8 @@ function runValidation(validation, ctx) {
       return requireHttpResult(ctx, validation, (hr) => validateHttpStatus(validation, hr));
     case "http_status_in":
       return requireHttpResult(ctx, validation, (hr) => validateHttpStatusIn(validation, hr));
+    case "probe_passed":
+      return requireHttpResult(ctx, validation, (hr) => validateProbePassed(validation, hr));
     case "on_401_require_header":
       return requireHttpResult(ctx, validation, (hr) => validateOn401RequireHeader(validation, hr));
     case "resource_equals_agent_url":
@@ -1210,6 +1212,20 @@ function validateHttpStatusIn(validation, hr) {
     actual: hr.status
   };
 }
+function validateProbePassed(validation, hr) {
+  if (!hr.error) {
+    return { check: "probe_passed", passed: true, description: validation.description };
+  }
+  return {
+    check: "probe_passed",
+    passed: false,
+    description: validation.description,
+    error: `Probe reported a failed grade: ${hr.error}`,
+    json_pointer: null,
+    expected: "probe grade with no error",
+    actual: hr.error
+  };
+}
 const SCHEMA_KEYWORD_RE = /invalid[_ ]?params|validation|schema|required|must be/i;
 const SCHEMA_CODE_RE = /VALIDATION|INVALID|SCHEMA|BAD_REQUEST/i;
 function looksLikeSchemaValidationBody(body) {

package/dist/lib/testing/storyboard/validations.mjs

@@ -82,6 +82,8 @@ function runValidation(validation, ctx) {
       return requireHttpResult(ctx, validation, (hr) => validateHttpStatus(validation, hr));
     case "http_status_in":
       return requireHttpResult(ctx, validation, (hr) => validateHttpStatusIn(validation, hr));
+    case "probe_passed":
+      return requireHttpResult(ctx, validation, (hr) => validateProbePassed(validation, hr));
     case "on_401_require_header":
       return requireHttpResult(ctx, validation, (hr) => validateOn401RequireHeader(validation, hr));
     case "resource_equals_agent_url":
@@ -1186,6 +1188,20 @@ function validateHttpStatusIn(validation, hr) {
     actual: hr.status
   };
 }
+function validateProbePassed(validation, hr) {
+  if (!hr.error) {
+    return { check: "probe_passed", passed: true, description: validation.description };
+  }
+  return {
+    check: "probe_passed",
+    passed: false,
+    description: validation.description,
+    error: `Probe reported a failed grade: ${hr.error}`,
+    json_pointer: null,
+    expected: "probe grade with no error",
+    actual: hr.error
+  };
+}
 const SCHEMA_KEYWORD_RE = /invalid[_ ]?params|validation|schema|required|must be/i;
 const SCHEMA_CODE_RE = /VALIDATION|INVALID|SCHEMA|BAD_REQUEST/i;
 function looksLikeSchemaValidationBody(body) {

package/dist/lib/testing/test-controller.d.mts

@@ -9,7 +9,7 @@ import type { AgentProfile, TaskResult, TestOptions } from './types.mjs';
 import type { RecordedCall } from '../upstream-recorder/types.mjs';
 import type { ComplyTestControllerResponse, ListScenariosSuccess, StateTransitionSuccess, SimulationSuccess, ControllerError } from '../types/tools.generated.mjs';
 /** Known canonical scenarios, retained as an autocomplete convenience. */
-export type ControllerScenario = 'force_creative_status' | 'force_account_status' | 'force_media_buy_status' | 'force_session_status' | 'force_create_media_buy_arm' | 'force_get_products_arm' | 'force_get_signals_arm' | 'force_task_completion' | 'simulate_delivery' | 'simulate_budget_spend'
+export type ControllerScenario = 'force_creative_status' | 'force_account_status' | 'force_media_buy_status' | 'force_session_status' | 'force_create_media_buy_arm' | 'force_get_products_arm' | 'force_get_signals_arm' | 'force_task_completion' | 'reset_state' | 'simulate_delivery' | 'simulate_budget_spend'
 /**
  * Returns outbound HTTP calls the agent has made since session start
  * (or since a caller-supplied timestamp). Backs the `upstream_traffic`

package/dist/lib/testing/test-controller.d.ts

@@ -9,7 +9,7 @@ import type { AgentProfile, TaskResult, TestOptions } from './types';
 import type { RecordedCall } from '../upstream-recorder/types';
 import type { ComplyTestControllerResponse, ListScenariosSuccess, StateTransitionSuccess, SimulationSuccess, ControllerError } from '../types/tools.generated';
 /** Known canonical scenarios, retained as an autocomplete convenience. */
-export type ControllerScenario = 'force_creative_status' | 'force_account_status' | 'force_media_buy_status' | 'force_session_status' | 'force_create_media_buy_arm' | 'force_get_products_arm' | 'force_get_signals_arm' | 'force_task_completion' | 'simulate_delivery' | 'simulate_budget_spend'
+export type ControllerScenario = 'force_creative_status' | 'force_account_status' | 'force_media_buy_status' | 'force_session_status' | 'force_create_media_buy_arm' | 'force_get_products_arm' | 'force_get_signals_arm' | 'force_task_completion' | 'reset_state' | 'simulate_delivery' | 'simulate_budget_spend'
 /**
  * Returns outbound HTTP calls the agent has made since session start
  * (or since a caller-supplied timestamp). Backs the `upstream_traffic`