@adcp/sdk 7.0.0 → 7.2.0

package/dist/lib/testing/storyboard/runner.js

@@ -32,6 +32,7 @@ const validations_1 = require("./validations");
 const parallel_dispatch_1 = require("./parallel-dispatch");
 const path_1 = require("./path");
 const redact_secrets_1 = require("../../utils/redact-secrets");
+const response_unwrapper_1 = require("../../utils/response-unwrapper");
 const test_controller_1 = require("../test-controller");
 const request_builder_1 = require("./request-builder");
 const client_2 = require("../client");
@@ -681,6 +682,7 @@ function buildCapabilityUnsupportedResult(agentUrls, storyboard, detail) {
             strict_only_failures: 0,
             lenient_also_failed: 0,
         },
+        notices: [],
     };
 }
 /**
@@ -701,6 +703,7 @@ const REQUIREMENT_TO_SKIP_REASON = {
     seeded_state: 'requirement_unmet',
     real_wire: 'requirement_unmet',
     webhook_receiver: 'requirement_unmet',
+    request_signer: 'not_applicable',
 };
 /**
  * Build a minimal StoryboardResult for a storyboard skipped because a
@@ -756,6 +759,7 @@ function buildRequirementUnmetResult(agentUrls, storyboard, requirement, detail)
             strict_only_failures: 0,
             lenient_also_failed: 0,
         },
+        notices: [],
     };
 }
 /**
@@ -777,8 +781,21 @@ function buildRequirementUnmetResult(agentUrls, storyboard, requirement, detail)
  *     Autodetected from step `sample_request` token presence (see
  *     `detectImplicitRequires`); authors don't write this tag manually.
  *     Spec: adcp-client#1678.
+ *   - `request_signer` — agent advertises `request_signing.supported: true`
+ *     in `get_adcp_capabilities`. Autodetected for any storyboard whose
+ *     id is `'signed_requests'` or that contains a `request_signing_probe`
+ *     step (see `detectImplicitRequires`); authors don't write this tag
+ *     manually. Absence-skip semantics are INVERTED from the general
+ *     `requires_capability` rule: the signed-requests universal
+ *     storyboard's own gating spec (signed-requests.yaml prerequisites)
+ *     declares "agents that do not advertise support are not tested
+ *     against this storyboard — absence of advertisement is not a
+ *     failure". When the runner can't read capabilities (no profile
+ *     threaded through), the gate is a no-op — the caller accepted
+ *     responsibility for capability compatibility by reusing an external
+ *     client. Spec: adcp-client#1702.
  */
-function checkRequires(requires, options) {
+function checkRequires(requires, options, profile) {
     for (const requirement of requires) {
         switch (requirement) {
             case 'controller': {
@@ -820,6 +837,31 @@ function checkRequires(requires, options) {
                 }
                 break;
             }
+            case 'request_signer': {
+                // Gate is a no-op when no profile is threaded through (external
+                // `_client` mode). The caller has accepted responsibility for
+                // capability compatibility; failing the gate here would surprise
+                // them with a skip they can't explain from CLI options alone.
+                if (!profile?.raw_capabilities)
+                    break;
+                const supported = resolveCapabilityPath(profile.raw_capabilities, 'request_signing.supported');
+                if (supported === true)
+                    break;
+                return {
+                    requirement,
+                    detail: 'Storyboard requires `request_signing.supported: true` in `get_adcp_capabilities`; ' +
+                        `agent declared ${supported === undefined ? 'no request_signing block' : JSON.stringify(supported)}. ` +
+                        'Per compliance/{version}/universal/signed-requests.yaml: "Agents that do not advertise ' +
+                        'support are not tested against this storyboard — absence of advertisement is not a ' +
+                        'failure, it is a declaration that the agent does not offer verified signed requests." ' +
+                        'To opt in, advertise `request_signing.supported: true` and pre-register the runner ' +
+                        'compliance test keypair per test-kits/signed-requests-runner.yaml. ' +
+                        'Forward-readiness: optional in AdCP 3.0; the schema (request_signing block description) ' +
+                        'declares request signing required for spend-committing operations in AdCP 4.0. Sellers ' +
+                        'that intend to support spend-committing tools SHOULD advertise the capability and ' +
+                        'register the test keypair before the 4.0 cut to avoid a hard compliance failure then.',
+                };
+            }
         }
     }
     return null;
@@ -854,23 +896,45 @@ function valueContainsWebhookToken(value) {
 }
 /**
  * Return the implicit requirements a storyboard needs based on its
- * structure (not its declared `requires:` list). Today: only
- * `'webhook_receiver'`, autodetected from `{{runner.webhook_url:…}}` /
- * `{{runner.webhook_base}}` token presence inside any step's
- * `sample_request`. Token presence is the authoring contract — a
- * storyboard that names the runner's webhook receiver cannot run
- * without one — so authors don't need to remember to add
- * `requires: [webhook_receiver]` separately. Spec: adcp-client#1678.
+ * structure (not its declared `requires:` list). Today:
+ *   - `'webhook_receiver'`, autodetected from `{{runner.webhook_url:…}}`
+ *     / `{{runner.webhook_base}}` token presence inside any step's
+ *     `sample_request`. Token presence is the authoring contract — a
+ *     storyboard that names the runner's webhook receiver cannot run
+ *     without one — so authors don't need to remember to add
+ *     `requires: [webhook_receiver]` separately. Spec: adcp-client#1678.
+ *   - `'request_signer'`, autodetected from `storyboard.id ===
+ *     'signed_requests'` or any step using the synthesized
+ *     `request_signing_probe` task. The signed-requests universal
+ *     storyboard's own prerequisites section declares the
+ *     `request_signing.supported: true` capability gate; the runner
+ *     enforces it here so adopters don't see false-negative vector
+ *     failures on bearer-only agents that never claimed signing.
+ *     Spec: adcp-client#1702.
  */
 function detectImplicitRequires(storyboard) {
+    const requires = [];
+    let needsWebhook = false;
+    let needsSigner = storyboard.id === 'signed_requests';
     for (const phase of storyboard.phases) {
         for (const step of phase.steps) {
-            if (step.sample_request && valueContainsWebhookToken(step.sample_request)) {
-                return ['webhook_receiver'];
+            if (!needsWebhook && step.sample_request && valueContainsWebhookToken(step.sample_request)) {
+                needsWebhook = true;
             }
+            if (!needsSigner && step.task === 'request_signing_probe') {
+                needsSigner = true;
+            }
+            if (needsWebhook && needsSigner)
+                break;
         }
+        if (needsWebhook && needsSigner)
+            break;
     }
-    return [];
+    if (needsWebhook)
+        requires.push('webhook_receiver');
+    if (needsSigner)
+        requires.push('request_signer');
+    return requires;
 }
 /**
  * Build a hard-failure StoryboardResult for when agent capability
@@ -927,6 +991,7 @@ function buildDiscoveryFailedResult(agentUrls, storyboard, discoveryStep) {
             strict_only_failures: 0,
             lenient_also_failed: 0,
         },
+        notices: [],
     };
 }
 /**
@@ -984,8 +1049,85 @@ function buildRequiredToolsMissingResult(agentUrls, storyboard, detail) {
             strict_only_failures: 0,
             lenient_also_failed: 0,
         },
+        notices: [],
     };
 }
+/**
+ * Collect protocol-compliance notices for a storyboard run based on the
+ * agent's declared capabilities. Returns an empty array when rawCaps is
+ * absent (standalone runner without pre-fetched profile).
+ *
+ * Currently emits two spec-grounded notices (adcp-client#1704):
+ *
+ * - `request_signing.required`: `request_signing.supported` is absent or
+ *   false on the signed_requests storyboard. Signing becomes required for
+ *   spend-committing operations in `effective_version: '4.0'`.
+ *
+ * - `webhook_signing.legacy_hmac_fallback.removed`: agent claims
+ *   `webhook_signing.legacy_hmac_fallback: true`, which is removed in
+ *   `effective_version: '4.0'`.
+ *
+ * Note: a third notice (`signed_requests_specialism_deprecated`) is deferred
+ * pending upstream deprecation of the `'signed-requests'` specialism value in
+ * adcontextprotocol/adcp#4418 — the value is still active in the spec.
+ */
+function collectCapabilityNotices(storyboard, rawCaps) {
+    const notices = [];
+    if (!rawCaps || typeof rawCaps !== 'object')
+        return notices;
+    const caps = rawCaps;
+    // Notice: request_signing required in AdCP 4.0.
+    // Fired when this is the signed_requests storyboard and the agent hasn't
+    // declared request_signing support. The storyboard is identified by id OR
+    // by the presence of a request_signing_probe step (mirrors the implicit-
+    // require detection in detectImplicitRequires for PR #1703's gate).
+    const isSignedRequestsStoryboard = storyboard.id === 'signed_requests' ||
+        storyboard.phases.some(p => p.steps.some(s => s.task === 'request_signing_probe'));
+    if (isSignedRequestsStoryboard) {
+        const requestSigning = caps['request_signing'];
+        if (requestSigning?.['supported'] !== true) {
+            notices.push({
+                severity: 'future_required',
+                code: 'request_signing.required',
+                message: 'RFC 9421 request signing (`request_signing.supported: true`) is not advertised. ' +
+                    'Required for spend-committing operations in AdCP 4.0 — declare the capability and ' +
+                    'pre-register the runner compliance test keypair before the 4.0 cut.',
+                effective_version: '4.0',
+                capability_path: 'request_signing.supported',
+                docs_url: 'https://adcontextprotocol.org/docs/building/implementation/security#signed-requests-transport-layer',
+                storyboard_ids: [storyboard.id],
+            });
+        }
+    }
+    // Notice: legacy_hmac_fallback removed in AdCP 4.0.
+    // Scoped via the WEBHOOK_STEP_TASKS step-task set rather than an id-regex.
+    // Step-task presence is the authoring contract — a storyboard that asserts
+    // webhook delivery uses one of these tasks. A storyboard id alone (e.g. a
+    // hypothetical `webhook_authoring_guide`) shouldn't trigger the notice
+    // unless it actually exercises the delivery path.
+    const WEBHOOK_STEP_TASKS = new Set([
+        'expect_webhook',
+        'expect_webhook_retry_keys_stable',
+        'expect_webhook_signature_valid',
+    ]);
+    const isWebhookRelatedStoryboard = storyboard.phases.some(p => p.steps.some(s => WEBHOOK_STEP_TASKS.has(s.task)));
+    if (isWebhookRelatedStoryboard) {
+        const webhookSigning = caps['webhook_signing'];
+        if (webhookSigning?.['legacy_hmac_fallback'] === true) {
+            notices.push({
+                severity: 'deprecation',
+                code: 'webhook_signing.legacy_hmac_fallback.removed',
+                message: '`webhook_signing.legacy_hmac_fallback: true` is deprecated and removed in AdCP 4.0. ' +
+                    'Migrate webhook signature verification to RFC 9421 before the 4.0 cut.',
+                effective_version: '4.0',
+                capability_path: 'webhook_signing.legacy_hmac_fallback',
+                docs_url: 'https://adcontextprotocol.org/docs/building/implementation/webhooks',
+                storyboard_ids: [storyboard.id],
+            });
+        }
+    }
+    return notices;
+}
 /**
  * Execute a single pass of the storyboard against the supplied replica URLs
  * using round-robin dispatch starting at `dispatchOffset`. Called directly
@@ -1108,15 +1250,24 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     // implicit gate, storyboards that name the receiver but run without
     // one would silently ship literal mustache tokens on the wire and
     // get rejected by 3.0-strict sellers as `INVALID_REQUEST: relative URL`.
+    // Pre-flight notice collection. Uses options._profile (set by the comply()
+    // pipeline before calling runStoryboard) so the notices are available on
+    // early-return results (requirement-unmet, capability-unsupported). In the
+    // standalone runner path options._profile may be undefined; notices will be
+    // collected again from the fully-fetched profile at result-build time.
+    const preflightNotices = collectCapabilityNotices(storyboard, options._profile?.raw_capabilities);
     const declared = storyboard.requires ?? [];
     const implicit = detectImplicitRequires(storyboard);
     const allRequires = [...declared, ...implicit.filter(r => !declared.includes(r))];
     if (allRequires.length) {
-        const unmet = checkRequires(allRequires, options);
+        const unmet = checkRequires(allRequires, options, profile);
         if (unmet) {
             if (!options._client)
                 await (0, protocols_1.closeConnections)(options.protocol);
-            return buildRequirementUnmetResult(agentUrls, storyboard, unmet.requirement, unmet.detail);
+            return {
+                ...buildRequirementUnmetResult(agentUrls, storyboard, unmet.requirement, unmet.detail),
+                notices: preflightNotices,
+            };
         }
     }
     // Evaluate requires_capability predicate before any phase setup.
@@ -1142,7 +1293,7 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
                     `agent declared ${JSON.stringify(actual)}.`;
                 if (!options._client)
                     await (0, protocols_1.closeConnections)(options.protocol);
-                return buildCapabilityUnsupportedResult(agentUrls, storyboard, detail);
+                return { ...buildCapabilityUnsupportedResult(agentUrls, storyboard, detail), notices: preflightNotices };
             }
         }
     }
@@ -1162,7 +1313,10 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
         if (!hasAnyRequired) {
             if (!options._client)
                 await (0, protocols_1.closeConnections)(options.protocol);
-            return buildRequiredToolsMissingResult(agentUrls, storyboard, `agent does not advertise any of [${storyboard.required_tools.join(', ')}]`);
+            return {
+                ...buildRequiredToolsMissingResult(agentUrls, storyboard, `agent does not advertise any of [${storyboard.required_tools.join(', ')}]`),
+                notices: preflightNotices,
+            };
         }
     }
     let context = { ...options.context };
@@ -1447,6 +1601,18 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
             });
             continue;
         }
+        // Pre-empt OAuth-metadata probes when the agent's capabilities never
+        // advertised OAuth at all (adcp-client#1702). Without this, an
+        // API-key-only agent (e.g. Wonderstruck) has its PRM endpoint
+        // probed; if the well-known path returns anything other than 404
+        // (the existing reactive cascade trigger in `executeProbeStep`),
+        // validations fail and `oauth_discovery` produces 5 false-negative
+        // step failures even though the phase is `optional: true`. The skip
+        // is phase-level — not storyboard-level — so the universal
+        // `unauth_rejection` and `mechanism_required` phases still run.
+        if (!phaseAbsent && phaseContainsOauthMetadataProbe(phase) && !agentAdvertisesOauth(profile)) {
+            phaseAbsent = true;
+        }
         // Reset alias cache at this phase boundary (#1657). $generate:uuid_v4#alias
         // is designed to be stable within a scenario — the initial call and its
         // idempotency replay share the same UUID — but aliases must NOT bleed across
@@ -1650,6 +1816,20 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
             }
             stepResults.push(result);
             priorStepResults.set(step.id, result);
+            // Schema-validation short-circuit (adcp-client#1709). When the
+            // response unwrapper rejected the agent's response against the SDK's
+            // Zod schema, the step-execution path synthesized a failing
+            // `response_schema` ValidationResult on `result.validations`.
+            // Running step-scope invariants against a response the SDK has
+            // already declared malformed produces noise — the invariants
+            // can't meaningfully grade a payload that didn't parse. Worse,
+            // before #1709 the invariants' failure entries crowded out the
+            // schema-validation entry in `extractFailures`, masking the root
+            // cause across BidMachine's 10+ deploys (see adcp#4419). Skip the
+            // invariant pass entirely on this step and emit a single skipped
+            // `assertion` entry per invariant so consumers can still see WHICH
+            // invariants were skipped and why.
+            const schemaInvalidResponse = result.validations.some(v => v.check === 'response_schema' && !v.passed);
             // Fire per-step assertions. Each result is appended to the step's
             // `validations[]` under `check: "assertion"` so existing UI renders
             // them alongside inline checks, and mirrored into `assertionResults`
@@ -1664,6 +1844,18 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
                 // behavior the invariant would flag (validated at runner start).
                 if ((0, assertions_1.stepDisablesAssertion)(step.invariants, spec.id))
                     continue;
+                if (schemaInvalidResponse) {
+                    // Emit a skipped marker so the report shows the invariant was
+                    // intentionally bypassed (not silently dropped). `passed: true`
+                    // keeps it from flipping `result.passed` — the schema-validation
+                    // failure already did that.
+                    result.validations.push({
+                        check: 'assertion',
+                        passed: true,
+                        description: `${spec.id}: skipped — response failed schema validation (adcp-client#1709)`,
+                    });
+                    continue;
+                }
                 const raw = await spec.onStep(assertionContexts.get(spec.id), result);
                 for (const r of raw) {
                     const full = { ...r, assertion_id: spec.id, scope: 'step', step_id: step.id };
@@ -2025,6 +2217,10 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     const schemasUsed = collectSchemasUsed(phaseResults);
     const strictSummary = summarizeStrictValidation(phaseResults);
     const validationsNotApplicable = countValidationsNotApplicable(phaseResults);
+    // Use the fully-fetched profile for notice detection; fall back to pre-flight
+    // notices (which used options._profile) when profile was not re-fetched in
+    // this pass (standalone runner with options._profile pre-set skips the fetch).
+    const notices = collectCapabilityNotices(storyboard, profile?.raw_capabilities ?? options._profile?.raw_capabilities);
     const result = {
         storyboard_id: storyboard.id,
         storyboard_title: storyboard.title,
@@ -2047,6 +2243,7 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
         ...(schemasUsed.length > 0 ? { schemas_used: schemasUsed } : {}),
         ...(assertionResults.length > 0 ? { assertions: assertionResults } : {}),
         strict_validation_summary: strictSummary,
+        notices,
     };
     // Close protocol connections when the runner created its own client. The
     // connection pool is keyed by URL+auth, so a single closeConnections() call
@@ -2123,6 +2320,8 @@ async function runMultiPass(agentUrls, storyboard, options) {
     // readers see a per-pass timeline; de-duplicating would hide a real
     // "passed on pass 1, failed on pass 2" divergence.
     const assertionsAgg = passResults.flatMap(r => r.assertions ?? []);
+    // Notices are identical across passes (same agent, same capabilities); deduplicate by code.
+    const noticesDedup = [...new Map(passResults.flatMap(r => r.notices).map(n => [n.code, n])).values()];
     return {
         storyboard_id: storyboard.id,
         storyboard_title: storyboard.title,
@@ -2141,6 +2340,7 @@ async function runMultiPass(agentUrls, storyboard, options) {
         tested_at: new Date().toISOString(),
         ...(schemasDedup.length > 0 ? { schemas_used: schemasDedup } : {}),
         ...(assertionsAgg.length > 0 ? { assertions: assertionsAgg } : {}),
+        notices: noticesDedup,
     };
 }
 /**
@@ -2614,6 +2814,11 @@ client, step, phaseId, context, allSteps, options, state) {
     const useRawProbe = rawProbeHeaders !== undefined;
     let taskResult;
     let stepResult;
+    // Raw caught error from the dispatch fn — preserved as `unknown` so the
+    // schema-validation attribution path below can `instanceof` it against
+    // `ResponseSchemaValidationError`. Undefined when the step succeeded or
+    // failed for a non-typed reason. Spec: adcp-client#1709.
+    let caughtError;
     let httpResult;
     let responseRecord;
     let a2aEnvelope;
@@ -2827,6 +3032,7 @@ client, step, phaseId, context, allSteps, options, state) {
             });
             taskResult = run.result;
             stepResult = run.step;
+            caughtError = run.caughtError;
             if (captureA2a && a2aCaptures) {
                 a2aEnvelope = parseLastA2aMessageSendCapture(a2aCaptures);
             }
@@ -2893,10 +3099,10 @@ client, step, phaseId, context, allSteps, options, state) {
     else {
         passed = stepResult.passed && (taskResult?.success ?? false);
     }
+    let validations = [];
     // Run validations. Resolve `$context.<key>` placeholders in `value` and
     // `allowed_values` fields so expected values can reference prior steps
     // (e.g., replay tests assert `media_buy_id === $context.initial_media_buy_id`).
-    let validations = [];
     if (step.validations?.length && (taskResult || httpResult)) {
         const resolvedValidations = step.validations.map(v => {
             const resolved = { ...v };
@@ -2989,6 +3195,41 @@ client, step, phaseId, context, allSteps, options, state) {
             }
         }
     }
+    // Schema-validation attribution (adcp-client#1709). When the response
+    // unwrapper rejected the agent's response against the SDK's Zod schema
+    // for the tool, it threw a typed `ResponseSchemaValidationError` that
+    // surfaced on `caughtError`. Without this attribution, the rejection
+    // would silently propagate into whichever step-scope invariant (e.g.
+    // `context.no_secret_echo`) happened to fire next — the BidMachine
+    // misdiagnosis trace from adcp-client#1709 / adcp#4419 ate 10+ deploys
+    // chasing the wrong cause.
+    //
+    // Synthesize a canonical `response_schema` ValidationResult and prepend
+    // it so `extractFailures.find(v => !v.passed)` resolves it before any
+    // invariant entry. Step-scope invariants downstream of this point will
+    // short-circuit on the schema-invalid response (see the invariant
+    // dispatch loop in `executeStoryboardPass`).
+    if (caughtError instanceof response_unwrapper_1.ResponseSchemaValidationError) {
+        const issues = caughtError.issues
+            .slice(0, 5)
+            .map(i => `${i.path.join('.') || '(root)'}: ${i.message}`)
+            .join('; ');
+        const firstIssue = caughtError.issues[0];
+        const jsonPointer = firstIssue ? '/' + firstIssue.path.map(s => String(s)).join('/') : null;
+        const synthSchemaResult = {
+            check: 'response_schema',
+            passed: false,
+            description: `Response schema validation for ${caughtError.toolName}`,
+            error: issues,
+            json_pointer: jsonPointer,
+            expected: `response schema for ${caughtError.toolName}`,
+            actual: caughtError.issues,
+        };
+        // Prepend so extractFailures picks it up before any inline validation
+        // entry that may also be failing (e.g. `field_present` checks that
+        // legitimately can't observe their target against an unparsed payload).
+        validations = [synthSchemaResult, ...validations];
+    }
     // Persist the captured A2A envelope keyed by step id so cross-step
     // validators (`a2a_context_continuity`) on subsequent steps can
     // compare against it. Only fires when this step actually captured
@@ -3418,6 +3659,33 @@ function isTaskShape(result) {
 // ────────────────────────────────────────────────────────────
 // Phase / step skip predicates
 // ────────────────────────────────────────────────────────────
+/**
+ * True when the phase contains an OAuth-metadata probe step
+ * (`protected_resource_metadata` or `oauth_auth_server_metadata`). Used
+ * to pre-emptively trigger the existing `phaseAbsent` cascade when the
+ * agent's capabilities never advertised OAuth in the first place, so we
+ * don't burn step failures probing a well-known path the agent doesn't
+ * claim. Spec: adcp-client#1702.
+ */
+function phaseContainsOauthMetadataProbe(phase) {
+    return phase.steps.some(s => s.task === 'protected_resource_metadata' || s.task === 'oauth_auth_server_metadata');
+}
+/**
+ * True when the agent's `get_adcp_capabilities` response declares an
+ * OAuth issuer (today: `account.authorization_endpoint`). When the
+ * profile or its capabilities aren't available — e.g. external
+ * `_client` mode — we conservatively return `true` so the runner falls
+ * through to the existing reactive cascade (`phaseAbsent` flips on a
+ * 404 from the PRM probe). That keeps backward compatibility for
+ * callers that haven't threaded a profile through. Spec: adcp-client#1702.
+ */
+function agentAdvertisesOauth(profile) {
+    const rawCaps = profile?.raw_capabilities;
+    if (rawCaps === undefined)
+        return true;
+    const endpoint = resolveCapabilityPath(rawCaps, 'account.authorization_endpoint');
+    return typeof endpoint === 'string' && endpoint.length > 0;
+}
 /**
  * Evaluate a phase's `skip_if` expression against the runtime options. Only
  * a tiny grammar is supported today; unknown expressions fail closed (phase runs).