@adcp/sdk 5.25.1 → 6.1.0

package/dist/lib/testing/storyboard/runner.js

@@ -12,6 +12,7 @@ exports.__redactSecretsForTest = __redactSecretsForTest;
 exports.__filterResponseHeadersForTest = __filterResponseHeadersForTest;
 exports.__defaultAuthHeadersForRawProbeForTest = __defaultAuthHeadersForRawProbeForTest;
 exports.runStoryboard = runStoryboard;
+exports.buildDiscoveryFailedResult = buildDiscoveryFailedResult;
 exports.listStrictOnlyFailures = listStrictOnlyFailures;
 exports.summarizeStrictValidation = summarizeStrictValidation;
 exports.runStoryboardStep = runStoryboardStep;
@@ -88,6 +89,27 @@ function resolveCapabilityPath(raw, dottedPath) {
 function buildSkip(reason, detail) {
     return { reason, detail: detail ?? SKIP_DETAILS[reason] };
 }
+/**
+ * True for skip reasons that imply state genuinely never materialized
+ * — no other code path could have established it. The runner trips
+ * the cascade immediately on these. `missing_tool` and
+ * `missing_test_controller` qualify: the agent doesn't advertise the
+ * tool, end of story.
+ *
+ * `not_applicable` is intentionally NOT in this set even though F6's
+ * first cut bundled it in. `not_applicable` means "this path doesn't
+ * apply to this agent", which is consistent with "a peer path in the
+ * same phase establishes equivalent state" — e.g. an explicit-mode
+ * seller skips `sync_accounts` as not_applicable because
+ * `list_accounts` is the canonical alternative for that account
+ * shape. Tripping the cascade on the first not_applicable would
+ * collapse this distinction and skip the substitute. The runner
+ * therefore defers `not_applicable`'s cascade decision to phase end —
+ * see `phasePendingNotApplicable` handling in the phase loop.
+ */
+function isHardMissingStateSkipReason(reason) {
+    return reason === 'missing_tool' || reason === 'missing_test_controller';
+}
 /**
  * Resolve each phase's branch-set membership, combining explicit
  * `branch_set: { id, semantics }` declarations with the implicit detection
@@ -383,6 +405,63 @@ function buildCapabilityUnsupportedResult(agentUrls, storyboard, detail) {
         },
     };
 }
+/**
+ * Build a hard-failure StoryboardResult for when agent capability
+ * discovery (`get_agent_info` / MCP `tools/list`) failed. Surfacing
+ * discovery errors as a hard storyboard failure prevents the silent
+ * "X/X clean, 100% skipped" failure mode where transport / auth
+ * misconfiguration produced an empty `agentTools: []` and every step
+ * skipped with `missing_tool`.
+ *
+ * @public — exported for direct testing of the failure-result shape;
+ * called from `runStoryboard` when discovery throws.
+ */
+function buildDiscoveryFailedResult(agentUrls, storyboard, discoveryStep) {
+    const detail = discoveryStep.error ?? 'Discovery failed (no agent tools advertised).';
+    const syntheticStep = {
+        storyboard_id: storyboard.id,
+        step_id: 'discovery_failed',
+        phase_id: 'discovery_failed',
+        title: 'Storyboard failed: agent capability discovery did not succeed',
+        task: '',
+        passed: false,
+        skipped: false,
+        duration_ms: discoveryStep.duration_ms,
+        validations: [],
+        context: {},
+        error: `Discovery failure: ${detail}. The runner refuses to proceed with empty agentTools — that mode produces silent "all clean" reports when the underlying transport / auth / network policy is broken. Fix the discovery error before re-running.`,
+        extraction: { path: 'none' },
+    };
+    return {
+        storyboard_id: storyboard.id,
+        storyboard_title: storyboard.title,
+        agent_url: agentUrls[0],
+        overall_passed: false,
+        phases: [
+            {
+                phase_id: 'discovery_failed',
+                phase_title: 'Discovery failed',
+                passed: false,
+                steps: [syntheticStep],
+                duration_ms: discoveryStep.duration_ms,
+            },
+        ],
+        context: {},
+        total_duration_ms: discoveryStep.duration_ms,
+        passed_count: 0,
+        failed_count: 1,
+        skipped_count: 0,
+        tested_at: new Date().toISOString(),
+        strict_validation_summary: {
+            observable: false,
+            checked: 0,
+            passed: 0,
+            failed: 0,
+            strict_only_failures: 0,
+            lenient_also_failed: 0,
+        },
+    };
+}
 /**
  * Build a minimal StoryboardResult for a storyboard skipped because the agent
  * does not advertise any of the tools listed in `required_tools`. The result
@@ -460,10 +539,26 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     let profile;
     if (!options._client) {
         const discovered = await (0, client_1.getOrDiscoverProfile)(clients[0], options);
+        // Discovery failure must surface as a HARD STORYBOARD FAILURE, not a
+        // silent empty `agentTools: []` that lets every step skip with
+        // `missing_tool`. The latter mode produces "X/X clean" summaries with
+        // 100% skipped — invisible CI failure when transport setup is broken
+        // (auth misconfig, MCP transport-fallback bugs, network policy, etc.).
+        // See: https://github.com/adcontextprotocol/adcp-client/issues/...
+        if (discovered.step.passed === false) {
+            if (!options._client)
+                await (0, protocols_1.closeConnections)(options.protocol);
+            return buildDiscoveryFailedResult(agentUrls, storyboard, discovered.step);
+        }
         profile = discovered.profile;
-        // Populate agentTools from discovered profile if not already set
+        // Populate agentTools and _profile from discovered profile if not already set.
+        // _profile is threaded into executeStep so capability-based skip gates
+        // (e.g. account-mode branching) can read raw_capabilities at step time.
         if (!options.agentTools && profile?.tools) {
-            options = { ...options, agentTools: profile.tools };
+            options = { ...options, agentTools: profile.tools, _profile: profile };
+        }
+        else if (profile && !options._profile) {
+            options = { ...options, _profile: profile };
         }
     }
     else {
@@ -533,6 +628,14 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
     let passedCount = 0;
     let failedCount = 0;
     let skippedCount = 0;
+    // Stateful-cascade flags live at storyboard scope, NOT phase scope —
+    // cross-phase storyboards (e.g. signal_marketplace/governance_denied:
+    // setup in phases 1-2, consumption in phase 3) need the cascade to
+    // survive phase boundaries. Once a stateful step in any phase failed
+    // or skipped to materialize state, every downstream stateful step
+    // stays cascade-skipped regardless of which phase it lives in.
+    let statefulFailed = false;
+    let statefulSkipTrigger = null;
     // Step results whose failures the main loop added to failedCount. The
     // branch-set post-pass decrements only for entries that were actually
     // counted, so an optional phase that hit `presenceDetected` (a PRM 2xx
@@ -681,7 +784,22 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
         const phaseStart = Date.now();
         const stepResults = [];
         let phasePassed = true;
-        let statefulFailed = false;
+        // `statefulFailed` and `statefulSkipTrigger` live at storyboard
+        // scope (declared above the phase loop) so the cascade survives
+        // cross-phase setup → assertion patterns. See declaration site for
+        // the rationale (signal_marketplace/governance_denied story).
+        //
+        // Phase-scoped substitution tracking (adcp-client#1005, round-9).
+        // A `not_applicable` skip on a stateful step doesn't mean state
+        // never materialized — it means *this* path doesn't apply, and the
+        // storyboard may have a peer step (e.g. `list_accounts` paired
+        // with `sync_accounts`) that establishes equivalent state. We
+        // record the trigger and defer the cascade decision to phase end:
+        // if any stateful step in this phase passed, the substitute did
+        // its job and we don't trip. If none did, we promote the pending
+        // trigger to a hard cascade so downstream phases skip cleanly.
+        let phasePendingNotApplicable = null;
+        let phaseEstablishedStatefulState = false;
         // PRM presence-probe state (adcp-client#677). `phaseAbsent` flips when
         // /.well-known/oauth-protected-resource returns 404 — subsequent steps
         // in this phase cascade-skip instead of failing their http_status:200
@@ -767,9 +885,15 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
                 skippedCount++;
                 continue;
             }
-            // Skip remaining steps if a stateful dependency failed
+            // Skip remaining steps if a stateful dependency failed (or
+            // skipped for a missing-state reason). The detail message
+            // distinguishes the two so adopters reading the cascade know
+            // whether to look at a real failure or a structural mismatch
+            // (storyboard step requires a tool the agent doesn't advertise).
             if (statefulFailed && step.stateful) {
-                const detail = 'Skipped: prior stateful step failed.';
+                const detail = statefulSkipTrigger
+                    ? `Skipped: prior stateful step "${statefulSkipTrigger.stepId}" skipped (${statefulSkipTrigger.reason}); state never materialized.`
+                    : 'Skipped: prior stateful step failed.';
                 stepResults.push({
                     storyboard_id: storyboard.id,
                     step_id: step.id,
@@ -874,10 +998,73 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
             if (result.skipped) {
                 skippedCount++;
                 context = result.context;
+                // Cascade-skip extension: a stateful step that SKIPS because the
+                // agent simply lacks the tool (`missing_tool`,
+                // `missing_test_controller`) is equivalent to a failed stateful
+                // step — state genuinely never materialized. Trip immediately so
+                // downstream stateful steps skip with `prerequisite_failed`
+                // instead of running against absent state and surfacing a
+                // misleading assertion failure.
+                //
+                // `not_applicable` is handled separately (adcp-client#1005,
+                // round-9). It signals "this path doesn't apply to this agent",
+                // and the storyboard may have a peer step in the same phase
+                // that establishes equivalent state — e.g. an explicit-mode
+                // seller's `list_accounts` substituting for `sync_accounts`.
+                // We record a pending trigger and let the rest of the phase
+                // run; the cascade decision is finalized at phase end based on
+                // whether any stateful peer passed.
+                //
+                // Skips that imply state DID materialize via another path
+                // (`peer_branch_taken`, `controller_seeding_failed` — handled
+                // by phase-level cascade, `oauth_not_advertised` — phase-absent
+                // path) deliberately don't trip the flag.
+                if (step.stateful) {
+                    if (isHardMissingStateSkipReason(result.skip_reason)) {
+                        statefulFailed = true;
+                        // Record provenance for the cascade detail message. First trip
+                        // wins — subsequent triggers don't overwrite, since the cascade
+                        // text references the originating diagnostic (the leftmost
+                        // missing-state stateful step in the phase).
+                        if (statefulSkipTrigger === null) {
+                            statefulSkipTrigger = { stepId: step.id, reason: result.skip_reason ?? 'missing_tool' };
+                        }
+                    }
+                    else if (result.skip_reason === 'not_applicable') {
+                        // Defer cascade decision until end of phase. Applies the
+                        // same first-wins rule used for `statefulSkipTrigger`: the
+                        // eventual cascade-detail message references the leftmost
+                        // not_applicable step in the phase.
+                        //
+                        // Scope note: this branch matches the canonical literal
+                        // `'not_applicable'` only. Detailed-form skip reasons that
+                        // canonicalize to not_applicable (`probe_skipped`,
+                        // `not_in_only_vectors`, `grader_skipped`,
+                        // `mcp_mode_flattens_url_edges`) carry the detailed form
+                        // on `result.skip_reason` and do NOT enter the deferred
+                        // path — they preserve the pre-fix behavior of not
+                        // tripping the cascade. `oauth_not_advertised` is handled
+                        // by the separate `phaseAbsent` code path before this
+                        // branch is reached. If a future detailed reason should
+                        // participate in substitute-aware deferral, extend this
+                        // condition deliberately rather than assuming canonical
+                        // mapping is enough.
+                        if (phasePendingNotApplicable === null) {
+                            phasePendingNotApplicable = { stepId: step.id, reason: 'not_applicable' };
+                        }
+                    }
+                }
             }
             else if (result.passed) {
                 context = result.context;
                 passedCount++;
+                // A passing stateful step counts as the phase establishing
+                // state. If a sibling skipped not_applicable earlier (or skips
+                // it later in the loop), this pass is the substitute path —
+                // the deferred cascade should NOT fire at phase end.
+                if (step.stateful) {
+                    phaseEstablishedStatefulState = true;
+                }
             }
             else {
                 phasePassed = false;
@@ -891,8 +1078,15 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
                     failedCount++;
                     countedAsFailed.add(result);
                 }
-                if (step.stateful)
+                if (step.stateful) {
                     statefulFailed = true;
+                    // Real failure takes precedence over a prior skip-trigger in
+                    // the cascade detail message — failures are the worse
+                    // diagnostic, so downstream cascade-skipped steps should
+                    // reference the failure rather than the earlier benign-ish
+                    // missing-state skip.
+                    statefulSkipTrigger = null;
+                }
                 // In multi-instance mode, annotate the failure with the cross-instance
                 // attribution block so CI readers pattern-match it as a deployment bug.
                 if (isMultiInstance) {
@@ -900,6 +1094,22 @@ async function executeStoryboardPass(agentUrls, storyboard, options, dispatchOff
                 }
             }
         }
+        // Phase-end cascade resolution for deferred `not_applicable` triggers.
+        // If a stateful step skipped not_applicable earlier in this phase and
+        // no stateful peer subsequently passed, the substitute path failed to
+        // establish state — promote to a hard cascade so downstream phases
+        // skip cleanly instead of running against absent state. A peer that
+        // passed (e.g. `list_accounts` substituting for an explicit-mode
+        // `sync_accounts`) cancels the trigger entirely. A peer that *failed*
+        // already tripped `statefulFailed` at the failure site with the
+        // worse-diagnostic real-failure message; we defer to that and don't
+        // overwrite.
+        if (phasePendingNotApplicable && !phaseEstablishedStatefulState && !statefulFailed) {
+            statefulFailed = true;
+            if (statefulSkipTrigger === null) {
+                statefulSkipTrigger = phasePendingNotApplicable;
+            }
+        }
         phaseResults.push({
             phase_id: phase.id,
             phase_title: phase.title,
@@ -1198,11 +1408,16 @@ async function runStoryboardStep(agentUrl, storyboard, stepId, options = {}) {
     const client = (0, client_1.getOrCreateClient)(agentUrl, options);
     // Discover agent profile for standalone step execution. Captured so the
     // executeStep call below can thread `library_version` through to
-    // shape-drift hint detection (issue #850).
+    // shape-drift hint detection (issue #850). Also threads _profile into
+    // options so capability-based skip gates in executeStep (e.g. account-mode
+    // branching) can read raw_capabilities, mirroring executeStoryboardPass.
     let profile;
     if (!options._client) {
         const discovered = await (0, client_1.getOrDiscoverProfile)(client, options);
         profile = discovered.profile;
+        if (profile && !options._profile) {
+            options = { ...options, _profile: profile };
+        }
     }
     else {
         profile = options._profile;
@@ -1315,6 +1530,44 @@ client, step, phaseId, context, allSteps, options, state) {
             extraction: { path: 'none' },
         };
     }
+    // Account-mode capability gate: sync_accounts is exclusive to implicit mode
+    // (require_operator_auth: false). When the seller declared explicit mode
+    // (require_operator_auth: true), sync_accounts does not apply — grade
+    // not_applicable rather than missing_tool so adopters can distinguish
+    // "your capability declaration says this path isn't yours" from "you forgot
+    // to implement a required tool."
+    //
+    // list_accounts is NOT gated here: it appears in audience_sync and other
+    // storyboard flows regardless of account mode, so it is always applicable.
+    //
+    // Requires _profile threaded from the discovery block in
+    // executeStoryboardPass or runStoryboardStep.
+    if (effectiveStep.task === 'sync_accounts') {
+        const rawCaps = options._profile?.raw_capabilities;
+        if (rawCaps !== undefined) {
+            const requireOperatorAuth = resolveCapabilityPath(rawCaps, 'account.require_operator_auth');
+            if (requireOperatorAuth === true) {
+                const detail = `Agent declared explicit account mode (require_operator_auth: true); ` +
+                    `sync_accounts is not applicable — list_accounts is the correct tool for this account shape.`;
+                const next = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);
+                return {
+                    step_id: step.id,
+                    phase_id: phaseId,
+                    title: step.title,
+                    task: step.task,
+                    passed: true,
+                    skipped: true,
+                    skip_reason: 'not_applicable',
+                    skip: buildSkip('not_applicable', detail),
+                    duration_ms: 0,
+                    validations: [],
+                    context,
+                    next,
+                    extraction: { path: 'none' },
+                };
+            }
+        }
+    }
     // Skip if agent doesn't implement the tool this step calls.
     if (options.agentTools && !options.agentTools.includes(effectiveStep.task)) {
         const next = getNextStepPreview(step.id, allSteps, context, runState.runnerVars);