@adcp/client 4.22.1 → 4.23.0

package/dist/lib/testing/compliance/comply.js

@@ -2,8 +2,10 @@
 /**
  * Compliance Engine
  *
- * Runs all applicable capability tracks against an agent
- * and reports results for every track — never stops at the first failure.
+ * Storyboard-driven compliance assessment. Storyboards are the routing
+ * mechanism; tracks are a reporting layer derived from storyboard results.
+ *
+ * Resolution priority: storyboards > platform_type > all applicable.
  */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
@@ -45,98 +47,15 @@ exports.formatComplianceResults = formatComplianceResults;
 exports.formatComplianceResultsJSON = formatComplianceResultsJSON;
 const client_1 = require("../client");
 const storyboard_tracks_1 = require("./storyboard-tracks");
+const runner_1 = require("../storyboard/runner");
+const loader_1 = require("../storyboard/loader");
+const platform_storyboards_1 = require("./platform-storyboards");
 const profiles_1 = require("./profiles");
 const mcp_1 = require("../../protocols/mcp");
 const test_controller_1 = require("../test-controller");
 /**
- * Maps each track to its constituent scenarios and a human-readable label.
- */
-const TRACK_DEFINITIONS = {
-    core: {
-        label: 'Core Protocol',
-        scenarios: [
-            'health_check',
-            'discovery',
-            'capability_discovery',
-            'schema_compliance',
-            'controller_validation',
-            'deterministic_account',
-        ],
-    },
-    products: {
-        label: 'Product Discovery',
-        scenarios: ['pricing_edge_cases', 'behavior_analysis', 'response_consistency'],
-    },
-    media_buy: {
-        label: 'Media Buy Lifecycle',
-        scenarios: [
-            'create_media_buy',
-            'full_sales_flow',
-            'creative_inline',
-            'temporal_validation',
-            'media_buy_lifecycle',
-            'terminal_state_enforcement',
-            'package_lifecycle',
-            'seller_governance_context',
-            'deterministic_media_buy',
-            'deterministic_budget',
-        ],
-    },
-    creative: {
-        label: 'Creative Management',
-        scenarios: ['creative_sync', 'creative_flow', 'deterministic_creative'],
-    },
-    reporting: {
-        label: 'Reporting',
-        scenarios: ['reporting_flow', 'deterministic_delivery'],
-    },
-    governance: {
-        label: 'Governance',
-        scenarios: ['governance_property_lists', 'governance_content_standards', 'property_list_filters'],
-    },
-    campaign_governance: {
-        label: 'Campaign Governance',
-        scenarios: [
-            'campaign_governance',
-            'campaign_governance_denied',
-            'campaign_governance_conditions',
-            'campaign_governance_delivery',
-        ],
-    },
-    signals: {
-        label: 'Signals',
-        scenarios: ['signals_flow'],
-    },
-    si: {
-        label: 'Sponsored Intelligence',
-        scenarios: ['si_session_lifecycle', 'si_availability', 'si_handoff', 'deterministic_session'],
-    },
-    audiences: {
-        label: 'Audience Management',
-        scenarios: ['sync_audiences'],
-    },
-    error_handling: {
-        label: 'Error Compliance',
-        scenarios: ['error_codes', 'error_structure', 'error_transport'],
-    },
-};
-/**
- * Which tools make a track "applicable" — if the agent has at least one
- * of these tools, the track should be attempted.
+ * All compliance tracks in display order.
  */
-const TRACK_RELEVANCE = {
-    core: [], // always applicable
-    products: ['get_products'],
-    media_buy: ['create_media_buy', 'update_media_buy', 'get_media_buys'],
-    creative: ['sync_creatives', 'build_creative', 'list_creative_formats'],
-    reporting: ['get_media_buy_delivery'],
-    governance: ['create_property_list', 'list_content_standards'],
-    campaign_governance: ['sync_plans', 'check_governance'],
-    signals: ['get_signals'],
-    si: ['si_initiate_session'],
-    audiences: ['sync_audiences'],
-    error_handling: ['create_media_buy'],
-};
 const TRACK_ORDER = [
     'core',
     'products',
@@ -150,51 +69,6 @@ const TRACK_ORDER = [
     'audiences',
     'error_handling',
 ];
-function isTrackApplicable(track, tools) {
-    const relevantTools = TRACK_RELEVANCE[track];
-    if (relevantTools.length === 0)
-        return true;
-    return relevantTools.some(t => tools.includes(t));
-}
-function isAuthError(step) {
-    if (!step.error || step.passed)
-        return false;
-    const e = step.error.toLowerCase();
-    return (e.includes('authentication') ||
-        e.includes('x-adcp-auth') ||
-        e.includes('unauthorized') ||
-        e.includes('missing auth') ||
-        e.includes('401'));
-}
-/**
- * Check if a scenario failed entirely due to auth errors.
- * Returns true if every failed step is an auth error.
- */
-function isAuthOnlyFailure(result) {
-    if (result.overall_passed)
-        return false;
-    const failedSteps = (result.steps ?? []).filter(s => !s.passed);
-    return failedSteps.length > 0 && failedSteps.every(isAuthError);
-}
-function computeTrackStatus(results, skippedCount, hasAuth) {
-    if (results.length === 0)
-        return 'skip';
-    // When running without auth, scenarios that failed only due to auth
-    // don't count as failures
-    const effectiveResults = results.map(r => {
-        if (!hasAuth && isAuthOnlyFailure(r)) {
-            return { ...r, _authSkipped: true, overall_passed: true };
-        }
-        return r;
-    });
-    const passed = effectiveResults.filter(r => r.overall_passed).length;
-    const total = effectiveResults.length;
-    if (passed === total)
-        return 'pass';
-    if (passed === 0)
-        return 'fail';
-    return 'partial';
-}
 /**
  * Collect advisory observations from test results.
  * Analyzes the actual data for quality signals that aren't pass/fail.
@@ -264,7 +138,6 @@ function collectObservations(track, results, profile) {
     // Media buy track observations
     if (track === 'media_buy') {
         // Check for valid_actions support (first match only)
-        // Only steps with observation_data are considered — snapshot-only steps don't set it.
         let checkedValidActions = false;
         for (const result of results) {
             if (checkedValidActions)
@@ -281,7 +154,6 @@ function collectObservations(track, results, profile) {
                                 'Without valid_actions, buyer agents must hardcode the state machine to know what operations are permitted.',
                         });
                     }
-                    // Check creative_deadline support
                     if (obs.has_creative_deadline === false) {
                         observations.push({
                             category: 'best_practice',
@@ -291,7 +163,6 @@ function collectObservations(track, results, profile) {
                                 'Buyers need to know when creative uploads must be finalized to avoid rejected submissions.',
                         });
                     }
-                    // Check history entry shape when present
                     if (obs.history_entries && obs.history_entries > 0 && obs.history_valid === false) {
                         observations.push({
                             category: 'best_practice',
@@ -301,7 +172,6 @@ function collectObservations(track, results, profile) {
                                 'History entries must include at least timestamp and action to be useful for audit.',
                         });
                     }
-                    // Check dry_run/sandbox confirmation
                     if (obs.sandbox === undefined || obs.sandbox === null) {
                         observations.push({
                             category: 'best_practice',
@@ -387,6 +257,15 @@ function collectObservations(track, results, profile) {
                                     'Buyers need to distinguish buyer-initiated from seller-initiated cancellations.',
                             });
                         }
+                        if (!obs.canceled_at) {
+                            observations.push({
+                                category: 'completeness',
+                                severity: 'warning',
+                                track,
+                                message: 'Agent transitions to canceled status but does not include canceled_at timestamp. ' +
+                                    'A cancellation timestamp is required for audit and reconciliation.',
+                            });
+                        }
                         checkedCancellation = true;
                     }
                 }
@@ -484,7 +363,16 @@ function collectObservations(track, results, profile) {
 }
 /**
  * Run compliance assessment against an agent.
- * Assesses all applicable tracks independently — never stops at first failure.
+ * Assesses all applicable storyboards and reports results grouped by track.
+ *
+ * Resolution priority:
+ * 1. options.storyboards — run exactly these storyboard IDs
+ * 2. options.platform_type (when tracks is not set) — resolve via PLATFORM_STORYBOARDS
+ * 3. options.tracks — run all storyboards for these tracks
+ * 4. Default — run all applicable storyboards
+ *
+ * When platform_type is set, it always drives coherence checking regardless
+ * of how the storyboard pool was resolved.
  */
 async function comply(agentUrl, options = {}) {
     try {
@@ -494,10 +382,106 @@ async function comply(agentUrl, options = {}) {
         await (0, mcp_1.closeMCPConnections)();
     }
 }
+// ────────────────────────────────────────────────────────────
+// Storyboard resolution
+// ────────────────────────────────────────────────────────────
+/**
+ * Resolve the storyboard pool based on options.
+ * Priority: storyboards > platform_type (when tracks is not set) > tracks > all bundled.
+ */
+function resolveStoryboards(options) {
+    // Explicit storyboard IDs — highest priority
+    if (options.storyboards?.length) {
+        const resolved = [];
+        for (const id of options.storyboards) {
+            const sb = (0, loader_1.getStoryboardById)(id);
+            if (!sb) {
+                throw new Error(`Unknown storyboard ID: "${id}". Use listStoryboards() to see available IDs.`);
+            }
+            resolved.push(sb);
+        }
+        return resolved;
+    }
+    // Platform type — resolve via PLATFORM_STORYBOARDS
+    if (options.platform_type && !options.tracks) {
+        const pt = options.platform_type;
+        const ids = platform_storyboards_1.PLATFORM_STORYBOARDS[pt];
+        if (ids) {
+            const resolved = [];
+            for (const id of ids) {
+                const sb = (0, loader_1.getStoryboardById)(id);
+                if (sb) {
+                    resolved.push(sb);
+                }
+                else {
+                    // Data integrity issue — storyboard declared in PLATFORM_STORYBOARDS
+                    // but not found in bundled set. This is a packaging bug.
+                    console.warn(`PLATFORM_STORYBOARDS[${pt}] references unknown storyboard "${id}"`);
+                }
+            }
+            // Also include universal storyboards (no platform_types) not already in the set
+            const resolvedIds = new Set(resolved.map(s => s.id));
+            for (const sb of (0, loader_1.loadBundledStoryboards)()) {
+                if (!sb.track)
+                    continue;
+                if (resolvedIds.has(sb.id))
+                    continue;
+                if (!sb.platform_types?.length) {
+                    resolved.push(sb);
+                }
+            }
+            return resolved;
+        }
+    }
+    // Track filter — run storyboards whose track field matches
+    if (options.tracks?.length) {
+        const trackSet = new Set(options.tracks);
+        return (0, loader_1.loadBundledStoryboards)().filter(sb => sb.track && trackSet.has(sb.track));
+    }
+    // Default — all compliance storyboards (those with a track field)
+    return (0, loader_1.loadBundledStoryboards)().filter(sb => sb.track);
+}
+/**
+ * Filter storyboards to those applicable for the agent's tools.
+ * A storyboard is applicable if the agent has at least one of its required_tools,
+ * or if it has no required_tools at all.
+ */
+function filterApplicable(storyboards, agentTools) {
+    return storyboards.filter(sb => {
+        if (!sb.required_tools?.length)
+            return true;
+        return sb.required_tools.some(tool => agentTools.includes(tool));
+    });
+}
+/**
+ * Group storyboard results by track.
+ */
+function groupByTrack(results, storyboards) {
+    // Build a storyboard ID → track lookup
+    const trackLookup = new Map();
+    for (const sb of storyboards) {
+        if (sb.track) {
+            trackLookup.set(sb.id, sb.track);
+        }
+    }
+    const grouped = new Map();
+    for (const result of results) {
+        const track = trackLookup.get(result.storyboard_id);
+        if (!track)
+            continue;
+        if (!grouped.has(track))
+            grouped.set(track, []);
+        grouped.get(track).push(result);
+    }
+    return grouped;
+}
+// ────────────────────────────────────────────────────────────
+// Core implementation
+// ────────────────────────────────────────────────────────────
 async function complyImpl(agentUrl, options) {
     const start = Date.now();
-    const { tracks: trackFilter, platform_type, timeout_ms, signal: externalSignal, ...testOptions } = options;
-    // Validate platform_type if provided (issue #402: accept string, validate internally)
+    const { storyboards: _storyboardIds, tracks: _trackFilter, platform_type, timeout_ms, signal: externalSignal, ...testOptions } = options;
+    // Validate platform_type if provided
     let platformProfile;
     if (platform_type) {
         const validTypes = (0, profiles_1.getAllPlatformTypes)();
@@ -537,14 +521,14 @@ async function complyImpl(agentUrl, options) {
         };
         // Check for abort before starting
         signal?.throwIfAborted();
-        // Collect observations across all tracks (declared early for tool discovery diagnostics)
+        // Collect observations across all tracks
         const allObservations = [];
-        // Discover agent capabilities once and share across all scenarios
+        // Discover agent capabilities once and share across all storyboards
         const client = (0, client_1.createTestClient)(agentUrl, effectiveOptions.protocol ?? 'mcp', effectiveOptions);
         const { profile, step: profileStep } = await (0, client_1.discoverAgentProfile)(client);
         effectiveOptions._client = client;
         effectiveOptions._profile = profile;
-        // Log discovered tools for diagnostic purposes
+        // Log discovered tools
         if (profileStep.passed) {
             allObservations.push({
                 category: 'tool_discovery',
@@ -562,145 +546,62 @@ async function complyImpl(agentUrl, options) {
             }
         }
         if (!profileStep.passed) {
-            const errorMsg = profileStep.error || 'Unknown error';
-            const observations = [];
-            // Check for auth errors — either explicit 401/Unauthorized or MCP SDK's generic
-            // "Failed to discover" which often wraps a 401
-            const isExplicitAuthError = errorMsg.includes('401') ||
-                errorMsg.includes('Unauthorized') ||
-                errorMsg.includes('unauthorized') ||
-                errorMsg.includes('authentication') ||
-                errorMsg.includes('JWS') ||
-                errorMsg.includes('JWT') ||
-                errorMsg.includes('signature verification');
-            // When MCP SDK wraps the error, probe the endpoint directly
-            let isAuthError = isExplicitAuthError;
-            if (!isAuthError && errorMsg.includes('Failed to discover')) {
-                try {
-                    const probe = await fetch(agentUrl, {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        signal,
-                    });
-                    if (probe.status === 401 || probe.status === 403) {
-                        isAuthError = true;
-                    }
-                }
-                catch {
-                    // Network error — not an auth issue
-                }
-            }
-            const headline = isAuthError ? `Authentication required` : `Agent unreachable — ${errorMsg}`;
-            if (isAuthError) {
-                // Check if agent supports OAuth
-                const { discoverOAuthMetadata } = await Promise.resolve().then(() => __importStar(require('../../auth/oauth/discovery')));
-                const oauthMeta = await discoverOAuthMetadata(agentUrl);
-                if (oauthMeta) {
-                    observations.push({
-                        category: 'auth',
-                        severity: 'error',
-                        message: `Agent requires OAuth (issuer: ${oauthMeta.issuer || 'unknown'}). Save credentials: adcp --save-auth <alias> ${agentUrl} --oauth`,
-                    });
-                }
-                else {
-                    observations.push({
-                        category: 'auth',
-                        severity: 'error',
-                        message: 'Agent returned 401. Check your --auth token.',
-                    });
-                }
-            }
-            return {
-                agent_url: agentUrl,
-                agent_profile: profile,
-                overall_status: (isAuthError ? 'auth_required' : 'unreachable'),
-                tracks: [],
-                tested_tracks: [],
-                skipped_tracks: [],
-                expected_tracks: [],
-                summary: {
-                    tracks_passed: 0,
-                    tracks_failed: 0,
-                    tracks_skipped: 0,
-                    tracks_partial: 0,
-                    tracks_expected: 0,
-                    headline,
-                },
-                observations,
-                tested_at: new Date().toISOString(),
-                total_duration_ms: Date.now() - start,
-                dry_run: effectiveOptions.dry_run !== false,
-            };
+            return buildUnreachableResult(agentUrl, profile, profileStep.error, start, effectiveOptions, signal);
         }
-        const tracksToRun = trackFilter ?? TRACK_ORDER;
-        const trackResults = [];
-        for (const track of tracksToRun) {
-            // Check for abort between tracks
+        // Resolve and filter storyboard pool
+        const allStoryboards = resolveStoryboards(options);
+        const applicableStoryboards = filterApplicable(allStoryboards, profile.tools);
+        // Run storyboards
+        const storyboardResults = [];
+        const runOptions = {
+            ...effectiveOptions,
+            agentTools: profile.tools,
+        };
+        for (const sb of applicableStoryboards) {
             signal?.throwIfAborted();
-            const def = TRACK_DEFINITIONS[track];
-            if (!def)
+            const result = await (0, runner_1.runStoryboard)(agentUrl, sb, runOptions);
+            storyboardResults.push(result);
+        }
+        // Group results by track and build TrackResults
+        const grouped = groupByTrack(storyboardResults, applicableStoryboards);
+        const trackResults = [];
+        // Determine which tracks had storyboards in the pool (even if filtered out by tools)
+        const poolTrackSet = new Set();
+        for (const sb of allStoryboards) {
+            if (sb.track)
+                poolTrackSet.add(sb.track);
+        }
+        for (const track of TRACK_ORDER) {
+            if (!poolTrackSet.has(track))
                 continue;
-            if (!isTrackApplicable(track, profile.tools)) {
+            const results = grouped.get(track) ?? [];
+            if (results.length > 0) {
+                const trackResult = (0, storyboard_tracks_1.mapStoryboardResultsToTrackResult)(track, results, profile);
+                const observations = collectObservations(track, trackResult.scenarios, profile);
+                trackResult.observations = observations;
+                allObservations.push(...observations);
+                trackResults.push(trackResult);
+            }
+            else {
+                // Track was in the pool but no storyboards ran (agent lacks tools)
                 const isExpected = track !== 'core' && (platformProfile?.expected_tracks.includes(track) ?? false);
-                const requiredTools = TRACK_RELEVANCE[track];
-                const trackObservations = [];
-                if (requiredTools.length > 0) {
-                    trackObservations.push({
-                        category: 'tool_discovery',
-                        severity: isExpected ? 'warning' : 'info',
-                        message: `Track "${track}" skipped: agent does not advertise any of [${requiredTools.join(', ')}]. ` +
-                            `Agent tools: [${profile.tools.join(', ')}]`,
-                        evidence: { expected_tools: requiredTools, agent_tool_count: profile.tools.length },
-                    });
-                }
-                allObservations.push(...trackObservations);
                 trackResults.push({
                     track,
                     status: isExpected ? 'expected' : 'skip',
-                    label: def.label,
-                    scenarios: [],
-                    skipped_scenarios: def.scenarios,
-                    observations: trackObservations,
-                    duration_ms: 0,
-                });
-                continue;
-            }
-            const trackStart = Date.now();
-            // Run compliance storyboards for this track
-            const storyboardResults = await (0, storyboard_tracks_1.runTrackStoryboards)(agentUrl, track, profile.tools, {
-                ...effectiveOptions,
-                agentTools: profile.tools,
-            });
-            let trackResult;
-            if (storyboardResults.length > 0) {
-                // Map storyboard results to TrackResult for backwards compat
-                trackResult = (0, storyboard_tracks_1.mapStoryboardResultsToTrackResult)(track, storyboardResults, profile);
-            }
-            else {
-                // No storyboards for this track — skip
-                trackResult = {
-                    track,
-                    status: 'skip',
-                    label: def.label,
+                    label: storyboard_tracks_1.TRACK_LABELS[track] || track,
                     scenarios: [],
                     skipped_scenarios: [],
                     observations: [],
                     duration_ms: 0,
-                };
+                });
             }
-            // Collect observations from track results and agent profile
-            const observations = collectObservations(track, trackResult.scenarios, profile);
-            trackResult.observations = observations;
-            trackResult.duration_ms = Date.now() - trackStart;
-            allObservations.push(...observations);
-            trackResults.push(trackResult);
         }
         // Build platform coherence result if platform type was declared
         let platformCoherence;
         if (platformProfile) {
             const findings = platformProfile.checkCoherence(profile);
-            const missingTracks = platformProfile.expected_tracks.filter(t => !isTrackApplicable(t, profile.tools) && t !== 'core');
-            // Add coherence findings as observations
+            const applicableTrackSet = new Set(trackResults.filter(t => t.status !== 'skip' && t.status !== 'expected').map(t => t.track));
+            const missingTracks = platformProfile.expected_tracks.filter(t => !applicableTrackSet.has(t) && t !== 'core');
             for (const finding of findings) {
                 allObservations.push({
                     category: 'coherence',
@@ -720,18 +621,14 @@ async function complyImpl(agentUrl, options) {
             };
         }
         const summary = buildSummary(trackResults);
-        // Partition tracks by disposition (issue #403)
         const testedTracks = trackResults.filter(t => t.status === 'pass' || t.status === 'fail' || t.status === 'partial');
         const skippedTracks = trackResults
             .filter(t => t.status === 'skip')
-            .map(t => {
-            const required = TRACK_RELEVANCE[t.track];
-            return {
-                track: t.track,
-                label: t.label,
-                reason: required.length > 0 ? `Agent lacks required tools: ${required.join(', ')}` : 'Agent lacks required tools',
-            };
-        });
+            .map(t => ({
+            track: t.track,
+            label: t.label,
+            reason: 'Agent lacks required tools for applicable storyboards',
+        }));
         const expectedTracks = trackResults
             .filter(t => t.status === 'expected')
             .map(t => ({
@@ -739,7 +636,6 @@ async function complyImpl(agentUrl, options) {
             label: t.label,
             reason: `Expected for ${platformCoherence?.label ?? 'declared platform type'}`,
         }));
-        // Compute overall status (issue #401)
         const overallStatus = computeOverallStatus(summary);
         return {
             agent_url: agentUrl,
@@ -767,6 +663,76 @@ async function complyImpl(agentUrl, options) {
         }
     }
 }
+/**
+ * Build result for an unreachable or auth-required agent.
+ */
+async function buildUnreachableResult(agentUrl, profile, errorMsg, start, effectiveOptions, signal) {
+    const err = errorMsg || 'Unknown error';
+    const observations = [];
+    const isExplicitAuthError = err.includes('401') ||
+        err.includes('Unauthorized') ||
+        err.includes('unauthorized') ||
+        err.includes('authentication') ||
+        err.includes('JWS') ||
+        err.includes('JWT') ||
+        err.includes('signature verification');
+    let isAuthError = isExplicitAuthError;
+    if (!isAuthError && err.includes('Failed to discover')) {
+        try {
+            const probe = await fetch(agentUrl, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                signal,
+            });
+            if (probe.status === 401 || probe.status === 403) {
+                isAuthError = true;
+            }
+        }
+        catch {
+            // Network error — not an auth issue
+        }
+    }
+    const headline = isAuthError ? `Authentication required` : `Agent unreachable — ${err}`;
+    if (isAuthError) {
+        const { discoverOAuthMetadata } = await Promise.resolve().then(() => __importStar(require('../../auth/oauth/discovery')));
+        const oauthMeta = await discoverOAuthMetadata(agentUrl);
+        if (oauthMeta) {
+            observations.push({
+                category: 'auth',
+                severity: 'error',
+                message: `Agent requires OAuth (issuer: ${oauthMeta.issuer || 'unknown'}). Save credentials: adcp --save-auth <alias> ${agentUrl} --oauth`,
+            });
+        }
+        else {
+            observations.push({
+                category: 'auth',
+                severity: 'error',
+                message: 'Agent returned 401. Check your --auth token.',
+            });
+        }
+    }
+    return {
+        agent_url: agentUrl,
+        agent_profile: profile,
+        overall_status: (isAuthError ? 'auth_required' : 'unreachable'),
+        tracks: [],
+        tested_tracks: [],
+        skipped_tracks: [],
+        expected_tracks: [],
+        summary: {
+            tracks_passed: 0,
+            tracks_failed: 0,
+            tracks_skipped: 0,
+            tracks_partial: 0,
+            tracks_expected: 0,
+            headline,
+        },
+        observations,
+        tested_at: new Date().toISOString(),
+        total_duration_ms: Date.now() - start,
+        dry_run: effectiveOptions.dry_run !== false,
+    };
+}
 /**
  * Compute overall status for a reachable agent.
  * auth_required and unreachable are set directly in the early-exit path.