@adcp/sdk 16.4.0 → 16.5.0

package/dist/lib/testing/client.mjs

@@ -1,10 +1,11 @@
+import { hasUnsignedCapabilityBootstrap } from "../signing/protocol-methods.mjs";
 import { ADCPMultiAgentClient } from "../core/ADCPMultiAgentClient.mjs";
 import { getBestUnionErrors } from "../utils/union-errors.mjs";
 import { getFormatAssets, usesDeprecatedAssetsField } from "../utils/format-assets.mjs";
 import { brandManifestToBrandReference } from "../types/compat.mjs";
 import { prepareResponseForSchemaValidation, TOOL_RESPONSE_SCHEMAS } from "../utils/response-schemas.mjs";
 import { injectLegacyEnvelopeStatus } from "../utils/envelope-status-compat.mjs";
-import { parseCapabilitiesResponse } from "../utils/capabilities.mjs";
+import { parseCapabilitiesResponse, looksLikeV3Capabilities } from "../utils/capabilities.mjs";
 import { classifyProbeUrl } from "../utils/probe-policy.mjs";
 import { SsrfRefusedError } from "../net/ssrf-fetch.mjs";
 import { ADCP_VERSION, toReleasePrecisionVersion } from "../version.mjs";
@@ -16,6 +17,13 @@ import { withServedValidationVersion } from "../validation/schema-loader.mjs";
 import { extractVersionUnsupportedDetails } from "../utils/error-extraction.mjs";
 import { buildAgentSigningContext } from "../signing/client.mjs";
 import { bindAgentStorage } from "../auth/oauth/storage-registry.mjs";
+import { isTerminalAdcpError, ResponseSchemaValidationError } from "../utils/response-unwrapper.mjs";
+import { redactDiscoveryEvidence } from "./discovery-evidence.mjs";
+import { redactCredentialPatterns } from "../utils/redact-credential-patterns.mjs";
+import {
+  NeedsAuthorizationError,
+  hasValidatedMcpAuthorizationRequirements
+} from "../auth/oauth/authorization-required.mjs";
 const TEST_CLIENT_VERSION_OPTIONS = /* @__PURE__ */ Symbol("adcp.testClientVersionOptions");
 const TEST_CLIENT_SCOPE_SALT = randomBytes(32);
 const testClientScope = new AsyncLocalStorage();
@@ -161,6 +169,8 @@ function createTestClient(agentUrl, protocol = "mcp", options = {}) {
       ...requestScopeMode !== void 0 && { requestScopeMode },
       ...requestSigningMode !== void 0 && { requestSigningMode },
       ...options.auth?.type === "oauth" && options.auth.storage && { oauthStorage: options.auth.storage },
+      connectionPartition: transport?.connectionPartition,
+      connectionReuse: transport?.connectionReuse,
       ...transport?.trustedFetchFn && { fetchFn: transport.trustedFetchFn },
       ...transport?.allowPrivateIp !== void 0 && { allowPrivateIp: transport.allowPrivateIp },
       ...transport?.maxResponseBytes !== void 0 && {
@@ -210,7 +220,7 @@ function testClientMatchesVersionOptions(client, agentUrl, options) {
   const expectedAdcpVersion = effectiveOptions.adcpVersion ?? ADCP_VERSION;
   const expectedWireAdcpVersion = effectiveOptions.wireAdcpVersion;
   const expectedVersionEnvelope = effectiveOptions.versionEnvelope ?? "auto";
-  return meta.agentUrl === agentUrl && meta.protocol === (effectiveOptions.protocol ?? "mcp") && meta.adcpVersion === expectedAdcpVersion && meta.wireAdcpVersion === expectedWireAdcpVersion && meta.versionEnvelope === expectedVersionEnvelope && meta.strictResponseSchemaValidation === (effectiveOptions.strictResponseSchemaValidation !== false) && meta.authMode === expectedAuthMode && meta.requestScopeMode === expectedRequestScopeMode && meta.requestSigningMode === expectedRequestSigningMode && meta.oauthStorage === expectedOAuthStorage && meta.fetchFn === expectedTransport?.trustedFetchFn && meta.allowPrivateIp === expectedTransport?.allowPrivateIp && meta.maxResponseBytes === expectedTransport?.maxResponseBytes && meta.requestTimeoutMs === expectedTransport?.requestTimeoutMs && meta.legacyCompatEnabled === expectedTransport?.legacyCompat?.enabled;
+  return meta.agentUrl === agentUrl && meta.protocol === (effectiveOptions.protocol ?? "mcp") && meta.adcpVersion === expectedAdcpVersion && meta.wireAdcpVersion === expectedWireAdcpVersion && meta.versionEnvelope === expectedVersionEnvelope && meta.strictResponseSchemaValidation === (effectiveOptions.strictResponseSchemaValidation !== false) && meta.authMode === expectedAuthMode && meta.requestScopeMode === expectedRequestScopeMode && meta.requestSigningMode === expectedRequestSigningMode && meta.oauthStorage === expectedOAuthStorage && meta.connectionPartition === expectedTransport?.connectionPartition && meta.connectionReuse === expectedTransport?.connectionReuse && meta.fetchFn === expectedTransport?.trustedFetchFn && meta.allowPrivateIp === expectedTransport?.allowPrivateIp && meta.maxResponseBytes === expectedTransport?.maxResponseBytes && meta.requestTimeoutMs === expectedTransport?.requestTimeoutMs && meta.legacyCompatEnabled === expectedTransport?.legacyCompat?.enabled;
 }
 function withTestKitAuthDefaults(options) {
   if (options.auth) return options;
@@ -283,7 +293,9 @@ function requestSigningReuseMode(config) {
       agentUrl: config.agent_url,
       jwksUri: config.jwks_uri,
       signSupported: config.sign_supported === true,
-      alwaysSign: config.always_sign ?? []
+      alwaysSign: config.always_sign ?? [],
+      alwaysSignProtocolMethods: config.always_sign_protocol_methods ?? [],
+      unsignedCapabilityBootstrap: hasUnsignedCapabilityBootstrap(config)
     })
   ).digest("hex").slice(0, 16);
   return `${kind}:${fingerprint}`;
@@ -294,7 +306,7 @@ async function getOrDiscoverProfile(client, options, schemaAdcpVersion) {
   schemaAdcpVersion ??= profileBinding?.schemaAdcpVersion ?? options.adcpVersion;
   options.signal?.throwIfAborted();
   const scopeKey = scope?.active ? client.getCapabilityEvidenceScope().scopeKey : void 0;
-  if (options._profile && (!scope?.active || scope.authRejectedProfiles.has(options._profile) || !options._profile.capabilities_probe_error && options._client === client && profileBinding?.scopeKey === scopeKey)) {
+  if (options._profile && (!scope?.active || scope.authRejectedProfiles.has(options._profile) || !options._profile.capabilities_probe_error && !options._profile.capabilities_schema_issues?.length && options._client === client && profileBinding?.scopeKey === scopeKey)) {
     seedTestClientSigningCapability(client, options._profile, schemaAdcpVersion);
     return {
       profile: options._profile,
@@ -321,7 +333,7 @@ async function getOrDiscoverProfile(client, options, schemaAdcpVersion) {
       (result) => {
         if (options.signal?.aborted) {
           evict();
-        } else if (!result.step.passed || result.profile.capabilities_probe_error) {
+        } else if (!result.step.passed || describeProfileDiscoveryFailure(result.profile)) {
           if (options._discoveryPass && discovery) discovery.failed = true;
           else evict();
         }
@@ -361,6 +373,7 @@ function seedTestClientSigningCapability(client, profile, adcpVersion) {
     }
     profiles.set(profile, { scopeKey: client.getCapabilityEvidenceScope().scopeKey, schemaAdcpVersion: adcpVersion });
   }
+  if (describeProfileDiscoveryFailure(profile)) return;
   const raw = profile.raw_capabilities;
   if (!raw || typeof raw !== "object") return;
   const clientAccess = client;
@@ -368,7 +381,10 @@ function seedTestClientSigningCapability(client, profile, adcpVersion) {
   const agent = clientAccess.getAgent.call(client);
   const meta = client[TEST_CLIENT_VERSION_OPTIONS];
   const effectiveVersion = meta?.wireAdcpVersion ?? meta?.adcpVersion ?? (typeof clientAccess.getAdcpVersion === "function" ? clientAccess.getAdcpVersion.call(client) : void 0) ?? adcpVersion;
-  const signingContext = buildAgentSigningContext(agent, { adcpVersion: effectiveVersion });
+  const signingContext = buildAgentSigningContext(agent, {
+    adcpVersion: effectiveVersion,
+    connectionPartition: meta?.connectionPartition
+  });
   if (!signingContext) return;
   signingContext.cache.set(signingContext.capabilityCacheKey, {
     requestSigning: raw.request_signing,
@@ -451,17 +467,44 @@ function describeVersionUnsupported(result, requestedVersion) {
   const supportedText = supported?.length ? `; seller supports ${supported.map((version) => JSON.stringify(version)).join(", ")}` : "; seller did not report supported_versions";
   return `VERSION_UNSUPPORTED: ${requestedText}${supportedText}`;
 }
+async function getOrDiscoverStoryboardProfile(client, options, schemaAdcpVersion) {
+  const result = await getOrDiscoverProfile(client, options, schemaAdcpVersion);
+  const failure = describeProfileDiscoveryFailure(
+    result.profile,
+    options._requireValidCapabilityEvidence === true,
+    options._allowCapabilityDiagnostic === true
+  );
+  return failure ? { ...result, step: failure } : result;
+}
+function describeProfileDiscoveryFailure(profile, requireValidSchema = true, allowCapabilityDiagnostic = false) {
+  if (testClientScope.getStore()?.authRejectedProfiles.has(profile)) return void 0;
+  const raw = profile.raw_capabilities;
+  if (allowCapabilityDiagnostic && profile.tools.includes("get_adcp_capabilities") && profile.capabilities_failure?.task === "get_adcp_capabilities" && !profile.capabilities_probe_error?.startsWith("VERSION_UNSUPPORTED:"))
+    return void 0;
+  const schemaOnly = !profile.capabilities_probe_error && Boolean(profile.capabilities_schema_issues?.length);
+  const error = profile.capabilities_probe_error ?? ((!schemaOnly || requireValidSchema) && profile.capabilities_failure?.passed === false ? profile.capabilities_failure.error ?? "Capability discovery failed" : void 0) ?? (requireValidSchema && profile.capabilities_schema_issues?.length ? `get_adcp_capabilities response schema validation failed: ${profile.capabilities_schema_issues.map((issue) => `${issue.pointer || "/"} ${issue.message}`).join("; ")}` : void 0) ?? (raw !== void 0 && (raw === null || typeof raw !== "object" || Array.isArray(raw)) ? "Supplied profile.raw_capabilities must be an object" : void 0) ?? (isTerminalAdcpError(raw, "get_adcp_capabilities") ? "Supplied capability evidence contains an AdCP error" : void 0);
+  if (!error) return void 0;
+  return {
+    ...redactDiscoveryEvidence(profile.capabilities_failure),
+    step: "Discover agent capabilities",
+    task: profile.capabilities_failure?.task ?? "get_adcp_capabilities",
+    passed: false,
+    duration_ms: profile.capabilities_failure?.duration_ms ?? 0,
+    error: redactCredentialPatterns(error, false).slice(0, 4096)
+  };
+}
 async function discoverAgentProfile(client, signal, schemaAdcpVersion) {
   const scopeKey = hasTestClientScope() ? client.getCapabilityEvidenceScope().scopeKey : void 0;
   const {
     result: agentInfo,
     step,
-    caughtError
+    caughtError: infoError
   } = await runStep(
     "Discover agent capabilities",
     "getAgentInfo",
     () => raceWithSignal(client.getAgentInfo({ signal }), signal)
   );
+  let caughtError = infoError;
   const profile = {
     name: agentInfo?.name || "Unknown",
     tools: agentInfo?.tools?.map((t) => t.name) || []
@@ -477,14 +520,37 @@ async function discoverAgentProfile(client, signal, schemaAdcpVersion) {
       2
     );
   }
-  if (profile.tools.includes("get_adcp_capabilities")) {
+  if (step.passed && profile.tools.includes("get_adcp_capabilities")) {
+    const probeStart = Date.now();
+    step.request = {
+      transport: client.getAgent?.().protocol === "a2a" ? "a2a" : "mcp",
+      operation: "get_adcp_capabilities",
+      payload: {}
+    };
     try {
-      const caps = await raceWithSignal(client.getAdcpCapabilities({}, void 0, { signal }), signal);
+      const caps = await raceWithSignal(
+        client.getAdcpCapabilities({}, void 0, { signal }),
+        signal
+      );
+      if (caps.errorInstance instanceof NeedsAuthorizationError) caughtError = caps.errorInstance;
+      const observed = caps.debug_logs?.slice().reverse().find(
+        (log) => typeof log.message === "string" && (log.message.startsWith("MCP: Tool get_adcp_capabilities response received") || log.message.startsWith("A2A: Response received")) && log.response !== void 0
+      )?.response;
+      if (observed !== void 0)
+        step.response_record = {
+          transport: step.request.transport,
+          payload: redactDiscoveryEvidence(observed),
+          duration_ms: Date.now() - probeStart
+        };
+      const adcpError = caps.adcpError ?? caps.adcp_error;
+      if (adcpError) step.adcp_error = redactDiscoveryEvidence(adcpError);
       const versionUnsupported = describeVersionUnsupported(caps, testClientWireAdcpVersion(client, schemaAdcpVersion));
       if (versionUnsupported) {
         profile.capabilities_probe_error = versionUnsupported;
         step.passed = false;
         step.error = versionUnsupported;
+      } else if (isTerminalAdcpError(caps.data, "get_adcp_capabilities") || !caps?.success && !looksLikeV3Capabilities(caps.data)) {
+        profile.capabilities_probe_error = caps.error ?? adcpError?.message ?? "get_adcp_capabilities was rejected";
       } else if (caps?.data) {
         profile.raw_capabilities = caps.data;
         const validation = withServedValidationVersion(
@@ -498,31 +564,57 @@ async function discoverAgentProfile(client, signal, schemaAdcpVersion) {
             message: issue.message
           }));
         }
-        const parsed = parseCapabilitiesResponse(caps.data);
-        profile.adcp_version = parsed.version;
-        profile.adcp_major_versions = parsed.majorVersions;
-        if (parsed.supportedVersions !== void 0) profile.adcp_supported_versions = parsed.supportedVersions;
-        if (parsed.buildVersion !== void 0) profile.adcp_build_version = parsed.buildVersion;
-        profile.supported_protocols = parsed.protocols;
-        profile.supports_governance = parsed.protocols.includes("governance");
-        profile.supports_si = parsed.protocols.includes("sponsored_intelligence");
-        const specialisms = caps.data.specialisms;
-        if (Array.isArray(specialisms)) {
-          profile.specialisms = specialisms.filter((s) => typeof s === "string");
+        if (looksLikeV3Capabilities(caps.data)) {
+          const parsed = parseCapabilitiesResponse(caps.data);
+          profile.adcp_version = parsed.version;
+          profile.adcp_major_versions = parsed.majorVersions;
+          if (parsed.supportedVersions !== void 0) profile.adcp_supported_versions = parsed.supportedVersions;
+          if (parsed.buildVersion !== void 0) profile.adcp_build_version = parsed.buildVersion;
+          profile.supported_protocols = parsed.protocols;
+          profile.supports_governance = parsed.protocols.includes("governance");
+          profile.supports_si = parsed.protocols.includes("sponsored_intelligence");
+          const specialisms = caps.data.specialisms;
+          if (Array.isArray(specialisms)) {
+            profile.specialisms = specialisms.filter((s) => typeof s === "string");
+          }
+          const libVersion = caps.data.library_version;
+          if (typeof libVersion === "string") profile.library_version = libVersion;
         }
-        const libVersion = caps.data.library_version;
-        if (typeof libVersion === "string") profile.library_version = libVersion;
       }
-      if (!versionUnsupported && (!caps?.success || !caps?.data) && !profile.capabilities_schema_issues?.length) {
+      if (!versionUnsupported && (!caps?.success || !caps?.data) && !profile.capabilities_schema_issues?.length && !profile.capabilities_probe_error) {
         profile.capabilities_probe_error = caps?.error || "get_adcp_capabilities returned no data";
       }
     } catch (err) {
+      caughtError = err;
       profile.capabilities_probe_error = err?.message || String(err);
+      if (err instanceof ResponseSchemaValidationError) {
+        step.response_record = { transport: step.request.transport, payload: redactDiscoveryEvidence(err.data) };
+      }
+    } finally {
+      step.duration_ms += Date.now() - probeStart;
     }
   }
-  if (scopeKey === void 0 || scopeKey === client.getCapabilityEvidenceScope().scopeKey) {
+  if (caughtError instanceof NeedsAuthorizationError && typeof client.getAgent === "function" && hasValidatedMcpAuthorizationRequirements(caughtError.requirements, client.getAgent().agent_uri)) {
+    step.authorization_required = redactDiscoveryEvidence(caughtError.requirements);
+  }
+  const failure = describeProfileDiscoveryFailure(profile);
+  if (!step.passed || failure) {
+    const evidence = { ...step, ...failure ?? {}, passed: false, duration_ms: step.duration_ms };
+    evidence.error = redactCredentialPatterns(evidence.error ?? "Agent discovery failed", false).slice(
+      0,
+      4096
+    );
+    profile.capabilities_failure = evidence;
+  }
+  if (!step.passed) step.error = redactCredentialPatterns(step.error ?? "Agent discovery failed", false);
+  if (!failure && step.passed && (scopeKey === void 0 || scopeKey === client.getCapabilityEvidenceScope().scopeKey)) {
     seedTestClientSigningCapability(client, profile, schemaAdcpVersion);
   }
+  if (step.passed && !failure) {
+    delete step.request;
+    delete step.response_record;
+    delete step.adcp_error;
+  }
   return { profile, step, ...caughtError !== void 0 && { caughtError } };
 }
 async function discoverAgentCapabilities(client, profile, options) {
@@ -823,6 +915,7 @@ function zodCodeToKeyword(code) {
 }
 export {
   createTestClient,
+  describeProfileDiscoveryFailure,
   describeVersionUnsupported,
   discoverAgentCapabilities,
   discoverAgentProfile,
@@ -832,6 +925,7 @@ export {
   getOrCreateClient,
   getOrCreateClientResolution,
   getOrDiscoverProfile,
+  getOrDiscoverStoryboardProfile,
   hasTestClientScope,
   resolveAccount,
   resolveAuthPrincipal,

package/dist/lib/testing/compliance/comply.js

@@ -38,6 +38,7 @@ var import_runner = require('../storyboard/runner.js');
 var import_native_a2a_compliance = require('../storyboard/native-a2a-compliance.js');
 var import_test_kit = require('../storyboard/test-kit.js');
 var import_protected_context = require('../storyboard/protected-context.js');
+var import_prepared_fixtures = require('../storyboard/prepared-fixtures.js');
 var import_spec_conformance = require('./spec-conformance.js');
 var import_default_invariants = require('../storyboard/default-invariants.js');
 var import_agent_routing = require('../storyboard/agent-routing.js');
@@ -419,6 +420,7 @@ function collectObservations(track, results, profile) {
   return observations;
 }
 async function comply(agentUrl, options = {}) {
+  (0, import_prepared_fixtures.assertNoPreparationAuthorizer)(options);
   if ("platform_type" in options) {
     throw new Error(
       'comply() no longer accepts platform_type. Agent selection is now driven by get_adcp_capabilities (supported_protocols + specialisms). Pass `storyboards: ["<bundle-or-id>"]` to target a specific bundle. See the changeset for migration notes.'
@@ -925,6 +927,13 @@ async function runComplyStoryboard(agentUrl, storyboard, runOptions, profile, ro
   } else {
     runOptions = { ...runOptions, _profile: void 0, agentTools: void 0 };
   }
+  if ((0, import_client.describeProfileDiscoveryFailure)(
+    discovered.profile,
+    (0, import_runner.storyboardRequiresCapabilityEvidence)(storyboard),
+    (0, import_runner.storyboardAllowsCapabilityDiagnostic)(storyboard)
+  )) {
+    return (0, import_runner.runStoryboard)(agentUrl, storyboard, { ...runOptions, _profile: discovered.profile });
+  }
   if (!routeStoryboard || (0, import_runner.storyboardCapabilityGateUnmet)(storyboard, runOptions) !== null) {
     return (0, import_runner.runStoryboard)(agentUrl, storyboard, runOptions);
   }
@@ -1283,16 +1292,15 @@ function extractFailures(results, storyboards, agentRef, fixOptions = {}) {
       }
     }
     for (const { assertion, owningStep } of (0, import_storyboard_tracks.collectDetachedAssertionFailures)(result)) {
-      if (!owningStep) continue;
       failures.push({
         track,
         storyboard_id: result.storyboard_id,
-        step_id: owningStep.step_id,
-        step_title: owningStep.title,
-        task: owningStep.task,
+        step_id: owningStep?.step_id ?? `${result.storyboard_id}/storyboard`,
+        step_title: owningStep?.title ?? sb?.title ?? result.storyboard_id,
+        task: owningStep?.task ?? "storyboard",
         error: assertion.error ?? assertion.description,
-        expected: findStoryboardStepExpected(sb, owningStep.step_id),
-        fix_command: buildFixCommand(agentRef, result.storyboard_id, owningStep.step_id, fixOptions),
+        expected: owningStep ? findStoryboardStepExpected(sb, owningStep.step_id) : void 0,
+        fix_command: buildFixCommand(agentRef, result.storyboard_id, owningStep?.step_id, fixOptions),
         validation: {
           check: "assertion",
           description: `${assertion.assertion_id}: ${assertion.description}`
@@ -1311,7 +1319,7 @@ function findStoryboardStepExpected(storyboard, stepId) {
   return void 0;
 }
 function buildFixCommand(agentRef, storyboardId, stepId, options) {
-  const parts = ["adcp", "storyboard", "step", agentRef, storyboardId, stepId, "--json"];
+  const parts = stepId === void 0 ? ["adcp", "storyboard", "run", agentRef, storyboardId, "--json"] : ["adcp", "storyboard", "step", agentRef, storyboardId, stepId, "--json"];
   if (options.complianceVersion) parts.push("--compliance-version", options.complianceVersion);
   if (options.complianceDir) parts.push("--compliance-dir", options.complianceDir);
   if (options.schemaRoot) parts.push("--schema-root", options.schemaRoot);
@@ -1444,7 +1452,7 @@ async function complyImpl(agentUrl, options) {
         allObservations.push({
           category: "tool_discovery",
           severity: "error",
-          message: `get_adcp_capabilities is advertised but the call failed. Only universal storyboards ran \u2014 domain and specialism bundles were skipped. Agent-reported error: ${fenceAgentText(profile.capabilities_probe_error)}`,
+          message: `get_adcp_capabilities is advertised but the call failed. Universal diagnostics were selected; failed discovery does not establish domain or specialism support. Agent-reported error: ${fenceAgentText(profile.capabilities_probe_error)}`,
           evidence: { agent_reported_error: profile.capabilities_probe_error },
           source: { kind: "profile", code: "capabilities-probe-failed" }
         });

package/dist/lib/testing/compliance/comply.mjs

@@ -2,6 +2,7 @@ import {
   createTestClient,
   getOrCreateClientResolution,
   getOrDiscoverProfile,
+  describeProfileDiscoveryFailure,
   seedTestClientSigningCapability,
   seedAuthRejectedTestClientProfile,
   withTestClientScope
@@ -12,11 +13,14 @@ import {
   collectCapabilityNotices,
   runStoryboard,
   storyboardCapabilityGateUnmet,
+  storyboardRequiresCapabilityEvidence,
+  storyboardAllowsCapabilityDiagnostic,
   storyboardCapabilityPredicates
 } from "../storyboard/runner.mjs";
 import { applyNativeA2AComplianceTransportOptions } from "../storyboard/native-a2a-compliance.mjs";
 import { validateTestKit } from "../storyboard/test-kit.mjs";
 import { isProtectedContextKey } from "../storyboard/protected-context.mjs";
+import { assertNoPreparationAuthorizer } from "../storyboard/prepared-fixtures.mjs";
 import { checkAccountDiscoveryGate, isAccountBearingSpecialism } from "./spec-conformance.mjs";
 import { registerDefaultInvariants } from "../storyboard/default-invariants.mjs";
 import { hasAnyRequiredTool } from "../storyboard/agent-routing.mjs";
@@ -409,6 +413,7 @@ function collectObservations(track, results, profile) {
   return observations;
 }
 async function comply(agentUrl, options = {}) {
+  assertNoPreparationAuthorizer(options);
   if ("platform_type" in options) {
     throw new Error(
       'comply() no longer accepts platform_type. Agent selection is now driven by get_adcp_capabilities (supported_protocols + specialisms). Pass `storyboards: ["<bundle-or-id>"]` to target a specific bundle. See the changeset for migration notes.'
@@ -915,6 +920,13 @@ async function runComplyStoryboard(agentUrl, storyboard, runOptions, profile, ro
   } else {
     runOptions = { ...runOptions, _profile: void 0, agentTools: void 0 };
   }
+  if (describeProfileDiscoveryFailure(
+    discovered.profile,
+    storyboardRequiresCapabilityEvidence(storyboard),
+    storyboardAllowsCapabilityDiagnostic(storyboard)
+  )) {
+    return runStoryboard(agentUrl, storyboard, { ...runOptions, _profile: discovered.profile });
+  }
   if (!routeStoryboard || storyboardCapabilityGateUnmet(storyboard, runOptions) !== null) {
     return runStoryboard(agentUrl, storyboard, runOptions);
   }
@@ -1273,16 +1285,15 @@ function extractFailures(results, storyboards, agentRef, fixOptions = {}) {
       }
     }
     for (const { assertion, owningStep } of collectDetachedAssertionFailures(result)) {
-      if (!owningStep) continue;
       failures.push({
         track,
         storyboard_id: result.storyboard_id,
-        step_id: owningStep.step_id,
-        step_title: owningStep.title,
-        task: owningStep.task,
+        step_id: owningStep?.step_id ?? `${result.storyboard_id}/storyboard`,
+        step_title: owningStep?.title ?? sb?.title ?? result.storyboard_id,
+        task: owningStep?.task ?? "storyboard",
         error: assertion.error ?? assertion.description,
-        expected: findStoryboardStepExpected(sb, owningStep.step_id),
-        fix_command: buildFixCommand(agentRef, result.storyboard_id, owningStep.step_id, fixOptions),
+        expected: owningStep ? findStoryboardStepExpected(sb, owningStep.step_id) : void 0,
+        fix_command: buildFixCommand(agentRef, result.storyboard_id, owningStep?.step_id, fixOptions),
         validation: {
           check: "assertion",
           description: `${assertion.assertion_id}: ${assertion.description}`
@@ -1301,7 +1312,7 @@ function findStoryboardStepExpected(storyboard, stepId) {
   return void 0;
 }
 function buildFixCommand(agentRef, storyboardId, stepId, options) {
-  const parts = ["adcp", "storyboard", "step", agentRef, storyboardId, stepId, "--json"];
+  const parts = stepId === void 0 ? ["adcp", "storyboard", "run", agentRef, storyboardId, "--json"] : ["adcp", "storyboard", "step", agentRef, storyboardId, stepId, "--json"];
   if (options.complianceVersion) parts.push("--compliance-version", options.complianceVersion);
   if (options.complianceDir) parts.push("--compliance-dir", options.complianceDir);
   if (options.schemaRoot) parts.push("--schema-root", options.schemaRoot);
@@ -1434,7 +1445,7 @@ async function complyImpl(agentUrl, options) {
         allObservations.push({
           category: "tool_discovery",
           severity: "error",
-          message: `get_adcp_capabilities is advertised but the call failed. Only universal storyboards ran \u2014 domain and specialism bundles were skipped. Agent-reported error: ${fenceAgentText(profile.capabilities_probe_error)}`,
+          message: `get_adcp_capabilities is advertised but the call failed. Universal diagnostics were selected; failed discovery does not establish domain or specialism support. Agent-reported error: ${fenceAgentText(profile.capabilities_probe_error)}`,
           evidence: { agent_reported_error: profile.capabilities_probe_error },
           source: { kind: "profile", code: "capabilities-probe-failed" }
         });

package/dist/lib/testing/compliance/index.d.mts

@@ -6,6 +6,6 @@
 export { comply, collectObservations, buildComplianceBundleResults, computeOverallStatus, formatComplianceResults, formatComplianceResultsJSON, rotateStoryboardsForOffset, } from './comply.mjs';
 export type { ComplyOptions, ComplianceBundleAssessmentOptions, ComplyRouteStoryboardContext, ComplyStoryboardRoute, ComplyStoryboardRouting, ComplyStoryboardSkip, } from './comply.mjs';
 export { buildComplianceSummary, buildCrashSummary, formatComplianceSummaryText, formatComplianceSummaryMarkdown, } from './summary.mjs';
-export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary.mjs';
+export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, ComplianceSummaryIncompleteStoryboard, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary.mjs';
 export { SAMPLE_BRIEFS, getBriefById, getBriefsByVertical } from './briefs.mjs';
 export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types.mjs';

package/dist/lib/testing/compliance/index.d.ts

@@ -6,6 +6,6 @@
 export { comply, collectObservations, buildComplianceBundleResults, computeOverallStatus, formatComplianceResults, formatComplianceResultsJSON, rotateStoryboardsForOffset, } from './comply';
 export type { ComplyOptions, ComplianceBundleAssessmentOptions, ComplyRouteStoryboardContext, ComplyStoryboardRoute, ComplyStoryboardRouting, ComplyStoryboardSkip, } from './comply';
 export { buildComplianceSummary, buildCrashSummary, formatComplianceSummaryText, formatComplianceSummaryMarkdown, } from './summary';
-export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary';
+export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, ComplianceSummaryIncompleteStoryboard, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary';
 export { SAMPLE_BRIEFS, getBriefById, getBriefsByVertical } from './briefs';
 export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types';

package/dist/lib/testing/compliance/storyboard-tracks.js

@@ -115,7 +115,7 @@ function mapStoryboardResultsToTrackResult(track, storyboardResults, _profile) {
       };
       scenarios.push(testResult);
     }
-    if (!sbResult.overall_passed && (sbResult.passed_count > 0 || assertionFailures.length > 0) && sbResult.phases.every((phase) => phase.passed)) {
+    if (!sbResult.overall_passed && (sbResult.failed_count === 0 || (sbResult.passed_count > 0 || assertionFailures.length > 0) && sbResult.phases.every((phase) => phase.passed))) {
       scenarios.push({
         agent_url: sbResult.agent_url,
         scenario: `${sbResult.storyboard_id}/storyboard`,

package/dist/lib/testing/compliance/storyboard-tracks.mjs

@@ -90,7 +90,7 @@ function mapStoryboardResultsToTrackResult(track, storyboardResults, _profile) {
       };
       scenarios.push(testResult);
     }
-    if (!sbResult.overall_passed && (sbResult.passed_count > 0 || assertionFailures.length > 0) && sbResult.phases.every((phase) => phase.passed)) {
+    if (!sbResult.overall_passed && (sbResult.failed_count === 0 || (sbResult.passed_count > 0 || assertionFailures.length > 0) && sbResult.phases.every((phase) => phase.passed))) {
       scenarios.push({
         agent_url: sbResult.agent_url,
         scenario: `${sbResult.storyboard_id}/storyboard`,

package/dist/lib/testing/compliance/summary.d.mts

@@ -60,6 +60,8 @@ export interface ComplianceSummaryArtifact {
     tested_at: string;
     storyboards_executed: string[];
     failures: ComplianceSummaryFailure[];
+    /** False storyboard verdicts without failed steps; coverage gaps remain distinct from seller failures. */
+    incomplete_storyboards?: ComplianceSummaryIncompleteStoryboard[];
     /** Actionable skip causes grouped by reason. Present only when skipped > 0. */
     skip_causes?: ComplianceSummarySkipCause[];
 }
@@ -75,6 +77,11 @@ export interface ComplianceSummaryFailure {
     reason: string;
     reason_kind: ComplianceSummaryFailureKind;
 }
+export interface ComplianceSummaryIncompleteStoryboard {
+    track: ComplianceTrack;
+    storyboard_id: string;
+    reason: string;
+}
 export interface BuildSummaryOptions {
     sdkVersion: string;
     adcpVersion: string;

package/dist/lib/testing/compliance/summary.d.ts

@@ -60,6 +60,8 @@ export interface ComplianceSummaryArtifact {
     tested_at: string;
     storyboards_executed: string[];
     failures: ComplianceSummaryFailure[];
+    /** False storyboard verdicts without failed steps; coverage gaps remain distinct from seller failures. */
+    incomplete_storyboards?: ComplianceSummaryIncompleteStoryboard[];
     /** Actionable skip causes grouped by reason. Present only when skipped > 0. */
     skip_causes?: ComplianceSummarySkipCause[];
 }
@@ -75,6 +77,11 @@ export interface ComplianceSummaryFailure {
     reason: string;
     reason_kind: ComplianceSummaryFailureKind;
 }
+export interface ComplianceSummaryIncompleteStoryboard {
+    track: ComplianceTrack;
+    storyboard_id: string;
+    reason: string;
+}
 export interface BuildSummaryOptions {
     sdkVersion: string;
     adcpVersion: string;

package/dist/lib/testing/compliance/summary.js

@@ -28,6 +28,7 @@ var import_types = require('../storyboard/types.js');
 const SUMMARY_SCHEMA_VERSION = 2;
 function buildComplianceSummary(result, opts) {
   const skipCauses = buildSkipCauses(result);
+  const incompleteStoryboards = collectIncompleteStoryboardVerdicts(result);
   return {
     schema_version: SUMMARY_SCHEMA_VERSION,
     agent_url: result.agent_url,
@@ -48,9 +49,31 @@ function buildComplianceSummary(result, opts) {
     tested_at: result.tested_at,
     storyboards_executed: result.storyboards_executed ?? [],
     failures: (result.failures ?? []).map(toSummaryFailure),
+    ...incompleteStoryboards.length > 0 ? { incomplete_storyboards: incompleteStoryboards } : {},
     ...skipCauses.length > 0 ? { skip_causes: skipCauses } : {}
   };
 }
+function collectIncompleteStoryboardVerdicts(result) {
+  const failedStoryboards = new Set((result.failures ?? []).map((failure) => failure.storyboard_id));
+  const seen = /* @__PURE__ */ new Set();
+  const incomplete = [];
+  const suffix = "/storyboard";
+  for (const track of result.tracks ?? []) {
+    for (const scenario of track.scenarios) {
+      if (scenario.overall_passed !== false || typeof scenario.scenario !== "string" || !scenario.scenario.endsWith(suffix) || (scenario.steps?.length ?? 0) > 0)
+        continue;
+      const id = scenario.scenario.slice(0, -suffix.length);
+      if (failedStoryboards.has(id) || seen.has(id)) continue;
+      seen.add(id);
+      incomplete.push({
+        track: track.track,
+        storyboard_id: id,
+        reason: truncate(scenario.summary || "Storyboard verdict failed without a failed step.", REASON_MAX_CHARS)
+      });
+    }
+  }
+  return incomplete;
+}
 function toSummaryFailure(f) {
   const { reason, reason_kind } = deriveReason(f);
   return {
@@ -264,6 +287,9 @@ function formatComplianceSummaryText(s) {
   }
   lines.push(`Duration:  ${(s.total_duration_ms / 1e3).toFixed(1)}s`);
   lines.push("");
+  for (const verdict of s.incomplete_storyboards ?? []) {
+    lines.push(`STORYBOARD-INCOMPLETE ${verdict.storyboard_id} [${verdict.track}]: ${verdict.reason}`);
+  }
   if (!isHardFail(s)) {
     if (s.completeness === "timed_out") {
       lines.push(`STORYBOARD-INCOMPLETE ${s.passed} steps passed before the timeout budget expired`);
@@ -306,6 +332,17 @@ function formatComplianceSummaryMarkdown(s) {
   if (skippedReasons) lines.push(`- **Skipped:** ${skippedReasons}`);
   lines.push(`- **Duration:** ${(s.total_duration_ms / 1e3).toFixed(1)}s`);
   lines.push("");
+  if (s.incomplete_storyboards?.length) {
+    lines.push("");
+    lines.push("### Incomplete storyboard verdicts");
+    lines.push("");
+    lines.push("| Storyboard | Track | Reason |", "|---|---|---|");
+    for (const verdict of s.incomplete_storyboards)
+      lines.push(
+        `| ${escapeTableCell(verdict.storyboard_id)} | ${escapeTableCell(verdict.track)} | ${escapeTableCell(verdict.reason)} |`
+      );
+    lines.push("");
+  }
   if (s.failures.length > 0) {
     lines.push("| Track | Storyboard | Step | Reason |");
     lines.push("| --- | --- | --- | --- |");

package/dist/lib/testing/compliance/summary.mjs

@@ -2,6 +2,7 @@ import { DETAILED_SKIP_TO_CANONICAL } from "../storyboard/types.mjs";
 const SUMMARY_SCHEMA_VERSION = 2;
 function buildComplianceSummary(result, opts) {
   const skipCauses = buildSkipCauses(result);
+  const incompleteStoryboards = collectIncompleteStoryboardVerdicts(result);
   return {
     schema_version: SUMMARY_SCHEMA_VERSION,
     agent_url: result.agent_url,
@@ -22,9 +23,31 @@ function buildComplianceSummary(result, opts) {
     tested_at: result.tested_at,
     storyboards_executed: result.storyboards_executed ?? [],
     failures: (result.failures ?? []).map(toSummaryFailure),
+    ...incompleteStoryboards.length > 0 ? { incomplete_storyboards: incompleteStoryboards } : {},
     ...skipCauses.length > 0 ? { skip_causes: skipCauses } : {}
   };
 }
+function collectIncompleteStoryboardVerdicts(result) {
+  const failedStoryboards = new Set((result.failures ?? []).map((failure) => failure.storyboard_id));
+  const seen = /* @__PURE__ */ new Set();
+  const incomplete = [];
+  const suffix = "/storyboard";
+  for (const track of result.tracks ?? []) {
+    for (const scenario of track.scenarios) {
+      if (scenario.overall_passed !== false || typeof scenario.scenario !== "string" || !scenario.scenario.endsWith(suffix) || (scenario.steps?.length ?? 0) > 0)
+        continue;
+      const id = scenario.scenario.slice(0, -suffix.length);
+      if (failedStoryboards.has(id) || seen.has(id)) continue;
+      seen.add(id);
+      incomplete.push({
+        track: track.track,
+        storyboard_id: id,
+        reason: truncate(scenario.summary || "Storyboard verdict failed without a failed step.", REASON_MAX_CHARS)
+      });
+    }
+  }
+  return incomplete;
+}
 function toSummaryFailure(f) {
   const { reason, reason_kind } = deriveReason(f);
   return {
@@ -238,6 +261,9 @@ function formatComplianceSummaryText(s) {
   }
   lines.push(`Duration:  ${(s.total_duration_ms / 1e3).toFixed(1)}s`);
   lines.push("");
+  for (const verdict of s.incomplete_storyboards ?? []) {
+    lines.push(`STORYBOARD-INCOMPLETE ${verdict.storyboard_id} [${verdict.track}]: ${verdict.reason}`);
+  }
   if (!isHardFail(s)) {
     if (s.completeness === "timed_out") {
       lines.push(`STORYBOARD-INCOMPLETE ${s.passed} steps passed before the timeout budget expired`);
@@ -280,6 +306,17 @@ function formatComplianceSummaryMarkdown(s) {
   if (skippedReasons) lines.push(`- **Skipped:** ${skippedReasons}`);
   lines.push(`- **Duration:** ${(s.total_duration_ms / 1e3).toFixed(1)}s`);
   lines.push("");
+  if (s.incomplete_storyboards?.length) {
+    lines.push("");
+    lines.push("### Incomplete storyboard verdicts");
+    lines.push("");
+    lines.push("| Storyboard | Track | Reason |", "|---|---|---|");
+    for (const verdict of s.incomplete_storyboards)
+      lines.push(
+        `| ${escapeTableCell(verdict.storyboard_id)} | ${escapeTableCell(verdict.track)} | ${escapeTableCell(verdict.reason)} |`
+      );
+    lines.push("");
+  }
   if (s.failures.length > 0) {
     lines.push("| Track | Storyboard | Step | Reason |");
     lines.push("| --- | --- | --- | --- |");

package/dist/lib/testing/discovery-evidence.d.mts

@@ -0,0 +1,2 @@
+/** Preserve protocol evidence while also scrubbing credentials inside text payloads. */
+export declare function redactDiscoveryEvidence(value: unknown): unknown;