@adcp/sdk 17.0.0-beta.27 → 17.0.0-beta.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/bin/adcp-storyboard-summary.js +47 -0
  2. package/bin/adcp.js +4 -0
  3. package/dist/lib/registry/types.generated.d.ts +3 -1
  4. package/dist/lib/testing/compliance/comply.d.mts +16 -1
  5. package/dist/lib/testing/compliance/comply.d.ts +16 -1
  6. package/dist/lib/testing/compliance/comply.js +69 -5
  7. package/dist/lib/testing/compliance/comply.mjs +75 -6
  8. package/dist/lib/testing/compliance/index.d.mts +1 -1
  9. package/dist/lib/testing/compliance/index.d.ts +1 -1
  10. package/dist/lib/testing/compliance/storyboard-tracks.d.mts +6 -0
  11. package/dist/lib/testing/compliance/storyboard-tracks.d.ts +6 -0
  12. package/dist/lib/testing/compliance/storyboard-tracks.js +26 -4
  13. package/dist/lib/testing/compliance/storyboard-tracks.mjs +24 -3
  14. package/dist/lib/testing/compliance/summary.d.mts +3 -1
  15. package/dist/lib/testing/compliance/summary.d.ts +3 -1
  16. package/dist/lib/testing/compliance/summary.js +2 -1
  17. package/dist/lib/testing/compliance/summary.mjs +2 -1
  18. package/dist/lib/testing/compliance/types.d.mts +30 -1
  19. package/dist/lib/testing/compliance/types.d.ts +30 -1
  20. package/dist/lib/testing/index.d.mts +1 -1
  21. package/dist/lib/testing/index.d.ts +1 -1
  22. package/dist/lib/testing/index.js +4 -0
  23. package/dist/lib/testing/index.mjs +4 -0
  24. package/dist/lib/testing/storyboard/index.d.mts +4 -2
  25. package/dist/lib/testing/storyboard/index.d.ts +4 -2
  26. package/dist/lib/testing/storyboard/index.js +7 -0
  27. package/dist/lib/testing/storyboard/index.mjs +6 -1
  28. package/dist/lib/testing/storyboard/runner.d.mts +7 -0
  29. package/dist/lib/testing/storyboard/runner.d.ts +7 -0
  30. package/dist/lib/testing/storyboard/runner.js +15 -0
  31. package/dist/lib/testing/storyboard/runner.mjs +14 -0
  32. package/dist/lib/testing/storyboard/types.d.mts +77 -0
  33. package/dist/lib/testing/storyboard/types.d.ts +77 -0
  34. package/dist/lib/testing/storyboard/validations.d.mts +1 -1
  35. package/dist/lib/testing/storyboard/validations.d.ts +1 -1
  36. package/dist/lib/testing/storyboard/verdict-explanation.d.mts +47 -0
  37. package/dist/lib/testing/storyboard/verdict-explanation.d.ts +47 -0
  38. package/dist/lib/testing/storyboard/verdict-explanation.js +309 -0
  39. package/dist/lib/testing/storyboard/verdict-explanation.mjs +283 -0
  40. package/dist/lib/testing/types.d.mts +11 -1
  41. package/dist/lib/testing/types.d.ts +11 -1
  42. package/dist/lib/version.d.mts +3 -3
  43. package/dist/lib/version.d.ts +3 -3
  44. package/dist/lib/version.js +3 -3
  45. package/dist/lib/version.mjs +3 -3
  46. package/docs/TYPE-SUMMARY.md +2 -2
  47. package/docs/guides/VALIDATE-YOUR-AGENT.md +28 -0
  48. package/docs/llms.txt +2 -2
  49. package/docs/migration-14.x-rc-worksheet.md +4 -4
  50. package/package.json +1 -1
@@ -147,9 +147,55 @@ function buildStoryboardSummaryMarkdown(results, agentUrl, overallPassed) {
147
147
  }
148
148
  lines.push('');
149
149
  }
150
+ // Failing storyboards no failed step explains (adcp#8576).
151
+ const storyboardLevel = results.filter(
152
+ r => r.overall_passed === false && r.verdict_detail && !failures.some(f => f.storyboard === (r.storyboard_id || ''))
153
+ );
154
+ if (storyboardLevel.length > 0) {
155
+ lines.push('## Storyboard-level verdicts');
156
+ lines.push('');
157
+ lines.push('| Storyboard | Reason | Detail |');
158
+ lines.push('|---|---|---|');
159
+ for (const r of storyboardLevel) {
160
+ lines.push(
161
+ `| ${escapeMarkdownCell(r.storyboard_id || '')} | ${escapeMarkdownCell(r.verdict_reason || 'unexplained')} | ${escapeMarkdownCell(r.verdict_detail)} |`
162
+ );
163
+ }
164
+ lines.push('');
165
+ }
150
166
  return lines.join('\n');
151
167
  }
152
168
 
169
+ const VERDICT_CONTRIBUTOR_LIMIT = 8;
170
+
171
+ /**
172
+ * Console lines explaining a failing storyboard verdict (adcp#8576): the
173
+ * runner's `verdict_reason`, `verdict_detail`, and the contributing steps,
174
+ * assertions and phases. Empty for a passing result or one from a runner
175
+ * that predates the fields. Every runner- or agent-sourced string is
176
+ * terminal-escaped.
177
+ */
178
+ function formatStoryboardVerdictLines(result, indent = ' ') {
179
+ if (!result || result.overall_passed !== false || !result.verdict_reason) return [];
180
+ const lines = [
181
+ `${indent}Why: [${escapeTerminalControlChars(result.verdict_reason)}] ${escapeTerminalControlChars(result.verdict_detail ?? '')}`,
182
+ ];
183
+ const rows = Array.isArray(result.verdict_contributing) ? result.verdict_contributing : [];
184
+ for (const row of rows.slice(0, VERDICT_CONTRIBUTOR_LIMIT)) {
185
+ const where = [row.phase_id && `phase ${row.phase_id}`, row.pass_index !== undefined && `pass ${row.pass_index}`]
186
+ .filter(Boolean)
187
+ .join(', ');
188
+ const skip = row.skip_reason ? ` ${row.skip_reason}${row.requirement ? `:${row.requirement}` : ''}` : '';
189
+ const head = `${row.kind} ${row.id} ${row.status}${skip}${where ? ` (${where})` : ''}`;
190
+ const message = row.message ? ` — ${row.message}` : '';
191
+ lines.push(`${indent} - ${escapeTerminalControlChars(head + message)}`);
192
+ }
193
+ if (rows.length > VERDICT_CONTRIBUTOR_LIMIT) {
194
+ lines.push(`${indent} … ${rows.length - VERDICT_CONTRIBUTOR_LIMIT} more (use --json for all)`);
195
+ }
196
+ return lines;
197
+ }
198
+
153
199
  /**
154
200
  * Writes `content` to `summaryFile`. Failures are downgraded to a stderr
155
201
  * warning rather than exiting the process — the markdown summary is a
@@ -474,6 +520,7 @@ module.exports = {
474
520
  formatJwtExpClaim,
475
521
  formatStepSkipLines,
476
522
  formatStepVerdictLines,
523
+ formatStoryboardVerdictLines,
477
524
  isCoverageGapRow,
478
525
  isCoverageUnavailableStep,
479
526
  unverifiedSigningCoverage,
package/bin/adcp.js CHANGED
@@ -52,6 +52,7 @@ const {
52
52
  formatJwtExpClaim,
53
53
  formatStepSkipLines,
54
54
  formatStepVerdictLines,
55
+ formatStoryboardVerdictLines,
55
56
  isCoverageUnavailableStep,
56
57
  unverifiedSigningCoverage,
57
58
  parseRequestSigningFlags,
@@ -3158,6 +3159,8 @@ async function handleStoryboardRun(args) {
3158
3159
  console.log(
3159
3160
  `${overallIcon} ${result.passed_count} passed, ${result.failed_count} failed, ${result.skipped_count} skipped (${result.total_duration_ms}ms)${hintTail}`
3160
3161
  );
3162
+ // A storyboard can fail with every visible step green; say why (adcp#8576).
3163
+ for (const line of formatStoryboardVerdictLines(result)) console.log(line);
3161
3164
  printStrictSummary(result.strict_validation_summary);
3162
3165
  printNotices(result.notices);
3163
3166
  }
@@ -4164,6 +4167,7 @@ async function handleLocalAgentStoryboardRun(modulePath, args, opts) {
4164
4167
  console.log(
4165
4168
  `${icon} ${sb.storyboard_title} (${sb.storyboard_id}) — ${sb.passed_count} passed, ${sb.failed_count} failed, ${sb.skipped_count} skipped`
4166
4169
  );
4170
+ for (const line of formatStoryboardVerdictLines(sb)) console.log(line);
4167
4171
  }
4168
4172
  if (result.not_applicable.length > 0) {
4169
4173
  console.log(`\n${result.not_applicable.length} storyboard(s) not applicable (out of scope):`);
@@ -6576,6 +6576,7 @@ export interface operations {
6576
6576
  properties_total: number;
6577
6577
  /** @description Canonical list of property_id strings the agent is authorized for under this publisher. Present iff the request included `?include=properties`. Same population as `properties_authorized` but surfaced as IDs for set-diff comparison. */
6578
6578
  property_ids?: string[];
6579
+ /** @description Scope-agnostic summary: true only when every in-window authorized_agents entry that matches the agent carries signing_keys. Verifiers compute the pin per message scope, so a pin can apply where this reads false. */
6579
6580
  signing_keys_pinned: boolean;
6580
6581
  /** @enum {string} */
6581
6582
  status: "authorized" | "revoked";
@@ -6658,6 +6659,7 @@ export interface operations {
6658
6659
  properties_total: number;
6659
6660
  /** @description Canonical list of property_id strings the agent is authorized for under this publisher. Present iff the request included `?include=properties`. Same population as `properties_authorized` but surfaced as IDs for set-diff comparison. */
6660
6661
  property_ids?: string[];
6662
+ /** @description Scope-agnostic summary: true only when every in-window authorized_agents entry that matches the agent carries signing_keys. Verifiers compute the pin per message scope, so a pin can apply where this reads false. */
6661
6663
  signing_keys_pinned: boolean;
6662
6664
  /** @enum {string} */
6663
6665
  status: "authorized" | "revoked";
@@ -8826,7 +8828,7 @@ export interface operations {
8826
8828
  created_at: string;
8827
8829
  /** Format: date-time */
8828
8830
  updated_at: string;
8829
- /** @description Publisher-pinned JWK set (authorized_agents[*].signing_keys) from the source adagents.json. Consumers verifying inbound TMP signatures key on kid → JWK. Null when the publisher declared no keys; consumers fall back to the agent-hosted JWKS per spec R-2 (docs/governance/property/adagents.mdx). */
8831
+ /** @description Publisher-pinned JWK set (authorized_agents[*].signing_keys) from the source adagents.json. Consumers verifying inbound TMP signatures key on kid → JWK. Null when the publisher declared no keys; consumers fall back to the agent-hosted JWKS per spec R-2 (docs/governance/property/adagents.mdx). This is one entry's pin. Rows from the same publisher combine per message scope: among the rows that authorize that scope, the publisher pins the agent only if every one carries keys, and the pin is the union of their keys (docs/governance/property/adagents.mdx#agent-url-matching). */
8830
8832
  signing_keys: components["schemas"]["SigningKey"][] | null;
8831
8833
  /** @description Collection constraints (authorized_agents[*].collections) from the source adagents.json. Null when the entry is unconstrained. When set, the row does NOT authorize the property unqualified — consumers MUST scope it to these selectors. A selector without collection_ids is a bulk grant for all collections declared at that publisher_domain. */
8832
8834
  collections: ({
@@ -10,7 +10,7 @@
10
10
  import type { TestOptions, TestResult, AgentProfile } from '../types.mjs';
11
11
  import type { NotApplicableStoryboard, ResolvedBundle } from '../storyboard/compliance.mjs';
12
12
  import type { AgentEntry, Storyboard, StoryboardContext, StoryboardResult, StoryboardRunOptions } from '../storyboard/types.mjs';
13
- import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types.mjs';
13
+ import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceVerdictExplanation, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types.mjs';
14
14
  import type { VersionEnvelopeMode } from '../../protocols/index.mjs';
15
15
  /**
16
16
  * Collect advisory observations from test results.
@@ -258,6 +258,20 @@ export declare function extractFailures(results: StoryboardResult[], storyboards
258
258
  hostedStableLineAlias?: string;
259
259
  testKitPath?: string;
260
260
  }): ComplianceFailure[];
261
+ /**
262
+ * One explanation per failing storyboard (adcp#8576), with a `fix_command`
263
+ * that re-runs the whole storyboard. `storyboard_level` marks the verdicts no
264
+ * `failures[]` row explains — the ones that used to surface only as
265
+ * "storyboard-level verdict failed".
266
+ */
267
+ export declare function extractVerdictExplanations(results: StoryboardResult[], storyboards: Storyboard[], agentRef: string, failures: readonly ComplianceFailure[], fixOptions?: Parameters<typeof buildFixCommand>[3]): ComplianceVerdictExplanation[];
268
+ declare function buildFixCommand(agentRef: string, storyboardId: string, stepId: string | undefined, options: {
269
+ complianceVersion?: string;
270
+ complianceDir?: string;
271
+ schemaRoot?: string;
272
+ hostedStableLineAlias?: string;
273
+ testKitPath?: string;
274
+ }): string;
261
275
  /**
262
276
  * Rotate a storyboard list to start at `offset % length`, preserving relative
263
277
  * order (the head wraps to the tail). Pure and deterministic: offset 0, an
@@ -287,3 +301,4 @@ export declare function computeOverallStatus(summary: ComplianceSummary): Overal
287
301
  */
288
302
  export declare function formatComplianceResults(result: ComplianceResult): string;
289
303
  export declare function formatComplianceResultsJSON(result: ComplianceResult): string;
304
+ export {};
@@ -10,7 +10,7 @@
10
10
  import type { TestOptions, TestResult, AgentProfile } from '../types';
11
11
  import type { NotApplicableStoryboard, ResolvedBundle } from '../storyboard/compliance';
12
12
  import type { AgentEntry, Storyboard, StoryboardContext, StoryboardResult, StoryboardRunOptions } from '../storyboard/types';
13
- import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types';
13
+ import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceVerdictExplanation, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types';
14
14
  import type { VersionEnvelopeMode } from '../../protocols';
15
15
  /**
16
16
  * Collect advisory observations from test results.
@@ -258,6 +258,20 @@ export declare function extractFailures(results: StoryboardResult[], storyboards
258
258
  hostedStableLineAlias?: string;
259
259
  testKitPath?: string;
260
260
  }): ComplianceFailure[];
261
+ /**
262
+ * One explanation per failing storyboard (adcp#8576), with a `fix_command`
263
+ * that re-runs the whole storyboard. `storyboard_level` marks the verdicts no
264
+ * `failures[]` row explains — the ones that used to surface only as
265
+ * "storyboard-level verdict failed".
266
+ */
267
+ export declare function extractVerdictExplanations(results: StoryboardResult[], storyboards: Storyboard[], agentRef: string, failures: readonly ComplianceFailure[], fixOptions?: Parameters<typeof buildFixCommand>[3]): ComplianceVerdictExplanation[];
268
+ declare function buildFixCommand(agentRef: string, storyboardId: string, stepId: string | undefined, options: {
269
+ complianceVersion?: string;
270
+ complianceDir?: string;
271
+ schemaRoot?: string;
272
+ hostedStableLineAlias?: string;
273
+ testKitPath?: string;
274
+ }): string;
261
275
  /**
262
276
  * Rotate a storyboard list to start at `offset % length`, preserving relative
263
277
  * order (the head wraps to the tail). Pure and deterministic: offset 0, an
@@ -287,3 +301,4 @@ export declare function computeOverallStatus(summary: ComplianceSummary): Overal
287
301
  */
288
302
  export declare function formatComplianceResults(result: ComplianceResult): string;
289
303
  export declare function formatComplianceResultsJSON(result: ComplianceResult): string;
304
+ export {};
@@ -25,6 +25,7 @@ __export(comply_exports, {
25
25
  computeOverallStatus: () => computeOverallStatus,
26
26
  detectAuthRejection: () => detectAuthRejection,
27
27
  extractFailures: () => extractFailures,
28
+ extractVerdictExplanations: () => extractVerdictExplanations,
28
29
  formatComplianceResults: () => formatComplianceResults,
29
30
  formatComplianceResultsJSON: () => formatComplianceResultsJSON,
30
31
  partitionStoryboardsByRequiredTools: () => partitionStoryboardsByRequiredTools,
@@ -964,7 +965,10 @@ async function runComplyStoryboard(agentUrl, storyboard, runOptions, profile, ro
964
965
  const routed = buildRoutedStoryboardRun(agentUrl, storyboard, runOptions, routing);
965
966
  if ("refused" in routed) return buildRouteSkippedStoryboardResult(agentUrl, storyboard, routed.refused, runOptions);
966
967
  const result = await (0, import_runner.runStoryboard)("", routed.storyboard, routed.options);
967
- return gradeRoutedResultForAgentUnderTest(result, routed.agentUnderTestIndexes, routed.keyByIndex);
968
+ return (0, import_runner.attachStoryboardVerdictExplanation)(
969
+ gradeRoutedResultForAgentUnderTest(result, routed.agentUnderTestIndexes, routed.keyByIndex),
970
+ routed.storyboard
971
+ );
968
972
  }
969
973
  function sameAgentUrl(a, b) {
970
974
  try {
@@ -1318,6 +1322,23 @@ function extractFailures(results, storyboards, agentRef, fixOptions = {}) {
1318
1322
  }
1319
1323
  return failures;
1320
1324
  }
1325
+ function extractVerdictExplanations(results, storyboards, agentRef, failures, fixOptions = {}) {
1326
+ const trackById = new Map(storyboards.map((sb) => [sb.id, sb.track ?? "core"]));
1327
+ const explained = new Set(failures.map((failure) => failure.storyboard_id));
1328
+ const explanations = [];
1329
+ for (const result of results) {
1330
+ const explanation = (0, import_storyboard_tracks.storyboardVerdictExplanation)(result);
1331
+ if (!explanation) continue;
1332
+ explanations.push({
1333
+ track: trackById.get(result.storyboard_id) ?? "core",
1334
+ storyboard_id: result.storyboard_id,
1335
+ ...explanation,
1336
+ storyboard_level: !explained.has(result.storyboard_id),
1337
+ fix_command: buildFixCommand(agentRef, result.storyboard_id, void 0, fixOptions)
1338
+ });
1339
+ }
1340
+ return explanations;
1341
+ }
1321
1342
  function findStoryboardStepExpected(storyboard, stepId) {
1322
1343
  if (!storyboard) return void 0;
1323
1344
  for (const phase of storyboard.phases) {
@@ -1647,13 +1668,21 @@ async function complyImpl(agentUrl, options) {
1647
1668
  const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
1648
1669
  const safeAgentUrl = (0, import_oauth_metadata_graph.redactOAuthUrlForOutput)(agentUrl);
1649
1670
  const agentRef = options.agent_alias || safeAgentUrl;
1650
- const failures = extractFailures(storyboardResults, runnableStoryboards, agentRef, {
1671
+ const fixOptions = {
1651
1672
  complianceVersion: complianceIndex.adcp_version,
1652
1673
  ...complianceDir !== void 0 && { complianceDir },
1653
1674
  ...schemaRoot !== void 0 && { schemaRoot },
1654
1675
  ...hostedStableLineAlias !== void 0 && { hostedStableLineAlias },
1655
1676
  ...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
1656
- });
1677
+ };
1678
+ const failures = extractFailures(storyboardResults, runnableStoryboards, agentRef, fixOptions);
1679
+ const verdictExplanations = extractVerdictExplanations(
1680
+ storyboardResults,
1681
+ runnableStoryboards,
1682
+ agentRef,
1683
+ failures,
1684
+ fixOptions
1685
+ );
1657
1686
  const aggregatedNotices = /* @__PURE__ */ new Map();
1658
1687
  for (const sbResult of storyboardResults) {
1659
1688
  for (const notice of sbResult.notices) {
@@ -1681,6 +1710,7 @@ async function complyImpl(agentUrl, options) {
1681
1710
  summary,
1682
1711
  observations: allObservations,
1683
1712
  failures: failures.length > 0 ? failures : void 0,
1713
+ ...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
1684
1714
  storyboards_executed: executedStoryboards.map((sb) => sb.id),
1685
1715
  ...resolvedBundles !== void 0 && {
1686
1716
  bundle_results: buildComplianceBundleResults(resolvedBundles, storyboardResults, {
@@ -1849,13 +1879,21 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
1849
1879
  const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
1850
1880
  const safeAgentUrl = (0, import_oauth_metadata_graph.redactOAuthUrlForOutput)(agentUrl);
1851
1881
  const agentRef = options.agent_alias || safeAgentUrl;
1852
- const failures = extractFailures(storyboardResults, storyboards, agentRef, {
1882
+ const fixOptions = {
1853
1883
  complianceVersion: adcpVersion,
1854
1884
  ...options.complianceDir !== void 0 && { complianceDir: options.complianceDir },
1855
1885
  ...options.schemaRoot !== void 0 && { schemaRoot: options.schemaRoot },
1856
1886
  ...options.hostedStableLineAlias !== void 0 && { hostedStableLineAlias: options.hostedStableLineAlias },
1857
1887
  ...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
1858
- });
1888
+ };
1889
+ const failures = extractFailures(storyboardResults, storyboards, agentRef, fixOptions);
1890
+ const verdictExplanations = extractVerdictExplanations(
1891
+ storyboardResults,
1892
+ storyboards,
1893
+ agentRef,
1894
+ failures,
1895
+ fixOptions
1896
+ );
1859
1897
  for (const t of trackResults) t._view = "canonical";
1860
1898
  const skippedTracks = trackResults.filter((t) => t.status === "skip").map((t) => ({
1861
1899
  track: t.track,
@@ -1874,6 +1912,7 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
1874
1912
  summary,
1875
1913
  observations: allObservations,
1876
1914
  failures: failures.length > 0 ? failures : void 0,
1915
+ ...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
1877
1916
  storyboards_executed: executedStoryboards.map((sb) => sb.id),
1878
1917
  controller_detected: false,
1879
1918
  tested_at: (/* @__PURE__ */ new Date()).toISOString(),
@@ -2125,6 +2164,10 @@ ${overallIcon} AdCP Compliance Report
2125
2164
  }
2126
2165
  if (failedSteps.length > 3) {
2127
2166
  output += ` ... and ${failedSteps.length - 3} more
2167
+ `;
2168
+ }
2169
+ if (failedSteps.length === 0 && scenario.verdict_detail) {
2170
+ output += ` Why: ${fenceAgentText(scenario.verdict_detail, 600)}
2128
2171
  `;
2129
2172
  }
2130
2173
  }
@@ -2171,6 +2214,26 @@ How to Fix
2171
2214
  }
2172
2215
  if (failuresWithExpected.length > 5) {
2173
2216
  output += ` ... and ${failuresWithExpected.length - 5} more (use --json for all)
2217
+ `;
2218
+ }
2219
+ }
2220
+ const storyboardLevel = (result.verdict_explanations ?? []).filter((explanation) => explanation.storyboard_level);
2221
+ if (storyboardLevel.length > 0) {
2222
+ output += `
2223
+ Storyboard-level Verdicts
2224
+ `;
2225
+ output += `${"\u2500".repeat(50)}
2226
+ `;
2227
+ for (const explanation of storyboardLevel.slice(0, 5)) {
2228
+ output += `\u274C ${explanation.storyboard_id} (${explanation.verdict_reason})
2229
+ `;
2230
+ output += ` Why: ${fenceAgentText(explanation.verdict_detail, 600)}
2231
+ `;
2232
+ output += ` Debug: ${explanation.fix_command}
2233
+ `;
2234
+ }
2235
+ if (storyboardLevel.length > 5) {
2236
+ output += ` ... and ${storyboardLevel.length - 5} more (use --json for all)
2174
2237
  `;
2175
2238
  }
2176
2239
  }
@@ -2283,6 +2346,7 @@ function formatMachineValue(value) {
2283
2346
  computeOverallStatus,
2284
2347
  detectAuthRejection,
2285
2348
  extractFailures,
2349
+ extractVerdictExplanations,
2286
2350
  formatComplianceResults,
2287
2351
  formatComplianceResultsJSON,
2288
2352
  partitionStoryboardsByRequiredTools,
@@ -7,9 +7,15 @@ import {
7
7
  seedAuthRejectedTestClientProfile,
8
8
  withTestClientScope
9
9
  } from "../client.mjs";
10
- import { collectDetachedAssertionFailures, mapStoryboardResultsToTrackResult, TRACK_LABELS } from "./storyboard-tracks.mjs";
10
+ import {
11
+ collectDetachedAssertionFailures,
12
+ mapStoryboardResultsToTrackResult,
13
+ storyboardVerdictExplanation,
14
+ TRACK_LABELS
15
+ } from "./storyboard-tracks.mjs";
11
16
  import {
12
17
  applyAdcpVersionRunOptions,
18
+ attachStoryboardVerdictExplanation,
13
19
  collectCapabilityNotices,
14
20
  runStoryboard,
15
21
  storyboardCapabilityGateUnmet,
@@ -958,7 +964,10 @@ async function runComplyStoryboard(agentUrl, storyboard, runOptions, profile, ro
958
964
  const routed = buildRoutedStoryboardRun(agentUrl, storyboard, runOptions, routing);
959
965
  if ("refused" in routed) return buildRouteSkippedStoryboardResult(agentUrl, storyboard, routed.refused, runOptions);
960
966
  const result = await runStoryboard("", routed.storyboard, routed.options);
961
- return gradeRoutedResultForAgentUnderTest(result, routed.agentUnderTestIndexes, routed.keyByIndex);
967
+ return attachStoryboardVerdictExplanation(
968
+ gradeRoutedResultForAgentUnderTest(result, routed.agentUnderTestIndexes, routed.keyByIndex),
969
+ routed.storyboard
970
+ );
962
971
  }
963
972
  function sameAgentUrl(a, b) {
964
973
  try {
@@ -1312,6 +1321,23 @@ function extractFailures(results, storyboards, agentRef, fixOptions = {}) {
1312
1321
  }
1313
1322
  return failures;
1314
1323
  }
1324
+ function extractVerdictExplanations(results, storyboards, agentRef, failures, fixOptions = {}) {
1325
+ const trackById = new Map(storyboards.map((sb) => [sb.id, sb.track ?? "core"]));
1326
+ const explained = new Set(failures.map((failure) => failure.storyboard_id));
1327
+ const explanations = [];
1328
+ for (const result of results) {
1329
+ const explanation = storyboardVerdictExplanation(result);
1330
+ if (!explanation) continue;
1331
+ explanations.push({
1332
+ track: trackById.get(result.storyboard_id) ?? "core",
1333
+ storyboard_id: result.storyboard_id,
1334
+ ...explanation,
1335
+ storyboard_level: !explained.has(result.storyboard_id),
1336
+ fix_command: buildFixCommand(agentRef, result.storyboard_id, void 0, fixOptions)
1337
+ });
1338
+ }
1339
+ return explanations;
1340
+ }
1315
1341
  function findStoryboardStepExpected(storyboard, stepId) {
1316
1342
  if (!storyboard) return void 0;
1317
1343
  for (const phase of storyboard.phases) {
@@ -1641,13 +1667,21 @@ async function complyImpl(agentUrl, options) {
1641
1667
  const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
1642
1668
  const safeAgentUrl = redactOAuthUrlForOutput(agentUrl);
1643
1669
  const agentRef = options.agent_alias || safeAgentUrl;
1644
- const failures = extractFailures(storyboardResults, runnableStoryboards, agentRef, {
1670
+ const fixOptions = {
1645
1671
  complianceVersion: complianceIndex.adcp_version,
1646
1672
  ...complianceDir !== void 0 && { complianceDir },
1647
1673
  ...schemaRoot !== void 0 && { schemaRoot },
1648
1674
  ...hostedStableLineAlias !== void 0 && { hostedStableLineAlias },
1649
1675
  ...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
1650
- });
1676
+ };
1677
+ const failures = extractFailures(storyboardResults, runnableStoryboards, agentRef, fixOptions);
1678
+ const verdictExplanations = extractVerdictExplanations(
1679
+ storyboardResults,
1680
+ runnableStoryboards,
1681
+ agentRef,
1682
+ failures,
1683
+ fixOptions
1684
+ );
1651
1685
  const aggregatedNotices = /* @__PURE__ */ new Map();
1652
1686
  for (const sbResult of storyboardResults) {
1653
1687
  for (const notice of sbResult.notices) {
@@ -1675,6 +1709,7 @@ async function complyImpl(agentUrl, options) {
1675
1709
  summary,
1676
1710
  observations: allObservations,
1677
1711
  failures: failures.length > 0 ? failures : void 0,
1712
+ ...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
1678
1713
  storyboards_executed: executedStoryboards.map((sb) => sb.id),
1679
1714
  ...resolvedBundles !== void 0 && {
1680
1715
  bundle_results: buildComplianceBundleResults(resolvedBundles, storyboardResults, {
@@ -1843,13 +1878,21 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
1843
1878
  const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
1844
1879
  const safeAgentUrl = redactOAuthUrlForOutput(agentUrl);
1845
1880
  const agentRef = options.agent_alias || safeAgentUrl;
1846
- const failures = extractFailures(storyboardResults, storyboards, agentRef, {
1881
+ const fixOptions = {
1847
1882
  complianceVersion: adcpVersion,
1848
1883
  ...options.complianceDir !== void 0 && { complianceDir: options.complianceDir },
1849
1884
  ...options.schemaRoot !== void 0 && { schemaRoot: options.schemaRoot },
1850
1885
  ...options.hostedStableLineAlias !== void 0 && { hostedStableLineAlias: options.hostedStableLineAlias },
1851
1886
  ...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
1852
- });
1887
+ };
1888
+ const failures = extractFailures(storyboardResults, storyboards, agentRef, fixOptions);
1889
+ const verdictExplanations = extractVerdictExplanations(
1890
+ storyboardResults,
1891
+ storyboards,
1892
+ agentRef,
1893
+ failures,
1894
+ fixOptions
1895
+ );
1853
1896
  for (const t of trackResults) t._view = "canonical";
1854
1897
  const skippedTracks = trackResults.filter((t) => t.status === "skip").map((t) => ({
1855
1898
  track: t.track,
@@ -1868,6 +1911,7 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
1868
1911
  summary,
1869
1912
  observations: allObservations,
1870
1913
  failures: failures.length > 0 ? failures : void 0,
1914
+ ...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
1871
1915
  storyboards_executed: executedStoryboards.map((sb) => sb.id),
1872
1916
  controller_detected: false,
1873
1917
  tested_at: (/* @__PURE__ */ new Date()).toISOString(),
@@ -2119,6 +2163,10 @@ ${overallIcon} AdCP Compliance Report
2119
2163
  }
2120
2164
  if (failedSteps.length > 3) {
2121
2165
  output += ` ... and ${failedSteps.length - 3} more
2166
+ `;
2167
+ }
2168
+ if (failedSteps.length === 0 && scenario.verdict_detail) {
2169
+ output += ` Why: ${fenceAgentText(scenario.verdict_detail, 600)}
2122
2170
  `;
2123
2171
  }
2124
2172
  }
@@ -2165,6 +2213,26 @@ How to Fix
2165
2213
  }
2166
2214
  if (failuresWithExpected.length > 5) {
2167
2215
  output += ` ... and ${failuresWithExpected.length - 5} more (use --json for all)
2216
+ `;
2217
+ }
2218
+ }
2219
+ const storyboardLevel = (result.verdict_explanations ?? []).filter((explanation) => explanation.storyboard_level);
2220
+ if (storyboardLevel.length > 0) {
2221
+ output += `
2222
+ Storyboard-level Verdicts
2223
+ `;
2224
+ output += `${"\u2500".repeat(50)}
2225
+ `;
2226
+ for (const explanation of storyboardLevel.slice(0, 5)) {
2227
+ output += `\u274C ${explanation.storyboard_id} (${explanation.verdict_reason})
2228
+ `;
2229
+ output += ` Why: ${fenceAgentText(explanation.verdict_detail, 600)}
2230
+ `;
2231
+ output += ` Debug: ${explanation.fix_command}
2232
+ `;
2233
+ }
2234
+ if (storyboardLevel.length > 5) {
2235
+ output += ` ... and ${storyboardLevel.length - 5} more (use --json for all)
2168
2236
  `;
2169
2237
  }
2170
2238
  }
@@ -2276,6 +2344,7 @@ export {
2276
2344
  computeOverallStatus,
2277
2345
  detectAuthRejection,
2278
2346
  extractFailures,
2347
+ extractVerdictExplanations,
2279
2348
  formatComplianceResults,
2280
2349
  formatComplianceResultsJSON,
2281
2350
  partitionStoryboardsByRequiredTools,
@@ -8,4 +8,4 @@ export type { ComplyOptions, ComplianceBundleAssessmentOptions, ComplyRouteStory
8
8
  export { buildComplianceSummary, buildCrashSummary, formatComplianceSummaryText, formatComplianceSummaryMarkdown, } from './summary.mjs';
9
9
  export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, ComplianceSummaryIncompleteStoryboard, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary.mjs';
10
10
  export { SAMPLE_BRIEFS, getBriefById, getBriefsByVertical } from './briefs.mjs';
11
- export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types.mjs';
11
+ export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceVerdictExplanation, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types.mjs';
@@ -8,4 +8,4 @@ export type { ComplyOptions, ComplianceBundleAssessmentOptions, ComplyRouteStory
8
8
  export { buildComplianceSummary, buildCrashSummary, formatComplianceSummaryText, formatComplianceSummaryMarkdown, } from './summary';
9
9
  export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, ComplianceSummaryIncompleteStoryboard, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary';
10
10
  export { SAMPLE_BRIEFS, getBriefById, getBriefsByVertical } from './briefs';
11
- export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types';
11
+ export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceVerdictExplanation, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types';
@@ -8,12 +8,18 @@
8
8
  import type { AgentProfile } from '../types.mjs';
9
9
  import type { ComplianceTrack, TrackResult } from './types.mjs';
10
10
  import type { AssertionResult, StoryboardResult, StoryboardStepResult } from '../storyboard/types.mjs';
11
+ import { type StoryboardVerdictExplanation } from '../storyboard/verdict-explanation.mjs';
11
12
  export interface DetachedAssertionFailure {
12
13
  assertion: AssertionResult;
13
14
  owningStep?: StoryboardStepResult;
14
15
  }
15
16
  /** Assertion failures that are not represented by a counted failed step. */
16
17
  export declare function collectDetachedAssertionFailures(result: StoryboardResult): DetachedAssertionFailure[];
18
+ /**
19
+ * The verdict explanation the runner attached, or one derived from the result
20
+ * when it carries none. `undefined` for a passing result.
21
+ */
22
+ export declare function storyboardVerdictExplanation(result: StoryboardResult): StoryboardVerdictExplanation | undefined;
17
23
  /** Labels for each compliance track. */
18
24
  export declare const TRACK_LABELS: Record<ComplianceTrack, string>;
19
25
  /**
@@ -8,12 +8,18 @@
8
8
  import type { AgentProfile } from '../types';
9
9
  import type { ComplianceTrack, TrackResult } from './types';
10
10
  import type { AssertionResult, StoryboardResult, StoryboardStepResult } from '../storyboard/types';
11
+ import { type StoryboardVerdictExplanation } from '../storyboard/verdict-explanation';
11
12
  export interface DetachedAssertionFailure {
12
13
  assertion: AssertionResult;
13
14
  owningStep?: StoryboardStepResult;
14
15
  }
15
16
  /** Assertion failures that are not represented by a counted failed step. */
16
17
  export declare function collectDetachedAssertionFailures(result: StoryboardResult): DetachedAssertionFailure[];
18
+ /**
19
+ * The verdict explanation the runner attached, or one derived from the result
20
+ * when it carries none. `undefined` for a passing result.
21
+ */
22
+ export declare function storyboardVerdictExplanation(result: StoryboardResult): StoryboardVerdictExplanation | undefined;
17
23
  /** Labels for each compliance track. */
18
24
  export declare const TRACK_LABELS: Record<ComplianceTrack, string>;
19
25
  /**
@@ -20,10 +20,12 @@ var storyboard_tracks_exports = {};
20
20
  __export(storyboard_tracks_exports, {
21
21
  TRACK_LABELS: () => TRACK_LABELS,
22
22
  collectDetachedAssertionFailures: () => collectDetachedAssertionFailures,
23
- mapStoryboardResultsToTrackResult: () => mapStoryboardResultsToTrackResult
23
+ mapStoryboardResultsToTrackResult: () => mapStoryboardResultsToTrackResult,
24
+ storyboardVerdictExplanation: () => storyboardVerdictExplanation
24
25
  });
25
26
  module.exports = __toCommonJS(storyboard_tracks_exports);
26
27
  var import_runner = require('../storyboard/runner.js');
28
+ var import_verdict_explanation = require('../storyboard/verdict-explanation.js');
27
29
  function collectDetachedAssertionFailures(result) {
28
30
  const failures = [];
29
31
  for (const assertion of result.assertions ?? []) {
@@ -40,6 +42,18 @@ function collectDetachedAssertionFailures(result) {
40
42
  }
41
43
  return failures;
42
44
  }
45
+ function storyboardVerdictExplanation(result) {
46
+ if (result.overall_passed) return void 0;
47
+ if (result.verdict_reason !== void 0 && result.verdict_detail !== void 0) {
48
+ return {
49
+ verdict_reason: result.verdict_reason,
50
+ verdict_detail: result.verdict_detail,
51
+ verdict_contributing: result.verdict_contributing ?? []
52
+ };
53
+ }
54
+ const steps = (result.passes?.flatMap((pass) => pass.phases) ?? result.phases).flatMap((phase) => phase.steps);
55
+ return (0, import_verdict_explanation.explainStoryboardVerdict)(result, { coverageUnverified: (0, import_runner.signingCoverageUnverified)(steps) });
56
+ }
43
57
  const TRACK_LABELS = {
44
58
  core: "Core Protocol",
45
59
  products: "Product Discovery",
@@ -116,14 +130,21 @@ function mapStoryboardResultsToTrackResult(track, storyboardResults, _profile) {
116
130
  scenarios.push(testResult);
117
131
  }
118
132
  if (!sbResult.overall_passed && (sbResult.failed_count === 0 || (sbResult.passed_count > 0 || assertionFailures.length > 0) && sbResult.phases.every((phase) => phase.passed))) {
133
+ const explanation = storyboardVerdictExplanation(sbResult);
134
+ const why = explanation ? `: ${explanation.verdict_detail}` : "";
119
135
  scenarios.push({
120
136
  agent_url: sbResult.agent_url,
121
137
  scenario: `${sbResult.storyboard_id}/storyboard`,
122
138
  overall_passed: false,
123
139
  steps: [],
124
- summary: assertionFailures.length > 0 ? `${sbResult.storyboard_title}: ${assertionFailures.length} assertion(s) failed` : `${sbResult.storyboard_title}: storyboard-level verdict failed`,
140
+ summary: assertionFailures.length > 0 ? `${sbResult.storyboard_title}: ${assertionFailures.length} assertion(s) failed${why}` : `${sbResult.storyboard_title}: storyboard-level verdict failed${why}`,
125
141
  total_duration_ms: 0,
126
- tested_at: sbResult.tested_at
142
+ tested_at: sbResult.tested_at,
143
+ ...explanation && {
144
+ verdict_reason: explanation.verdict_reason,
145
+ verdict_detail: explanation.verdict_detail,
146
+ verdict_contributing: explanation.verdict_contributing
147
+ }
127
148
  });
128
149
  }
129
150
  }
@@ -216,5 +237,6 @@ function computeTrackStatus(results) {
216
237
  0 && (module.exports = {
217
238
  TRACK_LABELS,
218
239
  collectDetachedAssertionFailures,
219
- mapStoryboardResultsToTrackResult
240
+ mapStoryboardResultsToTrackResult,
241
+ storyboardVerdictExplanation
220
242
  });