@adcp/sdk 17.0.0-beta.27 → 17.0.0-beta.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/adcp-storyboard-summary.js +47 -0
- package/bin/adcp.js +4 -0
- package/dist/lib/registry/types.generated.d.ts +3 -1
- package/dist/lib/testing/compliance/comply.d.mts +16 -1
- package/dist/lib/testing/compliance/comply.d.ts +16 -1
- package/dist/lib/testing/compliance/comply.js +69 -5
- package/dist/lib/testing/compliance/comply.mjs +75 -6
- package/dist/lib/testing/compliance/index.d.mts +1 -1
- package/dist/lib/testing/compliance/index.d.ts +1 -1
- package/dist/lib/testing/compliance/storyboard-tracks.d.mts +6 -0
- package/dist/lib/testing/compliance/storyboard-tracks.d.ts +6 -0
- package/dist/lib/testing/compliance/storyboard-tracks.js +26 -4
- package/dist/lib/testing/compliance/storyboard-tracks.mjs +24 -3
- package/dist/lib/testing/compliance/summary.d.mts +3 -1
- package/dist/lib/testing/compliance/summary.d.ts +3 -1
- package/dist/lib/testing/compliance/summary.js +2 -1
- package/dist/lib/testing/compliance/summary.mjs +2 -1
- package/dist/lib/testing/compliance/types.d.mts +30 -1
- package/dist/lib/testing/compliance/types.d.ts +30 -1
- package/dist/lib/testing/index.d.mts +1 -1
- package/dist/lib/testing/index.d.ts +1 -1
- package/dist/lib/testing/index.js +4 -0
- package/dist/lib/testing/index.mjs +4 -0
- package/dist/lib/testing/storyboard/index.d.mts +4 -2
- package/dist/lib/testing/storyboard/index.d.ts +4 -2
- package/dist/lib/testing/storyboard/index.js +7 -0
- package/dist/lib/testing/storyboard/index.mjs +6 -1
- package/dist/lib/testing/storyboard/runner.d.mts +7 -0
- package/dist/lib/testing/storyboard/runner.d.ts +7 -0
- package/dist/lib/testing/storyboard/runner.js +15 -0
- package/dist/lib/testing/storyboard/runner.mjs +14 -0
- package/dist/lib/testing/storyboard/types.d.mts +77 -0
- package/dist/lib/testing/storyboard/types.d.ts +77 -0
- package/dist/lib/testing/storyboard/validations.d.mts +1 -1
- package/dist/lib/testing/storyboard/validations.d.ts +1 -1
- package/dist/lib/testing/storyboard/verdict-explanation.d.mts +47 -0
- package/dist/lib/testing/storyboard/verdict-explanation.d.ts +47 -0
- package/dist/lib/testing/storyboard/verdict-explanation.js +309 -0
- package/dist/lib/testing/storyboard/verdict-explanation.mjs +283 -0
- package/dist/lib/testing/types.d.mts +11 -1
- package/dist/lib/testing/types.d.ts +11 -1
- package/dist/lib/version.d.mts +3 -3
- package/dist/lib/version.d.ts +3 -3
- package/dist/lib/version.js +3 -3
- package/dist/lib/version.mjs +3 -3
- package/docs/TYPE-SUMMARY.md +2 -2
- package/docs/guides/VALIDATE-YOUR-AGENT.md +28 -0
- package/docs/llms.txt +2 -2
- package/docs/migration-14.x-rc-worksheet.md +4 -4
- package/package.json +1 -1
|
@@ -147,9 +147,55 @@ function buildStoryboardSummaryMarkdown(results, agentUrl, overallPassed) {
|
|
|
147
147
|
}
|
|
148
148
|
lines.push('');
|
|
149
149
|
}
|
|
150
|
+
// Failing storyboards no failed step explains (adcp#8576).
|
|
151
|
+
const storyboardLevel = results.filter(
|
|
152
|
+
r => r.overall_passed === false && r.verdict_detail && !failures.some(f => f.storyboard === (r.storyboard_id || ''))
|
|
153
|
+
);
|
|
154
|
+
if (storyboardLevel.length > 0) {
|
|
155
|
+
lines.push('## Storyboard-level verdicts');
|
|
156
|
+
lines.push('');
|
|
157
|
+
lines.push('| Storyboard | Reason | Detail |');
|
|
158
|
+
lines.push('|---|---|---|');
|
|
159
|
+
for (const r of storyboardLevel) {
|
|
160
|
+
lines.push(
|
|
161
|
+
`| ${escapeMarkdownCell(r.storyboard_id || '')} | ${escapeMarkdownCell(r.verdict_reason || 'unexplained')} | ${escapeMarkdownCell(r.verdict_detail)} |`
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
lines.push('');
|
|
165
|
+
}
|
|
150
166
|
return lines.join('\n');
|
|
151
167
|
}
|
|
152
168
|
|
|
169
|
+
const VERDICT_CONTRIBUTOR_LIMIT = 8;
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Console lines explaining a failing storyboard verdict (adcp#8576): the
|
|
173
|
+
* runner's `verdict_reason`, `verdict_detail`, and the contributing steps,
|
|
174
|
+
* assertions and phases. Empty for a passing result or one from a runner
|
|
175
|
+
* that predates the fields. Every runner- or agent-sourced string is
|
|
176
|
+
* terminal-escaped.
|
|
177
|
+
*/
|
|
178
|
+
function formatStoryboardVerdictLines(result, indent = ' ') {
|
|
179
|
+
if (!result || result.overall_passed !== false || !result.verdict_reason) return [];
|
|
180
|
+
const lines = [
|
|
181
|
+
`${indent}Why: [${escapeTerminalControlChars(result.verdict_reason)}] ${escapeTerminalControlChars(result.verdict_detail ?? '')}`,
|
|
182
|
+
];
|
|
183
|
+
const rows = Array.isArray(result.verdict_contributing) ? result.verdict_contributing : [];
|
|
184
|
+
for (const row of rows.slice(0, VERDICT_CONTRIBUTOR_LIMIT)) {
|
|
185
|
+
const where = [row.phase_id && `phase ${row.phase_id}`, row.pass_index !== undefined && `pass ${row.pass_index}`]
|
|
186
|
+
.filter(Boolean)
|
|
187
|
+
.join(', ');
|
|
188
|
+
const skip = row.skip_reason ? ` ${row.skip_reason}${row.requirement ? `:${row.requirement}` : ''}` : '';
|
|
189
|
+
const head = `${row.kind} ${row.id} ${row.status}${skip}${where ? ` (${where})` : ''}`;
|
|
190
|
+
const message = row.message ? ` — ${row.message}` : '';
|
|
191
|
+
lines.push(`${indent} - ${escapeTerminalControlChars(head + message)}`);
|
|
192
|
+
}
|
|
193
|
+
if (rows.length > VERDICT_CONTRIBUTOR_LIMIT) {
|
|
194
|
+
lines.push(`${indent} … ${rows.length - VERDICT_CONTRIBUTOR_LIMIT} more (use --json for all)`);
|
|
195
|
+
}
|
|
196
|
+
return lines;
|
|
197
|
+
}
|
|
198
|
+
|
|
153
199
|
/**
|
|
154
200
|
* Writes `content` to `summaryFile`. Failures are downgraded to a stderr
|
|
155
201
|
* warning rather than exiting the process — the markdown summary is a
|
|
@@ -474,6 +520,7 @@ module.exports = {
|
|
|
474
520
|
formatJwtExpClaim,
|
|
475
521
|
formatStepSkipLines,
|
|
476
522
|
formatStepVerdictLines,
|
|
523
|
+
formatStoryboardVerdictLines,
|
|
477
524
|
isCoverageGapRow,
|
|
478
525
|
isCoverageUnavailableStep,
|
|
479
526
|
unverifiedSigningCoverage,
|
package/bin/adcp.js
CHANGED
|
@@ -52,6 +52,7 @@ const {
|
|
|
52
52
|
formatJwtExpClaim,
|
|
53
53
|
formatStepSkipLines,
|
|
54
54
|
formatStepVerdictLines,
|
|
55
|
+
formatStoryboardVerdictLines,
|
|
55
56
|
isCoverageUnavailableStep,
|
|
56
57
|
unverifiedSigningCoverage,
|
|
57
58
|
parseRequestSigningFlags,
|
|
@@ -3158,6 +3159,8 @@ async function handleStoryboardRun(args) {
|
|
|
3158
3159
|
console.log(
|
|
3159
3160
|
`${overallIcon} ${result.passed_count} passed, ${result.failed_count} failed, ${result.skipped_count} skipped (${result.total_duration_ms}ms)${hintTail}`
|
|
3160
3161
|
);
|
|
3162
|
+
// A storyboard can fail with every visible step green; say why (adcp#8576).
|
|
3163
|
+
for (const line of formatStoryboardVerdictLines(result)) console.log(line);
|
|
3161
3164
|
printStrictSummary(result.strict_validation_summary);
|
|
3162
3165
|
printNotices(result.notices);
|
|
3163
3166
|
}
|
|
@@ -4164,6 +4167,7 @@ async function handleLocalAgentStoryboardRun(modulePath, args, opts) {
|
|
|
4164
4167
|
console.log(
|
|
4165
4168
|
`${icon} ${sb.storyboard_title} (${sb.storyboard_id}) — ${sb.passed_count} passed, ${sb.failed_count} failed, ${sb.skipped_count} skipped`
|
|
4166
4169
|
);
|
|
4170
|
+
for (const line of formatStoryboardVerdictLines(sb)) console.log(line);
|
|
4167
4171
|
}
|
|
4168
4172
|
if (result.not_applicable.length > 0) {
|
|
4169
4173
|
console.log(`\n${result.not_applicable.length} storyboard(s) not applicable (out of scope):`);
|
|
@@ -6576,6 +6576,7 @@ export interface operations {
|
|
|
6576
6576
|
properties_total: number;
|
|
6577
6577
|
/** @description Canonical list of property_id strings the agent is authorized for under this publisher. Present iff the request included `?include=properties`. Same population as `properties_authorized` but surfaced as IDs for set-diff comparison. */
|
|
6578
6578
|
property_ids?: string[];
|
|
6579
|
+
/** @description Scope-agnostic summary: true only when every in-window authorized_agents entry that matches the agent carries signing_keys. Verifiers compute the pin per message scope, so a pin can apply where this reads false. */
|
|
6579
6580
|
signing_keys_pinned: boolean;
|
|
6580
6581
|
/** @enum {string} */
|
|
6581
6582
|
status: "authorized" | "revoked";
|
|
@@ -6658,6 +6659,7 @@ export interface operations {
|
|
|
6658
6659
|
properties_total: number;
|
|
6659
6660
|
/** @description Canonical list of property_id strings the agent is authorized for under this publisher. Present iff the request included `?include=properties`. Same population as `properties_authorized` but surfaced as IDs for set-diff comparison. */
|
|
6660
6661
|
property_ids?: string[];
|
|
6662
|
+
/** @description Scope-agnostic summary: true only when every in-window authorized_agents entry that matches the agent carries signing_keys. Verifiers compute the pin per message scope, so a pin can apply where this reads false. */
|
|
6661
6663
|
signing_keys_pinned: boolean;
|
|
6662
6664
|
/** @enum {string} */
|
|
6663
6665
|
status: "authorized" | "revoked";
|
|
@@ -8826,7 +8828,7 @@ export interface operations {
|
|
|
8826
8828
|
created_at: string;
|
|
8827
8829
|
/** Format: date-time */
|
|
8828
8830
|
updated_at: string;
|
|
8829
|
-
/** @description Publisher-pinned JWK set (authorized_agents[*].signing_keys) from the source adagents.json. Consumers verifying inbound TMP signatures key on kid → JWK. Null when the publisher declared no keys; consumers fall back to the agent-hosted JWKS per spec R-2 (docs/governance/property/adagents.mdx). */
|
|
8831
|
+
/** @description Publisher-pinned JWK set (authorized_agents[*].signing_keys) from the source adagents.json. Consumers verifying inbound TMP signatures key on kid → JWK. Null when the publisher declared no keys; consumers fall back to the agent-hosted JWKS per spec R-2 (docs/governance/property/adagents.mdx). This is one entry's pin. Rows from the same publisher combine per message scope: among the rows that authorize that scope, the publisher pins the agent only if every one carries keys, and the pin is the union of their keys (docs/governance/property/adagents.mdx#agent-url-matching). */
|
|
8830
8832
|
signing_keys: components["schemas"]["SigningKey"][] | null;
|
|
8831
8833
|
/** @description Collection constraints (authorized_agents[*].collections) from the source adagents.json. Null when the entry is unconstrained. When set, the row does NOT authorize the property unqualified — consumers MUST scope it to these selectors. A selector without collection_ids is a bulk grant for all collections declared at that publisher_domain. */
|
|
8832
8834
|
collections: ({
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import type { TestOptions, TestResult, AgentProfile } from '../types.mjs';
|
|
11
11
|
import type { NotApplicableStoryboard, ResolvedBundle } from '../storyboard/compliance.mjs';
|
|
12
12
|
import type { AgentEntry, Storyboard, StoryboardContext, StoryboardResult, StoryboardRunOptions } from '../storyboard/types.mjs';
|
|
13
|
-
import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types.mjs';
|
|
13
|
+
import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceVerdictExplanation, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types.mjs';
|
|
14
14
|
import type { VersionEnvelopeMode } from '../../protocols/index.mjs';
|
|
15
15
|
/**
|
|
16
16
|
* Collect advisory observations from test results.
|
|
@@ -258,6 +258,20 @@ export declare function extractFailures(results: StoryboardResult[], storyboards
|
|
|
258
258
|
hostedStableLineAlias?: string;
|
|
259
259
|
testKitPath?: string;
|
|
260
260
|
}): ComplianceFailure[];
|
|
261
|
+
/**
|
|
262
|
+
* One explanation per failing storyboard (adcp#8576), with a `fix_command`
|
|
263
|
+
* that re-runs the whole storyboard. `storyboard_level` marks the verdicts no
|
|
264
|
+
* `failures[]` row explains — the ones that used to surface only as
|
|
265
|
+
* "storyboard-level verdict failed".
|
|
266
|
+
*/
|
|
267
|
+
export declare function extractVerdictExplanations(results: StoryboardResult[], storyboards: Storyboard[], agentRef: string, failures: readonly ComplianceFailure[], fixOptions?: Parameters<typeof buildFixCommand>[3]): ComplianceVerdictExplanation[];
|
|
268
|
+
declare function buildFixCommand(agentRef: string, storyboardId: string, stepId: string | undefined, options: {
|
|
269
|
+
complianceVersion?: string;
|
|
270
|
+
complianceDir?: string;
|
|
271
|
+
schemaRoot?: string;
|
|
272
|
+
hostedStableLineAlias?: string;
|
|
273
|
+
testKitPath?: string;
|
|
274
|
+
}): string;
|
|
261
275
|
/**
|
|
262
276
|
* Rotate a storyboard list to start at `offset % length`, preserving relative
|
|
263
277
|
* order (the head wraps to the tail). Pure and deterministic: offset 0, an
|
|
@@ -287,3 +301,4 @@ export declare function computeOverallStatus(summary: ComplianceSummary): Overal
|
|
|
287
301
|
*/
|
|
288
302
|
export declare function formatComplianceResults(result: ComplianceResult): string;
|
|
289
303
|
export declare function formatComplianceResultsJSON(result: ComplianceResult): string;
|
|
304
|
+
export {};
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import type { TestOptions, TestResult, AgentProfile } from '../types';
|
|
11
11
|
import type { NotApplicableStoryboard, ResolvedBundle } from '../storyboard/compliance';
|
|
12
12
|
import type { AgentEntry, Storyboard, StoryboardContext, StoryboardResult, StoryboardRunOptions } from '../storyboard/types';
|
|
13
|
-
import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types';
|
|
13
|
+
import type { ComplianceBundleResult, ComplianceTrack, ComplianceFailure, ComplianceVerdictExplanation, ComplianceResult, ComplianceSummary, AdvisoryObservation, OverallStatus } from './types';
|
|
14
14
|
import type { VersionEnvelopeMode } from '../../protocols';
|
|
15
15
|
/**
|
|
16
16
|
* Collect advisory observations from test results.
|
|
@@ -258,6 +258,20 @@ export declare function extractFailures(results: StoryboardResult[], storyboards
|
|
|
258
258
|
hostedStableLineAlias?: string;
|
|
259
259
|
testKitPath?: string;
|
|
260
260
|
}): ComplianceFailure[];
|
|
261
|
+
/**
|
|
262
|
+
* One explanation per failing storyboard (adcp#8576), with a `fix_command`
|
|
263
|
+
* that re-runs the whole storyboard. `storyboard_level` marks the verdicts no
|
|
264
|
+
* `failures[]` row explains — the ones that used to surface only as
|
|
265
|
+
* "storyboard-level verdict failed".
|
|
266
|
+
*/
|
|
267
|
+
export declare function extractVerdictExplanations(results: StoryboardResult[], storyboards: Storyboard[], agentRef: string, failures: readonly ComplianceFailure[], fixOptions?: Parameters<typeof buildFixCommand>[3]): ComplianceVerdictExplanation[];
|
|
268
|
+
declare function buildFixCommand(agentRef: string, storyboardId: string, stepId: string | undefined, options: {
|
|
269
|
+
complianceVersion?: string;
|
|
270
|
+
complianceDir?: string;
|
|
271
|
+
schemaRoot?: string;
|
|
272
|
+
hostedStableLineAlias?: string;
|
|
273
|
+
testKitPath?: string;
|
|
274
|
+
}): string;
|
|
261
275
|
/**
|
|
262
276
|
* Rotate a storyboard list to start at `offset % length`, preserving relative
|
|
263
277
|
* order (the head wraps to the tail). Pure and deterministic: offset 0, an
|
|
@@ -287,3 +301,4 @@ export declare function computeOverallStatus(summary: ComplianceSummary): Overal
|
|
|
287
301
|
*/
|
|
288
302
|
export declare function formatComplianceResults(result: ComplianceResult): string;
|
|
289
303
|
export declare function formatComplianceResultsJSON(result: ComplianceResult): string;
|
|
304
|
+
export {};
|
|
@@ -25,6 +25,7 @@ __export(comply_exports, {
|
|
|
25
25
|
computeOverallStatus: () => computeOverallStatus,
|
|
26
26
|
detectAuthRejection: () => detectAuthRejection,
|
|
27
27
|
extractFailures: () => extractFailures,
|
|
28
|
+
extractVerdictExplanations: () => extractVerdictExplanations,
|
|
28
29
|
formatComplianceResults: () => formatComplianceResults,
|
|
29
30
|
formatComplianceResultsJSON: () => formatComplianceResultsJSON,
|
|
30
31
|
partitionStoryboardsByRequiredTools: () => partitionStoryboardsByRequiredTools,
|
|
@@ -964,7 +965,10 @@ async function runComplyStoryboard(agentUrl, storyboard, runOptions, profile, ro
|
|
|
964
965
|
const routed = buildRoutedStoryboardRun(agentUrl, storyboard, runOptions, routing);
|
|
965
966
|
if ("refused" in routed) return buildRouteSkippedStoryboardResult(agentUrl, storyboard, routed.refused, runOptions);
|
|
966
967
|
const result = await (0, import_runner.runStoryboard)("", routed.storyboard, routed.options);
|
|
967
|
-
return
|
|
968
|
+
return (0, import_runner.attachStoryboardVerdictExplanation)(
|
|
969
|
+
gradeRoutedResultForAgentUnderTest(result, routed.agentUnderTestIndexes, routed.keyByIndex),
|
|
970
|
+
routed.storyboard
|
|
971
|
+
);
|
|
968
972
|
}
|
|
969
973
|
function sameAgentUrl(a, b) {
|
|
970
974
|
try {
|
|
@@ -1318,6 +1322,23 @@ function extractFailures(results, storyboards, agentRef, fixOptions = {}) {
|
|
|
1318
1322
|
}
|
|
1319
1323
|
return failures;
|
|
1320
1324
|
}
|
|
1325
|
+
function extractVerdictExplanations(results, storyboards, agentRef, failures, fixOptions = {}) {
|
|
1326
|
+
const trackById = new Map(storyboards.map((sb) => [sb.id, sb.track ?? "core"]));
|
|
1327
|
+
const explained = new Set(failures.map((failure) => failure.storyboard_id));
|
|
1328
|
+
const explanations = [];
|
|
1329
|
+
for (const result of results) {
|
|
1330
|
+
const explanation = (0, import_storyboard_tracks.storyboardVerdictExplanation)(result);
|
|
1331
|
+
if (!explanation) continue;
|
|
1332
|
+
explanations.push({
|
|
1333
|
+
track: trackById.get(result.storyboard_id) ?? "core",
|
|
1334
|
+
storyboard_id: result.storyboard_id,
|
|
1335
|
+
...explanation,
|
|
1336
|
+
storyboard_level: !explained.has(result.storyboard_id),
|
|
1337
|
+
fix_command: buildFixCommand(agentRef, result.storyboard_id, void 0, fixOptions)
|
|
1338
|
+
});
|
|
1339
|
+
}
|
|
1340
|
+
return explanations;
|
|
1341
|
+
}
|
|
1321
1342
|
function findStoryboardStepExpected(storyboard, stepId) {
|
|
1322
1343
|
if (!storyboard) return void 0;
|
|
1323
1344
|
for (const phase of storyboard.phases) {
|
|
@@ -1647,13 +1668,21 @@ async function complyImpl(agentUrl, options) {
|
|
|
1647
1668
|
const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
|
|
1648
1669
|
const safeAgentUrl = (0, import_oauth_metadata_graph.redactOAuthUrlForOutput)(agentUrl);
|
|
1649
1670
|
const agentRef = options.agent_alias || safeAgentUrl;
|
|
1650
|
-
const
|
|
1671
|
+
const fixOptions = {
|
|
1651
1672
|
complianceVersion: complianceIndex.adcp_version,
|
|
1652
1673
|
...complianceDir !== void 0 && { complianceDir },
|
|
1653
1674
|
...schemaRoot !== void 0 && { schemaRoot },
|
|
1654
1675
|
...hostedStableLineAlias !== void 0 && { hostedStableLineAlias },
|
|
1655
1676
|
...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
|
|
1656
|
-
}
|
|
1677
|
+
};
|
|
1678
|
+
const failures = extractFailures(storyboardResults, runnableStoryboards, agentRef, fixOptions);
|
|
1679
|
+
const verdictExplanations = extractVerdictExplanations(
|
|
1680
|
+
storyboardResults,
|
|
1681
|
+
runnableStoryboards,
|
|
1682
|
+
agentRef,
|
|
1683
|
+
failures,
|
|
1684
|
+
fixOptions
|
|
1685
|
+
);
|
|
1657
1686
|
const aggregatedNotices = /* @__PURE__ */ new Map();
|
|
1658
1687
|
for (const sbResult of storyboardResults) {
|
|
1659
1688
|
for (const notice of sbResult.notices) {
|
|
@@ -1681,6 +1710,7 @@ async function complyImpl(agentUrl, options) {
|
|
|
1681
1710
|
summary,
|
|
1682
1711
|
observations: allObservations,
|
|
1683
1712
|
failures: failures.length > 0 ? failures : void 0,
|
|
1713
|
+
...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
|
|
1684
1714
|
storyboards_executed: executedStoryboards.map((sb) => sb.id),
|
|
1685
1715
|
...resolvedBundles !== void 0 && {
|
|
1686
1716
|
bundle_results: buildComplianceBundleResults(resolvedBundles, storyboardResults, {
|
|
@@ -1849,13 +1879,21 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
|
|
|
1849
1879
|
const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
|
|
1850
1880
|
const safeAgentUrl = (0, import_oauth_metadata_graph.redactOAuthUrlForOutput)(agentUrl);
|
|
1851
1881
|
const agentRef = options.agent_alias || safeAgentUrl;
|
|
1852
|
-
const
|
|
1882
|
+
const fixOptions = {
|
|
1853
1883
|
complianceVersion: adcpVersion,
|
|
1854
1884
|
...options.complianceDir !== void 0 && { complianceDir: options.complianceDir },
|
|
1855
1885
|
...options.schemaRoot !== void 0 && { schemaRoot: options.schemaRoot },
|
|
1856
1886
|
...options.hostedStableLineAlias !== void 0 && { hostedStableLineAlias: options.hostedStableLineAlias },
|
|
1857
1887
|
...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
|
|
1858
|
-
}
|
|
1888
|
+
};
|
|
1889
|
+
const failures = extractFailures(storyboardResults, storyboards, agentRef, fixOptions);
|
|
1890
|
+
const verdictExplanations = extractVerdictExplanations(
|
|
1891
|
+
storyboardResults,
|
|
1892
|
+
storyboards,
|
|
1893
|
+
agentRef,
|
|
1894
|
+
failures,
|
|
1895
|
+
fixOptions
|
|
1896
|
+
);
|
|
1859
1897
|
for (const t of trackResults) t._view = "canonical";
|
|
1860
1898
|
const skippedTracks = trackResults.filter((t) => t.status === "skip").map((t) => ({
|
|
1861
1899
|
track: t.track,
|
|
@@ -1874,6 +1912,7 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
|
|
|
1874
1912
|
summary,
|
|
1875
1913
|
observations: allObservations,
|
|
1876
1914
|
failures: failures.length > 0 ? failures : void 0,
|
|
1915
|
+
...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
|
|
1877
1916
|
storyboards_executed: executedStoryboards.map((sb) => sb.id),
|
|
1878
1917
|
controller_detected: false,
|
|
1879
1918
|
tested_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -2125,6 +2164,10 @@ ${overallIcon} AdCP Compliance Report
|
|
|
2125
2164
|
}
|
|
2126
2165
|
if (failedSteps.length > 3) {
|
|
2127
2166
|
output += ` ... and ${failedSteps.length - 3} more
|
|
2167
|
+
`;
|
|
2168
|
+
}
|
|
2169
|
+
if (failedSteps.length === 0 && scenario.verdict_detail) {
|
|
2170
|
+
output += ` Why: ${fenceAgentText(scenario.verdict_detail, 600)}
|
|
2128
2171
|
`;
|
|
2129
2172
|
}
|
|
2130
2173
|
}
|
|
@@ -2171,6 +2214,26 @@ How to Fix
|
|
|
2171
2214
|
}
|
|
2172
2215
|
if (failuresWithExpected.length > 5) {
|
|
2173
2216
|
output += ` ... and ${failuresWithExpected.length - 5} more (use --json for all)
|
|
2217
|
+
`;
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
const storyboardLevel = (result.verdict_explanations ?? []).filter((explanation) => explanation.storyboard_level);
|
|
2221
|
+
if (storyboardLevel.length > 0) {
|
|
2222
|
+
output += `
|
|
2223
|
+
Storyboard-level Verdicts
|
|
2224
|
+
`;
|
|
2225
|
+
output += `${"\u2500".repeat(50)}
|
|
2226
|
+
`;
|
|
2227
|
+
for (const explanation of storyboardLevel.slice(0, 5)) {
|
|
2228
|
+
output += `\u274C ${explanation.storyboard_id} (${explanation.verdict_reason})
|
|
2229
|
+
`;
|
|
2230
|
+
output += ` Why: ${fenceAgentText(explanation.verdict_detail, 600)}
|
|
2231
|
+
`;
|
|
2232
|
+
output += ` Debug: ${explanation.fix_command}
|
|
2233
|
+
`;
|
|
2234
|
+
}
|
|
2235
|
+
if (storyboardLevel.length > 5) {
|
|
2236
|
+
output += ` ... and ${storyboardLevel.length - 5} more (use --json for all)
|
|
2174
2237
|
`;
|
|
2175
2238
|
}
|
|
2176
2239
|
}
|
|
@@ -2283,6 +2346,7 @@ function formatMachineValue(value) {
|
|
|
2283
2346
|
computeOverallStatus,
|
|
2284
2347
|
detectAuthRejection,
|
|
2285
2348
|
extractFailures,
|
|
2349
|
+
extractVerdictExplanations,
|
|
2286
2350
|
formatComplianceResults,
|
|
2287
2351
|
formatComplianceResultsJSON,
|
|
2288
2352
|
partitionStoryboardsByRequiredTools,
|
|
@@ -7,9 +7,15 @@ import {
|
|
|
7
7
|
seedAuthRejectedTestClientProfile,
|
|
8
8
|
withTestClientScope
|
|
9
9
|
} from "../client.mjs";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
collectDetachedAssertionFailures,
|
|
12
|
+
mapStoryboardResultsToTrackResult,
|
|
13
|
+
storyboardVerdictExplanation,
|
|
14
|
+
TRACK_LABELS
|
|
15
|
+
} from "./storyboard-tracks.mjs";
|
|
11
16
|
import {
|
|
12
17
|
applyAdcpVersionRunOptions,
|
|
18
|
+
attachStoryboardVerdictExplanation,
|
|
13
19
|
collectCapabilityNotices,
|
|
14
20
|
runStoryboard,
|
|
15
21
|
storyboardCapabilityGateUnmet,
|
|
@@ -958,7 +964,10 @@ async function runComplyStoryboard(agentUrl, storyboard, runOptions, profile, ro
|
|
|
958
964
|
const routed = buildRoutedStoryboardRun(agentUrl, storyboard, runOptions, routing);
|
|
959
965
|
if ("refused" in routed) return buildRouteSkippedStoryboardResult(agentUrl, storyboard, routed.refused, runOptions);
|
|
960
966
|
const result = await runStoryboard("", routed.storyboard, routed.options);
|
|
961
|
-
return
|
|
967
|
+
return attachStoryboardVerdictExplanation(
|
|
968
|
+
gradeRoutedResultForAgentUnderTest(result, routed.agentUnderTestIndexes, routed.keyByIndex),
|
|
969
|
+
routed.storyboard
|
|
970
|
+
);
|
|
962
971
|
}
|
|
963
972
|
function sameAgentUrl(a, b) {
|
|
964
973
|
try {
|
|
@@ -1312,6 +1321,23 @@ function extractFailures(results, storyboards, agentRef, fixOptions = {}) {
|
|
|
1312
1321
|
}
|
|
1313
1322
|
return failures;
|
|
1314
1323
|
}
|
|
1324
|
+
function extractVerdictExplanations(results, storyboards, agentRef, failures, fixOptions = {}) {
|
|
1325
|
+
const trackById = new Map(storyboards.map((sb) => [sb.id, sb.track ?? "core"]));
|
|
1326
|
+
const explained = new Set(failures.map((failure) => failure.storyboard_id));
|
|
1327
|
+
const explanations = [];
|
|
1328
|
+
for (const result of results) {
|
|
1329
|
+
const explanation = storyboardVerdictExplanation(result);
|
|
1330
|
+
if (!explanation) continue;
|
|
1331
|
+
explanations.push({
|
|
1332
|
+
track: trackById.get(result.storyboard_id) ?? "core",
|
|
1333
|
+
storyboard_id: result.storyboard_id,
|
|
1334
|
+
...explanation,
|
|
1335
|
+
storyboard_level: !explained.has(result.storyboard_id),
|
|
1336
|
+
fix_command: buildFixCommand(agentRef, result.storyboard_id, void 0, fixOptions)
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
return explanations;
|
|
1340
|
+
}
|
|
1315
1341
|
function findStoryboardStepExpected(storyboard, stepId) {
|
|
1316
1342
|
if (!storyboard) return void 0;
|
|
1317
1343
|
for (const phase of storyboard.phases) {
|
|
@@ -1641,13 +1667,21 @@ async function complyImpl(agentUrl, options) {
|
|
|
1641
1667
|
const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
|
|
1642
1668
|
const safeAgentUrl = redactOAuthUrlForOutput(agentUrl);
|
|
1643
1669
|
const agentRef = options.agent_alias || safeAgentUrl;
|
|
1644
|
-
const
|
|
1670
|
+
const fixOptions = {
|
|
1645
1671
|
complianceVersion: complianceIndex.adcp_version,
|
|
1646
1672
|
...complianceDir !== void 0 && { complianceDir },
|
|
1647
1673
|
...schemaRoot !== void 0 && { schemaRoot },
|
|
1648
1674
|
...hostedStableLineAlias !== void 0 && { hostedStableLineAlias },
|
|
1649
1675
|
...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
|
|
1650
|
-
}
|
|
1676
|
+
};
|
|
1677
|
+
const failures = extractFailures(storyboardResults, runnableStoryboards, agentRef, fixOptions);
|
|
1678
|
+
const verdictExplanations = extractVerdictExplanations(
|
|
1679
|
+
storyboardResults,
|
|
1680
|
+
runnableStoryboards,
|
|
1681
|
+
agentRef,
|
|
1682
|
+
failures,
|
|
1683
|
+
fixOptions
|
|
1684
|
+
);
|
|
1651
1685
|
const aggregatedNotices = /* @__PURE__ */ new Map();
|
|
1652
1686
|
for (const sbResult of storyboardResults) {
|
|
1653
1687
|
for (const notice of sbResult.notices) {
|
|
@@ -1675,6 +1709,7 @@ async function complyImpl(agentUrl, options) {
|
|
|
1675
1709
|
summary,
|
|
1676
1710
|
observations: allObservations,
|
|
1677
1711
|
failures: failures.length > 0 ? failures : void 0,
|
|
1712
|
+
...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
|
|
1678
1713
|
storyboards_executed: executedStoryboards.map((sb) => sb.id),
|
|
1679
1714
|
...resolvedBundles !== void 0 && {
|
|
1680
1715
|
bundle_results: buildComplianceBundleResults(resolvedBundles, storyboardResults, {
|
|
@@ -1843,13 +1878,21 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
|
|
|
1843
1878
|
const overallStatus = stoppedForTimeoutBudget ? "partial" : computeOverallStatus(summary);
|
|
1844
1879
|
const safeAgentUrl = redactOAuthUrlForOutput(agentUrl);
|
|
1845
1880
|
const agentRef = options.agent_alias || safeAgentUrl;
|
|
1846
|
-
const
|
|
1881
|
+
const fixOptions = {
|
|
1847
1882
|
complianceVersion: adcpVersion,
|
|
1848
1883
|
...options.complianceDir !== void 0 && { complianceDir: options.complianceDir },
|
|
1849
1884
|
...options.schemaRoot !== void 0 && { schemaRoot: options.schemaRoot },
|
|
1850
1885
|
...options.hostedStableLineAlias !== void 0 && { hostedStableLineAlias: options.hostedStableLineAlias },
|
|
1851
1886
|
...options.testKitPath !== void 0 && { testKitPath: options.testKitPath }
|
|
1852
|
-
}
|
|
1887
|
+
};
|
|
1888
|
+
const failures = extractFailures(storyboardResults, storyboards, agentRef, fixOptions);
|
|
1889
|
+
const verdictExplanations = extractVerdictExplanations(
|
|
1890
|
+
storyboardResults,
|
|
1891
|
+
storyboards,
|
|
1892
|
+
agentRef,
|
|
1893
|
+
failures,
|
|
1894
|
+
fixOptions
|
|
1895
|
+
);
|
|
1853
1896
|
for (const t of trackResults) t._view = "canonical";
|
|
1854
1897
|
const skippedTracks = trackResults.filter((t) => t.status === "skip").map((t) => ({
|
|
1855
1898
|
track: t.track,
|
|
@@ -1868,6 +1911,7 @@ async function runWithDegradedProfile(agentUrl, profile, storyboards, options, e
|
|
|
1868
1911
|
summary,
|
|
1869
1912
|
observations: allObservations,
|
|
1870
1913
|
failures: failures.length > 0 ? failures : void 0,
|
|
1914
|
+
...verdictExplanations.length > 0 && { verdict_explanations: verdictExplanations },
|
|
1871
1915
|
storyboards_executed: executedStoryboards.map((sb) => sb.id),
|
|
1872
1916
|
controller_detected: false,
|
|
1873
1917
|
tested_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -2119,6 +2163,10 @@ ${overallIcon} AdCP Compliance Report
|
|
|
2119
2163
|
}
|
|
2120
2164
|
if (failedSteps.length > 3) {
|
|
2121
2165
|
output += ` ... and ${failedSteps.length - 3} more
|
|
2166
|
+
`;
|
|
2167
|
+
}
|
|
2168
|
+
if (failedSteps.length === 0 && scenario.verdict_detail) {
|
|
2169
|
+
output += ` Why: ${fenceAgentText(scenario.verdict_detail, 600)}
|
|
2122
2170
|
`;
|
|
2123
2171
|
}
|
|
2124
2172
|
}
|
|
@@ -2165,6 +2213,26 @@ How to Fix
|
|
|
2165
2213
|
}
|
|
2166
2214
|
if (failuresWithExpected.length > 5) {
|
|
2167
2215
|
output += ` ... and ${failuresWithExpected.length - 5} more (use --json for all)
|
|
2216
|
+
`;
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
const storyboardLevel = (result.verdict_explanations ?? []).filter((explanation) => explanation.storyboard_level);
|
|
2220
|
+
if (storyboardLevel.length > 0) {
|
|
2221
|
+
output += `
|
|
2222
|
+
Storyboard-level Verdicts
|
|
2223
|
+
`;
|
|
2224
|
+
output += `${"\u2500".repeat(50)}
|
|
2225
|
+
`;
|
|
2226
|
+
for (const explanation of storyboardLevel.slice(0, 5)) {
|
|
2227
|
+
output += `\u274C ${explanation.storyboard_id} (${explanation.verdict_reason})
|
|
2228
|
+
`;
|
|
2229
|
+
output += ` Why: ${fenceAgentText(explanation.verdict_detail, 600)}
|
|
2230
|
+
`;
|
|
2231
|
+
output += ` Debug: ${explanation.fix_command}
|
|
2232
|
+
`;
|
|
2233
|
+
}
|
|
2234
|
+
if (storyboardLevel.length > 5) {
|
|
2235
|
+
output += ` ... and ${storyboardLevel.length - 5} more (use --json for all)
|
|
2168
2236
|
`;
|
|
2169
2237
|
}
|
|
2170
2238
|
}
|
|
@@ -2276,6 +2344,7 @@ export {
|
|
|
2276
2344
|
computeOverallStatus,
|
|
2277
2345
|
detectAuthRejection,
|
|
2278
2346
|
extractFailures,
|
|
2347
|
+
extractVerdictExplanations,
|
|
2279
2348
|
formatComplianceResults,
|
|
2280
2349
|
formatComplianceResultsJSON,
|
|
2281
2350
|
partitionStoryboardsByRequiredTools,
|
|
@@ -8,4 +8,4 @@ export type { ComplyOptions, ComplianceBundleAssessmentOptions, ComplyRouteStory
|
|
|
8
8
|
export { buildComplianceSummary, buildCrashSummary, formatComplianceSummaryText, formatComplianceSummaryMarkdown, } from './summary.mjs';
|
|
9
9
|
export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, ComplianceSummaryIncompleteStoryboard, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary.mjs';
|
|
10
10
|
export { SAMPLE_BRIEFS, getBriefById, getBriefsByVertical } from './briefs.mjs';
|
|
11
|
-
export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types.mjs';
|
|
11
|
+
export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceVerdictExplanation, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types.mjs';
|
|
@@ -8,4 +8,4 @@ export type { ComplyOptions, ComplianceBundleAssessmentOptions, ComplyRouteStory
|
|
|
8
8
|
export { buildComplianceSummary, buildCrashSummary, formatComplianceSummaryText, formatComplianceSummaryMarkdown, } from './summary';
|
|
9
9
|
export type { ComplianceSummaryArtifact, ComplianceSummaryFailure, ComplianceSummaryFailureKind, ComplianceSummaryIncompleteStoryboard, BuildSummaryOptions, BuildCrashSummaryOptions, } from './summary';
|
|
10
10
|
export { SAMPLE_BRIEFS, getBriefById, getBriefsByVertical } from './briefs';
|
|
11
|
-
export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types';
|
|
11
|
+
export type { ComplianceTrack, TrackResult, TestedTrackEntry, TrackStatus, OverallStatus, ComplianceFailure, ComplianceVerdictExplanation, ComplianceBundleResult, ComplianceBundleStatus, ComplianceResult, ComplianceSummary, AdvisoryObservation, ObservationCategory, ObservationSeverity, ObservationSource, SampleBrief, } from './types';
|
|
@@ -8,12 +8,18 @@
|
|
|
8
8
|
import type { AgentProfile } from '../types.mjs';
|
|
9
9
|
import type { ComplianceTrack, TrackResult } from './types.mjs';
|
|
10
10
|
import type { AssertionResult, StoryboardResult, StoryboardStepResult } from '../storyboard/types.mjs';
|
|
11
|
+
import { type StoryboardVerdictExplanation } from '../storyboard/verdict-explanation.mjs';
|
|
11
12
|
export interface DetachedAssertionFailure {
|
|
12
13
|
assertion: AssertionResult;
|
|
13
14
|
owningStep?: StoryboardStepResult;
|
|
14
15
|
}
|
|
15
16
|
/** Assertion failures that are not represented by a counted failed step. */
|
|
16
17
|
export declare function collectDetachedAssertionFailures(result: StoryboardResult): DetachedAssertionFailure[];
|
|
18
|
+
/**
|
|
19
|
+
* The verdict explanation the runner attached, or one derived from the result
|
|
20
|
+
* when it carries none. `undefined` for a passing result.
|
|
21
|
+
*/
|
|
22
|
+
export declare function storyboardVerdictExplanation(result: StoryboardResult): StoryboardVerdictExplanation | undefined;
|
|
17
23
|
/** Labels for each compliance track. */
|
|
18
24
|
export declare const TRACK_LABELS: Record<ComplianceTrack, string>;
|
|
19
25
|
/**
|
|
@@ -8,12 +8,18 @@
|
|
|
8
8
|
import type { AgentProfile } from '../types';
|
|
9
9
|
import type { ComplianceTrack, TrackResult } from './types';
|
|
10
10
|
import type { AssertionResult, StoryboardResult, StoryboardStepResult } from '../storyboard/types';
|
|
11
|
+
import { type StoryboardVerdictExplanation } from '../storyboard/verdict-explanation';
|
|
11
12
|
export interface DetachedAssertionFailure {
|
|
12
13
|
assertion: AssertionResult;
|
|
13
14
|
owningStep?: StoryboardStepResult;
|
|
14
15
|
}
|
|
15
16
|
/** Assertion failures that are not represented by a counted failed step. */
|
|
16
17
|
export declare function collectDetachedAssertionFailures(result: StoryboardResult): DetachedAssertionFailure[];
|
|
18
|
+
/**
|
|
19
|
+
* The verdict explanation the runner attached, or one derived from the result
|
|
20
|
+
* when it carries none. `undefined` for a passing result.
|
|
21
|
+
*/
|
|
22
|
+
export declare function storyboardVerdictExplanation(result: StoryboardResult): StoryboardVerdictExplanation | undefined;
|
|
17
23
|
/** Labels for each compliance track. */
|
|
18
24
|
export declare const TRACK_LABELS: Record<ComplianceTrack, string>;
|
|
19
25
|
/**
|
|
@@ -20,10 +20,12 @@ var storyboard_tracks_exports = {};
|
|
|
20
20
|
__export(storyboard_tracks_exports, {
|
|
21
21
|
TRACK_LABELS: () => TRACK_LABELS,
|
|
22
22
|
collectDetachedAssertionFailures: () => collectDetachedAssertionFailures,
|
|
23
|
-
mapStoryboardResultsToTrackResult: () => mapStoryboardResultsToTrackResult
|
|
23
|
+
mapStoryboardResultsToTrackResult: () => mapStoryboardResultsToTrackResult,
|
|
24
|
+
storyboardVerdictExplanation: () => storyboardVerdictExplanation
|
|
24
25
|
});
|
|
25
26
|
module.exports = __toCommonJS(storyboard_tracks_exports);
|
|
26
27
|
var import_runner = require('../storyboard/runner.js');
|
|
28
|
+
var import_verdict_explanation = require('../storyboard/verdict-explanation.js');
|
|
27
29
|
function collectDetachedAssertionFailures(result) {
|
|
28
30
|
const failures = [];
|
|
29
31
|
for (const assertion of result.assertions ?? []) {
|
|
@@ -40,6 +42,18 @@ function collectDetachedAssertionFailures(result) {
|
|
|
40
42
|
}
|
|
41
43
|
return failures;
|
|
42
44
|
}
|
|
45
|
+
function storyboardVerdictExplanation(result) {
|
|
46
|
+
if (result.overall_passed) return void 0;
|
|
47
|
+
if (result.verdict_reason !== void 0 && result.verdict_detail !== void 0) {
|
|
48
|
+
return {
|
|
49
|
+
verdict_reason: result.verdict_reason,
|
|
50
|
+
verdict_detail: result.verdict_detail,
|
|
51
|
+
verdict_contributing: result.verdict_contributing ?? []
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
const steps = (result.passes?.flatMap((pass) => pass.phases) ?? result.phases).flatMap((phase) => phase.steps);
|
|
55
|
+
return (0, import_verdict_explanation.explainStoryboardVerdict)(result, { coverageUnverified: (0, import_runner.signingCoverageUnverified)(steps) });
|
|
56
|
+
}
|
|
43
57
|
const TRACK_LABELS = {
|
|
44
58
|
core: "Core Protocol",
|
|
45
59
|
products: "Product Discovery",
|
|
@@ -116,14 +130,21 @@ function mapStoryboardResultsToTrackResult(track, storyboardResults, _profile) {
|
|
|
116
130
|
scenarios.push(testResult);
|
|
117
131
|
}
|
|
118
132
|
if (!sbResult.overall_passed && (sbResult.failed_count === 0 || (sbResult.passed_count > 0 || assertionFailures.length > 0) && sbResult.phases.every((phase) => phase.passed))) {
|
|
133
|
+
const explanation = storyboardVerdictExplanation(sbResult);
|
|
134
|
+
const why = explanation ? `: ${explanation.verdict_detail}` : "";
|
|
119
135
|
scenarios.push({
|
|
120
136
|
agent_url: sbResult.agent_url,
|
|
121
137
|
scenario: `${sbResult.storyboard_id}/storyboard`,
|
|
122
138
|
overall_passed: false,
|
|
123
139
|
steps: [],
|
|
124
|
-
summary: assertionFailures.length > 0 ? `${sbResult.storyboard_title}: ${assertionFailures.length} assertion(s) failed` : `${sbResult.storyboard_title}: storyboard-level verdict failed`,
|
|
140
|
+
summary: assertionFailures.length > 0 ? `${sbResult.storyboard_title}: ${assertionFailures.length} assertion(s) failed${why}` : `${sbResult.storyboard_title}: storyboard-level verdict failed${why}`,
|
|
125
141
|
total_duration_ms: 0,
|
|
126
|
-
tested_at: sbResult.tested_at
|
|
142
|
+
tested_at: sbResult.tested_at,
|
|
143
|
+
...explanation && {
|
|
144
|
+
verdict_reason: explanation.verdict_reason,
|
|
145
|
+
verdict_detail: explanation.verdict_detail,
|
|
146
|
+
verdict_contributing: explanation.verdict_contributing
|
|
147
|
+
}
|
|
127
148
|
});
|
|
128
149
|
}
|
|
129
150
|
}
|
|
@@ -216,5 +237,6 @@ function computeTrackStatus(results) {
|
|
|
216
237
|
0 && (module.exports = {
|
|
217
238
|
TRACK_LABELS,
|
|
218
239
|
collectDetachedAssertionFailures,
|
|
219
|
-
mapStoryboardResultsToTrackResult
|
|
240
|
+
mapStoryboardResultsToTrackResult,
|
|
241
|
+
storyboardVerdictExplanation
|
|
220
242
|
});
|