agent-scenario-loop 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +119 -0
- package/app/profile-session.ts +812 -0
- package/core/config-template.json +41 -0
- package/dist/core/agent-summary.d.ts +15 -0
- package/dist/core/agent-summary.js +177 -0
- package/dist/core/artifact-contract.d.ts +151 -0
- package/dist/core/artifact-contract.js +897 -0
- package/dist/core/artifact-layout.d.ts +56 -0
- package/dist/core/artifact-layout.js +61 -0
- package/dist/core/artifact-writer.d.ts +44 -0
- package/dist/core/artifact-writer.js +55 -0
- package/dist/core/comparison.d.ts +133 -0
- package/dist/core/comparison.js +294 -0
- package/dist/core/evidence-interpreter.d.ts +28 -0
- package/dist/core/evidence-interpreter.js +69 -0
- package/dist/core/execution-plan.d.ts +44 -0
- package/dist/core/execution-plan.js +95 -0
- package/dist/core/planner.d.ts +132 -0
- package/dist/core/planner.js +812 -0
- package/dist/core/ports.d.ts +198 -0
- package/dist/core/ports.js +146 -0
- package/dist/core/run-index.d.ts +62 -0
- package/dist/core/run-index.js +143 -0
- package/dist/core/schema-validator.d.ts +86 -0
- package/dist/core/schema-validator.js +407 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +27 -0
- package/dist/runner/agent-device-driver.d.ts +126 -0
- package/dist/runner/agent-device-driver.js +168 -0
- package/dist/runner/agent-device.d.ts +295 -0
- package/dist/runner/agent-device.js +1271 -0
- package/dist/runner/android-adb-driver.d.ts +175 -0
- package/dist/runner/android-adb-driver.js +399 -0
- package/dist/runner/android-adb.d.ts +254 -0
- package/dist/runner/android-adb.js +1618 -0
- package/dist/runner/argent-driver.d.ts +183 -0
- package/dist/runner/argent-driver.js +297 -0
- package/dist/runner/argent.d.ts +349 -0
- package/dist/runner/argent.js +1211 -0
- package/dist/runner/check-plan.d.ts +45 -0
- package/dist/runner/check-plan.js +210 -0
- package/dist/runner/cli.d.ts +20 -0
- package/dist/runner/cli.js +23 -0
- package/dist/runner/compare-latest.d.ts +99 -0
- package/dist/runner/compare-latest.js +233 -0
- package/dist/runner/compare.d.ts +58 -0
- package/dist/runner/compare.js +157 -0
- package/dist/runner/demo-loop.d.ts +45 -0
- package/dist/runner/demo-loop.js +170 -0
- package/dist/runner/example-android-live.d.ts +137 -0
- package/dist/runner/example-android-live.js +454 -0
- package/dist/runner/example-ios-live.d.ts +137 -0
- package/dist/runner/example-ios-live.js +471 -0
- package/dist/runner/host-doctor.d.ts +131 -0
- package/dist/runner/host-doctor.js +628 -0
- package/dist/runner/init-project.d.ts +88 -0
- package/dist/runner/init-project.js +263 -0
- package/dist/runner/ios-simctl-driver.d.ts +69 -0
- package/dist/runner/ios-simctl-driver.js +97 -0
- package/dist/runner/ios-simctl.d.ts +254 -0
- package/dist/runner/ios-simctl.js +1415 -0
- package/dist/runner/live-android.d.ts +137 -0
- package/dist/runner/live-android.js +539 -0
- package/dist/runner/live-comparison.d.ts +67 -0
- package/dist/runner/live-comparison.js +147 -0
- package/dist/runner/live-ios.d.ts +137 -0
- package/dist/runner/live-ios.js +460 -0
- package/dist/runner/live-proof-summary.d.ts +263 -0
- package/dist/runner/live-proof-summary.js +465 -0
- package/dist/runner/live-proof.d.ts +467 -0
- package/dist/runner/live-proof.js +920 -0
- package/dist/runner/local-env.d.ts +64 -0
- package/dist/runner/local-env.js +155 -0
- package/dist/runner/profile-android.d.ts +82 -0
- package/dist/runner/profile-android.js +671 -0
- package/dist/runner/profile-ios.d.ts +108 -0
- package/dist/runner/profile-ios.js +532 -0
- package/dist/runner/profile-mobile.d.ts +254 -0
- package/dist/runner/profile-mobile.js +1307 -0
- package/dist/runner/validate-project.d.ts +273 -0
- package/dist/runner/validate-project.js +1501 -0
- package/docs/adapters.md +145 -0
- package/docs/api.md +94 -0
- package/docs/authoring.md +196 -0
- package/docs/concepts.md +136 -0
- package/docs/consumer-rehearsal.md +115 -0
- package/docs/contracts.md +267 -0
- package/docs/live-proofs.md +270 -0
- package/docs/principles.md +46 -0
- package/examples/event-logs/app-startup-baseline.log +4 -0
- package/examples/event-logs/app-startup-current.log +4 -0
- package/examples/minimal-app/README.md +70 -0
- package/examples/mobile-app/README.md +302 -0
- package/examples/mobile-app/app.json +22 -0
- package/examples/mobile-app/asl/package-scripts.json +32 -0
- package/examples/mobile-app/asl.config.json +37 -0
- package/examples/mobile-app/event-logs/android-app-startup.log +4 -0
- package/examples/mobile-app/event-logs/android-open-close-cycle.log +12 -0
- package/examples/mobile-app/event-logs/android-scroll-settle.log +12 -0
- package/examples/mobile-app/event-logs/app-startup.log +4 -0
- package/examples/mobile-app/event-logs/open-close-cycle.log +12 -0
- package/examples/mobile-app/event-logs/scroll-settle.log +12 -0
- package/examples/mobile-app/index.ts +20 -0
- package/examples/mobile-app/metro.config.js +20 -0
- package/examples/mobile-app/package.json +62 -0
- package/examples/mobile-app/patches/expo-modules-jsi@56.0.10.patch +19 -0
- package/examples/mobile-app/plugins/with-ios-build-compat.js +271 -0
- package/examples/mobile-app/pnpm-lock.yaml +4440 -0
- package/examples/mobile-app/runner-manifests/evidence-provider.json +79 -0
- package/examples/mobile-app/runner-manifests/primary-runner.json +19 -0
- package/examples/mobile-app/scenarios/android/app-startup-video.json +73 -0
- package/examples/mobile-app/scenarios/android/app-startup.json +44 -0
- package/examples/mobile-app/scenarios/android/open-close-cycle.json +54 -0
- package/examples/mobile-app/scenarios/android/scroll-settle.json +49 -0
- package/examples/mobile-app/scenarios/ios/app-startup.json +44 -0
- package/examples/mobile-app/scenarios/ios/open-close-cycle.json +54 -0
- package/examples/mobile-app/scenarios/ios/scroll-settle.json +49 -0
- package/examples/mobile-app/scenarios/mobile/app-startup.json +91 -0
- package/examples/mobile-app/scenarios/mobile/open-close-cycle.json +160 -0
- package/examples/mobile-app/scenarios/mobile/scroll-settle.json +148 -0
- package/examples/mobile-app/scripts/asl-capture-accessibility-provider.mjs +112 -0
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +127 -0
- package/examples/mobile-app/src/devtools/profile-session.ts +7 -0
- package/examples/mobile-app/src/example-screen.tsx +322 -0
- package/examples/mobile-app/tsconfig.json +16 -0
- package/examples/mobile-app/tsconfig.typecheck.json +13 -0
- package/examples/runners/README.md +44 -0
- package/examples/runners/adb-android.json +25 -0
- package/examples/runners/agent-device-android.json +27 -0
- package/examples/runners/agent-device-ios.json +27 -0
- package/examples/runners/argent-android.json +32 -0
- package/examples/runners/argent-ios.json +32 -0
- package/examples/runners/argent-react-profiler-provider.json +15 -0
- package/examples/runners/axe-accessibility-provider.json +24 -0
- package/examples/runners/manual-log-ingest.json +9 -0
- package/examples/runners/rozenite-profiler-provider.json +9 -0
- package/examples/runners/script-accessibility-provider.json +24 -0
- package/examples/runners/script-memory-provider.json +24 -0
- package/examples/runners/script-network-provider.json +24 -0
- package/examples/runners/script-profiler-provider.json +30 -0
- package/examples/runners/xcodebuildmcp-ios.json +29 -0
- package/examples/scenarios/ios/app-startup.json +28 -0
- package/examples/scenarios/ios/open-close-cycle.json +35 -0
- package/examples/scenarios/mobile/app-startup.json +72 -0
- package/examples/scenarios/mobile/media-open-close.json +141 -0
- package/examples/scenarios/mobile/open-close-cycle.json +135 -0
- package/examples/scenarios/mobile/scroll-settle.json +106 -0
- package/package.json +240 -0
- package/schemas/budget-verdict.schema.json +115 -0
- package/schemas/causal-run.schema.json +279 -0
- package/schemas/comparison.schema.json +196 -0
- package/schemas/health.schema.json +108 -0
- package/schemas/live-proof-set.schema.json +195 -0
- package/schemas/live-proof.schema.json +413 -0
- package/schemas/manifest.schema.json +204 -0
- package/schemas/metrics.schema.json +137 -0
- package/schemas/project-validation.schema.json +343 -0
- package/schemas/runner-capabilities.schema.json +217 -0
- package/schemas/scenario.schema.json +400 -0
- package/schemas/verdict.schema.json +88 -0
- package/templates/evidence-provider.json +83 -0
- package/templates/gitignore-snippet +9 -0
- package/templates/integration-readme.md +125 -0
- package/templates/mobile-scenario.json +133 -0
- package/templates/package-scripts.json +32 -0
- package/templates/primary-runner.json +19 -0
- package/templates/project.config.json +37 -0
- package/templates/scripts/asl-capture-accessibility-provider.mjs +112 -0
- package/templates/scripts/asl-capture-profiler-provider.mjs +127 -0
|
@@ -0,0 +1,920 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.assertLiveProofArtifactPointers = assertLiveProofArtifactPointers;
|
|
5
|
+
exports.assertLiveProofAggregateSignals = assertLiveProofAggregateSignals;
|
|
6
|
+
exports.assertLiveProofComparisonCounts = assertLiveProofComparisonCounts;
|
|
7
|
+
exports.assertLiveProofSetRequiredPlatforms = assertLiveProofSetRequiredPlatforms;
|
|
8
|
+
exports.buildLiveProofSetArtifact = buildLiveProofSetArtifact;
|
|
9
|
+
exports.buildLiveProofSetFailureReasons = buildLiveProofSetFailureReasons;
|
|
10
|
+
exports.buildLiveProofSetNextAction = buildLiveProofSetNextAction;
|
|
11
|
+
exports.collectLiveProofArtifactPointerIssues = collectLiveProofArtifactPointerIssues;
|
|
12
|
+
exports.countLiveProofComparisons = countLiveProofComparisons;
|
|
13
|
+
exports.deriveLiveProofComparisonStatus = deriveLiveProofComparisonStatus;
|
|
14
|
+
exports.expectedLiveProofNextActionCode = expectedLiveProofNextActionCode;
|
|
15
|
+
exports.formatComparisonPointerMetrics = formatComparisonPointerMetrics;
|
|
16
|
+
exports.formatInteractionProofCaptures = formatInteractionProofCaptures;
|
|
17
|
+
exports.formatInteractionProofWarningDetails = formatInteractionProofWarningDetails;
|
|
18
|
+
exports.formatInteractionProofWarnings = formatInteractionProofWarnings;
|
|
19
|
+
exports.formatLiveProofSetWarningDetails = formatLiveProofSetWarningDetails;
|
|
20
|
+
exports.formatLiveProof = formatLiveProof;
|
|
21
|
+
exports.formatLiveProofSet = formatLiveProofSet;
|
|
22
|
+
exports.formatLiveProofSetArtifactMarkdown = formatLiveProofSetArtifactMarkdown;
|
|
23
|
+
exports.main = main;
|
|
24
|
+
exports.parseArgs = parseArgs;
|
|
25
|
+
exports.parseRequiredPlatforms = parseRequiredPlatforms;
|
|
26
|
+
exports.readLiveProof = readLiveProof;
|
|
27
|
+
exports.readLiveProofSet = readLiveProofSet;
|
|
28
|
+
exports.resolveLiveProofSetOutputDir = resolveLiveProofSetOutputDir;
|
|
29
|
+
exports.resolveLiveProofArtifactBaseDir = resolveLiveProofArtifactBaseDir;
|
|
30
|
+
exports.resolveLiveProofFiles = resolveLiveProofFiles;
|
|
31
|
+
exports.resolveLiveProofSetRunId = resolveLiveProofSetRunId;
|
|
32
|
+
exports.shouldRequireArtifacts = shouldRequireArtifacts;
|
|
33
|
+
exports.shouldFailLiveProofSet = shouldFailLiveProofSet;
|
|
34
|
+
exports.shouldFailOnRegression = shouldFailOnRegression;
|
|
35
|
+
exports.usage = usage;
|
|
36
|
+
exports.writeLiveProofSetArtifact = writeLiveProofSetArtifact;
|
|
37
|
+
const fs = require('node:fs');
|
|
38
|
+
const path = require('node:path');
|
|
39
|
+
const { createArtifactLayout } = require('../core/artifact-layout');
|
|
40
|
+
const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
|
|
41
|
+
const { SCHEMAS, assertValidJson } = require('../core/schema-validator');
|
|
42
|
+
const { hasHelpFlag, writeUsage } = require('./cli');
|
|
43
|
+
/**
|
|
44
|
+
* Prints CLI usage.
|
|
45
|
+
*
|
|
46
|
+
* @param {{write: (message: string) => unknown}} [output]
|
|
47
|
+
* @returns {void}
|
|
48
|
+
*/
|
|
49
|
+
function usage(output = process.stderr) {
|
|
50
|
+
writeUsage([
|
|
51
|
+
'Usage: asl-live-proof --file <live-proof.json> [--file <live-proof.json> ...] [--require-platforms android,ios] [--out <dir>] [--run-id <id>] [--fail-on-regression]',
|
|
52
|
+
'',
|
|
53
|
+
'Validates one or more aggregate live-proof artifacts and prints status and next action details.',
|
|
54
|
+
'Use --require-platforms to fail when a platform proof is missing from a multi-artifact gate.',
|
|
55
|
+
'Use --require-artifacts to fail when live-proof pointers reference missing local evidence files.',
|
|
56
|
+
'Use --artifact-base-dir <dir> to resolve relative artifact pointers from a directory other than cwd.',
|
|
57
|
+
'Use --out to write live-proof-set.json and agent-summary.md for a durable platform-set gate.',
|
|
58
|
+
'Use --fail-on-regression to exit nonzero when comparisonStatus is regressed.',
|
|
59
|
+
], output);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Adds one parsed CLI argument, preserving repeated --file flags.
|
|
63
|
+
*
|
|
64
|
+
* @param {CliArgs} args
|
|
65
|
+
* @param {string} key
|
|
66
|
+
* @param {string | boolean} value
|
|
67
|
+
* @returns {void}
|
|
68
|
+
*/
|
|
69
|
+
function assignArg(args, key, value) {
|
|
70
|
+
if (key !== 'file' || typeof value !== 'string') {
|
|
71
|
+
args[key] = value;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (Array.isArray(args.file)) {
|
|
75
|
+
args.file.push(value);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (typeof args.file === 'string') {
|
|
79
|
+
args.file = [args.file, value];
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
args.file = value;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Parses a small flag set for the live-proof CLI.
|
|
86
|
+
*
|
|
87
|
+
* @param {string[]} argv
|
|
88
|
+
* @returns {CliArgs}
|
|
89
|
+
*/
|
|
90
|
+
function parseArgs(argv) {
|
|
91
|
+
const args = {};
|
|
92
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
93
|
+
const token = argv[index];
|
|
94
|
+
if (!token) {
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (!token.startsWith('--')) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const key = token.slice(2);
|
|
101
|
+
const next = argv[index + 1];
|
|
102
|
+
if (!next || next.startsWith('--')) {
|
|
103
|
+
assignArg(args, key, true);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
assignArg(args, key, next);
|
|
107
|
+
index += 1;
|
|
108
|
+
}
|
|
109
|
+
return args;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Resolves one or more --file values into a stable list.
|
|
113
|
+
*
|
|
114
|
+
* @param {CliArgs} args
|
|
115
|
+
* @returns {string[]}
|
|
116
|
+
*/
|
|
117
|
+
function resolveLiveProofFiles(args) {
|
|
118
|
+
if (Array.isArray(args.file)) {
|
|
119
|
+
return args.file;
|
|
120
|
+
}
|
|
121
|
+
return typeof args.file === 'string' ? [args.file] : [];
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Parses required platform names for a live-proof set gate.
|
|
125
|
+
*
|
|
126
|
+
* @param {string | boolean | undefined} value
|
|
127
|
+
* @returns {LiveProofPlatform[]}
|
|
128
|
+
*/
|
|
129
|
+
function parseRequiredPlatforms(value) {
|
|
130
|
+
if (value === undefined) {
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
133
|
+
if (typeof value !== 'string') {
|
|
134
|
+
throw new Error('--require-platforms expects a comma-separated platform list such as android,ios.');
|
|
135
|
+
}
|
|
136
|
+
const platforms = value
|
|
137
|
+
.split(',')
|
|
138
|
+
.map((platform) => platform.trim())
|
|
139
|
+
.filter((platform) => platform.length > 0);
|
|
140
|
+
const invalid = platforms.filter((platform) => platform !== 'android' && platform !== 'ios');
|
|
141
|
+
if (invalid.length > 0) {
|
|
142
|
+
throw new Error(`Unsupported required live-proof platform(s): ${invalid.join(', ')}.`);
|
|
143
|
+
}
|
|
144
|
+
return Array.from(new Set(platforms));
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Counts comparison outcomes from one live-proof comparison list.
|
|
148
|
+
*
|
|
149
|
+
* @param {Array<{status?: string}>} comparisons
|
|
150
|
+
* @returns {LiveProofComparisonCounts}
|
|
151
|
+
*/
|
|
152
|
+
function countLiveProofComparisons(comparisons) {
|
|
153
|
+
const counts = {
|
|
154
|
+
better: 0,
|
|
155
|
+
inconclusive: 0,
|
|
156
|
+
mixed: 0,
|
|
157
|
+
skipped: 0,
|
|
158
|
+
unchanged: 0,
|
|
159
|
+
worse: 0,
|
|
160
|
+
};
|
|
161
|
+
for (const comparison of comparisons) {
|
|
162
|
+
const status = comparison.status;
|
|
163
|
+
if (status && Object.prototype.hasOwnProperty.call(counts, status)) {
|
|
164
|
+
counts[status] += 1;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return counts;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Collapses live-proof comparison pointers into the expected aggregate status.
|
|
171
|
+
*
|
|
172
|
+
* @param {Array<{status?: string}>} comparisons
|
|
173
|
+
* @returns {LiveProofAggregateStatus}
|
|
174
|
+
*/
|
|
175
|
+
function deriveLiveProofComparisonStatus(comparisons) {
|
|
176
|
+
if (comparisons.length === 0) {
|
|
177
|
+
return 'not_compared';
|
|
178
|
+
}
|
|
179
|
+
const statuses = comparisons.map((comparison) => comparison.status);
|
|
180
|
+
if (statuses.includes('worse')) {
|
|
181
|
+
return 'regressed';
|
|
182
|
+
}
|
|
183
|
+
if (statuses.includes('inconclusive')) {
|
|
184
|
+
return 'inconclusive';
|
|
185
|
+
}
|
|
186
|
+
if (statuses.every((status) => status === 'skipped')) {
|
|
187
|
+
return 'baseline_missing';
|
|
188
|
+
}
|
|
189
|
+
if (statuses.includes('skipped')) {
|
|
190
|
+
return 'inconclusive';
|
|
191
|
+
}
|
|
192
|
+
if (statuses.includes('mixed')) {
|
|
193
|
+
return 'mixed';
|
|
194
|
+
}
|
|
195
|
+
if (statuses.includes('better')) {
|
|
196
|
+
return 'improved';
|
|
197
|
+
}
|
|
198
|
+
return 'unchanged';
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Resolves the expected next-action code for an aggregate proof status.
|
|
202
|
+
*
|
|
203
|
+
* @param {LiveProofAggregateStatus} comparisonStatus
|
|
204
|
+
* @param {string} [status]
|
|
205
|
+
* @returns {LiveProofNextActionCode}
|
|
206
|
+
*/
|
|
207
|
+
function expectedLiveProofNextActionCode(comparisonStatus, status = 'passed') {
|
|
208
|
+
if (status === 'failed') {
|
|
209
|
+
return 'inspect_failed_run';
|
|
210
|
+
}
|
|
211
|
+
if (comparisonStatus === 'regressed') {
|
|
212
|
+
return 'inspect_regressions';
|
|
213
|
+
}
|
|
214
|
+
if (comparisonStatus === 'baseline_missing') {
|
|
215
|
+
return 'establish_baseline';
|
|
216
|
+
}
|
|
217
|
+
if (comparisonStatus === 'inconclusive') {
|
|
218
|
+
return 'inspect_inconclusive';
|
|
219
|
+
}
|
|
220
|
+
if (comparisonStatus === 'mixed') {
|
|
221
|
+
return 'inspect_mixed';
|
|
222
|
+
}
|
|
223
|
+
return 'inspect_summary';
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Verifies that aggregate comparison counts match the comparison pointers.
|
|
227
|
+
*
|
|
228
|
+
* @param {LiveProofArtifact} proof
|
|
229
|
+
* @returns {void}
|
|
230
|
+
*/
|
|
231
|
+
function assertLiveProofComparisonCounts(proof) {
|
|
232
|
+
const actual = countLiveProofComparisons(proof.comparisons);
|
|
233
|
+
for (const key of Object.keys(actual)) {
|
|
234
|
+
if (actual[key] !== proof.comparisonCounts[key]) {
|
|
235
|
+
throw new Error(`Live proof artifact comparisonCounts.${key} expected ${actual[key]} from comparisons but found ${proof.comparisonCounts[key]}.`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Verifies that aggregate comparison status and next action match the pointers.
|
|
241
|
+
*
|
|
242
|
+
* @param {LiveProofArtifact} proof
|
|
243
|
+
* @returns {void}
|
|
244
|
+
*/
|
|
245
|
+
function assertLiveProofAggregateSignals(proof) {
|
|
246
|
+
const expectedStatus = deriveLiveProofComparisonStatus(proof.comparisons);
|
|
247
|
+
if (proof.comparisonStatus !== expectedStatus) {
|
|
248
|
+
throw new Error(`Live proof artifact comparisonStatus expected ${expectedStatus} from comparisons but found ${proof.comparisonStatus}.`);
|
|
249
|
+
}
|
|
250
|
+
const expectedAction = expectedLiveProofNextActionCode(expectedStatus, proof.status);
|
|
251
|
+
if (proof.nextAction.code !== expectedAction) {
|
|
252
|
+
throw new Error(`Live proof artifact nextAction.code expected ${expectedAction} for ${proof.status}/${expectedStatus} but found ${proof.nextAction.code}.`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Resolves a pointer path using the CLI working directory by default.
|
|
257
|
+
*
|
|
258
|
+
* @param {string} pointerPath
|
|
259
|
+
* @param {string} baseDir
|
|
260
|
+
* @returns {string}
|
|
261
|
+
*/
|
|
262
|
+
function resolveLiveProofPointerPath(pointerPath, baseDir) {
|
|
263
|
+
return path.isAbsolute(pointerPath) ? pointerPath : path.resolve(baseDir, pointerPath);
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Records a pointer issue when a referenced local artifact is missing.
|
|
267
|
+
*
|
|
268
|
+
* @param {LiveProofArtifactPointerIssue[]} issues
|
|
269
|
+
* @param {{baseDir: string, expected: 'directory' | 'file', label: string, pointerPath?: string | null | undefined}} options
|
|
270
|
+
* @returns {void}
|
|
271
|
+
*/
|
|
272
|
+
function collectExistingPointerIssue(issues, { baseDir, expected, label, pointerPath, }) {
|
|
273
|
+
if (!pointerPath) {
|
|
274
|
+
issues.push({
|
|
275
|
+
expected,
|
|
276
|
+
label,
|
|
277
|
+
path: '<missing pointer>',
|
|
278
|
+
reason: 'pointer is empty',
|
|
279
|
+
});
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
const resolvedPath = resolveLiveProofPointerPath(pointerPath, baseDir);
|
|
283
|
+
let stats;
|
|
284
|
+
try {
|
|
285
|
+
stats = fs.statSync(resolvedPath);
|
|
286
|
+
}
|
|
287
|
+
catch {
|
|
288
|
+
issues.push({
|
|
289
|
+
expected,
|
|
290
|
+
label,
|
|
291
|
+
path: resolvedPath,
|
|
292
|
+
reason: 'path does not exist',
|
|
293
|
+
});
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
if (expected === 'directory' && !stats.isDirectory()) {
|
|
297
|
+
issues.push({
|
|
298
|
+
expected,
|
|
299
|
+
label,
|
|
300
|
+
path: resolvedPath,
|
|
301
|
+
reason: 'path is not a directory',
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
if (expected === 'file' && !stats.isFile()) {
|
|
305
|
+
issues.push({
|
|
306
|
+
expected,
|
|
307
|
+
label,
|
|
308
|
+
path: resolvedPath,
|
|
309
|
+
reason: 'path is not a file',
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Collects missing local evidence pointers from one live-proof artifact.
|
|
315
|
+
*
|
|
316
|
+
* @param {LiveProofArtifact} proof
|
|
317
|
+
* @param {{artifactBaseDir?: string}} [options]
|
|
318
|
+
* @returns {LiveProofArtifactPointerIssue[]}
|
|
319
|
+
*/
|
|
320
|
+
function collectLiveProofArtifactPointerIssues(proof, { artifactBaseDir = process.cwd() } = {}) {
|
|
321
|
+
const issues = [];
|
|
322
|
+
const baseDir = path.resolve(artifactBaseDir);
|
|
323
|
+
collectExistingPointerIssue(issues, {
|
|
324
|
+
baseDir,
|
|
325
|
+
expected: 'directory',
|
|
326
|
+
label: `preflight ${proof.preflight.runId} runDir`,
|
|
327
|
+
pointerPath: proof.preflight.runDir,
|
|
328
|
+
});
|
|
329
|
+
collectExistingPointerIssue(issues, {
|
|
330
|
+
baseDir,
|
|
331
|
+
expected: 'file',
|
|
332
|
+
label: `preflight ${proof.preflight.runId} summaryPath`,
|
|
333
|
+
pointerPath: proof.preflight.summaryPath,
|
|
334
|
+
});
|
|
335
|
+
for (const profile of proof.profiles) {
|
|
336
|
+
collectExistingPointerIssue(issues, {
|
|
337
|
+
baseDir,
|
|
338
|
+
expected: 'directory',
|
|
339
|
+
label: `profile ${profile.label} runDir`,
|
|
340
|
+
pointerPath: profile.runDir,
|
|
341
|
+
});
|
|
342
|
+
collectExistingPointerIssue(issues, {
|
|
343
|
+
baseDir,
|
|
344
|
+
expected: 'file',
|
|
345
|
+
label: `profile ${profile.label} summaryPath`,
|
|
346
|
+
pointerPath: profile.summaryPath,
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
for (const interactionProof of proof.interactionProofs ?? []) {
|
|
350
|
+
collectExistingPointerIssue(issues, {
|
|
351
|
+
baseDir,
|
|
352
|
+
expected: 'directory',
|
|
353
|
+
label: `interaction ${interactionProof.label} runDir`,
|
|
354
|
+
pointerPath: interactionProof.runDir,
|
|
355
|
+
});
|
|
356
|
+
collectExistingPointerIssue(issues, {
|
|
357
|
+
baseDir,
|
|
358
|
+
expected: 'file',
|
|
359
|
+
label: `interaction ${interactionProof.label} summaryPath`,
|
|
360
|
+
pointerPath: interactionProof.summaryPath,
|
|
361
|
+
});
|
|
362
|
+
for (const screenshotPath of interactionProof.captures?.screenshots ?? []) {
|
|
363
|
+
collectExistingPointerIssue(issues, {
|
|
364
|
+
baseDir,
|
|
365
|
+
expected: 'file',
|
|
366
|
+
label: `interaction ${interactionProof.label} screenshot`,
|
|
367
|
+
pointerPath: path.isAbsolute(screenshotPath)
|
|
368
|
+
? screenshotPath
|
|
369
|
+
: path.join(interactionProof.runDir, screenshotPath),
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
for (const comparison of proof.comparisons) {
|
|
374
|
+
if (comparison.status === 'skipped') {
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
collectExistingPointerIssue(issues, {
|
|
378
|
+
baseDir,
|
|
379
|
+
expected: 'directory',
|
|
380
|
+
label: `comparison ${comparison.label ?? 'comparison'} baselineDir`,
|
|
381
|
+
pointerPath: comparison.baselineDir,
|
|
382
|
+
});
|
|
383
|
+
collectExistingPointerIssue(issues, {
|
|
384
|
+
baseDir,
|
|
385
|
+
expected: 'directory',
|
|
386
|
+
label: `comparison ${comparison.label ?? 'comparison'} comparisonDir`,
|
|
387
|
+
pointerPath: comparison.comparisonDir,
|
|
388
|
+
});
|
|
389
|
+
collectExistingPointerIssue(issues, {
|
|
390
|
+
baseDir,
|
|
391
|
+
expected: 'file',
|
|
392
|
+
label: `comparison ${comparison.label ?? 'comparison'} summaryPath`,
|
|
393
|
+
pointerPath: comparison.summaryPath,
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
return issues;
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* Fails when a live-proof artifact references missing local evidence.
|
|
400
|
+
*
|
|
401
|
+
* @param {LiveProofArtifact} proof
|
|
402
|
+
* @param {{artifactBaseDir?: string}} [options]
|
|
403
|
+
* @returns {void}
|
|
404
|
+
*/
|
|
405
|
+
function assertLiveProofArtifactPointers(proof, options = {}) {
|
|
406
|
+
const issues = collectLiveProofArtifactPointerIssues(proof, options);
|
|
407
|
+
if (issues.length === 0) {
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
const preview = issues
|
|
411
|
+
.slice(0, 8)
|
|
412
|
+
.map((issue) => `${issue.label} -> ${issue.path} (${issue.reason})`)
|
|
413
|
+
.join('; ');
|
|
414
|
+
const suffix = issues.length > 8 ? `; ${issues.length - 8} more` : '';
|
|
415
|
+
throw new Error(`Live proof artifact pointers missing: ${preview}${suffix}.`);
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Formats one metric highlight from a comparison pointer.
|
|
419
|
+
*
|
|
420
|
+
* @param {{delta?: number | null, name?: string, status?: string, unit?: string}} metric
|
|
421
|
+
* @returns {string}
|
|
422
|
+
*/
|
|
423
|
+
function formatMetricHighlight(metric) {
|
|
424
|
+
const delta = typeof metric.delta === 'number' ? `${metric.delta}${metric.unit ?? ''}` : 'n/a';
|
|
425
|
+
return `${metric.name ?? 'unknown metric'} ${metric.status ?? 'unknown'} (${delta})`;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Formats compact metric summary details for one comparison pointer.
|
|
429
|
+
*
|
|
430
|
+
* @param {LiveProofComparisonPointer} comparison
|
|
431
|
+
* @returns {string}
|
|
432
|
+
*/
|
|
433
|
+
function formatComparisonPointerMetrics(comparison) {
|
|
434
|
+
const counts = comparison.metricSummary?.counts;
|
|
435
|
+
if (!counts) {
|
|
436
|
+
return '';
|
|
437
|
+
}
|
|
438
|
+
const highlights = comparison.metricSummary?.notableMetrics ?? [];
|
|
439
|
+
const highlightText = highlights.length > 0
|
|
440
|
+
? `; notable: ${highlights.map(formatMetricHighlight).join(', ')}`
|
|
441
|
+
: '';
|
|
442
|
+
return ` (metrics better=${counts.better} worse=${counts.worse} unchanged=${counts.unchanged} inconclusive=${counts.inconclusive}${highlightText})`;
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Formats capture counts for one interaction proof pointer.
|
|
446
|
+
*
|
|
447
|
+
* @param {{captures?: {screenshots?: string[]}}} proofPointer
|
|
448
|
+
* @returns {string}
|
|
449
|
+
*/
|
|
450
|
+
function formatInteractionProofCaptures(proofPointer) {
|
|
451
|
+
const screenshotCount = proofPointer.captures?.screenshots?.length ?? 0;
|
|
452
|
+
return screenshotCount > 0 ? ` screenshots=${screenshotCount}` : '';
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* Formats warning counts for one interaction proof pointer.
|
|
456
|
+
*
|
|
457
|
+
* @param {{warnings?: {count?: number}}} proofPointer
|
|
458
|
+
* @returns {string}
|
|
459
|
+
*/
|
|
460
|
+
function formatInteractionProofWarnings(proofPointer) {
|
|
461
|
+
const warningCount = proofPointer.warnings?.count ?? 0;
|
|
462
|
+
return warningCount > 0 ? ` warnings=${warningCount}` : '';
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Formats warning check details for one interaction proof pointer.
|
|
466
|
+
*
|
|
467
|
+
* @param {{warnings?: {checks?: Array<{code?: string, message?: string, name?: string, nextAction?: {code?: string, summary?: string}}>}}} proofPointer
|
|
468
|
+
* @returns {string[]}
|
|
469
|
+
*/
|
|
470
|
+
function formatInteractionProofWarningDetails(proofPointer) {
|
|
471
|
+
return (proofPointer.warnings?.checks ?? []).map((warning) => {
|
|
472
|
+
const nextAction = warning.nextAction?.code || warning.nextAction?.summary
|
|
473
|
+
? ` next=${warning.nextAction?.code ?? 'inspect_interaction_warning'}${warning.nextAction?.summary ? ` - ${warning.nextAction.summary}` : ''}`
|
|
474
|
+
: '';
|
|
475
|
+
return ` warning ${warning.name ?? 'interaction_warning'}: ${warning.code ?? 'warning'} - ${warning.message ?? 'Interaction proof emitted a warning.'}${nextAction}`;
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Reads and validates a live-proof artifact.
|
|
480
|
+
*
|
|
481
|
+
* @param {string} filePath
|
|
482
|
+
* @param {LiveProofArtifactReadOptions} [options]
|
|
483
|
+
* @returns {LiveProofArtifact}
|
|
484
|
+
*/
|
|
485
|
+
function readLiveProof(filePath, options = {}) {
|
|
486
|
+
const proof = JSON.parse(fs.readFileSync(path.resolve(filePath), 'utf8'));
|
|
487
|
+
const validated = assertValidJson(proof, SCHEMAS.liveProof, 'Live proof artifact');
|
|
488
|
+
assertLiveProofComparisonCounts(validated);
|
|
489
|
+
assertLiveProofAggregateSignals(validated);
|
|
490
|
+
if (options.requireArtifacts) {
|
|
491
|
+
assertLiveProofArtifactPointers(validated, options.artifactBaseDir === undefined ? {} : { artifactBaseDir: options.artifactBaseDir });
|
|
492
|
+
}
|
|
493
|
+
return validated;
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* Verifies that a multi-proof gate includes every required platform.
|
|
497
|
+
*
|
|
498
|
+
* @param {LiveProofArtifact[]} proofs
|
|
499
|
+
* @param {LiveProofPlatform[]} requiredPlatforms
|
|
500
|
+
* @returns {void}
|
|
501
|
+
*/
|
|
502
|
+
function assertLiveProofSetRequiredPlatforms(proofs, requiredPlatforms) {
|
|
503
|
+
if (requiredPlatforms.length === 0) {
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
const present = new Set(proofs.map((proof) => proof.platform));
|
|
507
|
+
const missing = requiredPlatforms.filter((platform) => !present.has(platform));
|
|
508
|
+
if (missing.length === 0) {
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
const presentText = Array.from(present).sort().join(', ') || 'none';
|
|
512
|
+
throw new Error(`Live proof set missing required platform(s): ${missing.join(', ')}. Present platform(s): ${presentText}.`);
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Reads and validates a platform proof set.
|
|
516
|
+
*
|
|
517
|
+
* @param {{artifactBaseDir?: string, files: string[], requiredPlatforms?: LiveProofPlatform[], requireArtifacts?: boolean}} options
|
|
518
|
+
* @returns {LiveProofArtifact[]}
|
|
519
|
+
*/
|
|
520
|
+
function readLiveProofSet({ artifactBaseDir, files, requiredPlatforms = [], requireArtifacts = false, }) {
|
|
521
|
+
const readOptions = { requireArtifacts };
|
|
522
|
+
if (artifactBaseDir !== undefined) {
|
|
523
|
+
readOptions.artifactBaseDir = artifactBaseDir;
|
|
524
|
+
}
|
|
525
|
+
const proofs = files.map((file) => readLiveProof(file, readOptions));
|
|
526
|
+
assertLiveProofSetRequiredPlatforms(proofs, requiredPlatforms);
|
|
527
|
+
return proofs;
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Resolves the optional proof-set artifact output directory.
|
|
531
|
+
*
|
|
532
|
+
* @param {CliArgs} args
|
|
533
|
+
* @returns {string | null}
|
|
534
|
+
*/
|
|
535
|
+
function resolveLiveProofSetOutputDir(args) {
|
|
536
|
+
return typeof args.out === 'string' ? path.resolve(args.out) : null;
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Resolves the proof-set run id used for written artifacts.
|
|
540
|
+
*
|
|
541
|
+
* @param {CliArgs} args
|
|
542
|
+
* @returns {string}
|
|
543
|
+
*/
|
|
544
|
+
function resolveLiveProofSetRunId(args) {
|
|
545
|
+
if (typeof args['run-id'] !== 'string') {
|
|
546
|
+
return 'live-proof-set';
|
|
547
|
+
}
|
|
548
|
+
const normalized = args['run-id']
|
|
549
|
+
.trim()
|
|
550
|
+
.toLowerCase()
|
|
551
|
+
.replace(/[^a-z0-9._-]+/gu, '-')
|
|
552
|
+
.replace(/^-+|-+$/gu, '')
|
|
553
|
+
.slice(0, 96);
|
|
554
|
+
return normalized.length > 0 ? normalized : 'live-proof-set';
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* Resolves the base directory for local artifact pointer checks.
|
|
558
|
+
*
|
|
559
|
+
* @param {CliArgs} args
|
|
560
|
+
* @returns {string | undefined}
|
|
561
|
+
*/
|
|
562
|
+
function resolveLiveProofArtifactBaseDir(args) {
|
|
563
|
+
if (args['artifact-base-dir'] === undefined) {
|
|
564
|
+
return undefined;
|
|
565
|
+
}
|
|
566
|
+
if (typeof args['artifact-base-dir'] !== 'string') {
|
|
567
|
+
throw new Error('--artifact-base-dir expects a directory path.');
|
|
568
|
+
}
|
|
569
|
+
return path.resolve(args['artifact-base-dir']);
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Returns whether the caller requested local artifact pointer checks.
|
|
573
|
+
*
|
|
574
|
+
* @param {CliArgs} args
|
|
575
|
+
* @returns {boolean}
|
|
576
|
+
*/
|
|
577
|
+
function shouldRequireArtifacts(args) {
|
|
578
|
+
return args['require-artifacts'] === true || args['require-artifacts'] === 'true';
|
|
579
|
+
}
|
|
580
|
+
/**
|
|
581
|
+
* Counts interaction warnings preserved by one live-proof artifact.
|
|
582
|
+
*
|
|
583
|
+
* @param {LiveProofArtifact} proof
|
|
584
|
+
* @returns {number}
|
|
585
|
+
*/
|
|
586
|
+
function countInteractionWarnings(proof) {
|
|
587
|
+
return (proof.interactionProofs ?? []).reduce((sum, interactionProof) => (sum + (interactionProof.warnings?.count ?? 0)), 0);
|
|
588
|
+
}
|
|
589
|
+
/**
|
|
590
|
+
* Builds warning detail pointers from one platform live-proof artifact.
|
|
591
|
+
*
|
|
592
|
+
* @param {LiveProofArtifact} proof
|
|
593
|
+
* @returns {LiveProofSetInteractionWarningPointer[]}
|
|
594
|
+
*/
|
|
595
|
+
function buildLiveProofSetInteractionWarnings(proof) {
|
|
596
|
+
return (proof.interactionProofs ?? [])
|
|
597
|
+
.filter((interactionProof) => (interactionProof.warnings?.checks?.length ?? 0) > 0)
|
|
598
|
+
.map((interactionProof) => ({
|
|
599
|
+
checks: (interactionProof.warnings?.checks ?? []).map((warning) => ({
|
|
600
|
+
code: warning.code ?? 'warning',
|
|
601
|
+
message: warning.message ?? 'Interaction proof emitted a warning.',
|
|
602
|
+
name: warning.name ?? 'interaction_warning',
|
|
603
|
+
...(warning.nextAction?.code || warning.nextAction?.summary
|
|
604
|
+
? {
|
|
605
|
+
nextAction: {
|
|
606
|
+
code: warning.nextAction?.code ?? 'inspect_interaction_warning',
|
|
607
|
+
summary: warning.nextAction?.summary ?? 'Inspect the interaction proof warning.',
|
|
608
|
+
},
|
|
609
|
+
}
|
|
610
|
+
: {}),
|
|
611
|
+
})),
|
|
612
|
+
label: interactionProof.label,
|
|
613
|
+
runId: interactionProof.runId,
|
|
614
|
+
runnerId: interactionProof.runnerId,
|
|
615
|
+
scenarioId: interactionProof.scenarioId,
|
|
616
|
+
}));
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Builds the compact pointer for one platform live-proof artifact.
|
|
620
|
+
*
|
|
621
|
+
* @param {{filePath: string, proof: LiveProofArtifact}} options
|
|
622
|
+
* @returns {LiveProofSetProofPointer}
|
|
623
|
+
*/
|
|
624
|
+
function buildLiveProofSetProofPointer({ filePath, proof, }) {
|
|
625
|
+
const interactionWarnings = buildLiveProofSetInteractionWarnings(proof);
|
|
626
|
+
return {
|
|
627
|
+
comparisonStatus: proof.comparisonStatus,
|
|
628
|
+
filePath,
|
|
629
|
+
interactionProofCount: proof.interactionProofs?.length ?? 0,
|
|
630
|
+
...(interactionWarnings.length > 0 ? { interactionWarnings } : {}),
|
|
631
|
+
interactionWarningCount: countInteractionWarnings(proof),
|
|
632
|
+
nextAction: proof.nextAction,
|
|
633
|
+
platform: proof.platform,
|
|
634
|
+
profileCount: proof.profiles.length,
|
|
635
|
+
runId: proof.runId,
|
|
636
|
+
status: proof.status,
|
|
637
|
+
summaryPath: path.join(path.dirname(path.resolve(filePath)), 'agent-summary.md'),
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
/**
|
|
641
|
+
* Builds human-readable failure reasons for one proof set.
|
|
642
|
+
*
|
|
643
|
+
* @param {{failOnRegression: boolean, missingPlatforms: LiveProofPlatform[], proofs: LiveProofArtifact[]}} options
|
|
644
|
+
* @returns {string[]}
|
|
645
|
+
*/
|
|
646
|
+
function buildLiveProofSetFailureReasons({ failOnRegression, missingPlatforms, proofs, }) {
|
|
647
|
+
return [
|
|
648
|
+
...missingPlatforms.map((platform) => `Missing required platform proof: ${platform}.`),
|
|
649
|
+
...proofs
|
|
650
|
+
.filter((proof) => proof.status === 'failed')
|
|
651
|
+
.map((proof) => `${proof.platform} proof ${proof.runId} failed.`),
|
|
652
|
+
...(failOnRegression
|
|
653
|
+
? proofs
|
|
654
|
+
.filter((proof) => proof.comparisonStatus === 'regressed')
|
|
655
|
+
.map((proof) => `${proof.platform} proof ${proof.runId} regressed.`)
|
|
656
|
+
: []),
|
|
657
|
+
];
|
|
658
|
+
}
|
|
659
|
+
/**
|
|
660
|
+
* Builds the next action for a proof-set artifact.
|
|
661
|
+
*
|
|
662
|
+
* @param {{failureReasons: string[], missingPlatforms: LiveProofPlatform[], proofs: LiveProofArtifact[]}} options
|
|
663
|
+
* @returns {{code: string, summary: string}}
|
|
664
|
+
*/
|
|
665
|
+
function buildLiveProofSetNextAction({ failureReasons, missingPlatforms, proofs, }) {
|
|
666
|
+
if (missingPlatforms.length > 0) {
|
|
667
|
+
return {
|
|
668
|
+
code: 'collect_missing_platform_proofs',
|
|
669
|
+
summary: `Run the missing platform proof(s): ${missingPlatforms.join(', ')}.`,
|
|
670
|
+
};
|
|
671
|
+
}
|
|
672
|
+
if (proofs.some((proof) => proof.status === 'failed')) {
|
|
673
|
+
return {
|
|
674
|
+
code: 'inspect_failed_run',
|
|
675
|
+
summary: 'Inspect failed live-proof artifacts before trusting the platform set.',
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
if (proofs.some((proof) => proof.comparisonStatus === 'regressed')) {
|
|
679
|
+
return {
|
|
680
|
+
code: 'inspect_regressions',
|
|
681
|
+
summary: 'Inspect regressed platform proof comparisons before claiming improvement.',
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
if (failureReasons.length > 0) {
|
|
685
|
+
return {
|
|
686
|
+
code: 'inspect_failed_set',
|
|
687
|
+
summary: 'Inspect failed proof-set reasons before trusting the platform set.',
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
return {
|
|
691
|
+
code: 'inspect_summary',
|
|
692
|
+
summary: 'Platform proof set is complete; inspect linked artifacts for detail.',
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Builds the durable platform proof-set artifact.
|
|
697
|
+
*
|
|
698
|
+
* @param {{failOnRegression: boolean, files: string[], proofs: LiveProofArtifact[], requiredPlatforms?: LiveProofPlatform[], runId?: string}} options
|
|
699
|
+
* @returns {LiveProofSetArtifact}
|
|
700
|
+
*/
|
|
701
|
+
function buildLiveProofSetArtifact({ failOnRegression, files, proofs, requiredPlatforms = [], runId = 'live-proof-set', }) {
|
|
702
|
+
const presentPlatforms = Array.from(new Set(proofs.map((proof) => proof.platform))).sort();
|
|
703
|
+
const missingPlatforms = requiredPlatforms.filter((platform) => !presentPlatforms.includes(platform));
|
|
704
|
+
const failureReasons = buildLiveProofSetFailureReasons({
|
|
705
|
+
failOnRegression,
|
|
706
|
+
missingPlatforms,
|
|
707
|
+
proofs,
|
|
708
|
+
});
|
|
709
|
+
const status = failureReasons.length > 0 ? 'failed' : 'passed';
|
|
710
|
+
const nextAction = buildLiveProofSetNextAction({
|
|
711
|
+
failureReasons,
|
|
712
|
+
missingPlatforms,
|
|
713
|
+
proofs,
|
|
714
|
+
});
|
|
715
|
+
return {
|
|
716
|
+
failureReasons,
|
|
717
|
+
missingPlatforms,
|
|
718
|
+
nextAction,
|
|
719
|
+
presentPlatforms,
|
|
720
|
+
proofCount: proofs.length,
|
|
721
|
+
proofs: proofs.map((proof, index) => buildLiveProofSetProofPointer({
|
|
722
|
+
filePath: path.resolve(files[index] ?? ''),
|
|
723
|
+
proof,
|
|
724
|
+
})),
|
|
725
|
+
requiredPlatforms,
|
|
726
|
+
runId,
|
|
727
|
+
schemaVersion: '1.0.0',
|
|
728
|
+
status,
|
|
729
|
+
summary: status === 'passed'
|
|
730
|
+
? `live proof set passed for ${presentPlatforms.join(', ')}.`
|
|
731
|
+
: `live proof set failed: ${failureReasons.join(' ')}`,
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
/**
|
|
735
|
+
* Formats proof-set interaction warning details for agent-readable markdown.
|
|
736
|
+
*
|
|
737
|
+
* @param {LiveProofSetProofPointer} proof
|
|
738
|
+
* @returns {string[]}
|
|
739
|
+
*/
|
|
740
|
+
function formatLiveProofSetWarningDetails(proof) {
|
|
741
|
+
return (proof.interactionWarnings ?? []).flatMap((interactionWarning) => (interactionWarning.checks.map((warning) => {
|
|
742
|
+
const nextAction = warning.nextAction
|
|
743
|
+
? ` Next action: ${warning.nextAction.code} - ${warning.nextAction.summary}`
|
|
744
|
+
: '';
|
|
745
|
+
return ` - warning ${proof.platform}/${interactionWarning.label} (${interactionWarning.runnerId}/${interactionWarning.scenarioId}/${interactionWarning.runId}): ${warning.name} ${warning.code} - ${warning.message}${nextAction}`;
|
|
746
|
+
})));
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Formats a proof-set artifact for agent-readable markdown.
|
|
750
|
+
*
|
|
751
|
+
* @param {LiveProofSetArtifact} artifact
|
|
752
|
+
* @returns {string}
|
|
753
|
+
*/
|
|
754
|
+
function formatLiveProofSetArtifactMarkdown(artifact) {
|
|
755
|
+
return [
|
|
756
|
+
`# Live Proof Set ${artifact.runId}`,
|
|
757
|
+
'',
|
|
758
|
+
`Status: ${artifact.status}`,
|
|
759
|
+
`Required platforms: ${artifact.requiredPlatforms.join(', ') || 'none'}`,
|
|
760
|
+
`Present platforms: ${artifact.presentPlatforms.join(', ') || 'none'}`,
|
|
761
|
+
`Missing platforms: ${artifact.missingPlatforms.join(', ') || 'none'}`,
|
|
762
|
+
`Proofs: ${artifact.proofCount}`,
|
|
763
|
+
...artifact.proofs.flatMap((proof) => [
|
|
764
|
+
`- ${proof.platform} ${proof.runId}: status=${proof.status} comparison=${proof.comparisonStatus} profiles=${proof.profileCount} interactionProofs=${proof.interactionProofCount} warnings=${proof.interactionWarningCount} summary=${proof.summaryPath}`,
|
|
765
|
+
...formatLiveProofSetWarningDetails(proof),
|
|
766
|
+
]),
|
|
767
|
+
`Failure reasons: ${artifact.failureReasons.length > 0 ? artifact.failureReasons.join(' ') : 'none'}`,
|
|
768
|
+
`Next action: ${artifact.nextAction.code} - ${artifact.nextAction.summary}`,
|
|
769
|
+
'',
|
|
770
|
+
artifact.summary,
|
|
771
|
+
].join('\n');
|
|
772
|
+
}
|
|
773
|
+
/**
|
|
774
|
+
* Writes durable proof-set artifacts under an output directory.
|
|
775
|
+
*
|
|
776
|
+
* @param {{artifact: LiveProofSetArtifact, outputDir: string}} options
|
|
777
|
+
* @returns {Promise<{liveProofSetPath: string, summaryPath: string}>}
|
|
778
|
+
*/
|
|
779
|
+
async function writeLiveProofSetArtifact({ artifact, outputDir, }) {
|
|
780
|
+
const layout = createArtifactLayout({ outputDir });
|
|
781
|
+
await writeJsonArtifact({
|
|
782
|
+
filePath: layout.liveProofSet,
|
|
783
|
+
label: 'Live proof set artifact',
|
|
784
|
+
schema: SCHEMAS.liveProofSet,
|
|
785
|
+
value: artifact,
|
|
786
|
+
});
|
|
787
|
+
await writeTextArtifact({
|
|
788
|
+
filePath: layout.agentSummary,
|
|
789
|
+
content: formatLiveProofSetArtifactMarkdown(artifact),
|
|
790
|
+
});
|
|
791
|
+
return {
|
|
792
|
+
liveProofSetPath: layout.liveProofSet,
|
|
793
|
+
summaryPath: layout.agentSummary,
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Formats a live-proof artifact for CLI output.
|
|
798
|
+
*
|
|
799
|
+
* @param {LiveProofArtifact} proof
|
|
800
|
+
* @returns {string}
|
|
801
|
+
*/
|
|
802
|
+
function formatLiveProof(proof) {
|
|
803
|
+
return [
|
|
804
|
+
`Live proof: ${proof.platform} ${proof.runId}`,
|
|
805
|
+
`Status: ${proof.status}`,
|
|
806
|
+
`Comparison status: ${proof.comparisonStatus}`,
|
|
807
|
+
`Preflight: ${proof.preflight.runId} health=${proof.preflight.healthStatus} verdict=${proof.preflight.verdictStatus}`,
|
|
808
|
+
`Profiles: ${proof.profiles.length}`,
|
|
809
|
+
...proof.profiles.map((profile) => (`- ${profile.label} (${profile.scenarioId}/${profile.runId}): health=${profile.healthStatus} verdict=${profile.verdictStatus}`)),
|
|
810
|
+
`Interaction proofs: ${proof.interactionProofs?.length ?? 0}`,
|
|
811
|
+
...(proof.interactionProofs ?? []).flatMap((proofPointer) => [
|
|
812
|
+
`- ${proofPointer.label} (${proofPointer.runnerId}/${proofPointer.scenarioId}/${proofPointer.runId}): health=${proofPointer.healthStatus} verdict=${proofPointer.verdictStatus}${formatInteractionProofCaptures(proofPointer)}${formatInteractionProofWarnings(proofPointer)}`,
|
|
813
|
+
...formatInteractionProofWarningDetails(proofPointer),
|
|
814
|
+
]),
|
|
815
|
+
`Skipped interaction proofs: ${proof.skippedInteractionProofs?.length ?? 0}`,
|
|
816
|
+
...(proof.skippedInteractionProofs ?? []).map((proofPointer) => (`- ${proofPointer.label} (${proofPointer.runnerId}/${proofPointer.scenarioId}/${proofPointer.runId}): ${proofPointer.reason} next=${proofPointer.nextAction.code}`)),
|
|
817
|
+
`Comparisons: ${proof.comparisons.length}`,
|
|
818
|
+
`Comparison counts: better=${proof.comparisonCounts.better} worse=${proof.comparisonCounts.worse} unchanged=${proof.comparisonCounts.unchanged} mixed=${proof.comparisonCounts.mixed} inconclusive=${proof.comparisonCounts.inconclusive} skipped=${proof.comparisonCounts.skipped}`,
|
|
819
|
+
...proof.comparisons.map((comparison) => (`- ${comparison.label ?? 'comparison'} (${comparison.scenarioId ?? 'unknown-scenario'}/${comparison.runId ?? 'unknown-run'}): ${comparison.status ?? 'unknown'}${formatComparisonPointerMetrics(comparison)}`)),
|
|
820
|
+
`Next action: ${proof.nextAction.code} - ${proof.nextAction.summary}`,
|
|
821
|
+
`Summary: ${proof.summary}`,
|
|
822
|
+
].join('\n');
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Formats one or more live-proof artifacts for CLI output.
|
|
826
|
+
*
|
|
827
|
+
* @param {{proofs: LiveProofArtifact[], requiredPlatforms?: LiveProofPlatform[]}} options
|
|
828
|
+
* @returns {string}
|
|
829
|
+
*/
|
|
830
|
+
function formatLiveProofSet({ proofs, requiredPlatforms = [], }) {
|
|
831
|
+
if (proofs.length === 1 && requiredPlatforms.length === 0) {
|
|
832
|
+
const [proof] = proofs;
|
|
833
|
+
if (proof) {
|
|
834
|
+
return formatLiveProof(proof);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
return [
|
|
838
|
+
`Live proof set: ${proofs.length} artifact(s)`,
|
|
839
|
+
requiredPlatforms.length > 0 ? `Required platforms: ${requiredPlatforms.join(', ')}` : null,
|
|
840
|
+
`Present platforms: ${Array.from(new Set(proofs.map((proof) => proof.platform))).sort().join(', ')}`,
|
|
841
|
+
'',
|
|
842
|
+
...proofs.map(formatLiveProof),
|
|
843
|
+
].filter((line) => line !== null).join('\n');
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Returns whether the caller requested a regression gate and the proof regressed.
|
|
847
|
+
*
|
|
848
|
+
* @param {{failOnRegression: boolean, proof: LiveProofArtifact}} options
|
|
849
|
+
* @returns {boolean}
|
|
850
|
+
*/
|
|
851
|
+
function shouldFailOnRegression({ failOnRegression, proof, }) {
|
|
852
|
+
return failOnRegression && proof.comparisonStatus === 'regressed';
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Returns whether a live-proof set should make the CLI exit nonzero.
|
|
856
|
+
*
|
|
857
|
+
* @param {{failOnRegression: boolean, proofs: LiveProofArtifact[]}} options
|
|
858
|
+
* @returns {boolean}
|
|
859
|
+
*/
|
|
860
|
+
function shouldFailLiveProofSet({ failOnRegression, proofs, }) {
|
|
861
|
+
return proofs.some((proof) => proof.status === 'failed' || shouldFailOnRegression({ failOnRegression, proof }));
|
|
862
|
+
}
|
|
863
|
+
/**
|
|
864
|
+
* Runs the live-proof inspection CLI.
|
|
865
|
+
*
|
|
866
|
+
* @returns {Promise<void>}
|
|
867
|
+
*/
|
|
868
|
+
async function main() {
|
|
869
|
+
const argv = process.argv.slice(2);
|
|
870
|
+
if (hasHelpFlag(argv)) {
|
|
871
|
+
usage(process.stdout);
|
|
872
|
+
return;
|
|
873
|
+
}
|
|
874
|
+
const args = parseArgs(argv);
|
|
875
|
+
const files = resolveLiveProofFiles(args);
|
|
876
|
+
if (files.length === 0) {
|
|
877
|
+
usage();
|
|
878
|
+
process.exitCode = 1;
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
const requiredPlatforms = parseRequiredPlatforms(args['require-platforms']);
|
|
882
|
+
const requireArtifacts = shouldRequireArtifacts(args);
|
|
883
|
+
const artifactBaseDir = resolveLiveProofArtifactBaseDir(args);
|
|
884
|
+
const readOptions = { requireArtifacts };
|
|
885
|
+
if (artifactBaseDir !== undefined) {
|
|
886
|
+
readOptions.artifactBaseDir = artifactBaseDir;
|
|
887
|
+
}
|
|
888
|
+
const proofs = files.map((file) => readLiveProof(file, readOptions));
|
|
889
|
+
const failOnRegression = args['fail-on-regression'] === true || args['fail-on-regression'] === 'true';
|
|
890
|
+
const proofSet = buildLiveProofSetArtifact({
|
|
891
|
+
failOnRegression,
|
|
892
|
+
files,
|
|
893
|
+
proofs,
|
|
894
|
+
requiredPlatforms,
|
|
895
|
+
runId: resolveLiveProofSetRunId(args),
|
|
896
|
+
});
|
|
897
|
+
process.stdout.write(`${formatLiveProofSet({ proofs, requiredPlatforms })}\n`);
|
|
898
|
+
if (proofSet.failureReasons.length > 0) {
|
|
899
|
+
process.stdout.write(`Proof set status: ${proofSet.status}\n`);
|
|
900
|
+
for (const reason of proofSet.failureReasons) {
|
|
901
|
+
process.stdout.write(`- ${reason}\n`);
|
|
902
|
+
}
|
|
903
|
+
process.stdout.write(`Next action: ${proofSet.nextAction.code} - ${proofSet.nextAction.summary}\n`);
|
|
904
|
+
}
|
|
905
|
+
const outputDir = resolveLiveProofSetOutputDir(args);
|
|
906
|
+
if (outputDir) {
|
|
907
|
+
const written = await writeLiveProofSetArtifact({ artifact: proofSet, outputDir });
|
|
908
|
+
process.stdout.write(`Live proof set artifact: ${written.liveProofSetPath}\n`);
|
|
909
|
+
process.stdout.write(`Live proof set summary: ${written.summaryPath}\n`);
|
|
910
|
+
}
|
|
911
|
+
if (proofSet.status === 'failed') {
|
|
912
|
+
process.exitCode = 2;
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
if (require.main === module) {
|
|
916
|
+
main().catch((error) => {
|
|
917
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
918
|
+
process.exitCode = 1;
|
|
919
|
+
});
|
|
920
|
+
}
|