agent-scenario-loop 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/app/profile-session.ts +352 -12
- package/dist/core/agent-summary.d.ts +3 -2
- package/dist/core/agent-summary.js +44 -2
- package/dist/core/artifact-contract.d.ts +28 -8
- package/dist/core/artifact-contract.js +676 -26
- package/dist/core/comparison.d.ts +57 -3
- package/dist/core/comparison.js +113 -1
- package/dist/core/planner.d.ts +32 -1
- package/dist/core/planner.js +144 -0
- package/dist/core/run-index.d.ts +4 -0
- package/dist/core/run-index.js +55 -1
- package/dist/core/schema-validator.d.ts +2 -0
- package/dist/core/schema-validator.js +2 -0
- package/dist/runner/android-adb-driver.d.ts +7 -2
- package/dist/runner/android-adb-driver.js +7 -1
- package/dist/runner/android-adb.d.ts +40 -5
- package/dist/runner/android-adb.js +1046 -664
- package/dist/runner/compare-latest.d.ts +8 -4
- package/dist/runner/compare-latest.js +24 -5
- package/dist/runner/example-android-live.d.ts +10 -1
- package/dist/runner/example-android-live.js +55 -0
- package/dist/runner/example-ios-live.d.ts +10 -1
- package/dist/runner/example-ios-live.js +55 -0
- package/dist/runner/ios-simctl.d.ts +6 -0
- package/dist/runner/ios-simctl.js +7 -0
- package/dist/runner/live-comparison.d.ts +2 -2
- package/dist/runner/live-comparison.js +2 -1
- package/dist/runner/live-proof-summary.d.ts +5 -4
- package/dist/runner/live-proof-summary.js +12 -2
- package/dist/runner/live-proof.d.ts +3 -2
- package/dist/runner/live-proof.js +9 -2
- package/dist/runner/profile-android.d.ts +16 -1
- package/dist/runner/profile-android.js +364 -26
- package/dist/runner/profile-ios.d.ts +13 -2
- package/dist/runner/profile-ios.js +341 -19
- package/dist/runner/profile-mobile.d.ts +39 -3
- package/dist/runner/profile-mobile.js +1054 -42
- package/dist/runner/validate-project.js +3 -0
- package/dist/scripts/consumer-rehearsal.d.ts +119 -0
- package/dist/scripts/consumer-rehearsal.js +757 -0
- package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
- package/dist/scripts/downstream-local-package-gate.js +264 -0
- package/dist/scripts/package-smoke.d.ts +96 -0
- package/dist/scripts/package-smoke.js +2282 -0
- package/dist/scripts/release-readiness.d.ts +2 -0
- package/dist/scripts/release-readiness.js +520 -0
- package/docs/adapters.md +7 -1
- package/docs/api.md +2 -2
- package/docs/architecture.md +90 -0
- package/docs/authoring.md +39 -3
- package/docs/concepts.md +3 -24
- package/docs/consumer-rehearsal.md +31 -1
- package/docs/contracts.md +45 -101
- package/docs/external-adapter-protocol.md +219 -0
- package/docs/live-proofs.md +86 -3
- package/docs/principles.md +9 -15
- package/examples/mobile-app/README.md +12 -0
- package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
- package/examples/mobile-app/runner-manifests/primary-runner.json +1 -0
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
- package/examples/runners/README.md +4 -3
- package/examples/runners/adb-android.json +1 -0
- package/examples/runners/agent-device-android.json +1 -0
- package/examples/runners/agent-device-ios.json +1 -0
- package/examples/runners/argent-android.json +1 -0
- package/examples/runners/argent-ios.json +1 -0
- package/examples/runners/axe-accessibility-provider.json +2 -2
- package/examples/runners/script-accessibility-provider.json +2 -2
- package/examples/runners/script-memory-provider.json +2 -2
- package/examples/runners/script-network-provider.json +2 -2
- package/examples/runners/script-profiler-provider.json +2 -2
- package/examples/runners/xcodebuildmcp-ios.json +1 -0
- package/package.json +12 -3
- package/schemas/causal-run.schema.json +85 -2
- package/schemas/comparison.schema.json +130 -2
- package/schemas/external-adapter-message.schema.json +693 -0
- package/schemas/health.schema.json +72 -0
- package/schemas/live-proof-set.schema.json +1 -1
- package/schemas/live-proof.schema.json +14 -6
- package/schemas/manifest.schema.json +515 -4
- package/schemas/profiler.schema.json +243 -0
- package/schemas/runner-capabilities.schema.json +28 -2
- package/schemas/scenario.schema.json +34 -2
- package/templates/evidence-provider.json +3 -3
- package/templates/primary-runner.json +1 -0
- package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
|
@@ -0,0 +1,2282 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.createSmokeEnv = createSmokeEnv;
|
|
5
|
+
exports.listFiles = listFiles;
|
|
6
|
+
exports.main = main;
|
|
7
|
+
exports.isAllowedPackedFile = isAllowedPackedFile;
|
|
8
|
+
exports.packageBinPath = packageBinPath;
|
|
9
|
+
exports.run = run;
|
|
10
|
+
exports.runExpectFailure = runExpectFailure;
|
|
11
|
+
exports.typescriptBinPath = typescriptBinPath;
|
|
12
|
+
exports.writeFakeAdb = writeFakeAdb;
|
|
13
|
+
exports.writeSmokeRun = writeSmokeRun;
|
|
14
|
+
const assert = require('node:assert/strict');
|
|
15
|
+
const crypto = require('node:crypto');
|
|
16
|
+
const fs = require('node:fs');
|
|
17
|
+
const os = require('node:os');
|
|
18
|
+
const path = require('node:path');
|
|
19
|
+
const { execFileSync } = require('node:child_process');
|
|
20
|
+
const DEFAULT_COMMAND_TIMEOUT_MS = 180_000;
|
|
21
|
+
/**
|
|
22
|
+
* Hashes a package-smoke fixture file.
|
|
23
|
+
*
|
|
24
|
+
* @param {string} filePath
|
|
25
|
+
* @returns {string}
|
|
26
|
+
*/
|
|
27
|
+
function sha256File(filePath) {
|
|
28
|
+
return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
|
29
|
+
}
|
|
30
|
+
const EXAMPLE_PROFILE_RUNS = [
|
|
31
|
+
{
|
|
32
|
+
binaryName: 'asl-profile-ios',
|
|
33
|
+
eventLog: 'app-startup.log',
|
|
34
|
+
platform: 'ios',
|
|
35
|
+
runId: 'example-startup',
|
|
36
|
+
scenario: 'app-startup.json',
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
binaryName: 'asl-profile-ios',
|
|
40
|
+
eventLog: 'open-close-cycle.log',
|
|
41
|
+
platform: 'ios',
|
|
42
|
+
runId: 'example-open-close',
|
|
43
|
+
scenario: 'open-close-cycle.json',
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
binaryName: 'asl-profile-ios',
|
|
47
|
+
eventLog: 'scroll-settle.log',
|
|
48
|
+
platform: 'ios',
|
|
49
|
+
runId: 'example-scroll',
|
|
50
|
+
scenario: 'scroll-settle.json',
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
binaryName: 'asl-profile-android',
|
|
54
|
+
eventLog: 'android-app-startup.log',
|
|
55
|
+
platform: 'android',
|
|
56
|
+
runId: 'android-example-startup',
|
|
57
|
+
scenario: 'app-startup.json',
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
binaryName: 'asl-profile-android',
|
|
61
|
+
eventLog: 'android-open-close-cycle.log',
|
|
62
|
+
platform: 'android',
|
|
63
|
+
runId: 'android-example-open-close',
|
|
64
|
+
scenario: 'open-close-cycle.json',
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
binaryName: 'asl-profile-android',
|
|
68
|
+
eventLog: 'android-scroll-settle.log',
|
|
69
|
+
platform: 'android',
|
|
70
|
+
runId: 'android-example-scroll',
|
|
71
|
+
scenario: 'scroll-settle.json',
|
|
72
|
+
},
|
|
73
|
+
];
|
|
74
|
+
const PACKED_FILE_ALLOWLIST = [
|
|
75
|
+
/^LICENSE$/u,
|
|
76
|
+
/^README\.md$/u,
|
|
77
|
+
/^package\.json$/u,
|
|
78
|
+
/^app\/profile-session\.ts$/u,
|
|
79
|
+
/^core\/config-template\.json$/u,
|
|
80
|
+
/^dist\/index\.(?:js|d\.ts)$/u,
|
|
81
|
+
/^dist\/core\/[a-z-]+\.(?:js|d\.ts)$/u,
|
|
82
|
+
/^dist\/runner\/[a-z-]+\.(?:js|d\.ts)$/u,
|
|
83
|
+
/^dist\/scripts\/[a-z-]+\.(?:js|d\.ts)$/u,
|
|
84
|
+
/^docs\/[a-z-]+\.md$/u,
|
|
85
|
+
/^examples\/.+/u,
|
|
86
|
+
/^schemas\/[a-z-]+\.schema\.json$/u,
|
|
87
|
+
/^templates\/[a-z0-9.-]+(?:\.(?:json|md))?$/u,
|
|
88
|
+
/^templates\/skills\/agent-scenario-loop\/SKILL\.md$/u,
|
|
89
|
+
/^templates\/skills\/agent-scenario-loop\/references\/[a-z-]+\.md$/u,
|
|
90
|
+
/^templates\/scripts\/[a-z0-9.-]+\.mjs$/u,
|
|
91
|
+
];
|
|
92
|
+
/**
|
|
93
|
+
* Lists every file under a directory using relative POSIX-style paths.
|
|
94
|
+
*
|
|
95
|
+
* @param {string} rootDir
|
|
96
|
+
* @param {string} [relativeDir]
|
|
97
|
+
* @returns {string[]}
|
|
98
|
+
*/
|
|
99
|
+
function listFiles(rootDir, relativeDir = '') {
|
|
100
|
+
const absoluteDir = path.join(rootDir, relativeDir);
|
|
101
|
+
return fs.readdirSync(absoluteDir, { withFileTypes: true }).flatMap((entry) => {
|
|
102
|
+
const relativePath = path.join(relativeDir, entry.name);
|
|
103
|
+
if (entry.isDirectory()) {
|
|
104
|
+
return listFiles(rootDir, relativePath);
|
|
105
|
+
}
|
|
106
|
+
if (!entry.isFile()) {
|
|
107
|
+
return [];
|
|
108
|
+
}
|
|
109
|
+
return [relativePath.split(path.sep).join('/')];
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Returns whether a packed file belongs to the intentional public package surface.
|
|
114
|
+
*
|
|
115
|
+
* @param {string} filePath
|
|
116
|
+
* @returns {boolean}
|
|
117
|
+
*/
|
|
118
|
+
function isAllowedPackedFile(filePath) {
|
|
119
|
+
return PACKED_FILE_ALLOWLIST.some((pattern) => pattern.test(filePath));
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Creates a clean npm environment for repeatable package smoke checks.
|
|
123
|
+
*
|
|
124
|
+
* @param {string} tempRoot
|
|
125
|
+
* @returns {NodeJS.ProcessEnv}
|
|
126
|
+
*/
|
|
127
|
+
function createSmokeEnv(tempRoot) {
|
|
128
|
+
const env = { ...process.env };
|
|
129
|
+
for (const key of Object.keys(env)) {
|
|
130
|
+
if (key.toLowerCase().startsWith('npm_config_')) {
|
|
131
|
+
delete env[key];
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
env.npm_config_audit = 'false';
|
|
135
|
+
env.npm_config_cache = path.join(tempRoot, 'npm-cache');
|
|
136
|
+
env.npm_config_fund = 'false';
|
|
137
|
+
env.npm_config_update_notifier = 'false';
|
|
138
|
+
return env;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Resolves the per-command timeout for package gate child processes.
|
|
142
|
+
*
|
|
143
|
+
* @param {NodeJS.ProcessEnv} env
|
|
144
|
+
* @returns {number}
|
|
145
|
+
*/
|
|
146
|
+
function resolveCommandTimeoutMs(env) {
|
|
147
|
+
const timeoutMs = Number.parseInt(env.ASL_PACKAGE_GATE_TIMEOUT_MS ?? '', 10);
|
|
148
|
+
return Number.isFinite(timeoutMs) && timeoutMs > 0
|
|
149
|
+
? timeoutMs
|
|
150
|
+
: DEFAULT_COMMAND_TIMEOUT_MS;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Runs a command and returns stdout while preserving stderr for failures.
|
|
154
|
+
*
|
|
155
|
+
* @param {string} command
|
|
156
|
+
* @param {string[]} args
|
|
157
|
+
* @param {RunOptions} options
|
|
158
|
+
* @returns {string}
|
|
159
|
+
*/
|
|
160
|
+
function run(command, args, options) {
|
|
161
|
+
return execFileSync(command, args, {
|
|
162
|
+
cwd: options.cwd,
|
|
163
|
+
encoding: 'utf8',
|
|
164
|
+
env: options.env,
|
|
165
|
+
stdio: ['ignore', 'pipe', 'inherit'],
|
|
166
|
+
timeout: resolveCommandTimeoutMs(options.env),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Runs a command that is expected to fail and returns captured output.
|
|
171
|
+
*
|
|
172
|
+
* @param {string} command
|
|
173
|
+
* @param {string[]} args
|
|
174
|
+
* @param {RunOptions} options
|
|
175
|
+
* @returns {FailedRunOutput}
|
|
176
|
+
*/
|
|
177
|
+
function runExpectFailure(command, args, options) {
|
|
178
|
+
try {
|
|
179
|
+
const stdout = execFileSync(command, args, {
|
|
180
|
+
cwd: options.cwd,
|
|
181
|
+
encoding: 'utf8',
|
|
182
|
+
env: options.env,
|
|
183
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
184
|
+
timeout: resolveCommandTimeoutMs(options.env),
|
|
185
|
+
});
|
|
186
|
+
throw new Error(`Expected command to fail, but it passed with stdout: ${stdout}`);
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
const failed = error;
|
|
190
|
+
if (failed.message.startsWith('Expected command to fail')) {
|
|
191
|
+
throw failed;
|
|
192
|
+
}
|
|
193
|
+
return {
|
|
194
|
+
status: failed.status ?? null,
|
|
195
|
+
stderr: Buffer.isBuffer(failed.stderr) ? failed.stderr.toString('utf8') : String(failed.stderr ?? ''),
|
|
196
|
+
stdout: Buffer.isBuffer(failed.stdout) ? failed.stdout.toString('utf8') : String(failed.stdout ?? ''),
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Writes a tiny adb-compatible command for installed-package smoke tests.
|
|
202
|
+
*
|
|
203
|
+
* @param {{filePath: string, logcatText: string, packageName: string}} options
|
|
204
|
+
* @returns {void}
|
|
205
|
+
*/
|
|
206
|
+
function writeFakeAdb({ filePath, logcatText, packageName, }) {
|
|
207
|
+
const scriptPath = filePath.endsWith('.cmd') ? filePath.replace(/\.cmd$/u, '.js') : filePath;
|
|
208
|
+
const script = [
|
|
209
|
+
'#!/usr/bin/env node',
|
|
210
|
+
"const args = process.argv.slice(2);",
|
|
211
|
+
"const key = args.join(' ');",
|
|
212
|
+
`const logcatText = ${JSON.stringify(logcatText)};`,
|
|
213
|
+
`const packageName = ${JSON.stringify(packageName)};`,
|
|
214
|
+
"const responses = new Map([",
|
|
215
|
+
" ['version', { stdout: 'Android Debug Bridge version 1.0.41\\n' }],",
|
|
216
|
+
" ['devices -l', { stdout: 'List of devices attached\\nemulator-5554 device product:sdk_gphone model:Pixel_6 device:emu64\\n' }],",
|
|
217
|
+
" ['-s emulator-5554 shell getprop ro.product.model', { stdout: 'Pixel 6\\n' }],",
|
|
218
|
+
" ['-s emulator-5554 shell getprop ro.build.version.release', { stdout: '15\\n' }],",
|
|
219
|
+
" ['-s emulator-5554 shell getprop ro.build.version.sdk', { stdout: '35\\n' }],",
|
|
220
|
+
" [`-s emulator-5554 shell pm path ${packageName}`, { stdout: `package:/data/app/${packageName}/base.apk\\n` }],",
|
|
221
|
+
" ['-s emulator-5554 logcat -c', { stdout: '' }],",
|
|
222
|
+
" [`-s emulator-5554 shell monkey -p ${packageName} -c android.intent.category.LAUNCHER 1`, { stdout: 'Events injected: 1\\n' }],",
|
|
223
|
+
" [`-s emulator-5554 shell pidof ${packageName}`, { stdout: '1234\\n' }],",
|
|
224
|
+
" ['-s emulator-5554 shell rm -f /sdcard/agent-scenario-loop-ui.xml; uiautomator dump /sdcard/agent-scenario-loop-ui.xml >/dev/null; cat /sdcard/agent-scenario-loop-ui.xml; status=$?; rm -f /sdcard/agent-scenario-loop-ui.xml; exit $status', { stdout: '<?xml version=\"1.0\" encoding=\"UTF-8\"?><hierarchy><node resource-id=\"dev.agentscenarioloop.example:id/asl-example-title\" text=\"Example Mobile App\" bounds=\"[10,20][300,80]\" /></hierarchy>\\n' }],",
|
|
225
|
+
" ['-s emulator-5554 exec-out screencap -p', { stdout: 'fake png' }],",
|
|
226
|
+
" ['-s emulator-5554 logcat -d -v time -t 25', { stdout: logcatText }],",
|
|
227
|
+
" ['-s emulator-5554 logcat -d -v time -t 1000', { stdout: logcatText }],",
|
|
228
|
+
"]);",
|
|
229
|
+
"const response = responses.get(key);",
|
|
230
|
+
"if (!response) {",
|
|
231
|
+
" process.stderr.write(`unexpected fake adb command: ${key}\\n`);",
|
|
232
|
+
" process.exit(1);",
|
|
233
|
+
"}",
|
|
234
|
+
"process.stdout.write(response.stdout ?? '');",
|
|
235
|
+
"process.stderr.write(response.stderr ?? '');",
|
|
236
|
+
"process.exit(response.exitCode ?? 0);",
|
|
237
|
+
'',
|
|
238
|
+
].join('\n');
|
|
239
|
+
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
|
|
240
|
+
if (filePath.endsWith('.cmd')) {
|
|
241
|
+
fs.writeFileSync(filePath, `@echo off\r\n"${process.execPath}" "%~dp0${path.basename(scriptPath)}" %*\r\n`, {
|
|
242
|
+
mode: 0o755,
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Writes a tiny adb-compatible command for the installed example-live proof.
|
|
248
|
+
*
|
|
249
|
+
* @param {{filePath: string, fixtures: Record<string, ExampleLiveFixture>, packageName: string}} options
|
|
250
|
+
* @returns {void}
|
|
251
|
+
*/
|
|
252
|
+
function writeFakeExampleLiveAdb({ filePath, fixtures, packageName, }) {
|
|
253
|
+
const scriptPath = filePath.endsWith('.cmd') ? filePath.replace(/\.cmd$/u, '.js') : filePath;
|
|
254
|
+
const script = [
|
|
255
|
+
'#!/usr/bin/env node',
|
|
256
|
+
"const fs = require('node:fs');",
|
|
257
|
+
"const args = process.argv.slice(2);",
|
|
258
|
+
"const key = args.join(' ');",
|
|
259
|
+
`const fixtures = ${JSON.stringify(fixtures)};`,
|
|
260
|
+
`const packageName = ${JSON.stringify(packageName)};`,
|
|
261
|
+
`const statePath = ${JSON.stringify(`${scriptPath}.state.json`)};`,
|
|
262
|
+
"function readState() {",
|
|
263
|
+
" try {",
|
|
264
|
+
" return JSON.parse(fs.readFileSync(statePath, 'utf8'));",
|
|
265
|
+
" } catch {",
|
|
266
|
+
" return { currentRunId: 'android-live-startup', currentScenario: 'app-startup' };",
|
|
267
|
+
" }",
|
|
268
|
+
"}",
|
|
269
|
+
"function writeState(state) {",
|
|
270
|
+
" fs.writeFileSync(statePath, `${JSON.stringify(state)}\\n`, 'utf8');",
|
|
271
|
+
"}",
|
|
272
|
+
"const state = readState();",
|
|
273
|
+
"function ok(stdout = '') { process.stdout.write(stdout); process.exit(0); }",
|
|
274
|
+
"if (key === 'version') ok('Android Debug Bridge version 1.0.41\\n');",
|
|
275
|
+
"if (key === 'devices -l') ok('List of devices attached\\nemulator-5554 device product:sdk_gphone model:Pixel_6 device:emu64\\n');",
|
|
276
|
+
"if (key.endsWith('shell getprop ro.product.model')) ok('Pixel 6\\n');",
|
|
277
|
+
"if (key.endsWith('shell getprop ro.build.version.release')) ok('15\\n');",
|
|
278
|
+
"if (key.endsWith('shell getprop ro.build.version.sdk')) ok('35\\n');",
|
|
279
|
+
"if (key.endsWith(`shell pm path ${packageName}`)) ok(`package:/data/app/${packageName}/base.apk\\n`);",
|
|
280
|
+
"if (key === '-s emulator-5554 reverse tcp:8097 tcp:8097') ok('');",
|
|
281
|
+
"if (key.includes(`shell run-as ${packageName} sh -c`) && key.includes('debug_http_host') && key.includes('localhost:8097')) ok('');",
|
|
282
|
+
"if (key.endsWith(`shell monkey -p ${packageName} -c android.intent.category.LAUNCHER 1`)) ok('Events injected: 1\\n');",
|
|
283
|
+
"if (key.endsWith(`shell pidof ${packageName}`)) ok('1234\\n');",
|
|
284
|
+
"if (key.endsWith('shell rm -f /sdcard/agent-scenario-loop-ui.xml; uiautomator dump /sdcard/agent-scenario-loop-ui.xml >/dev/null; cat /sdcard/agent-scenario-loop-ui.xml; status=$?; rm -f /sdcard/agent-scenario-loop-ui.xml; exit $status')) ok('<?xml version=\"1.0\" encoding=\"UTF-8\"?><hierarchy><node resource-id=\"dev.agentscenarioloop.example:id/asl-example-title\" text=\"Example Mobile App\" bounds=\"[10,20][300,80]\" /></hierarchy>\\n');",
|
|
285
|
+
"if (key.endsWith('exec-out screencap -p')) ok('fake png');",
|
|
286
|
+
"if (key.endsWith('logcat -c')) ok('');",
|
|
287
|
+
"if (key.includes('profile-session/start')) {",
|
|
288
|
+
" state.currentRunId = /runId=([^&']+)/u.exec(key)?.[1] ?? state.currentRunId;",
|
|
289
|
+
" state.currentScenario = /scenario=([^&']+)/u.exec(key)?.[1] ?? state.currentScenario;",
|
|
290
|
+
" writeState(state);",
|
|
291
|
+
" ok('Starting: Intent\\n');",
|
|
292
|
+
"}",
|
|
293
|
+
"if (key.includes('profile-session/command')) ok('Starting: Intent\\n');",
|
|
294
|
+
"if (key.endsWith('logcat -d -v time -t 1000')) {",
|
|
295
|
+
" const fixture = fixtures[state.currentScenario];",
|
|
296
|
+
" if (!fixture) {",
|
|
297
|
+
" process.stderr.write(`missing fixture for ${state.currentScenario}\\n`);",
|
|
298
|
+
" process.exit(1);",
|
|
299
|
+
" }",
|
|
300
|
+
" ok(fixture.logcatText.replaceAll(fixture.fixtureRunId, state.currentRunId));",
|
|
301
|
+
"}",
|
|
302
|
+
"process.stderr.write(`unexpected fake adb command: ${key}\\n`);",
|
|
303
|
+
"process.exit(1);",
|
|
304
|
+
'',
|
|
305
|
+
].join('\n');
|
|
306
|
+
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
|
|
307
|
+
if (filePath.endsWith('.cmd')) {
|
|
308
|
+
fs.writeFileSync(filePath, `@echo off\r\n"${process.execPath}" "%~dp0${path.basename(scriptPath)}" %*\r\n`, {
|
|
309
|
+
mode: 0o755,
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Writes a tiny agent-device-compatible command for installed aggregate proof smoke tests.
|
|
315
|
+
*
|
|
316
|
+
* @param {string} filePath
|
|
317
|
+
* @returns {void}
|
|
318
|
+
*/
|
|
319
|
+
function writeFakeAgentDevice(filePath) {
|
|
320
|
+
const scriptPath = filePath.endsWith('.cmd') ? filePath.replace(/\.cmd$/u, '.js') : filePath;
|
|
321
|
+
const script = [
|
|
322
|
+
'#!/usr/bin/env node',
|
|
323
|
+
"const fs = require('node:fs');",
|
|
324
|
+
"const path = require('node:path');",
|
|
325
|
+
"const args = process.argv.slice(2);",
|
|
326
|
+
"function ok(data = {}) {",
|
|
327
|
+
" process.stdout.write(`${JSON.stringify({ success: true, data })}\\n`);",
|
|
328
|
+
" process.exit(0);",
|
|
329
|
+
"}",
|
|
330
|
+
"if (args[0] === 'open' && typeof args[1] === 'string') {",
|
|
331
|
+
" ok({ opened: args[1] });",
|
|
332
|
+
"}",
|
|
333
|
+
"if (args[0] === 'is' && args[1] === 'visible' && typeof args[2] === 'string') {",
|
|
334
|
+
" ok({ selector: args[2], visible: true });",
|
|
335
|
+
"}",
|
|
336
|
+
"if (args[0] === 'screenshot' && typeof args[1] === 'string') {",
|
|
337
|
+
" fs.mkdirSync(path.dirname(args[1]), { recursive: true });",
|
|
338
|
+
" fs.writeFileSync(args[1], 'fake screenshot', 'utf8');",
|
|
339
|
+
" ok({ path: args[1] });",
|
|
340
|
+
"}",
|
|
341
|
+
"process.stderr.write(`unexpected fake agent-device command: ${args.join(' ')}\\n`);",
|
|
342
|
+
"process.exit(1);",
|
|
343
|
+
'',
|
|
344
|
+
].join('\n');
|
|
345
|
+
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
|
|
346
|
+
if (filePath.endsWith('.cmd')) {
|
|
347
|
+
fs.writeFileSync(filePath, `@echo off\r\n"${process.execPath}" "%~dp0${path.basename(scriptPath)}" %*\r\n`, {
|
|
348
|
+
mode: 0o755,
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Writes a tiny Argent-compatible command for installed-package smoke tests.
|
|
354
|
+
*
|
|
355
|
+
* @param {string} filePath
|
|
356
|
+
* @returns {void}
|
|
357
|
+
*/
|
|
358
|
+
function writeFakeArgent(filePath) {
|
|
359
|
+
const scriptPath = filePath.endsWith('.cmd') ? filePath.replace(/\.cmd$/u, '.js') : filePath;
|
|
360
|
+
const script = [
|
|
361
|
+
'#!/usr/bin/env node',
|
|
362
|
+
"const fs = require('node:fs');",
|
|
363
|
+
"const path = require('node:path');",
|
|
364
|
+
"const args = process.argv.slice(2);",
|
|
365
|
+
"const key = args.join(' ');",
|
|
366
|
+
"function ok(stdout = '{\"description\":\"Home Ready asl-example-title Example Mobile App\"}\\n') { process.stdout.write(stdout); process.exit(0); }",
|
|
367
|
+
"if (args.includes('launch-app')) ok('{\"success\":true}\\n');",
|
|
368
|
+
"if (args.includes('describe')) ok();",
|
|
369
|
+
"if (args.includes('screenshot')) {",
|
|
370
|
+
" const screenshotPath = path.join(path.dirname(process.argv[1]), 'fake-argent-screenshot.png');",
|
|
371
|
+
" fs.writeFileSync(screenshotPath, 'fake screenshot', 'utf8');",
|
|
372
|
+
" ok(`Saved screenshot: ${screenshotPath}\\n`);",
|
|
373
|
+
"}",
|
|
374
|
+
"if (args.includes('gesture-tap')) ok('{\"success\":true}\\n');",
|
|
375
|
+
"if (args.includes('gesture-swipe')) ok('{\"success\":true}\\n');",
|
|
376
|
+
"process.stderr.write(`unexpected fake Argent command: ${key}\\n`);",
|
|
377
|
+
"process.exit(1);",
|
|
378
|
+
'',
|
|
379
|
+
].join('\n');
|
|
380
|
+
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
|
|
381
|
+
if (filePath.endsWith('.cmd')) {
|
|
382
|
+
fs.writeFileSync(filePath, `@echo off\r\n"${process.execPath}" "%~dp0${path.basename(scriptPath)}" %*\r\n`, {
|
|
383
|
+
mode: 0o755,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Writes a tiny xcrun-compatible command for the installed iOS example-live proof.
|
|
389
|
+
*
|
|
390
|
+
* @param {{bundleId: string, deviceId: string, filePath: string, fixtures: Record<string, ExampleLiveFixture>}} options
|
|
391
|
+
* @returns {void}
|
|
392
|
+
*/
|
|
393
|
+
function writeFakeExampleLiveXcrun({ bundleId, deviceId, filePath, fixtures, }) {
|
|
394
|
+
const scriptPath = filePath.endsWith('.cmd') ? filePath.replace(/\.cmd$/u, '.js') : filePath;
|
|
395
|
+
const dataContainer = `${scriptPath}.data`;
|
|
396
|
+
const script = [
|
|
397
|
+
'#!/usr/bin/env node',
|
|
398
|
+
"const fs = require('node:fs');",
|
|
399
|
+
"const path = require('node:path');",
|
|
400
|
+
"const args = process.argv.slice(2);",
|
|
401
|
+
"const key = args.join(' ');",
|
|
402
|
+
`const bundleId = ${JSON.stringify(bundleId)};`,
|
|
403
|
+
`const dataContainer = ${JSON.stringify(dataContainer)};`,
|
|
404
|
+
`const deviceId = ${JSON.stringify(deviceId)};`,
|
|
405
|
+
`const fixtures = ${JSON.stringify(fixtures)};`,
|
|
406
|
+
"function ok(stdout = '') { process.stdout.write(stdout); process.exit(0); }",
|
|
407
|
+
"function storageDir() {",
|
|
408
|
+
" return path.join(dataContainer, 'Library', 'Application Support', bundleId, 'RCTAsyncLocalStorage_V1');",
|
|
409
|
+
"}",
|
|
410
|
+
"function readSession() {",
|
|
411
|
+
" const manifestPath = path.join(storageDir(), 'manifest.json');",
|
|
412
|
+
" const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));",
|
|
413
|
+
" return { manifest, manifestPath, session: JSON.parse(manifest['agent-scenario-loop.profile-session.1']) };",
|
|
414
|
+
"}",
|
|
415
|
+
"function parseFixtureEvents(logText, fixtureRunId, runId) {",
|
|
416
|
+
" return logText.split(/\\r?\\n/u).filter(Boolean).map((line) => {",
|
|
417
|
+
" const payload = line.slice(line.indexOf('[profile-event]') + '[profile-event]'.length).trim();",
|
|
418
|
+
" return { ...JSON.parse(payload), runId };",
|
|
419
|
+
" });",
|
|
420
|
+
"}",
|
|
421
|
+
"function writeCurrentEvents() {",
|
|
422
|
+
" const current = readSession();",
|
|
423
|
+
" const fixture = fixtures[current.session.scenario];",
|
|
424
|
+
" if (!fixture) {",
|
|
425
|
+
" process.stderr.write(`missing fixture for ${current.session.scenario}\\n`);",
|
|
426
|
+
" process.exit(1);",
|
|
427
|
+
" }",
|
|
428
|
+
" current.manifest['agent-scenario-loop.profile-events.1'] = JSON.stringify(parseFixtureEvents(fixture.logcatText, fixture.fixtureRunId, current.session.runId));",
|
|
429
|
+
" fs.writeFileSync(current.manifestPath, JSON.stringify(current.manifest), 'utf8');",
|
|
430
|
+
"}",
|
|
431
|
+
"if (key === 'simctl list devices') ok(`== Devices ==\\n-- iOS 26.3 --\\n iPhone 17 Pro Max (${deviceId}) (Booted)\\n`);",
|
|
432
|
+
"if (key === `simctl get_app_container ${deviceId} ${bundleId} app`) ok('/tmp/ASLExampleMobile.app\\n');",
|
|
433
|
+
"if (key === `simctl get_app_container ${deviceId} ${bundleId} data`) { fs.mkdirSync(storageDir(), { recursive: true }); ok(`${dataContainer}\\n`); }",
|
|
434
|
+
"if (key === `simctl terminate ${deviceId} ${bundleId}`) ok('');",
|
|
435
|
+
"if (key === `simctl launch ${deviceId} ${bundleId}`) { writeCurrentEvents(); ok(`${bundleId}: 1234\\n`); }",
|
|
436
|
+
"if (args[0] === 'simctl' && args[1] === 'io' && args[2] === deviceId && args[3] === 'screenshot' && typeof args[4] === 'string') {",
|
|
437
|
+
" fs.mkdirSync(path.dirname(args[4]), { recursive: true });",
|
|
438
|
+
" fs.writeFileSync(args[4], 'fake screenshot', 'utf8');",
|
|
439
|
+
" ok('');",
|
|
440
|
+
"}",
|
|
441
|
+
"if (key === `simctl spawn ${deviceId} log show --style compact --last 2m --predicate eventMessage CONTAINS \"[profile-event]\" OR eventMessage CONTAINS \"[profile-session]\"`) ok('Timestamp Ty Process[PID:TID]\\n');",
|
|
442
|
+
"process.stderr.write(`unexpected fake xcrun command: ${key}\\n`);",
|
|
443
|
+
"process.exit(1);",
|
|
444
|
+
'',
|
|
445
|
+
].join('\n');
|
|
446
|
+
fs.writeFileSync(scriptPath, script, { mode: 0o755 });
|
|
447
|
+
if (filePath.endsWith('.cmd')) {
|
|
448
|
+
fs.writeFileSync(filePath, `@echo off\r\n"${process.execPath}" "%~dp0${path.basename(scriptPath)}" %*\r\n`, {
|
|
449
|
+
mode: 0o755,
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Writes a minimal passed run directory for installed-package comparison smoke tests.
|
|
455
|
+
*
|
|
456
|
+
* @param {SmokeRunOptions} options
|
|
457
|
+
* @returns {string}
|
|
458
|
+
*/
|
|
459
|
+
function writeSmokeRun({ actual, endedAt, root, runId, }) {
|
|
460
|
+
const runDir = path.join(root, 'app-startup', runId);
|
|
461
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
462
|
+
fs.writeFileSync(path.join(runDir, 'manifest.json'), `${JSON.stringify({
|
|
463
|
+
schemaVersion: '1.0.0',
|
|
464
|
+
runId,
|
|
465
|
+
scenario: 'app-startup',
|
|
466
|
+
platform: 'android',
|
|
467
|
+
interactionDriver: 'adb-logcat',
|
|
468
|
+
startedAt: '2026-06-16T10:00:00.000Z',
|
|
469
|
+
endedAt,
|
|
470
|
+
durationMs: 1000,
|
|
471
|
+
artifacts: { raw: {} },
|
|
472
|
+
})}\n`, 'utf8');
|
|
473
|
+
fs.writeFileSync(path.join(runDir, 'health.json'), `${JSON.stringify({
|
|
474
|
+
schemaVersion: '1.0.0',
|
|
475
|
+
scenarioId: 'app-startup',
|
|
476
|
+
flowId: 'app-startup',
|
|
477
|
+
runId,
|
|
478
|
+
healthStatus: 'passed',
|
|
479
|
+
checks: [{ name: 'truth_events_complete', status: 'passed', source: 'truth' }],
|
|
480
|
+
})}\n`, 'utf8');
|
|
481
|
+
fs.writeFileSync(path.join(runDir, 'verdict.json'), `${JSON.stringify({
|
|
482
|
+
schemaVersion: '1.0.0',
|
|
483
|
+
scenarioId: 'app-startup',
|
|
484
|
+
flowId: 'app-startup',
|
|
485
|
+
runId,
|
|
486
|
+
healthStatus: 'passed',
|
|
487
|
+
verdictStatus: 'passed',
|
|
488
|
+
budgetChecks: [
|
|
489
|
+
{
|
|
490
|
+
name: 'startup p95',
|
|
491
|
+
source: 'milestone',
|
|
492
|
+
metric: 'p95',
|
|
493
|
+
unit: 'ms',
|
|
494
|
+
expected: 1000,
|
|
495
|
+
actual,
|
|
496
|
+
pass: actual <= 1000,
|
|
497
|
+
},
|
|
498
|
+
],
|
|
499
|
+
})}\n`, 'utf8');
|
|
500
|
+
return runDir;
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Asserts that an installed-package comparison carries latest-trusted provenance.
|
|
504
|
+
*
|
|
505
|
+
* @param {{artifactRoot: string, baselineRunDir: string, baselineRunId: string, comparison: Record<string, any>, currentRunDir: string, currentRunId: string}} options
|
|
506
|
+
* @returns {void}
|
|
507
|
+
*/
|
|
508
|
+
function assertLatestTrustedComparisonBasis({ artifactRoot, baselineRunDir, baselineRunId, comparison, currentRunDir, currentRunId, }) {
|
|
509
|
+
assert.equal(comparison.comparisonBasis.strategy, 'latest_trusted_prior');
|
|
510
|
+
assert.equal(comparison.comparisonBasis.baseline.runId, baselineRunId);
|
|
511
|
+
assert.equal(comparison.comparisonBasis.baseline.runDir, baselineRunDir);
|
|
512
|
+
assert.equal(comparison.comparisonBasis.current.runId, currentRunId);
|
|
513
|
+
assert.equal(comparison.comparisonBasis.current.runDir, currentRunDir);
|
|
514
|
+
assert.equal(comparison.comparisonBasis.selection.artifactRoot, artifactRoot);
|
|
515
|
+
assert.equal(comparison.comparisonBasis.selection.selectedRunId, baselineRunId);
|
|
516
|
+
assert.equal(comparison.comparisonBasis.selection.selectedRunDir, baselineRunDir);
|
|
517
|
+
assert.equal(comparison.comparisonBasis.selection.scenarioId, comparison.scenarioId);
|
|
518
|
+
assert.equal(comparison.comparisonBasis.selection.skippedCurrentRun, true);
|
|
519
|
+
assert.equal(comparison.comparisonBasis.selection.trustedPriorCandidates >= 1, true);
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* Returns the platform-specific path for a package binary in a temp install.
|
|
523
|
+
*
|
|
524
|
+
* @param {string} installDir
|
|
525
|
+
* @param {string} name
|
|
526
|
+
* @returns {string}
|
|
527
|
+
*/
|
|
528
|
+
function packageBinPath(installDir, name) {
|
|
529
|
+
const suffix = process.platform === 'win32' ? '.cmd' : '';
|
|
530
|
+
return path.join(installDir, 'node_modules', '.bin', `${name}${suffix}`);
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Returns the platform-specific path for the repo-local TypeScript compiler.
|
|
534
|
+
*
|
|
535
|
+
* @param {string} repoRoot
|
|
536
|
+
* @returns {string}
|
|
537
|
+
*/
|
|
538
|
+
function typescriptBinPath(repoRoot) {
|
|
539
|
+
const suffix = process.platform === 'win32' ? '.cmd' : '';
|
|
540
|
+
return path.join(repoRoot, 'node_modules', '.bin', `tsc${suffix}`);
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Packs, installs, and exercises the public package surface from a temp project.
|
|
544
|
+
*
|
|
545
|
+
* @returns {void}
|
|
546
|
+
*/
|
|
547
|
+
function main() {
|
|
548
|
+
const repoRoot = path.resolve(__dirname, '..', '..');
|
|
549
|
+
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'asl-package-smoke-'));
|
|
550
|
+
const packDir = path.join(tempRoot, 'pack');
|
|
551
|
+
const installDir = path.join(tempRoot, 'install');
|
|
552
|
+
const artifactDir = path.join(tempRoot, 'artifacts', 'plan', 'app-startup');
|
|
553
|
+
const env = createSmokeEnv(tempRoot);
|
|
554
|
+
fs.mkdirSync(packDir, { recursive: true });
|
|
555
|
+
fs.mkdirSync(installDir, { recursive: true });
|
|
556
|
+
try {
|
|
557
|
+
assert.equal(fs.existsSync(path.join(repoRoot, '.npmignore')), true, 'root .npmignore must exist so npm pack never falls back to .gitignore');
|
|
558
|
+
assert.equal(fs.existsSync(path.join(repoRoot, 'examples', 'mobile-app', '.npmignore')), true, 'example app .npmignore must exist so npm pack never falls back to the app-local .gitignore');
|
|
559
|
+
const packOutput = run('npm', ['pack', '--pack-destination', packDir], {
|
|
560
|
+
cwd: repoRoot,
|
|
561
|
+
env,
|
|
562
|
+
});
|
|
563
|
+
const tarballName = packOutput.trim().split(/\n/u).pop();
|
|
564
|
+
assert.ok(tarballName, 'npm pack did not print a tarball name');
|
|
565
|
+
const tarballPath = path.join(packDir, tarballName);
|
|
566
|
+
assert.equal(fs.existsSync(tarballPath), true, `missing packed tarball: ${tarballPath}`);
|
|
567
|
+
run('npm', ['init', '-y'], {
|
|
568
|
+
cwd: installDir,
|
|
569
|
+
env,
|
|
570
|
+
});
|
|
571
|
+
run('npm', ['install', tarballPath, '--ignore-scripts'], {
|
|
572
|
+
cwd: installDir,
|
|
573
|
+
env,
|
|
574
|
+
});
|
|
575
|
+
const packageRoot = path.join(installDir, 'node_modules', 'agent-scenario-loop');
|
|
576
|
+
const commonJsSmokeScript = [
|
|
577
|
+
"const assert = require('node:assert/strict');",
|
|
578
|
+
"const asl = require('agent-scenario-loop');",
|
|
579
|
+
"assert.equal(typeof asl.buildRunIndex, 'function');",
|
|
580
|
+
"assert.equal(typeof asl.createArtifactLayout, 'function');",
|
|
581
|
+
"assert.equal(typeof asl.validateJson, 'function');",
|
|
582
|
+
].join('\n');
|
|
583
|
+
run(process.execPath, ['-e', commonJsSmokeScript], {
|
|
584
|
+
cwd: installDir,
|
|
585
|
+
env,
|
|
586
|
+
});
|
|
587
|
+
const esmSmokeScriptPath = path.join(installDir, 'package-smoke-esm.mjs');
|
|
588
|
+
fs.writeFileSync(esmSmokeScriptPath, [
|
|
589
|
+
"import assert from 'node:assert/strict';",
|
|
590
|
+
"import * as asl from 'agent-scenario-loop';",
|
|
591
|
+
"import * as demoLoop from 'agent-scenario-loop/runner/demo-loop';",
|
|
592
|
+
"assert.equal(typeof asl.buildRunIndex, 'function');",
|
|
593
|
+
"assert.equal(typeof asl.createArtifactLayout, 'function');",
|
|
594
|
+
"assert.equal(typeof asl.validateJson, 'function');",
|
|
595
|
+
"assert.equal(typeof demoLoop.runDemoLoop, 'function');",
|
|
596
|
+
].join('\n'), 'utf8');
|
|
597
|
+
run(process.execPath, [esmSmokeScriptPath], {
|
|
598
|
+
cwd: installDir,
|
|
599
|
+
env,
|
|
600
|
+
});
|
|
601
|
+
const realInstallDir = fs.realpathSync(installDir);
|
|
602
|
+
const demoOutputDir = path.join(realInstallDir, 'artifacts', 'asl', 'demo');
|
|
603
|
+
const demoOutput = run('npx', [
|
|
604
|
+
'--no-install',
|
|
605
|
+
'asl-demo-loop',
|
|
606
|
+
'--out',
|
|
607
|
+
path.join('artifacts', 'asl', 'demo'),
|
|
608
|
+
], {
|
|
609
|
+
cwd: installDir,
|
|
610
|
+
env,
|
|
611
|
+
});
|
|
612
|
+
const demoResult = JSON.parse(demoOutput);
|
|
613
|
+
assert.equal(demoResult.outputDir, demoOutputDir);
|
|
614
|
+
assert.equal(fs.existsSync(path.join(demoOutputDir, 'preflight', 'app-startup', 'health.json')), true);
|
|
615
|
+
assert.equal(fs.existsSync(path.join(demoOutputDir, 'preflight', 'app-startup', 'agent-summary.md')), true);
|
|
616
|
+
assert.equal(fs.existsSync(path.join(demoOutputDir, 'profile-runs', 'app-startup', 'demo-baseline', 'health.json')), true);
|
|
617
|
+
assert.equal(fs.existsSync(path.join(demoOutputDir, 'profile-runs', 'app-startup', 'demo-current', 'health.json')), true);
|
|
618
|
+
assert.equal(fs.existsSync(path.join(demoOutputDir, 'profile-runs', 'app-startup', 'demo-current', 'comparison.json')), true);
|
|
619
|
+
assert.equal(fs.existsSync(path.join(demoOutputDir, 'profile-runs', 'app-startup', 'demo-current', 'agent-summary.md')), true);
|
|
620
|
+
assert.equal(demoResult.baselineRunDir, path.join(demoOutputDir, 'profile-runs', 'app-startup', 'demo-baseline'));
|
|
621
|
+
assert.equal(demoResult.currentRunDir, path.join(demoOutputDir, 'profile-runs', 'app-startup', 'demo-current'));
|
|
622
|
+
assert.equal(demoResult.preflightDir, path.join(demoOutputDir, 'preflight', 'app-startup'));
|
|
623
|
+
const packedFiles = listFiles(packageRoot);
|
|
624
|
+
assert.equal(packedFiles.includes('examples/mobile-app/scenarios/android/app-startup.json'), true, 'packed package must include Android example app scenarios');
|
|
625
|
+
assert.equal(packedFiles.includes('examples/mobile-app/scenarios/ios/app-startup.json'), true, 'packed package must include iOS example app scenarios');
|
|
626
|
+
const packedExampleScripts = JSON.parse(fs.readFileSync(path.join(packageRoot, 'examples', 'mobile-app', 'asl', 'package-scripts.json'), 'utf8'));
|
|
627
|
+
assert.match(packedExampleScripts['asl:argent:ios'], /ASL_ARGENT_IOS_SIMCTL_SCREENSHOT_FALLBACK/u);
|
|
628
|
+
assert.match(packedExampleScripts['asl:argent:ios'], /ASL_XCRUN_PATH/u);
|
|
629
|
+
const forbiddenPathPatterns = [
|
|
630
|
+
/^\.github\//u,
|
|
631
|
+
/^artifacts\//u,
|
|
632
|
+
/^dist\/.*__tests__\//u,
|
|
633
|
+
/^examples\/mobile-app\/(?:\.expo|android|artifacts|ios|node_modules)(?:\/|$)/u,
|
|
634
|
+
/^node_modules\//u,
|
|
635
|
+
/^runner\/__tests__\//u,
|
|
636
|
+
/^scripts\//u,
|
|
637
|
+
new RegExp(['internal', 'roadmap'].join('-'), 'u'),
|
|
638
|
+
];
|
|
639
|
+
for (const filePath of packedFiles) {
|
|
640
|
+
assert.equal(isAllowedPackedFile(filePath), true, `unexpected public package path: ${filePath}`);
|
|
641
|
+
assert.equal(forbiddenPathPatterns.some((pattern) => pattern.test(filePath)), false, `unexpected file shipped in package: ${filePath}`);
|
|
642
|
+
}
|
|
643
|
+
for (const filePath of packedFiles) {
|
|
644
|
+
const content = fs.readFileSync(path.join(packageRoot, filePath), 'utf8');
|
|
645
|
+
const productName = ['Help', 'Bnk'].join('');
|
|
646
|
+
const productSlug = ['help', 'bnk'].join('');
|
|
647
|
+
const applicationsPath = ['/', 'Applications'].join('');
|
|
648
|
+
const localUserPath = ['/', 'Users', 'sensei'].join('/');
|
|
649
|
+
const privateTempPath = ['private', 'tmp'].join('/');
|
|
650
|
+
const legacyStorageSuffix = ['\\.', 'v', '1', '\\b'].join('');
|
|
651
|
+
const forbiddenContentPattern = new RegExp(`${productName}|${productSlug}|${applicationsPath}|${localUserPath}|${privateTempPath}|${legacyStorageSuffix}`, 'u');
|
|
652
|
+
assert.equal(forbiddenContentPattern.test(content), false, `local or product-specific content leaked into ${filePath}`);
|
|
653
|
+
}
|
|
654
|
+
const scenarioPath = path.join(packageRoot, 'examples', 'scenarios', 'mobile', 'app-startup.json');
|
|
655
|
+
const runnerPath = path.join(packageRoot, 'examples', 'runners', 'xcodebuildmcp-ios.json');
|
|
656
|
+
const exampleAppRoot = path.join(packageRoot, 'examples', 'mobile-app');
|
|
657
|
+
run(packageBinPath(installDir, 'asl-check-plan'), [
|
|
658
|
+
'--scenario',
|
|
659
|
+
scenarioPath,
|
|
660
|
+
'--runner',
|
|
661
|
+
runnerPath,
|
|
662
|
+
'--platform',
|
|
663
|
+
'ios',
|
|
664
|
+
'--run-id',
|
|
665
|
+
'package-smoke',
|
|
666
|
+
'--out',
|
|
667
|
+
artifactDir,
|
|
668
|
+
], {
|
|
669
|
+
cwd: installDir,
|
|
670
|
+
env,
|
|
671
|
+
});
|
|
672
|
+
run(packageBinPath(installDir, 'asl-check-plan'), [
|
|
673
|
+
'--scenario',
|
|
674
|
+
path.join(packageRoot, 'templates', 'mobile-scenario.json'),
|
|
675
|
+
'--runner',
|
|
676
|
+
path.join(packageRoot, 'templates', 'primary-runner.json'),
|
|
677
|
+
'--platform',
|
|
678
|
+
'ios',
|
|
679
|
+
'--run-id',
|
|
680
|
+
'template-smoke',
|
|
681
|
+
'--out',
|
|
682
|
+
path.join(tempRoot, 'template-plan'),
|
|
683
|
+
], {
|
|
684
|
+
cwd: installDir,
|
|
685
|
+
env,
|
|
686
|
+
});
|
|
687
|
+
const packageJson = JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf8'));
|
|
688
|
+
assert.equal(packageJson.private, false);
|
|
689
|
+
assert.equal(packageJson.publishConfig?.access, 'public');
|
|
690
|
+
assert.equal(packageJson.scripts?.prepublishOnly, 'pnpm release:check');
|
|
691
|
+
const initOutputDir = path.join(tempRoot, 'initialized-app');
|
|
692
|
+
const initOutput = run(packageBinPath(installDir, 'asl-init'), [
|
|
693
|
+
'--out',
|
|
694
|
+
initOutputDir,
|
|
695
|
+
'--scenario',
|
|
696
|
+
'Checkout Submit',
|
|
697
|
+
], {
|
|
698
|
+
cwd: installDir,
|
|
699
|
+
env,
|
|
700
|
+
});
|
|
701
|
+
assert.match(initOutput, /Agent Scenario Loop files initialized/u);
|
|
702
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'asl.config.json')), true);
|
|
703
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'scenarios', 'mobile', 'checkout-submit.json')), true);
|
|
704
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'runner-manifests', 'primary-runner.json')), true);
|
|
705
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'runner-manifests', 'evidence-provider.json')), true);
|
|
706
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'asl', 'README.md')), true);
|
|
707
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'asl', 'gitignore-snippet')), true);
|
|
708
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'asl', 'package-scripts.json')), true);
|
|
709
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, '.agents', 'skills', 'agent-scenario-loop', 'SKILL.md')), false);
|
|
710
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'src', 'devtools', 'profile-session.ts')), true);
|
|
711
|
+
const initWithSkillOutputDir = path.join(tempRoot, 'initialized-app-with-skill');
|
|
712
|
+
const initWithSkillOutput = run(packageBinPath(installDir, 'asl-init'), [
|
|
713
|
+
'--out',
|
|
714
|
+
initWithSkillOutputDir,
|
|
715
|
+
'--scenario',
|
|
716
|
+
'Checkout Submit',
|
|
717
|
+
'--with-agent-skill',
|
|
718
|
+
], {
|
|
719
|
+
cwd: installDir,
|
|
720
|
+
env,
|
|
721
|
+
});
|
|
722
|
+
assert.match(initWithSkillOutput, /Agent Scenario Loop files initialized/u);
|
|
723
|
+
assert.equal(fs.existsSync(path.join(initWithSkillOutputDir, '.agents', 'skills', 'agent-scenario-loop', 'SKILL.md')), true);
|
|
724
|
+
assert.equal(fs.existsSync(path.join(initWithSkillOutputDir, '.agents', 'skills', 'agent-scenario-loop', 'references', 'artifact-interpretation.md')), true);
|
|
725
|
+
assert.equal(fs.existsSync(path.join(initWithSkillOutputDir, '.agents', 'skills', 'agent-scenario-loop', 'references', 'adoption-checklist.md')), true);
|
|
726
|
+
assert.match(fs.readFileSync(path.join(initWithSkillOutputDir, '.agents', 'skills', 'agent-scenario-loop', 'SKILL.md'), 'utf8'), /Treat passed health plus failed verdict as trustworthy evidence of failure/u);
|
|
727
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(initOutputDir, 'asl.config.json'), 'utf8')).projectName, 'replace-me');
|
|
728
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(initOutputDir, 'scenarios', 'mobile', 'checkout-submit.json'), 'utf8')).id, 'checkout-submit');
|
|
729
|
+
assert.match(fs.readFileSync(path.join(initOutputDir, 'asl', 'README.md'), 'utf8'), /checkout-submit/u);
|
|
730
|
+
assert.equal(fs.readFileSync(path.join(initOutputDir, 'asl', 'README.md'), 'utf8').includes('--current <run-dir>'), false);
|
|
731
|
+
assert.match(fs.readFileSync(path.join(initOutputDir, 'asl', 'README.md'), 'utf8'), /ASL_COMPARE_ANDROID_CURRENT=/u);
|
|
732
|
+
assert.match(fs.readFileSync(path.join(initOutputDir, 'asl', 'README.md'), 'utf8'), /Keep deterministic validation and live device proof as separate lanes/u);
|
|
733
|
+
assert.match(fs.readFileSync(path.join(initOutputDir, 'asl', 'README.md'), 'utf8'), /host\/device access/u);
|
|
734
|
+
const initializedScripts = JSON.parse(fs.readFileSync(path.join(initOutputDir, 'asl', 'package-scripts.json'), 'utf8'));
|
|
735
|
+
const initializedConfig = JSON.parse(fs.readFileSync(path.join(initOutputDir, 'asl.config.json'), 'utf8'));
|
|
736
|
+
assert.deepEqual(initializedConfig.drivers?.supported, ['fixture-log-ingest', 'adb', 'ios-simctl', 'agent-device', 'argent', 'xcodebuildmcp']);
|
|
737
|
+
fs.writeFileSync(path.join(initOutputDir, 'package.json'), `${JSON.stringify({
|
|
738
|
+
name: 'initialized-app',
|
|
739
|
+
private: true,
|
|
740
|
+
scripts: initializedScripts,
|
|
741
|
+
}, null, 2)}\n`, 'utf8');
|
|
742
|
+
assert.deepEqual(Object.keys(initializedScripts).sort(), [
|
|
743
|
+
'asl:agent-device:android',
|
|
744
|
+
'asl:agent-device:check',
|
|
745
|
+
'asl:agent-device:ios',
|
|
746
|
+
'asl:android:live',
|
|
747
|
+
'asl:android:live:agent-device',
|
|
748
|
+
'asl:android:live:argent',
|
|
749
|
+
'asl:android:live:runners',
|
|
750
|
+
'asl:argent:android',
|
|
751
|
+
'asl:argent:check',
|
|
752
|
+
'asl:argent:ios',
|
|
753
|
+
'asl:check:android',
|
|
754
|
+
'asl:check:ios',
|
|
755
|
+
'asl:compare:android',
|
|
756
|
+
'asl:compare:ios',
|
|
757
|
+
'asl:host:doctor',
|
|
758
|
+
'asl:ios:live',
|
|
759
|
+
'asl:ios:live:agent-device',
|
|
760
|
+
'asl:ios:live:argent',
|
|
761
|
+
'asl:ios:live:runners',
|
|
762
|
+
'asl:live-proof',
|
|
763
|
+
'asl:live-proof:android',
|
|
764
|
+
'asl:live-proof:both',
|
|
765
|
+
'asl:live-proof:ios',
|
|
766
|
+
'asl:profile:android',
|
|
767
|
+
'asl:profile:android:live',
|
|
768
|
+
'asl:profile:android:provider',
|
|
769
|
+
'asl:profile:ios',
|
|
770
|
+
'asl:profile:ios:live',
|
|
771
|
+
'asl:profile:ios:provider',
|
|
772
|
+
'asl:validate',
|
|
773
|
+
]);
|
|
774
|
+
assert.match(initializedScripts['asl:check:ios'], /checkout-submit/u);
|
|
775
|
+
assert.match(initializedScripts['asl:check:ios'], /--provider runner-manifests\/evidence-provider\.json/u);
|
|
776
|
+
assert.match(initializedScripts['asl:check:android'], /--provider runner-manifests\/evidence-provider\.json/u);
|
|
777
|
+
assert.match(initializedScripts['asl:host:doctor'], /^asl-host-doctor --out artifacts\/asl\/host-doctor$/u);
|
|
778
|
+
assert.match(initializedScripts['asl:host:doctor'], /--out artifacts\/asl\/host-doctor/u);
|
|
779
|
+
assert.match(initializedScripts['asl:profile:ios'], /\$\{ASL_PROFILE_IOS_EVENTS:\+--events \$ASL_PROFILE_IOS_EVENTS\}/u);
|
|
780
|
+
assert.match(initializedScripts['asl:profile:ios:provider'], /--provider runner-manifests\/evidence-provider\.json/u);
|
|
781
|
+
assert.match(initializedScripts['asl:profile:ios:provider'], /--comparison-lane checkout-submit-ios-provider/u);
|
|
782
|
+
assert.match(initializedScripts['asl:profile:ios'], /--comparison-lane checkout-submit-ios-fixture/u);
|
|
783
|
+
assert.match(initializedScripts['asl:profile:android'], /\$\{ASL_PROFILE_ANDROID_EVENTS:\+--events \$ASL_PROFILE_ANDROID_EVENTS\}/u);
|
|
784
|
+
assert.match(initializedScripts['asl:profile:android'], /--comparison-lane checkout-submit-android-fixture/u);
|
|
785
|
+
assert.match(initializedScripts['asl:profile:android:provider'], /--provider runner-manifests\/evidence-provider\.json/u);
|
|
786
|
+
assert.match(initializedScripts['asl:profile:android:provider'], /--comparison-lane checkout-submit-android-provider/u);
|
|
787
|
+
assert.match(initializedScripts['asl:agent-device:ios'], /checkout-submit-ios-agent-device/u);
|
|
788
|
+
assert.match(initializedScripts['asl:agent-device:android'], /checkout-submit-android-agent-device/u);
|
|
789
|
+
assert.match(initializedScripts['asl:agent-device:ios'], /ASL_AGENT_DEVICE_COMMAND_TIMEOUT_MS/u);
|
|
790
|
+
assert.match(initializedScripts['asl:agent-device:android'], /ASL_AGENT_DEVICE_COMMAND_TIMEOUT_MS/u);
|
|
791
|
+
assert.match(initializedScripts['asl:agent-device:check'], /^asl-agent-device --check --out artifacts\/asl\/agent-device-check$/u);
|
|
792
|
+
assert.match(initializedScripts['asl:agent-device:check'], /--out artifacts\/asl\/agent-device-check/u);
|
|
793
|
+
assert.match(initializedScripts['asl:argent:check'], /^asl-argent --check --out artifacts\/asl\/argent-check$/u);
|
|
794
|
+
assert.match(initializedScripts['asl:argent:check'], /--out artifacts\/asl\/argent-check/u);
|
|
795
|
+
assert.match(initializedScripts['asl:argent:ios'], /checkout-submit-ios-argent/u);
|
|
796
|
+
assert.match(initializedScripts['asl:argent:android'], /checkout-submit-android-argent/u);
|
|
797
|
+
assert.match(initializedScripts['asl:argent:ios'], /ASL_ARGENT_BASE_ARGS/u);
|
|
798
|
+
assert.match(initializedScripts['asl:argent:android'], /ASL_ARGENT_BASE_ARGS/u);
|
|
799
|
+
assert.match(initializedScripts['asl:argent:ios'], /ASL_ARGENT_COMMAND_TIMEOUT_MS/u);
|
|
800
|
+
assert.match(initializedScripts['asl:argent:android'], /ASL_ARGENT_COMMAND_TIMEOUT_MS/u);
|
|
801
|
+
assert.match(initializedScripts['asl:argent:ios'], /ASL_ARGENT_IOS_SIMCTL_SCREENSHOT_FALLBACK/u);
|
|
802
|
+
assert.match(initializedScripts['asl:argent:ios'], /ASL_XCRUN_PATH/u);
|
|
803
|
+
assert.match(initializedScripts['asl:ios:live'], /^asl-live-ios /u);
|
|
804
|
+
assert.match(initializedScripts['asl:ios:live'], /--scenario scenarios\/mobile\/checkout-submit\.json/u);
|
|
805
|
+
assert.match(initializedScripts['asl:ios:live'], /--compare-latest --fail-on-regression/u);
|
|
806
|
+
assert.match(initializedScripts['asl:android:live'], /^asl-live-android /u);
|
|
807
|
+
assert.match(initializedScripts['asl:android:live'], /--scenario scenarios\/mobile\/checkout-submit\.json/u);
|
|
808
|
+
assert.match(initializedScripts['asl:android:live'], /--compare-latest --fail-on-regression/u);
|
|
809
|
+
assert.match(initializedScripts['asl:ios:live:agent-device'], /ASL_IOS_AGENT_DEVICE_SESSION:\+--agent-device-session/u);
|
|
810
|
+
assert.match(initializedScripts['asl:ios:live:agent-device'], /ASL_IOS_AGENT_DEVICE_SESSION_MODE:\+--agent-device-session-mode/u);
|
|
811
|
+
assert.match(initializedScripts['asl:ios:live:agent-device'], /--run-suffix agent-device --agent-device-proof/u);
|
|
812
|
+
assert.match(initializedScripts['asl:android:live:agent-device'], /ASL_ANDROID_AGENT_DEVICE_SESSION:\+--agent-device-session/u);
|
|
813
|
+
assert.match(initializedScripts['asl:android:live:agent-device'], /ASL_ANDROID_AGENT_DEVICE_SESSION_MODE:\+--agent-device-session-mode/u);
|
|
814
|
+
assert.match(initializedScripts['asl:android:live:agent-device'], /--run-suffix agent-device --agent-device-proof/u);
|
|
815
|
+
assert.match(initializedScripts['asl:ios:live:argent'], /--run-suffix argent --argent-proof/u);
|
|
816
|
+
assert.match(initializedScripts['asl:android:live:argent'], /--run-suffix argent --argent-proof/u);
|
|
817
|
+
assert.match(initializedScripts['asl:ios:live:runners'], /ASL_IOS_AGENT_DEVICE_SESSION:\+--agent-device-session/u);
|
|
818
|
+
assert.match(initializedScripts['asl:ios:live:runners'], /ASL_IOS_AGENT_DEVICE_SESSION_MODE:\+--agent-device-session-mode/u);
|
|
819
|
+
assert.match(initializedScripts['asl:android:live:runners'], /ASL_ANDROID_AGENT_DEVICE_SESSION:\+--agent-device-session/u);
|
|
820
|
+
assert.match(initializedScripts['asl:android:live:runners'], /ASL_ANDROID_AGENT_DEVICE_SESSION_MODE:\+--agent-device-session-mode/u);
|
|
821
|
+
assert.match(initializedScripts['asl:ios:live:runners'], /--run-suffix runners --agent-device-proof --argent-proof --compare-latest --fail-on-regression/u);
|
|
822
|
+
assert.match(initializedScripts['asl:android:live:runners'], /--run-suffix runners --agent-device-proof --argent-proof --compare-latest --fail-on-regression/u);
|
|
823
|
+
assert.equal(Object.values(initializedScripts).some((script) => String(script).startsWith('asl-example-')), false);
|
|
824
|
+
assert.match(initializedScripts['asl:compare:ios'], /--fail-on-regression/u);
|
|
825
|
+
assert.match(initializedScripts['asl:compare:android'], /--fail-on-regression/u);
|
|
826
|
+
assert.match(initializedScripts['asl:compare:ios'], /\$\{ASL_COMPARE_IOS_CURRENT:\?set_ASL_COMPARE_IOS_CURRENT\}/u);
|
|
827
|
+
assert.match(initializedScripts['asl:compare:android'], /\$\{ASL_COMPARE_ANDROID_CURRENT:\?set_ASL_COMPARE_ANDROID_CURRENT\}/u);
|
|
828
|
+
assert.match(initializedScripts['asl:live-proof:ios'], /artifacts\/asl\/ios-live\/_live-proof\/ios-live-proof\/live-proof\.json/u);
|
|
829
|
+
assert.match(initializedScripts['asl:live-proof:android'], /artifacts\/asl\/android-live\/_live-proof\/android-live-proof\/live-proof\.json/u);
|
|
830
|
+
assert.match(initializedScripts['asl:live-proof:both'], /--require-platforms android,ios --out artifacts\/asl\/live-proof-set --fail-on-regression/u);
|
|
831
|
+
assert.match(initializedScripts['asl:live-proof'], /\$\{ASL_LIVE_PROOF:\?set_ASL_LIVE_PROOF\}/u);
|
|
832
|
+
assert.equal(Object.values(initializedScripts).some((script) => String(script).includes('<run-dir>')), false);
|
|
833
|
+
assert.equal(Object.values(initializedScripts).some((script) => String(script).includes('<live-proof.json>')), false);
|
|
834
|
+
assert.match(initializedScripts['asl:profile:ios:live'], /checkout-submit-ios-live/u);
|
|
835
|
+
assert.match(initializedScripts['asl:profile:ios:live'], /--comparison-lane checkout-submit-ios-live/u);
|
|
836
|
+
assert.match(initializedScripts['asl:profile:android:live'], /checkout-submit-android-live/u);
|
|
837
|
+
assert.match(initializedScripts['asl:profile:android:live'], /--comparison-lane checkout-submit-android-live/u);
|
|
838
|
+
assert.match(fs.readFileSync(path.join(initOutputDir, 'asl', 'gitignore-snippet'), 'utf8'), /artifacts\/asl\//u);
|
|
839
|
+
assert.match(fs.readFileSync(path.join(initOutputDir, 'asl', 'gitignore-snippet'), 'utf8'), /\.asl\.local\.env/u);
|
|
840
|
+
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(initOutputDir, 'runner-manifests', 'evidence-provider.json'), 'utf8')).capabilities, ['accessibility', 'memory', 'network', 'profiler']);
|
|
841
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'scripts', 'asl-capture-accessibility-provider.mjs')), true);
|
|
842
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'scripts', 'asl-capture-profiler-provider.mjs')), true);
|
|
843
|
+
for (const platform of ['ios', 'android']) {
|
|
844
|
+
run(packageBinPath(installDir, 'asl-check-plan'), [
|
|
845
|
+
'--scenario',
|
|
846
|
+
path.join(initOutputDir, 'scenarios', 'mobile', 'checkout-submit.json'),
|
|
847
|
+
'--runner',
|
|
848
|
+
path.join(initOutputDir, 'runner-manifests', 'primary-runner.json'),
|
|
849
|
+
'--platform',
|
|
850
|
+
platform,
|
|
851
|
+
'--run-id',
|
|
852
|
+
`initialized-${platform}`,
|
|
853
|
+
'--out',
|
|
854
|
+
path.join(initOutputDir, 'artifacts', 'asl', 'plan', `checkout-submit-${platform}`),
|
|
855
|
+
], {
|
|
856
|
+
cwd: initOutputDir,
|
|
857
|
+
env,
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
const initializedValidationOutput = run(packageBinPath(installDir, 'asl-validate-project'), [
|
|
861
|
+
'--root',
|
|
862
|
+
initOutputDir,
|
|
863
|
+
'--platform',
|
|
864
|
+
'all',
|
|
865
|
+
'--out',
|
|
866
|
+
path.join(initOutputDir, 'artifacts', 'asl', 'project-validation'),
|
|
867
|
+
], {
|
|
868
|
+
cwd: initOutputDir,
|
|
869
|
+
env,
|
|
870
|
+
});
|
|
871
|
+
assert.match(initializedValidationOutput, /project validation passed/u);
|
|
872
|
+
assert.equal(fs.existsSync(path.join(initOutputDir, 'artifacts', 'asl', 'project-validation', 'project-validation.json')), true);
|
|
873
|
+
const initializedValidation = JSON.parse(fs.readFileSync(path.join(initOutputDir, 'artifacts', 'asl', 'project-validation', 'project-validation.json'), 'utf8'));
|
|
874
|
+
assert.equal(initializedValidation.scripts.status, 'present');
|
|
875
|
+
assert.equal(initializedValidation.scripts.packageJsonStatus, 'present');
|
|
876
|
+
assert.deepEqual(initializedValidation.config.customDrivers, []);
|
|
877
|
+
assert.deepEqual(initializedValidation.config.externalTargetDrivers, ['xcodebuildmcp']);
|
|
878
|
+
assert.deepEqual(initializedValidation.config.packageSupportedDrivers, ['adb', 'agent-device', 'argent', 'fixture-log-ingest', 'ios-simctl']);
|
|
879
|
+
assert.deepEqual(initializedValidation.config.supportedDrivers, ['adb', 'agent-device', 'argent', 'fixture-log-ingest', 'ios-simctl', 'xcodebuildmcp']);
|
|
880
|
+
assert.deepEqual(initializedValidation.config.missingSupportedDrivers, []);
|
|
881
|
+
assert.equal(initializedValidation.nextActions.some((action) => action.code === 'replace_config_placeholders'), true);
|
|
882
|
+
assert.equal(initializedValidation.nextActions.some((action) => action.code === 'ignore_runtime_artifacts'), true);
|
|
883
|
+
assert.equal(initializedValidation.warnings.some((warning) => warning.includes('projectName')), true);
|
|
884
|
+
assert.equal(initializedValidation.warnings.some((warning) => warning.includes('Runtime artifact gitignore')), true);
|
|
885
|
+
for (const binaryName of Object.keys(packageJson.bin).sort()) {
|
|
886
|
+
const helpText = run(packageBinPath(installDir, binaryName), ['--help'], {
|
|
887
|
+
cwd: installDir,
|
|
888
|
+
env,
|
|
889
|
+
});
|
|
890
|
+
assert.match(helpText, /Usage:/u, `${binaryName} did not print usage`);
|
|
891
|
+
if (binaryName === 'asl-android-adb' || binaryName === 'asl-profile-android') {
|
|
892
|
+
assert.match(helpText, /--launch-wait-ms <ms>/u, `${binaryName} did not expose Android launch wait help`);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
const fakeArgentPath = path.join(tempRoot, process.platform === 'win32' ? 'fake-argent.cmd' : 'fake-argent');
|
|
896
|
+
const argentRunRoot = path.join(tempRoot, 'argent-run');
|
|
897
|
+
writeFakeArgent(fakeArgentPath);
|
|
898
|
+
const argentRunOutput = run(packageBinPath(installDir, 'asl-argent'), [
|
|
899
|
+
'--argent',
|
|
900
|
+
fakeArgentPath,
|
|
901
|
+
'--platform',
|
|
902
|
+
'ios',
|
|
903
|
+
'--scenario',
|
|
904
|
+
path.join(exampleAppRoot, 'scenarios', 'mobile', 'app-startup.json'),
|
|
905
|
+
'--app',
|
|
906
|
+
'dev.agent-scenario-loop.example',
|
|
907
|
+
'--device',
|
|
908
|
+
'SIM-123',
|
|
909
|
+
'--out',
|
|
910
|
+
argentRunRoot,
|
|
911
|
+
'--run-id',
|
|
912
|
+
'package-smoke-argent',
|
|
913
|
+
], {
|
|
914
|
+
cwd: installDir,
|
|
915
|
+
env,
|
|
916
|
+
});
|
|
917
|
+
const argentRunDir = argentRunOutput.trim();
|
|
918
|
+
assert.equal(argentRunDir, argentRunRoot);
|
|
919
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(argentRunDir, 'health.json'), 'utf8')).healthStatus, 'passed');
|
|
920
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(argentRunDir, 'verdict.json'), 'utf8')).verdictStatus, 'not_evaluated');
|
|
921
|
+
assert.equal(fs.existsSync(path.join(argentRunDir, 'raw', 'argent-metadata.json')), true);
|
|
922
|
+
assert.equal(fs.existsSync(path.join(argentRunDir, 'captures', 'fake-argent-screenshot.png')), true);
|
|
923
|
+
for (const exampleRun of EXAMPLE_PROFILE_RUNS) {
|
|
924
|
+
const output = run(packageBinPath(installDir, exampleRun.binaryName), [
|
|
925
|
+
'--config',
|
|
926
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
927
|
+
'--scenario',
|
|
928
|
+
path.join(exampleAppRoot, 'scenarios', 'mobile', exampleRun.scenario),
|
|
929
|
+
'--events',
|
|
930
|
+
path.join(exampleAppRoot, 'event-logs', exampleRun.eventLog),
|
|
931
|
+
'--out',
|
|
932
|
+
path.join(tempRoot, 'example-mobile-app-artifacts', exampleRun.platform),
|
|
933
|
+
'--run-id',
|
|
934
|
+
exampleRun.runId,
|
|
935
|
+
], {
|
|
936
|
+
cwd: installDir,
|
|
937
|
+
env,
|
|
938
|
+
});
|
|
939
|
+
const runDir = output.trim();
|
|
940
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(runDir, 'manifest.json'), 'utf8'));
|
|
941
|
+
const health = JSON.parse(fs.readFileSync(path.join(runDir, 'health.json'), 'utf8'));
|
|
942
|
+
const verdict = JSON.parse(fs.readFileSync(path.join(runDir, 'verdict.json'), 'utf8'));
|
|
943
|
+
assert.equal(manifest.platform, exampleRun.platform);
|
|
944
|
+
assert.equal(health.healthStatus, 'passed');
|
|
945
|
+
assert.equal(verdict.verdictStatus, 'passed');
|
|
946
|
+
}
|
|
947
|
+
const providerEvidenceRoot = path.join(tempRoot, 'provider-evidence');
|
|
948
|
+
const providerProfileRoot = path.join(tempRoot, 'provider-evidence-profile');
|
|
949
|
+
fs.mkdirSync(providerEvidenceRoot, { recursive: true });
|
|
950
|
+
fs.writeFileSync(path.join(providerEvidenceRoot, 'js-profile.json'), '{"samples":[]}\n', 'utf8');
|
|
951
|
+
fs.writeFileSync(path.join(providerEvidenceRoot, 'network-capture.har'), '{"log":{"entries":[]}}\n', 'utf8');
|
|
952
|
+
fs.writeFileSync(path.join(providerEvidenceRoot, 'ui-tree.json'), '{"tree":[]}\n', 'utf8');
|
|
953
|
+
const providerProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
954
|
+
'--config',
|
|
955
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
956
|
+
'--scenario',
|
|
957
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
958
|
+
'--events',
|
|
959
|
+
path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'),
|
|
960
|
+
'--signal',
|
|
961
|
+
`js:${path.join(providerEvidenceRoot, 'js-profile.json')}`,
|
|
962
|
+
'--signal',
|
|
963
|
+
`network:${path.join(providerEvidenceRoot, 'network-capture.har')}`,
|
|
964
|
+
'--capture',
|
|
965
|
+
`uiTree:${path.join(providerEvidenceRoot, 'ui-tree.json')}`,
|
|
966
|
+
'--out',
|
|
967
|
+
providerProfileRoot,
|
|
968
|
+
'--run-id',
|
|
969
|
+
'android-example-startup',
|
|
970
|
+
], {
|
|
971
|
+
cwd: installDir,
|
|
972
|
+
env,
|
|
973
|
+
});
|
|
974
|
+
const providerProfileRunDir = providerProfileOutput.trim();
|
|
975
|
+
const providerManifest = JSON.parse(fs.readFileSync(path.join(providerProfileRunDir, 'manifest.json'), 'utf8'));
|
|
976
|
+
const providerJsSignal = path.join(providerEvidenceRoot, 'js-profile.json');
|
|
977
|
+
const providerNetworkSignal = path.join(providerEvidenceRoot, 'network-capture.har');
|
|
978
|
+
const providerUiTree = path.join(providerEvidenceRoot, 'ui-tree.json');
|
|
979
|
+
assert.deepEqual(providerManifest.artifacts.signals.js, ['signals/js/js-profile.json']);
|
|
980
|
+
assert.deepEqual(providerManifest.artifacts.signals.network, ['signals/network/network-capture.har']);
|
|
981
|
+
assert.equal(providerManifest.artifacts.captures.uiTree, 'captures/ui-tree.json');
|
|
982
|
+
assert.deepEqual(providerManifest.artifacts.evidenceAttachments, [
|
|
983
|
+
{
|
|
984
|
+
channel: 'signal',
|
|
985
|
+
completenessStatus: 'complete',
|
|
986
|
+
corruptionStatus: 'valid',
|
|
987
|
+
kind: 'js',
|
|
988
|
+
path: 'signals/js/js-profile.json',
|
|
989
|
+
redactionStatus: 'not-redacted',
|
|
990
|
+
sha256: sha256File(providerJsSignal),
|
|
991
|
+
sizeBytes: fs.statSync(providerJsSignal).size,
|
|
992
|
+
sourceFileName: 'js-profile.json',
|
|
993
|
+
transformations: ['copied'],
|
|
994
|
+
},
|
|
995
|
+
{
|
|
996
|
+
channel: 'signal',
|
|
997
|
+
completenessStatus: 'complete',
|
|
998
|
+
corruptionStatus: 'valid',
|
|
999
|
+
kind: 'network',
|
|
1000
|
+
path: 'signals/network/network-capture.har',
|
|
1001
|
+
redactionStatus: 'not-redacted',
|
|
1002
|
+
sha256: sha256File(providerNetworkSignal),
|
|
1003
|
+
sizeBytes: fs.statSync(providerNetworkSignal).size,
|
|
1004
|
+
sourceFileName: 'network-capture.har',
|
|
1005
|
+
transformations: ['copied'],
|
|
1006
|
+
},
|
|
1007
|
+
{
|
|
1008
|
+
channel: 'capture',
|
|
1009
|
+
completenessStatus: 'complete',
|
|
1010
|
+
corruptionStatus: 'valid',
|
|
1011
|
+
kind: 'uiTree',
|
|
1012
|
+
path: 'captures/ui-tree.json',
|
|
1013
|
+
redactionStatus: 'not-redacted',
|
|
1014
|
+
sha256: sha256File(providerUiTree),
|
|
1015
|
+
sizeBytes: fs.statSync(providerUiTree).size,
|
|
1016
|
+
sourceFileName: 'ui-tree.json',
|
|
1017
|
+
transformations: ['copied'],
|
|
1018
|
+
},
|
|
1019
|
+
]);
|
|
1020
|
+
assert.equal(fs.existsSync(path.join(providerProfileRunDir, 'signals', 'js', 'js-profile.json')), true);
|
|
1021
|
+
assert.equal(fs.existsSync(path.join(providerProfileRunDir, 'signals', 'network', 'network-capture.har')), true);
|
|
1022
|
+
assert.equal(fs.existsSync(path.join(providerProfileRunDir, 'captures', 'ui-tree.json')), true);
|
|
1023
|
+
const providerCommandRoot = path.join(tempRoot, 'provider-command');
|
|
1024
|
+
const providerCommandProfileRoot = path.join(tempRoot, 'provider-command-profile');
|
|
1025
|
+
fs.mkdirSync(providerCommandRoot, { recursive: true });
|
|
1026
|
+
const providerCommandScript = path.join(providerCommandRoot, 'write-accessibility.js');
|
|
1027
|
+
fs.writeFileSync(providerCommandScript, [
|
|
1028
|
+
"const fs = require('node:fs');",
|
|
1029
|
+
"const path = require('node:path');",
|
|
1030
|
+
"const outputPath = process.argv[2];",
|
|
1031
|
+
"fs.mkdirSync(path.dirname(outputPath), { recursive: true });",
|
|
1032
|
+
"fs.writeFileSync(outputPath, JSON.stringify({ violations: [] }) + '\\n');",
|
|
1033
|
+
].join('\n'));
|
|
1034
|
+
const providerCommandManifestPath = path.join(providerCommandRoot, 'provider.json');
|
|
1035
|
+
fs.writeFileSync(providerCommandManifestPath, `${JSON.stringify({
|
|
1036
|
+
schemaVersion: '1.0.0',
|
|
1037
|
+
runnerId: 'smoke-accessibility-provider',
|
|
1038
|
+
kind: 'evidenceProvider',
|
|
1039
|
+
platforms: ['android'],
|
|
1040
|
+
capabilities: ['accessibility'],
|
|
1041
|
+
artifactOutputs: ['accessibility'],
|
|
1042
|
+
lifecycle: ['capture'],
|
|
1043
|
+
providerCommands: [
|
|
1044
|
+
{
|
|
1045
|
+
id: 'capture-accessibility',
|
|
1046
|
+
phase: 'capture',
|
|
1047
|
+
command: process.execPath,
|
|
1048
|
+
args: [providerCommandScript, '{providerDir}/accessibility.json'],
|
|
1049
|
+
outputs: [
|
|
1050
|
+
{
|
|
1051
|
+
channel: 'provider',
|
|
1052
|
+
kind: 'accessibility',
|
|
1053
|
+
path: '{providerDir}/accessibility.json',
|
|
1054
|
+
},
|
|
1055
|
+
],
|
|
1056
|
+
},
|
|
1057
|
+
],
|
|
1058
|
+
}, null, 2)}\n`);
|
|
1059
|
+
const providerCommandProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1060
|
+
'--config',
|
|
1061
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1062
|
+
'--scenario',
|
|
1063
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
1064
|
+
'--events',
|
|
1065
|
+
path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'),
|
|
1066
|
+
'--provider',
|
|
1067
|
+
providerCommandManifestPath,
|
|
1068
|
+
'--out',
|
|
1069
|
+
providerCommandProfileRoot,
|
|
1070
|
+
'--run-id',
|
|
1071
|
+
'android-provider-command',
|
|
1072
|
+
], {
|
|
1073
|
+
cwd: installDir,
|
|
1074
|
+
env,
|
|
1075
|
+
});
|
|
1076
|
+
const providerCommandRunDir = providerCommandProfileOutput.trim();
|
|
1077
|
+
const providerCommandManifest = JSON.parse(fs.readFileSync(path.join(providerCommandRunDir, 'manifest.json'), 'utf8'));
|
|
1078
|
+
const providerCommandOutput = path.join(providerCommandRunDir, 'raw', 'providers', 'smoke-accessibility-provider', 'accessibility.json');
|
|
1079
|
+
assert.equal(fs.existsSync(providerCommandOutput), true);
|
|
1080
|
+
assert.equal(fs.existsSync(path.join(providerCommandRunDir, 'raw', 'provider-commands', 'smoke-accessibility-provider-capture-accessibility.json')), true);
|
|
1081
|
+
assert.deepEqual(providerCommandManifest.artifacts.evidenceAttachments, [
|
|
1082
|
+
{
|
|
1083
|
+
channel: 'provider',
|
|
1084
|
+
completenessStatus: 'complete',
|
|
1085
|
+
corruptionStatus: 'valid',
|
|
1086
|
+
kind: 'accessibility',
|
|
1087
|
+
path: 'raw/providers/smoke-accessibility-provider/accessibility.json',
|
|
1088
|
+
redactionStatus: 'not-redacted',
|
|
1089
|
+
sha256: sha256File(providerCommandOutput),
|
|
1090
|
+
sizeBytes: fs.statSync(providerCommandOutput).size,
|
|
1091
|
+
sourceFileName: 'accessibility.json',
|
|
1092
|
+
transformations: ['copied'],
|
|
1093
|
+
},
|
|
1094
|
+
]);
|
|
1095
|
+
const unsupportedProviderRoot = path.join(tempRoot, 'provider-command-unsupported-platform');
|
|
1096
|
+
const unsupportedProviderProfileRoot = path.join(tempRoot, 'provider-command-unsupported-platform-profile');
|
|
1097
|
+
fs.mkdirSync(unsupportedProviderRoot, { recursive: true });
|
|
1098
|
+
const unsupportedProviderMarkerPath = path.join(unsupportedProviderRoot, 'provider-ran.txt');
|
|
1099
|
+
const unsupportedProviderManifestPath = path.join(unsupportedProviderRoot, 'provider.json');
|
|
1100
|
+
fs.writeFileSync(unsupportedProviderManifestPath, `${JSON.stringify({
|
|
1101
|
+
schemaVersion: '1.0.0',
|
|
1102
|
+
runnerId: 'smoke-ios-only-provider',
|
|
1103
|
+
kind: 'evidenceProvider',
|
|
1104
|
+
platforms: ['ios'],
|
|
1105
|
+
capabilities: ['accessibility'],
|
|
1106
|
+
artifactOutputs: ['accessibility'],
|
|
1107
|
+
lifecycle: ['capture'],
|
|
1108
|
+
providerCommands: [
|
|
1109
|
+
{
|
|
1110
|
+
id: 'capture-accessibility',
|
|
1111
|
+
phase: 'capture',
|
|
1112
|
+
command: process.execPath,
|
|
1113
|
+
args: ['-e', `require('node:fs').writeFileSync(${JSON.stringify(unsupportedProviderMarkerPath)}, 'ran\\n')`],
|
|
1114
|
+
outputs: [
|
|
1115
|
+
{
|
|
1116
|
+
channel: 'provider',
|
|
1117
|
+
kind: 'accessibility',
|
|
1118
|
+
path: '{providerDir}/accessibility.json',
|
|
1119
|
+
},
|
|
1120
|
+
],
|
|
1121
|
+
},
|
|
1122
|
+
],
|
|
1123
|
+
}, null, 2)}\n`);
|
|
1124
|
+
const unsupportedProviderProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1125
|
+
'--config',
|
|
1126
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1127
|
+
'--scenario',
|
|
1128
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
1129
|
+
'--events',
|
|
1130
|
+
path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'),
|
|
1131
|
+
'--provider',
|
|
1132
|
+
unsupportedProviderManifestPath,
|
|
1133
|
+
'--out',
|
|
1134
|
+
unsupportedProviderProfileRoot,
|
|
1135
|
+
'--run-id',
|
|
1136
|
+
'android-provider-unsupported-platform',
|
|
1137
|
+
], {
|
|
1138
|
+
cwd: installDir,
|
|
1139
|
+
env,
|
|
1140
|
+
});
|
|
1141
|
+
const unsupportedProviderRunDir = unsupportedProviderProfileOutput.trim();
|
|
1142
|
+
const unsupportedProviderHealth = JSON.parse(fs.readFileSync(path.join(unsupportedProviderRunDir, 'health.json'), 'utf8'));
|
|
1143
|
+
const unsupportedProviderSummary = fs.readFileSync(path.join(unsupportedProviderRunDir, 'agent-summary.md'), 'utf8');
|
|
1144
|
+
assert.equal(fs.existsSync(unsupportedProviderMarkerPath), false);
|
|
1145
|
+
assert.equal(unsupportedProviderHealth.healthStatus, 'failed');
|
|
1146
|
+
assert.equal(unsupportedProviderHealth.checks.some((check) => check.code === 'provider_platform_unsupported' &&
|
|
1147
|
+
check.metadata?.nextActionCode === 'select_supported_provider_platform' &&
|
|
1148
|
+
check.metadata?.providerId === 'smoke-ios-only-provider'), true);
|
|
1149
|
+
assert.equal(fs.existsSync(path.join(unsupportedProviderRunDir, 'raw', 'provider-commands', 'smoke-ios-only-provider-capture-accessibility.json')), false);
|
|
1150
|
+
assert.match(unsupportedProviderSummary, /Next action `select_supported_provider_platform`/u);
|
|
1151
|
+
const failingProviderCommandRoot = path.join(tempRoot, 'provider-command-failure');
|
|
1152
|
+
const failingProviderCommandProfileRoot = path.join(tempRoot, 'provider-command-failure-profile');
|
|
1153
|
+
fs.mkdirSync(failingProviderCommandRoot, { recursive: true });
|
|
1154
|
+
const failingProviderCommandScript = path.join(failingProviderCommandRoot, 'fail-provider.js');
|
|
1155
|
+
fs.writeFileSync(failingProviderCommandScript, "process.stderr.write('provider unavailable\\n'); process.exit(7);\n");
|
|
1156
|
+
const failingProviderCommandManifestPath = path.join(failingProviderCommandRoot, 'provider.json');
|
|
1157
|
+
fs.writeFileSync(failingProviderCommandManifestPath, `${JSON.stringify({
|
|
1158
|
+
schemaVersion: '1.0.0',
|
|
1159
|
+
runnerId: 'smoke-failing-provider',
|
|
1160
|
+
kind: 'evidenceProvider',
|
|
1161
|
+
platforms: ['android'],
|
|
1162
|
+
capabilities: ['accessibility'],
|
|
1163
|
+
artifactOutputs: ['accessibility'],
|
|
1164
|
+
lifecycle: ['capture'],
|
|
1165
|
+
providerCommands: [
|
|
1166
|
+
{
|
|
1167
|
+
id: 'capture-accessibility',
|
|
1168
|
+
phase: 'capture',
|
|
1169
|
+
command: process.execPath,
|
|
1170
|
+
args: [failingProviderCommandScript],
|
|
1171
|
+
outputs: [
|
|
1172
|
+
{
|
|
1173
|
+
channel: 'provider',
|
|
1174
|
+
kind: 'accessibility',
|
|
1175
|
+
path: '{providerDir}/accessibility.json',
|
|
1176
|
+
},
|
|
1177
|
+
],
|
|
1178
|
+
},
|
|
1179
|
+
],
|
|
1180
|
+
}, null, 2)}\n`);
|
|
1181
|
+
const failingProviderCommandProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1182
|
+
'--config',
|
|
1183
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1184
|
+
'--scenario',
|
|
1185
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
1186
|
+
'--events',
|
|
1187
|
+
path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'),
|
|
1188
|
+
'--provider',
|
|
1189
|
+
failingProviderCommandManifestPath,
|
|
1190
|
+
'--out',
|
|
1191
|
+
failingProviderCommandProfileRoot,
|
|
1192
|
+
'--run-id',
|
|
1193
|
+
'android-provider-failure',
|
|
1194
|
+
], {
|
|
1195
|
+
cwd: installDir,
|
|
1196
|
+
env,
|
|
1197
|
+
});
|
|
1198
|
+
const failingProviderCommandRunDir = failingProviderCommandProfileOutput.trim();
|
|
1199
|
+
const failingProviderCommandHealth = JSON.parse(fs.readFileSync(path.join(failingProviderCommandRunDir, 'health.json'), 'utf8'));
|
|
1200
|
+
const failingProviderCommandSummary = fs.readFileSync(path.join(failingProviderCommandRunDir, 'agent-summary.md'), 'utf8');
|
|
1201
|
+
const failingProviderCommandRecord = JSON.parse(fs.readFileSync(path.join(failingProviderCommandRunDir, 'raw', 'provider-commands', 'smoke-failing-provider-capture-accessibility.json'), 'utf8'));
|
|
1202
|
+
assert.equal(failingProviderCommandHealth.healthStatus, 'failed');
|
|
1203
|
+
assert.equal(failingProviderCommandRecord.exitCode, 7);
|
|
1204
|
+
assert.equal(failingProviderCommandHealth.checks.some((check) => check.code === 'provider_command_failed' && check.metadata?.nextActionCode === 'fix_provider_command'), true);
|
|
1205
|
+
assert.match(failingProviderCommandSummary, /Next action `fix_provider_command`/u);
|
|
1206
|
+
const adbArtifactRoot = path.join(tempRoot, 'adb-artifacts');
|
|
1207
|
+
fs.mkdirSync(path.join(adbArtifactRoot, 'raw'), { recursive: true });
|
|
1208
|
+
fs.copyFileSync(path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'), path.join(adbArtifactRoot, 'raw', 'adb-logcat.txt'));
|
|
1209
|
+
const adbArtifactProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1210
|
+
'--config',
|
|
1211
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1212
|
+
'--scenario',
|
|
1213
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
1214
|
+
'--adb-artifacts',
|
|
1215
|
+
adbArtifactRoot,
|
|
1216
|
+
'--out',
|
|
1217
|
+
path.join(tempRoot, 'example-mobile-app-adb-artifact-profile'),
|
|
1218
|
+
'--run-id',
|
|
1219
|
+
'android-example-startup',
|
|
1220
|
+
], {
|
|
1221
|
+
cwd: installDir,
|
|
1222
|
+
env,
|
|
1223
|
+
});
|
|
1224
|
+
const adbArtifactProfileRunDir = adbArtifactProfileOutput.trim();
|
|
1225
|
+
const adbArtifactManifest = JSON.parse(fs.readFileSync(path.join(adbArtifactProfileRunDir, 'manifest.json'), 'utf8'));
|
|
1226
|
+
const adbArtifactCausalRun = JSON.parse(fs.readFileSync(path.join(adbArtifactProfileRunDir, 'causal-run.json'), 'utf8'));
|
|
1227
|
+
const adbArtifactHealth = JSON.parse(fs.readFileSync(path.join(adbArtifactProfileRunDir, 'health.json'), 'utf8'));
|
|
1228
|
+
const adbArtifactVerdict = JSON.parse(fs.readFileSync(path.join(adbArtifactProfileRunDir, 'verdict.json'), 'utf8'));
|
|
1229
|
+
assert.equal(adbArtifactManifest.artifacts.raw.interactionLog, 'raw/adb-logcat.txt');
|
|
1230
|
+
assert.equal(adbArtifactManifest.interactionDriver, 'adb-logcat');
|
|
1231
|
+
assert.equal(adbArtifactCausalRun.scenario.driver, 'adb-logcat');
|
|
1232
|
+
assert.equal(adbArtifactHealth.healthStatus, 'passed');
|
|
1233
|
+
assert.equal(adbArtifactVerdict.verdictStatus, 'passed');
|
|
1234
|
+
const adbCaptureRunId = 'package-smoke-adb-capture';
|
|
1235
|
+
const androidPackageName = 'dev.agentscenarioloop.example';
|
|
1236
|
+
const fakeAdbPath = path.join(tempRoot, process.platform === 'win32' ? 'fake-adb.cmd' : 'fake-adb');
|
|
1237
|
+
const adbCaptureProfileRoot = path.join(tempRoot, 'adb-capture-profile');
|
|
1238
|
+
writeFakeAdb({
|
|
1239
|
+
filePath: fakeAdbPath,
|
|
1240
|
+
logcatText: fs
|
|
1241
|
+
.readFileSync(path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'), 'utf8')
|
|
1242
|
+
.replace(/android-example-startup/gu, adbCaptureRunId),
|
|
1243
|
+
packageName: androidPackageName,
|
|
1244
|
+
});
|
|
1245
|
+
const adbCaptureProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1246
|
+
'--config',
|
|
1247
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1248
|
+
'--scenario',
|
|
1249
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
1250
|
+
'--adb-capture',
|
|
1251
|
+
'--adb',
|
|
1252
|
+
fakeAdbPath,
|
|
1253
|
+
'--clear-logcat',
|
|
1254
|
+
'--launch',
|
|
1255
|
+
'--wait-ms',
|
|
1256
|
+
'1',
|
|
1257
|
+
'--out',
|
|
1258
|
+
adbCaptureProfileRoot,
|
|
1259
|
+
'--run-id',
|
|
1260
|
+
adbCaptureRunId,
|
|
1261
|
+
], {
|
|
1262
|
+
cwd: installDir,
|
|
1263
|
+
env,
|
|
1264
|
+
});
|
|
1265
|
+
const adbCaptureProfileRunDir = adbCaptureProfileOutput.trim();
|
|
1266
|
+
const adbCaptureRoot = path.join(adbCaptureProfileRoot, '_adb-captures', adbCaptureRunId);
|
|
1267
|
+
const adbCaptureManifest = JSON.parse(fs.readFileSync(path.join(adbCaptureProfileRunDir, 'manifest.json'), 'utf8'));
|
|
1268
|
+
const adbCaptureCausalRun = JSON.parse(fs.readFileSync(path.join(adbCaptureProfileRunDir, 'causal-run.json'), 'utf8'));
|
|
1269
|
+
const adbCaptureHealth = JSON.parse(fs.readFileSync(path.join(adbCaptureProfileRunDir, 'health.json'), 'utf8'));
|
|
1270
|
+
const adbCaptureVerdict = JSON.parse(fs.readFileSync(path.join(adbCaptureProfileRunDir, 'verdict.json'), 'utf8'));
|
|
1271
|
+
const adbCaptureRunnerHealth = JSON.parse(fs.readFileSync(path.join(adbCaptureRoot, 'health.json'), 'utf8'));
|
|
1272
|
+
assert.equal(adbCaptureProfileRunDir, path.join(adbCaptureProfileRoot, 'app-startup', adbCaptureRunId));
|
|
1273
|
+
assert.equal(adbCaptureManifest.artifacts.raw.interactionLog, 'raw/adb-logcat.txt');
|
|
1274
|
+
assert.equal(adbCaptureManifest.bundleId, androidPackageName);
|
|
1275
|
+
assert.equal(adbCaptureManifest.interactionDriver, 'adb-logcat');
|
|
1276
|
+
assert.equal(adbCaptureCausalRun.scenario.driver, 'adb-logcat');
|
|
1277
|
+
assert.equal(adbCaptureHealth.healthStatus, 'passed');
|
|
1278
|
+
assert.equal(adbCaptureVerdict.verdictStatus, 'passed');
|
|
1279
|
+
assert.equal(adbCaptureRunnerHealth.healthStatus, 'passed');
|
|
1280
|
+
assert.equal(fs.existsSync(path.join(adbCaptureProfileRunDir, 'raw', 'adb-logcat.txt')), true);
|
|
1281
|
+
assert.equal(fs.existsSync(path.join(adbCaptureRoot, 'raw', 'adb-app-pidof-after-launch.txt')), true);
|
|
1282
|
+
assert.equal(fs.existsSync(path.join(adbCaptureRoot, 'raw', 'adb-app-pidof-after-capture.txt')), true);
|
|
1283
|
+
assert.equal(fs.existsSync(path.join(adbCaptureRoot, 'raw', 'adb-app-lifecycle-log.txt')), true);
|
|
1284
|
+
const adbDriverStepRunId = 'package-smoke-adb-driver-step';
|
|
1285
|
+
const fakeAdbDriverStepPath = path.join(tempRoot, process.platform === 'win32' ? 'fake-adb-driver-step.cmd' : 'fake-adb-driver-step');
|
|
1286
|
+
const adbDriverStepProfileRoot = path.join(tempRoot, 'adb-driver-step-profile');
|
|
1287
|
+
const adbDriverStepScenarioPath = path.join(tempRoot, 'android-app-startup-readlogs.json');
|
|
1288
|
+
const adbDriverStepScenario = JSON.parse(fs.readFileSync(path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'), 'utf8'));
|
|
1289
|
+
adbDriverStepScenario.steps = [
|
|
1290
|
+
{
|
|
1291
|
+
id: 'capture-log-window',
|
|
1292
|
+
kind: 'captureEvidence',
|
|
1293
|
+
artifact: 'logs',
|
|
1294
|
+
driverAction: 'readLogs',
|
|
1295
|
+
adapterOptions: {
|
|
1296
|
+
androidAdb: {
|
|
1297
|
+
logcatLines: 25,
|
|
1298
|
+
rawFileName: 'adb-logcat.txt',
|
|
1299
|
+
},
|
|
1300
|
+
},
|
|
1301
|
+
},
|
|
1302
|
+
];
|
|
1303
|
+
fs.writeFileSync(adbDriverStepScenarioPath, `${JSON.stringify(adbDriverStepScenario, null, 2)}\n`, 'utf8');
|
|
1304
|
+
writeFakeAdb({
|
|
1305
|
+
filePath: fakeAdbDriverStepPath,
|
|
1306
|
+
logcatText: fs
|
|
1307
|
+
.readFileSync(path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'), 'utf8')
|
|
1308
|
+
.replace(/android-example-startup/gu, adbDriverStepRunId),
|
|
1309
|
+
packageName: androidPackageName,
|
|
1310
|
+
});
|
|
1311
|
+
const adbDriverStepProfileOutput = run(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1312
|
+
'--config',
|
|
1313
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1314
|
+
'--scenario',
|
|
1315
|
+
adbDriverStepScenarioPath,
|
|
1316
|
+
'--adb-capture',
|
|
1317
|
+
'--adb',
|
|
1318
|
+
fakeAdbDriverStepPath,
|
|
1319
|
+
'--out',
|
|
1320
|
+
adbDriverStepProfileRoot,
|
|
1321
|
+
'--run-id',
|
|
1322
|
+
adbDriverStepRunId,
|
|
1323
|
+
], {
|
|
1324
|
+
cwd: installDir,
|
|
1325
|
+
env,
|
|
1326
|
+
});
|
|
1327
|
+
const adbDriverStepProfileRunDir = adbDriverStepProfileOutput.trim();
|
|
1328
|
+
const adbDriverStepCaptureRoot = path.join(adbDriverStepProfileRoot, '_adb-captures', adbDriverStepRunId);
|
|
1329
|
+
const adbDriverStepMetadata = JSON.parse(fs.readFileSync(path.join(adbDriverStepCaptureRoot, 'raw', 'android-metadata.json'), 'utf8'));
|
|
1330
|
+
const adbDriverStepHealth = JSON.parse(fs.readFileSync(path.join(adbDriverStepProfileRunDir, 'health.json'), 'utf8'));
|
|
1331
|
+
assert.equal(adbDriverStepHealth.healthStatus, 'passed');
|
|
1332
|
+
assert.equal(adbDriverStepMetadata.logcat.rawPath, 'raw/adb-logcat.txt');
|
|
1333
|
+
assert.equal(adbDriverStepMetadata.logcat.stepId, 'capture-log-window');
|
|
1334
|
+
const exampleLiveRoot = path.join(tempRoot, 'example-android-live-proof');
|
|
1335
|
+
const fakeExampleLiveAdbPath = path.join(tempRoot, process.platform === 'win32' ? 'fake-example-live-adb.cmd' : 'fake-example-live-adb');
|
|
1336
|
+
const fakeAgentDevicePath = path.join(tempRoot, process.platform === 'win32' ? 'fake-agent-device.cmd' : 'fake-agent-device');
|
|
1337
|
+
writeFakeExampleLiveAdb({
|
|
1338
|
+
filePath: fakeExampleLiveAdbPath,
|
|
1339
|
+
fixtures: {
|
|
1340
|
+
'app-startup': {
|
|
1341
|
+
fixtureRunId: 'android-example-startup',
|
|
1342
|
+
logcatText: fs.readFileSync(path.join(exampleAppRoot, 'event-logs', 'android-app-startup.log'), 'utf8'),
|
|
1343
|
+
},
|
|
1344
|
+
'open-close-cycle': {
|
|
1345
|
+
fixtureRunId: 'android-example-open-close',
|
|
1346
|
+
logcatText: fs.readFileSync(path.join(exampleAppRoot, 'event-logs', 'android-open-close-cycle.log'), 'utf8'),
|
|
1347
|
+
},
|
|
1348
|
+
'scroll-settle': {
|
|
1349
|
+
fixtureRunId: 'android-example-scroll',
|
|
1350
|
+
logcatText: fs.readFileSync(path.join(exampleAppRoot, 'event-logs', 'android-scroll-settle.log'), 'utf8'),
|
|
1351
|
+
},
|
|
1352
|
+
},
|
|
1353
|
+
packageName: androidPackageName,
|
|
1354
|
+
});
|
|
1355
|
+
writeFakeAgentDevice(fakeAgentDevicePath);
|
|
1356
|
+
const genericAndroidLiveRoot = path.join(tempRoot, 'generic-android-live-proof');
|
|
1357
|
+
const genericAndroidLiveOutput = run(packageBinPath(installDir, 'asl-live-android'), [
|
|
1358
|
+
'--config',
|
|
1359
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1360
|
+
'--scenario',
|
|
1361
|
+
path.join(exampleAppRoot, 'scenarios', 'mobile', 'app-startup.json'),
|
|
1362
|
+
'--adb',
|
|
1363
|
+
fakeExampleLiveAdbPath,
|
|
1364
|
+
'--package',
|
|
1365
|
+
androidPackageName,
|
|
1366
|
+
'--out',
|
|
1367
|
+
genericAndroidLiveRoot,
|
|
1368
|
+
'--wait-ms',
|
|
1369
|
+
'1',
|
|
1370
|
+
'--command-wait-ms',
|
|
1371
|
+
'1',
|
|
1372
|
+
'--agent-device-proof',
|
|
1373
|
+
'--agent-device',
|
|
1374
|
+
fakeAgentDevicePath,
|
|
1375
|
+
'--agent-device-session',
|
|
1376
|
+
'package-smoke-android',
|
|
1377
|
+
'--agent-device-session-mode',
|
|
1378
|
+
'bind',
|
|
1379
|
+
'--argent-proof',
|
|
1380
|
+
], {
|
|
1381
|
+
cwd: installDir,
|
|
1382
|
+
env: {
|
|
1383
|
+
...env,
|
|
1384
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1385
|
+
},
|
|
1386
|
+
});
|
|
1387
|
+
assert.match(genericAndroidLiveOutput, /_live-proof\/android-live-proof\/agent-summary\.md/u);
|
|
1388
|
+
assert.equal(fs.existsSync(path.join(genericAndroidLiveRoot, '_live-proof', 'android-live-proof', 'live-proof.json')), true);
|
|
1389
|
+
const genericAndroidLiveCompareOutput = run(packageBinPath(installDir, 'asl-live-android'), [
|
|
1390
|
+
'--config',
|
|
1391
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1392
|
+
'--scenario',
|
|
1393
|
+
path.join(exampleAppRoot, 'scenarios', 'mobile', 'app-startup.json'),
|
|
1394
|
+
'--adb',
|
|
1395
|
+
fakeExampleLiveAdbPath,
|
|
1396
|
+
'--package',
|
|
1397
|
+
androidPackageName,
|
|
1398
|
+
'--out',
|
|
1399
|
+
genericAndroidLiveRoot,
|
|
1400
|
+
'--wait-ms',
|
|
1401
|
+
'1',
|
|
1402
|
+
'--command-wait-ms',
|
|
1403
|
+
'1',
|
|
1404
|
+
'--run-suffix',
|
|
1405
|
+
'smoke',
|
|
1406
|
+
'--compare-latest',
|
|
1407
|
+
'--fail-on-regression',
|
|
1408
|
+
'--agent-device-proof',
|
|
1409
|
+
'--agent-device',
|
|
1410
|
+
fakeAgentDevicePath,
|
|
1411
|
+
'--agent-device-session',
|
|
1412
|
+
'package-smoke-android',
|
|
1413
|
+
'--agent-device-session-mode',
|
|
1414
|
+
'bind',
|
|
1415
|
+
'--argent-proof',
|
|
1416
|
+
], {
|
|
1417
|
+
cwd: installDir,
|
|
1418
|
+
env: {
|
|
1419
|
+
...env,
|
|
1420
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1421
|
+
},
|
|
1422
|
+
});
|
|
1423
|
+
assert.match(genericAndroidLiveCompareOutput, /_live-proof\/android-live-proof-smoke\/agent-summary\.md/u);
|
|
1424
|
+
const genericAndroidProofOutput = run(packageBinPath(installDir, 'asl-live-proof'), [
|
|
1425
|
+
'--file',
|
|
1426
|
+
path.join(genericAndroidLiveRoot, '_live-proof', 'android-live-proof-smoke', 'live-proof.json'),
|
|
1427
|
+
'--fail-on-regression',
|
|
1428
|
+
], {
|
|
1429
|
+
cwd: installDir,
|
|
1430
|
+
env,
|
|
1431
|
+
});
|
|
1432
|
+
assert.match(genericAndroidProofOutput, /Comparison status: unchanged/u);
|
|
1433
|
+
assert.match(genericAndroidProofOutput, /app-startup \(app-startup\/app-startup-android-live-smoke\): unchanged/u);
|
|
1434
|
+
assert.match(genericAndroidProofOutput, /interaction-agent-device \(agent-device\/app-startup\/app-startup-android-agent-device-smoke\): health=passed verdict=not_evaluated/u);
|
|
1435
|
+
assert.match(genericAndroidProofOutput, /interaction-argent \(argent\/app-startup\/app-startup-android-argent-smoke\): health=passed verdict=not_evaluated/u);
|
|
1436
|
+
const exampleLiveOutput = run(packageBinPath(installDir, 'asl-example-android-live'), [
|
|
1437
|
+
'--adb',
|
|
1438
|
+
fakeExampleLiveAdbPath,
|
|
1439
|
+
'--out',
|
|
1440
|
+
exampleLiveRoot,
|
|
1441
|
+
'--wait-ms',
|
|
1442
|
+
'1',
|
|
1443
|
+
'--command-wait-ms',
|
|
1444
|
+
'1',
|
|
1445
|
+
'--agent-device-proof',
|
|
1446
|
+
'--agent-device',
|
|
1447
|
+
fakeAgentDevicePath,
|
|
1448
|
+
'--agent-device-session',
|
|
1449
|
+
'package-smoke-android',
|
|
1450
|
+
'--agent-device-session-mode',
|
|
1451
|
+
'bind',
|
|
1452
|
+
'--argent-proof',
|
|
1453
|
+
], {
|
|
1454
|
+
cwd: installDir,
|
|
1455
|
+
env: {
|
|
1456
|
+
...env,
|
|
1457
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1458
|
+
},
|
|
1459
|
+
});
|
|
1460
|
+
assert.match(exampleLiveOutput, /Android example live proof passed/u);
|
|
1461
|
+
const exampleLivePreflightHealth = JSON.parse(fs.readFileSync(path.join(exampleLiveRoot, '_preflight', 'android-live-preflight', 'health.json'), 'utf8'));
|
|
1462
|
+
assert.equal(exampleLivePreflightHealth.healthStatus, 'passed');
|
|
1463
|
+
assert.equal(fs.existsSync(path.join(exampleLiveRoot, '_live-proof', 'android-live-proof', 'live-proof.json')), true);
|
|
1464
|
+
assert.equal(fs.existsSync(path.join(exampleLiveRoot, '_live-proof', 'android-live-proof', 'agent-summary.md')), true);
|
|
1465
|
+
assert.equal(fs.existsSync(path.join(exampleLiveRoot, '_preflight', 'android-live-preflight', 'raw', 'adb-react-native-reverse.txt')), true);
|
|
1466
|
+
assert.equal(fs.existsSync(path.join(exampleLiveRoot, '_preflight', 'android-live-preflight', 'raw', 'adb-react-native-debug-host.txt')), true);
|
|
1467
|
+
const exampleLiveCompareOutput = run(packageBinPath(installDir, 'asl-example-android-live'), [
|
|
1468
|
+
'--adb',
|
|
1469
|
+
fakeExampleLiveAdbPath,
|
|
1470
|
+
'--out',
|
|
1471
|
+
exampleLiveRoot,
|
|
1472
|
+
'--wait-ms',
|
|
1473
|
+
'1',
|
|
1474
|
+
'--command-wait-ms',
|
|
1475
|
+
'1',
|
|
1476
|
+
'--run-suffix',
|
|
1477
|
+
'smoke',
|
|
1478
|
+
'--compare-latest',
|
|
1479
|
+
'--fail-on-regression',
|
|
1480
|
+
'--agent-device-proof',
|
|
1481
|
+
'--agent-device',
|
|
1482
|
+
fakeAgentDevicePath,
|
|
1483
|
+
'--agent-device-session',
|
|
1484
|
+
'package-smoke-android',
|
|
1485
|
+
'--agent-device-session-mode',
|
|
1486
|
+
'bind',
|
|
1487
|
+
'--argent-proof',
|
|
1488
|
+
], {
|
|
1489
|
+
cwd: installDir,
|
|
1490
|
+
env: {
|
|
1491
|
+
...env,
|
|
1492
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1493
|
+
},
|
|
1494
|
+
});
|
|
1495
|
+
assert.match(exampleLiveCompareOutput, /Comparisons:/u);
|
|
1496
|
+
assert.equal(fs.existsSync(path.join(exampleLiveRoot, '_live-proof', 'android-live-proof-smoke', 'live-proof.json')), true);
|
|
1497
|
+
const exampleLiveProofOutput = run(packageBinPath(installDir, 'asl-live-proof'), [
|
|
1498
|
+
'--file',
|
|
1499
|
+
path.join(exampleLiveRoot, '_live-proof', 'android-live-proof-smoke', 'live-proof.json'),
|
|
1500
|
+
'--fail-on-regression',
|
|
1501
|
+
], {
|
|
1502
|
+
cwd: installDir,
|
|
1503
|
+
env,
|
|
1504
|
+
});
|
|
1505
|
+
assert.match(exampleLiveProofOutput, /Comparison status: unchanged/u);
|
|
1506
|
+
assert.match(exampleLiveProofOutput, /Preflight: android-live-preflight-smoke health=passed verdict=not_evaluated/u);
|
|
1507
|
+
assert.match(exampleLiveProofOutput, /Comparison counts: better=0 worse=0 unchanged=3 mixed=0 inconclusive=0 low_confidence=0 skipped=0/u);
|
|
1508
|
+
assert.match(exampleLiveProofOutput, /startup \(app-startup\/android-live-startup-smoke\): unchanged \(metrics better=0 worse=0 unchanged=1 inconclusive=0 low_confidence=0\)/u);
|
|
1509
|
+
assert.match(exampleLiveProofOutput, /startup \(app-startup\/android-live-startup-smoke\): health=passed verdict=passed/u);
|
|
1510
|
+
assert.match(exampleLiveProofOutput, /startup-ui \(agent-device\/app-startup\/android-agent-device-startup-smoke\): health=passed verdict=not_evaluated/u);
|
|
1511
|
+
assert.match(exampleLiveProofOutput, /startup-ui-argent \(argent\/app-startup\/android-argent-startup-smoke\): health=passed verdict=not_evaluated/u);
|
|
1512
|
+
assert.match(exampleLiveProofOutput, /Next action: inspect_summary/u);
|
|
1513
|
+
const exampleLiveProof = JSON.parse(fs.readFileSync(path.join(exampleLiveRoot, '_live-proof', 'android-live-proof-smoke', 'live-proof.json'), 'utf8'));
|
|
1514
|
+
assert.deepEqual({
|
|
1515
|
+
healthStatus: exampleLiveProof.preflight.healthStatus,
|
|
1516
|
+
verdictStatus: exampleLiveProof.preflight.verdictStatus,
|
|
1517
|
+
}, { healthStatus: 'passed', verdictStatus: 'not_evaluated' });
|
|
1518
|
+
assert.deepEqual(exampleLiveProof.interactionProofs.map((proof) => ({
|
|
1519
|
+
healthStatus: proof.healthStatus,
|
|
1520
|
+
runnerId: proof.runnerId,
|
|
1521
|
+
})), [
|
|
1522
|
+
{ healthStatus: 'passed', runnerId: 'agent-device' },
|
|
1523
|
+
{ healthStatus: 'passed', runnerId: 'argent' },
|
|
1524
|
+
]);
|
|
1525
|
+
assert.deepEqual(exampleLiveProof.comparisons.map((comparison) => ({
|
|
1526
|
+
notableMetrics: comparison.metricSummary?.notableMetrics?.length ?? 0,
|
|
1527
|
+
unchanged: comparison.metricSummary?.counts?.unchanged,
|
|
1528
|
+
})), [
|
|
1529
|
+
{ notableMetrics: 0, unchanged: 1 },
|
|
1530
|
+
{ notableMetrics: 0, unchanged: 3 },
|
|
1531
|
+
{ notableMetrics: 0, unchanged: 3 },
|
|
1532
|
+
]);
|
|
1533
|
+
for (const [scenarioDir, runId] of [
|
|
1534
|
+
['app-startup', 'android-live-startup'],
|
|
1535
|
+
['open-close-cycle', 'android-live-open-close'],
|
|
1536
|
+
['scroll-settle', 'android-live-scroll'],
|
|
1537
|
+
]) {
|
|
1538
|
+
const runDir = path.join(exampleLiveRoot, scenarioDir, runId);
|
|
1539
|
+
const health = JSON.parse(fs.readFileSync(path.join(runDir, 'health.json'), 'utf8'));
|
|
1540
|
+
const verdict = JSON.parse(fs.readFileSync(path.join(runDir, 'verdict.json'), 'utf8'));
|
|
1541
|
+
assert.equal(health.healthStatus, 'passed');
|
|
1542
|
+
assert.equal(verdict.verdictStatus, 'passed');
|
|
1543
|
+
assert.equal(fs.existsSync(path.join(runDir, 'agent-summary.md')), true);
|
|
1544
|
+
}
|
|
1545
|
+
for (const [scenarioDir, runId] of [
|
|
1546
|
+
['app-startup', 'android-live-startup-smoke'],
|
|
1547
|
+
['open-close-cycle', 'android-live-open-close-smoke'],
|
|
1548
|
+
['scroll-settle', 'android-live-scroll-smoke'],
|
|
1549
|
+
]) {
|
|
1550
|
+
const baselineRunId = runId.replace(/-smoke$/u, '');
|
|
1551
|
+
const comparisonDir = path.join(exampleLiveRoot, 'comparisons', scenarioDir, runId);
|
|
1552
|
+
const comparisonPath = path.join(comparisonDir, 'comparison.json');
|
|
1553
|
+
const comparison = JSON.parse(fs.readFileSync(comparisonPath, 'utf8'));
|
|
1554
|
+
assertLatestTrustedComparisonBasis({
|
|
1555
|
+
artifactRoot: exampleLiveRoot,
|
|
1556
|
+
baselineRunDir: path.join(exampleLiveRoot, scenarioDir, baselineRunId),
|
|
1557
|
+
baselineRunId,
|
|
1558
|
+
comparison,
|
|
1559
|
+
currentRunDir: path.join(exampleLiveRoot, scenarioDir, runId),
|
|
1560
|
+
currentRunId: runId,
|
|
1561
|
+
});
|
|
1562
|
+
assert.equal(fs.existsSync(path.join(comparisonDir, 'agent-summary.md')), true);
|
|
1563
|
+
}
|
|
1564
|
+
const exampleIosLiveRoot = path.join(tempRoot, 'example-ios-live-proof');
|
|
1565
|
+
const fakeExampleLiveXcrunPath = path.join(tempRoot, process.platform === 'win32' ? 'fake-example-live-xcrun.cmd' : 'fake-example-live-xcrun');
|
|
1566
|
+
const iosDeviceId = 'A692ED28-893E-453F-8866-C69331AE757F';
|
|
1567
|
+
const iosBundleId = 'dev.agent-scenario-loop.example';
|
|
1568
|
+
writeFakeExampleLiveXcrun({
|
|
1569
|
+
bundleId: iosBundleId,
|
|
1570
|
+
deviceId: iosDeviceId,
|
|
1571
|
+
filePath: fakeExampleLiveXcrunPath,
|
|
1572
|
+
fixtures: {
|
|
1573
|
+
'app-startup': {
|
|
1574
|
+
fixtureRunId: 'example-startup',
|
|
1575
|
+
logcatText: fs.readFileSync(path.join(exampleAppRoot, 'event-logs', 'app-startup.log'), 'utf8'),
|
|
1576
|
+
},
|
|
1577
|
+
'open-close-cycle': {
|
|
1578
|
+
fixtureRunId: 'example-open-close',
|
|
1579
|
+
logcatText: fs.readFileSync(path.join(exampleAppRoot, 'event-logs', 'open-close-cycle.log'), 'utf8'),
|
|
1580
|
+
},
|
|
1581
|
+
'scroll-settle': {
|
|
1582
|
+
fixtureRunId: 'example-scroll',
|
|
1583
|
+
logcatText: fs.readFileSync(path.join(exampleAppRoot, 'event-logs', 'scroll-settle.log'), 'utf8'),
|
|
1584
|
+
},
|
|
1585
|
+
},
|
|
1586
|
+
});
|
|
1587
|
+
const genericIosLiveRoot = path.join(tempRoot, 'generic-ios-live-proof');
|
|
1588
|
+
const genericIosLiveOutput = run(packageBinPath(installDir, 'asl-live-ios'), [
|
|
1589
|
+
'--config',
|
|
1590
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1591
|
+
'--scenario',
|
|
1592
|
+
path.join(exampleAppRoot, 'scenarios', 'mobile', 'app-startup.json'),
|
|
1593
|
+
'--xcrun',
|
|
1594
|
+
fakeExampleLiveXcrunPath,
|
|
1595
|
+
'--device',
|
|
1596
|
+
iosDeviceId,
|
|
1597
|
+
'--bundle',
|
|
1598
|
+
iosBundleId,
|
|
1599
|
+
'--out',
|
|
1600
|
+
genericIosLiveRoot,
|
|
1601
|
+
'--wait-ms',
|
|
1602
|
+
'1',
|
|
1603
|
+
'--agent-device-proof',
|
|
1604
|
+
'--agent-device',
|
|
1605
|
+
fakeAgentDevicePath,
|
|
1606
|
+
'--agent-device-session',
|
|
1607
|
+
'package-smoke-ios',
|
|
1608
|
+
'--agent-device-session-mode',
|
|
1609
|
+
'bind',
|
|
1610
|
+
'--argent-proof',
|
|
1611
|
+
], {
|
|
1612
|
+
cwd: installDir,
|
|
1613
|
+
env: {
|
|
1614
|
+
...env,
|
|
1615
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1616
|
+
},
|
|
1617
|
+
});
|
|
1618
|
+
assert.match(genericIosLiveOutput, /_live-proof\/ios-live-proof\/agent-summary\.md/u);
|
|
1619
|
+
assert.equal(fs.existsSync(path.join(genericIosLiveRoot, '_live-proof', 'ios-live-proof', 'live-proof.json')), true);
|
|
1620
|
+
const genericIosLiveCompareOutput = run(packageBinPath(installDir, 'asl-live-ios'), [
|
|
1621
|
+
'--config',
|
|
1622
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1623
|
+
'--scenario',
|
|
1624
|
+
path.join(exampleAppRoot, 'scenarios', 'mobile', 'app-startup.json'),
|
|
1625
|
+
'--xcrun',
|
|
1626
|
+
fakeExampleLiveXcrunPath,
|
|
1627
|
+
'--device',
|
|
1628
|
+
iosDeviceId,
|
|
1629
|
+
'--bundle',
|
|
1630
|
+
iosBundleId,
|
|
1631
|
+
'--out',
|
|
1632
|
+
genericIosLiveRoot,
|
|
1633
|
+
'--wait-ms',
|
|
1634
|
+
'1',
|
|
1635
|
+
'--run-suffix',
|
|
1636
|
+
'smoke',
|
|
1637
|
+
'--compare-latest',
|
|
1638
|
+
'--fail-on-regression',
|
|
1639
|
+
'--agent-device-proof',
|
|
1640
|
+
'--agent-device',
|
|
1641
|
+
fakeAgentDevicePath,
|
|
1642
|
+
'--agent-device-session',
|
|
1643
|
+
'package-smoke-ios',
|
|
1644
|
+
'--agent-device-session-mode',
|
|
1645
|
+
'bind',
|
|
1646
|
+
'--argent-proof',
|
|
1647
|
+
], {
|
|
1648
|
+
cwd: installDir,
|
|
1649
|
+
env: {
|
|
1650
|
+
...env,
|
|
1651
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1652
|
+
},
|
|
1653
|
+
});
|
|
1654
|
+
assert.match(genericIosLiveCompareOutput, /_live-proof\/ios-live-proof-smoke\/agent-summary\.md/u);
|
|
1655
|
+
const genericIosProofOutput = run(packageBinPath(installDir, 'asl-live-proof'), [
|
|
1656
|
+
'--file',
|
|
1657
|
+
path.join(genericIosLiveRoot, '_live-proof', 'ios-live-proof-smoke', 'live-proof.json'),
|
|
1658
|
+
'--fail-on-regression',
|
|
1659
|
+
], {
|
|
1660
|
+
cwd: installDir,
|
|
1661
|
+
env,
|
|
1662
|
+
});
|
|
1663
|
+
assert.match(genericIosProofOutput, /Comparison status: unchanged/u);
|
|
1664
|
+
assert.match(genericIosProofOutput, /app-startup \(app-startup\/app-startup-ios-live-smoke\): unchanged/u);
|
|
1665
|
+
assert.match(genericIosProofOutput, /interaction-agent-device \(agent-device\/app-startup\/app-startup-ios-agent-device-smoke\): health=passed verdict=not_evaluated/u);
|
|
1666
|
+
assert.match(genericIosProofOutput, /interaction-argent \(argent\/app-startup\/app-startup-ios-argent-smoke\): health=passed verdict=not_evaluated/u);
|
|
1667
|
+
const exampleIosLiveOutput = run(packageBinPath(installDir, 'asl-example-ios-live'), [
|
|
1668
|
+
'--xcrun',
|
|
1669
|
+
fakeExampleLiveXcrunPath,
|
|
1670
|
+
'--device',
|
|
1671
|
+
iosDeviceId,
|
|
1672
|
+
'--out',
|
|
1673
|
+
exampleIosLiveRoot,
|
|
1674
|
+
'--wait-ms',
|
|
1675
|
+
'1',
|
|
1676
|
+
'--agent-device-proof',
|
|
1677
|
+
'--agent-device',
|
|
1678
|
+
fakeAgentDevicePath,
|
|
1679
|
+
'--agent-device-session',
|
|
1680
|
+
'package-smoke-ios',
|
|
1681
|
+
'--agent-device-session-mode',
|
|
1682
|
+
'bind',
|
|
1683
|
+
'--argent-proof',
|
|
1684
|
+
], {
|
|
1685
|
+
cwd: installDir,
|
|
1686
|
+
env: {
|
|
1687
|
+
...env,
|
|
1688
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1689
|
+
},
|
|
1690
|
+
});
|
|
1691
|
+
assert.match(exampleIosLiveOutput, /iOS example live proof passed/u);
|
|
1692
|
+
const exampleIosLivePreflightHealth = JSON.parse(fs.readFileSync(path.join(exampleIosLiveRoot, '_preflight', 'ios-live-preflight', 'health.json'), 'utf8'));
|
|
1693
|
+
assert.equal(exampleIosLivePreflightHealth.healthStatus, 'passed');
|
|
1694
|
+
assert.equal(fs.existsSync(path.join(exampleIosLiveRoot, '_live-proof', 'ios-live-proof', 'live-proof.json')), true);
|
|
1695
|
+
assert.equal(fs.existsSync(path.join(exampleIosLiveRoot, '_live-proof', 'ios-live-proof', 'agent-summary.md')), true);
|
|
1696
|
+
const exampleIosLiveCompareOutput = run(packageBinPath(installDir, 'asl-example-ios-live'), [
|
|
1697
|
+
'--xcrun',
|
|
1698
|
+
fakeExampleLiveXcrunPath,
|
|
1699
|
+
'--device',
|
|
1700
|
+
iosDeviceId,
|
|
1701
|
+
'--out',
|
|
1702
|
+
exampleIosLiveRoot,
|
|
1703
|
+
'--wait-ms',
|
|
1704
|
+
'1',
|
|
1705
|
+
'--run-suffix',
|
|
1706
|
+
'smoke',
|
|
1707
|
+
'--compare-latest',
|
|
1708
|
+
'--fail-on-regression',
|
|
1709
|
+
'--agent-device-proof',
|
|
1710
|
+
'--agent-device',
|
|
1711
|
+
fakeAgentDevicePath,
|
|
1712
|
+
'--agent-device-session',
|
|
1713
|
+
'package-smoke-ios',
|
|
1714
|
+
'--agent-device-session-mode',
|
|
1715
|
+
'bind',
|
|
1716
|
+
'--argent-proof',
|
|
1717
|
+
], {
|
|
1718
|
+
cwd: installDir,
|
|
1719
|
+
env: {
|
|
1720
|
+
...env,
|
|
1721
|
+
ASL_ARGENT_BIN: fakeArgentPath,
|
|
1722
|
+
},
|
|
1723
|
+
});
|
|
1724
|
+
assert.match(exampleIosLiveCompareOutput, /Comparisons:/u);
|
|
1725
|
+
assert.equal(fs.existsSync(path.join(exampleIosLiveRoot, '_live-proof', 'ios-live-proof-smoke', 'live-proof.json')), true);
|
|
1726
|
+
const exampleIosLiveProofOutput = run(packageBinPath(installDir, 'asl-live-proof'), [
|
|
1727
|
+
'--file',
|
|
1728
|
+
path.join(exampleIosLiveRoot, '_live-proof', 'ios-live-proof-smoke', 'live-proof.json'),
|
|
1729
|
+
'--fail-on-regression',
|
|
1730
|
+
], {
|
|
1731
|
+
cwd: installDir,
|
|
1732
|
+
env,
|
|
1733
|
+
});
|
|
1734
|
+
assert.match(exampleIosLiveProofOutput, /Comparison status: unchanged/u);
|
|
1735
|
+
assert.match(exampleIosLiveProofOutput, /Preflight: ios-live-preflight-smoke health=passed verdict=not_evaluated/u);
|
|
1736
|
+
assert.match(exampleIosLiveProofOutput, /startup \(app-startup\/ios-live-startup-smoke\): unchanged \(metrics better=0 worse=0 unchanged=1 inconclusive=0 low_confidence=0\)/u);
|
|
1737
|
+
assert.match(exampleIosLiveProofOutput, /startup-ui \(agent-device\/app-startup\/ios-agent-device-startup-smoke\): health=passed verdict=not_evaluated/u);
|
|
1738
|
+
assert.match(exampleIosLiveProofOutput, /startup-ui-argent \(argent\/app-startup\/ios-argent-startup-smoke\): health=passed verdict=not_evaluated/u);
|
|
1739
|
+
const exampleIosLiveProof = JSON.parse(fs.readFileSync(path.join(exampleIosLiveRoot, '_live-proof', 'ios-live-proof-smoke', 'live-proof.json'), 'utf8'));
|
|
1740
|
+
assert.deepEqual({
|
|
1741
|
+
healthStatus: exampleIosLiveProof.preflight.healthStatus,
|
|
1742
|
+
verdictStatus: exampleIosLiveProof.preflight.verdictStatus,
|
|
1743
|
+
}, { healthStatus: 'passed', verdictStatus: 'not_evaluated' });
|
|
1744
|
+
assert.deepEqual(exampleIosLiveProof.interactionProofs.map((proof) => ({
|
|
1745
|
+
healthStatus: proof.healthStatus,
|
|
1746
|
+
runnerId: proof.runnerId,
|
|
1747
|
+
})), [
|
|
1748
|
+
{ healthStatus: 'passed', runnerId: 'agent-device' },
|
|
1749
|
+
{ healthStatus: 'passed', runnerId: 'argent' },
|
|
1750
|
+
]);
|
|
1751
|
+
assert.deepEqual(exampleIosLiveProof.comparisons.map((comparison) => ({
|
|
1752
|
+
notableMetrics: comparison.metricSummary?.notableMetrics?.length ?? 0,
|
|
1753
|
+
unchanged: comparison.metricSummary?.counts?.unchanged,
|
|
1754
|
+
})), [
|
|
1755
|
+
{ notableMetrics: 0, unchanged: 1 },
|
|
1756
|
+
{ notableMetrics: 0, unchanged: 3 },
|
|
1757
|
+
{ notableMetrics: 0, unchanged: 3 },
|
|
1758
|
+
]);
|
|
1759
|
+
for (const [scenarioDir, runId] of [
|
|
1760
|
+
['app-startup', 'ios-live-startup'],
|
|
1761
|
+
['open-close-cycle', 'ios-live-open-close'],
|
|
1762
|
+
['scroll-settle', 'ios-live-scroll'],
|
|
1763
|
+
]) {
|
|
1764
|
+
const runDir = path.join(exampleIosLiveRoot, scenarioDir, runId);
|
|
1765
|
+
const health = JSON.parse(fs.readFileSync(path.join(runDir, 'health.json'), 'utf8'));
|
|
1766
|
+
const verdict = JSON.parse(fs.readFileSync(path.join(runDir, 'verdict.json'), 'utf8'));
|
|
1767
|
+
assert.equal(health.healthStatus, 'passed');
|
|
1768
|
+
assert.equal(verdict.verdictStatus, 'passed');
|
|
1769
|
+
assert.equal(fs.existsSync(path.join(runDir, 'agent-summary.md')), true);
|
|
1770
|
+
assert.equal(fs.existsSync(path.join(runDir, 'raw', 'ios-profile-events.log')), true);
|
|
1771
|
+
}
|
|
1772
|
+
for (const [scenarioDir, runId] of [
|
|
1773
|
+
['app-startup', 'ios-live-startup-smoke'],
|
|
1774
|
+
['open-close-cycle', 'ios-live-open-close-smoke'],
|
|
1775
|
+
['scroll-settle', 'ios-live-scroll-smoke'],
|
|
1776
|
+
]) {
|
|
1777
|
+
const baselineRunId = runId.replace(/-smoke$/u, '');
|
|
1778
|
+
const comparisonDir = path.join(exampleIosLiveRoot, 'comparisons', scenarioDir, runId);
|
|
1779
|
+
const comparisonPath = path.join(comparisonDir, 'comparison.json');
|
|
1780
|
+
const comparison = JSON.parse(fs.readFileSync(comparisonPath, 'utf8'));
|
|
1781
|
+
assertLatestTrustedComparisonBasis({
|
|
1782
|
+
artifactRoot: exampleIosLiveRoot,
|
|
1783
|
+
baselineRunDir: path.join(exampleIosLiveRoot, scenarioDir, baselineRunId),
|
|
1784
|
+
baselineRunId,
|
|
1785
|
+
comparison,
|
|
1786
|
+
currentRunDir: path.join(exampleIosLiveRoot, scenarioDir, runId),
|
|
1787
|
+
currentRunId: runId,
|
|
1788
|
+
});
|
|
1789
|
+
assert.equal(fs.existsSync(path.join(comparisonDir, 'agent-summary.md')), true);
|
|
1790
|
+
}
|
|
1791
|
+
const missingAdbRunId = 'package-smoke-missing-adb';
|
|
1792
|
+
const missingAdbProfileRoot = path.join(tempRoot, 'missing-adb-profile');
|
|
1793
|
+
const missingAdb = runExpectFailure(packageBinPath(installDir, 'asl-profile-android'), [
|
|
1794
|
+
'--config',
|
|
1795
|
+
path.join(exampleAppRoot, 'asl.config.json'),
|
|
1796
|
+
'--scenario',
|
|
1797
|
+
path.join(exampleAppRoot, 'scenarios', 'android', 'app-startup.json'),
|
|
1798
|
+
'--adb-capture',
|
|
1799
|
+
'--adb',
|
|
1800
|
+
path.join(tempRoot, 'missing-adb'),
|
|
1801
|
+
'--clear-logcat',
|
|
1802
|
+
'--launch',
|
|
1803
|
+
'--wait-ms',
|
|
1804
|
+
'1',
|
|
1805
|
+
'--out',
|
|
1806
|
+
missingAdbProfileRoot,
|
|
1807
|
+
'--run-id',
|
|
1808
|
+
missingAdbRunId,
|
|
1809
|
+
], {
|
|
1810
|
+
cwd: installDir,
|
|
1811
|
+
env,
|
|
1812
|
+
});
|
|
1813
|
+
const missingAdbCaptureRoot = path.join(missingAdbProfileRoot, '_adb-captures', missingAdbRunId);
|
|
1814
|
+
const missingAdbHealth = JSON.parse(fs.readFileSync(path.join(missingAdbCaptureRoot, 'health.json'), 'utf8'));
|
|
1815
|
+
const missingAdbSummary = fs.readFileSync(path.join(missingAdbCaptureRoot, 'agent-summary.md'), 'utf8');
|
|
1816
|
+
assert.equal(missingAdb.status, 1);
|
|
1817
|
+
assert.match(missingAdb.stderr, /Android adb capture failed/u);
|
|
1818
|
+
assert.equal(missingAdbHealth.healthStatus, 'failed');
|
|
1819
|
+
assert.equal(missingAdbHealth.checks.some((check) => check.code === 'adb_unavailable'), true);
|
|
1820
|
+
assert.equal(missingAdbHealth.checks.some((check) => check.code === 'adb_unavailable' && check.metadata?.nextActionCode === 'fix_adb_command'), true);
|
|
1821
|
+
assert.match(missingAdbSummary, /Next action `fix_adb_command`/u);
|
|
1822
|
+
const health = JSON.parse(fs.readFileSync(path.join(artifactDir, 'health.json'), 'utf8'));
|
|
1823
|
+
assert.equal(health.scenarioId, 'app-startup');
|
|
1824
|
+
assert.equal(health.runId, 'package-smoke');
|
|
1825
|
+
assert.equal(health.healthStatus, 'passed');
|
|
1826
|
+
const latestCompareRoot = path.join(tempRoot, 'latest-compare-runs');
|
|
1827
|
+
writeSmokeRun({
|
|
1828
|
+
root: latestCompareRoot,
|
|
1829
|
+
runId: 'baseline-run',
|
|
1830
|
+
actual: 950,
|
|
1831
|
+
endedAt: '2026-06-16T10:00:00.000Z',
|
|
1832
|
+
});
|
|
1833
|
+
const latestCompareCurrentDir = writeSmokeRun({
|
|
1834
|
+
root: latestCompareRoot,
|
|
1835
|
+
runId: 'current-run',
|
|
1836
|
+
actual: 850,
|
|
1837
|
+
endedAt: '2026-06-16T10:05:00.000Z',
|
|
1838
|
+
});
|
|
1839
|
+
const latestCompareOutputDir = path.join(tempRoot, 'latest-compare-output');
|
|
1840
|
+
const latestCompareOutput = run(packageBinPath(installDir, 'asl-compare-latest'), [
|
|
1841
|
+
'--root',
|
|
1842
|
+
latestCompareRoot,
|
|
1843
|
+
'--scenario',
|
|
1844
|
+
'app-startup',
|
|
1845
|
+
'--current',
|
|
1846
|
+
latestCompareCurrentDir,
|
|
1847
|
+
'--out',
|
|
1848
|
+
latestCompareOutputDir,
|
|
1849
|
+
'--fail-on-regression',
|
|
1850
|
+
], {
|
|
1851
|
+
cwd: installDir,
|
|
1852
|
+
env,
|
|
1853
|
+
});
|
|
1854
|
+
const latestComparison = JSON.parse(fs.readFileSync(path.join(latestCompareOutputDir, 'comparison.json'), 'utf8'));
|
|
1855
|
+
assert.equal(latestCompareOutput.trim(), latestCompareOutputDir);
|
|
1856
|
+
assert.equal(latestComparison.baselineRunId, 'baseline-run');
|
|
1857
|
+
assert.equal(latestComparison.runId, 'current-run');
|
|
1858
|
+
assert.equal(latestComparison.comparisonStatus, 'better');
|
|
1859
|
+
assertLatestTrustedComparisonBasis({
|
|
1860
|
+
artifactRoot: latestCompareRoot,
|
|
1861
|
+
baselineRunDir: path.join(latestCompareRoot, 'app-startup', 'baseline-run'),
|
|
1862
|
+
baselineRunId: 'baseline-run',
|
|
1863
|
+
comparison: latestComparison,
|
|
1864
|
+
currentRunDir: latestCompareCurrentDir,
|
|
1865
|
+
currentRunId: 'current-run',
|
|
1866
|
+
});
|
|
1867
|
+
assert.equal(fs.existsSync(path.join(latestCompareOutputDir, 'agent-summary.md')), true);
|
|
1868
|
+
const resolveSmokeScript = [
|
|
1869
|
+
"const assert = require('node:assert/strict');",
|
|
1870
|
+
"const fs = require('node:fs');",
|
|
1871
|
+
"const path = require('node:path');",
|
|
1872
|
+
"const asl = require('agent-scenario-loop');",
|
|
1873
|
+
"const packageRoot = path.join(process.cwd(), 'node_modules', 'agent-scenario-loop');",
|
|
1874
|
+
"const packageJson = require('agent-scenario-loop/package.json');",
|
|
1875
|
+
"function readJson(relativePath) { return JSON.parse(fs.readFileSync(path.join(packageRoot, relativePath), 'utf8')); }",
|
|
1876
|
+
"function listJsonFiles(relativeDir) { return fs.readdirSync(path.join(packageRoot, relativeDir)).filter((name) => name.endsWith('.json')).sort().map((name) => path.join(relativeDir, name)); }",
|
|
1877
|
+
"function documentedRunnerSubpaths() {",
|
|
1878
|
+
" const apiDocs = fs.readFileSync(path.join(packageRoot, 'docs/api.md'), 'utf8');",
|
|
1879
|
+
" return Array.from(new Set(Array.from(apiDocs.matchAll(/`(agent-scenario-loop\\/runner\\/[^`]+)`/gu), (match) => match[1]))).sort();",
|
|
1880
|
+
"}",
|
|
1881
|
+
"function exportedRunnerSubpaths() {",
|
|
1882
|
+
" return Object.keys(packageJson.exports).filter((name) => name.startsWith('./runner/')).map((name) => `agent-scenario-loop/${name.slice(2)}`).sort();",
|
|
1883
|
+
"}",
|
|
1884
|
+
"for (const subpath of Object.keys(packageJson.exports).filter((name) => !name.includes('*'))) {",
|
|
1885
|
+
" const specifier = subpath === '.' ? 'agent-scenario-loop' : `agent-scenario-loop/${subpath.slice(2)}`;",
|
|
1886
|
+
" require.resolve(specifier);",
|
|
1887
|
+
"}",
|
|
1888
|
+
"assert.deepEqual(documentedRunnerSubpaths(), exportedRunnerSubpaths(), 'installed docs/api.md runner subpaths must match package exports');",
|
|
1889
|
+
"assert.equal(asl.ARTIFACT_LAYOUT_VERSION, '1.0.0');",
|
|
1890
|
+
"assert.equal(asl.ARTIFACT_FILENAMES.health, 'health.json');",
|
|
1891
|
+
"assert.deepEqual(asl.ARTIFACT_FILENAMES, { agentSummary: 'agent-summary.md', comparison: 'comparison.json', health: 'health.json', liveProof: 'live-proof.json', liveProofSet: 'live-proof-set.json', plannerCompatibility: 'planner-compatibility.json', projectValidation: 'project-validation.json', verdict: 'verdict.json' });",
|
|
1892
|
+
"assert.equal(asl.PROFILE_ARTIFACT_FILENAMES.metrics, 'metrics.json');",
|
|
1893
|
+
"assert.deepEqual(asl.PROFILE_ARTIFACT_FILENAMES, { budgetVerdict: 'budget-verdict.json', causalRun: 'causal-run.json', manifest: 'manifest.json', metrics: 'metrics.json', summary: 'summary.md' });",
|
|
1894
|
+
"assert.equal(typeof asl.createArtifactLayout, 'function');",
|
|
1895
|
+
"const layout = asl.createArtifactLayout({ outputDir: 'run' });",
|
|
1896
|
+
"assert.equal(layout.health, 'run/health.json');",
|
|
1897
|
+
"assert.equal(layout.verdict, 'run/verdict.json');",
|
|
1898
|
+
"assert.equal(layout.comparison, 'run/comparison.json');",
|
|
1899
|
+
"assert.equal(layout.agentSummary, 'run/agent-summary.md');",
|
|
1900
|
+
"assert.equal(layout.liveProof, 'run/live-proof.json');",
|
|
1901
|
+
"assert.equal(layout.liveProofSet, 'run/live-proof-set.json');",
|
|
1902
|
+
"assert.equal(layout.plannerCompatibility, 'run/planner-compatibility.json');",
|
|
1903
|
+
"assert.equal(layout.projectValidation, 'run/project-validation.json');",
|
|
1904
|
+
"assert.equal(layout.raw, 'run/raw');",
|
|
1905
|
+
"assert.equal(layout.captures, 'run/captures');",
|
|
1906
|
+
"assert.equal(layout.signals.js, 'run/signals/js');",
|
|
1907
|
+
"assert.equal(layout.signals.memory, 'run/signals/memory');",
|
|
1908
|
+
"assert.equal(layout.signals.network, 'run/signals/network');",
|
|
1909
|
+
"assert.equal(layout.profile.manifest, 'run/manifest.json');",
|
|
1910
|
+
"assert.equal(layout.profile.metrics, 'run/metrics.json');",
|
|
1911
|
+
"assert.equal(layout.profile.causalRun, 'run/causal-run.json');",
|
|
1912
|
+
"assert.equal(layout.profile.budgetVerdict, 'run/budget-verdict.json');",
|
|
1913
|
+
"assert.equal(layout.profile.summary, 'run/summary.md');",
|
|
1914
|
+
"assert.equal(typeof asl.buildAgentSummaryMarkdown, 'function');",
|
|
1915
|
+
"assert.equal(typeof asl.extractProfileEvents, 'function');",
|
|
1916
|
+
"assert.equal(typeof asl.buildMetricsFromProfileEvents, 'function');",
|
|
1917
|
+
"assert.equal(typeof asl.buildManifest, 'function');",
|
|
1918
|
+
"assert.equal(typeof asl.writeJsonArtifact, 'function');",
|
|
1919
|
+
"assert.equal(typeof asl.writeTextArtifact, 'function');",
|
|
1920
|
+
"assert.equal(typeof asl.buildComparisonArtifact, 'function');",
|
|
1921
|
+
"assert.equal(typeof asl.compareBudgetCheck, 'function');",
|
|
1922
|
+
"assert.equal(typeof asl.interpretEvidence, 'function');",
|
|
1923
|
+
"assert.equal(typeof asl.isTimingEvidenceTrusted, 'function');",
|
|
1924
|
+
"assert.equal(typeof asl.buildScenarioExecutionPlan, 'function');",
|
|
1925
|
+
"assert.equal(typeof asl.evaluateRunnerCompatibility, 'function');",
|
|
1926
|
+
"assert.equal(typeof asl.collectScenarioDriverActions, 'function');",
|
|
1927
|
+
"assert.equal(typeof asl.buildRunIndex, 'function');",
|
|
1928
|
+
"assert.equal(typeof asl.findLatestTrustedRun, 'function');",
|
|
1929
|
+
"assert.equal(typeof asl.validateJson, 'function');",
|
|
1930
|
+
"assert.equal(typeof asl.assertValidJson, 'function');",
|
|
1931
|
+
"assert.deepEqual(asl.PRIMARY_RUNNER_PORT, ['prepare', 'launch', 'startSession', 'executeStep', 'waitForTruthEvent', 'captureEvidence', 'stopSession', 'finalize']);",
|
|
1932
|
+
"assert.deepEqual(asl.EVIDENCE_PROVIDER_PORT, ['prepare', 'startWindow', 'capture', 'stopWindow', 'finalize']);",
|
|
1933
|
+
"assert.deepEqual(asl.DRIVER_PORT, ['tap', 'scroll', 'assertVisible', 'inspectTree', 'screenshot', 'record', 'readLogs', 'collectPerfSignals']);",
|
|
1934
|
+
"assert.deepEqual(asl.ARTIFACT_WRITER_PORT, ['writeJson', 'writeText', 'copyRaw']);",
|
|
1935
|
+
"assert.deepEqual(asl.INTERPRETER_PORT, ['interpret']);",
|
|
1936
|
+
"assert.equal(typeof asl.validatePortImplementation, 'function');",
|
|
1937
|
+
"assert.equal(typeof asl.assertPortImplementation, 'function');",
|
|
1938
|
+
"assert.equal(typeof asl.dispatchDriverAction, 'function');",
|
|
1939
|
+
"assert.equal(asl.isDriverActionName('tap'), true);",
|
|
1940
|
+
"assert.equal(asl.isDriverActionName('pinch'), false);",
|
|
1941
|
+
"const driverValidation = asl.validatePortImplementation({ name: 'driver', implementation: { tap() {}, screenshot() {} }, requiredMethods: asl.DRIVER_PORT });",
|
|
1942
|
+
"assert.equal(driverValidation.valid, false);",
|
|
1943
|
+
"assert.deepEqual(driverValidation.expectedMethods, asl.DRIVER_PORT);",
|
|
1944
|
+
"assert.deepEqual(driverValidation.implementedMethods, ['screenshot', 'tap']);",
|
|
1945
|
+
"assert.deepEqual(driverValidation.missingMethods, ['scroll', 'assertVisible', 'inspectTree', 'record', 'readLogs', 'collectPerfSignals']);",
|
|
1946
|
+
"assert.equal(driverValidation.message, 'driver is missing required method(s): scroll, assertVisible, inspectTree, record, readLogs, collectPerfSignals');",
|
|
1947
|
+
"assert.deepEqual(asl.collectScenarioDriverActions({ steps: [{ kind: 'gesture', driverAction: 'scroll' }, { kind: 'captureEvidence', driverAction: 'screenshot', required: false }] }), { required: ['scroll'], optional: ['screenshot'] });",
|
|
1948
|
+
"assert.deepEqual(asl.buildScenarioExecutionPlan({ id: 'scroll', steps: [{ kind: 'gesture', driverAction: 'scroll' }] }).steps[0], { id: '01-gesture', index: 0, kind: 'gesture', portMethod: 'executeStep', required: true, driverAction: 'scroll' });",
|
|
1949
|
+
"const legacyLayoutVersionExport = ['V', '1', '_ARTIFACT_LAYOUT_VERSION'].join('');",
|
|
1950
|
+
"const legacyFilenamesExport = ['V', '1', '_ARTIFACT_FILENAMES'].join('');",
|
|
1951
|
+
"assert.equal(Object.prototype.hasOwnProperty.call(asl, legacyLayoutVersionExport), false);",
|
|
1952
|
+
"assert.equal(Object.prototype.hasOwnProperty.call(asl, legacyFilenamesExport), false);",
|
|
1953
|
+
"assert.equal(Object.prototype.hasOwnProperty.call(asl, 'TRANSITION_ARTIFACT_FILENAMES'), false);",
|
|
1954
|
+
"assert.equal(Object.prototype.hasOwnProperty.call(layout, 'transition'), false);",
|
|
1955
|
+
"require.resolve('agent-scenario-loop/schemas/budget-verdict.schema.json');",
|
|
1956
|
+
"require.resolve('agent-scenario-loop/schemas/causal-run.schema.json');",
|
|
1957
|
+
"require.resolve('agent-scenario-loop/schemas/comparison.schema.json');",
|
|
1958
|
+
"require.resolve('agent-scenario-loop/schemas/external-adapter-message.schema.json');",
|
|
1959
|
+
"require.resolve('agent-scenario-loop/schemas/health.schema.json');",
|
|
1960
|
+
"require.resolve('agent-scenario-loop/schemas/live-proof.schema.json');",
|
|
1961
|
+
"require.resolve('agent-scenario-loop/schemas/live-proof-set.schema.json');",
|
|
1962
|
+
"require.resolve('agent-scenario-loop/schemas/scenario.schema.json');",
|
|
1963
|
+
"require.resolve('agent-scenario-loop/schemas/manifest.schema.json');",
|
|
1964
|
+
"require.resolve('agent-scenario-loop/schemas/metrics.schema.json');",
|
|
1965
|
+
"require.resolve('agent-scenario-loop/schemas/project-validation.schema.json');",
|
|
1966
|
+
"require.resolve('agent-scenario-loop/schemas/runner-capabilities.schema.json');",
|
|
1967
|
+
"require.resolve('agent-scenario-loop/schemas/verdict.schema.json');",
|
|
1968
|
+
"require.resolve('agent-scenario-loop/examples/scenarios/mobile/app-startup.json');",
|
|
1969
|
+
"require.resolve('agent-scenario-loop/examples/mobile-app/asl.config.json');",
|
|
1970
|
+
"require.resolve('agent-scenario-loop/examples/runners/README.md');",
|
|
1971
|
+
"require.resolve('agent-scenario-loop/examples/runners/script-accessibility-provider.json');",
|
|
1972
|
+
"require.resolve('agent-scenario-loop/examples/runners/script-memory-provider.json');",
|
|
1973
|
+
"require.resolve('agent-scenario-loop/examples/runners/script-network-provider.json');",
|
|
1974
|
+
"require.resolve('agent-scenario-loop/examples/runners/script-profiler-provider.json');",
|
|
1975
|
+
"require.resolve('agent-scenario-loop/templates/project.config.json');",
|
|
1976
|
+
"require.resolve('agent-scenario-loop/templates/mobile-scenario.json');",
|
|
1977
|
+
"require.resolve('agent-scenario-loop/templates/primary-runner.json');",
|
|
1978
|
+
"require.resolve('agent-scenario-loop/templates/evidence-provider.json');",
|
|
1979
|
+
"require.resolve('agent-scenario-loop/templates/gitignore-snippet');",
|
|
1980
|
+
"require.resolve('agent-scenario-loop/templates/integration-readme.md');",
|
|
1981
|
+
"require.resolve('agent-scenario-loop/templates/package-scripts.json');",
|
|
1982
|
+
"require.resolve('agent-scenario-loop/templates/skills/agent-scenario-loop/SKILL.md');",
|
|
1983
|
+
"require.resolve('agent-scenario-loop/templates/skills/agent-scenario-loop/references/artifact-interpretation.md');",
|
|
1984
|
+
"require.resolve('agent-scenario-loop/templates/skills/agent-scenario-loop/references/adoption-checklist.md');",
|
|
1985
|
+
"require.resolve('agent-scenario-loop/templates/scripts/asl-capture-accessibility-provider.mjs');",
|
|
1986
|
+
"require.resolve('agent-scenario-loop/templates/scripts/asl-capture-profiler-provider.mjs');",
|
|
1987
|
+
"for (const scenarioFixture of listJsonFiles('examples/scenarios/mobile')) {",
|
|
1988
|
+
" const result = asl.validateJson(readJson(scenarioFixture), asl.SCHEMAS.scenario, scenarioFixture);",
|
|
1989
|
+
" assert.equal(result.valid, true, result.message);",
|
|
1990
|
+
"}",
|
|
1991
|
+
"assert.equal(asl.validateJson(readJson('templates/mobile-scenario.json'), asl.SCHEMAS.scenario, 'templates/mobile-scenario.json').valid, true);",
|
|
1992
|
+
"for (const runnerFixture of listJsonFiles('examples/runners')) {",
|
|
1993
|
+
" const result = asl.validateJson(readJson(runnerFixture), asl.SCHEMAS.runnerCapabilities, runnerFixture);",
|
|
1994
|
+
" assert.equal(result.valid, true, result.message);",
|
|
1995
|
+
"}",
|
|
1996
|
+
"const runnerMatrix = fs.readFileSync(path.join(packageRoot, 'examples/runners/README.md'), 'utf8');",
|
|
1997
|
+
"for (const runnerFixture of listJsonFiles('examples/runners')) {",
|
|
1998
|
+
" const runnerFileName = path.basename(runnerFixture);",
|
|
1999
|
+
" assert.equal(runnerMatrix.includes('`' + runnerFileName + '`'), true, `${runnerFileName} missing from runner matrix`);",
|
|
2000
|
+
"}",
|
|
2001
|
+
"assert.equal(asl.validateJson(readJson('templates/primary-runner.json'), asl.SCHEMAS.runnerCapabilities, 'templates/primary-runner.json').valid, true);",
|
|
2002
|
+
"assert.equal(asl.validateJson(readJson('templates/evidence-provider.json'), asl.SCHEMAS.runnerCapabilities, 'templates/evidence-provider.json').valid, true);",
|
|
2003
|
+
"const appJson = readJson('examples/mobile-app/app.json');",
|
|
2004
|
+
"const aslConfig = readJson('examples/mobile-app/asl.config.json');",
|
|
2005
|
+
"assert.equal(aslConfig.app.androidPackage, appJson.expo.android.package);",
|
|
2006
|
+
"assert.equal(aslConfig.app.iosBundleId, appJson.expo.ios.bundleIdentifier);",
|
|
2007
|
+
"assert.equal(fs.existsSync('node_modules/agent-scenario-loop/app/profile-session.ts'), true);",
|
|
2008
|
+
"require.resolve('agent-scenario-loop/app/profile-session');",
|
|
2009
|
+
"assert.equal(fs.existsSync('node_modules/agent-scenario-loop/core/config-template.json'), true);",
|
|
2010
|
+
"assert.equal(fs.existsSync('node_modules/agent-scenario-loop/docs/api.md'), true);",
|
|
2011
|
+
"assert.equal(fs.existsSync('node_modules/agent-scenario-loop/docs/authoring.md'), true);",
|
|
2012
|
+
"require.resolve('agent-scenario-loop/runner/agent-device');",
|
|
2013
|
+
"require.resolve('agent-scenario-loop/runner/agent-device-driver');",
|
|
2014
|
+
"require.resolve('agent-scenario-loop/runner/argent');",
|
|
2015
|
+
"require.resolve('agent-scenario-loop/runner/argent-driver');",
|
|
2016
|
+
"require.resolve('agent-scenario-loop/runner/ios-simctl-driver');",
|
|
2017
|
+
"require.resolve('agent-scenario-loop/runner/live-proof');",
|
|
2018
|
+
"require.resolve('agent-scenario-loop/runner/validate-project');",
|
|
2019
|
+
].join('\n');
|
|
2020
|
+
run(process.execPath, ['-e', resolveSmokeScript], {
|
|
2021
|
+
cwd: installDir,
|
|
2022
|
+
env,
|
|
2023
|
+
});
|
|
2024
|
+
const typeSmokeSource = [
|
|
2025
|
+
"import {",
|
|
2026
|
+
" ARTIFACT_LAYOUT_VERSION,",
|
|
2027
|
+
" DRIVER_PORT,",
|
|
2028
|
+
" SCHEMAS,",
|
|
2029
|
+
" buildAgentSummaryMarkdown,",
|
|
2030
|
+
" buildComparisonArtifact,",
|
|
2031
|
+
" buildScenarioExecutionPlan,",
|
|
2032
|
+
" createArtifactLayout,",
|
|
2033
|
+
" collectScenarioDriverActions,",
|
|
2034
|
+
" dispatchDriverAction,",
|
|
2035
|
+
" buildRunIndex,",
|
|
2036
|
+
" findLatestTrustedRun,",
|
|
2037
|
+
" validateJson,",
|
|
2038
|
+
" validatePortImplementation,",
|
|
2039
|
+
" validateScenarioAdapterOptions,",
|
|
2040
|
+
" type ArtifactLayout,",
|
|
2041
|
+
" type ArtifactWriterPort,",
|
|
2042
|
+
" type DriverActionName,",
|
|
2043
|
+
" type DriverPort,",
|
|
2044
|
+
" type EvidenceProviderPort,",
|
|
2045
|
+
" type InterpreterPort,",
|
|
2046
|
+
" type PortResult,",
|
|
2047
|
+
" type PortValidationResult,",
|
|
2048
|
+
" type PrimaryRunnerPort,",
|
|
2049
|
+
"} from 'agent-scenario-loop';",
|
|
2050
|
+
"import { resolveAgentDeviceDriverSteps, runAgentDeviceCapture } from 'agent-scenario-loop/runner/agent-device';",
|
|
2051
|
+
"import { resolveArgentDriverSteps, runArgentCapture } from 'agent-scenario-loop/runner/argent';",
|
|
2052
|
+
"import { buildAndroidScrollCoordinatesFromBounds, createAndroidAdbDriver, resolveAndroidSelectorFromUiTree } from 'agent-scenario-loop/runner/android-adb-driver';",
|
|
2053
|
+
"import { createAgentDeviceDriver, formatAgentDeviceSelector } from 'agent-scenario-loop/runner/agent-device-driver';",
|
|
2054
|
+
"import { createArgentDriver, normalizeArgentPoint } from 'agent-scenario-loop/runner/argent-driver';",
|
|
2055
|
+
"import { createIosSimctlDriver } from 'agent-scenario-loop/runner/ios-simctl-driver';",
|
|
2056
|
+
"import { runExampleAndroidLiveProof } from 'agent-scenario-loop/runner/example-android-live';",
|
|
2057
|
+
"import { runExampleIosLiveProof } from 'agent-scenario-loop/runner/example-ios-live';",
|
|
2058
|
+
"import { initProject } from 'agent-scenario-loop/runner/init-project';",
|
|
2059
|
+
"import { compareLatestTrustedRun } from 'agent-scenario-loop/runner/compare-latest';",
|
|
2060
|
+
"import { runIosSimctlCapture } from 'agent-scenario-loop/runner/ios-simctl';",
|
|
2061
|
+
"import { readLiveProof } from 'agent-scenario-loop/runner/live-proof';",
|
|
2062
|
+
"import { resolveAndroidAdbDriverSteps, resolveAndroidAdbProfileCommands, runProfileAndroid } from 'agent-scenario-loop/runner/profile-android';",
|
|
2063
|
+
"import { runProfileIos, type CliArgs } from 'agent-scenario-loop/runner/profile-ios';",
|
|
2064
|
+
"import { validateProject } from 'agent-scenario-loop/runner/validate-project';",
|
|
2065
|
+
'',
|
|
2066
|
+
"const layout: ArtifactLayout = createArtifactLayout({ outputDir: 'run' });",
|
|
2067
|
+
"const validation: PortValidationResult = validatePortImplementation({",
|
|
2068
|
+
" name: 'driver',",
|
|
2069
|
+
' implementation: { tap() {}, screenshot() {} },',
|
|
2070
|
+
' requiredMethods: DRIVER_PORT,',
|
|
2071
|
+
'});',
|
|
2072
|
+
"const passedResult: PortResult = { status: 'passed' };",
|
|
2073
|
+
'const primaryRunner: PrimaryRunnerPort = {',
|
|
2074
|
+
' prepare: async () => passedResult,',
|
|
2075
|
+
' launch: async () => passedResult,',
|
|
2076
|
+
' startSession: async () => passedResult,',
|
|
2077
|
+
' executeStep: async () => passedResult,',
|
|
2078
|
+
' waitForTruthEvent: async () => passedResult,',
|
|
2079
|
+
' captureEvidence: async () => passedResult,',
|
|
2080
|
+
' stopSession: async () => passedResult,',
|
|
2081
|
+
' finalize: async () => passedResult,',
|
|
2082
|
+
'};',
|
|
2083
|
+
'const evidenceProvider: EvidenceProviderPort = {',
|
|
2084
|
+
' prepare: async () => passedResult,',
|
|
2085
|
+
' startWindow: async () => passedResult,',
|
|
2086
|
+
' capture: async () => passedResult,',
|
|
2087
|
+
' stopWindow: async () => passedResult,',
|
|
2088
|
+
' finalize: async () => passedResult,',
|
|
2089
|
+
'};',
|
|
2090
|
+
'const driverPort: DriverPort = {',
|
|
2091
|
+
' tap: async () => passedResult,',
|
|
2092
|
+
' scroll: async () => passedResult,',
|
|
2093
|
+
' assertVisible: async () => passedResult,',
|
|
2094
|
+
' inspectTree: async () => passedResult,',
|
|
2095
|
+
' screenshot: async () => passedResult,',
|
|
2096
|
+
' record: async () => passedResult,',
|
|
2097
|
+
' readLogs: async () => passedResult,',
|
|
2098
|
+
' collectPerfSignals: async () => passedResult,',
|
|
2099
|
+
'};',
|
|
2100
|
+
"const driverActionName: DriverActionName = 'tap';",
|
|
2101
|
+
"const dispatchedDriverAction = dispatchDriverAction({ driver: driverPort, input: { action: driverActionName, platform: 'android' } });",
|
|
2102
|
+
'const artifactWriter: ArtifactWriterPort = {',
|
|
2103
|
+
" writeJson: async () => {},",
|
|
2104
|
+
" writeText: async () => {},",
|
|
2105
|
+
" copyRaw: async () => 'raw/output.txt',",
|
|
2106
|
+
'};',
|
|
2107
|
+
'const interpreter: InterpreterPort = {',
|
|
2108
|
+
" interpret: async () => ({ likelyCauses: [], notes: [], summary: 'passed', trusted: true }),",
|
|
2109
|
+
'};',
|
|
2110
|
+
"const driverActions = collectScenarioDriverActions({ steps: [{ kind: 'gesture', driverAction: 'scroll' }] });",
|
|
2111
|
+
"const executionPlan = buildScenarioExecutionPlan({ id: 'scroll', steps: [{ kind: 'gesture', driverAction: 'scroll' }] });",
|
|
2112
|
+
"const runIndex = buildRunIndex({ rootDir: 'missing-artifacts' });",
|
|
2113
|
+
"const latestTrusted = findLatestTrustedRun(runIndex, 'startup');",
|
|
2114
|
+
"const androidDriver = createAndroidAdbDriver({",
|
|
2115
|
+
" adbPath: 'adb',",
|
|
2116
|
+
" deviceSerial: 'emulator-5554',",
|
|
2117
|
+
" executor: async (command, commandArgs) => ({",
|
|
2118
|
+
' args: commandArgs,',
|
|
2119
|
+
' command,',
|
|
2120
|
+
' exitCode: 0,',
|
|
2121
|
+
" stderr: '',",
|
|
2122
|
+
" stdout: '',",
|
|
2123
|
+
' }),',
|
|
2124
|
+
'});',
|
|
2125
|
+
"const androidTap = androidDriver.tap({ x: 10, y: 20 });",
|
|
2126
|
+
"const androidScroll = androidDriver.scroll({ startX: 100, startY: 800, endX: 100, endY: 200 });",
|
|
2127
|
+
"const androidTree = androidDriver.inspectTree();",
|
|
2128
|
+
"const androidScreenshot = androidDriver.screenshot();",
|
|
2129
|
+
"const androidSelector = resolveAndroidSelectorFromUiTree({",
|
|
2130
|
+
" selector: { kind: 'testId', value: 'open-card' },",
|
|
2131
|
+
" uiTreeXml: '<hierarchy><node resource-id=\"dev.example:id/open-card\" bounds=\"[0,0][100,200]\" /></hierarchy>',",
|
|
2132
|
+
'});',
|
|
2133
|
+
"const androidSelectorScroll = buildAndroidScrollCoordinatesFromBounds({ bottom: 200, left: 0, right: 100, top: 0 });",
|
|
2134
|
+
"const androidDriverSteps = resolveAndroidAdbDriverSteps({ steps: [{ kind: 'captureEvidence', driverAction: 'readLogs', artifact: 'logs' }] });",
|
|
2135
|
+
"const agentDeviceDriver = createAgentDeviceDriver({",
|
|
2136
|
+
" agentDevicePath: 'agent-device',",
|
|
2137
|
+
" platform: 'ios',",
|
|
2138
|
+
" udid: 'booted',",
|
|
2139
|
+
" executor: async (command, commandArgs) => ({",
|
|
2140
|
+
' args: commandArgs,',
|
|
2141
|
+
' command,',
|
|
2142
|
+
' exitCode: 0,',
|
|
2143
|
+
" stderr: '',",
|
|
2144
|
+
" stdout: '',",
|
|
2145
|
+
' }),',
|
|
2146
|
+
'});',
|
|
2147
|
+
"const agentDeviceTap = agentDeviceDriver.tap({ selector: { kind: 'accessibilityLabel', value: 'Open' } });",
|
|
2148
|
+
"const agentDeviceSelector = formatAgentDeviceSelector({ kind: 'text', value: 'Ready' });",
|
|
2149
|
+
"const agentDeviceSteps = resolveAgentDeviceDriverSteps({ steps: [{ kind: 'captureEvidence', artifact: 'screenshot', driverAction: 'screenshot' }] });",
|
|
2150
|
+
"const argentPoint = normalizeArgentPoint({ screenSize: { width: 1000, height: 2000 }, x: 500, y: 400 });",
|
|
2151
|
+
"const argentDriver = createArgentDriver({",
|
|
2152
|
+
" argentCommand: 'argent',",
|
|
2153
|
+
" appId: 'dev.example.app',",
|
|
2154
|
+
" deviceId: 'booted',",
|
|
2155
|
+
" executor: async (command, commandArgs) => ({",
|
|
2156
|
+
' args: commandArgs,',
|
|
2157
|
+
' command,',
|
|
2158
|
+
' exitCode: 0,',
|
|
2159
|
+
" stderr: '',",
|
|
2160
|
+
" stdout: '',",
|
|
2161
|
+
' }),',
|
|
2162
|
+
'});',
|
|
2163
|
+
"const argentTap = argentDriver.tap({ x: 0.5, y: 0.25 });",
|
|
2164
|
+
"const argentSteps = resolveArgentDriverSteps({ steps: [{ kind: 'gesture', driverAction: 'tap', adapterOptions: { argent: { x: 0.5, y: 0.25 } } }] });",
|
|
2165
|
+
"const argentCapture = runArgentCapture({",
|
|
2166
|
+
" deviceId: 'booted',",
|
|
2167
|
+
" executor: async (command, commandArgs) => ({",
|
|
2168
|
+
' args: commandArgs,',
|
|
2169
|
+
' command,',
|
|
2170
|
+
' exitCode: 0,',
|
|
2171
|
+
" stderr: '',",
|
|
2172
|
+
" stdout: '',",
|
|
2173
|
+
' }),',
|
|
2174
|
+
" platform: 'ios',",
|
|
2175
|
+
" runId: 'argent-smoke',",
|
|
2176
|
+
" scenario: { id: 'argent-smoke', steps: [] },",
|
|
2177
|
+
'});',
|
|
2178
|
+
"const agentDeviceCapture = runAgentDeviceCapture({",
|
|
2179
|
+
" driverSteps: [],",
|
|
2180
|
+
" executor: async (command, commandArgs) => ({",
|
|
2181
|
+
' args: commandArgs,',
|
|
2182
|
+
' command,',
|
|
2183
|
+
' exitCode: 0,',
|
|
2184
|
+
" stderr: '',",
|
|
2185
|
+
" stdout: '',",
|
|
2186
|
+
' }),',
|
|
2187
|
+
" platform: 'ios',",
|
|
2188
|
+
" runId: 'agent-device-smoke',",
|
|
2189
|
+
"});",
|
|
2190
|
+
"const iosDriver = createIosSimctlDriver({",
|
|
2191
|
+
" deviceUdid: 'booted',",
|
|
2192
|
+
" xcrunPath: 'xcrun',",
|
|
2193
|
+
" executor: async (command, commandArgs) => ({",
|
|
2194
|
+
' args: commandArgs,',
|
|
2195
|
+
' command,',
|
|
2196
|
+
' exitCode: 0,',
|
|
2197
|
+
" stderr: '',",
|
|
2198
|
+
" stdout: '',",
|
|
2199
|
+
' }),',
|
|
2200
|
+
'});',
|
|
2201
|
+
"const iosLogs = iosDriver.readLogs({ last: '30s' });",
|
|
2202
|
+
"const iosScreenshot = iosDriver.screenshot({ display: 'Internal-1', imageType: 'jpeg', mask: 'black', outputPath: 'ios-screenshot.jpeg' });",
|
|
2203
|
+
"const args: CliArgs = { config: 'config.json', scenario: 'scenario.json' };",
|
|
2204
|
+
'const comparison = buildComparisonArtifact({',
|
|
2205
|
+
" baselineHealth: { schemaVersion: '1.0.0', scenarioId: 'startup', runId: 'before', healthStatus: 'passed', checks: [] },",
|
|
2206
|
+
" baselineVerdict: { schemaVersion: '1.0.0', scenarioId: 'startup', runId: 'before', healthStatus: 'passed', verdictStatus: 'passed' },",
|
|
2207
|
+
" currentHealth: { schemaVersion: '1.0.0', scenarioId: 'startup', runId: 'after', healthStatus: 'passed', checks: [] },",
|
|
2208
|
+
" currentVerdict: { schemaVersion: '1.0.0', scenarioId: 'startup', runId: 'after', healthStatus: 'passed', verdictStatus: 'passed' },",
|
|
2209
|
+
'});',
|
|
2210
|
+
"const summary = buildAgentSummaryMarkdown({ health: comparison, verdict: comparison });",
|
|
2211
|
+
"validateJson({ schemaVersion: '1.0.0' }, SCHEMAS.health, 'health');",
|
|
2212
|
+
"validateScenarioAdapterOptions({ effectivePlatforms: ['android'], errors: [], scenario: { id: 'startup', steps: [] } });",
|
|
2213
|
+
"void ARTIFACT_LAYOUT_VERSION;",
|
|
2214
|
+
'void layout;',
|
|
2215
|
+
'void validation;',
|
|
2216
|
+
'void primaryRunner;',
|
|
2217
|
+
'void evidenceProvider;',
|
|
2218
|
+
'void driverPort;',
|
|
2219
|
+
'void driverActionName;',
|
|
2220
|
+
'void dispatchedDriverAction;',
|
|
2221
|
+
'void artifactWriter;',
|
|
2222
|
+
'void interpreter;',
|
|
2223
|
+
'void driverActions;',
|
|
2224
|
+
'void executionPlan;',
|
|
2225
|
+
'void runIndex;',
|
|
2226
|
+
'void latestTrusted;',
|
|
2227
|
+
'void androidDriver;',
|
|
2228
|
+
'void androidTap;',
|
|
2229
|
+
'void androidScroll;',
|
|
2230
|
+
'void androidTree;',
|
|
2231
|
+
'void androidScreenshot;',
|
|
2232
|
+
'void androidSelector;',
|
|
2233
|
+
'void androidSelectorScroll;',
|
|
2234
|
+
'void androidDriverSteps;',
|
|
2235
|
+
'void argentDriver;',
|
|
2236
|
+
'void argentPoint;',
|
|
2237
|
+
'void argentTap;',
|
|
2238
|
+
'void argentSteps;',
|
|
2239
|
+
'void argentCapture;',
|
|
2240
|
+
'void iosDriver;',
|
|
2241
|
+
'void iosLogs;',
|
|
2242
|
+
'void iosScreenshot;',
|
|
2243
|
+
'void args;',
|
|
2244
|
+
'void summary;',
|
|
2245
|
+
'void compareLatestTrustedRun;',
|
|
2246
|
+
'void runExampleAndroidLiveProof;',
|
|
2247
|
+
'void runExampleIosLiveProof;',
|
|
2248
|
+
'void initProject;',
|
|
2249
|
+
'void runIosSimctlCapture;',
|
|
2250
|
+
'void readLiveProof;',
|
|
2251
|
+
'void resolveAndroidAdbProfileCommands;',
|
|
2252
|
+
'void runProfileAndroid;',
|
|
2253
|
+
'void runProfileIos;',
|
|
2254
|
+
'void validateProject;',
|
|
2255
|
+
'',
|
|
2256
|
+
].join('\n');
|
|
2257
|
+
fs.writeFileSync(path.join(installDir, 'package-smoke-types.ts'), typeSmokeSource, 'utf8');
|
|
2258
|
+
fs.writeFileSync(path.join(installDir, 'tsconfig.json'), `${JSON.stringify({
|
|
2259
|
+
compilerOptions: {
|
|
2260
|
+
module: 'Node16',
|
|
2261
|
+
moduleResolution: 'Node16',
|
|
2262
|
+
noEmit: true,
|
|
2263
|
+
strict: true,
|
|
2264
|
+
target: 'ES2022',
|
|
2265
|
+
},
|
|
2266
|
+
include: ['package-smoke-types.ts'],
|
|
2267
|
+
}, null, 2)}\n`, 'utf8');
|
|
2268
|
+
run(typescriptBinPath(repoRoot), ['-p', installDir], {
|
|
2269
|
+
cwd: installDir,
|
|
2270
|
+
env,
|
|
2271
|
+
});
|
|
2272
|
+
process.stdout.write(`package smoke passed: ${tarballPath}\n`);
|
|
2273
|
+
fs.rmSync(tempRoot, { recursive: true, force: true });
|
|
2274
|
+
}
|
|
2275
|
+
catch (error) {
|
|
2276
|
+
console.error(`package smoke temp kept at: ${tempRoot}`);
|
|
2277
|
+
throw error;
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
if (require.main === module) {
|
|
2281
|
+
main();
|
|
2282
|
+
}
|