agent-scenario-loop 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/profile-session.ts +263 -17
- package/dist/core/artifact-contract.d.ts +6 -4
- package/dist/core/artifact-contract.js +164 -15
- package/dist/core/schema-validator.d.ts +1 -0
- package/dist/core/schema-validator.js +1 -0
- package/dist/runner/android-adb-driver.d.ts +7 -2
- package/dist/runner/android-adb-driver.js +7 -1
- package/dist/runner/android-adb.d.ts +40 -5
- package/dist/runner/android-adb.js +1046 -664
- package/dist/runner/ios-simctl.d.ts +1 -0
- package/dist/runner/ios-simctl.js +1 -0
- package/dist/runner/profile-android.d.ts +11 -1
- package/dist/runner/profile-android.js +230 -16
- package/dist/runner/profile-ios.d.ts +3 -2
- package/dist/runner/profile-ios.js +223 -20
- package/dist/runner/profile-mobile.d.ts +31 -3
- package/dist/runner/profile-mobile.js +793 -20
- package/dist/runner/validate-project.js +3 -0
- package/dist/scripts/consumer-rehearsal.d.ts +119 -0
- package/dist/scripts/consumer-rehearsal.js +757 -0
- package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
- package/dist/scripts/downstream-local-package-gate.js +264 -0
- package/dist/scripts/package-smoke.d.ts +96 -0
- package/dist/scripts/package-smoke.js +2282 -0
- package/dist/scripts/release-readiness.d.ts +2 -0
- package/dist/scripts/release-readiness.js +520 -0
- package/docs/adapters.md +3 -1
- package/docs/api.md +2 -2
- package/docs/authoring.md +34 -2
- package/docs/consumer-rehearsal.md +27 -1
- package/docs/contracts.md +16 -2
- package/docs/live-proofs.md +5 -3
- package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
- package/examples/runners/README.md +3 -3
- package/examples/runners/axe-accessibility-provider.json +2 -2
- package/examples/runners/script-accessibility-provider.json +2 -2
- package/examples/runners/script-memory-provider.json +2 -2
- package/examples/runners/script-network-provider.json +2 -2
- package/examples/runners/script-profiler-provider.json +2 -2
- package/package.json +11 -3
- package/schemas/manifest.schema.json +73 -3
- package/schemas/profiler.schema.json +243 -0
- package/schemas/runner-capabilities.schema.json +8 -2
- package/schemas/scenario.schema.json +18 -2
- package/templates/evidence-provider.json +3 -3
- package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
const assert = require('node:assert/strict');
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const path = require('node:path');
|
|
7
|
+
const { execFileSync } = require('node:child_process');
|
|
8
|
+
const REQUIRED_BIN_TARGETS = {
|
|
9
|
+
'agent-scenario-loop': 'dist/runner/check-plan.js',
|
|
10
|
+
'asl-agent-device': 'dist/runner/agent-device.js',
|
|
11
|
+
'asl-android-adb': 'dist/runner/android-adb.js',
|
|
12
|
+
'asl-argent': 'dist/runner/argent.js',
|
|
13
|
+
'asl-check-plan': 'dist/runner/check-plan.js',
|
|
14
|
+
'asl-compare': 'dist/runner/compare.js',
|
|
15
|
+
'asl-compare-latest': 'dist/runner/compare-latest.js',
|
|
16
|
+
'asl-demo-loop': 'dist/runner/demo-loop.js',
|
|
17
|
+
'asl-example-android-live': 'dist/runner/example-android-live.js',
|
|
18
|
+
'asl-example-ios-live': 'dist/runner/example-ios-live.js',
|
|
19
|
+
'asl-host-doctor': 'dist/runner/host-doctor.js',
|
|
20
|
+
'asl-init': 'dist/runner/init-project.js',
|
|
21
|
+
'asl-ios-simctl': 'dist/runner/ios-simctl.js',
|
|
22
|
+
'asl-live-android': 'dist/runner/live-android.js',
|
|
23
|
+
'asl-live-ios': 'dist/runner/live-ios.js',
|
|
24
|
+
'asl-live-proof': 'dist/runner/live-proof.js',
|
|
25
|
+
'asl-profile-android': 'dist/runner/profile-android.js',
|
|
26
|
+
'asl-profile-ios': 'dist/runner/profile-ios.js',
|
|
27
|
+
'asl-validate-project': 'dist/runner/validate-project.js',
|
|
28
|
+
};
|
|
29
|
+
const REQUIRED_EXPORTS = {
|
|
30
|
+
'.': {
|
|
31
|
+
require: './dist/index.js',
|
|
32
|
+
types: './dist/index.d.ts',
|
|
33
|
+
import: './dist/index.js',
|
|
34
|
+
default: './dist/index.js',
|
|
35
|
+
},
|
|
36
|
+
'./app/profile-session': {
|
|
37
|
+
require: './app/profile-session.ts',
|
|
38
|
+
types: './app/profile-session.ts',
|
|
39
|
+
import: './app/profile-session.ts',
|
|
40
|
+
default: './app/profile-session.ts',
|
|
41
|
+
},
|
|
42
|
+
'./examples/*': './examples/*',
|
|
43
|
+
'./package.json': './package.json',
|
|
44
|
+
'./runner/agent-device': {
|
|
45
|
+
require: './dist/runner/agent-device.js',
|
|
46
|
+
types: './dist/runner/agent-device.d.ts',
|
|
47
|
+
import: './dist/runner/agent-device.js',
|
|
48
|
+
default: './dist/runner/agent-device.js',
|
|
49
|
+
},
|
|
50
|
+
'./runner/agent-device-driver': {
|
|
51
|
+
require: './dist/runner/agent-device-driver.js',
|
|
52
|
+
types: './dist/runner/agent-device-driver.d.ts',
|
|
53
|
+
import: './dist/runner/agent-device-driver.js',
|
|
54
|
+
default: './dist/runner/agent-device-driver.js',
|
|
55
|
+
},
|
|
56
|
+
'./runner/argent-driver': {
|
|
57
|
+
require: './dist/runner/argent-driver.js',
|
|
58
|
+
types: './dist/runner/argent-driver.d.ts',
|
|
59
|
+
import: './dist/runner/argent-driver.js',
|
|
60
|
+
default: './dist/runner/argent-driver.js',
|
|
61
|
+
},
|
|
62
|
+
'./runner/argent': {
|
|
63
|
+
require: './dist/runner/argent.js',
|
|
64
|
+
types: './dist/runner/argent.d.ts',
|
|
65
|
+
import: './dist/runner/argent.js',
|
|
66
|
+
default: './dist/runner/argent.js',
|
|
67
|
+
},
|
|
68
|
+
'./runner/android-adb': {
|
|
69
|
+
require: './dist/runner/android-adb.js',
|
|
70
|
+
types: './dist/runner/android-adb.d.ts',
|
|
71
|
+
import: './dist/runner/android-adb.js',
|
|
72
|
+
default: './dist/runner/android-adb.js',
|
|
73
|
+
},
|
|
74
|
+
'./runner/android-adb-driver': {
|
|
75
|
+
require: './dist/runner/android-adb-driver.js',
|
|
76
|
+
types: './dist/runner/android-adb-driver.d.ts',
|
|
77
|
+
import: './dist/runner/android-adb-driver.js',
|
|
78
|
+
default: './dist/runner/android-adb-driver.js',
|
|
79
|
+
},
|
|
80
|
+
'./runner/check-plan': {
|
|
81
|
+
require: './dist/runner/check-plan.js',
|
|
82
|
+
types: './dist/runner/check-plan.d.ts',
|
|
83
|
+
import: './dist/runner/check-plan.js',
|
|
84
|
+
default: './dist/runner/check-plan.js',
|
|
85
|
+
},
|
|
86
|
+
'./runner/compare': {
|
|
87
|
+
require: './dist/runner/compare.js',
|
|
88
|
+
types: './dist/runner/compare.d.ts',
|
|
89
|
+
import: './dist/runner/compare.js',
|
|
90
|
+
default: './dist/runner/compare.js',
|
|
91
|
+
},
|
|
92
|
+
'./runner/compare-latest': {
|
|
93
|
+
require: './dist/runner/compare-latest.js',
|
|
94
|
+
types: './dist/runner/compare-latest.d.ts',
|
|
95
|
+
import: './dist/runner/compare-latest.js',
|
|
96
|
+
default: './dist/runner/compare-latest.js',
|
|
97
|
+
},
|
|
98
|
+
'./runner/demo-loop': {
|
|
99
|
+
require: './dist/runner/demo-loop.js',
|
|
100
|
+
types: './dist/runner/demo-loop.d.ts',
|
|
101
|
+
import: './dist/runner/demo-loop.js',
|
|
102
|
+
default: './dist/runner/demo-loop.js',
|
|
103
|
+
},
|
|
104
|
+
'./runner/example-android-live': {
|
|
105
|
+
require: './dist/runner/example-android-live.js',
|
|
106
|
+
types: './dist/runner/example-android-live.d.ts',
|
|
107
|
+
import: './dist/runner/example-android-live.js',
|
|
108
|
+
default: './dist/runner/example-android-live.js',
|
|
109
|
+
},
|
|
110
|
+
'./runner/example-ios-live': {
|
|
111
|
+
require: './dist/runner/example-ios-live.js',
|
|
112
|
+
types: './dist/runner/example-ios-live.d.ts',
|
|
113
|
+
import: './dist/runner/example-ios-live.js',
|
|
114
|
+
default: './dist/runner/example-ios-live.js',
|
|
115
|
+
},
|
|
116
|
+
'./runner/host-doctor': {
|
|
117
|
+
require: './dist/runner/host-doctor.js',
|
|
118
|
+
types: './dist/runner/host-doctor.d.ts',
|
|
119
|
+
import: './dist/runner/host-doctor.js',
|
|
120
|
+
default: './dist/runner/host-doctor.js',
|
|
121
|
+
},
|
|
122
|
+
'./runner/init-project': {
|
|
123
|
+
require: './dist/runner/init-project.js',
|
|
124
|
+
types: './dist/runner/init-project.d.ts',
|
|
125
|
+
import: './dist/runner/init-project.js',
|
|
126
|
+
default: './dist/runner/init-project.js',
|
|
127
|
+
},
|
|
128
|
+
'./runner/ios-simctl': {
|
|
129
|
+
require: './dist/runner/ios-simctl.js',
|
|
130
|
+
types: './dist/runner/ios-simctl.d.ts',
|
|
131
|
+
import: './dist/runner/ios-simctl.js',
|
|
132
|
+
default: './dist/runner/ios-simctl.js',
|
|
133
|
+
},
|
|
134
|
+
'./runner/ios-simctl-driver': {
|
|
135
|
+
require: './dist/runner/ios-simctl-driver.js',
|
|
136
|
+
types: './dist/runner/ios-simctl-driver.d.ts',
|
|
137
|
+
import: './dist/runner/ios-simctl-driver.js',
|
|
138
|
+
default: './dist/runner/ios-simctl-driver.js',
|
|
139
|
+
},
|
|
140
|
+
'./runner/live-android': {
|
|
141
|
+
require: './dist/runner/live-android.js',
|
|
142
|
+
types: './dist/runner/live-android.d.ts',
|
|
143
|
+
import: './dist/runner/live-android.js',
|
|
144
|
+
default: './dist/runner/live-android.js',
|
|
145
|
+
},
|
|
146
|
+
'./runner/live-ios': {
|
|
147
|
+
require: './dist/runner/live-ios.js',
|
|
148
|
+
types: './dist/runner/live-ios.d.ts',
|
|
149
|
+
import: './dist/runner/live-ios.js',
|
|
150
|
+
default: './dist/runner/live-ios.js',
|
|
151
|
+
},
|
|
152
|
+
'./runner/live-proof': {
|
|
153
|
+
require: './dist/runner/live-proof.js',
|
|
154
|
+
types: './dist/runner/live-proof.d.ts',
|
|
155
|
+
import: './dist/runner/live-proof.js',
|
|
156
|
+
default: './dist/runner/live-proof.js',
|
|
157
|
+
},
|
|
158
|
+
'./runner/profile-android': {
|
|
159
|
+
require: './dist/runner/profile-android.js',
|
|
160
|
+
types: './dist/runner/profile-android.d.ts',
|
|
161
|
+
import: './dist/runner/profile-android.js',
|
|
162
|
+
default: './dist/runner/profile-android.js',
|
|
163
|
+
},
|
|
164
|
+
'./runner/profile-ios': {
|
|
165
|
+
require: './dist/runner/profile-ios.js',
|
|
166
|
+
types: './dist/runner/profile-ios.d.ts',
|
|
167
|
+
import: './dist/runner/profile-ios.js',
|
|
168
|
+
default: './dist/runner/profile-ios.js',
|
|
169
|
+
},
|
|
170
|
+
'./runner/validate-project': {
|
|
171
|
+
require: './dist/runner/validate-project.js',
|
|
172
|
+
types: './dist/runner/validate-project.d.ts',
|
|
173
|
+
import: './dist/runner/validate-project.js',
|
|
174
|
+
default: './dist/runner/validate-project.js',
|
|
175
|
+
},
|
|
176
|
+
'./schemas/*': './schemas/*',
|
|
177
|
+
'./templates/*': './templates/*',
|
|
178
|
+
};
|
|
179
|
+
const REQUIRED_PACKAGE_FILES = [
|
|
180
|
+
'LICENSE',
|
|
181
|
+
'README.md',
|
|
182
|
+
'app/profile-session.ts',
|
|
183
|
+
'core/config-template.json',
|
|
184
|
+
'dist',
|
|
185
|
+
'docs',
|
|
186
|
+
'examples',
|
|
187
|
+
'schemas',
|
|
188
|
+
'templates',
|
|
189
|
+
];
|
|
190
|
+
const REQUIRED_PACKAGE_EXCLUSIONS = [
|
|
191
|
+
'!dist/**/__tests__',
|
|
192
|
+
'!examples/mobile-app/.expo',
|
|
193
|
+
'!examples/mobile-app/.expo/**',
|
|
194
|
+
'!examples/mobile-app/android',
|
|
195
|
+
'!examples/mobile-app/android/**',
|
|
196
|
+
'!examples/mobile-app/artifacts',
|
|
197
|
+
'!examples/mobile-app/artifacts/**',
|
|
198
|
+
'!examples/mobile-app/ios',
|
|
199
|
+
'!examples/mobile-app/ios/**',
|
|
200
|
+
'!examples/mobile-app/node_modules',
|
|
201
|
+
'!examples/mobile-app/node_modules/**',
|
|
202
|
+
];
|
|
203
|
+
const REQUIRED_GITIGNORE_ENTRIES = [
|
|
204
|
+
'node_modules/',
|
|
205
|
+
'.agents/',
|
|
206
|
+
'dist/',
|
|
207
|
+
'core/artifacts/',
|
|
208
|
+
'artifacts/',
|
|
209
|
+
'examples/mobile-app/.expo/',
|
|
210
|
+
'examples/mobile-app/android/',
|
|
211
|
+
'examples/mobile-app/artifacts/',
|
|
212
|
+
'examples/mobile-app/ios/',
|
|
213
|
+
'examples/mobile-app/node_modules/',
|
|
214
|
+
];
|
|
215
|
+
const FORBIDDEN_TRACKED_PATH_PATTERNS = [
|
|
216
|
+
/^artifacts\//u,
|
|
217
|
+
/^core\/artifacts\//u,
|
|
218
|
+
/^dist\//u,
|
|
219
|
+
/^examples\/mobile-app\/(?:\.expo|android|artifacts|ios|node_modules)(?:\/|$)/u,
|
|
220
|
+
];
|
|
221
|
+
const REQUIRED_CONFIG_STRING_FIELDS = [
|
|
222
|
+
['app', 'profileSessionScheme'],
|
|
223
|
+
['app', 'iosBundleId'],
|
|
224
|
+
['app', 'androidPackage'],
|
|
225
|
+
['paths', 'artifactRoot'],
|
|
226
|
+
['paths', 'iosArtifactsRoot'],
|
|
227
|
+
['paths', 'androidArtifactsRoot'],
|
|
228
|
+
['paths', 'scenarioRoot'],
|
|
229
|
+
];
|
|
230
|
+
const REQUIRED_PLATFORM_SCRIPT_PAIRS = [
|
|
231
|
+
['asl:check:ios', 'asl:check:android'],
|
|
232
|
+
['asl:profile:ios', 'asl:profile:android'],
|
|
233
|
+
['asl:profile:ios:provider', 'asl:profile:android:provider'],
|
|
234
|
+
['asl:profile:ios:live', 'asl:profile:android:live'],
|
|
235
|
+
['asl:ios:live', 'asl:android:live'],
|
|
236
|
+
['asl:ios:live:agent-device', 'asl:android:live:agent-device'],
|
|
237
|
+
['asl:ios:live:argent', 'asl:android:live:argent'],
|
|
238
|
+
['asl:ios:live:runners', 'asl:android:live:runners'],
|
|
239
|
+
['asl:agent-device:ios', 'asl:agent-device:android'],
|
|
240
|
+
['asl:argent:ios', 'asl:argent:android'],
|
|
241
|
+
['asl:compare:ios', 'asl:compare:android'],
|
|
242
|
+
['asl:live-proof:ios', 'asl:live-proof:android'],
|
|
243
|
+
];
|
|
244
|
+
const REQUIRED_STRICT_EXAMPLE_LIVE_SCRIPTS = [
|
|
245
|
+
'example:android:live',
|
|
246
|
+
'example:android:live:agent-device',
|
|
247
|
+
'example:android:live:argent',
|
|
248
|
+
'example:android:live:runners',
|
|
249
|
+
'example:ios:live',
|
|
250
|
+
'example:ios:live:agent-device',
|
|
251
|
+
'example:ios:live:argent',
|
|
252
|
+
'example:ios:live:runners',
|
|
253
|
+
];
|
|
254
|
+
/**
|
|
255
|
+
* Reads and parses a JSON object from disk.
|
|
256
|
+
*
|
|
257
|
+
* @param {string} filePath
|
|
258
|
+
* @returns {Record<string, unknown>}
|
|
259
|
+
*/
|
|
260
|
+
function readJsonObject(filePath) {
|
|
261
|
+
const value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
262
|
+
assert.equal(typeof value, 'object', `${filePath} must contain a JSON object`);
|
|
263
|
+
assert.notEqual(value, null, `${filePath} must contain a JSON object`);
|
|
264
|
+
assert.equal(Array.isArray(value), false, `${filePath} must contain a JSON object`);
|
|
265
|
+
return value;
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Reads a nested value from a JSON object.
|
|
269
|
+
*
|
|
270
|
+
* @param {Record<string, unknown>} object
|
|
271
|
+
* @param {string[]} pathSegments
|
|
272
|
+
* @returns {unknown}
|
|
273
|
+
*/
|
|
274
|
+
function readNestedValue(object, pathSegments) {
|
|
275
|
+
let value = object;
|
|
276
|
+
for (const segment of pathSegments) {
|
|
277
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
278
|
+
return undefined;
|
|
279
|
+
}
|
|
280
|
+
value = value[segment];
|
|
281
|
+
}
|
|
282
|
+
return value;
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Returns a string map property from a JSON object.
|
|
286
|
+
*
|
|
287
|
+
* @param {Record<string, unknown>} object
|
|
288
|
+
* @param {string} key
|
|
289
|
+
* @returns {Record<string, string>}
|
|
290
|
+
*/
|
|
291
|
+
function getStringMap(object, key) {
|
|
292
|
+
const value = object[key];
|
|
293
|
+
assert.equal(typeof value, 'object', `${key} must be an object`);
|
|
294
|
+
assert.notEqual(value, null, `${key} must be an object`);
|
|
295
|
+
assert.equal(Array.isArray(value), false, `${key} must be an object`);
|
|
296
|
+
for (const [entryKey, entryValue] of Object.entries(value)) {
|
|
297
|
+
assert.equal(typeof entryValue, 'string', `${key}.${entryKey} must be a string`);
|
|
298
|
+
}
|
|
299
|
+
return value;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Returns an object map property from a JSON object.
|
|
303
|
+
*
|
|
304
|
+
* @param {Record<string, unknown>} object
|
|
305
|
+
* @param {string} key
|
|
306
|
+
* @returns {Record<string, unknown>}
|
|
307
|
+
*/
|
|
308
|
+
function getObjectMap(object, key) {
|
|
309
|
+
const value = object[key];
|
|
310
|
+
assert.equal(typeof value, 'object', `${key} must be an object`);
|
|
311
|
+
assert.notEqual(value, null, `${key} must be an object`);
|
|
312
|
+
assert.equal(Array.isArray(value), false, `${key} must be an object`);
|
|
313
|
+
return value;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Returns a string array property from a JSON object.
|
|
317
|
+
*
|
|
318
|
+
* @param {Record<string, unknown>} object
|
|
319
|
+
* @param {string} key
|
|
320
|
+
* @returns {string[]}
|
|
321
|
+
*/
|
|
322
|
+
function getStringArray(object, key) {
|
|
323
|
+
const value = object[key];
|
|
324
|
+
assert.equal(Array.isArray(value), true, `${key} must be an array`);
|
|
325
|
+
for (const entry of value) {
|
|
326
|
+
assert.equal(typeof entry, 'string', `${key} entries must be strings`);
|
|
327
|
+
}
|
|
328
|
+
return value;
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Asserts that every expected value appears in an actual string list.
|
|
332
|
+
*
|
|
333
|
+
* @param {string[]} actual
|
|
334
|
+
* @param {string[]} expected
|
|
335
|
+
* @param {string} label
|
|
336
|
+
* @returns {void}
|
|
337
|
+
*/
|
|
338
|
+
function assertIncludesAll(actual, expected, label) {
|
|
339
|
+
for (const entry of expected) {
|
|
340
|
+
assert.equal(actual.includes(entry), true, `${label} is missing ${entry}`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Asserts that package metadata still describes the intended public package.
|
|
345
|
+
*
|
|
346
|
+
* @param {Record<string, unknown>} packageJson
|
|
347
|
+
* @returns {void}
|
|
348
|
+
*/
|
|
349
|
+
function assertPublicPackageMetadata(packageJson) {
|
|
350
|
+
assert.equal(packageJson.name, 'agent-scenario-loop');
|
|
351
|
+
assert.match(String(packageJson.version), /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/u);
|
|
352
|
+
assert.equal(packageJson.private, false);
|
|
353
|
+
assert.equal(packageJson.license, 'MIT');
|
|
354
|
+
assert.equal(packageJson.type, 'commonjs');
|
|
355
|
+
assert.equal(packageJson.main, 'dist/index.js');
|
|
356
|
+
assert.equal(packageJson.types, 'dist/index.d.ts');
|
|
357
|
+
assert.equal(getObjectMap(packageJson, 'publishConfig').access, 'public');
|
|
358
|
+
assert.equal(getObjectMap(packageJson, 'engines').node, '>=20');
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Asserts that release scripts keep npm publishing behind the full validation gate.
|
|
362
|
+
*
|
|
363
|
+
* @param {Record<string, unknown>} packageJson
|
|
364
|
+
* @returns {void}
|
|
365
|
+
*/
|
|
366
|
+
function assertReleaseScripts(packageJson) {
|
|
367
|
+
const scripts = getStringMap(packageJson, 'scripts');
|
|
368
|
+
assert.equal(scripts.prepublishOnly, 'pnpm release:check');
|
|
369
|
+
assert.equal(scripts['release:readiness'], 'pnpm build && node dist/scripts/release-readiness.js');
|
|
370
|
+
assert.equal(scripts['release:check'], 'pnpm test && pnpm release:readiness && pnpm package:smoke && pnpm consumer:rehearse');
|
|
371
|
+
assert.equal(scripts['package:smoke'], 'pnpm build && node dist/scripts/package-smoke.js');
|
|
372
|
+
assert.equal(scripts['consumer:rehearse'], 'pnpm build && node dist/scripts/consumer-rehearsal.js');
|
|
373
|
+
assert.equal(scripts['downstream:local-package'], 'pnpm build && node dist/scripts/downstream-local-package-gate.js');
|
|
374
|
+
for (const scriptName of REQUIRED_STRICT_EXAMPLE_LIVE_SCRIPTS) {
|
|
375
|
+
assert.match(scripts[scriptName], /--compare-latest/u, `${scriptName} must write comparison evidence by default`);
|
|
376
|
+
assert.match(scripts[scriptName], /--fail-on-regression/u, `${scriptName} must fail on regressions by default`);
|
|
377
|
+
}
|
|
378
|
+
assert.match(scripts['example:mobile:live-proof'], /--require-platforms android,ios/u);
|
|
379
|
+
assert.match(scripts['example:mobile:live-proof'], /--out artifacts\/example-mobile-app\/live-proof-set/u);
|
|
380
|
+
assert.match(scripts['example:mobile:live-proof'], /--fail-on-regression/u);
|
|
381
|
+
assert.match(scripts['example:app:start:isolated'], /--port 8097 --host localhost --clear/u);
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Asserts that public binaries and package subpath exports map to built files.
|
|
385
|
+
*
|
|
386
|
+
* @param {Record<string, unknown>} packageJson
|
|
387
|
+
* @returns {void}
|
|
388
|
+
*/
|
|
389
|
+
function assertPublicEntrypoints(packageJson) {
|
|
390
|
+
assert.deepEqual(getStringMap(packageJson, 'bin'), REQUIRED_BIN_TARGETS);
|
|
391
|
+
assert.deepEqual(getObjectMap(packageJson, 'exports'), REQUIRED_EXPORTS);
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Extracts documented runner subpath specifiers from the public API guide.
|
|
395
|
+
*
|
|
396
|
+
* @param {string} repoRoot
|
|
397
|
+
* @returns {string[]}
|
|
398
|
+
*/
|
|
399
|
+
function readDocumentedRunnerSubpaths(repoRoot) {
|
|
400
|
+
const apiDocs = fs.readFileSync(path.join(repoRoot, 'docs', 'api.md'), 'utf8');
|
|
401
|
+
const subpaths = new Set();
|
|
402
|
+
for (const match of apiDocs.matchAll(/`(agent-scenario-loop\/runner\/[^`]+)`/gu)) {
|
|
403
|
+
subpaths.add(match[1]);
|
|
404
|
+
}
|
|
405
|
+
return Array.from(subpaths).sort();
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Extracts concrete runner subpath specifiers from package exports.
|
|
409
|
+
*
|
|
410
|
+
* @param {Record<string, unknown>} packageJson
|
|
411
|
+
* @returns {string[]}
|
|
412
|
+
*/
|
|
413
|
+
function readExportedRunnerSubpaths(packageJson) {
|
|
414
|
+
return Object.keys(getObjectMap(packageJson, 'exports'))
|
|
415
|
+
.filter((subpath) => subpath.startsWith('./runner/'))
|
|
416
|
+
.map((subpath) => `agent-scenario-loop/${subpath.slice(2)}`)
|
|
417
|
+
.sort();
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Asserts that the public API guide and package exports describe the same runner surface.
|
|
421
|
+
*
|
|
422
|
+
* @param {Record<string, unknown>} packageJson
|
|
423
|
+
* @param {string} repoRoot
|
|
424
|
+
* @returns {void}
|
|
425
|
+
*/
|
|
426
|
+
function assertPublicApiDocs(packageJson, repoRoot) {
|
|
427
|
+
assert.deepEqual(readDocumentedRunnerSubpaths(repoRoot), readExportedRunnerSubpaths(packageJson), 'docs/api.md runner subpaths must match package.json exports');
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* Asserts that package include/exclude metadata protects generated local state.
|
|
431
|
+
*
|
|
432
|
+
* @param {Record<string, unknown>} packageJson
|
|
433
|
+
* @returns {void}
|
|
434
|
+
*/
|
|
435
|
+
function assertPackageFileList(packageJson) {
|
|
436
|
+
const files = getStringArray(packageJson, 'files');
|
|
437
|
+
assertIncludesAll(files, REQUIRED_PACKAGE_FILES, 'package files');
|
|
438
|
+
assertIncludesAll(files, REQUIRED_PACKAGE_EXCLUSIONS, 'package files');
|
|
439
|
+
for (const forbidden of ['.github', 'scripts', 'runner', 'node_modules', 'artifacts']) {
|
|
440
|
+
assert.equal(files.includes(forbidden), false, `package files must not include ${forbidden}`);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Asserts that ignored local state paths remain explicit in the repository.
|
|
445
|
+
*
|
|
446
|
+
* @param {string} repoRoot
|
|
447
|
+
* @returns {void}
|
|
448
|
+
*/
|
|
449
|
+
function assertGitignoreState(repoRoot) {
|
|
450
|
+
const gitignore = fs.readFileSync(path.join(repoRoot, '.gitignore'), 'utf8').split(/\r?\n/u);
|
|
451
|
+
assertIncludesAll(gitignore, REQUIRED_GITIGNORE_ENTRIES, '.gitignore');
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Asserts that generated runtime/native state is not tracked by git.
|
|
455
|
+
*
|
|
456
|
+
* @param {string} repoRoot
|
|
457
|
+
* @returns {void}
|
|
458
|
+
*/
|
|
459
|
+
function assertNoTrackedGeneratedState(repoRoot) {
|
|
460
|
+
const output = execFileSync('git', ['ls-files', '-z'], {
|
|
461
|
+
cwd: repoRoot,
|
|
462
|
+
encoding: 'utf8',
|
|
463
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
464
|
+
});
|
|
465
|
+
const trackedPaths = output.split('\0').filter(Boolean);
|
|
466
|
+
for (const trackedPath of trackedPaths) {
|
|
467
|
+
assert.equal(FORBIDDEN_TRACKED_PATH_PATTERNS.some((pattern) => pattern.test(trackedPath)), false, `generated state must not be tracked: ${trackedPath}`);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Asserts that shipped config templates contain identifiers needed by both mobile platforms.
|
|
472
|
+
*
|
|
473
|
+
* @param {string} repoRoot
|
|
474
|
+
* @returns {void}
|
|
475
|
+
*/
|
|
476
|
+
function assertConfigTemplates(repoRoot) {
|
|
477
|
+
for (const relativePath of ['core/config-template.json', 'templates/project.config.json']) {
|
|
478
|
+
const config = readJsonObject(path.join(repoRoot, relativePath));
|
|
479
|
+
for (const fieldPath of REQUIRED_CONFIG_STRING_FIELDS) {
|
|
480
|
+
const value = readNestedValue(config, fieldPath);
|
|
481
|
+
assert.equal(typeof value, 'string', `${relativePath} must include ${fieldPath.join('.')}`);
|
|
482
|
+
assert.notEqual(String(value).trim(), '', `${relativePath} must include non-empty ${fieldPath.join('.')}`);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* Asserts that shipped package-script snippets expose paired iOS and Android lanes.
|
|
488
|
+
*
|
|
489
|
+
* @param {string} repoRoot
|
|
490
|
+
* @returns {void}
|
|
491
|
+
*/
|
|
492
|
+
function assertPlatformPackageScripts(repoRoot) {
|
|
493
|
+
for (const relativePath of ['templates/package-scripts.json', 'examples/mobile-app/asl/package-scripts.json']) {
|
|
494
|
+
const scripts = readJsonObject(path.join(repoRoot, relativePath));
|
|
495
|
+
for (const [iosScriptName, androidScriptName] of REQUIRED_PLATFORM_SCRIPT_PAIRS) {
|
|
496
|
+
assert.equal(typeof scripts[iosScriptName], 'string', `${relativePath} must include ${iosScriptName}`);
|
|
497
|
+
assert.equal(typeof scripts[androidScriptName], 'string', `${relativePath} must include ${androidScriptName}`);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Runs the release-readiness assertions for the current repository checkout.
|
|
503
|
+
*
|
|
504
|
+
* @returns {void}
|
|
505
|
+
*/
|
|
506
|
+
function main() {
|
|
507
|
+
const repoRoot = path.resolve(__dirname, '..', '..');
|
|
508
|
+
const packageJson = readJsonObject(path.join(repoRoot, 'package.json'));
|
|
509
|
+
assertPublicPackageMetadata(packageJson);
|
|
510
|
+
assertReleaseScripts(packageJson);
|
|
511
|
+
assertPublicEntrypoints(packageJson);
|
|
512
|
+
assertPublicApiDocs(packageJson, repoRoot);
|
|
513
|
+
assertPackageFileList(packageJson);
|
|
514
|
+
assertGitignoreState(repoRoot);
|
|
515
|
+
assertNoTrackedGeneratedState(repoRoot);
|
|
516
|
+
assertConfigTemplates(repoRoot);
|
|
517
|
+
assertPlatformPackageScripts(repoRoot);
|
|
518
|
+
process.stdout.write('release readiness passed\n');
|
|
519
|
+
}
|
|
520
|
+
main();
|
package/docs/adapters.md
CHANGED
|
@@ -102,6 +102,8 @@ The built-in adb and simctl adapters show the expected boundary:
|
|
|
102
102
|
|
|
103
103
|
External tools such as agent-device, Argent, XcodeBuildMCP, axe, profilers, and custom scripts should plug in behind the same shape. The tactical tool can change; the scenario and artifact contract should not.
|
|
104
104
|
|
|
105
|
+
Prefer capability-based orchestration over forcing one tool to own every surface. Use adb and simctl as the primary live profile capture lanes for app launch, logs, screenshots, profile-session truth, and causal timelines. Attach heavier or tool-specific diagnostics after the active profile window through provider commands or rehydration. Agent Device is a good fit for Android snapshots and cross-platform network/performance evidence when its session is bound to the target. Argent is a good fit for iOS accessibility descriptions when `describe` can return AXRuntime evidence; native hierarchy, video, trace, React DevTools, and long profiler captures should be explicit heavy lanes until a runner/provider maps them into stable ASL artifacts.
|
|
106
|
+
|
|
105
107
|
## Preserve Evidence
|
|
106
108
|
|
|
107
109
|
Every run should leave agent-readable proof:
|
|
@@ -133,7 +135,7 @@ asl-profile-android \
|
|
|
133
135
|
--capture screenshot:artifacts/provider/final-screen.png
|
|
134
136
|
```
|
|
135
137
|
|
|
136
|
-
If the provider should run during
|
|
138
|
+
If the provider should run during profile artifact assembly, declare `providerCommands` in its manifest. Profile runners execute provider commands after the selected platform evidence source is available, including live `--adb-capture` / `--simctl-capture` runs and later `--adb-artifacts` / `--simctl-artifacts` rehydration runs. Use `phase: "afterCapture"` for diagnostics collected from an existing capture sidecar; use `phase: "postRun"` for evidence that should be understood as post-profile enrichment. Commands run without a shell, preserve stdout/stderr/exit code, and inventory outputs in `manifest.artifacts.evidenceAttachments`. Provider command outputs may set `required: true` so the matching diagnostic inventory entry is required when the provider successfully captures that output; scenario-authored required artifacts and capabilities remain canonical too. Runtime profiles reject a provider whose `platforms` do not include the selected platform before command execution, preserving the same active-provider semantics used by planner compatibility.
|
|
137
139
|
|
|
138
140
|
## Acceptance Checklist
|
|
139
141
|
|
package/docs/api.md
CHANGED
|
@@ -80,11 +80,11 @@ For concrete runner and evidence-provider integration steps, see [Adapter Onboar
|
|
|
80
80
|
|
|
81
81
|
## App Helper
|
|
82
82
|
|
|
83
|
-
`app/profile-session
|
|
83
|
+
`agent-scenario-loop/app/profile-session` is shipped as React Native source. Apps can copy `app/profile-session.ts` into their own codebase or re-export the package subpath from an app-local helper module. It is not a compiled CommonJS runtime export because it depends on app-side React Native modules, app bundling, and platform storage behavior.
|
|
84
84
|
|
|
85
85
|
The intended integration is:
|
|
86
86
|
|
|
87
|
-
1. Copy `app/profile-session.ts` into the app.
|
|
87
|
+
1. Copy `app/profile-session.ts` into the app, or re-export `agent-scenario-loop/app/profile-session` from an app-local helper.
|
|
88
88
|
2. Wire `useProfileSessionBootstrap()` once near the app root.
|
|
89
89
|
3. Emit app-owned truth events with `emitProfileEvent()`.
|
|
90
90
|
4. Register optional command targets with `registerProfileCommandTargetHandler()`.
|
package/docs/authoring.md
CHANGED
|
@@ -80,12 +80,14 @@ Preferred fields:
|
|
|
80
80
|
- `journey`: human-readable intent, actor, start state, and end state
|
|
81
81
|
- `comparisonLane`: default historical baseline lane for runs of this scenario
|
|
82
82
|
- `milestones`: named event checkpoints with phases and timeouts
|
|
83
|
-
- `cycles`: iteration count and
|
|
83
|
+
- `cycles`: iteration count, stop policy, and optional setup/body step ids
|
|
84
84
|
- `budgets`: thresholds to evaluate only after truth-event health passes
|
|
85
85
|
- `artifacts`: required and optional evidence outputs
|
|
86
86
|
|
|
87
87
|
Use `comparisonLane` when a scenario should always compare within one stable proof mode, such as `feed-open-android-live`. Profile CLIs can also receive `--comparison-lane`; the CLI flag wins when one-off runs need a different lane.
|
|
88
88
|
|
|
89
|
+
For repeated scenarios, separate setup from the measured body. Commands that clear state, navigate home, dismiss modals, or establish readiness should not be measured every iteration unless that cleanup is the journey under test. Use `cycles.setupStepIds` for leading setup commands that run once, or `cycles.bodyStepIds` to name the repeated command body. If neither is provided, ASL profile-session runners infer a conservative setup prefix from readiness waits and measured milestone budgets, but explicit ids are clearer for complex flows.
|
|
90
|
+
|
|
89
91
|
## Truth Events
|
|
90
92
|
|
|
91
93
|
Treat truth events as app-owned facts, not runner observations. The app should emit them from the code path that actually represents the journey state.
|
|
@@ -105,6 +107,32 @@ Weak truth events:
|
|
|
105
107
|
|
|
106
108
|
Timing is not trusted unless scenario health passes. If a required truth event is missing, the run can still write artifacts, but verdicts and comparisons must remain inconclusive.
|
|
107
109
|
|
|
110
|
+
### Resume Scenarios
|
|
111
|
+
|
|
112
|
+
`--lifecycle-phase resume` and related runner controls assert runner-owned lifecycle setup in `manifest.environment`; they do not create product truth events. If a scenario waits for `app_resumed`, `feed_restored_after_resume`, or another resumed-state milestone, the app must emit that event from the code path that proves resumed product readiness.
|
|
113
|
+
|
|
114
|
+
## Budget Intervals
|
|
115
|
+
|
|
116
|
+
Milestone budgets measure the interval the scenario names. A budget with only `toMilestone` measures elapsed time from the run or session clock origin to each matching milestone occurrence. That is correct for startup and first-usable-screen budgets, but it is cumulative for repeated interactions.
|
|
117
|
+
|
|
118
|
+
For transition or gesture budgets, provide both ends of the interval:
|
|
119
|
+
|
|
120
|
+
```json
|
|
121
|
+
{
|
|
122
|
+
"name": "surface transition p95",
|
|
123
|
+
"source": "milestone",
|
|
124
|
+
"metric": "p95",
|
|
125
|
+
"unit": "ms",
|
|
126
|
+
"limit": 300,
|
|
127
|
+
"fromMilestone": "surfaceTransitionRequested",
|
|
128
|
+
"toMilestone": "surfaceSettled"
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Use app-owned truth events for both milestones. Do not use a command-delivered event as the start point unless that command delivery is the product fact being measured.
|
|
133
|
+
|
|
134
|
+
When the start event is useful only as a timing anchor, keep it optional and keep scenario health tied to the completion truth. For repeated flows, set `metricEvents.milestone` or the completion-oriented cycle events to the truth that proves the iteration completed, then use the optional intent milestone as `fromMilestone` in the budget.
|
|
135
|
+
|
|
108
136
|
## Steps
|
|
109
137
|
|
|
110
138
|
Use steps to describe intent and required adapter actions:
|
|
@@ -118,6 +146,8 @@ Use steps to describe intent and required adapter actions:
|
|
|
118
146
|
|
|
119
147
|
Use `driverAction` only when the scenario truly requires a concrete operation such as `tap`, `scroll`, `assertVisible`, `screenshot`, `readLogs`, or `collectPerfSignals`. The planner fails early when no active runner or provider can satisfy a required driver action.
|
|
120
148
|
|
|
149
|
+
For profile-session command transport, platform `waitMs` metadata is queue pacing. ASL preserves it in storage and deep-link command envelopes and waits before releasing the next queued command. App-owned milestones still provide the truth that a command produced the intended product state.
|
|
150
|
+
|
|
121
151
|
Use `selector` to describe the intended app target without committing the scenario to one driver. Supported selector kinds are `testId`, `accessibilityId`, `accessibilityLabel`, `text`, `resourceId`, and `xpath`.
|
|
122
152
|
|
|
123
153
|
```json
|
|
@@ -161,10 +191,12 @@ asl-profile-android \
|
|
|
161
191
|
|
|
162
192
|
Signals are copied into `signals/js`, `signals/memory`, or `signals/network` and listed in `manifest.json`. Captures are copied into `captures`; screenshots are listed in `artifacts.captures.screenshots`, while video and UI tree captures replace the matching named capture path in the manifest. Every attached file is also listed in `artifacts.evidenceAttachments` with kind, run-relative path, source filename, byte size, sha256 hash, completeness status, corruption status, redaction status, and transformation list. Attached provider evidence is preserved as proof, but timing verdicts still come from app-owned truth events and budgets.
|
|
163
193
|
|
|
164
|
-
Provider manifests can also declare `providerCommands`. Profile runners execute those commands when passed with `--provider <manifest>`, but only when the provider manifest includes the selected platform. A provider with `platforms: ["ios"]` passed to an Android profile writes failed `health.json` with `provider_platform_unsupported` and does not run the command. Commands run without a shell, can use placeholders such as `{providerDir}`, `{runDir}`, `{runId}`, `{scenarioId}`, and `{platform}`, and must declare their output files. Provider-channel outputs are copied or preserved under `raw/providers/<provider-id>/` and inventoried in `artifacts.evidenceAttachments`; signal and capture outputs can still map into the standard `signals/*` or `captures/` folders. Command stdout, stderr, exit code, phase, and argv are preserved under `raw/provider-commands/`. When a provider command exits nonzero, the runner writes failed `health.json`, inconclusive `verdict.json`, and `agent-summary.md` with a next-action hint instead of making timing claims.
|
|
194
|
+
Provider manifests can also declare `providerCommands`. Profile runners execute those commands when passed with `--provider <manifest>`, but only when the provider manifest includes the selected platform. Commands run after the platform evidence source has been collected or supplied, so heavy diagnostics can be attached in a post-loop rehydration run with `--adb-artifacts` or `--simctl-artifacts` instead of perturbing the measured command window. Use `phase: "afterCapture"` for capture-sidecar diagnostics and `phase: "postRun"` for post-profile enrichment; older `capture` phase values remain accepted for existing manifests. A provider with `platforms: ["ios"]` passed to an Android profile writes failed `health.json` with `provider_platform_unsupported` and does not run the command. Commands run without a shell, can use placeholders such as `{providerDir}`, `{runDir}`, `{runId}`, `{scenarioId}`, and `{platform}`, and must declare their output files. Provider-channel outputs are copied or preserved under `raw/providers/<provider-id>/` and inventoried in `artifacts.evidenceAttachments`; signal and capture outputs can still map into the standard `signals/*` or `captures/` folders. An output can set `required: true` when the provider treats that file as required evidence; matching entries in `manifest.artifacts.diagnostics` then remain marked required in addition to scenario-authored `artifacts.required` and `requiredCapabilities`. Command stdout, stderr, exit code, phase, and argv are preserved under `raw/provider-commands/`. When a provider command exits nonzero, the runner writes failed `health.json`, inconclusive `verdict.json`, and `agent-summary.md` with a next-action hint instead of making timing claims.
|
|
165
195
|
|
|
166
196
|
The `examples/runners/script-*.json` manifests show package-neutral wrappers for accessibility, profiler, memory, and network evidence. They intentionally reference placeholder commands such as `capture-accessibility` or `capture-memory`; replace those with your project-local script, binary, or agent command. The contract that matters is the declared output path and evidence kind, not the specific tool used to create the file.
|
|
167
197
|
|
|
198
|
+
For React Native profiling, prefer a provider that emits both the raw profiler export and a structured JSON summary. JSON outputs with `kind: "profiler"` are validated against ASL's profiler evidence schema, so include the provider id, platform, run id, scenario id, tool metadata, completeness status, and at least one content surface such as samples, metrics, events, traces, a profile object, summary, or attachment references. If profiler evidence depends on explicit start/stop commands, model it as lifecycle-owned evidence: declare `captureMode`, `profileKind`, `lifecycle`, `targetBinding`, and `comparability` so agents can distinguish passive existing reports from session captures, inline captures that may perturb budgets, and after-capture or rehydrated diagnostics. CPU summaries derived from a prior profiler session should not be attached as passive evidence unless the provider also preserves the session provenance and raw attachments. If your profiler only produces a native trace or flamegraph, attach it as preserved evidence and avoid making performance claims until a provider translates the relevant facts into structured metrics.
|
|
199
|
+
|
|
168
200
|
## Artifacts
|
|
169
201
|
|
|
170
202
|
A completed profile run should leave the standard artifact set:
|