agent-scenario-loop 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/app/profile-session.ts +352 -12
- package/dist/core/agent-summary.d.ts +3 -2
- package/dist/core/agent-summary.js +44 -2
- package/dist/core/artifact-contract.d.ts +28 -8
- package/dist/core/artifact-contract.js +676 -26
- package/dist/core/comparison.d.ts +57 -3
- package/dist/core/comparison.js +113 -1
- package/dist/core/planner.d.ts +32 -1
- package/dist/core/planner.js +144 -0
- package/dist/core/run-index.d.ts +4 -0
- package/dist/core/run-index.js +55 -1
- package/dist/core/schema-validator.d.ts +2 -0
- package/dist/core/schema-validator.js +2 -0
- package/dist/runner/android-adb-driver.d.ts +7 -2
- package/dist/runner/android-adb-driver.js +7 -1
- package/dist/runner/android-adb.d.ts +40 -5
- package/dist/runner/android-adb.js +1046 -664
- package/dist/runner/compare-latest.d.ts +8 -4
- package/dist/runner/compare-latest.js +24 -5
- package/dist/runner/example-android-live.d.ts +10 -1
- package/dist/runner/example-android-live.js +55 -0
- package/dist/runner/example-ios-live.d.ts +10 -1
- package/dist/runner/example-ios-live.js +55 -0
- package/dist/runner/ios-simctl.d.ts +6 -0
- package/dist/runner/ios-simctl.js +7 -0
- package/dist/runner/live-comparison.d.ts +2 -2
- package/dist/runner/live-comparison.js +2 -1
- package/dist/runner/live-proof-summary.d.ts +5 -4
- package/dist/runner/live-proof-summary.js +12 -2
- package/dist/runner/live-proof.d.ts +3 -2
- package/dist/runner/live-proof.js +9 -2
- package/dist/runner/profile-android.d.ts +16 -1
- package/dist/runner/profile-android.js +364 -26
- package/dist/runner/profile-ios.d.ts +13 -2
- package/dist/runner/profile-ios.js +341 -19
- package/dist/runner/profile-mobile.d.ts +39 -3
- package/dist/runner/profile-mobile.js +1054 -42
- package/dist/runner/validate-project.js +3 -0
- package/dist/scripts/consumer-rehearsal.d.ts +119 -0
- package/dist/scripts/consumer-rehearsal.js +757 -0
- package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
- package/dist/scripts/downstream-local-package-gate.js +264 -0
- package/dist/scripts/package-smoke.d.ts +96 -0
- package/dist/scripts/package-smoke.js +2282 -0
- package/dist/scripts/release-readiness.d.ts +2 -0
- package/dist/scripts/release-readiness.js +520 -0
- package/docs/adapters.md +7 -1
- package/docs/api.md +2 -2
- package/docs/architecture.md +90 -0
- package/docs/authoring.md +39 -3
- package/docs/concepts.md +3 -24
- package/docs/consumer-rehearsal.md +31 -1
- package/docs/contracts.md +45 -101
- package/docs/external-adapter-protocol.md +219 -0
- package/docs/live-proofs.md +86 -3
- package/docs/principles.md +9 -15
- package/examples/mobile-app/README.md +12 -0
- package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
- package/examples/mobile-app/runner-manifests/primary-runner.json +1 -0
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
- package/examples/runners/README.md +4 -3
- package/examples/runners/adb-android.json +1 -0
- package/examples/runners/agent-device-android.json +1 -0
- package/examples/runners/agent-device-ios.json +1 -0
- package/examples/runners/argent-android.json +1 -0
- package/examples/runners/argent-ios.json +1 -0
- package/examples/runners/axe-accessibility-provider.json +2 -2
- package/examples/runners/script-accessibility-provider.json +2 -2
- package/examples/runners/script-memory-provider.json +2 -2
- package/examples/runners/script-network-provider.json +2 -2
- package/examples/runners/script-profiler-provider.json +2 -2
- package/examples/runners/xcodebuildmcp-ios.json +1 -0
- package/package.json +12 -3
- package/schemas/causal-run.schema.json +85 -2
- package/schemas/comparison.schema.json +130 -2
- package/schemas/external-adapter-message.schema.json +693 -0
- package/schemas/health.schema.json +72 -0
- package/schemas/live-proof-set.schema.json +1 -1
- package/schemas/live-proof.schema.json +14 -6
- package/schemas/manifest.schema.json +515 -4
- package/schemas/profiler.schema.json +243 -0
- package/schemas/runner-capabilities.schema.json +28 -2
- package/schemas/scenario.schema.json +34 -2
- package/templates/evidence-provider.json +3 -3
- package/templates/primary-runner.json +1 -0
- package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
|
@@ -28,12 +28,30 @@ const { runIosSimctlCapture } = require('./ios-simctl');
|
|
|
28
28
|
const { runAgentDeviceCapture } = require('./agent-device');
|
|
29
29
|
const { loadAslLocalEnv, readStringArgOrEnv } = require('./local-env');
|
|
30
30
|
const PROFILE_SESSION_CAPTURE_BOOTSTRAP_MS = 1000;
|
|
31
|
-
const
|
|
31
|
+
const PROFILE_SESSION_CAPTURE_COMMAND_OVERHEAD_MS = 250;
|
|
32
|
+
const PROFILE_SESSION_CAPTURE_BUFFER_MIN_MS = 2000;
|
|
33
|
+
const PROFILE_SESSION_CAPTURE_BUFFER_RATIO = 0.2;
|
|
34
|
+
const PROFILE_SESSION_CAPTURE_MAX_MS = 10 * 60 * 1000;
|
|
32
35
|
const DEFAULT_IOS_PROFILE_SESSION_STORAGE_KEY = 'agent-scenario-loop.profile-session.1';
|
|
33
36
|
const DEFAULT_IOS_PROFILE_COMMAND_STORAGE_KEY = 'agent-scenario-loop.profile-commands.1';
|
|
34
37
|
const DEFAULT_IOS_PROFILE_EVENT_STORAGE_KEY = 'agent-scenario-loop.profile-events.1';
|
|
35
38
|
const DEFAULT_IOS_PROFILE_SIGNAL_STORAGE_KEY = 'agent-scenario-loop.profile-signals.1';
|
|
36
39
|
const DEFAULT_IOS_PROFILE_SESSION_ENTRIES_STORAGE_KEY = 'agent-scenario-loop.profile-session-entries.1';
|
|
40
|
+
const MANIFEST_LIFECYCLE_PHASES = new Set([
|
|
41
|
+
'cold-launch',
|
|
42
|
+
'warm-launch',
|
|
43
|
+
'hot-launch',
|
|
44
|
+
'resume',
|
|
45
|
+
'foreground',
|
|
46
|
+
'background',
|
|
47
|
+
'force-stop',
|
|
48
|
+
'process-death',
|
|
49
|
+
'scene-recreation',
|
|
50
|
+
'activity-recreation',
|
|
51
|
+
'os-reclaim',
|
|
52
|
+
'reboot',
|
|
53
|
+
'relaunch',
|
|
54
|
+
]);
|
|
37
55
|
/**
|
|
38
56
|
* Reads and parses a JSON object from disk.
|
|
39
57
|
*
|
|
@@ -63,6 +81,22 @@ function readPositiveInteger(value, fallback) {
|
|
|
63
81
|
const parsed = typeof value === 'string' ? Number(value) : value;
|
|
64
82
|
return typeof parsed === 'number' && Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
65
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Resolves the lifecycle phase this runner is prepared to assert.
|
|
86
|
+
*
|
|
87
|
+
* @param {import('./profile-mobile').CliArgs} args
|
|
88
|
+
* @returns {string}
|
|
89
|
+
*/
|
|
90
|
+
function resolveManifestLifecyclePhase(args) {
|
|
91
|
+
const lifecyclePhase = readScalarArg(args['lifecycle-phase']);
|
|
92
|
+
if (lifecyclePhase === undefined) {
|
|
93
|
+
return 'cold-launch';
|
|
94
|
+
}
|
|
95
|
+
if (typeof lifecyclePhase !== 'string' || !MANIFEST_LIFECYCLE_PHASES.has(lifecyclePhase)) {
|
|
96
|
+
throw new Error(`Unsupported --lifecycle-phase "${String(lifecyclePhase)}". Expected one of ${Array.from(MANIFEST_LIFECYCLE_PHASES).join(', ')}.`);
|
|
97
|
+
}
|
|
98
|
+
return lifecyclePhase;
|
|
99
|
+
}
|
|
66
100
|
/**
|
|
67
101
|
* Reads the number of scenario iterations that can emit app-owned truth events.
|
|
68
102
|
*
|
|
@@ -140,7 +174,7 @@ function resolveIosConflictingBundleIds(config) {
|
|
|
140
174
|
* @param {{config: Record<string, unknown>, action: 'start' | 'command', scenario: string, runId: string, command?: string}} options
|
|
141
175
|
* @returns {string}
|
|
142
176
|
*/
|
|
143
|
-
function buildProfileSessionUrl({ action, command, config, runId, scenario, }) {
|
|
177
|
+
function buildProfileSessionUrl({ action, command, commandId, config, queueId, runId, scenario, sequence, waitForMilestone, waitMs, waitTimeoutMs, }) {
|
|
144
178
|
const scheme = typeof config.app?.profileSessionScheme === 'string'
|
|
145
179
|
? config.app.profileSessionScheme
|
|
146
180
|
: typeof config.app?.scheme === 'string'
|
|
@@ -149,6 +183,24 @@ function buildProfileSessionUrl({ action, command, config, runId, scenario, }) {
|
|
|
149
183
|
const params = new URLSearchParams({ runId, scenario });
|
|
150
184
|
if (action === 'command' && command) {
|
|
151
185
|
params.set('command', command);
|
|
186
|
+
if (commandId) {
|
|
187
|
+
params.set('commandId', commandId);
|
|
188
|
+
}
|
|
189
|
+
if (typeof sequence === 'number') {
|
|
190
|
+
params.set('sequence', String(sequence));
|
|
191
|
+
}
|
|
192
|
+
if (queueId) {
|
|
193
|
+
params.set('queueId', queueId);
|
|
194
|
+
}
|
|
195
|
+
if (waitForMilestone) {
|
|
196
|
+
params.set('waitForMilestone', waitForMilestone);
|
|
197
|
+
}
|
|
198
|
+
if (typeof waitMs === 'number') {
|
|
199
|
+
params.set('waitMs', String(waitMs));
|
|
200
|
+
}
|
|
201
|
+
if (typeof waitTimeoutMs === 'number') {
|
|
202
|
+
params.set('waitTimeoutMs', String(waitTimeoutMs));
|
|
203
|
+
}
|
|
152
204
|
}
|
|
153
205
|
return `${scheme}://profile-session/${action}?${params.toString()}`;
|
|
154
206
|
}
|
|
@@ -169,24 +221,49 @@ function readStepWaitMs(step) {
|
|
|
169
221
|
return readPositiveInteger(step.timeoutMs, 0);
|
|
170
222
|
}
|
|
171
223
|
/**
|
|
172
|
-
*
|
|
224
|
+
* Reads wait time from a profile-session command.
|
|
225
|
+
*
|
|
226
|
+
* @param {IosSimctlProfileCommand} command
|
|
227
|
+
* @returns {number}
|
|
228
|
+
*/
|
|
229
|
+
function readProfileCommandWindowMs(command) {
|
|
230
|
+
return readPositiveInteger(command.waitMs, 0) +
|
|
231
|
+
readPositiveInteger(command.waitTimeoutMs, 0) +
|
|
232
|
+
PROFILE_SESSION_CAPTURE_COMMAND_OVERHEAD_MS;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Reads execution-plan waits that are not already attached to a profile-session command.
|
|
173
236
|
*
|
|
174
237
|
* @param {Record<string, unknown>} scenario
|
|
175
238
|
* @returns {number}
|
|
176
239
|
*/
|
|
177
|
-
function
|
|
240
|
+
function readUnattachedExecutionWaitMs(scenario) {
|
|
178
241
|
const executionPlan = buildScenarioExecutionPlan(scenario);
|
|
179
242
|
const iterations = readScenarioIterationCount(scenario);
|
|
180
|
-
const perIterationWaitMs = executionPlan.steps.reduce((total, step) => {
|
|
181
|
-
if (step.
|
|
182
|
-
return total
|
|
243
|
+
const perIterationWaitMs = executionPlan.steps.reduce((total, step, index) => {
|
|
244
|
+
if (step.portMethod !== 'waitForTruthEvent') {
|
|
245
|
+
return total;
|
|
183
246
|
}
|
|
184
|
-
|
|
185
|
-
|
|
247
|
+
const previousStep = executionPlan.steps[index - 1];
|
|
248
|
+
if (previousStep?.portMethod === 'executeStep') {
|
|
249
|
+
return total;
|
|
186
250
|
}
|
|
187
|
-
return total;
|
|
251
|
+
return total + readPositiveInteger(step.timeoutMs, 0);
|
|
188
252
|
}, 0);
|
|
189
|
-
|
|
253
|
+
return perIterationWaitMs * iterations;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Derives a storage-backed profile capture window from scenario waits, command gates, and cycles.
|
|
257
|
+
*
|
|
258
|
+
* @param {Record<string, unknown>} scenario
|
|
259
|
+
* @returns {number}
|
|
260
|
+
*/
|
|
261
|
+
function deriveProfileSessionCaptureWaitMs(scenario) {
|
|
262
|
+
const commands = resolveIosSimctlProfileCommands(scenario);
|
|
263
|
+
const commandWindowMs = commands.reduce((total, command) => total + readProfileCommandWindowMs(command), 0);
|
|
264
|
+
const executionWindowMs = commandWindowMs + readUnattachedExecutionWaitMs(scenario);
|
|
265
|
+
const bufferMs = Math.max(PROFILE_SESSION_CAPTURE_BUFFER_MIN_MS, Math.ceil(executionWindowMs * PROFILE_SESSION_CAPTURE_BUFFER_RATIO));
|
|
266
|
+
const derivedWaitMs = PROFILE_SESSION_CAPTURE_BOOTSTRAP_MS + executionWindowMs + bufferMs;
|
|
190
267
|
return Math.min(Math.max(derivedWaitMs, PROFILE_SESSION_CAPTURE_BOOTSTRAP_MS), PROFILE_SESSION_CAPTURE_MAX_MS);
|
|
191
268
|
}
|
|
192
269
|
/**
|
|
@@ -211,14 +288,196 @@ function resolveProfileSessionCaptureWaitMs({ args, profileSessionEnabled, scena
|
|
|
211
288
|
function resolveExecutionPlanProfileCommands(scenario) {
|
|
212
289
|
const executionPlan = buildScenarioExecutionPlan(scenario);
|
|
213
290
|
const repeat = readPositiveInteger(scenario.defaultIterations, readPositiveInteger(scenario.cycles?.iterations, 1));
|
|
214
|
-
const commands =
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
291
|
+
const commands = [];
|
|
292
|
+
for (const [index, step] of executionPlan.steps.entries()) {
|
|
293
|
+
if (step.portMethod !== 'executeStep' || typeof step.command !== 'string') {
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
const nextStep = executionPlan.steps[index + 1];
|
|
297
|
+
commands.push({
|
|
298
|
+
command: step.command,
|
|
299
|
+
commandId: step.id,
|
|
300
|
+
label: step.id,
|
|
301
|
+
queueId: scenario.id ?? scenario.name,
|
|
302
|
+
waitMs: readStepWaitMs(step),
|
|
303
|
+
...(nextStep?.portMethod === 'waitForTruthEvent' && typeof nextStep.milestone === 'string'
|
|
304
|
+
? {
|
|
305
|
+
waitForMilestone: resolveMilestoneEventName(scenario, nextStep.milestone),
|
|
306
|
+
waitTimeoutMs: readPositiveInteger(nextStep.timeoutMs, 0),
|
|
307
|
+
}
|
|
308
|
+
: {}),
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
return expandProfileCommandCycles(scenario, commands, repeat);
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Returns true when a command is part of the setup prefix that establishes app readiness before repeated cycle work.
|
|
315
|
+
*
|
|
316
|
+
* @param {Record<string, unknown>} scenario
|
|
317
|
+
* @param {IosSimctlProfileCommand} command
|
|
318
|
+
* @returns {boolean}
|
|
319
|
+
*/
|
|
320
|
+
function isReadinessSetupProfileCommand(scenario, command) {
|
|
321
|
+
if (typeof command.waitForMilestone !== 'string') {
|
|
322
|
+
return false;
|
|
323
|
+
}
|
|
324
|
+
const readyEvent = resolveScenarioReadinessEvent(scenario);
|
|
325
|
+
return typeof readyEvent === 'string' && command.waitForMilestone === readyEvent;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Reads a string id list from scenario cycles metadata.
|
|
329
|
+
*
|
|
330
|
+
* @param {unknown} value
|
|
331
|
+
* @returns {Set<string>}
|
|
332
|
+
*/
|
|
333
|
+
function readCycleStepIdSet(value) {
|
|
334
|
+
return new Set(Array.isArray(value) ? value.filter((entry) => typeof entry === 'string') : []);
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Resolves the milestone ids that represent measured cycle boundaries.
|
|
338
|
+
*
|
|
339
|
+
* @param {Record<string, unknown>} scenario
|
|
340
|
+
* @returns {Set<string>}
|
|
341
|
+
*/
|
|
342
|
+
function resolveMeasuredCycleMilestoneEvents(scenario) {
|
|
343
|
+
const milestones = new Set();
|
|
344
|
+
for (const budget of Array.isArray(scenario.budgets) ? scenario.budgets : []) {
|
|
345
|
+
if (!budget || typeof budget !== 'object' || budget.source !== 'milestone') {
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
if (typeof budget.fromMilestone === 'string') {
|
|
349
|
+
milestones.add(resolveMilestoneEventName(scenario, budget.fromMilestone));
|
|
350
|
+
}
|
|
351
|
+
if (typeof budget.toMilestone === 'string') {
|
|
352
|
+
milestones.add(resolveMilestoneEventName(scenario, budget.toMilestone));
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return milestones;
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Resolves how many leading commands are setup-only before repeated cycle work.
|
|
359
|
+
*
|
|
360
|
+
* @param {Record<string, unknown>} scenario
|
|
361
|
+
* @param {IosSimctlProfileCommand[]} commands
|
|
362
|
+
* @returns {number}
|
|
363
|
+
*/
|
|
364
|
+
function resolveSetupCommandCount(scenario, commands) {
|
|
365
|
+
const explicitSetupStepIds = readCycleStepIdSet(scenario.cycles?.setupStepIds);
|
|
366
|
+
if (explicitSetupStepIds.size > 0) {
|
|
367
|
+
let count = 0;
|
|
368
|
+
for (const command of commands) {
|
|
369
|
+
if (!command.commandId || !explicitSetupStepIds.has(command.commandId)) {
|
|
370
|
+
break;
|
|
371
|
+
}
|
|
372
|
+
count += 1;
|
|
373
|
+
}
|
|
374
|
+
return count;
|
|
375
|
+
}
|
|
376
|
+
const explicitBodyStepIds = readCycleStepIdSet(scenario.cycles?.bodyStepIds);
|
|
377
|
+
if (explicitBodyStepIds.size > 0) {
|
|
378
|
+
const firstBodyIndex = commands.findIndex((command) => (typeof command.commandId === 'string' && explicitBodyStepIds.has(command.commandId)));
|
|
379
|
+
return firstBodyIndex > 0 ? firstBodyIndex : 0;
|
|
380
|
+
}
|
|
381
|
+
let readinessSetupCommandCount = 0;
|
|
382
|
+
for (const command of commands) {
|
|
383
|
+
if (!isReadinessSetupProfileCommand(scenario, command)) {
|
|
384
|
+
break;
|
|
385
|
+
}
|
|
386
|
+
readinessSetupCommandCount += 1;
|
|
387
|
+
}
|
|
388
|
+
if (readinessSetupCommandCount > 0) {
|
|
389
|
+
return readinessSetupCommandCount;
|
|
390
|
+
}
|
|
391
|
+
const measuredMilestones = resolveMeasuredCycleMilestoneEvents(scenario);
|
|
392
|
+
if (measuredMilestones.size === 0) {
|
|
393
|
+
return 0;
|
|
394
|
+
}
|
|
395
|
+
const firstMeasuredCommandIndex = commands.findIndex((command) => (typeof command.waitForMilestone === 'string' && measuredMilestones.has(command.waitForMilestone)));
|
|
396
|
+
return firstMeasuredCommandIndex > 0 ? firstMeasuredCommandIndex : 0;
|
|
397
|
+
}
|
|
398
|
+
/**
|
|
399
|
+
* Expands commands so setup/readiness commands execute once while cycle-body commands repeat.
|
|
400
|
+
*
|
|
401
|
+
* @param {Record<string, unknown>} scenario
|
|
402
|
+
* @param {IosSimctlProfileCommand[]} commands
|
|
403
|
+
* @param {number} repeat
|
|
404
|
+
* @returns {IosSimctlProfileCommand[]}
|
|
405
|
+
*/
|
|
406
|
+
function expandProfileCommandCycles(scenario, commands, repeat) {
|
|
407
|
+
const setupCommandCount = resolveSetupCommandCount(scenario, commands);
|
|
408
|
+
const setupCommands = commands.slice(0, setupCommandCount);
|
|
409
|
+
const cycleCommands = commands.slice(setupCommandCount);
|
|
410
|
+
const expandedCommands = cycleCommands.length === 0
|
|
411
|
+
? setupCommands
|
|
412
|
+
: [
|
|
413
|
+
...setupCommands,
|
|
414
|
+
...Array.from({ length: repeat }).flatMap(() => cycleCommands),
|
|
415
|
+
];
|
|
416
|
+
return expandedCommands.map((command, index) => ({
|
|
417
|
+
...command,
|
|
418
|
+
sequence: index + 1,
|
|
220
419
|
}));
|
|
221
|
-
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Resolves a portable milestone id to the app truth event that releases command sequencing.
|
|
423
|
+
*
|
|
424
|
+
* @param {Record<string, unknown>} scenario
|
|
425
|
+
* @param {string} milestone
|
|
426
|
+
* @returns {string}
|
|
427
|
+
*/
|
|
428
|
+
function resolveMilestoneEventName(scenario, milestone) {
|
|
429
|
+
const milestoneEntry = Array.isArray(scenario.milestones)
|
|
430
|
+
? scenario.milestones.find((entry) => entry?.id === milestone)
|
|
431
|
+
: undefined;
|
|
432
|
+
if (typeof milestoneEntry?.event === 'string' && milestoneEntry.event.length > 0) {
|
|
433
|
+
return milestoneEntry.event;
|
|
434
|
+
}
|
|
435
|
+
const metricEvent = scenario.metricEvents?.[milestone];
|
|
436
|
+
return typeof metricEvent === 'string' && metricEvent.length > 0 ? metricEvent : milestone;
|
|
437
|
+
}
|
|
438
|
+
/**
|
|
439
|
+
* Resolves the scenario truth event that represents initial app readiness.
|
|
440
|
+
*
|
|
441
|
+
* @param {Record<string, unknown>} scenario
|
|
442
|
+
* @returns {string | null}
|
|
443
|
+
*/
|
|
444
|
+
function resolveScenarioReadinessEvent(scenario) {
|
|
445
|
+
const explicitReadyEvent = scenario.truthEvents?.ready?.event;
|
|
446
|
+
if (typeof explicitReadyEvent === 'string' && explicitReadyEvent.length > 0) {
|
|
447
|
+
return explicitReadyEvent;
|
|
448
|
+
}
|
|
449
|
+
const milestoneEntry = Array.isArray(scenario.milestones)
|
|
450
|
+
? scenario.milestones.find((entry) => (String(entry?.event ?? '').includes('ready')))
|
|
451
|
+
: undefined;
|
|
452
|
+
return typeof milestoneEntry?.event === 'string' && milestoneEntry.event.length > 0
|
|
453
|
+
? milestoneEntry.event
|
|
454
|
+
: null;
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Applies wait gates from the normalized execution plan to platform-declared commands.
|
|
458
|
+
*
|
|
459
|
+
* @param {Record<string, unknown>} scenario
|
|
460
|
+
* @param {IosSimctlProfileCommand[]} commands
|
|
461
|
+
* @returns {IosSimctlProfileCommand[]}
|
|
462
|
+
*/
|
|
463
|
+
function applyExecutionPlanCommandGates(scenario, commands) {
|
|
464
|
+
const planCommands = resolveExecutionPlanProfileCommands(scenario);
|
|
465
|
+
if (planCommands.length === 0) {
|
|
466
|
+
return commands;
|
|
467
|
+
}
|
|
468
|
+
return commands.map((command, index) => {
|
|
469
|
+
const planCommand = planCommands[index];
|
|
470
|
+
if (!planCommand || typeof planCommand.waitForMilestone !== 'string' || typeof command.waitForMilestone === 'string') {
|
|
471
|
+
return command;
|
|
472
|
+
}
|
|
473
|
+
return {
|
|
474
|
+
...command,
|
|
475
|
+
waitForMilestone: planCommand.waitForMilestone,
|
|
476
|
+
...(typeof command.waitTimeoutMs === 'number'
|
|
477
|
+
? {}
|
|
478
|
+
: { waitTimeoutMs: readPositiveInteger(planCommand.waitTimeoutMs, 0) }),
|
|
479
|
+
};
|
|
480
|
+
});
|
|
222
481
|
}
|
|
223
482
|
/**
|
|
224
483
|
* Expands scenario-declared iOS commands for a simctl capture profile session.
|
|
@@ -240,12 +499,21 @@ function resolveIosSimctlProfileCommands(scenario) {
|
|
|
240
499
|
}
|
|
241
500
|
commands.push({
|
|
242
501
|
command: command.command,
|
|
502
|
+
commandId: typeof command.id === 'string'
|
|
503
|
+
? command.id
|
|
504
|
+
: typeof command.commandId === 'string'
|
|
505
|
+
? command.commandId
|
|
506
|
+
: typeof command.label === 'string'
|
|
507
|
+
? command.label
|
|
508
|
+
: command.command,
|
|
243
509
|
...(typeof command.label === 'string' ? { label: command.label } : {}),
|
|
510
|
+
queueId: scenario.id ?? scenario.name,
|
|
511
|
+
sequence: commands.length + 1,
|
|
244
512
|
waitMs: readPositiveInteger(command.waitMs, 0),
|
|
245
513
|
});
|
|
246
514
|
}
|
|
247
515
|
}
|
|
248
|
-
return commands;
|
|
516
|
+
return applyExecutionPlanCommandGates(scenario, commands);
|
|
249
517
|
}
|
|
250
518
|
/**
|
|
251
519
|
* Returns true when the scenario asks iOS simctl capture to preserve a screenshot.
|
|
@@ -327,6 +595,7 @@ function appendAgentDeviceCaptureArgs({ args, capture, }) {
|
|
|
327
595
|
async function runProfileIos(args, options = {}) {
|
|
328
596
|
if (!isEnabled(args['simctl-capture']) && !isEnabled(args['agent-device-capture'])) {
|
|
329
597
|
return runProfileMobile(args, {
|
|
598
|
+
commandTransport: typeof args.events === 'string' ? 'fixture-log-ingest' : 'simctl-artifacts',
|
|
330
599
|
...(options.comparisonLane ? { comparisonLane: options.comparisonLane } : {}),
|
|
331
600
|
defaultDriver: 'ios-simctl',
|
|
332
601
|
...(typeof args['simctl-artifacts'] === 'string' ? { interactionDriver: 'ios-simctl' } : {}),
|
|
@@ -397,9 +666,15 @@ async function runProfileIos(args, options = {}) {
|
|
|
397
666
|
url: buildProfileSessionUrl({
|
|
398
667
|
action: 'command',
|
|
399
668
|
command: profileCommand.command,
|
|
669
|
+
...(typeof profileCommand.commandId === 'string' ? { commandId: profileCommand.commandId } : {}),
|
|
400
670
|
config,
|
|
401
671
|
runId,
|
|
402
672
|
scenario: scenarioName,
|
|
673
|
+
...(typeof profileCommand.queueId === 'string' ? { queueId: profileCommand.queueId } : {}),
|
|
674
|
+
...(typeof profileCommand.sequence === 'number' ? { sequence: profileCommand.sequence } : {}),
|
|
675
|
+
...(typeof profileCommand.waitForMilestone === 'string' ? { waitForMilestone: profileCommand.waitForMilestone } : {}),
|
|
676
|
+
...(typeof profileCommand.waitMs === 'number' ? { waitMs: profileCommand.waitMs } : {}),
|
|
677
|
+
...(typeof profileCommand.waitTimeoutMs === 'number' ? { waitTimeoutMs: profileCommand.waitTimeoutMs } : {}),
|
|
403
678
|
}),
|
|
404
679
|
waitMs: profileCommand.waitMs,
|
|
405
680
|
})),
|
|
@@ -431,8 +706,14 @@ async function runProfileIos(args, options = {}) {
|
|
|
431
706
|
profileSessionStorage: {
|
|
432
707
|
commands: profileSessionCommands.map((profileCommand, index) => ({
|
|
433
708
|
command: profileCommand.command,
|
|
709
|
+
...(typeof profileCommand.commandId === 'string' ? { commandId: profileCommand.commandId } : {}),
|
|
434
710
|
id: `ios-storage-command-${index + 1}`,
|
|
435
711
|
...(typeof profileCommand.label === 'string' ? { label: profileCommand.label } : {}),
|
|
712
|
+
...(typeof profileCommand.queueId === 'string' ? { queueId: profileCommand.queueId } : {}),
|
|
713
|
+
...(typeof profileCommand.sequence === 'number' ? { sequence: profileCommand.sequence } : {}),
|
|
714
|
+
...(typeof profileCommand.waitForMilestone === 'string' ? { waitForMilestone: profileCommand.waitForMilestone } : {}),
|
|
715
|
+
...(typeof profileCommand.waitMs === 'number' ? { waitMs: profileCommand.waitMs } : {}),
|
|
716
|
+
...(typeof profileCommand.waitTimeoutMs === 'number' ? { waitTimeoutMs: profileCommand.waitTimeoutMs } : {}),
|
|
436
717
|
})),
|
|
437
718
|
runId,
|
|
438
719
|
scenario: scenarioName,
|
|
@@ -496,9 +777,50 @@ async function runProfileIos(args, options = {}) {
|
|
|
496
777
|
const profileArgs = agentDeviceCapture
|
|
497
778
|
? appendAgentDeviceCaptureArgs({ args: baseProfileArgs, capture: agentDeviceCapture })
|
|
498
779
|
: baseProfileArgs;
|
|
780
|
+
const lifecyclePhase = resolveManifestLifecyclePhase(args);
|
|
781
|
+
const environmentSource = agentDeviceCapture ? 'agent-device' : 'simctl';
|
|
782
|
+
const copiedSimctlLogArtifact = simctlCapture &&
|
|
783
|
+
!fs.existsSync(path.join(simctlCapture.runDir, 'raw', 'ios-profile-events.log')) &&
|
|
784
|
+
fs.existsSync(path.join(simctlCapture.runDir, 'raw', 'ios-simctl-log.txt'))
|
|
785
|
+
? 'raw/ios-simctl-log.txt'
|
|
786
|
+
: undefined;
|
|
499
787
|
return runProfileMobile(profileArgs, {
|
|
788
|
+
commandTransport: agentDeviceCapture
|
|
789
|
+
? 'agent-device'
|
|
790
|
+
: profileSessionEnabled && !profileSessionStorageEnabled
|
|
791
|
+
? 'profile-session-deeplink'
|
|
792
|
+
: profileSessionEnabled
|
|
793
|
+
? 'profile-session-storage'
|
|
794
|
+
: 'simctl-capture',
|
|
500
795
|
...(options.comparisonLane ? { comparisonLane: options.comparisonLane } : {}),
|
|
501
796
|
defaultDriver: 'ios-simctl',
|
|
797
|
+
environmentPostconditions: {
|
|
798
|
+
appState: {
|
|
799
|
+
value: 'foreground',
|
|
800
|
+
evidence: 'asserted',
|
|
801
|
+
source: environmentSource,
|
|
802
|
+
...(copiedSimctlLogArtifact ? { artifact: copiedSimctlLogArtifact } : {}),
|
|
803
|
+
},
|
|
804
|
+
lifecyclePhase: {
|
|
805
|
+
value: 'foreground',
|
|
806
|
+
evidence: 'asserted',
|
|
807
|
+
source: environmentSource,
|
|
808
|
+
...(copiedSimctlLogArtifact ? { artifact: copiedSimctlLogArtifact } : {}),
|
|
809
|
+
},
|
|
810
|
+
},
|
|
811
|
+
environmentPreconditions: {
|
|
812
|
+
foregroundState: {
|
|
813
|
+
value: 'controlled-by-runner',
|
|
814
|
+
evidence: 'asserted',
|
|
815
|
+
source: environmentSource,
|
|
816
|
+
},
|
|
817
|
+
lifecyclePhase: {
|
|
818
|
+
value: lifecyclePhase,
|
|
819
|
+
evidence: 'asserted',
|
|
820
|
+
source: environmentSource,
|
|
821
|
+
...(copiedSimctlLogArtifact ? { artifact: copiedSimctlLogArtifact } : {}),
|
|
822
|
+
},
|
|
823
|
+
},
|
|
502
824
|
interactionDriver: agentDeviceCapture ? 'agent-device' : 'ios-simctl',
|
|
503
825
|
platform: 'ios',
|
|
504
826
|
});
|
|
@@ -21,31 +21,61 @@ type ProfileRunResult = {
|
|
|
21
21
|
};
|
|
22
22
|
type ProfilePlatform = 'android' | 'ios';
|
|
23
23
|
type ProfileMobileOptions = {
|
|
24
|
+
commandTransport?: string;
|
|
24
25
|
comparisonLane?: string;
|
|
25
26
|
defaultDriver: string;
|
|
27
|
+
environmentPostconditions?: Record<string, unknown>;
|
|
28
|
+
environmentPreconditions?: Record<string, unknown>;
|
|
26
29
|
interactionDriver?: string;
|
|
27
30
|
platform: ProfilePlatform;
|
|
31
|
+
provenanceCohort?: Record<string, unknown>;
|
|
28
32
|
};
|
|
29
33
|
type CaptureEvidenceKind = 'screenshot' | 'uiTree' | 'video';
|
|
30
34
|
type ProviderEvidenceKind = 'accessibility' | 'logs' | 'profiler';
|
|
31
35
|
type SignalEvidenceKind = 'js' | 'memory' | 'network';
|
|
32
36
|
type EvidenceChannel = 'capture' | 'provider' | 'signal';
|
|
33
37
|
type EvidenceKind = CaptureEvidenceKind | ProviderEvidenceKind | SignalEvidenceKind;
|
|
38
|
+
type DiagnosticStatus = 'captured' | 'not_requested' | 'not_supported' | 'unavailable' | 'failed' | 'skipped' | 'missing';
|
|
39
|
+
type DiagnosticKind = EvidenceKind | 'logs';
|
|
40
|
+
type DiagnosticInventoryEntry = {
|
|
41
|
+
kind: DiagnosticKind;
|
|
42
|
+
status: DiagnosticStatus;
|
|
43
|
+
required: boolean;
|
|
44
|
+
name?: string;
|
|
45
|
+
provider?: string;
|
|
46
|
+
runnerId?: string;
|
|
47
|
+
path?: string;
|
|
48
|
+
reason?: string;
|
|
49
|
+
nextAction?: string;
|
|
50
|
+
sidecarRoot?: string;
|
|
51
|
+
evidenceDependency?: {
|
|
52
|
+
kind: string;
|
|
53
|
+
root?: 'run' | 'sidecar';
|
|
54
|
+
path: string;
|
|
55
|
+
};
|
|
56
|
+
};
|
|
34
57
|
type EvidenceAttachment = {
|
|
35
58
|
channel: EvidenceChannel;
|
|
59
|
+
completenessStatus: 'complete';
|
|
60
|
+
corruptionStatus: 'valid';
|
|
36
61
|
destinationPath: string;
|
|
37
62
|
kind: EvidenceKind;
|
|
38
63
|
manifestPath: string;
|
|
64
|
+
redactionStatus: 'not-redacted';
|
|
65
|
+
required: boolean;
|
|
39
66
|
sha256: string;
|
|
40
67
|
sourcePath: string;
|
|
41
68
|
sourceFileName: string;
|
|
42
69
|
sizeBytes: number;
|
|
70
|
+
transformations: readonly ['copied'];
|
|
43
71
|
};
|
|
44
72
|
type EvidenceAttachmentInput = {
|
|
45
73
|
channel: EvidenceChannel;
|
|
46
74
|
destinationPath: string;
|
|
47
75
|
kind: EvidenceKind;
|
|
48
76
|
manifestPath: string;
|
|
77
|
+
providerId?: string;
|
|
78
|
+
required?: boolean;
|
|
49
79
|
sourcePath: string;
|
|
50
80
|
};
|
|
51
81
|
type AttachedEvidence = {
|
|
@@ -62,6 +92,7 @@ type ProviderCommandOutput = {
|
|
|
62
92
|
channel: EvidenceChannel;
|
|
63
93
|
kind: EvidenceKind;
|
|
64
94
|
path: string;
|
|
95
|
+
required?: boolean;
|
|
65
96
|
};
|
|
66
97
|
type ProviderCommand = {
|
|
67
98
|
args?: string[];
|
|
@@ -70,7 +101,7 @@ type ProviderCommand = {
|
|
|
70
101
|
env?: Record<string, string>;
|
|
71
102
|
id: string;
|
|
72
103
|
outputs: ProviderCommandOutput[];
|
|
73
|
-
phase: 'prepare' | 'startWindow' | 'capture' | 'stopWindow' | 'finalize';
|
|
104
|
+
phase: 'prepare' | 'startWindow' | 'capture' | 'stopWindow' | 'afterCapture' | 'postRun' | 'finalize';
|
|
74
105
|
};
|
|
75
106
|
type ProviderCommandFailure = {
|
|
76
107
|
commandId: string;
|
|
@@ -131,13 +162,18 @@ declare function resolveAttachedEvidence({ args, layout, providerInputs, }: {
|
|
|
131
162
|
/**
|
|
132
163
|
* Builds scenario health from profile metrics.
|
|
133
164
|
*
|
|
134
|
-
* @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>}} options
|
|
165
|
+
* @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>, diagnostics?: DiagnosticInventoryEntry[], profileEventCount?: number, profileSessionEntryCount?: number, commandTransport?: string, sessionEntries?: Record<string, unknown>[]}} options
|
|
135
166
|
* @returns {Record<string, unknown>}
|
|
136
167
|
*/
|
|
137
|
-
declare function buildProfileHealth({ scenario, runId, metrics, }: {
|
|
168
|
+
declare function buildProfileHealth({ scenario, runId, metrics, diagnostics, profileEventCount, profileSessionEntryCount, commandTransport, sessionEntries, }: {
|
|
138
169
|
scenario: Record<string, any>;
|
|
139
170
|
runId: string;
|
|
140
171
|
metrics: Record<string, any>;
|
|
172
|
+
diagnostics?: DiagnosticInventoryEntry[];
|
|
173
|
+
profileEventCount?: number;
|
|
174
|
+
profileSessionEntryCount?: number;
|
|
175
|
+
commandTransport?: string;
|
|
176
|
+
sessionEntries?: Record<string, any>[];
|
|
141
177
|
}): Record<string, unknown>;
|
|
142
178
|
/**
|
|
143
179
|
* Builds failed scenario health from evidence-provider command failures.
|