agent-scenario-loop 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/app/profile-session.ts +352 -12
- package/dist/core/agent-summary.d.ts +3 -2
- package/dist/core/agent-summary.js +44 -2
- package/dist/core/artifact-contract.d.ts +28 -8
- package/dist/core/artifact-contract.js +676 -26
- package/dist/core/comparison.d.ts +57 -3
- package/dist/core/comparison.js +113 -1
- package/dist/core/planner.d.ts +32 -1
- package/dist/core/planner.js +144 -0
- package/dist/core/run-index.d.ts +4 -0
- package/dist/core/run-index.js +55 -1
- package/dist/core/schema-validator.d.ts +2 -0
- package/dist/core/schema-validator.js +2 -0
- package/dist/runner/android-adb-driver.d.ts +7 -2
- package/dist/runner/android-adb-driver.js +7 -1
- package/dist/runner/android-adb.d.ts +40 -5
- package/dist/runner/android-adb.js +1046 -664
- package/dist/runner/compare-latest.d.ts +8 -4
- package/dist/runner/compare-latest.js +24 -5
- package/dist/runner/example-android-live.d.ts +10 -1
- package/dist/runner/example-android-live.js +55 -0
- package/dist/runner/example-ios-live.d.ts +10 -1
- package/dist/runner/example-ios-live.js +55 -0
- package/dist/runner/ios-simctl.d.ts +6 -0
- package/dist/runner/ios-simctl.js +7 -0
- package/dist/runner/live-comparison.d.ts +2 -2
- package/dist/runner/live-comparison.js +2 -1
- package/dist/runner/live-proof-summary.d.ts +5 -4
- package/dist/runner/live-proof-summary.js +12 -2
- package/dist/runner/live-proof.d.ts +3 -2
- package/dist/runner/live-proof.js +9 -2
- package/dist/runner/profile-android.d.ts +16 -1
- package/dist/runner/profile-android.js +364 -26
- package/dist/runner/profile-ios.d.ts +13 -2
- package/dist/runner/profile-ios.js +341 -19
- package/dist/runner/profile-mobile.d.ts +39 -3
- package/dist/runner/profile-mobile.js +1054 -42
- package/dist/runner/validate-project.js +3 -0
- package/dist/scripts/consumer-rehearsal.d.ts +119 -0
- package/dist/scripts/consumer-rehearsal.js +757 -0
- package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
- package/dist/scripts/downstream-local-package-gate.js +264 -0
- package/dist/scripts/package-smoke.d.ts +96 -0
- package/dist/scripts/package-smoke.js +2282 -0
- package/dist/scripts/release-readiness.d.ts +2 -0
- package/dist/scripts/release-readiness.js +520 -0
- package/docs/adapters.md +7 -1
- package/docs/api.md +2 -2
- package/docs/architecture.md +90 -0
- package/docs/authoring.md +39 -3
- package/docs/concepts.md +3 -24
- package/docs/consumer-rehearsal.md +31 -1
- package/docs/contracts.md +45 -101
- package/docs/external-adapter-protocol.md +219 -0
- package/docs/live-proofs.md +86 -3
- package/docs/principles.md +9 -15
- package/examples/mobile-app/README.md +12 -0
- package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
- package/examples/mobile-app/runner-manifests/primary-runner.json +1 -0
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
- package/examples/runners/README.md +4 -3
- package/examples/runners/adb-android.json +1 -0
- package/examples/runners/agent-device-android.json +1 -0
- package/examples/runners/agent-device-ios.json +1 -0
- package/examples/runners/argent-android.json +1 -0
- package/examples/runners/argent-ios.json +1 -0
- package/examples/runners/axe-accessibility-provider.json +2 -2
- package/examples/runners/script-accessibility-provider.json +2 -2
- package/examples/runners/script-memory-provider.json +2 -2
- package/examples/runners/script-network-provider.json +2 -2
- package/examples/runners/script-profiler-provider.json +2 -2
- package/examples/runners/xcodebuildmcp-ios.json +1 -0
- package/package.json +12 -3
- package/schemas/causal-run.schema.json +85 -2
- package/schemas/comparison.schema.json +130 -2
- package/schemas/external-adapter-message.schema.json +693 -0
- package/schemas/health.schema.json +72 -0
- package/schemas/live-proof-set.schema.json +1 -1
- package/schemas/live-proof.schema.json +14 -6
- package/schemas/manifest.schema.json +515 -4
- package/schemas/profiler.schema.json +243 -0
- package/schemas/runner-capabilities.schema.json +28 -2
- package/schemas/scenario.schema.json +34 -2
- package/templates/evidence-provider.json +3 -3
- package/templates/primary-runner.json +1 -0
- package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
|
@@ -26,7 +26,7 @@ const crypto = require('node:crypto');
|
|
|
26
26
|
const { buildAgentSummaryMarkdown } = require('../core/agent-summary');
|
|
27
27
|
const { createArtifactLayout } = require('../core/artifact-layout');
|
|
28
28
|
const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
|
|
29
|
-
const { buildBudgetVerdict, buildCausalRun, buildCausalTimeline, buildManifest, buildMetricsFromProfileEvents, buildSummaryMarkdown, extractProfileEvents, } = require('../core/artifact-contract');
|
|
29
|
+
const { buildBudgetVerdict, buildCausalRun, buildCausalTimeline, buildManifest, buildMetricsFromProfileEvents, buildSummaryMarkdown, extractProfileEvents, extractProfileSessionEntries, } = require('../core/artifact-contract');
|
|
30
30
|
const { SCHEMAS, assertValidJson } = require('../core/schema-validator');
|
|
31
31
|
const { writeUsage } = require('./cli');
|
|
32
32
|
const CAPTURE_EVIDENCE_KINDS = new Set(['screenshot', 'uiTree', 'video']);
|
|
@@ -49,7 +49,7 @@ function usage({ binaryName, output = process.stderr, platform, }) {
|
|
|
49
49
|
];
|
|
50
50
|
if (platform === 'android') {
|
|
51
51
|
lines.push('Use --adb-artifacts <dir> to read raw/adb-logcat.txt from a prior asl-android-adb capture.');
|
|
52
|
-
lines.push('Use --adb-capture [--clear-logcat] [--launch] [--launch-wait-ms <ms>] [--wait-ms <ms>] to capture adb logcat before profiling.');
|
|
52
|
+
lines.push('Use --adb-capture [--clear-logcat] [--launch] [--launch-wait-ms <ms>] [--wait-ms <ms>] [--adb-command-timeout-ms <ms>] to capture adb logcat before profiling.');
|
|
53
53
|
lines.push('Use --android-dev-client-url <url> [--android-dev-client-wait-ms <ms>] [--android-dev-client-ready-pattern <pattern>] with --adb-capture to open an Expo dev-client session before profile-session deep links.');
|
|
54
54
|
lines.push('Use --android-profile-session-storage with --profile-session to seed startup control through Android AsyncStorage.');
|
|
55
55
|
lines.push('Use --profile-session with --adb-capture to start the app profile session and execute scenario-declared Android commands.');
|
|
@@ -62,6 +62,7 @@ function usage({ binaryName, output = process.stderr, platform, }) {
|
|
|
62
62
|
}
|
|
63
63
|
lines.push('Use --agent-device-capture to execute scenario-declared portable driver actions through agent-device and attach its captures.');
|
|
64
64
|
lines.push('Use --agent-device-session-mode bind when a named agent-device session should still receive the configured Android serial or iOS UDID.');
|
|
65
|
+
lines.push('Use --lifecycle-phase <phase> when the runner can explicitly assert a non-cold lifecycle precondition such as warm-launch or resume.');
|
|
65
66
|
writeUsage(lines, output);
|
|
66
67
|
}
|
|
67
68
|
/**
|
|
@@ -250,6 +251,8 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
|
|
|
250
251
|
destinationPath: path.join(layout.signals[kind], fileName),
|
|
251
252
|
kind,
|
|
252
253
|
manifestPath: `signals/${kind}/${fileName}`,
|
|
254
|
+
providerId,
|
|
255
|
+
...(typeof output.required === 'boolean' ? { required: output.required } : {}),
|
|
253
256
|
sourcePath,
|
|
254
257
|
};
|
|
255
258
|
}
|
|
@@ -262,6 +265,8 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
|
|
|
262
265
|
destinationPath: path.join(layout.captures, fileName),
|
|
263
266
|
kind: output.kind,
|
|
264
267
|
manifestPath: `captures/${fileName}`,
|
|
268
|
+
providerId,
|
|
269
|
+
...(typeof output.required === 'boolean' ? { required: output.required } : {}),
|
|
265
270
|
sourcePath,
|
|
266
271
|
};
|
|
267
272
|
}
|
|
@@ -273,9 +278,33 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
|
|
|
273
278
|
destinationPath: path.join(layout.raw, 'providers', providerId, fileName),
|
|
274
279
|
kind: output.kind,
|
|
275
280
|
manifestPath: `raw/providers/${providerId}/${fileName}`,
|
|
281
|
+
providerId,
|
|
282
|
+
...(typeof output.required === 'boolean' ? { required: output.required } : {}),
|
|
276
283
|
sourcePath,
|
|
277
284
|
};
|
|
278
285
|
}
|
|
286
|
+
/**
|
|
287
|
+
* Validates structured profiler evidence when a provider emits JSON.
|
|
288
|
+
*
|
|
289
|
+
* Native traces and flamegraph files may be attached as profiler evidence, but
|
|
290
|
+
* JSON profiler files must carry enough envelope metadata for agents to reason
|
|
291
|
+
* about source, target, and completeness.
|
|
292
|
+
*
|
|
293
|
+
* @param {{kind: EvidenceKind, sourcePath: string}} options
|
|
294
|
+
* @returns {void}
|
|
295
|
+
*/
|
|
296
|
+
function validateStructuredProfilerEvidence({ kind, sourcePath, }) {
|
|
297
|
+
if (kind !== 'profiler' || path.extname(sourcePath).toLowerCase() !== '.json') {
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
assertValidJson(readJson(sourcePath), SCHEMAS.profiler, 'Profiler evidence artifact');
|
|
302
|
+
}
|
|
303
|
+
catch (error) {
|
|
304
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
305
|
+
throw new Error(`Profiler evidence artifact is invalid: ${sourcePath}. ${detail}`);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
279
308
|
/**
|
|
280
309
|
* Fails when provider command ids would collide in raw command records.
|
|
281
310
|
*
|
|
@@ -300,9 +329,10 @@ function assertUniqueProviderCommandIds({ providerCommands = [], providerId, })
|
|
|
300
329
|
async function executeProviderCommands({ args, layout, platform, runDir, runId, scenarioId, }) {
|
|
301
330
|
const failures = [];
|
|
302
331
|
const inputs = [];
|
|
332
|
+
const providers = [];
|
|
303
333
|
const providerManifestPaths = readRepeatableArgValues(args, 'provider');
|
|
304
334
|
if (providerManifestPaths.length === 0) {
|
|
305
|
-
return { failures, inputs };
|
|
335
|
+
return { failures, inputs, providers };
|
|
306
336
|
}
|
|
307
337
|
const commandRecordDir = path.join(layout.raw, 'provider-commands');
|
|
308
338
|
await ensureDir(commandRecordDir);
|
|
@@ -314,6 +344,10 @@ async function executeProviderCommands({ args, layout, platform, runDir, runId,
|
|
|
314
344
|
throw new Error(`Provider manifest must use kind "evidenceProvider": ${absoluteManifestPath}`);
|
|
315
345
|
}
|
|
316
346
|
const providerId = safeProviderSegment(String(provider.runnerId ?? path.basename(absoluteManifestPath, '.json')));
|
|
347
|
+
providers.push({
|
|
348
|
+
name: providerId,
|
|
349
|
+
...(typeof provider.version === 'string' ? { version: provider.version } : {}),
|
|
350
|
+
});
|
|
317
351
|
if (Array.isArray(provider.platforms) && !provider.platforms.includes(platform)) {
|
|
318
352
|
failures.push({
|
|
319
353
|
commandId: 'platform-compatibility',
|
|
@@ -392,7 +426,7 @@ async function executeProviderCommands({ args, layout, platform, runDir, runId,
|
|
|
392
426
|
}
|
|
393
427
|
}
|
|
394
428
|
}
|
|
395
|
-
return { failures, inputs };
|
|
429
|
+
return { failures, inputs, providers };
|
|
396
430
|
}
|
|
397
431
|
/**
|
|
398
432
|
* Converts internal attachment copy plans into manifest-safe metadata.
|
|
@@ -403,11 +437,15 @@ async function executeProviderCommands({ args, layout, platform, runDir, runId,
|
|
|
403
437
|
function buildEvidenceAttachmentManifest(attachments) {
|
|
404
438
|
return attachments.map((attachment) => ({
|
|
405
439
|
channel: attachment.channel,
|
|
440
|
+
completenessStatus: attachment.completenessStatus,
|
|
441
|
+
corruptionStatus: attachment.corruptionStatus,
|
|
406
442
|
kind: attachment.kind,
|
|
407
443
|
path: attachment.manifestPath,
|
|
444
|
+
redactionStatus: attachment.redactionStatus,
|
|
408
445
|
sha256: attachment.sha256,
|
|
409
446
|
sizeBytes: attachment.sizeBytes,
|
|
410
447
|
sourceFileName: attachment.sourceFileName,
|
|
448
|
+
transformations: attachment.transformations,
|
|
411
449
|
}));
|
|
412
450
|
}
|
|
413
451
|
/**
|
|
@@ -432,7 +470,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
|
|
|
432
470
|
},
|
|
433
471
|
};
|
|
434
472
|
const destinationPaths = new Set();
|
|
435
|
-
const addCopy = async ({ channel, destinationPath, kind, manifestPath, sourcePath, }) => {
|
|
473
|
+
const addCopy = async ({ channel, destinationPath, kind, manifestPath, required = false, sourcePath, }) => {
|
|
436
474
|
const stat = await fsp.stat(sourcePath).catch(() => null);
|
|
437
475
|
if (!stat?.isFile()) {
|
|
438
476
|
throw new Error(`Evidence artifact does not exist or is not a file: ${sourcePath}`);
|
|
@@ -440,16 +478,22 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
|
|
|
440
478
|
if (destinationPaths.has(destinationPath)) {
|
|
441
479
|
throw new Error(`Duplicate evidence artifact destination: ${manifestPath}`);
|
|
442
480
|
}
|
|
481
|
+
validateStructuredProfilerEvidence({ kind, sourcePath });
|
|
443
482
|
destinationPaths.add(destinationPath);
|
|
444
483
|
const attachment = {
|
|
445
484
|
channel,
|
|
485
|
+
completenessStatus: 'complete',
|
|
486
|
+
corruptionStatus: 'valid',
|
|
446
487
|
destinationPath,
|
|
447
488
|
kind,
|
|
448
489
|
manifestPath,
|
|
490
|
+
redactionStatus: 'not-redacted',
|
|
491
|
+
required,
|
|
449
492
|
sha256: await hashFileSha256(sourcePath),
|
|
450
493
|
sourceFileName: path.basename(sourcePath),
|
|
451
494
|
sourcePath,
|
|
452
495
|
sizeBytes: stat.size,
|
|
496
|
+
transformations: ['copied'],
|
|
453
497
|
};
|
|
454
498
|
attached.attachments.push(attachment);
|
|
455
499
|
attached.copies.push(attachment);
|
|
@@ -564,20 +608,447 @@ function toPortablePathReference(targetPath) {
|
|
|
564
608
|
}
|
|
565
609
|
return path.basename(targetPath);
|
|
566
610
|
}
|
|
611
|
+
/**
|
|
612
|
+
* Returns a path reference from one run folder to an external sidecar.
|
|
613
|
+
*
|
|
614
|
+
* @param {{runDir: string, targetPath: string}} options
|
|
615
|
+
* @returns {string}
|
|
616
|
+
*/
|
|
617
|
+
function toRunPathReference({ runDir, targetPath }) {
|
|
618
|
+
const relativePath = path.relative(runDir, targetPath);
|
|
619
|
+
return relativePath.length > 0 ? relativePath : path.basename(targetPath);
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* Returns a sidecar dependency path that stays readable in rehydrated artifacts.
|
|
623
|
+
*
|
|
624
|
+
* @param {{runDir: string, sidecarRoot: string, targetPath: string}} options
|
|
625
|
+
* @returns {SidecarEvidenceDependency}
|
|
626
|
+
*/
|
|
627
|
+
function toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath, }) {
|
|
628
|
+
const sidecarRelativePath = path.relative(sidecarRoot, targetPath);
|
|
629
|
+
if (sidecarRelativePath.length > 0 &&
|
|
630
|
+
!sidecarRelativePath.startsWith('..') &&
|
|
631
|
+
!path.isAbsolute(sidecarRelativePath)) {
|
|
632
|
+
return {
|
|
633
|
+
kind: 'sidecar',
|
|
634
|
+
root: 'sidecar',
|
|
635
|
+
path: sidecarRelativePath,
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
return {
|
|
639
|
+
kind: 'sidecar',
|
|
640
|
+
path: toRunPathReference({ runDir, targetPath }),
|
|
641
|
+
};
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Reads scenario string-list declarations into a set.
|
|
645
|
+
*
|
|
646
|
+
* @param {Record<string, unknown>} scenario
|
|
647
|
+
* @param {string[]} pathSegments
|
|
648
|
+
* @returns {Set<string>}
|
|
649
|
+
*/
|
|
650
|
+
function readScenarioStringSet(scenario, pathSegments) {
|
|
651
|
+
const values = pathSegments.reduce((current, segment) => (current && typeof current === 'object' && !Array.isArray(current)
|
|
652
|
+
? current[segment]
|
|
653
|
+
: undefined), scenario);
|
|
654
|
+
return new Set(Array.isArray(values)
|
|
655
|
+
? values.filter((value) => typeof value === 'string')
|
|
656
|
+
: []);
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Returns true when a scenario artifact declaration matches a diagnostic kind.
|
|
660
|
+
*
|
|
661
|
+
* @param {Set<string>} artifacts
|
|
662
|
+
* @param {string[]} aliases
|
|
663
|
+
* @returns {boolean}
|
|
664
|
+
*/
|
|
665
|
+
function artifactSetHasAny(artifacts, aliases) {
|
|
666
|
+
return aliases.some((alias) => artifacts.has(alias));
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Resolves common aliases used by scenario artifact contracts.
|
|
670
|
+
*
|
|
671
|
+
* @param {DiagnosticKind} kind
|
|
672
|
+
* @returns {string[]}
|
|
673
|
+
*/
|
|
674
|
+
function diagnosticArtifactAliases(kind) {
|
|
675
|
+
const aliases = {
|
|
676
|
+
accessibility: ['accessibility'],
|
|
677
|
+
js: ['js', 'profileEvents', 'profileSession'],
|
|
678
|
+
logs: ['logs', 'deviceLog', 'interactionLog'],
|
|
679
|
+
memory: ['memory'],
|
|
680
|
+
network: ['network'],
|
|
681
|
+
profiler: ['profiler', 'profile'],
|
|
682
|
+
screenshot: ['screenshot', 'screenshots'],
|
|
683
|
+
uiTree: ['uiTree', 'ui-tree', 'accessibilityTree'],
|
|
684
|
+
video: ['video', 'recording'],
|
|
685
|
+
};
|
|
686
|
+
return aliases[kind];
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* Resolves common aliases used by runner capability declarations.
|
|
690
|
+
*
|
|
691
|
+
* @param {DiagnosticKind} kind
|
|
692
|
+
* @returns {string[]}
|
|
693
|
+
*/
|
|
694
|
+
function diagnosticCapabilityAliases(kind) {
|
|
695
|
+
const aliases = {
|
|
696
|
+
accessibility: ['accessibility', 'accessibilityCapture'],
|
|
697
|
+
js: ['js', 'profileSession', 'profileEvents'],
|
|
698
|
+
logs: ['logCapture', 'logs', 'deviceLog'],
|
|
699
|
+
memory: ['memory', 'memoryCapture'],
|
|
700
|
+
network: ['network', 'networkCapture'],
|
|
701
|
+
profiler: ['profiler', 'profile', 'profilerCapture'],
|
|
702
|
+
screenshot: ['screenshot', 'screenshots'],
|
|
703
|
+
uiTree: ['uiTree', 'ui-tree', 'accessibilityTree'],
|
|
704
|
+
video: ['video', 'recording'],
|
|
705
|
+
};
|
|
706
|
+
return aliases[kind];
|
|
707
|
+
}
|
|
708
|
+
/**
|
|
709
|
+
* Returns requirement/request metadata for one diagnostic kind.
|
|
710
|
+
*
|
|
711
|
+
* @param {{kind: DiagnosticKind, optionalArtifacts: Set<string>, optionalCapabilities: Set<string>, requiredArtifacts: Set<string>, requiredCapabilities: Set<string>}} options
|
|
712
|
+
* @returns {{required: boolean, requested: boolean}}
|
|
713
|
+
*/
|
|
714
|
+
function resolveDiagnosticRequest({ kind, optionalArtifacts, optionalCapabilities, requiredArtifacts, requiredCapabilities, }) {
|
|
715
|
+
const artifactAliases = diagnosticArtifactAliases(kind);
|
|
716
|
+
const capabilityAliases = diagnosticCapabilityAliases(kind);
|
|
717
|
+
const required = artifactSetHasAny(requiredArtifacts, artifactAliases) ||
|
|
718
|
+
artifactSetHasAny(requiredCapabilities, capabilityAliases);
|
|
719
|
+
return {
|
|
720
|
+
required,
|
|
721
|
+
requested: required ||
|
|
722
|
+
artifactSetHasAny(optionalArtifacts, artifactAliases) ||
|
|
723
|
+
artifactSetHasAny(optionalCapabilities, capabilityAliases),
|
|
724
|
+
};
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* Builds a status entry for one diagnostic surface.
|
|
728
|
+
*
|
|
729
|
+
* @param {DiagnosticInventoryEntry & {requested?: boolean}} entry
|
|
730
|
+
* @returns {DiagnosticInventoryEntry}
|
|
731
|
+
*/
|
|
732
|
+
function buildDiagnosticEntry(entry) {
|
|
733
|
+
const { requested = true, ...diagnostic } = entry;
|
|
734
|
+
if (diagnostic.status === 'captured' || requested || diagnostic.required) {
|
|
735
|
+
return diagnostic;
|
|
736
|
+
}
|
|
737
|
+
return {
|
|
738
|
+
...diagnostic,
|
|
739
|
+
status: 'not_requested',
|
|
740
|
+
reason: diagnostic.reason ?? 'Scenario did not request this optional diagnostic surface.',
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Builds the product-neutral diagnostic inventory for a profile run.
|
|
745
|
+
*
|
|
746
|
+
* @param {{args: CliArgs, attachedEvidence: AttachedEvidence, eventLogPath: string | null, platform: ProfilePlatform, profileSessionEntriesPath: string | null, runDir: string, scenario: Record<string, unknown>}} options
|
|
747
|
+
* @returns {DiagnosticInventoryEntry[]}
|
|
748
|
+
*/
|
|
749
|
+
function buildDiagnosticInventory({ args, attachedEvidence, eventLogPath, platform, profileSessionEntriesPath, runDir, scenario, }) {
|
|
750
|
+
const requiredArtifacts = readScenarioStringSet(scenario, ['artifacts', 'required']);
|
|
751
|
+
const optionalArtifacts = readScenarioStringSet(scenario, ['artifacts', 'optional']);
|
|
752
|
+
const requiredCapabilities = readScenarioStringSet(scenario, ['requiredCapabilities']);
|
|
753
|
+
const optionalCapabilities = readScenarioStringSet(scenario, ['optionalCapabilities']);
|
|
754
|
+
const requiredProviderDiagnostics = new Set(attachedEvidence.attachments
|
|
755
|
+
.filter((attachment) => attachment.required)
|
|
756
|
+
.map((attachment) => attachment.kind));
|
|
757
|
+
const sidecarRoot = typeof args['adb-artifacts'] === 'string'
|
|
758
|
+
? path.resolve(args['adb-artifacts'])
|
|
759
|
+
: typeof args['simctl-artifacts'] === 'string'
|
|
760
|
+
? path.resolve(args['simctl-artifacts'])
|
|
761
|
+
: null;
|
|
762
|
+
const sidecarRootRef = sidecarRoot ? toRunPathReference({ runDir, targetPath: sidecarRoot }) : undefined;
|
|
763
|
+
const adbScreenshotDependency = platform === 'android'
|
|
764
|
+
? resolveAndroidAdbScreenshotDependency({ runDir, sidecarRoot })
|
|
765
|
+
: null;
|
|
766
|
+
const eventLogBaseName = eventLogPath ? path.basename(eventLogPath) : undefined;
|
|
767
|
+
const eventLogManifestPath = eventLogBaseName ? `raw/${eventLogBaseName}` : undefined;
|
|
768
|
+
const eventLogIsIosProfileEvents = platform === 'ios' && eventLogBaseName === 'ios-profile-events.log';
|
|
769
|
+
const simctlRuntimeLogPath = typeof args['simctl-artifacts'] === 'string'
|
|
770
|
+
? path.resolve(args['simctl-artifacts'], 'raw', 'ios-simctl-log.txt')
|
|
771
|
+
: null;
|
|
772
|
+
const simctlRuntimeLogExists = Boolean(simctlRuntimeLogPath && fs.existsSync(simctlRuntimeLogPath));
|
|
773
|
+
const simctlRuntimeLogDependency = simctlRuntimeLogPath && simctlRuntimeLogExists
|
|
774
|
+
? toSidecarEvidenceDependency({ runDir, sidecarRoot: path.resolve(args['simctl-artifacts']), targetPath: simctlRuntimeLogPath })
|
|
775
|
+
: undefined;
|
|
776
|
+
const copiedSimctlLogManifestPath = platform === 'ios' && eventLogPath && path.basename(eventLogPath) === 'ios-simctl-log.txt'
|
|
777
|
+
? eventLogManifestPath
|
|
778
|
+
: undefined;
|
|
779
|
+
const explicitIosRuntimeLogManifestPath = platform === 'ios' &&
|
|
780
|
+
typeof args.events === 'string' &&
|
|
781
|
+
eventLogManifestPath &&
|
|
782
|
+
!eventLogIsIosProfileEvents
|
|
783
|
+
? eventLogManifestPath
|
|
784
|
+
: undefined;
|
|
785
|
+
const eventLogDependency = eventLogPath && sidecarRoot
|
|
786
|
+
? toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath: eventLogPath })
|
|
787
|
+
: undefined;
|
|
788
|
+
const jsProfilePath = attachedEvidence.signals.js[0] ?? eventLogManifestPath;
|
|
789
|
+
const profileSessionEntriesManifestPath = profileSessionEntriesPath
|
|
790
|
+
? `raw/${path.basename(profileSessionEntriesPath)}`
|
|
791
|
+
: undefined;
|
|
792
|
+
const entries = [];
|
|
793
|
+
const pushDiagnostic = (kind, entry) => {
|
|
794
|
+
const request = resolveDiagnosticRequest({
|
|
795
|
+
kind,
|
|
796
|
+
optionalArtifacts,
|
|
797
|
+
optionalCapabilities,
|
|
798
|
+
requiredArtifacts,
|
|
799
|
+
requiredCapabilities,
|
|
800
|
+
});
|
|
801
|
+
entries.push(buildDiagnosticEntry({
|
|
802
|
+
kind,
|
|
803
|
+
...entry,
|
|
804
|
+
required: request.required || requiredProviderDiagnostics.has(kind) || Boolean(entry.required),
|
|
805
|
+
requested: request.requested || requiredProviderDiagnostics.has(kind) || Boolean(entry.requested),
|
|
806
|
+
}));
|
|
807
|
+
};
|
|
808
|
+
const logCaptured = platform === 'ios'
|
|
809
|
+
? Boolean(copiedSimctlLogManifestPath || simctlRuntimeLogDependency || explicitIosRuntimeLogManifestPath)
|
|
810
|
+
: Boolean(eventLogManifestPath);
|
|
811
|
+
pushDiagnostic('logs', {
|
|
812
|
+
name: platform === 'ios' ? 'simulator-runtime-log' : 'device-log',
|
|
813
|
+
...(typeof args['adb-artifacts'] === 'string'
|
|
814
|
+
? { provider: 'adb', runnerId: 'android-adb' }
|
|
815
|
+
: typeof args['simctl-artifacts'] === 'string'
|
|
816
|
+
? { provider: 'simctl', runnerId: 'ios-simctl' }
|
|
817
|
+
: typeof args.events === 'string'
|
|
818
|
+
? { provider: 'fixture-log-ingest' }
|
|
819
|
+
: {}),
|
|
820
|
+
status: logCaptured ? 'captured' : 'unavailable',
|
|
821
|
+
...(platform === 'ios'
|
|
822
|
+
? copiedSimctlLogManifestPath
|
|
823
|
+
? { path: copiedSimctlLogManifestPath }
|
|
824
|
+
: simctlRuntimeLogDependency
|
|
825
|
+
? { path: simctlRuntimeLogDependency.path }
|
|
826
|
+
: explicitIosRuntimeLogManifestPath
|
|
827
|
+
? { path: explicitIosRuntimeLogManifestPath }
|
|
828
|
+
: {}
|
|
829
|
+
: eventLogManifestPath
|
|
830
|
+
? { path: eventLogManifestPath }
|
|
831
|
+
: {}),
|
|
832
|
+
...(sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
|
|
833
|
+
...(platform === 'ios'
|
|
834
|
+
? simctlRuntimeLogDependency
|
|
835
|
+
? { evidenceDependency: simctlRuntimeLogDependency }
|
|
836
|
+
: {}
|
|
837
|
+
: eventLogDependency
|
|
838
|
+
? { evidenceDependency: eventLogDependency }
|
|
839
|
+
: {}),
|
|
840
|
+
...(logCaptured
|
|
841
|
+
? {
|
|
842
|
+
reason: platform === 'ios'
|
|
843
|
+
? 'iOS simulator runtime log evidence was available from the simctl capture sidecar.'
|
|
844
|
+
: 'Device or fixture log evidence was available to the profile runner.',
|
|
845
|
+
}
|
|
846
|
+
: {
|
|
847
|
+
reason: platform === 'ios'
|
|
848
|
+
? 'No iOS simulator runtime log was available in the selected simctl capture sidecar.'
|
|
849
|
+
: 'No device log source was supplied to this profile run.',
|
|
850
|
+
nextAction: platform === 'ios'
|
|
851
|
+
? 'Run with --simctl-capture or provide --simctl-artifacts containing raw/ios-simctl-log.txt.'
|
|
852
|
+
: 'Run with --events, --adb-artifacts, --adb-capture, or provide a runtime log artifact.',
|
|
853
|
+
}),
|
|
854
|
+
});
|
|
855
|
+
pushDiagnostic('js', {
|
|
856
|
+
name: 'profile-session-evidence',
|
|
857
|
+
status: eventLogManifestPath || attachedEvidence.signals.js.length > 0 ? 'captured' : 'unavailable',
|
|
858
|
+
...(jsProfilePath ? { path: jsProfilePath } : {}),
|
|
859
|
+
...(profileSessionEntriesManifestPath
|
|
860
|
+
? {
|
|
861
|
+
evidenceDependency: {
|
|
862
|
+
kind: 'profile-session-entries',
|
|
863
|
+
path: profileSessionEntriesManifestPath,
|
|
864
|
+
},
|
|
865
|
+
}
|
|
866
|
+
: eventLogDependency
|
|
867
|
+
? { evidenceDependency: eventLogDependency }
|
|
868
|
+
: {}),
|
|
869
|
+
...(sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
|
|
870
|
+
...(eventLogManifestPath || attachedEvidence.signals.js.length > 0
|
|
871
|
+
? { reason: 'Profile or JS evidence was captured from runner input.' }
|
|
872
|
+
: {
|
|
873
|
+
reason: 'No profile-session event log or JS signal attachment was available.',
|
|
874
|
+
nextAction: 'Attach JS evidence with --signal js:<path> or run a profile-session capture that emits profile events.',
|
|
875
|
+
}),
|
|
876
|
+
});
|
|
877
|
+
const attachedScreenshotPath = attachedEvidence.captures.screenshots[0];
|
|
878
|
+
const sidecarScreenshotDependency = attachedScreenshotPath ? null : adbScreenshotDependency;
|
|
879
|
+
pushDiagnostic('screenshot', {
|
|
880
|
+
...(sidecarScreenshotDependency ? { provider: 'adb', runnerId: 'android-adb' } : {}),
|
|
881
|
+
status: attachedScreenshotPath || sidecarScreenshotDependency ? 'captured' : 'unavailable',
|
|
882
|
+
...(attachedScreenshotPath
|
|
883
|
+
? { path: attachedScreenshotPath }
|
|
884
|
+
: sidecarScreenshotDependency
|
|
885
|
+
? { path: sidecarScreenshotDependency.path }
|
|
886
|
+
: {}),
|
|
887
|
+
...(sidecarScreenshotDependency && sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
|
|
888
|
+
...(sidecarScreenshotDependency ? { evidenceDependency: sidecarScreenshotDependency.dependency } : {}),
|
|
889
|
+
...(attachedScreenshotPath || sidecarScreenshotDependency
|
|
890
|
+
? {
|
|
891
|
+
reason: sidecarScreenshotDependency
|
|
892
|
+
? 'Screenshot evidence was available from the adb capture sidecar.'
|
|
893
|
+
: 'Screenshot capture was attached to the run.',
|
|
894
|
+
}
|
|
895
|
+
: {
|
|
896
|
+
reason: 'No screenshot capture was produced by the selected runner/provider set.',
|
|
897
|
+
nextAction: 'Use --capture screenshot:<path> or a runner/provider that produces screenshots.',
|
|
898
|
+
}),
|
|
899
|
+
});
|
|
900
|
+
pushDiagnostic('uiTree', {
|
|
901
|
+
status: attachedEvidence.captures.uiTree ? 'captured' : 'unavailable',
|
|
902
|
+
...(attachedEvidence.captures.uiTree ? { path: attachedEvidence.captures.uiTree } : {}),
|
|
903
|
+
...(attachedEvidence.captures.uiTree
|
|
904
|
+
? { reason: 'UI tree capture was attached to the run.' }
|
|
905
|
+
: {
|
|
906
|
+
reason: 'No UI tree capture was produced by the selected runner/provider set.',
|
|
907
|
+
nextAction: 'Use --capture uiTree:<path> or add an accessibility/UI-tree provider.',
|
|
908
|
+
}),
|
|
909
|
+
});
|
|
910
|
+
pushDiagnostic('video', {
|
|
911
|
+
status: attachedEvidence.captures.video ? 'captured' : 'unavailable',
|
|
912
|
+
...(attachedEvidence.captures.video ? { path: attachedEvidence.captures.video } : {}),
|
|
913
|
+
...(attachedEvidence.captures.video
|
|
914
|
+
? { reason: 'Video capture was attached to the run.' }
|
|
915
|
+
: {
|
|
916
|
+
reason: 'No video capture was produced by the selected runner/provider set.',
|
|
917
|
+
nextAction: 'Use --capture video:<path> or run a capture provider that records video.',
|
|
918
|
+
}),
|
|
919
|
+
});
|
|
920
|
+
for (const kind of ['memory', 'network']) {
|
|
921
|
+
pushDiagnostic(kind, {
|
|
922
|
+
status: attachedEvidence.signals[kind].length > 0 ? 'captured' : 'unavailable',
|
|
923
|
+
...(attachedEvidence.signals[kind][0] ? { path: attachedEvidence.signals[kind][0] } : {}),
|
|
924
|
+
...(attachedEvidence.signals[kind].length > 0
|
|
925
|
+
? { reason: `${kind} signal evidence was attached to the run.` }
|
|
926
|
+
: {
|
|
927
|
+
reason: `No ${kind} signal evidence was produced by the selected provider set.`,
|
|
928
|
+
nextAction: `Attach ${kind} evidence with --signal ${kind}:<path> or add a provider command that emits it.`,
|
|
929
|
+
}),
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
for (const kind of ['accessibility', 'profiler']) {
|
|
933
|
+
const attachment = attachedEvidence.attachments.find((item) => item.kind === kind);
|
|
934
|
+
pushDiagnostic(kind, {
|
|
935
|
+
...(attachment?.channel === 'provider' ? { provider: 'evidence-provider' } : {}),
|
|
936
|
+
status: attachment ? 'captured' : 'unavailable',
|
|
937
|
+
...(attachment ? { path: attachment.manifestPath } : {}),
|
|
938
|
+
...(attachment
|
|
939
|
+
? { reason: `${kind} provider evidence was attached to the run.` }
|
|
940
|
+
: {
|
|
941
|
+
reason: `No ${kind} provider attachment was produced by the selected provider set.`,
|
|
942
|
+
nextAction: `Declare a provider command or attach ${kind} evidence before expecting this diagnostic.`,
|
|
943
|
+
}),
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
return entries.map((entry) => {
|
|
947
|
+
const cleaned = Object.entries(entry).filter(([, value]) => value !== undefined);
|
|
948
|
+
return Object.fromEntries(cleaned);
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Converts uncaptured required diagnostics into health checks.
|
|
953
|
+
*
|
|
954
|
+
* @param {DiagnosticInventoryEntry[]} diagnostics
|
|
955
|
+
* @returns {Record<string, unknown>[]}
|
|
956
|
+
*/
|
|
957
|
+
function buildRequiredDiagnosticHealthChecks(diagnostics = []) {
|
|
958
|
+
return diagnostics
|
|
959
|
+
.filter((diagnostic) => diagnostic.required && diagnostic.status !== 'captured')
|
|
960
|
+
.map((diagnostic) => ({
|
|
961
|
+
name: `required_${diagnostic.kind}_diagnostic`,
|
|
962
|
+
status: 'failed',
|
|
963
|
+
source: 'evidence',
|
|
964
|
+
code: 'required_diagnostic_not_captured',
|
|
965
|
+
message: diagnostic.reason ?? `Required ${diagnostic.kind} diagnostic was not captured.`,
|
|
966
|
+
metadata: {
|
|
967
|
+
kind: diagnostic.kind,
|
|
968
|
+
status: diagnostic.status,
|
|
969
|
+
...(diagnostic.name ? { name: diagnostic.name } : {}),
|
|
970
|
+
...(diagnostic.nextAction ? { nextAction: diagnostic.nextAction } : {}),
|
|
971
|
+
...(diagnostic.provider ? { provider: diagnostic.provider } : {}),
|
|
972
|
+
...(diagnostic.runnerId ? { runnerId: diagnostic.runnerId } : {}),
|
|
973
|
+
},
|
|
974
|
+
}));
|
|
975
|
+
}
|
|
567
976
|
/**
|
|
568
977
|
* Builds scenario health from profile metrics.
|
|
569
978
|
*
|
|
570
|
-
* @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>}} options
|
|
979
|
+
* @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>, diagnostics?: DiagnosticInventoryEntry[], profileEventCount?: number, profileSessionEntryCount?: number, commandTransport?: string, sessionEntries?: Record<string, unknown>[]}} options
|
|
571
980
|
* @returns {Record<string, unknown>}
|
|
572
981
|
*/
|
|
573
|
-
function buildProfileHealth({ scenario, runId, metrics, }) {
|
|
982
|
+
function buildProfileHealth({ scenario, runId, metrics, diagnostics = [], profileEventCount, profileSessionEntryCount, commandTransport, sessionEntries = [], }) {
|
|
574
983
|
const passed = metrics.status === 'passed';
|
|
984
|
+
const metadata = {
|
|
985
|
+
failures: typeof metrics.failures === 'number' ? metrics.failures : null,
|
|
986
|
+
timeouts: typeof metrics.timeouts === 'number' ? metrics.timeouts : null,
|
|
987
|
+
};
|
|
988
|
+
if (typeof profileEventCount === 'number') {
|
|
989
|
+
metadata.profileEventCount = profileEventCount;
|
|
990
|
+
}
|
|
991
|
+
if (typeof profileSessionEntryCount === 'number') {
|
|
992
|
+
metadata.profileSessionEntryCount = profileSessionEntryCount;
|
|
993
|
+
}
|
|
994
|
+
if (typeof commandTransport === 'string' && commandTransport.length > 0) {
|
|
995
|
+
metadata.commandTransport = commandTransport;
|
|
996
|
+
}
|
|
997
|
+
if (!passed &&
|
|
998
|
+
profileEventCount === 0 &&
|
|
999
|
+
profileSessionEntryCount === 0 &&
|
|
1000
|
+
typeof commandTransport === 'string' &&
|
|
1001
|
+
commandTransport.startsWith('profile-session')) {
|
|
1002
|
+
metadata.nextActionCode = 'verify_profile_session_bootstrap';
|
|
1003
|
+
metadata.nextAction =
|
|
1004
|
+
'Verify the app loaded the expected bundle, mounted the profile-session bootstrap near the app root, and uses the configured storage keys or deep-link scheme before treating this as a product failure.';
|
|
1005
|
+
}
|
|
1006
|
+
const skippedCommands = sessionEntries.filter((entry) => (entry?.kind === 'command' && entry.status === 'skipped'));
|
|
1007
|
+
const firstSkippedCommand = skippedCommands[0];
|
|
1008
|
+
const firstSkippedReason = typeof firstSkippedCommand?.reason === 'string'
|
|
1009
|
+
? firstSkippedCommand.reason
|
|
1010
|
+
: undefined;
|
|
1011
|
+
const commandFailureCode = firstSkippedReason === 'wait-for-milestone-timeout'
|
|
1012
|
+
? 'profile_command_gate_timeout'
|
|
1013
|
+
: 'profile_command_skipped';
|
|
1014
|
+
const commandFailureMessage = firstSkippedReason === 'wait-for-milestone-timeout'
|
|
1015
|
+
? 'One or more profile-session commands waited for a milestone that was not observed before timeout.'
|
|
1016
|
+
: 'One or more profile-session commands were skipped before the scenario completed.';
|
|
1017
|
+
const commandChecks = skippedCommands.length > 0
|
|
1018
|
+
? [
|
|
1019
|
+
{
|
|
1020
|
+
name: 'profile_command_sequence',
|
|
1021
|
+
status: 'failed',
|
|
1022
|
+
source: 'runner',
|
|
1023
|
+
code: commandFailureCode,
|
|
1024
|
+
message: commandFailureMessage,
|
|
1025
|
+
metadata: {
|
|
1026
|
+
skippedCommandCount: skippedCommands.length,
|
|
1027
|
+
...(typeof firstSkippedCommand?.command === 'string' ? { command: firstSkippedCommand.command } : {}),
|
|
1028
|
+
...(typeof firstSkippedCommand?.commandId === 'string' ? { commandId: firstSkippedCommand.commandId } : {}),
|
|
1029
|
+
...(typeof firstSkippedCommand?.queueId === 'string' ? { queueId: firstSkippedCommand.queueId } : {}),
|
|
1030
|
+
...(typeof firstSkippedCommand?.reason === 'string' ? { reason: firstSkippedCommand.reason } : {}),
|
|
1031
|
+
...(typeof firstSkippedCommand?.sequence === 'number' ? { sequence: firstSkippedCommand.sequence } : {}),
|
|
1032
|
+
...(typeof firstSkippedCommand?.waitForMilestone === 'string'
|
|
1033
|
+
? { waitForMilestone: firstSkippedCommand.waitForMilestone }
|
|
1034
|
+
: {}),
|
|
1035
|
+
...(typeof firstSkippedCommand?.waitTimeoutMs === 'number'
|
|
1036
|
+
? { waitTimeoutMs: firstSkippedCommand.waitTimeoutMs }
|
|
1037
|
+
: {}),
|
|
1038
|
+
},
|
|
1039
|
+
},
|
|
1040
|
+
]
|
|
1041
|
+
: [];
|
|
1042
|
+
const commandChecksPassed = commandChecks.every((check) => check.status === 'passed');
|
|
1043
|
+
const diagnosticChecks = buildRequiredDiagnosticHealthChecks(diagnostics);
|
|
1044
|
+
const diagnosticChecksPassed = diagnosticChecks.every((check) => check.status === 'passed');
|
|
1045
|
+
const healthPassed = passed && commandChecksPassed && diagnosticChecksPassed;
|
|
575
1046
|
return assertValidJson({
|
|
576
1047
|
schemaVersion: '1.0.0',
|
|
577
1048
|
scenarioId: scenario.name,
|
|
578
1049
|
...(typeof scenario.flowId === 'string' ? { flowId: scenario.flowId } : {}),
|
|
579
1050
|
runId,
|
|
580
|
-
healthStatus:
|
|
1051
|
+
healthStatus: healthPassed ? 'passed' : 'failed',
|
|
581
1052
|
checks: [
|
|
582
1053
|
{
|
|
583
1054
|
name: 'truth_events_complete',
|
|
@@ -587,14 +1058,83 @@ function buildProfileHealth({ scenario, runId, metrics, }) {
|
|
|
587
1058
|
message: passed
|
|
588
1059
|
? 'Profile events completed every expected iteration.'
|
|
589
1060
|
: 'Profile events did not complete every expected iteration.',
|
|
590
|
-
metadata
|
|
591
|
-
failures: typeof metrics.failures === 'number' ? metrics.failures : null,
|
|
592
|
-
timeouts: typeof metrics.timeouts === 'number' ? metrics.timeouts : null,
|
|
593
|
-
},
|
|
1061
|
+
metadata,
|
|
594
1062
|
},
|
|
1063
|
+
...commandChecks,
|
|
1064
|
+
...diagnosticChecks,
|
|
595
1065
|
],
|
|
596
1066
|
}, SCHEMAS.health, 'Health artifact');
|
|
597
1067
|
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Derives the terminal state for one profile artifact attempt.
|
|
1070
|
+
*
|
|
1071
|
+
* @param {Record<string, unknown>} metrics
|
|
1072
|
+
* @returns {string}
|
|
1073
|
+
*/
|
|
1074
|
+
function buildAttemptTerminalState(metrics) {
|
|
1075
|
+
if (metrics.status === 'passed') {
|
|
1076
|
+
return 'passed';
|
|
1077
|
+
}
|
|
1078
|
+
if (typeof metrics.timeouts === 'number' && metrics.timeouts > 0) {
|
|
1079
|
+
return 'timeout';
|
|
1080
|
+
}
|
|
1081
|
+
return 'failed';
|
|
1082
|
+
}
|
|
1083
|
+
/**
|
|
1084
|
+
* Classifies one profile artifact attempt without product-specific vocabulary.
|
|
1085
|
+
*
|
|
1086
|
+
* @param {Record<string, unknown>} metrics
|
|
1087
|
+
* @returns {Record<string, unknown>}
|
|
1088
|
+
*/
|
|
1089
|
+
function buildAttemptClassification(metrics) {
|
|
1090
|
+
if (metrics.status === 'passed') {
|
|
1091
|
+
return {
|
|
1092
|
+
category: 'none',
|
|
1093
|
+
};
|
|
1094
|
+
}
|
|
1095
|
+
if (typeof metrics.timeouts === 'number' && metrics.timeouts > 0) {
|
|
1096
|
+
return {
|
|
1097
|
+
category: 'timeout',
|
|
1098
|
+
code: 'profile_truth_event_timeout',
|
|
1099
|
+
message: `Profile run recorded ${metrics.timeouts} timeout(s) before all expected truth events completed.`,
|
|
1100
|
+
retryable: true,
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
return {
|
|
1104
|
+
category: 'evidence',
|
|
1105
|
+
code: 'profile_truth_events_incomplete',
|
|
1106
|
+
message: 'Profile run did not capture every expected truth event.',
|
|
1107
|
+
retryable: true,
|
|
1108
|
+
};
|
|
1109
|
+
}
|
|
1110
|
+
/**
|
|
1111
|
+
* Records whether the written artifact set is valid for diagnosis when a run fails.
|
|
1112
|
+
*
|
|
1113
|
+
* @param {{artifacts: Record<string, unknown>, metrics: Record<string, unknown>}} options
|
|
1114
|
+
* @returns {Record<string, unknown>}
|
|
1115
|
+
*/
|
|
1116
|
+
function buildAttemptPartialArtifacts({ artifacts, metrics, }) {
|
|
1117
|
+
if (metrics.status === 'passed') {
|
|
1118
|
+
return {
|
|
1119
|
+
valid: false,
|
|
1120
|
+
reason: 'complete successful run artifacts are present',
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
const paths = [
|
|
1124
|
+
artifacts.manifest,
|
|
1125
|
+
'health.json',
|
|
1126
|
+
artifacts.metrics,
|
|
1127
|
+
artifacts.causalRun,
|
|
1128
|
+
artifacts.summary,
|
|
1129
|
+
artifacts.raw?.interactionLog,
|
|
1130
|
+
artifacts.raw?.deviceLog,
|
|
1131
|
+
].filter((item) => typeof item === 'string' && item.length > 0);
|
|
1132
|
+
return {
|
|
1133
|
+
valid: true,
|
|
1134
|
+
reason: 'failed profile run artifacts are preserved for diagnosis and are not a product proof until scenario health passes',
|
|
1135
|
+
paths,
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
598
1138
|
/**
|
|
599
1139
|
* Builds failed scenario health from evidence-provider command failures.
|
|
600
1140
|
*
|
|
@@ -760,6 +1300,134 @@ function resolveEventLogPath({ args, platform }) {
|
|
|
760
1300
|
}
|
|
761
1301
|
return null;
|
|
762
1302
|
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Resolves the optional profile-session entry artifact path for command acknowledgement evidence.
|
|
1305
|
+
*
|
|
1306
|
+
* @param {{args: CliArgs, platform: ProfilePlatform}} options
|
|
1307
|
+
* @returns {string | null}
|
|
1308
|
+
*/
|
|
1309
|
+
function resolveProfileSessionEntriesPath({ args, platform }) {
|
|
1310
|
+
if (platform === 'ios' && typeof args['simctl-artifacts'] === 'string') {
|
|
1311
|
+
const storedEntriesPath = path.resolve(args['simctl-artifacts'], 'raw', 'ios-profile-session-entries.json');
|
|
1312
|
+
return fs.existsSync(storedEntriesPath) ? storedEntriesPath : null;
|
|
1313
|
+
}
|
|
1314
|
+
return null;
|
|
1315
|
+
}
|
|
1316
|
+
/**
|
|
1317
|
+
* Resolves the run id used by rehydrated sidecar evidence.
|
|
1318
|
+
*
|
|
1319
|
+
* A rehydrated artifact can intentionally have a new run id while ingesting a
|
|
1320
|
+
* previously captured adb/simctl sidecar. Keep live runs strict, but allow an
|
|
1321
|
+
* explicit sidecar with exactly one source run id for the scenario to provide
|
|
1322
|
+
* the event filter.
|
|
1323
|
+
*
|
|
1324
|
+
* @param {{args: CliArgs, eventLogText: string, profileSessionEntriesPath: string | null, runId: string, scenarioName: string}} options
|
|
1325
|
+
* @returns {string}
|
|
1326
|
+
*/
|
|
1327
|
+
function resolveEvidenceFilterRunId({ args, eventLogText, profileSessionEntriesPath, runId, scenarioName, }) {
|
|
1328
|
+
const isRehydratedSidecar = typeof args['adb-artifacts'] === 'string' || typeof args['simctl-artifacts'] === 'string';
|
|
1329
|
+
if (!isRehydratedSidecar) {
|
|
1330
|
+
return runId;
|
|
1331
|
+
}
|
|
1332
|
+
const scenarioEvents = extractProfileEvents(eventLogText, { scenario: scenarioName });
|
|
1333
|
+
const currentRunEvents = scenarioEvents.filter((event) => event.runId === runId);
|
|
1334
|
+
if (currentRunEvents.length > 0) {
|
|
1335
|
+
return runId;
|
|
1336
|
+
}
|
|
1337
|
+
const sourceRunIds = new Set(scenarioEvents
|
|
1338
|
+
.map((event) => event.runId)
|
|
1339
|
+
.filter((sourceRunId) => typeof sourceRunId === 'string' && sourceRunId.length > 0));
|
|
1340
|
+
if (profileSessionEntriesPath && fs.existsSync(profileSessionEntriesPath)) {
|
|
1341
|
+
const storedEntries = JSON.parse(fs.readFileSync(profileSessionEntriesPath, 'utf8'));
|
|
1342
|
+
if (Array.isArray(storedEntries)) {
|
|
1343
|
+
for (const entry of storedEntries) {
|
|
1344
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
1345
|
+
continue;
|
|
1346
|
+
}
|
|
1347
|
+
const record = entry;
|
|
1348
|
+
if (record.scenario !== scenarioName) {
|
|
1349
|
+
continue;
|
|
1350
|
+
}
|
|
1351
|
+
if (typeof record.runId === 'string' && record.runId.length > 0) {
|
|
1352
|
+
sourceRunIds.add(record.runId);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
return sourceRunIds.size === 1 ? [...sourceRunIds][0] : runId;
|
|
1358
|
+
}
|
|
1359
|
+
/**
|
|
1360
|
+
* Returns the first usable adb screenshot file from sidecar metadata.
|
|
1361
|
+
*
|
|
1362
|
+
* ADB can produce a valid PNG even when command metadata records a nonzero
|
|
1363
|
+
* exit status from the host process. Treat the binary artifact as the capture
|
|
1364
|
+
* authority, but only after validating the PNG signature and sidecar boundary.
|
|
1365
|
+
*
|
|
1366
|
+
* @param {{runDir: string, sidecarRoot: string | null}} options
|
|
1367
|
+
* @returns {{dependency: SidecarEvidenceDependency, path: string} | null}
|
|
1368
|
+
*/
|
|
1369
|
+
function resolveAndroidAdbScreenshotDependency({ runDir, sidecarRoot, }) {
|
|
1370
|
+
if (!sidecarRoot) {
|
|
1371
|
+
return null;
|
|
1372
|
+
}
|
|
1373
|
+
const metadata = readOptionalJsonObject(path.resolve(sidecarRoot, 'raw', 'android-metadata.json'));
|
|
1374
|
+
const actions = Array.isArray(metadata?.driverActions) ? metadata.driverActions : [];
|
|
1375
|
+
for (const action of actions) {
|
|
1376
|
+
if (!action || typeof action !== 'object' || Array.isArray(action)) {
|
|
1377
|
+
continue;
|
|
1378
|
+
}
|
|
1379
|
+
const record = action;
|
|
1380
|
+
if (record.driverAction !== 'screenshot') {
|
|
1381
|
+
continue;
|
|
1382
|
+
}
|
|
1383
|
+
const sidecarRelativePath = typeof record.capturePath === 'string'
|
|
1384
|
+
? record.capturePath
|
|
1385
|
+
: typeof record.rawPath === 'string'
|
|
1386
|
+
? record.rawPath
|
|
1387
|
+
: null;
|
|
1388
|
+
if (!sidecarRelativePath || path.isAbsolute(sidecarRelativePath)) {
|
|
1389
|
+
continue;
|
|
1390
|
+
}
|
|
1391
|
+
const screenshotPath = path.resolve(sidecarRoot, sidecarRelativePath);
|
|
1392
|
+
const relativeToSidecar = path.relative(sidecarRoot, screenshotPath);
|
|
1393
|
+
if (relativeToSidecar.length === 0 ||
|
|
1394
|
+
relativeToSidecar.startsWith('..') ||
|
|
1395
|
+
path.isAbsolute(relativeToSidecar) ||
|
|
1396
|
+
!isPngFile(screenshotPath)) {
|
|
1397
|
+
continue;
|
|
1398
|
+
}
|
|
1399
|
+
const sidecarDependency = toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath: screenshotPath });
|
|
1400
|
+
return {
|
|
1401
|
+
dependency: sidecarDependency,
|
|
1402
|
+
path: sidecarDependency.path,
|
|
1403
|
+
};
|
|
1404
|
+
}
|
|
1405
|
+
return null;
|
|
1406
|
+
}
|
|
1407
|
+
/**
|
|
1408
|
+
* Checks whether a file starts with the PNG signature.
|
|
1409
|
+
*
|
|
1410
|
+
* @param {string} filePath
|
|
1411
|
+
* @returns {boolean}
|
|
1412
|
+
*/
|
|
1413
|
+
function isPngFile(filePath) {
|
|
1414
|
+
let signature;
|
|
1415
|
+
try {
|
|
1416
|
+
signature = fs.readFileSync(filePath, { flag: 'r' }).subarray(0, 8);
|
|
1417
|
+
}
|
|
1418
|
+
catch {
|
|
1419
|
+
return false;
|
|
1420
|
+
}
|
|
1421
|
+
return signature.length === 8 &&
|
|
1422
|
+
signature[0] === 0x89 &&
|
|
1423
|
+
signature[1] === 0x50 &&
|
|
1424
|
+
signature[2] === 0x4e &&
|
|
1425
|
+
signature[3] === 0x47 &&
|
|
1426
|
+
signature[4] === 0x0d &&
|
|
1427
|
+
signature[5] === 0x0a &&
|
|
1428
|
+
signature[6] === 0x1a &&
|
|
1429
|
+
signature[7] === 0x0a;
|
|
1430
|
+
}
|
|
763
1431
|
/**
|
|
764
1432
|
* Reads a JSON artifact if it exists and contains an object.
|
|
765
1433
|
*
|
|
@@ -948,6 +1616,46 @@ function findMilestoneEvent(scenario, milestoneId) {
|
|
|
948
1616
|
}
|
|
949
1617
|
return null;
|
|
950
1618
|
}
|
|
1619
|
+
/**
|
|
1620
|
+
* Returns true when a milestone is explicitly optional.
|
|
1621
|
+
*
|
|
1622
|
+
* @param {Record<string, unknown>} scenario
|
|
1623
|
+
* @param {unknown} milestoneId
|
|
1624
|
+
* @returns {boolean}
|
|
1625
|
+
*/
|
|
1626
|
+
function isOptionalMilestone(scenario, milestoneId) {
|
|
1627
|
+
if (typeof milestoneId !== 'string' || !Array.isArray(scenario.milestones)) {
|
|
1628
|
+
return false;
|
|
1629
|
+
}
|
|
1630
|
+
for (const milestone of scenario.milestones) {
|
|
1631
|
+
if (!isRecord(milestone)) {
|
|
1632
|
+
continue;
|
|
1633
|
+
}
|
|
1634
|
+
if (milestone.id === milestoneId) {
|
|
1635
|
+
return milestone.required === false;
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
return false;
|
|
1639
|
+
}
|
|
1640
|
+
/**
|
|
1641
|
+
* Returns true when a milestone represents one-time scenario readiness rather than a repeated cycle edge.
|
|
1642
|
+
*
|
|
1643
|
+
* @param {Record<string, unknown>} scenario
|
|
1644
|
+
* @param {unknown} milestoneId
|
|
1645
|
+
* @param {string | null} milestoneEvent
|
|
1646
|
+
* @returns {boolean}
|
|
1647
|
+
*/
|
|
1648
|
+
function isReadinessMilestone(scenario, milestoneId, milestoneEvent) {
|
|
1649
|
+
const readyEvent = isRecord(scenario.truthEvents) && isRecord(scenario.truthEvents.ready)
|
|
1650
|
+
? scenario.truthEvents.ready.event
|
|
1651
|
+
: undefined;
|
|
1652
|
+
if (typeof readyEvent === 'string' && milestoneEvent === readyEvent) {
|
|
1653
|
+
return true;
|
|
1654
|
+
}
|
|
1655
|
+
const id = typeof milestoneId === 'string' ? milestoneId.toLowerCase() : '';
|
|
1656
|
+
const event = typeof milestoneEvent === 'string' ? milestoneEvent.toLowerCase() : '';
|
|
1657
|
+
return id.includes('ready') || event.includes('ready');
|
|
1658
|
+
}
|
|
951
1659
|
/**
|
|
952
1660
|
* Builds a milestone-id to event-name lookup for schema-era scenarios.
|
|
953
1661
|
*
|
|
@@ -1003,6 +1711,16 @@ function resolveProfileMetricEvents(scenario) {
|
|
|
1003
1711
|
milestone: toEvent,
|
|
1004
1712
|
};
|
|
1005
1713
|
}
|
|
1714
|
+
if (fromEvent && toEvent && isReadinessMilestone(scenario, budget.fromMilestone, fromEvent)) {
|
|
1715
|
+
return {
|
|
1716
|
+
milestone: toEvent,
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
if (fromEvent && toEvent && isOptionalMilestone(scenario, budget.fromMilestone)) {
|
|
1720
|
+
return {
|
|
1721
|
+
milestone: toEvent,
|
|
1722
|
+
};
|
|
1723
|
+
}
|
|
1006
1724
|
if (fromEvent && toEvent) {
|
|
1007
1725
|
return {
|
|
1008
1726
|
closeRequested: toEvent,
|
|
@@ -1015,7 +1733,59 @@ function resolveProfileMetricEvents(scenario) {
|
|
|
1015
1733
|
return null;
|
|
1016
1734
|
}
|
|
1017
1735
|
/**
|
|
1018
|
-
*
|
|
1736
|
+
* Resolves how many repeated completion milestone events prove one cycle body.
|
|
1737
|
+
*
|
|
1738
|
+
* @param {Record<string, unknown>} scenario
|
|
1739
|
+
* @param {Record<string, string> | null} metricEvents
|
|
1740
|
+
* @returns {number}
|
|
1741
|
+
*/
|
|
1742
|
+
function resolveMilestoneEventsPerIteration(scenario, metricEvents) {
|
|
1743
|
+
if (!metricEvents || typeof metricEvents.milestone !== 'string' || !Array.isArray(scenario.steps)) {
|
|
1744
|
+
return 1;
|
|
1745
|
+
}
|
|
1746
|
+
const milestoneEvents = buildMilestoneEventLookup(scenario);
|
|
1747
|
+
const matchingWaitStepIds = new Set();
|
|
1748
|
+
for (const step of scenario.steps) {
|
|
1749
|
+
if (!isRecord(step) || step.kind !== 'waitForMilestone' || typeof step.milestone !== 'string') {
|
|
1750
|
+
continue;
|
|
1751
|
+
}
|
|
1752
|
+
const event = milestoneEvents[step.milestone] ?? step.milestone;
|
|
1753
|
+
if (event === metricEvents.milestone && typeof step.id === 'string') {
|
|
1754
|
+
matchingWaitStepIds.add(step.id);
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
if (matchingWaitStepIds.size === 0) {
|
|
1758
|
+
return 1;
|
|
1759
|
+
}
|
|
1760
|
+
const bodyStepIds = isRecord(scenario.cycles) && Array.isArray(scenario.cycles.bodyStepIds)
|
|
1761
|
+
? new Set(scenario.cycles.bodyStepIds.filter((entry) => typeof entry === 'string'))
|
|
1762
|
+
: null;
|
|
1763
|
+
if (bodyStepIds && bodyStepIds.size > 0) {
|
|
1764
|
+
let count = 0;
|
|
1765
|
+
let bodyCommandPending = false;
|
|
1766
|
+
for (const step of scenario.steps) {
|
|
1767
|
+
if (!isRecord(step)) {
|
|
1768
|
+
continue;
|
|
1769
|
+
}
|
|
1770
|
+
if (typeof step.id === 'string' && bodyStepIds.has(step.id) && step.kind === 'command') {
|
|
1771
|
+
bodyCommandPending = true;
|
|
1772
|
+
continue;
|
|
1773
|
+
}
|
|
1774
|
+
if (bodyCommandPending && typeof step.id === 'string' && matchingWaitStepIds.has(step.id)) {
|
|
1775
|
+
count += 1;
|
|
1776
|
+
bodyCommandPending = false;
|
|
1777
|
+
continue;
|
|
1778
|
+
}
|
|
1779
|
+
if (bodyCommandPending && step.kind === 'command') {
|
|
1780
|
+
bodyCommandPending = false;
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
return count > 1 ? count : 1;
|
|
1784
|
+
}
|
|
1785
|
+
return matchingWaitStepIds.size > 1 ? matchingWaitStepIds.size : 1;
|
|
1786
|
+
}
|
|
1787
|
+
/**
|
|
1788
|
+
* Maps shorthand milestone budget fields to aggregate profile budget keys.
|
|
1019
1789
|
*
|
|
1020
1790
|
* @param {{budget: Record<string, unknown>, metric: string}} options
|
|
1021
1791
|
* @returns {string | null}
|
|
@@ -1047,11 +1817,24 @@ function resolveProfileBudgets(scenario) {
|
|
|
1047
1817
|
return null;
|
|
1048
1818
|
}
|
|
1049
1819
|
const pass = {};
|
|
1820
|
+
const intervals = [];
|
|
1050
1821
|
for (const budget of scenario.budgets) {
|
|
1051
1822
|
if (!isRecord(budget) || typeof budget.limit !== 'number') {
|
|
1052
1823
|
continue;
|
|
1053
1824
|
}
|
|
1054
1825
|
if (budget.metric === 'p95' || budget.metric === 'p50') {
|
|
1826
|
+
const fromEvent = findMilestoneEvent(scenario, budget.fromMilestone);
|
|
1827
|
+
const toEvent = findMilestoneEvent(scenario, budget.toMilestone);
|
|
1828
|
+
if (fromEvent && toEvent) {
|
|
1829
|
+
intervals.push({
|
|
1830
|
+
name: typeof budget.name === 'string' ? budget.name : `${String(budget.fromMilestone)} to ${String(budget.toMilestone)}`,
|
|
1831
|
+
metric: budget.metric,
|
|
1832
|
+
limit: budget.limit,
|
|
1833
|
+
fromEvent,
|
|
1834
|
+
toEvent,
|
|
1835
|
+
});
|
|
1836
|
+
continue;
|
|
1837
|
+
}
|
|
1055
1838
|
const budgetKey = resolveProfileBudgetKey({ budget, metric: budget.metric });
|
|
1056
1839
|
if (budgetKey) {
|
|
1057
1840
|
pass[budgetKey] = budget.limit;
|
|
@@ -1064,13 +1847,113 @@ function resolveProfileBudgets(scenario) {
|
|
|
1064
1847
|
pass.timeouts = budget.limit;
|
|
1065
1848
|
}
|
|
1066
1849
|
}
|
|
1067
|
-
return Object.keys(pass).length > 0
|
|
1850
|
+
return Object.keys(pass).length > 0 || intervals.length > 0
|
|
1068
1851
|
? {
|
|
1069
1852
|
metric: 'milestone budget',
|
|
1070
1853
|
pass,
|
|
1854
|
+
...(intervals.length > 0 ? { intervals } : {}),
|
|
1071
1855
|
}
|
|
1072
1856
|
: null;
|
|
1073
1857
|
}
|
|
1858
|
+
/**
|
|
1859
|
+
* Reads the installed package version for run provenance.
|
|
1860
|
+
*
|
|
1861
|
+
* @returns {string}
|
|
1862
|
+
*/
|
|
1863
|
+
function readAslPackageVersion() {
|
|
1864
|
+
try {
|
|
1865
|
+
const packageJsonPath = path.resolve(__dirname, '..', '..', 'package.json');
|
|
1866
|
+
const packageJson = readJson(packageJsonPath);
|
|
1867
|
+
return typeof packageJson.version === 'string' ? packageJson.version : 'unknown';
|
|
1868
|
+
}
|
|
1869
|
+
catch {
|
|
1870
|
+
return 'unknown';
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
/**
|
|
1874
|
+
* Infers the command transport used for profile-session or fixture evidence.
|
|
1875
|
+
*
|
|
1876
|
+
* @param {{args: CliArgs, interactionDriver: string, options: ProfileMobileOptions}} options
|
|
1877
|
+
* @returns {string}
|
|
1878
|
+
*/
|
|
1879
|
+
function resolveCommandTransport({ args, interactionDriver, options, }) {
|
|
1880
|
+
if (typeof options.commandTransport === 'string' && options.commandTransport.length > 0) {
|
|
1881
|
+
return options.commandTransport;
|
|
1882
|
+
}
|
|
1883
|
+
if (typeof args.events === 'string') {
|
|
1884
|
+
return 'fixture-log-ingest';
|
|
1885
|
+
}
|
|
1886
|
+
if (typeof args['ios-profile-session-transport'] === 'string') {
|
|
1887
|
+
return `profile-session-${args['ios-profile-session-transport']}`;
|
|
1888
|
+
}
|
|
1889
|
+
if (args['android-profile-session-storage'] || args['ios-profile-session-storage']) {
|
|
1890
|
+
return 'profile-session-storage';
|
|
1891
|
+
}
|
|
1892
|
+
if (args['profile-session']) {
|
|
1893
|
+
return 'profile-session-deeplink';
|
|
1894
|
+
}
|
|
1895
|
+
if (typeof args['adb-artifacts'] === 'string') {
|
|
1896
|
+
return 'adb-artifacts';
|
|
1897
|
+
}
|
|
1898
|
+
if (typeof args['simctl-artifacts'] === 'string') {
|
|
1899
|
+
return 'simctl-artifacts';
|
|
1900
|
+
}
|
|
1901
|
+
return interactionDriver;
|
|
1902
|
+
}
|
|
1903
|
+
/**
|
|
1904
|
+
* Builds product-neutral provenance cohort metadata for the run manifest.
|
|
1905
|
+
*
|
|
1906
|
+
* @param {{args: CliArgs, appId: string, interactionDriver: string, options: ProfileMobileOptions, providerExecution: ProviderCommandExecution}} options
|
|
1907
|
+
* @returns {Record<string, unknown>}
|
|
1908
|
+
*/
|
|
1909
|
+
function buildProfileProvenanceCohort({ appId, args, interactionDriver, options, providerExecution, }) {
|
|
1910
|
+
return {
|
|
1911
|
+
appId,
|
|
1912
|
+
commandTransport: resolveCommandTransport({ args, interactionDriver, options }),
|
|
1913
|
+
platform: options.platform,
|
|
1914
|
+
providers: providerExecution.providers,
|
|
1915
|
+
runnerName: interactionDriver,
|
|
1916
|
+
runnerVersion: readAslPackageVersion(),
|
|
1917
|
+
...options.provenanceCohort,
|
|
1918
|
+
};
|
|
1919
|
+
}
|
|
1920
|
+
/**
|
|
1921
|
+
* Builds an environment assertion for manifest pre/postconditions.
|
|
1922
|
+
*
|
|
1923
|
+
* @param {{artifact?: string, evidence?: string, source: string, value: unknown}} options
|
|
1924
|
+
* @returns {Record<string, unknown>}
|
|
1925
|
+
*/
|
|
1926
|
+
function environmentAssertion({ artifact, evidence = 'asserted', source, value, }) {
|
|
1927
|
+
return {
|
|
1928
|
+
value,
|
|
1929
|
+
evidence,
|
|
1930
|
+
source,
|
|
1931
|
+
...(artifact ? { artifact } : {}),
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1934
|
+
/**
|
|
1935
|
+
* Builds postconditions that ASL can truthfully assert after writing profile artifacts.
|
|
1936
|
+
*
|
|
1937
|
+
* @param {{metrics: Record<string, unknown>, options: ProfileMobileOptions}} options
|
|
1938
|
+
* @returns {Record<string, unknown>}
|
|
1939
|
+
*/
|
|
1940
|
+
function buildProfileEnvironmentPostconditions({ metrics, options, }) {
|
|
1941
|
+
const runPassed = metrics.status === 'passed';
|
|
1942
|
+
return {
|
|
1943
|
+
artifactState: environmentAssertion({
|
|
1944
|
+
value: runPassed ? 'complete' : 'partial',
|
|
1945
|
+
evidence: 'asserted',
|
|
1946
|
+
source: 'asl-profile-runner',
|
|
1947
|
+
artifact: 'manifest.json',
|
|
1948
|
+
}),
|
|
1949
|
+
cleanupState: environmentAssertion({
|
|
1950
|
+
value: 'not-required',
|
|
1951
|
+
evidence: 'asserted',
|
|
1952
|
+
source: 'asl-profile-runner',
|
|
1953
|
+
}),
|
|
1954
|
+
...options.environmentPostconditions,
|
|
1955
|
+
};
|
|
1956
|
+
}
|
|
1074
1957
|
/**
|
|
1075
1958
|
* Runs the mobile log-ingest profile artifact pipeline.
|
|
1076
1959
|
*
|
|
@@ -1091,6 +1974,7 @@ async function runProfileMobile(args, options) {
|
|
|
1091
1974
|
const scenarioHash = hashScenarioContract(profileScenario);
|
|
1092
1975
|
const expectedIterations = resolveExpectedIterations(profileScenario);
|
|
1093
1976
|
const profileMetricEvents = resolveProfileMetricEvents(profileScenario);
|
|
1977
|
+
const milestoneEventsPerIteration = resolveMilestoneEventsPerIteration(profileScenario, profileMetricEvents);
|
|
1094
1978
|
const profileBudgets = resolveProfileBudgets(profileScenario);
|
|
1095
1979
|
const runId = typeof args['run-id'] === 'string' ? args['run-id'] : createRunId();
|
|
1096
1980
|
const artifactRoot = resolveArtifactRoot({ args, config, configPath, platform: options.platform });
|
|
@@ -1100,6 +1984,7 @@ async function runProfileMobile(args, options) {
|
|
|
1100
1984
|
const capturesDir = layout.captures;
|
|
1101
1985
|
const startedAt = new Date().toISOString();
|
|
1102
1986
|
const eventLogPath = resolveEventLogPath({ args, platform: options.platform });
|
|
1987
|
+
const profileSessionEntriesPath = resolveProfileSessionEntriesPath({ args, platform: options.platform });
|
|
1103
1988
|
const interactionDriver = resolveInteractionDriver({ config, options, scenario });
|
|
1104
1989
|
const comparisonLane = resolveComparisonLane({ args, options, scenario });
|
|
1105
1990
|
await ensureDir(rawDir);
|
|
@@ -1145,12 +2030,86 @@ async function runProfileMobile(args, options) {
|
|
|
1145
2030
|
verdict,
|
|
1146
2031
|
};
|
|
1147
2032
|
}
|
|
1148
|
-
|
|
2033
|
+
let attachedEvidence;
|
|
2034
|
+
try {
|
|
2035
|
+
attachedEvidence = await resolveAttachedEvidence({ args, layout, providerInputs: providerExecution.inputs });
|
|
2036
|
+
}
|
|
2037
|
+
catch (error) {
|
|
2038
|
+
const providerInput = providerExecution.inputs.find((input) => error instanceof Error && error.message.includes(input.sourcePath));
|
|
2039
|
+
const health = buildProviderCommandFailureHealth({
|
|
2040
|
+
failures: [
|
|
2041
|
+
{
|
|
2042
|
+
commandId: 'provider-evidence',
|
|
2043
|
+
code: 'provider_evidence_invalid',
|
|
2044
|
+
exitCode: null,
|
|
2045
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2046
|
+
name: 'evidence_provider_output_valid',
|
|
2047
|
+
nextAction: 'Fix the provider output so it satisfies the ASL evidence contract, then rerun the profile.',
|
|
2048
|
+
nextActionCode: 'fix_provider_evidence_output',
|
|
2049
|
+
phase: 'afterCapture',
|
|
2050
|
+
providerId: providerInput?.providerId ?? 'unknown-provider',
|
|
2051
|
+
...(providerInput?.manifestPath ? { rawPath: providerInput.manifestPath } : {}),
|
|
2052
|
+
},
|
|
2053
|
+
],
|
|
2054
|
+
runId,
|
|
2055
|
+
scenario: profileScenario,
|
|
2056
|
+
});
|
|
2057
|
+
const verdict = buildProfileVerdict({ scenario: profileScenario, runId, health, metrics: {} });
|
|
2058
|
+
const agentSummary = buildAgentSummaryMarkdown({ health, verdict });
|
|
2059
|
+
await writeJsonArtifact({
|
|
2060
|
+
filePath: layout.health,
|
|
2061
|
+
value: health,
|
|
2062
|
+
schema: SCHEMAS.health,
|
|
2063
|
+
label: 'Health artifact',
|
|
2064
|
+
});
|
|
2065
|
+
await writeJsonArtifact({
|
|
2066
|
+
filePath: layout.verdict,
|
|
2067
|
+
value: verdict,
|
|
2068
|
+
schema: SCHEMAS.verdict,
|
|
2069
|
+
label: 'Verdict artifact',
|
|
2070
|
+
});
|
|
2071
|
+
await writeTextArtifact({
|
|
2072
|
+
filePath: layout.agentSummary,
|
|
2073
|
+
content: agentSummary,
|
|
2074
|
+
});
|
|
2075
|
+
return {
|
|
2076
|
+
runDir,
|
|
2077
|
+
health,
|
|
2078
|
+
verdict,
|
|
2079
|
+
};
|
|
2080
|
+
}
|
|
1149
2081
|
const eventLogText = eventLogPath ? await fsp.readFile(eventLogPath, 'utf8') : '';
|
|
2082
|
+
const evidenceFilterRunId = resolveEvidenceFilterRunId({
|
|
2083
|
+
args,
|
|
2084
|
+
eventLogText,
|
|
2085
|
+
profileSessionEntriesPath,
|
|
2086
|
+
runId,
|
|
2087
|
+
scenarioName,
|
|
2088
|
+
});
|
|
1150
2089
|
const events = extractProfileEvents(eventLogText, {
|
|
1151
2090
|
scenario: scenarioName,
|
|
1152
|
-
runId,
|
|
2091
|
+
runId: evidenceFilterRunId,
|
|
1153
2092
|
});
|
|
2093
|
+
const logSessionEntries = extractProfileSessionEntries(eventLogText, {
|
|
2094
|
+
scenario: scenarioName,
|
|
2095
|
+
runId: evidenceFilterRunId,
|
|
2096
|
+
});
|
|
2097
|
+
const storedSessionEntries = profileSessionEntriesPath
|
|
2098
|
+
? JSON.parse(await fsp.readFile(profileSessionEntriesPath, 'utf8'))
|
|
2099
|
+
: [];
|
|
2100
|
+
const sessionEntries = [
|
|
2101
|
+
...logSessionEntries,
|
|
2102
|
+
...(Array.isArray(storedSessionEntries)
|
|
2103
|
+
? storedSessionEntries.filter((entry) => {
|
|
2104
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
2105
|
+
return false;
|
|
2106
|
+
}
|
|
2107
|
+
const record = entry;
|
|
2108
|
+
return ((!('scenario' in record) || record.scenario === scenarioName) &&
|
|
2109
|
+
(!('runId' in record) || record.runId === evidenceFilterRunId));
|
|
2110
|
+
})
|
|
2111
|
+
: []),
|
|
2112
|
+
];
|
|
1154
2113
|
const runtimeTarget = resolveRuntimeTarget({ args, platform: options.platform });
|
|
1155
2114
|
const metrics = buildMetricsFromProfileEvents({
|
|
1156
2115
|
scenario: scenarioName,
|
|
@@ -1159,53 +2118,94 @@ async function runProfileMobile(args, options) {
|
|
|
1159
2118
|
expectedIterations,
|
|
1160
2119
|
budgets: profileBudgets,
|
|
1161
2120
|
cycleEventNames: profileMetricEvents,
|
|
2121
|
+
milestoneEventsPerIteration,
|
|
1162
2122
|
artifacts: {
|
|
1163
2123
|
captures: attachedEvidence.captures,
|
|
1164
2124
|
signals: attachedEvidence.signals,
|
|
1165
2125
|
},
|
|
1166
2126
|
});
|
|
2127
|
+
const eventLogRawPath = eventLogPath ? `raw/${path.basename(eventLogPath)}` : undefined;
|
|
2128
|
+
const eventLogIsProfileSessionEvidenceOnly = options.platform === 'ios' &&
|
|
2129
|
+
eventLogPath &&
|
|
2130
|
+
path.basename(eventLogPath) === 'ios-profile-events.log';
|
|
2131
|
+
const manifestArtifacts = {
|
|
2132
|
+
causalRun: 'causal-run.json',
|
|
2133
|
+
budgetVerdict: 'budget-verdict.json',
|
|
2134
|
+
manifest: 'manifest.json',
|
|
2135
|
+
metrics: 'metrics.json',
|
|
2136
|
+
summary: 'summary.md',
|
|
2137
|
+
scenario: toPortablePathReference(scenarioPath),
|
|
2138
|
+
raw: {
|
|
2139
|
+
...(eventLogRawPath && !eventLogIsProfileSessionEvidenceOnly
|
|
2140
|
+
? {
|
|
2141
|
+
interactionLog: eventLogRawPath,
|
|
2142
|
+
deviceLog: eventLogRawPath,
|
|
2143
|
+
}
|
|
2144
|
+
: {}),
|
|
2145
|
+
...(profileSessionEntriesPath
|
|
2146
|
+
? { profileSessionEntries: `raw/${path.basename(profileSessionEntriesPath)}` }
|
|
2147
|
+
: {}),
|
|
2148
|
+
},
|
|
2149
|
+
captures: {
|
|
2150
|
+
screenshots: attachedEvidence.captures.screenshots,
|
|
2151
|
+
...(attachedEvidence.captures.video ? { video: attachedEvidence.captures.video } : {}),
|
|
2152
|
+
...(attachedEvidence.captures.uiTree ? { uiTree: attachedEvidence.captures.uiTree } : {}),
|
|
2153
|
+
},
|
|
2154
|
+
signals: {
|
|
2155
|
+
js: attachedEvidence.signals.js,
|
|
2156
|
+
memory: attachedEvidence.signals.memory,
|
|
2157
|
+
network: attachedEvidence.signals.network,
|
|
2158
|
+
},
|
|
2159
|
+
evidenceAttachments: buildEvidenceAttachmentManifest(attachedEvidence.attachments),
|
|
2160
|
+
diagnostics: buildDiagnosticInventory({
|
|
2161
|
+
args,
|
|
2162
|
+
attachedEvidence,
|
|
2163
|
+
eventLogPath,
|
|
2164
|
+
platform: options.platform,
|
|
2165
|
+
profileSessionEntriesPath,
|
|
2166
|
+
runDir,
|
|
2167
|
+
scenario: profileScenario,
|
|
2168
|
+
}),
|
|
2169
|
+
};
|
|
2170
|
+
const appId = resolveAppId({ config, platform: options.platform });
|
|
2171
|
+
const commandTransport = resolveCommandTransport({ args, interactionDriver, options });
|
|
2172
|
+
const provenanceCohort = buildProfileProvenanceCohort({
|
|
2173
|
+
appId,
|
|
2174
|
+
args,
|
|
2175
|
+
interactionDriver,
|
|
2176
|
+
options,
|
|
2177
|
+
providerExecution,
|
|
2178
|
+
});
|
|
1167
2179
|
const manifest = buildManifest({
|
|
1168
2180
|
scenario: scenarioName,
|
|
1169
2181
|
scenarioHash,
|
|
1170
2182
|
runId,
|
|
1171
2183
|
platform: options.platform,
|
|
1172
2184
|
status: metrics.status,
|
|
2185
|
+
terminalState: buildAttemptTerminalState(metrics),
|
|
1173
2186
|
endedAt: new Date().toISOString(),
|
|
1174
2187
|
interactionDriver,
|
|
1175
2188
|
comparisonLane,
|
|
2189
|
+
classification: buildAttemptClassification(metrics),
|
|
2190
|
+
cleanup: {
|
|
2191
|
+
status: 'not-required',
|
|
2192
|
+
},
|
|
2193
|
+
partialArtifacts: buildAttemptPartialArtifacts({ artifacts: manifestArtifacts, metrics }),
|
|
2194
|
+
preconditions: options.environmentPreconditions,
|
|
2195
|
+
postconditions: buildProfileEnvironmentPostconditions({ metrics, options }),
|
|
1176
2196
|
startedAt,
|
|
1177
2197
|
simulator: runtimeTarget,
|
|
1178
|
-
bundleId:
|
|
2198
|
+
bundleId: appId,
|
|
1179
2199
|
gitSha: 'unknown',
|
|
1180
2200
|
toolVersions: {
|
|
1181
2201
|
node: process.version,
|
|
1182
2202
|
},
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
budgetVerdict: 'budget-verdict.json',
|
|
1186
|
-
manifest: 'manifest.json',
|
|
1187
|
-
metrics: 'metrics.json',
|
|
1188
|
-
summary: 'summary.md',
|
|
1189
|
-
scenario: toPortablePathReference(scenarioPath),
|
|
1190
|
-
raw: {
|
|
1191
|
-
interactionLog: eventLogPath ? `raw/${path.basename(eventLogPath)}` : 'raw/interaction.log',
|
|
1192
|
-
deviceLog: 'raw/device.log',
|
|
1193
|
-
},
|
|
1194
|
-
captures: {
|
|
1195
|
-
screenshots: attachedEvidence.captures.screenshots,
|
|
1196
|
-
video: attachedEvidence.captures.video ?? 'captures/run.mp4',
|
|
1197
|
-
uiTree: attachedEvidence.captures.uiTree ?? 'captures/ui-tree.json',
|
|
1198
|
-
},
|
|
1199
|
-
signals: {
|
|
1200
|
-
js: attachedEvidence.signals.js,
|
|
1201
|
-
memory: attachedEvidence.signals.memory,
|
|
1202
|
-
network: attachedEvidence.signals.network,
|
|
1203
|
-
},
|
|
1204
|
-
evidenceAttachments: buildEvidenceAttachmentManifest(attachedEvidence.attachments),
|
|
1205
|
-
},
|
|
2203
|
+
cohort: provenanceCohort,
|
|
2204
|
+
artifacts: manifestArtifacts,
|
|
1206
2205
|
});
|
|
1207
2206
|
const timeline = buildCausalTimeline({
|
|
1208
2207
|
events,
|
|
2208
|
+
sessionEntries,
|
|
1209
2209
|
startedAt,
|
|
1210
2210
|
phaseMap: scenario.timelinePhases ?? null,
|
|
1211
2211
|
owner: scenario.flowId ?? scenarioName,
|
|
@@ -1229,9 +2229,18 @@ async function runProfileMobile(args, options) {
|
|
|
1229
2229
|
runId,
|
|
1230
2230
|
budgetEvaluation: metrics.budgetEvaluation ?? null,
|
|
1231
2231
|
});
|
|
1232
|
-
const health = buildProfileHealth({
|
|
2232
|
+
const health = buildProfileHealth({
|
|
2233
|
+
scenario: profileScenario,
|
|
2234
|
+
runId,
|
|
2235
|
+
metrics,
|
|
2236
|
+
diagnostics: manifestArtifacts.diagnostics,
|
|
2237
|
+
profileEventCount: events.length,
|
|
2238
|
+
profileSessionEntryCount: sessionEntries.length,
|
|
2239
|
+
commandTransport,
|
|
2240
|
+
sessionEntries,
|
|
2241
|
+
});
|
|
1233
2242
|
const verdict = buildProfileVerdict({ scenario: profileScenario, runId, health, metrics });
|
|
1234
|
-
const agentSummary = buildAgentSummaryMarkdown({ health, verdict });
|
|
2243
|
+
const agentSummary = buildAgentSummaryMarkdown({ health, verdict, manifest });
|
|
1235
2244
|
const summary = buildSummaryMarkdown({ manifest, metrics });
|
|
1236
2245
|
await writeJsonArtifact({
|
|
1237
2246
|
filePath: layout.health,
|
|
@@ -1282,6 +2291,9 @@ async function runProfileMobile(args, options) {
|
|
|
1282
2291
|
if (eventLogPath) {
|
|
1283
2292
|
await fsp.copyFile(eventLogPath, path.join(rawDir, path.basename(eventLogPath)));
|
|
1284
2293
|
}
|
|
2294
|
+
if (profileSessionEntriesPath) {
|
|
2295
|
+
await fsp.copyFile(profileSessionEntriesPath, path.join(rawDir, path.basename(profileSessionEntriesPath)));
|
|
2296
|
+
}
|
|
1285
2297
|
await copyAttachedEvidence(attachedEvidence.copies);
|
|
1286
2298
|
return {
|
|
1287
2299
|
runDir,
|