agent-scenario-loop 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/profile-session.ts +263 -17
- package/dist/core/artifact-contract.d.ts +6 -4
- package/dist/core/artifact-contract.js +164 -15
- package/dist/core/artifact-layout.d.ts +2 -0
- package/dist/core/artifact-layout.js +2 -0
- package/dist/core/planner.js +4 -3
- package/dist/core/schema-validator.d.ts +1 -0
- package/dist/core/schema-validator.js +1 -0
- package/dist/runner/android-adb-driver.d.ts +7 -2
- package/dist/runner/android-adb-driver.js +7 -1
- package/dist/runner/android-adb.d.ts +40 -5
- package/dist/runner/android-adb.js +1046 -664
- package/dist/runner/ios-simctl.d.ts +1 -0
- package/dist/runner/ios-simctl.js +1 -0
- package/dist/runner/profile-android.d.ts +11 -1
- package/dist/runner/profile-android.js +266 -25
- package/dist/runner/profile-ios.d.ts +3 -2
- package/dist/runner/profile-ios.js +252 -22
- package/dist/runner/profile-mobile.d.ts +63 -4
- package/dist/runner/profile-mobile.js +1002 -20
- package/dist/runner/validate-project.js +3 -0
- package/dist/scripts/consumer-rehearsal.d.ts +127 -0
- package/dist/scripts/consumer-rehearsal.js +774 -0
- package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
- package/dist/scripts/downstream-local-package-gate.js +264 -0
- package/dist/scripts/package-smoke.d.ts +104 -0
- package/dist/scripts/package-smoke.js +2304 -0
- package/dist/scripts/release-check.d.ts +47 -0
- package/dist/scripts/release-check.js +117 -0
- package/dist/scripts/release-readiness.d.ts +2 -0
- package/dist/scripts/release-readiness.js +539 -0
- package/docs/adapters.md +3 -1
- package/docs/api.md +2 -2
- package/docs/authoring.md +34 -2
- package/docs/consumer-rehearsal.md +33 -1
- package/docs/contracts.md +16 -2
- package/docs/live-proofs.md +12 -4
- package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
- package/examples/runners/README.md +3 -3
- package/examples/runners/axe-accessibility-provider.json +2 -2
- package/examples/runners/script-accessibility-provider.json +2 -2
- package/examples/runners/script-memory-provider.json +2 -2
- package/examples/runners/script-network-provider.json +2 -2
- package/examples/runners/script-profiler-provider.json +2 -2
- package/package.json +12 -4
- package/schemas/manifest.schema.json +73 -3
- package/schemas/profiler.schema.json +243 -0
- package/schemas/runner-capabilities.schema.json +8 -2
- package/schemas/scenario.schema.json +18 -2
- package/templates/evidence-provider.json +3 -3
- package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
|
@@ -14,6 +14,8 @@ exports.resolveAttachedEvidence = resolveAttachedEvidence;
|
|
|
14
14
|
exports.resolveComparisonLane = resolveComparisonLane;
|
|
15
15
|
exports.resolveEventLogPath = resolveEventLogPath;
|
|
16
16
|
exports.resolveInteractionDriver = resolveInteractionDriver;
|
|
17
|
+
exports.resolveProfileScenarioName = resolveProfileScenarioName;
|
|
18
|
+
exports.runProfileCompatibilityPreflight = runProfileCompatibilityPreflight;
|
|
17
19
|
exports.runProfileCli = runProfileCli;
|
|
18
20
|
exports.runProfileMobile = runProfileMobile;
|
|
19
21
|
exports.hashScenarioContract = hashScenarioContract;
|
|
@@ -26,6 +28,7 @@ const crypto = require('node:crypto');
|
|
|
26
28
|
const { buildAgentSummaryMarkdown } = require('../core/agent-summary');
|
|
27
29
|
const { createArtifactLayout } = require('../core/artifact-layout');
|
|
28
30
|
const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
|
|
31
|
+
const { buildCompatibilityHealth, buildUnevaluatedVerdict, evaluateRunnerCompatibility, } = require('../core/planner');
|
|
29
32
|
const { buildBudgetVerdict, buildCausalRun, buildCausalTimeline, buildManifest, buildMetricsFromProfileEvents, buildSummaryMarkdown, extractProfileEvents, extractProfileSessionEntries, } = require('../core/artifact-contract');
|
|
30
33
|
const { SCHEMAS, assertValidJson } = require('../core/schema-validator');
|
|
31
34
|
const { writeUsage } = require('./cli');
|
|
@@ -49,7 +52,7 @@ function usage({ binaryName, output = process.stderr, platform, }) {
|
|
|
49
52
|
];
|
|
50
53
|
if (platform === 'android') {
|
|
51
54
|
lines.push('Use --adb-artifacts <dir> to read raw/adb-logcat.txt from a prior asl-android-adb capture.');
|
|
52
|
-
lines.push('Use --adb-capture [--clear-logcat] [--launch] [--launch-wait-ms <ms>] [--wait-ms <ms>] to capture adb logcat before profiling.');
|
|
55
|
+
lines.push('Use --adb-capture [--clear-logcat] [--launch] [--launch-wait-ms <ms>] [--wait-ms <ms>] [--adb-command-timeout-ms <ms>] to capture adb logcat before profiling.');
|
|
53
56
|
lines.push('Use --android-dev-client-url <url> [--android-dev-client-wait-ms <ms>] [--android-dev-client-ready-pattern <pattern>] with --adb-capture to open an Expo dev-client session before profile-session deep links.');
|
|
54
57
|
lines.push('Use --android-profile-session-storage with --profile-session to seed startup control through Android AsyncStorage.');
|
|
55
58
|
lines.push('Use --profile-session with --adb-capture to start the app profile session and execute scenario-declared Android commands.');
|
|
@@ -145,6 +148,15 @@ function readRepeatableArgValues(args, key) {
|
|
|
145
148
|
return entry;
|
|
146
149
|
});
|
|
147
150
|
}
|
|
151
|
+
/**
|
|
152
|
+
* Reads whether a boolean-style CLI flag was supplied.
|
|
153
|
+
*
|
|
154
|
+
* @param {CliArgValue | undefined} value
|
|
155
|
+
* @returns {boolean}
|
|
156
|
+
*/
|
|
157
|
+
function isEnabled(value) {
|
|
158
|
+
return value === true || value === 'true';
|
|
159
|
+
}
|
|
148
160
|
/**
|
|
149
161
|
* Parses a `kind:path` evidence attachment value.
|
|
150
162
|
*
|
|
@@ -251,6 +263,8 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
|
|
|
251
263
|
destinationPath: path.join(layout.signals[kind], fileName),
|
|
252
264
|
kind,
|
|
253
265
|
manifestPath: `signals/${kind}/${fileName}`,
|
|
266
|
+
providerId,
|
|
267
|
+
...(typeof output.required === 'boolean' ? { required: output.required } : {}),
|
|
254
268
|
sourcePath,
|
|
255
269
|
};
|
|
256
270
|
}
|
|
@@ -263,6 +277,8 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
|
|
|
263
277
|
destinationPath: path.join(layout.captures, fileName),
|
|
264
278
|
kind: output.kind,
|
|
265
279
|
manifestPath: `captures/${fileName}`,
|
|
280
|
+
providerId,
|
|
281
|
+
...(typeof output.required === 'boolean' ? { required: output.required } : {}),
|
|
266
282
|
sourcePath,
|
|
267
283
|
};
|
|
268
284
|
}
|
|
@@ -274,9 +290,33 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
|
|
|
274
290
|
destinationPath: path.join(layout.raw, 'providers', providerId, fileName),
|
|
275
291
|
kind: output.kind,
|
|
276
292
|
manifestPath: `raw/providers/${providerId}/${fileName}`,
|
|
293
|
+
providerId,
|
|
294
|
+
...(typeof output.required === 'boolean' ? { required: output.required } : {}),
|
|
277
295
|
sourcePath,
|
|
278
296
|
};
|
|
279
297
|
}
|
|
298
|
+
/**
|
|
299
|
+
* Validates structured profiler evidence when a provider emits JSON.
|
|
300
|
+
*
|
|
301
|
+
* Native traces and flamegraph files may be attached as profiler evidence, but
|
|
302
|
+
* JSON profiler files must carry enough envelope metadata for agents to reason
|
|
303
|
+
* about source, target, and completeness.
|
|
304
|
+
*
|
|
305
|
+
* @param {{kind: EvidenceKind, sourcePath: string}} options
|
|
306
|
+
* @returns {void}
|
|
307
|
+
*/
|
|
308
|
+
function validateStructuredProfilerEvidence({ kind, sourcePath, }) {
|
|
309
|
+
if (kind !== 'profiler' || path.extname(sourcePath).toLowerCase() !== '.json') {
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
try {
|
|
313
|
+
assertValidJson(readJson(sourcePath), SCHEMAS.profiler, 'Profiler evidence artifact');
|
|
314
|
+
}
|
|
315
|
+
catch (error) {
|
|
316
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
317
|
+
throw new Error(`Profiler evidence artifact is invalid: ${sourcePath}. ${detail}`);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
280
320
|
/**
|
|
281
321
|
* Fails when provider command ids would collide in raw command records.
|
|
282
322
|
*
|
|
@@ -442,7 +482,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
|
|
|
442
482
|
},
|
|
443
483
|
};
|
|
444
484
|
const destinationPaths = new Set();
|
|
445
|
-
const addCopy = async ({ channel, destinationPath, kind, manifestPath, sourcePath, }) => {
|
|
485
|
+
const addCopy = async ({ channel, destinationPath, kind, manifestPath, required = false, sourcePath, }) => {
|
|
446
486
|
const stat = await fsp.stat(sourcePath).catch(() => null);
|
|
447
487
|
if (!stat?.isFile()) {
|
|
448
488
|
throw new Error(`Evidence artifact does not exist or is not a file: ${sourcePath}`);
|
|
@@ -450,6 +490,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
|
|
|
450
490
|
if (destinationPaths.has(destinationPath)) {
|
|
451
491
|
throw new Error(`Duplicate evidence artifact destination: ${manifestPath}`);
|
|
452
492
|
}
|
|
493
|
+
validateStructuredProfilerEvidence({ kind, sourcePath });
|
|
453
494
|
destinationPaths.add(destinationPath);
|
|
454
495
|
const attachment = {
|
|
455
496
|
channel,
|
|
@@ -459,6 +500,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
|
|
|
459
500
|
kind,
|
|
460
501
|
manifestPath,
|
|
461
502
|
redactionStatus: 'not-redacted',
|
|
503
|
+
required,
|
|
462
504
|
sha256: await hashFileSha256(sourcePath),
|
|
463
505
|
sourceFileName: path.basename(sourcePath),
|
|
464
506
|
sourcePath,
|
|
@@ -578,20 +620,568 @@ function toPortablePathReference(targetPath) {
|
|
|
578
620
|
}
|
|
579
621
|
return path.basename(targetPath);
|
|
580
622
|
}
|
|
623
|
+
/**
|
|
624
|
+
* Resolves the evidence input mode before profile parsing starts.
|
|
625
|
+
*
|
|
626
|
+
* @param {{args: CliArgs, platform: ProfilePlatform}} options
|
|
627
|
+
* @returns {string}
|
|
628
|
+
*/
|
|
629
|
+
function resolveProfileInputMode({ args, platform }) {
|
|
630
|
+
if (typeof args.events === 'string') {
|
|
631
|
+
return 'fixture-event-log';
|
|
632
|
+
}
|
|
633
|
+
if (platform === 'android') {
|
|
634
|
+
if (typeof args['adb-artifacts'] === 'string') {
|
|
635
|
+
return 'adb-sidecar';
|
|
636
|
+
}
|
|
637
|
+
if (isEnabled(args['adb-capture'])) {
|
|
638
|
+
return 'adb-live-capture';
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
if (typeof args['simctl-artifacts'] === 'string') {
|
|
642
|
+
return 'simctl-sidecar';
|
|
643
|
+
}
|
|
644
|
+
if (isEnabled(args['simctl-capture'])) {
|
|
645
|
+
return 'simctl-live-capture';
|
|
646
|
+
}
|
|
647
|
+
return 'no-profile-evidence';
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Reads unique scenario step kinds for early operator visibility.
|
|
651
|
+
*
|
|
652
|
+
* @param {Record<string, unknown>} scenario
|
|
653
|
+
* @returns {string[]}
|
|
654
|
+
*/
|
|
655
|
+
function readScenarioStepKinds(scenario) {
|
|
656
|
+
if (!Array.isArray(scenario.steps)) {
|
|
657
|
+
return [];
|
|
658
|
+
}
|
|
659
|
+
return Array.from(new Set(scenario.steps
|
|
660
|
+
.filter(isRecord)
|
|
661
|
+
.map((step) => step.kind)
|
|
662
|
+
.filter((kind) => typeof kind === 'string')))
|
|
663
|
+
.sort();
|
|
664
|
+
}
|
|
665
|
+
/**
|
|
666
|
+
* Reads wait milestone ids from scenario steps.
|
|
667
|
+
*
|
|
668
|
+
* @param {Record<string, unknown>} scenario
|
|
669
|
+
* @returns {string[]}
|
|
670
|
+
*/
|
|
671
|
+
function readScenarioWaitMilestones(scenario) {
|
|
672
|
+
if (!Array.isArray(scenario.steps)) {
|
|
673
|
+
return [];
|
|
674
|
+
}
|
|
675
|
+
return Array.from(new Set(scenario.steps
|
|
676
|
+
.filter(isRecord)
|
|
677
|
+
.filter((step) => step.kind === 'waitForMilestone')
|
|
678
|
+
.map((step) => step.milestone)
|
|
679
|
+
.filter((milestone) => typeof milestone === 'string')))
|
|
680
|
+
.sort();
|
|
681
|
+
}
|
|
682
|
+
/**
|
|
683
|
+
* Builds the early run plan artifact before provider commands or event parsing.
|
|
684
|
+
*
|
|
685
|
+
* @param {{args: CliArgs, artifactRoot: string, comparisonLane?: string | undefined, expectedIterations: number, interactionDriver: string, layout: ReturnType<typeof createArtifactLayout>, milestoneEventsPerIteration: number, options: ProfileMobileOptions, profileScenario: Record<string, unknown>, runDir: string, runId: string, scenarioHash: string, scenarioPath: string}} options
|
|
686
|
+
* @returns {ProfileRunPlan}
|
|
687
|
+
*/
|
|
688
|
+
function buildProfileRunPlan({ args, artifactRoot, comparisonLane, expectedIterations, interactionDriver, layout, milestoneEventsPerIteration, options, profileScenario, runDir, runId, scenarioHash, scenarioPath, }) {
|
|
689
|
+
return {
|
|
690
|
+
artifactVersion: '1.0.0',
|
|
691
|
+
runId,
|
|
692
|
+
scenarioId: resolveProfileScenarioName({ scenario: profileScenario, scenarioPath }),
|
|
693
|
+
scenarioHash,
|
|
694
|
+
platform: options.platform,
|
|
695
|
+
inputMode: resolveProfileInputMode({ args, platform: options.platform }),
|
|
696
|
+
artifactRoot,
|
|
697
|
+
runDir,
|
|
698
|
+
interactionDriver,
|
|
699
|
+
...(comparisonLane ? { comparisonLane } : {}),
|
|
700
|
+
expectedIterations,
|
|
701
|
+
milestoneEventsPerIteration,
|
|
702
|
+
commandTransport: resolveCommandTransport({ args, interactionDriver, options }),
|
|
703
|
+
providers: readRepeatableArgValues(args, 'provider').map((providerPath) => ({
|
|
704
|
+
path: toPortablePathReference(path.resolve(providerPath)),
|
|
705
|
+
})),
|
|
706
|
+
requestedDiagnostics: {
|
|
707
|
+
required: Array.from(readScenarioStringSet(profileScenario, ['artifacts', 'required'])).sort(),
|
|
708
|
+
optional: Array.from(readScenarioStringSet(profileScenario, ['artifacts', 'optional'])).sort(),
|
|
709
|
+
},
|
|
710
|
+
scenarioShape: {
|
|
711
|
+
budgets: Array.isArray(profileScenario.budgets) ? profileScenario.budgets.length : 0,
|
|
712
|
+
steps: Array.isArray(profileScenario.steps) ? profileScenario.steps.length : 0,
|
|
713
|
+
stepKinds: readScenarioStepKinds(profileScenario),
|
|
714
|
+
waitForMilestones: readScenarioWaitMilestones(profileScenario),
|
|
715
|
+
},
|
|
716
|
+
evidenceSources: {
|
|
717
|
+
...(typeof args.events === 'string' ? { events: toPortablePathReference(path.resolve(args.events)) } : {}),
|
|
718
|
+
...(typeof args['profile-session-entries'] === 'string'
|
|
719
|
+
? { profileSessionEntries: toPortablePathReference(path.resolve(args['profile-session-entries'])) }
|
|
720
|
+
: {}),
|
|
721
|
+
...(typeof args['adb-artifacts'] === 'string'
|
|
722
|
+
? { adbArtifacts: toPortablePathReference(path.resolve(args['adb-artifacts'])) }
|
|
723
|
+
: {}),
|
|
724
|
+
...(typeof args['simctl-artifacts'] === 'string'
|
|
725
|
+
? { simctlArtifacts: toPortablePathReference(path.resolve(args['simctl-artifacts'])) }
|
|
726
|
+
: {}),
|
|
727
|
+
adbCapture: isEnabled(args['adb-capture']),
|
|
728
|
+
simctlCapture: isEnabled(args['simctl-capture']),
|
|
729
|
+
signals: readRepeatableArgValues(args, 'signal').length,
|
|
730
|
+
captures: readRepeatableArgValues(args, 'capture').length,
|
|
731
|
+
},
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
/**
|
|
735
|
+
* Writes the early profile run plan artifact and a compact status heartbeat.
|
|
736
|
+
*
|
|
737
|
+
* @param {{layout: ReturnType<typeof createArtifactLayout>, plan: ProfileRunPlan}} options
|
|
738
|
+
* @returns {Promise<void>}
|
|
739
|
+
*/
|
|
740
|
+
async function writeProfileRunPlan({ layout, plan, }) {
|
|
741
|
+
await fsp.writeFile(layout.runPlan, `${JSON.stringify(plan, null, 2)}\n`, 'utf8');
|
|
742
|
+
process.stderr.write(`profile run plan: ${plan.platform}/${plan.scenarioId} mode=${plan.inputMode} providers=${plan.providers.length} requiredDiagnostics=${plan.requestedDiagnostics.required.length} runPlan=${path.relative(process.cwd(), layout.runPlan)}\n`);
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Returns a path reference from one run folder to an external sidecar.
|
|
746
|
+
*
|
|
747
|
+
* @param {{runDir: string, targetPath: string}} options
|
|
748
|
+
* @returns {string}
|
|
749
|
+
*/
|
|
750
|
+
function toRunPathReference({ runDir, targetPath }) {
|
|
751
|
+
const relativePath = path.relative(runDir, targetPath);
|
|
752
|
+
return relativePath.length > 0 ? relativePath : path.basename(targetPath);
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* Returns a sidecar dependency path that stays readable in rehydrated artifacts.
|
|
756
|
+
*
|
|
757
|
+
* @param {{runDir: string, sidecarRoot: string, targetPath: string}} options
|
|
758
|
+
* @returns {SidecarEvidenceDependency}
|
|
759
|
+
*/
|
|
760
|
+
function toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath, }) {
|
|
761
|
+
const sidecarRelativePath = path.relative(sidecarRoot, targetPath);
|
|
762
|
+
if (sidecarRelativePath.length > 0 &&
|
|
763
|
+
!sidecarRelativePath.startsWith('..') &&
|
|
764
|
+
!path.isAbsolute(sidecarRelativePath)) {
|
|
765
|
+
return {
|
|
766
|
+
kind: 'sidecar',
|
|
767
|
+
root: 'sidecar',
|
|
768
|
+
path: sidecarRelativePath,
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
return {
|
|
772
|
+
kind: 'sidecar',
|
|
773
|
+
path: toRunPathReference({ runDir, targetPath }),
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Reads scenario string-list declarations into a set.
|
|
778
|
+
*
|
|
779
|
+
* @param {Record<string, unknown>} scenario
|
|
780
|
+
* @param {string[]} pathSegments
|
|
781
|
+
* @returns {Set<string>}
|
|
782
|
+
*/
|
|
783
|
+
function readScenarioStringSet(scenario, pathSegments) {
|
|
784
|
+
const values = pathSegments.reduce((current, segment) => (current && typeof current === 'object' && !Array.isArray(current)
|
|
785
|
+
? current[segment]
|
|
786
|
+
: undefined), scenario);
|
|
787
|
+
return new Set(Array.isArray(values)
|
|
788
|
+
? values.filter((value) => typeof value === 'string')
|
|
789
|
+
: []);
|
|
790
|
+
}
|
|
791
|
+
/**
|
|
792
|
+
* Returns true when a scenario artifact declaration matches a diagnostic kind.
|
|
793
|
+
*
|
|
794
|
+
* @param {Set<string>} artifacts
|
|
795
|
+
* @param {string[]} aliases
|
|
796
|
+
* @returns {boolean}
|
|
797
|
+
*/
|
|
798
|
+
function artifactSetHasAny(artifacts, aliases) {
|
|
799
|
+
return aliases.some((alias) => artifacts.has(alias));
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Resolves common aliases used by scenario artifact contracts.
|
|
803
|
+
*
|
|
804
|
+
* @param {DiagnosticKind} kind
|
|
805
|
+
* @returns {string[]}
|
|
806
|
+
*/
|
|
807
|
+
function diagnosticArtifactAliases(kind) {
|
|
808
|
+
const aliases = {
|
|
809
|
+
accessibility: ['accessibility'],
|
|
810
|
+
js: ['js', 'profileEvents', 'profileSession'],
|
|
811
|
+
logs: ['logs', 'deviceLog', 'interactionLog'],
|
|
812
|
+
memory: ['memory'],
|
|
813
|
+
network: ['network'],
|
|
814
|
+
profiler: ['profiler', 'profile'],
|
|
815
|
+
screenshot: ['screenshot', 'screenshots'],
|
|
816
|
+
uiTree: ['uiTree', 'ui-tree', 'accessibilityTree'],
|
|
817
|
+
video: ['video', 'recording'],
|
|
818
|
+
};
|
|
819
|
+
return aliases[kind];
|
|
820
|
+
}
|
|
821
|
+
/**
|
|
822
|
+
* Resolves common aliases used by runner capability declarations.
|
|
823
|
+
*
|
|
824
|
+
* @param {DiagnosticKind} kind
|
|
825
|
+
* @returns {string[]}
|
|
826
|
+
*/
|
|
827
|
+
function diagnosticCapabilityAliases(kind) {
|
|
828
|
+
const aliases = {
|
|
829
|
+
accessibility: ['accessibility', 'accessibilityCapture'],
|
|
830
|
+
js: ['js', 'profileSession', 'profileEvents'],
|
|
831
|
+
logs: ['logCapture', 'logs', 'deviceLog'],
|
|
832
|
+
memory: ['memory', 'memoryCapture'],
|
|
833
|
+
network: ['network', 'networkCapture'],
|
|
834
|
+
profiler: ['profiler', 'profile', 'profilerCapture'],
|
|
835
|
+
screenshot: ['screenshot', 'screenshots'],
|
|
836
|
+
uiTree: ['uiTree', 'ui-tree', 'accessibilityTree'],
|
|
837
|
+
video: ['video', 'recording'],
|
|
838
|
+
};
|
|
839
|
+
return aliases[kind];
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* Returns requirement/request metadata for one diagnostic kind.
|
|
843
|
+
*
|
|
844
|
+
* @param {{kind: DiagnosticKind, optionalArtifacts: Set<string>, optionalCapabilities: Set<string>, requiredArtifacts: Set<string>, requiredCapabilities: Set<string>}} options
|
|
845
|
+
* @returns {{required: boolean, requested: boolean}}
|
|
846
|
+
*/
|
|
847
|
+
function resolveDiagnosticRequest({ kind, optionalArtifacts, optionalCapabilities, requiredArtifacts, requiredCapabilities, }) {
|
|
848
|
+
const artifactAliases = diagnosticArtifactAliases(kind);
|
|
849
|
+
const capabilityAliases = diagnosticCapabilityAliases(kind);
|
|
850
|
+
const required = artifactSetHasAny(requiredArtifacts, artifactAliases) ||
|
|
851
|
+
artifactSetHasAny(requiredCapabilities, capabilityAliases);
|
|
852
|
+
return {
|
|
853
|
+
required,
|
|
854
|
+
requested: required ||
|
|
855
|
+
artifactSetHasAny(optionalArtifacts, artifactAliases) ||
|
|
856
|
+
artifactSetHasAny(optionalCapabilities, capabilityAliases),
|
|
857
|
+
};
|
|
858
|
+
}
|
|
859
|
+
/**
|
|
860
|
+
* Builds a status entry for one diagnostic surface.
|
|
861
|
+
*
|
|
862
|
+
* @param {DiagnosticInventoryEntry & {requested?: boolean}} entry
|
|
863
|
+
* @returns {DiagnosticInventoryEntry}
|
|
864
|
+
*/
|
|
865
|
+
function buildDiagnosticEntry(entry) {
|
|
866
|
+
const { requested = true, ...diagnostic } = entry;
|
|
867
|
+
if (diagnostic.status === 'captured' || requested || diagnostic.required) {
|
|
868
|
+
return diagnostic;
|
|
869
|
+
}
|
|
870
|
+
return {
|
|
871
|
+
...diagnostic,
|
|
872
|
+
status: 'not_requested',
|
|
873
|
+
reason: diagnostic.reason ?? 'Scenario did not request this optional diagnostic surface.',
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* Builds the product-neutral diagnostic inventory for a profile run.
|
|
878
|
+
*
|
|
879
|
+
* @param {{args: CliArgs, attachedEvidence: AttachedEvidence, eventLogPath: string | null, platform: ProfilePlatform, profileSessionEntriesPath: string | null, runDir: string, scenario: Record<string, unknown>}} options
|
|
880
|
+
* @returns {DiagnosticInventoryEntry[]}
|
|
881
|
+
*/
|
|
882
|
+
function buildDiagnosticInventory({ args, attachedEvidence, eventLogPath, platform, profileSessionEntriesPath, runDir, scenario, }) {
|
|
883
|
+
const requiredArtifacts = readScenarioStringSet(scenario, ['artifacts', 'required']);
|
|
884
|
+
const optionalArtifacts = readScenarioStringSet(scenario, ['artifacts', 'optional']);
|
|
885
|
+
const requiredCapabilities = readScenarioStringSet(scenario, ['requiredCapabilities']);
|
|
886
|
+
const optionalCapabilities = readScenarioStringSet(scenario, ['optionalCapabilities']);
|
|
887
|
+
const requiredProviderDiagnostics = new Set(attachedEvidence.attachments
|
|
888
|
+
.filter((attachment) => attachment.required)
|
|
889
|
+
.map((attachment) => attachment.kind));
|
|
890
|
+
const sidecarRoot = typeof args['adb-artifacts'] === 'string'
|
|
891
|
+
? path.resolve(args['adb-artifacts'])
|
|
892
|
+
: typeof args['simctl-artifacts'] === 'string'
|
|
893
|
+
? path.resolve(args['simctl-artifacts'])
|
|
894
|
+
: null;
|
|
895
|
+
const sidecarRootRef = sidecarRoot ? toRunPathReference({ runDir, targetPath: sidecarRoot }) : undefined;
|
|
896
|
+
const adbScreenshotDependency = platform === 'android'
|
|
897
|
+
? resolveAndroidAdbScreenshotDependency({ runDir, sidecarRoot })
|
|
898
|
+
: null;
|
|
899
|
+
const eventLogBaseName = eventLogPath ? path.basename(eventLogPath) : undefined;
|
|
900
|
+
const eventLogManifestPath = eventLogBaseName ? `raw/${eventLogBaseName}` : undefined;
|
|
901
|
+
const eventLogIsIosProfileEvents = platform === 'ios' && eventLogBaseName === 'ios-profile-events.log';
|
|
902
|
+
const simctlRuntimeLogPath = typeof args['simctl-artifacts'] === 'string'
|
|
903
|
+
? path.resolve(args['simctl-artifacts'], 'raw', 'ios-simctl-log.txt')
|
|
904
|
+
: null;
|
|
905
|
+
const simctlRuntimeLogExists = Boolean(simctlRuntimeLogPath && fs.existsSync(simctlRuntimeLogPath));
|
|
906
|
+
const simctlRuntimeLogDependency = simctlRuntimeLogPath && simctlRuntimeLogExists
|
|
907
|
+
? toSidecarEvidenceDependency({ runDir, sidecarRoot: path.resolve(args['simctl-artifacts']), targetPath: simctlRuntimeLogPath })
|
|
908
|
+
: undefined;
|
|
909
|
+
const copiedSimctlLogManifestPath = platform === 'ios' && eventLogPath && path.basename(eventLogPath) === 'ios-simctl-log.txt'
|
|
910
|
+
? eventLogManifestPath
|
|
911
|
+
: undefined;
|
|
912
|
+
const explicitIosRuntimeLogManifestPath = platform === 'ios' &&
|
|
913
|
+
typeof args.events === 'string' &&
|
|
914
|
+
eventLogManifestPath &&
|
|
915
|
+
!eventLogIsIosProfileEvents
|
|
916
|
+
? eventLogManifestPath
|
|
917
|
+
: undefined;
|
|
918
|
+
const eventLogDependency = eventLogPath && sidecarRoot
|
|
919
|
+
? toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath: eventLogPath })
|
|
920
|
+
: undefined;
|
|
921
|
+
const jsProfilePath = attachedEvidence.signals.js[0] ?? eventLogManifestPath;
|
|
922
|
+
const profileSessionEntriesManifestPath = profileSessionEntriesPath
|
|
923
|
+
? `raw/${path.basename(profileSessionEntriesPath)}`
|
|
924
|
+
: undefined;
|
|
925
|
+
const entries = [];
|
|
926
|
+
const pushDiagnostic = (kind, entry) => {
|
|
927
|
+
const request = resolveDiagnosticRequest({
|
|
928
|
+
kind,
|
|
929
|
+
optionalArtifacts,
|
|
930
|
+
optionalCapabilities,
|
|
931
|
+
requiredArtifacts,
|
|
932
|
+
requiredCapabilities,
|
|
933
|
+
});
|
|
934
|
+
entries.push(buildDiagnosticEntry({
|
|
935
|
+
kind,
|
|
936
|
+
...entry,
|
|
937
|
+
required: request.required || requiredProviderDiagnostics.has(kind) || Boolean(entry.required),
|
|
938
|
+
requested: request.requested || requiredProviderDiagnostics.has(kind) || Boolean(entry.requested),
|
|
939
|
+
}));
|
|
940
|
+
};
|
|
941
|
+
const logCaptured = platform === 'ios'
|
|
942
|
+
? Boolean(copiedSimctlLogManifestPath || simctlRuntimeLogDependency || explicitIosRuntimeLogManifestPath)
|
|
943
|
+
: Boolean(eventLogManifestPath);
|
|
944
|
+
pushDiagnostic('logs', {
|
|
945
|
+
name: platform === 'ios' ? 'simulator-runtime-log' : 'device-log',
|
|
946
|
+
...(typeof args['adb-artifacts'] === 'string'
|
|
947
|
+
? { provider: 'adb', runnerId: 'android-adb' }
|
|
948
|
+
: typeof args['simctl-artifacts'] === 'string'
|
|
949
|
+
? { provider: 'simctl', runnerId: 'ios-simctl' }
|
|
950
|
+
: typeof args.events === 'string'
|
|
951
|
+
? { provider: 'fixture-log-ingest' }
|
|
952
|
+
: {}),
|
|
953
|
+
status: logCaptured ? 'captured' : 'unavailable',
|
|
954
|
+
...(platform === 'ios'
|
|
955
|
+
? copiedSimctlLogManifestPath
|
|
956
|
+
? { path: copiedSimctlLogManifestPath }
|
|
957
|
+
: simctlRuntimeLogDependency
|
|
958
|
+
? { path: simctlRuntimeLogDependency.path }
|
|
959
|
+
: explicitIosRuntimeLogManifestPath
|
|
960
|
+
? { path: explicitIosRuntimeLogManifestPath }
|
|
961
|
+
: {}
|
|
962
|
+
: eventLogManifestPath
|
|
963
|
+
? { path: eventLogManifestPath }
|
|
964
|
+
: {}),
|
|
965
|
+
...(sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
|
|
966
|
+
...(platform === 'ios'
|
|
967
|
+
? simctlRuntimeLogDependency
|
|
968
|
+
? { evidenceDependency: simctlRuntimeLogDependency }
|
|
969
|
+
: {}
|
|
970
|
+
: eventLogDependency
|
|
971
|
+
? { evidenceDependency: eventLogDependency }
|
|
972
|
+
: {}),
|
|
973
|
+
...(logCaptured
|
|
974
|
+
? {
|
|
975
|
+
reason: platform === 'ios'
|
|
976
|
+
? 'iOS simulator runtime log evidence was available from the simctl capture sidecar.'
|
|
977
|
+
: 'Device or fixture log evidence was available to the profile runner.',
|
|
978
|
+
}
|
|
979
|
+
: {
|
|
980
|
+
reason: platform === 'ios'
|
|
981
|
+
? 'No iOS simulator runtime log was available in the selected simctl capture sidecar.'
|
|
982
|
+
: 'No device log source was supplied to this profile run.',
|
|
983
|
+
nextAction: platform === 'ios'
|
|
984
|
+
? 'Run with --simctl-capture or provide --simctl-artifacts containing raw/ios-simctl-log.txt.'
|
|
985
|
+
: 'Run with --events, --adb-artifacts, --adb-capture, or provide a runtime log artifact.',
|
|
986
|
+
}),
|
|
987
|
+
});
|
|
988
|
+
pushDiagnostic('js', {
|
|
989
|
+
name: 'profile-session-evidence',
|
|
990
|
+
status: eventLogManifestPath || attachedEvidence.signals.js.length > 0 ? 'captured' : 'unavailable',
|
|
991
|
+
...(jsProfilePath ? { path: jsProfilePath } : {}),
|
|
992
|
+
...(profileSessionEntriesManifestPath
|
|
993
|
+
? {
|
|
994
|
+
evidenceDependency: {
|
|
995
|
+
kind: 'profile-session-entries',
|
|
996
|
+
path: profileSessionEntriesManifestPath,
|
|
997
|
+
},
|
|
998
|
+
}
|
|
999
|
+
: eventLogDependency
|
|
1000
|
+
? { evidenceDependency: eventLogDependency }
|
|
1001
|
+
: {}),
|
|
1002
|
+
...(sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
|
|
1003
|
+
...(eventLogManifestPath || attachedEvidence.signals.js.length > 0
|
|
1004
|
+
? { reason: 'Profile or JS evidence was captured from runner input.' }
|
|
1005
|
+
: {
|
|
1006
|
+
reason: 'No profile-session event log or JS signal attachment was available.',
|
|
1007
|
+
nextAction: 'Attach JS evidence with --signal js:<path> or run a profile-session capture that emits profile events.',
|
|
1008
|
+
}),
|
|
1009
|
+
});
|
|
1010
|
+
const attachedScreenshotPath = attachedEvidence.captures.screenshots[0];
|
|
1011
|
+
const sidecarScreenshotDependency = attachedScreenshotPath ? null : adbScreenshotDependency;
|
|
1012
|
+
pushDiagnostic('screenshot', {
|
|
1013
|
+
...(sidecarScreenshotDependency ? { provider: 'adb', runnerId: 'android-adb' } : {}),
|
|
1014
|
+
status: attachedScreenshotPath || sidecarScreenshotDependency ? 'captured' : 'unavailable',
|
|
1015
|
+
...(attachedScreenshotPath
|
|
1016
|
+
? { path: attachedScreenshotPath }
|
|
1017
|
+
: sidecarScreenshotDependency
|
|
1018
|
+
? { path: sidecarScreenshotDependency.path }
|
|
1019
|
+
: {}),
|
|
1020
|
+
...(sidecarScreenshotDependency && sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
|
|
1021
|
+
...(sidecarScreenshotDependency ? { evidenceDependency: sidecarScreenshotDependency.dependency } : {}),
|
|
1022
|
+
...(attachedScreenshotPath || sidecarScreenshotDependency
|
|
1023
|
+
? {
|
|
1024
|
+
reason: sidecarScreenshotDependency
|
|
1025
|
+
? 'Screenshot evidence was available from the adb capture sidecar.'
|
|
1026
|
+
: 'Screenshot capture was attached to the run.',
|
|
1027
|
+
}
|
|
1028
|
+
: {
|
|
1029
|
+
reason: 'No screenshot capture was produced by the selected runner/provider set.',
|
|
1030
|
+
nextAction: 'Use --capture screenshot:<path> or a runner/provider that produces screenshots.',
|
|
1031
|
+
}),
|
|
1032
|
+
});
|
|
1033
|
+
pushDiagnostic('uiTree', {
|
|
1034
|
+
status: attachedEvidence.captures.uiTree ? 'captured' : 'unavailable',
|
|
1035
|
+
...(attachedEvidence.captures.uiTree ? { path: attachedEvidence.captures.uiTree } : {}),
|
|
1036
|
+
...(attachedEvidence.captures.uiTree
|
|
1037
|
+
? { reason: 'UI tree capture was attached to the run.' }
|
|
1038
|
+
: {
|
|
1039
|
+
reason: 'No UI tree capture was produced by the selected runner/provider set.',
|
|
1040
|
+
nextAction: 'Use --capture uiTree:<path> or add an accessibility/UI-tree provider.',
|
|
1041
|
+
}),
|
|
1042
|
+
});
|
|
1043
|
+
pushDiagnostic('video', {
|
|
1044
|
+
status: attachedEvidence.captures.video ? 'captured' : 'unavailable',
|
|
1045
|
+
...(attachedEvidence.captures.video ? { path: attachedEvidence.captures.video } : {}),
|
|
1046
|
+
...(attachedEvidence.captures.video
|
|
1047
|
+
? { reason: 'Video capture was attached to the run.' }
|
|
1048
|
+
: {
|
|
1049
|
+
reason: 'No video capture was produced by the selected runner/provider set.',
|
|
1050
|
+
nextAction: 'Use --capture video:<path> or run a capture provider that records video.',
|
|
1051
|
+
}),
|
|
1052
|
+
});
|
|
1053
|
+
for (const kind of ['memory', 'network']) {
|
|
1054
|
+
pushDiagnostic(kind, {
|
|
1055
|
+
status: attachedEvidence.signals[kind].length > 0 ? 'captured' : 'unavailable',
|
|
1056
|
+
...(attachedEvidence.signals[kind][0] ? { path: attachedEvidence.signals[kind][0] } : {}),
|
|
1057
|
+
...(attachedEvidence.signals[kind].length > 0
|
|
1058
|
+
? { reason: `${kind} signal evidence was attached to the run.` }
|
|
1059
|
+
: {
|
|
1060
|
+
reason: `No ${kind} signal evidence was produced by the selected provider set.`,
|
|
1061
|
+
nextAction: `Attach ${kind} evidence with --signal ${kind}:<path> or add a provider command that emits it.`,
|
|
1062
|
+
}),
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
for (const kind of ['accessibility', 'profiler']) {
|
|
1066
|
+
const attachment = attachedEvidence.attachments.find((item) => item.kind === kind);
|
|
1067
|
+
pushDiagnostic(kind, {
|
|
1068
|
+
...(attachment?.channel === 'provider' ? { provider: 'evidence-provider' } : {}),
|
|
1069
|
+
status: attachment ? 'captured' : 'unavailable',
|
|
1070
|
+
...(attachment ? { path: attachment.manifestPath } : {}),
|
|
1071
|
+
...(attachment
|
|
1072
|
+
? { reason: `${kind} provider evidence was attached to the run.` }
|
|
1073
|
+
: {
|
|
1074
|
+
reason: `No ${kind} provider attachment was produced by the selected provider set.`,
|
|
1075
|
+
nextAction: `Declare a provider command or attach ${kind} evidence before expecting this diagnostic.`,
|
|
1076
|
+
}),
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
return entries.map((entry) => {
|
|
1080
|
+
const cleaned = Object.entries(entry).filter(([, value]) => value !== undefined);
|
|
1081
|
+
return Object.fromEntries(cleaned);
|
|
1082
|
+
});
|
|
1083
|
+
}
|
|
1084
|
+
/**
|
|
1085
|
+
* Converts uncaptured required diagnostics into health checks.
|
|
1086
|
+
*
|
|
1087
|
+
* @param {DiagnosticInventoryEntry[]} diagnostics
|
|
1088
|
+
* @returns {Record<string, unknown>[]}
|
|
1089
|
+
*/
|
|
1090
|
+
function buildRequiredDiagnosticHealthChecks(diagnostics = []) {
|
|
1091
|
+
return diagnostics
|
|
1092
|
+
.filter((diagnostic) => diagnostic.required && diagnostic.status !== 'captured')
|
|
1093
|
+
.map((diagnostic) => ({
|
|
1094
|
+
name: `required_${diagnostic.kind}_diagnostic`,
|
|
1095
|
+
status: 'failed',
|
|
1096
|
+
source: 'evidence',
|
|
1097
|
+
code: 'required_diagnostic_not_captured',
|
|
1098
|
+
message: diagnostic.reason ?? `Required ${diagnostic.kind} diagnostic was not captured.`,
|
|
1099
|
+
metadata: {
|
|
1100
|
+
kind: diagnostic.kind,
|
|
1101
|
+
status: diagnostic.status,
|
|
1102
|
+
...(diagnostic.name ? { name: diagnostic.name } : {}),
|
|
1103
|
+
...(diagnostic.nextAction ? { nextAction: diagnostic.nextAction } : {}),
|
|
1104
|
+
...(diagnostic.provider ? { provider: diagnostic.provider } : {}),
|
|
1105
|
+
...(diagnostic.runnerId ? { runnerId: diagnostic.runnerId } : {}),
|
|
1106
|
+
},
|
|
1107
|
+
}));
|
|
1108
|
+
}
|
|
581
1109
|
/**
|
|
582
1110
|
* Builds scenario health from profile metrics.
|
|
583
1111
|
*
|
|
584
|
-
* @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>}} options
|
|
1112
|
+
* @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>, diagnostics?: DiagnosticInventoryEntry[], profileEventCount?: number, profileSessionEntryCount?: number, commandTransport?: string, sessionEntries?: Record<string, unknown>[]}} options
|
|
585
1113
|
* @returns {Record<string, unknown>}
|
|
586
1114
|
*/
|
|
587
|
-
function buildProfileHealth({ scenario, runId, metrics, }) {
|
|
1115
|
+
function buildProfileHealth({ scenario, runId, metrics, diagnostics = [], profileEventCount, profileSessionEntryCount, commandTransport, sessionEntries = [], }) {
|
|
588
1116
|
const passed = metrics.status === 'passed';
|
|
1117
|
+
const metadata = {
|
|
1118
|
+
failures: typeof metrics.failures === 'number' ? metrics.failures : null,
|
|
1119
|
+
timeouts: typeof metrics.timeouts === 'number' ? metrics.timeouts : null,
|
|
1120
|
+
};
|
|
1121
|
+
if (typeof profileEventCount === 'number') {
|
|
1122
|
+
metadata.profileEventCount = profileEventCount;
|
|
1123
|
+
}
|
|
1124
|
+
if (typeof profileSessionEntryCount === 'number') {
|
|
1125
|
+
metadata.profileSessionEntryCount = profileSessionEntryCount;
|
|
1126
|
+
}
|
|
1127
|
+
if (typeof commandTransport === 'string' && commandTransport.length > 0) {
|
|
1128
|
+
metadata.commandTransport = commandTransport;
|
|
1129
|
+
}
|
|
1130
|
+
if (!passed &&
|
|
1131
|
+
profileEventCount === 0 &&
|
|
1132
|
+
profileSessionEntryCount === 0 &&
|
|
1133
|
+
typeof commandTransport === 'string' &&
|
|
1134
|
+
commandTransport.startsWith('profile-session')) {
|
|
1135
|
+
metadata.nextActionCode = 'verify_profile_session_bootstrap';
|
|
1136
|
+
metadata.nextAction =
|
|
1137
|
+
'Verify the app loaded the expected bundle, mounted the profile-session bootstrap near the app root, and uses the configured storage keys or deep-link scheme before treating this as a product failure.';
|
|
1138
|
+
}
|
|
1139
|
+
const skippedCommands = sessionEntries.filter((entry) => (entry?.kind === 'command' && entry.status === 'skipped'));
|
|
1140
|
+
const firstSkippedCommand = skippedCommands[0];
|
|
1141
|
+
const firstSkippedReason = typeof firstSkippedCommand?.reason === 'string'
|
|
1142
|
+
? firstSkippedCommand.reason
|
|
1143
|
+
: undefined;
|
|
1144
|
+
const commandFailureCode = firstSkippedReason === 'wait-for-milestone-timeout'
|
|
1145
|
+
? 'profile_command_gate_timeout'
|
|
1146
|
+
: 'profile_command_skipped';
|
|
1147
|
+
const commandFailureMessage = firstSkippedReason === 'wait-for-milestone-timeout'
|
|
1148
|
+
? 'One or more profile-session commands waited for a milestone that was not observed before timeout.'
|
|
1149
|
+
: 'One or more profile-session commands were skipped before the scenario completed.';
|
|
1150
|
+
const commandChecks = skippedCommands.length > 0
|
|
1151
|
+
? [
|
|
1152
|
+
{
|
|
1153
|
+
name: 'profile_command_sequence',
|
|
1154
|
+
status: 'failed',
|
|
1155
|
+
source: 'runner',
|
|
1156
|
+
code: commandFailureCode,
|
|
1157
|
+
message: commandFailureMessage,
|
|
1158
|
+
metadata: {
|
|
1159
|
+
skippedCommandCount: skippedCommands.length,
|
|
1160
|
+
...(typeof firstSkippedCommand?.command === 'string' ? { command: firstSkippedCommand.command } : {}),
|
|
1161
|
+
...(typeof firstSkippedCommand?.commandId === 'string' ? { commandId: firstSkippedCommand.commandId } : {}),
|
|
1162
|
+
...(typeof firstSkippedCommand?.queueId === 'string' ? { queueId: firstSkippedCommand.queueId } : {}),
|
|
1163
|
+
...(typeof firstSkippedCommand?.reason === 'string' ? { reason: firstSkippedCommand.reason } : {}),
|
|
1164
|
+
...(typeof firstSkippedCommand?.sequence === 'number' ? { sequence: firstSkippedCommand.sequence } : {}),
|
|
1165
|
+
...(typeof firstSkippedCommand?.waitForMilestone === 'string'
|
|
1166
|
+
? { waitForMilestone: firstSkippedCommand.waitForMilestone }
|
|
1167
|
+
: {}),
|
|
1168
|
+
...(typeof firstSkippedCommand?.waitTimeoutMs === 'number'
|
|
1169
|
+
? { waitTimeoutMs: firstSkippedCommand.waitTimeoutMs }
|
|
1170
|
+
: {}),
|
|
1171
|
+
},
|
|
1172
|
+
},
|
|
1173
|
+
]
|
|
1174
|
+
: [];
|
|
1175
|
+
const commandChecksPassed = commandChecks.every((check) => check.status === 'passed');
|
|
1176
|
+
const diagnosticChecks = buildRequiredDiagnosticHealthChecks(diagnostics);
|
|
1177
|
+
const diagnosticChecksPassed = diagnosticChecks.every((check) => check.status === 'passed');
|
|
1178
|
+
const healthPassed = passed && commandChecksPassed && diagnosticChecksPassed;
|
|
589
1179
|
return assertValidJson({
|
|
590
1180
|
schemaVersion: '1.0.0',
|
|
591
1181
|
scenarioId: scenario.name,
|
|
592
1182
|
...(typeof scenario.flowId === 'string' ? { flowId: scenario.flowId } : {}),
|
|
593
1183
|
runId,
|
|
594
|
-
healthStatus:
|
|
1184
|
+
healthStatus: healthPassed ? 'passed' : 'failed',
|
|
595
1185
|
checks: [
|
|
596
1186
|
{
|
|
597
1187
|
name: 'truth_events_complete',
|
|
@@ -601,11 +1191,10 @@ function buildProfileHealth({ scenario, runId, metrics, }) {
|
|
|
601
1191
|
message: passed
|
|
602
1192
|
? 'Profile events completed every expected iteration.'
|
|
603
1193
|
: 'Profile events did not complete every expected iteration.',
|
|
604
|
-
metadata
|
|
605
|
-
failures: typeof metrics.failures === 'number' ? metrics.failures : null,
|
|
606
|
-
timeouts: typeof metrics.timeouts === 'number' ? metrics.timeouts : null,
|
|
607
|
-
},
|
|
1194
|
+
metadata,
|
|
608
1195
|
},
|
|
1196
|
+
...commandChecks,
|
|
1197
|
+
...diagnosticChecks,
|
|
609
1198
|
],
|
|
610
1199
|
}, SCHEMAS.health, 'Health artifact');
|
|
611
1200
|
}
|
|
@@ -857,6 +1446,121 @@ function resolveProfileSessionEntriesPath({ args, platform }) {
|
|
|
857
1446
|
}
|
|
858
1447
|
return null;
|
|
859
1448
|
}
|
|
1449
|
+
/**
|
|
1450
|
+
* Resolves the run id used by rehydrated sidecar evidence.
|
|
1451
|
+
*
|
|
1452
|
+
* A rehydrated artifact can intentionally have a new run id while ingesting a
|
|
1453
|
+
* previously captured adb/simctl sidecar. Keep live runs strict, but allow an
|
|
1454
|
+
* explicit sidecar with exactly one source run id for the scenario to provide
|
|
1455
|
+
* the event filter.
|
|
1456
|
+
*
|
|
1457
|
+
* @param {{args: CliArgs, eventLogText: string, profileSessionEntriesPath: string | null, runId: string, scenarioName: string}} options
|
|
1458
|
+
* @returns {string}
|
|
1459
|
+
*/
|
|
1460
|
+
function resolveEvidenceFilterRunId({ args, eventLogText, profileSessionEntriesPath, runId, scenarioName, }) {
|
|
1461
|
+
const isRehydratedSidecar = typeof args['adb-artifacts'] === 'string' || typeof args['simctl-artifacts'] === 'string';
|
|
1462
|
+
if (!isRehydratedSidecar) {
|
|
1463
|
+
return runId;
|
|
1464
|
+
}
|
|
1465
|
+
const scenarioEvents = extractProfileEvents(eventLogText, { scenario: scenarioName });
|
|
1466
|
+
const currentRunEvents = scenarioEvents.filter((event) => event.runId === runId);
|
|
1467
|
+
if (currentRunEvents.length > 0) {
|
|
1468
|
+
return runId;
|
|
1469
|
+
}
|
|
1470
|
+
const sourceRunIds = new Set(scenarioEvents
|
|
1471
|
+
.map((event) => event.runId)
|
|
1472
|
+
.filter((sourceRunId) => typeof sourceRunId === 'string' && sourceRunId.length > 0));
|
|
1473
|
+
if (profileSessionEntriesPath && fs.existsSync(profileSessionEntriesPath)) {
|
|
1474
|
+
const storedEntries = JSON.parse(fs.readFileSync(profileSessionEntriesPath, 'utf8'));
|
|
1475
|
+
if (Array.isArray(storedEntries)) {
|
|
1476
|
+
for (const entry of storedEntries) {
|
|
1477
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
1478
|
+
continue;
|
|
1479
|
+
}
|
|
1480
|
+
const record = entry;
|
|
1481
|
+
if (record.scenario !== scenarioName) {
|
|
1482
|
+
continue;
|
|
1483
|
+
}
|
|
1484
|
+
if (typeof record.runId === 'string' && record.runId.length > 0) {
|
|
1485
|
+
sourceRunIds.add(record.runId);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
return sourceRunIds.size === 1 ? [...sourceRunIds][0] : runId;
|
|
1491
|
+
}
|
|
1492
|
+
/**
|
|
1493
|
+
* Returns the first usable adb screenshot file from sidecar metadata.
|
|
1494
|
+
*
|
|
1495
|
+
* ADB can produce a valid PNG even when command metadata records a nonzero
|
|
1496
|
+
* exit status from the host process. Treat the binary artifact as the capture
|
|
1497
|
+
* authority, but only after validating the PNG signature and sidecar boundary.
|
|
1498
|
+
*
|
|
1499
|
+
* @param {{runDir: string, sidecarRoot: string | null}} options
|
|
1500
|
+
* @returns {{dependency: SidecarEvidenceDependency, path: string} | null}
|
|
1501
|
+
*/
|
|
1502
|
+
function resolveAndroidAdbScreenshotDependency({ runDir, sidecarRoot, }) {
|
|
1503
|
+
if (!sidecarRoot) {
|
|
1504
|
+
return null;
|
|
1505
|
+
}
|
|
1506
|
+
const metadata = readOptionalJsonObject(path.resolve(sidecarRoot, 'raw', 'android-metadata.json'));
|
|
1507
|
+
const actions = Array.isArray(metadata?.driverActions) ? metadata.driverActions : [];
|
|
1508
|
+
for (const action of actions) {
|
|
1509
|
+
if (!action || typeof action !== 'object' || Array.isArray(action)) {
|
|
1510
|
+
continue;
|
|
1511
|
+
}
|
|
1512
|
+
const record = action;
|
|
1513
|
+
if (record.driverAction !== 'screenshot') {
|
|
1514
|
+
continue;
|
|
1515
|
+
}
|
|
1516
|
+
const sidecarRelativePath = typeof record.capturePath === 'string'
|
|
1517
|
+
? record.capturePath
|
|
1518
|
+
: typeof record.rawPath === 'string'
|
|
1519
|
+
? record.rawPath
|
|
1520
|
+
: null;
|
|
1521
|
+
if (!sidecarRelativePath || path.isAbsolute(sidecarRelativePath)) {
|
|
1522
|
+
continue;
|
|
1523
|
+
}
|
|
1524
|
+
const screenshotPath = path.resolve(sidecarRoot, sidecarRelativePath);
|
|
1525
|
+
const relativeToSidecar = path.relative(sidecarRoot, screenshotPath);
|
|
1526
|
+
if (relativeToSidecar.length === 0 ||
|
|
1527
|
+
relativeToSidecar.startsWith('..') ||
|
|
1528
|
+
path.isAbsolute(relativeToSidecar) ||
|
|
1529
|
+
!isPngFile(screenshotPath)) {
|
|
1530
|
+
continue;
|
|
1531
|
+
}
|
|
1532
|
+
const sidecarDependency = toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath: screenshotPath });
|
|
1533
|
+
return {
|
|
1534
|
+
dependency: sidecarDependency,
|
|
1535
|
+
path: sidecarDependency.path,
|
|
1536
|
+
};
|
|
1537
|
+
}
|
|
1538
|
+
return null;
|
|
1539
|
+
}
|
|
1540
|
+
/**
|
|
1541
|
+
* Checks whether a file starts with the PNG signature.
|
|
1542
|
+
*
|
|
1543
|
+
* @param {string} filePath
|
|
1544
|
+
* @returns {boolean}
|
|
1545
|
+
*/
|
|
1546
|
+
function isPngFile(filePath) {
|
|
1547
|
+
let signature;
|
|
1548
|
+
try {
|
|
1549
|
+
signature = fs.readFileSync(filePath, { flag: 'r' }).subarray(0, 8);
|
|
1550
|
+
}
|
|
1551
|
+
catch {
|
|
1552
|
+
return false;
|
|
1553
|
+
}
|
|
1554
|
+
return signature.length === 8 &&
|
|
1555
|
+
signature[0] === 0x89 &&
|
|
1556
|
+
signature[1] === 0x50 &&
|
|
1557
|
+
signature[2] === 0x4e &&
|
|
1558
|
+
signature[3] === 0x47 &&
|
|
1559
|
+
signature[4] === 0x0d &&
|
|
1560
|
+
signature[5] === 0x0a &&
|
|
1561
|
+
signature[6] === 0x1a &&
|
|
1562
|
+
signature[7] === 0x0a;
|
|
1563
|
+
}
|
|
860
1564
|
/**
|
|
861
1565
|
* Reads a JSON artifact if it exists and contains an object.
|
|
862
1566
|
*
|
|
@@ -964,6 +1668,66 @@ function resolveProfileScenarioName({ scenario, scenarioPath, }) {
|
|
|
964
1668
|
}
|
|
965
1669
|
return path.basename(scenarioPath, '.json');
|
|
966
1670
|
}
|
|
1671
|
+
/**
|
|
1672
|
+
* Reads evidence-provider manifests for profile compatibility preflight.
|
|
1673
|
+
*
|
|
1674
|
+
* @param {CliArgs} args
|
|
1675
|
+
* @returns {Record<string, unknown>[]}
|
|
1676
|
+
*/
|
|
1677
|
+
function readEvidenceProviderManifests(args) {
|
|
1678
|
+
return readRepeatableArgValues(args, 'provider').map((providerPath, index) => assertValidJson(readJson(path.resolve(providerPath)), SCHEMAS.runnerCapabilities, `Evidence provider manifest ${index + 1}`));
|
|
1679
|
+
}
|
|
1680
|
+
/**
|
|
1681
|
+
* Runs planner compatibility before a live profile capture starts.
|
|
1682
|
+
*
|
|
1683
|
+
* Failed compatibility writes classified artifacts in the profile run folder so
|
|
1684
|
+
* agents can stop before adb, simctl, or provider work consumes runtime time.
|
|
1685
|
+
*
|
|
1686
|
+
* @param {CompatibilityPreflightOptions} options
|
|
1687
|
+
* @returns {Promise<void>}
|
|
1688
|
+
*/
|
|
1689
|
+
async function runProfileCompatibilityPreflight({ args, artifactRoot, platform, primaryRunner, runDir, runId, scenario, scenarioName, }) {
|
|
1690
|
+
const layout = createArtifactLayout({ outputDir: runDir });
|
|
1691
|
+
const compatibility = evaluateRunnerCompatibility({
|
|
1692
|
+
scenario,
|
|
1693
|
+
runner: primaryRunner,
|
|
1694
|
+
evidenceProviders: readEvidenceProviderManifests(args),
|
|
1695
|
+
platform,
|
|
1696
|
+
});
|
|
1697
|
+
await writeJsonArtifact({
|
|
1698
|
+
filePath: layout.plannerCompatibility,
|
|
1699
|
+
value: compatibility,
|
|
1700
|
+
schema: {
|
|
1701
|
+
type: 'object',
|
|
1702
|
+
additionalProperties: true,
|
|
1703
|
+
},
|
|
1704
|
+
label: 'Planner compatibility artifact',
|
|
1705
|
+
});
|
|
1706
|
+
if (compatibility.compatible) {
|
|
1707
|
+
process.stderr.write(`profile preflight passed: ${platform}/${scenarioName} artifactRoot=${artifactRoot} planner=${path.relative(process.cwd(), layout.plannerCompatibility)}\n`);
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
const health = buildCompatibilityHealth({ scenario, runId, compatibility });
|
|
1711
|
+
const verdict = buildUnevaluatedVerdict({ scenario, runId, health });
|
|
1712
|
+
const agentSummary = buildAgentSummaryMarkdown({ health, verdict });
|
|
1713
|
+
await writeJsonArtifact({
|
|
1714
|
+
filePath: layout.health,
|
|
1715
|
+
value: health,
|
|
1716
|
+
schema: SCHEMAS.health,
|
|
1717
|
+
label: 'Health artifact',
|
|
1718
|
+
});
|
|
1719
|
+
await writeJsonArtifact({
|
|
1720
|
+
filePath: layout.verdict,
|
|
1721
|
+
value: verdict,
|
|
1722
|
+
schema: SCHEMAS.verdict,
|
|
1723
|
+
label: 'Verdict artifact',
|
|
1724
|
+
});
|
|
1725
|
+
await writeTextArtifact({
|
|
1726
|
+
filePath: layout.agentSummary,
|
|
1727
|
+
content: agentSummary,
|
|
1728
|
+
});
|
|
1729
|
+
throw new Error(`Profile compatibility preflight failed; inspect ${runDir}/agent-summary.md.`);
|
|
1730
|
+
}
|
|
967
1731
|
/**
|
|
968
1732
|
* Serializes JSON with stable object key ordering for reproducible hashes.
|
|
969
1733
|
*
|
|
@@ -1045,6 +1809,46 @@ function findMilestoneEvent(scenario, milestoneId) {
|
|
|
1045
1809
|
}
|
|
1046
1810
|
return null;
|
|
1047
1811
|
}
|
|
1812
|
+
/**
|
|
1813
|
+
* Returns true when a milestone is explicitly optional.
|
|
1814
|
+
*
|
|
1815
|
+
* @param {Record<string, unknown>} scenario
|
|
1816
|
+
* @param {unknown} milestoneId
|
|
1817
|
+
* @returns {boolean}
|
|
1818
|
+
*/
|
|
1819
|
+
function isOptionalMilestone(scenario, milestoneId) {
|
|
1820
|
+
if (typeof milestoneId !== 'string' || !Array.isArray(scenario.milestones)) {
|
|
1821
|
+
return false;
|
|
1822
|
+
}
|
|
1823
|
+
for (const milestone of scenario.milestones) {
|
|
1824
|
+
if (!isRecord(milestone)) {
|
|
1825
|
+
continue;
|
|
1826
|
+
}
|
|
1827
|
+
if (milestone.id === milestoneId) {
|
|
1828
|
+
return milestone.required === false;
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
return false;
|
|
1832
|
+
}
|
|
1833
|
+
/**
|
|
1834
|
+
* Returns true when a milestone represents one-time scenario readiness rather than a repeated cycle edge.
|
|
1835
|
+
*
|
|
1836
|
+
* @param {Record<string, unknown>} scenario
|
|
1837
|
+
* @param {unknown} milestoneId
|
|
1838
|
+
* @param {string | null} milestoneEvent
|
|
1839
|
+
* @returns {boolean}
|
|
1840
|
+
*/
|
|
1841
|
+
function isReadinessMilestone(scenario, milestoneId, milestoneEvent) {
|
|
1842
|
+
const readyEvent = isRecord(scenario.truthEvents) && isRecord(scenario.truthEvents.ready)
|
|
1843
|
+
? scenario.truthEvents.ready.event
|
|
1844
|
+
: undefined;
|
|
1845
|
+
if (typeof readyEvent === 'string' && milestoneEvent === readyEvent) {
|
|
1846
|
+
return true;
|
|
1847
|
+
}
|
|
1848
|
+
const id = typeof milestoneId === 'string' ? milestoneId.toLowerCase() : '';
|
|
1849
|
+
const event = typeof milestoneEvent === 'string' ? milestoneEvent.toLowerCase() : '';
|
|
1850
|
+
return id.includes('ready') || event.includes('ready');
|
|
1851
|
+
}
|
|
1048
1852
|
/**
|
|
1049
1853
|
* Builds a milestone-id to event-name lookup for schema-era scenarios.
|
|
1050
1854
|
*
|
|
@@ -1100,6 +1904,16 @@ function resolveProfileMetricEvents(scenario) {
|
|
|
1100
1904
|
milestone: toEvent,
|
|
1101
1905
|
};
|
|
1102
1906
|
}
|
|
1907
|
+
if (fromEvent && toEvent && isReadinessMilestone(scenario, budget.fromMilestone, fromEvent)) {
|
|
1908
|
+
return {
|
|
1909
|
+
milestone: toEvent,
|
|
1910
|
+
};
|
|
1911
|
+
}
|
|
1912
|
+
if (fromEvent && toEvent && isOptionalMilestone(scenario, budget.fromMilestone)) {
|
|
1913
|
+
return {
|
|
1914
|
+
milestone: toEvent,
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1103
1917
|
if (fromEvent && toEvent) {
|
|
1104
1918
|
return {
|
|
1105
1919
|
closeRequested: toEvent,
|
|
@@ -1112,7 +1926,59 @@ function resolveProfileMetricEvents(scenario) {
|
|
|
1112
1926
|
return null;
|
|
1113
1927
|
}
|
|
1114
1928
|
/**
|
|
1115
|
-
*
|
|
1929
|
+
* Resolves how many repeated completion milestone events prove one cycle body.
|
|
1930
|
+
*
|
|
1931
|
+
* @param {Record<string, unknown>} scenario
|
|
1932
|
+
* @param {Record<string, string> | null} metricEvents
|
|
1933
|
+
* @returns {number}
|
|
1934
|
+
*/
|
|
1935
|
+
function resolveMilestoneEventsPerIteration(scenario, metricEvents) {
|
|
1936
|
+
if (!metricEvents || typeof metricEvents.milestone !== 'string' || !Array.isArray(scenario.steps)) {
|
|
1937
|
+
return 1;
|
|
1938
|
+
}
|
|
1939
|
+
const milestoneEvents = buildMilestoneEventLookup(scenario);
|
|
1940
|
+
const matchingWaitStepIds = new Set();
|
|
1941
|
+
for (const step of scenario.steps) {
|
|
1942
|
+
if (!isRecord(step) || step.kind !== 'waitForMilestone' || typeof step.milestone !== 'string') {
|
|
1943
|
+
continue;
|
|
1944
|
+
}
|
|
1945
|
+
const event = milestoneEvents[step.milestone] ?? step.milestone;
|
|
1946
|
+
if (event === metricEvents.milestone && typeof step.id === 'string') {
|
|
1947
|
+
matchingWaitStepIds.add(step.id);
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
if (matchingWaitStepIds.size === 0) {
|
|
1951
|
+
return 1;
|
|
1952
|
+
}
|
|
1953
|
+
const bodyStepIds = isRecord(scenario.cycles) && Array.isArray(scenario.cycles.bodyStepIds)
|
|
1954
|
+
? new Set(scenario.cycles.bodyStepIds.filter((entry) => typeof entry === 'string'))
|
|
1955
|
+
: null;
|
|
1956
|
+
if (bodyStepIds && bodyStepIds.size > 0) {
|
|
1957
|
+
let count = 0;
|
|
1958
|
+
let bodyCommandPending = false;
|
|
1959
|
+
for (const step of scenario.steps) {
|
|
1960
|
+
if (!isRecord(step)) {
|
|
1961
|
+
continue;
|
|
1962
|
+
}
|
|
1963
|
+
if (typeof step.id === 'string' && bodyStepIds.has(step.id) && step.kind === 'command') {
|
|
1964
|
+
bodyCommandPending = true;
|
|
1965
|
+
continue;
|
|
1966
|
+
}
|
|
1967
|
+
if (bodyCommandPending && typeof step.id === 'string' && matchingWaitStepIds.has(step.id)) {
|
|
1968
|
+
count += 1;
|
|
1969
|
+
bodyCommandPending = false;
|
|
1970
|
+
continue;
|
|
1971
|
+
}
|
|
1972
|
+
if (bodyCommandPending && step.kind === 'command') {
|
|
1973
|
+
bodyCommandPending = false;
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
return count > 1 ? count : 1;
|
|
1977
|
+
}
|
|
1978
|
+
return matchingWaitStepIds.size > 1 ? matchingWaitStepIds.size : 1;
|
|
1979
|
+
}
|
|
1980
|
+
/**
|
|
1981
|
+
* Maps shorthand milestone budget fields to aggregate profile budget keys.
|
|
1116
1982
|
*
|
|
1117
1983
|
* @param {{budget: Record<string, unknown>, metric: string}} options
|
|
1118
1984
|
* @returns {string | null}
|
|
@@ -1144,11 +2010,24 @@ function resolveProfileBudgets(scenario) {
|
|
|
1144
2010
|
return null;
|
|
1145
2011
|
}
|
|
1146
2012
|
const pass = {};
|
|
2013
|
+
const intervals = [];
|
|
1147
2014
|
for (const budget of scenario.budgets) {
|
|
1148
2015
|
if (!isRecord(budget) || typeof budget.limit !== 'number') {
|
|
1149
2016
|
continue;
|
|
1150
2017
|
}
|
|
1151
2018
|
if (budget.metric === 'p95' || budget.metric === 'p50') {
|
|
2019
|
+
const fromEvent = findMilestoneEvent(scenario, budget.fromMilestone);
|
|
2020
|
+
const toEvent = findMilestoneEvent(scenario, budget.toMilestone);
|
|
2021
|
+
if (fromEvent && toEvent) {
|
|
2022
|
+
intervals.push({
|
|
2023
|
+
name: typeof budget.name === 'string' ? budget.name : `${String(budget.fromMilestone)} to ${String(budget.toMilestone)}`,
|
|
2024
|
+
metric: budget.metric,
|
|
2025
|
+
limit: budget.limit,
|
|
2026
|
+
fromEvent,
|
|
2027
|
+
toEvent,
|
|
2028
|
+
});
|
|
2029
|
+
continue;
|
|
2030
|
+
}
|
|
1152
2031
|
const budgetKey = resolveProfileBudgetKey({ budget, metric: budget.metric });
|
|
1153
2032
|
if (budgetKey) {
|
|
1154
2033
|
pass[budgetKey] = budget.limit;
|
|
@@ -1161,10 +2040,11 @@ function resolveProfileBudgets(scenario) {
|
|
|
1161
2040
|
pass.timeouts = budget.limit;
|
|
1162
2041
|
}
|
|
1163
2042
|
}
|
|
1164
|
-
return Object.keys(pass).length > 0
|
|
2043
|
+
return Object.keys(pass).length > 0 || intervals.length > 0
|
|
1165
2044
|
? {
|
|
1166
2045
|
metric: 'milestone budget',
|
|
1167
2046
|
pass,
|
|
2047
|
+
...(intervals.length > 0 ? { intervals } : {}),
|
|
1168
2048
|
}
|
|
1169
2049
|
: null;
|
|
1170
2050
|
}
|
|
@@ -1287,6 +2167,7 @@ async function runProfileMobile(args, options) {
|
|
|
1287
2167
|
const scenarioHash = hashScenarioContract(profileScenario);
|
|
1288
2168
|
const expectedIterations = resolveExpectedIterations(profileScenario);
|
|
1289
2169
|
const profileMetricEvents = resolveProfileMetricEvents(profileScenario);
|
|
2170
|
+
const milestoneEventsPerIteration = resolveMilestoneEventsPerIteration(profileScenario, profileMetricEvents);
|
|
1290
2171
|
const profileBudgets = resolveProfileBudgets(profileScenario);
|
|
1291
2172
|
const runId = typeof args['run-id'] === 'string' ? args['run-id'] : createRunId();
|
|
1292
2173
|
const artifactRoot = resolveArtifactRoot({ args, config, configPath, platform: options.platform });
|
|
@@ -1304,6 +2185,22 @@ async function runProfileMobile(args, options) {
|
|
|
1304
2185
|
await ensureDir(layout.signals.js);
|
|
1305
2186
|
await ensureDir(layout.signals.memory);
|
|
1306
2187
|
await ensureDir(layout.signals.network);
|
|
2188
|
+
const runPlan = buildProfileRunPlan({
|
|
2189
|
+
args,
|
|
2190
|
+
artifactRoot,
|
|
2191
|
+
comparisonLane,
|
|
2192
|
+
expectedIterations,
|
|
2193
|
+
interactionDriver,
|
|
2194
|
+
layout,
|
|
2195
|
+
milestoneEventsPerIteration,
|
|
2196
|
+
options,
|
|
2197
|
+
profileScenario,
|
|
2198
|
+
runDir,
|
|
2199
|
+
runId,
|
|
2200
|
+
scenarioHash,
|
|
2201
|
+
scenarioPath,
|
|
2202
|
+
});
|
|
2203
|
+
await writeProfileRunPlan({ layout, plan: runPlan });
|
|
1307
2204
|
const providerExecution = await executeProviderCommands({
|
|
1308
2205
|
args,
|
|
1309
2206
|
layout,
|
|
@@ -1342,15 +2239,69 @@ async function runProfileMobile(args, options) {
|
|
|
1342
2239
|
verdict,
|
|
1343
2240
|
};
|
|
1344
2241
|
}
|
|
1345
|
-
|
|
2242
|
+
let attachedEvidence;
|
|
2243
|
+
try {
|
|
2244
|
+
attachedEvidence = await resolveAttachedEvidence({ args, layout, providerInputs: providerExecution.inputs });
|
|
2245
|
+
}
|
|
2246
|
+
catch (error) {
|
|
2247
|
+
const providerInput = providerExecution.inputs.find((input) => error instanceof Error && error.message.includes(input.sourcePath));
|
|
2248
|
+
const health = buildProviderCommandFailureHealth({
|
|
2249
|
+
failures: [
|
|
2250
|
+
{
|
|
2251
|
+
commandId: 'provider-evidence',
|
|
2252
|
+
code: 'provider_evidence_invalid',
|
|
2253
|
+
exitCode: null,
|
|
2254
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2255
|
+
name: 'evidence_provider_output_valid',
|
|
2256
|
+
nextAction: 'Fix the provider output so it satisfies the ASL evidence contract, then rerun the profile.',
|
|
2257
|
+
nextActionCode: 'fix_provider_evidence_output',
|
|
2258
|
+
phase: 'afterCapture',
|
|
2259
|
+
providerId: providerInput?.providerId ?? 'unknown-provider',
|
|
2260
|
+
...(providerInput?.manifestPath ? { rawPath: providerInput.manifestPath } : {}),
|
|
2261
|
+
},
|
|
2262
|
+
],
|
|
2263
|
+
runId,
|
|
2264
|
+
scenario: profileScenario,
|
|
2265
|
+
});
|
|
2266
|
+
const verdict = buildProfileVerdict({ scenario: profileScenario, runId, health, metrics: {} });
|
|
2267
|
+
const agentSummary = buildAgentSummaryMarkdown({ health, verdict });
|
|
2268
|
+
await writeJsonArtifact({
|
|
2269
|
+
filePath: layout.health,
|
|
2270
|
+
value: health,
|
|
2271
|
+
schema: SCHEMAS.health,
|
|
2272
|
+
label: 'Health artifact',
|
|
2273
|
+
});
|
|
2274
|
+
await writeJsonArtifact({
|
|
2275
|
+
filePath: layout.verdict,
|
|
2276
|
+
value: verdict,
|
|
2277
|
+
schema: SCHEMAS.verdict,
|
|
2278
|
+
label: 'Verdict artifact',
|
|
2279
|
+
});
|
|
2280
|
+
await writeTextArtifact({
|
|
2281
|
+
filePath: layout.agentSummary,
|
|
2282
|
+
content: agentSummary,
|
|
2283
|
+
});
|
|
2284
|
+
return {
|
|
2285
|
+
runDir,
|
|
2286
|
+
health,
|
|
2287
|
+
verdict,
|
|
2288
|
+
};
|
|
2289
|
+
}
|
|
1346
2290
|
const eventLogText = eventLogPath ? await fsp.readFile(eventLogPath, 'utf8') : '';
|
|
2291
|
+
const evidenceFilterRunId = resolveEvidenceFilterRunId({
|
|
2292
|
+
args,
|
|
2293
|
+
eventLogText,
|
|
2294
|
+
profileSessionEntriesPath,
|
|
2295
|
+
runId,
|
|
2296
|
+
scenarioName,
|
|
2297
|
+
});
|
|
1347
2298
|
const events = extractProfileEvents(eventLogText, {
|
|
1348
2299
|
scenario: scenarioName,
|
|
1349
|
-
runId,
|
|
2300
|
+
runId: evidenceFilterRunId,
|
|
1350
2301
|
});
|
|
1351
2302
|
const logSessionEntries = extractProfileSessionEntries(eventLogText, {
|
|
1352
2303
|
scenario: scenarioName,
|
|
1353
|
-
runId,
|
|
2304
|
+
runId: evidenceFilterRunId,
|
|
1354
2305
|
});
|
|
1355
2306
|
const storedSessionEntries = profileSessionEntriesPath
|
|
1356
2307
|
? JSON.parse(await fsp.readFile(profileSessionEntriesPath, 'utf8'))
|
|
@@ -1364,7 +2315,7 @@ async function runProfileMobile(args, options) {
|
|
|
1364
2315
|
}
|
|
1365
2316
|
const record = entry;
|
|
1366
2317
|
return ((!('scenario' in record) || record.scenario === scenarioName) &&
|
|
1367
|
-
(!('runId' in record) || record.runId ===
|
|
2318
|
+
(!('runId' in record) || record.runId === evidenceFilterRunId));
|
|
1368
2319
|
})
|
|
1369
2320
|
: []),
|
|
1370
2321
|
];
|
|
@@ -1376,11 +2327,16 @@ async function runProfileMobile(args, options) {
|
|
|
1376
2327
|
expectedIterations,
|
|
1377
2328
|
budgets: profileBudgets,
|
|
1378
2329
|
cycleEventNames: profileMetricEvents,
|
|
2330
|
+
milestoneEventsPerIteration,
|
|
1379
2331
|
artifacts: {
|
|
1380
2332
|
captures: attachedEvidence.captures,
|
|
1381
2333
|
signals: attachedEvidence.signals,
|
|
1382
2334
|
},
|
|
1383
2335
|
});
|
|
2336
|
+
const eventLogRawPath = eventLogPath ? `raw/${path.basename(eventLogPath)}` : undefined;
|
|
2337
|
+
const eventLogIsProfileSessionEvidenceOnly = options.platform === 'ios' &&
|
|
2338
|
+
eventLogPath &&
|
|
2339
|
+
path.basename(eventLogPath) === 'ios-profile-events.log';
|
|
1384
2340
|
const manifestArtifacts = {
|
|
1385
2341
|
causalRun: 'causal-run.json',
|
|
1386
2342
|
budgetVerdict: 'budget-verdict.json',
|
|
@@ -1389,13 +2345,20 @@ async function runProfileMobile(args, options) {
|
|
|
1389
2345
|
summary: 'summary.md',
|
|
1390
2346
|
scenario: toPortablePathReference(scenarioPath),
|
|
1391
2347
|
raw: {
|
|
1392
|
-
|
|
1393
|
-
|
|
2348
|
+
...(eventLogRawPath && !eventLogIsProfileSessionEvidenceOnly
|
|
2349
|
+
? {
|
|
2350
|
+
interactionLog: eventLogRawPath,
|
|
2351
|
+
deviceLog: eventLogRawPath,
|
|
2352
|
+
}
|
|
2353
|
+
: {}),
|
|
2354
|
+
...(profileSessionEntriesPath
|
|
2355
|
+
? { profileSessionEntries: `raw/${path.basename(profileSessionEntriesPath)}` }
|
|
2356
|
+
: {}),
|
|
1394
2357
|
},
|
|
1395
2358
|
captures: {
|
|
1396
2359
|
screenshots: attachedEvidence.captures.screenshots,
|
|
1397
|
-
video: attachedEvidence.captures.video
|
|
1398
|
-
uiTree: attachedEvidence.captures.uiTree
|
|
2360
|
+
...(attachedEvidence.captures.video ? { video: attachedEvidence.captures.video } : {}),
|
|
2361
|
+
...(attachedEvidence.captures.uiTree ? { uiTree: attachedEvidence.captures.uiTree } : {}),
|
|
1399
2362
|
},
|
|
1400
2363
|
signals: {
|
|
1401
2364
|
js: attachedEvidence.signals.js,
|
|
@@ -1403,8 +2366,18 @@ async function runProfileMobile(args, options) {
|
|
|
1403
2366
|
network: attachedEvidence.signals.network,
|
|
1404
2367
|
},
|
|
1405
2368
|
evidenceAttachments: buildEvidenceAttachmentManifest(attachedEvidence.attachments),
|
|
2369
|
+
diagnostics: buildDiagnosticInventory({
|
|
2370
|
+
args,
|
|
2371
|
+
attachedEvidence,
|
|
2372
|
+
eventLogPath,
|
|
2373
|
+
platform: options.platform,
|
|
2374
|
+
profileSessionEntriesPath,
|
|
2375
|
+
runDir,
|
|
2376
|
+
scenario: profileScenario,
|
|
2377
|
+
}),
|
|
1406
2378
|
};
|
|
1407
2379
|
const appId = resolveAppId({ config, platform: options.platform });
|
|
2380
|
+
const commandTransport = resolveCommandTransport({ args, interactionDriver, options });
|
|
1408
2381
|
const provenanceCohort = buildProfileProvenanceCohort({
|
|
1409
2382
|
appId,
|
|
1410
2383
|
args,
|
|
@@ -1465,7 +2438,16 @@ async function runProfileMobile(args, options) {
|
|
|
1465
2438
|
runId,
|
|
1466
2439
|
budgetEvaluation: metrics.budgetEvaluation ?? null,
|
|
1467
2440
|
});
|
|
1468
|
-
const health = buildProfileHealth({
|
|
2441
|
+
const health = buildProfileHealth({
|
|
2442
|
+
scenario: profileScenario,
|
|
2443
|
+
runId,
|
|
2444
|
+
metrics,
|
|
2445
|
+
diagnostics: manifestArtifacts.diagnostics,
|
|
2446
|
+
profileEventCount: events.length,
|
|
2447
|
+
profileSessionEntryCount: sessionEntries.length,
|
|
2448
|
+
commandTransport,
|
|
2449
|
+
sessionEntries,
|
|
2450
|
+
});
|
|
1469
2451
|
const verdict = buildProfileVerdict({ scenario: profileScenario, runId, health, metrics });
|
|
1470
2452
|
const agentSummary = buildAgentSummaryMarkdown({ health, verdict, manifest });
|
|
1471
2453
|
const summary = buildSummaryMarkdown({ manifest, metrics });
|