agent-scenario-loop 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/app/profile-session.ts +263 -17
  2. package/dist/core/artifact-contract.d.ts +6 -4
  3. package/dist/core/artifact-contract.js +164 -15
  4. package/dist/core/artifact-layout.d.ts +2 -0
  5. package/dist/core/artifact-layout.js +2 -0
  6. package/dist/core/planner.js +4 -3
  7. package/dist/core/schema-validator.d.ts +1 -0
  8. package/dist/core/schema-validator.js +1 -0
  9. package/dist/runner/android-adb-driver.d.ts +7 -2
  10. package/dist/runner/android-adb-driver.js +7 -1
  11. package/dist/runner/android-adb.d.ts +40 -5
  12. package/dist/runner/android-adb.js +1046 -664
  13. package/dist/runner/ios-simctl.d.ts +1 -0
  14. package/dist/runner/ios-simctl.js +1 -0
  15. package/dist/runner/profile-android.d.ts +11 -1
  16. package/dist/runner/profile-android.js +266 -25
  17. package/dist/runner/profile-ios.d.ts +3 -2
  18. package/dist/runner/profile-ios.js +252 -22
  19. package/dist/runner/profile-mobile.d.ts +63 -4
  20. package/dist/runner/profile-mobile.js +1002 -20
  21. package/dist/runner/validate-project.js +3 -0
  22. package/dist/scripts/consumer-rehearsal.d.ts +127 -0
  23. package/dist/scripts/consumer-rehearsal.js +774 -0
  24. package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
  25. package/dist/scripts/downstream-local-package-gate.js +264 -0
  26. package/dist/scripts/package-smoke.d.ts +104 -0
  27. package/dist/scripts/package-smoke.js +2304 -0
  28. package/dist/scripts/release-check.d.ts +47 -0
  29. package/dist/scripts/release-check.js +117 -0
  30. package/dist/scripts/release-readiness.d.ts +2 -0
  31. package/dist/scripts/release-readiness.js +539 -0
  32. package/docs/adapters.md +3 -1
  33. package/docs/api.md +2 -2
  34. package/docs/authoring.md +34 -2
  35. package/docs/consumer-rehearsal.md +33 -1
  36. package/docs/contracts.md +16 -2
  37. package/docs/live-proofs.md +12 -4
  38. package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
  39. package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
  40. package/examples/runners/README.md +3 -3
  41. package/examples/runners/axe-accessibility-provider.json +2 -2
  42. package/examples/runners/script-accessibility-provider.json +2 -2
  43. package/examples/runners/script-memory-provider.json +2 -2
  44. package/examples/runners/script-network-provider.json +2 -2
  45. package/examples/runners/script-profiler-provider.json +2 -2
  46. package/package.json +12 -4
  47. package/schemas/manifest.schema.json +73 -3
  48. package/schemas/profiler.schema.json +243 -0
  49. package/schemas/runner-capabilities.schema.json +8 -2
  50. package/schemas/scenario.schema.json +18 -2
  51. package/templates/evidence-provider.json +3 -3
  52. package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
@@ -14,6 +14,8 @@ exports.resolveAttachedEvidence = resolveAttachedEvidence;
14
14
  exports.resolveComparisonLane = resolveComparisonLane;
15
15
  exports.resolveEventLogPath = resolveEventLogPath;
16
16
  exports.resolveInteractionDriver = resolveInteractionDriver;
17
+ exports.resolveProfileScenarioName = resolveProfileScenarioName;
18
+ exports.runProfileCompatibilityPreflight = runProfileCompatibilityPreflight;
17
19
  exports.runProfileCli = runProfileCli;
18
20
  exports.runProfileMobile = runProfileMobile;
19
21
  exports.hashScenarioContract = hashScenarioContract;
@@ -26,6 +28,7 @@ const crypto = require('node:crypto');
26
28
  const { buildAgentSummaryMarkdown } = require('../core/agent-summary');
27
29
  const { createArtifactLayout } = require('../core/artifact-layout');
28
30
  const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
31
+ const { buildCompatibilityHealth, buildUnevaluatedVerdict, evaluateRunnerCompatibility, } = require('../core/planner');
29
32
  const { buildBudgetVerdict, buildCausalRun, buildCausalTimeline, buildManifest, buildMetricsFromProfileEvents, buildSummaryMarkdown, extractProfileEvents, extractProfileSessionEntries, } = require('../core/artifact-contract');
30
33
  const { SCHEMAS, assertValidJson } = require('../core/schema-validator');
31
34
  const { writeUsage } = require('./cli');
@@ -49,7 +52,7 @@ function usage({ binaryName, output = process.stderr, platform, }) {
49
52
  ];
50
53
  if (platform === 'android') {
51
54
  lines.push('Use --adb-artifacts <dir> to read raw/adb-logcat.txt from a prior asl-android-adb capture.');
52
- lines.push('Use --adb-capture [--clear-logcat] [--launch] [--launch-wait-ms <ms>] [--wait-ms <ms>] to capture adb logcat before profiling.');
55
+ lines.push('Use --adb-capture [--clear-logcat] [--launch] [--launch-wait-ms <ms>] [--wait-ms <ms>] [--adb-command-timeout-ms <ms>] to capture adb logcat before profiling.');
53
56
  lines.push('Use --android-dev-client-url <url> [--android-dev-client-wait-ms <ms>] [--android-dev-client-ready-pattern <pattern>] with --adb-capture to open an Expo dev-client session before profile-session deep links.');
54
57
  lines.push('Use --android-profile-session-storage with --profile-session to seed startup control through Android AsyncStorage.');
55
58
  lines.push('Use --profile-session with --adb-capture to start the app profile session and execute scenario-declared Android commands.');
@@ -145,6 +148,15 @@ function readRepeatableArgValues(args, key) {
145
148
  return entry;
146
149
  });
147
150
  }
151
+ /**
152
+ * Reads whether a boolean-style CLI flag was supplied.
153
+ *
154
+ * @param {CliArgValue | undefined} value
155
+ * @returns {boolean}
156
+ */
157
+ function isEnabled(value) {
158
+ return value === true || value === 'true';
159
+ }
148
160
  /**
149
161
  * Parses a `kind:path` evidence attachment value.
150
162
  *
@@ -251,6 +263,8 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
251
263
  destinationPath: path.join(layout.signals[kind], fileName),
252
264
  kind,
253
265
  manifestPath: `signals/${kind}/${fileName}`,
266
+ providerId,
267
+ ...(typeof output.required === 'boolean' ? { required: output.required } : {}),
254
268
  sourcePath,
255
269
  };
256
270
  }
@@ -263,6 +277,8 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
263
277
  destinationPath: path.join(layout.captures, fileName),
264
278
  kind: output.kind,
265
279
  manifestPath: `captures/${fileName}`,
280
+ providerId,
281
+ ...(typeof output.required === 'boolean' ? { required: output.required } : {}),
266
282
  sourcePath,
267
283
  };
268
284
  }
@@ -274,9 +290,33 @@ function buildProviderEvidenceInput({ layout, output, providerId, sourcePath, })
274
290
  destinationPath: path.join(layout.raw, 'providers', providerId, fileName),
275
291
  kind: output.kind,
276
292
  manifestPath: `raw/providers/${providerId}/${fileName}`,
293
+ providerId,
294
+ ...(typeof output.required === 'boolean' ? { required: output.required } : {}),
277
295
  sourcePath,
278
296
  };
279
297
  }
298
+ /**
299
+ * Validates structured profiler evidence when a provider emits JSON.
300
+ *
301
+ * Native traces and flamegraph files may be attached as profiler evidence, but
302
+ * JSON profiler files must carry enough envelope metadata for agents to reason
303
+ * about source, target, and completeness.
304
+ *
305
+ * @param {{kind: EvidenceKind, sourcePath: string}} options
306
+ * @returns {void}
307
+ */
308
+ function validateStructuredProfilerEvidence({ kind, sourcePath, }) {
309
+ if (kind !== 'profiler' || path.extname(sourcePath).toLowerCase() !== '.json') {
310
+ return;
311
+ }
312
+ try {
313
+ assertValidJson(readJson(sourcePath), SCHEMAS.profiler, 'Profiler evidence artifact');
314
+ }
315
+ catch (error) {
316
+ const detail = error instanceof Error ? error.message : String(error);
317
+ throw new Error(`Profiler evidence artifact is invalid: ${sourcePath}. ${detail}`);
318
+ }
319
+ }
280
320
  /**
281
321
  * Fails when provider command ids would collide in raw command records.
282
322
  *
@@ -442,7 +482,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
442
482
  },
443
483
  };
444
484
  const destinationPaths = new Set();
445
- const addCopy = async ({ channel, destinationPath, kind, manifestPath, sourcePath, }) => {
485
+ const addCopy = async ({ channel, destinationPath, kind, manifestPath, required = false, sourcePath, }) => {
446
486
  const stat = await fsp.stat(sourcePath).catch(() => null);
447
487
  if (!stat?.isFile()) {
448
488
  throw new Error(`Evidence artifact does not exist or is not a file: ${sourcePath}`);
@@ -450,6 +490,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
450
490
  if (destinationPaths.has(destinationPath)) {
451
491
  throw new Error(`Duplicate evidence artifact destination: ${manifestPath}`);
452
492
  }
493
+ validateStructuredProfilerEvidence({ kind, sourcePath });
453
494
  destinationPaths.add(destinationPath);
454
495
  const attachment = {
455
496
  channel,
@@ -459,6 +500,7 @@ async function resolveAttachedEvidence({ args, layout, providerInputs = [], }) {
459
500
  kind,
460
501
  manifestPath,
461
502
  redactionStatus: 'not-redacted',
503
+ required,
462
504
  sha256: await hashFileSha256(sourcePath),
463
505
  sourceFileName: path.basename(sourcePath),
464
506
  sourcePath,
@@ -578,20 +620,568 @@ function toPortablePathReference(targetPath) {
578
620
  }
579
621
  return path.basename(targetPath);
580
622
  }
623
+ /**
624
+ * Resolves the evidence input mode before profile parsing starts.
625
+ *
626
+ * @param {{args: CliArgs, platform: ProfilePlatform}} options
627
+ * @returns {string}
628
+ */
629
+ function resolveProfileInputMode({ args, platform }) {
630
+ if (typeof args.events === 'string') {
631
+ return 'fixture-event-log';
632
+ }
633
+ if (platform === 'android') {
634
+ if (typeof args['adb-artifacts'] === 'string') {
635
+ return 'adb-sidecar';
636
+ }
637
+ if (isEnabled(args['adb-capture'])) {
638
+ return 'adb-live-capture';
639
+ }
640
+ }
641
+ if (typeof args['simctl-artifacts'] === 'string') {
642
+ return 'simctl-sidecar';
643
+ }
644
+ if (isEnabled(args['simctl-capture'])) {
645
+ return 'simctl-live-capture';
646
+ }
647
+ return 'no-profile-evidence';
648
+ }
649
+ /**
650
+ * Reads unique scenario step kinds for early operator visibility.
651
+ *
652
+ * @param {Record<string, unknown>} scenario
653
+ * @returns {string[]}
654
+ */
655
+ function readScenarioStepKinds(scenario) {
656
+ if (!Array.isArray(scenario.steps)) {
657
+ return [];
658
+ }
659
+ return Array.from(new Set(scenario.steps
660
+ .filter(isRecord)
661
+ .map((step) => step.kind)
662
+ .filter((kind) => typeof kind === 'string')))
663
+ .sort();
664
+ }
665
+ /**
666
+ * Reads wait milestone ids from scenario steps.
667
+ *
668
+ * @param {Record<string, unknown>} scenario
669
+ * @returns {string[]}
670
+ */
671
+ function readScenarioWaitMilestones(scenario) {
672
+ if (!Array.isArray(scenario.steps)) {
673
+ return [];
674
+ }
675
+ return Array.from(new Set(scenario.steps
676
+ .filter(isRecord)
677
+ .filter((step) => step.kind === 'waitForMilestone')
678
+ .map((step) => step.milestone)
679
+ .filter((milestone) => typeof milestone === 'string')))
680
+ .sort();
681
+ }
682
+ /**
683
+ * Builds the early run plan artifact before provider commands or event parsing.
684
+ *
685
+ * @param {{args: CliArgs, artifactRoot: string, comparisonLane?: string | undefined, expectedIterations: number, interactionDriver: string, layout: ReturnType<typeof createArtifactLayout>, milestoneEventsPerIteration: number, options: ProfileMobileOptions, profileScenario: Record<string, unknown>, runDir: string, runId: string, scenarioHash: string, scenarioPath: string}} options
686
+ * @returns {ProfileRunPlan}
687
+ */
688
+ function buildProfileRunPlan({ args, artifactRoot, comparisonLane, expectedIterations, interactionDriver, layout, milestoneEventsPerIteration, options, profileScenario, runDir, runId, scenarioHash, scenarioPath, }) {
689
+ return {
690
+ artifactVersion: '1.0.0',
691
+ runId,
692
+ scenarioId: resolveProfileScenarioName({ scenario: profileScenario, scenarioPath }),
693
+ scenarioHash,
694
+ platform: options.platform,
695
+ inputMode: resolveProfileInputMode({ args, platform: options.platform }),
696
+ artifactRoot,
697
+ runDir,
698
+ interactionDriver,
699
+ ...(comparisonLane ? { comparisonLane } : {}),
700
+ expectedIterations,
701
+ milestoneEventsPerIteration,
702
+ commandTransport: resolveCommandTransport({ args, interactionDriver, options }),
703
+ providers: readRepeatableArgValues(args, 'provider').map((providerPath) => ({
704
+ path: toPortablePathReference(path.resolve(providerPath)),
705
+ })),
706
+ requestedDiagnostics: {
707
+ required: Array.from(readScenarioStringSet(profileScenario, ['artifacts', 'required'])).sort(),
708
+ optional: Array.from(readScenarioStringSet(profileScenario, ['artifacts', 'optional'])).sort(),
709
+ },
710
+ scenarioShape: {
711
+ budgets: Array.isArray(profileScenario.budgets) ? profileScenario.budgets.length : 0,
712
+ steps: Array.isArray(profileScenario.steps) ? profileScenario.steps.length : 0,
713
+ stepKinds: readScenarioStepKinds(profileScenario),
714
+ waitForMilestones: readScenarioWaitMilestones(profileScenario),
715
+ },
716
+ evidenceSources: {
717
+ ...(typeof args.events === 'string' ? { events: toPortablePathReference(path.resolve(args.events)) } : {}),
718
+ ...(typeof args['profile-session-entries'] === 'string'
719
+ ? { profileSessionEntries: toPortablePathReference(path.resolve(args['profile-session-entries'])) }
720
+ : {}),
721
+ ...(typeof args['adb-artifacts'] === 'string'
722
+ ? { adbArtifacts: toPortablePathReference(path.resolve(args['adb-artifacts'])) }
723
+ : {}),
724
+ ...(typeof args['simctl-artifacts'] === 'string'
725
+ ? { simctlArtifacts: toPortablePathReference(path.resolve(args['simctl-artifacts'])) }
726
+ : {}),
727
+ adbCapture: isEnabled(args['adb-capture']),
728
+ simctlCapture: isEnabled(args['simctl-capture']),
729
+ signals: readRepeatableArgValues(args, 'signal').length,
730
+ captures: readRepeatableArgValues(args, 'capture').length,
731
+ },
732
+ };
733
+ }
734
+ /**
735
+ * Writes the early profile run plan artifact and a compact status heartbeat.
736
+ *
737
+ * @param {{layout: ReturnType<typeof createArtifactLayout>, plan: ProfileRunPlan}} options
738
+ * @returns {Promise<void>}
739
+ */
740
+ async function writeProfileRunPlan({ layout, plan, }) {
741
+ await fsp.writeFile(layout.runPlan, `${JSON.stringify(plan, null, 2)}\n`, 'utf8');
742
+ process.stderr.write(`profile run plan: ${plan.platform}/${plan.scenarioId} mode=${plan.inputMode} providers=${plan.providers.length} requiredDiagnostics=${plan.requestedDiagnostics.required.length} runPlan=${path.relative(process.cwd(), layout.runPlan)}\n`);
743
+ }
744
+ /**
745
+ * Returns a path reference from one run folder to an external sidecar.
746
+ *
747
+ * @param {{runDir: string, targetPath: string}} options
748
+ * @returns {string}
749
+ */
750
+ function toRunPathReference({ runDir, targetPath }) {
751
+ const relativePath = path.relative(runDir, targetPath);
752
+ return relativePath.length > 0 ? relativePath : path.basename(targetPath);
753
+ }
754
+ /**
755
+ * Returns a sidecar dependency path that stays readable in rehydrated artifacts.
756
+ *
757
+ * @param {{runDir: string, sidecarRoot: string, targetPath: string}} options
758
+ * @returns {SidecarEvidenceDependency}
759
+ */
760
+ function toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath, }) {
761
+ const sidecarRelativePath = path.relative(sidecarRoot, targetPath);
762
+ if (sidecarRelativePath.length > 0 &&
763
+ !sidecarRelativePath.startsWith('..') &&
764
+ !path.isAbsolute(sidecarRelativePath)) {
765
+ return {
766
+ kind: 'sidecar',
767
+ root: 'sidecar',
768
+ path: sidecarRelativePath,
769
+ };
770
+ }
771
+ return {
772
+ kind: 'sidecar',
773
+ path: toRunPathReference({ runDir, targetPath }),
774
+ };
775
+ }
776
+ /**
777
+ * Reads scenario string-list declarations into a set.
778
+ *
779
+ * @param {Record<string, unknown>} scenario
780
+ * @param {string[]} pathSegments
781
+ * @returns {Set<string>}
782
+ */
783
+ function readScenarioStringSet(scenario, pathSegments) {
784
+ const values = pathSegments.reduce((current, segment) => (current && typeof current === 'object' && !Array.isArray(current)
785
+ ? current[segment]
786
+ : undefined), scenario);
787
+ return new Set(Array.isArray(values)
788
+ ? values.filter((value) => typeof value === 'string')
789
+ : []);
790
+ }
791
+ /**
792
+ * Returns true when a scenario artifact declaration matches a diagnostic kind.
793
+ *
794
+ * @param {Set<string>} artifacts
795
+ * @param {string[]} aliases
796
+ * @returns {boolean}
797
+ */
798
+ function artifactSetHasAny(artifacts, aliases) {
799
+ return aliases.some((alias) => artifacts.has(alias));
800
+ }
801
+ /**
802
+ * Resolves common aliases used by scenario artifact contracts.
803
+ *
804
+ * @param {DiagnosticKind} kind
805
+ * @returns {string[]}
806
+ */
807
+ function diagnosticArtifactAliases(kind) {
808
+ const aliases = {
809
+ accessibility: ['accessibility'],
810
+ js: ['js', 'profileEvents', 'profileSession'],
811
+ logs: ['logs', 'deviceLog', 'interactionLog'],
812
+ memory: ['memory'],
813
+ network: ['network'],
814
+ profiler: ['profiler', 'profile'],
815
+ screenshot: ['screenshot', 'screenshots'],
816
+ uiTree: ['uiTree', 'ui-tree', 'accessibilityTree'],
817
+ video: ['video', 'recording'],
818
+ };
819
+ return aliases[kind];
820
+ }
821
+ /**
822
+ * Resolves common aliases used by runner capability declarations.
823
+ *
824
+ * @param {DiagnosticKind} kind
825
+ * @returns {string[]}
826
+ */
827
+ function diagnosticCapabilityAliases(kind) {
828
+ const aliases = {
829
+ accessibility: ['accessibility', 'accessibilityCapture'],
830
+ js: ['js', 'profileSession', 'profileEvents'],
831
+ logs: ['logCapture', 'logs', 'deviceLog'],
832
+ memory: ['memory', 'memoryCapture'],
833
+ network: ['network', 'networkCapture'],
834
+ profiler: ['profiler', 'profile', 'profilerCapture'],
835
+ screenshot: ['screenshot', 'screenshots'],
836
+ uiTree: ['uiTree', 'ui-tree', 'accessibilityTree'],
837
+ video: ['video', 'recording'],
838
+ };
839
+ return aliases[kind];
840
+ }
841
+ /**
842
+ * Returns requirement/request metadata for one diagnostic kind.
843
+ *
844
+ * @param {{kind: DiagnosticKind, optionalArtifacts: Set<string>, optionalCapabilities: Set<string>, requiredArtifacts: Set<string>, requiredCapabilities: Set<string>}} options
845
+ * @returns {{required: boolean, requested: boolean}}
846
+ */
847
+ function resolveDiagnosticRequest({ kind, optionalArtifacts, optionalCapabilities, requiredArtifacts, requiredCapabilities, }) {
848
+ const artifactAliases = diagnosticArtifactAliases(kind);
849
+ const capabilityAliases = diagnosticCapabilityAliases(kind);
850
+ const required = artifactSetHasAny(requiredArtifacts, artifactAliases) ||
851
+ artifactSetHasAny(requiredCapabilities, capabilityAliases);
852
+ return {
853
+ required,
854
+ requested: required ||
855
+ artifactSetHasAny(optionalArtifacts, artifactAliases) ||
856
+ artifactSetHasAny(optionalCapabilities, capabilityAliases),
857
+ };
858
+ }
859
+ /**
860
+ * Builds a status entry for one diagnostic surface.
861
+ *
862
+ * @param {DiagnosticInventoryEntry & {requested?: boolean}} entry
863
+ * @returns {DiagnosticInventoryEntry}
864
+ */
865
+ function buildDiagnosticEntry(entry) {
866
+ const { requested = true, ...diagnostic } = entry;
867
+ if (diagnostic.status === 'captured' || requested || diagnostic.required) {
868
+ return diagnostic;
869
+ }
870
+ return {
871
+ ...diagnostic,
872
+ status: 'not_requested',
873
+ reason: diagnostic.reason ?? 'Scenario did not request this optional diagnostic surface.',
874
+ };
875
+ }
876
+ /**
877
+ * Builds the product-neutral diagnostic inventory for a profile run.
878
+ *
879
+ * @param {{args: CliArgs, attachedEvidence: AttachedEvidence, eventLogPath: string | null, platform: ProfilePlatform, profileSessionEntriesPath: string | null, runDir: string, scenario: Record<string, unknown>}} options
880
+ * @returns {DiagnosticInventoryEntry[]}
881
+ */
882
+ function buildDiagnosticInventory({ args, attachedEvidence, eventLogPath, platform, profileSessionEntriesPath, runDir, scenario, }) {
883
+ const requiredArtifacts = readScenarioStringSet(scenario, ['artifacts', 'required']);
884
+ const optionalArtifacts = readScenarioStringSet(scenario, ['artifacts', 'optional']);
885
+ const requiredCapabilities = readScenarioStringSet(scenario, ['requiredCapabilities']);
886
+ const optionalCapabilities = readScenarioStringSet(scenario, ['optionalCapabilities']);
887
+ const requiredProviderDiagnostics = new Set(attachedEvidence.attachments
888
+ .filter((attachment) => attachment.required)
889
+ .map((attachment) => attachment.kind));
890
+ const sidecarRoot = typeof args['adb-artifacts'] === 'string'
891
+ ? path.resolve(args['adb-artifacts'])
892
+ : typeof args['simctl-artifacts'] === 'string'
893
+ ? path.resolve(args['simctl-artifacts'])
894
+ : null;
895
+ const sidecarRootRef = sidecarRoot ? toRunPathReference({ runDir, targetPath: sidecarRoot }) : undefined;
896
+ const adbScreenshotDependency = platform === 'android'
897
+ ? resolveAndroidAdbScreenshotDependency({ runDir, sidecarRoot })
898
+ : null;
899
+ const eventLogBaseName = eventLogPath ? path.basename(eventLogPath) : undefined;
900
+ const eventLogManifestPath = eventLogBaseName ? `raw/${eventLogBaseName}` : undefined;
901
+ const eventLogIsIosProfileEvents = platform === 'ios' && eventLogBaseName === 'ios-profile-events.log';
902
+ const simctlRuntimeLogPath = typeof args['simctl-artifacts'] === 'string'
903
+ ? path.resolve(args['simctl-artifacts'], 'raw', 'ios-simctl-log.txt')
904
+ : null;
905
+ const simctlRuntimeLogExists = Boolean(simctlRuntimeLogPath && fs.existsSync(simctlRuntimeLogPath));
906
+ const simctlRuntimeLogDependency = simctlRuntimeLogPath && simctlRuntimeLogExists
907
+ ? toSidecarEvidenceDependency({ runDir, sidecarRoot: path.resolve(args['simctl-artifacts']), targetPath: simctlRuntimeLogPath })
908
+ : undefined;
909
+ const copiedSimctlLogManifestPath = platform === 'ios' && eventLogPath && path.basename(eventLogPath) === 'ios-simctl-log.txt'
910
+ ? eventLogManifestPath
911
+ : undefined;
912
+ const explicitIosRuntimeLogManifestPath = platform === 'ios' &&
913
+ typeof args.events === 'string' &&
914
+ eventLogManifestPath &&
915
+ !eventLogIsIosProfileEvents
916
+ ? eventLogManifestPath
917
+ : undefined;
918
+ const eventLogDependency = eventLogPath && sidecarRoot
919
+ ? toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath: eventLogPath })
920
+ : undefined;
921
+ const jsProfilePath = attachedEvidence.signals.js[0] ?? eventLogManifestPath;
922
+ const profileSessionEntriesManifestPath = profileSessionEntriesPath
923
+ ? `raw/${path.basename(profileSessionEntriesPath)}`
924
+ : undefined;
925
+ const entries = [];
926
+ const pushDiagnostic = (kind, entry) => {
927
+ const request = resolveDiagnosticRequest({
928
+ kind,
929
+ optionalArtifacts,
930
+ optionalCapabilities,
931
+ requiredArtifacts,
932
+ requiredCapabilities,
933
+ });
934
+ entries.push(buildDiagnosticEntry({
935
+ kind,
936
+ ...entry,
937
+ required: request.required || requiredProviderDiagnostics.has(kind) || Boolean(entry.required),
938
+ requested: request.requested || requiredProviderDiagnostics.has(kind) || Boolean(entry.requested),
939
+ }));
940
+ };
941
+ const logCaptured = platform === 'ios'
942
+ ? Boolean(copiedSimctlLogManifestPath || simctlRuntimeLogDependency || explicitIosRuntimeLogManifestPath)
943
+ : Boolean(eventLogManifestPath);
944
+ pushDiagnostic('logs', {
945
+ name: platform === 'ios' ? 'simulator-runtime-log' : 'device-log',
946
+ ...(typeof args['adb-artifacts'] === 'string'
947
+ ? { provider: 'adb', runnerId: 'android-adb' }
948
+ : typeof args['simctl-artifacts'] === 'string'
949
+ ? { provider: 'simctl', runnerId: 'ios-simctl' }
950
+ : typeof args.events === 'string'
951
+ ? { provider: 'fixture-log-ingest' }
952
+ : {}),
953
+ status: logCaptured ? 'captured' : 'unavailable',
954
+ ...(platform === 'ios'
955
+ ? copiedSimctlLogManifestPath
956
+ ? { path: copiedSimctlLogManifestPath }
957
+ : simctlRuntimeLogDependency
958
+ ? { path: simctlRuntimeLogDependency.path }
959
+ : explicitIosRuntimeLogManifestPath
960
+ ? { path: explicitIosRuntimeLogManifestPath }
961
+ : {}
962
+ : eventLogManifestPath
963
+ ? { path: eventLogManifestPath }
964
+ : {}),
965
+ ...(sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
966
+ ...(platform === 'ios'
967
+ ? simctlRuntimeLogDependency
968
+ ? { evidenceDependency: simctlRuntimeLogDependency }
969
+ : {}
970
+ : eventLogDependency
971
+ ? { evidenceDependency: eventLogDependency }
972
+ : {}),
973
+ ...(logCaptured
974
+ ? {
975
+ reason: platform === 'ios'
976
+ ? 'iOS simulator runtime log evidence was available from the simctl capture sidecar.'
977
+ : 'Device or fixture log evidence was available to the profile runner.',
978
+ }
979
+ : {
980
+ reason: platform === 'ios'
981
+ ? 'No iOS simulator runtime log was available in the selected simctl capture sidecar.'
982
+ : 'No device log source was supplied to this profile run.',
983
+ nextAction: platform === 'ios'
984
+ ? 'Run with --simctl-capture or provide --simctl-artifacts containing raw/ios-simctl-log.txt.'
985
+ : 'Run with --events, --adb-artifacts, --adb-capture, or provide a runtime log artifact.',
986
+ }),
987
+ });
988
+ pushDiagnostic('js', {
989
+ name: 'profile-session-evidence',
990
+ status: eventLogManifestPath || attachedEvidence.signals.js.length > 0 ? 'captured' : 'unavailable',
991
+ ...(jsProfilePath ? { path: jsProfilePath } : {}),
992
+ ...(profileSessionEntriesManifestPath
993
+ ? {
994
+ evidenceDependency: {
995
+ kind: 'profile-session-entries',
996
+ path: profileSessionEntriesManifestPath,
997
+ },
998
+ }
999
+ : eventLogDependency
1000
+ ? { evidenceDependency: eventLogDependency }
1001
+ : {}),
1002
+ ...(sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
1003
+ ...(eventLogManifestPath || attachedEvidence.signals.js.length > 0
1004
+ ? { reason: 'Profile or JS evidence was captured from runner input.' }
1005
+ : {
1006
+ reason: 'No profile-session event log or JS signal attachment was available.',
1007
+ nextAction: 'Attach JS evidence with --signal js:<path> or run a profile-session capture that emits profile events.',
1008
+ }),
1009
+ });
1010
+ const attachedScreenshotPath = attachedEvidence.captures.screenshots[0];
1011
+ const sidecarScreenshotDependency = attachedScreenshotPath ? null : adbScreenshotDependency;
1012
+ pushDiagnostic('screenshot', {
1013
+ ...(sidecarScreenshotDependency ? { provider: 'adb', runnerId: 'android-adb' } : {}),
1014
+ status: attachedScreenshotPath || sidecarScreenshotDependency ? 'captured' : 'unavailable',
1015
+ ...(attachedScreenshotPath
1016
+ ? { path: attachedScreenshotPath }
1017
+ : sidecarScreenshotDependency
1018
+ ? { path: sidecarScreenshotDependency.path }
1019
+ : {}),
1020
+ ...(sidecarScreenshotDependency && sidecarRootRef ? { sidecarRoot: sidecarRootRef } : {}),
1021
+ ...(sidecarScreenshotDependency ? { evidenceDependency: sidecarScreenshotDependency.dependency } : {}),
1022
+ ...(attachedScreenshotPath || sidecarScreenshotDependency
1023
+ ? {
1024
+ reason: sidecarScreenshotDependency
1025
+ ? 'Screenshot evidence was available from the adb capture sidecar.'
1026
+ : 'Screenshot capture was attached to the run.',
1027
+ }
1028
+ : {
1029
+ reason: 'No screenshot capture was produced by the selected runner/provider set.',
1030
+ nextAction: 'Use --capture screenshot:<path> or a runner/provider that produces screenshots.',
1031
+ }),
1032
+ });
1033
+ pushDiagnostic('uiTree', {
1034
+ status: attachedEvidence.captures.uiTree ? 'captured' : 'unavailable',
1035
+ ...(attachedEvidence.captures.uiTree ? { path: attachedEvidence.captures.uiTree } : {}),
1036
+ ...(attachedEvidence.captures.uiTree
1037
+ ? { reason: 'UI tree capture was attached to the run.' }
1038
+ : {
1039
+ reason: 'No UI tree capture was produced by the selected runner/provider set.',
1040
+ nextAction: 'Use --capture uiTree:<path> or add an accessibility/UI-tree provider.',
1041
+ }),
1042
+ });
1043
+ pushDiagnostic('video', {
1044
+ status: attachedEvidence.captures.video ? 'captured' : 'unavailable',
1045
+ ...(attachedEvidence.captures.video ? { path: attachedEvidence.captures.video } : {}),
1046
+ ...(attachedEvidence.captures.video
1047
+ ? { reason: 'Video capture was attached to the run.' }
1048
+ : {
1049
+ reason: 'No video capture was produced by the selected runner/provider set.',
1050
+ nextAction: 'Use --capture video:<path> or run a capture provider that records video.',
1051
+ }),
1052
+ });
1053
+ for (const kind of ['memory', 'network']) {
1054
+ pushDiagnostic(kind, {
1055
+ status: attachedEvidence.signals[kind].length > 0 ? 'captured' : 'unavailable',
1056
+ ...(attachedEvidence.signals[kind][0] ? { path: attachedEvidence.signals[kind][0] } : {}),
1057
+ ...(attachedEvidence.signals[kind].length > 0
1058
+ ? { reason: `${kind} signal evidence was attached to the run.` }
1059
+ : {
1060
+ reason: `No ${kind} signal evidence was produced by the selected provider set.`,
1061
+ nextAction: `Attach ${kind} evidence with --signal ${kind}:<path> or add a provider command that emits it.`,
1062
+ }),
1063
+ });
1064
+ }
1065
+ for (const kind of ['accessibility', 'profiler']) {
1066
+ const attachment = attachedEvidence.attachments.find((item) => item.kind === kind);
1067
+ pushDiagnostic(kind, {
1068
+ ...(attachment?.channel === 'provider' ? { provider: 'evidence-provider' } : {}),
1069
+ status: attachment ? 'captured' : 'unavailable',
1070
+ ...(attachment ? { path: attachment.manifestPath } : {}),
1071
+ ...(attachment
1072
+ ? { reason: `${kind} provider evidence was attached to the run.` }
1073
+ : {
1074
+ reason: `No ${kind} provider attachment was produced by the selected provider set.`,
1075
+ nextAction: `Declare a provider command or attach ${kind} evidence before expecting this diagnostic.`,
1076
+ }),
1077
+ });
1078
+ }
1079
+ return entries.map((entry) => {
1080
+ const cleaned = Object.entries(entry).filter(([, value]) => value !== undefined);
1081
+ return Object.fromEntries(cleaned);
1082
+ });
1083
+ }
1084
+ /**
1085
+ * Converts uncaptured required diagnostics into health checks.
1086
+ *
1087
+ * @param {DiagnosticInventoryEntry[]} diagnostics
1088
+ * @returns {Record<string, unknown>[]}
1089
+ */
1090
+ function buildRequiredDiagnosticHealthChecks(diagnostics = []) {
1091
+ return diagnostics
1092
+ .filter((diagnostic) => diagnostic.required && diagnostic.status !== 'captured')
1093
+ .map((diagnostic) => ({
1094
+ name: `required_${diagnostic.kind}_diagnostic`,
1095
+ status: 'failed',
1096
+ source: 'evidence',
1097
+ code: 'required_diagnostic_not_captured',
1098
+ message: diagnostic.reason ?? `Required ${diagnostic.kind} diagnostic was not captured.`,
1099
+ metadata: {
1100
+ kind: diagnostic.kind,
1101
+ status: diagnostic.status,
1102
+ ...(diagnostic.name ? { name: diagnostic.name } : {}),
1103
+ ...(diagnostic.nextAction ? { nextAction: diagnostic.nextAction } : {}),
1104
+ ...(diagnostic.provider ? { provider: diagnostic.provider } : {}),
1105
+ ...(diagnostic.runnerId ? { runnerId: diagnostic.runnerId } : {}),
1106
+ },
1107
+ }));
1108
+ }
581
1109
  /**
582
1110
  * Builds scenario health from profile metrics.
583
1111
  *
584
- * @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>}} options
1112
+ * @param {{scenario: Record<string, unknown>, runId: string, metrics: Record<string, unknown>, diagnostics?: DiagnosticInventoryEntry[], profileEventCount?: number, profileSessionEntryCount?: number, commandTransport?: string, sessionEntries?: Record<string, unknown>[]}} options
585
1113
  * @returns {Record<string, unknown>}
586
1114
  */
587
- function buildProfileHealth({ scenario, runId, metrics, }) {
1115
+ function buildProfileHealth({ scenario, runId, metrics, diagnostics = [], profileEventCount, profileSessionEntryCount, commandTransport, sessionEntries = [], }) {
588
1116
  const passed = metrics.status === 'passed';
1117
+ const metadata = {
1118
+ failures: typeof metrics.failures === 'number' ? metrics.failures : null,
1119
+ timeouts: typeof metrics.timeouts === 'number' ? metrics.timeouts : null,
1120
+ };
1121
+ if (typeof profileEventCount === 'number') {
1122
+ metadata.profileEventCount = profileEventCount;
1123
+ }
1124
+ if (typeof profileSessionEntryCount === 'number') {
1125
+ metadata.profileSessionEntryCount = profileSessionEntryCount;
1126
+ }
1127
+ if (typeof commandTransport === 'string' && commandTransport.length > 0) {
1128
+ metadata.commandTransport = commandTransport;
1129
+ }
1130
+ if (!passed &&
1131
+ profileEventCount === 0 &&
1132
+ profileSessionEntryCount === 0 &&
1133
+ typeof commandTransport === 'string' &&
1134
+ commandTransport.startsWith('profile-session')) {
1135
+ metadata.nextActionCode = 'verify_profile_session_bootstrap';
1136
+ metadata.nextAction =
1137
+ 'Verify the app loaded the expected bundle, mounted the profile-session bootstrap near the app root, and uses the configured storage keys or deep-link scheme before treating this as a product failure.';
1138
+ }
1139
+ const skippedCommands = sessionEntries.filter((entry) => (entry?.kind === 'command' && entry.status === 'skipped'));
1140
+ const firstSkippedCommand = skippedCommands[0];
1141
+ const firstSkippedReason = typeof firstSkippedCommand?.reason === 'string'
1142
+ ? firstSkippedCommand.reason
1143
+ : undefined;
1144
+ const commandFailureCode = firstSkippedReason === 'wait-for-milestone-timeout'
1145
+ ? 'profile_command_gate_timeout'
1146
+ : 'profile_command_skipped';
1147
+ const commandFailureMessage = firstSkippedReason === 'wait-for-milestone-timeout'
1148
+ ? 'One or more profile-session commands waited for a milestone that was not observed before timeout.'
1149
+ : 'One or more profile-session commands were skipped before the scenario completed.';
1150
+ const commandChecks = skippedCommands.length > 0
1151
+ ? [
1152
+ {
1153
+ name: 'profile_command_sequence',
1154
+ status: 'failed',
1155
+ source: 'runner',
1156
+ code: commandFailureCode,
1157
+ message: commandFailureMessage,
1158
+ metadata: {
1159
+ skippedCommandCount: skippedCommands.length,
1160
+ ...(typeof firstSkippedCommand?.command === 'string' ? { command: firstSkippedCommand.command } : {}),
1161
+ ...(typeof firstSkippedCommand?.commandId === 'string' ? { commandId: firstSkippedCommand.commandId } : {}),
1162
+ ...(typeof firstSkippedCommand?.queueId === 'string' ? { queueId: firstSkippedCommand.queueId } : {}),
1163
+ ...(typeof firstSkippedCommand?.reason === 'string' ? { reason: firstSkippedCommand.reason } : {}),
1164
+ ...(typeof firstSkippedCommand?.sequence === 'number' ? { sequence: firstSkippedCommand.sequence } : {}),
1165
+ ...(typeof firstSkippedCommand?.waitForMilestone === 'string'
1166
+ ? { waitForMilestone: firstSkippedCommand.waitForMilestone }
1167
+ : {}),
1168
+ ...(typeof firstSkippedCommand?.waitTimeoutMs === 'number'
1169
+ ? { waitTimeoutMs: firstSkippedCommand.waitTimeoutMs }
1170
+ : {}),
1171
+ },
1172
+ },
1173
+ ]
1174
+ : [];
1175
+ const commandChecksPassed = commandChecks.every((check) => check.status === 'passed');
1176
+ const diagnosticChecks = buildRequiredDiagnosticHealthChecks(diagnostics);
1177
+ const diagnosticChecksPassed = diagnosticChecks.every((check) => check.status === 'passed');
1178
+ const healthPassed = passed && commandChecksPassed && diagnosticChecksPassed;
589
1179
  return assertValidJson({
590
1180
  schemaVersion: '1.0.0',
591
1181
  scenarioId: scenario.name,
592
1182
  ...(typeof scenario.flowId === 'string' ? { flowId: scenario.flowId } : {}),
593
1183
  runId,
594
- healthStatus: passed ? 'passed' : 'failed',
1184
+ healthStatus: healthPassed ? 'passed' : 'failed',
595
1185
  checks: [
596
1186
  {
597
1187
  name: 'truth_events_complete',
@@ -601,11 +1191,10 @@ function buildProfileHealth({ scenario, runId, metrics, }) {
601
1191
  message: passed
602
1192
  ? 'Profile events completed every expected iteration.'
603
1193
  : 'Profile events did not complete every expected iteration.',
604
- metadata: {
605
- failures: typeof metrics.failures === 'number' ? metrics.failures : null,
606
- timeouts: typeof metrics.timeouts === 'number' ? metrics.timeouts : null,
607
- },
1194
+ metadata,
608
1195
  },
1196
+ ...commandChecks,
1197
+ ...diagnosticChecks,
609
1198
  ],
610
1199
  }, SCHEMAS.health, 'Health artifact');
611
1200
  }
@@ -857,6 +1446,121 @@ function resolveProfileSessionEntriesPath({ args, platform }) {
857
1446
  }
858
1447
  return null;
859
1448
  }
1449
+ /**
1450
+ * Resolves the run id used by rehydrated sidecar evidence.
1451
+ *
1452
+ * A rehydrated artifact can intentionally have a new run id while ingesting a
1453
+ * previously captured adb/simctl sidecar. Keep live runs strict, but allow an
1454
+ * explicit sidecar with exactly one source run id for the scenario to provide
1455
+ * the event filter.
1456
+ *
1457
+ * @param {{args: CliArgs, eventLogText: string, profileSessionEntriesPath: string | null, runId: string, scenarioName: string}} options
1458
+ * @returns {string}
1459
+ */
1460
+ function resolveEvidenceFilterRunId({ args, eventLogText, profileSessionEntriesPath, runId, scenarioName, }) {
1461
+ const isRehydratedSidecar = typeof args['adb-artifacts'] === 'string' || typeof args['simctl-artifacts'] === 'string';
1462
+ if (!isRehydratedSidecar) {
1463
+ return runId;
1464
+ }
1465
+ const scenarioEvents = extractProfileEvents(eventLogText, { scenario: scenarioName });
1466
+ const currentRunEvents = scenarioEvents.filter((event) => event.runId === runId);
1467
+ if (currentRunEvents.length > 0) {
1468
+ return runId;
1469
+ }
1470
+ const sourceRunIds = new Set(scenarioEvents
1471
+ .map((event) => event.runId)
1472
+ .filter((sourceRunId) => typeof sourceRunId === 'string' && sourceRunId.length > 0));
1473
+ if (profileSessionEntriesPath && fs.existsSync(profileSessionEntriesPath)) {
1474
+ const storedEntries = JSON.parse(fs.readFileSync(profileSessionEntriesPath, 'utf8'));
1475
+ if (Array.isArray(storedEntries)) {
1476
+ for (const entry of storedEntries) {
1477
+ if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
1478
+ continue;
1479
+ }
1480
+ const record = entry;
1481
+ if (record.scenario !== scenarioName) {
1482
+ continue;
1483
+ }
1484
+ if (typeof record.runId === 'string' && record.runId.length > 0) {
1485
+ sourceRunIds.add(record.runId);
1486
+ }
1487
+ }
1488
+ }
1489
+ }
1490
+ return sourceRunIds.size === 1 ? [...sourceRunIds][0] : runId;
1491
+ }
1492
+ /**
1493
+ * Returns the first usable adb screenshot file from sidecar metadata.
1494
+ *
1495
+ * ADB can produce a valid PNG even when command metadata records a nonzero
1496
+ * exit status from the host process. Treat the binary artifact as the capture
1497
+ * authority, but only after validating the PNG signature and sidecar boundary.
1498
+ *
1499
+ * @param {{runDir: string, sidecarRoot: string | null}} options
1500
+ * @returns {{dependency: SidecarEvidenceDependency, path: string} | null}
1501
+ */
1502
+ function resolveAndroidAdbScreenshotDependency({ runDir, sidecarRoot, }) {
1503
+ if (!sidecarRoot) {
1504
+ return null;
1505
+ }
1506
+ const metadata = readOptionalJsonObject(path.resolve(sidecarRoot, 'raw', 'android-metadata.json'));
1507
+ const actions = Array.isArray(metadata?.driverActions) ? metadata.driverActions : [];
1508
+ for (const action of actions) {
1509
+ if (!action || typeof action !== 'object' || Array.isArray(action)) {
1510
+ continue;
1511
+ }
1512
+ const record = action;
1513
+ if (record.driverAction !== 'screenshot') {
1514
+ continue;
1515
+ }
1516
+ const sidecarRelativePath = typeof record.capturePath === 'string'
1517
+ ? record.capturePath
1518
+ : typeof record.rawPath === 'string'
1519
+ ? record.rawPath
1520
+ : null;
1521
+ if (!sidecarRelativePath || path.isAbsolute(sidecarRelativePath)) {
1522
+ continue;
1523
+ }
1524
+ const screenshotPath = path.resolve(sidecarRoot, sidecarRelativePath);
1525
+ const relativeToSidecar = path.relative(sidecarRoot, screenshotPath);
1526
+ if (relativeToSidecar.length === 0 ||
1527
+ relativeToSidecar.startsWith('..') ||
1528
+ path.isAbsolute(relativeToSidecar) ||
1529
+ !isPngFile(screenshotPath)) {
1530
+ continue;
1531
+ }
1532
+ const sidecarDependency = toSidecarEvidenceDependency({ runDir, sidecarRoot, targetPath: screenshotPath });
1533
+ return {
1534
+ dependency: sidecarDependency,
1535
+ path: sidecarDependency.path,
1536
+ };
1537
+ }
1538
+ return null;
1539
+ }
1540
+ /**
1541
+ * Checks whether a file starts with the PNG signature.
1542
+ *
1543
+ * @param {string} filePath
1544
+ * @returns {boolean}
1545
+ */
1546
+ function isPngFile(filePath) {
1547
+ let signature;
1548
+ try {
1549
+ signature = fs.readFileSync(filePath, { flag: 'r' }).subarray(0, 8);
1550
+ }
1551
+ catch {
1552
+ return false;
1553
+ }
1554
+ return signature.length === 8 &&
1555
+ signature[0] === 0x89 &&
1556
+ signature[1] === 0x50 &&
1557
+ signature[2] === 0x4e &&
1558
+ signature[3] === 0x47 &&
1559
+ signature[4] === 0x0d &&
1560
+ signature[5] === 0x0a &&
1561
+ signature[6] === 0x1a &&
1562
+ signature[7] === 0x0a;
1563
+ }
860
1564
  /**
861
1565
  * Reads a JSON artifact if it exists and contains an object.
862
1566
  *
@@ -964,6 +1668,66 @@ function resolveProfileScenarioName({ scenario, scenarioPath, }) {
964
1668
  }
965
1669
  return path.basename(scenarioPath, '.json');
966
1670
  }
1671
+ /**
1672
+ * Reads evidence-provider manifests for profile compatibility preflight.
1673
+ *
1674
+ * @param {CliArgs} args
1675
+ * @returns {Record<string, unknown>[]}
1676
+ */
1677
+ function readEvidenceProviderManifests(args) {
1678
+ return readRepeatableArgValues(args, 'provider').map((providerPath, index) => assertValidJson(readJson(path.resolve(providerPath)), SCHEMAS.runnerCapabilities, `Evidence provider manifest ${index + 1}`));
1679
+ }
1680
+ /**
1681
+ * Runs planner compatibility before a live profile capture starts.
1682
+ *
1683
+ * Failed compatibility writes classified artifacts in the profile run folder so
1684
+ * agents can stop before adb, simctl, or provider work consumes runtime time.
1685
+ *
1686
+ * @param {CompatibilityPreflightOptions} options
1687
+ * @returns {Promise<void>}
1688
+ */
1689
+ async function runProfileCompatibilityPreflight({ args, artifactRoot, platform, primaryRunner, runDir, runId, scenario, scenarioName, }) {
1690
+ const layout = createArtifactLayout({ outputDir: runDir });
1691
+ const compatibility = evaluateRunnerCompatibility({
1692
+ scenario,
1693
+ runner: primaryRunner,
1694
+ evidenceProviders: readEvidenceProviderManifests(args),
1695
+ platform,
1696
+ });
1697
+ await writeJsonArtifact({
1698
+ filePath: layout.plannerCompatibility,
1699
+ value: compatibility,
1700
+ schema: {
1701
+ type: 'object',
1702
+ additionalProperties: true,
1703
+ },
1704
+ label: 'Planner compatibility artifact',
1705
+ });
1706
+ if (compatibility.compatible) {
1707
+ process.stderr.write(`profile preflight passed: ${platform}/${scenarioName} artifactRoot=${artifactRoot} planner=${path.relative(process.cwd(), layout.plannerCompatibility)}\n`);
1708
+ return;
1709
+ }
1710
+ const health = buildCompatibilityHealth({ scenario, runId, compatibility });
1711
+ const verdict = buildUnevaluatedVerdict({ scenario, runId, health });
1712
+ const agentSummary = buildAgentSummaryMarkdown({ health, verdict });
1713
+ await writeJsonArtifact({
1714
+ filePath: layout.health,
1715
+ value: health,
1716
+ schema: SCHEMAS.health,
1717
+ label: 'Health artifact',
1718
+ });
1719
+ await writeJsonArtifact({
1720
+ filePath: layout.verdict,
1721
+ value: verdict,
1722
+ schema: SCHEMAS.verdict,
1723
+ label: 'Verdict artifact',
1724
+ });
1725
+ await writeTextArtifact({
1726
+ filePath: layout.agentSummary,
1727
+ content: agentSummary,
1728
+ });
1729
+ throw new Error(`Profile compatibility preflight failed; inspect ${runDir}/agent-summary.md.`);
1730
+ }
967
1731
  /**
968
1732
  * Serializes JSON with stable object key ordering for reproducible hashes.
969
1733
  *
@@ -1045,6 +1809,46 @@ function findMilestoneEvent(scenario, milestoneId) {
1045
1809
  }
1046
1810
  return null;
1047
1811
  }
1812
+ /**
1813
+ * Returns true when a milestone is explicitly optional.
1814
+ *
1815
+ * @param {Record<string, unknown>} scenario
1816
+ * @param {unknown} milestoneId
1817
+ * @returns {boolean}
1818
+ */
1819
+ function isOptionalMilestone(scenario, milestoneId) {
1820
+ if (typeof milestoneId !== 'string' || !Array.isArray(scenario.milestones)) {
1821
+ return false;
1822
+ }
1823
+ for (const milestone of scenario.milestones) {
1824
+ if (!isRecord(milestone)) {
1825
+ continue;
1826
+ }
1827
+ if (milestone.id === milestoneId) {
1828
+ return milestone.required === false;
1829
+ }
1830
+ }
1831
+ return false;
1832
+ }
1833
+ /**
1834
+ * Returns true when a milestone represents one-time scenario readiness rather than a repeated cycle edge.
1835
+ *
1836
+ * @param {Record<string, unknown>} scenario
1837
+ * @param {unknown} milestoneId
1838
+ * @param {string | null} milestoneEvent
1839
+ * @returns {boolean}
1840
+ */
1841
+ function isReadinessMilestone(scenario, milestoneId, milestoneEvent) {
1842
+ const readyEvent = isRecord(scenario.truthEvents) && isRecord(scenario.truthEvents.ready)
1843
+ ? scenario.truthEvents.ready.event
1844
+ : undefined;
1845
+ if (typeof readyEvent === 'string' && milestoneEvent === readyEvent) {
1846
+ return true;
1847
+ }
1848
+ const id = typeof milestoneId === 'string' ? milestoneId.toLowerCase() : '';
1849
+ const event = typeof milestoneEvent === 'string' ? milestoneEvent.toLowerCase() : '';
1850
+ return id.includes('ready') || event.includes('ready');
1851
+ }
1048
1852
  /**
1049
1853
  * Builds a milestone-id to event-name lookup for schema-era scenarios.
1050
1854
  *
@@ -1100,6 +1904,16 @@ function resolveProfileMetricEvents(scenario) {
1100
1904
  milestone: toEvent,
1101
1905
  };
1102
1906
  }
1907
+ if (fromEvent && toEvent && isReadinessMilestone(scenario, budget.fromMilestone, fromEvent)) {
1908
+ return {
1909
+ milestone: toEvent,
1910
+ };
1911
+ }
1912
+ if (fromEvent && toEvent && isOptionalMilestone(scenario, budget.fromMilestone)) {
1913
+ return {
1914
+ milestone: toEvent,
1915
+ };
1916
+ }
1103
1917
  if (fromEvent && toEvent) {
1104
1918
  return {
1105
1919
  closeRequested: toEvent,
@@ -1112,7 +1926,59 @@ function resolveProfileMetricEvents(scenario) {
1112
1926
  return null;
1113
1927
  }
1114
1928
  /**
1115
- * Maps schema-era milestone budget fields to the legacy profile budget keys.
1929
+ * Resolves how many repeated completion milestone events prove one cycle body.
1930
+ *
1931
+ * @param {Record<string, unknown>} scenario
1932
+ * @param {Record<string, string> | null} metricEvents
1933
+ * @returns {number}
1934
+ */
1935
+ function resolveMilestoneEventsPerIteration(scenario, metricEvents) {
1936
+ if (!metricEvents || typeof metricEvents.milestone !== 'string' || !Array.isArray(scenario.steps)) {
1937
+ return 1;
1938
+ }
1939
+ const milestoneEvents = buildMilestoneEventLookup(scenario);
1940
+ const matchingWaitStepIds = new Set();
1941
+ for (const step of scenario.steps) {
1942
+ if (!isRecord(step) || step.kind !== 'waitForMilestone' || typeof step.milestone !== 'string') {
1943
+ continue;
1944
+ }
1945
+ const event = milestoneEvents[step.milestone] ?? step.milestone;
1946
+ if (event === metricEvents.milestone && typeof step.id === 'string') {
1947
+ matchingWaitStepIds.add(step.id);
1948
+ }
1949
+ }
1950
+ if (matchingWaitStepIds.size === 0) {
1951
+ return 1;
1952
+ }
1953
+ const bodyStepIds = isRecord(scenario.cycles) && Array.isArray(scenario.cycles.bodyStepIds)
1954
+ ? new Set(scenario.cycles.bodyStepIds.filter((entry) => typeof entry === 'string'))
1955
+ : null;
1956
+ if (bodyStepIds && bodyStepIds.size > 0) {
1957
+ let count = 0;
1958
+ let bodyCommandPending = false;
1959
+ for (const step of scenario.steps) {
1960
+ if (!isRecord(step)) {
1961
+ continue;
1962
+ }
1963
+ if (typeof step.id === 'string' && bodyStepIds.has(step.id) && step.kind === 'command') {
1964
+ bodyCommandPending = true;
1965
+ continue;
1966
+ }
1967
+ if (bodyCommandPending && typeof step.id === 'string' && matchingWaitStepIds.has(step.id)) {
1968
+ count += 1;
1969
+ bodyCommandPending = false;
1970
+ continue;
1971
+ }
1972
+ if (bodyCommandPending && step.kind === 'command') {
1973
+ bodyCommandPending = false;
1974
+ }
1975
+ }
1976
+ return count > 1 ? count : 1;
1977
+ }
1978
+ return matchingWaitStepIds.size > 1 ? matchingWaitStepIds.size : 1;
1979
+ }
1980
+ /**
1981
+ * Maps shorthand milestone budget fields to aggregate profile budget keys.
1116
1982
  *
1117
1983
  * @param {{budget: Record<string, unknown>, metric: string}} options
1118
1984
  * @returns {string | null}
@@ -1144,11 +2010,24 @@ function resolveProfileBudgets(scenario) {
1144
2010
  return null;
1145
2011
  }
1146
2012
  const pass = {};
2013
+ const intervals = [];
1147
2014
  for (const budget of scenario.budgets) {
1148
2015
  if (!isRecord(budget) || typeof budget.limit !== 'number') {
1149
2016
  continue;
1150
2017
  }
1151
2018
  if (budget.metric === 'p95' || budget.metric === 'p50') {
2019
+ const fromEvent = findMilestoneEvent(scenario, budget.fromMilestone);
2020
+ const toEvent = findMilestoneEvent(scenario, budget.toMilestone);
2021
+ if (fromEvent && toEvent) {
2022
+ intervals.push({
2023
+ name: typeof budget.name === 'string' ? budget.name : `${String(budget.fromMilestone)} to ${String(budget.toMilestone)}`,
2024
+ metric: budget.metric,
2025
+ limit: budget.limit,
2026
+ fromEvent,
2027
+ toEvent,
2028
+ });
2029
+ continue;
2030
+ }
1152
2031
  const budgetKey = resolveProfileBudgetKey({ budget, metric: budget.metric });
1153
2032
  if (budgetKey) {
1154
2033
  pass[budgetKey] = budget.limit;
@@ -1161,10 +2040,11 @@ function resolveProfileBudgets(scenario) {
1161
2040
  pass.timeouts = budget.limit;
1162
2041
  }
1163
2042
  }
1164
- return Object.keys(pass).length > 0
2043
+ return Object.keys(pass).length > 0 || intervals.length > 0
1165
2044
  ? {
1166
2045
  metric: 'milestone budget',
1167
2046
  pass,
2047
+ ...(intervals.length > 0 ? { intervals } : {}),
1168
2048
  }
1169
2049
  : null;
1170
2050
  }
@@ -1287,6 +2167,7 @@ async function runProfileMobile(args, options) {
1287
2167
  const scenarioHash = hashScenarioContract(profileScenario);
1288
2168
  const expectedIterations = resolveExpectedIterations(profileScenario);
1289
2169
  const profileMetricEvents = resolveProfileMetricEvents(profileScenario);
2170
+ const milestoneEventsPerIteration = resolveMilestoneEventsPerIteration(profileScenario, profileMetricEvents);
1290
2171
  const profileBudgets = resolveProfileBudgets(profileScenario);
1291
2172
  const runId = typeof args['run-id'] === 'string' ? args['run-id'] : createRunId();
1292
2173
  const artifactRoot = resolveArtifactRoot({ args, config, configPath, platform: options.platform });
@@ -1304,6 +2185,22 @@ async function runProfileMobile(args, options) {
1304
2185
  await ensureDir(layout.signals.js);
1305
2186
  await ensureDir(layout.signals.memory);
1306
2187
  await ensureDir(layout.signals.network);
2188
+ const runPlan = buildProfileRunPlan({
2189
+ args,
2190
+ artifactRoot,
2191
+ comparisonLane,
2192
+ expectedIterations,
2193
+ interactionDriver,
2194
+ layout,
2195
+ milestoneEventsPerIteration,
2196
+ options,
2197
+ profileScenario,
2198
+ runDir,
2199
+ runId,
2200
+ scenarioHash,
2201
+ scenarioPath,
2202
+ });
2203
+ await writeProfileRunPlan({ layout, plan: runPlan });
1307
2204
  const providerExecution = await executeProviderCommands({
1308
2205
  args,
1309
2206
  layout,
@@ -1342,15 +2239,69 @@ async function runProfileMobile(args, options) {
1342
2239
  verdict,
1343
2240
  };
1344
2241
  }
1345
- const attachedEvidence = await resolveAttachedEvidence({ args, layout, providerInputs: providerExecution.inputs });
2242
+ let attachedEvidence;
2243
+ try {
2244
+ attachedEvidence = await resolveAttachedEvidence({ args, layout, providerInputs: providerExecution.inputs });
2245
+ }
2246
+ catch (error) {
2247
+ const providerInput = providerExecution.inputs.find((input) => error instanceof Error && error.message.includes(input.sourcePath));
2248
+ const health = buildProviderCommandFailureHealth({
2249
+ failures: [
2250
+ {
2251
+ commandId: 'provider-evidence',
2252
+ code: 'provider_evidence_invalid',
2253
+ exitCode: null,
2254
+ message: error instanceof Error ? error.message : String(error),
2255
+ name: 'evidence_provider_output_valid',
2256
+ nextAction: 'Fix the provider output so it satisfies the ASL evidence contract, then rerun the profile.',
2257
+ nextActionCode: 'fix_provider_evidence_output',
2258
+ phase: 'afterCapture',
2259
+ providerId: providerInput?.providerId ?? 'unknown-provider',
2260
+ ...(providerInput?.manifestPath ? { rawPath: providerInput.manifestPath } : {}),
2261
+ },
2262
+ ],
2263
+ runId,
2264
+ scenario: profileScenario,
2265
+ });
2266
+ const verdict = buildProfileVerdict({ scenario: profileScenario, runId, health, metrics: {} });
2267
+ const agentSummary = buildAgentSummaryMarkdown({ health, verdict });
2268
+ await writeJsonArtifact({
2269
+ filePath: layout.health,
2270
+ value: health,
2271
+ schema: SCHEMAS.health,
2272
+ label: 'Health artifact',
2273
+ });
2274
+ await writeJsonArtifact({
2275
+ filePath: layout.verdict,
2276
+ value: verdict,
2277
+ schema: SCHEMAS.verdict,
2278
+ label: 'Verdict artifact',
2279
+ });
2280
+ await writeTextArtifact({
2281
+ filePath: layout.agentSummary,
2282
+ content: agentSummary,
2283
+ });
2284
+ return {
2285
+ runDir,
2286
+ health,
2287
+ verdict,
2288
+ };
2289
+ }
1346
2290
  const eventLogText = eventLogPath ? await fsp.readFile(eventLogPath, 'utf8') : '';
2291
+ const evidenceFilterRunId = resolveEvidenceFilterRunId({
2292
+ args,
2293
+ eventLogText,
2294
+ profileSessionEntriesPath,
2295
+ runId,
2296
+ scenarioName,
2297
+ });
1347
2298
  const events = extractProfileEvents(eventLogText, {
1348
2299
  scenario: scenarioName,
1349
- runId,
2300
+ runId: evidenceFilterRunId,
1350
2301
  });
1351
2302
  const logSessionEntries = extractProfileSessionEntries(eventLogText, {
1352
2303
  scenario: scenarioName,
1353
- runId,
2304
+ runId: evidenceFilterRunId,
1354
2305
  });
1355
2306
  const storedSessionEntries = profileSessionEntriesPath
1356
2307
  ? JSON.parse(await fsp.readFile(profileSessionEntriesPath, 'utf8'))
@@ -1364,7 +2315,7 @@ async function runProfileMobile(args, options) {
1364
2315
  }
1365
2316
  const record = entry;
1366
2317
  return ((!('scenario' in record) || record.scenario === scenarioName) &&
1367
- (!('runId' in record) || record.runId === runId));
2318
+ (!('runId' in record) || record.runId === evidenceFilterRunId));
1368
2319
  })
1369
2320
  : []),
1370
2321
  ];
@@ -1376,11 +2327,16 @@ async function runProfileMobile(args, options) {
1376
2327
  expectedIterations,
1377
2328
  budgets: profileBudgets,
1378
2329
  cycleEventNames: profileMetricEvents,
2330
+ milestoneEventsPerIteration,
1379
2331
  artifacts: {
1380
2332
  captures: attachedEvidence.captures,
1381
2333
  signals: attachedEvidence.signals,
1382
2334
  },
1383
2335
  });
2336
+ const eventLogRawPath = eventLogPath ? `raw/${path.basename(eventLogPath)}` : undefined;
2337
+ const eventLogIsProfileSessionEvidenceOnly = options.platform === 'ios' &&
2338
+ eventLogPath &&
2339
+ path.basename(eventLogPath) === 'ios-profile-events.log';
1384
2340
  const manifestArtifacts = {
1385
2341
  causalRun: 'causal-run.json',
1386
2342
  budgetVerdict: 'budget-verdict.json',
@@ -1389,13 +2345,20 @@ async function runProfileMobile(args, options) {
1389
2345
  summary: 'summary.md',
1390
2346
  scenario: toPortablePathReference(scenarioPath),
1391
2347
  raw: {
1392
- interactionLog: eventLogPath ? `raw/${path.basename(eventLogPath)}` : 'raw/interaction.log',
1393
- deviceLog: 'raw/device.log',
2348
+ ...(eventLogRawPath && !eventLogIsProfileSessionEvidenceOnly
2349
+ ? {
2350
+ interactionLog: eventLogRawPath,
2351
+ deviceLog: eventLogRawPath,
2352
+ }
2353
+ : {}),
2354
+ ...(profileSessionEntriesPath
2355
+ ? { profileSessionEntries: `raw/${path.basename(profileSessionEntriesPath)}` }
2356
+ : {}),
1394
2357
  },
1395
2358
  captures: {
1396
2359
  screenshots: attachedEvidence.captures.screenshots,
1397
- video: attachedEvidence.captures.video ?? 'captures/run.mp4',
1398
- uiTree: attachedEvidence.captures.uiTree ?? 'captures/ui-tree.json',
2360
+ ...(attachedEvidence.captures.video ? { video: attachedEvidence.captures.video } : {}),
2361
+ ...(attachedEvidence.captures.uiTree ? { uiTree: attachedEvidence.captures.uiTree } : {}),
1399
2362
  },
1400
2363
  signals: {
1401
2364
  js: attachedEvidence.signals.js,
@@ -1403,8 +2366,18 @@ async function runProfileMobile(args, options) {
1403
2366
  network: attachedEvidence.signals.network,
1404
2367
  },
1405
2368
  evidenceAttachments: buildEvidenceAttachmentManifest(attachedEvidence.attachments),
2369
+ diagnostics: buildDiagnosticInventory({
2370
+ args,
2371
+ attachedEvidence,
2372
+ eventLogPath,
2373
+ platform: options.platform,
2374
+ profileSessionEntriesPath,
2375
+ runDir,
2376
+ scenario: profileScenario,
2377
+ }),
1406
2378
  };
1407
2379
  const appId = resolveAppId({ config, platform: options.platform });
2380
+ const commandTransport = resolveCommandTransport({ args, interactionDriver, options });
1408
2381
  const provenanceCohort = buildProfileProvenanceCohort({
1409
2382
  appId,
1410
2383
  args,
@@ -1465,7 +2438,16 @@ async function runProfileMobile(args, options) {
1465
2438
  runId,
1466
2439
  budgetEvaluation: metrics.budgetEvaluation ?? null,
1467
2440
  });
1468
- const health = buildProfileHealth({ scenario: profileScenario, runId, metrics });
2441
+ const health = buildProfileHealth({
2442
+ scenario: profileScenario,
2443
+ runId,
2444
+ metrics,
2445
+ diagnostics: manifestArtifacts.diagnostics,
2446
+ profileEventCount: events.length,
2447
+ profileSessionEntryCount: sessionEntries.length,
2448
+ commandTransport,
2449
+ sessionEntries,
2450
+ });
1469
2451
  const verdict = buildProfileVerdict({ scenario: profileScenario, runId, health, metrics });
1470
2452
  const agentSummary = buildAgentSummaryMarkdown({ health, verdict, manifest });
1471
2453
  const summary = buildSummaryMarkdown({ manifest, metrics });