agent-scenario-loop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +119 -0
  3. package/app/profile-session.ts +812 -0
  4. package/core/config-template.json +41 -0
  5. package/dist/core/agent-summary.d.ts +15 -0
  6. package/dist/core/agent-summary.js +177 -0
  7. package/dist/core/artifact-contract.d.ts +151 -0
  8. package/dist/core/artifact-contract.js +897 -0
  9. package/dist/core/artifact-layout.d.ts +56 -0
  10. package/dist/core/artifact-layout.js +61 -0
  11. package/dist/core/artifact-writer.d.ts +44 -0
  12. package/dist/core/artifact-writer.js +55 -0
  13. package/dist/core/comparison.d.ts +133 -0
  14. package/dist/core/comparison.js +294 -0
  15. package/dist/core/evidence-interpreter.d.ts +28 -0
  16. package/dist/core/evidence-interpreter.js +69 -0
  17. package/dist/core/execution-plan.d.ts +44 -0
  18. package/dist/core/execution-plan.js +95 -0
  19. package/dist/core/planner.d.ts +132 -0
  20. package/dist/core/planner.js +812 -0
  21. package/dist/core/ports.d.ts +198 -0
  22. package/dist/core/ports.js +146 -0
  23. package/dist/core/run-index.d.ts +62 -0
  24. package/dist/core/run-index.js +143 -0
  25. package/dist/core/schema-validator.d.ts +86 -0
  26. package/dist/core/schema-validator.js +407 -0
  27. package/dist/index.d.ts +11 -0
  28. package/dist/index.js +27 -0
  29. package/dist/runner/agent-device-driver.d.ts +126 -0
  30. package/dist/runner/agent-device-driver.js +168 -0
  31. package/dist/runner/agent-device.d.ts +295 -0
  32. package/dist/runner/agent-device.js +1271 -0
  33. package/dist/runner/android-adb-driver.d.ts +175 -0
  34. package/dist/runner/android-adb-driver.js +399 -0
  35. package/dist/runner/android-adb.d.ts +254 -0
  36. package/dist/runner/android-adb.js +1618 -0
  37. package/dist/runner/argent-driver.d.ts +183 -0
  38. package/dist/runner/argent-driver.js +297 -0
  39. package/dist/runner/argent.d.ts +349 -0
  40. package/dist/runner/argent.js +1211 -0
  41. package/dist/runner/check-plan.d.ts +45 -0
  42. package/dist/runner/check-plan.js +210 -0
  43. package/dist/runner/cli.d.ts +20 -0
  44. package/dist/runner/cli.js +23 -0
  45. package/dist/runner/compare-latest.d.ts +99 -0
  46. package/dist/runner/compare-latest.js +233 -0
  47. package/dist/runner/compare.d.ts +58 -0
  48. package/dist/runner/compare.js +157 -0
  49. package/dist/runner/demo-loop.d.ts +45 -0
  50. package/dist/runner/demo-loop.js +170 -0
  51. package/dist/runner/example-android-live.d.ts +137 -0
  52. package/dist/runner/example-android-live.js +454 -0
  53. package/dist/runner/example-ios-live.d.ts +137 -0
  54. package/dist/runner/example-ios-live.js +471 -0
  55. package/dist/runner/host-doctor.d.ts +131 -0
  56. package/dist/runner/host-doctor.js +628 -0
  57. package/dist/runner/init-project.d.ts +88 -0
  58. package/dist/runner/init-project.js +263 -0
  59. package/dist/runner/ios-simctl-driver.d.ts +69 -0
  60. package/dist/runner/ios-simctl-driver.js +97 -0
  61. package/dist/runner/ios-simctl.d.ts +254 -0
  62. package/dist/runner/ios-simctl.js +1415 -0
  63. package/dist/runner/live-android.d.ts +137 -0
  64. package/dist/runner/live-android.js +539 -0
  65. package/dist/runner/live-comparison.d.ts +67 -0
  66. package/dist/runner/live-comparison.js +147 -0
  67. package/dist/runner/live-ios.d.ts +137 -0
  68. package/dist/runner/live-ios.js +460 -0
  69. package/dist/runner/live-proof-summary.d.ts +263 -0
  70. package/dist/runner/live-proof-summary.js +465 -0
  71. package/dist/runner/live-proof.d.ts +467 -0
  72. package/dist/runner/live-proof.js +920 -0
  73. package/dist/runner/local-env.d.ts +64 -0
  74. package/dist/runner/local-env.js +155 -0
  75. package/dist/runner/profile-android.d.ts +82 -0
  76. package/dist/runner/profile-android.js +671 -0
  77. package/dist/runner/profile-ios.d.ts +108 -0
  78. package/dist/runner/profile-ios.js +532 -0
  79. package/dist/runner/profile-mobile.d.ts +254 -0
  80. package/dist/runner/profile-mobile.js +1307 -0
  81. package/dist/runner/validate-project.d.ts +273 -0
  82. package/dist/runner/validate-project.js +1501 -0
  83. package/docs/adapters.md +145 -0
  84. package/docs/api.md +94 -0
  85. package/docs/authoring.md +196 -0
  86. package/docs/concepts.md +136 -0
  87. package/docs/consumer-rehearsal.md +115 -0
  88. package/docs/contracts.md +267 -0
  89. package/docs/live-proofs.md +270 -0
  90. package/docs/principles.md +46 -0
  91. package/examples/event-logs/app-startup-baseline.log +4 -0
  92. package/examples/event-logs/app-startup-current.log +4 -0
  93. package/examples/minimal-app/README.md +70 -0
  94. package/examples/mobile-app/README.md +302 -0
  95. package/examples/mobile-app/app.json +22 -0
  96. package/examples/mobile-app/asl/package-scripts.json +32 -0
  97. package/examples/mobile-app/asl.config.json +37 -0
  98. package/examples/mobile-app/event-logs/android-app-startup.log +4 -0
  99. package/examples/mobile-app/event-logs/android-open-close-cycle.log +12 -0
  100. package/examples/mobile-app/event-logs/android-scroll-settle.log +12 -0
  101. package/examples/mobile-app/event-logs/app-startup.log +4 -0
  102. package/examples/mobile-app/event-logs/open-close-cycle.log +12 -0
  103. package/examples/mobile-app/event-logs/scroll-settle.log +12 -0
  104. package/examples/mobile-app/index.ts +20 -0
  105. package/examples/mobile-app/metro.config.js +20 -0
  106. package/examples/mobile-app/package.json +62 -0
  107. package/examples/mobile-app/patches/expo-modules-jsi@56.0.10.patch +19 -0
  108. package/examples/mobile-app/plugins/with-ios-build-compat.js +271 -0
  109. package/examples/mobile-app/pnpm-lock.yaml +4440 -0
  110. package/examples/mobile-app/runner-manifests/evidence-provider.json +79 -0
  111. package/examples/mobile-app/runner-manifests/primary-runner.json +19 -0
  112. package/examples/mobile-app/scenarios/android/app-startup-video.json +73 -0
  113. package/examples/mobile-app/scenarios/android/app-startup.json +44 -0
  114. package/examples/mobile-app/scenarios/android/open-close-cycle.json +54 -0
  115. package/examples/mobile-app/scenarios/android/scroll-settle.json +49 -0
  116. package/examples/mobile-app/scenarios/ios/app-startup.json +44 -0
  117. package/examples/mobile-app/scenarios/ios/open-close-cycle.json +54 -0
  118. package/examples/mobile-app/scenarios/ios/scroll-settle.json +49 -0
  119. package/examples/mobile-app/scenarios/mobile/app-startup.json +91 -0
  120. package/examples/mobile-app/scenarios/mobile/open-close-cycle.json +160 -0
  121. package/examples/mobile-app/scenarios/mobile/scroll-settle.json +148 -0
  122. package/examples/mobile-app/scripts/asl-capture-accessibility-provider.mjs +112 -0
  123. package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +127 -0
  124. package/examples/mobile-app/src/devtools/profile-session.ts +7 -0
  125. package/examples/mobile-app/src/example-screen.tsx +322 -0
  126. package/examples/mobile-app/tsconfig.json +16 -0
  127. package/examples/mobile-app/tsconfig.typecheck.json +13 -0
  128. package/examples/runners/README.md +44 -0
  129. package/examples/runners/adb-android.json +25 -0
  130. package/examples/runners/agent-device-android.json +27 -0
  131. package/examples/runners/agent-device-ios.json +27 -0
  132. package/examples/runners/argent-android.json +32 -0
  133. package/examples/runners/argent-ios.json +32 -0
  134. package/examples/runners/argent-react-profiler-provider.json +15 -0
  135. package/examples/runners/axe-accessibility-provider.json +24 -0
  136. package/examples/runners/manual-log-ingest.json +9 -0
  137. package/examples/runners/rozenite-profiler-provider.json +9 -0
  138. package/examples/runners/script-accessibility-provider.json +24 -0
  139. package/examples/runners/script-memory-provider.json +24 -0
  140. package/examples/runners/script-network-provider.json +24 -0
  141. package/examples/runners/script-profiler-provider.json +30 -0
  142. package/examples/runners/xcodebuildmcp-ios.json +29 -0
  143. package/examples/scenarios/ios/app-startup.json +28 -0
  144. package/examples/scenarios/ios/open-close-cycle.json +35 -0
  145. package/examples/scenarios/mobile/app-startup.json +72 -0
  146. package/examples/scenarios/mobile/media-open-close.json +141 -0
  147. package/examples/scenarios/mobile/open-close-cycle.json +135 -0
  148. package/examples/scenarios/mobile/scroll-settle.json +106 -0
  149. package/package.json +240 -0
  150. package/schemas/budget-verdict.schema.json +115 -0
  151. package/schemas/causal-run.schema.json +279 -0
  152. package/schemas/comparison.schema.json +196 -0
  153. package/schemas/health.schema.json +108 -0
  154. package/schemas/live-proof-set.schema.json +195 -0
  155. package/schemas/live-proof.schema.json +413 -0
  156. package/schemas/manifest.schema.json +204 -0
  157. package/schemas/metrics.schema.json +137 -0
  158. package/schemas/project-validation.schema.json +343 -0
  159. package/schemas/runner-capabilities.schema.json +217 -0
  160. package/schemas/scenario.schema.json +400 -0
  161. package/schemas/verdict.schema.json +88 -0
  162. package/templates/evidence-provider.json +83 -0
  163. package/templates/gitignore-snippet +9 -0
  164. package/templates/integration-readme.md +125 -0
  165. package/templates/mobile-scenario.json +133 -0
  166. package/templates/package-scripts.json +32 -0
  167. package/templates/primary-runner.json +19 -0
  168. package/templates/project.config.json +37 -0
  169. package/templates/scripts/asl-capture-accessibility-provider.mjs +112 -0
  170. package/templates/scripts/asl-capture-profiler-provider.mjs +127 -0
@@ -0,0 +1,897 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.PROFILE_EVENT_PREFIX = void 0;
4
+ exports.buildBudgetVerdict = buildBudgetVerdict;
5
+ exports.buildCausalRun = buildCausalRun;
6
+ exports.buildCausalTimeline = buildCausalTimeline;
7
+ exports.buildManifest = buildManifest;
8
+ exports.buildMetricsFromProfileEvents = buildMetricsFromProfileEvents;
9
+ exports.buildSummaryMarkdown = buildSummaryMarkdown;
10
+ exports.evaluateUiContract = evaluateUiContract;
11
+ exports.evaluateProfileBudgets = evaluateProfileBudgets;
12
+ exports.extractCandidateIdentifiers = extractCandidateIdentifiers;
13
+ exports.extractProfileEvents = extractProfileEvents;
14
+ exports.findMatchingIdentifier = findMatchingIdentifier;
15
+ exports.percentile = percentile;
16
+ exports.sortValue = sortValue;
17
+ const PROFILE_EVENT_PREFIX = '[profile-event]';
18
+ exports.PROFILE_EVENT_PREFIX = PROFILE_EVENT_PREFIX;
19
+ /**
20
+ * Converts finite numeric strings to numbers while preserving invalid input as `null`.
21
+ *
22
+ * @param {unknown} value
23
+ * @returns {number | null}
24
+ */
25
+ function coerceNumber(value) {
26
+ if (typeof value === 'number' && Number.isFinite(value)) {
27
+ return value;
28
+ }
29
+ if (typeof value !== 'string' || value.length === 0) {
30
+ return null;
31
+ }
32
+ const parsed = Number(value);
33
+ return Number.isFinite(parsed) ? parsed : null;
34
+ }
35
+ /**
36
+ * Parses legacy key/value `[profile-event]` payloads into structured event objects.
37
+ *
38
+ * @param {string} payload
39
+ * @returns {Record<string, unknown> | null}
40
+ */
41
+ function parseKeyValueProfileEvent(payload) {
42
+ const matches = payload.match(/(?:[^\s=]+)=(?:"[^"]*"|'[^']*'|[^\s]+)/gu) ?? [];
43
+ if (matches.length === 0) {
44
+ return null;
45
+ }
46
+ const event = {};
47
+ for (const match of matches) {
48
+ const separatorIndex = match.indexOf('=');
49
+ if (separatorIndex <= 0) {
50
+ continue;
51
+ }
52
+ const key = match.slice(0, separatorIndex);
53
+ let value = match.slice(separatorIndex + 1);
54
+ if ((value.startsWith('"') && value.endsWith('"')) ||
55
+ (value.startsWith("'") && value.endsWith("'"))) {
56
+ value = value.slice(1, -1);
57
+ }
58
+ event[key] = value;
59
+ }
60
+ if (typeof event.event !== 'string' ||
61
+ typeof event.scenario !== 'string' ||
62
+ typeof event.runId !== 'string') {
63
+ return null;
64
+ }
65
+ const iteration = coerceNumber(event.iteration);
66
+ const atMs = coerceNumber(event.atMs) ?? coerceNumber(event.timestamp);
67
+ if (iteration !== null) {
68
+ event.iteration = iteration;
69
+ }
70
+ if (atMs !== null) {
71
+ event.atMs = atMs;
72
+ }
73
+ return event;
74
+ }
75
+ /**
76
+ * Rounds millisecond values to a stable artifact precision.
77
+ *
78
+ * @param {number} value
79
+ * @returns {number}
80
+ */
81
+ function roundMs(value) {
82
+ return Math.round(value * 1000) / 1000;
83
+ }
84
+ /**
85
+ * Returns a nearest-rank percentile for numeric measurements.
86
+ *
87
+ * @param {number[]} values
88
+ * @param {number} percentileValue
89
+ * @returns {number | null}
90
+ */
91
+ function percentile(values, percentileValue) {
92
+ if (!Array.isArray(values) || values.length === 0) {
93
+ return null;
94
+ }
95
+ const sorted = [...values].sort((left, right) => left - right);
96
+ const rank = Math.ceil((percentileValue / 100) * sorted.length) - 1;
97
+ const index = Math.min(sorted.length - 1, Math.max(0, rank));
98
+ const value = sorted[index];
99
+ return typeof value === 'number' ? roundMs(value) : null;
100
+ }
101
+ /**
102
+ * Extracts structured profile events from device logs.
103
+ *
104
+ * Supports both JSON payloads and the older key/value payload format.
105
+ *
106
+ * @param {string} logText
107
+ * @param {{runId?: string, scenario?: string}} [filters]
108
+ * @returns {Record<string, unknown>[]}
109
+ */
110
+ function extractProfileEvents(logText, filters = {}) {
111
+ const { runId, scenario } = filters;
112
+ return String(logText)
113
+ .split(/\r?\n/u)
114
+ .flatMap((line) => {
115
+ const prefixIndex = line.indexOf(PROFILE_EVENT_PREFIX);
116
+ if (prefixIndex === -1) {
117
+ return [];
118
+ }
119
+ const payload = line.slice(prefixIndex + PROFILE_EVENT_PREFIX.length).trim();
120
+ if (!payload) {
121
+ return [];
122
+ }
123
+ try {
124
+ const event = JSON.parse(payload);
125
+ if (!event || typeof event !== 'object') {
126
+ return [];
127
+ }
128
+ if (runId && event.runId !== runId) {
129
+ return [];
130
+ }
131
+ if (scenario && event.scenario !== scenario) {
132
+ return [];
133
+ }
134
+ return [event];
135
+ }
136
+ catch {
137
+ const event = parseKeyValueProfileEvent(payload);
138
+ if (!event) {
139
+ return [];
140
+ }
141
+ if (runId && event.runId !== runId) {
142
+ return [];
143
+ }
144
+ if (scenario && event.scenario !== scenario) {
145
+ return [];
146
+ }
147
+ return [event];
148
+ }
149
+ });
150
+ }
151
+ /**
152
+ * Builds timing metrics from app-emitted profile events.
153
+ *
154
+ * @param {{scenario: string, runId: string, events: Record<string, unknown>[], expectedIterations: number, timeoutCount?: number, artifacts?: Record<string, unknown>, cycleEventNames?: Record<string, string> | null, budgets?: Record<string, unknown> | null}} options
155
+ * @returns {Record<string, unknown>}
156
+ */
157
+ function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterations, timeoutCount = 0, artifacts = {}, cycleEventNames = null, budgets = null, }) {
158
+ const resolvedCycleEventNames = {
159
+ openRequested: cycleEventNames?.openRequested ?? 'surface_open_requested',
160
+ opened: cycleEventNames?.opened ?? 'surface_opened',
161
+ closeRequested: cycleEventNames?.closeRequested ?? 'surface_close_requested',
162
+ dismissed: cycleEventNames?.dismissed ?? 'surface_dismissed',
163
+ milestone: cycleEventNames?.milestone,
164
+ };
165
+ const usesMilestoneOnlyCycle = typeof resolvedCycleEventNames.milestone === 'string';
166
+ const iterations = new Map();
167
+ for (const event of [...events].sort((left, right) => {
168
+ const leftAt = typeof left.atMs === 'number' ? left.atMs : Number.POSITIVE_INFINITY;
169
+ const rightAt = typeof right.atMs === 'number' ? right.atMs : Number.POSITIVE_INFINITY;
170
+ return leftAt - rightAt;
171
+ })) {
172
+ const eventIteration = typeof event.iteration === 'number'
173
+ ? event.iteration
174
+ : expectedIterations === 1
175
+ ? 1
176
+ : null;
177
+ if (eventIteration === null) {
178
+ continue;
179
+ }
180
+ if (typeof event.atMs !== 'number') {
181
+ continue;
182
+ }
183
+ const current = iterations.get(eventIteration) ?? {};
184
+ if (event.event === resolvedCycleEventNames.openRequested &&
185
+ typeof current.presentRequestedAt !== 'number') {
186
+ current.presentRequestedAt = event.atMs;
187
+ }
188
+ if (event.event === resolvedCycleEventNames.opened &&
189
+ typeof current.openedAt !== 'number' &&
190
+ typeof current.presentRequestedAt === 'number' &&
191
+ event.atMs >= current.presentRequestedAt) {
192
+ current.openedAt = event.atMs;
193
+ }
194
+ if (event.event === resolvedCycleEventNames.closeRequested &&
195
+ typeof current.closeRequestedAt !== 'number') {
196
+ current.closeRequestedAt = event.atMs;
197
+ }
198
+ if (event.event === resolvedCycleEventNames.dismissed &&
199
+ typeof current.dismissedAt !== 'number' &&
200
+ typeof current.presentRequestedAt === 'number' &&
201
+ typeof current.closeRequestedAt === 'number' &&
202
+ event.atMs >= current.presentRequestedAt &&
203
+ event.atMs >= current.closeRequestedAt) {
204
+ current.dismissedAt = event.atMs;
205
+ }
206
+ if (event.event === resolvedCycleEventNames.milestone &&
207
+ typeof current.milestoneAt !== 'number') {
208
+ current.milestoneAt = event.atMs;
209
+ }
210
+ iterations.set(eventIteration, current);
211
+ }
212
+ const durationsMs = [];
213
+ const openDurationsMs = [];
214
+ const closeDurationsMs = [];
215
+ const incompleteIterations = [];
216
+ let failures = 0;
217
+ for (let iteration = 1; iteration <= expectedIterations; iteration += 1) {
218
+ const record = iterations.get(iteration);
219
+ if (!record) {
220
+ failures += 1;
221
+ incompleteIterations.push(iteration);
222
+ continue;
223
+ }
224
+ const hasCycleDuration = typeof record.presentRequestedAt === 'number' &&
225
+ typeof record.dismissedAt === 'number' &&
226
+ record.dismissedAt >= record.presentRequestedAt;
227
+ const hasOpenDuration = typeof record.presentRequestedAt === 'number' &&
228
+ typeof record.openedAt === 'number' &&
229
+ record.openedAt >= record.presentRequestedAt;
230
+ const hasCloseDuration = typeof record.closeRequestedAt === 'number' &&
231
+ typeof record.dismissedAt === 'number' &&
232
+ record.dismissedAt >= record.closeRequestedAt;
233
+ const hasMilestoneDuration = usesMilestoneOnlyCycle &&
234
+ typeof record.milestoneAt === 'number' &&
235
+ record.milestoneAt >= 0;
236
+ if (hasMilestoneDuration) {
237
+ durationsMs.push(roundMs(record.milestoneAt));
238
+ openDurationsMs.push(roundMs(record.milestoneAt));
239
+ }
240
+ else if (hasCycleDuration) {
241
+ durationsMs.push(roundMs(record.dismissedAt - record.presentRequestedAt));
242
+ }
243
+ if (hasOpenDuration) {
244
+ openDurationsMs.push(roundMs(record.openedAt - record.presentRequestedAt));
245
+ }
246
+ if (hasCloseDuration) {
247
+ closeDurationsMs.push(roundMs(record.dismissedAt - record.closeRequestedAt));
248
+ }
249
+ const iterationComplete = usesMilestoneOnlyCycle
250
+ ? hasMilestoneDuration
251
+ : hasCycleDuration && hasOpenDuration && hasCloseDuration;
252
+ if (!iterationComplete) {
253
+ failures += 1;
254
+ incompleteIterations.push(iteration);
255
+ }
256
+ }
257
+ const metrics = {
258
+ scenario,
259
+ runId,
260
+ status: failures === 0 && timeoutCount === 0 ? 'passed' : 'failed',
261
+ iterations: expectedIterations,
262
+ durationsMs,
263
+ p50Ms: percentile(durationsMs, 50),
264
+ p95Ms: percentile(durationsMs, 95),
265
+ failures,
266
+ timeouts: timeoutCount,
267
+ openDurationsMs,
268
+ closeDurationsMs,
269
+ incompleteIterations,
270
+ artifacts: sortValue(artifacts),
271
+ };
272
+ const budgetEvaluation = evaluateProfileBudgets({ metrics, budgets });
273
+ if (budgetEvaluation) {
274
+ metrics.budgetEvaluation = sortValue(budgetEvaluation);
275
+ }
276
+ return metrics;
277
+ }
278
+ /**
279
+ * Evaluates one numeric budget threshold.
280
+ *
281
+ * @param {{name: string, actual: unknown, limit: unknown}} options
282
+ * @returns {{name: string, actual: unknown, limit: number, pass: boolean, unit: string} | null}
283
+ */
284
+ function evaluateBudgetCheck({ name, actual, limit }) {
285
+ if (typeof limit !== 'number') {
286
+ return null;
287
+ }
288
+ const pass = typeof actual === 'number' && actual <= limit;
289
+ return {
290
+ name,
291
+ actual,
292
+ limit,
293
+ pass,
294
+ unit: 'ms',
295
+ };
296
+ }
297
+ /**
298
+ * Evaluates configured profile budgets against generated metrics.
299
+ *
300
+ * @param {{metrics: Record<string, unknown>, budgets?: Record<string, unknown> | null}} options
301
+ * @returns {Record<string, unknown> | null}
302
+ */
303
+ function evaluateProfileBudgets({ metrics, budgets }) {
304
+ if (!budgets?.pass || typeof budgets.pass !== 'object') {
305
+ return null;
306
+ }
307
+ const checks = [
308
+ evaluateBudgetCheck({
309
+ name: 'cycle p50',
310
+ actual: metrics.p50Ms,
311
+ limit: budgets.pass.cycleP50Ms,
312
+ }),
313
+ evaluateBudgetCheck({
314
+ name: 'cycle p95',
315
+ actual: metrics.p95Ms,
316
+ limit: budgets.pass.cycleP95Ms,
317
+ }),
318
+ evaluateBudgetCheck({
319
+ name: 'open p50',
320
+ actual: percentile(metrics.openDurationsMs, 50),
321
+ limit: budgets.pass.openP50Ms,
322
+ }),
323
+ evaluateBudgetCheck({
324
+ name: 'open p95',
325
+ actual: percentile(metrics.openDurationsMs, 95),
326
+ limit: budgets.pass.openP95Ms,
327
+ }),
328
+ evaluateBudgetCheck({
329
+ name: 'close p50',
330
+ actual: percentile(metrics.closeDurationsMs, 50),
331
+ limit: budgets.pass.closeP50Ms,
332
+ }),
333
+ evaluateBudgetCheck({
334
+ name: 'close p95',
335
+ actual: percentile(metrics.closeDurationsMs, 95),
336
+ limit: budgets.pass.closeP95Ms,
337
+ }),
338
+ evaluateBudgetCheck({
339
+ name: 'scroll p50',
340
+ actual: metrics.p50Ms,
341
+ limit: budgets.pass.scrollP50Ms,
342
+ }),
343
+ evaluateBudgetCheck({
344
+ name: 'scroll p95',
345
+ actual: metrics.p95Ms,
346
+ limit: budgets.pass.scrollP95Ms,
347
+ }),
348
+ evaluateBudgetCheck({
349
+ name: 'first visible p50',
350
+ actual: metrics.firstVisibleP50Ms,
351
+ limit: budgets.pass.firstVisibleP50Ms,
352
+ }),
353
+ evaluateBudgetCheck({
354
+ name: 'first visible p95',
355
+ actual: metrics.firstVisibleP95Ms,
356
+ limit: budgets.pass.firstVisibleP95Ms,
357
+ }),
358
+ ].filter((check) => Boolean(check));
359
+ const thresholdChecks = [
360
+ typeof budgets.pass.failures === 'number'
361
+ ? {
362
+ name: 'failures',
363
+ actual: metrics.failures,
364
+ limit: budgets.pass.failures,
365
+ pass: metrics.failures <= budgets.pass.failures,
366
+ unit: 'count',
367
+ }
368
+ : null,
369
+ typeof budgets.pass.timeouts === 'number'
370
+ ? {
371
+ name: 'timeouts',
372
+ actual: metrics.timeouts,
373
+ limit: budgets.pass.timeouts,
374
+ pass: metrics.timeouts <= budgets.pass.timeouts,
375
+ unit: 'count',
376
+ }
377
+ : null,
378
+ ].filter((check) => Boolean(check));
379
+ const allChecks = [...thresholdChecks, ...checks];
380
+ if (allChecks.length === 0) {
381
+ return null;
382
+ }
383
+ return {
384
+ metric: budgets.metric ?? metrics.measurement ?? 'profile budget',
385
+ pass: allChecks.every((check) => check.pass),
386
+ checks: allChecks,
387
+ failedChecks: allChecks.filter((check) => !check.pass).map((check) => check.name),
388
+ };
389
+ }
390
+ /**
391
+ * Recursively sorts object keys and array values for stable JSON artifacts.
392
+ *
393
+ * @param {unknown} value
394
+ * @returns {unknown}
395
+ */
396
+ function sortValue(value) {
397
+ if (Array.isArray(value)) {
398
+ return [...value].sort().map(sortValue);
399
+ }
400
+ if (value && typeof value === 'object') {
401
+ return Object.keys(value)
402
+ .sort()
403
+ .reduce((result, key) => {
404
+ result[key] = sortValue(value[key]);
405
+ return result;
406
+ }, {});
407
+ }
408
+ return value;
409
+ }
410
+ /**
411
+ * Normalizes event timestamps to milliseconds since run start.
412
+ *
413
+ * @param {{event: Record<string, unknown>, startedAt?: string}} options
414
+ * @returns {number | null}
415
+ */
416
+ function normalizeEventTimestamp({ event, startedAt }) {
417
+ if (!event || typeof event !== 'object') {
418
+ return null;
419
+ }
420
+ if (typeof event.atMs === 'number' && Number.isFinite(event.atMs)) {
421
+ return roundMs(event.atMs);
422
+ }
423
+ const eventTimestamp = typeof event.timestamp === 'number'
424
+ ? event.timestamp
425
+ : typeof event.timestamp === 'string' && event.timestamp.length > 0
426
+ ? Number(event.timestamp)
427
+ : null;
428
+ const startedAtEpochMs = typeof startedAt === 'string' && startedAt.length > 0 ? Date.parse(startedAt) : Number.NaN;
429
+ if (eventTimestamp !== null && Number.isFinite(eventTimestamp) && Number.isFinite(startedAtEpochMs)) {
430
+ return roundMs(Math.max(0, eventTimestamp - startedAtEpochMs));
431
+ }
432
+ return null;
433
+ }
434
+ /**
435
+ * Infers a causal timeline phase from an event name when no explicit phase is provided.
436
+ *
437
+ * @param {unknown} eventName
438
+ * @returns {string}
439
+ */
440
+ function inferTimelinePhase(eventName) {
441
+ if (typeof eventName !== 'string' || eventName.length === 0) {
442
+ return 'domain';
443
+ }
444
+ const normalized = eventName.toLowerCase();
445
+ if (normalized.includes('requested') ||
446
+ normalized.includes('tapped') ||
447
+ normalized.includes('tap') ||
448
+ normalized.includes('intent')) {
449
+ return 'intent';
450
+ }
451
+ if (normalized.includes('route') ||
452
+ normalized.includes('presented') ||
453
+ normalized.includes('dismissed') ||
454
+ normalized.includes('opened')) {
455
+ return 'navigation';
456
+ }
457
+ if (normalized.includes('query') || normalized.includes('cache')) {
458
+ return 'query';
459
+ }
460
+ if (normalized.includes('request') ||
461
+ normalized.includes('response') ||
462
+ normalized.includes('network')) {
463
+ return 'network';
464
+ }
465
+ if (normalized.includes('render') ||
466
+ normalized.includes('mounted') ||
467
+ normalized.includes('shell_ready')) {
468
+ return 'render';
469
+ }
470
+ if (normalized.includes('native') ||
471
+ normalized.includes('frame') ||
472
+ normalized.includes('hitch')) {
473
+ return 'native';
474
+ }
475
+ if (normalized.includes('visible') ||
476
+ normalized.includes('usable') ||
477
+ normalized.includes('committed')) {
478
+ return 'visual';
479
+ }
480
+ if (normalized.includes('finished') ||
481
+ normalized.includes('completed') ||
482
+ normalized.includes('settled')) {
483
+ return 'completion';
484
+ }
485
+ return 'domain';
486
+ }
487
+ /**
488
+ * Infers timeline status from an event name when no explicit status is provided.
489
+ *
490
+ * @param {unknown} eventName
491
+ * @returns {string}
492
+ */
493
+ function inferTimelineStatus(eventName) {
494
+ if (typeof eventName !== 'string' || eventName.length === 0) {
495
+ return 'observed';
496
+ }
497
+ const normalized = eventName.toLowerCase();
498
+ if (normalized.includes('failed')) {
499
+ return 'failed';
500
+ }
501
+ if (normalized.includes('requested') || normalized.includes('started')) {
502
+ return 'started';
503
+ }
504
+ if (normalized.includes('opened') ||
505
+ normalized.includes('dismissed') ||
506
+ normalized.includes('committed') ||
507
+ normalized.includes('visible') ||
508
+ normalized.includes('ready') ||
509
+ normalized.includes('settled') ||
510
+ normalized.includes('completed') ||
511
+ normalized.includes('finished')) {
512
+ return 'completed';
513
+ }
514
+ return 'observed';
515
+ }
516
+ /**
517
+ * Builds a causal timeline from app-owned profile events.
518
+ *
519
+ * @param {{events: Record<string, unknown>[], startedAt?: string, phaseMap?: Record<string, string> | null, owner?: string | null}} options
520
+ * @returns {Record<string, unknown>[]}
521
+ */
522
+ function buildCausalTimeline({ events, startedAt, phaseMap = null, owner = null, }) {
523
+ return [...(Array.isArray(events) ? events : [])]
524
+ .map((event) => {
525
+ if (!event || typeof event !== 'object' || typeof event.event !== 'string') {
526
+ return null;
527
+ }
528
+ const atMs = normalizeEventTimestamp({
529
+ event,
530
+ ...(typeof startedAt === 'string' ? { startedAt } : {}),
531
+ });
532
+ if (atMs === null) {
533
+ return null;
534
+ }
535
+ const eventMetadata = event.metadata && typeof event.metadata === 'object' && !Array.isArray(event.metadata)
536
+ ? event.metadata
537
+ : {};
538
+ const explicitPhase = typeof event.phase === 'string'
539
+ ? event.phase
540
+ : phaseMap && typeof phaseMap[event.event] === 'string'
541
+ ? phaseMap[event.event]
542
+ : null;
543
+ const metadata = {
544
+ ...eventMetadata,
545
+ ...(typeof event.flowId === 'string' ? { flowId: event.flowId } : {}),
546
+ ...(typeof event.route === 'string' ? { route: event.route } : {}),
547
+ ...(typeof event.iteration === 'number' ? { iteration: event.iteration } : {}),
548
+ };
549
+ return sortValue({
550
+ phase: explicitPhase ?? inferTimelinePhase(event.event),
551
+ name: event.event,
552
+ atMs,
553
+ status: typeof event.status === 'string' ? event.status : inferTimelineStatus(event.event),
554
+ ...((typeof event.owner === 'string' && event.owner.length > 0) || owner
555
+ ? { owner: event.owner || owner }
556
+ : {}),
557
+ ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
558
+ });
559
+ })
560
+ .filter(Boolean)
561
+ .sort((left, right) => left.atMs - right.atMs);
562
+ }
563
+ /**
564
+ * Builds the `budget-verdict.json` profile artifact from budget evaluation.
565
+ *
566
+ * @param {{flowId: string, runId: string, budgetEvaluation?: Record<string, unknown> | null, visualOutcome?: Record<string, unknown> | null, baselineRunId?: string | null}} options
567
+ * @returns {Record<string, unknown> | null}
568
+ */
569
+ function buildBudgetVerdict({ flowId, runId, budgetEvaluation, visualOutcome = null, baselineRunId = null, }) {
570
+ if (!budgetEvaluation) {
571
+ return null;
572
+ }
573
+ const hasManualVisualReview = visualOutcome &&
574
+ Array.isArray(visualOutcome.checks) &&
575
+ visualOutcome.checks.some((check) => check.status === 'manual-review-needed');
576
+ return sortValue({
577
+ schemaVersion: '1.0.0',
578
+ flowId,
579
+ runId,
580
+ status: hasManualVisualReview
581
+ ? 'partial'
582
+ : budgetEvaluation.pass
583
+ ? 'passed'
584
+ : 'failed',
585
+ checks: (budgetEvaluation.checks ?? []).map((check) => ({
586
+ name: check.name,
587
+ metric: budgetEvaluation.metric ?? 'profile budget',
588
+ unit: check.unit,
589
+ expected: check.limit,
590
+ actual: check.actual ?? null,
591
+ pass: check.pass,
592
+ })),
593
+ ...(visualOutcome ? { visualOutcome } : {}),
594
+ ...(baselineRunId
595
+ ? {
596
+ regression: {
597
+ baselineRunId,
598
+ status: 'unknown',
599
+ summary: 'Regression status is unknown until comparison.json is produced for the baseline/current run folders.',
600
+ },
601
+ }
602
+ : {}),
603
+ });
604
+ }
605
+ /**
606
+ * Infers the display unit for legacy budget keys.
607
+ *
608
+ * @param {unknown} name
609
+ * @returns {string}
610
+ */
611
+ function inferBudgetUnit(name) {
612
+ if (typeof name !== 'string') {
613
+ return 'count';
614
+ }
615
+ if (/ms$/u.test(name) || /p50|p95|duration|cycle|visible|open|close/u.test(name)) {
616
+ return 'ms';
617
+ }
618
+ return 'count';
619
+ }
620
+ /**
621
+ * Normalizes legacy budget config into the causal-run budget shape.
622
+ *
623
+ * @param {Record<string, unknown> | null | undefined} budgets
624
+ * @returns {Record<string, unknown>}
625
+ */
626
+ function normalizeBudgetsForCausalRun(budgets) {
627
+ if (!budgets || typeof budgets !== 'object') {
628
+ return {};
629
+ }
630
+ return Object.keys(budgets).reduce((result, key) => {
631
+ const limit = budgets[key];
632
+ if (typeof limit !== 'number' && typeof limit !== 'boolean') {
633
+ return result;
634
+ }
635
+ result[key] = {
636
+ metric: key,
637
+ unit: inferBudgetUnit(key),
638
+ limit,
639
+ };
640
+ return result;
641
+ }, {});
642
+ }
643
+ /**
644
+ * Builds the `causal-run.json` profile artifact.
645
+ *
646
+ * @param {Record<string, unknown>} options
647
+ * @returns {Record<string, unknown>}
648
+ */
649
+ function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor = 'unknown', interactionDriver, trigger = null, budgets = null, timeline = [], artifacts, manifest, metrics, }) {
650
+ return sortValue({
651
+ schemaVersion: '1.0.0',
652
+ flowId,
653
+ runId,
654
+ platform,
655
+ buildFlavor,
656
+ scenario: {
657
+ id: scenario.name,
658
+ driver: interactionDriver,
659
+ ...(typeof metrics?.iterations === 'number' ? { iterations: metrics.iterations } : {}),
660
+ },
661
+ trigger: trigger && typeof trigger === 'object'
662
+ ? trigger
663
+ : {
664
+ kind: 'unknown',
665
+ label: scenario.description ?? scenario.name,
666
+ },
667
+ budgets: normalizeBudgetsForCausalRun(budgets),
668
+ timeline,
669
+ artifacts: {
670
+ summary: artifacts.summary,
671
+ metrics: artifacts.metrics,
672
+ manifest: artifacts.manifest,
673
+ video: artifacts.captures?.video,
674
+ screenshot: Array.isArray(artifacts.captures?.screenshots)
675
+ ? artifacts.captures.screenshots[0] ?? null
676
+ : null,
677
+ signals: artifacts.signals,
678
+ evidenceAttachments: Array.isArray(artifacts.evidenceAttachments) ? artifacts.evidenceAttachments : [],
679
+ },
680
+ notes: [
681
+ `Manifest status: ${manifest.status}`,
682
+ `Metrics status: ${metrics.status}`,
683
+ ],
684
+ });
685
+ }
686
+ /**
687
+ * Builds the run manifest artifact.
688
+ *
689
+ * @param {Record<string, unknown>} options
690
+ * @returns {Record<string, unknown>}
691
+ */
692
+ function buildManifest({ scenario, scenarioHash, runId, platform = 'ios', status, startedAt, endedAt, interactionDriver, comparisonLane, simulator, bundleId, gitSha, toolVersions, artifacts, failureReason = null, }) {
693
+ return {
694
+ scenario,
695
+ ...(typeof scenarioHash === 'string' && scenarioHash.length > 0 ? { scenarioHash } : {}),
696
+ runId,
697
+ platform,
698
+ status,
699
+ startedAt,
700
+ endedAt,
701
+ durationMs: roundMs(Math.max(0, Date.parse(endedAt) - Date.parse(startedAt))),
702
+ interactionDriver,
703
+ ...(typeof comparisonLane === 'string' && comparisonLane.length > 0 ? { comparisonLane } : {}),
704
+ simulator: sortValue(simulator),
705
+ bundleId,
706
+ gitSha,
707
+ toolVersions: sortValue(toolVersions),
708
+ artifacts: sortValue(artifacts),
709
+ failureReason,
710
+ };
711
+ }
712
+ /**
713
+ * Builds the human-readable profile summary.
714
+ *
715
+ * @param {{manifest: Record<string, unknown>, metrics: Record<string, unknown>}} options
716
+ * @returns {string}
717
+ */
718
+ function buildSummaryMarkdown({ manifest, metrics }) {
719
+ const runtimeLabel = manifest.platform === 'android' ? 'Device' : 'Simulator';
720
+ const signalLines = [
721
+ `- JS: ${manifest.artifacts.signals.js.length > 0
722
+ ? manifest.artifacts.signals.js.map((item) => `\`${item}\``).join(', ')
723
+ : 'none'}`,
724
+ `- Memory: ${manifest.artifacts.signals.memory.length > 0
725
+ ? manifest.artifacts.signals.memory.map((item) => `\`${item}\``).join(', ')
726
+ : 'none'}`,
727
+ `- Network: ${manifest.artifacts.signals.network.length > 0
728
+ ? manifest.artifacts.signals.network.map((item) => `\`${item}\``).join(', ')
729
+ : 'none'}`,
730
+ ];
731
+ const screenshots = Array.isArray(manifest.artifacts.captures.screenshots)
732
+ ? manifest.artifacts.captures.screenshots
733
+ : [];
734
+ const evidenceAttachments = Array.isArray(manifest.artifacts.evidenceAttachments)
735
+ ? manifest.artifacts.evidenceAttachments
736
+ : [];
737
+ const evidenceAttachmentLines = evidenceAttachments.length > 0
738
+ ? evidenceAttachments.map((attachment) => `- ${attachment.channel}/${attachment.kind}: \`${attachment.path}\` (${attachment.sizeBytes} bytes, sha256 ${attachment.sha256})`)
739
+ : ['- none'];
740
+ const lines = [
741
+ `# ${String(manifest.platform || 'ios').toUpperCase()} profile run: ${manifest.scenario}`,
742
+ '',
743
+ `- Status: ${manifest.status}`,
744
+ `- Run ID: \`${manifest.runId}\``,
745
+ `- Interaction driver: \`${manifest.interactionDriver}\``,
746
+ ...(typeof manifest.comparisonLane === 'string'
747
+ ? [`- Comparison lane: \`${manifest.comparisonLane}\``]
748
+ : []),
749
+ `- ${runtimeLabel}: ${manifest.simulator.name} (${manifest.simulator.udid})`,
750
+ `- Bundle ID: \`${manifest.bundleId}\``,
751
+ `- Iterations: ${metrics.iterations}`,
752
+ `- Completed cycles: ${metrics.durationsMs.length}/${metrics.iterations}`,
753
+ `- Failures: ${metrics.failures}`,
754
+ `- Timeouts: ${metrics.timeouts}`,
755
+ `- p50 cycle: ${metrics.p50Ms === null ? 'n/a' : `${metrics.p50Ms}ms`}`,
756
+ `- p95 cycle: ${metrics.p95Ms === null ? 'n/a' : `${metrics.p95Ms}ms`}`,
757
+ '',
758
+ '## Artifact paths',
759
+ '',
760
+ `- Causal run: \`${manifest.artifacts.causalRun}\``,
761
+ `- Budget verdict: ${metrics.budgetEvaluation ? `\`${manifest.artifacts.budgetVerdict}\`` : 'none (no budgets configured)'}`,
762
+ `- Manifest: \`${manifest.artifacts.manifest}\``,
763
+ `- Scenario: \`${manifest.artifacts.scenario}\``,
764
+ `- Metrics: \`${manifest.artifacts.metrics}\``,
765
+ `- Interaction log: \`${manifest.artifacts.raw.interactionLog}\``,
766
+ `- Device log: \`${manifest.artifacts.raw.deviceLog}\``,
767
+ `- Video: \`${manifest.artifacts.captures.video}\``,
768
+ `- UI tree: \`${manifest.artifacts.captures.uiTree}\``,
769
+ `- Screenshots: ${screenshots.length > 0
770
+ ? screenshots.map((item) => `\`${item}\``).join(', ')
771
+ : 'none'}`,
772
+ '',
773
+ '## Signal attachments',
774
+ '',
775
+ ...signalLines,
776
+ '',
777
+ '## Evidence attachments',
778
+ '',
779
+ ...evidenceAttachmentLines,
780
+ ];
781
+ if (metrics.budgetEvaluation) {
782
+ lines.push('', '## Budget', '', `- Metric: ${metrics.budgetEvaluation.metric}`, `- Status: ${metrics.budgetEvaluation.pass ? 'pass' : 'fail'}`);
783
+ for (const check of metrics.budgetEvaluation.checks) {
784
+ const formatValue = (value) => {
785
+ if (typeof value !== 'number') {
786
+ return value ?? 'n/a';
787
+ }
788
+ return check.unit === 'ms' ? `${value}ms` : String(value);
789
+ };
790
+ lines.push(`- ${check.name}: ${formatValue(check.actual)} / ${formatValue(check.limit)}`);
791
+ }
792
+ }
793
+ if (metrics.openDurationsMs.length > 0) {
794
+ lines.push('', `- Open durations (ms): ${metrics.openDurationsMs.join(', ')}`);
795
+ }
796
+ if (metrics.closeDurationsMs.length > 0) {
797
+ lines.push(`- Close durations (ms): ${metrics.closeDurationsMs.join(', ')}`);
798
+ }
799
+ if (metrics.incompleteIterations.length > 0) {
800
+ lines.push(`- Incomplete iterations: ${metrics.incompleteIterations.join(', ')}`);
801
+ }
802
+ if (manifest.failureReason) {
803
+ lines.push('', '## Failure', '', manifest.failureReason);
804
+ }
805
+ return lines.join('\n');
806
+ }
807
+ /**
808
+ * Extracts possible accessibility or test identifiers from a UI tree dump.
809
+ *
810
+ * @param {string} rawDescription
811
+ * @returns {string[]}
812
+ */
813
+ function extractCandidateIdentifiers(rawDescription) {
814
+ const seen = new Set();
815
+ const identifiers = [];
816
+ const push = (value) => {
817
+ if (typeof value !== 'string') {
818
+ return;
819
+ }
820
+ const normalized = value.trim();
821
+ if (!normalized || seen.has(normalized)) {
822
+ return;
823
+ }
824
+ seen.add(normalized);
825
+ identifiers.push(normalized);
826
+ };
827
+ const visit = (value) => {
828
+ if (Array.isArray(value)) {
829
+ for (const item of value) {
830
+ visit(item);
831
+ }
832
+ return;
833
+ }
834
+ if (!value || typeof value !== 'object') {
835
+ return;
836
+ }
837
+ for (const [key, child] of Object.entries(value)) {
838
+ if (key === 'id' ||
839
+ key === 'identifier' ||
840
+ key === 'accessibilityIdentifier' ||
841
+ key === 'AXUniqueId' ||
842
+ key === 'AXLabel' ||
843
+ key === 'testID' ||
844
+ key === 'testId') {
845
+ push(child);
846
+ }
847
+ visit(child);
848
+ }
849
+ };
850
+ try {
851
+ visit(JSON.parse(rawDescription));
852
+ }
853
+ catch {
854
+ // Fall back to a raw regex scan when the driver output is not JSON.
855
+ }
856
+ const matches = String(rawDescription).match(/[A-Za-z0-9][A-Za-z0-9._:-]{2,}/gu) ?? [];
857
+ for (const match of matches) {
858
+ push(match);
859
+ }
860
+ return identifiers;
861
+ }
862
+ /**
863
+ * Finds the first UI identifier matching a required pattern.
864
+ *
865
+ * @param {string} rawDescription
866
+ * @param {string} pattern
867
+ * @returns {string | null}
868
+ */
869
+ function findMatchingIdentifier(rawDescription, pattern) {
870
+ const regex = new RegExp(pattern, 'u');
871
+ const identifier = extractCandidateIdentifiers(rawDescription).find((candidate) => regex.test(candidate));
872
+ if (identifier) {
873
+ return identifier;
874
+ }
875
+ return String(rawDescription).match(regex)?.[0] ?? null;
876
+ }
877
+ /**
878
+ * Evaluates a UI contract from required identifier patterns.
879
+ *
880
+ * @param {{rawDescription: string, requiredIdentifierPatterns?: string[]}} options
881
+ * @returns {{pass: boolean, checks: Record<string, unknown>[], missingPatterns: string[]}}
882
+ */
883
+ function evaluateUiContract({ rawDescription, requiredIdentifierPatterns = [], }) {
884
+ const checks = requiredIdentifierPatterns.map((pattern) => {
885
+ const matchedIdentifier = findMatchingIdentifier(rawDescription, pattern);
886
+ return {
887
+ pattern,
888
+ pass: Boolean(matchedIdentifier),
889
+ matchedIdentifier,
890
+ };
891
+ });
892
+ return {
893
+ pass: checks.every((check) => check.pass),
894
+ checks,
895
+ missingPatterns: checks.filter((check) => !check.pass).map((check) => check.pattern),
896
+ };
897
+ }