agent-scenario-loop 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +119 -0
- package/app/profile-session.ts +812 -0
- package/core/config-template.json +41 -0
- package/dist/core/agent-summary.d.ts +15 -0
- package/dist/core/agent-summary.js +177 -0
- package/dist/core/artifact-contract.d.ts +151 -0
- package/dist/core/artifact-contract.js +897 -0
- package/dist/core/artifact-layout.d.ts +56 -0
- package/dist/core/artifact-layout.js +61 -0
- package/dist/core/artifact-writer.d.ts +44 -0
- package/dist/core/artifact-writer.js +55 -0
- package/dist/core/comparison.d.ts +133 -0
- package/dist/core/comparison.js +294 -0
- package/dist/core/evidence-interpreter.d.ts +28 -0
- package/dist/core/evidence-interpreter.js +69 -0
- package/dist/core/execution-plan.d.ts +44 -0
- package/dist/core/execution-plan.js +95 -0
- package/dist/core/planner.d.ts +132 -0
- package/dist/core/planner.js +812 -0
- package/dist/core/ports.d.ts +198 -0
- package/dist/core/ports.js +146 -0
- package/dist/core/run-index.d.ts +62 -0
- package/dist/core/run-index.js +143 -0
- package/dist/core/schema-validator.d.ts +86 -0
- package/dist/core/schema-validator.js +407 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +27 -0
- package/dist/runner/agent-device-driver.d.ts +126 -0
- package/dist/runner/agent-device-driver.js +168 -0
- package/dist/runner/agent-device.d.ts +295 -0
- package/dist/runner/agent-device.js +1271 -0
- package/dist/runner/android-adb-driver.d.ts +175 -0
- package/dist/runner/android-adb-driver.js +399 -0
- package/dist/runner/android-adb.d.ts +254 -0
- package/dist/runner/android-adb.js +1618 -0
- package/dist/runner/argent-driver.d.ts +183 -0
- package/dist/runner/argent-driver.js +297 -0
- package/dist/runner/argent.d.ts +349 -0
- package/dist/runner/argent.js +1211 -0
- package/dist/runner/check-plan.d.ts +45 -0
- package/dist/runner/check-plan.js +210 -0
- package/dist/runner/cli.d.ts +20 -0
- package/dist/runner/cli.js +23 -0
- package/dist/runner/compare-latest.d.ts +99 -0
- package/dist/runner/compare-latest.js +233 -0
- package/dist/runner/compare.d.ts +58 -0
- package/dist/runner/compare.js +157 -0
- package/dist/runner/demo-loop.d.ts +45 -0
- package/dist/runner/demo-loop.js +170 -0
- package/dist/runner/example-android-live.d.ts +137 -0
- package/dist/runner/example-android-live.js +454 -0
- package/dist/runner/example-ios-live.d.ts +137 -0
- package/dist/runner/example-ios-live.js +471 -0
- package/dist/runner/host-doctor.d.ts +131 -0
- package/dist/runner/host-doctor.js +628 -0
- package/dist/runner/init-project.d.ts +88 -0
- package/dist/runner/init-project.js +263 -0
- package/dist/runner/ios-simctl-driver.d.ts +69 -0
- package/dist/runner/ios-simctl-driver.js +97 -0
- package/dist/runner/ios-simctl.d.ts +254 -0
- package/dist/runner/ios-simctl.js +1415 -0
- package/dist/runner/live-android.d.ts +137 -0
- package/dist/runner/live-android.js +539 -0
- package/dist/runner/live-comparison.d.ts +67 -0
- package/dist/runner/live-comparison.js +147 -0
- package/dist/runner/live-ios.d.ts +137 -0
- package/dist/runner/live-ios.js +460 -0
- package/dist/runner/live-proof-summary.d.ts +263 -0
- package/dist/runner/live-proof-summary.js +465 -0
- package/dist/runner/live-proof.d.ts +467 -0
- package/dist/runner/live-proof.js +920 -0
- package/dist/runner/local-env.d.ts +64 -0
- package/dist/runner/local-env.js +155 -0
- package/dist/runner/profile-android.d.ts +82 -0
- package/dist/runner/profile-android.js +671 -0
- package/dist/runner/profile-ios.d.ts +108 -0
- package/dist/runner/profile-ios.js +532 -0
- package/dist/runner/profile-mobile.d.ts +254 -0
- package/dist/runner/profile-mobile.js +1307 -0
- package/dist/runner/validate-project.d.ts +273 -0
- package/dist/runner/validate-project.js +1501 -0
- package/docs/adapters.md +145 -0
- package/docs/api.md +94 -0
- package/docs/authoring.md +196 -0
- package/docs/concepts.md +136 -0
- package/docs/consumer-rehearsal.md +115 -0
- package/docs/contracts.md +267 -0
- package/docs/live-proofs.md +270 -0
- package/docs/principles.md +46 -0
- package/examples/event-logs/app-startup-baseline.log +4 -0
- package/examples/event-logs/app-startup-current.log +4 -0
- package/examples/minimal-app/README.md +70 -0
- package/examples/mobile-app/README.md +302 -0
- package/examples/mobile-app/app.json +22 -0
- package/examples/mobile-app/asl/package-scripts.json +32 -0
- package/examples/mobile-app/asl.config.json +37 -0
- package/examples/mobile-app/event-logs/android-app-startup.log +4 -0
- package/examples/mobile-app/event-logs/android-open-close-cycle.log +12 -0
- package/examples/mobile-app/event-logs/android-scroll-settle.log +12 -0
- package/examples/mobile-app/event-logs/app-startup.log +4 -0
- package/examples/mobile-app/event-logs/open-close-cycle.log +12 -0
- package/examples/mobile-app/event-logs/scroll-settle.log +12 -0
- package/examples/mobile-app/index.ts +20 -0
- package/examples/mobile-app/metro.config.js +20 -0
- package/examples/mobile-app/package.json +62 -0
- package/examples/mobile-app/patches/expo-modules-jsi@56.0.10.patch +19 -0
- package/examples/mobile-app/plugins/with-ios-build-compat.js +271 -0
- package/examples/mobile-app/pnpm-lock.yaml +4440 -0
- package/examples/mobile-app/runner-manifests/evidence-provider.json +79 -0
- package/examples/mobile-app/runner-manifests/primary-runner.json +19 -0
- package/examples/mobile-app/scenarios/android/app-startup-video.json +73 -0
- package/examples/mobile-app/scenarios/android/app-startup.json +44 -0
- package/examples/mobile-app/scenarios/android/open-close-cycle.json +54 -0
- package/examples/mobile-app/scenarios/android/scroll-settle.json +49 -0
- package/examples/mobile-app/scenarios/ios/app-startup.json +44 -0
- package/examples/mobile-app/scenarios/ios/open-close-cycle.json +54 -0
- package/examples/mobile-app/scenarios/ios/scroll-settle.json +49 -0
- package/examples/mobile-app/scenarios/mobile/app-startup.json +91 -0
- package/examples/mobile-app/scenarios/mobile/open-close-cycle.json +160 -0
- package/examples/mobile-app/scenarios/mobile/scroll-settle.json +148 -0
- package/examples/mobile-app/scripts/asl-capture-accessibility-provider.mjs +112 -0
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +127 -0
- package/examples/mobile-app/src/devtools/profile-session.ts +7 -0
- package/examples/mobile-app/src/example-screen.tsx +322 -0
- package/examples/mobile-app/tsconfig.json +16 -0
- package/examples/mobile-app/tsconfig.typecheck.json +13 -0
- package/examples/runners/README.md +44 -0
- package/examples/runners/adb-android.json +25 -0
- package/examples/runners/agent-device-android.json +27 -0
- package/examples/runners/agent-device-ios.json +27 -0
- package/examples/runners/argent-android.json +32 -0
- package/examples/runners/argent-ios.json +32 -0
- package/examples/runners/argent-react-profiler-provider.json +15 -0
- package/examples/runners/axe-accessibility-provider.json +24 -0
- package/examples/runners/manual-log-ingest.json +9 -0
- package/examples/runners/rozenite-profiler-provider.json +9 -0
- package/examples/runners/script-accessibility-provider.json +24 -0
- package/examples/runners/script-memory-provider.json +24 -0
- package/examples/runners/script-network-provider.json +24 -0
- package/examples/runners/script-profiler-provider.json +30 -0
- package/examples/runners/xcodebuildmcp-ios.json +29 -0
- package/examples/scenarios/ios/app-startup.json +28 -0
- package/examples/scenarios/ios/open-close-cycle.json +35 -0
- package/examples/scenarios/mobile/app-startup.json +72 -0
- package/examples/scenarios/mobile/media-open-close.json +141 -0
- package/examples/scenarios/mobile/open-close-cycle.json +135 -0
- package/examples/scenarios/mobile/scroll-settle.json +106 -0
- package/package.json +240 -0
- package/schemas/budget-verdict.schema.json +115 -0
- package/schemas/causal-run.schema.json +279 -0
- package/schemas/comparison.schema.json +196 -0
- package/schemas/health.schema.json +108 -0
- package/schemas/live-proof-set.schema.json +195 -0
- package/schemas/live-proof.schema.json +413 -0
- package/schemas/manifest.schema.json +204 -0
- package/schemas/metrics.schema.json +137 -0
- package/schemas/project-validation.schema.json +343 -0
- package/schemas/runner-capabilities.schema.json +217 -0
- package/schemas/scenario.schema.json +400 -0
- package/schemas/verdict.schema.json +88 -0
- package/templates/evidence-provider.json +83 -0
- package/templates/gitignore-snippet +9 -0
- package/templates/integration-readme.md +125 -0
- package/templates/mobile-scenario.json +133 -0
- package/templates/package-scripts.json +32 -0
- package/templates/primary-runner.json +19 -0
- package/templates/project.config.json +37 -0
- package/templates/scripts/asl-capture-accessibility-provider.mjs +112 -0
- package/templates/scripts/asl-capture-profiler-provider.mjs +127 -0
|
@@ -0,0 +1,897 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PROFILE_EVENT_PREFIX = void 0;
|
|
4
|
+
exports.buildBudgetVerdict = buildBudgetVerdict;
|
|
5
|
+
exports.buildCausalRun = buildCausalRun;
|
|
6
|
+
exports.buildCausalTimeline = buildCausalTimeline;
|
|
7
|
+
exports.buildManifest = buildManifest;
|
|
8
|
+
exports.buildMetricsFromProfileEvents = buildMetricsFromProfileEvents;
|
|
9
|
+
exports.buildSummaryMarkdown = buildSummaryMarkdown;
|
|
10
|
+
exports.evaluateUiContract = evaluateUiContract;
|
|
11
|
+
exports.evaluateProfileBudgets = evaluateProfileBudgets;
|
|
12
|
+
exports.extractCandidateIdentifiers = extractCandidateIdentifiers;
|
|
13
|
+
exports.extractProfileEvents = extractProfileEvents;
|
|
14
|
+
exports.findMatchingIdentifier = findMatchingIdentifier;
|
|
15
|
+
exports.percentile = percentile;
|
|
16
|
+
exports.sortValue = sortValue;
|
|
17
|
+
const PROFILE_EVENT_PREFIX = '[profile-event]';
|
|
18
|
+
exports.PROFILE_EVENT_PREFIX = PROFILE_EVENT_PREFIX;
|
|
19
|
+
/**
|
|
20
|
+
* Converts finite numeric strings to numbers while preserving invalid input as `null`.
|
|
21
|
+
*
|
|
22
|
+
* @param {unknown} value
|
|
23
|
+
* @returns {number | null}
|
|
24
|
+
*/
|
|
25
|
+
function coerceNumber(value) {
|
|
26
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
const parsed = Number(value);
|
|
33
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Parses legacy key/value `[profile-event]` payloads into structured event objects.
|
|
37
|
+
*
|
|
38
|
+
* @param {string} payload
|
|
39
|
+
* @returns {Record<string, unknown> | null}
|
|
40
|
+
*/
|
|
41
|
+
function parseKeyValueProfileEvent(payload) {
|
|
42
|
+
const matches = payload.match(/(?:[^\s=]+)=(?:"[^"]*"|'[^']*'|[^\s]+)/gu) ?? [];
|
|
43
|
+
if (matches.length === 0) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
const event = {};
|
|
47
|
+
for (const match of matches) {
|
|
48
|
+
const separatorIndex = match.indexOf('=');
|
|
49
|
+
if (separatorIndex <= 0) {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
const key = match.slice(0, separatorIndex);
|
|
53
|
+
let value = match.slice(separatorIndex + 1);
|
|
54
|
+
if ((value.startsWith('"') && value.endsWith('"')) ||
|
|
55
|
+
(value.startsWith("'") && value.endsWith("'"))) {
|
|
56
|
+
value = value.slice(1, -1);
|
|
57
|
+
}
|
|
58
|
+
event[key] = value;
|
|
59
|
+
}
|
|
60
|
+
if (typeof event.event !== 'string' ||
|
|
61
|
+
typeof event.scenario !== 'string' ||
|
|
62
|
+
typeof event.runId !== 'string') {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
const iteration = coerceNumber(event.iteration);
|
|
66
|
+
const atMs = coerceNumber(event.atMs) ?? coerceNumber(event.timestamp);
|
|
67
|
+
if (iteration !== null) {
|
|
68
|
+
event.iteration = iteration;
|
|
69
|
+
}
|
|
70
|
+
if (atMs !== null) {
|
|
71
|
+
event.atMs = atMs;
|
|
72
|
+
}
|
|
73
|
+
return event;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Rounds millisecond values to a stable artifact precision.
|
|
77
|
+
*
|
|
78
|
+
* @param {number} value
|
|
79
|
+
* @returns {number}
|
|
80
|
+
*/
|
|
81
|
+
function roundMs(value) {
|
|
82
|
+
return Math.round(value * 1000) / 1000;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Returns a nearest-rank percentile for numeric measurements.
|
|
86
|
+
*
|
|
87
|
+
* @param {number[]} values
|
|
88
|
+
* @param {number} percentileValue
|
|
89
|
+
* @returns {number | null}
|
|
90
|
+
*/
|
|
91
|
+
function percentile(values, percentileValue) {
|
|
92
|
+
if (!Array.isArray(values) || values.length === 0) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
96
|
+
const rank = Math.ceil((percentileValue / 100) * sorted.length) - 1;
|
|
97
|
+
const index = Math.min(sorted.length - 1, Math.max(0, rank));
|
|
98
|
+
const value = sorted[index];
|
|
99
|
+
return typeof value === 'number' ? roundMs(value) : null;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Extracts structured profile events from device logs.
|
|
103
|
+
*
|
|
104
|
+
* Supports both JSON payloads and the older key/value payload format.
|
|
105
|
+
*
|
|
106
|
+
* @param {string} logText
|
|
107
|
+
* @param {{runId?: string, scenario?: string}} [filters]
|
|
108
|
+
* @returns {Record<string, unknown>[]}
|
|
109
|
+
*/
|
|
110
|
+
function extractProfileEvents(logText, filters = {}) {
|
|
111
|
+
const { runId, scenario } = filters;
|
|
112
|
+
return String(logText)
|
|
113
|
+
.split(/\r?\n/u)
|
|
114
|
+
.flatMap((line) => {
|
|
115
|
+
const prefixIndex = line.indexOf(PROFILE_EVENT_PREFIX);
|
|
116
|
+
if (prefixIndex === -1) {
|
|
117
|
+
return [];
|
|
118
|
+
}
|
|
119
|
+
const payload = line.slice(prefixIndex + PROFILE_EVENT_PREFIX.length).trim();
|
|
120
|
+
if (!payload) {
|
|
121
|
+
return [];
|
|
122
|
+
}
|
|
123
|
+
try {
|
|
124
|
+
const event = JSON.parse(payload);
|
|
125
|
+
if (!event || typeof event !== 'object') {
|
|
126
|
+
return [];
|
|
127
|
+
}
|
|
128
|
+
if (runId && event.runId !== runId) {
|
|
129
|
+
return [];
|
|
130
|
+
}
|
|
131
|
+
if (scenario && event.scenario !== scenario) {
|
|
132
|
+
return [];
|
|
133
|
+
}
|
|
134
|
+
return [event];
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
const event = parseKeyValueProfileEvent(payload);
|
|
138
|
+
if (!event) {
|
|
139
|
+
return [];
|
|
140
|
+
}
|
|
141
|
+
if (runId && event.runId !== runId) {
|
|
142
|
+
return [];
|
|
143
|
+
}
|
|
144
|
+
if (scenario && event.scenario !== scenario) {
|
|
145
|
+
return [];
|
|
146
|
+
}
|
|
147
|
+
return [event];
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Builds timing metrics from app-emitted profile events.
|
|
153
|
+
*
|
|
154
|
+
* @param {{scenario: string, runId: string, events: Record<string, unknown>[], expectedIterations: number, timeoutCount?: number, artifacts?: Record<string, unknown>, cycleEventNames?: Record<string, string> | null, budgets?: Record<string, unknown> | null}} options
|
|
155
|
+
* @returns {Record<string, unknown>}
|
|
156
|
+
*/
|
|
157
|
+
function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterations, timeoutCount = 0, artifacts = {}, cycleEventNames = null, budgets = null, }) {
|
|
158
|
+
const resolvedCycleEventNames = {
|
|
159
|
+
openRequested: cycleEventNames?.openRequested ?? 'surface_open_requested',
|
|
160
|
+
opened: cycleEventNames?.opened ?? 'surface_opened',
|
|
161
|
+
closeRequested: cycleEventNames?.closeRequested ?? 'surface_close_requested',
|
|
162
|
+
dismissed: cycleEventNames?.dismissed ?? 'surface_dismissed',
|
|
163
|
+
milestone: cycleEventNames?.milestone,
|
|
164
|
+
};
|
|
165
|
+
const usesMilestoneOnlyCycle = typeof resolvedCycleEventNames.milestone === 'string';
|
|
166
|
+
const iterations = new Map();
|
|
167
|
+
for (const event of [...events].sort((left, right) => {
|
|
168
|
+
const leftAt = typeof left.atMs === 'number' ? left.atMs : Number.POSITIVE_INFINITY;
|
|
169
|
+
const rightAt = typeof right.atMs === 'number' ? right.atMs : Number.POSITIVE_INFINITY;
|
|
170
|
+
return leftAt - rightAt;
|
|
171
|
+
})) {
|
|
172
|
+
const eventIteration = typeof event.iteration === 'number'
|
|
173
|
+
? event.iteration
|
|
174
|
+
: expectedIterations === 1
|
|
175
|
+
? 1
|
|
176
|
+
: null;
|
|
177
|
+
if (eventIteration === null) {
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (typeof event.atMs !== 'number') {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
const current = iterations.get(eventIteration) ?? {};
|
|
184
|
+
if (event.event === resolvedCycleEventNames.openRequested &&
|
|
185
|
+
typeof current.presentRequestedAt !== 'number') {
|
|
186
|
+
current.presentRequestedAt = event.atMs;
|
|
187
|
+
}
|
|
188
|
+
if (event.event === resolvedCycleEventNames.opened &&
|
|
189
|
+
typeof current.openedAt !== 'number' &&
|
|
190
|
+
typeof current.presentRequestedAt === 'number' &&
|
|
191
|
+
event.atMs >= current.presentRequestedAt) {
|
|
192
|
+
current.openedAt = event.atMs;
|
|
193
|
+
}
|
|
194
|
+
if (event.event === resolvedCycleEventNames.closeRequested &&
|
|
195
|
+
typeof current.closeRequestedAt !== 'number') {
|
|
196
|
+
current.closeRequestedAt = event.atMs;
|
|
197
|
+
}
|
|
198
|
+
if (event.event === resolvedCycleEventNames.dismissed &&
|
|
199
|
+
typeof current.dismissedAt !== 'number' &&
|
|
200
|
+
typeof current.presentRequestedAt === 'number' &&
|
|
201
|
+
typeof current.closeRequestedAt === 'number' &&
|
|
202
|
+
event.atMs >= current.presentRequestedAt &&
|
|
203
|
+
event.atMs >= current.closeRequestedAt) {
|
|
204
|
+
current.dismissedAt = event.atMs;
|
|
205
|
+
}
|
|
206
|
+
if (event.event === resolvedCycleEventNames.milestone &&
|
|
207
|
+
typeof current.milestoneAt !== 'number') {
|
|
208
|
+
current.milestoneAt = event.atMs;
|
|
209
|
+
}
|
|
210
|
+
iterations.set(eventIteration, current);
|
|
211
|
+
}
|
|
212
|
+
const durationsMs = [];
|
|
213
|
+
const openDurationsMs = [];
|
|
214
|
+
const closeDurationsMs = [];
|
|
215
|
+
const incompleteIterations = [];
|
|
216
|
+
let failures = 0;
|
|
217
|
+
for (let iteration = 1; iteration <= expectedIterations; iteration += 1) {
|
|
218
|
+
const record = iterations.get(iteration);
|
|
219
|
+
if (!record) {
|
|
220
|
+
failures += 1;
|
|
221
|
+
incompleteIterations.push(iteration);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
const hasCycleDuration = typeof record.presentRequestedAt === 'number' &&
|
|
225
|
+
typeof record.dismissedAt === 'number' &&
|
|
226
|
+
record.dismissedAt >= record.presentRequestedAt;
|
|
227
|
+
const hasOpenDuration = typeof record.presentRequestedAt === 'number' &&
|
|
228
|
+
typeof record.openedAt === 'number' &&
|
|
229
|
+
record.openedAt >= record.presentRequestedAt;
|
|
230
|
+
const hasCloseDuration = typeof record.closeRequestedAt === 'number' &&
|
|
231
|
+
typeof record.dismissedAt === 'number' &&
|
|
232
|
+
record.dismissedAt >= record.closeRequestedAt;
|
|
233
|
+
const hasMilestoneDuration = usesMilestoneOnlyCycle &&
|
|
234
|
+
typeof record.milestoneAt === 'number' &&
|
|
235
|
+
record.milestoneAt >= 0;
|
|
236
|
+
if (hasMilestoneDuration) {
|
|
237
|
+
durationsMs.push(roundMs(record.milestoneAt));
|
|
238
|
+
openDurationsMs.push(roundMs(record.milestoneAt));
|
|
239
|
+
}
|
|
240
|
+
else if (hasCycleDuration) {
|
|
241
|
+
durationsMs.push(roundMs(record.dismissedAt - record.presentRequestedAt));
|
|
242
|
+
}
|
|
243
|
+
if (hasOpenDuration) {
|
|
244
|
+
openDurationsMs.push(roundMs(record.openedAt - record.presentRequestedAt));
|
|
245
|
+
}
|
|
246
|
+
if (hasCloseDuration) {
|
|
247
|
+
closeDurationsMs.push(roundMs(record.dismissedAt - record.closeRequestedAt));
|
|
248
|
+
}
|
|
249
|
+
const iterationComplete = usesMilestoneOnlyCycle
|
|
250
|
+
? hasMilestoneDuration
|
|
251
|
+
: hasCycleDuration && hasOpenDuration && hasCloseDuration;
|
|
252
|
+
if (!iterationComplete) {
|
|
253
|
+
failures += 1;
|
|
254
|
+
incompleteIterations.push(iteration);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
const metrics = {
|
|
258
|
+
scenario,
|
|
259
|
+
runId,
|
|
260
|
+
status: failures === 0 && timeoutCount === 0 ? 'passed' : 'failed',
|
|
261
|
+
iterations: expectedIterations,
|
|
262
|
+
durationsMs,
|
|
263
|
+
p50Ms: percentile(durationsMs, 50),
|
|
264
|
+
p95Ms: percentile(durationsMs, 95),
|
|
265
|
+
failures,
|
|
266
|
+
timeouts: timeoutCount,
|
|
267
|
+
openDurationsMs,
|
|
268
|
+
closeDurationsMs,
|
|
269
|
+
incompleteIterations,
|
|
270
|
+
artifacts: sortValue(artifacts),
|
|
271
|
+
};
|
|
272
|
+
const budgetEvaluation = evaluateProfileBudgets({ metrics, budgets });
|
|
273
|
+
if (budgetEvaluation) {
|
|
274
|
+
metrics.budgetEvaluation = sortValue(budgetEvaluation);
|
|
275
|
+
}
|
|
276
|
+
return metrics;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Evaluates one numeric budget threshold.
|
|
280
|
+
*
|
|
281
|
+
* @param {{name: string, actual: unknown, limit: unknown}} options
|
|
282
|
+
* @returns {{name: string, actual: unknown, limit: number, pass: boolean, unit: string} | null}
|
|
283
|
+
*/
|
|
284
|
+
function evaluateBudgetCheck({ name, actual, limit }) {
|
|
285
|
+
if (typeof limit !== 'number') {
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
const pass = typeof actual === 'number' && actual <= limit;
|
|
289
|
+
return {
|
|
290
|
+
name,
|
|
291
|
+
actual,
|
|
292
|
+
limit,
|
|
293
|
+
pass,
|
|
294
|
+
unit: 'ms',
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Evaluates configured profile budgets against generated metrics.
|
|
299
|
+
*
|
|
300
|
+
* @param {{metrics: Record<string, unknown>, budgets?: Record<string, unknown> | null}} options
|
|
301
|
+
* @returns {Record<string, unknown> | null}
|
|
302
|
+
*/
|
|
303
|
+
function evaluateProfileBudgets({ metrics, budgets }) {
|
|
304
|
+
if (!budgets?.pass || typeof budgets.pass !== 'object') {
|
|
305
|
+
return null;
|
|
306
|
+
}
|
|
307
|
+
const checks = [
|
|
308
|
+
evaluateBudgetCheck({
|
|
309
|
+
name: 'cycle p50',
|
|
310
|
+
actual: metrics.p50Ms,
|
|
311
|
+
limit: budgets.pass.cycleP50Ms,
|
|
312
|
+
}),
|
|
313
|
+
evaluateBudgetCheck({
|
|
314
|
+
name: 'cycle p95',
|
|
315
|
+
actual: metrics.p95Ms,
|
|
316
|
+
limit: budgets.pass.cycleP95Ms,
|
|
317
|
+
}),
|
|
318
|
+
evaluateBudgetCheck({
|
|
319
|
+
name: 'open p50',
|
|
320
|
+
actual: percentile(metrics.openDurationsMs, 50),
|
|
321
|
+
limit: budgets.pass.openP50Ms,
|
|
322
|
+
}),
|
|
323
|
+
evaluateBudgetCheck({
|
|
324
|
+
name: 'open p95',
|
|
325
|
+
actual: percentile(metrics.openDurationsMs, 95),
|
|
326
|
+
limit: budgets.pass.openP95Ms,
|
|
327
|
+
}),
|
|
328
|
+
evaluateBudgetCheck({
|
|
329
|
+
name: 'close p50',
|
|
330
|
+
actual: percentile(metrics.closeDurationsMs, 50),
|
|
331
|
+
limit: budgets.pass.closeP50Ms,
|
|
332
|
+
}),
|
|
333
|
+
evaluateBudgetCheck({
|
|
334
|
+
name: 'close p95',
|
|
335
|
+
actual: percentile(metrics.closeDurationsMs, 95),
|
|
336
|
+
limit: budgets.pass.closeP95Ms,
|
|
337
|
+
}),
|
|
338
|
+
evaluateBudgetCheck({
|
|
339
|
+
name: 'scroll p50',
|
|
340
|
+
actual: metrics.p50Ms,
|
|
341
|
+
limit: budgets.pass.scrollP50Ms,
|
|
342
|
+
}),
|
|
343
|
+
evaluateBudgetCheck({
|
|
344
|
+
name: 'scroll p95',
|
|
345
|
+
actual: metrics.p95Ms,
|
|
346
|
+
limit: budgets.pass.scrollP95Ms,
|
|
347
|
+
}),
|
|
348
|
+
evaluateBudgetCheck({
|
|
349
|
+
name: 'first visible p50',
|
|
350
|
+
actual: metrics.firstVisibleP50Ms,
|
|
351
|
+
limit: budgets.pass.firstVisibleP50Ms,
|
|
352
|
+
}),
|
|
353
|
+
evaluateBudgetCheck({
|
|
354
|
+
name: 'first visible p95',
|
|
355
|
+
actual: metrics.firstVisibleP95Ms,
|
|
356
|
+
limit: budgets.pass.firstVisibleP95Ms,
|
|
357
|
+
}),
|
|
358
|
+
].filter((check) => Boolean(check));
|
|
359
|
+
const thresholdChecks = [
|
|
360
|
+
typeof budgets.pass.failures === 'number'
|
|
361
|
+
? {
|
|
362
|
+
name: 'failures',
|
|
363
|
+
actual: metrics.failures,
|
|
364
|
+
limit: budgets.pass.failures,
|
|
365
|
+
pass: metrics.failures <= budgets.pass.failures,
|
|
366
|
+
unit: 'count',
|
|
367
|
+
}
|
|
368
|
+
: null,
|
|
369
|
+
typeof budgets.pass.timeouts === 'number'
|
|
370
|
+
? {
|
|
371
|
+
name: 'timeouts',
|
|
372
|
+
actual: metrics.timeouts,
|
|
373
|
+
limit: budgets.pass.timeouts,
|
|
374
|
+
pass: metrics.timeouts <= budgets.pass.timeouts,
|
|
375
|
+
unit: 'count',
|
|
376
|
+
}
|
|
377
|
+
: null,
|
|
378
|
+
].filter((check) => Boolean(check));
|
|
379
|
+
const allChecks = [...thresholdChecks, ...checks];
|
|
380
|
+
if (allChecks.length === 0) {
|
|
381
|
+
return null;
|
|
382
|
+
}
|
|
383
|
+
return {
|
|
384
|
+
metric: budgets.metric ?? metrics.measurement ?? 'profile budget',
|
|
385
|
+
pass: allChecks.every((check) => check.pass),
|
|
386
|
+
checks: allChecks,
|
|
387
|
+
failedChecks: allChecks.filter((check) => !check.pass).map((check) => check.name),
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Recursively sorts object keys and array values for stable JSON artifacts.
|
|
392
|
+
*
|
|
393
|
+
* @param {unknown} value
|
|
394
|
+
* @returns {unknown}
|
|
395
|
+
*/
|
|
396
|
+
function sortValue(value) {
|
|
397
|
+
if (Array.isArray(value)) {
|
|
398
|
+
return [...value].sort().map(sortValue);
|
|
399
|
+
}
|
|
400
|
+
if (value && typeof value === 'object') {
|
|
401
|
+
return Object.keys(value)
|
|
402
|
+
.sort()
|
|
403
|
+
.reduce((result, key) => {
|
|
404
|
+
result[key] = sortValue(value[key]);
|
|
405
|
+
return result;
|
|
406
|
+
}, {});
|
|
407
|
+
}
|
|
408
|
+
return value;
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Normalizes event timestamps to milliseconds since run start.
|
|
412
|
+
*
|
|
413
|
+
* @param {{event: Record<string, unknown>, startedAt?: string}} options
|
|
414
|
+
* @returns {number | null}
|
|
415
|
+
*/
|
|
416
|
+
function normalizeEventTimestamp({ event, startedAt }) {
|
|
417
|
+
if (!event || typeof event !== 'object') {
|
|
418
|
+
return null;
|
|
419
|
+
}
|
|
420
|
+
if (typeof event.atMs === 'number' && Number.isFinite(event.atMs)) {
|
|
421
|
+
return roundMs(event.atMs);
|
|
422
|
+
}
|
|
423
|
+
const eventTimestamp = typeof event.timestamp === 'number'
|
|
424
|
+
? event.timestamp
|
|
425
|
+
: typeof event.timestamp === 'string' && event.timestamp.length > 0
|
|
426
|
+
? Number(event.timestamp)
|
|
427
|
+
: null;
|
|
428
|
+
const startedAtEpochMs = typeof startedAt === 'string' && startedAt.length > 0 ? Date.parse(startedAt) : Number.NaN;
|
|
429
|
+
if (eventTimestamp !== null && Number.isFinite(eventTimestamp) && Number.isFinite(startedAtEpochMs)) {
|
|
430
|
+
return roundMs(Math.max(0, eventTimestamp - startedAtEpochMs));
|
|
431
|
+
}
|
|
432
|
+
return null;
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Infers a causal timeline phase from an event name when no explicit phase is provided.
|
|
436
|
+
*
|
|
437
|
+
* @param {unknown} eventName
|
|
438
|
+
* @returns {string}
|
|
439
|
+
*/
|
|
440
|
+
function inferTimelinePhase(eventName) {
|
|
441
|
+
if (typeof eventName !== 'string' || eventName.length === 0) {
|
|
442
|
+
return 'domain';
|
|
443
|
+
}
|
|
444
|
+
const normalized = eventName.toLowerCase();
|
|
445
|
+
if (normalized.includes('requested') ||
|
|
446
|
+
normalized.includes('tapped') ||
|
|
447
|
+
normalized.includes('tap') ||
|
|
448
|
+
normalized.includes('intent')) {
|
|
449
|
+
return 'intent';
|
|
450
|
+
}
|
|
451
|
+
if (normalized.includes('route') ||
|
|
452
|
+
normalized.includes('presented') ||
|
|
453
|
+
normalized.includes('dismissed') ||
|
|
454
|
+
normalized.includes('opened')) {
|
|
455
|
+
return 'navigation';
|
|
456
|
+
}
|
|
457
|
+
if (normalized.includes('query') || normalized.includes('cache')) {
|
|
458
|
+
return 'query';
|
|
459
|
+
}
|
|
460
|
+
if (normalized.includes('request') ||
|
|
461
|
+
normalized.includes('response') ||
|
|
462
|
+
normalized.includes('network')) {
|
|
463
|
+
return 'network';
|
|
464
|
+
}
|
|
465
|
+
if (normalized.includes('render') ||
|
|
466
|
+
normalized.includes('mounted') ||
|
|
467
|
+
normalized.includes('shell_ready')) {
|
|
468
|
+
return 'render';
|
|
469
|
+
}
|
|
470
|
+
if (normalized.includes('native') ||
|
|
471
|
+
normalized.includes('frame') ||
|
|
472
|
+
normalized.includes('hitch')) {
|
|
473
|
+
return 'native';
|
|
474
|
+
}
|
|
475
|
+
if (normalized.includes('visible') ||
|
|
476
|
+
normalized.includes('usable') ||
|
|
477
|
+
normalized.includes('committed')) {
|
|
478
|
+
return 'visual';
|
|
479
|
+
}
|
|
480
|
+
if (normalized.includes('finished') ||
|
|
481
|
+
normalized.includes('completed') ||
|
|
482
|
+
normalized.includes('settled')) {
|
|
483
|
+
return 'completion';
|
|
484
|
+
}
|
|
485
|
+
return 'domain';
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Infers timeline status from an event name when no explicit status is provided.
|
|
489
|
+
*
|
|
490
|
+
* @param {unknown} eventName
|
|
491
|
+
* @returns {string}
|
|
492
|
+
*/
|
|
493
|
+
function inferTimelineStatus(eventName) {
|
|
494
|
+
if (typeof eventName !== 'string' || eventName.length === 0) {
|
|
495
|
+
return 'observed';
|
|
496
|
+
}
|
|
497
|
+
const normalized = eventName.toLowerCase();
|
|
498
|
+
if (normalized.includes('failed')) {
|
|
499
|
+
return 'failed';
|
|
500
|
+
}
|
|
501
|
+
if (normalized.includes('requested') || normalized.includes('started')) {
|
|
502
|
+
return 'started';
|
|
503
|
+
}
|
|
504
|
+
if (normalized.includes('opened') ||
|
|
505
|
+
normalized.includes('dismissed') ||
|
|
506
|
+
normalized.includes('committed') ||
|
|
507
|
+
normalized.includes('visible') ||
|
|
508
|
+
normalized.includes('ready') ||
|
|
509
|
+
normalized.includes('settled') ||
|
|
510
|
+
normalized.includes('completed') ||
|
|
511
|
+
normalized.includes('finished')) {
|
|
512
|
+
return 'completed';
|
|
513
|
+
}
|
|
514
|
+
return 'observed';
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Builds a causal timeline from app-owned profile events.
|
|
518
|
+
*
|
|
519
|
+
* @param {{events: Record<string, unknown>[], startedAt?: string, phaseMap?: Record<string, string> | null, owner?: string | null}} options
|
|
520
|
+
* @returns {Record<string, unknown>[]}
|
|
521
|
+
*/
|
|
522
|
+
function buildCausalTimeline({ events, startedAt, phaseMap = null, owner = null, }) {
|
|
523
|
+
return [...(Array.isArray(events) ? events : [])]
|
|
524
|
+
.map((event) => {
|
|
525
|
+
if (!event || typeof event !== 'object' || typeof event.event !== 'string') {
|
|
526
|
+
return null;
|
|
527
|
+
}
|
|
528
|
+
const atMs = normalizeEventTimestamp({
|
|
529
|
+
event,
|
|
530
|
+
...(typeof startedAt === 'string' ? { startedAt } : {}),
|
|
531
|
+
});
|
|
532
|
+
if (atMs === null) {
|
|
533
|
+
return null;
|
|
534
|
+
}
|
|
535
|
+
const eventMetadata = event.metadata && typeof event.metadata === 'object' && !Array.isArray(event.metadata)
|
|
536
|
+
? event.metadata
|
|
537
|
+
: {};
|
|
538
|
+
const explicitPhase = typeof event.phase === 'string'
|
|
539
|
+
? event.phase
|
|
540
|
+
: phaseMap && typeof phaseMap[event.event] === 'string'
|
|
541
|
+
? phaseMap[event.event]
|
|
542
|
+
: null;
|
|
543
|
+
const metadata = {
|
|
544
|
+
...eventMetadata,
|
|
545
|
+
...(typeof event.flowId === 'string' ? { flowId: event.flowId } : {}),
|
|
546
|
+
...(typeof event.route === 'string' ? { route: event.route } : {}),
|
|
547
|
+
...(typeof event.iteration === 'number' ? { iteration: event.iteration } : {}),
|
|
548
|
+
};
|
|
549
|
+
return sortValue({
|
|
550
|
+
phase: explicitPhase ?? inferTimelinePhase(event.event),
|
|
551
|
+
name: event.event,
|
|
552
|
+
atMs,
|
|
553
|
+
status: typeof event.status === 'string' ? event.status : inferTimelineStatus(event.event),
|
|
554
|
+
...((typeof event.owner === 'string' && event.owner.length > 0) || owner
|
|
555
|
+
? { owner: event.owner || owner }
|
|
556
|
+
: {}),
|
|
557
|
+
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
|
558
|
+
});
|
|
559
|
+
})
|
|
560
|
+
.filter(Boolean)
|
|
561
|
+
.sort((left, right) => left.atMs - right.atMs);
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Builds the `budget-verdict.json` profile artifact from budget evaluation.
|
|
565
|
+
*
|
|
566
|
+
* @param {{flowId: string, runId: string, budgetEvaluation?: Record<string, unknown> | null, visualOutcome?: Record<string, unknown> | null, baselineRunId?: string | null}} options
|
|
567
|
+
* @returns {Record<string, unknown> | null}
|
|
568
|
+
*/
|
|
569
|
+
function buildBudgetVerdict({ flowId, runId, budgetEvaluation, visualOutcome = null, baselineRunId = null, }) {
|
|
570
|
+
if (!budgetEvaluation) {
|
|
571
|
+
return null;
|
|
572
|
+
}
|
|
573
|
+
const hasManualVisualReview = visualOutcome &&
|
|
574
|
+
Array.isArray(visualOutcome.checks) &&
|
|
575
|
+
visualOutcome.checks.some((check) => check.status === 'manual-review-needed');
|
|
576
|
+
return sortValue({
|
|
577
|
+
schemaVersion: '1.0.0',
|
|
578
|
+
flowId,
|
|
579
|
+
runId,
|
|
580
|
+
status: hasManualVisualReview
|
|
581
|
+
? 'partial'
|
|
582
|
+
: budgetEvaluation.pass
|
|
583
|
+
? 'passed'
|
|
584
|
+
: 'failed',
|
|
585
|
+
checks: (budgetEvaluation.checks ?? []).map((check) => ({
|
|
586
|
+
name: check.name,
|
|
587
|
+
metric: budgetEvaluation.metric ?? 'profile budget',
|
|
588
|
+
unit: check.unit,
|
|
589
|
+
expected: check.limit,
|
|
590
|
+
actual: check.actual ?? null,
|
|
591
|
+
pass: check.pass,
|
|
592
|
+
})),
|
|
593
|
+
...(visualOutcome ? { visualOutcome } : {}),
|
|
594
|
+
...(baselineRunId
|
|
595
|
+
? {
|
|
596
|
+
regression: {
|
|
597
|
+
baselineRunId,
|
|
598
|
+
status: 'unknown',
|
|
599
|
+
summary: 'Regression status is unknown until comparison.json is produced for the baseline/current run folders.',
|
|
600
|
+
},
|
|
601
|
+
}
|
|
602
|
+
: {}),
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Infers the display unit for legacy budget keys.
|
|
607
|
+
*
|
|
608
|
+
* @param {unknown} name
|
|
609
|
+
* @returns {string}
|
|
610
|
+
*/
|
|
611
|
+
function inferBudgetUnit(name) {
|
|
612
|
+
if (typeof name !== 'string') {
|
|
613
|
+
return 'count';
|
|
614
|
+
}
|
|
615
|
+
if (/ms$/u.test(name) || /p50|p95|duration|cycle|visible|open|close/u.test(name)) {
|
|
616
|
+
return 'ms';
|
|
617
|
+
}
|
|
618
|
+
return 'count';
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Normalizes legacy budget config into the causal-run budget shape.
|
|
622
|
+
*
|
|
623
|
+
* @param {Record<string, unknown> | null | undefined} budgets
|
|
624
|
+
* @returns {Record<string, unknown>}
|
|
625
|
+
*/
|
|
626
|
+
function normalizeBudgetsForCausalRun(budgets) {
|
|
627
|
+
if (!budgets || typeof budgets !== 'object') {
|
|
628
|
+
return {};
|
|
629
|
+
}
|
|
630
|
+
return Object.keys(budgets).reduce((result, key) => {
|
|
631
|
+
const limit = budgets[key];
|
|
632
|
+
if (typeof limit !== 'number' && typeof limit !== 'boolean') {
|
|
633
|
+
return result;
|
|
634
|
+
}
|
|
635
|
+
result[key] = {
|
|
636
|
+
metric: key,
|
|
637
|
+
unit: inferBudgetUnit(key),
|
|
638
|
+
limit,
|
|
639
|
+
};
|
|
640
|
+
return result;
|
|
641
|
+
}, {});
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Builds the `causal-run.json` profile artifact.
|
|
645
|
+
*
|
|
646
|
+
* @param {Record<string, unknown>} options
|
|
647
|
+
* @returns {Record<string, unknown>}
|
|
648
|
+
*/
|
|
649
|
+
function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor = 'unknown', interactionDriver, trigger = null, budgets = null, timeline = [], artifacts, manifest, metrics, }) {
|
|
650
|
+
return sortValue({
|
|
651
|
+
schemaVersion: '1.0.0',
|
|
652
|
+
flowId,
|
|
653
|
+
runId,
|
|
654
|
+
platform,
|
|
655
|
+
buildFlavor,
|
|
656
|
+
scenario: {
|
|
657
|
+
id: scenario.name,
|
|
658
|
+
driver: interactionDriver,
|
|
659
|
+
...(typeof metrics?.iterations === 'number' ? { iterations: metrics.iterations } : {}),
|
|
660
|
+
},
|
|
661
|
+
trigger: trigger && typeof trigger === 'object'
|
|
662
|
+
? trigger
|
|
663
|
+
: {
|
|
664
|
+
kind: 'unknown',
|
|
665
|
+
label: scenario.description ?? scenario.name,
|
|
666
|
+
},
|
|
667
|
+
budgets: normalizeBudgetsForCausalRun(budgets),
|
|
668
|
+
timeline,
|
|
669
|
+
artifacts: {
|
|
670
|
+
summary: artifacts.summary,
|
|
671
|
+
metrics: artifacts.metrics,
|
|
672
|
+
manifest: artifacts.manifest,
|
|
673
|
+
video: artifacts.captures?.video,
|
|
674
|
+
screenshot: Array.isArray(artifacts.captures?.screenshots)
|
|
675
|
+
? artifacts.captures.screenshots[0] ?? null
|
|
676
|
+
: null,
|
|
677
|
+
signals: artifacts.signals,
|
|
678
|
+
evidenceAttachments: Array.isArray(artifacts.evidenceAttachments) ? artifacts.evidenceAttachments : [],
|
|
679
|
+
},
|
|
680
|
+
notes: [
|
|
681
|
+
`Manifest status: ${manifest.status}`,
|
|
682
|
+
`Metrics status: ${metrics.status}`,
|
|
683
|
+
],
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Builds the run manifest artifact.
|
|
688
|
+
*
|
|
689
|
+
* @param {Record<string, unknown>} options
|
|
690
|
+
* @returns {Record<string, unknown>}
|
|
691
|
+
*/
|
|
692
|
+
function buildManifest({ scenario, scenarioHash, runId, platform = 'ios', status, startedAt, endedAt, interactionDriver, comparisonLane, simulator, bundleId, gitSha, toolVersions, artifacts, failureReason = null, }) {
|
|
693
|
+
return {
|
|
694
|
+
scenario,
|
|
695
|
+
...(typeof scenarioHash === 'string' && scenarioHash.length > 0 ? { scenarioHash } : {}),
|
|
696
|
+
runId,
|
|
697
|
+
platform,
|
|
698
|
+
status,
|
|
699
|
+
startedAt,
|
|
700
|
+
endedAt,
|
|
701
|
+
durationMs: roundMs(Math.max(0, Date.parse(endedAt) - Date.parse(startedAt))),
|
|
702
|
+
interactionDriver,
|
|
703
|
+
...(typeof comparisonLane === 'string' && comparisonLane.length > 0 ? { comparisonLane } : {}),
|
|
704
|
+
simulator: sortValue(simulator),
|
|
705
|
+
bundleId,
|
|
706
|
+
gitSha,
|
|
707
|
+
toolVersions: sortValue(toolVersions),
|
|
708
|
+
artifacts: sortValue(artifacts),
|
|
709
|
+
failureReason,
|
|
710
|
+
};
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Builds the human-readable profile summary.
|
|
714
|
+
*
|
|
715
|
+
* @param {{manifest: Record<string, unknown>, metrics: Record<string, unknown>}} options
|
|
716
|
+
* @returns {string}
|
|
717
|
+
*/
|
|
718
|
+
function buildSummaryMarkdown({ manifest, metrics }) {
|
|
719
|
+
const runtimeLabel = manifest.platform === 'android' ? 'Device' : 'Simulator';
|
|
720
|
+
const signalLines = [
|
|
721
|
+
`- JS: ${manifest.artifacts.signals.js.length > 0
|
|
722
|
+
? manifest.artifacts.signals.js.map((item) => `\`${item}\``).join(', ')
|
|
723
|
+
: 'none'}`,
|
|
724
|
+
`- Memory: ${manifest.artifacts.signals.memory.length > 0
|
|
725
|
+
? manifest.artifacts.signals.memory.map((item) => `\`${item}\``).join(', ')
|
|
726
|
+
: 'none'}`,
|
|
727
|
+
`- Network: ${manifest.artifacts.signals.network.length > 0
|
|
728
|
+
? manifest.artifacts.signals.network.map((item) => `\`${item}\``).join(', ')
|
|
729
|
+
: 'none'}`,
|
|
730
|
+
];
|
|
731
|
+
const screenshots = Array.isArray(manifest.artifacts.captures.screenshots)
|
|
732
|
+
? manifest.artifacts.captures.screenshots
|
|
733
|
+
: [];
|
|
734
|
+
const evidenceAttachments = Array.isArray(manifest.artifacts.evidenceAttachments)
|
|
735
|
+
? manifest.artifacts.evidenceAttachments
|
|
736
|
+
: [];
|
|
737
|
+
const evidenceAttachmentLines = evidenceAttachments.length > 0
|
|
738
|
+
? evidenceAttachments.map((attachment) => `- ${attachment.channel}/${attachment.kind}: \`${attachment.path}\` (${attachment.sizeBytes} bytes, sha256 ${attachment.sha256})`)
|
|
739
|
+
: ['- none'];
|
|
740
|
+
const lines = [
|
|
741
|
+
`# ${String(manifest.platform || 'ios').toUpperCase()} profile run: ${manifest.scenario}`,
|
|
742
|
+
'',
|
|
743
|
+
`- Status: ${manifest.status}`,
|
|
744
|
+
`- Run ID: \`${manifest.runId}\``,
|
|
745
|
+
`- Interaction driver: \`${manifest.interactionDriver}\``,
|
|
746
|
+
...(typeof manifest.comparisonLane === 'string'
|
|
747
|
+
? [`- Comparison lane: \`${manifest.comparisonLane}\``]
|
|
748
|
+
: []),
|
|
749
|
+
`- ${runtimeLabel}: ${manifest.simulator.name} (${manifest.simulator.udid})`,
|
|
750
|
+
`- Bundle ID: \`${manifest.bundleId}\``,
|
|
751
|
+
`- Iterations: ${metrics.iterations}`,
|
|
752
|
+
`- Completed cycles: ${metrics.durationsMs.length}/${metrics.iterations}`,
|
|
753
|
+
`- Failures: ${metrics.failures}`,
|
|
754
|
+
`- Timeouts: ${metrics.timeouts}`,
|
|
755
|
+
`- p50 cycle: ${metrics.p50Ms === null ? 'n/a' : `${metrics.p50Ms}ms`}`,
|
|
756
|
+
`- p95 cycle: ${metrics.p95Ms === null ? 'n/a' : `${metrics.p95Ms}ms`}`,
|
|
757
|
+
'',
|
|
758
|
+
'## Artifact paths',
|
|
759
|
+
'',
|
|
760
|
+
`- Causal run: \`${manifest.artifacts.causalRun}\``,
|
|
761
|
+
`- Budget verdict: ${metrics.budgetEvaluation ? `\`${manifest.artifacts.budgetVerdict}\`` : 'none (no budgets configured)'}`,
|
|
762
|
+
`- Manifest: \`${manifest.artifacts.manifest}\``,
|
|
763
|
+
`- Scenario: \`${manifest.artifacts.scenario}\``,
|
|
764
|
+
`- Metrics: \`${manifest.artifacts.metrics}\``,
|
|
765
|
+
`- Interaction log: \`${manifest.artifacts.raw.interactionLog}\``,
|
|
766
|
+
`- Device log: \`${manifest.artifacts.raw.deviceLog}\``,
|
|
767
|
+
`- Video: \`${manifest.artifacts.captures.video}\``,
|
|
768
|
+
`- UI tree: \`${manifest.artifacts.captures.uiTree}\``,
|
|
769
|
+
`- Screenshots: ${screenshots.length > 0
|
|
770
|
+
? screenshots.map((item) => `\`${item}\``).join(', ')
|
|
771
|
+
: 'none'}`,
|
|
772
|
+
'',
|
|
773
|
+
'## Signal attachments',
|
|
774
|
+
'',
|
|
775
|
+
...signalLines,
|
|
776
|
+
'',
|
|
777
|
+
'## Evidence attachments',
|
|
778
|
+
'',
|
|
779
|
+
...evidenceAttachmentLines,
|
|
780
|
+
];
|
|
781
|
+
if (metrics.budgetEvaluation) {
|
|
782
|
+
lines.push('', '## Budget', '', `- Metric: ${metrics.budgetEvaluation.metric}`, `- Status: ${metrics.budgetEvaluation.pass ? 'pass' : 'fail'}`);
|
|
783
|
+
for (const check of metrics.budgetEvaluation.checks) {
|
|
784
|
+
const formatValue = (value) => {
|
|
785
|
+
if (typeof value !== 'number') {
|
|
786
|
+
return value ?? 'n/a';
|
|
787
|
+
}
|
|
788
|
+
return check.unit === 'ms' ? `${value}ms` : String(value);
|
|
789
|
+
};
|
|
790
|
+
lines.push(`- ${check.name}: ${formatValue(check.actual)} / ${formatValue(check.limit)}`);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
if (metrics.openDurationsMs.length > 0) {
|
|
794
|
+
lines.push('', `- Open durations (ms): ${metrics.openDurationsMs.join(', ')}`);
|
|
795
|
+
}
|
|
796
|
+
if (metrics.closeDurationsMs.length > 0) {
|
|
797
|
+
lines.push(`- Close durations (ms): ${metrics.closeDurationsMs.join(', ')}`);
|
|
798
|
+
}
|
|
799
|
+
if (metrics.incompleteIterations.length > 0) {
|
|
800
|
+
lines.push(`- Incomplete iterations: ${metrics.incompleteIterations.join(', ')}`);
|
|
801
|
+
}
|
|
802
|
+
if (manifest.failureReason) {
|
|
803
|
+
lines.push('', '## Failure', '', manifest.failureReason);
|
|
804
|
+
}
|
|
805
|
+
return lines.join('\n');
|
|
806
|
+
}
|
|
807
|
+
/**
|
|
808
|
+
* Extracts possible accessibility or test identifiers from a UI tree dump.
|
|
809
|
+
*
|
|
810
|
+
* @param {string} rawDescription
|
|
811
|
+
* @returns {string[]}
|
|
812
|
+
*/
|
|
813
|
+
function extractCandidateIdentifiers(rawDescription) {
|
|
814
|
+
const seen = new Set();
|
|
815
|
+
const identifiers = [];
|
|
816
|
+
const push = (value) => {
|
|
817
|
+
if (typeof value !== 'string') {
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
const normalized = value.trim();
|
|
821
|
+
if (!normalized || seen.has(normalized)) {
|
|
822
|
+
return;
|
|
823
|
+
}
|
|
824
|
+
seen.add(normalized);
|
|
825
|
+
identifiers.push(normalized);
|
|
826
|
+
};
|
|
827
|
+
const visit = (value) => {
|
|
828
|
+
if (Array.isArray(value)) {
|
|
829
|
+
for (const item of value) {
|
|
830
|
+
visit(item);
|
|
831
|
+
}
|
|
832
|
+
return;
|
|
833
|
+
}
|
|
834
|
+
if (!value || typeof value !== 'object') {
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
for (const [key, child] of Object.entries(value)) {
|
|
838
|
+
if (key === 'id' ||
|
|
839
|
+
key === 'identifier' ||
|
|
840
|
+
key === 'accessibilityIdentifier' ||
|
|
841
|
+
key === 'AXUniqueId' ||
|
|
842
|
+
key === 'AXLabel' ||
|
|
843
|
+
key === 'testID' ||
|
|
844
|
+
key === 'testId') {
|
|
845
|
+
push(child);
|
|
846
|
+
}
|
|
847
|
+
visit(child);
|
|
848
|
+
}
|
|
849
|
+
};
|
|
850
|
+
try {
|
|
851
|
+
visit(JSON.parse(rawDescription));
|
|
852
|
+
}
|
|
853
|
+
catch {
|
|
854
|
+
// Fall back to a raw regex scan when the driver output is not JSON.
|
|
855
|
+
}
|
|
856
|
+
const matches = String(rawDescription).match(/[A-Za-z0-9][A-Za-z0-9._:-]{2,}/gu) ?? [];
|
|
857
|
+
for (const match of matches) {
|
|
858
|
+
push(match);
|
|
859
|
+
}
|
|
860
|
+
return identifiers;
|
|
861
|
+
}
|
|
862
|
+
/**
|
|
863
|
+
* Finds the first UI identifier matching a required pattern.
|
|
864
|
+
*
|
|
865
|
+
* @param {string} rawDescription
|
|
866
|
+
* @param {string} pattern
|
|
867
|
+
* @returns {string | null}
|
|
868
|
+
*/
|
|
869
|
+
function findMatchingIdentifier(rawDescription, pattern) {
|
|
870
|
+
const regex = new RegExp(pattern, 'u');
|
|
871
|
+
const identifier = extractCandidateIdentifiers(rawDescription).find((candidate) => regex.test(candidate));
|
|
872
|
+
if (identifier) {
|
|
873
|
+
return identifier;
|
|
874
|
+
}
|
|
875
|
+
return String(rawDescription).match(regex)?.[0] ?? null;
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Evaluates a UI contract from required identifier patterns.
|
|
879
|
+
*
|
|
880
|
+
* @param {{rawDescription: string, requiredIdentifierPatterns?: string[]}} options
|
|
881
|
+
* @returns {{pass: boolean, checks: Record<string, unknown>[], missingPatterns: string[]}}
|
|
882
|
+
*/
|
|
883
|
+
function evaluateUiContract({ rawDescription, requiredIdentifierPatterns = [], }) {
|
|
884
|
+
const checks = requiredIdentifierPatterns.map((pattern) => {
|
|
885
|
+
const matchedIdentifier = findMatchingIdentifier(rawDescription, pattern);
|
|
886
|
+
return {
|
|
887
|
+
pattern,
|
|
888
|
+
pass: Boolean(matchedIdentifier),
|
|
889
|
+
matchedIdentifier,
|
|
890
|
+
};
|
|
891
|
+
});
|
|
892
|
+
return {
|
|
893
|
+
pass: checks.every((check) => check.pass),
|
|
894
|
+
checks,
|
|
895
|
+
missingPatterns: checks.filter((check) => !check.pass).map((check) => check.pattern),
|
|
896
|
+
};
|
|
897
|
+
}
|