agent-scenario-loop 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/app/profile-session.ts +352 -12
- package/dist/core/agent-summary.d.ts +3 -2
- package/dist/core/agent-summary.js +44 -2
- package/dist/core/artifact-contract.d.ts +28 -8
- package/dist/core/artifact-contract.js +676 -26
- package/dist/core/comparison.d.ts +57 -3
- package/dist/core/comparison.js +113 -1
- package/dist/core/planner.d.ts +32 -1
- package/dist/core/planner.js +144 -0
- package/dist/core/run-index.d.ts +4 -0
- package/dist/core/run-index.js +55 -1
- package/dist/core/schema-validator.d.ts +2 -0
- package/dist/core/schema-validator.js +2 -0
- package/dist/runner/android-adb-driver.d.ts +7 -2
- package/dist/runner/android-adb-driver.js +7 -1
- package/dist/runner/android-adb.d.ts +40 -5
- package/dist/runner/android-adb.js +1046 -664
- package/dist/runner/compare-latest.d.ts +8 -4
- package/dist/runner/compare-latest.js +24 -5
- package/dist/runner/example-android-live.d.ts +10 -1
- package/dist/runner/example-android-live.js +55 -0
- package/dist/runner/example-ios-live.d.ts +10 -1
- package/dist/runner/example-ios-live.js +55 -0
- package/dist/runner/ios-simctl.d.ts +6 -0
- package/dist/runner/ios-simctl.js +7 -0
- package/dist/runner/live-comparison.d.ts +2 -2
- package/dist/runner/live-comparison.js +2 -1
- package/dist/runner/live-proof-summary.d.ts +5 -4
- package/dist/runner/live-proof-summary.js +12 -2
- package/dist/runner/live-proof.d.ts +3 -2
- package/dist/runner/live-proof.js +9 -2
- package/dist/runner/profile-android.d.ts +16 -1
- package/dist/runner/profile-android.js +364 -26
- package/dist/runner/profile-ios.d.ts +13 -2
- package/dist/runner/profile-ios.js +341 -19
- package/dist/runner/profile-mobile.d.ts +39 -3
- package/dist/runner/profile-mobile.js +1054 -42
- package/dist/runner/validate-project.js +3 -0
- package/dist/scripts/consumer-rehearsal.d.ts +119 -0
- package/dist/scripts/consumer-rehearsal.js +757 -0
- package/dist/scripts/downstream-local-package-gate.d.ts +2 -0
- package/dist/scripts/downstream-local-package-gate.js +264 -0
- package/dist/scripts/package-smoke.d.ts +96 -0
- package/dist/scripts/package-smoke.js +2282 -0
- package/dist/scripts/release-readiness.d.ts +2 -0
- package/dist/scripts/release-readiness.js +520 -0
- package/docs/adapters.md +7 -1
- package/docs/api.md +2 -2
- package/docs/architecture.md +90 -0
- package/docs/authoring.md +39 -3
- package/docs/concepts.md +3 -24
- package/docs/consumer-rehearsal.md +31 -1
- package/docs/contracts.md +45 -101
- package/docs/external-adapter-protocol.md +219 -0
- package/docs/live-proofs.md +86 -3
- package/docs/principles.md +9 -15
- package/examples/mobile-app/README.md +12 -0
- package/examples/mobile-app/runner-manifests/evidence-provider.json +3 -3
- package/examples/mobile-app/runner-manifests/primary-runner.json +1 -0
- package/examples/mobile-app/scripts/asl-capture-profiler-provider.mjs +25 -0
- package/examples/runners/README.md +4 -3
- package/examples/runners/adb-android.json +1 -0
- package/examples/runners/agent-device-android.json +1 -0
- package/examples/runners/agent-device-ios.json +1 -0
- package/examples/runners/argent-android.json +1 -0
- package/examples/runners/argent-ios.json +1 -0
- package/examples/runners/axe-accessibility-provider.json +2 -2
- package/examples/runners/script-accessibility-provider.json +2 -2
- package/examples/runners/script-memory-provider.json +2 -2
- package/examples/runners/script-network-provider.json +2 -2
- package/examples/runners/script-profiler-provider.json +2 -2
- package/examples/runners/xcodebuildmcp-ios.json +1 -0
- package/package.json +12 -3
- package/schemas/causal-run.schema.json +85 -2
- package/schemas/comparison.schema.json +130 -2
- package/schemas/external-adapter-message.schema.json +693 -0
- package/schemas/health.schema.json +72 -0
- package/schemas/live-proof-set.schema.json +1 -1
- package/schemas/live-proof.schema.json +14 -6
- package/schemas/manifest.schema.json +515 -4
- package/schemas/profiler.schema.json +243 -0
- package/schemas/runner-capabilities.schema.json +28 -2
- package/schemas/scenario.schema.json +34 -2
- package/templates/evidence-provider.json +3 -3
- package/templates/primary-runner.json +1 -0
- package/templates/scripts/asl-capture-profiler-provider.mjs +20 -0
|
@@ -11,11 +11,75 @@ exports.evaluateUiContract = evaluateUiContract;
|
|
|
11
11
|
exports.evaluateProfileBudgets = evaluateProfileBudgets;
|
|
12
12
|
exports.extractCandidateIdentifiers = extractCandidateIdentifiers;
|
|
13
13
|
exports.extractProfileEvents = extractProfileEvents;
|
|
14
|
+
exports.extractProfileSessionEntries = extractProfileSessionEntries;
|
|
14
15
|
exports.findMatchingIdentifier = findMatchingIdentifier;
|
|
15
16
|
exports.percentile = percentile;
|
|
16
17
|
exports.sortValue = sortValue;
|
|
18
|
+
const crypto = require('node:crypto');
|
|
17
19
|
const PROFILE_EVENT_PREFIX = '[profile-event]';
|
|
18
20
|
exports.PROFILE_EVENT_PREFIX = PROFILE_EVENT_PREFIX;
|
|
21
|
+
const PROFILE_SESSION_PREFIX = '[profile-session]';
|
|
22
|
+
const CAUSAL_TIMELINE_PHASES = new Set([
|
|
23
|
+
'intent',
|
|
24
|
+
'navigation',
|
|
25
|
+
'domain',
|
|
26
|
+
'query',
|
|
27
|
+
'network',
|
|
28
|
+
'render',
|
|
29
|
+
'native',
|
|
30
|
+
'visual',
|
|
31
|
+
'completion',
|
|
32
|
+
]);
|
|
33
|
+
const CAUSAL_TIMELINE_STATUSES = new Set([
|
|
34
|
+
'started',
|
|
35
|
+
'completed',
|
|
36
|
+
'failed',
|
|
37
|
+
'skipped',
|
|
38
|
+
'observed',
|
|
39
|
+
]);
|
|
40
|
+
const UNKNOWN_LIFECYCLE_ASSERTION = Object.freeze({
|
|
41
|
+
value: 'unknown',
|
|
42
|
+
evidence: 'not-asserted',
|
|
43
|
+
});
|
|
44
|
+
const DEFAULT_ENVIRONMENT_PRECONDITIONS = Object.freeze({
|
|
45
|
+
installedState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
46
|
+
appDataState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
47
|
+
authState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
48
|
+
initialRoute: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
49
|
+
foregroundState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
50
|
+
lifecyclePhase: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
51
|
+
deviceLockState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
52
|
+
permissions: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
53
|
+
locale: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
54
|
+
timezone: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
55
|
+
theme: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
56
|
+
fontScale: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
57
|
+
orientation: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
58
|
+
networkState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
59
|
+
animations: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
60
|
+
});
|
|
61
|
+
const DEFAULT_ENVIRONMENT_POSTCONDITIONS = Object.freeze({
|
|
62
|
+
appState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
63
|
+
lifecyclePhase: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
64
|
+
cleanupState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
65
|
+
dataState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
66
|
+
artifactState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
67
|
+
});
|
|
68
|
+
const FAILURE_TERMINAL_STATES = new Set([
|
|
69
|
+
'failed',
|
|
70
|
+
'timeout',
|
|
71
|
+
'cancelled',
|
|
72
|
+
'aborted',
|
|
73
|
+
'inconclusive',
|
|
74
|
+
'unsupported',
|
|
75
|
+
'unhealthy',
|
|
76
|
+
]);
|
|
77
|
+
const TERMINAL_CLASSIFICATION_CATEGORIES = Object.freeze({
|
|
78
|
+
timeout: 'timeout',
|
|
79
|
+
cancelled: 'cancelled',
|
|
80
|
+
aborted: 'cancelled',
|
|
81
|
+
unsupported: 'runner',
|
|
82
|
+
});
|
|
19
83
|
/**
|
|
20
84
|
* Converts finite numeric strings to numbers while preserving invalid input as `null`.
|
|
21
85
|
*
|
|
@@ -72,6 +136,58 @@ function parseKeyValueProfileEvent(payload) {
|
|
|
72
136
|
}
|
|
73
137
|
return event;
|
|
74
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* Parses key/value `[profile-session]` payloads into structured session entries.
|
|
141
|
+
*
|
|
142
|
+
* @param {string} payload
|
|
143
|
+
* @returns {Record<string, unknown> | null}
|
|
144
|
+
*/
|
|
145
|
+
function parseKeyValueProfileSessionEntry(payload) {
|
|
146
|
+
const matches = payload.match(/(?:[^\s=]+)=(?:"[^"]*"|'[^']*'|[^\s]+)/gu) ?? [];
|
|
147
|
+
if (matches.length === 0) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const entry = {};
|
|
151
|
+
for (const match of matches) {
|
|
152
|
+
const separatorIndex = match.indexOf('=');
|
|
153
|
+
if (separatorIndex <= 0) {
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
const key = match.slice(0, separatorIndex);
|
|
157
|
+
let value = match.slice(separatorIndex + 1);
|
|
158
|
+
if ((value.startsWith('"') && value.endsWith('"')) ||
|
|
159
|
+
(value.startsWith("'") && value.endsWith("'"))) {
|
|
160
|
+
value = value.slice(1, -1);
|
|
161
|
+
}
|
|
162
|
+
entry[key] = value;
|
|
163
|
+
}
|
|
164
|
+
if (typeof entry.kind !== 'string' ||
|
|
165
|
+
typeof entry.scenario !== 'string' ||
|
|
166
|
+
typeof entry.runId !== 'string') {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
const timestamp = coerceNumber(entry.timestamp);
|
|
170
|
+
const atMs = coerceNumber(entry.atMs);
|
|
171
|
+
const sequence = coerceNumber(entry.sequence);
|
|
172
|
+
const waitMs = coerceNumber(entry.waitMs);
|
|
173
|
+
const waitTimeoutMs = coerceNumber(entry.waitTimeoutMs);
|
|
174
|
+
if (atMs !== null) {
|
|
175
|
+
entry.atMs = atMs;
|
|
176
|
+
}
|
|
177
|
+
if (timestamp !== null) {
|
|
178
|
+
entry.timestamp = timestamp;
|
|
179
|
+
}
|
|
180
|
+
if (sequence !== null) {
|
|
181
|
+
entry.sequence = sequence;
|
|
182
|
+
}
|
|
183
|
+
if (waitMs !== null) {
|
|
184
|
+
entry.waitMs = waitMs;
|
|
185
|
+
}
|
|
186
|
+
if (waitTimeoutMs !== null) {
|
|
187
|
+
entry.waitTimeoutMs = waitTimeoutMs;
|
|
188
|
+
}
|
|
189
|
+
return entry;
|
|
190
|
+
}
|
|
75
191
|
/**
|
|
76
192
|
* Rounds millisecond values to a stable artifact precision.
|
|
77
193
|
*
|
|
@@ -81,6 +197,59 @@ function parseKeyValueProfileEvent(payload) {
|
|
|
81
197
|
function roundMs(value) {
|
|
82
198
|
return Math.round(value * 1000) / 1000;
|
|
83
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* Asserts cross-field attempt semantics that JSON Schema alone does not express.
|
|
202
|
+
*
|
|
203
|
+
* @param {Record<string, any>} attempt
|
|
204
|
+
* @returns {void}
|
|
205
|
+
*/
|
|
206
|
+
function assertAttemptInvariants(attempt) {
|
|
207
|
+
const { attemptId, attemptNumber, classification, cleanup, maxAttempts, partialArtifacts, retryOfAttemptId, retryReason, status, terminalState, } = attempt;
|
|
208
|
+
if (status === 'passed' && terminalState !== 'passed') {
|
|
209
|
+
throw new Error(`Passed attempts must use terminalState "passed", received "${terminalState}".`);
|
|
210
|
+
}
|
|
211
|
+
if (status === 'failed' && !FAILURE_TERMINAL_STATES.has(String(terminalState))) {
|
|
212
|
+
throw new Error(`Failed attempts must use a failure terminalState, received "${terminalState}".`);
|
|
213
|
+
}
|
|
214
|
+
const expectedCategory = TERMINAL_CLASSIFICATION_CATEGORIES[String(terminalState)];
|
|
215
|
+
if (expectedCategory && classification?.category !== expectedCategory) {
|
|
216
|
+
throw new Error(`terminalState "${terminalState}" requires classification.category "${expectedCategory}".`);
|
|
217
|
+
}
|
|
218
|
+
if (terminalState === 'cancelled' || terminalState === 'aborted' || terminalState === 'timeout') {
|
|
219
|
+
if (partialArtifacts?.valid !== true) {
|
|
220
|
+
throw new Error(`terminalState "${terminalState}" must preserve valid partialArtifacts for diagnosis.`);
|
|
221
|
+
}
|
|
222
|
+
if (!Array.isArray(partialArtifacts.paths) || partialArtifacts.paths.length === 0) {
|
|
223
|
+
throw new Error(`terminalState "${terminalState}" must record partialArtifacts.paths.`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (cleanup?.status && cleanup.status !== 'not-required' && cleanup.status !== 'unknown' && typeof cleanup.message !== 'string') {
|
|
227
|
+
throw new Error(`cleanup.status "${cleanup.status}" must include a cleanup message.`);
|
|
228
|
+
}
|
|
229
|
+
if (!Number.isInteger(attemptNumber) || attemptNumber < 1) {
|
|
230
|
+
throw new Error(`attemptNumber must be an integer greater than or equal to 1.`);
|
|
231
|
+
}
|
|
232
|
+
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
233
|
+
throw new Error(`maxAttempts must be an integer greater than or equal to 1.`);
|
|
234
|
+
}
|
|
235
|
+
if (attemptNumber > maxAttempts) {
|
|
236
|
+
throw new Error(`attemptNumber ${attemptNumber} cannot exceed maxAttempts ${maxAttempts}.`);
|
|
237
|
+
}
|
|
238
|
+
if (attemptNumber > 1) {
|
|
239
|
+
if (typeof retryOfAttemptId !== 'string' || retryOfAttemptId.length === 0) {
|
|
240
|
+
throw new Error(`retry attempts must record retryOfAttemptId.`);
|
|
241
|
+
}
|
|
242
|
+
if (retryOfAttemptId === attemptId) {
|
|
243
|
+
throw new Error(`retryOfAttemptId must not equal attemptId.`);
|
|
244
|
+
}
|
|
245
|
+
if (typeof retryReason !== 'string' || retryReason.length === 0) {
|
|
246
|
+
throw new Error(`retry attempts must record retryReason.`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (attemptNumber === 1 && (typeof retryOfAttemptId === 'string' || typeof retryReason === 'string')) {
|
|
250
|
+
throw new Error(`first attempts must not record retry lineage.`);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
84
253
|
/**
|
|
85
254
|
* Returns a nearest-rank percentile for numeric measurements.
|
|
86
255
|
*
|
|
@@ -148,13 +317,61 @@ function extractProfileEvents(logText, filters = {}) {
|
|
|
148
317
|
}
|
|
149
318
|
});
|
|
150
319
|
}
|
|
320
|
+
/**
|
|
321
|
+
* Extracts structured profile-session entries from device logs.
|
|
322
|
+
*
|
|
323
|
+
* @param {string} logText
|
|
324
|
+
* @param {{runId?: string, scenario?: string}} [filters]
|
|
325
|
+
* @returns {Record<string, unknown>[]}
|
|
326
|
+
*/
|
|
327
|
+
function extractProfileSessionEntries(logText, filters = {}) {
|
|
328
|
+
const { runId, scenario } = filters;
|
|
329
|
+
return String(logText)
|
|
330
|
+
.split(/\r?\n/u)
|
|
331
|
+
.flatMap((line) => {
|
|
332
|
+
const prefixIndex = line.indexOf(PROFILE_SESSION_PREFIX);
|
|
333
|
+
if (prefixIndex === -1) {
|
|
334
|
+
return [];
|
|
335
|
+
}
|
|
336
|
+
const payload = line.slice(prefixIndex + PROFILE_SESSION_PREFIX.length).trim();
|
|
337
|
+
if (!payload) {
|
|
338
|
+
return [];
|
|
339
|
+
}
|
|
340
|
+
try {
|
|
341
|
+
const entry = JSON.parse(payload);
|
|
342
|
+
if (!entry || typeof entry !== 'object') {
|
|
343
|
+
return [];
|
|
344
|
+
}
|
|
345
|
+
if (runId && entry.runId !== runId) {
|
|
346
|
+
return [];
|
|
347
|
+
}
|
|
348
|
+
if (scenario && entry.scenario !== scenario) {
|
|
349
|
+
return [];
|
|
350
|
+
}
|
|
351
|
+
return [entry];
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
const entry = parseKeyValueProfileSessionEntry(payload);
|
|
355
|
+
if (!entry) {
|
|
356
|
+
return [];
|
|
357
|
+
}
|
|
358
|
+
if (runId && entry.runId !== runId) {
|
|
359
|
+
return [];
|
|
360
|
+
}
|
|
361
|
+
if (scenario && entry.scenario !== scenario) {
|
|
362
|
+
return [];
|
|
363
|
+
}
|
|
364
|
+
return [entry];
|
|
365
|
+
}
|
|
366
|
+
});
|
|
367
|
+
}
|
|
151
368
|
/**
|
|
152
369
|
* Builds timing metrics from app-emitted profile events.
|
|
153
370
|
*
|
|
154
|
-
* @param {{scenario: string, runId: string, events: Record<string, unknown>[], expectedIterations: number, timeoutCount?: number, artifacts?: Record<string, unknown>, cycleEventNames?: Record<string, string> | null, budgets?: Record<string, unknown> | null}} options
|
|
371
|
+
* @param {{scenario: string, runId: string, events: Record<string, unknown>[], expectedIterations: number, timeoutCount?: number, artifacts?: Record<string, unknown>, cycleEventNames?: Record<string, string> | null, milestoneEventsPerIteration?: number, budgets?: Record<string, unknown> | null}} options
|
|
155
372
|
* @returns {Record<string, unknown>}
|
|
156
373
|
*/
|
|
157
|
-
function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterations, timeoutCount = 0, artifacts = {}, cycleEventNames = null, budgets = null, }) {
|
|
374
|
+
function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterations, timeoutCount = 0, artifacts = {}, cycleEventNames = null, milestoneEventsPerIteration = 1, budgets = null, }) {
|
|
158
375
|
const resolvedCycleEventNames = {
|
|
159
376
|
openRequested: cycleEventNames?.openRequested ?? 'surface_open_requested',
|
|
160
377
|
opened: cycleEventNames?.opened ?? 'surface_opened',
|
|
@@ -163,17 +380,35 @@ function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterat
|
|
|
163
380
|
milestone: cycleEventNames?.milestone,
|
|
164
381
|
};
|
|
165
382
|
const usesMilestoneOnlyCycle = typeof resolvedCycleEventNames.milestone === 'string';
|
|
383
|
+
const requiredMilestoneEventsPerIteration = usesMilestoneOnlyCycle &&
|
|
384
|
+
Number.isInteger(milestoneEventsPerIteration) &&
|
|
385
|
+
milestoneEventsPerIteration > 1
|
|
386
|
+
? milestoneEventsPerIteration
|
|
387
|
+
: 1;
|
|
166
388
|
const iterations = new Map();
|
|
389
|
+
let nextImplicitMilestoneIteration = 1;
|
|
390
|
+
let nextImplicitMilestoneCount = 0;
|
|
167
391
|
for (const event of [...events].sort((left, right) => {
|
|
168
392
|
const leftAt = typeof left.atMs === 'number' ? left.atMs : Number.POSITIVE_INFINITY;
|
|
169
393
|
const rightAt = typeof right.atMs === 'number' ? right.atMs : Number.POSITIVE_INFINITY;
|
|
170
394
|
return leftAt - rightAt;
|
|
171
395
|
})) {
|
|
172
|
-
|
|
396
|
+
let eventIteration = typeof event.iteration === 'number'
|
|
173
397
|
? event.iteration
|
|
174
398
|
: expectedIterations === 1
|
|
175
399
|
? 1
|
|
176
400
|
: null;
|
|
401
|
+
if (eventIteration === null &&
|
|
402
|
+
usesMilestoneOnlyCycle &&
|
|
403
|
+
event.event === resolvedCycleEventNames.milestone &&
|
|
404
|
+
nextImplicitMilestoneIteration <= expectedIterations) {
|
|
405
|
+
eventIteration = nextImplicitMilestoneIteration;
|
|
406
|
+
nextImplicitMilestoneCount += 1;
|
|
407
|
+
if (nextImplicitMilestoneCount >= requiredMilestoneEventsPerIteration) {
|
|
408
|
+
nextImplicitMilestoneIteration += 1;
|
|
409
|
+
nextImplicitMilestoneCount = 0;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
177
412
|
if (eventIteration === null) {
|
|
178
413
|
continue;
|
|
179
414
|
}
|
|
@@ -203,8 +438,10 @@ function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterat
|
|
|
203
438
|
event.atMs >= current.closeRequestedAt) {
|
|
204
439
|
current.dismissedAt = event.atMs;
|
|
205
440
|
}
|
|
206
|
-
if (event.event === resolvedCycleEventNames.milestone
|
|
207
|
-
typeof current.
|
|
441
|
+
if (event.event === resolvedCycleEventNames.milestone) {
|
|
442
|
+
current.milestoneCount = typeof current.milestoneCount === 'number'
|
|
443
|
+
? current.milestoneCount + 1
|
|
444
|
+
: 1;
|
|
208
445
|
current.milestoneAt = event.atMs;
|
|
209
446
|
}
|
|
210
447
|
iterations.set(eventIteration, current);
|
|
@@ -232,6 +469,9 @@ function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterat
|
|
|
232
469
|
record.dismissedAt >= record.closeRequestedAt;
|
|
233
470
|
const hasMilestoneDuration = usesMilestoneOnlyCycle &&
|
|
234
471
|
typeof record.milestoneAt === 'number' &&
|
|
472
|
+
(requiredMilestoneEventsPerIteration <= 1 ||
|
|
473
|
+
(typeof record.milestoneCount === 'number' &&
|
|
474
|
+
record.milestoneCount >= requiredMilestoneEventsPerIteration)) &&
|
|
235
475
|
record.milestoneAt >= 0;
|
|
236
476
|
if (hasMilestoneDuration) {
|
|
237
477
|
durationsMs.push(roundMs(record.milestoneAt));
|
|
@@ -269,7 +509,8 @@ function buildMetricsFromProfileEvents({ scenario, runId, events, expectedIterat
|
|
|
269
509
|
incompleteIterations,
|
|
270
510
|
artifacts: sortValue(artifacts),
|
|
271
511
|
};
|
|
272
|
-
const
|
|
512
|
+
const intervalBudgetChecks = evaluateIntervalBudgetChecks({ events, expectedIterations, budgets });
|
|
513
|
+
const budgetEvaluation = evaluateProfileBudgets({ metrics, budgets, extraChecks: intervalBudgetChecks });
|
|
273
514
|
if (budgetEvaluation) {
|
|
274
515
|
metrics.budgetEvaluation = sortValue(budgetEvaluation);
|
|
275
516
|
}
|
|
@@ -294,15 +535,111 @@ function evaluateBudgetCheck({ name, actual, limit }) {
|
|
|
294
535
|
unit: 'ms',
|
|
295
536
|
};
|
|
296
537
|
}
|
|
538
|
+
/**
|
|
539
|
+
* Collects duration samples for one named interval budget.
|
|
540
|
+
*
|
|
541
|
+
* @param {{events: ProfileEvent[], fromEvent: string, toEvent: string, expectedIterations: number}} options
|
|
542
|
+
* @returns {number[]}
|
|
543
|
+
*/
|
|
544
|
+
function collectIntervalDurations({ events, fromEvent, toEvent, expectedIterations, }) {
|
|
545
|
+
const sortedEvents = [...events].sort((left, right) => {
|
|
546
|
+
const leftAt = typeof left.atMs === 'number' ? left.atMs : Number.POSITIVE_INFINITY;
|
|
547
|
+
const rightAt = typeof right.atMs === 'number' ? right.atMs : Number.POSITIVE_INFINITY;
|
|
548
|
+
return leftAt - rightAt;
|
|
549
|
+
});
|
|
550
|
+
const hasIterationPairs = sortedEvents.some((event) => (typeof event.iteration === 'number' && (event.event === fromEvent || event.event === toEvent)));
|
|
551
|
+
if (hasIterationPairs) {
|
|
552
|
+
const durations = [];
|
|
553
|
+
for (let iteration = 1; iteration <= expectedIterations; iteration += 1) {
|
|
554
|
+
const from = sortedEvents.find((event) => (event.event === fromEvent &&
|
|
555
|
+
event.iteration === iteration &&
|
|
556
|
+
typeof event.atMs === 'number'));
|
|
557
|
+
const to = sortedEvents.find((event) => (event.event === toEvent &&
|
|
558
|
+
event.iteration === iteration &&
|
|
559
|
+
typeof event.atMs === 'number' &&
|
|
560
|
+
typeof from?.atMs === 'number' &&
|
|
561
|
+
event.atMs >= from.atMs));
|
|
562
|
+
if (typeof from?.atMs === 'number' && typeof to?.atMs === 'number') {
|
|
563
|
+
durations.push(roundMs(to.atMs - from.atMs));
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
return durations;
|
|
567
|
+
}
|
|
568
|
+
const durations = [];
|
|
569
|
+
let pendingFrom = null;
|
|
570
|
+
for (const event of sortedEvents) {
|
|
571
|
+
if (typeof event.atMs !== 'number') {
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
if (event.event === fromEvent && pendingFrom === null) {
|
|
575
|
+
pendingFrom = event.atMs;
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
if (event.event === toEvent && pendingFrom !== null && event.atMs >= pendingFrom) {
|
|
579
|
+
durations.push(roundMs(event.atMs - pendingFrom));
|
|
580
|
+
pendingFrom = null;
|
|
581
|
+
if (durations.length >= expectedIterations) {
|
|
582
|
+
break;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
return durations;
|
|
587
|
+
}
|
|
588
|
+
/**
|
|
589
|
+
* Evaluates named interval budgets that should not define scenario health completeness.
|
|
590
|
+
*
|
|
591
|
+
* @param {{events: ProfileEvent[], expectedIterations: number, budgets?: Record<string, unknown> | null}} options
|
|
592
|
+
* @returns {BudgetCheck[]}
|
|
593
|
+
*/
|
|
594
|
+
function evaluateIntervalBudgetChecks({ events, expectedIterations, budgets, }) {
|
|
595
|
+
if (!Array.isArray(budgets?.intervals)) {
|
|
596
|
+
return [];
|
|
597
|
+
}
|
|
598
|
+
return budgets.intervals
|
|
599
|
+
.map((budget) => {
|
|
600
|
+
if (typeof budget.name !== 'string' ||
|
|
601
|
+
typeof budget.metric !== 'string' ||
|
|
602
|
+
typeof budget.limit !== 'number' ||
|
|
603
|
+
typeof budget.fromEvent !== 'string' ||
|
|
604
|
+
typeof budget.toEvent !== 'string') {
|
|
605
|
+
return null;
|
|
606
|
+
}
|
|
607
|
+
const durations = collectIntervalDurations({
|
|
608
|
+
events,
|
|
609
|
+
fromEvent: budget.fromEvent,
|
|
610
|
+
toEvent: budget.toEvent,
|
|
611
|
+
expectedIterations,
|
|
612
|
+
});
|
|
613
|
+
const actual = budget.metric === 'p50'
|
|
614
|
+
? percentile(durations, 50)
|
|
615
|
+
: budget.metric === 'p95'
|
|
616
|
+
? percentile(durations, 95)
|
|
617
|
+
: null;
|
|
618
|
+
return evaluateBudgetCheck({
|
|
619
|
+
name: budget.name,
|
|
620
|
+
actual,
|
|
621
|
+
limit: budget.limit,
|
|
622
|
+
});
|
|
623
|
+
})
|
|
624
|
+
.filter((check) => Boolean(check));
|
|
625
|
+
}
|
|
297
626
|
/**
|
|
298
627
|
* Evaluates configured profile budgets against generated metrics.
|
|
299
628
|
*
|
|
300
|
-
* @param {{metrics: Record<string, unknown>, budgets?: Record<string, unknown> | null}} options
|
|
629
|
+
* @param {{metrics: Record<string, unknown>, budgets?: Record<string, unknown> | null, extraChecks?: BudgetCheck[]}} options
|
|
301
630
|
* @returns {Record<string, unknown> | null}
|
|
302
631
|
*/
|
|
303
|
-
function evaluateProfileBudgets({ metrics, budgets }) {
|
|
632
|
+
function evaluateProfileBudgets({ metrics, budgets, extraChecks = [], }) {
|
|
304
633
|
if (!budgets?.pass || typeof budgets.pass !== 'object') {
|
|
305
|
-
|
|
634
|
+
if (extraChecks.length === 0) {
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
return {
|
|
638
|
+
metric: budgets?.metric ?? metrics.measurement ?? 'profile budget',
|
|
639
|
+
pass: extraChecks.every((check) => check.pass),
|
|
640
|
+
checks: extraChecks,
|
|
641
|
+
failedChecks: extraChecks.filter((check) => !check.pass).map((check) => check.name),
|
|
642
|
+
};
|
|
306
643
|
}
|
|
307
644
|
const checks = [
|
|
308
645
|
evaluateBudgetCheck({
|
|
@@ -376,7 +713,7 @@ function evaluateProfileBudgets({ metrics, budgets }) {
|
|
|
376
713
|
}
|
|
377
714
|
: null,
|
|
378
715
|
].filter((check) => Boolean(check));
|
|
379
|
-
const allChecks = [...thresholdChecks, ...checks];
|
|
716
|
+
const allChecks = [...thresholdChecks, ...checks, ...extraChecks];
|
|
380
717
|
if (allChecks.length === 0) {
|
|
381
718
|
return null;
|
|
382
719
|
}
|
|
@@ -407,6 +744,54 @@ function sortValue(value) {
|
|
|
407
744
|
}
|
|
408
745
|
return value;
|
|
409
746
|
}
|
|
747
|
+
/**
|
|
748
|
+
* Returns a deterministic SHA-256 hash for a JSON-compatible value.
|
|
749
|
+
*
|
|
750
|
+
* @param {unknown} value
|
|
751
|
+
* @returns {string}
|
|
752
|
+
*/
|
|
753
|
+
function hashStableValue(value) {
|
|
754
|
+
return crypto.createHash('sha256').update(JSON.stringify(sortValue(value))).digest('hex');
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* Normalizes optional run cohort provenance into schema-safe scalar fields.
|
|
758
|
+
*
|
|
759
|
+
* @param {Record<string, unknown> | null | undefined} cohort
|
|
760
|
+
* @returns {Record<string, unknown> | null}
|
|
761
|
+
*/
|
|
762
|
+
function normalizeProvenanceCohort(cohort) {
|
|
763
|
+
if (!cohort || typeof cohort !== 'object') {
|
|
764
|
+
return null;
|
|
765
|
+
}
|
|
766
|
+
const normalized = {
|
|
767
|
+
...(typeof cohort.appId === 'string' ? { appId: cohort.appId } : {}),
|
|
768
|
+
...(typeof cohort.appVersion === 'string' ? { appVersion: cohort.appVersion } : {}),
|
|
769
|
+
...(typeof cohort.buildId === 'string' ? { buildId: cohort.buildId } : {}),
|
|
770
|
+
...(typeof cohort.buildMode === 'string' ? { buildMode: cohort.buildMode } : {}),
|
|
771
|
+
...(typeof cohort.commandTransport === 'string' ? { commandTransport: cohort.commandTransport } : {}),
|
|
772
|
+
...(typeof cohort.deviceClass === 'string' ? { deviceClass: cohort.deviceClass } : {}),
|
|
773
|
+
...(typeof cohort.osVersion === 'string' ? { osVersion: cohort.osVersion } : {}),
|
|
774
|
+
...(typeof cohort.platform === 'string' ? { platform: cohort.platform } : {}),
|
|
775
|
+
...(typeof cohort.runnerName === 'string' ? { runnerName: cohort.runnerName } : {}),
|
|
776
|
+
...(typeof cohort.runnerVersion === 'string' ? { runnerVersion: cohort.runnerVersion } : {}),
|
|
777
|
+
...(typeof cohort.seedIdentity === 'string' ? { seedIdentity: cohort.seedIdentity } : {}),
|
|
778
|
+
...(cohort.featureFlags && typeof cohort.featureFlags === 'object' && !Array.isArray(cohort.featureFlags)
|
|
779
|
+
? { featureFlags: sortValue(cohort.featureFlags) }
|
|
780
|
+
: {}),
|
|
781
|
+
...(Array.isArray(cohort.providers)
|
|
782
|
+
? {
|
|
783
|
+
providers: cohort.providers
|
|
784
|
+
.filter((provider) => provider && typeof provider === 'object')
|
|
785
|
+
.map((provider) => sortValue({
|
|
786
|
+
...(typeof provider.name === 'string' ? { name: provider.name } : {}),
|
|
787
|
+
...(typeof provider.version === 'string' ? { version: provider.version } : {}),
|
|
788
|
+
}))
|
|
789
|
+
.filter((provider) => typeof provider.name === 'string' || typeof provider.version === 'string'),
|
|
790
|
+
}
|
|
791
|
+
: {}),
|
|
792
|
+
};
|
|
793
|
+
return Object.keys(normalized).length > 0 ? sortValue(normalized) : null;
|
|
794
|
+
}
|
|
410
795
|
/**
|
|
411
796
|
* Normalizes event timestamps to milliseconds since run start.
|
|
412
797
|
*
|
|
@@ -513,14 +898,101 @@ function inferTimelineStatus(eventName) {
|
|
|
513
898
|
}
|
|
514
899
|
return 'observed';
|
|
515
900
|
}
|
|
901
|
+
/**
|
|
902
|
+
* Keeps causal-run timeline values within the public artifact schema.
|
|
903
|
+
*
|
|
904
|
+
* App-owned events may carry richer phase/status vocabulary than ASL's stable
|
|
905
|
+
* artifact contract. Preserve that vocabulary in metadata, but emit only schema
|
|
906
|
+
* values at the timeline top level.
|
|
907
|
+
*
|
|
908
|
+
* @param {{phase: string, status: string, metadata: Record<string, unknown>}} options
|
|
909
|
+
* @returns {{phase: string, status: string, metadata: Record<string, unknown>}}
|
|
910
|
+
*/
|
|
911
|
+
function normalizeTimelineContractValues({ metadata, phase, status, }) {
|
|
912
|
+
const normalizedMetadata = { ...metadata };
|
|
913
|
+
let normalizedPhase = phase;
|
|
914
|
+
let normalizedStatus = status;
|
|
915
|
+
if (!CAUSAL_TIMELINE_PHASES.has(normalizedPhase)) {
|
|
916
|
+
normalizedMetadata.appPhase = normalizedPhase;
|
|
917
|
+
normalizedPhase = 'domain';
|
|
918
|
+
}
|
|
919
|
+
if (!CAUSAL_TIMELINE_STATUSES.has(normalizedStatus)) {
|
|
920
|
+
normalizedMetadata.appStatus = normalizedStatus;
|
|
921
|
+
normalizedStatus = 'observed';
|
|
922
|
+
}
|
|
923
|
+
return {
|
|
924
|
+
metadata: normalizedMetadata,
|
|
925
|
+
phase: normalizedPhase,
|
|
926
|
+
status: normalizedStatus,
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* Converts profile-session command control entries into causal timeline events.
|
|
931
|
+
*
|
|
932
|
+
* These are ASL control-plane acknowledgements, not product truth events. They
|
|
933
|
+
* let agents verify command ordering and delivery while keeping product verdicts
|
|
934
|
+
* based on app-owned profile events.
|
|
935
|
+
*
|
|
936
|
+
* @param {{entries: Record<string, unknown>[], startedAt?: string}} options
|
|
937
|
+
* @returns {Record<string, unknown>[]}
|
|
938
|
+
*/
|
|
939
|
+
function buildCommandAcknowledgementTimeline({ entries, startedAt, }) {
|
|
940
|
+
return [...(Array.isArray(entries) ? entries : [])]
|
|
941
|
+
.map((entry) => {
|
|
942
|
+
if (!entry || typeof entry !== 'object' || entry.kind !== 'command') {
|
|
943
|
+
return null;
|
|
944
|
+
}
|
|
945
|
+
const atMs = normalizeEventTimestamp({
|
|
946
|
+
event: entry,
|
|
947
|
+
...(typeof startedAt === 'string' ? { startedAt } : {}),
|
|
948
|
+
});
|
|
949
|
+
if (atMs === null) {
|
|
950
|
+
return null;
|
|
951
|
+
}
|
|
952
|
+
const commandStatus = typeof entry.status === 'string' ? entry.status : 'observed';
|
|
953
|
+
const status = commandStatus === 'completed' || commandStatus === 'delivered'
|
|
954
|
+
? 'completed'
|
|
955
|
+
: commandStatus === 'skipped'
|
|
956
|
+
? 'skipped'
|
|
957
|
+
: commandStatus === 'failed'
|
|
958
|
+
? 'failed'
|
|
959
|
+
: commandStatus === 'received' || commandStatus === 'queued'
|
|
960
|
+
? 'started'
|
|
961
|
+
: 'observed';
|
|
962
|
+
const metadata = {
|
|
963
|
+
...(typeof entry.command === 'string' ? { command: entry.command } : {}),
|
|
964
|
+
...(typeof entry.commandId === 'string' ? { commandId: entry.commandId } : {}),
|
|
965
|
+
...(typeof entry.id === 'string' ? { entryId: entry.id } : {}),
|
|
966
|
+
...(typeof entry.queueId === 'string' ? { queueId: entry.queueId } : {}),
|
|
967
|
+
...(typeof entry.sequence === 'number' ? { sequence: entry.sequence } : {}),
|
|
968
|
+
...(typeof entry.source === 'string' ? { source: entry.source } : {}),
|
|
969
|
+
commandStatus,
|
|
970
|
+
...(typeof entry.result === 'string' ? { result: entry.result } : {}),
|
|
971
|
+
...(typeof entry.reason === 'string' ? { reason: entry.reason } : {}),
|
|
972
|
+
...(typeof entry.waitForMilestone === 'string' ? { waitForMilestone: entry.waitForMilestone } : {}),
|
|
973
|
+
...(typeof entry.waitMs === 'number' ? { waitMs: entry.waitMs } : {}),
|
|
974
|
+
...(typeof entry.waitTimeoutMs === 'number' ? { waitTimeoutMs: entry.waitTimeoutMs } : {}),
|
|
975
|
+
};
|
|
976
|
+
return sortValue({
|
|
977
|
+
phase: 'intent',
|
|
978
|
+
name: `profile_command_${commandStatus}`,
|
|
979
|
+
atMs,
|
|
980
|
+
status,
|
|
981
|
+
owner: 'asl-command-transport',
|
|
982
|
+
metadata,
|
|
983
|
+
});
|
|
984
|
+
})
|
|
985
|
+
.filter(Boolean)
|
|
986
|
+
.sort((left, right) => left.atMs - right.atMs);
|
|
987
|
+
}
|
|
516
988
|
/**
|
|
517
989
|
* Builds a causal timeline from app-owned profile events.
|
|
518
990
|
*
|
|
519
991
|
* @param {{events: Record<string, unknown>[], startedAt?: string, phaseMap?: Record<string, string> | null, owner?: string | null}} options
|
|
520
992
|
* @returns {Record<string, unknown>[]}
|
|
521
993
|
*/
|
|
522
|
-
function buildCausalTimeline({ events, startedAt, phaseMap = null, owner = null, }) {
|
|
523
|
-
|
|
994
|
+
function buildCausalTimeline({ events, sessionEntries = [], startedAt, phaseMap = null, owner = null, }) {
|
|
995
|
+
const eventTimeline = [...(Array.isArray(events) ? events : [])]
|
|
524
996
|
.map((event) => {
|
|
525
997
|
if (!event || typeof event !== 'object' || typeof event.event !== 'string') {
|
|
526
998
|
return null;
|
|
@@ -545,20 +1017,78 @@ function buildCausalTimeline({ events, startedAt, phaseMap = null, owner = null,
|
|
|
545
1017
|
...(typeof event.flowId === 'string' ? { flowId: event.flowId } : {}),
|
|
546
1018
|
...(typeof event.route === 'string' ? { route: event.route } : {}),
|
|
547
1019
|
...(typeof event.iteration === 'number' ? { iteration: event.iteration } : {}),
|
|
1020
|
+
...(typeof event.sequence === 'number' ? { sequence: event.sequence } : {}),
|
|
1021
|
+
...(typeof event.queueId === 'string' ? { queueId: event.queueId } : {}),
|
|
1022
|
+
...(typeof event.commandId === 'string' ? { commandId: event.commandId } : {}),
|
|
1023
|
+
...(typeof event.operationId === 'string' ? { operationId: event.operationId } : {}),
|
|
1024
|
+
...(typeof event.attemptId === 'string' ? { attemptId: event.attemptId } : {}),
|
|
1025
|
+
...(typeof event.clockDomain === 'string' ? { clockDomain: event.clockDomain } : {}),
|
|
548
1026
|
};
|
|
549
|
-
|
|
1027
|
+
const timelineValues = normalizeTimelineContractValues({
|
|
1028
|
+
metadata,
|
|
550
1029
|
phase: explicitPhase ?? inferTimelinePhase(event.event),
|
|
1030
|
+
status: typeof event.status === 'string' ? event.status : inferTimelineStatus(event.event),
|
|
1031
|
+
});
|
|
1032
|
+
return sortValue({
|
|
1033
|
+
phase: timelineValues.phase,
|
|
551
1034
|
name: event.event,
|
|
552
1035
|
atMs,
|
|
553
|
-
status:
|
|
1036
|
+
status: timelineValues.status,
|
|
554
1037
|
...((typeof event.owner === 'string' && event.owner.length > 0) || owner
|
|
555
1038
|
? { owner: event.owner || owner }
|
|
556
1039
|
: {}),
|
|
557
|
-
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
|
1040
|
+
...(Object.keys(timelineValues.metadata).length > 0 ? { metadata: timelineValues.metadata } : {}),
|
|
558
1041
|
});
|
|
559
1042
|
})
|
|
560
|
-
.filter(Boolean)
|
|
561
|
-
|
|
1043
|
+
.filter(Boolean);
|
|
1044
|
+
const commandTimeline = buildCommandAcknowledgementTimeline({
|
|
1045
|
+
entries: sessionEntries,
|
|
1046
|
+
...(typeof startedAt === 'string' ? { startedAt } : {}),
|
|
1047
|
+
});
|
|
1048
|
+
return [...eventTimeline, ...commandTimeline].sort((left, right) => left.atMs - right.atMs);
|
|
1049
|
+
}
|
|
1050
|
+
/**
|
|
1051
|
+
* Summarizes repeated scenario accounting for causal artifacts.
|
|
1052
|
+
*
|
|
1053
|
+
* Metrics already decide product health; this summary makes the iteration
|
|
1054
|
+
* evidence explicit for agents reading `causal-run.json`.
|
|
1055
|
+
*
|
|
1056
|
+
* @param {Record<string, unknown>} metrics
|
|
1057
|
+
* @returns {Record<string, unknown> | null}
|
|
1058
|
+
*/
|
|
1059
|
+
function buildIterationSummary(metrics) {
|
|
1060
|
+
if (typeof metrics.iterations !== 'number' || metrics.iterations < 1) {
|
|
1061
|
+
return null;
|
|
1062
|
+
}
|
|
1063
|
+
const expected = Math.trunc(metrics.iterations);
|
|
1064
|
+
const failed = typeof metrics.failures === 'number' && metrics.failures > 0
|
|
1065
|
+
? Math.trunc(metrics.failures)
|
|
1066
|
+
: 0;
|
|
1067
|
+
const timeouts = typeof metrics.timeouts === 'number' && metrics.timeouts > 0
|
|
1068
|
+
? Math.trunc(metrics.timeouts)
|
|
1069
|
+
: 0;
|
|
1070
|
+
const incomplete = Array.isArray(metrics.incompleteIterations)
|
|
1071
|
+
? [...new Set(metrics.incompleteIterations.filter((iteration) => (typeof iteration === 'number' &&
|
|
1072
|
+
Number.isInteger(iteration) &&
|
|
1073
|
+
iteration >= 1 &&
|
|
1074
|
+
iteration <= expected)))].sort((left, right) => left - right)
|
|
1075
|
+
: [];
|
|
1076
|
+
const completed = Math.max(0, expected - incomplete.length);
|
|
1077
|
+
const status = timeouts > 0
|
|
1078
|
+
? 'timeout'
|
|
1079
|
+
: failed === 0 && incomplete.length === 0
|
|
1080
|
+
? 'complete'
|
|
1081
|
+
: completed > 0
|
|
1082
|
+
? 'partial'
|
|
1083
|
+
: 'failed';
|
|
1084
|
+
return {
|
|
1085
|
+
completed,
|
|
1086
|
+
expected,
|
|
1087
|
+
failed,
|
|
1088
|
+
incomplete,
|
|
1089
|
+
status,
|
|
1090
|
+
timeouts,
|
|
1091
|
+
};
|
|
562
1092
|
}
|
|
563
1093
|
/**
|
|
564
1094
|
* Builds the `budget-verdict.json` profile artifact from budget evaluation.
|
|
@@ -647,6 +1177,8 @@ function normalizeBudgetsForCausalRun(budgets) {
|
|
|
647
1177
|
* @returns {Record<string, unknown>}
|
|
648
1178
|
*/
|
|
649
1179
|
function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor = 'unknown', interactionDriver, trigger = null, budgets = null, timeline = [], artifacts, manifest, metrics, }) {
|
|
1180
|
+
const iterationSummary = buildIterationSummary(metrics);
|
|
1181
|
+
const videoPath = typeof artifacts.captures?.video === 'string' ? artifacts.captures.video : null;
|
|
650
1182
|
return sortValue({
|
|
651
1183
|
schemaVersion: '1.0.0',
|
|
652
1184
|
flowId,
|
|
@@ -664,13 +1196,21 @@ function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor
|
|
|
664
1196
|
kind: 'unknown',
|
|
665
1197
|
label: scenario.description ?? scenario.name,
|
|
666
1198
|
},
|
|
1199
|
+
provenanceRef: {
|
|
1200
|
+
manifest: artifacts.manifest,
|
|
1201
|
+
runId,
|
|
1202
|
+
...(typeof manifest.scenarioHash === 'string' && manifest.scenarioHash.length > 0
|
|
1203
|
+
? { scenarioHash: manifest.scenarioHash }
|
|
1204
|
+
: {}),
|
|
1205
|
+
},
|
|
667
1206
|
budgets: normalizeBudgetsForCausalRun(budgets),
|
|
668
1207
|
timeline,
|
|
1208
|
+
...(iterationSummary ? { iterationSummary } : {}),
|
|
669
1209
|
artifacts: {
|
|
670
1210
|
summary: artifacts.summary,
|
|
671
1211
|
metrics: artifacts.metrics,
|
|
672
1212
|
manifest: artifacts.manifest,
|
|
673
|
-
video:
|
|
1213
|
+
...(videoPath ? { video: videoPath } : {}),
|
|
674
1214
|
screenshot: Array.isArray(artifacts.captures?.screenshots)
|
|
675
1215
|
? artifacts.captures.screenshots[0] ?? null
|
|
676
1216
|
: null,
|
|
@@ -689,7 +1229,61 @@ function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor
|
|
|
689
1229
|
* @param {Record<string, unknown>} options
|
|
690
1230
|
* @returns {Record<string, unknown>}
|
|
691
1231
|
*/
|
|
692
|
-
function buildManifest({ scenario, scenarioHash, runId, platform = 'ios', status, startedAt, endedAt, interactionDriver, comparisonLane, simulator, bundleId, gitSha, toolVersions, artifacts, failureReason = null, }) {
|
|
1232
|
+
function buildManifest({ scenario, scenarioHash, runId, attemptId, attemptNumber, maxAttempts, retryOfAttemptId, retryReason, platform = 'ios', status, terminalState, startedAt, endedAt, interactionDriver, comparisonLane, classification, cleanup, partialArtifacts, preconditions, postconditions, simulator, bundleId, gitSha, toolVersions, cohort, artifacts, failureReason = null, }) {
|
|
1233
|
+
const durationMs = roundMs(Math.max(0, Date.parse(endedAt) - Date.parse(startedAt)));
|
|
1234
|
+
const sortedToolVersions = sortValue(toolVersions);
|
|
1235
|
+
const sortedSimulator = sortValue(simulator);
|
|
1236
|
+
const normalizedCohort = normalizeProvenanceCohort(cohort);
|
|
1237
|
+
const cohortHash = normalizedCohort ? hashStableValue(normalizedCohort) : null;
|
|
1238
|
+
const resolvedTerminalState = typeof terminalState === 'string' && terminalState.length > 0 ? terminalState : status;
|
|
1239
|
+
const resolvedAttemptNumber = Number.isInteger(attemptNumber) ? attemptNumber : 1;
|
|
1240
|
+
const resolvedMaxAttempts = Number.isInteger(maxAttempts) ? maxAttempts : Math.max(1, resolvedAttemptNumber);
|
|
1241
|
+
const resolvedAttemptId = typeof attemptId === 'string' && attemptId.length > 0 ? attemptId : runId;
|
|
1242
|
+
const resolvedClassification = classification
|
|
1243
|
+
? sortValue(classification)
|
|
1244
|
+
: {
|
|
1245
|
+
category: status === 'passed' ? 'none' : 'unknown',
|
|
1246
|
+
};
|
|
1247
|
+
const resolvedCleanup = cleanup
|
|
1248
|
+
? sortValue(cleanup)
|
|
1249
|
+
: {
|
|
1250
|
+
status: 'not-required',
|
|
1251
|
+
};
|
|
1252
|
+
const resolvedPartialArtifacts = partialArtifacts
|
|
1253
|
+
? sortValue(partialArtifacts)
|
|
1254
|
+
: {
|
|
1255
|
+
valid: status !== 'passed',
|
|
1256
|
+
reason: status === 'passed'
|
|
1257
|
+
? 'complete successful run artifacts are present'
|
|
1258
|
+
: 'failed run artifacts are preserved for diagnosis and must not be treated as product proof unless health passes',
|
|
1259
|
+
};
|
|
1260
|
+
const resolvedAttempt = {
|
|
1261
|
+
attemptId: resolvedAttemptId,
|
|
1262
|
+
attemptNumber: resolvedAttemptNumber,
|
|
1263
|
+
maxAttempts: resolvedMaxAttempts,
|
|
1264
|
+
...(typeof retryOfAttemptId === 'string' && retryOfAttemptId.length > 0 ? { retryOfAttemptId } : {}),
|
|
1265
|
+
...(typeof retryReason === 'string' && retryReason.length > 0 ? { retryReason } : {}),
|
|
1266
|
+
runId,
|
|
1267
|
+
status,
|
|
1268
|
+
terminalState: resolvedTerminalState,
|
|
1269
|
+
startedAt,
|
|
1270
|
+
endedAt,
|
|
1271
|
+
durationMs,
|
|
1272
|
+
interactionDriver,
|
|
1273
|
+
...(typeof comparisonLane === 'string' && comparisonLane.length > 0 ? { comparisonLane } : {}),
|
|
1274
|
+
classification: resolvedClassification,
|
|
1275
|
+
cleanup: resolvedCleanup,
|
|
1276
|
+
partialArtifacts: resolvedPartialArtifacts,
|
|
1277
|
+
};
|
|
1278
|
+
const resolvedPreconditions = sortValue({
|
|
1279
|
+
...DEFAULT_ENVIRONMENT_PRECONDITIONS,
|
|
1280
|
+
...(preconditions && typeof preconditions === 'object' ? preconditions : {}),
|
|
1281
|
+
});
|
|
1282
|
+
const resolvedPostconditions = sortValue({
|
|
1283
|
+
...DEFAULT_ENVIRONMENT_POSTCONDITIONS,
|
|
1284
|
+
...(postconditions && typeof postconditions === 'object' ? postconditions : {}),
|
|
1285
|
+
});
|
|
1286
|
+
assertAttemptInvariants(resolvedAttempt);
|
|
693
1287
|
return {
|
|
694
1288
|
scenario,
|
|
695
1289
|
...(typeof scenarioHash === 'string' && scenarioHash.length > 0 ? { scenarioHash } : {}),
|
|
@@ -698,13 +1292,28 @@ function buildManifest({ scenario, scenarioHash, runId, platform = 'ios', status
|
|
|
698
1292
|
status,
|
|
699
1293
|
startedAt,
|
|
700
1294
|
endedAt,
|
|
701
|
-
durationMs
|
|
1295
|
+
durationMs,
|
|
702
1296
|
interactionDriver,
|
|
703
1297
|
...(typeof comparisonLane === 'string' && comparisonLane.length > 0 ? { comparisonLane } : {}),
|
|
704
|
-
|
|
1298
|
+
provenance: {
|
|
1299
|
+
...(typeof scenarioHash === 'string' && scenarioHash.length > 0 ? { scenarioHash } : {}),
|
|
1300
|
+
...(normalizedCohort ? { cohort: normalizedCohort, cohortHash } : {}),
|
|
1301
|
+
gitSha,
|
|
1302
|
+
toolVersions: sortedToolVersions,
|
|
1303
|
+
},
|
|
1304
|
+
attempt: resolvedAttempt,
|
|
1305
|
+
environment: {
|
|
1306
|
+
platform,
|
|
1307
|
+
bundleId,
|
|
1308
|
+
runtimeTarget: sortedSimulator,
|
|
1309
|
+
...(typeof sortedToolVersions.node === 'string' ? { nodeVersion: sortedToolVersions.node } : {}),
|
|
1310
|
+
preconditions: resolvedPreconditions,
|
|
1311
|
+
postconditions: resolvedPostconditions,
|
|
1312
|
+
},
|
|
1313
|
+
simulator: sortedSimulator,
|
|
705
1314
|
bundleId,
|
|
706
1315
|
gitSha,
|
|
707
|
-
toolVersions:
|
|
1316
|
+
toolVersions: sortedToolVersions,
|
|
708
1317
|
artifacts: sortValue(artifacts),
|
|
709
1318
|
failureReason,
|
|
710
1319
|
};
|
|
@@ -734,9 +1343,34 @@ function buildSummaryMarkdown({ manifest, metrics }) {
|
|
|
734
1343
|
const evidenceAttachments = Array.isArray(manifest.artifacts.evidenceAttachments)
|
|
735
1344
|
? manifest.artifacts.evidenceAttachments
|
|
736
1345
|
: [];
|
|
1346
|
+
const diagnostics = Array.isArray(manifest.artifacts.diagnostics)
|
|
1347
|
+
? manifest.artifacts.diagnostics
|
|
1348
|
+
: [];
|
|
737
1349
|
const evidenceAttachmentLines = evidenceAttachments.length > 0
|
|
738
1350
|
? evidenceAttachments.map((attachment) => `- ${attachment.channel}/${attachment.kind}: \`${attachment.path}\` (${attachment.sizeBytes} bytes, sha256 ${attachment.sha256})`)
|
|
739
1351
|
: ['- none'];
|
|
1352
|
+
const diagnosticLines = diagnostics.length > 0
|
|
1353
|
+
? diagnostics.map((diagnostic) => {
|
|
1354
|
+
const label = diagnostic.name ? `${diagnostic.kind}/${diagnostic.name}` : diagnostic.kind;
|
|
1355
|
+
const pathLabel = typeof diagnostic.path === 'string' ? ` \`${diagnostic.path}\`` : '';
|
|
1356
|
+
const reasonLabel = typeof diagnostic.reason === 'string' ? ` - ${diagnostic.reason}` : '';
|
|
1357
|
+
return `- ${label}: ${diagnostic.status}${diagnostic.required ? ' (required)' : ''}${pathLabel}${reasonLabel}`;
|
|
1358
|
+
})
|
|
1359
|
+
: ['- none'];
|
|
1360
|
+
const attempt = manifest.attempt && typeof manifest.attempt === 'object' ? manifest.attempt : {};
|
|
1361
|
+
const attemptLines = [
|
|
1362
|
+
`- Attempt ID: \`${attempt.attemptId ?? manifest.runId}\``,
|
|
1363
|
+
`- Attempt number: ${attempt.attemptNumber ?? 1}/${attempt.maxAttempts ?? 1}`,
|
|
1364
|
+
`- Terminal state: ${attempt.terminalState ?? manifest.status}`,
|
|
1365
|
+
...(typeof attempt.retryOfAttemptId === 'string'
|
|
1366
|
+
? [`- Retry of: \`${attempt.retryOfAttemptId}\``]
|
|
1367
|
+
: []),
|
|
1368
|
+
...(typeof attempt.retryReason === 'string'
|
|
1369
|
+
? [`- Retry reason: ${attempt.retryReason}`]
|
|
1370
|
+
: []),
|
|
1371
|
+
`- Cleanup: ${attempt.cleanup?.status ?? 'unknown'}`,
|
|
1372
|
+
`- Partial artifacts valid: ${attempt.partialArtifacts?.valid === true ? 'yes' : 'no'}`,
|
|
1373
|
+
];
|
|
740
1374
|
const lines = [
|
|
741
1375
|
`# ${String(manifest.platform || 'ios').toUpperCase()} profile run: ${manifest.scenario}`,
|
|
742
1376
|
'',
|
|
@@ -755,6 +1389,10 @@ function buildSummaryMarkdown({ manifest, metrics }) {
|
|
|
755
1389
|
`- p50 cycle: ${metrics.p50Ms === null ? 'n/a' : `${metrics.p50Ms}ms`}`,
|
|
756
1390
|
`- p95 cycle: ${metrics.p95Ms === null ? 'n/a' : `${metrics.p95Ms}ms`}`,
|
|
757
1391
|
'',
|
|
1392
|
+
'## Attempt',
|
|
1393
|
+
'',
|
|
1394
|
+
...attemptLines,
|
|
1395
|
+
'',
|
|
758
1396
|
'## Artifact paths',
|
|
759
1397
|
'',
|
|
760
1398
|
`- Causal run: \`${manifest.artifacts.causalRun}\``,
|
|
@@ -762,10 +1400,18 @@ function buildSummaryMarkdown({ manifest, metrics }) {
|
|
|
762
1400
|
`- Manifest: \`${manifest.artifacts.manifest}\``,
|
|
763
1401
|
`- Scenario: \`${manifest.artifacts.scenario}\``,
|
|
764
1402
|
`- Metrics: \`${manifest.artifacts.metrics}\``,
|
|
765
|
-
`- Interaction log:
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
`-
|
|
1403
|
+
`- Interaction log: ${typeof manifest.artifacts.raw.interactionLog === 'string'
|
|
1404
|
+
? `\`${manifest.artifacts.raw.interactionLog}\``
|
|
1405
|
+
: 'none'}`,
|
|
1406
|
+
`- Device log: ${typeof manifest.artifacts.raw.deviceLog === 'string'
|
|
1407
|
+
? `\`${manifest.artifacts.raw.deviceLog}\``
|
|
1408
|
+
: 'none'}`,
|
|
1409
|
+
`- Video: ${typeof manifest.artifacts.captures.video === 'string'
|
|
1410
|
+
? `\`${manifest.artifacts.captures.video}\``
|
|
1411
|
+
: 'none'}`,
|
|
1412
|
+
`- UI tree: ${typeof manifest.artifacts.captures.uiTree === 'string'
|
|
1413
|
+
? `\`${manifest.artifacts.captures.uiTree}\``
|
|
1414
|
+
: 'none'}`,
|
|
769
1415
|
`- Screenshots: ${screenshots.length > 0
|
|
770
1416
|
? screenshots.map((item) => `\`${item}\``).join(', ')
|
|
771
1417
|
: 'none'}`,
|
|
@@ -777,6 +1423,10 @@ function buildSummaryMarkdown({ manifest, metrics }) {
|
|
|
777
1423
|
'## Evidence attachments',
|
|
778
1424
|
'',
|
|
779
1425
|
...evidenceAttachmentLines,
|
|
1426
|
+
'',
|
|
1427
|
+
'## Diagnostic inventory',
|
|
1428
|
+
'',
|
|
1429
|
+
...diagnosticLines,
|
|
780
1430
|
];
|
|
781
1431
|
if (metrics.budgetEvaluation) {
|
|
782
1432
|
lines.push('', '## Budget', '', `- Metric: ${metrics.budgetEvaluation.metric}`, `- Status: ${metrics.budgetEvaluation.pass ? 'pass' : 'fail'}`);
|