agent-scenario-loop 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/app/profile-session.ts +98 -4
- package/dist/core/agent-summary.d.ts +3 -2
- package/dist/core/agent-summary.js +44 -2
- package/dist/core/artifact-contract.d.ts +22 -4
- package/dist/core/artifact-contract.js +512 -11
- package/dist/core/comparison.d.ts +57 -3
- package/dist/core/comparison.js +113 -1
- package/dist/core/planner.d.ts +32 -1
- package/dist/core/planner.js +144 -0
- package/dist/core/run-index.d.ts +4 -0
- package/dist/core/run-index.js +55 -1
- package/dist/core/schema-validator.d.ts +1 -0
- package/dist/core/schema-validator.js +1 -0
- package/dist/runner/compare-latest.d.ts +8 -4
- package/dist/runner/compare-latest.js +24 -5
- package/dist/runner/example-android-live.d.ts +10 -1
- package/dist/runner/example-android-live.js +55 -0
- package/dist/runner/example-ios-live.d.ts +10 -1
- package/dist/runner/example-ios-live.js +55 -0
- package/dist/runner/ios-simctl.d.ts +5 -0
- package/dist/runner/ios-simctl.js +6 -0
- package/dist/runner/live-comparison.d.ts +2 -2
- package/dist/runner/live-comparison.js +2 -1
- package/dist/runner/live-proof-summary.d.ts +5 -4
- package/dist/runner/live-proof-summary.js +12 -2
- package/dist/runner/live-proof.d.ts +3 -2
- package/dist/runner/live-proof.js +9 -2
- package/dist/runner/profile-android.d.ts +5 -0
- package/dist/runner/profile-android.js +148 -24
- package/dist/runner/profile-ios.d.ts +11 -1
- package/dist/runner/profile-ios.js +128 -9
- package/dist/runner/profile-mobile.d.ts +8 -0
- package/dist/runner/profile-mobile.js +267 -28
- package/docs/adapters.md +4 -0
- package/docs/architecture.md +90 -0
- package/docs/authoring.md +5 -1
- package/docs/concepts.md +3 -24
- package/docs/consumer-rehearsal.md +4 -0
- package/docs/contracts.md +30 -100
- package/docs/external-adapter-protocol.md +219 -0
- package/docs/live-proofs.md +83 -2
- package/docs/principles.md +9 -15
- package/examples/mobile-app/README.md +12 -0
- package/examples/mobile-app/runner-manifests/primary-runner.json +1 -0
- package/examples/runners/README.md +1 -0
- package/examples/runners/adb-android.json +1 -0
- package/examples/runners/agent-device-android.json +1 -0
- package/examples/runners/agent-device-ios.json +1 -0
- package/examples/runners/argent-android.json +1 -0
- package/examples/runners/argent-ios.json +1 -0
- package/examples/runners/xcodebuildmcp-ios.json +1 -0
- package/package.json +2 -1
- package/schemas/causal-run.schema.json +85 -2
- package/schemas/comparison.schema.json +130 -2
- package/schemas/external-adapter-message.schema.json +693 -0
- package/schemas/health.schema.json +72 -0
- package/schemas/live-proof-set.schema.json +1 -1
- package/schemas/live-proof.schema.json +14 -6
- package/schemas/manifest.schema.json +442 -1
- package/schemas/runner-capabilities.schema.json +20 -0
- package/schemas/scenario.schema.json +16 -0
- package/templates/primary-runner.json +1 -0
|
@@ -11,11 +11,75 @@ exports.evaluateUiContract = evaluateUiContract;
|
|
|
11
11
|
exports.evaluateProfileBudgets = evaluateProfileBudgets;
|
|
12
12
|
exports.extractCandidateIdentifiers = extractCandidateIdentifiers;
|
|
13
13
|
exports.extractProfileEvents = extractProfileEvents;
|
|
14
|
+
exports.extractProfileSessionEntries = extractProfileSessionEntries;
|
|
14
15
|
exports.findMatchingIdentifier = findMatchingIdentifier;
|
|
15
16
|
exports.percentile = percentile;
|
|
16
17
|
exports.sortValue = sortValue;
|
|
18
|
+
const crypto = require('node:crypto');
|
|
17
19
|
const PROFILE_EVENT_PREFIX = '[profile-event]';
|
|
18
20
|
exports.PROFILE_EVENT_PREFIX = PROFILE_EVENT_PREFIX;
|
|
21
|
+
const PROFILE_SESSION_PREFIX = '[profile-session]';
|
|
22
|
+
const CAUSAL_TIMELINE_PHASES = new Set([
|
|
23
|
+
'intent',
|
|
24
|
+
'navigation',
|
|
25
|
+
'domain',
|
|
26
|
+
'query',
|
|
27
|
+
'network',
|
|
28
|
+
'render',
|
|
29
|
+
'native',
|
|
30
|
+
'visual',
|
|
31
|
+
'completion',
|
|
32
|
+
]);
|
|
33
|
+
const CAUSAL_TIMELINE_STATUSES = new Set([
|
|
34
|
+
'started',
|
|
35
|
+
'completed',
|
|
36
|
+
'failed',
|
|
37
|
+
'skipped',
|
|
38
|
+
'observed',
|
|
39
|
+
]);
|
|
40
|
+
const UNKNOWN_LIFECYCLE_ASSERTION = Object.freeze({
|
|
41
|
+
value: 'unknown',
|
|
42
|
+
evidence: 'not-asserted',
|
|
43
|
+
});
|
|
44
|
+
const DEFAULT_ENVIRONMENT_PRECONDITIONS = Object.freeze({
|
|
45
|
+
installedState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
46
|
+
appDataState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
47
|
+
authState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
48
|
+
initialRoute: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
49
|
+
foregroundState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
50
|
+
lifecyclePhase: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
51
|
+
deviceLockState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
52
|
+
permissions: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
53
|
+
locale: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
54
|
+
timezone: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
55
|
+
theme: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
56
|
+
fontScale: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
57
|
+
orientation: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
58
|
+
networkState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
59
|
+
animations: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
60
|
+
});
|
|
61
|
+
const DEFAULT_ENVIRONMENT_POSTCONDITIONS = Object.freeze({
|
|
62
|
+
appState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
63
|
+
lifecyclePhase: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
64
|
+
cleanupState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
65
|
+
dataState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
66
|
+
artifactState: UNKNOWN_LIFECYCLE_ASSERTION,
|
|
67
|
+
});
|
|
68
|
+
const FAILURE_TERMINAL_STATES = new Set([
|
|
69
|
+
'failed',
|
|
70
|
+
'timeout',
|
|
71
|
+
'cancelled',
|
|
72
|
+
'aborted',
|
|
73
|
+
'inconclusive',
|
|
74
|
+
'unsupported',
|
|
75
|
+
'unhealthy',
|
|
76
|
+
]);
|
|
77
|
+
const TERMINAL_CLASSIFICATION_CATEGORIES = Object.freeze({
|
|
78
|
+
timeout: 'timeout',
|
|
79
|
+
cancelled: 'cancelled',
|
|
80
|
+
aborted: 'cancelled',
|
|
81
|
+
unsupported: 'runner',
|
|
82
|
+
});
|
|
19
83
|
/**
|
|
20
84
|
* Converts finite numeric strings to numbers while preserving invalid input as `null`.
|
|
21
85
|
*
|
|
@@ -72,6 +136,54 @@ function parseKeyValueProfileEvent(payload) {
|
|
|
72
136
|
}
|
|
73
137
|
return event;
|
|
74
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* Parses key/value `[profile-session]` payloads into structured session entries.
|
|
141
|
+
*
|
|
142
|
+
* @param {string} payload
|
|
143
|
+
* @returns {Record<string, unknown> | null}
|
|
144
|
+
*/
|
|
145
|
+
function parseKeyValueProfileSessionEntry(payload) {
|
|
146
|
+
const matches = payload.match(/(?:[^\s=]+)=(?:"[^"]*"|'[^']*'|[^\s]+)/gu) ?? [];
|
|
147
|
+
if (matches.length === 0) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const entry = {};
|
|
151
|
+
for (const match of matches) {
|
|
152
|
+
const separatorIndex = match.indexOf('=');
|
|
153
|
+
if (separatorIndex <= 0) {
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
const key = match.slice(0, separatorIndex);
|
|
157
|
+
let value = match.slice(separatorIndex + 1);
|
|
158
|
+
if ((value.startsWith('"') && value.endsWith('"')) ||
|
|
159
|
+
(value.startsWith("'") && value.endsWith("'"))) {
|
|
160
|
+
value = value.slice(1, -1);
|
|
161
|
+
}
|
|
162
|
+
entry[key] = value;
|
|
163
|
+
}
|
|
164
|
+
if (typeof entry.kind !== 'string' ||
|
|
165
|
+
typeof entry.scenario !== 'string' ||
|
|
166
|
+
typeof entry.runId !== 'string') {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
const timestamp = coerceNumber(entry.timestamp);
|
|
170
|
+
const atMs = coerceNumber(entry.atMs);
|
|
171
|
+
const sequence = coerceNumber(entry.sequence);
|
|
172
|
+
const waitTimeoutMs = coerceNumber(entry.waitTimeoutMs);
|
|
173
|
+
if (atMs !== null) {
|
|
174
|
+
entry.atMs = atMs;
|
|
175
|
+
}
|
|
176
|
+
if (timestamp !== null) {
|
|
177
|
+
entry.timestamp = timestamp;
|
|
178
|
+
}
|
|
179
|
+
if (sequence !== null) {
|
|
180
|
+
entry.sequence = sequence;
|
|
181
|
+
}
|
|
182
|
+
if (waitTimeoutMs !== null) {
|
|
183
|
+
entry.waitTimeoutMs = waitTimeoutMs;
|
|
184
|
+
}
|
|
185
|
+
return entry;
|
|
186
|
+
}
|
|
75
187
|
/**
|
|
76
188
|
* Rounds millisecond values to a stable artifact precision.
|
|
77
189
|
*
|
|
@@ -81,6 +193,59 @@ function parseKeyValueProfileEvent(payload) {
|
|
|
81
193
|
function roundMs(value) {
|
|
82
194
|
return Math.round(value * 1000) / 1000;
|
|
83
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Asserts cross-field attempt semantics that JSON Schema alone does not express.
|
|
198
|
+
*
|
|
199
|
+
* @param {Record<string, any>} attempt
|
|
200
|
+
* @returns {void}
|
|
201
|
+
*/
|
|
202
|
+
function assertAttemptInvariants(attempt) {
|
|
203
|
+
const { attemptId, attemptNumber, classification, cleanup, maxAttempts, partialArtifacts, retryOfAttemptId, retryReason, status, terminalState, } = attempt;
|
|
204
|
+
if (status === 'passed' && terminalState !== 'passed') {
|
|
205
|
+
throw new Error(`Passed attempts must use terminalState "passed", received "${terminalState}".`);
|
|
206
|
+
}
|
|
207
|
+
if (status === 'failed' && !FAILURE_TERMINAL_STATES.has(String(terminalState))) {
|
|
208
|
+
throw new Error(`Failed attempts must use a failure terminalState, received "${terminalState}".`);
|
|
209
|
+
}
|
|
210
|
+
const expectedCategory = TERMINAL_CLASSIFICATION_CATEGORIES[String(terminalState)];
|
|
211
|
+
if (expectedCategory && classification?.category !== expectedCategory) {
|
|
212
|
+
throw new Error(`terminalState "${terminalState}" requires classification.category "${expectedCategory}".`);
|
|
213
|
+
}
|
|
214
|
+
if (terminalState === 'cancelled' || terminalState === 'aborted' || terminalState === 'timeout') {
|
|
215
|
+
if (partialArtifacts?.valid !== true) {
|
|
216
|
+
throw new Error(`terminalState "${terminalState}" must preserve valid partialArtifacts for diagnosis.`);
|
|
217
|
+
}
|
|
218
|
+
if (!Array.isArray(partialArtifacts.paths) || partialArtifacts.paths.length === 0) {
|
|
219
|
+
throw new Error(`terminalState "${terminalState}" must record partialArtifacts.paths.`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (cleanup?.status && cleanup.status !== 'not-required' && cleanup.status !== 'unknown' && typeof cleanup.message !== 'string') {
|
|
223
|
+
throw new Error(`cleanup.status "${cleanup.status}" must include a cleanup message.`);
|
|
224
|
+
}
|
|
225
|
+
if (!Number.isInteger(attemptNumber) || attemptNumber < 1) {
|
|
226
|
+
throw new Error(`attemptNumber must be an integer greater than or equal to 1.`);
|
|
227
|
+
}
|
|
228
|
+
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
|
229
|
+
throw new Error(`maxAttempts must be an integer greater than or equal to 1.`);
|
|
230
|
+
}
|
|
231
|
+
if (attemptNumber > maxAttempts) {
|
|
232
|
+
throw new Error(`attemptNumber ${attemptNumber} cannot exceed maxAttempts ${maxAttempts}.`);
|
|
233
|
+
}
|
|
234
|
+
if (attemptNumber > 1) {
|
|
235
|
+
if (typeof retryOfAttemptId !== 'string' || retryOfAttemptId.length === 0) {
|
|
236
|
+
throw new Error(`retry attempts must record retryOfAttemptId.`);
|
|
237
|
+
}
|
|
238
|
+
if (retryOfAttemptId === attemptId) {
|
|
239
|
+
throw new Error(`retryOfAttemptId must not equal attemptId.`);
|
|
240
|
+
}
|
|
241
|
+
if (typeof retryReason !== 'string' || retryReason.length === 0) {
|
|
242
|
+
throw new Error(`retry attempts must record retryReason.`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (attemptNumber === 1 && (typeof retryOfAttemptId === 'string' || typeof retryReason === 'string')) {
|
|
246
|
+
throw new Error(`first attempts must not record retry lineage.`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
84
249
|
/**
|
|
85
250
|
* Returns a nearest-rank percentile for numeric measurements.
|
|
86
251
|
*
|
|
@@ -148,6 +313,54 @@ function extractProfileEvents(logText, filters = {}) {
|
|
|
148
313
|
}
|
|
149
314
|
});
|
|
150
315
|
}
|
|
316
|
+
/**
|
|
317
|
+
* Extracts structured profile-session entries from device logs.
|
|
318
|
+
*
|
|
319
|
+
* @param {string} logText
|
|
320
|
+
* @param {{runId?: string, scenario?: string}} [filters]
|
|
321
|
+
* @returns {Record<string, unknown>[]}
|
|
322
|
+
*/
|
|
323
|
+
function extractProfileSessionEntries(logText, filters = {}) {
|
|
324
|
+
const { runId, scenario } = filters;
|
|
325
|
+
return String(logText)
|
|
326
|
+
.split(/\r?\n/u)
|
|
327
|
+
.flatMap((line) => {
|
|
328
|
+
const prefixIndex = line.indexOf(PROFILE_SESSION_PREFIX);
|
|
329
|
+
if (prefixIndex === -1) {
|
|
330
|
+
return [];
|
|
331
|
+
}
|
|
332
|
+
const payload = line.slice(prefixIndex + PROFILE_SESSION_PREFIX.length).trim();
|
|
333
|
+
if (!payload) {
|
|
334
|
+
return [];
|
|
335
|
+
}
|
|
336
|
+
try {
|
|
337
|
+
const entry = JSON.parse(payload);
|
|
338
|
+
if (!entry || typeof entry !== 'object') {
|
|
339
|
+
return [];
|
|
340
|
+
}
|
|
341
|
+
if (runId && entry.runId !== runId) {
|
|
342
|
+
return [];
|
|
343
|
+
}
|
|
344
|
+
if (scenario && entry.scenario !== scenario) {
|
|
345
|
+
return [];
|
|
346
|
+
}
|
|
347
|
+
return [entry];
|
|
348
|
+
}
|
|
349
|
+
catch {
|
|
350
|
+
const entry = parseKeyValueProfileSessionEntry(payload);
|
|
351
|
+
if (!entry) {
|
|
352
|
+
return [];
|
|
353
|
+
}
|
|
354
|
+
if (runId && entry.runId !== runId) {
|
|
355
|
+
return [];
|
|
356
|
+
}
|
|
357
|
+
if (scenario && entry.scenario !== scenario) {
|
|
358
|
+
return [];
|
|
359
|
+
}
|
|
360
|
+
return [entry];
|
|
361
|
+
}
|
|
362
|
+
});
|
|
363
|
+
}
|
|
151
364
|
/**
|
|
152
365
|
* Builds timing metrics from app-emitted profile events.
|
|
153
366
|
*
|
|
@@ -407,6 +620,54 @@ function sortValue(value) {
|
|
|
407
620
|
}
|
|
408
621
|
return value;
|
|
409
622
|
}
|
|
623
|
+
/**
|
|
624
|
+
* Returns a deterministic SHA-256 hash for a JSON-compatible value.
|
|
625
|
+
*
|
|
626
|
+
* @param {unknown} value
|
|
627
|
+
* @returns {string}
|
|
628
|
+
*/
|
|
629
|
+
function hashStableValue(value) {
|
|
630
|
+
return crypto.createHash('sha256').update(JSON.stringify(sortValue(value))).digest('hex');
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Normalizes optional run cohort provenance into schema-safe scalar fields.
|
|
634
|
+
*
|
|
635
|
+
* @param {Record<string, unknown> | null | undefined} cohort
|
|
636
|
+
* @returns {Record<string, unknown> | null}
|
|
637
|
+
*/
|
|
638
|
+
function normalizeProvenanceCohort(cohort) {
|
|
639
|
+
if (!cohort || typeof cohort !== 'object') {
|
|
640
|
+
return null;
|
|
641
|
+
}
|
|
642
|
+
const normalized = {
|
|
643
|
+
...(typeof cohort.appId === 'string' ? { appId: cohort.appId } : {}),
|
|
644
|
+
...(typeof cohort.appVersion === 'string' ? { appVersion: cohort.appVersion } : {}),
|
|
645
|
+
...(typeof cohort.buildId === 'string' ? { buildId: cohort.buildId } : {}),
|
|
646
|
+
...(typeof cohort.buildMode === 'string' ? { buildMode: cohort.buildMode } : {}),
|
|
647
|
+
...(typeof cohort.commandTransport === 'string' ? { commandTransport: cohort.commandTransport } : {}),
|
|
648
|
+
...(typeof cohort.deviceClass === 'string' ? { deviceClass: cohort.deviceClass } : {}),
|
|
649
|
+
...(typeof cohort.osVersion === 'string' ? { osVersion: cohort.osVersion } : {}),
|
|
650
|
+
...(typeof cohort.platform === 'string' ? { platform: cohort.platform } : {}),
|
|
651
|
+
...(typeof cohort.runnerName === 'string' ? { runnerName: cohort.runnerName } : {}),
|
|
652
|
+
...(typeof cohort.runnerVersion === 'string' ? { runnerVersion: cohort.runnerVersion } : {}),
|
|
653
|
+
...(typeof cohort.seedIdentity === 'string' ? { seedIdentity: cohort.seedIdentity } : {}),
|
|
654
|
+
...(cohort.featureFlags && typeof cohort.featureFlags === 'object' && !Array.isArray(cohort.featureFlags)
|
|
655
|
+
? { featureFlags: sortValue(cohort.featureFlags) }
|
|
656
|
+
: {}),
|
|
657
|
+
...(Array.isArray(cohort.providers)
|
|
658
|
+
? {
|
|
659
|
+
providers: cohort.providers
|
|
660
|
+
.filter((provider) => provider && typeof provider === 'object')
|
|
661
|
+
.map((provider) => sortValue({
|
|
662
|
+
...(typeof provider.name === 'string' ? { name: provider.name } : {}),
|
|
663
|
+
...(typeof provider.version === 'string' ? { version: provider.version } : {}),
|
|
664
|
+
}))
|
|
665
|
+
.filter((provider) => typeof provider.name === 'string' || typeof provider.version === 'string'),
|
|
666
|
+
}
|
|
667
|
+
: {}),
|
|
668
|
+
};
|
|
669
|
+
return Object.keys(normalized).length > 0 ? sortValue(normalized) : null;
|
|
670
|
+
}
|
|
410
671
|
/**
|
|
411
672
|
* Normalizes event timestamps to milliseconds since run start.
|
|
412
673
|
*
|
|
@@ -513,14 +774,100 @@ function inferTimelineStatus(eventName) {
|
|
|
513
774
|
}
|
|
514
775
|
return 'observed';
|
|
515
776
|
}
|
|
777
|
+
/**
|
|
778
|
+
* Keeps causal-run timeline values within the public artifact schema.
|
|
779
|
+
*
|
|
780
|
+
* App-owned events may carry richer phase/status vocabulary than ASL's stable
|
|
781
|
+
* artifact contract. Preserve that vocabulary in metadata, but emit only schema
|
|
782
|
+
* values at the timeline top level.
|
|
783
|
+
*
|
|
784
|
+
* @param {{phase: string, status: string, metadata: Record<string, unknown>}} options
|
|
785
|
+
* @returns {{phase: string, status: string, metadata: Record<string, unknown>}}
|
|
786
|
+
*/
|
|
787
|
+
function normalizeTimelineContractValues({ metadata, phase, status, }) {
|
|
788
|
+
const normalizedMetadata = { ...metadata };
|
|
789
|
+
let normalizedPhase = phase;
|
|
790
|
+
let normalizedStatus = status;
|
|
791
|
+
if (!CAUSAL_TIMELINE_PHASES.has(normalizedPhase)) {
|
|
792
|
+
normalizedMetadata.appPhase = normalizedPhase;
|
|
793
|
+
normalizedPhase = 'domain';
|
|
794
|
+
}
|
|
795
|
+
if (!CAUSAL_TIMELINE_STATUSES.has(normalizedStatus)) {
|
|
796
|
+
normalizedMetadata.appStatus = normalizedStatus;
|
|
797
|
+
normalizedStatus = 'observed';
|
|
798
|
+
}
|
|
799
|
+
return {
|
|
800
|
+
metadata: normalizedMetadata,
|
|
801
|
+
phase: normalizedPhase,
|
|
802
|
+
status: normalizedStatus,
|
|
803
|
+
};
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Converts profile-session command control entries into causal timeline events.
|
|
807
|
+
*
|
|
808
|
+
* These are ASL control-plane acknowledgements, not product truth events. They
|
|
809
|
+
* let agents verify command ordering and delivery while keeping product verdicts
|
|
810
|
+
* based on app-owned profile events.
|
|
811
|
+
*
|
|
812
|
+
* @param {{entries: Record<string, unknown>[], startedAt?: string}} options
|
|
813
|
+
* @returns {Record<string, unknown>[]}
|
|
814
|
+
*/
|
|
815
|
+
function buildCommandAcknowledgementTimeline({ entries, startedAt, }) {
|
|
816
|
+
return [...(Array.isArray(entries) ? entries : [])]
|
|
817
|
+
.map((entry) => {
|
|
818
|
+
if (!entry || typeof entry !== 'object' || entry.kind !== 'command') {
|
|
819
|
+
return null;
|
|
820
|
+
}
|
|
821
|
+
const atMs = normalizeEventTimestamp({
|
|
822
|
+
event: entry,
|
|
823
|
+
...(typeof startedAt === 'string' ? { startedAt } : {}),
|
|
824
|
+
});
|
|
825
|
+
if (atMs === null) {
|
|
826
|
+
return null;
|
|
827
|
+
}
|
|
828
|
+
const commandStatus = typeof entry.status === 'string' ? entry.status : 'observed';
|
|
829
|
+
const status = commandStatus === 'completed' || commandStatus === 'delivered'
|
|
830
|
+
? 'completed'
|
|
831
|
+
: commandStatus === 'skipped'
|
|
832
|
+
? 'skipped'
|
|
833
|
+
: commandStatus === 'failed'
|
|
834
|
+
? 'failed'
|
|
835
|
+
: commandStatus === 'received' || commandStatus === 'queued'
|
|
836
|
+
? 'started'
|
|
837
|
+
: 'observed';
|
|
838
|
+
const metadata = {
|
|
839
|
+
...(typeof entry.command === 'string' ? { command: entry.command } : {}),
|
|
840
|
+
...(typeof entry.commandId === 'string' ? { commandId: entry.commandId } : {}),
|
|
841
|
+
...(typeof entry.id === 'string' ? { entryId: entry.id } : {}),
|
|
842
|
+
...(typeof entry.queueId === 'string' ? { queueId: entry.queueId } : {}),
|
|
843
|
+
...(typeof entry.sequence === 'number' ? { sequence: entry.sequence } : {}),
|
|
844
|
+
...(typeof entry.source === 'string' ? { source: entry.source } : {}),
|
|
845
|
+
commandStatus,
|
|
846
|
+
...(typeof entry.result === 'string' ? { result: entry.result } : {}),
|
|
847
|
+
...(typeof entry.reason === 'string' ? { reason: entry.reason } : {}),
|
|
848
|
+
...(typeof entry.waitForMilestone === 'string' ? { waitForMilestone: entry.waitForMilestone } : {}),
|
|
849
|
+
...(typeof entry.waitTimeoutMs === 'number' ? { waitTimeoutMs: entry.waitTimeoutMs } : {}),
|
|
850
|
+
};
|
|
851
|
+
return sortValue({
|
|
852
|
+
phase: 'intent',
|
|
853
|
+
name: `profile_command_${commandStatus}`,
|
|
854
|
+
atMs,
|
|
855
|
+
status,
|
|
856
|
+
owner: 'asl-command-transport',
|
|
857
|
+
metadata,
|
|
858
|
+
});
|
|
859
|
+
})
|
|
860
|
+
.filter(Boolean)
|
|
861
|
+
.sort((left, right) => left.atMs - right.atMs);
|
|
862
|
+
}
|
|
516
863
|
/**
|
|
517
864
|
* Builds a causal timeline from app-owned profile events.
|
|
518
865
|
*
|
|
519
866
|
* @param {{events: Record<string, unknown>[], startedAt?: string, phaseMap?: Record<string, string> | null, owner?: string | null}} options
|
|
520
867
|
* @returns {Record<string, unknown>[]}
|
|
521
868
|
*/
|
|
522
|
-
function buildCausalTimeline({ events, startedAt, phaseMap = null, owner = null, }) {
|
|
523
|
-
|
|
869
|
+
function buildCausalTimeline({ events, sessionEntries = [], startedAt, phaseMap = null, owner = null, }) {
|
|
870
|
+
const eventTimeline = [...(Array.isArray(events) ? events : [])]
|
|
524
871
|
.map((event) => {
|
|
525
872
|
if (!event || typeof event !== 'object' || typeof event.event !== 'string') {
|
|
526
873
|
return null;
|
|
@@ -545,20 +892,78 @@ function buildCausalTimeline({ events, startedAt, phaseMap = null, owner = null,
|
|
|
545
892
|
...(typeof event.flowId === 'string' ? { flowId: event.flowId } : {}),
|
|
546
893
|
...(typeof event.route === 'string' ? { route: event.route } : {}),
|
|
547
894
|
...(typeof event.iteration === 'number' ? { iteration: event.iteration } : {}),
|
|
895
|
+
...(typeof event.sequence === 'number' ? { sequence: event.sequence } : {}),
|
|
896
|
+
...(typeof event.queueId === 'string' ? { queueId: event.queueId } : {}),
|
|
897
|
+
...(typeof event.commandId === 'string' ? { commandId: event.commandId } : {}),
|
|
898
|
+
...(typeof event.operationId === 'string' ? { operationId: event.operationId } : {}),
|
|
899
|
+
...(typeof event.attemptId === 'string' ? { attemptId: event.attemptId } : {}),
|
|
900
|
+
...(typeof event.clockDomain === 'string' ? { clockDomain: event.clockDomain } : {}),
|
|
548
901
|
};
|
|
549
|
-
|
|
902
|
+
const timelineValues = normalizeTimelineContractValues({
|
|
903
|
+
metadata,
|
|
550
904
|
phase: explicitPhase ?? inferTimelinePhase(event.event),
|
|
905
|
+
status: typeof event.status === 'string' ? event.status : inferTimelineStatus(event.event),
|
|
906
|
+
});
|
|
907
|
+
return sortValue({
|
|
908
|
+
phase: timelineValues.phase,
|
|
551
909
|
name: event.event,
|
|
552
910
|
atMs,
|
|
553
|
-
status:
|
|
911
|
+
status: timelineValues.status,
|
|
554
912
|
...((typeof event.owner === 'string' && event.owner.length > 0) || owner
|
|
555
913
|
? { owner: event.owner || owner }
|
|
556
914
|
: {}),
|
|
557
|
-
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
|
915
|
+
...(Object.keys(timelineValues.metadata).length > 0 ? { metadata: timelineValues.metadata } : {}),
|
|
558
916
|
});
|
|
559
917
|
})
|
|
560
|
-
.filter(Boolean)
|
|
561
|
-
|
|
918
|
+
.filter(Boolean);
|
|
919
|
+
const commandTimeline = buildCommandAcknowledgementTimeline({
|
|
920
|
+
entries: sessionEntries,
|
|
921
|
+
...(typeof startedAt === 'string' ? { startedAt } : {}),
|
|
922
|
+
});
|
|
923
|
+
return [...eventTimeline, ...commandTimeline].sort((left, right) => left.atMs - right.atMs);
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* Summarizes repeated scenario accounting for causal artifacts.
|
|
927
|
+
*
|
|
928
|
+
* Metrics already decide product health; this summary makes the iteration
|
|
929
|
+
* evidence explicit for agents reading `causal-run.json`.
|
|
930
|
+
*
|
|
931
|
+
* @param {Record<string, unknown>} metrics
|
|
932
|
+
* @returns {Record<string, unknown> | null}
|
|
933
|
+
*/
|
|
934
|
+
function buildIterationSummary(metrics) {
|
|
935
|
+
if (typeof metrics.iterations !== 'number' || metrics.iterations < 1) {
|
|
936
|
+
return null;
|
|
937
|
+
}
|
|
938
|
+
const expected = Math.trunc(metrics.iterations);
|
|
939
|
+
const failed = typeof metrics.failures === 'number' && metrics.failures > 0
|
|
940
|
+
? Math.trunc(metrics.failures)
|
|
941
|
+
: 0;
|
|
942
|
+
const timeouts = typeof metrics.timeouts === 'number' && metrics.timeouts > 0
|
|
943
|
+
? Math.trunc(metrics.timeouts)
|
|
944
|
+
: 0;
|
|
945
|
+
const incomplete = Array.isArray(metrics.incompleteIterations)
|
|
946
|
+
? [...new Set(metrics.incompleteIterations.filter((iteration) => (typeof iteration === 'number' &&
|
|
947
|
+
Number.isInteger(iteration) &&
|
|
948
|
+
iteration >= 1 &&
|
|
949
|
+
iteration <= expected)))].sort((left, right) => left - right)
|
|
950
|
+
: [];
|
|
951
|
+
const completed = Math.max(0, expected - incomplete.length);
|
|
952
|
+
const status = timeouts > 0
|
|
953
|
+
? 'timeout'
|
|
954
|
+
: failed === 0 && incomplete.length === 0
|
|
955
|
+
? 'complete'
|
|
956
|
+
: completed > 0
|
|
957
|
+
? 'partial'
|
|
958
|
+
: 'failed';
|
|
959
|
+
return {
|
|
960
|
+
completed,
|
|
961
|
+
expected,
|
|
962
|
+
failed,
|
|
963
|
+
incomplete,
|
|
964
|
+
status,
|
|
965
|
+
timeouts,
|
|
966
|
+
};
|
|
562
967
|
}
|
|
563
968
|
/**
|
|
564
969
|
* Builds the `budget-verdict.json` profile artifact from budget evaluation.
|
|
@@ -647,6 +1052,7 @@ function normalizeBudgetsForCausalRun(budgets) {
|
|
|
647
1052
|
* @returns {Record<string, unknown>}
|
|
648
1053
|
*/
|
|
649
1054
|
function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor = 'unknown', interactionDriver, trigger = null, budgets = null, timeline = [], artifacts, manifest, metrics, }) {
|
|
1055
|
+
const iterationSummary = buildIterationSummary(metrics);
|
|
650
1056
|
return sortValue({
|
|
651
1057
|
schemaVersion: '1.0.0',
|
|
652
1058
|
flowId,
|
|
@@ -664,8 +1070,16 @@ function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor
|
|
|
664
1070
|
kind: 'unknown',
|
|
665
1071
|
label: scenario.description ?? scenario.name,
|
|
666
1072
|
},
|
|
1073
|
+
provenanceRef: {
|
|
1074
|
+
manifest: artifacts.manifest,
|
|
1075
|
+
runId,
|
|
1076
|
+
...(typeof manifest.scenarioHash === 'string' && manifest.scenarioHash.length > 0
|
|
1077
|
+
? { scenarioHash: manifest.scenarioHash }
|
|
1078
|
+
: {}),
|
|
1079
|
+
},
|
|
667
1080
|
budgets: normalizeBudgetsForCausalRun(budgets),
|
|
668
1081
|
timeline,
|
|
1082
|
+
...(iterationSummary ? { iterationSummary } : {}),
|
|
669
1083
|
artifacts: {
|
|
670
1084
|
summary: artifacts.summary,
|
|
671
1085
|
metrics: artifacts.metrics,
|
|
@@ -689,7 +1103,61 @@ function buildCausalRun({ scenario, flowId, runId, platform = 'ios', buildFlavor
|
|
|
689
1103
|
* @param {Record<string, unknown>} options
|
|
690
1104
|
* @returns {Record<string, unknown>}
|
|
691
1105
|
*/
|
|
692
|
-
function buildManifest({ scenario, scenarioHash, runId, platform = 'ios', status, startedAt, endedAt, interactionDriver, comparisonLane, simulator, bundleId, gitSha, toolVersions, artifacts, failureReason = null, }) {
|
|
1106
|
+
function buildManifest({ scenario, scenarioHash, runId, attemptId, attemptNumber, maxAttempts, retryOfAttemptId, retryReason, platform = 'ios', status, terminalState, startedAt, endedAt, interactionDriver, comparisonLane, classification, cleanup, partialArtifacts, preconditions, postconditions, simulator, bundleId, gitSha, toolVersions, cohort, artifacts, failureReason = null, }) {
|
|
1107
|
+
const durationMs = roundMs(Math.max(0, Date.parse(endedAt) - Date.parse(startedAt)));
|
|
1108
|
+
const sortedToolVersions = sortValue(toolVersions);
|
|
1109
|
+
const sortedSimulator = sortValue(simulator);
|
|
1110
|
+
const normalizedCohort = normalizeProvenanceCohort(cohort);
|
|
1111
|
+
const cohortHash = normalizedCohort ? hashStableValue(normalizedCohort) : null;
|
|
1112
|
+
const resolvedTerminalState = typeof terminalState === 'string' && terminalState.length > 0 ? terminalState : status;
|
|
1113
|
+
const resolvedAttemptNumber = Number.isInteger(attemptNumber) ? attemptNumber : 1;
|
|
1114
|
+
const resolvedMaxAttempts = Number.isInteger(maxAttempts) ? maxAttempts : Math.max(1, resolvedAttemptNumber);
|
|
1115
|
+
const resolvedAttemptId = typeof attemptId === 'string' && attemptId.length > 0 ? attemptId : runId;
|
|
1116
|
+
const resolvedClassification = classification
|
|
1117
|
+
? sortValue(classification)
|
|
1118
|
+
: {
|
|
1119
|
+
category: status === 'passed' ? 'none' : 'unknown',
|
|
1120
|
+
};
|
|
1121
|
+
const resolvedCleanup = cleanup
|
|
1122
|
+
? sortValue(cleanup)
|
|
1123
|
+
: {
|
|
1124
|
+
status: 'not-required',
|
|
1125
|
+
};
|
|
1126
|
+
const resolvedPartialArtifacts = partialArtifacts
|
|
1127
|
+
? sortValue(partialArtifacts)
|
|
1128
|
+
: {
|
|
1129
|
+
valid: status !== 'passed',
|
|
1130
|
+
reason: status === 'passed'
|
|
1131
|
+
? 'complete successful run artifacts are present'
|
|
1132
|
+
: 'failed run artifacts are preserved for diagnosis and must not be treated as product proof unless health passes',
|
|
1133
|
+
};
|
|
1134
|
+
const resolvedAttempt = {
|
|
1135
|
+
attemptId: resolvedAttemptId,
|
|
1136
|
+
attemptNumber: resolvedAttemptNumber,
|
|
1137
|
+
maxAttempts: resolvedMaxAttempts,
|
|
1138
|
+
...(typeof retryOfAttemptId === 'string' && retryOfAttemptId.length > 0 ? { retryOfAttemptId } : {}),
|
|
1139
|
+
...(typeof retryReason === 'string' && retryReason.length > 0 ? { retryReason } : {}),
|
|
1140
|
+
runId,
|
|
1141
|
+
status,
|
|
1142
|
+
terminalState: resolvedTerminalState,
|
|
1143
|
+
startedAt,
|
|
1144
|
+
endedAt,
|
|
1145
|
+
durationMs,
|
|
1146
|
+
interactionDriver,
|
|
1147
|
+
...(typeof comparisonLane === 'string' && comparisonLane.length > 0 ? { comparisonLane } : {}),
|
|
1148
|
+
classification: resolvedClassification,
|
|
1149
|
+
cleanup: resolvedCleanup,
|
|
1150
|
+
partialArtifacts: resolvedPartialArtifacts,
|
|
1151
|
+
};
|
|
1152
|
+
const resolvedPreconditions = sortValue({
|
|
1153
|
+
...DEFAULT_ENVIRONMENT_PRECONDITIONS,
|
|
1154
|
+
...(preconditions && typeof preconditions === 'object' ? preconditions : {}),
|
|
1155
|
+
});
|
|
1156
|
+
const resolvedPostconditions = sortValue({
|
|
1157
|
+
...DEFAULT_ENVIRONMENT_POSTCONDITIONS,
|
|
1158
|
+
...(postconditions && typeof postconditions === 'object' ? postconditions : {}),
|
|
1159
|
+
});
|
|
1160
|
+
assertAttemptInvariants(resolvedAttempt);
|
|
693
1161
|
return {
|
|
694
1162
|
scenario,
|
|
695
1163
|
...(typeof scenarioHash === 'string' && scenarioHash.length > 0 ? { scenarioHash } : {}),
|
|
@@ -698,13 +1166,28 @@ function buildManifest({ scenario, scenarioHash, runId, platform = 'ios', status
|
|
|
698
1166
|
status,
|
|
699
1167
|
startedAt,
|
|
700
1168
|
endedAt,
|
|
701
|
-
durationMs
|
|
1169
|
+
durationMs,
|
|
702
1170
|
interactionDriver,
|
|
703
1171
|
...(typeof comparisonLane === 'string' && comparisonLane.length > 0 ? { comparisonLane } : {}),
|
|
704
|
-
|
|
1172
|
+
provenance: {
|
|
1173
|
+
...(typeof scenarioHash === 'string' && scenarioHash.length > 0 ? { scenarioHash } : {}),
|
|
1174
|
+
...(normalizedCohort ? { cohort: normalizedCohort, cohortHash } : {}),
|
|
1175
|
+
gitSha,
|
|
1176
|
+
toolVersions: sortedToolVersions,
|
|
1177
|
+
},
|
|
1178
|
+
attempt: resolvedAttempt,
|
|
1179
|
+
environment: {
|
|
1180
|
+
platform,
|
|
1181
|
+
bundleId,
|
|
1182
|
+
runtimeTarget: sortedSimulator,
|
|
1183
|
+
...(typeof sortedToolVersions.node === 'string' ? { nodeVersion: sortedToolVersions.node } : {}),
|
|
1184
|
+
preconditions: resolvedPreconditions,
|
|
1185
|
+
postconditions: resolvedPostconditions,
|
|
1186
|
+
},
|
|
1187
|
+
simulator: sortedSimulator,
|
|
705
1188
|
bundleId,
|
|
706
1189
|
gitSha,
|
|
707
|
-
toolVersions:
|
|
1190
|
+
toolVersions: sortedToolVersions,
|
|
708
1191
|
artifacts: sortValue(artifacts),
|
|
709
1192
|
failureReason,
|
|
710
1193
|
};
|
|
@@ -737,6 +1220,20 @@ function buildSummaryMarkdown({ manifest, metrics }) {
|
|
|
737
1220
|
const evidenceAttachmentLines = evidenceAttachments.length > 0
|
|
738
1221
|
? evidenceAttachments.map((attachment) => `- ${attachment.channel}/${attachment.kind}: \`${attachment.path}\` (${attachment.sizeBytes} bytes, sha256 ${attachment.sha256})`)
|
|
739
1222
|
: ['- none'];
|
|
1223
|
+
const attempt = manifest.attempt && typeof manifest.attempt === 'object' ? manifest.attempt : {};
|
|
1224
|
+
const attemptLines = [
|
|
1225
|
+
`- Attempt ID: \`${attempt.attemptId ?? manifest.runId}\``,
|
|
1226
|
+
`- Attempt number: ${attempt.attemptNumber ?? 1}/${attempt.maxAttempts ?? 1}`,
|
|
1227
|
+
`- Terminal state: ${attempt.terminalState ?? manifest.status}`,
|
|
1228
|
+
...(typeof attempt.retryOfAttemptId === 'string'
|
|
1229
|
+
? [`- Retry of: \`${attempt.retryOfAttemptId}\``]
|
|
1230
|
+
: []),
|
|
1231
|
+
...(typeof attempt.retryReason === 'string'
|
|
1232
|
+
? [`- Retry reason: ${attempt.retryReason}`]
|
|
1233
|
+
: []),
|
|
1234
|
+
`- Cleanup: ${attempt.cleanup?.status ?? 'unknown'}`,
|
|
1235
|
+
`- Partial artifacts valid: ${attempt.partialArtifacts?.valid === true ? 'yes' : 'no'}`,
|
|
1236
|
+
];
|
|
740
1237
|
const lines = [
|
|
741
1238
|
`# ${String(manifest.platform || 'ios').toUpperCase()} profile run: ${manifest.scenario}`,
|
|
742
1239
|
'',
|
|
@@ -755,6 +1252,10 @@ function buildSummaryMarkdown({ manifest, metrics }) {
|
|
|
755
1252
|
`- p50 cycle: ${metrics.p50Ms === null ? 'n/a' : `${metrics.p50Ms}ms`}`,
|
|
756
1253
|
`- p95 cycle: ${metrics.p95Ms === null ? 'n/a' : `${metrics.p95Ms}ms`}`,
|
|
757
1254
|
'',
|
|
1255
|
+
'## Attempt',
|
|
1256
|
+
'',
|
|
1257
|
+
...attemptLines,
|
|
1258
|
+
'',
|
|
758
1259
|
'## Artifact paths',
|
|
759
1260
|
'',
|
|
760
1261
|
`- Causal run: \`${manifest.artifacts.causalRun}\``,
|