@nebulacomponents/citable 1.18.2 → 1.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/dist/universal/.agents/skills/citable/SKILL.md +1 -1
  3. package/dist/universal/.agents/skills/citable/VERSION +1 -1
  4. package/dist/universal/.agents/skills/citable/manifest.json +9 -6
  5. package/dist/universal/.agents/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  6. package/dist/universal/.agents/skills/citable/schemas/browser-network-events.schema.json +142 -0
  7. package/dist/universal/.agents/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  8. package/dist/universal/.agents/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  9. package/dist/universal/.claude/agents/citable/manifest.json +2 -2
  10. package/dist/universal/.claude/skills/citable/SKILL.md +1 -1
  11. package/dist/universal/.claude/skills/citable/VERSION +1 -1
  12. package/dist/universal/.claude/skills/citable/manifest.json +9 -6
  13. package/dist/universal/.claude/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  14. package/dist/universal/.claude/skills/citable/schemas/browser-network-events.schema.json +142 -0
  15. package/dist/universal/.claude/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  16. package/dist/universal/.claude/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  17. package/dist/universal/.cursor/skills/citable/SKILL.md +1 -1
  18. package/dist/universal/.cursor/skills/citable/VERSION +1 -1
  19. package/dist/universal/.cursor/skills/citable/manifest.json +9 -6
  20. package/dist/universal/.cursor/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  21. package/dist/universal/.cursor/skills/citable/schemas/browser-network-events.schema.json +142 -0
  22. package/dist/universal/.cursor/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  23. package/dist/universal/.cursor/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  24. package/dist/universal/.gemini/skills/citable/SKILL.md +1 -1
  25. package/dist/universal/.gemini/skills/citable/VERSION +1 -1
  26. package/dist/universal/.gemini/skills/citable/manifest.json +9 -6
  27. package/dist/universal/.gemini/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  28. package/dist/universal/.gemini/skills/citable/schemas/browser-network-events.schema.json +142 -0
  29. package/dist/universal/.gemini/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  30. package/dist/universal/.gemini/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  31. package/dist/universal/.github/skills/citable/SKILL.md +1 -1
  32. package/dist/universal/.github/skills/citable/VERSION +1 -1
  33. package/dist/universal/.github/skills/citable/manifest.json +9 -6
  34. package/dist/universal/.github/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  35. package/dist/universal/.github/skills/citable/schemas/browser-network-events.schema.json +142 -0
  36. package/dist/universal/.github/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  37. package/dist/universal/.github/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  38. package/dist/universal/.kiro/skills/citable/SKILL.md +1 -1
  39. package/dist/universal/.kiro/skills/citable/VERSION +1 -1
  40. package/dist/universal/.kiro/skills/citable/manifest.json +9 -6
  41. package/dist/universal/.kiro/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  42. package/dist/universal/.kiro/skills/citable/schemas/browser-network-events.schema.json +142 -0
  43. package/dist/universal/.kiro/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  44. package/dist/universal/.kiro/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  45. package/dist/universal/.opencode/skills/citable/SKILL.md +1 -1
  46. package/dist/universal/.opencode/skills/citable/VERSION +1 -1
  47. package/dist/universal/.opencode/skills/citable/manifest.json +9 -6
  48. package/dist/universal/.opencode/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  49. package/dist/universal/.opencode/skills/citable/schemas/browser-network-events.schema.json +142 -0
  50. package/dist/universal/.opencode/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  51. package/dist/universal/.opencode/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  52. package/dist/universal/.pi/agent/skills/citable/SKILL.md +1 -1
  53. package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
  54. package/dist/universal/.pi/agent/skills/citable/manifest.json +9 -6
  55. package/dist/universal/.pi/agent/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  56. package/dist/universal/.pi/agent/skills/citable/schemas/browser-network-events.schema.json +142 -0
  57. package/dist/universal/.pi/agent/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  58. package/dist/universal/.pi/agent/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  59. package/dist/universal/.qoder/skills/citable/SKILL.md +1 -1
  60. package/dist/universal/.qoder/skills/citable/VERSION +1 -1
  61. package/dist/universal/.qoder/skills/citable/manifest.json +9 -6
  62. package/dist/universal/.qoder/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  63. package/dist/universal/.qoder/skills/citable/schemas/browser-network-events.schema.json +142 -0
  64. package/dist/universal/.qoder/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  65. package/dist/universal/.qoder/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  66. package/dist/universal/.rovodev/skills/citable/SKILL.md +1 -1
  67. package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
  68. package/dist/universal/.rovodev/skills/citable/manifest.json +9 -6
  69. package/dist/universal/.rovodev/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  70. package/dist/universal/.rovodev/skills/citable/schemas/browser-network-events.schema.json +142 -0
  71. package/dist/universal/.rovodev/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  72. package/dist/universal/.rovodev/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  73. package/dist/universal/.trae/skills/citable/SKILL.md +1 -1
  74. package/dist/universal/.trae/skills/citable/VERSION +1 -1
  75. package/dist/universal/.trae/skills/citable/manifest.json +9 -6
  76. package/dist/universal/.trae/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  77. package/dist/universal/.trae/skills/citable/schemas/browser-network-events.schema.json +142 -0
  78. package/dist/universal/.trae/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  79. package/dist/universal/.trae/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  80. package/dist/universal/.trae-cn/skills/citable/SKILL.md +1 -1
  81. package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
  82. package/dist/universal/.trae-cn/skills/citable/manifest.json +9 -6
  83. package/dist/universal/.trae-cn/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
  84. package/dist/universal/.trae-cn/skills/citable/schemas/browser-network-events.schema.json +142 -0
  85. package/dist/universal/.trae-cn/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
  86. package/dist/universal/.trae-cn/skills/citable/schemas/browser-state-observations.schema.json +97 -0
  87. package/dist/universal/manifest.json +112 -76
  88. package/package.json +1 -1
  89. package/schemas/browser-evidence-plan.schema.json +186 -3
  90. package/schemas/browser-network-events.schema.json +142 -0
  91. package/schemas/browser-runtime-events.schema.json +112 -0
  92. package/schemas/browser-state-observations.schema.json +97 -0
  93. package/skill/SKILL.md +1 -1
  94. package/src/observations/browser/correlation.js +83 -0
  95. package/src/observations/browser/networkCollector.js +413 -0
  96. package/src/observations/browser/runtimeEventCollector.js +337 -0
  97. package/src/observations/browser/sanitizer.js +322 -0
  98. package/src/observations/browser/stateCollector.js +386 -0
  99. package/src/observations/browserJourney.js +131 -3
  100. package/src/reporting/executiveCroReport.js +1 -1
  101. package/src/reporting/executiveSearchReport.js +1 -1
  102. package/src/sow/generateSow.js +1 -1
@@ -0,0 +1,386 @@
1
+ import { sha256 } from '../../shared/io.js';
2
+ import {
3
+ sanitizeQueryParams,
4
+ isSensitiveKey,
5
+ isSensitiveValue,
6
+ REDACTED_MARKER,
7
+ } from './sanitizer.js';
8
+
9
+ export class StateCollector {
10
+ constructor(options = {}) {
11
+ const {
12
+ statePolicy = {},
13
+ correlationTracker,
14
+ planId,
15
+ profileId,
16
+ } = options;
17
+
18
+ this.policy = statePolicy;
19
+ this.enabled = Boolean(this.policy.enabled);
20
+ this.planId = planId || 'plan-unspecified';
21
+ this.profileId = profileId || 'profile-unspecified';
22
+ this.tracker = correlationTracker;
23
+
24
+ this.checkpointsConfig = this.policy.checkpoints || ['final'];
25
+ this.captureUrl = this.policy.capture_url !== false;
26
+ this.allowedQueryParams = this.policy.allowed_query_params || null;
27
+ this.captureReferrer = Boolean(this.policy.capture_referrer);
28
+ this.captureNavigation = Boolean(this.policy.capture_navigation);
29
+ this.cookiesPolicy = this.policy.cookies || null;
30
+ this.localStoragePolicy = this.policy.local_storage || null;
31
+ this.sessionStoragePolicy = this.policy.session_storage || null;
32
+ this.globalsConfig = this.policy.globals || [];
33
+ this.domQueriesConfig = this.policy.dom_queries || [];
34
+ this.maxObservations = Math.min(this.policy.max_observations || 100, 500);
35
+ this.maxValueLength = Math.min(this.policy.max_value_length || 1024, 4096);
36
+
37
+ this.checkpoints = [];
38
+ this.summary = {
39
+ total_checkpoints: 0,
40
+ total_observations: 0,
41
+ truncated: false,
42
+ truncation_reason: null,
43
+ };
44
+ }
45
+
46
+ shouldCaptureCheckpoint(type) {
47
+ if (!this.enabled) return false;
48
+ return this.checkpointsConfig.includes(type);
49
+ }
50
+
51
+ async observeCheckpoint(page, context, checkpointId, checkpointType) {
52
+ if (!this.enabled) return null;
53
+
54
+ if (this.checkpoints.length >= this.maxObservations) {
55
+ this.summary.truncated = true;
56
+ this.summary.truncation_reason = 'max_observations_exceeded';
57
+ return null;
58
+ }
59
+
60
+ const seq = this.tracker.nextSequence();
61
+ const timing = this.tracker.timing();
62
+ const correlation = this.tracker.getCorrelation(checkpointId);
63
+ let redactedCount = 0;
64
+ const unavailableItems = [];
65
+
66
+ // 1. URL & query params
67
+ const rawUrl = page.url();
68
+ let sanitizedQuery = null;
69
+ if (this.allowedQueryParams) {
70
+ const queryResult = sanitizeQueryParams(rawUrl, this.allowedQueryParams);
71
+ sanitizedQuery = queryResult.query;
72
+ redactedCount += queryResult.redactedCount;
73
+ }
74
+
75
+ // 2. Referrer & Navigation
76
+ let referrer = null;
77
+ let navType = null;
78
+ try {
79
+ if (this.captureReferrer || this.captureNavigation) {
80
+ const meta = await page.evaluate(() => ({
81
+ referrer: document.referrer || null,
82
+ navType: (performance.getEntriesByType('navigation')[0] && performance.getEntriesByType('navigation')[0].type) || null,
83
+ }));
84
+ if (this.captureReferrer) referrer = meta.referrer;
85
+ if (this.captureNavigation) navType = meta.navType;
86
+ }
87
+ } catch {
88
+ unavailableItems.push('referrer_or_navigation');
89
+ }
90
+
91
+ // 3. Cookies (explicitly governed only)
92
+ let cookiesObs = null;
93
+ if (this.cookiesPolicy && Array.isArray(this.cookiesPolicy.allowed_names) && this.cookiesPolicy.allowed_names.length > 0) {
94
+ cookiesObs = {};
95
+ const mode = this.cookiesPolicy.mode || 'presence_only';
96
+ try {
97
+ const cookies = await context.cookies([rawUrl]);
98
+ const cookieMap = new Map(cookies.map((c) => [c.name, c]));
99
+ for (const name of this.cookiesPolicy.allowed_names) {
100
+ if (!cookieMap.has(name)) {
101
+ cookiesObs[name] = { exists: false };
102
+ continue;
103
+ }
104
+ const c = cookieMap.get(name);
105
+ const val = c.value ?? '';
106
+ const len = val.length;
107
+ const isHttpOnly = Boolean(c.httpOnly);
108
+ const isSecure = Boolean(c.secure);
109
+ const sameSite = c.sameSite || null;
110
+
111
+ const baseInfo = {
112
+ exists: true,
113
+ length: len,
114
+ http_only: isHttpOnly,
115
+ secure: isSecure,
116
+ same_site: sameSite,
117
+ };
118
+
119
+ if (mode === 'presence_only') {
120
+ cookiesObs[name] = baseInfo;
121
+ } else if (mode === 'hash_only') {
122
+ cookiesObs[name] = { ...baseInfo, value_hash: sha256(val) };
123
+ } else if (mode === 'allowlist_values') {
124
+ if (isHttpOnly) {
125
+ // Policy restricts raw HttpOnly values from plaintext exposure
126
+ cookiesObs[name] = { ...baseInfo, value: '[POLICY_RESTRICTED_HTTPONLY]', disposition: 'policy_restricted' };
127
+ } else if (isSensitiveKey(name) || isSensitiveValue(val)) {
128
+ cookiesObs[name] = { ...baseInfo, value: REDACTED_MARKER };
129
+ redactedCount++;
130
+ } else {
131
+ const bounded = val.length > this.maxValueLength ? val.slice(0, this.maxValueLength) + '...[TRUNCATED]' : val;
132
+ cookiesObs[name] = { ...baseInfo, value: bounded };
133
+ }
134
+ }
135
+ }
136
+ } catch {
137
+ unavailableItems.push('cookies:context_error');
138
+ cookiesObs = null;
139
+ }
140
+ }
141
+
142
+ // 4. Local Storage (explicitly governed only)
143
+ let localObs = null;
144
+ if (this.localStoragePolicy && Array.isArray(this.localStoragePolicy.allowed_keys) && this.localStoragePolicy.allowed_keys.length > 0) {
145
+ localObs = {};
146
+ const mode = this.localStoragePolicy.mode || 'presence_only';
147
+ try {
148
+ const entries = await page.evaluate((keys) => {
149
+ const res = {};
150
+ for (const k of keys) {
151
+ try {
152
+ res[k] = window.localStorage.getItem(k);
153
+ } catch {
154
+ res[k] = null;
155
+ }
156
+ }
157
+ return res;
158
+ }, this.localStoragePolicy.allowed_keys);
159
+
160
+ for (const key of this.localStoragePolicy.allowed_keys) {
161
+ const val = entries[key];
162
+ if (val === null || val === undefined) {
163
+ localObs[key] = { exists: false };
164
+ continue;
165
+ }
166
+ const len = val.length;
167
+ if (mode === 'presence_only') {
168
+ localObs[key] = { exists: true, length: len };
169
+ } else if (mode === 'hash_only') {
170
+ localObs[key] = { exists: true, length: len, value_hash: sha256(val) };
171
+ } else if (mode === 'allowlist_values') {
172
+ if (isSensitiveKey(key) || isSensitiveValue(val)) {
173
+ localObs[key] = { exists: true, length: len, value: REDACTED_MARKER };
174
+ redactedCount++;
175
+ } else {
176
+ const bounded = val.length > this.maxValueLength ? val.slice(0, this.maxValueLength) + '...[TRUNCATED]' : val;
177
+ localObs[key] = { exists: true, length: len, value: bounded };
178
+ }
179
+ }
180
+ }
181
+ } catch {
182
+ unavailableItems.push('local_storage');
183
+ localObs = null;
184
+ }
185
+ }
186
+
187
+ // 5. Session Storage (explicitly governed only)
188
+ let sessionObs = null;
189
+ if (this.sessionStoragePolicy && Array.isArray(this.sessionStoragePolicy.allowed_keys) && this.sessionStoragePolicy.allowed_keys.length > 0) {
190
+ sessionObs = {};
191
+ const mode = this.sessionStoragePolicy.mode || 'presence_only';
192
+ try {
193
+ const entries = await page.evaluate((keys) => {
194
+ const res = {};
195
+ for (const k of keys) {
196
+ try {
197
+ res[k] = window.sessionStorage.getItem(k);
198
+ } catch {
199
+ res[k] = null;
200
+ }
201
+ }
202
+ return res;
203
+ }, this.sessionStoragePolicy.allowed_keys);
204
+
205
+ for (const key of this.sessionStoragePolicy.allowed_keys) {
206
+ const val = entries[key];
207
+ if (val === null || val === undefined) {
208
+ sessionObs[key] = { exists: false };
209
+ continue;
210
+ }
211
+ const len = val.length;
212
+ if (mode === 'presence_only') {
213
+ sessionObs[key] = { exists: true, length: len };
214
+ } else if (mode === 'hash_only') {
215
+ sessionObs[key] = { exists: true, length: len, value_hash: sha256(val) };
216
+ } else if (mode === 'allowlist_values') {
217
+ if (isSensitiveKey(key) || isSensitiveValue(val)) {
218
+ sessionObs[key] = { exists: true, length: len, value: REDACTED_MARKER };
219
+ redactedCount++;
220
+ } else {
221
+ const bounded = val.length > this.maxValueLength ? val.slice(0, this.maxValueLength) + '...[TRUNCATED]' : val;
222
+ sessionObs[key] = { exists: true, length: len, value: bounded };
223
+ }
224
+ }
225
+ }
226
+ } catch {
227
+ unavailableItems.push('session_storage');
228
+ sessionObs = null;
229
+ }
230
+ }
231
+
232
+ // 6. Globals (safe expressions only)
233
+ let globalsObs = null;
234
+ if (this.globalsConfig && this.globalsConfig.length > 0) {
235
+ globalsObs = {};
236
+ for (const item of this.globalsConfig) {
237
+ // Enforce safe expression regex
238
+ if (!/^[a-zA-Z0-9_$.[\]'"]+$/.test(item.expression)) {
239
+ unavailableItems.push(`global:${item.name}:unsafe_expression`);
240
+ continue;
241
+ }
242
+ try {
243
+ const evalRes = await page.evaluate((expr) => {
244
+ try {
245
+ // Safe evaluation using Function or window path traversal
246
+ const parts = expr.replace(/^window\./, '').split('.');
247
+ let curr = window;
248
+ for (const part of parts) {
249
+ if (curr == null) return { exists: false };
250
+ curr = curr[part];
251
+ }
252
+ if (curr === undefined) return { exists: false };
253
+ const type = typeof curr;
254
+ if (type === 'string' || type === 'number' || type === 'boolean') {
255
+ return { exists: true, type, value: String(curr) };
256
+ }
257
+ if (curr === null) return { exists: true, type: 'null', value: 'null' };
258
+ return { exists: true, type, value: JSON.stringify(curr) };
259
+ } catch {
260
+ return { exists: false };
261
+ }
262
+ }, item.expression);
263
+
264
+ if (!evalRes.exists) {
265
+ globalsObs[item.name] = { exists: false };
266
+ continue;
267
+ }
268
+
269
+ const mode = item.mode || 'presence_only';
270
+ if (mode === 'presence_only') {
271
+ globalsObs[item.name] = { exists: true };
272
+ } else if (mode === 'type_only') {
273
+ globalsObs[item.name] = { exists: true, type: evalRes.type };
274
+ } else if (mode === 'hash_only') {
275
+ globalsObs[item.name] = { exists: true, type: evalRes.type, value_hash: sha256(evalRes.value) };
276
+ } else if (mode === 'scalar_value') {
277
+ if (isSensitiveKey(item.name) || isSensitiveValue(evalRes.value)) {
278
+ globalsObs[item.name] = { exists: true, type: evalRes.type, value: REDACTED_MARKER };
279
+ redactedCount++;
280
+ } else {
281
+ const bounded = evalRes.value.length > this.maxValueLength ? evalRes.value.slice(0, this.maxValueLength) + '...[TRUNCATED]' : evalRes.value;
282
+ globalsObs[item.name] = { exists: true, type: evalRes.type, value: bounded };
283
+ }
284
+ }
285
+ } catch {
286
+ unavailableItems.push(`global:${item.name}`);
287
+ }
288
+ }
289
+ }
290
+
291
+ // 7. DOM Queries
292
+ let domObs = null;
293
+ if (this.domQueriesConfig && this.domQueriesConfig.length > 0) {
294
+ domObs = {};
295
+ for (const query of this.domQueriesConfig) {
296
+ try {
297
+ const locator = page.locator(query.selector).first();
298
+ const count = await locator.count();
299
+ if (count === 0) {
300
+ domObs[query.name] = { exists: false };
301
+ continue;
302
+ }
303
+
304
+ if (query.property === 'presence') {
305
+ domObs[query.name] = { exists: true };
306
+ } else if (query.property === 'visible') {
307
+ const isVisible = await locator.isVisible();
308
+ domObs[query.name] = { exists: true, visible: isVisible };
309
+ } else if (query.property === 'text_content') {
310
+ const text = await locator.innerText();
311
+ if (isSensitiveValue(text)) {
312
+ domObs[query.name] = { exists: true, text: REDACTED_MARKER };
313
+ redactedCount++;
314
+ } else {
315
+ const bounded = text.length > this.maxValueLength ? text.slice(0, this.maxValueLength) + '...[TRUNCATED]' : text;
316
+ domObs[query.name] = { exists: true, text: bounded };
317
+ }
318
+ } else if (query.property === 'text_length') {
319
+ const text = await locator.innerText();
320
+ domObs[query.name] = { exists: true, text_length: text.length };
321
+ } else if (query.property === 'text_hash') {
322
+ const text = await locator.innerText();
323
+ domObs[query.name] = { exists: true, text_hash: sha256(text) };
324
+ } else if (query.property === 'attribute_value' && query.attribute_name) {
325
+ const attr = await locator.getAttribute(query.attribute_name);
326
+ if (attr === null) {
327
+ domObs[query.name] = { exists: true, attribute: null };
328
+ } else if (isSensitiveKey(query.attribute_name) || isSensitiveValue(attr)) {
329
+ domObs[query.name] = { exists: true, attribute: REDACTED_MARKER };
330
+ redactedCount++;
331
+ } else {
332
+ const bounded = attr.length > this.maxValueLength ? attr.slice(0, this.maxValueLength) + '...[TRUNCATED]' : attr;
333
+ domObs[query.name] = { exists: true, attribute: bounded };
334
+ }
335
+ }
336
+ } catch {
337
+ unavailableItems.push(`dom:${query.name}`);
338
+ }
339
+ }
340
+ }
341
+
342
+ const checkpointRecord = {
343
+ checkpoint_id: checkpointId,
344
+ sequence: seq,
345
+ timing,
346
+ correlation: {
347
+ step_id: correlation.step_id,
348
+ step_sequence: correlation.step_sequence,
349
+ },
350
+ state: {
351
+ url: rawUrl,
352
+ query_params: sanitizedQuery,
353
+ referrer,
354
+ navigation_type: navType,
355
+ cookies: cookiesObs,
356
+ local_storage: localObs,
357
+ session_storage: sessionObs,
358
+ globals: globalsObs,
359
+ dom_queries: domObs,
360
+ },
361
+ disposition: {
362
+ observation_status: unavailableItems.length > 0 ? 'partially_observed' : 'observed',
363
+ redacted_count: redactedCount,
364
+ unavailable_items: unavailableItems,
365
+ },
366
+ };
367
+
368
+ this.checkpoints.push(checkpointRecord);
369
+ this.summary.total_checkpoints++;
370
+ this.summary.total_observations += 1;
371
+ return checkpointRecord;
372
+ }
373
+
374
+ toArtifact() {
375
+ this.summary.collector_status = this.summary.truncated ? 'truncated' : 'complete';
376
+ return {
377
+ schema_version: 1,
378
+ plan_id: this.planId,
379
+ profile_id: this.profileId,
380
+ collected_at: this.tracker.timing().timestamp,
381
+ capture_policy: this.policy,
382
+ summary: this.summary,
383
+ checkpoints: this.checkpoints,
384
+ };
385
+ }
386
+ }
@@ -3,8 +3,13 @@ import { fetchUrl, handlePublicBrowserRoute, validatePublicUrl } from '../crawle
3
3
  import { sha256 } from '../shared/io.js';
4
4
  import { validateAgainst } from '../shared/schemaValidator.js';
5
5
  import { analyzeRenderParity } from './renderParity.js';
6
+ import { CorrelationTracker } from './browser/correlation.js';
7
+ import { NetworkCollector } from './browser/networkCollector.js';
8
+ import { StateCollector } from './browser/stateCollector.js';
9
+ import { RuntimeEventCollector } from './browser/runtimeEventCollector.js';
6
10
 
7
11
  async function executeStep(page, step, lookup) {
12
+ if (step.action === 'checkpoint') return;
8
13
  const locator = step.locator ? page.locator(step.locator).first() : null;
9
14
  if (step.action === 'navigate') {
10
15
  await validatePublicUrl(step.url, lookup ? { lookup } : {});
@@ -30,6 +35,34 @@ async function playwrightCapture(profile, plan, playwright, lookup) {
30
35
  if (profile.authentication_state !== 'anonymous') throw new Error(`authentication state ${profile.authentication_state} requires a disclosed custom adapter`);
31
36
  const browserType = playwright[profile.browser.engine];
32
37
  if (!browserType) throw new Error(`Playwright engine ${profile.browser.engine} is unavailable`);
38
+
39
+ const correlationTracker = new CorrelationTracker();
40
+ const networkPolicy = profile.network_policy || plan.network_policy || { enabled: false };
41
+ const statePolicy = profile.state_policy || plan.state_policy || { enabled: false };
42
+ const runtimeEventPolicy = profile.runtime_event_policy || plan.runtime_event_policy || { enabled: false };
43
+
44
+ const networkCollector = new NetworkCollector({
45
+ networkPolicy,
46
+ targetUrl: plan.target,
47
+ correlationTracker,
48
+ planId: plan.plan_id,
49
+ profileId: profile.profile_id,
50
+ });
51
+
52
+ const stateCollector = new StateCollector({
53
+ statePolicy,
54
+ correlationTracker,
55
+ planId: plan.plan_id,
56
+ profileId: profile.profile_id,
57
+ });
58
+
59
+ const runtimeEventCollector = new RuntimeEventCollector({
60
+ runtimeEventPolicy,
61
+ correlationTracker,
62
+ planId: plan.plan_id,
63
+ profileId: profile.profile_id,
64
+ });
65
+
33
66
  const browser = await browserType.launch({ headless: true });
34
67
  try {
35
68
  const context = await browser.newContext({
@@ -40,28 +73,86 @@ async function playwrightCapture(profile, plan, playwright, lookup) {
40
73
  });
41
74
  try {
42
75
  await context.route('**/*', (route) => handlePublicBrowserRoute(route, lookup ? { lookup } : {}));
76
+
77
+ const initScript = runtimeEventCollector.getInitScript();
78
+ if (initScript && typeof context.addInitScript === 'function') {
79
+ await context.addInitScript(initScript);
80
+ }
81
+
43
82
  const page = await context.newPage();
83
+ networkCollector.attachToPage(page);
84
+
44
85
  const consoleErrors = [], networkFailures = [], steps = [], stepScreenshots = {};
45
86
  page.on('console', (message) => { if (message.type() === 'error') consoleErrors.push(message.text()); });
46
87
  page.on('requestfailed', (request) => networkFailures.push({ url: request.url(), error: request.failure()?.errorText || 'unknown' }));
88
+
47
89
  const response = await page.goto(plan.target, { waitUntil: 'networkidle' });
48
- for (const step of profile.steps) {
90
+ await runtimeEventCollector.installOnPage(page);
91
+ await runtimeEventCollector.drainEvents(page);
92
+
93
+ if (stateCollector.shouldCaptureCheckpoint('initial')) {
94
+ await stateCollector.observeCheckpoint(page, context, 'initial', 'initial');
95
+ }
96
+
97
+ for (let i = 0; i < profile.steps.length; i++) {
98
+ const step = profile.steps[i];
99
+ const stepSequence = i + 1;
100
+ correlationTracker.startStep(step, stepSequence);
49
101
  const record = { step_id: step.step_id, action: step.action, status: 'completed', failure: null, screenshot_ref: null };
50
102
  try {
51
- await executeStep(page, step, lookup);
103
+ if (step.action === 'checkpoint') {
104
+ const chkId = step.checkpoint_id || `step_${step.step_id}`;
105
+ await stateCollector.observeCheckpoint(page, context, chkId, 'step');
106
+ } else {
107
+ await executeStep(page, step, lookup);
108
+ }
109
+
52
110
  if (step.capture_screenshot) {
53
111
  record.screenshot_ref = `journeys/${profile.profile_id}/steps/${step.step_id}.png`;
54
112
  stepScreenshots[record.screenshot_ref] = await page.screenshot({ fullPage: true });
55
113
  }
114
+
115
+ await runtimeEventCollector.drainEvents(page);
116
+
117
+ if (step.action !== 'checkpoint' && (step.checkpoint_id || step.state_checkpoint || stateCollector.shouldCaptureCheckpoint('step'))) {
118
+ const chkId = step.checkpoint_id || `step_${step.step_id}`;
119
+ await stateCollector.observeCheckpoint(page, context, chkId, 'step');
120
+ }
56
121
  } catch (error) {
57
122
  record.status = 'failed';
58
123
  record.failure = error.message;
59
124
  steps.push(record);
60
- if (step.required) throw Object.assign(new Error(`required step ${step.step_id} failed: ${error.message}`), { steps, stepScreenshots });
125
+ correlationTracker.endStep(step, stepSequence);
126
+ if (step.required) {
127
+ throw Object.assign(new Error(`required step ${step.step_id} failed: ${error.message}`), {
128
+ steps,
129
+ stepScreenshots,
130
+ network_events: networkCollector.enabled ? networkCollector.toArtifact() : null,
131
+ state_observations: stateCollector.enabled ? stateCollector.toArtifact() : null,
132
+ runtime_events: runtimeEventCollector.enabled ? runtimeEventCollector.toArtifact() : null,
133
+ });
134
+ }
61
135
  continue;
62
136
  }
63
137
  steps.push(record);
138
+ correlationTracker.endStep(step, stepSequence);
64
139
  }
140
+
141
+ correlationTracker.finishJourney();
142
+ await runtimeEventCollector.drainEvents(page);
143
+
144
+ const settlingTimeout = networkPolicy.settling_timeout_ms || 0;
145
+ if (settlingTimeout > 0) {
146
+ try {
147
+ await page.waitForTimeout(Math.min(settlingTimeout, 5000));
148
+ await runtimeEventCollector.drainEvents(page);
149
+ } catch {}
150
+ }
151
+
152
+ if (stateCollector.shouldCaptureCheckpoint('final')) {
153
+ await stateCollector.observeCheckpoint(page, context, 'final', 'final');
154
+ }
155
+
65
156
  const dom = await page.content();
66
157
  const text = await page.locator('body').innerText();
67
158
  const accessibilityTree = typeof page.locator('body').ariaSnapshot === 'function' ? await page.locator('body').ariaSnapshot() : null;
@@ -69,6 +160,9 @@ async function playwrightCapture(profile, plan, playwright, lookup) {
69
160
  final_url: page.url(), status: response?.status() ?? null, browser_version: browser.version(),
70
161
  dom, text, accessibility_tree: accessibilityTree, screenshot: await page.screenshot({ fullPage: true }),
71
162
  console_errors: consoleErrors, network_failures: networkFailures, steps, step_screenshots: stepScreenshots,
163
+ network_events: networkCollector.enabled ? networkCollector.toArtifact() : null,
164
+ state_observations: stateCollector.enabled ? stateCollector.toArtifact() : null,
165
+ runtime_events: runtimeEventCollector.enabled ? runtimeEventCollector.toArtifact() : null,
72
166
  };
73
167
  } finally {
74
168
  await context.close();
@@ -95,6 +189,8 @@ export async function observeBrowserPlan(root, options) {
95
189
  if (step.action === 'navigate') {
96
190
  if (!step.url) throw new Error(`${profile.profile_id}/${step.step_id}: navigate requires url`);
97
191
  if (new URL(step.url).origin !== targetOrigin) throw new Error(`${profile.profile_id}/${step.step_id}: cross-origin navigation is not allowed`);
192
+ } else if (step.action === 'checkpoint') {
193
+ // checkpoint does not require locator
98
194
  } else if (!step.locator) throw new Error(`${profile.profile_id}/${step.step_id}: ${step.action} requires locator`);
99
195
  if (['fill', 'select_option'].includes(step.action) && !step.value_env) throw new Error(`${profile.profile_id}/${step.step_id}: ${step.action} requires value_env`);
100
196
  if (step.action === 'press' && !step.key) throw new Error(`${profile.profile_id}/${step.step_id}: press requires key`);
@@ -131,6 +227,9 @@ export async function observeBrowserPlan(root, options) {
131
227
  screenshot: `${prefix}/final.png`,
132
228
  failures: `${prefix}/failures.json`,
133
229
  interactions: `${prefix}/steps.json`,
230
+ network_events: capture.network_events ? `${prefix}/network-events.json` : null,
231
+ browser_state: capture.state_observations ? `${prefix}/state-observations.json` : null,
232
+ runtime_events: capture.runtime_events ? `${prefix}/runtime-events.json` : null,
134
233
  };
135
234
  artifacts[refs.rendered_dom] = capture.dom;
136
235
  artifacts[refs.rendered_text] = capture.text;
@@ -138,6 +237,23 @@ export async function observeBrowserPlan(root, options) {
138
237
  artifacts[refs.screenshot] = capture.screenshot;
139
238
  artifacts[refs.failures] = { console_errors: capture.console_errors || [], network_failures: capture.network_failures || [] };
140
239
  artifacts[refs.interactions] = capture.steps || [];
240
+
241
+ if (capture.network_events) {
242
+ const checkNet = validateAgainst('browser-network-events.schema.json', capture.network_events);
243
+ if (!checkNet.valid) throw new Error(`network events artifact violates contract: ${checkNet.errors.join('; ')}`);
244
+ artifacts[refs.network_events] = capture.network_events;
245
+ }
246
+ if (capture.state_observations) {
247
+ const checkState = validateAgainst('browser-state-observations.schema.json', capture.state_observations);
248
+ if (!checkState.valid) throw new Error(`state observations artifact violates contract: ${checkState.errors.join('; ')}`);
249
+ artifacts[refs.browser_state] = capture.state_observations;
250
+ }
251
+ if (capture.runtime_events) {
252
+ const checkEvents = validateAgainst('browser-runtime-events.schema.json', capture.runtime_events);
253
+ if (!checkEvents.valid) throw new Error(`runtime events artifact violates contract: ${checkEvents.errors.join('; ')}`);
254
+ artifacts[refs.runtime_events] = capture.runtime_events;
255
+ }
256
+
141
257
  for (const [name, value] of Object.entries(capture.step_screenshots || {})) {
142
258
  if (!name.startsWith(`${prefix}/steps/`) || name.includes('..')) throw new Error(`browser adapter returned unsafe artifact path: ${name}`);
143
259
  artifacts[name] = value;
@@ -151,6 +267,9 @@ export async function observeBrowserPlan(root, options) {
151
267
  consent_state: profile.consent_state, authentication_state: profile.authentication_state,
152
268
  final_url: capture.final_url, status: capture.status, dom_hash: sha256(capture.dom), text_hash: sha256(capture.text),
153
269
  steps: capture.steps || [], artifact_refs: refs,
270
+ network_summary: capture.network_events ? capture.network_events.summary : null,
271
+ state_summary: capture.state_observations ? capture.state_observations.summary : null,
272
+ runtime_events_summary: capture.runtime_events ? capture.runtime_events.summary : null,
154
273
  render_parity: renderParity,
155
274
  interpretation_boundary: 'Observed differences do not establish semantic, retrieval, citation, ranking, or business impact.',
156
275
  };
@@ -161,6 +280,15 @@ export async function observeBrowserPlan(root, options) {
161
280
  incomplete.push(`${profile.profile_id}: ${error.message}`);
162
281
  const failureRef = `${prefix}/failure.json`;
163
282
  artifacts[failureRef] = { profile_id: profile.profile_id, error: error.message, completed_steps: error.steps || [] };
283
+ if (error.network_events) {
284
+ artifacts[`${prefix}/network-events.json`] = error.network_events;
285
+ }
286
+ if (error.state_observations) {
287
+ artifacts[`${prefix}/state-observations.json`] = error.state_observations;
288
+ }
289
+ if (error.runtime_events) {
290
+ artifacts[`${prefix}/runtime-events.json`] = error.runtime_events;
291
+ }
164
292
  for (const [name, value] of Object.entries(error.stepScreenshots || {})) artifacts[name] = value;
165
293
  observations.push(envelope('browser_journey', {
166
294
  plan_id: plan.plan_id, target: plan.target, profile_id: profile.profile_id, browser: profile.browser,
@@ -522,7 +522,7 @@ export async function buildExecutiveCroReport(root, options = {}) {
522
522
 
523
523
  const generationProvenance = {
524
524
  generated_at: generatedAt,
525
- generator_version: '1.18.2',
525
+ generator_version: '1.19.0',
526
526
  generation_mode: resolved.generation_mode,
527
527
  source_type: resolved.source_type,
528
528
  source_identifier: resolved.source_identifier,
@@ -613,7 +613,7 @@ export async function buildExecutiveSearchReport(root, options = {}) {
613
613
 
614
614
  const generationProvenance = {
615
615
  generated_at: generatedAt,
616
- generator_version: '1.18.2',
616
+ generator_version: '1.19.0',
617
617
  generation_mode: resolved.generation_mode,
618
618
  source_type: resolved.source_type,
619
619
  source_identifier: resolved.source_identifier,
@@ -763,7 +763,7 @@ export async function generateSow(root, {
763
763
  // Machine-readable Provenance Envelope
764
764
  generation_provenance: {
765
765
  generated_at: generatedAt,
766
- generator_version: '1.18.2',
766
+ generator_version: '1.19.0',
767
767
  generation_mode: generationMode,
768
768
  source_type: sourceProvenance.source_type || 'UNKNOWN',
769
769
  source_identifier: sourceProvenance.source_identifier || null,