@nebulacomponents/citable 1.18.2 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/dist/universal/.agents/skills/citable/SKILL.md +1 -1
- package/dist/universal/.agents/skills/citable/VERSION +1 -1
- package/dist/universal/.agents/skills/citable/manifest.json +9 -6
- package/dist/universal/.agents/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.agents/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.agents/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.agents/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.claude/agents/citable/manifest.json +2 -2
- package/dist/universal/.claude/skills/citable/SKILL.md +1 -1
- package/dist/universal/.claude/skills/citable/VERSION +1 -1
- package/dist/universal/.claude/skills/citable/manifest.json +9 -6
- package/dist/universal/.claude/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.claude/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.claude/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.claude/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.cursor/skills/citable/SKILL.md +1 -1
- package/dist/universal/.cursor/skills/citable/VERSION +1 -1
- package/dist/universal/.cursor/skills/citable/manifest.json +9 -6
- package/dist/universal/.cursor/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.cursor/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.cursor/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.cursor/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.gemini/skills/citable/SKILL.md +1 -1
- package/dist/universal/.gemini/skills/citable/VERSION +1 -1
- package/dist/universal/.gemini/skills/citable/manifest.json +9 -6
- package/dist/universal/.gemini/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.gemini/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.gemini/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.gemini/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.github/skills/citable/SKILL.md +1 -1
- package/dist/universal/.github/skills/citable/VERSION +1 -1
- package/dist/universal/.github/skills/citable/manifest.json +9 -6
- package/dist/universal/.github/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.github/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.github/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.github/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.kiro/skills/citable/SKILL.md +1 -1
- package/dist/universal/.kiro/skills/citable/VERSION +1 -1
- package/dist/universal/.kiro/skills/citable/manifest.json +9 -6
- package/dist/universal/.kiro/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.kiro/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.kiro/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.kiro/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.opencode/skills/citable/SKILL.md +1 -1
- package/dist/universal/.opencode/skills/citable/VERSION +1 -1
- package/dist/universal/.opencode/skills/citable/manifest.json +9 -6
- package/dist/universal/.opencode/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.opencode/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.opencode/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.opencode/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.pi/agent/skills/citable/SKILL.md +1 -1
- package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
- package/dist/universal/.pi/agent/skills/citable/manifest.json +9 -6
- package/dist/universal/.pi/agent/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.pi/agent/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.pi/agent/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.pi/agent/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.qoder/skills/citable/SKILL.md +1 -1
- package/dist/universal/.qoder/skills/citable/VERSION +1 -1
- package/dist/universal/.qoder/skills/citable/manifest.json +9 -6
- package/dist/universal/.qoder/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.qoder/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.qoder/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.qoder/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.rovodev/skills/citable/SKILL.md +1 -1
- package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
- package/dist/universal/.rovodev/skills/citable/manifest.json +9 -6
- package/dist/universal/.rovodev/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.rovodev/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.rovodev/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.rovodev/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.trae/skills/citable/SKILL.md +1 -1
- package/dist/universal/.trae/skills/citable/VERSION +1 -1
- package/dist/universal/.trae/skills/citable/manifest.json +9 -6
- package/dist/universal/.trae/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.trae/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.trae/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.trae/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/.trae-cn/skills/citable/SKILL.md +1 -1
- package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
- package/dist/universal/.trae-cn/skills/citable/manifest.json +9 -6
- package/dist/universal/.trae-cn/skills/citable/schemas/browser-evidence-plan.schema.json +186 -3
- package/dist/universal/.trae-cn/skills/citable/schemas/browser-network-events.schema.json +142 -0
- package/dist/universal/.trae-cn/skills/citable/schemas/browser-runtime-events.schema.json +112 -0
- package/dist/universal/.trae-cn/skills/citable/schemas/browser-state-observations.schema.json +97 -0
- package/dist/universal/manifest.json +112 -76
- package/package.json +1 -1
- package/schemas/browser-evidence-plan.schema.json +186 -3
- package/schemas/browser-network-events.schema.json +142 -0
- package/schemas/browser-runtime-events.schema.json +112 -0
- package/schemas/browser-state-observations.schema.json +97 -0
- package/skill/SKILL.md +1 -1
- package/src/observations/browser/correlation.js +83 -0
- package/src/observations/browser/networkCollector.js +413 -0
- package/src/observations/browser/runtimeEventCollector.js +337 -0
- package/src/observations/browser/sanitizer.js +322 -0
- package/src/observations/browser/stateCollector.js +386 -0
- package/src/observations/browserJourney.js +131 -3
- package/src/reporting/executiveCroReport.js +1 -1
- package/src/reporting/executiveSearchReport.js +1 -1
- package/src/sow/generateSow.js +1 -1
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
import { sha256 } from '../../shared/io.js';
|
|
2
|
+
import {
|
|
3
|
+
sanitizeQueryParams,
|
|
4
|
+
isSensitiveKey,
|
|
5
|
+
isSensitiveValue,
|
|
6
|
+
REDACTED_MARKER,
|
|
7
|
+
} from './sanitizer.js';
|
|
8
|
+
|
|
9
|
+
export class StateCollector {
|
|
10
|
+
constructor(options = {}) {
|
|
11
|
+
const {
|
|
12
|
+
statePolicy = {},
|
|
13
|
+
correlationTracker,
|
|
14
|
+
planId,
|
|
15
|
+
profileId,
|
|
16
|
+
} = options;
|
|
17
|
+
|
|
18
|
+
this.policy = statePolicy;
|
|
19
|
+
this.enabled = Boolean(this.policy.enabled);
|
|
20
|
+
this.planId = planId || 'plan-unspecified';
|
|
21
|
+
this.profileId = profileId || 'profile-unspecified';
|
|
22
|
+
this.tracker = correlationTracker;
|
|
23
|
+
|
|
24
|
+
this.checkpointsConfig = this.policy.checkpoints || ['final'];
|
|
25
|
+
this.captureUrl = this.policy.capture_url !== false;
|
|
26
|
+
this.allowedQueryParams = this.policy.allowed_query_params || null;
|
|
27
|
+
this.captureReferrer = Boolean(this.policy.capture_referrer);
|
|
28
|
+
this.captureNavigation = Boolean(this.policy.capture_navigation);
|
|
29
|
+
this.cookiesPolicy = this.policy.cookies || null;
|
|
30
|
+
this.localStoragePolicy = this.policy.local_storage || null;
|
|
31
|
+
this.sessionStoragePolicy = this.policy.session_storage || null;
|
|
32
|
+
this.globalsConfig = this.policy.globals || [];
|
|
33
|
+
this.domQueriesConfig = this.policy.dom_queries || [];
|
|
34
|
+
this.maxObservations = Math.min(this.policy.max_observations || 100, 500);
|
|
35
|
+
this.maxValueLength = Math.min(this.policy.max_value_length || 1024, 4096);
|
|
36
|
+
|
|
37
|
+
this.checkpoints = [];
|
|
38
|
+
this.summary = {
|
|
39
|
+
total_checkpoints: 0,
|
|
40
|
+
total_observations: 0,
|
|
41
|
+
truncated: false,
|
|
42
|
+
truncation_reason: null,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
shouldCaptureCheckpoint(type) {
|
|
47
|
+
if (!this.enabled) return false;
|
|
48
|
+
return this.checkpointsConfig.includes(type);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async observeCheckpoint(page, context, checkpointId, checkpointType) {
|
|
52
|
+
if (!this.enabled) return null;
|
|
53
|
+
|
|
54
|
+
if (this.checkpoints.length >= this.maxObservations) {
|
|
55
|
+
this.summary.truncated = true;
|
|
56
|
+
this.summary.truncation_reason = 'max_observations_exceeded';
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const seq = this.tracker.nextSequence();
|
|
61
|
+
const timing = this.tracker.timing();
|
|
62
|
+
const correlation = this.tracker.getCorrelation(checkpointId);
|
|
63
|
+
let redactedCount = 0;
|
|
64
|
+
const unavailableItems = [];
|
|
65
|
+
|
|
66
|
+
// 1. URL & query params
|
|
67
|
+
const rawUrl = page.url();
|
|
68
|
+
let sanitizedQuery = null;
|
|
69
|
+
if (this.allowedQueryParams) {
|
|
70
|
+
const queryResult = sanitizeQueryParams(rawUrl, this.allowedQueryParams);
|
|
71
|
+
sanitizedQuery = queryResult.query;
|
|
72
|
+
redactedCount += queryResult.redactedCount;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// 2. Referrer & Navigation
|
|
76
|
+
let referrer = null;
|
|
77
|
+
let navType = null;
|
|
78
|
+
try {
|
|
79
|
+
if (this.captureReferrer || this.captureNavigation) {
|
|
80
|
+
const meta = await page.evaluate(() => ({
|
|
81
|
+
referrer: document.referrer || null,
|
|
82
|
+
navType: (performance.getEntriesByType('navigation')[0] && performance.getEntriesByType('navigation')[0].type) || null,
|
|
83
|
+
}));
|
|
84
|
+
if (this.captureReferrer) referrer = meta.referrer;
|
|
85
|
+
if (this.captureNavigation) navType = meta.navType;
|
|
86
|
+
}
|
|
87
|
+
} catch {
|
|
88
|
+
unavailableItems.push('referrer_or_navigation');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// 3. Cookies (explicitly governed only)
|
|
92
|
+
let cookiesObs = null;
|
|
93
|
+
if (this.cookiesPolicy && Array.isArray(this.cookiesPolicy.allowed_names) && this.cookiesPolicy.allowed_names.length > 0) {
|
|
94
|
+
cookiesObs = {};
|
|
95
|
+
const mode = this.cookiesPolicy.mode || 'presence_only';
|
|
96
|
+
try {
|
|
97
|
+
const cookies = await context.cookies([rawUrl]);
|
|
98
|
+
const cookieMap = new Map(cookies.map((c) => [c.name, c]));
|
|
99
|
+
for (const name of this.cookiesPolicy.allowed_names) {
|
|
100
|
+
if (!cookieMap.has(name)) {
|
|
101
|
+
cookiesObs[name] = { exists: false };
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
const c = cookieMap.get(name);
|
|
105
|
+
const val = c.value ?? '';
|
|
106
|
+
const len = val.length;
|
|
107
|
+
const isHttpOnly = Boolean(c.httpOnly);
|
|
108
|
+
const isSecure = Boolean(c.secure);
|
|
109
|
+
const sameSite = c.sameSite || null;
|
|
110
|
+
|
|
111
|
+
const baseInfo = {
|
|
112
|
+
exists: true,
|
|
113
|
+
length: len,
|
|
114
|
+
http_only: isHttpOnly,
|
|
115
|
+
secure: isSecure,
|
|
116
|
+
same_site: sameSite,
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
if (mode === 'presence_only') {
|
|
120
|
+
cookiesObs[name] = baseInfo;
|
|
121
|
+
} else if (mode === 'hash_only') {
|
|
122
|
+
cookiesObs[name] = { ...baseInfo, value_hash: sha256(val) };
|
|
123
|
+
} else if (mode === 'allowlist_values') {
|
|
124
|
+
if (isHttpOnly) {
|
|
125
|
+
// Policy restricts raw HttpOnly values from plaintext exposure
|
|
126
|
+
cookiesObs[name] = { ...baseInfo, value: '[POLICY_RESTRICTED_HTTPONLY]', disposition: 'policy_restricted' };
|
|
127
|
+
} else if (isSensitiveKey(name) || isSensitiveValue(val)) {
|
|
128
|
+
cookiesObs[name] = { ...baseInfo, value: REDACTED_MARKER };
|
|
129
|
+
redactedCount++;
|
|
130
|
+
} else {
|
|
131
|
+
const bounded = val.length > this.maxValueLength ? val.slice(0, this.maxValueLength) + '...[TRUNCATED]' : val;
|
|
132
|
+
cookiesObs[name] = { ...baseInfo, value: bounded };
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
} catch {
|
|
137
|
+
unavailableItems.push('cookies:context_error');
|
|
138
|
+
cookiesObs = null;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// 4. Local Storage (explicitly governed only)
|
|
143
|
+
let localObs = null;
|
|
144
|
+
if (this.localStoragePolicy && Array.isArray(this.localStoragePolicy.allowed_keys) && this.localStoragePolicy.allowed_keys.length > 0) {
|
|
145
|
+
localObs = {};
|
|
146
|
+
const mode = this.localStoragePolicy.mode || 'presence_only';
|
|
147
|
+
try {
|
|
148
|
+
const entries = await page.evaluate((keys) => {
|
|
149
|
+
const res = {};
|
|
150
|
+
for (const k of keys) {
|
|
151
|
+
try {
|
|
152
|
+
res[k] = window.localStorage.getItem(k);
|
|
153
|
+
} catch {
|
|
154
|
+
res[k] = null;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return res;
|
|
158
|
+
}, this.localStoragePolicy.allowed_keys);
|
|
159
|
+
|
|
160
|
+
for (const key of this.localStoragePolicy.allowed_keys) {
|
|
161
|
+
const val = entries[key];
|
|
162
|
+
if (val === null || val === undefined) {
|
|
163
|
+
localObs[key] = { exists: false };
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
const len = val.length;
|
|
167
|
+
if (mode === 'presence_only') {
|
|
168
|
+
localObs[key] = { exists: true, length: len };
|
|
169
|
+
} else if (mode === 'hash_only') {
|
|
170
|
+
localObs[key] = { exists: true, length: len, value_hash: sha256(val) };
|
|
171
|
+
} else if (mode === 'allowlist_values') {
|
|
172
|
+
if (isSensitiveKey(key) || isSensitiveValue(val)) {
|
|
173
|
+
localObs[key] = { exists: true, length: len, value: REDACTED_MARKER };
|
|
174
|
+
redactedCount++;
|
|
175
|
+
} else {
|
|
176
|
+
const bounded = val.length > this.maxValueLength ? val.slice(0, this.maxValueLength) + '...[TRUNCATED]' : val;
|
|
177
|
+
localObs[key] = { exists: true, length: len, value: bounded };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
} catch {
|
|
182
|
+
unavailableItems.push('local_storage');
|
|
183
|
+
localObs = null;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// 5. Session Storage (explicitly governed only)
|
|
188
|
+
let sessionObs = null;
|
|
189
|
+
if (this.sessionStoragePolicy && Array.isArray(this.sessionStoragePolicy.allowed_keys) && this.sessionStoragePolicy.allowed_keys.length > 0) {
|
|
190
|
+
sessionObs = {};
|
|
191
|
+
const mode = this.sessionStoragePolicy.mode || 'presence_only';
|
|
192
|
+
try {
|
|
193
|
+
const entries = await page.evaluate((keys) => {
|
|
194
|
+
const res = {};
|
|
195
|
+
for (const k of keys) {
|
|
196
|
+
try {
|
|
197
|
+
res[k] = window.sessionStorage.getItem(k);
|
|
198
|
+
} catch {
|
|
199
|
+
res[k] = null;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return res;
|
|
203
|
+
}, this.sessionStoragePolicy.allowed_keys);
|
|
204
|
+
|
|
205
|
+
for (const key of this.sessionStoragePolicy.allowed_keys) {
|
|
206
|
+
const val = entries[key];
|
|
207
|
+
if (val === null || val === undefined) {
|
|
208
|
+
sessionObs[key] = { exists: false };
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
const len = val.length;
|
|
212
|
+
if (mode === 'presence_only') {
|
|
213
|
+
sessionObs[key] = { exists: true, length: len };
|
|
214
|
+
} else if (mode === 'hash_only') {
|
|
215
|
+
sessionObs[key] = { exists: true, length: len, value_hash: sha256(val) };
|
|
216
|
+
} else if (mode === 'allowlist_values') {
|
|
217
|
+
if (isSensitiveKey(key) || isSensitiveValue(val)) {
|
|
218
|
+
sessionObs[key] = { exists: true, length: len, value: REDACTED_MARKER };
|
|
219
|
+
redactedCount++;
|
|
220
|
+
} else {
|
|
221
|
+
const bounded = val.length > this.maxValueLength ? val.slice(0, this.maxValueLength) + '...[TRUNCATED]' : val;
|
|
222
|
+
sessionObs[key] = { exists: true, length: len, value: bounded };
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
} catch {
|
|
227
|
+
unavailableItems.push('session_storage');
|
|
228
|
+
sessionObs = null;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// 6. Globals (safe expressions only)
|
|
233
|
+
let globalsObs = null;
|
|
234
|
+
if (this.globalsConfig && this.globalsConfig.length > 0) {
|
|
235
|
+
globalsObs = {};
|
|
236
|
+
for (const item of this.globalsConfig) {
|
|
237
|
+
// Enforce safe expression regex
|
|
238
|
+
if (!/^[a-zA-Z0-9_$.[\]'"]+$/.test(item.expression)) {
|
|
239
|
+
unavailableItems.push(`global:${item.name}:unsafe_expression`);
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
try {
|
|
243
|
+
const evalRes = await page.evaluate((expr) => {
|
|
244
|
+
try {
|
|
245
|
+
// Safe evaluation using Function or window path traversal
|
|
246
|
+
const parts = expr.replace(/^window\./, '').split('.');
|
|
247
|
+
let curr = window;
|
|
248
|
+
for (const part of parts) {
|
|
249
|
+
if (curr == null) return { exists: false };
|
|
250
|
+
curr = curr[part];
|
|
251
|
+
}
|
|
252
|
+
if (curr === undefined) return { exists: false };
|
|
253
|
+
const type = typeof curr;
|
|
254
|
+
if (type === 'string' || type === 'number' || type === 'boolean') {
|
|
255
|
+
return { exists: true, type, value: String(curr) };
|
|
256
|
+
}
|
|
257
|
+
if (curr === null) return { exists: true, type: 'null', value: 'null' };
|
|
258
|
+
return { exists: true, type, value: JSON.stringify(curr) };
|
|
259
|
+
} catch {
|
|
260
|
+
return { exists: false };
|
|
261
|
+
}
|
|
262
|
+
}, item.expression);
|
|
263
|
+
|
|
264
|
+
if (!evalRes.exists) {
|
|
265
|
+
globalsObs[item.name] = { exists: false };
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const mode = item.mode || 'presence_only';
|
|
270
|
+
if (mode === 'presence_only') {
|
|
271
|
+
globalsObs[item.name] = { exists: true };
|
|
272
|
+
} else if (mode === 'type_only') {
|
|
273
|
+
globalsObs[item.name] = { exists: true, type: evalRes.type };
|
|
274
|
+
} else if (mode === 'hash_only') {
|
|
275
|
+
globalsObs[item.name] = { exists: true, type: evalRes.type, value_hash: sha256(evalRes.value) };
|
|
276
|
+
} else if (mode === 'scalar_value') {
|
|
277
|
+
if (isSensitiveKey(item.name) || isSensitiveValue(evalRes.value)) {
|
|
278
|
+
globalsObs[item.name] = { exists: true, type: evalRes.type, value: REDACTED_MARKER };
|
|
279
|
+
redactedCount++;
|
|
280
|
+
} else {
|
|
281
|
+
const bounded = evalRes.value.length > this.maxValueLength ? evalRes.value.slice(0, this.maxValueLength) + '...[TRUNCATED]' : evalRes.value;
|
|
282
|
+
globalsObs[item.name] = { exists: true, type: evalRes.type, value: bounded };
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
} catch {
|
|
286
|
+
unavailableItems.push(`global:${item.name}`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// 7. DOM Queries
|
|
292
|
+
let domObs = null;
|
|
293
|
+
if (this.domQueriesConfig && this.domQueriesConfig.length > 0) {
|
|
294
|
+
domObs = {};
|
|
295
|
+
for (const query of this.domQueriesConfig) {
|
|
296
|
+
try {
|
|
297
|
+
const locator = page.locator(query.selector).first();
|
|
298
|
+
const count = await locator.count();
|
|
299
|
+
if (count === 0) {
|
|
300
|
+
domObs[query.name] = { exists: false };
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
if (query.property === 'presence') {
|
|
305
|
+
domObs[query.name] = { exists: true };
|
|
306
|
+
} else if (query.property === 'visible') {
|
|
307
|
+
const isVisible = await locator.isVisible();
|
|
308
|
+
domObs[query.name] = { exists: true, visible: isVisible };
|
|
309
|
+
} else if (query.property === 'text_content') {
|
|
310
|
+
const text = await locator.innerText();
|
|
311
|
+
if (isSensitiveValue(text)) {
|
|
312
|
+
domObs[query.name] = { exists: true, text: REDACTED_MARKER };
|
|
313
|
+
redactedCount++;
|
|
314
|
+
} else {
|
|
315
|
+
const bounded = text.length > this.maxValueLength ? text.slice(0, this.maxValueLength) + '...[TRUNCATED]' : text;
|
|
316
|
+
domObs[query.name] = { exists: true, text: bounded };
|
|
317
|
+
}
|
|
318
|
+
} else if (query.property === 'text_length') {
|
|
319
|
+
const text = await locator.innerText();
|
|
320
|
+
domObs[query.name] = { exists: true, text_length: text.length };
|
|
321
|
+
} else if (query.property === 'text_hash') {
|
|
322
|
+
const text = await locator.innerText();
|
|
323
|
+
domObs[query.name] = { exists: true, text_hash: sha256(text) };
|
|
324
|
+
} else if (query.property === 'attribute_value' && query.attribute_name) {
|
|
325
|
+
const attr = await locator.getAttribute(query.attribute_name);
|
|
326
|
+
if (attr === null) {
|
|
327
|
+
domObs[query.name] = { exists: true, attribute: null };
|
|
328
|
+
} else if (isSensitiveKey(query.attribute_name) || isSensitiveValue(attr)) {
|
|
329
|
+
domObs[query.name] = { exists: true, attribute: REDACTED_MARKER };
|
|
330
|
+
redactedCount++;
|
|
331
|
+
} else {
|
|
332
|
+
const bounded = attr.length > this.maxValueLength ? attr.slice(0, this.maxValueLength) + '...[TRUNCATED]' : attr;
|
|
333
|
+
domObs[query.name] = { exists: true, attribute: bounded };
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
} catch {
|
|
337
|
+
unavailableItems.push(`dom:${query.name}`);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const checkpointRecord = {
|
|
343
|
+
checkpoint_id: checkpointId,
|
|
344
|
+
sequence: seq,
|
|
345
|
+
timing,
|
|
346
|
+
correlation: {
|
|
347
|
+
step_id: correlation.step_id,
|
|
348
|
+
step_sequence: correlation.step_sequence,
|
|
349
|
+
},
|
|
350
|
+
state: {
|
|
351
|
+
url: rawUrl,
|
|
352
|
+
query_params: sanitizedQuery,
|
|
353
|
+
referrer,
|
|
354
|
+
navigation_type: navType,
|
|
355
|
+
cookies: cookiesObs,
|
|
356
|
+
local_storage: localObs,
|
|
357
|
+
session_storage: sessionObs,
|
|
358
|
+
globals: globalsObs,
|
|
359
|
+
dom_queries: domObs,
|
|
360
|
+
},
|
|
361
|
+
disposition: {
|
|
362
|
+
observation_status: unavailableItems.length > 0 ? 'partially_observed' : 'observed',
|
|
363
|
+
redacted_count: redactedCount,
|
|
364
|
+
unavailable_items: unavailableItems,
|
|
365
|
+
},
|
|
366
|
+
};
|
|
367
|
+
|
|
368
|
+
this.checkpoints.push(checkpointRecord);
|
|
369
|
+
this.summary.total_checkpoints++;
|
|
370
|
+
this.summary.total_observations += 1;
|
|
371
|
+
return checkpointRecord;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
toArtifact() {
|
|
375
|
+
this.summary.collector_status = this.summary.truncated ? 'truncated' : 'complete';
|
|
376
|
+
return {
|
|
377
|
+
schema_version: 1,
|
|
378
|
+
plan_id: this.planId,
|
|
379
|
+
profile_id: this.profileId,
|
|
380
|
+
collected_at: this.tracker.timing().timestamp,
|
|
381
|
+
capture_policy: this.policy,
|
|
382
|
+
summary: this.summary,
|
|
383
|
+
checkpoints: this.checkpoints,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
}
|
|
@@ -3,8 +3,13 @@ import { fetchUrl, handlePublicBrowserRoute, validatePublicUrl } from '../crawle
|
|
|
3
3
|
import { sha256 } from '../shared/io.js';
|
|
4
4
|
import { validateAgainst } from '../shared/schemaValidator.js';
|
|
5
5
|
import { analyzeRenderParity } from './renderParity.js';
|
|
6
|
+
import { CorrelationTracker } from './browser/correlation.js';
|
|
7
|
+
import { NetworkCollector } from './browser/networkCollector.js';
|
|
8
|
+
import { StateCollector } from './browser/stateCollector.js';
|
|
9
|
+
import { RuntimeEventCollector } from './browser/runtimeEventCollector.js';
|
|
6
10
|
|
|
7
11
|
async function executeStep(page, step, lookup) {
|
|
12
|
+
if (step.action === 'checkpoint') return;
|
|
8
13
|
const locator = step.locator ? page.locator(step.locator).first() : null;
|
|
9
14
|
if (step.action === 'navigate') {
|
|
10
15
|
await validatePublicUrl(step.url, lookup ? { lookup } : {});
|
|
@@ -30,6 +35,34 @@ async function playwrightCapture(profile, plan, playwright, lookup) {
|
|
|
30
35
|
if (profile.authentication_state !== 'anonymous') throw new Error(`authentication state ${profile.authentication_state} requires a disclosed custom adapter`);
|
|
31
36
|
const browserType = playwright[profile.browser.engine];
|
|
32
37
|
if (!browserType) throw new Error(`Playwright engine ${profile.browser.engine} is unavailable`);
|
|
38
|
+
|
|
39
|
+
const correlationTracker = new CorrelationTracker();
|
|
40
|
+
const networkPolicy = profile.network_policy || plan.network_policy || { enabled: false };
|
|
41
|
+
const statePolicy = profile.state_policy || plan.state_policy || { enabled: false };
|
|
42
|
+
const runtimeEventPolicy = profile.runtime_event_policy || plan.runtime_event_policy || { enabled: false };
|
|
43
|
+
|
|
44
|
+
const networkCollector = new NetworkCollector({
|
|
45
|
+
networkPolicy,
|
|
46
|
+
targetUrl: plan.target,
|
|
47
|
+
correlationTracker,
|
|
48
|
+
planId: plan.plan_id,
|
|
49
|
+
profileId: profile.profile_id,
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
const stateCollector = new StateCollector({
|
|
53
|
+
statePolicy,
|
|
54
|
+
correlationTracker,
|
|
55
|
+
planId: plan.plan_id,
|
|
56
|
+
profileId: profile.profile_id,
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
const runtimeEventCollector = new RuntimeEventCollector({
|
|
60
|
+
runtimeEventPolicy,
|
|
61
|
+
correlationTracker,
|
|
62
|
+
planId: plan.plan_id,
|
|
63
|
+
profileId: profile.profile_id,
|
|
64
|
+
});
|
|
65
|
+
|
|
33
66
|
const browser = await browserType.launch({ headless: true });
|
|
34
67
|
try {
|
|
35
68
|
const context = await browser.newContext({
|
|
@@ -40,28 +73,86 @@ async function playwrightCapture(profile, plan, playwright, lookup) {
|
|
|
40
73
|
});
|
|
41
74
|
try {
|
|
42
75
|
await context.route('**/*', (route) => handlePublicBrowserRoute(route, lookup ? { lookup } : {}));
|
|
76
|
+
|
|
77
|
+
const initScript = runtimeEventCollector.getInitScript();
|
|
78
|
+
if (initScript && typeof context.addInitScript === 'function') {
|
|
79
|
+
await context.addInitScript(initScript);
|
|
80
|
+
}
|
|
81
|
+
|
|
43
82
|
const page = await context.newPage();
|
|
83
|
+
networkCollector.attachToPage(page);
|
|
84
|
+
|
|
44
85
|
const consoleErrors = [], networkFailures = [], steps = [], stepScreenshots = {};
|
|
45
86
|
page.on('console', (message) => { if (message.type() === 'error') consoleErrors.push(message.text()); });
|
|
46
87
|
page.on('requestfailed', (request) => networkFailures.push({ url: request.url(), error: request.failure()?.errorText || 'unknown' }));
|
|
88
|
+
|
|
47
89
|
const response = await page.goto(plan.target, { waitUntil: 'networkidle' });
|
|
48
|
-
|
|
90
|
+
await runtimeEventCollector.installOnPage(page);
|
|
91
|
+
await runtimeEventCollector.drainEvents(page);
|
|
92
|
+
|
|
93
|
+
if (stateCollector.shouldCaptureCheckpoint('initial')) {
|
|
94
|
+
await stateCollector.observeCheckpoint(page, context, 'initial', 'initial');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
for (let i = 0; i < profile.steps.length; i++) {
|
|
98
|
+
const step = profile.steps[i];
|
|
99
|
+
const stepSequence = i + 1;
|
|
100
|
+
correlationTracker.startStep(step, stepSequence);
|
|
49
101
|
const record = { step_id: step.step_id, action: step.action, status: 'completed', failure: null, screenshot_ref: null };
|
|
50
102
|
try {
|
|
51
|
-
|
|
103
|
+
if (step.action === 'checkpoint') {
|
|
104
|
+
const chkId = step.checkpoint_id || `step_${step.step_id}`;
|
|
105
|
+
await stateCollector.observeCheckpoint(page, context, chkId, 'step');
|
|
106
|
+
} else {
|
|
107
|
+
await executeStep(page, step, lookup);
|
|
108
|
+
}
|
|
109
|
+
|
|
52
110
|
if (step.capture_screenshot) {
|
|
53
111
|
record.screenshot_ref = `journeys/${profile.profile_id}/steps/${step.step_id}.png`;
|
|
54
112
|
stepScreenshots[record.screenshot_ref] = await page.screenshot({ fullPage: true });
|
|
55
113
|
}
|
|
114
|
+
|
|
115
|
+
await runtimeEventCollector.drainEvents(page);
|
|
116
|
+
|
|
117
|
+
if (step.action !== 'checkpoint' && (step.checkpoint_id || step.state_checkpoint || stateCollector.shouldCaptureCheckpoint('step'))) {
|
|
118
|
+
const chkId = step.checkpoint_id || `step_${step.step_id}`;
|
|
119
|
+
await stateCollector.observeCheckpoint(page, context, chkId, 'step');
|
|
120
|
+
}
|
|
56
121
|
} catch (error) {
|
|
57
122
|
record.status = 'failed';
|
|
58
123
|
record.failure = error.message;
|
|
59
124
|
steps.push(record);
|
|
60
|
-
|
|
125
|
+
correlationTracker.endStep(step, stepSequence);
|
|
126
|
+
if (step.required) {
|
|
127
|
+
throw Object.assign(new Error(`required step ${step.step_id} failed: ${error.message}`), {
|
|
128
|
+
steps,
|
|
129
|
+
stepScreenshots,
|
|
130
|
+
network_events: networkCollector.enabled ? networkCollector.toArtifact() : null,
|
|
131
|
+
state_observations: stateCollector.enabled ? stateCollector.toArtifact() : null,
|
|
132
|
+
runtime_events: runtimeEventCollector.enabled ? runtimeEventCollector.toArtifact() : null,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
61
135
|
continue;
|
|
62
136
|
}
|
|
63
137
|
steps.push(record);
|
|
138
|
+
correlationTracker.endStep(step, stepSequence);
|
|
64
139
|
}
|
|
140
|
+
|
|
141
|
+
correlationTracker.finishJourney();
|
|
142
|
+
await runtimeEventCollector.drainEvents(page);
|
|
143
|
+
|
|
144
|
+
const settlingTimeout = networkPolicy.settling_timeout_ms || 0;
|
|
145
|
+
if (settlingTimeout > 0) {
|
|
146
|
+
try {
|
|
147
|
+
await page.waitForTimeout(Math.min(settlingTimeout, 5000));
|
|
148
|
+
await runtimeEventCollector.drainEvents(page);
|
|
149
|
+
} catch {}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (stateCollector.shouldCaptureCheckpoint('final')) {
|
|
153
|
+
await stateCollector.observeCheckpoint(page, context, 'final', 'final');
|
|
154
|
+
}
|
|
155
|
+
|
|
65
156
|
const dom = await page.content();
|
|
66
157
|
const text = await page.locator('body').innerText();
|
|
67
158
|
const accessibilityTree = typeof page.locator('body').ariaSnapshot === 'function' ? await page.locator('body').ariaSnapshot() : null;
|
|
@@ -69,6 +160,9 @@ async function playwrightCapture(profile, plan, playwright, lookup) {
|
|
|
69
160
|
final_url: page.url(), status: response?.status() ?? null, browser_version: browser.version(),
|
|
70
161
|
dom, text, accessibility_tree: accessibilityTree, screenshot: await page.screenshot({ fullPage: true }),
|
|
71
162
|
console_errors: consoleErrors, network_failures: networkFailures, steps, step_screenshots: stepScreenshots,
|
|
163
|
+
network_events: networkCollector.enabled ? networkCollector.toArtifact() : null,
|
|
164
|
+
state_observations: stateCollector.enabled ? stateCollector.toArtifact() : null,
|
|
165
|
+
runtime_events: runtimeEventCollector.enabled ? runtimeEventCollector.toArtifact() : null,
|
|
72
166
|
};
|
|
73
167
|
} finally {
|
|
74
168
|
await context.close();
|
|
@@ -95,6 +189,8 @@ export async function observeBrowserPlan(root, options) {
|
|
|
95
189
|
if (step.action === 'navigate') {
|
|
96
190
|
if (!step.url) throw new Error(`${profile.profile_id}/${step.step_id}: navigate requires url`);
|
|
97
191
|
if (new URL(step.url).origin !== targetOrigin) throw new Error(`${profile.profile_id}/${step.step_id}: cross-origin navigation is not allowed`);
|
|
192
|
+
} else if (step.action === 'checkpoint') {
|
|
193
|
+
// checkpoint does not require locator
|
|
98
194
|
} else if (!step.locator) throw new Error(`${profile.profile_id}/${step.step_id}: ${step.action} requires locator`);
|
|
99
195
|
if (['fill', 'select_option'].includes(step.action) && !step.value_env) throw new Error(`${profile.profile_id}/${step.step_id}: ${step.action} requires value_env`);
|
|
100
196
|
if (step.action === 'press' && !step.key) throw new Error(`${profile.profile_id}/${step.step_id}: press requires key`);
|
|
@@ -131,6 +227,9 @@ export async function observeBrowserPlan(root, options) {
|
|
|
131
227
|
screenshot: `${prefix}/final.png`,
|
|
132
228
|
failures: `${prefix}/failures.json`,
|
|
133
229
|
interactions: `${prefix}/steps.json`,
|
|
230
|
+
network_events: capture.network_events ? `${prefix}/network-events.json` : null,
|
|
231
|
+
browser_state: capture.state_observations ? `${prefix}/state-observations.json` : null,
|
|
232
|
+
runtime_events: capture.runtime_events ? `${prefix}/runtime-events.json` : null,
|
|
134
233
|
};
|
|
135
234
|
artifacts[refs.rendered_dom] = capture.dom;
|
|
136
235
|
artifacts[refs.rendered_text] = capture.text;
|
|
@@ -138,6 +237,23 @@ export async function observeBrowserPlan(root, options) {
|
|
|
138
237
|
artifacts[refs.screenshot] = capture.screenshot;
|
|
139
238
|
artifacts[refs.failures] = { console_errors: capture.console_errors || [], network_failures: capture.network_failures || [] };
|
|
140
239
|
artifacts[refs.interactions] = capture.steps || [];
|
|
240
|
+
|
|
241
|
+
if (capture.network_events) {
|
|
242
|
+
const checkNet = validateAgainst('browser-network-events.schema.json', capture.network_events);
|
|
243
|
+
if (!checkNet.valid) throw new Error(`network events artifact violates contract: ${checkNet.errors.join('; ')}`);
|
|
244
|
+
artifacts[refs.network_events] = capture.network_events;
|
|
245
|
+
}
|
|
246
|
+
if (capture.state_observations) {
|
|
247
|
+
const checkState = validateAgainst('browser-state-observations.schema.json', capture.state_observations);
|
|
248
|
+
if (!checkState.valid) throw new Error(`state observations artifact violates contract: ${checkState.errors.join('; ')}`);
|
|
249
|
+
artifacts[refs.browser_state] = capture.state_observations;
|
|
250
|
+
}
|
|
251
|
+
if (capture.runtime_events) {
|
|
252
|
+
const checkEvents = validateAgainst('browser-runtime-events.schema.json', capture.runtime_events);
|
|
253
|
+
if (!checkEvents.valid) throw new Error(`runtime events artifact violates contract: ${checkEvents.errors.join('; ')}`);
|
|
254
|
+
artifacts[refs.runtime_events] = capture.runtime_events;
|
|
255
|
+
}
|
|
256
|
+
|
|
141
257
|
for (const [name, value] of Object.entries(capture.step_screenshots || {})) {
|
|
142
258
|
if (!name.startsWith(`${prefix}/steps/`) || name.includes('..')) throw new Error(`browser adapter returned unsafe artifact path: ${name}`);
|
|
143
259
|
artifacts[name] = value;
|
|
@@ -151,6 +267,9 @@ export async function observeBrowserPlan(root, options) {
|
|
|
151
267
|
consent_state: profile.consent_state, authentication_state: profile.authentication_state,
|
|
152
268
|
final_url: capture.final_url, status: capture.status, dom_hash: sha256(capture.dom), text_hash: sha256(capture.text),
|
|
153
269
|
steps: capture.steps || [], artifact_refs: refs,
|
|
270
|
+
network_summary: capture.network_events ? capture.network_events.summary : null,
|
|
271
|
+
state_summary: capture.state_observations ? capture.state_observations.summary : null,
|
|
272
|
+
runtime_events_summary: capture.runtime_events ? capture.runtime_events.summary : null,
|
|
154
273
|
render_parity: renderParity,
|
|
155
274
|
interpretation_boundary: 'Observed differences do not establish semantic, retrieval, citation, ranking, or business impact.',
|
|
156
275
|
};
|
|
@@ -161,6 +280,15 @@ export async function observeBrowserPlan(root, options) {
|
|
|
161
280
|
incomplete.push(`${profile.profile_id}: ${error.message}`);
|
|
162
281
|
const failureRef = `${prefix}/failure.json`;
|
|
163
282
|
artifacts[failureRef] = { profile_id: profile.profile_id, error: error.message, completed_steps: error.steps || [] };
|
|
283
|
+
if (error.network_events) {
|
|
284
|
+
artifacts[`${prefix}/network-events.json`] = error.network_events;
|
|
285
|
+
}
|
|
286
|
+
if (error.state_observations) {
|
|
287
|
+
artifacts[`${prefix}/state-observations.json`] = error.state_observations;
|
|
288
|
+
}
|
|
289
|
+
if (error.runtime_events) {
|
|
290
|
+
artifacts[`${prefix}/runtime-events.json`] = error.runtime_events;
|
|
291
|
+
}
|
|
164
292
|
for (const [name, value] of Object.entries(error.stepScreenshots || {})) artifacts[name] = value;
|
|
165
293
|
observations.push(envelope('browser_journey', {
|
|
166
294
|
plan_id: plan.plan_id, target: plan.target, profile_id: profile.profile_id, browser: profile.browser,
|
|
@@ -522,7 +522,7 @@ export async function buildExecutiveCroReport(root, options = {}) {
|
|
|
522
522
|
|
|
523
523
|
const generationProvenance = {
|
|
524
524
|
generated_at: generatedAt,
|
|
525
|
-
generator_version: '1.
|
|
525
|
+
generator_version: '1.19.0',
|
|
526
526
|
generation_mode: resolved.generation_mode,
|
|
527
527
|
source_type: resolved.source_type,
|
|
528
528
|
source_identifier: resolved.source_identifier,
|
|
@@ -613,7 +613,7 @@ export async function buildExecutiveSearchReport(root, options = {}) {
|
|
|
613
613
|
|
|
614
614
|
const generationProvenance = {
|
|
615
615
|
generated_at: generatedAt,
|
|
616
|
-
generator_version: '1.
|
|
616
|
+
generator_version: '1.19.0',
|
|
617
617
|
generation_mode: resolved.generation_mode,
|
|
618
618
|
source_type: resolved.source_type,
|
|
619
619
|
source_identifier: resolved.source_identifier,
|
package/src/sow/generateSow.js
CHANGED
|
@@ -763,7 +763,7 @@ export async function generateSow(root, {
|
|
|
763
763
|
// Machine-readable Provenance Envelope
|
|
764
764
|
generation_provenance: {
|
|
765
765
|
generated_at: generatedAt,
|
|
766
|
-
generator_version: '1.
|
|
766
|
+
generator_version: '1.19.0',
|
|
767
767
|
generation_mode: generationMode,
|
|
768
768
|
source_type: sourceProvenance.source_type || 'UNKNOWN',
|
|
769
769
|
source_identifier: sourceProvenance.source_identifier || null,
|