@rezkam/browser-tools 0.2.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,485 @@
1
+ import { createRequire } from 'node:module';
2
+ import {
3
+ hasFlag,
4
+ optionValue,
5
+ parsePositiveIntegerOption,
6
+ sleep,
7
+ } from './browser-control.mjs';
8
+ import {
9
+ isSensitiveName,
10
+ matchesPatterns,
11
+ matchesStatus,
12
+ normalizeResourceTypes,
13
+ optionValues,
14
+ patternOptionValues,
15
+ parseStatusSelectors,
16
+ redactBodyText,
17
+ redactHeaders,
18
+ redactSensitive,
19
+ redactUrl,
20
+ resourceTypesForPreset,
21
+ writePrivateJson,
22
+ } from './cdp-common.mjs';
23
+ import { parseBoundedNumber } from './gif-recorder.mjs';
24
+
25
+ const { version: BROWSER_TOOLS_VERSION } = createRequire(import.meta.url)('../package.json');
26
+
27
+ export const DEFAULT_HAR_MAX_BODY_BYTES = 1024 * 1024;
28
+ export const DEFAULT_HAR_MAX_DURATION_SECONDS = 300;
29
+ export const DEFAULT_HAR_IDLE_MS = 500;
30
+ export const DEFAULT_HAR_DRAIN_TIMEOUT_MS = 5000;
31
+ export const HAR_CAPTURE_COMPONENTS = ['headers', 'bodies', 'timing'];
32
+
33
+ function headerValue(headers, wanted) {
34
+ const match = Object.entries(headers || {}).find(([name]) => name.toLowerCase() === wanted.toLowerCase());
35
+ return match ? String(match[1]) : '';
36
+ }
37
+
38
+ function queryString(url, redact) {
39
+ try {
40
+ return [...new URL(url).searchParams.entries()].map(([name, value]) => ({
41
+ name,
42
+ value: redact && isSensitiveName(name)
43
+ ? '<redacted>'
44
+ : value,
45
+ }));
46
+ } catch {
47
+ return [];
48
+ }
49
+ }
50
+
51
+ function harHttpVersion(protocol) {
52
+ const value = String(protocol || '').toLowerCase();
53
+ if (value === 'h2') return 'HTTP/2';
54
+ if (value === 'h3') return 'HTTP/3';
55
+ if (value.startsWith('http/')) return value.toUpperCase();
56
+ return protocol || 'HTTP/1.1';
57
+ }
58
+
59
+ function bodyMimeType(headers) {
60
+ return headerValue(headers, 'content-type').split(';')[0] || 'application/octet-stream';
61
+ }
62
+
63
+ function mergedHeaders(primary = {}, extra = null) {
64
+ return { ...primary, ...(extra?.headers || {}) };
65
+ }
66
+
67
+ function requestCookies(extraInfo, redact) {
68
+ return (extraInfo?.associatedCookies || [])
69
+ .filter((associated) => !(associated.blockedReasons || []).length)
70
+ .map(({ cookie }) => ({
71
+ name: cookie.name,
72
+ value: redact ? '<redacted>' : cookie.value,
73
+ path: cookie.path || '/',
74
+ domain: cookie.domain || '',
75
+ expires: Number(cookie.expires) > 0 ? new Date(cookie.expires * 1000).toISOString() : undefined,
76
+ httpOnly: Boolean(cookie.httpOnly),
77
+ secure: Boolean(cookie.secure),
78
+ sameSite: cookie.sameSite || undefined,
79
+ }));
80
+ }
81
+
82
+ function nonNegativeDuration(start, end) {
83
+ if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start) return -1;
84
+ return end - start;
85
+ }
86
+
87
+ function harTimings(entry, captureTiming) {
88
+ if (!captureTiming) {
89
+ return { blocked: -1, dns: -1, connect: -1, ssl: -1, send: -1, wait: -1, receive: -1 };
90
+ }
91
+ const timing = entry.response?.timing || {};
92
+ const hasProtocolTiming = Number.isFinite(timing.requestTime) && timing.receiveHeadersEnd >= 0;
93
+ const wait = hasProtocolTiming
94
+ ? nonNegativeDuration(timing.sendEnd, timing.receiveHeadersEnd)
95
+ : (entry.responseTimestamp ? Math.max(0, (entry.responseTimestamp - entry.requestTimestamp) * 1000) : -1);
96
+ const headersAt = hasProtocolTiming ? timing.requestTime + timing.receiveHeadersEnd / 1000 : entry.responseTimestamp;
97
+ return {
98
+ blocked: -1,
99
+ dns: nonNegativeDuration(timing.dnsStart, timing.dnsEnd),
100
+ connect: nonNegativeDuration(timing.connectStart, timing.connectEnd),
101
+ ssl: nonNegativeDuration(timing.sslStart, timing.sslEnd),
102
+ send: nonNegativeDuration(timing.sendStart, timing.sendEnd),
103
+ wait,
104
+ receive: entry.finishedTimestamp && headersAt
105
+ ? Math.max(0, (entry.finishedTimestamp - headersAt) * 1000)
106
+ : -1,
107
+ };
108
+ }
109
+
110
+ function totalTime(timings) {
111
+ const values = Object.values(timings).filter((value) => value >= 0);
112
+ return values.length ? values.reduce((sum, value) => sum + value, 0) : 0;
113
+ }
114
+
115
+ function truncateText(text, maxBytes) {
116
+ if (text === undefined || text === null) return { text, truncated: false };
117
+ const buffer = Buffer.from(String(text), 'utf-8');
118
+ if (buffer.length <= maxBytes) return { text: String(text), truncated: false };
119
+ return { text: buffer.subarray(0, maxBytes).toString('utf-8'), truncated: true };
120
+ }
121
+
122
+ export function parseHarCaptureOptions(args) {
123
+ const preset = String(optionValue(args, '--preset', 'all')).toLowerCase();
124
+ const explicitTypes = normalizeResourceTypes(optionValues(args, '--resource-type'));
125
+ const presetTypes = explicitTypes.length && !args.includes('--preset') ? [] : resourceTypesForPreset(preset);
126
+ const resourceTypes = [...new Set([...presetTypes, ...explicitTypes])];
127
+ const excludedResourceTypes = normalizeResourceTypes(optionValues(args, '--exclude-resource-type'));
128
+ const methods = optionValues(args, '--method').map((method) => method.toUpperCase());
129
+ const excludedMethods = optionValues(args, '--exclude-method').map((method) => method.toUpperCase());
130
+ const captureValues = optionValues(args, '--capture');
131
+ const capture = captureValues.length ? captureValues : [...HAR_CAPTURE_COMPONENTS];
132
+ for (const component of capture) {
133
+ if (!HAR_CAPTURE_COMPONENTS.includes(component)) {
134
+ throw new Error(`Unknown --capture component: ${component}. Expected: ${HAR_CAPTURE_COMPONENTS.join(', ')}`);
135
+ }
136
+ }
137
+ const minSize = parseBoundedNumber(optionValue(args, '--min-size', null), '--min-size', { fallback: 0, min: 0, max: Number.MAX_SAFE_INTEGER, integer: true });
138
+ const maxSize = parseBoundedNumber(optionValue(args, '--max-size', null), '--max-size', { fallback: Number.MAX_SAFE_INTEGER, min: 0, max: Number.MAX_SAFE_INTEGER, integer: true });
139
+ if (maxSize < minSize) throw new Error('--max-size must be greater than or equal to --min-size');
140
+
141
+ return {
142
+ preset,
143
+ resourceTypes,
144
+ excludedResourceTypes,
145
+ urlPatterns: patternOptionValues(args, '--url-pattern'),
146
+ excludedUrlPatterns: patternOptionValues(args, '--exclude-url-pattern'),
147
+ methods,
148
+ excludedMethods,
149
+ statuses: parseStatusSelectors(optionValues(args, '--status')),
150
+ excludedStatuses: parseStatusSelectors(optionValues(args, '--exclude-status')),
151
+ mimeTypes: optionValues(args, '--mime-type'),
152
+ excludedMimeTypes: optionValues(args, '--exclude-mime-type'),
153
+ minSize,
154
+ maxSize,
155
+ capture: [...new Set(capture)],
156
+ maxBodyBytes: parsePositiveIntegerOption(args, '--max-body-bytes', DEFAULT_HAR_MAX_BODY_BYTES),
157
+ maxDurationSeconds: parsePositiveIntegerOption(args, '--max-duration', DEFAULT_HAR_MAX_DURATION_SECONDS),
158
+ idleMs: parsePositiveIntegerOption(args, '--idle-ms', DEFAULT_HAR_IDLE_MS),
159
+ drainTimeoutMs: parsePositiveIntegerOption(args, '--drain-timeout-ms', DEFAULT_HAR_DRAIN_TIMEOUT_MS),
160
+ redact: hasFlag(args, '--redact'),
161
+ };
162
+ }
163
+
164
+ export class HarCollector {
165
+ constructor(options, { pageId = 'page_0', pageUrl = '', pageTitle = '' } = {}) {
166
+ this.options = options;
167
+ this.pageId = pageId;
168
+ this.pageUrl = pageUrl;
169
+ this.pageTitle = pageTitle;
170
+ this.entries = [];
171
+ this.currentByRequestId = new Map();
172
+ this.entriesByRequestId = new Map();
173
+ this.requestIndexes = new Map();
174
+ this.requestExtraIndexes = new Map();
175
+ this.responseExtraIndexes = new Map();
176
+ this.pendingRequestExtra = new Map();
177
+ this.pendingResponseExtra = new Map();
178
+ this.activeRequestIds = new Set();
179
+ this.lastActivityAt = Date.now();
180
+ }
181
+
182
+ requestMatches(event) {
183
+ const method = String(event.request.method).toUpperCase();
184
+ const resourceType = event.type || 'Other';
185
+ if (!this.options.resourceTypes.includes(resourceType)) return false;
186
+ if (this.options.excludedResourceTypes.includes(resourceType)) return false;
187
+ if (this.options.methods.length && !this.options.methods.includes(method)) return false;
188
+ if (this.options.excludedMethods.includes(method)) return false;
189
+ return matchesPatterns(event.request.url, this.options.urlPatterns, this.options.excludedUrlPatterns);
190
+ }
191
+
192
+ responseMatches(entry) {
193
+ const status = entry.responseExtraInfo?.statusCode ?? entry.response?.status ?? 0;
194
+ if (!matchesStatus(status, this.options.statuses, this.options.excludedStatuses)) return false;
195
+ const mimeType = entry.response?.mimeType || '';
196
+ if (!matchesPatterns(mimeType, this.options.mimeTypes, this.options.excludedMimeTypes)) return false;
197
+ const size = entry.encodedDataLength ?? 0;
198
+ return size >= this.options.minSize && size <= this.options.maxSize;
199
+ }
200
+
201
+ requestWillBeSent(event) {
202
+ this.lastActivityAt = Date.now();
203
+ const entryIndex = this.requestIndexes.get(event.requestId) || 0;
204
+ this.requestIndexes.set(event.requestId, entryIndex + 1);
205
+ if (event.redirectResponse) {
206
+ const redirected = this.currentByRequestId.get(event.requestId);
207
+ if (redirected) {
208
+ redirected.response = event.redirectResponse;
209
+ redirected.responseTimestamp = event.timestamp;
210
+ redirected.finishedTimestamp = event.timestamp;
211
+ redirected.encodedDataLength = Number(event.redirectResponse.encodedDataLength || 0);
212
+ }
213
+ }
214
+
215
+ if (!this.requestMatches(event)) {
216
+ this.currentByRequestId.delete(event.requestId);
217
+ this.activeRequestIds.delete(event.requestId);
218
+ return;
219
+ }
220
+
221
+ const requestEntries = this.entriesByRequestId.get(event.requestId) || [];
222
+ const entry = {
223
+ requestId: event.requestId,
224
+ request: event.request,
225
+ resourceType: event.type || 'Other',
226
+ requestTimestamp: event.timestamp,
227
+ wallTime: event.wallTime,
228
+ initiator: event.initiator,
229
+ response: null,
230
+ responseTimestamp: null,
231
+ finishedTimestamp: null,
232
+ encodedDataLength: 0,
233
+ responseBody: null,
234
+ failed: null,
235
+ servedFromCache: false,
236
+ requestExtraInfo: this.pendingRequestExtra.get(event.requestId)?.get(entryIndex) || null,
237
+ responseExtraInfo: this.pendingResponseExtra.get(event.requestId)?.get(entryIndex) || null,
238
+ };
239
+ requestEntries[entryIndex] = entry;
240
+ this.entriesByRequestId.set(event.requestId, requestEntries);
241
+ this.entries.push(entry);
242
+ this.currentByRequestId.set(event.requestId, entry);
243
+ this.activeRequestIds.add(event.requestId);
244
+ }
245
+
246
+ requestWillBeSentExtraInfo(event) {
247
+ this.assignExtraInfo(event, 'request');
248
+ }
249
+
250
+ responseReceivedExtraInfo(event) {
251
+ this.assignExtraInfo(event, 'response');
252
+ }
253
+
254
+ assignExtraInfo(event, kind) {
255
+ this.lastActivityAt = Date.now();
256
+ const indexes = kind === 'request' ? this.requestExtraIndexes : this.responseExtraIndexes;
257
+ const pending = kind === 'request' ? this.pendingRequestExtra : this.pendingResponseExtra;
258
+ const property = kind === 'request' ? 'requestExtraInfo' : 'responseExtraInfo';
259
+ const index = indexes.get(event.requestId) || 0;
260
+ indexes.set(event.requestId, index + 1);
261
+ const entry = this.entriesByRequestId.get(event.requestId)?.[index];
262
+ if (entry) {
263
+ entry[property] = event;
264
+ return;
265
+ }
266
+ if (!pending.has(event.requestId)) pending.set(event.requestId, new Map());
267
+ pending.get(event.requestId).set(index, event);
268
+ }
269
+
270
+ responseReceived(event) {
271
+ this.lastActivityAt = Date.now();
272
+ const entry = this.currentByRequestId.get(event.requestId);
273
+ if (!entry) return;
274
+ entry.response = event.response;
275
+ entry.responseTimestamp = event.timestamp;
276
+ if (event.type) entry.resourceType = event.type;
277
+ }
278
+
279
+ requestServedFromCache(event) {
280
+ const entry = this.currentByRequestId.get(event.requestId);
281
+ if (entry) entry.servedFromCache = true;
282
+ }
283
+
284
+ async loadingFinished(event, session) {
285
+ this.lastActivityAt = Date.now();
286
+ const entry = this.currentByRequestId.get(event.requestId);
287
+ this.activeRequestIds.delete(event.requestId);
288
+ if (!entry) return;
289
+ entry.finishedTimestamp = event.timestamp;
290
+ entry.encodedDataLength = Number(event.encodedDataLength || 0);
291
+ if (!this.options.capture.includes('bodies') || !this.responseMatches(entry)) return;
292
+ try {
293
+ entry.responseBody = await session.send('Network.getResponseBody', { requestId: event.requestId });
294
+ } catch (error) {
295
+ entry.responseBodyError = error.message;
296
+ }
297
+ }
298
+
299
+ loadingFailed(event) {
300
+ this.lastActivityAt = Date.now();
301
+ const entry = this.currentByRequestId.get(event.requestId);
302
+ this.activeRequestIds.delete(event.requestId);
303
+ if (!entry) return;
304
+ entry.finishedTimestamp = event.timestamp;
305
+ entry.failed = {
306
+ errorText: event.errorText,
307
+ canceled: Boolean(event.canceled),
308
+ blockedReason: event.blockedReason || null,
309
+ };
310
+ }
311
+
312
+ async waitForDrain({ idleMs = this.options.idleMs, timeoutMs = this.options.drainTimeoutMs } = {}) {
313
+ const deadline = Date.now() + timeoutMs;
314
+ while (Date.now() < deadline) {
315
+ if (this.activeRequestIds.size === 0 && Date.now() - this.lastActivityAt >= idleMs) return true;
316
+ await sleep(50);
317
+ }
318
+ return false;
319
+ }
320
+
321
+ toHarEntry(entry) {
322
+ const redact = this.options.redact;
323
+ const includeSensitive = !redact;
324
+ const captureHeaders = this.options.capture.includes('headers');
325
+ const rawRequestHeaders = mergedHeaders(entry.request.headers, entry.requestExtraInfo);
326
+ const rawResponseHeaders = mergedHeaders(entry.response?.headers, entry.responseExtraInfo);
327
+ const requestHeaders = captureHeaders ? redactHeaders(rawRequestHeaders, { includeSensitive }) : [];
328
+ const responseHeaders = captureHeaders ? redactHeaders(rawResponseHeaders, { includeSensitive }) : [];
329
+ const requestMimeType = bodyMimeType(rawRequestHeaders);
330
+ const requestBody = this.options.capture.includes('bodies') && entry.request.postData !== undefined
331
+ ? truncateText(redactBodyText(entry.request.postData, requestMimeType, { includeSensitive }), this.options.maxBodyBytes)
332
+ : { text: undefined, truncated: false };
333
+
334
+ let responseText;
335
+ let responseEncoding;
336
+ let responseTruncated = false;
337
+ if (entry.responseBody) {
338
+ responseEncoding = entry.responseBody.base64Encoded ? 'base64' : undefined;
339
+ const rawText = entry.responseBody.body;
340
+ const redacted = responseEncoding
341
+ ? rawText
342
+ : redactBodyText(rawText, entry.response?.mimeType, { includeSensitive });
343
+ const truncated = truncateText(redacted, this.options.maxBodyBytes);
344
+ responseText = truncated.text;
345
+ responseTruncated = truncated.truncated;
346
+ }
347
+
348
+ const timings = harTimings(entry, this.options.capture.includes('timing'));
349
+ const request = {
350
+ method: entry.request.method,
351
+ url: redactUrl(entry.request.url, { includeSensitive }),
352
+ httpVersion: harHttpVersion(entry.response?.protocol),
353
+ headers: requestHeaders,
354
+ queryString: queryString(entry.request.url, redact),
355
+ cookies: captureHeaders ? requestCookies(entry.requestExtraInfo, redact) : [],
356
+ headersSize: -1,
357
+ bodySize: entry.request.postData ? Buffer.byteLength(entry.request.postData) : 0,
358
+ };
359
+ if (requestBody.text !== undefined) {
360
+ request.postData = {
361
+ mimeType: requestMimeType,
362
+ text: requestBody.text,
363
+ ...(requestBody.truncated ? { _truncated: true } : {}),
364
+ };
365
+ }
366
+
367
+ const response = {
368
+ status: Number(entry.responseExtraInfo?.statusCode ?? entry.response?.status ?? 0),
369
+ statusText: entry.response?.statusText || (entry.failed?.errorText ?? ''),
370
+ httpVersion: harHttpVersion(entry.response?.protocol),
371
+ headers: responseHeaders,
372
+ cookies: [],
373
+ content: {
374
+ size: entry.responseBody?.body ? Buffer.byteLength(entry.responseBody.body) : Number(entry.encodedDataLength || 0),
375
+ mimeType: entry.response?.mimeType || 'application/octet-stream',
376
+ ...(responseText !== undefined ? { text: responseText } : {}),
377
+ ...(responseEncoding ? { encoding: responseEncoding } : {}),
378
+ ...(responseTruncated ? { _truncated: true } : {}),
379
+ },
380
+ redirectURL: redactUrl(headerValue(rawResponseHeaders, 'location'), { includeSensitive }),
381
+ headersSize: -1,
382
+ bodySize: Number(entry.encodedDataLength || 0),
383
+ };
384
+
385
+ return {
386
+ pageref: this.pageId,
387
+ startedDateTime: new Date((entry.wallTime || Date.now() / 1000) * 1000).toISOString(),
388
+ time: totalTime(timings),
389
+ request,
390
+ response,
391
+ cache: entry.servedFromCache ? { _servedFromCache: true } : {},
392
+ timings,
393
+ serverIPAddress: entry.response?.remoteIPAddress || '',
394
+ connection: entry.response?.connectionId ? String(entry.response.connectionId) : '',
395
+ _resourceType: entry.resourceType,
396
+ _requestId: entry.requestId,
397
+ _initiator: redact ? redactSensitive(entry.initiator) : entry.initiator,
398
+ ...(entry.failed ? { _failure: entry.failed } : {}),
399
+ ...(entry.responseBodyError ? { _responseBodyError: entry.responseBodyError } : {}),
400
+ ...(redact ? { _redacted: true } : {}),
401
+ };
402
+ }
403
+
404
+ build() {
405
+ const entries = this.entries
406
+ .filter((entry) => entry.response || entry.failed)
407
+ .filter((entry) => this.responseMatches(entry))
408
+ .map((entry) => this.toHarEntry(entry));
409
+ const startedDateTime = entries[0]?.startedDateTime || new Date().toISOString();
410
+ return {
411
+ log: {
412
+ version: '1.2',
413
+ creator: { name: '@rezkam/browser-tools', version: BROWSER_TOOLS_VERSION },
414
+ pages: [{
415
+ startedDateTime,
416
+ id: this.pageId,
417
+ title: this.pageTitle,
418
+ pageTimings: {},
419
+ _url: redactUrl(this.pageUrl, { includeSensitive: !this.options.redact }),
420
+ }],
421
+ entries,
422
+ _capture: {
423
+ preset: this.options.preset,
424
+ resourceTypes: this.options.resourceTypes,
425
+ filters: {
426
+ urlPatterns: this.options.urlPatterns,
427
+ excludedUrlPatterns: this.options.excludedUrlPatterns,
428
+ methods: this.options.methods,
429
+ statuses: this.options.statuses,
430
+ mimeTypes: this.options.mimeTypes,
431
+ minSize: this.options.minSize,
432
+ maxSize: this.options.maxSize,
433
+ },
434
+ components: this.options.capture,
435
+ redacted: this.options.redact,
436
+ },
437
+ },
438
+ };
439
+ }
440
+ }
441
+
442
+ export async function runHarCapture({
443
+ session,
444
+ page,
445
+ options,
446
+ shouldStop,
447
+ onPhase = () => {},
448
+ }) {
449
+ const collector = new HarCollector(options, {
450
+ pageUrl: page.url(),
451
+ pageTitle: await page.title().catch(() => ''),
452
+ });
453
+ const pending = new Set();
454
+ const track = (promise) => {
455
+ pending.add(promise);
456
+ promise.finally(() => pending.delete(promise));
457
+ };
458
+
459
+ session.on('Network.requestWillBeSent', (event) => collector.requestWillBeSent(event));
460
+ session.on('Network.requestWillBeSentExtraInfo', (event) => collector.requestWillBeSentExtraInfo(event));
461
+ session.on('Network.responseReceived', (event) => collector.responseReceived(event));
462
+ session.on('Network.responseReceivedExtraInfo', (event) => collector.responseReceivedExtraInfo(event));
463
+ session.on('Network.requestServedFromCache', (event) => collector.requestServedFromCache(event));
464
+ session.on('Network.loadingFinished', (event) => track(collector.loadingFinished(event, session)));
465
+ session.on('Network.loadingFailed', (event) => collector.loadingFailed(event));
466
+
467
+ await session.send('Network.enable', {
468
+ maxTotalBufferSize: Math.max(options.maxBodyBytes * 4, 4 * 1024 * 1024),
469
+ maxResourceBufferSize: options.maxBodyBytes,
470
+ maxPostDataSize: options.maxBodyBytes,
471
+ });
472
+ await onPhase('recording');
473
+
474
+ const deadline = Date.now() + options.maxDurationSeconds * 1000;
475
+ while (!shouldStop() && Date.now() < deadline) await sleep(100);
476
+ const reason = shouldStop() ? 'requested' : 'max-duration';
477
+ await onPhase('draining', reason);
478
+ await collector.waitForDrain();
479
+ await Promise.allSettled([...pending]);
480
+ return { reason, har: collector.build(), eventCount: collector.entries.length };
481
+ }
482
+
483
+ export function writeHar(file, har) {
484
+ writePrivateJson(file, har);
485
+ }