@obsrviq/tracker 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +639 -103
- package/dist/index.js.map +1 -1
- package/dist/obsrviq.global.js +40 -40
- package/dist/obsrviq.global.js.map +1 -1
- package/package.json +2 -2
- package/src/index.ts +49 -1
- package/src/mask.ts +127 -6
- package/src/network.ts +81 -17
- package/src/ratelimit.ts +24 -0
- package/src/recorder.ts +20 -1
- package/src/serialize.ts +63 -7
- package/src/transport.ts +357 -64
- package/src/util.ts +15 -1
- package/src/websocket.ts +143 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@obsrviq/tracker",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"@rrweb/types": "2.0.1",
|
|
28
28
|
"fflate": "^0.8.2",
|
|
29
29
|
"rrweb": "2.0.1",
|
|
30
|
-
"@obsrviq/types": "0.
|
|
30
|
+
"@obsrviq/types": "0.5.0"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"tsup": "^8.3.5",
|
package/src/index.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { IngestSessionMeta, ObsrviqEvent, ObsrviqInitConfig } from '@obsrviq/types';
|
|
2
2
|
import type { InstrumentCtx, ResolvedConfig, Teardown } from './context.js';
|
|
3
3
|
import { Transport } from './transport.js';
|
|
4
|
-
import { instrumentDom } from './recorder.js';
|
|
4
|
+
import { instrumentDom, takeFullSnapshot } from './recorder.js';
|
|
5
5
|
import { instrumentConsole } from './console.js';
|
|
6
6
|
import { instrumentNetwork } from './network.js';
|
|
7
|
+
import { instrumentWebSocket } from './websocket.js';
|
|
8
|
+
import { setCustomMaskKeys, markMaskConfigReady } from './mask.js';
|
|
7
9
|
import { instrumentErrors } from './errors.js';
|
|
8
10
|
import { instrumentVitals } from './vitals.js';
|
|
9
11
|
import { instrumentInteractions } from './interactions.js';
|
|
@@ -97,6 +99,14 @@ class ObsrviqClient {
|
|
|
97
99
|
// askPermission = "show a built-in consent prompt"; it implies a consent gate.
|
|
98
100
|
if (this.config.askPermission) this.config.requireConsent = true;
|
|
99
101
|
|
|
102
|
+
// Custom masking keys: apply any static ones NOW, then fetch the per-site list
|
|
103
|
+
// (GET /v1/config) and merge. Fired before the network instrumenter is installed
|
|
104
|
+
// so this request is never self-recorded; the transport also re-masks custom
|
|
105
|
+
// keys at send time so nothing captured before the fetch resolves can leak.
|
|
106
|
+
const staticMaskKeys = Array.isArray(config.maskKeys) ? config.maskKeys : [];
|
|
107
|
+
setCustomMaskKeys(staticMaskKeys);
|
|
108
|
+
void this.loadRemoteMaskKeys(staticMaskKeys);
|
|
109
|
+
|
|
100
110
|
this.startSession();
|
|
101
111
|
|
|
102
112
|
// Init-time identity (rarely known this early — usually set later via identify()).
|
|
@@ -124,6 +134,39 @@ class ObsrviqClient {
|
|
|
124
134
|
}
|
|
125
135
|
}
|
|
126
136
|
|
|
137
|
+
/** Fetch the per-site masking config (GET /v1/config) and merge its keys with any
|
|
138
|
+
* static ones. Best-effort: on failure the built-in masking still applies. */
|
|
139
|
+
private async loadRemoteMaskKeys(staticKeys: string[]): Promise<void> {
|
|
140
|
+
const cfg = this.config;
|
|
141
|
+
if (!cfg || typeof fetch === 'undefined') {
|
|
142
|
+
markMaskConfigReady();
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
const ctrl = new AbortController();
|
|
147
|
+
const timer = setTimeout(() => ctrl.abort(), 2000);
|
|
148
|
+
try {
|
|
149
|
+
const url = `${cfg.endpoint.replace(/\/$/, '')}/v1/config?key=${encodeURIComponent(cfg.siteKey)}`;
|
|
150
|
+
const res = await fetch(url, { credentials: 'omit', mode: 'cors', signal: ctrl.signal });
|
|
151
|
+
if (res.ok) {
|
|
152
|
+
const data = (await res.json()) as { maskKeys?: unknown };
|
|
153
|
+
const remote = Array.isArray(data.maskKeys)
|
|
154
|
+
? data.maskKeys.filter((k): k is string => typeof k === 'string')
|
|
155
|
+
: [];
|
|
156
|
+
setCustomMaskKeys([...staticKeys, ...remote]);
|
|
157
|
+
}
|
|
158
|
+
} finally {
|
|
159
|
+
clearTimeout(timer);
|
|
160
|
+
}
|
|
161
|
+
} catch {
|
|
162
|
+
/* offline / blocked / timeout / bad JSON — built-in masking still protects. */
|
|
163
|
+
} finally {
|
|
164
|
+
// Always release the transport's hold — a failed/slow fetch must never block
|
|
165
|
+
// sending forever (built-in + static-key masking still applies).
|
|
166
|
+
markMaskConfigReady();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
127
170
|
/** (Re)create the session id, clock, meta, and transport. Used by init() and
|
|
128
171
|
* reset(). Does not start recording — callers decide via maybeStart().
|
|
129
172
|
* @param seedInitUser seed identity from the init-time `cfg.userId`. True for
|
|
@@ -172,6 +215,10 @@ class ObsrviqClient {
|
|
|
172
215
|
this.nextSeq = next;
|
|
173
216
|
this.persistSession();
|
|
174
217
|
},
|
|
218
|
+
// Backpressure dropped DOM deltas → re-anchor the rrweb stream immediately
|
|
219
|
+
// so replay stays coherent (a delta stream with a hole is corrupt until
|
|
220
|
+
// the next 30s checkout).
|
|
221
|
+
onDomDrop: () => takeFullSnapshot(),
|
|
175
222
|
onFlush: ({ bytes, events, ok }) => {
|
|
176
223
|
if (ok) {
|
|
177
224
|
this.diag.flushes++;
|
|
@@ -237,6 +284,7 @@ class ObsrviqClient {
|
|
|
237
284
|
// before app code runs more requests; DOM last so its snapshot is current.
|
|
238
285
|
this.teardowns.push(instrumentConsole(ctx));
|
|
239
286
|
this.teardowns.push(instrumentNetwork(ctx));
|
|
287
|
+
this.teardowns.push(instrumentWebSocket(ctx));
|
|
240
288
|
this.teardowns.push(instrumentErrors(ctx));
|
|
241
289
|
this.teardowns.push(instrumentVitals(ctx));
|
|
242
290
|
this.teardowns.push(instrumentInteractions(ctx));
|
package/src/mask.ts
CHANGED
|
@@ -2,14 +2,34 @@
|
|
|
2
2
|
// anything leaves the page. Heuristics catch the obvious PII; tenants tighten
|
|
3
3
|
// with maskTextSelector / beforeSend.
|
|
4
4
|
|
|
5
|
+
import type { ObsrviqEvent } from '@obsrviq/types';
|
|
6
|
+
|
|
5
7
|
const EMAIL = /[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi;
|
|
6
8
|
const CC = /\b(?:\d[ -]*?){13,19}\b/g; // candidate card numbers
|
|
7
9
|
const SSN = /\b\d{3}-\d{2}-\d{4}\b/g;
|
|
8
10
|
const LONG_DIGITS = /\b\d{9,}\b/g; // phone-ish / account-ish
|
|
9
11
|
|
|
10
|
-
// Keys whose values we always redact regardless of content.
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
// Keys whose values we always redact, regardless of content. Two tiers so we
|
|
13
|
+
// catch real secrets without over-redacting innocent fields:
|
|
14
|
+
// STRONG — long/unambiguous tokens, safe to match as a substring of the key
|
|
15
|
+
// name (e.g. "accessToken", "x-api-key", "clientSecret").
|
|
16
|
+
// WEAK — short/ambiguous tokens that we only honour when they stand alone as
|
|
17
|
+
// a key SEGMENT, so "shipping"(pin) / "discard"(card) / "author"(auth)
|
|
18
|
+
// aren't caught but "pin" / "cardNumber" / "authToken" are.
|
|
19
|
+
const SENSITIVE_STRONG =
|
|
20
|
+
/(password|passwd|passphrase|secret|token|authoriz|apikey|api[_-]key|creditcard|cardnumber|cardholder|cookie|cvv|cvc)/i;
|
|
21
|
+
const SENSITIVE_WEAK = new Set(['pass', 'pwd', 'auth', 'card', 'pin', 'otp', 'ssn', 'cvv', 'cvc']);
|
|
22
|
+
|
|
23
|
+
/** Split a key name into lowercase word segments on delimiters AND camelCase
|
|
24
|
+
* boundaries: "accessToken" → [access, token], "api_key" → [api, key]. */
|
|
25
|
+
function keySegments(key: string): string[] {
|
|
26
|
+
return key
|
|
27
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
28
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
|
|
29
|
+
.split(/[^A-Za-z0-9]+/)
|
|
30
|
+
.filter(Boolean)
|
|
31
|
+
.map((s) => s.toLowerCase());
|
|
32
|
+
}
|
|
13
33
|
|
|
14
34
|
export function maskString(input: string): string {
|
|
15
35
|
return input
|
|
@@ -24,8 +44,109 @@ function luhnish(s: string): boolean {
|
|
|
24
44
|
return digits.length >= 13 && digits.length <= 19;
|
|
25
45
|
}
|
|
26
46
|
|
|
27
|
-
|
|
28
|
-
|
|
47
|
+
// Customer-defined keys to mask (case-insensitive exact match), delivered via the
|
|
48
|
+
// per-site config (GET /v1/config) and/or the init `maskKeys` option. Additive to
|
|
49
|
+
// the built-in list above — you can only add protections, never remove them.
|
|
50
|
+
let customKeys = new Set<string>();
|
|
51
|
+
export function setCustomMaskKeys(keys: string[]): void {
|
|
52
|
+
customKeys = new Set(keys.map((k) => String(k).trim().toLowerCase()).filter(Boolean));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Whether the per-site mask config (GET /v1/config) has settled. Until it does,
|
|
56
|
+
// remote-only custom keys aren't known yet, so the transport holds non-unload
|
|
57
|
+
// sends (below) to avoid leaking a custom-keyed value in the brief init window.
|
|
58
|
+
let maskConfigSettled = false;
|
|
59
|
+
export function markMaskConfigReady(): void {
|
|
60
|
+
maskConfigSettled = true;
|
|
61
|
+
}
|
|
62
|
+
export function isMaskConfigPending(): boolean {
|
|
63
|
+
return !maskConfigSettled;
|
|
64
|
+
}
|
|
65
|
+
export function hasCustomMaskKeys(): boolean {
|
|
66
|
+
return customKeys.size > 0;
|
|
67
|
+
}
|
|
68
|
+
function isCustomKey(key: string): boolean {
|
|
69
|
+
return customKeys.has(key.toLowerCase());
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Key-based sensitivity test: built-in STRONG/WEAK tiers PLUS any customer-
|
|
73
|
+
* configured keys (exact, case-insensitive). Boundary-aware so innocent fields
|
|
74
|
+
* like "shipping" / "discard" / "author" aren't redacted. */
|
|
75
|
+
export function isSensitiveKeyName(key: string): boolean {
|
|
76
|
+
if (isCustomKey(key)) return true;
|
|
77
|
+
if (SENSITIVE_STRONG.test(key)) return true;
|
|
78
|
+
return keySegments(key).some((seg) => SENSITIVE_WEAK.has(seg));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Redact sensitive-key values in a parsed object (built-in list + custom keys).
|
|
82
|
+
* The send-time safety net that guarantees a `{"password": "…"}` style body can
|
|
83
|
+
* never leave the page unmasked, even for non-DOM network/WS payloads (whose
|
|
84
|
+
* capture-time masking is PII-pattern only, not key-aware). */
|
|
85
|
+
function redactSensitiveInValue(value: unknown, depth = 0): unknown {
|
|
86
|
+
if (depth > 24 || value === null || typeof value !== 'object') return value;
|
|
87
|
+
if (Array.isArray(value)) return value.map((v) => redactSensitiveInValue(v, depth + 1));
|
|
88
|
+
const out: Record<string, unknown> = {};
|
|
89
|
+
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
90
|
+
out[k] = isSensitiveKeyName(k) ? '«redacted»' : redactSensitiveInValue(v, depth + 1);
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** String-level key redaction for payloads that don't parse as JSON — non-JSON
|
|
96
|
+
* bodies (form-urlencoded / query strings) AND truncated JSON, where the
|
|
97
|
+
* structured path bails. Captures each `"key": value` (JSON-ish, tolerant of a
|
|
98
|
+
* cut-off tail) and `key=value` (form / query) key name and redacts the value
|
|
99
|
+
* when the key is sensitive. */
|
|
100
|
+
function redactSensitiveInRawText(text: string): string {
|
|
101
|
+
let out = text.replace(
|
|
102
|
+
/"([A-Za-z0-9_.\-]+)"\s*:\s*("(?:[^"\\]|\\.)*"|[^,}\]\s]+)/g,
|
|
103
|
+
(m, key: string) => (isSensitiveKeyName(key) ? '"' + key + '": "«redacted»"' : m),
|
|
104
|
+
);
|
|
105
|
+
out = out.replace(
|
|
106
|
+
/(^|[?&])([A-Za-z0-9_.\-[\]]+)=([^&\s]*)/g,
|
|
107
|
+
(m, pre: string, key: string) => (isSensitiveKeyName(key) ? pre + key + '=«redacted»' : m),
|
|
108
|
+
);
|
|
109
|
+
return out;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Redact sensitive-key values in a stored body/arg string. Tries structured
|
|
113
|
+
* JSON first; falls back to a string scan for non-JSON or truncated payloads so a
|
|
114
|
+
* sensitive key can't slip through in `email=a&password=b` or a cut-off body. */
|
|
115
|
+
function redactSensitiveInText(text: string | undefined): string | undefined {
|
|
116
|
+
if (!text) return text;
|
|
117
|
+
const trimmed = text.trim();
|
|
118
|
+
if (trimmed[0] === '{' || trimmed[0] === '[') {
|
|
119
|
+
try {
|
|
120
|
+
return JSON.stringify(redactSensitiveInValue(JSON.parse(trimmed)));
|
|
121
|
+
} catch {
|
|
122
|
+
/* truncated / invalid JSON — fall through to the string scan */
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return redactSensitiveInRawText(text);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Send-time pass: mask sensitive keys (built-in list + custom keys) across a
|
|
129
|
+
* batch's bodies / frames / console args / custom props (mutates in place).
|
|
130
|
+
* Always runs — the built-in list applies with no per-site config needed. */
|
|
131
|
+
export function redactBatchSensitiveKeys(events: ObsrviqEvent[]): void {
|
|
132
|
+
for (const e of events) {
|
|
133
|
+
if (e.type === 'network' || e.type === 'ws') {
|
|
134
|
+
const anyE = e as unknown as {
|
|
135
|
+
requestBody?: { preview?: string };
|
|
136
|
+
responseBody?: { preview?: string };
|
|
137
|
+
body?: { preview?: string };
|
|
138
|
+
};
|
|
139
|
+
if (anyE.requestBody?.preview) anyE.requestBody.preview = redactSensitiveInText(anyE.requestBody.preview);
|
|
140
|
+
if (anyE.responseBody?.preview) anyE.responseBody.preview = redactSensitiveInText(anyE.responseBody.preview);
|
|
141
|
+
if (anyE.body?.preview) anyE.body.preview = redactSensitiveInText(anyE.body.preview);
|
|
142
|
+
} else if (e.type === 'custom') {
|
|
143
|
+
const c = e as unknown as { props?: Record<string, unknown> };
|
|
144
|
+
if (c.props) c.props = redactSensitiveInValue(c.props) as Record<string, unknown>;
|
|
145
|
+
} else if (e.type === 'console') {
|
|
146
|
+
const args = (e as unknown as { args?: Array<{ kind: string; json?: string }> }).args;
|
|
147
|
+
if (args) for (const a of args) if (a.kind === 'object' && a.json) a.json = redactSensitiveInText(a.json);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
29
150
|
}
|
|
30
151
|
|
|
31
152
|
/** Recursively mask an object's string values and sensitive keys. */
|
|
@@ -36,7 +157,7 @@ export function maskValue(value: unknown, depth = 0): unknown {
|
|
|
36
157
|
if (Array.isArray(value)) return value.map((v) => maskValue(v, depth + 1));
|
|
37
158
|
const out: Record<string, unknown> = {};
|
|
38
159
|
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
39
|
-
out[k] =
|
|
160
|
+
out[k] = isSensitiveKeyName(k) ? '«redacted»' : maskValue(v, depth + 1);
|
|
40
161
|
}
|
|
41
162
|
return out;
|
|
42
163
|
}
|
package/src/network.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import type { NetworkEvent, NetworkResourceType, NetworkTiming } from '@obsrviq/types';
|
|
2
2
|
import type { InstrumentCtx, Teardown } from './context.js';
|
|
3
3
|
import { maskString } from './mask.js';
|
|
4
|
+
import { tokenBucket } from './ratelimit.js';
|
|
4
5
|
import { uuid } from './util.js';
|
|
5
6
|
|
|
6
7
|
const BODY_CAP = 4 * 1024; // 4 KB body preview cap
|
|
8
|
+
const NET_RATE = 50; // sustained network events/sec per page
|
|
9
|
+
const NET_BURST = 100; // burst capacity
|
|
7
10
|
// Response/request header allowlist — everything else is dropped.
|
|
8
11
|
const HEADER_ALLOW = new Set([
|
|
9
12
|
'content-type',
|
|
@@ -56,16 +59,56 @@ function bodyPreview(
|
|
|
56
59
|
contentType: string | undefined,
|
|
57
60
|
): NetworkEvent['requestBody'] {
|
|
58
61
|
if (text == null) return undefined;
|
|
59
|
-
|
|
60
|
-
const
|
|
62
|
+
// Pre-cut giant bodies (2× slack) so masking never regex-scans a multi-MB string.
|
|
63
|
+
const cut = text.length > BODY_CAP * 2 ? text.slice(0, BODY_CAP * 2) : text;
|
|
64
|
+
const masked = maskString(cut);
|
|
65
|
+
const truncated = cut !== text || masked.length > BODY_CAP;
|
|
61
66
|
return {
|
|
62
67
|
contentType,
|
|
63
68
|
preview: truncated ? masked.slice(0, BODY_CAP) : masked,
|
|
64
69
|
truncated,
|
|
65
|
-
redacted: masked !==
|
|
70
|
+
redacted: masked !== cut,
|
|
66
71
|
};
|
|
67
72
|
}
|
|
68
73
|
|
|
74
|
+
/** Read at most `cap`+1 bytes of a Response body via its stream reader (then
|
|
75
|
+
* cancel), so a multi-MB response is never fully buffered just to keep a 4 KB
|
|
76
|
+
* preview. `overflow` = the body had more bytes than `cap`. Falls back to
|
|
77
|
+
* text() where a stream reader isn't available (older fetch polyfills). */
|
|
78
|
+
async function readTextCapped(res: Response, cap: number): Promise<{ text: string; overflow: boolean }> {
|
|
79
|
+
const body = res.body;
|
|
80
|
+
if (!body || typeof body.getReader !== 'function') {
|
|
81
|
+
return { text: await res.text(), overflow: false };
|
|
82
|
+
}
|
|
83
|
+
const reader = body.getReader();
|
|
84
|
+
const decoder = new TextDecoder(); // Response.text() decodes UTF-8 — match it
|
|
85
|
+
let text = '';
|
|
86
|
+
let bytes = 0;
|
|
87
|
+
let overflow = false;
|
|
88
|
+
try {
|
|
89
|
+
for (;;) {
|
|
90
|
+
const { done, value } = await reader.read();
|
|
91
|
+
if (done || !value) {
|
|
92
|
+
text += decoder.decode(); // flush any trailing multi-byte sequence
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
const remaining = cap + 1 - bytes;
|
|
96
|
+
if (value.byteLength >= remaining) {
|
|
97
|
+
// Keep only up to cap+1 bytes total — the +1 detects truncation.
|
|
98
|
+
text += decoder.decode(value.subarray(0, remaining), { stream: true });
|
|
99
|
+
overflow = true;
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
text += decoder.decode(value, { stream: true });
|
|
103
|
+
bytes += value.byteLength;
|
|
104
|
+
}
|
|
105
|
+
} finally {
|
|
106
|
+
// Release the tee'd clone branch so it doesn't buffer the rest of the response.
|
|
107
|
+
void reader.cancel().catch(() => {});
|
|
108
|
+
}
|
|
109
|
+
return { text, overflow };
|
|
110
|
+
}
|
|
111
|
+
|
|
69
112
|
function perfNow(): number {
|
|
70
113
|
return typeof performance !== 'undefined' ? performance.now() : 0;
|
|
71
114
|
}
|
|
@@ -239,8 +282,26 @@ function createTimingResolver(): TimingResolver {
|
|
|
239
282
|
};
|
|
240
283
|
}
|
|
241
284
|
|
|
285
|
+
/** A rate-gated NetworkEvent emitter shared by all capture paths on the page. */
|
|
286
|
+
type EmitNet = (ev: NetworkEvent) => void;
|
|
287
|
+
|
|
242
288
|
export function instrumentNetwork(ctx: InstrumentCtx): Teardown {
|
|
243
289
|
const timing = createTimingResolver();
|
|
290
|
+
// Per-page event throttle — a polling loop / request storm must not flood the
|
|
291
|
+
// buffer. Dropped events are counted and reported on the next captured one.
|
|
292
|
+
const bucket = tokenBucket(NET_RATE, NET_BURST);
|
|
293
|
+
let dropped = 0;
|
|
294
|
+
const emitNet: EmitNet = (ev) => {
|
|
295
|
+
if (!bucket.take()) {
|
|
296
|
+
dropped++;
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
if (dropped > 0) {
|
|
300
|
+
ev.droppedEvents = dropped;
|
|
301
|
+
dropped = 0;
|
|
302
|
+
}
|
|
303
|
+
ctx.emit(ev);
|
|
304
|
+
};
|
|
244
305
|
// On page hide, settle any request whose Resource Timing hasn't arrived yet so
|
|
245
306
|
// its event lands in the transport buffer before the unload flush.
|
|
246
307
|
const onPageHide = () => timing.flushPending();
|
|
@@ -251,10 +312,10 @@ export function instrumentNetwork(ctx: InstrumentCtx): Teardown {
|
|
|
251
312
|
if (typeof document !== 'undefined') document.addEventListener('visibilitychange', onVisibility);
|
|
252
313
|
|
|
253
314
|
const teardowns: Teardown[] = [];
|
|
254
|
-
teardowns.push(wrapFetch(ctx, timing));
|
|
255
|
-
teardowns.push(wrapXhr(ctx, timing));
|
|
256
|
-
teardowns.push(wrapBeacon(ctx));
|
|
257
|
-
teardowns.push(observeResources(ctx));
|
|
315
|
+
teardowns.push(wrapFetch(ctx, timing, emitNet));
|
|
316
|
+
teardowns.push(wrapXhr(ctx, timing, emitNet));
|
|
317
|
+
teardowns.push(wrapBeacon(ctx, emitNet));
|
|
318
|
+
teardowns.push(observeResources(ctx, emitNet));
|
|
258
319
|
teardowns.push(() => {
|
|
259
320
|
if (typeof window !== 'undefined') window.removeEventListener('pagehide', onPageHide);
|
|
260
321
|
if (typeof document !== 'undefined') document.removeEventListener('visibilitychange', onVisibility);
|
|
@@ -264,13 +325,14 @@ export function instrumentNetwork(ctx: InstrumentCtx): Teardown {
|
|
|
264
325
|
}
|
|
265
326
|
|
|
266
327
|
// ───────────────────────────── fetch ────────────────────────────────
|
|
267
|
-
function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
|
|
328
|
+
function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver, emitNet: EmitNet): Teardown {
|
|
268
329
|
const orig = window.fetch;
|
|
269
330
|
if (!orig) return () => {};
|
|
270
331
|
window.fetch = async function (input: RequestInfo | URL, init?: RequestInit) {
|
|
271
332
|
const rawUrl = input instanceof Request ? input.url : String(input);
|
|
272
333
|
// Never record our own telemetry round-trips to the ingest endpoint.
|
|
273
|
-
if (rawUrl.includes('/v1/batch')
|
|
334
|
+
if (rawUrl.includes('/v1/batch') || rawUrl.includes('/v1/config'))
|
|
335
|
+
return orig.call(window, input as RequestInfo, init);
|
|
274
336
|
const startT = ctx.clock.now();
|
|
275
337
|
const perfStart = perfNow();
|
|
276
338
|
const method = (init?.method || (input instanceof Request ? input.method : 'GET')).toUpperCase();
|
|
@@ -303,7 +365,7 @@ function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
|
|
|
303
365
|
timing,
|
|
304
366
|
...partial,
|
|
305
367
|
};
|
|
306
|
-
|
|
368
|
+
emitNet(ev);
|
|
307
369
|
});
|
|
308
370
|
};
|
|
309
371
|
|
|
@@ -315,8 +377,10 @@ function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
|
|
|
315
377
|
let responseBody: NetworkEvent['responseBody'];
|
|
316
378
|
if (ctx.config.captureNetworkBodies && isTexty(res.headers.get('content-type'))) {
|
|
317
379
|
try {
|
|
318
|
-
|
|
380
|
+
// Stream-read the clone up to the preview cap — never buffer a huge body.
|
|
381
|
+
const { text, overflow } = await readTextCapped(res.clone(), BODY_CAP);
|
|
319
382
|
responseBody = bodyPreview(text, res.headers.get('content-type') || undefined);
|
|
383
|
+
if (responseBody && overflow) responseBody.truncated = true;
|
|
320
384
|
} catch {
|
|
321
385
|
/* ignore */
|
|
322
386
|
}
|
|
@@ -363,7 +427,7 @@ interface XhrMeta {
|
|
|
363
427
|
body?: string;
|
|
364
428
|
}
|
|
365
429
|
|
|
366
|
-
function wrapXhr(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
|
|
430
|
+
function wrapXhr(ctx: InstrumentCtx, tr: TimingResolver, emitNet: EmitNet): Teardown {
|
|
367
431
|
const XHR = XMLHttpRequest.prototype;
|
|
368
432
|
const origOpen = XHR.open;
|
|
369
433
|
const origSend = XHR.send;
|
|
@@ -444,7 +508,7 @@ function wrapXhr(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
|
|
|
444
508
|
timing,
|
|
445
509
|
error: error || classifyHttp(status),
|
|
446
510
|
};
|
|
447
|
-
|
|
511
|
+
emitNet(ev);
|
|
448
512
|
});
|
|
449
513
|
};
|
|
450
514
|
this.addEventListener('loadend', () => finish());
|
|
@@ -483,7 +547,7 @@ function parseRawHeaders(ctx: InstrumentCtx, raw: string): Record<string, string
|
|
|
483
547
|
}
|
|
484
548
|
|
|
485
549
|
// ──────────────────────────── sendBeacon ────────────────────────────
|
|
486
|
-
function wrapBeacon(ctx: InstrumentCtx): Teardown {
|
|
550
|
+
function wrapBeacon(ctx: InstrumentCtx, emitNet: EmitNet): Teardown {
|
|
487
551
|
const orig = navigator.sendBeacon;
|
|
488
552
|
if (!orig) return () => {};
|
|
489
553
|
navigator.sendBeacon = function (url: string | URL, data?: BodyInit | null): boolean {
|
|
@@ -510,7 +574,7 @@ function wrapBeacon(ctx: InstrumentCtx): Teardown {
|
|
|
510
574
|
timing: { startT, endT: ctx.clock.now(), duration: 0 },
|
|
511
575
|
error: ok ? undefined : 'network',
|
|
512
576
|
};
|
|
513
|
-
|
|
577
|
+
emitNet(ev);
|
|
514
578
|
}
|
|
515
579
|
}
|
|
516
580
|
return ok;
|
|
@@ -533,7 +597,7 @@ const RESOURCE_TYPE_MAP: Record<string, NetworkResourceType> = {
|
|
|
533
597
|
track: 'media',
|
|
534
598
|
};
|
|
535
599
|
|
|
536
|
-
function observeResources(ctx: InstrumentCtx): Teardown {
|
|
600
|
+
function observeResources(ctx: InstrumentCtx, emitNet: EmitNet): Teardown {
|
|
537
601
|
if (typeof PerformanceObserver === 'undefined') return () => {};
|
|
538
602
|
let obs: PerformanceObserver | undefined;
|
|
539
603
|
try {
|
|
@@ -570,7 +634,7 @@ function observeResources(ctx: InstrumentCtx): Teardown {
|
|
|
570
634
|
responseSize: entry.transferSize || entry.encodedBodySize || undefined,
|
|
571
635
|
timing,
|
|
572
636
|
};
|
|
573
|
-
|
|
637
|
+
emitNet(ev);
|
|
574
638
|
}
|
|
575
639
|
});
|
|
576
640
|
obs.observe({ type: 'resource', buffered: true });
|
package/src/ratelimit.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Shared token-bucket rate limiter for capture paths (network / WebSocket
|
|
2
|
+
// event floods). Kept tiny on purpose — callers count their own drops.
|
|
3
|
+
|
|
4
|
+
export interface TokenBucket {
|
|
5
|
+
/** Spend one token if available; false = the caller should drop the event. */
|
|
6
|
+
take(): boolean;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** A token bucket: refills at `rate` tokens/sec (sustained), holds at most
|
|
10
|
+
* `burst` tokens (burst capacity). */
|
|
11
|
+
export function tokenBucket(rate: number, burst: number): TokenBucket {
|
|
12
|
+
let tokens = burst;
|
|
13
|
+
let last = Date.now();
|
|
14
|
+
return {
|
|
15
|
+
take() {
|
|
16
|
+
const now = Date.now();
|
|
17
|
+
tokens = Math.min(burst, tokens + ((now - last) / 1000) * rate);
|
|
18
|
+
last = now;
|
|
19
|
+
if (tokens < 1) return false;
|
|
20
|
+
tokens -= 1;
|
|
21
|
+
return true;
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
package/src/recorder.ts
CHANGED
|
@@ -10,11 +10,18 @@ const INCREMENTAL = 3;
|
|
|
10
10
|
export function instrumentDom(ctx: InstrumentCtx): Teardown {
|
|
11
11
|
const stop = record({
|
|
12
12
|
emit(event: eventWithTime) {
|
|
13
|
+
const t = ctx.clock.fromPerf(performanceNow());
|
|
14
|
+
// ONE CLOCK: rrweb stamps events with Date.now(), while everything else in
|
|
15
|
+
// the session runs on the monotonic perf clock (`t`). Rewrite the rrweb
|
|
16
|
+
// timestamp onto that same clock (startedAt + t) so an NTP step / manual
|
|
17
|
+
// clock change mid-session can't desync replay from the console/network
|
|
18
|
+
// lanes — the player only ever uses timestamp DELTAS, so this is safe.
|
|
19
|
+
event.timestamp = ctx.clock.startedAt + t;
|
|
13
20
|
const dom: DomEvent = {
|
|
14
21
|
id: uuid(),
|
|
15
22
|
sessionId: ctx.sessionId,
|
|
16
23
|
type: 'dom',
|
|
17
|
-
t
|
|
24
|
+
t,
|
|
18
25
|
ts: ctx.clock.wall(),
|
|
19
26
|
data: event,
|
|
20
27
|
};
|
|
@@ -48,6 +55,18 @@ export function instrumentDom(ctx: InstrumentCtx): Teardown {
|
|
|
48
55
|
};
|
|
49
56
|
}
|
|
50
57
|
|
|
58
|
+
/** Force a fresh FullSnapshot checkpoint NOW. Used after transport backpressure
|
|
59
|
+
* had to drop buffered DOM deltas — the new snapshot re-anchors the stream so
|
|
60
|
+
* replay stays coherent instead of playing broken deltas until the next 30s
|
|
61
|
+
* checkout. Safe no-op when recording isn't active. */
|
|
62
|
+
export function takeFullSnapshot(): void {
|
|
63
|
+
try {
|
|
64
|
+
record.takeFullSnapshot(true);
|
|
65
|
+
} catch {
|
|
66
|
+
/* not recording */
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
51
70
|
function performanceNow(): number {
|
|
52
71
|
return typeof performance !== 'undefined' ? performance.now() : Date.now();
|
|
53
72
|
}
|
package/src/serialize.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { SerializedArg } from '@obsrviq/types';
|
|
2
|
-
import { maskString,
|
|
2
|
+
import { maskString, isSensitiveKeyName } from './mask.js';
|
|
3
3
|
|
|
4
4
|
const DEFAULT_MAX_BYTES = 8 * 1024; // 8 KB string cap (§6.1)
|
|
5
|
+
const MAX_WALK_NODES = 2000; // deep-walk node-visit cap (§6.1)
|
|
5
6
|
|
|
6
7
|
function byteLen(s: string): number {
|
|
7
8
|
return new Blob([s]).size;
|
|
@@ -24,9 +25,12 @@ export function serializeArg(arg: unknown, maxBytes = DEFAULT_MAX_BYTES): Serial
|
|
|
24
25
|
}
|
|
25
26
|
|
|
26
27
|
if (type === 'string') {
|
|
27
|
-
|
|
28
|
+
// Pre-cut giant strings (2× slack) so masking never regex-scans a multi-MB arg.
|
|
29
|
+
const s = arg as string;
|
|
30
|
+
const cut = s.length > maxBytes * 2 ? s.slice(0, maxBytes * 2) : s;
|
|
31
|
+
const masked = maskString(cut);
|
|
28
32
|
const { value, truncated } = truncate(masked, maxBytes);
|
|
29
|
-
return { kind: 'string', value, truncated };
|
|
33
|
+
return { kind: 'string', value, truncated: truncated || cut !== s };
|
|
30
34
|
}
|
|
31
35
|
|
|
32
36
|
if (arg instanceof Error) {
|
|
@@ -43,24 +47,76 @@ export function serializeArg(arg: unknown, maxBytes = DEFAULT_MAX_BYTES): Serial
|
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
if (type === 'object' || type === 'function' || type === 'symbol' || type === 'bigint') {
|
|
46
|
-
|
|
50
|
+
// Budgets are enforced DURING the walk (node-visit count + ~2× the byte cap)
|
|
51
|
+
// so a huge object never burns CPU/memory before the final truncate applies.
|
|
52
|
+
const budget: WalkBudget = { nodes: MAX_WALK_NODES, bytes: maxBytes * 2, exhausted: false };
|
|
53
|
+
const masked = maskValueBudgeted(arg, budget);
|
|
47
54
|
let json: string | undefined;
|
|
48
|
-
let truncated =
|
|
55
|
+
let truncated = budget.exhausted;
|
|
49
56
|
try {
|
|
50
57
|
const full = JSON.stringify(masked, safeReplacer());
|
|
51
58
|
const cut = truncate(full ?? 'undefined', maxBytes);
|
|
52
59
|
json = cut.value;
|
|
53
|
-
truncated = cut.truncated;
|
|
60
|
+
truncated = truncated || cut.truncated;
|
|
54
61
|
} catch {
|
|
55
62
|
json = undefined;
|
|
56
63
|
}
|
|
57
|
-
return { kind: 'object', preview: previewOf(
|
|
64
|
+
return { kind: 'object', preview: previewOf(arg), json, truncated };
|
|
58
65
|
}
|
|
59
66
|
|
|
60
67
|
// undefined / fallback
|
|
61
68
|
return { kind: 'primitive', value: String(arg) };
|
|
62
69
|
}
|
|
63
70
|
|
|
71
|
+
interface WalkBudget {
|
|
72
|
+
nodes: number; // node visits left
|
|
73
|
+
bytes: number; // approximate serialized bytes left
|
|
74
|
+
exhausted: boolean;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** `maskValue` with hard budgets enforced DURING the walk — a node-visit cap and a
|
|
78
|
+
* running approximate byte budget — so a huge arg (e.g. a multi-MB store) is never
|
|
79
|
+
* fully cloned/stringified just to be truncated afterwards. When a budget runs out
|
|
80
|
+
* we stop descending and substitute a '«truncated»' placeholder. Byte-identical to
|
|
81
|
+
* the unbudgeted walk for args that fit the budget. */
|
|
82
|
+
function maskValueBudgeted(value: unknown, budget: WalkBudget, depth = 0): unknown {
|
|
83
|
+
if (budget.exhausted) return '«truncated»';
|
|
84
|
+
if (depth > 6) return '«depth»';
|
|
85
|
+
if (--budget.nodes <= 0) budget.exhausted = true;
|
|
86
|
+
if (typeof value === 'string') {
|
|
87
|
+
// Pre-cut giant strings so masking never regex-scans more than the budget.
|
|
88
|
+
const cut = value.length > budget.bytes ? value.slice(0, Math.max(0, budget.bytes)) : value;
|
|
89
|
+
if (cut !== value) budget.exhausted = true;
|
|
90
|
+
const masked = maskString(cut);
|
|
91
|
+
if ((budget.bytes -= masked.length + 2) <= 0) budget.exhausted = true;
|
|
92
|
+
return masked;
|
|
93
|
+
}
|
|
94
|
+
if (value === null || typeof value !== 'object') return value;
|
|
95
|
+
if (Array.isArray(value)) {
|
|
96
|
+
const out: unknown[] = [];
|
|
97
|
+
for (const v of value) {
|
|
98
|
+
if (budget.exhausted) {
|
|
99
|
+
out.push('«truncated»');
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
out.push(maskValueBudgeted(v, budget, depth + 1));
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
const out: Record<string, unknown> = {};
|
|
107
|
+
for (const k of Object.keys(value)) {
|
|
108
|
+
if (budget.exhausted) {
|
|
109
|
+
out['…'] = '«truncated»';
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
if ((budget.bytes -= k.length + 4) <= 0) budget.exhausted = true;
|
|
113
|
+
out[k] = isSensitiveKeyName(k)
|
|
114
|
+
? '«redacted»'
|
|
115
|
+
: maskValueBudgeted((value as Record<string, unknown>)[k], budget, depth + 1);
|
|
116
|
+
}
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
|
|
64
120
|
function safeReplacer() {
|
|
65
121
|
const seen = new WeakSet();
|
|
66
122
|
return (_key: string, value: unknown) => {
|