@enricai/barnacle 1.12.49 → 1.12.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/recon/capture-filters.d.ts +124 -0
- package/dist/recon/capture-filters.d.ts.map +1 -1
- package/dist/recon/capture-filters.js +209 -0
- package/dist/recon/capture-filters.js.map +1 -1
- package/dist/scraper/captcha-callback-capture.d.ts +11 -22
- package/dist/scraper/captcha-callback-capture.d.ts.map +1 -1
- package/dist/scraper/captcha-callback-capture.js +3 -89
- package/dist/scraper/captcha-callback-capture.js.map +1 -1
- package/dist/scraper/cdp-frame-init-script.d.ts +16 -0
- package/dist/scraper/cdp-frame-init-script.d.ts.map +1 -0
- package/dist/scraper/cdp-frame-init-script.js +67 -0
- package/dist/scraper/cdp-frame-init-script.js.map +1 -0
- package/dist/scraper/flow-runner.d.ts +14 -1
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +23 -2
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/session.d.ts.map +1 -1
- package/dist/scraper/session.js +3 -3
- package/dist/scraper/session.js.map +1 -1
- package/dist/scripts/recon-generate-multicall-fixture.d.ts +14 -0
- package/dist/scripts/recon-generate-multicall-fixture.d.ts.map +1 -1
- package/dist/scripts/recon-generate-multicall-fixture.js +24 -0
- package/dist/scripts/recon-generate-multicall-fixture.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +77 -39
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +834 -100
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +1 -1
|
@@ -47,6 +47,22 @@ export declare function isNoiseUrl(url: string): boolean;
|
|
|
47
47
|
* public suffixes (e.g. `co.uk`), which this codebase does not target.
|
|
48
48
|
*/
|
|
49
49
|
export declare function registrableDomain(hostname: string): string;
|
|
50
|
+
/**
|
|
51
|
+
* Splits a URL path into lowercase word tokens, breaking on non-alphanumeric
|
|
52
|
+
* separators (`/`, `-`, `_`, `.`) and camelCase boundaries, then drops tokens
|
|
53
|
+
* that are too short (<3 chars) or too generic ({@link GENERIC_PATH_TOKENS})
|
|
54
|
+
* to signal endpoint-family relatedness on their own.
|
|
55
|
+
*
|
|
56
|
+
* Only tokens from a *compound* path segment (one that itself splits into 2+
|
|
57
|
+
* words, e.g. `listing-avail-vas`) are kept. A whole segment that is a single
|
|
58
|
+
* plain word (e.g. `search`, `list`, `detail`, `widget`) is dropped entirely:
|
|
59
|
+
* such words recur across unrelated endpoint families on the same host, so
|
|
60
|
+
* treating them as a relatedness signal produces false positives (a marketing
|
|
61
|
+
* `/promotions/search` call "matching" a booking flow's `.../v1/search` just
|
|
62
|
+
* because both happen to end in the common word "search"). Compound segments
|
|
63
|
+
* are where a real endpoint-family identifier lives.
|
|
64
|
+
*/
|
|
65
|
+
export declare function pathStructuralTokens(path: string): Set<string>;
|
|
50
66
|
/**
|
|
51
67
|
* True when `candidatePath` shares enough path-segment tokens with at least
|
|
52
68
|
* one path in `referencePaths` to be judged part of the same endpoint
|
|
@@ -121,6 +137,114 @@ export declare function isStructurallyIsolatedCapture(candidatePath: string, poo
|
|
|
121
137
|
* can reuse the identical relatedness rule instead of re-deriving it.
|
|
122
138
|
*/
|
|
123
139
|
export declare function isSamePathFamily(pathA: string, pathB: string): boolean;
|
|
140
|
+
/**
|
|
141
|
+
* True when `capture`'s response carries no business-relevant state: a
|
|
142
|
+
* non-JSON (or absent) content-type, a null/undefined body, a JSON body
|
|
143
|
+
* with no keys, or a JSON body whose every leaf value is already present in
|
|
144
|
+
* the request's own URL (query string or path). A page-load sensor/
|
|
145
|
+
* analytics beacon (an Akamai-style session-authenticator pixel, a polled
|
|
146
|
+
* non-JSON status ping, or one that echoes back the `clientId`/`siteId` the
|
|
147
|
+
* caller just sent it) answers every call with exactly this shape — unlike a
|
|
148
|
+
* real API response, which carries data a caller could not have already
|
|
149
|
+
* known before firing the request.
|
|
150
|
+
*
|
|
151
|
+
* The "echoes its own request" branch is what closes the gap a bare
|
|
152
|
+
* empty-body check misses: a response is technically non-empty JSON but
|
|
153
|
+
* every leaf is one of the fixed query's own values (or a path segment),
|
|
154
|
+
* so nothing in it is new information relative to what the caller already
|
|
155
|
+
* sent — it is not business-relevant just because it happens to be
|
|
156
|
+
* non-empty.
|
|
157
|
+
*
|
|
158
|
+
* Missing response metadata (unit-test callers that construct a capture
|
|
159
|
+
* without `responseHeaders`/`responseBody`) reads as "no business-relevant
|
|
160
|
+
* state" too: this predicate only ever narrows an already-recurring
|
|
161
|
+
* candidate — {@link isZeroVarianceRepeatCapture} (already-fixed-query) and
|
|
162
|
+
* `recon-generate.ts`'s own-repeat structural-isolation pass (already
|
|
163
|
+
* repeats identically, regardless of query shape) — so defaulting to the
|
|
164
|
+
* noise reading there costs nothing except in the caller that deliberately
|
|
165
|
+
* supplies a JSON response to prove the opposite.
|
|
166
|
+
*
|
|
167
|
+
* Exported (not just used internally by {@link isZeroVarianceRepeatCapture})
|
|
168
|
+
* because a same-host, fixed-request endpoint with NO query string at all
|
|
169
|
+
* (e.g. a polled feature-toggle feed with a single-compound-segment path)
|
|
170
|
+
* can be genuinely zero-business-value too, and query-key matching alone —
|
|
171
|
+
* this file's `hasFixedKey` signal — has nothing to key off when there is no
|
|
172
|
+
* query. Response business-value is the general, path/query-shape-independent
|
|
173
|
+
* signal for "this repeat carries nothing a caller could not already know,"
|
|
174
|
+
* so `recon-generate.ts` reuses it directly instead of the query-shape logic
|
|
175
|
+
* that cannot apply to a query-less endpoint.
|
|
176
|
+
*/
|
|
177
|
+
export declare function hasNoBusinessRelevantResponseState(capture: {
|
|
178
|
+
url: string;
|
|
179
|
+
responseHeaders?: Record<string, string>;
|
|
180
|
+
responseBody?: unknown;
|
|
181
|
+
}): boolean;
|
|
182
|
+
/**
|
|
183
|
+
* True when `candidate` carries at least one query key whose value stays
|
|
184
|
+
* identical across every occurrence of the same endpoint in `allCaptures`,
|
|
185
|
+
* and recurs at least once elsewhere at the same method and endpoint
|
|
186
|
+
* (origin + pathname) — proof the endpoint's identifying signal is a fixed
|
|
187
|
+
* query key, not the full query string. Any OTHER key that differs between
|
|
188
|
+
* occurrences (a cache-buster, a session nonce) is incidental and does not
|
|
189
|
+
* prevent the match, AND either the request body is also byte-identical
|
|
190
|
+
* across every occurrence, or the response carries no business-relevant
|
|
191
|
+
* state ({@link hasNoBusinessRelevantResponseState}).
|
|
192
|
+
*
|
|
193
|
+
* Grouping by the full URL string (byte-identical query) is too strict: a
|
|
194
|
+
* real beacon can carry one incidental varying query key (a cache-buster or
|
|
195
|
+
* session nonce that is never literally equal across calls) alongside its
|
|
196
|
+
* genuinely fixed identifying keys (`clientId`, `environment`, `siteId`),
|
|
197
|
+
* and requiring the entire string to match lets it escape exclusion
|
|
198
|
+
* entirely. Per-key comparison against the candidate's own keys — rather
|
|
199
|
+
* than requiring the whole set of keys or values to agree — is what
|
|
200
|
+
* recovers the beacon: the fixed keys still prove the endpoint's identity,
|
|
201
|
+
* the varying key is simply ignored because it never advances past
|
|
202
|
+
* "matches on at least one key."
|
|
203
|
+
*
|
|
204
|
+
* A key is "fixed" when the candidate's own value for it is the STRICT
|
|
205
|
+
* MAJORITY value across every same-endpoint occurrence in `allCaptures`
|
|
206
|
+
* (more than half, with at least two supporting occurrences) — not
|
|
207
|
+
* necessarily every single one. `allCaptures` is the full raw capture list,
|
|
208
|
+
* which can include an incidental earlier/differently-scoped occurrence of
|
|
209
|
+
* the same endpoint (a page load before the flow proper starts) that
|
|
210
|
+
* legitimately carries a different value for every candidate key. Requiring
|
|
211
|
+
* literal unanimity would let that one outlier veto the proof for the
|
|
212
|
+
* entire flow's worth of genuinely fixed repeats; requiring only a majority
|
|
213
|
+
* still refuses to flag a key that varies freely (no value would ever reach
|
|
214
|
+
* a majority) while tolerating the minority-outlier shape a real archive
|
|
215
|
+
* produces.
|
|
216
|
+
*
|
|
217
|
+
* The requirement that at least one key be fixed is deliberate, not an
|
|
218
|
+
* extension/host special-case: it is what separates this from a genuinely
|
|
219
|
+
* no-argument own endpoint that a flow legitimately polls with an identical
|
|
220
|
+
* body every time (a feature-toggle feed, an availability heartbeat) —
|
|
221
|
+
* those endpoints carry real business meaning and folding their repeats
|
|
222
|
+
* into a single call is the collapse mechanism's job, not this predicate's.
|
|
223
|
+
* A marketing/analytics beacon's own fixed identifying query is the tell a
|
|
224
|
+
* plain body-repetition check can't see on its own, and is exactly the
|
|
225
|
+
* shape {@link isStructurallyIsolatedCapture} can be fooled by — that
|
|
226
|
+
* check's reference pool is every OTHER admitted capture, so N copies of
|
|
227
|
+
* the same fixed-query request "vouch" for each other's path tokens and
|
|
228
|
+
* none of them reads as isolated.
|
|
229
|
+
*
|
|
230
|
+
* The response-state fallback exists because a byte-identical-body
|
|
231
|
+
* requirement alone misses a beacon whose body embeds a fingerprint,
|
|
232
|
+
* timestamp, or session nonce that differs on every fire even though the
|
|
233
|
+
* fixed query is the only signal that actually identifies the endpoint —
|
|
234
|
+
* the body varies, but the response never carries anything the flow could
|
|
235
|
+
* not already derive from the request itself.
|
|
236
|
+
*/
|
|
237
|
+
export declare function isZeroVarianceRepeatCapture(candidate: {
|
|
238
|
+
method: string;
|
|
239
|
+
url: string;
|
|
240
|
+
requestPostData: string | null;
|
|
241
|
+
responseHeaders?: Record<string, string>;
|
|
242
|
+
responseBody?: unknown;
|
|
243
|
+
}, allCaptures: readonly {
|
|
244
|
+
method: string;
|
|
245
|
+
url: string;
|
|
246
|
+
requestPostData: string | null;
|
|
247
|
+
}[]): boolean;
|
|
124
248
|
/**
|
|
125
249
|
* True when `hostname` is allowed as a fixture host: an exact match against
|
|
126
250
|
* `ownBackendHostnames` when the flow declares any, otherwise a
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capture-filters.d.ts","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,EAAE,CAW/C;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,QAAyB,CAAC;AAmC9D;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAc/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM1D;
|
|
1
|
+
{"version":3,"file":"capture-filters.d.ts","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,EAAE,CAW/C;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,QAAyB,CAAC;AAmC9D;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAc/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM1D;AAKD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAc9D;AA+BD;;;;;;;;;;;;GAYG;AACH,wBAAgB,6BAA6B,CAC3C,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,SAAS,MAAM,EAAE,GAChC,OAAO,CAUT;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,6BAA6B,CAC3C,aAAa,EAAE,MAAM,EACrB,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,OAAO,CAWT;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAWtE;AAwCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,EAAE;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,GAAG,OAAO,CAeV;AAoBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,wBAAgB,2BAA2B,CACzC,SAAS,EAAE;IACT,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,EACD,WAAW,EAAE,SAAS;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,EAAE,GACtF,OAAO,CAkCT;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,mBAAmB,EAAE,MAAM,EAAE,EAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,GAC5B,OAAO,CAKT"}
|
|
@@ -15,9 +15,12 @@ exports.ERROR_SINK_PATH_SEGMENT = void 0;
|
|
|
15
15
|
exports.telemetryUrlPatterns = telemetryUrlPatterns;
|
|
16
16
|
exports.isNoiseUrl = isNoiseUrl;
|
|
17
17
|
exports.registrableDomain = registrableDomain;
|
|
18
|
+
exports.pathStructuralTokens = pathStructuralTokens;
|
|
18
19
|
exports.isStructurallyRelevantCapture = isStructurallyRelevantCapture;
|
|
19
20
|
exports.isStructurallyIsolatedCapture = isStructurallyIsolatedCapture;
|
|
20
21
|
exports.isSamePathFamily = isSamePathFamily;
|
|
22
|
+
exports.hasNoBusinessRelevantResponseState = hasNoBusinessRelevantResponseState;
|
|
23
|
+
exports.isZeroVarianceRepeatCapture = isZeroVarianceRepeatCapture;
|
|
21
24
|
exports.isAllowedFixtureHost = isAllowedFixtureHost;
|
|
22
25
|
/**
|
|
23
26
|
* Path/URL substrings we always treat as analytics or logging noise. Site-specific
|
|
@@ -293,6 +296,212 @@ function isSamePathFamily(pathA, pathB) {
|
|
|
293
296
|
const segmentsA = new Set(meaningfulPathSegments(pathA));
|
|
294
297
|
return meaningfulPathSegments(pathB).some((segment) => segmentsA.has(segment));
|
|
295
298
|
}
|
|
299
|
+
/**
|
|
300
|
+
* Every string/number/boolean leaf reachable from `value`, stringified, with
|
|
301
|
+
* `null`/`undefined` leaves skipped (they carry no derivable identifier to
|
|
302
|
+
* compare against the request).
|
|
303
|
+
*/
|
|
304
|
+
function collectLeafValues(value, out) {
|
|
305
|
+
if (value === null || value === undefined)
|
|
306
|
+
return;
|
|
307
|
+
if (typeof value === "object") {
|
|
308
|
+
for (const child of Array.isArray(value)
|
|
309
|
+
? value
|
|
310
|
+
: Object.values(value)) {
|
|
311
|
+
collectLeafValues(child, out);
|
|
312
|
+
}
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
out.push(String(value));
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* The set of raw values a request's own URL already exposes to the caller:
|
|
319
|
+
* every query-parameter value and every non-empty path segment. A response
|
|
320
|
+
* leaf that only ever echoes one of these tells the caller nothing it could
|
|
321
|
+
* not already derive from the request it just sent.
|
|
322
|
+
*/
|
|
323
|
+
function urlOwnValues(url) {
|
|
324
|
+
const values = new Set();
|
|
325
|
+
try {
|
|
326
|
+
const parsed = new URL(url);
|
|
327
|
+
for (const value of parsed.searchParams.values())
|
|
328
|
+
values.add(value);
|
|
329
|
+
for (const segment of parsed.pathname.split("/")) {
|
|
330
|
+
if (segment.length > 0)
|
|
331
|
+
values.add(decodeURIComponent(segment));
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
catch {
|
|
335
|
+
// unparsable URL contributes no derivable values
|
|
336
|
+
}
|
|
337
|
+
return values;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* True when `capture`'s response carries no business-relevant state: a
|
|
341
|
+
* non-JSON (or absent) content-type, a null/undefined body, a JSON body
|
|
342
|
+
* with no keys, or a JSON body whose every leaf value is already present in
|
|
343
|
+
* the request's own URL (query string or path). A page-load sensor/
|
|
344
|
+
* analytics beacon (an Akamai-style session-authenticator pixel, a polled
|
|
345
|
+
* non-JSON status ping, or one that echoes back the `clientId`/`siteId` the
|
|
346
|
+
* caller just sent it) answers every call with exactly this shape — unlike a
|
|
347
|
+
* real API response, which carries data a caller could not have already
|
|
348
|
+
* known before firing the request.
|
|
349
|
+
*
|
|
350
|
+
* The "echoes its own request" branch is what closes the gap a bare
|
|
351
|
+
* empty-body check misses: a response is technically non-empty JSON but
|
|
352
|
+
* every leaf is one of the fixed query's own values (or a path segment),
|
|
353
|
+
* so nothing in it is new information relative to what the caller already
|
|
354
|
+
* sent — it is not business-relevant just because it happens to be
|
|
355
|
+
* non-empty.
|
|
356
|
+
*
|
|
357
|
+
* Missing response metadata (unit-test callers that construct a capture
|
|
358
|
+
* without `responseHeaders`/`responseBody`) reads as "no business-relevant
|
|
359
|
+
* state" too: this predicate only ever narrows an already-recurring
|
|
360
|
+
* candidate — {@link isZeroVarianceRepeatCapture} (already-fixed-query) and
|
|
361
|
+
* `recon-generate.ts`'s own-repeat structural-isolation pass (already
|
|
362
|
+
* repeats identically, regardless of query shape) — so defaulting to the
|
|
363
|
+
* noise reading there costs nothing except in the caller that deliberately
|
|
364
|
+
* supplies a JSON response to prove the opposite.
|
|
365
|
+
*
|
|
366
|
+
* Exported (not just used internally by {@link isZeroVarianceRepeatCapture})
|
|
367
|
+
* because a same-host, fixed-request endpoint with NO query string at all
|
|
368
|
+
* (e.g. a polled feature-toggle feed with a single-compound-segment path)
|
|
369
|
+
* can be genuinely zero-business-value too, and query-key matching alone —
|
|
370
|
+
* this file's `hasFixedKey` signal — has nothing to key off when there is no
|
|
371
|
+
* query. Response business-value is the general, path/query-shape-independent
|
|
372
|
+
* signal for "this repeat carries nothing a caller could not already know,"
|
|
373
|
+
* so `recon-generate.ts` reuses it directly instead of the query-shape logic
|
|
374
|
+
* that cannot apply to a query-less endpoint.
|
|
375
|
+
*/
|
|
376
|
+
function hasNoBusinessRelevantResponseState(capture) {
|
|
377
|
+
const headers = capture.responseHeaders ?? {};
|
|
378
|
+
const contentType = (Object.entries(headers).find(([key]) => key.toLowerCase() === "content-type")?.[1] ?? "").toLowerCase();
|
|
379
|
+
if (!contentType.includes("json"))
|
|
380
|
+
return true;
|
|
381
|
+
const body = capture.responseBody;
|
|
382
|
+
if (body === null || body === undefined)
|
|
383
|
+
return true;
|
|
384
|
+
if (typeof body !== "object")
|
|
385
|
+
return false;
|
|
386
|
+
if (Object.keys(body).length === 0)
|
|
387
|
+
return true;
|
|
388
|
+
const leaves = [];
|
|
389
|
+
collectLeafValues(body, leaves);
|
|
390
|
+
if (leaves.length === 0)
|
|
391
|
+
return true;
|
|
392
|
+
const ownValues = urlOwnValues(capture.url);
|
|
393
|
+
return leaves.every((leaf) => ownValues.has(leaf));
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Identity of the endpoint a URL addresses, ignoring its query string, so
|
|
397
|
+
* recurrences of the same endpoint with a differently-ordered or
|
|
398
|
+
* incidentally-varying query string still group as "the same endpoint"
|
|
399
|
+
* instead of requiring the whole URL to be byte-identical. Mirrors
|
|
400
|
+
* `recon-generate.ts`'s own `endpointKey` (origin + pathname) — the same
|
|
401
|
+
* definition of "same endpoint" this codebase already uses for collapse and
|
|
402
|
+
* variance reasoning elsewhere.
|
|
403
|
+
*/
|
|
404
|
+
function endpointOrigin(url) {
|
|
405
|
+
try {
|
|
406
|
+
const parsed = new URL(url);
|
|
407
|
+
return `${parsed.origin}${parsed.pathname}`;
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
return null;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* True when `candidate` carries at least one query key whose value stays
|
|
415
|
+
* identical across every occurrence of the same endpoint in `allCaptures`,
|
|
416
|
+
* and recurs at least once elsewhere at the same method and endpoint
|
|
417
|
+
* (origin + pathname) — proof the endpoint's identifying signal is a fixed
|
|
418
|
+
* query key, not the full query string. Any OTHER key that differs between
|
|
419
|
+
* occurrences (a cache-buster, a session nonce) is incidental and does not
|
|
420
|
+
* prevent the match, AND either the request body is also byte-identical
|
|
421
|
+
* across every occurrence, or the response carries no business-relevant
|
|
422
|
+
* state ({@link hasNoBusinessRelevantResponseState}).
|
|
423
|
+
*
|
|
424
|
+
* Grouping by the full URL string (byte-identical query) is too strict: a
|
|
425
|
+
* real beacon can carry one incidental varying query key (a cache-buster or
|
|
426
|
+
* session nonce that is never literally equal across calls) alongside its
|
|
427
|
+
* genuinely fixed identifying keys (`clientId`, `environment`, `siteId`),
|
|
428
|
+
* and requiring the entire string to match lets it escape exclusion
|
|
429
|
+
* entirely. Per-key comparison against the candidate's own keys — rather
|
|
430
|
+
* than requiring the whole set of keys or values to agree — is what
|
|
431
|
+
* recovers the beacon: the fixed keys still prove the endpoint's identity,
|
|
432
|
+
* the varying key is simply ignored because it never advances past
|
|
433
|
+
* "matches on at least one key."
|
|
434
|
+
*
|
|
435
|
+
* A key is "fixed" when the candidate's own value for it is the STRICT
|
|
436
|
+
* MAJORITY value across every same-endpoint occurrence in `allCaptures`
|
|
437
|
+
* (more than half, with at least two supporting occurrences) — not
|
|
438
|
+
* necessarily every single one. `allCaptures` is the full raw capture list,
|
|
439
|
+
* which can include an incidental earlier/differently-scoped occurrence of
|
|
440
|
+
* the same endpoint (a page load before the flow proper starts) that
|
|
441
|
+
* legitimately carries a different value for every candidate key. Requiring
|
|
442
|
+
* literal unanimity would let that one outlier veto the proof for the
|
|
443
|
+
* entire flow's worth of genuinely fixed repeats; requiring only a majority
|
|
444
|
+
* still refuses to flag a key that varies freely (no value would ever reach
|
|
445
|
+
* a majority) while tolerating the minority-outlier shape a real archive
|
|
446
|
+
* produces.
|
|
447
|
+
*
|
|
448
|
+
* The requirement that at least one key be fixed is deliberate, not an
|
|
449
|
+
* extension/host special-case: it is what separates this from a genuinely
|
|
450
|
+
* no-argument own endpoint that a flow legitimately polls with an identical
|
|
451
|
+
* body every time (a feature-toggle feed, an availability heartbeat) —
|
|
452
|
+
* those endpoints carry real business meaning and folding their repeats
|
|
453
|
+
* into a single call is the collapse mechanism's job, not this predicate's.
|
|
454
|
+
* A marketing/analytics beacon's own fixed identifying query is the tell a
|
|
455
|
+
* plain body-repetition check can't see on its own, and is exactly the
|
|
456
|
+
* shape {@link isStructurallyIsolatedCapture} can be fooled by — that
|
|
457
|
+
* check's reference pool is every OTHER admitted capture, so N copies of
|
|
458
|
+
* the same fixed-query request "vouch" for each other's path tokens and
|
|
459
|
+
* none of them reads as isolated.
|
|
460
|
+
*
|
|
461
|
+
* The response-state fallback exists because a byte-identical-body
|
|
462
|
+
* requirement alone misses a beacon whose body embeds a fingerprint,
|
|
463
|
+
* timestamp, or session nonce that differs on every fire even though the
|
|
464
|
+
* fixed query is the only signal that actually identifies the endpoint —
|
|
465
|
+
* the body varies, but the response never carries anything the flow could
|
|
466
|
+
* not already derive from the request itself.
|
|
467
|
+
*/
|
|
468
|
+
function isZeroVarianceRepeatCapture(candidate, allCaptures) {
|
|
469
|
+
let candidateUrl;
|
|
470
|
+
try {
|
|
471
|
+
candidateUrl = new URL(candidate.url);
|
|
472
|
+
}
|
|
473
|
+
catch {
|
|
474
|
+
return false;
|
|
475
|
+
}
|
|
476
|
+
const candidateEndpoint = endpointOrigin(candidate.url);
|
|
477
|
+
if (candidateEndpoint === null)
|
|
478
|
+
return false;
|
|
479
|
+
const candidateKeys = [...candidateUrl.searchParams.keys()];
|
|
480
|
+
if (candidateKeys.length === 0)
|
|
481
|
+
return false;
|
|
482
|
+
const sameEndpoint = allCaptures.filter((c) => c.method === candidate.method && endpointOrigin(c.url) === candidateEndpoint);
|
|
483
|
+
if (sameEndpoint.length < 2)
|
|
484
|
+
return false;
|
|
485
|
+
const sameEndpointUrls = sameEndpoint
|
|
486
|
+
.map((c) => {
|
|
487
|
+
try {
|
|
488
|
+
return new URL(c.url);
|
|
489
|
+
}
|
|
490
|
+
catch {
|
|
491
|
+
return null;
|
|
492
|
+
}
|
|
493
|
+
})
|
|
494
|
+
.filter((u) => u !== null);
|
|
495
|
+
const hasFixedKey = candidateKeys.some((key) => {
|
|
496
|
+
const candidateValue = candidateUrl.searchParams.get(key);
|
|
497
|
+
const matchCount = sameEndpointUrls.filter((u) => u.searchParams.get(key) === candidateValue).length;
|
|
498
|
+
return matchCount >= 2 && matchCount > sameEndpointUrls.length / 2;
|
|
499
|
+
});
|
|
500
|
+
if (!hasFixedKey)
|
|
501
|
+
return false;
|
|
502
|
+
const bodyIdentical = sameEndpoint.every((c) => c.requestPostData === candidate.requestPostData);
|
|
503
|
+
return bodyIdentical || hasNoBusinessRelevantResponseState(candidate);
|
|
504
|
+
}
|
|
296
505
|
/**
|
|
297
506
|
* True when `hostname` is allowed as a fixture host: an exact match against
|
|
298
507
|
* `ownBackendHostnames` when the flow declares any, otherwise a
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capture-filters.js","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG
|
|
1
|
+
{"version":3,"file":"capture-filters.js","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;;;;;;;;AAEH;;;;;;;GAOG;AACH;IACE,OAAO;QACL,4BAA4B;QAC5B,aAAa;QACb,yBAAyB;QACzB,sBAAsB;QACtB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,EAAE,CAAC;aAChD,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;KACnB,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACU,QAAA,uBAAuB,GAAG,sBAAsB,CAAC;AAE9D;;;;;;;;GAQG;AACH,MAAM,uBAAuB,GAAG;IAC9B,eAAe;IACf,YAAY;IACZ,YAAY;IACZ,iBAAiB;IACjB,sBAAsB;IACtB,sBAAsB;IACtB,cAAc;IACd,cAAc;IACd,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,eAAe;IACf,mBAAmB;IACnB,cAAc;IACd,YAAY;IACZ,YAAY;IACZ,gBAAgB;IAChB,uBAAuB;CACxB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,eAAe,GAAG,gEAAgE,CAAC;AAEzF;;;;;;;GAOG;AACH,oBAA2B,GAAW;IACpC,MAAM,QAAQ,GAAG,oBAAoB,EAAE,CAAC;IACxC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3F,IAAI,QAAA,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/D,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,2BAAkC,QAAgB;IAChD,MAAM,MAAM,GAAG,QAAQ;SACpB,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5E,CAAC;AAED,0FAA0F;AAC1F,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEpG;;;;;;;;;;;;;;GAcG;AACH,8BAAqC,IAAY;IAC/C,MAAM,KAAK,GAAG,IAAI;SACf,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,OAAO,CAAC;SACf,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACnB,MAAM,KAAK,GAAG,OAAO;aAClB,KAAK,CAAC,eAAe,CAAC;aACtB,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;aACvD,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;SACjC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACxE,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI;SACR,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;SACvC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACnF,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,4BAA4B,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC9C,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,uCACE,aAAqB,EACrB,cAAiC;IAEjC,MAAM,eAAe,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5D,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE;QAC3C,MAAM,eAAe,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC5D,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;QAC9C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,uCACE,aAAqB,EACrB,SAA4B;IAE5B,MAAM,eAAe,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5D,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC;QAAE,OAAO,CAAC,6BAA6B,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAC9F,IAAI,CAAC,4BAA4B,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,SAAS;SACN,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC;SAC7D,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAC3D,CAAC;IACF,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,0BAAiC,KAAa,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;QACtC,CAAC;IACH,CAAC;IACD,IAAI,CAAC,4BAA4B,CAAC,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,OAAO,sBAAsB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,KAAc,EAAE,GAAa;IACtD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACtC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAgC,CAAC,EAAE,CAAC;YACpD,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,OAAO;IACT,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;YAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpE,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;IACnD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,4CAAmD,OAIlD;IACC,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC;IAC9C,MAAM,WAAW,GAAG,CAClB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CACzF,CAAC,WAAW,EAAE,CAAC;IAChB,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACrD,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC3C,IAAI,MAAM,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,qCACE,SAMC,EACD,WAAuF;IAEvF,IAAI,YAAiB,CAAC;IACtB,IAAI,CAAC;QACH,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,iBAAiB,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACxD,IAAI,iBAAiB,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,aAAa,GAAG,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,iBAAiB,CACpF,CAAC;IACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,MAAM,gBAAgB,GAAG,YAAY;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAY,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7C,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CACxC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,cAAc,CAClD,CAAC,MAAM,CAAC;QACT,OAAO,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,eAAe,CAAC,CAAC;IACjG,OAAO,aAAa,IAAI,kCAAkC,CAAC,SAAS,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;;GAOG;AACH,8BACE,QAAgB,EAChB,mBAA6B,EAC7B,cAA6B;IAE7B,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAClF,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC;QAAE,OAAO,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,OAAO,cAAc,KAAK,IAAI,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,cAAc,CAAC;AAC/E,CAAC"}
|
|
@@ -1,4 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Site-agnostic capture of hCaptcha's programmatic render config. When a
|
|
3
|
+
* site calls `hcaptcha.render(container, { sitekey, callback })` rather than
|
|
4
|
+
* declaring `data-callback` on the widget element, the callback only ever
|
|
5
|
+
* lives inside hCaptcha's own closure — nothing in the DOM names it. This
|
|
6
|
+
* module builds a page-init script (for `Page.addInitScript`) that
|
|
7
|
+
* monkeypatches `hcaptcha.render` before any site script runs, so every
|
|
8
|
+
* render call's `{ sitekey, widgetId, callback }` lands in a page-global
|
|
9
|
+
* registry a flow hook can query later, regardless of which plugin or site
|
|
10
|
+
* triggered the render.
|
|
11
|
+
*/
|
|
2
12
|
/** Page-global property name the capture script stores its registry under. */
|
|
3
13
|
export declare const HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = "__barnacleHcaptchaCallbacks";
|
|
4
14
|
/**
|
|
@@ -9,25 +19,4 @@ export declare const HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = "__barnacleHcaptchaCall
|
|
|
9
19
|
* value or side effects.
|
|
10
20
|
*/
|
|
11
21
|
export declare function buildHcaptchaCallbackCaptureScript(): string;
|
|
12
|
-
/**
|
|
13
|
-
* Re-asserts the capture script into every frame's own realm as it attaches
|
|
14
|
-
* or navigates, closing the race where a same-origin iframe's own script
|
|
15
|
-
* assigns `window.hcaptcha` and calls `render` before `context.addInitScript`'s
|
|
16
|
-
* effect is observably in place in that frame's realm (a CDP round-trip race
|
|
17
|
-
* under some session providers). Complements, rather than replaces, the
|
|
18
|
-
* context-level install — that install remains the first line of defense for
|
|
19
|
-
* the common case (fast frames, no race).
|
|
20
|
-
*
|
|
21
|
-
* Frame-agnostic: this listens for `Page.frameAttached`/`Page.frameNavigated`
|
|
22
|
-
* on the main frame's CDP session and evaluates the (idempotent) capture
|
|
23
|
-
* script into whichever frame each event names, regardless of which site or
|
|
24
|
-
* plugin owns that frame.
|
|
25
|
-
*
|
|
26
|
-
* `Page.frameAttached`/`Page.frameNavigated` can fire before that frame's
|
|
27
|
-
* main-world execution context exists, so the first `evaluate()` rejects
|
|
28
|
-
* with "main world not ready for frame ...". Each fresh `evaluate()` call
|
|
29
|
-
* re-arms stagehand's own wait for that context, so retrying the call
|
|
30
|
-
* (rather than adding a bespoke listener) is what closes the race.
|
|
31
|
-
*/
|
|
32
|
-
export declare function installHcaptchaCallbackCaptureOnAllFrames(page: Page): void;
|
|
33
22
|
//# sourceMappingURL=captcha-callback-capture.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"captcha-callback-capture.d.ts","sourceRoot":"","sources":["../../src/scraper/captcha-callback-capture.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"captcha-callback-capture.d.ts","sourceRoot":"","sources":["../../src/scraper/captcha-callback-capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,8EAA8E;AAC9E,eAAO,MAAM,iCAAiC,gCAAgC,CAAC;AAE/E;;;;;;GAMG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAuE3D"}
|
|
@@ -1,45 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = void 0;
|
|
37
|
-
exports.buildHcaptchaCallbackCaptureScript = buildHcaptchaCallbackCaptureScript;
|
|
38
|
-
exports.installHcaptchaCallbackCaptureOnAllFrames = installHcaptchaCallbackCaptureOnAllFrames;
|
|
39
|
-
const p_retry_1 = __importStar(require("p-retry"));
|
|
40
|
-
const logging_1 = require("../lib/logging");
|
|
41
|
-
/** Matches stagehand's `ExecutionContextRegistry.waitForMainWorld` rejection message. */
|
|
42
|
-
const MAIN_WORLD_NOT_READY_PATTERN = /main world not ready for frame/i;
|
|
43
2
|
/**
|
|
44
3
|
* Site-agnostic capture of hCaptcha's programmatic render config. When a
|
|
45
4
|
* site calls `hcaptcha.render(container, { sitekey, callback })` rather than
|
|
@@ -51,7 +10,9 @@ const MAIN_WORLD_NOT_READY_PATTERN = /main world not ready for frame/i;
|
|
|
51
10
|
* registry a flow hook can query later, regardless of which plugin or site
|
|
52
11
|
* triggered the render.
|
|
53
12
|
*/
|
|
54
|
-
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = void 0;
|
|
15
|
+
exports.buildHcaptchaCallbackCaptureScript = buildHcaptchaCallbackCaptureScript;
|
|
55
16
|
/** Page-global property name the capture script stores its registry under. */
|
|
56
17
|
exports.HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = "__barnacleHcaptchaCallbacks";
|
|
57
18
|
/**
|
|
@@ -133,51 +94,4 @@ function buildHcaptchaCallbackCaptureScript() {
|
|
|
133
94
|
}
|
|
134
95
|
})();`;
|
|
135
96
|
}
|
|
136
|
-
/**
|
|
137
|
-
* Re-asserts the capture script into every frame's own realm as it attaches
|
|
138
|
-
* or navigates, closing the race where a same-origin iframe's own script
|
|
139
|
-
* assigns `window.hcaptcha` and calls `render` before `context.addInitScript`'s
|
|
140
|
-
* effect is observably in place in that frame's realm (a CDP round-trip race
|
|
141
|
-
* under some session providers). Complements, rather than replaces, the
|
|
142
|
-
* context-level install — that install remains the first line of defense for
|
|
143
|
-
* the common case (fast frames, no race).
|
|
144
|
-
*
|
|
145
|
-
* Frame-agnostic: this listens for `Page.frameAttached`/`Page.frameNavigated`
|
|
146
|
-
* on the main frame's CDP session and evaluates the (idempotent) capture
|
|
147
|
-
* script into whichever frame each event names, regardless of which site or
|
|
148
|
-
* plugin owns that frame.
|
|
149
|
-
*
|
|
150
|
-
* `Page.frameAttached`/`Page.frameNavigated` can fire before that frame's
|
|
151
|
-
* main-world execution context exists, so the first `evaluate()` rejects
|
|
152
|
-
* with "main world not ready for frame ...". Each fresh `evaluate()` call
|
|
153
|
-
* re-arms stagehand's own wait for that context, so retrying the call
|
|
154
|
-
* (rather than adding a bespoke listener) is what closes the race.
|
|
155
|
-
*/
|
|
156
|
-
function installHcaptchaCallbackCaptureOnAllFrames(page) {
|
|
157
|
-
const session = page.getSessionForFrame(page.mainFrameId());
|
|
158
|
-
const script = buildHcaptchaCallbackCaptureScript();
|
|
159
|
-
const evaluateIntoFrame = (frameId) => {
|
|
160
|
-
(0, p_retry_1.default)(() => page
|
|
161
|
-
.frameForId(frameId)
|
|
162
|
-
.evaluate(script)
|
|
163
|
-
.catch((err) => {
|
|
164
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
165
|
-
if (!MAIN_WORLD_NOT_READY_PATTERN.test(error.message)) {
|
|
166
|
-
throw new p_retry_1.AbortError(error);
|
|
167
|
-
}
|
|
168
|
-
throw error;
|
|
169
|
-
}), { retries: 5, factor: 1, minTimeout: 100, maxTimeout: 100 }).catch((err) => {
|
|
170
|
-
logger.warn(`hcaptcha callback capture: per-frame re-assert failed: ${String(err)}`);
|
|
171
|
-
});
|
|
172
|
-
};
|
|
173
|
-
session.send("Page.enable").catch((err) => {
|
|
174
|
-
logger.warn(`hcaptcha callback capture: Page.enable failed: ${String(err)}`);
|
|
175
|
-
});
|
|
176
|
-
session.on("Page.frameAttached", (params) => {
|
|
177
|
-
evaluateIntoFrame(params.frameId);
|
|
178
|
-
});
|
|
179
|
-
session.on("Page.frameNavigated", (params) => {
|
|
180
|
-
evaluateIntoFrame(params.frame.id);
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
97
|
//# sourceMappingURL=captcha-callback-capture.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"captcha-callback-capture.js","sourceRoot":"","sources":["../../src/scraper/captcha-callback-capture.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"captcha-callback-capture.js","sourceRoot":"","sources":["../../src/scraper/captcha-callback-capture.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;AAEH,8EAA8E;AACjE,QAAA,iCAAiC,GAAG,6BAA6B,CAAC;AAE/E;;;;;;GAMG;AACH;IACE,OAAO;2BACkB,IAAI,CAAC,SAAS,CAAC,QAAA,iCAAiC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAoEpE,CAAC;AACT,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Page } from "@browserbasehq/stagehand";
|
|
2
|
+
/**
|
|
3
|
+
* Installs `source` into every frame's realm before that frame's own scripts
|
|
4
|
+
* execute — including a newly-attached same-origin child iframe (covered by
|
|
5
|
+
* `Page.addScriptToEvaluateOnNewDocument`, a Page-domain command the CDP
|
|
6
|
+
* contract applies to every frame of the target it's sent to) and a
|
|
7
|
+
* newly-attached cross-origin child target (paused via `Target.setAutoAttach`
|
|
8
|
+
* before any of its scripts run, installed into its own session, and only
|
|
9
|
+
* then resumed via `Runtime.runIfWaitingForDebugger`).
|
|
10
|
+
*
|
|
11
|
+
* Must be wired as the sole per-target install path for a given script — the
|
|
12
|
+
* race this closes only stays closed if nothing else independently resumes a
|
|
13
|
+
* paused target before this module's install lands.
|
|
14
|
+
*/
|
|
15
|
+
export declare function installInitScriptOnAllFrames(page: Page, source: string): Promise<void>;
|
|
16
|
+
//# sourceMappingURL=cdp-frame-init-script.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cdp-frame-init-script.d.ts","sourceRoot":"","sources":["../../src/scraper/cdp-frame-init-script.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAwErD;;;;;;;;;;;;GAYG;AACH,wBAAsB,4BAA4B,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI5F"}
|