@enricai/barnacle 1.12.48 → 1.12.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/recon/capture-filters.d.ts +172 -5
- package/dist/recon/capture-filters.d.ts.map +1 -1
- package/dist/recon/capture-filters.js +308 -8
- package/dist/recon/capture-filters.js.map +1 -1
- package/dist/scraper/captcha-callback-capture.d.ts +11 -22
- package/dist/scraper/captcha-callback-capture.d.ts.map +1 -1
- package/dist/scraper/captcha-callback-capture.js +3 -89
- package/dist/scraper/captcha-callback-capture.js.map +1 -1
- package/dist/scraper/cdp-frame-init-script.d.ts +16 -0
- package/dist/scraper/cdp-frame-init-script.d.ts.map +1 -0
- package/dist/scraper/cdp-frame-init-script.js +67 -0
- package/dist/scraper/cdp-frame-init-script.js.map +1 -0
- package/dist/scraper/flow-runner.d.ts +58 -1
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +95 -9
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/session.d.ts.map +1 -1
- package/dist/scraper/session.js +3 -3
- package/dist/scraper/session.js.map +1 -1
- package/dist/scripts/recon-generate-multicall-fixture.d.ts +31 -0
- package/dist/scripts/recon-generate-multicall-fixture.d.ts.map +1 -1
- package/dist/scripts/recon-generate-multicall-fixture.js +78 -0
- package/dist/scripts/recon-generate-multicall-fixture.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +90 -40
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +945 -82
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +1 -1
|
@@ -47,13 +47,29 @@ export declare function isNoiseUrl(url: string): boolean;
|
|
|
47
47
|
* public suffixes (e.g. `co.uk`), which this codebase does not target.
|
|
48
48
|
*/
|
|
49
49
|
export declare function registrableDomain(hostname: string): string;
|
|
50
|
+
/**
|
|
51
|
+
* Splits a URL path into lowercase word tokens, breaking on non-alphanumeric
|
|
52
|
+
* separators (`/`, `-`, `_`, `.`) and camelCase boundaries, then drops tokens
|
|
53
|
+
* that are too short (<3 chars) or too generic ({@link GENERIC_PATH_TOKENS})
|
|
54
|
+
* to signal endpoint-family relatedness on their own.
|
|
55
|
+
*
|
|
56
|
+
* Only tokens from a *compound* path segment (one that itself splits into 2+
|
|
57
|
+
* words, e.g. `listing-avail-vas`) are kept. A whole segment that is a single
|
|
58
|
+
* plain word (e.g. `search`, `list`, `detail`, `widget`) is dropped entirely:
|
|
59
|
+
* such words recur across unrelated endpoint families on the same host, so
|
|
60
|
+
* treating them as a relatedness signal produces false positives (a marketing
|
|
61
|
+
* `/promotions/search` call "matching" a booking flow's `.../v1/search` just
|
|
62
|
+
* because both happen to end in the common word "search"). Compound segments
|
|
63
|
+
* are where a real endpoint-family identifier lives.
|
|
64
|
+
*/
|
|
65
|
+
export declare function pathStructuralTokens(path: string): Set<string>;
|
|
50
66
|
/**
|
|
51
67
|
* True when `candidatePath` shares enough path-segment tokens with at least
|
|
52
68
|
* one path in `referencePaths` to be judged part of the same endpoint
|
|
53
69
|
* family, false when it is structurally unrelated to all of them.
|
|
54
70
|
*
|
|
55
71
|
* A literal prefix/substring check is too strict: real same-flow endpoint
|
|
56
|
-
* families (e.g. `.../
|
|
72
|
+
* families (e.g. `.../listing-avail-vas/...` and `.../item-detail-vas/...`)
|
|
57
73
|
* do not share a string prefix segment-for-segment, but do share the
|
|
58
74
|
* `-vas` suffix and surrounding path structure once tokenized. Token overlap
|
|
59
75
|
* on segment words — ignoring short/generic tokens — catches that relation
|
|
@@ -65,10 +81,36 @@ export declare function isStructurallyRelevantCapture(candidatePath: string, ref
|
|
|
65
81
|
* True when `candidatePath` has a compound (multi-word) path segment whose
|
|
66
82
|
* tokens share nothing with ANY other path in `poolPaths` — a same-host
|
|
67
83
|
* capture whose own path structurally isolates it from every other member of
|
|
68
|
-
* the pool it was admitted into.
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
84
|
+
* the pool it was admitted into.
|
|
85
|
+
*
|
|
86
|
+
* A path with no compound segment (empty token set) falls back to a raw
|
|
87
|
+
* segment-overlap check instead of an automatic pass, but only when the path
|
|
88
|
+
* has a {@link hasRepeatedMeaningfulSegment repeated segment} of its own
|
|
89
|
+
* (e.g. `widget` recurring in `/widget/api/promotions/widget/default`): a chain's
|
|
90
|
+
* own steps are plain-word paths that name a distinct action per step
|
|
91
|
+
* (`/applicant`, `/sections/name`, or a 3-segment `/user/profile/edit`) and
|
|
92
|
+
* so essentially never repeat a segment against themselves, even when they
|
|
93
|
+
* share no tokens — or even raw segments — with sibling steps (e.g. a real
|
|
94
|
+
* `create` -> `sections/name` -> `submit` sequence, none of whose plain-word
|
|
95
|
+
* steps share a segment with either of the others). A same-host
|
|
96
|
+
* marketing/promotions capture is commonly self-referential — its own
|
|
97
|
+
* resource identifier shows up twice in its own path — which is the signal
|
|
98
|
+
* that marks it as a templated/generated noise path rather than a genuine
|
|
99
|
+
* short (or not-so-short) single-word chain step, independent of segment
|
|
100
|
+
* count. Requiring it to also share no raw path segment with the pool closes
|
|
101
|
+
* the gap without penalizing genuine single-word chain steps of any depth.
|
|
102
|
+
* The overlap check drops the same short/generic segments
|
|
103
|
+
* {@link pathStructuralTokens} already excludes ({@link GENERIC_PATH_TOKENS},
|
|
104
|
+
* <3 chars): otherwise two completely unrelated endpoint families sharing
|
|
105
|
+
* only a boilerplate `/api/` segment would be judged "related" and the
|
|
106
|
+
* isolated capture would dodge exclusion the same way the empty-token-set
|
|
107
|
+
* exemption originally let it. Pool paths that are themselves self-referential
|
|
108
|
+
* ({@link hasRepeatedMeaningfulSegment}) are excluded from contributing to
|
|
109
|
+
* this overlap set: otherwise two co-occurring same-noise-family path
|
|
110
|
+
* variants (e.g. a POST and a GET/default variant of the same templated
|
|
111
|
+
* marketing endpoint) would mutually "vouch" for each other's segments and
|
|
112
|
+
* neither would be recognized as isolated — only a genuinely non-self-
|
|
113
|
+
* referential path can vouch for a candidate's relatedness.
|
|
72
114
|
*
|
|
73
115
|
* Anchored on {@link isStructurallyRelevantCapture}'s own token overlap rule
|
|
74
116
|
* so "isolated" is exactly "not relevant to anything else in the pool" —
|
|
@@ -78,6 +120,131 @@ export declare function isStructurallyRelevantCapture(candidatePath: string, ref
|
|
|
78
120
|
* unlike {@link isStructurallyRelevantCapture} which requires one.
|
|
79
121
|
*/
|
|
80
122
|
export declare function isStructurallyIsolatedCapture(candidatePath: string, poolPaths: readonly string[]): boolean;
|
|
123
|
+
/**
|
|
124
|
+
* True when `pathA` and `pathB` belong to the same structural path family:
|
|
125
|
+
* either they share a compound-segment token ({@link pathStructuralTokens}),
|
|
126
|
+
* or both are self-referential ({@link hasRepeatedMeaningfulSegment}) and
|
|
127
|
+
* share a raw meaningful segment (e.g. `widget` in both `/widget/api/promotions/widget`
|
|
128
|
+
* and `/widget/api/promotions/widget/default`).
|
|
129
|
+
*
|
|
130
|
+
* Generalizes {@link isStructurallyRelevantCapture}'s token-overlap rule to
|
|
131
|
+
* also cover the all-single-word-segment, self-referential shape that rule
|
|
132
|
+
* alone can't see (its token set is empty for a path with no compound
|
|
133
|
+
* segment) — the same shape {@link isStructurallyIsolatedCapture} already
|
|
134
|
+
* recognizes for pool-isolation. Exported so a caller reasoning about
|
|
135
|
+
* "is this OTHER capture part of the SAME noise family as an already-known
|
|
136
|
+
* noise capture" (rather than "is this capture isolated from a whole pool")
|
|
137
|
+
* can reuse the identical relatedness rule instead of re-deriving it.
|
|
138
|
+
*/
|
|
139
|
+
export declare function isSamePathFamily(pathA: string, pathB: string): boolean;
|
|
140
|
+
/**
|
|
141
|
+
* True when `capture`'s response carries no business-relevant state: a
|
|
142
|
+
* non-JSON (or absent) content-type, a null/undefined body, a JSON body
|
|
143
|
+
* with no keys, or a JSON body whose every leaf value is already present in
|
|
144
|
+
* the request's own URL (query string or path). A page-load sensor/
|
|
145
|
+
* analytics beacon (an Akamai-style session-authenticator pixel, a polled
|
|
146
|
+
* non-JSON status ping, or one that echoes back the `clientId`/`siteId` the
|
|
147
|
+
* caller just sent it) answers every call with exactly this shape — unlike a
|
|
148
|
+
* real API response, which carries data a caller could not have already
|
|
149
|
+
* known before firing the request.
|
|
150
|
+
*
|
|
151
|
+
* The "echoes its own request" branch is what closes the gap a bare
|
|
152
|
+
* empty-body check misses: a response is technically non-empty JSON but
|
|
153
|
+
* every leaf is one of the fixed query's own values (or a path segment),
|
|
154
|
+
* so nothing in it is new information relative to what the caller already
|
|
155
|
+
* sent — it is not business-relevant just because it happens to be
|
|
156
|
+
* non-empty.
|
|
157
|
+
*
|
|
158
|
+
* Missing response metadata (unit-test callers that construct a capture
|
|
159
|
+
* without `responseHeaders`/`responseBody`) reads as "no business-relevant
|
|
160
|
+
* state" too: this predicate only ever narrows an already-recurring
|
|
161
|
+
* candidate — {@link isZeroVarianceRepeatCapture} (already-fixed-query) and
|
|
162
|
+
* `recon-generate.ts`'s own-repeat structural-isolation pass (already
|
|
163
|
+
* repeats identically, regardless of query shape) — so defaulting to the
|
|
164
|
+
* noise reading there costs nothing except in the caller that deliberately
|
|
165
|
+
* supplies a JSON response to prove the opposite.
|
|
166
|
+
*
|
|
167
|
+
* Exported (not just used internally by {@link isZeroVarianceRepeatCapture})
|
|
168
|
+
* because a same-host, fixed-request endpoint with NO query string at all
|
|
169
|
+
* (e.g. a polled feature-toggle feed with a single-compound-segment path)
|
|
170
|
+
* can be genuinely zero-business-value too, and query-key matching alone —
|
|
171
|
+
* this file's `hasFixedKey` signal — has nothing to key off when there is no
|
|
172
|
+
* query. Response business-value is the general, path/query-shape-independent
|
|
173
|
+
* signal for "this repeat carries nothing a caller could not already know,"
|
|
174
|
+
* so `recon-generate.ts` reuses it directly instead of the query-shape logic
|
|
175
|
+
* that cannot apply to a query-less endpoint.
|
|
176
|
+
*/
|
|
177
|
+
export declare function hasNoBusinessRelevantResponseState(capture: {
|
|
178
|
+
url: string;
|
|
179
|
+
responseHeaders?: Record<string, string>;
|
|
180
|
+
responseBody?: unknown;
|
|
181
|
+
}): boolean;
|
|
182
|
+
/**
|
|
183
|
+
* True when `candidate` carries at least one query key whose value stays
|
|
184
|
+
* identical across every occurrence of the same endpoint in `allCaptures`,
|
|
185
|
+
* and recurs at least once elsewhere at the same method and endpoint
|
|
186
|
+
* (origin + pathname) — proof the endpoint's identifying signal is a fixed
|
|
187
|
+
* query key, not the full query string. Any OTHER key that differs between
|
|
188
|
+
* occurrences (a cache-buster, a session nonce) is incidental and does not
|
|
189
|
+
* prevent the match, AND either the request body is also byte-identical
|
|
190
|
+
* across every occurrence, or the response carries no business-relevant
|
|
191
|
+
* state ({@link hasNoBusinessRelevantResponseState}).
|
|
192
|
+
*
|
|
193
|
+
* Grouping by the full URL string (byte-identical query) is too strict: a
|
|
194
|
+
* real beacon can carry one incidental varying query key (a cache-buster or
|
|
195
|
+
* session nonce that is never literally equal across calls) alongside its
|
|
196
|
+
* genuinely fixed identifying keys (`clientId`, `environment`, `siteId`),
|
|
197
|
+
* and requiring the entire string to match lets it escape exclusion
|
|
198
|
+
* entirely. Per-key comparison against the candidate's own keys — rather
|
|
199
|
+
* than requiring the whole set of keys or values to agree — is what
|
|
200
|
+
* recovers the beacon: the fixed keys still prove the endpoint's identity,
|
|
201
|
+
* the varying key is simply ignored because it never advances past
|
|
202
|
+
* "matches on at least one key."
|
|
203
|
+
*
|
|
204
|
+
* A key is "fixed" when the candidate's own value for it is the STRICT
|
|
205
|
+
* MAJORITY value across every same-endpoint occurrence in `allCaptures`
|
|
206
|
+
* (more than half, with at least two supporting occurrences) — not
|
|
207
|
+
* necessarily every single one. `allCaptures` is the full raw capture list,
|
|
208
|
+
* which can include an incidental earlier/differently-scoped occurrence of
|
|
209
|
+
* the same endpoint (a page load before the flow proper starts) that
|
|
210
|
+
* legitimately carries a different value for every candidate key. Requiring
|
|
211
|
+
* literal unanimity would let that one outlier veto the proof for the
|
|
212
|
+
* entire flow's worth of genuinely fixed repeats; requiring only a majority
|
|
213
|
+
* still refuses to flag a key that varies freely (no value would ever reach
|
|
214
|
+
* a majority) while tolerating the minority-outlier shape a real archive
|
|
215
|
+
* produces.
|
|
216
|
+
*
|
|
217
|
+
* The requirement that at least one key be fixed is deliberate, not an
|
|
218
|
+
* extension/host special-case: it is what separates this from a genuinely
|
|
219
|
+
* no-argument own endpoint that a flow legitimately polls with an identical
|
|
220
|
+
* body every time (a feature-toggle feed, an availability heartbeat) —
|
|
221
|
+
* those endpoints carry real business meaning and folding their repeats
|
|
222
|
+
* into a single call is the collapse mechanism's job, not this predicate's.
|
|
223
|
+
* A marketing/analytics beacon's own fixed identifying query is the tell a
|
|
224
|
+
* plain body-repetition check can't see on its own, and is exactly the
|
|
225
|
+
* shape {@link isStructurallyIsolatedCapture} can be fooled by — that
|
|
226
|
+
* check's reference pool is every OTHER admitted capture, so N copies of
|
|
227
|
+
* the same fixed-query request "vouch" for each other's path tokens and
|
|
228
|
+
* none of them reads as isolated.
|
|
229
|
+
*
|
|
230
|
+
* The response-state fallback exists because a byte-identical-body
|
|
231
|
+
* requirement alone misses a beacon whose body embeds a fingerprint,
|
|
232
|
+
* timestamp, or session nonce that differs on every fire even though the
|
|
233
|
+
* fixed query is the only signal that actually identifies the endpoint —
|
|
234
|
+
* the body varies, but the response never carries anything the flow could
|
|
235
|
+
* not already derive from the request itself.
|
|
236
|
+
*/
|
|
237
|
+
export declare function isZeroVarianceRepeatCapture(candidate: {
|
|
238
|
+
method: string;
|
|
239
|
+
url: string;
|
|
240
|
+
requestPostData: string | null;
|
|
241
|
+
responseHeaders?: Record<string, string>;
|
|
242
|
+
responseBody?: unknown;
|
|
243
|
+
}, allCaptures: readonly {
|
|
244
|
+
method: string;
|
|
245
|
+
url: string;
|
|
246
|
+
requestPostData: string | null;
|
|
247
|
+
}[]): boolean;
|
|
81
248
|
/**
|
|
82
249
|
* True when `hostname` is allowed as a fixture host: an exact match against
|
|
83
250
|
* `ownBackendHostnames` when the flow declares any, otherwise a
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capture-filters.d.ts","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,EAAE,CAW/C;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,QAAyB,CAAC;AAmC9D;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAc/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM1D;
|
|
1
|
+
{"version":3,"file":"capture-filters.d.ts","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,EAAE,CAW/C;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,QAAyB,CAAC;AAmC9D;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAc/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM1D;AAKD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAc9D;AA+BD;;;;;;;;;;;;GAYG;AACH,wBAAgB,6BAA6B,CAC3C,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,SAAS,MAAM,EAAE,GAChC,OAAO,CAUT;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,6BAA6B,CAC3C,aAAa,EAAE,MAAM,EACrB,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,OAAO,CAWT;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAWtE;AAwCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,EAAE;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,GAAG,OAAO,CAeV;AAoBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,wBAAgB,2BAA2B,CACzC,SAAS,EAAE;IACT,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,EACD,WAAW,EAAE,SAAS;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,EAAE,GACtF,OAAO,CAkCT;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,mBAAmB,EAAE,MAAM,EAAE,EAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,GAC5B,OAAO,CAKT"}
|
|
@@ -15,8 +15,12 @@ exports.ERROR_SINK_PATH_SEGMENT = void 0;
|
|
|
15
15
|
exports.telemetryUrlPatterns = telemetryUrlPatterns;
|
|
16
16
|
exports.isNoiseUrl = isNoiseUrl;
|
|
17
17
|
exports.registrableDomain = registrableDomain;
|
|
18
|
+
exports.pathStructuralTokens = pathStructuralTokens;
|
|
18
19
|
exports.isStructurallyRelevantCapture = isStructurallyRelevantCapture;
|
|
19
20
|
exports.isStructurallyIsolatedCapture = isStructurallyIsolatedCapture;
|
|
21
|
+
exports.isSamePathFamily = isSamePathFamily;
|
|
22
|
+
exports.hasNoBusinessRelevantResponseState = hasNoBusinessRelevantResponseState;
|
|
23
|
+
exports.isZeroVarianceRepeatCapture = isZeroVarianceRepeatCapture;
|
|
20
24
|
exports.isAllowedFixtureHost = isAllowedFixtureHost;
|
|
21
25
|
/**
|
|
22
26
|
* Path/URL substrings we always treat as analytics or logging noise. Site-specific
|
|
@@ -132,7 +136,7 @@ const GENERIC_PATH_TOKENS = new Set(["api", "app", "apps", "v1", "v2", "v3", "co
|
|
|
132
136
|
* to signal endpoint-family relatedness on their own.
|
|
133
137
|
*
|
|
134
138
|
* Only tokens from a *compound* path segment (one that itself splits into 2+
|
|
135
|
-
* words, e.g. `
|
|
139
|
+
* words, e.g. `listing-avail-vas`) are kept. A whole segment that is a single
|
|
136
140
|
* plain word (e.g. `search`, `list`, `detail`, `widget`) is dropped entirely:
|
|
137
141
|
* such words recur across unrelated endpoint families on the same host, so
|
|
138
142
|
* treating them as a relatedness signal produces false positives (a marketing
|
|
@@ -155,13 +159,40 @@ function pathStructuralTokens(path) {
|
|
|
155
159
|
.filter((word) => word.length >= 3 && !GENERIC_PATH_TOKENS.has(word));
|
|
156
160
|
return new Set(words);
|
|
157
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* Lowercased raw path segments, dropping the same short (<3 chars) and
|
|
164
|
+
* generic ({@link GENERIC_PATH_TOKENS}) ones {@link pathStructuralTokens}
|
|
165
|
+
* excludes from compound-segment tokens — so a raw-segment overlap check
|
|
166
|
+
* can't be satisfied by two paths sharing nothing but a boilerplate `/api/`
|
|
167
|
+
* or `/v1/` segment.
|
|
168
|
+
*/
|
|
169
|
+
function meaningfulPathSegments(path) {
|
|
170
|
+
return path
|
|
171
|
+
.split("/")
|
|
172
|
+
.filter(Boolean)
|
|
173
|
+
.map((segment) => segment.toLowerCase())
|
|
174
|
+
.filter((segment) => segment.length >= 3 && !GENERIC_PATH_TOKENS.has(segment));
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* True when some meaningful segment of `path` recurs elsewhere in the same
|
|
178
|
+
* path (e.g. `widget` in `/widget/api/promotions/widget/default`). Real action
|
|
179
|
+
* chains name each step for what it does (`/user/profile/edit`) and so
|
|
180
|
+
* rarely repeat a segment; a same-host marketing/promotions endpoint is
|
|
181
|
+
* commonly self-referential — its resource identifier shows up twice in its
|
|
182
|
+
* own path — which is what marks it as a templated/generated noise path
|
|
183
|
+
* rather than a genuine, if short, chain step.
|
|
184
|
+
*/
|
|
185
|
+
function hasRepeatedMeaningfulSegment(path) {
|
|
186
|
+
const segments = meaningfulPathSegments(path);
|
|
187
|
+
return new Set(segments).size < segments.length;
|
|
188
|
+
}
|
|
158
189
|
/**
|
|
159
190
|
* True when `candidatePath` shares enough path-segment tokens with at least
|
|
160
191
|
* one path in `referencePaths` to be judged part of the same endpoint
|
|
161
192
|
* family, false when it is structurally unrelated to all of them.
|
|
162
193
|
*
|
|
163
194
|
* A literal prefix/substring check is too strict: real same-flow endpoint
|
|
164
|
-
* families (e.g. `.../
|
|
195
|
+
* families (e.g. `.../listing-avail-vas/...` and `.../item-detail-vas/...`)
|
|
165
196
|
* do not share a string prefix segment-for-segment, but do share the
|
|
166
197
|
* `-vas` suffix and surrounding path structure once tokenized. Token overlap
|
|
167
198
|
* on segment words — ignoring short/generic tokens — catches that relation
|
|
@@ -185,10 +216,36 @@ function isStructurallyRelevantCapture(candidatePath, referencePaths) {
|
|
|
185
216
|
* True when `candidatePath` has a compound (multi-word) path segment whose
|
|
186
217
|
* tokens share nothing with ANY other path in `poolPaths` — a same-host
|
|
187
218
|
* capture whose own path structurally isolates it from every other member of
|
|
188
|
-
* the pool it was admitted into.
|
|
189
|
-
*
|
|
190
|
-
*
|
|
191
|
-
*
|
|
219
|
+
* the pool it was admitted into.
|
|
220
|
+
*
|
|
221
|
+
* A path with no compound segment (empty token set) falls back to a raw
|
|
222
|
+
* segment-overlap check instead of an automatic pass, but only when the path
|
|
223
|
+
* has a {@link hasRepeatedMeaningfulSegment repeated segment} of its own
|
|
224
|
+
* (e.g. `widget` recurring in `/widget/api/promotions/widget/default`): a chain's
|
|
225
|
+
* own steps are plain-word paths that name a distinct action per step
|
|
226
|
+
* (`/applicant`, `/sections/name`, or a 3-segment `/user/profile/edit`) and
|
|
227
|
+
* so essentially never repeat a segment against themselves, even when they
|
|
228
|
+
* share no tokens — or even raw segments — with sibling steps (e.g. a real
|
|
229
|
+
* `create` -> `sections/name` -> `submit` sequence, none of whose plain-word
|
|
230
|
+
* steps share a segment with either of the others). A same-host
|
|
231
|
+
* marketing/promotions capture is commonly self-referential — its own
|
|
232
|
+
* resource identifier shows up twice in its own path — which is the signal
|
|
233
|
+
* that marks it as a templated/generated noise path rather than a genuine
|
|
234
|
+
* short (or not-so-short) single-word chain step, independent of segment
|
|
235
|
+
* count. Requiring it to also share no raw path segment with the pool closes
|
|
236
|
+
* the gap without penalizing genuine single-word chain steps of any depth.
|
|
237
|
+
* The overlap check drops the same short/generic segments
|
|
238
|
+
* {@link pathStructuralTokens} already excludes ({@link GENERIC_PATH_TOKENS},
|
|
239
|
+
* <3 chars): otherwise two completely unrelated endpoint families sharing
|
|
240
|
+
* only a boilerplate `/api/` segment would be judged "related" and the
|
|
241
|
+
* isolated capture would dodge exclusion the same way the empty-token-set
|
|
242
|
+
* exemption originally let it. Pool paths that are themselves self-referential
|
|
243
|
+
* ({@link hasRepeatedMeaningfulSegment}) are excluded from contributing to
|
|
244
|
+
* this overlap set: otherwise two co-occurring same-noise-family path
|
|
245
|
+
* variants (e.g. a POST and a GET/default variant of the same templated
|
|
246
|
+
* marketing endpoint) would mutually "vouch" for each other's segments and
|
|
247
|
+
* neither would be recognized as isolated — only a genuinely non-self-
|
|
248
|
+
* referential path can vouch for a candidate's relatedness.
|
|
192
249
|
*
|
|
193
250
|
* Anchored on {@link isStructurallyRelevantCapture}'s own token overlap rule
|
|
194
251
|
* so "isolated" is exactly "not relevant to anything else in the pool" —
|
|
@@ -198,9 +255,252 @@ function isStructurallyRelevantCapture(candidatePath, referencePaths) {
|
|
|
198
255
|
* unlike {@link isStructurallyRelevantCapture} which requires one.
|
|
199
256
|
*/
|
|
200
257
|
function isStructurallyIsolatedCapture(candidatePath, poolPaths) {
|
|
201
|
-
|
|
258
|
+
const candidateTokens = pathStructuralTokens(candidatePath);
|
|
259
|
+
if (candidateTokens.size > 0)
|
|
260
|
+
return !isStructurallyRelevantCapture(candidatePath, poolPaths);
|
|
261
|
+
if (!hasRepeatedMeaningfulSegment(candidatePath))
|
|
262
|
+
return false;
|
|
263
|
+
const candidateSegments = meaningfulPathSegments(candidatePath);
|
|
264
|
+
const poolSegments = new Set(poolPaths
|
|
265
|
+
.filter((poolPath) => !hasRepeatedMeaningfulSegment(poolPath))
|
|
266
|
+
.flatMap((poolPath) => meaningfulPathSegments(poolPath)));
|
|
267
|
+
return !candidateSegments.some((segment) => poolSegments.has(segment));
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* True when `pathA` and `pathB` belong to the same structural path family:
|
|
271
|
+
* either they share a compound-segment token ({@link pathStructuralTokens}),
|
|
272
|
+
* or both are self-referential ({@link hasRepeatedMeaningfulSegment}) and
|
|
273
|
+
* share a raw meaningful segment (e.g. `widget` in both `/widget/api/promotions/widget`
|
|
274
|
+
* and `/widget/api/promotions/widget/default`).
|
|
275
|
+
*
|
|
276
|
+
* Generalizes {@link isStructurallyRelevantCapture}'s token-overlap rule to
|
|
277
|
+
* also cover the all-single-word-segment, self-referential shape that rule
|
|
278
|
+
* alone can't see (its token set is empty for a path with no compound
|
|
279
|
+
* segment) — the same shape {@link isStructurallyIsolatedCapture} already
|
|
280
|
+
* recognizes for pool-isolation. Exported so a caller reasoning about
|
|
281
|
+
* "is this OTHER capture part of the SAME noise family as an already-known
|
|
282
|
+
* noise capture" (rather than "is this capture isolated from a whole pool")
|
|
283
|
+
* can reuse the identical relatedness rule instead of re-deriving it.
|
|
284
|
+
*/
|
|
285
|
+
function isSamePathFamily(pathA, pathB) {
|
|
286
|
+
const tokensA = pathStructuralTokens(pathA);
|
|
287
|
+
const tokensB = pathStructuralTokens(pathB);
|
|
288
|
+
if (tokensA.size > 0 && tokensB.size > 0) {
|
|
289
|
+
for (const token of tokensA) {
|
|
290
|
+
if (tokensB.has(token))
|
|
291
|
+
return true;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
if (!hasRepeatedMeaningfulSegment(pathA) || !hasRepeatedMeaningfulSegment(pathB))
|
|
295
|
+
return false;
|
|
296
|
+
const segmentsA = new Set(meaningfulPathSegments(pathA));
|
|
297
|
+
return meaningfulPathSegments(pathB).some((segment) => segmentsA.has(segment));
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Every string/number/boolean leaf reachable from `value`, stringified, with
|
|
301
|
+
* `null`/`undefined` leaves skipped (they carry no derivable identifier to
|
|
302
|
+
* compare against the request).
|
|
303
|
+
*/
|
|
304
|
+
function collectLeafValues(value, out) {
|
|
305
|
+
if (value === null || value === undefined)
|
|
306
|
+
return;
|
|
307
|
+
if (typeof value === "object") {
|
|
308
|
+
for (const child of Array.isArray(value)
|
|
309
|
+
? value
|
|
310
|
+
: Object.values(value)) {
|
|
311
|
+
collectLeafValues(child, out);
|
|
312
|
+
}
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
out.push(String(value));
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* The set of raw values a request's own URL already exposes to the caller:
|
|
319
|
+
* every query-parameter value and every non-empty path segment. A response
|
|
320
|
+
* leaf that only ever echoes one of these tells the caller nothing it could
|
|
321
|
+
* not already derive from the request it just sent.
|
|
322
|
+
*/
|
|
323
|
+
function urlOwnValues(url) {
|
|
324
|
+
const values = new Set();
|
|
325
|
+
try {
|
|
326
|
+
const parsed = new URL(url);
|
|
327
|
+
for (const value of parsed.searchParams.values())
|
|
328
|
+
values.add(value);
|
|
329
|
+
for (const segment of parsed.pathname.split("/")) {
|
|
330
|
+
if (segment.length > 0)
|
|
331
|
+
values.add(decodeURIComponent(segment));
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
catch {
|
|
335
|
+
// unparsable URL contributes no derivable values
|
|
336
|
+
}
|
|
337
|
+
return values;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* True when `capture`'s response carries no business-relevant state: a
|
|
341
|
+
* non-JSON (or absent) content-type, a null/undefined body, a JSON body
|
|
342
|
+
* with no keys, or a JSON body whose every leaf value is already present in
|
|
343
|
+
* the request's own URL (query string or path). A page-load sensor/
|
|
344
|
+
* analytics beacon (an Akamai-style session-authenticator pixel, a polled
|
|
345
|
+
* non-JSON status ping, or one that echoes back the `clientId`/`siteId` the
|
|
346
|
+
* caller just sent it) answers every call with exactly this shape — unlike a
|
|
347
|
+
* real API response, which carries data a caller could not have already
|
|
348
|
+
* known before firing the request.
|
|
349
|
+
*
|
|
350
|
+
* The "echoes its own request" branch is what closes the gap a bare
|
|
351
|
+
* empty-body check misses: a response is technically non-empty JSON but
|
|
352
|
+
* every leaf is one of the fixed query's own values (or a path segment),
|
|
353
|
+
* so nothing in it is new information relative to what the caller already
|
|
354
|
+
* sent — it is not business-relevant just because it happens to be
|
|
355
|
+
* non-empty.
|
|
356
|
+
*
|
|
357
|
+
* Missing response metadata (unit-test callers that construct a capture
|
|
358
|
+
* without `responseHeaders`/`responseBody`) reads as "no business-relevant
|
|
359
|
+
* state" too: this predicate only ever narrows an already-recurring
|
|
360
|
+
* candidate — {@link isZeroVarianceRepeatCapture} (already-fixed-query) and
|
|
361
|
+
* `recon-generate.ts`'s own-repeat structural-isolation pass (already
|
|
362
|
+
* repeats identically, regardless of query shape) — so defaulting to the
|
|
363
|
+
* noise reading there costs nothing except in the caller that deliberately
|
|
364
|
+
* supplies a JSON response to prove the opposite.
|
|
365
|
+
*
|
|
366
|
+
* Exported (not just used internally by {@link isZeroVarianceRepeatCapture})
|
|
367
|
+
* because a same-host, fixed-request endpoint with NO query string at all
|
|
368
|
+
* (e.g. a polled feature-toggle feed with a single-compound-segment path)
|
|
369
|
+
* can be genuinely zero-business-value too, and query-key matching alone —
|
|
370
|
+
* this file's `hasFixedKey` signal — has nothing to key off when there is no
|
|
371
|
+
* query. Response business-value is the general, path/query-shape-independent
|
|
372
|
+
* signal for "this repeat carries nothing a caller could not already know,"
|
|
373
|
+
* so `recon-generate.ts` reuses it directly instead of the query-shape logic
|
|
374
|
+
* that cannot apply to a query-less endpoint.
|
|
375
|
+
*/
|
|
376
|
+
function hasNoBusinessRelevantResponseState(capture) {
|
|
377
|
+
const headers = capture.responseHeaders ?? {};
|
|
378
|
+
const contentType = (Object.entries(headers).find(([key]) => key.toLowerCase() === "content-type")?.[1] ?? "").toLowerCase();
|
|
379
|
+
if (!contentType.includes("json"))
|
|
380
|
+
return true;
|
|
381
|
+
const body = capture.responseBody;
|
|
382
|
+
if (body === null || body === undefined)
|
|
383
|
+
return true;
|
|
384
|
+
if (typeof body !== "object")
|
|
385
|
+
return false;
|
|
386
|
+
if (Object.keys(body).length === 0)
|
|
387
|
+
return true;
|
|
388
|
+
const leaves = [];
|
|
389
|
+
collectLeafValues(body, leaves);
|
|
390
|
+
if (leaves.length === 0)
|
|
391
|
+
return true;
|
|
392
|
+
const ownValues = urlOwnValues(capture.url);
|
|
393
|
+
return leaves.every((leaf) => ownValues.has(leaf));
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Identity of the endpoint a URL addresses, ignoring its query string, so
|
|
397
|
+
* recurrences of the same endpoint with a differently-ordered or
|
|
398
|
+
* incidentally-varying query string still group as "the same endpoint"
|
|
399
|
+
* instead of requiring the whole URL to be byte-identical. Mirrors
|
|
400
|
+
* `recon-generate.ts`'s own `endpointKey` (origin + pathname) — the same
|
|
401
|
+
* definition of "same endpoint" this codebase already uses for collapse and
|
|
402
|
+
* variance reasoning elsewhere.
|
|
403
|
+
*/
|
|
404
|
+
function endpointOrigin(url) {
|
|
405
|
+
try {
|
|
406
|
+
const parsed = new URL(url);
|
|
407
|
+
return `${parsed.origin}${parsed.pathname}`;
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
return null;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* True when `candidate` carries at least one query key whose value stays
|
|
415
|
+
* identical across every occurrence of the same endpoint in `allCaptures`,
|
|
416
|
+
* and recurs at least once elsewhere at the same method and endpoint
|
|
417
|
+
* (origin + pathname) — proof the endpoint's identifying signal is a fixed
|
|
418
|
+
* query key, not the full query string. Any OTHER key that differs between
|
|
419
|
+
* occurrences (a cache-buster, a session nonce) is incidental and does not
|
|
420
|
+
* prevent the match, AND either the request body is also byte-identical
|
|
421
|
+
* across every occurrence, or the response carries no business-relevant
|
|
422
|
+
* state ({@link hasNoBusinessRelevantResponseState}).
|
|
423
|
+
*
|
|
424
|
+
* Grouping by the full URL string (byte-identical query) is too strict: a
|
|
425
|
+
* real beacon can carry one incidental varying query key (a cache-buster or
|
|
426
|
+
* session nonce that is never literally equal across calls) alongside its
|
|
427
|
+
* genuinely fixed identifying keys (`clientId`, `environment`, `siteId`),
|
|
428
|
+
* and requiring the entire string to match lets it escape exclusion
|
|
429
|
+
* entirely. Per-key comparison against the candidate's own keys — rather
|
|
430
|
+
* than requiring the whole set of keys or values to agree — is what
|
|
431
|
+
* recovers the beacon: the fixed keys still prove the endpoint's identity,
|
|
432
|
+
* the varying key is simply ignored because it never advances past
|
|
433
|
+
* "matches on at least one key."
|
|
434
|
+
*
|
|
435
|
+
* A key is "fixed" when the candidate's own value for it is the STRICT
|
|
436
|
+
* MAJORITY value across every same-endpoint occurrence in `allCaptures`
|
|
437
|
+
* (more than half, with at least two supporting occurrences) — not
|
|
438
|
+
* necessarily every single one. `allCaptures` is the full raw capture list,
|
|
439
|
+
* which can include an incidental earlier/differently-scoped occurrence of
|
|
440
|
+
* the same endpoint (a page load before the flow proper starts) that
|
|
441
|
+
* legitimately carries a different value for every candidate key. Requiring
|
|
442
|
+
* literal unanimity would let that one outlier veto the proof for the
|
|
443
|
+
* entire flow's worth of genuinely fixed repeats; requiring only a majority
|
|
444
|
+
* still refuses to flag a key that varies freely (no value would ever reach
|
|
445
|
+
* a majority) while tolerating the minority-outlier shape a real archive
|
|
446
|
+
* produces.
|
|
447
|
+
*
|
|
448
|
+
* The requirement that at least one key be fixed is deliberate, not an
|
|
449
|
+
* extension/host special-case: it is what separates this from a genuinely
|
|
450
|
+
* no-argument own endpoint that a flow legitimately polls with an identical
|
|
451
|
+
* body every time (a feature-toggle feed, an availability heartbeat) —
|
|
452
|
+
* those endpoints carry real business meaning and folding their repeats
|
|
453
|
+
* into a single call is the collapse mechanism's job, not this predicate's.
|
|
454
|
+
* A marketing/analytics beacon's own fixed identifying query is the tell a
|
|
455
|
+
* plain body-repetition check can't see on its own, and is exactly the
|
|
456
|
+
* shape {@link isStructurallyIsolatedCapture} can be fooled by — that
|
|
457
|
+
* check's reference pool is every OTHER admitted capture, so N copies of
|
|
458
|
+
* the same fixed-query request "vouch" for each other's path tokens and
|
|
459
|
+
* none of them reads as isolated.
|
|
460
|
+
*
|
|
461
|
+
* The response-state fallback exists because a byte-identical-body
|
|
462
|
+
* requirement alone misses a beacon whose body embeds a fingerprint,
|
|
463
|
+
* timestamp, or session nonce that differs on every fire even though the
|
|
464
|
+
* fixed query is the only signal that actually identifies the endpoint —
|
|
465
|
+
* the body varies, but the response never carries anything the flow could
|
|
466
|
+
* not already derive from the request itself.
|
|
467
|
+
*/
|
|
468
|
+
function isZeroVarianceRepeatCapture(candidate, allCaptures) {
|
|
469
|
+
let candidateUrl;
|
|
470
|
+
try {
|
|
471
|
+
candidateUrl = new URL(candidate.url);
|
|
472
|
+
}
|
|
473
|
+
catch {
|
|
474
|
+
return false;
|
|
475
|
+
}
|
|
476
|
+
const candidateEndpoint = endpointOrigin(candidate.url);
|
|
477
|
+
if (candidateEndpoint === null)
|
|
478
|
+
return false;
|
|
479
|
+
const candidateKeys = [...candidateUrl.searchParams.keys()];
|
|
480
|
+
if (candidateKeys.length === 0)
|
|
481
|
+
return false;
|
|
482
|
+
const sameEndpoint = allCaptures.filter((c) => c.method === candidate.method && endpointOrigin(c.url) === candidateEndpoint);
|
|
483
|
+
if (sameEndpoint.length < 2)
|
|
484
|
+
return false;
|
|
485
|
+
const sameEndpointUrls = sameEndpoint
|
|
486
|
+
.map((c) => {
|
|
487
|
+
try {
|
|
488
|
+
return new URL(c.url);
|
|
489
|
+
}
|
|
490
|
+
catch {
|
|
491
|
+
return null;
|
|
492
|
+
}
|
|
493
|
+
})
|
|
494
|
+
.filter((u) => u !== null);
|
|
495
|
+
const hasFixedKey = candidateKeys.some((key) => {
|
|
496
|
+
const candidateValue = candidateUrl.searchParams.get(key);
|
|
497
|
+
const matchCount = sameEndpointUrls.filter((u) => u.searchParams.get(key) === candidateValue).length;
|
|
498
|
+
return matchCount >= 2 && matchCount > sameEndpointUrls.length / 2;
|
|
499
|
+
});
|
|
500
|
+
if (!hasFixedKey)
|
|
202
501
|
return false;
|
|
203
|
-
|
|
502
|
+
const bodyIdentical = sameEndpoint.every((c) => c.requestPostData === candidate.requestPostData);
|
|
503
|
+
return bodyIdentical || hasNoBusinessRelevantResponseState(candidate);
|
|
204
504
|
}
|
|
205
505
|
/**
|
|
206
506
|
* True when `hostname` is allowed as a fixture host: an exact match against
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capture-filters.js","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG
|
|
1
|
+
{"version":3,"file":"capture-filters.js","sourceRoot":"","sources":["../../src/recon/capture-filters.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;;;;;;;;;;;AAEH;;;;;;;GAOG;AACH;IACE,OAAO;QACL,4BAA4B;QAC5B,aAAa;QACb,yBAAyB;QACzB,sBAAsB;QACtB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,EAAE,CAAC;aAChD,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;KACnB,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACU,QAAA,uBAAuB,GAAG,sBAAsB,CAAC;AAE9D;;;;;;;;GAQG;AACH,MAAM,uBAAuB,GAAG;IAC9B,eAAe;IACf,YAAY;IACZ,YAAY;IACZ,iBAAiB;IACjB,sBAAsB;IACtB,sBAAsB;IACtB,cAAc;IACd,cAAc;IACd,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,eAAe;IACf,mBAAmB;IACnB,cAAc;IACd,YAAY;IACZ,YAAY;IACZ,gBAAgB;IAChB,uBAAuB;CACxB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,eAAe,GAAG,gEAAgE,CAAC;AAEzF;;;;;;;GAOG;AACH,oBAA2B,GAAW;IACpC,MAAM,QAAQ,GAAG,oBAAoB,EAAE,CAAC;IACxC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3F,IAAI,QAAA,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/D,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,2BAAkC,QAAgB;IAChD,MAAM,MAAM,GAAG,QAAQ;SACpB,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5E,CAAC;AAED,0FAA0F;AAC1F,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEpG;;;;;;;;;;;;;;GAcG;AACH,8BAAqC,IAAY;IAC/C,MAAM,KAAK,GAAG,IAAI;SACf,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,OAAO,CAAC;SACf,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACnB,MAAM,KAAK,GAAG,OAAO;aAClB,KAAK,CAAC,eAAe,CAAC;aACtB,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;aACvD,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;SACjC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACxE,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,IAAI;SACR,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;SACvC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACnF,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,4BAA4B,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;IAC9C,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,uCACE,aAAqB,EACrB,cAAiC;IAEjC,MAAM,eAAe,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5D,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE;QAC3C,MAAM,eAAe,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC5D,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;QAC9C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,uCACE,aAAqB,EACrB,SAA4B;IAE5B,MAAM,eAAe,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC5D,IAAI,eAAe,CAAC,IAAI,GAAG,CAAC;QAAE,OAAO,CAAC,6BAA6B,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAC9F,IAAI,CAAC,4BAA4B,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,SAAS;SACN,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC;SAC7D,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAC3D,CAAC;IACF,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,0BAAiC,KAAa,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;QACtC,CAAC;IACH,CAAC;IACD,IAAI,CAAC,4BAA4B,CAAC,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,OAAO,sBAAsB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,KAAc,EAAE,GAAa;IACtD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACtC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAgC,CAAC,EAAE,CAAC;YACpD,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,OAAO;IACT,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;YAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpE,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;IACnD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,4CAAmD,OAIlD;IACC,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC;IAC9C,MAAM,WAAW,GAAG,CAClB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CACzF,CAAC,WAAW,EAAE,CAAC;IAChB,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACrD,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC3C,IAAI,MAAM,CAAC,IAAI,CAAC,IAA+B,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,qCACE,SAMC,EACD,WAAuF;IAEvF,IAAI,YAAiB,CAAC;IACtB,IAAI,CAAC;QACH,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,iBAAiB,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACxD,IAAI,iBAAiB,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,aAAa,GAAG,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,iBAAiB,CACpF,CAAC;IACF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,MAAM,gBAAgB,GAAG,YAAY;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAY,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7C,MAAM,cAAc,GAAG,YAAY,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CACxC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,cAAc,CAClD,CAAC,MAAM,CAAC;QACT,OAAO,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,eAAe,CAAC,CAAC;IACjG,OAAO,aAAa,IAAI,kCAAkC,CAAC,SAAS,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;;GAOG;AACH,8BACE,QAAgB,EAChB,mBAA6B,EAC7B,cAA6B;IAE7B,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAClF,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC;QAAE,OAAO,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,OAAO,cAAc,KAAK,IAAI,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,cAAc,CAAC;AAC/E,CAAC"}
|
|
@@ -1,4 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Site-agnostic capture of hCaptcha's programmatic render config. When a
|
|
3
|
+
* site calls `hcaptcha.render(container, { sitekey, callback })` rather than
|
|
4
|
+
* declaring `data-callback` on the widget element, the callback only ever
|
|
5
|
+
* lives inside hCaptcha's own closure — nothing in the DOM names it. This
|
|
6
|
+
* module builds a page-init script (for `Page.addInitScript`) that
|
|
7
|
+
* monkeypatches `hcaptcha.render` before any site script runs, so every
|
|
8
|
+
* render call's `{ sitekey, widgetId, callback }` lands in a page-global
|
|
9
|
+
* registry a flow hook can query later, regardless of which plugin or site
|
|
10
|
+
* triggered the render.
|
|
11
|
+
*/
|
|
2
12
|
/** Page-global property name the capture script stores its registry under. */
|
|
3
13
|
export declare const HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = "__barnacleHcaptchaCallbacks";
|
|
4
14
|
/**
|
|
@@ -9,25 +19,4 @@ export declare const HCAPTCHA_CALLBACK_REGISTRY_GLOBAL = "__barnacleHcaptchaCall
|
|
|
9
19
|
* value or side effects.
|
|
10
20
|
*/
|
|
11
21
|
export declare function buildHcaptchaCallbackCaptureScript(): string;
|
|
12
|
-
/**
|
|
13
|
-
* Re-asserts the capture script into every frame's own realm as it attaches
|
|
14
|
-
* or navigates, closing the race where a same-origin iframe's own script
|
|
15
|
-
* assigns `window.hcaptcha` and calls `render` before `context.addInitScript`'s
|
|
16
|
-
* effect is observably in place in that frame's realm (a CDP round-trip race
|
|
17
|
-
* under some session providers). Complements, rather than replaces, the
|
|
18
|
-
* context-level install — that install remains the first line of defense for
|
|
19
|
-
* the common case (fast frames, no race).
|
|
20
|
-
*
|
|
21
|
-
* Frame-agnostic: this listens for `Page.frameAttached`/`Page.frameNavigated`
|
|
22
|
-
* on the main frame's CDP session and evaluates the (idempotent) capture
|
|
23
|
-
* script into whichever frame each event names, regardless of which site or
|
|
24
|
-
* plugin owns that frame.
|
|
25
|
-
*
|
|
26
|
-
* `Page.frameAttached`/`Page.frameNavigated` can fire before that frame's
|
|
27
|
-
* main-world execution context exists, so the first `evaluate()` rejects
|
|
28
|
-
* with "main world not ready for frame ...". Each fresh `evaluate()` call
|
|
29
|
-
* re-arms stagehand's own wait for that context, so retrying the call
|
|
30
|
-
* (rather than adding a bespoke listener) is what closes the race.
|
|
31
|
-
*/
|
|
32
|
-
export declare function installHcaptchaCallbackCaptureOnAllFrames(page: Page): void;
|
|
33
22
|
//# sourceMappingURL=captcha-callback-capture.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"captcha-callback-capture.d.ts","sourceRoot":"","sources":["../../src/scraper/captcha-callback-capture.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"captcha-callback-capture.d.ts","sourceRoot":"","sources":["../../src/scraper/captcha-callback-capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,8EAA8E;AAC9E,eAAO,MAAM,iCAAiC,gCAAgC,CAAC;AAE/E;;;;;;GAMG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAuE3D"}
|