@enricai/barnacle 1.4.1 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scraper/http-client.d.ts +41 -0
- package/dist/scraper/http-client.d.ts.map +1 -1
- package/dist/scraper/http-client.js +49 -2
- package/dist/scraper/http-client.js.map +1 -1
- package/dist/scraper/session-browserbase.d.ts +19 -4
- package/dist/scraper/session-browserbase.d.ts.map +1 -1
- package/dist/scraper/session-browserbase.js +21 -6
- package/dist/scraper/session-browserbase.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +86 -5
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +194 -16
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/site-plugin.d.ts +5 -3
- package/dist/site-plugin.d.ts.map +1 -1
- package/package.json +44 -24
|
@@ -10,6 +10,36 @@ export interface HttpResponseInfo {
|
|
|
10
10
|
headers: Headers;
|
|
11
11
|
url: string;
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* Declares a single named value to capture from a response and echo back as
|
|
15
|
+
* a request header on later calls made by the same client instance. Models
|
|
16
|
+
* the disneycruise-style pattern where one call mints a bearer via
|
|
17
|
+
* `Set-Cookie` and a later stateful call 401s without it — extends the
|
|
18
|
+
* response-body state-threading the emitter already speaks (`produces`) to
|
|
19
|
+
* response headers, instead of a general cookie jar (see http-client.ts
|
|
20
|
+
* module docblock and CLAUDE.md "battle-tested libraries only": a real jar
|
|
21
|
+
* with domain/path/expiry matching belongs to a library like tough-cookie,
|
|
22
|
+
* not a hand-rolled one, and the sites this closes only need a single opaque
|
|
23
|
+
* value echoed back verbatim).
|
|
24
|
+
*/
|
|
25
|
+
export interface HttpResponseBinding {
|
|
26
|
+
/**
|
|
27
|
+
* Response header to read from. Use `"set-cookie"` (case-insensitive) to
|
|
28
|
+
* read from `Set-Cookie` specifically — multiple `Set-Cookie` entries are
|
|
29
|
+
* read via `Headers.getSetCookie()` so they aren't comma-joined the way
|
|
30
|
+
* `Headers.get()` joins ordinary repeated headers.
|
|
31
|
+
*/
|
|
32
|
+
sourceHeader: string;
|
|
33
|
+
/**
|
|
34
|
+
* For `sourceHeader: "set-cookie"`, the cookie name to extract (e.g.
|
|
35
|
+
* `"__pa"`) — the binding reads that cookie's value out of whichever
|
|
36
|
+
* `Set-Cookie` entry defines it. Ignored for non-cookie source headers,
|
|
37
|
+
* whose raw value is bound as-is.
|
|
38
|
+
*/
|
|
39
|
+
cookieName?: string;
|
|
40
|
+
/** Outbound request header to populate on subsequent calls, e.g. `"Cookie"`. */
|
|
41
|
+
targetHeader: string;
|
|
42
|
+
}
|
|
13
43
|
/**
|
|
14
44
|
* Static configuration passed to `createHttpClient` once per plugin. Bundles
|
|
15
45
|
* the Zod response schema, the Bottleneck rate limiter, and the load-bearing
|
|
@@ -29,6 +59,17 @@ export interface HttpClientOptions<TResponse> {
|
|
|
29
59
|
* or capture audit data without bypassing the core client.
|
|
30
60
|
*/
|
|
31
61
|
onResponse?: (info: HttpResponseInfo) => void;
|
|
62
|
+
/**
|
|
63
|
+
* Named response-header/cookie values to capture on one call and forward as
|
|
64
|
+
* a request header on subsequent calls from the same client instance (e.g.
|
|
65
|
+
* a token-mint call's `Set-Cookie` echoed back on the next stateful call).
|
|
66
|
+
* Evaluated at the same point as `onResponse` — every fetch outcome,
|
|
67
|
+
* including non-2xx. A binding whose source is absent from a given
|
|
68
|
+
* response leaves any previously-bound value untouched; if nothing has
|
|
69
|
+
* ever been bound, the target header is simply omitted from later
|
|
70
|
+
* requests rather than sent empty.
|
|
71
|
+
*/
|
|
72
|
+
bind?: HttpResponseBinding[];
|
|
32
73
|
}
|
|
33
74
|
/**
|
|
34
75
|
* Per-call overrides forwarded to the underlying `fetch()`. A strict subset of
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,UAAU,MAAM,YAAY,CAAC;AAEzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AA0BtC;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB,CAAC,SAAS;IAC1C,+DAA+D;IAC/D,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,yEAAyE;IACzE,UAAU,EAAE,UAAU,CAAC;IACvB,oFAAoF;IACpF,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,UAAU,MAAM,YAAY,CAAC;AAEzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AA0BtC;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB,CAAC,SAAS;IAC1C,+DAA+D;IAC/D,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,yEAAyE;IACzE,UAAU,EAAE,UAAU,CAAC;IACvB,oFAAoF;IACpF,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC9C;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,mBAAmB,EAAE,CAAC;CAC9B;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAoDD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EACxC,OAAO,EAAE,iBAAiB,CAAC,SAAS,CAAC,GACpC,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,SAAS,CAAC,CA4J7D"}
|
|
@@ -66,6 +66,39 @@ function parseJsonOrThrowRetryable(rawText, url) {
|
|
|
66
66
|
throw new errors_1.UnknownScraperError(`response body is not valid JSON: ${String(err)}`);
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Extracts one cookie's value from a set of raw `Set-Cookie` header strings.
|
|
71
|
+
* Reads via `Headers.getSetCookie()` (not `.get("set-cookie")`, which joins
|
|
72
|
+
* multiple entries with commas and would corrupt the individual cookie-pair
|
|
73
|
+
* boundaries) so multi-cookie responses resolve correctly.
|
|
74
|
+
*/
|
|
75
|
+
function extractCookieValue(setCookieHeaders, cookieName) {
|
|
76
|
+
for (const entry of setCookieHeaders) {
|
|
77
|
+
const pair = entry.split(";", 1)[0] ?? "";
|
|
78
|
+
const eq = pair.indexOf("=");
|
|
79
|
+
if (eq === -1)
|
|
80
|
+
continue;
|
|
81
|
+
if (pair.slice(0, eq).trim() === cookieName) {
|
|
82
|
+
return pair.slice(eq + 1).trim();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Resolves one {@link HttpResponseBinding} against a response's headers.
|
|
89
|
+
* Returns `undefined` on a miss (source header/cookie absent) so the caller
|
|
90
|
+
* can leave a previously-bound value untouched instead of overwriting it
|
|
91
|
+
* with an empty string — see `HttpClientOptions.bind` for why a miss must
|
|
92
|
+
* not fabricate a value.
|
|
93
|
+
*/
|
|
94
|
+
function resolveBinding(binding, headers) {
|
|
95
|
+
if (binding.sourceHeader.toLowerCase() === "set-cookie") {
|
|
96
|
+
return binding.cookieName
|
|
97
|
+
? extractCookieValue(headers.getSetCookie(), binding.cookieName)
|
|
98
|
+
: (headers.getSetCookie()[0] ?? undefined);
|
|
99
|
+
}
|
|
100
|
+
return headers.get(binding.sourceHeader) ?? undefined;
|
|
101
|
+
}
|
|
69
102
|
/**
|
|
70
103
|
* Factory that creates a typed direct-HTTP request function pre-wired with
|
|
71
104
|
* the plugin's Bottleneck limiter, p-retry for transient network failures,
|
|
@@ -88,11 +121,15 @@ function parseJsonOrThrowRetryable(rawText, url) {
|
|
|
88
121
|
* 2 times with exponential backoff before propagating.
|
|
89
122
|
*/
|
|
90
123
|
function createHttpClient(options) {
|
|
91
|
-
const { schema, bottleneck, baseHeaders, onResponse } = options;
|
|
124
|
+
const { schema, bottleneck, baseHeaders, onResponse, bind = [] } = options;
|
|
125
|
+
// Values bound from prior responses (e.g. a minted auth cookie), keyed by
|
|
126
|
+
// targetHeader. Lives for the lifetime of this client instance so a later
|
|
127
|
+
// call can pick up what an earlier call captured — see HttpResponseBinding.
|
|
128
|
+
const boundHeaders = {};
|
|
92
129
|
return async (url, init = {}) => {
|
|
93
130
|
return bottleneck.schedule(() => (0, p_retry_1.default)(async () => {
|
|
94
131
|
const method = init.method ?? "GET";
|
|
95
|
-
const headers = { ...baseHeaders, ...(init.headers ?? {}) };
|
|
132
|
+
const headers = { ...baseHeaders, ...boundHeaders, ...(init.headers ?? {}) };
|
|
96
133
|
let response;
|
|
97
134
|
try {
|
|
98
135
|
response = await fetch(url, {
|
|
@@ -115,6 +152,16 @@ function createHttpClient(options) {
|
|
|
115
152
|
throw new errors_1.UnknownScraperError(`http fetch failed: ${String(err)}`);
|
|
116
153
|
}
|
|
117
154
|
onResponse?.({ status: response.status, headers: response.headers, url });
|
|
155
|
+
for (const binding of bind) {
|
|
156
|
+
const value = resolveBinding(binding, response.headers);
|
|
157
|
+
// A miss leaves any previously-bound value in place rather than
|
|
158
|
+
// clearing it — and if nothing has ever been bound, the target
|
|
159
|
+
// header stays absent from `boundHeaders` entirely, so later
|
|
160
|
+
// requests never send it as an empty string.
|
|
161
|
+
if (value !== undefined) {
|
|
162
|
+
boundHeaders[binding.targetHeader] = value;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
118
165
|
if (response.status === 401 || response.status === 403) {
|
|
119
166
|
// Bot challenge / auth wall — not a transient failure, abort retry.
|
|
120
167
|
throw new p_retry_1.AbortError(new errors_1.HttpBotChallengeError(`http ${response.status} from ${url} — bot challenge or auth required`));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAmD;AACnD,yCAAiC;AAGjC,mDAA6C;AAG7C,2CAA0C;AAC1C,6CAQ0B;AAC1B,iEAAoE;AAEpE,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,IAAI,mBAAmB,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAmD;AACnD,yCAAiC;AAGjC,mDAA6C;AAG7C,2CAA0C;AAC1C,6CAQ0B;AAC1B,iEAAoE;AAEpE,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAiG5B;;;;;;GAMG;AACH,SAAS,yBAAyB,CAAC,OAAe,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,uBAAuB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9F,MAAM,IAAI,4BAAmB,CAAC,oCAAoC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,gBAA0B,EAAE,UAAkB;IACxE,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,SAAS;QACxB,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,UAAU,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,OAA4B,EAAE,OAAgB;IACpE,IAAI,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,YAAY,EAAE,CAAC;QACxD,OAAO,OAAO,CAAC,UAAU;YACvB,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC;YAChE,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AACxD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,0BACE,OAAqC;IAErC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAC3E,0EAA0E;IAC1E,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,YAAY,GAA2B,EAAE,CAAC;IAEhD,OAAO,KAAK,EAAE,GAAW,EAAE,IAAI,GAAoB,EAAE,EAAsB,EAAE;QAC3E,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,CAC9B,IAAA,iBAAM,EACJ,KAAK,IAAI,EAAE;YACT,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;YACpC,MAAM,OAAO,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,YAAY,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;YAE7E,IAAI,QAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAC1B,MAAM;oBACN,OAAO;oBACP,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,+DAA+D;gBAC/D,+DAA+D;gBAC/D,gEAAgE;gBAChE,+DAA+D;gBAC/D,uBAAuB;gBACvB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACtD,MAAM,IAAI,oBAAU,CAAC,GAAG,CAAC,CAAC;gBAC5B,CAAC;gBACD,+DAA+D;gBAC/D,MAAM,IAAI,4BAAmB,CAAC,sBAAsB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1E,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACxD,gEAAgE;gBAChE,+DAA+D;gBAC/D,6DAA6D;gBAC7D,6CAA6C;gBAC7C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,YAAY,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC;gBAC7C,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACvD,oEAAoE;gBACpE,MAAM,IAAI,oBAAU,CAClB,IAAI,8BAAqB,CACvB,QAAQ,QAAQ,CAAC,MAAM,SAAS,GAAG,mCAAmC,CACvE,CACF,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,qDAAqD;gBACrD,MAAM,IAAI,oBAAU,CAClB,IAAI,2BAAkB,CAAC,iBAAiB,GAAG,wBAAwB,CAAC,CACrE,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC3B,kEAAkE;gBAClE,uCAAuC;gBACvC,MAAM,IAAI,oBAAU,CAAC,IAAI,wBAAe,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,GAAG,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,sBAAsB,CAAC;oBACtE,IAAA,mBAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBAC3D,MAAM,IAAI,GACR,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ;yBAClB,KAAK,CAAC,GAAG,CAAC;yBACV,MAAM,CAAC,OAAO,CAAC;yBACf,KAAK,CAAC,CAAC,CAAC,CAAC;yBACT,IAAI,CAAC,GAAG,CAAC;yBACT,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC;oBAChD,MAAM,MAAM,GAAG,IAAA,gBAAI,EAAC,GAAG,EAAE,GAAG,GAAG,IAAI,MAAM,IAAI,IAAI,OAAO,CAAC,CAAC;oBAC1D,IAAA,uBAAa,EACX,MAAM,EACN,IAAI,CAAC,SAAS,CACZ;wBACE,MAAM;wBACN,GAAG;wBACH,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,cAAc,EAAE,OAAO;wBACvB,WAAW,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;qBAC/B,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,2BAA2B,MAAM,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,iCAAiC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACpF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,MAAM,YAAY,GAAG,IAAA,yCAAsB,EAAC,OAAO,CAAC,CAAC;YACrD,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;gBAC9B,mEAAmE;gBACnE,MAAM,IAAI,oBAAU,CAClB,IAAI,2BAAkB,CAAC,2CAA2C,GAAG,EAAE,CAAC,CACzE,CAAC;YACJ,CAAC;YACD,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;gBACjC,kEAAkE;gBAClE,mEAAmE;gBACnE,2DAA2D;gBAC3D,MAAM,IAAI,gCAAuB,CAC/B,6BAA6B,GAAG,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACpE,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,yBAAyB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CACT,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5F,CAAC;gBACF,gDAAgD;gBAChD,MAAM,IAAI,oBAAU,CAClB,IAAI,wBAAe,CACjB,6BAA6B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC,EACD;YACE,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,GAAG;YACf,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,CAAC,IAAI,CACT,yBAAyB,OAAO,CAAC,aAAa,YAAY,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,WAAW,eAAe,CACvH,CAAC;YACJ,CAAC;SACF,CACF,CACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -22,10 +22,25 @@ import { type BrowserSession, type BrowserSessionOptions } from "../scraper/sess
|
|
|
22
22
|
* available via the array form (not used here — out of scope until needed).
|
|
23
23
|
*/
|
|
24
24
|
/**
|
|
25
|
-
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile.
|
|
26
|
-
* `
|
|
27
|
-
*
|
|
28
|
-
*
|
|
25
|
+
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile. When
|
|
26
|
+
* enabled we also force `solveCaptchas: true` (explicit; Browserbase defaults
|
|
27
|
+
* it on) and pin a Windows desktop fingerprint — DataDome-protected sites react
|
|
28
|
+
* significantly better to Windows OS signals than the default mac/linux mix.
|
|
29
|
+
* The combination mirrors a production Stagehand preset validated against such
|
|
30
|
+
* sites, not Browserbase's defaults.
|
|
31
|
+
*
|
|
32
|
+
* `browserbaseSessionCreateParams` forwards caller-supplied Browserbase session
|
|
33
|
+
* create params (`timeout` being the intended knob — seconds, per Browserbase's
|
|
34
|
+
* `SessionCreateParams`). They are spread first so `proxies` and
|
|
35
|
+
* `browserSettings.fingerprint` land after and win — ordering is the only thing
|
|
36
|
+
* keeping those ours, so keep the caller spread above them.
|
|
37
|
+
*
|
|
38
|
+
* `projectId` is dropped from the caller params on top of being re-set below —
|
|
39
|
+
* belt and suspenders. Stagehand resolves it as `overrideProjectId ?? projectId`,
|
|
40
|
+
* so a caller value that reached the spread would beat the top-level one and
|
|
41
|
+
* silently route the session into a different Browserbase project. The re-set
|
|
42
|
+
* already prevents that; the strip is what keeps it prevented if the re-set is
|
|
43
|
+
* ever moved or dropped.
|
|
29
44
|
*/
|
|
30
45
|
export declare function createBrowserbaseBrowserSession(opts?: Pick<BrowserSessionOptions, "advancedStealth" | "browserbaseSessionCreateParams">): Promise<BrowserSession>;
|
|
31
46
|
//# sourceMappingURL=session-browserbase.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AA6ElC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH
|
|
1
|
+
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AA6ElC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,+BAA+B,CACnD,IAAI,CAAC,EAAE,IAAI,CAAC,qBAAqB,EAAE,iBAAiB,GAAG,gCAAgC,CAAC,GACvF,OAAO,CAAC,cAAc,CAAC,CAkHzB"}
|
|
@@ -87,10 +87,25 @@ function makeFilteredStagehandLogger(pinoLogger) {
|
|
|
87
87
|
* available via the array form (not used here — out of scope until needed).
|
|
88
88
|
*/
|
|
89
89
|
/**
|
|
90
|
-
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile.
|
|
91
|
-
* `
|
|
92
|
-
*
|
|
93
|
-
*
|
|
90
|
+
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile. When
|
|
91
|
+
* enabled we also force `solveCaptchas: true` (explicit; Browserbase defaults
|
|
92
|
+
* it on) and pin a Windows desktop fingerprint — DataDome-protected sites react
|
|
93
|
+
* significantly better to Windows OS signals than the default mac/linux mix.
|
|
94
|
+
* The combination mirrors a production Stagehand preset validated against such
|
|
95
|
+
* sites, not Browserbase's defaults.
|
|
96
|
+
*
|
|
97
|
+
* `browserbaseSessionCreateParams` forwards caller-supplied Browserbase session
|
|
98
|
+
* create params (`timeout` being the intended knob — seconds, per Browserbase's
|
|
99
|
+
* `SessionCreateParams`). They are spread first so `proxies` and
|
|
100
|
+
* `browserSettings.fingerprint` land after and win — ordering is the only thing
|
|
101
|
+
* keeping those ours, so keep the caller spread above them.
|
|
102
|
+
*
|
|
103
|
+
* `projectId` is dropped from the caller params on top of being re-set below —
|
|
104
|
+
* belt and suspenders. Stagehand resolves it as `overrideProjectId ?? projectId`,
|
|
105
|
+
* so a caller value that reached the spread would beat the top-level one and
|
|
106
|
+
* silently route the session into a different Browserbase project. The re-set
|
|
107
|
+
* already prevents that; the strip is what keeps it prevented if the re-set is
|
|
108
|
+
* ever moved or dropped.
|
|
94
109
|
*/
|
|
95
110
|
async function createBrowserbaseBrowserSession(opts) {
|
|
96
111
|
if (!config_1.config.scraper.browserbaseApiKey) {
|
|
@@ -105,7 +120,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
105
120
|
const viewport = (0, session_shared_1.pickRandomViewport)();
|
|
106
121
|
const useResidentialProxy = config_1.config.scraper.proxyType.toLowerCase() === "residential";
|
|
107
122
|
const advancedStealth = opts?.advancedStealth === true;
|
|
108
|
-
const customSessionParams = opts?.browserbaseSessionCreateParams;
|
|
123
|
+
const { projectId: _callerProjectId, ...customSessionParams } = opts?.browserbaseSessionCreateParams ?? {};
|
|
109
124
|
if (config_1.config.scraper.useBedrock) {
|
|
110
125
|
logger.info(`using bedrock model ${config_1.config.bedrock.model} in region ${config_1.config.bedrock.region}`);
|
|
111
126
|
}
|
|
@@ -142,7 +157,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
142
157
|
projectId: config_1.config.scraper.browserbaseProjectId,
|
|
143
158
|
proxies: useResidentialProxy,
|
|
144
159
|
browserSettings: {
|
|
145
|
-
...customSessionParams
|
|
160
|
+
...customSessionParams.browserSettings,
|
|
146
161
|
...(advancedStealth ? { advancedStealth: true, solveCaptchas: true } : {}),
|
|
147
162
|
fingerprint,
|
|
148
163
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;AAAA,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,6DAKkC;AAClC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAclE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,2BAA2B,CAAC,UAAkB;IAIrD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,4FAA4F,CAC5I,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH
|
|
1
|
+
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;AAAA,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,6DAKkC;AAClC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAclE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,2BAA2B,CAAC,UAAkB;IAIrD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,4FAA4F,CAC5I,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACI,KAAK,0CACV,IAAwF;IAExF,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,QAAQ,GAAG,IAAA,mCAAkB,GAAE,CAAC;IACtC,MAAM,mBAAmB,GAAG,eAAM,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC;IACrF,MAAM,eAAe,GAAG,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IACvD,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,GAAG,mBAAmB,EAAE,GAC3D,IAAI,EAAE,8BAA8B,IAAI,EAAE,CAAC;IAE7C,IAAI,eAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,uBAAuB,eAAM,CAAC,OAAO,CAAC,KAAK,cAAc,eAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,SAAS,GAAG,eAAM,CAAC,OAAO,CAAC,UAAU;QACzC,CAAC,CAAC,IAAI,uBAAW,CAAC,EAAE,KAAK,EAAE,IAAA,4BAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,CAAC,CAAC,SAAS,CAAC;IAEd,2EAA2E;IAC3E,uEAAuE;IACvE,4BAA4B;IAC5B,MAAM,eAAe,GAAG;QACtB,MAAM,EAAE;YACN,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;SAC3B;KACF,CAAC;IACF,MAAM,WAAW,GAAG,eAAe;QACjC,CAAC,CAAC;YACE,GAAG,eAAe;YAClB,OAAO,EAAE,CAAC,SAAkB,CAAC;YAC7B,gBAAgB,EAAE,CAAC,SAAkB,CAAC;SACvC;QACH,CAAC,CAAC,eAAe,CAAC;IAEpB,MAAM,EAAE,QAAQ,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,2BAA2B,EAAE,GACxF,2BAA2B,CAAC,MAAM,CAAC,CAAC;IAEtC,IAAI,SAAgC,CAAC;IACrC,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,qBAAS,CAAC;YACxB,GAAG,EAAE,aAAa;YAClB,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,iBAAiB;YACxC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;YAC9C,8BAA8B,EAAE;gBAC9B,GAAG,mBAAmB;gBACtB,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;gBAC9C,OAAO,EAAE,mBAAmB;gBAC5B,eAAe,EAAE;oBACf,GAAG,mBAAmB,CAAC,eAAe;oBACtC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1E,WAAW;iBACZ;aACF;YACD,KAAK,EAAE,eAAM,CAAC,OAAO,CAAC,UAAU;gBAC9B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAE;oBACC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,KAAK;oBAC/B,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,eAAe;oBACtC,gFAAgF;oBAChF,KAAK,EAAE,IAAA,mCAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;iBACtC;YAC5B,SAAS;YACT,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,CAAC;YACV,wDAAwD;YACxD,mDAAmD;YACnD,MAAM,EAAE,uBAAuB;SAChC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,IAAI,SAAS;gBAAE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACzC,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAA,uBAAc,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAG,SAAS,CAAC,oBAAoB,IAAI,SAAS,CAAC;IAC9D,MAAM,CAAC,IAAI,CACT,+BAA+B,SAAS,aAAa,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,YAAY,mBAAmB,oBAAoB,eAAe,EAAE,CAC3J,CAAC;IAEF,MAAM,OAAO,GAAG,IAAA,+BAAoB,GAAE,CAAC;IAEvC,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,SAAS,KAAK,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,2BAA2B,EAAE,CAAC;IAChC,CAAC,CAAC;IAEF,OAAO;QACL,SAAS;QACT,OAAO;QACP,SAAS;QACT,QAAQ,EAAE,aAAa;QACvB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
|
@@ -72,6 +72,26 @@ export declare function inferZodSchemaFromSamples(samples: readonly unknown[], d
|
|
|
72
72
|
export declare function selectPayloadAction<T extends {
|
|
73
73
|
capture: Capture;
|
|
74
74
|
}>(steps: readonly T[]): T | null;
|
|
75
|
+
interface ActionCapture {
|
|
76
|
+
capture: Capture;
|
|
77
|
+
index: number;
|
|
78
|
+
}
|
|
79
|
+
interface StateValue {
|
|
80
|
+
/** The raw string that appears in some response and is reused downstream. */
|
|
81
|
+
value: string;
|
|
82
|
+
/** Index of the capture whose response is the EARLIEST origin of this value. */
|
|
83
|
+
originIndex: number;
|
|
84
|
+
/** JSON path within the origin response (e.g. ["Auth", "Token"]). Empty for
|
|
85
|
+
* a header/cookie-origin value — see `headerOrigin`. */
|
|
86
|
+
path: string[];
|
|
87
|
+
/** Set when `value` originates in a response header/cookie rather than a
|
|
88
|
+
* body JSON leaf (e.g. disneycruise's `Set-Cookie: __pa=<jwt>` token mint).
|
|
89
|
+
* `path` is empty in this case since there is no body accessor. */
|
|
90
|
+
headerOrigin?: {
|
|
91
|
+
sourceHeader: string;
|
|
92
|
+
cookieName?: string;
|
|
93
|
+
};
|
|
94
|
+
}
|
|
75
95
|
interface ErrorSignals {
|
|
76
96
|
/** Top-level string-valued key whose presence in a response signals an
|
|
77
97
|
* error. Emitted by the generator as a `typeof obj.X === "string"` guard. */
|
|
@@ -109,6 +129,53 @@ interface FieldOptionsMapping {
|
|
|
109
129
|
}>;
|
|
110
130
|
}
|
|
111
131
|
type FieldOptionsMap = Map<string, FieldOptionsMapping>;
|
|
132
|
+
/**
|
|
133
|
+
* Walks every capture's response (including GETs — formHistoryId-style values
|
|
134
|
+
* may originate in a state-load GET, not a POST). Indexes every string leaf
|
|
135
|
+
* whose length is in [MIN, MAX], recording the EARLIEST capture index that
|
|
136
|
+
* produced it. Later occurrences of the same value reuse the earliest origin.
|
|
137
|
+
*
|
|
138
|
+
* Also indexes response-header/cookie-origin values (e.g. a `Set-Cookie`
|
|
139
|
+
* auth token) the same way, tagged with `headerOrigin` instead of a body
|
|
140
|
+
* `path` — this is what lets a stateful API's token-mint response feed a
|
|
141
|
+
* later call's `Cookie` header via `compileActionSteps`.
|
|
142
|
+
*
|
|
143
|
+
* The index is intentionally permissive — it doesn't try to shape-match
|
|
144
|
+
* "what looks like a token" because token shapes are an open set across the
|
|
145
|
+
* web. Authoritative filtering happens downstream in `compileActionSteps`,
|
|
146
|
+
* which only emits produces[] entries for values that ALSO appear in some
|
|
147
|
+
* downstream URL/headers/body (i.e. real cross-step reuse).
|
|
148
|
+
*
|
|
149
|
+
* Exception: values in `PLACEHOLDER_STATE_VALUES` are skipped entirely so
|
|
150
|
+
* the LATER non-placeholder occurrence at the same JSON path becomes the
|
|
151
|
+
* canonical binding instead.
|
|
152
|
+
*/
|
|
153
|
+
/** Exported for unit testing — lets tests exercise the produces[] walk (body
|
|
154
|
+
* AND header/cookie origins) directly against synthetic Capture sequences. */
|
|
155
|
+
export declare function indexStateValues(captures: Capture[], shieldedUuids?: Set<string>, actionCaptureIndices?: Set<number>): Map<string, StateValue>;
|
|
156
|
+
/** A state value produced by a step's response body — read via a JSON
|
|
157
|
+
* accessor on the response variable (e.g. `r6.products["0"].productId`). */
|
|
158
|
+
interface BodyProduce {
|
|
159
|
+
kind: "body";
|
|
160
|
+
name: string;
|
|
161
|
+
pathExpr: string;
|
|
162
|
+
path: string[];
|
|
163
|
+
}
|
|
164
|
+
/** A state value produced by a step's response header/cookie (e.g. a
|
|
165
|
+
* `Set-Cookie`-minted auth token). Unlike `BodyProduce` this has no JS
|
|
166
|
+
* accessor — the value never surfaces in emitted code at all, because
|
|
167
|
+
* `createHttpClient`'s `bind` option (see http-client.ts) captures and
|
|
168
|
+
* forwards it internally. Carried here only so the emitter knows to render
|
|
169
|
+
* a `bind` entry and which request header on the CONSUMING step observed it,
|
|
170
|
+
* i.e. `targetHeader`. */
|
|
171
|
+
interface HeaderProduce {
|
|
172
|
+
kind: "header";
|
|
173
|
+
name: string;
|
|
174
|
+
sourceHeader: string;
|
|
175
|
+
cookieName?: string;
|
|
176
|
+
targetHeader: string;
|
|
177
|
+
}
|
|
178
|
+
type Produce = BodyProduce | HeaderProduce;
|
|
112
179
|
interface ActionStep {
|
|
113
180
|
/** The capture this step corresponds to. */
|
|
114
181
|
capture: Capture;
|
|
@@ -117,16 +184,19 @@ interface ActionStep {
|
|
|
117
184
|
/** Camelcase state values this step's response produces, ready for destructure.
|
|
118
185
|
* `path` is the JSON path inside the response (used by the emitter to build
|
|
119
186
|
* a narrow per-binding assertion type so the emitted access stays `any`-free). */
|
|
120
|
-
produces:
|
|
121
|
-
name: string;
|
|
122
|
-
pathExpr: string;
|
|
123
|
-
path: string[];
|
|
124
|
-
}>;
|
|
187
|
+
produces: Produce[];
|
|
125
188
|
/** Whether the request body is multipart (body bytes not in capture). */
|
|
126
189
|
isMultipart: boolean;
|
|
127
190
|
/** True when the capture's host differs from the immediately-preceding action. */
|
|
128
191
|
isCrossDomain: boolean;
|
|
129
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* Walks the action sequence and decorates each step with: a unique response
|
|
195
|
+
* var name, the state values its response produces (used by downstream steps),
|
|
196
|
+
* and a multipart flag (request body bytes not captured).
|
|
197
|
+
*/
|
|
198
|
+
/** Exported for unit testing — see `indexStateValues`. */
|
|
199
|
+
export declare function compileActionSteps(actions: ActionCapture[], stateIndex: Map<string, StateValue>): ActionStep[];
|
|
130
200
|
/**
|
|
131
201
|
* Maps a site's screening-question prompts to the payload field that answers
|
|
132
202
|
* them, as `{ FieldName: [keyword, …] }`.
|
|
@@ -188,6 +258,11 @@ export declare function emitContractTs(opts: {
|
|
|
188
258
|
* emitBrowserFlowTs so schema and flow can never drift. */
|
|
189
259
|
payloadFieldNames?: Set<string>;
|
|
190
260
|
base64ContentHelper?: string;
|
|
261
|
+
/** Response-header/cookie-origin state bindings collected from the action
|
|
262
|
+
* sequence's produces[] (see `collectHeaderBindings`) — rendered as
|
|
263
|
+
* `createHttpClient`'s `bind` option so a value like a `Set-Cookie`-minted
|
|
264
|
+
* auth token actually reaches the stateful call that needs it. */
|
|
265
|
+
headerBindings?: HeaderProduce[];
|
|
191
266
|
}): string;
|
|
192
267
|
/** A flow step as read from recon-flow.json, carrying the optional splicer hints. */
|
|
193
268
|
type FlowStepInput = string | {
|
|
@@ -231,5 +306,11 @@ export declare function emitBrowserFlowTs(opts: {
|
|
|
231
306
|
code: string;
|
|
232
307
|
payloadFieldNames: Set<string>;
|
|
233
308
|
};
|
|
309
|
+
/** Generates the site's index.ts barrel — exported so the out-of-tree e2e
|
|
310
|
+
* test can drive the emitter directly without spawning the CLI. */
|
|
311
|
+
export declare function emitIndexTs(opts: {
|
|
312
|
+
siteId: string;
|
|
313
|
+
pascal: string;
|
|
314
|
+
}): string;
|
|
234
315
|
export {};
|
|
235
316
|
//# sourceMappingURL=recon-generate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recon-generate.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-generate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"recon-generate.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-generate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiBH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAGL,KAAK,OAAO,EAKb,MAAM,wBAAwB,CAAC;AA6DhC;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,MAAM,EACjB,SAAS,CAAC,EAAE,OAAO,EACnB,UAAU,GAAE,eAAmD,GAC9D,MAAM,GAAG,IAAI,CAoBf;AAUD,UAAU,SAAS;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,SAAS,OAAO,EAAE,EAC3B,KAAK,SAAI,EACT,MAAM,SAAK,EACX,IAAI,GAAE,SAAc,GACnB,MAAM,CA+DR;AAoCD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAqBjG;AA6ID,UAAU,aAAa;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAsDD,UAAU,UAAU;IAClB,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB;4DACwD;IACxD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;uEAEmE;IACnE,YAAY,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D;AAmFD,UAAU,YAAY;IACpB;iFAC6E;IAC7E,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;;uDAEmD;IACnD,gBAAgB,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrE;AAiFD;;;;;GAKG;AACH,KAAK,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAExC;;;;;;;;;GASG;AACH,UAAU,mBAAmB;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrD;AACD,KAAK,eAAe,GAAG,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;AAmfxD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;8EAC8E;AAC9E,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,OAAO,EAAE,EACnB,aAAa,GAAE,GAAG,CAAC,MAAM,CAAa,EACtC,oBAAoB,GAAE,GAAG,CAAC,MAAM,CAAa,GAC5C,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CA2DzB;AAED;4EAC4E;AAC5E,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;;;;;0BAM0B;AAC1B,UAAU,aAAa;IACrB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,KAAK,OAAO,GAAG,WAAW,GAAG,aAAa,CAAC;AAE3C,UAAU,UAAU;IAClB,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;IACjB,mEAAmE;IACnE,OAAO,EAAE,MAAM,CAAC;IAChB;;sFAEkF;IAClF,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,yEAAyE;IACzE,WAAW,EAAE,OAAO,CAAC;IACrB,kFAAkF;IAClF,aAAa,EAAE,OAAO,CAAC;CACxB;AAmDD;;;;GAIG;AACH,0DAA0D;AAC1D,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,aAAa,EAAE,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,GAClC,UAAU,EAAE,CAoGd;AAwKD;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CASrE;AA2QD;6DAC6D;AAC7D,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,UAAU,EAAE,EACrB,SAAS,EAAE,OAAO,EAClB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY,EAC1B,mBAAmB,EAAE,GAAG,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,eAAe,EAChC,yBAAyB,EAAE,GAAG,CAAC,MAAM,CAAC,EACtC,4BAA4B,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjD,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,EAC7E,OAAO,EAAE,MAAM,EACf,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,sBAAsB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC3C,mBAAmB,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,GACnD,MAAM,CAkTR;AAkCD;yEACyE;AACzE,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,GAAG,EAAE,OAAO,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iGAAiG;IACjG,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mGAAmG;IACnG,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;qDAEiD;IACjD,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC;;;mCAG+B;IAC/B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;sEACkE;IAClE,sBAAsB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC;;;;sDAIkD;IAClD,yBAAyB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD;;4EAEwE;IACxE,4BAA4B,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;IAC5E;;;+DAG2D;IAC3D,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;sEAGkE;IAClE,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;CAClC,GAAG,MAAM,CA4TT;AAED,qFAAqF;AACrF,KAAK,aAAa,GACd,MAAM,GACN;IACE,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAwEN;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B,GAAG,MAAM,CAmDT;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAAE,CAiInD;AAED;mEACmE;AACnE,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAgB5E"}
|