@enricai/barnacle 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scraper/flow-runner.d.ts +12 -0
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +37 -0
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/http-client.d.ts +41 -0
- package/dist/scraper/http-client.d.ts.map +1 -1
- package/dist/scraper/http-client.js +49 -2
- package/dist/scraper/http-client.js.map +1 -1
- package/dist/scraper/session-browserbase.d.ts +19 -4
- package/dist/scraper/session-browserbase.d.ts.map +1 -1
- package/dist/scraper/session-browserbase.js +21 -6
- package/dist/scraper/session-browserbase.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +108 -7
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +315 -41
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/site-plugin.d.ts +5 -3
- package/dist/site-plugin.d.ts.map +1 -1
- package/package.json +48 -24
|
@@ -66,6 +66,39 @@ function parseJsonOrThrowRetryable(rawText, url) {
|
|
|
66
66
|
throw new errors_1.UnknownScraperError(`response body is not valid JSON: ${String(err)}`);
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
|
+
/**
|
|
70
|
+
* Extracts one cookie's value from a set of raw `Set-Cookie` header strings.
|
|
71
|
+
* Reads via `Headers.getSetCookie()` (not `.get("set-cookie")`, which joins
|
|
72
|
+
* multiple entries with commas and would corrupt the individual cookie-pair
|
|
73
|
+
* boundaries) so multi-cookie responses resolve correctly.
|
|
74
|
+
*/
|
|
75
|
+
function extractCookieValue(setCookieHeaders, cookieName) {
|
|
76
|
+
for (const entry of setCookieHeaders) {
|
|
77
|
+
const pair = entry.split(";", 1)[0] ?? "";
|
|
78
|
+
const eq = pair.indexOf("=");
|
|
79
|
+
if (eq === -1)
|
|
80
|
+
continue;
|
|
81
|
+
if (pair.slice(0, eq).trim() === cookieName) {
|
|
82
|
+
return pair.slice(eq + 1).trim();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Resolves one {@link HttpResponseBinding} against a response's headers.
|
|
89
|
+
* Returns `undefined` on a miss (source header/cookie absent) so the caller
|
|
90
|
+
* can leave a previously-bound value untouched instead of overwriting it
|
|
91
|
+
* with an empty string — see `HttpClientOptions.bind` for why a miss must
|
|
92
|
+
* not fabricate a value.
|
|
93
|
+
*/
|
|
94
|
+
function resolveBinding(binding, headers) {
|
|
95
|
+
if (binding.sourceHeader.toLowerCase() === "set-cookie") {
|
|
96
|
+
return binding.cookieName
|
|
97
|
+
? extractCookieValue(headers.getSetCookie(), binding.cookieName)
|
|
98
|
+
: (headers.getSetCookie()[0] ?? undefined);
|
|
99
|
+
}
|
|
100
|
+
return headers.get(binding.sourceHeader) ?? undefined;
|
|
101
|
+
}
|
|
69
102
|
/**
|
|
70
103
|
* Factory that creates a typed direct-HTTP request function pre-wired with
|
|
71
104
|
* the plugin's Bottleneck limiter, p-retry for transient network failures,
|
|
@@ -88,11 +121,15 @@ function parseJsonOrThrowRetryable(rawText, url) {
|
|
|
88
121
|
* 2 times with exponential backoff before propagating.
|
|
89
122
|
*/
|
|
90
123
|
function createHttpClient(options) {
|
|
91
|
-
const { schema, bottleneck, baseHeaders, onResponse } = options;
|
|
124
|
+
const { schema, bottleneck, baseHeaders, onResponse, bind = [] } = options;
|
|
125
|
+
// Values bound from prior responses (e.g. a minted auth cookie), keyed by
|
|
126
|
+
// targetHeader. Lives for the lifetime of this client instance so a later
|
|
127
|
+
// call can pick up what an earlier call captured — see HttpResponseBinding.
|
|
128
|
+
const boundHeaders = {};
|
|
92
129
|
return async (url, init = {}) => {
|
|
93
130
|
return bottleneck.schedule(() => (0, p_retry_1.default)(async () => {
|
|
94
131
|
const method = init.method ?? "GET";
|
|
95
|
-
const headers = { ...baseHeaders, ...(init.headers ?? {}) };
|
|
132
|
+
const headers = { ...baseHeaders, ...boundHeaders, ...(init.headers ?? {}) };
|
|
96
133
|
let response;
|
|
97
134
|
try {
|
|
98
135
|
response = await fetch(url, {
|
|
@@ -115,6 +152,16 @@ function createHttpClient(options) {
|
|
|
115
152
|
throw new errors_1.UnknownScraperError(`http fetch failed: ${String(err)}`);
|
|
116
153
|
}
|
|
117
154
|
onResponse?.({ status: response.status, headers: response.headers, url });
|
|
155
|
+
for (const binding of bind) {
|
|
156
|
+
const value = resolveBinding(binding, response.headers);
|
|
157
|
+
// A miss leaves any previously-bound value in place rather than
|
|
158
|
+
// clearing it — and if nothing has ever been bound, the target
|
|
159
|
+
// header stays absent from `boundHeaders` entirely, so later
|
|
160
|
+
// requests never send it as an empty string.
|
|
161
|
+
if (value !== undefined) {
|
|
162
|
+
boundHeaders[binding.targetHeader] = value;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
118
165
|
if (response.status === 401 || response.status === 403) {
|
|
119
166
|
// Bot challenge / auth wall — not a transient failure, abort retry.
|
|
120
167
|
throw new p_retry_1.AbortError(new errors_1.HttpBotChallengeError(`http ${response.status} from ${url} — bot challenge or auth required`));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAmD;AACnD,yCAAiC;AAGjC,mDAA6C;AAG7C,2CAA0C;AAC1C,6CAQ0B;AAC1B,iEAAoE;AAEpE,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,IAAI,mBAAmB,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAmD;AACnD,yCAAiC;AAGjC,mDAA6C;AAG7C,2CAA0C;AAC1C,6CAQ0B;AAC1B,iEAAoE;AAEpE,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAiG5B;;;;;;GAMG;AACH,SAAS,yBAAyB,CAAC,OAAe,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,uBAAuB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9F,MAAM,IAAI,4BAAmB,CAAC,oCAAoC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,gBAA0B,EAAE,UAAkB;IACxE,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,SAAS;QACxB,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,UAAU,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,OAA4B,EAAE,OAAgB;IACpE,IAAI,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,YAAY,EAAE,CAAC;QACxD,OAAO,OAAO,CAAC,UAAU;YACvB,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC;YAChE,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AACxD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,0BACE,OAAqC;IAErC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAC3E,0EAA0E;IAC1E,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,YAAY,GAA2B,EAAE,CAAC;IAEhD,OAAO,KAAK,EAAE,GAAW,EAAE,IAAI,GAAoB,EAAE,EAAsB,EAAE;QAC3E,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,CAC9B,IAAA,iBAAM,EACJ,KAAK,IAAI,EAAE;YACT,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;YACpC,MAAM,OAAO,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,YAAY,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;YAE7E,IAAI,QAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAC1B,MAAM;oBACN,OAAO;oBACP,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,+DAA+D;gBAC/D,+DAA+D;gBAC/D,gEAAgE;gBAChE,+DAA+D;gBAC/D,uBAAuB;gBACvB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACtD,MAAM,IAAI,oBAAU,CAAC,GAAG,CAAC,CAAC;gBAC5B,CAAC;gBACD,+DAA+D;gBAC/D,MAAM,IAAI,4BAAmB,CAAC,sBAAsB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1E,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACxD,gEAAgE;gBAChE,+DAA+D;gBAC/D,6DAA6D;gBAC7D,6CAA6C;gBAC7C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,YAAY,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC;gBAC7C,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACvD,oEAAoE;gBACpE,MAAM,IAAI,oBAAU,CAClB,IAAI,8BAAqB,CACvB,QAAQ,QAAQ,CAAC,MAAM,SAAS,GAAG,mCAAmC,CACvE,CACF,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,qDAAqD;gBACrD,MAAM,IAAI,oBAAU,CAClB,IAAI,2BAAkB,CAAC,iBAAiB,GAAG,wBAAwB,CAAC,CACrE,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC3B,kEAAkE;gBAClE,uCAAuC;gBACvC,MAAM,IAAI,oBAAU,CAAC,IAAI,wBAAe,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,GAAG,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,sBAAsB,CAAC;oBACtE,IAAA,mBAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBAC3D,MAAM,IAAI,GACR,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ;yBAClB,KAAK,CAAC,GAAG,CAAC;yBACV,MAAM,CAAC,OAAO,CAAC;yBACf,KAAK,CAAC,CAAC,CAAC,CAAC;yBACT,IAAI,CAAC,GAAG,CAAC;yBACT,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC;oBAChD,MAAM,MAAM,GAAG,IAAA,gBAAI,EAAC,GAAG,EAAE,GAAG,GAAG,IAAI,MAAM,IAAI,IAAI,OAAO,CAAC,CAAC;oBAC1D,IAAA,uBAAa,EACX,MAAM,EACN,IAAI,CAAC,SAAS,CACZ;wBACE,MAAM;wBACN,GAAG;wBACH,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,cAAc,EAAE,OAAO;wBACvB,WAAW,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;qBAC/B,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,2BAA2B,MAAM,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,iCAAiC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACpF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,MAAM,YAAY,GAAG,IAAA,yCAAsB,EAAC,OAAO,CAAC,CAAC;YACrD,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;gBAC9B,mEAAmE;gBACnE,MAAM,IAAI,oBAAU,CAClB,IAAI,2BAAkB,CAAC,2CAA2C,GAAG,EAAE,CAAC,CACzE,CAAC;YACJ,CAAC;YACD,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;gBACjC,kEAAkE;gBAClE,mEAAmE;gBACnE,2DAA2D;gBAC3D,MAAM,IAAI,gCAAuB,CAC/B,6BAA6B,GAAG,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACpE,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,yBAAyB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CACT,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5F,CAAC;gBACF,gDAAgD;gBAChD,MAAM,IAAI,oBAAU,CAClB,IAAI,wBAAe,CACjB,6BAA6B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC,EACD;YACE,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,GAAG;YACf,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,CAAC,IAAI,CACT,yBAAyB,OAAO,CAAC,aAAa,YAAY,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,WAAW,eAAe,CACvH,CAAC;YACJ,CAAC;SACF,CACF,CACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -22,10 +22,25 @@ import { type BrowserSession, type BrowserSessionOptions } from "../scraper/sess
|
|
|
22
22
|
* available via the array form (not used here — out of scope until needed).
|
|
23
23
|
*/
|
|
24
24
|
/**
|
|
25
|
-
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile.
|
|
26
|
-
* `
|
|
27
|
-
*
|
|
28
|
-
*
|
|
25
|
+
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile. When
|
|
26
|
+
* enabled we also force `solveCaptchas: true` (explicit; Browserbase defaults
|
|
27
|
+
* it on) and pin a Windows desktop fingerprint — DataDome-protected sites react
|
|
28
|
+
* significantly better to Windows OS signals than the default mac/linux mix.
|
|
29
|
+
* The combination mirrors a production Stagehand preset validated against such
|
|
30
|
+
* sites, not Browserbase's defaults.
|
|
31
|
+
*
|
|
32
|
+
* `browserbaseSessionCreateParams` forwards caller-supplied Browserbase session
|
|
33
|
+
* create params (`timeout` being the intended knob — seconds, per Browserbase's
|
|
34
|
+
* `SessionCreateParams`). They are spread first so `proxies` and
|
|
35
|
+
* `browserSettings.fingerprint` land after and win — ordering is the only thing
|
|
36
|
+
* keeping those ours, so keep the caller spread above them.
|
|
37
|
+
*
|
|
38
|
+
* `projectId` is dropped from the caller params on top of being re-set below —
|
|
39
|
+
* belt and suspenders. Stagehand resolves it as `overrideProjectId ?? projectId`,
|
|
40
|
+
* so a caller value that reached the spread would beat the top-level one and
|
|
41
|
+
* silently route the session into a different Browserbase project. The re-set
|
|
42
|
+
* already prevents that; the strip is what keeps it prevented if the re-set is
|
|
43
|
+
* ever moved or dropped.
|
|
29
44
|
*/
|
|
30
45
|
export declare function createBrowserbaseBrowserSession(opts?: Pick<BrowserSessionOptions, "advancedStealth" | "browserbaseSessionCreateParams">): Promise<BrowserSession>;
|
|
31
46
|
//# sourceMappingURL=session-browserbase.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AA6ElC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH
|
|
1
|
+
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AA6ElC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,+BAA+B,CACnD,IAAI,CAAC,EAAE,IAAI,CAAC,qBAAqB,EAAE,iBAAiB,GAAG,gCAAgC,CAAC,GACvF,OAAO,CAAC,cAAc,CAAC,CAkHzB"}
|
|
@@ -87,10 +87,25 @@ function makeFilteredStagehandLogger(pinoLogger) {
|
|
|
87
87
|
* available via the array form (not used here — out of scope until needed).
|
|
88
88
|
*/
|
|
89
89
|
/**
|
|
90
|
-
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile.
|
|
91
|
-
* `
|
|
92
|
-
*
|
|
93
|
-
*
|
|
90
|
+
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile. When
|
|
91
|
+
* enabled we also force `solveCaptchas: true` (explicit; Browserbase defaults
|
|
92
|
+
* it on) and pin a Windows desktop fingerprint — DataDome-protected sites react
|
|
93
|
+
* significantly better to Windows OS signals than the default mac/linux mix.
|
|
94
|
+
* The combination mirrors a production Stagehand preset validated against such
|
|
95
|
+
* sites, not Browserbase's defaults.
|
|
96
|
+
*
|
|
97
|
+
* `browserbaseSessionCreateParams` forwards caller-supplied Browserbase session
|
|
98
|
+
* create params (`timeout` being the intended knob — seconds, per Browserbase's
|
|
99
|
+
* `SessionCreateParams`). They are spread first so `proxies` and
|
|
100
|
+
* `browserSettings.fingerprint` land after and win — ordering is the only thing
|
|
101
|
+
* keeping those ours, so keep the caller spread above them.
|
|
102
|
+
*
|
|
103
|
+
* `projectId` is dropped from the caller params on top of being re-set below —
|
|
104
|
+
* belt and suspenders. Stagehand resolves it as `overrideProjectId ?? projectId`,
|
|
105
|
+
* so a caller value that reached the spread would beat the top-level one and
|
|
106
|
+
* silently route the session into a different Browserbase project. The re-set
|
|
107
|
+
* already prevents that; the strip is what keeps it prevented if the re-set is
|
|
108
|
+
* ever moved or dropped.
|
|
94
109
|
*/
|
|
95
110
|
async function createBrowserbaseBrowserSession(opts) {
|
|
96
111
|
if (!config_1.config.scraper.browserbaseApiKey) {
|
|
@@ -105,7 +120,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
105
120
|
const viewport = (0, session_shared_1.pickRandomViewport)();
|
|
106
121
|
const useResidentialProxy = config_1.config.scraper.proxyType.toLowerCase() === "residential";
|
|
107
122
|
const advancedStealth = opts?.advancedStealth === true;
|
|
108
|
-
const customSessionParams = opts?.browserbaseSessionCreateParams;
|
|
123
|
+
const { projectId: _callerProjectId, ...customSessionParams } = opts?.browserbaseSessionCreateParams ?? {};
|
|
109
124
|
if (config_1.config.scraper.useBedrock) {
|
|
110
125
|
logger.info(`using bedrock model ${config_1.config.bedrock.model} in region ${config_1.config.bedrock.region}`);
|
|
111
126
|
}
|
|
@@ -142,7 +157,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
142
157
|
projectId: config_1.config.scraper.browserbaseProjectId,
|
|
143
158
|
proxies: useResidentialProxy,
|
|
144
159
|
browserSettings: {
|
|
145
|
-
...customSessionParams
|
|
160
|
+
...customSessionParams.browserSettings,
|
|
146
161
|
...(advancedStealth ? { advancedStealth: true, solveCaptchas: true } : {}),
|
|
147
162
|
fingerprint,
|
|
148
163
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;AAAA,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,6DAKkC;AAClC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAclE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,2BAA2B,CAAC,UAAkB;IAIrD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,4FAA4F,CAC5I,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH
|
|
1
|
+
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;AAAA,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,6DAKkC;AAClC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAclE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,2BAA2B,CAAC,UAAkB;IAIrD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,4FAA4F,CAC5I,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACI,KAAK,0CACV,IAAwF;IAExF,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,QAAQ,GAAG,IAAA,mCAAkB,GAAE,CAAC;IACtC,MAAM,mBAAmB,GAAG,eAAM,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC;IACrF,MAAM,eAAe,GAAG,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IACvD,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,GAAG,mBAAmB,EAAE,GAC3D,IAAI,EAAE,8BAA8B,IAAI,EAAE,CAAC;IAE7C,IAAI,eAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,uBAAuB,eAAM,CAAC,OAAO,CAAC,KAAK,cAAc,eAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,SAAS,GAAG,eAAM,CAAC,OAAO,CAAC,UAAU;QACzC,CAAC,CAAC,IAAI,uBAAW,CAAC,EAAE,KAAK,EAAE,IAAA,4BAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,CAAC,CAAC,SAAS,CAAC;IAEd,2EAA2E;IAC3E,uEAAuE;IACvE,4BAA4B;IAC5B,MAAM,eAAe,GAAG;QACtB,MAAM,EAAE;YACN,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;SAC3B;KACF,CAAC;IACF,MAAM,WAAW,GAAG,eAAe;QACjC,CAAC,CAAC;YACE,GAAG,eAAe;YAClB,OAAO,EAAE,CAAC,SAAkB,CAAC;YAC7B,gBAAgB,EAAE,CAAC,SAAkB,CAAC;SACvC;QACH,CAAC,CAAC,eAAe,CAAC;IAEpB,MAAM,EAAE,QAAQ,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,2BAA2B,EAAE,GACxF,2BAA2B,CAAC,MAAM,CAAC,CAAC;IAEtC,IAAI,SAAgC,CAAC;IACrC,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,qBAAS,CAAC;YACxB,GAAG,EAAE,aAAa;YAClB,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,iBAAiB;YACxC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;YAC9C,8BAA8B,EAAE;gBAC9B,GAAG,mBAAmB;gBACtB,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;gBAC9C,OAAO,EAAE,mBAAmB;gBAC5B,eAAe,EAAE;oBACf,GAAG,mBAAmB,CAAC,eAAe;oBACtC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1E,WAAW;iBACZ;aACF;YACD,KAAK,EAAE,eAAM,CAAC,OAAO,CAAC,UAAU;gBAC9B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAE;oBACC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,KAAK;oBAC/B,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,eAAe;oBACtC,gFAAgF;oBAChF,KAAK,EAAE,IAAA,mCAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;iBACtC;YAC5B,SAAS;YACT,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,CAAC;YACV,wDAAwD;YACxD,mDAAmD;YACnD,MAAM,EAAE,uBAAuB;SAChC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,IAAI,SAAS;gBAAE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACzC,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAA,uBAAc,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAG,SAAS,CAAC,oBAAoB,IAAI,SAAS,CAAC;IAC9D,MAAM,CAAC,IAAI,CACT,+BAA+B,SAAS,aAAa,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,YAAY,mBAAmB,oBAAoB,eAAe,EAAE,CAC3J,CAAC;IAEF,MAAM,OAAO,GAAG,IAAA,+BAAoB,GAAE,CAAC;IAEvC,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,SAAS,KAAK,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,2BAA2B,EAAE,CAAC;IAChC,CAAC,CAAC;IAEF,OAAO;QACL,SAAS;QACT,OAAO;QACP,SAAS;QACT,QAAQ,EAAE,aAAa;QACvB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
|
@@ -72,6 +72,36 @@ export declare function inferZodSchemaFromSamples(samples: readonly unknown[], d
|
|
|
72
72
|
export declare function selectPayloadAction<T extends {
|
|
73
73
|
capture: Capture;
|
|
74
74
|
}>(steps: readonly T[]): T | null;
|
|
75
|
+
interface ActionCapture {
|
|
76
|
+
capture: Capture;
|
|
77
|
+
index: number;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Extracts the ordered sequence of meaningful POSTs that represent the
|
|
81
|
+
* transactional flow: same-host 2xx POSTs, minus telemetry and error-reporting
|
|
82
|
+
* sinks. Assets need no filter of their own — they arrive as GETs.
|
|
83
|
+
*
|
|
84
|
+
* Exported for tests: this predicate decides what a generated plugin will POST
|
|
85
|
+
* at a live site, and it is the only gate between a browser's incidental
|
|
86
|
+
* chatter and the emitted hot path.
|
|
87
|
+
*/
|
|
88
|
+
export declare function extractActionSequence(captures: Capture[], baseUrl: string): ActionCapture[];
|
|
89
|
+
interface StateValue {
|
|
90
|
+
/** The raw string that appears in some response and is reused downstream. */
|
|
91
|
+
value: string;
|
|
92
|
+
/** Index of the capture whose response is the EARLIEST origin of this value. */
|
|
93
|
+
originIndex: number;
|
|
94
|
+
/** JSON path within the origin response (e.g. ["Auth", "Token"]). Empty for
|
|
95
|
+
* a header/cookie-origin value — see `headerOrigin`. */
|
|
96
|
+
path: string[];
|
|
97
|
+
/** Set when `value` originates in a response header/cookie rather than a
|
|
98
|
+
* body JSON leaf (e.g. disneycruise's `Set-Cookie: __pa=<jwt>` token mint).
|
|
99
|
+
* `path` is empty in this case since there is no body accessor. */
|
|
100
|
+
headerOrigin?: {
|
|
101
|
+
sourceHeader: string;
|
|
102
|
+
cookieName?: string;
|
|
103
|
+
};
|
|
104
|
+
}
|
|
75
105
|
interface ErrorSignals {
|
|
76
106
|
/** Top-level string-valued key whose presence in a response signals an
|
|
77
107
|
* error. Emitted by the generator as a `typeof obj.X === "string"` guard. */
|
|
@@ -109,6 +139,53 @@ interface FieldOptionsMapping {
|
|
|
109
139
|
}>;
|
|
110
140
|
}
|
|
111
141
|
type FieldOptionsMap = Map<string, FieldOptionsMapping>;
|
|
142
|
+
/**
|
|
143
|
+
* Walks every capture's response (including GETs — formHistoryId-style values
|
|
144
|
+
* may originate in a state-load GET, not a POST). Indexes every string leaf
|
|
145
|
+
* whose length is in [MIN, MAX], recording the EARLIEST capture index that
|
|
146
|
+
* produced it. Later occurrences of the same value reuse the earliest origin.
|
|
147
|
+
*
|
|
148
|
+
* Also indexes response-header/cookie-origin values (e.g. a `Set-Cookie`
|
|
149
|
+
* auth token) the same way, tagged with `headerOrigin` instead of a body
|
|
150
|
+
* `path` — this is what lets a stateful API's token-mint response feed a
|
|
151
|
+
* later call's `Cookie` header via `compileActionSteps`.
|
|
152
|
+
*
|
|
153
|
+
* The index is intentionally permissive — it doesn't try to shape-match
|
|
154
|
+
* "what looks like a token" because token shapes are an open set across the
|
|
155
|
+
* web. Authoritative filtering happens downstream in `compileActionSteps`,
|
|
156
|
+
* which only emits produces[] entries for values that ALSO appear in some
|
|
157
|
+
* downstream URL/headers/body (i.e. real cross-step reuse).
|
|
158
|
+
*
|
|
159
|
+
* Exception: values in `PLACEHOLDER_STATE_VALUES` are skipped entirely so
|
|
160
|
+
* the LATER non-placeholder occurrence at the same JSON path becomes the
|
|
161
|
+
* canonical binding instead.
|
|
162
|
+
*/
|
|
163
|
+
/** Exported for unit testing — lets tests exercise the produces[] walk (body
|
|
164
|
+
* AND header/cookie origins) directly against synthetic Capture sequences. */
|
|
165
|
+
export declare function indexStateValues(captures: Capture[], shieldedUuids?: Set<string>, actionCaptureIndices?: Set<number>): Map<string, StateValue>;
|
|
166
|
+
/** A state value produced by a step's response body — read via a JSON
|
|
167
|
+
* accessor on the response variable (e.g. `r6.products["0"].productId`). */
|
|
168
|
+
interface BodyProduce {
|
|
169
|
+
kind: "body";
|
|
170
|
+
name: string;
|
|
171
|
+
pathExpr: string;
|
|
172
|
+
path: string[];
|
|
173
|
+
}
|
|
174
|
+
/** A state value produced by a step's response header/cookie (e.g. a
|
|
175
|
+
* `Set-Cookie`-minted auth token). Unlike `BodyProduce` this has no JS
|
|
176
|
+
* accessor — the value never surfaces in emitted code at all, because
|
|
177
|
+
* `createHttpClient`'s `bind` option (see http-client.ts) captures and
|
|
178
|
+
* forwards it internally. Carried here only so the emitter knows to render
|
|
179
|
+
* a `bind` entry and which request header on the CONSUMING step observed it,
|
|
180
|
+
* i.e. `targetHeader`. */
|
|
181
|
+
interface HeaderProduce {
|
|
182
|
+
kind: "header";
|
|
183
|
+
name: string;
|
|
184
|
+
sourceHeader: string;
|
|
185
|
+
cookieName?: string;
|
|
186
|
+
targetHeader: string;
|
|
187
|
+
}
|
|
188
|
+
type Produce = BodyProduce | HeaderProduce;
|
|
112
189
|
interface ActionStep {
|
|
113
190
|
/** The capture this step corresponds to. */
|
|
114
191
|
capture: Capture;
|
|
@@ -117,16 +194,19 @@ interface ActionStep {
|
|
|
117
194
|
/** Camelcase state values this step's response produces, ready for destructure.
|
|
118
195
|
* `path` is the JSON path inside the response (used by the emitter to build
|
|
119
196
|
* a narrow per-binding assertion type so the emitted access stays `any`-free). */
|
|
120
|
-
produces:
|
|
121
|
-
name: string;
|
|
122
|
-
pathExpr: string;
|
|
123
|
-
path: string[];
|
|
124
|
-
}>;
|
|
197
|
+
produces: Produce[];
|
|
125
198
|
/** Whether the request body is multipart (body bytes not in capture). */
|
|
126
199
|
isMultipart: boolean;
|
|
127
200
|
/** True when the capture's host differs from the immediately-preceding action. */
|
|
128
201
|
isCrossDomain: boolean;
|
|
129
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* Walks the action sequence and decorates each step with: a unique response
|
|
205
|
+
* var name, the state values its response produces (used by downstream steps),
|
|
206
|
+
* and a multipart flag (request body bytes not captured).
|
|
207
|
+
*/
|
|
208
|
+
/** Exported for unit testing — see `indexStateValues`. */
|
|
209
|
+
export declare function compileActionSteps(actions: ActionCapture[], stateIndex: Map<string, StateValue>): ActionStep[];
|
|
130
210
|
/**
|
|
131
211
|
* Maps a site's screening-question prompts to the payload field that answers
|
|
132
212
|
* them, as `{ FieldName: [keyword, …] }`.
|
|
@@ -188,6 +268,11 @@ export declare function emitContractTs(opts: {
|
|
|
188
268
|
* emitBrowserFlowTs so schema and flow can never drift. */
|
|
189
269
|
payloadFieldNames?: Set<string>;
|
|
190
270
|
base64ContentHelper?: string;
|
|
271
|
+
/** Response-header/cookie-origin state bindings collected from the action
|
|
272
|
+
* sequence's produces[] (see `collectHeaderBindings`) — rendered as
|
|
273
|
+
* `createHttpClient`'s `bind` option so a value like a `Set-Cookie`-minted
|
|
274
|
+
* auth token actually reaches the stateful call that needs it. */
|
|
275
|
+
headerBindings?: HeaderProduce[];
|
|
191
276
|
}): string;
|
|
192
277
|
/** A flow step as read from recon-flow.json, carrying the optional splicer hints. */
|
|
193
278
|
type FlowStepInput = string | {
|
|
@@ -203,8 +288,14 @@ type FlowStepInput = string | {
|
|
|
203
288
|
* flow, as an alternative to the `.ts` trio for browser-only sites. Reuses the
|
|
204
289
|
* SAME `resolveStepPayloadField` splice logic as the browser-flow emitter, so
|
|
205
290
|
* every `{{ .request.<field> }}` reference also lands in the manifest's request
|
|
206
|
-
* schema — the two cannot drift.
|
|
207
|
-
*
|
|
291
|
+
* schema — the two cannot drift.
|
|
292
|
+
*
|
|
293
|
+
* `recovered` carries the request contract the `.ts` path infers from real
|
|
294
|
+
* captures — the first POST body's fields plus form-schema discoveries — so
|
|
295
|
+
* `--emit config` no longer throws that away and emit a request schema built
|
|
296
|
+
* only from the handful of flow-step splice hints. The direct-HTTP hot path is
|
|
297
|
+
* still omitted; a site that needs it keeps the `.ts` path or wires
|
|
298
|
+
* `spec.httpModule` by hand.
|
|
208
299
|
*/
|
|
209
300
|
export declare function emitConfigManifest(opts: {
|
|
210
301
|
siteId: string;
|
|
@@ -212,6 +303,10 @@ export declare function emitConfigManifest(opts: {
|
|
|
212
303
|
baseUrl: string;
|
|
213
304
|
flowSteps: FlowStepInput[];
|
|
214
305
|
vocabulary?: ReconVocabulary;
|
|
306
|
+
/** First action body: its top-level keys are the caller's real request fields. */
|
|
307
|
+
inputBody?: unknown;
|
|
308
|
+
/** Form-schema fields the recon recovered, added as caller-supplied strings. */
|
|
309
|
+
recoveredFields?: Iterable<string>;
|
|
215
310
|
}): string;
|
|
216
311
|
/**
|
|
217
312
|
* Emit the generated browser-flow module and return the accumulated set of
|
|
@@ -231,5 +326,11 @@ export declare function emitBrowserFlowTs(opts: {
|
|
|
231
326
|
code: string;
|
|
232
327
|
payloadFieldNames: Set<string>;
|
|
233
328
|
};
|
|
329
|
+
/** Generates the site's index.ts barrel — exported so the out-of-tree e2e
|
|
330
|
+
* test can drive the emitter directly without spawning the CLI. */
|
|
331
|
+
export declare function emitIndexTs(opts: {
|
|
332
|
+
siteId: string;
|
|
333
|
+
pascal: string;
|
|
334
|
+
}): string;
|
|
234
335
|
export {};
|
|
235
336
|
//# sourceMappingURL=recon-generate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recon-generate.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-generate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"recon-generate.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-generate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiBH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAGL,KAAK,OAAO,EAKb,MAAM,wBAAwB,CAAC;AA2EhC;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,MAAM,EACjB,SAAS,CAAC,EAAE,OAAO,EACnB,UAAU,GAAE,eAAmD,GAC9D,MAAM,GAAG,IAAI,CAoBf;AAUD,UAAU,SAAS;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,SAAS,OAAO,EAAE,EAC3B,KAAK,SAAI,EACT,MAAM,SAAK,EACX,IAAI,GAAE,SAAc,GACnB,MAAM,CA+DR;AAoCD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAqBjG;AA6JD,UAAU,aAAa;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CA4B3F;AAyBD,UAAU,UAAU;IAClB,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB;4DACwD;IACxD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;uEAEmE;IACnE,YAAY,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D;AAmFD,UAAU,YAAY;IACpB;iFAC6E;IAC7E,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;;uDAEmD;IACnD,gBAAgB,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrE;AAiFD;;;;;GAKG;AACH,KAAK,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAExC;;;;;;;;;GASG;AACH,UAAU,mBAAmB;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrD;AACD,KAAK,eAAe,GAAG,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;AAsfxD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;8EAC8E;AAC9E,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,OAAO,EAAE,EACnB,aAAa,GAAE,GAAG,CAAC,MAAM,CAAa,EACtC,oBAAoB,GAAE,GAAG,CAAC,MAAM,CAAa,GAC5C,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CA2DzB;AAED;4EAC4E;AAC5E,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;;;;;0BAM0B;AAC1B,UAAU,aAAa;IACrB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,KAAK,OAAO,GAAG,WAAW,GAAG,aAAa,CAAC;AAE3C,UAAU,UAAU;IAClB,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;IACjB,mEAAmE;IACnE,OAAO,EAAE,MAAM,CAAC;IAChB;;sFAEkF;IAClF,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,yEAAyE;IACzE,WAAW,EAAE,OAAO,CAAC;IACrB,kFAAkF;IAClF,aAAa,EAAE,OAAO,CAAC;CACxB;AAmDD;;;;GAIG;AACH,0DAA0D;AAC1D,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,aAAa,EAAE,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,GAClC,UAAU,EAAE,CAoGd;AAwKD;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CASrE;AA2QD;6DAC6D;AAC7D,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,UAAU,EAAE,EACrB,SAAS,EAAE,OAAO,EAClB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY,EAC1B,mBAAmB,EAAE,GAAG,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,eAAe,EAChC,yBAAyB,EAAE,GAAG,CAAC,MAAM,CAAC,EACtC,4BAA4B,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjD,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,EAC7E,OAAO,EAAE,MAAM,EACf,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,sBAAsB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC3C,mBAAmB,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,GACnD,MAAM,CAkTR;AAkCD;yEACyE;AACzE,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,GAAG,EAAE,OAAO,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iGAAiG;IACjG,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mGAAmG;IACnG,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;qDAEiD;IACjD,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC;;;mCAG+B;IAC/B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;sEACkE;IAClE,sBAAsB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC;;;;sDAIkD;IAClD,yBAAyB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD;;4EAEwE;IACxE,4BAA4B,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;IAC5E;;;+DAG2D;IAC3D,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;sEAGkE;IAClE,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;CAClC,GAAG,MAAM,CAmUT;AAED,qFAAqF;AACrF,KAAK,aAAa,GACd,MAAM,GACN;IACE,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAyFN;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,kFAAkF;IAClF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gFAAgF;IAChF,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;CACpC,GAAG,MAAM,CAiET;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAAE,CAiInD;AAED;mEACmE;AACnE,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAgB5E"}
|