@enricai/barnacle 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -0
- package/dist/api/errors.d.ts +2 -1
- package/dist/api/errors.d.ts.map +1 -1
- package/dist/api/errors.js +3 -2
- package/dist/api/errors.js.map +1 -1
- package/dist/lib/llm/judge.d.ts +1 -1
- package/dist/lib/llm/judge.js +1 -1
- package/dist/recon/capture-filters.d.ts +40 -0
- package/dist/recon/capture-filters.d.ts.map +1 -0
- package/dist/recon/capture-filters.js +106 -0
- package/dist/recon/capture-filters.js.map +1 -0
- package/dist/recon/form-schema.d.ts +51 -0
- package/dist/recon/form-schema.d.ts.map +1 -0
- package/dist/recon/form-schema.js +20 -0
- package/dist/recon/form-schema.js.map +1 -0
- package/dist/recon/load-form-schema.d.ts +24 -0
- package/dist/recon/load-form-schema.d.ts.map +1 -0
- package/dist/recon/load-form-schema.js +66 -0
- package/dist/recon/load-form-schema.js.map +1 -0
- package/dist/scraper/cookie-jar.d.ts +19 -0
- package/dist/scraper/cookie-jar.d.ts.map +1 -0
- package/dist/scraper/cookie-jar.js +43 -0
- package/dist/scraper/cookie-jar.js.map +1 -0
- package/dist/scraper/errors.d.ts +14 -14
- package/dist/scraper/errors.d.ts.map +1 -1
- package/dist/scraper/errors.js +15 -15
- package/dist/scraper/errors.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +28 -8
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/http-client.d.ts +12 -0
- package/dist/scraper/http-client.d.ts.map +1 -1
- package/dist/scraper/http-client.js +11 -15
- package/dist/scraper/http-client.js.map +1 -1
- package/dist/scraper/parse-json-response.d.ts +7 -5
- package/dist/scraper/parse-json-response.d.ts.map +1 -1
- package/dist/scraper/parse-json-response.js +9 -8
- package/dist/scraper/parse-json-response.js.map +1 -1
- package/dist/scraper/rate-limited-json-client.d.ts +8 -1
- package/dist/scraper/rate-limited-json-client.d.ts.map +1 -1
- package/dist/scraper/rate-limited-json-client.js +6 -1
- package/dist/scraper/rate-limited-json-client.js.map +1 -1
- package/dist/scraper/session-browserbase.d.ts +6 -0
- package/dist/scraper/session-browserbase.d.ts.map +1 -1
- package/dist/scraper/session-browserbase.js +6 -0
- package/dist/scraper/session-browserbase.js.map +1 -1
- package/dist/scripts/recon-browser.d.ts +19 -2
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +40 -5
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +50 -21
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +190 -182
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/scripts/recon-http.d.ts +5 -2
- package/dist/scripts/recon-http.d.ts.map +1 -1
- package/dist/scripts/recon-http.js +16 -4
- package/dist/scripts/recon-http.js.map +1 -1
- package/dist/scripts/recon-shared.d.ts +33 -0
- package/dist/scripts/recon-shared.d.ts.map +1 -1
- package/dist/scripts/recon-shared.js +2 -1
- package/dist/scripts/recon-shared.js.map +1 -1
- package/dist/testing/persona-fixture.d.ts +2 -2
- package/dist/testing/persona-fixture.d.ts.map +1 -1
- package/dist/testing/persona-fixture.js +2 -2
- package/dist/testing/persona-fixture.js.map +1 -1
- package/package.json +13 -1
- package/dist/scraper/oracle-sentinels.d.ts +0 -22
- package/dist/scraper/oracle-sentinels.d.ts.map +0 -1
- package/dist/scraper/oracle-sentinels.js +0 -40
- package/dist/scraper/oracle-sentinels.js.map +0 -1
|
@@ -39,7 +39,6 @@ const node_path_1 = require("node:path");
|
|
|
39
39
|
const p_retry_1 = __importStar(require("p-retry"));
|
|
40
40
|
const logging_1 = require("../lib/logging");
|
|
41
41
|
const errors_1 = require("../scraper/errors");
|
|
42
|
-
const oracle_sentinels_1 = require("../scraper/oracle-sentinels");
|
|
43
42
|
const logger = (0, logging_1.getLogger)({ name: "scraper/http-client" });
|
|
44
43
|
/**
|
|
45
44
|
* One-shot diagnostic: when `CAPTURE_BASELINE_BODIES=1`, write each successful
|
|
@@ -51,9 +50,9 @@ const logger = (0, logging_1.getLogger)({ name: "scraper/http-client" });
|
|
|
51
50
|
*/
|
|
52
51
|
let baselineCallCounter = 0;
|
|
53
52
|
/**
|
|
54
|
-
* Parses `rawText` as JSON.
|
|
55
|
-
*
|
|
56
|
-
* On any JSON parse failure for
|
|
53
|
+
* Parses `rawText` as JSON. Any plugin-supplied `classifyResponseBody` runs
|
|
54
|
+
* before this, so a recognized non-JSON sentinel is already handled.
|
|
55
|
+
* On any JSON parse failure for an unclassified body, logs the first 200 chars
|
|
57
56
|
* and throws `UnknownScraperError` (retryable) so p-retry re-issues the
|
|
58
57
|
* request rather than hard-aborting with `HttpSchemaError`.
|
|
59
58
|
*/
|
|
@@ -121,7 +120,7 @@ function resolveBinding(binding, headers) {
|
|
|
121
120
|
* 2 times with exponential backoff before propagating.
|
|
122
121
|
*/
|
|
123
122
|
function createHttpClient(options) {
|
|
124
|
-
const { schema, bottleneck, baseHeaders, onResponse, bind = [] } = options;
|
|
123
|
+
const { schema, bottleneck, baseHeaders, onResponse, bind = [], classifyResponseBody } = options;
|
|
125
124
|
// Values bound from prior responses (e.g. a minted auth cookie), keyed by
|
|
126
125
|
// targetHeader. Lives for the lifetime of this client instance so a later
|
|
127
126
|
// call can pick up what an earlier call captured — see HttpResponseBinding.
|
|
@@ -201,16 +200,13 @@ function createHttpClient(options) {
|
|
|
201
200
|
}
|
|
202
201
|
}
|
|
203
202
|
const rawText = await response.text();
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
}
|
|
209
|
-
if (
|
|
210
|
-
|
|
211
|
-
// but kept distinct from UnknownScraperError so the encompass flow
|
|
212
|
-
// can catch exactly this class and re-mint the AccessCode.
|
|
213
|
-
throw new errors_1.OracleTokenExpiredError(`oracle token expired from ${url}: ${rawText.trim().slice(0, 200)}`);
|
|
203
|
+
// A plugin may recognize a non-JSON response body the engine can't
|
|
204
|
+
// (a vendor sentinel). Its verdict drives retry via the error's
|
|
205
|
+
// `retryable` flag: non-retryable aborts the p-retry loop, retryable
|
|
206
|
+
// is re-thrown so p-retry re-issues. `undefined` falls through.
|
|
207
|
+
const classified = classifyResponseBody?.(rawText, { url });
|
|
208
|
+
if (classified !== undefined) {
|
|
209
|
+
throw classified.retryable ? classified : new p_retry_1.AbortError(classified);
|
|
214
210
|
}
|
|
215
211
|
const body = parseJsonOrThrowRetryable(rawText, url);
|
|
216
212
|
const parsed = schema.safeParse(body);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAmD;AACnD,yCAAiC;AAGjC,mDAA6C;AAG7C,2CAA0C;AAC1C,
|
|
1
|
+
{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/scraper/http-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAmD;AACnD,yCAAiC;AAGjC,mDAA6C;AAG7C,2CAA0C;AAC1C,6CAO0B;AAE1B,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAE1D;;;;;;;GAOG;AACH,IAAI,mBAAmB,GAAG,CAAC,CAAC;AA0G5B;;;;;;GAMG;AACH,SAAS,yBAAyB,CAAC,OAAe,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,uBAAuB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9F,MAAM,IAAI,4BAAmB,CAAC,oCAAoC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,gBAA0B,EAAE,UAAkB;IACxE,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,SAAS;QACxB,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,UAAU,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,OAA4B,EAAE,OAAgB;IACpE,IAAI,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,YAAY,EAAE,CAAC;QACxD,OAAO,OAAO,CAAC,UAAU;YACvB,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC;YAChE,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AACxD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,0BACE,OAAqC;IAErC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,GAAG,EAAE,EAAE,oBAAoB,EAAE,GAAG,OAAO,CAAC;IACjG,0EAA0E;IAC1E,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,YAAY,GAA2B,EAAE,CAAC;IAEhD,OAAO,KAAK,EAAE,GAAW,EAAE,IAAI,GAAoB,EAAE,EAAsB,EAAE;QAC3E,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,CAC9B,IAAA,iBAAM,EACJ,KAAK,IAAI,EAAE;YACT,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;YACpC,MAAM,OAAO,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,YAAY,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;YAE7E,IAAI,QAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAC1B,MAAM;oBACN,OAAO;oBACP,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,+DAA+D;gBAC/D,+DAA+D;gBAC/D,gEAAgE;gBAChE,+DAA+D;gBAC/D,uBAAuB;gBACvB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACtD,MAAM,IAAI,oBAAU,CAAC,GAAG,CAAC,CAAC;gBAC5B,CAAC;gBACD,+DAA+D;gBAC/D,MAAM,IAAI,4BAAmB,CAAC,sBAAsB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1E,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACxD,gEAAgE;gBAChE,+DAA+D;gBAC/D,6DAA6D;gBAC7D,6CAA6C;gBAC7C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,YAAY,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC;gBAC7C,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACvD,oEAAoE;gBACpE,MAAM,IAAI,oBAAU,CAClB,IAAI,8BAAqB,CACvB,QAAQ,QAAQ,CAAC,MAAM,SAAS,GAAG,mCAAmC,CACvE,CACF,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,qDAAqD;gBACrD,MAAM,IAAI,oBAAU,CAClB,IAAI,2BAAkB,CAAC,iBAAiB,GAAG,wBAAwB,CAAC,CACrE,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC3B,kEAAkE;gBAClE,uCAAuC;gBACvC,MAAM,IAAI,oBAAU,CAAC,IAAI,wBAAe,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,GAAG,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,sBAAsB,CAAC;oBACtE,IAAA,mBAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpC,MAAM,GAAG,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBAC3D,MAAM,IAAI,GACR,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ;yBAClB,KAAK,CAAC,GAAG,CAAC;yBACV,MAAM,CAAC,OAAO,CAAC;yBACf,KAAK,CAAC,CAAC,CAAC,CAAC;yBACT,IAAI,CAAC,GAAG,CAAC;yBACT,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC;oBAChD,MAAM,MAAM,GAAG,IAAA,gBAAI,EAAC,GAAG,EAAE,GAAG,GAAG,IAAI,MAAM,IAAI,IAAI,OAAO,CAAC,CAAC;oBAC1D,IAAA,uBAAa,EACX,MAAM,EACN,IAAI,CAAC,SAAS,CACZ;wBACE,MAAM;wBACN,GAAG;wBACH,MAAM,EAAE,QAAQ,CAAC,MAAM;wBACvB,cAAc,EAAE,OAAO;wBACvB,WAAW,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;qBAC/B,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,2BAA2B,MAAM,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,iCAAiC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACpF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,mEAAmE;YACnE,gEAAgE;YAChE,qEAAqE;YACrE,gEAAgE;YAChE,MAAM,UAAU,GAAG,oBAAoB,EAAE,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5D,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,oBAAU,CAAC,UAAU,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,IAAI,GAAG,yBAAyB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAErD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CACT,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5F,CAAC;gBACF,gDAAgD;gBAChD,MAAM,IAAI,oBAAU,CAClB,IAAI,wBAAe,CACjB,6BAA6B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC,EACD;YACE,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,GAAG;YACf,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,CAAC,IAAI,CACT,yBAAyB,OAAO,CAAC,aAAa,YAAY,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,WAAW,eAAe,CACvH,CAAC;YACJ,CAAC;SACF,CACF,CACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
* message shapes and slice constants across plugins.
|
|
6
6
|
*/
|
|
7
7
|
import type { ZodType } from "zod/v4";
|
|
8
|
+
import { type ScraperError } from "../scraper/errors";
|
|
8
9
|
/**
|
|
9
10
|
* Parses `rawBody` as JSON, validates it against `schema`, and returns the
|
|
10
|
-
* narrowed result.
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* narrowed result. On a JSON parse failure, an optional `classifyBody` gets
|
|
12
|
+
* first refusal — a plugin can recognize a non-JSON vendor sentinel and return
|
|
13
|
+
* the error to throw (e.g. a locked-URL signal) — otherwise this throws
|
|
14
|
+
* {@link HttpSchemaError}, with `contextLabel` prefixed so error messages are
|
|
15
|
+
* immediately actionable in logs without additional wrapping.
|
|
14
16
|
*/
|
|
15
|
-
export declare function parseJsonResponse<T>(rawBody: string, schema: ZodType<T>, contextLabel: string): T;
|
|
17
|
+
export declare function parseJsonResponse<T>(rawBody: string, schema: ZodType<T>, contextLabel: string, classifyBody?: (rawBody: string) => ScraperError | undefined): T;
|
|
16
18
|
//# sourceMappingURL=parse-json-response.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-json-response.d.ts","sourceRoot":"","sources":["../../src/scraper/parse-json-response.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"parse-json-response.d.ts","sourceRoot":"","sources":["../../src/scraper/parse-json-response.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAGtC,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEtE;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EACjC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,EAClB,YAAY,EAAE,MAAM,EACpB,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,GAC3D,CAAC,CAsBH"}
|
|
@@ -9,22 +9,23 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
9
9
|
exports.parseJsonResponse = parseJsonResponse;
|
|
10
10
|
const errors_1 = require("../lib/errors");
|
|
11
11
|
const errors_2 = require("../scraper/errors");
|
|
12
|
-
const oracle_sentinels_1 = require("../scraper/oracle-sentinels");
|
|
13
12
|
/**
|
|
14
13
|
* Parses `rawBody` as JSON, validates it against `schema`, and returns the
|
|
15
|
-
* narrowed result.
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
14
|
+
* narrowed result. On a JSON parse failure, an optional `classifyBody` gets
|
|
15
|
+
* first refusal — a plugin can recognize a non-JSON vendor sentinel and return
|
|
16
|
+
* the error to throw (e.g. a locked-URL signal) — otherwise this throws
|
|
17
|
+
* {@link HttpSchemaError}, with `contextLabel` prefixed so error messages are
|
|
18
|
+
* immediately actionable in logs without additional wrapping.
|
|
19
19
|
*/
|
|
20
|
-
function parseJsonResponse(rawBody, schema, contextLabel) {
|
|
20
|
+
function parseJsonResponse(rawBody, schema, contextLabel, classifyBody) {
|
|
21
21
|
let parsed;
|
|
22
22
|
try {
|
|
23
23
|
parsed = JSON.parse(rawBody);
|
|
24
24
|
}
|
|
25
25
|
catch (err) {
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
const classified = classifyBody?.(rawBody);
|
|
27
|
+
if (classified !== undefined) {
|
|
28
|
+
throw classified;
|
|
28
29
|
}
|
|
29
30
|
throw new errors_2.HttpSchemaError(`${contextLabel} non-JSON body: ${(0, errors_1.toErrorMessage)(err)} (first 200B: ${rawBody.slice(0, 200)})`);
|
|
30
31
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-json-response.js","sourceRoot":"","sources":["../../src/scraper/parse-json-response.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAIH,yCAA8C;AAC9C,
|
|
1
|
+
{"version":3,"file":"parse-json-response.js","sourceRoot":"","sources":["../../src/scraper/parse-json-response.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAIH,yCAA8C;AAC9C,6CAAsE;AAEtE;;;;;;;GAOG;AACH,2BACE,OAAe,EACf,MAAkB,EAClB,YAAoB,EACpB,YAA4D;IAE5D,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,UAAU,CAAC;QACnB,CAAC;QACD,MAAM,IAAI,wBAAe,CACvB,GAAG,YAAY,mBAAmB,IAAA,uBAAc,EAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAC/F,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,wBAAe,CACvB,GAAG,YAAY,2BAA2B,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC/E,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ZodType } from "zod/v4";
|
|
2
|
-
import type { HttpRequestInit } from "../scraper/http-client";
|
|
2
|
+
import type { HttpClientOptions, HttpRequestInit } from "../scraper/http-client";
|
|
3
3
|
/**
|
|
4
4
|
* Options for building a Chromium-flavoured rate-limited JSON client.
|
|
5
5
|
* Bundles the Bottleneck configuration, Chromium client-hint values, and any
|
|
@@ -23,6 +23,13 @@ export interface RateLimitedJsonClientOptions<TResponse> {
|
|
|
23
23
|
extraHeaders?: Record<string, string>;
|
|
24
24
|
/** Zod schema that validates and narrows each response body. */
|
|
25
25
|
schema: ZodType<TResponse>;
|
|
26
|
+
/**
|
|
27
|
+
* Optional plugin-supplied response-body classifier, forwarded verbatim to
|
|
28
|
+
* {@link createHttpClient} — lets a plugin recognize a vendor sentinel body
|
|
29
|
+
* (e.g. a plain-text token) without the engine knowing that vendor's wire
|
|
30
|
+
* format. See {@link HttpClientOptions.classifyResponseBody}.
|
|
31
|
+
*/
|
|
32
|
+
classifyResponseBody?: HttpClientOptions<TResponse>["classifyResponseBody"];
|
|
26
33
|
}
|
|
27
34
|
/**
|
|
28
35
|
* Site-agnostic factory that wires together Bottleneck + chromiumClientHints +
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limited-json-client.d.ts","sourceRoot":"","sources":["../../src/scraper/rate-limited-json-client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAGtC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"rate-limited-json-client.d.ts","sourceRoot":"","sources":["../../src/scraper/rate-limited-json-client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAGtC,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAGhF;;;;;GAKG;AACH,MAAM,WAAW,4BAA4B,CAAC,SAAS;IACrD,qFAAqF;IACrF,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,gEAAgE;IAChE,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC;CAC7E;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,SAAS,EACnD,IAAI,EAAE,4BAA4B,CAAC,SAAS,CAAC,GAC5C,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,SAAS,CAAC,CAgB7D"}
|
|
@@ -24,6 +24,11 @@ function createRateLimitedJsonClient(opts) {
|
|
|
24
24
|
}),
|
|
25
25
|
...(opts.extraHeaders ?? {}),
|
|
26
26
|
};
|
|
27
|
-
return (0, http_client_1.createHttpClient)({
|
|
27
|
+
return (0, http_client_1.createHttpClient)({
|
|
28
|
+
schema: opts.schema,
|
|
29
|
+
bottleneck,
|
|
30
|
+
baseHeaders,
|
|
31
|
+
classifyResponseBody: opts.classifyResponseBody,
|
|
32
|
+
});
|
|
28
33
|
}
|
|
29
34
|
//# sourceMappingURL=rate-limited-json-client.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rate-limited-json-client.js","sourceRoot":"","sources":["../../src/scraper/rate-limited-json-client.ts"],"names":[],"mappings":";;;;;;AAAA,4DAAoC;AAGpC,uEAAkE;AAElE,uDAAyD;
|
|
1
|
+
{"version":3,"file":"rate-limited-json-client.js","sourceRoot":"","sources":["../../src/scraper/rate-limited-json-client.ts"],"names":[],"mappings":";;;;;;AAAA,4DAAoC;AAGpC,uEAAkE;AAElE,uDAAyD;AAkCzD;;;;;;GAMG;AACH,qCACE,IAA6C;IAE7C,MAAM,UAAU,GAAG,IAAI,oBAAU,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/D,MAAM,WAAW,GAA2B;QAC1C,GAAG,IAAA,2CAAmB,EAAC;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;KAC7B,CAAC;IACF,OAAO,IAAA,8BAAgB,EAAC;QACtB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU;QACV,WAAW;QACX,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;KAChD,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -20,6 +20,12 @@ import { type BrowserSession, type BrowserSessionOptions } from "../scraper/sess
|
|
|
20
20
|
* Proxies: `proxies: true` enables Browserbase's residential proxy pool. The
|
|
21
21
|
* boolean form takes Browserbase's default region; per-region routing is
|
|
22
22
|
* available via the array form (not used here — out of scope until needed).
|
|
23
|
+
*
|
|
24
|
+
* Non-finding (recorded so it is not re-investigated): a Queue-it virtual-waiting-
|
|
25
|
+
* room interstitial has never blocked a Browserbase run in practice. It gates the
|
|
26
|
+
* HTML/browser path for every locale tried but not the underlying JSON API path,
|
|
27
|
+
* and across the runs observed it simply never appeared. Treat a Queue-it page as
|
|
28
|
+
* a site-path signal (prefer the API), not a Browserbase capability gap.
|
|
23
29
|
*/
|
|
24
30
|
/**
|
|
25
31
|
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile. When
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AA6ElC
|
|
1
|
+
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AA6ElC;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,+BAA+B,CACnD,IAAI,CAAC,EAAE,IAAI,CAAC,qBAAqB,EAAE,iBAAiB,GAAG,gCAAgC,CAAC,GACvF,OAAO,CAAC,cAAc,CAAC,CAkHzB"}
|
|
@@ -85,6 +85,12 @@ function makeFilteredStagehandLogger(pinoLogger) {
|
|
|
85
85
|
* Proxies: `proxies: true` enables Browserbase's residential proxy pool. The
|
|
86
86
|
* boolean form takes Browserbase's default region; per-region routing is
|
|
87
87
|
* available via the array form (not used here — out of scope until needed).
|
|
88
|
+
*
|
|
89
|
+
* Non-finding (recorded so it is not re-investigated): a Queue-it virtual-waiting-
|
|
90
|
+
* room interstitial has never blocked a Browserbase run in practice. It gates the
|
|
91
|
+
* HTML/browser path for every locale tried but not the underlying JSON API path,
|
|
92
|
+
* and across the runs observed it simply never appeared. Treat a Queue-it page as
|
|
93
|
+
* a site-path signal (prefer the API), not a Browserbase capability gap.
|
|
88
94
|
*/
|
|
89
95
|
/**
|
|
90
96
|
* `advancedStealth` opts into Browserbase's Scale Plan stealth profile. When
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;AAAA,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,6DAKkC;AAClC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAclE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,2BAA2B,CAAC,UAAkB;IAIrD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,4FAA4F,CAC5I,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;AAAA,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,6DAKkC;AAClC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAclE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,2BAA2B,CAAC,UAAkB;IAIrD,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,4FAA4F,CAC5I,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACI,KAAK,0CACV,IAAwF;IAExF,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,QAAQ,GAAG,IAAA,mCAAkB,GAAE,CAAC;IACtC,MAAM,mBAAmB,GAAG,eAAM,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC;IACrF,MAAM,eAAe,GAAG,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IACvD,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,GAAG,mBAAmB,EAAE,GAC3D,IAAI,EAAE,8BAA8B,IAAI,EAAE,CAAC;IAE7C,IAAI,eAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,uBAAuB,eAAM,CAAC,OAAO,CAAC,KAAK,cAAc,eAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,SAAS,GAAG,eAAM,CAAC,OAAO,CAAC,UAAU;QACzC,CAAC,CAAC,IAAI,uBAAW,CAAC,EAAE,KAAK,EAAE,IAAA,4BAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,CAAC,CAAC,SAAS,CAAC;IAEd,2EAA2E;IAC3E,uEAAuE;IACvE,4BAA4B;IAC5B,MAAM,eAAe,GAAG;QACtB,MAAM,EAAE;YACN,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;SAC3B;KACF,CAAC;IACF,MAAM,WAAW,GAAG,eAAe;QACjC,CAAC,CAAC;YACE,GAAG,eAAe;YAClB,OAAO,EAAE,CAAC,SAAkB,CAAC;YAC7B,gBAAgB,EAAE,CAAC,SAAkB,CAAC;SACvC;QACH,CAAC,CAAC,eAAe,CAAC;IAEpB,MAAM,EAAE,QAAQ,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,2BAA2B,EAAE,GACxF,2BAA2B,CAAC,MAAM,CAAC,CAAC;IAEtC,IAAI,SAAgC,CAAC;IACrC,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,qBAAS,CAAC;YACxB,GAAG,EAAE,aAAa;YAClB,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,iBAAiB;YACxC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;YAC9C,8BAA8B,EAAE;gBAC9B,GAAG,mBAAmB;gBACtB,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;gBAC9C,OAAO,EAAE,mBAAmB;gBAC5B,eAAe,EAAE;oBACf,GAAG,mBAAmB,CAAC,eAAe;oBACtC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1E,WAAW;iBACZ;aACF;YACD,KAAK,EAAE,eAAM,CAAC,OAAO,CAAC,UAAU;gBAC9B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAE;oBACC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,KAAK;oBAC/B,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,eAAe;oBACtC,gFAAgF;oBAChF,KAAK,EAAE,IAAA,mCAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;iBACtC;YAC5B,SAAS;YACT,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,CAAC;YACV,wDAAwD;YACxD,mDAAmD;YACnD,MAAM,EAAE,uBAAuB;SAChC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,IAAI,SAAS;gBAAE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACzC,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAA,uBAAc,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAG,SAAS,CAAC,oBAAoB,IAAI,SAAS,CAAC;IAC9D,MAAM,CAAC,IAAI,CACT,+BAA+B,SAAS,aAAa,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,YAAY,mBAAmB,oBAAoB,eAAe,EAAE,CAC3J,CAAC;IAEF,MAAM,OAAO,GAAG,IAAA,+BAAoB,GAAE,CAAC;IAEvC,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,SAAS,KAAK,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,2BAA2B,EAAE,CAAC;IAChC,CAAC,CAAC;IAEF,OAAO;QACL,SAAS;QACT,OAAO;QACP,SAAS;QACT,QAAQ,EAAE,aAAa;QACvB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
* Phase 1 recon: drives a real browser through a user-defined flow while
|
|
3
3
|
* wiretapping every network response. Captures are written to
|
|
4
4
|
* /tmp/recon/graphql/<NNN>-<phase>-<operationName>.json — one file per call,
|
|
5
|
-
* diffable and greppable.
|
|
5
|
+
* diffable and greppable. The browser's full cookie jar is snapshotted at
|
|
6
|
+
* each phase boundary (initial goto, pre-step, post-step, run completion) and
|
|
7
|
+
* written to /tmp/recon/cookies/<NNN>-<label>-<phase>.json, so the jar state
|
|
8
|
+
* can be diffed across the journey (e.g. what a click-tracker traversal
|
|
9
|
+
* establishes vs. what a later step adds).
|
|
6
10
|
*
|
|
7
11
|
* Recovery model — each flow step runs through a 4-attempt self-healing cascade
|
|
8
12
|
* (act → observe+act → observe+act with ignoreSelectors → Anthropic-SDK rephrase),
|
|
@@ -56,6 +60,19 @@ interface ReplanEvent {
|
|
|
56
60
|
htmlLength: number;
|
|
57
61
|
};
|
|
58
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* Reads the browser's full cookie jar for the given phase and writes it under
|
|
65
|
+
* `COOKIES_DIR`, so a run's snapshots land in the same append-only, diffable
|
|
66
|
+
* layout as `wireNetworkCapture`'s network captures. `counter` indexes
|
|
67
|
+
* filenames chronologically (zero-padded, shared convention with
|
|
68
|
+
* flow-runner's capture counter) so snapshots sort in the order the phases
|
|
69
|
+
* actually occurred. Never throws — cookie telemetry is best-effort, matching
|
|
70
|
+
* the existing capture-write behavior: a write failure logs a warning and the
|
|
71
|
+
* recon run continues.
|
|
72
|
+
*/
|
|
73
|
+
declare function snapshotAndPersistCookieJar(page: Page, counter: {
|
|
74
|
+
n: number;
|
|
75
|
+
}, label: string, phase: string, stepIndex: number): Promise<void>;
|
|
59
76
|
/**
|
|
60
77
|
* Validates the RECON_GOTO_WAIT_UNTIL override before it reaches Stagehand,
|
|
61
78
|
* which accepts a narrower set of load states than Playwright does — `commit`
|
|
@@ -307,5 +324,5 @@ declare function replanRemainingFlow(params: {
|
|
|
307
324
|
export type { Html5DateFillResult, InvalidFormControl, LeafInvalidField, RadioGroupCandidate, ValidationRejectionPair, VerifyFillReadbackResult, } from "../scraper/flow-runner";
|
|
308
325
|
export { buildRadioIdXPath, capturesAfterIndex, chooseRequiredSelectOption, describeAttemptEffectSignals, extractGaEventEvidence, extractSubmitFailureEvidence, fillHtml5DateTimeInput, findRecentBackendError, findRecentPageTransition, formatValidationRejectedReason, hasBillingErrorBeenLogged, isAdvanceStalled, isAdvanceStep, isDomOnlyAdvanceVerified, isSubmitRevealedInvalid, isUploadAffordanceLabel, isWizardExitAction, latestCaptureIndex, logBillingErrorIfPresent, narrowInvalidFormControl, normalizeDateValue, pairInvalidWithErrors, parseCaptureIndex, parseRadioStep, parseSelectStep, pollEnumerate, probeLeafInvalidContainers, probeStepBeforeAttempts, renderLeafInvalidFields, renderUnfocusedObserve, rephraseWithLLM, resetBillingErrorFlagForTests, selectBodyExcerpt, selectRadioGroupOption, shouldSkipTechnique, shouldVetoFallbackAdvance, verifyFillReadback, waitForTransitionBody, windowHasAdvanceTransition, windowHasTransitionBody, writeFixtureToTempFile, } from "../scraper/flow-runner";
|
|
309
326
|
export type { NormalizedStep, ReplanEvent };
|
|
310
|
-
export { dedupeConsecutiveIdentical, denormalizeStep, persistReplannedFlow, readFailureDumpEvidence, replanRemainingFlow, };
|
|
327
|
+
export { dedupeConsecutiveIdentical, denormalizeStep, persistReplannedFlow, readFailureDumpEvidence, replanRemainingFlow, snapshotAndPersistCookieJar, };
|
|
311
328
|
//# sourceMappingURL=recon-browser.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recon-browser.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-browser.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"recon-browser.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAMH,OAAO,KAAK,SAAS,MAAM,mBAAmB,CAAC;AAE/C,OAAO,KAAK,EAAU,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAyBnF,OAAO,EACL,KAAK,aAAa,EAElB,KAAK,SAAS,EAiBf,MAAM,uBAAuB,CAAC;AAM/B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAM9C,UAAU,WAAW;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,cAAc,GAAG,mBAAmB,CAAC;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,SAAS,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD;AAkED;;;;;;;;;GASG;AACH,iBAAe,2BAA2B,CACxC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IAAE,CAAC,EAAE,MAAM,CAAA;CAAE,EACtB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CASf;AAWD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,MAAe,GAAG,SAAS,CAQ7F;AAiBD,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,OAAO,GAAG;IAC5D,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CA4BA;AA0HD,wBAAgB,uBAAuB,CAAC,MAAM,EAAE;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7C,GAAG,MAAM,GAAG,IAAI,CAmBhB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,SAAS;IACjB,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,EAAE,GACF,OAAO,CAGT;AAED,wBAAgB,2BAA2B,CAAC,MAAM,EAAE;IAClD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,MAAM,CAoBT;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,YAAY,EAAE,SAAS,WAAW,EAAE,EACpC,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,YAAY,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAChD,OAAO,CAaT;AA+GD;;;;;;;GAOG;AACH,UAAU,cAAc;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAOhB,UAAU,CAAC,EAAE,OAAO,CAAC;IAMrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,MAAM,EAAE,UAAU,GAAG,QAAQ,CAAC;CAC/B;AAuCD;;;;;GAKG;AACH,iBAAS,eAAe,CAAC,IAAI,EAAE,cAAc,GACzC,MAAM,GACN;IACE,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,IAAI,CAAC;CACzB,CAqBJ;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,iBAAS,0BAA0B,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAStD;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,UAAU,EAAE,MAAM,GACjB,cAAc,EAAE,CAGlB;AAOD;;;;;;;;;;GAUG;AACH,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,UAAU,EAAE,MAAM,GACjB,OAAO,CAIT;AAED;;;;;;;;;;;GAWG;AACH,iBAAS,oBAAoB,CAAC,MAAM,EAAE;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;IACnC,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,qFAAqF;IACrF,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;IACnC,4FAA4F;IAC5F,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,oGAAoG;IACpG,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,uGAAuG;IACvG,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;IACjC,oGAAoG;IACpG,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,uGAAuG;IACvG,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;CACpC,GAAG,IAAI,CA6FP;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7C,MAAM,CAkBR;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,SAAS,WAAW,EAAE,GACnC,MAAM,CAcR;AAED,iBAAe,uBAAuB,CACpC,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE;IACR,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC1B,uBAAuB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,IAAI,CAAC;CACb,GACA,OAAO,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAChC,CAAC,CA8ED;AAED;;;;GAIG;AACH,iBAAe,mBAAmB,CAAC,MAAM,EAAE;IACzC,MAAM,EAAE,SAAS,CAAC;IAClB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,SAAS;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;KAAE,EAAE,CAAC;IACvF;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,CA0SnC;AAy6BD,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,0BAA0B,EAC1B,4BAA4B,EAC5B,sBAAsB,EACtB,4BAA4B,EAC5B,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,8BAA8B,EAC9B,yBAAyB,EACzB,gBAAgB,EAChB,aAAa,EACb,wBAAwB,EACxB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,qBAAqB,EACrB,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,aAAa,EACb,0BAA0B,EAC1B,uBAAuB,EACvB,uBAAuB,EACvB,sBAAsB,EACtB,eAAe,EACf,6BAA6B,EAC7B,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,kBAAkB,EAClB,qBAAqB,EACrB,0BAA0B,EAC1B,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;AAG5C,OAAO,EACL,0BAA0B,EAC1B,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,GAC5B,CAAC"}
|
|
@@ -3,7 +3,11 @@
|
|
|
3
3
|
* Phase 1 recon: drives a real browser through a user-defined flow while
|
|
4
4
|
* wiretapping every network response. Captures are written to
|
|
5
5
|
* /tmp/recon/graphql/<NNN>-<phase>-<operationName>.json — one file per call,
|
|
6
|
-
* diffable and greppable.
|
|
6
|
+
* diffable and greppable. The browser's full cookie jar is snapshotted at
|
|
7
|
+
* each phase boundary (initial goto, pre-step, post-step, run completion) and
|
|
8
|
+
* written to /tmp/recon/cookies/<NNN>-<label>-<phase>.json, so the jar state
|
|
9
|
+
* can be diffed across the journey (e.g. what a click-tracker traversal
|
|
10
|
+
* establishes vs. what a later step adds).
|
|
7
11
|
*
|
|
8
12
|
* Recovery model — each flow step runs through a 4-attempt self-healing cascade
|
|
9
13
|
* (act → observe+act → observe+act with ignoreSelectors → Anthropic-SDK rephrase),
|
|
@@ -51,6 +55,7 @@ exports.denormalizeStep = denormalizeStep;
|
|
|
51
55
|
exports.persistReplannedFlow = persistReplannedFlow;
|
|
52
56
|
exports.readFailureDumpEvidence = readFailureDumpEvidence;
|
|
53
57
|
exports.replanRemainingFlow = replanRemainingFlow;
|
|
58
|
+
exports.snapshotAndPersistCookieJar = snapshotAndPersistCookieJar;
|
|
54
59
|
const node_crypto_1 = require("node:crypto");
|
|
55
60
|
const node_fs_1 = require("node:fs");
|
|
56
61
|
const node_path_1 = require("node:path");
|
|
@@ -68,6 +73,7 @@ const logging_1 = require("../lib/logging");
|
|
|
68
73
|
const call_capture_1 = require("../lib/telemetry/call-capture");
|
|
69
74
|
const call_types_1 = require("../lib/telemetry/call-types");
|
|
70
75
|
const telemetry_paths_1 = require("../lib/telemetry/telemetry-paths");
|
|
76
|
+
const cookie_jar_1 = require("../scraper/cookie-jar");
|
|
71
77
|
const errors_2 = require("../scraper/errors");
|
|
72
78
|
const flow_runner_1 = require("../scraper/flow-runner");
|
|
73
79
|
const session_1 = require("../scraper/session");
|
|
@@ -126,6 +132,27 @@ function wireNetworkCapture(page, counter, signalCounter, recentCaptures, recent
|
|
|
126
132
|
},
|
|
127
133
|
});
|
|
128
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* Reads the browser's full cookie jar for the given phase and writes it under
|
|
137
|
+
* `COOKIES_DIR`, so a run's snapshots land in the same append-only, diffable
|
|
138
|
+
* layout as `wireNetworkCapture`'s network captures. `counter` indexes
|
|
139
|
+
* filenames chronologically (zero-padded, shared convention with
|
|
140
|
+
* flow-runner's capture counter) so snapshots sort in the order the phases
|
|
141
|
+
* actually occurred. Never throws — cookie telemetry is best-effort, matching
|
|
142
|
+
* the existing capture-write behavior: a write failure logs a warning and the
|
|
143
|
+
* recon run continues.
|
|
144
|
+
*/
|
|
145
|
+
async function snapshotAndPersistCookieJar(page, counter, label, phase, stepIndex) {
|
|
146
|
+
const snapshot = await (0, cookie_jar_1.captureCookieJarSnapshot)(page, label, phase, stepIndex);
|
|
147
|
+
const idx = String(counter.n++).padStart(3, "0");
|
|
148
|
+
const filename = `${idx}-${label}-${phase}.json`;
|
|
149
|
+
try {
|
|
150
|
+
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(recon_shared_1.COOKIES_DIR, filename), JSON.stringify(snapshot, null, 2));
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
logger.warn(`cookie-snapshot-write skipped for ${filename}: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
129
156
|
const MAX_PROBE_REPLANS = 5;
|
|
130
157
|
/**
|
|
131
158
|
* Replans triggered by the full self-healing cascade exhausting all 4
|
|
@@ -420,8 +447,7 @@ const RECON_FLOW_FILE_SCHEMA = v4_1.z.union([
|
|
|
420
447
|
* URL after the click contains any of these, that's one of the strong
|
|
421
448
|
* signals required for verified=true. Examples: ["/applied",
|
|
422
449
|
* "/applyboard/applied", "/confirmation", "/thank-you"]. Site-specific
|
|
423
|
-
* since URL conventions vary across
|
|
424
|
-
* tenants.
|
|
450
|
+
* since URL conventions vary across ATS tenants.
|
|
425
451
|
*/
|
|
426
452
|
successUrlFragments: v4_1.z.array(v4_1.z.string().min(1)).optional(),
|
|
427
453
|
/**
|
|
@@ -438,8 +464,8 @@ const RECON_FLOW_FILE_SCHEMA = v4_1.z.union([
|
|
|
438
464
|
* hostnames as a corroborating network signal — anything else (e.g.
|
|
439
465
|
* analytics, third-party trackers, CDNs) is ignored. Without this
|
|
440
466
|
* list the judge falls back to the URL alone, which is weaker.
|
|
441
|
-
* Examples: ["apply.appcast.io"], ["careers
|
|
442
|
-
* "<tenant>.
|
|
467
|
+
* Examples: ["apply.appcast.io"], ["careers.<ats>.com",
|
|
468
|
+
* "<tenant>.<ats>.com"].
|
|
443
469
|
*/
|
|
444
470
|
ownBackendHostnames: v4_1.z.array(v4_1.z.string().min(1)).optional(),
|
|
445
471
|
/**
|
|
@@ -1316,6 +1342,7 @@ async function main() {
|
|
|
1316
1342
|
}
|
|
1317
1343
|
const flow = substituteFlowEnvVars(rawFlow);
|
|
1318
1344
|
(0, node_fs_1.mkdirSync)(recon_shared_1.CAPTURES_DIR, { recursive: true });
|
|
1345
|
+
(0, node_fs_1.mkdirSync)(recon_shared_1.COOKIES_DIR, { recursive: true });
|
|
1319
1346
|
const resumeFixture = loadResumeFixture(resumeFixturePath);
|
|
1320
1347
|
// Per-URL partition under the flow file's site directory. Without a flow
|
|
1321
1348
|
// file (inline --flow mode), telemetry has no durable home and is dropped
|
|
@@ -1346,6 +1373,10 @@ async function main() {
|
|
|
1346
1373
|
// wireNetworkCapture for the rationale.
|
|
1347
1374
|
const counter = { n: 0 };
|
|
1348
1375
|
const signalCounter = { n: 0 };
|
|
1376
|
+
// Indexes cookie-jar snapshot filenames chronologically, separate from
|
|
1377
|
+
// `counter` (network captures) so a phase with zero network activity still
|
|
1378
|
+
// gets a snapshot without skipping capture indices.
|
|
1379
|
+
const jarCounter = { n: 0 };
|
|
1349
1380
|
const recentCaptures = [];
|
|
1350
1381
|
// Parallel tracker of recent non-GET captures' method + status. Used by
|
|
1351
1382
|
// the Tier 1 trailing-optional-step grace: a verification failure on an
|
|
@@ -1395,6 +1426,7 @@ async function main() {
|
|
|
1395
1426
|
catch (err) {
|
|
1396
1427
|
logger.warn(`navigation wait (${GOTO_WAIT_UNTIL}) did not settle: ${(0, errors_1.toErrorMessage)(err)} — continuing; the SPA readiness probe below decides whether the page is usable`);
|
|
1397
1428
|
}
|
|
1429
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "goto", currentPhase, -1);
|
|
1398
1430
|
const SPA_READINESS_TIMEOUT_MS = 15_000;
|
|
1399
1431
|
const SPA_READINESS_POLL_MS = 500;
|
|
1400
1432
|
const SPA_MIN_BODY_LENGTH = 5_000;
|
|
@@ -1454,6 +1486,7 @@ async function main() {
|
|
|
1454
1486
|
.replace(/^-|-$/g, "")
|
|
1455
1487
|
.slice(0, 24) || `step-${i}`;
|
|
1456
1488
|
logger.info(`step ${i + 1}/${plan.length} [${currentPhase}]${step.optional ? " (optional)" : ""}: ${step.instruction}`);
|
|
1489
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "pre-step", currentPhase, i);
|
|
1457
1490
|
// Re-gate on SPA hydration when the origin changed since the last step —
|
|
1458
1491
|
// the wizard SPA (e.g. apply.talemetry.com after the Apply click) boots on
|
|
1459
1492
|
// a new origin the initial-goto readiness gate never covered, so wait for
|
|
@@ -1517,6 +1550,7 @@ async function main() {
|
|
|
1517
1550
|
captureFn,
|
|
1518
1551
|
onStepFailure: dumpStepFailure,
|
|
1519
1552
|
});
|
|
1553
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "post-step", currentPhase, i);
|
|
1520
1554
|
// Wizard-restart detection: if a configured restart-signal URL (e.g.
|
|
1521
1555
|
// Talemetry's `init-apply?...&application_canceled=true`) landed during
|
|
1522
1556
|
// this step, the multi-page wizard reset to page 1. Remaining steps now
|
|
@@ -1770,6 +1804,7 @@ async function main() {
|
|
|
1770
1804
|
}
|
|
1771
1805
|
logger.info(`end-of-run audit PASSED: at least one captured 2xx matched submitEndpointPattern with clean response body`);
|
|
1772
1806
|
}
|
|
1807
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "run-complete", currentPhase, plan.length - 1);
|
|
1773
1808
|
logger.info(`recon complete — ${counter.n} captures written to ${recon_shared_1.CAPTURES_DIR}`);
|
|
1774
1809
|
}
|
|
1775
1810
|
finally {
|