@enricai/barnacle 1.12.18 → 1.12.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1211
- package/dist/scraper/errors.d.ts +7 -7
- package/dist/scraper/errors.js +7 -7
- package/dist/scraper/fake-timer-advance.d.ts +35 -8
- package/dist/scraper/fake-timer-advance.d.ts.map +1 -1
- package/dist/scraper/fake-timer-advance.js +29 -9
- package/dist/scraper/fake-timer-advance.js.map +1 -1
- package/dist/scraper/http-client.d.ts +1 -1
- package/dist/scraper/http-client.js +1 -1
- package/dist/scraper/pool.d.ts +3 -3
- package/dist/scraper/pool.js +5 -5
- package/dist/scripts/recon-browser.d.ts +2 -1
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +2 -1
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +27 -0
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +56 -9
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/scripts/recon-heal.d.ts +3 -3
- package/dist/scripts/recon-heal.js +4 -4
- package/dist/scripts/recon-heal.js.map +1 -1
- package/dist/server.js +1 -1
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
package/dist/scraper/errors.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* session pool key off of. Some failures abort the whole call (no retry
|
|
8
8
|
* can resolve them); others are retryable in place because a fresh AI
|
|
9
9
|
* resolution or DOM re-settle usually fixes them; some need a brand-new
|
|
10
|
-
*
|
|
10
|
+
* session before retrying, in which case retry.ts invokes its
|
|
11
11
|
* onRestart callback. See each class's TSDoc for its specific policy
|
|
12
12
|
* and the diagnostic it represents.
|
|
13
13
|
*/
|
|
@@ -21,8 +21,8 @@ export declare abstract class ScraperError extends Error {
|
|
|
21
21
|
constructor(message: string, retryable: boolean);
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
|
-
* The scraper encountered a CAPTCHA challenge
|
|
25
|
-
* behalf. Propagate to the caller as error code 2004.
|
|
24
|
+
* The scraper encountered a CAPTCHA challenge the session provider's built-in
|
|
25
|
+
* solver couldn't solve on our behalf. Propagate to the caller as error code 2004.
|
|
26
26
|
*/
|
|
27
27
|
export declare class CaptchaError extends ScraperError {
|
|
28
28
|
constructor(message?: string);
|
|
@@ -57,7 +57,7 @@ export declare class SelectorFailureError extends ScraperError {
|
|
|
57
57
|
constructor(message?: string);
|
|
58
58
|
}
|
|
59
59
|
/**
|
|
60
|
-
* The
|
|
60
|
+
* The browser session timed out or closed unexpectedly.
|
|
61
61
|
* Needs a fresh session to retry — the session pool wrapper in retry.ts
|
|
62
62
|
* honors this by invoking its `onRestart` hook before re-running.
|
|
63
63
|
*/
|
|
@@ -130,7 +130,7 @@ export type StepVerificationErrorKind = "cascade-exhausted" | "phantom-click-exh
|
|
|
130
130
|
* backend-error-unrecoverable.
|
|
131
131
|
* - "flow-timeout": runHealingFlow's per-run deadline (`maxFlowMs`) elapsed
|
|
132
132
|
* while stepping through the flow. Distinct from {@link SessionTimeoutError},
|
|
133
|
-
* which means the
|
|
133
|
+
* which means the session/browser itself died — this means the
|
|
134
134
|
* browser is fine but the flow overran its own budget, so the caller can
|
|
135
135
|
* fail fast well under the coarser pool-level `taskTimeoutMs` ceiling
|
|
136
136
|
* instead of grinding for the full session lifetime.
|
|
@@ -179,7 +179,7 @@ export declare function isHttpServerError(err: unknown): err is HttpServerError;
|
|
|
179
179
|
* The direct-HTTP hot path received a 429 rate-limit response. Non-retryable
|
|
180
180
|
* and NOT a fallback trigger — a 429 means the configured rps ceiling is too
|
|
181
181
|
* high; the right response is to back off and surface the metric, not burn a
|
|
182
|
-
*
|
|
182
|
+
* session.
|
|
183
183
|
*/
|
|
184
184
|
export declare class HttpRateLimitError extends ScraperError {
|
|
185
185
|
constructor(message?: string);
|
|
@@ -192,7 +192,7 @@ export declare function isHttpRateLimitError(err: unknown): err is HttpRateLimit
|
|
|
192
192
|
* resource after repeated requests. Non-retryable and NOT a browser-fallback
|
|
193
193
|
* trigger — the lock is at the target's end, so neither a retry of the identical
|
|
194
194
|
* HTTP request nor a fresh Stagehand browser session can succeed. The caller must
|
|
195
|
-
* back off and surface a "retry later" state rather than burning a
|
|
195
|
+
* back off and surface a "retry later" state rather than burning a session.
|
|
196
196
|
* Kept distinct from HttpRateLimitError so metrics/logs (classifyDispatchError)
|
|
197
197
|
* can tell a resource lock apart from a self-inflicted 429 rate limit.
|
|
198
198
|
*/
|
package/dist/scraper/errors.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* session pool key off of. Some failures abort the whole call (no retry
|
|
9
9
|
* can resolve them); others are retryable in place because a fresh AI
|
|
10
10
|
* resolution or DOM re-settle usually fixes them; some need a brand-new
|
|
11
|
-
*
|
|
11
|
+
* session before retrying, in which case retry.ts invokes its
|
|
12
12
|
* onRestart callback. See each class's TSDoc for its specific policy
|
|
13
13
|
* and the diagnostic it represents.
|
|
14
14
|
*/
|
|
@@ -38,8 +38,8 @@ class ScraperError extends Error {
|
|
|
38
38
|
}
|
|
39
39
|
exports.ScraperError = ScraperError;
|
|
40
40
|
/**
|
|
41
|
-
* The scraper encountered a CAPTCHA challenge
|
|
42
|
-
* behalf. Propagate to the caller as error code 2004.
|
|
41
|
+
* The scraper encountered a CAPTCHA challenge the session provider's built-in
|
|
42
|
+
* solver couldn't solve on our behalf. Propagate to the caller as error code 2004.
|
|
43
43
|
*/
|
|
44
44
|
class CaptchaError extends ScraperError {
|
|
45
45
|
constructor(message = "captcha challenge encountered") {
|
|
@@ -87,7 +87,7 @@ class SelectorFailureError extends ScraperError {
|
|
|
87
87
|
}
|
|
88
88
|
exports.SelectorFailureError = SelectorFailureError;
|
|
89
89
|
/**
|
|
90
|
-
* The
|
|
90
|
+
* The browser session timed out or closed unexpectedly.
|
|
91
91
|
* Needs a fresh session to retry — the session pool wrapper in retry.ts
|
|
92
92
|
* honors this by invoking its `onRestart` hook before re-running.
|
|
93
93
|
*/
|
|
@@ -170,7 +170,7 @@ exports.UnknownScraperError = UnknownScraperError;
|
|
|
170
170
|
* backend-error-unrecoverable.
|
|
171
171
|
* - "flow-timeout": runHealingFlow's per-run deadline (`maxFlowMs`) elapsed
|
|
172
172
|
* while stepping through the flow. Distinct from {@link SessionTimeoutError},
|
|
173
|
-
* which means the
|
|
173
|
+
* which means the session/browser itself died — this means the
|
|
174
174
|
* browser is fine but the flow overran its own budget, so the caller can
|
|
175
175
|
* fail fast well under the coarser pool-level `taskTimeoutMs` ceiling
|
|
176
176
|
* instead of grinding for the full session lifetime.
|
|
@@ -239,7 +239,7 @@ function isHttpServerError(err) {
|
|
|
239
239
|
* The direct-HTTP hot path received a 429 rate-limit response. Non-retryable
|
|
240
240
|
* and NOT a fallback trigger — a 429 means the configured rps ceiling is too
|
|
241
241
|
* high; the right response is to back off and surface the metric, not burn a
|
|
242
|
-
*
|
|
242
|
+
* session.
|
|
243
243
|
*/
|
|
244
244
|
class HttpRateLimitError extends ScraperError {
|
|
245
245
|
constructor(message = "http 429 rate limit exceeded") {
|
|
@@ -257,7 +257,7 @@ function isHttpRateLimitError(err) {
|
|
|
257
257
|
* resource after repeated requests. Non-retryable and NOT a browser-fallback
|
|
258
258
|
* trigger — the lock is at the target's end, so neither a retry of the identical
|
|
259
259
|
* HTTP request nor a fresh Stagehand browser session can succeed. The caller must
|
|
260
|
-
* back off and surface a "retry later" state rather than burning a
|
|
260
|
+
* back off and surface a "retry later" state rather than burning a session.
|
|
261
261
|
* Kept distinct from HttpRateLimitError so metrics/logs (classifyDispatchError)
|
|
262
262
|
* can tell a resource lock apart from a self-inflicted 429 rate limit.
|
|
263
263
|
*/
|
|
@@ -5,13 +5,22 @@
|
|
|
5
5
|
*/
|
|
6
6
|
export declare const DEFAULT_ADVANCE_STEP_MS = 1000;
|
|
7
7
|
/**
|
|
8
|
-
* Default cap on ticks {@link advanceUntilSettled} will
|
|
9
|
-
*
|
|
8
|
+
* Default cap on ticks {@link advanceUntilSettled} will give up before when a
|
|
9
|
+
* caller omits `maxIterations` — matches both existing call sites'
|
|
10
10
|
* `MAX_ADVANCE_ITERATIONS`/300-iteration bound before this module replaced
|
|
11
11
|
* them (300s of virtual time, comfortably past every watchdog this repo
|
|
12
|
-
* currently pins).
|
|
12
|
+
* currently pins). Only clock-advancing ticks count against it; see
|
|
13
|
+
* {@link DEFAULT_MAX_IDLE_ITERATIONS}.
|
|
13
14
|
*/
|
|
14
15
|
export declare const DEFAULT_MAX_ADVANCE_ITERATIONS = 300;
|
|
16
|
+
/**
|
|
17
|
+
* Default cap on consecutive zero-advance ticks (see
|
|
18
|
+
* {@link AdvanceUntilSettledOptions.getTimerCount}) before
|
|
19
|
+
* {@link advanceUntilSettled} gives up. Bounds the real-time cost of a
|
|
20
|
+
* promise that never settles while nothing is scheduled — each such tick is
|
|
21
|
+
* one real event-loop turn, so this is worth roughly a second of wall clock.
|
|
22
|
+
*/
|
|
23
|
+
export declare const DEFAULT_MAX_IDLE_ITERATIONS = 1000;
|
|
15
24
|
/**
|
|
16
25
|
* Options for {@link advanceUntilSettled}. `advanceTimersByTimeAsync` is
|
|
17
26
|
* injected rather than imported from `vitest` at module scope: `src/**` is
|
|
@@ -23,10 +32,28 @@ export declare const DEFAULT_MAX_ADVANCE_ITERATIONS = 300;
|
|
|
23
32
|
export interface AdvanceUntilSettledOptions {
|
|
24
33
|
/** A fake-timer advance function, e.g. vitest's `vi.advanceTimersByTimeAsync`. */
|
|
25
34
|
readonly advanceTimersByTimeAsync: (ms: number) => Promise<unknown>;
|
|
26
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Pending-fake-timer count, e.g. vitest's `vi.getTimerCount`. Supply it
|
|
37
|
+
* whenever the promise under test is driven by a deep real await chain
|
|
38
|
+
* (`runHealingFlow` and friends) rather than by timers alone.
|
|
39
|
+
*
|
|
40
|
+
* Sinon's `tickAsync` drains only a bounded number of turns per call, so
|
|
41
|
+
* such a chain needs many ticks purely to get its awaits to run — and
|
|
42
|
+
* blindly advancing `stepMs` on each of those charges the virtual clock for
|
|
43
|
+
* work that never waited on a timer. Worse, how many turns a chain needs
|
|
44
|
+
* varies with real machine speed, which makes any `Date.now()`-based
|
|
45
|
+
* elapsed-time assertion downstream flaky. With this supplied, ticks taken
|
|
46
|
+
* while nothing is scheduled advance by 0ms — still yielding a real
|
|
47
|
+
* event-loop turn, since `tickAsync(0)` does — so virtual elapsed time
|
|
48
|
+
* reflects only the delays the code under test actually scheduled.
|
|
49
|
+
*/
|
|
50
|
+
readonly getTimerCount?: () => number;
|
|
51
|
+
/** Virtual milliseconds advanced per clock-advancing tick. Defaults to {@link DEFAULT_ADVANCE_STEP_MS}. */
|
|
27
52
|
readonly stepMs?: number;
|
|
28
|
-
/** Maximum
|
|
53
|
+
/** Maximum clock-advancing ticks before giving up. Defaults to {@link DEFAULT_MAX_ADVANCE_ITERATIONS}. */
|
|
29
54
|
readonly maxIterations?: number;
|
|
55
|
+
/** Maximum consecutive zero-advance ticks before giving up. Defaults to {@link DEFAULT_MAX_IDLE_ITERATIONS}. */
|
|
56
|
+
readonly maxIdleIterations?: number;
|
|
30
57
|
}
|
|
31
58
|
/** Result of {@link advanceUntilSettled}: whether `promise` settled before the tick budget ran out. */
|
|
32
59
|
export interface AdvanceUntilSettledResult {
|
|
@@ -39,9 +66,9 @@ export interface AdvanceUntilSettledResult {
|
|
|
39
66
|
* inflate any `Date.now()`-based elapsed-time measurement a caller takes
|
|
40
67
|
* immediately after, since a fake clock has no notion of "idle" and just
|
|
41
68
|
* keeps advancing however far it's told to. Also stops at `maxIterations`
|
|
42
|
-
* so a promise that never settles (e.g. a genuinely
|
|
43
|
-
* can't loop forever, reporting `settled: false` in
|
|
44
|
-
* throwing.
|
|
69
|
+
* (or `maxIdleIterations`) so a promise that never settles (e.g. a genuinely
|
|
70
|
+
* wedged call under test) can't loop forever, reporting `settled: false` in
|
|
71
|
+
* that case instead of throwing.
|
|
45
72
|
*/
|
|
46
73
|
export declare function advanceUntilSettled(promise: Promise<unknown>, options: AdvanceUntilSettledOptions): Promise<AdvanceUntilSettledResult>;
|
|
47
74
|
//# sourceMappingURL=fake-timer-advance.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fake-timer-advance.d.ts","sourceRoot":"","sources":["../../src/scraper/fake-timer-advance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,OAAQ,CAAC;AAE7C
|
|
1
|
+
{"version":3,"file":"fake-timer-advance.d.ts","sourceRoot":"","sources":["../../src/scraper/fake-timer-advance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,OAAQ,CAAC;AAE7C;;;;;;;GAOG;AACH,eAAO,MAAM,8BAA8B,MAAM,CAAC;AAElD;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,OAAQ,CAAC;AAEjD;;;;;;;GAOG;AACH,MAAM,WAAW,0BAA0B;IACzC,kFAAkF;IAClF,QAAQ,CAAC,wBAAwB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACpE;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,MAAM,CAAC;IACtC,2GAA2G;IAC3G,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,0GAA0G;IAC1G,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,gHAAgH;IAChH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,uGAAuG;AACvG,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,EACzB,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,yBAAyB,CAAC,CA0BpC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DEFAULT_MAX_ADVANCE_ITERATIONS = exports.DEFAULT_ADVANCE_STEP_MS = void 0;
|
|
3
|
+
exports.DEFAULT_MAX_IDLE_ITERATIONS = exports.DEFAULT_MAX_ADVANCE_ITERATIONS = exports.DEFAULT_ADVANCE_STEP_MS = void 0;
|
|
4
4
|
exports.advanceUntilSettled = advanceUntilSettled;
|
|
5
5
|
/**
|
|
6
6
|
* Default per-tick virtual-time step {@link advanceUntilSettled} advances by
|
|
@@ -9,13 +9,22 @@ exports.advanceUntilSettled = advanceUntilSettled;
|
|
|
9
9
|
*/
|
|
10
10
|
exports.DEFAULT_ADVANCE_STEP_MS = 1_000;
|
|
11
11
|
/**
|
|
12
|
-
* Default cap on ticks {@link advanceUntilSettled} will
|
|
13
|
-
*
|
|
12
|
+
* Default cap on ticks {@link advanceUntilSettled} will give up before when a
|
|
13
|
+
* caller omits `maxIterations` — matches both existing call sites'
|
|
14
14
|
* `MAX_ADVANCE_ITERATIONS`/300-iteration bound before this module replaced
|
|
15
15
|
* them (300s of virtual time, comfortably past every watchdog this repo
|
|
16
|
-
* currently pins).
|
|
16
|
+
* currently pins). Only clock-advancing ticks count against it; see
|
|
17
|
+
* {@link DEFAULT_MAX_IDLE_ITERATIONS}.
|
|
17
18
|
*/
|
|
18
19
|
exports.DEFAULT_MAX_ADVANCE_ITERATIONS = 300;
|
|
20
|
+
/**
|
|
21
|
+
* Default cap on consecutive zero-advance ticks (see
|
|
22
|
+
* {@link AdvanceUntilSettledOptions.getTimerCount}) before
|
|
23
|
+
* {@link advanceUntilSettled} gives up. Bounds the real-time cost of a
|
|
24
|
+
* promise that never settles while nothing is scheduled — each such tick is
|
|
25
|
+
* one real event-loop turn, so this is worth roughly a second of wall clock.
|
|
26
|
+
*/
|
|
27
|
+
exports.DEFAULT_MAX_IDLE_ITERATIONS = 1_000;
|
|
19
28
|
/**
|
|
20
29
|
* Advances a fake clock in bounded increments until `promise` settles
|
|
21
30
|
* (resolves or rejects), stopping on the tick after settlement rather than
|
|
@@ -23,21 +32,32 @@ exports.DEFAULT_MAX_ADVANCE_ITERATIONS = 300;
|
|
|
23
32
|
* inflate any `Date.now()`-based elapsed-time measurement a caller takes
|
|
24
33
|
* immediately after, since a fake clock has no notion of "idle" and just
|
|
25
34
|
* keeps advancing however far it's told to. Also stops at `maxIterations`
|
|
26
|
-
* so a promise that never settles (e.g. a genuinely
|
|
27
|
-
* can't loop forever, reporting `settled: false` in
|
|
28
|
-
* throwing.
|
|
35
|
+
* (or `maxIdleIterations`) so a promise that never settles (e.g. a genuinely
|
|
36
|
+
* wedged call under test) can't loop forever, reporting `settled: false` in
|
|
37
|
+
* that case instead of throwing.
|
|
29
38
|
*/
|
|
30
39
|
async function advanceUntilSettled(promise, options) {
|
|
31
40
|
const stepMs = options.stepMs ?? exports.DEFAULT_ADVANCE_STEP_MS;
|
|
32
41
|
const maxIterations = options.maxIterations ?? exports.DEFAULT_MAX_ADVANCE_ITERATIONS;
|
|
42
|
+
const maxIdleIterations = options.maxIdleIterations ?? exports.DEFAULT_MAX_IDLE_ITERATIONS;
|
|
33
43
|
let settled = false;
|
|
34
44
|
promise.then(() => {
|
|
35
45
|
settled = true;
|
|
36
46
|
}, () => {
|
|
37
47
|
settled = true;
|
|
38
48
|
});
|
|
39
|
-
|
|
40
|
-
|
|
49
|
+
let advanced = 0;
|
|
50
|
+
let idle = 0;
|
|
51
|
+
while (!settled && advanced < maxIterations && idle < maxIdleIterations) {
|
|
52
|
+
const nothingScheduled = options.getTimerCount?.() === 0;
|
|
53
|
+
if (nothingScheduled) {
|
|
54
|
+
idle += 1;
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
advanced += 1;
|
|
58
|
+
idle = 0;
|
|
59
|
+
}
|
|
60
|
+
await options.advanceTimersByTimeAsync(nothingScheduled ? 0 : stepMs);
|
|
41
61
|
}
|
|
42
62
|
return { settled };
|
|
43
63
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fake-timer-advance.js","sourceRoot":"","sources":["../../src/scraper/fake-timer-advance.ts"],"names":[],"mappings":";;;;AAAA;;;;GAIG;AACU,QAAA,uBAAuB,GAAG,KAAK,CAAC;AAE7C
|
|
1
|
+
{"version":3,"file":"fake-timer-advance.js","sourceRoot":"","sources":["../../src/scraper/fake-timer-advance.ts"],"names":[],"mappings":";;;;AAAA;;;;GAIG;AACU,QAAA,uBAAuB,GAAG,KAAK,CAAC;AAE7C;;;;;;;GAOG;AACU,QAAA,8BAA8B,GAAG,GAAG,CAAC;AAElD;;;;;;GAMG;AACU,QAAA,2BAA2B,GAAG,KAAK,CAAC;AA0CjD;;;;;;;;;;GAUG;AACI,KAAK,8BACV,OAAyB,EACzB,OAAmC;IAEnC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,QAAA,uBAAuB,CAAC;IACzD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAA,8BAA8B,CAAC;IAC9E,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,QAAA,2BAA2B,CAAC;IACnF,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,OAAO,CAAC,IAAI,CACV,GAAG,EAAE;QACH,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC,EACD,GAAG,EAAE;QACH,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC,CACF,CAAC;IACF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,OAAO,CAAC,OAAO,IAAI,QAAQ,GAAG,aAAa,IAAI,IAAI,GAAG,iBAAiB,EAAE,CAAC;QACxE,MAAM,gBAAgB,GAAG,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,CAAC,CAAC;QACzD,IAAI,gBAAgB,EAAE,CAAC;YACrB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,QAAQ,IAAI,CAAC,CAAC;YACd,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;QACD,MAAM,OAAO,CAAC,wBAAwB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC;AACrB,CAAC"}
|
|
@@ -142,7 +142,7 @@ export interface HttpRequestInit<TOverride = unknown> {
|
|
|
142
142
|
* Throws `HttpServerError` on 5xx — also a fallback trigger; a server-side
|
|
143
143
|
* outage is not the same as a bot block but the recovery is identical.
|
|
144
144
|
* Throws `HttpRateLimitError` on 429 — NOT a fallback trigger; the caller
|
|
145
|
-
* should back off, not burn a
|
|
145
|
+
* should back off, not burn a session.
|
|
146
146
|
* Wraps transient network errors in `UnknownScraperError` and retries up to
|
|
147
147
|
* 2 times with exponential backoff before propagating.
|
|
148
148
|
*/
|
|
@@ -167,7 +167,7 @@ function resolveBinding(binding, headers) {
|
|
|
167
167
|
* Throws `HttpServerError` on 5xx — also a fallback trigger; a server-side
|
|
168
168
|
* outage is not the same as a bot block but the recovery is identical.
|
|
169
169
|
* Throws `HttpRateLimitError` on 429 — NOT a fallback trigger; the caller
|
|
170
|
-
* should back off, not burn a
|
|
170
|
+
* should back off, not burn a session.
|
|
171
171
|
* Wraps transient network errors in `UnknownScraperError` and retries up to
|
|
172
172
|
* 2 times with exponential backoff before propagating.
|
|
173
173
|
*/
|
package/dist/scraper/pool.d.ts
CHANGED
|
@@ -24,9 +24,9 @@ export declare function poolStats(): {
|
|
|
24
24
|
};
|
|
25
25
|
/**
|
|
26
26
|
* Drains the scraper queue on graceful shutdown — pauses new intake,
|
|
27
|
-
* waits for in-flight tasks' `finally` blocks to close their
|
|
28
|
-
*
|
|
29
|
-
*
|
|
27
|
+
* waits for in-flight tasks' `finally` blocks to close their sessions,
|
|
28
|
+
* then resolves. Leaving sessions open past process exit means the
|
|
29
|
+
* provider keeps billing until their own timeout kicks in.
|
|
30
30
|
*
|
|
31
31
|
* Bounded by `timeoutMs` so a hung scrape can't block SIGTERM forever;
|
|
32
32
|
* the orchestrator's grace period is usually 30s, so 20s is a safe
|
package/dist/scraper/pool.js
CHANGED
|
@@ -24,8 +24,8 @@ const TASK_TIMEOUT_MS = 60 * 60 * 1_000; // 60 minutes
|
|
|
24
24
|
/**
|
|
25
25
|
* Shared queue that limits how many scraper tasks run concurrently across
|
|
26
26
|
* the whole process. Sessions are created on demand inside each queued
|
|
27
|
-
* task, not pre-warmed, so
|
|
28
|
-
* traffic.
|
|
27
|
+
* task, not pre-warmed, so provider billing (Browserbase by default, Steel
|
|
28
|
+
* for the opt-in fallback) stays proportional to actual traffic.
|
|
29
29
|
*/
|
|
30
30
|
const queue = new p_queue_1.default({ concurrency: config_1.config.scraper.poolSize });
|
|
31
31
|
/**
|
|
@@ -88,9 +88,9 @@ function poolStats() {
|
|
|
88
88
|
}
|
|
89
89
|
/**
|
|
90
90
|
* Drains the scraper queue on graceful shutdown — pauses new intake,
|
|
91
|
-
* waits for in-flight tasks' `finally` blocks to close their
|
|
92
|
-
*
|
|
93
|
-
*
|
|
91
|
+
* waits for in-flight tasks' `finally` blocks to close their sessions,
|
|
92
|
+
* then resolves. Leaving sessions open past process exit means the
|
|
93
|
+
* provider keeps billing until their own timeout kicks in.
|
|
94
94
|
*
|
|
95
95
|
* Bounded by `timeoutMs` so a hung scrape can't block SIGTERM forever;
|
|
96
96
|
* the orchestrator's grace period is usually 30s, so 20s is a safe
|
|
@@ -39,7 +39,8 @@
|
|
|
39
39
|
* # recon:generate. Use grep against /tmp/recon/graphql/ for specific endpoints.
|
|
40
40
|
* pnpm tsx src/scripts/recon-browser.ts --url https://example.com --allow-empty-flow
|
|
41
41
|
*
|
|
42
|
-
* The script needs
|
|
42
|
+
* The script needs BROWSERBASE_API_KEY/BROWSERBASE_PROJECT_ID (or STEEL_API_KEY
|
|
43
|
+
* when passing --provider steel) and either ANTHROPIC_API_KEY or USE_BEDROCK=true
|
|
43
44
|
* in the environment (same vars as the main server).
|
|
44
45
|
*
|
|
45
46
|
* Runtime: varies — ~20–40 min for a full flow (STEP_PAUSE_MS × N steps + LLM latency
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recon-browser.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-browser.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"recon-browser.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAMH,OAAO,KAAK,SAAS,MAAM,mBAAmB,CAAC;AAE/C,OAAO,KAAK,EAAU,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAEnF,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AA4B3B,OAAO,EACL,KAAK,aAAa,EAElB,KAAK,SAAS,EAmBf,MAAM,uBAAuB,CAAC;AAO/B,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAQ5E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAM9C,UAAU,WAAW;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,cAAc,GAAG,mBAAmB,CAAC;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,SAAS,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD;AAoED;;;;;;;;;GASG;AACH,iBAAe,2BAA2B,CACxC,IAAI,EAAE,IAAI,EACV,OAAO,EAAE;IAAE,CAAC,EAAE,MAAM,CAAA;CAAE,EACtB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAUf;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,MAAe,GAAG,SAAS,CAQ7F;AAiBD,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,OAAO,GAAG;IAC5D,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CA4BA;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE;IACtC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB,EAAE,OAAO,CAAC;CACjC,GAAG,OAAO,CAEV;AA+GD;;;;;;;;;;;;;;GAcG;AACH,iBAAS,mBAAmB,CAAC,MAAM,EAAE;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,IAAI,CAuCP;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7C,GAAG,MAAM,GAAG,IAAI,CAmBhB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,SAAS;IACjB,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,EAAE,GACF,OAAO,CAGT;AAED,wBAAgB,2BAA2B,CAAC,MAAM,EAAE;IAClD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,MAAM,CAoBT;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,YAAY,EAAE,SAAS,WAAW,EAAE,EACpC,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,YAAY,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAChD,OAAO,CAaT;AAED,QAAA,MAAM,iBAAiB;;;;;;;;;;;;oBAAyC,CAAC;AAEjE;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAuGjC,CAAC;AAEH;;;;;;;GAOG;AACH,UAAU,cAAc;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAOhB,UAAU,CAAC,EAAE,OAAO,CAAC;IAMrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAI3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,UAAU,GAAG,QAAQ,CAAC;CAC/B;AAED,iBAAS,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,GAAG,cAAc,EAAE,CAgBjF;AAuBD;;;;;GAKG;AACH,iBAAS,eAAe,CAAC,IAAI,EAAE,cAAc,GACzC,MAAM,GACN;IACE,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,IAAI,CAAC;IAChB,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,IAAI,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,QAAQ,CAAC;CACnB,CA8BJ;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAUvD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAWnF;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,UAAU,EAAE,MAAM,GACjB,cAAc,EAAE,CAGlB;AAgDD;;;;;;;;;;GAUG;AACH,wBAAgB,mCAAmC,CACjD,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,iBAAiB,EAAE,SAAS,cAAc,EAAE,GAC3C,cAAc,EAAE,CAMlB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,UAAU,EAAE,MAAM,GACjB,OAAO,CAIT;AAaD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oCAAoC,CAClD,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,cAAc,EAAE,SAAS,MAAM,EAAE,GAChC,OAAO,CAUT;AAqCD;;;;;;;;;;;GAWG;AACH,iBAAS,oBAAoB,CAAC,MAAM,EAAE;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;IACnC,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,qFAAqF;IACrF,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;IACnC,4FAA4F;IAC5F,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,oGAAoG;IACpG,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,uGAAuG;IACvG,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;IACjC,oGAAoG;IACpG,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,uGAAuG;IACvG,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;CACpC,GAAG,IAAI,CAqGP;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7C,MAAM,CAkBR;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,SAAS,WAAW,EAAE,GACnC,MAAM,CAcR;AAED,iBAAe,uBAAuB,CACpC,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE;IACR,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC1B,uBAAuB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,GACA,OAAO,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAChC,CAAC,CAmFD;AAED;;;;GAIG;AACH,iBAAe,mBAAmB,CAAC,MAAM,EAAE;IACzC,MAAM,EAAE,SAAS,CAAC;IAClB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,SAAS;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;KAAE,EAAE,CAAC;IACvF;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,CAiTnC;AAiFD,iBAAS,QAAQ,IAAI;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,cAAc,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,YAAY,GAAG,SAAS,CAAC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,eAAe,EAAE,OAAO,CAAC;IACzB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,0BAA0B,EAAE,OAAO,CAAC;IACpC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,sBAAsB,EAAE,MAAM,EAAE,CAAC;IACjC,wBAAwB,EAAE,MAAM,EAAE,CAAC;IACnC,aAAa,EAAE,OAAO,GAAG,QAAQ,CAAC;IAClC,cAAc,EAAE,OAAO,CAAC;CACzB,CAmNA;AA6BD,iBAAe,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAm2BnC;AAgBD,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,GACzB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,0BAA0B,EAC1B,4BAA4B,EAC5B,sBAAsB,EACtB,4BAA4B,EAC5B,sBAAsB,EACtB,uBAAuB,EACvB,sBAAsB,EACtB,wBAAwB,EACxB,8BAA8B,EAC9B,yBAAyB,EACzB,gBAAgB,EAChB,aAAa,EACb,wBAAwB,EACxB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,qBAAqB,EACrB,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,aAAa,EACb,0BAA0B,EAC1B,uBAAuB,EACvB,uBAAuB,EACvB,sBAAsB,EACtB,eAAe,EACf,6BAA6B,EAC7B,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,+BAA+B,EAC/B,kBAAkB,EAClB,qBAAqB,EACrB,0BAA0B,EAC1B,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;AAI5C,OAAO,EACL,eAAe,EACf,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,EACnB,2BAA2B,EAC3B,mBAAmB,GACpB,CAAC"}
|
|
@@ -40,7 +40,8 @@
|
|
|
40
40
|
* # recon:generate. Use grep against /tmp/recon/graphql/ for specific endpoints.
|
|
41
41
|
* pnpm tsx src/scripts/recon-browser.ts --url https://example.com --allow-empty-flow
|
|
42
42
|
*
|
|
43
|
-
* The script needs
|
|
43
|
+
* The script needs BROWSERBASE_API_KEY/BROWSERBASE_PROJECT_ID (or STEEL_API_KEY
|
|
44
|
+
* when passing --provider steel) and either ANTHROPIC_API_KEY or USE_BEDROCK=true
|
|
44
45
|
* in the environment (same vars as the main server).
|
|
45
46
|
*
|
|
46
47
|
* Runtime: varies — ~20–40 min for a full flow (STEP_PAUSE_MS × N steps + LLM latency
|