@enricai/barnacle 1.12.79 → 1.12.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/helpers/reply.d.ts.map +1 -1
- package/dist/api/helpers/reply.js +5 -2
- package/dist/api/helpers/reply.js.map +1 -1
- package/dist/config.d.ts +10 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +2 -0
- package/dist/config.js.map +1 -1
- package/dist/lib/llm/judge.d.ts.map +1 -1
- package/dist/lib/llm/judge.js +6 -1
- package/dist/lib/llm/judge.js.map +1 -1
- package/dist/lib/llm/judges/verify-step-advanced.d.ts +30 -0
- package/dist/lib/llm/judges/verify-step-advanced.d.ts.map +1 -0
- package/dist/lib/llm/judges/verify-step-advanced.js +62 -0
- package/dist/lib/llm/judges/verify-step-advanced.js.map +1 -0
- package/dist/lib/llm/judges/verify-submit.d.ts +7 -1
- package/dist/lib/llm/judges/verify-submit.d.ts.map +1 -1
- package/dist/lib/llm/judges/verify-submit.js +29 -8
- package/dist/lib/llm/judges/verify-submit.js.map +1 -1
- package/dist/lib/llm/schemas.d.ts +8 -0
- package/dist/lib/llm/schemas.d.ts.map +1 -1
- package/dist/lib/llm/schemas.js +21 -13
- package/dist/lib/llm/schemas.js.map +1 -1
- package/dist/lib/logging.d.ts +7 -0
- package/dist/lib/logging.d.ts.map +1 -1
- package/dist/lib/logging.js +19 -1
- package/dist/lib/logging.js.map +1 -1
- package/dist/lib/pii-redaction.d.ts +104 -0
- package/dist/lib/pii-redaction.d.ts.map +1 -0
- package/dist/lib/pii-redaction.js +397 -0
- package/dist/lib/pii-redaction.js.map +1 -0
- package/dist/lib/random.d.ts +3 -1
- package/dist/lib/random.d.ts.map +1 -1
- package/dist/lib/random.js +7 -2
- package/dist/lib/random.js.map +1 -1
- package/dist/lib/telemetry/beacon-capture.d.ts.map +1 -1
- package/dist/lib/telemetry/beacon-capture.js +2 -0
- package/dist/lib/telemetry/beacon-capture.js.map +1 -1
- package/dist/lib/telemetry/call-capture.d.ts.map +1 -1
- package/dist/lib/telemetry/call-capture.js +4 -1
- package/dist/lib/telemetry/call-capture.js.map +1 -1
- package/dist/lib/telemetry/submission-capture.d.ts +11 -1
- package/dist/lib/telemetry/submission-capture.d.ts.map +1 -1
- package/dist/lib/telemetry/submission-capture.js +32 -3
- package/dist/lib/telemetry/submission-capture.js.map +1 -1
- package/dist/plugins/config-plugin.d.ts +8 -0
- package/dist/plugins/config-plugin.d.ts.map +1 -1
- package/dist/plugins/config-plugin.js +4 -0
- package/dist/plugins/config-plugin.js.map +1 -1
- package/dist/plugins/discover.d.ts.map +1 -1
- package/dist/plugins/discover.js +2 -0
- package/dist/plugins/discover.js.map +1 -1
- package/dist/plugins/loader.d.ts.map +1 -1
- package/dist/plugins/loader.js +23 -6
- package/dist/plugins/loader.js.map +1 -1
- package/dist/scraper/active-page.d.ts +18 -0
- package/dist/scraper/active-page.d.ts.map +1 -0
- package/dist/scraper/active-page.js +53 -0
- package/dist/scraper/active-page.js.map +1 -0
- package/dist/scraper/click-settle.d.ts +19 -0
- package/dist/scraper/click-settle.d.ts.map +1 -0
- package/dist/scraper/click-settle.js +65 -0
- package/dist/scraper/click-settle.js.map +1 -0
- package/dist/scraper/deep-locator-candidates.d.ts +20 -1
- package/dist/scraper/deep-locator-candidates.d.ts.map +1 -1
- package/dist/scraper/deep-locator-candidates.js +75 -11
- package/dist/scraper/deep-locator-candidates.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts +321 -6
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +2087 -341
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/navigate.d.ts +3 -3
- package/dist/scraper/navigate.js +4 -4
- package/dist/scraper/navigate.js.map +1 -1
- package/dist/scraper/phantom-click.d.ts +34 -2
- package/dist/scraper/phantom-click.d.ts.map +1 -1
- package/dist/scraper/phantom-click.js +80 -6
- package/dist/scraper/phantom-click.js.map +1 -1
- package/dist/scraper/quoted-spans.d.ts +35 -0
- package/dist/scraper/quoted-spans.d.ts.map +1 -0
- package/dist/scraper/quoted-spans.js +83 -0
- package/dist/scraper/quoted-spans.js.map +1 -0
- package/dist/scraper/session-browserbase.d.ts +4 -2
- package/dist/scraper/session-browserbase.d.ts.map +1 -1
- package/dist/scraper/session-browserbase.js +36 -5
- package/dist/scraper/session-browserbase.js.map +1 -1
- package/dist/scraper/session-steel.d.ts.map +1 -1
- package/dist/scraper/session-steel.js +16 -2
- package/dist/scraper/session-steel.js.map +1 -1
- package/dist/scraper/session-teardown.d.ts +1 -0
- package/dist/scraper/session-teardown.d.ts.map +1 -1
- package/dist/scraper/session-teardown.js +19 -8
- package/dist/scraper/session-teardown.js.map +1 -1
- package/dist/scraper/spa-readiness.d.ts +13 -1
- package/dist/scraper/spa-readiness.d.ts.map +1 -1
- package/dist/scraper/spa-readiness.js +31 -1
- package/dist/scraper/spa-readiness.js.map +1 -1
- package/dist/scraper/stagehand-guard.d.ts +8 -3
- package/dist/scraper/stagehand-guard.d.ts.map +1 -1
- package/dist/scraper/stagehand-guard.js +181 -13
- package/dist/scraper/stagehand-guard.js.map +1 -1
- package/dist/scraper/transient-provider-error.d.ts +19 -0
- package/dist/scraper/transient-provider-error.d.ts.map +1 -0
- package/dist/scraper/transient-provider-error.js +59 -0
- package/dist/scraper/transient-provider-error.js.map +1 -0
- package/dist/scraper/trusted-click.d.ts +122 -0
- package/dist/scraper/trusted-click.d.ts.map +1 -0
- package/dist/scraper/trusted-click.js +201 -0
- package/dist/scraper/trusted-click.js.map +1 -0
- package/dist/scraper/upload-affordance-expr.d.ts +33 -0
- package/dist/scraper/upload-affordance-expr.d.ts.map +1 -0
- package/dist/scraper/upload-affordance-expr.js +91 -0
- package/dist/scraper/upload-affordance-expr.js.map +1 -0
- package/dist/scripts/recon-browser.d.ts +26 -1
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +108 -21
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-capture-rebind.d.ts +11 -0
- package/dist/scripts/recon-capture-rebind.d.ts.map +1 -0
- package/dist/scripts/recon-capture-rebind.js +26 -0
- package/dist/scripts/recon-capture-rebind.js.map +1 -0
- package/dist/scripts/recon-generate.d.ts +112 -9
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +889 -181
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/scripts/recon-shared.d.ts +7 -0
- package/dist/scripts/recon-shared.d.ts.map +1 -1
- package/dist/scripts/recon-shared.js +19 -0
- package/dist/scripts/recon-shared.js.map +1 -1
- package/dist/site-plugin.d.ts +8 -0
- package/dist/site-plugin.d.ts.map +1 -1
- package/dist/types/pii-redaction.d.ts +39 -0
- package/dist/types/pii-redaction.d.ts.map +1 -0
- package/dist/types/pii-redaction.js +3 -0
- package/dist/types/pii-redaction.js.map +1 -0
- package/package.json +14 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared apply-page navigation helper. Centralises the two-step pattern that
|
|
3
3
|
* every site browser-flow repeats: await the active Stagehand page, then
|
|
4
|
-
* navigate to the apply URL with
|
|
5
|
-
*
|
|
4
|
+
* navigate to the apply URL with domcontentloaded. Chatty pages never reach
|
|
5
|
+
* networkidle, so hydration is left to waitForSpaReady in the caller.
|
|
6
6
|
*
|
|
7
7
|
* Scripts that interleave CDP listener wiring between awaitActivePage and goto
|
|
8
8
|
* (recon-browser.ts, recon-generate.ts, etc.) must NOT use this helper — they
|
|
@@ -12,7 +12,7 @@ import type { Page, Stagehand } from "@browserbasehq/stagehand";
|
|
|
12
12
|
import type { MetricsCollector } from "../lib/dispatch-metrics";
|
|
13
13
|
/**
|
|
14
14
|
* Awaits the active Stagehand page and navigates to `url` with
|
|
15
|
-
* `waitUntil: "
|
|
15
|
+
* `waitUntil: "domcontentloaded"`. When a `MetricsCollector` is supplied, wraps
|
|
16
16
|
* the goto in a `"navigate"` step so the caller's metrics include navigation
|
|
17
17
|
* timing without each site having to repeat the bookkeeping.
|
|
18
18
|
*/
|
package/dist/scraper/navigate.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Shared apply-page navigation helper. Centralises the two-step pattern that
|
|
4
4
|
* every site browser-flow repeats: await the active Stagehand page, then
|
|
5
|
-
* navigate to the apply URL with
|
|
6
|
-
*
|
|
5
|
+
* navigate to the apply URL with domcontentloaded. Chatty pages never reach
|
|
6
|
+
* networkidle, so hydration is left to waitForSpaReady in the caller.
|
|
7
7
|
*
|
|
8
8
|
* Scripts that interleave CDP listener wiring between awaitActivePage and goto
|
|
9
9
|
* (recon-browser.ts, recon-generate.ts, etc.) must NOT use this helper — they
|
|
@@ -13,7 +13,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
13
13
|
exports.navigateActivePage = navigateActivePage;
|
|
14
14
|
/**
|
|
15
15
|
* Awaits the active Stagehand page and navigates to `url` with
|
|
16
|
-
* `waitUntil: "
|
|
16
|
+
* `waitUntil: "domcontentloaded"`. When a `MetricsCollector` is supplied, wraps
|
|
17
17
|
* the goto in a `"navigate"` step so the caller's metrics include navigation
|
|
18
18
|
* timing without each site having to repeat the bookkeeping.
|
|
19
19
|
*/
|
|
@@ -21,7 +21,7 @@ async function navigateActivePage(stagehand, url, collector, timeoutMs) {
|
|
|
21
21
|
const page = await stagehand.context.awaitActivePage();
|
|
22
22
|
collector?.startStep("navigate");
|
|
23
23
|
await page.goto(url, {
|
|
24
|
-
waitUntil: "
|
|
24
|
+
waitUntil: "domcontentloaded",
|
|
25
25
|
...(timeoutMs !== undefined && { timeoutMs }),
|
|
26
26
|
});
|
|
27
27
|
collector?.endStep("success");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"navigate.js","sourceRoot":"","sources":["../../src/scraper/navigate.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;AAMH;;;;;GAKG;AACI,KAAK,6BACV,SAAoB,EACpB,GAAW,EACX,SAA4B,EAC5B,SAAkB;IAElB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;IACvD,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;QACnB,SAAS,EAAE,
|
|
1
|
+
{"version":3,"file":"navigate.js","sourceRoot":"","sources":["../../src/scraper/navigate.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;AAMH;;;;;GAKG;AACI,KAAK,6BACV,SAAoB,EACpB,GAAW,EACX,SAA4B,EAC5B,SAAkB;IAElB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;IACvD,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;QACnB,SAAS,EAAE,kBAAkB;QAC7B,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAC9C,CAAC,CAAC;IACH,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -11,6 +11,8 @@ export interface PhantomClickSnapshot {
|
|
|
11
11
|
url: string;
|
|
12
12
|
/** `document.body.outerHTML.length`. */
|
|
13
13
|
bodyHtmlLength: number;
|
|
14
|
+
/** `<len>:<first-200-chars>` of visible text, as in flow-runner's StepSnapshot. */
|
|
15
|
+
visibleTextSignature?: string;
|
|
14
16
|
}
|
|
15
17
|
export interface PhantomClickAttempt {
|
|
16
18
|
/** Stagehand's own verdict for the attempt — did it believe it acted? */
|
|
@@ -39,6 +41,14 @@ export interface PhantomClickAttempt {
|
|
|
39
41
|
* so non-submit callers and existing tests are unchanged; defaults to false.
|
|
40
42
|
*/
|
|
41
43
|
isSubmitShapedStep?: boolean;
|
|
44
|
+
/**
|
|
45
|
+
* True when a submit-shaped step is NOT the flow's final submit (e.g. an
|
|
46
|
+
* account-creation transition in a wizard). Such a step is credited when
|
|
47
|
+
* {@link hasPageAdvanced} holds even with no network/URL/title signal.
|
|
48
|
+
* Implies `isSubmitShapedStep`: a mid-flow submit is submit-shaped by
|
|
49
|
+
* definition, so callers need not set both for it to be held to submit rules.
|
|
50
|
+
*/
|
|
51
|
+
isMidFlowSubmitStep?: boolean;
|
|
42
52
|
/**
|
|
43
53
|
* Mirrors flow-runner's own `isPlausibleStepDestination` gate on its
|
|
44
54
|
* `urlChanged`/`retryUrlChanged` signals: false when the post-URL landed
|
|
@@ -83,9 +93,12 @@ export declare function splitIntoActionClauses(instruction: string): string[];
|
|
|
83
93
|
* mistaken for a step that is itself about signing in. Fails open (returns
|
|
84
94
|
* true) on an unparseable postUrl, matching {@link hasOriginOrPathChanged}'s
|
|
85
95
|
* own try/catch style, since an unparseable URL gives no basis for a
|
|
86
|
-
* sign-in-shaped veto.
|
|
96
|
+
* sign-in-shaped veto. When `preUrl` is given and the origin/path did not
|
|
97
|
+
* change, the page did not navigate, so no destination veto applies: a
|
|
98
|
+
* mid-flow account-creation step on an unchanged sign-in URL is still judged
|
|
99
|
+
* on its own evidence.
|
|
87
100
|
*/
|
|
88
|
-
export declare function isPlausibleStepDestination(stepInstruction: string, postUrl: string): boolean;
|
|
101
|
+
export declare function isPlausibleStepDestination(stepInstruction: string, postUrl: string, preUrl?: string): boolean;
|
|
89
102
|
export type PhantomClickVerdict =
|
|
90
103
|
/** Stagehand reported success but pre/post shows no observable effect — a no-op click. */
|
|
91
104
|
"phantom"
|
|
@@ -101,6 +114,25 @@ export type PhantomClickVerdict =
|
|
|
101
114
|
* phantom, not effective.
|
|
102
115
|
*/
|
|
103
116
|
export declare const TRIVIAL_DOM_DELTA_BYTES = 500;
|
|
117
|
+
/** Minimum visible-text length change for a mid-flow submit to count as a page advance. */
|
|
118
|
+
export declare const PAGE_ADVANCE_MIN_TEXT_LENGTH_DELTA = 100;
|
|
119
|
+
/** Relative visible-text length change that also counts as an advance on a short page. */
|
|
120
|
+
export declare const PAGE_ADVANCE_MIN_TEXT_LENGTH_RATIO = 0.3;
|
|
121
|
+
/** Floor under the relative rule so a tooltip on a tiny page is not an advance. */
|
|
122
|
+
export declare const PAGE_ADVANCE_MIN_RELATIVE_TEXT_LENGTH_DELTA = 30;
|
|
123
|
+
/** Word-set similarity below which the visible-text head counts as replaced. */
|
|
124
|
+
export declare const PAGE_ADVANCE_MAX_HEAD_SIMILARITY = 0.5;
|
|
125
|
+
/**
|
|
126
|
+
* Decides whether the page demonstrably advanced (a wizard step swapped
|
|
127
|
+
* content) from pre/post snapshots alone, so a network-free, URL-stable
|
|
128
|
+
* transition is still credited. Visible text is the evidence: a wizard swap
|
|
129
|
+
* can leave the HTML size nearly unchanged (same-size markup, different
|
|
130
|
+
* copy), so byte delta is deliberately not required. The text must have
|
|
131
|
+
* moved substantially — a large absolute or relative length change, or a
|
|
132
|
+
* head whose words were mostly replaced — so a tooltip or inline validation
|
|
133
|
+
* line (small, mostly-shared text) is not an advance.
|
|
134
|
+
*/
|
|
135
|
+
export declare function hasPageAdvanced(pre: PhantomClickSnapshot, post: PhantomClickSnapshot): boolean;
|
|
104
136
|
/**
|
|
105
137
|
* Classifies one cascade attempt as `phantom` (Stagehand claimed success but
|
|
106
138
|
* pre/post shows zero network, zero URL change, and only trivial DOM
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"phantom-click.d.ts","sourceRoot":"","sources":["../../src/scraper/phantom-click.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,qFAAqF;AACrF,MAAM,WAAW,oBAAoB;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"phantom-click.d.ts","sourceRoot":"","sources":["../../src/scraper/phantom-click.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,qFAAqF;AACrF,MAAM,WAAW,oBAAoB;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,mFAAmF;IACnF,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,yEAAyE;IACzE,gBAAgB,EAAE,OAAO,GAAG,IAAI,CAAC;IACjC,GAAG,EAAE,oBAAoB,CAAC;IAC1B,IAAI,EAAE,oBAAoB,CAAC;IAC3B;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;;OAMG;IACH,oBAAoB,EAAE,OAAO,CAAC;CAC/B;AAWD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAE/E;AAED,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,UAA0C,CAAC;AAwDxE;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAapE;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAgBT;AAED,MAAM,MAAM,mBAAmB;AAC7B,0FAA0F;AACxF,SAAS;AACX,mEAAmE;GACjE,WAAW;AACb,kKAAkK;GAChK,YAAY,CAAC;AAEjB;;;;;;GAMG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,2FAA2F;AAC3F,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,0FAA0F;AAC1F,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,mFAAmF;AACnF,eAAO,MAAM,2CAA2C,KAAK,CAAC;AAE9D,gFAAgF;AAChF,eAAO,MAAM,gCAAgC,MAAM,CAAC;AAsBpD;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,oBAAoB,EAAE,IAAI,EAAE,oBAAoB,GAAG,OAAO,CAmB9F;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,mBAAmB,GAAG,mBAAmB,CA0BtF"}
|
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
* same way (see recon-submit-phantom-click bug report).
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.TRIVIAL_DOM_DELTA_BYTES = exports.SIGN_IN_PATTERNS = void 0;
|
|
10
|
+
exports.PAGE_ADVANCE_MAX_HEAD_SIMILARITY = exports.PAGE_ADVANCE_MIN_RELATIVE_TEXT_LENGTH_DELTA = exports.PAGE_ADVANCE_MIN_TEXT_LENGTH_RATIO = exports.PAGE_ADVANCE_MIN_TEXT_LENGTH_DELTA = exports.TRIVIAL_DOM_DELTA_BYTES = exports.SIGN_IN_PATTERNS = void 0;
|
|
11
11
|
exports.hasOriginOrPathChanged = hasOriginOrPathChanged;
|
|
12
12
|
exports.splitIntoActionClauses = splitIntoActionClauses;
|
|
13
13
|
exports.isPlausibleStepDestination = isPlausibleStepDestination;
|
|
14
|
+
exports.hasPageAdvanced = hasPageAdvanced;
|
|
14
15
|
exports.classifyPhantomClick = classifyPhantomClick;
|
|
15
16
|
function originAndPathOf(url) {
|
|
16
17
|
try {
|
|
@@ -119,9 +120,16 @@ function splitIntoActionClauses(instruction) {
|
|
|
119
120
|
* mistaken for a step that is itself about signing in. Fails open (returns
|
|
120
121
|
* true) on an unparseable postUrl, matching {@link hasOriginOrPathChanged}'s
|
|
121
122
|
* own try/catch style, since an unparseable URL gives no basis for a
|
|
122
|
-
* sign-in-shaped veto.
|
|
123
|
+
* sign-in-shaped veto. When `preUrl` is given and the origin/path did not
|
|
124
|
+
* change, the page did not navigate, so no destination veto applies: a
|
|
125
|
+
* mid-flow account-creation step on an unchanged sign-in URL is still judged
|
|
126
|
+
* on its own evidence.
|
|
123
127
|
*/
|
|
124
|
-
function isPlausibleStepDestination(stepInstruction, postUrl) {
|
|
128
|
+
function isPlausibleStepDestination(stepInstruction, postUrl, preUrl) {
|
|
129
|
+
// A click that never left its page cannot have bounced to an auth gate; an
|
|
130
|
+
// account-creation step on a sign-in URL is the page's own transition.
|
|
131
|
+
if (preUrl !== undefined && !hasOriginOrPathChanged(preUrl, postUrl))
|
|
132
|
+
return true;
|
|
125
133
|
const pathname = (() => {
|
|
126
134
|
try {
|
|
127
135
|
return new URL(postUrl).pathname;
|
|
@@ -146,6 +154,65 @@ function isPlausibleStepDestination(stepInstruction, postUrl) {
|
|
|
146
154
|
* phantom, not effective.
|
|
147
155
|
*/
|
|
148
156
|
exports.TRIVIAL_DOM_DELTA_BYTES = 500;
|
|
157
|
+
/** Minimum visible-text length change for a mid-flow submit to count as a page advance. */
|
|
158
|
+
exports.PAGE_ADVANCE_MIN_TEXT_LENGTH_DELTA = 100;
|
|
159
|
+
/** Relative visible-text length change that also counts as an advance on a short page. */
|
|
160
|
+
exports.PAGE_ADVANCE_MIN_TEXT_LENGTH_RATIO = 0.3;
|
|
161
|
+
/** Floor under the relative rule so a tooltip on a tiny page is not an advance. */
|
|
162
|
+
exports.PAGE_ADVANCE_MIN_RELATIVE_TEXT_LENGTH_DELTA = 30;
|
|
163
|
+
/** Word-set similarity below which the visible-text head counts as replaced. */
|
|
164
|
+
exports.PAGE_ADVANCE_MAX_HEAD_SIMILARITY = 0.5;
|
|
165
|
+
function textLengthOf(signature) {
|
|
166
|
+
if (signature === undefined)
|
|
167
|
+
return null;
|
|
168
|
+
const match = /^(\d+):/.exec(signature);
|
|
169
|
+
return match ? Number(match[1]) : null;
|
|
170
|
+
}
|
|
171
|
+
function textHeadWords(signature) {
|
|
172
|
+
const head = signature.slice(signature.indexOf(":") + 1).toLowerCase();
|
|
173
|
+
return new Set(head.split(/\W+/).filter((w) => w !== ""));
|
|
174
|
+
}
|
|
175
|
+
function headWordSimilarity(pre, post) {
|
|
176
|
+
const preWords = textHeadWords(pre);
|
|
177
|
+
const postWords = textHeadWords(post);
|
|
178
|
+
const union = new Set([...preWords, ...postWords]);
|
|
179
|
+
if (union.size === 0)
|
|
180
|
+
return 1;
|
|
181
|
+
const shared = [...preWords].filter((w) => postWords.has(w)).length;
|
|
182
|
+
return shared / union.size;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Decides whether the page demonstrably advanced (a wizard step swapped
|
|
186
|
+
* content) from pre/post snapshots alone, so a network-free, URL-stable
|
|
187
|
+
* transition is still credited. Visible text is the evidence: a wizard swap
|
|
188
|
+
* can leave the HTML size nearly unchanged (same-size markup, different
|
|
189
|
+
* copy), so byte delta is deliberately not required. The text must have
|
|
190
|
+
* moved substantially — a large absolute or relative length change, or a
|
|
191
|
+
* head whose words were mostly replaced — so a tooltip or inline validation
|
|
192
|
+
* line (small, mostly-shared text) is not an advance.
|
|
193
|
+
*/
|
|
194
|
+
function hasPageAdvanced(pre, post) {
|
|
195
|
+
const preSig = pre.visibleTextSignature;
|
|
196
|
+
const postSig = post.visibleTextSignature;
|
|
197
|
+
if (preSig === undefined || postSig === undefined || preSig === postSig)
|
|
198
|
+
return false;
|
|
199
|
+
const preLen = textLengthOf(preSig);
|
|
200
|
+
const postLen = textLengthOf(postSig);
|
|
201
|
+
if (preLen === null || postLen === null)
|
|
202
|
+
return false;
|
|
203
|
+
const lengthDelta = Math.abs(postLen - preLen);
|
|
204
|
+
if (lengthDelta >= exports.PAGE_ADVANCE_MIN_TEXT_LENGTH_DELTA)
|
|
205
|
+
return true;
|
|
206
|
+
const similarity = headWordSimilarity(preSig, postSig);
|
|
207
|
+
// Identical head words mean whitespace/layout reflow, not new copy.
|
|
208
|
+
if (similarity === 1)
|
|
209
|
+
return false;
|
|
210
|
+
if (lengthDelta >= exports.PAGE_ADVANCE_MIN_RELATIVE_TEXT_LENGTH_DELTA &&
|
|
211
|
+
lengthDelta / Math.max(preLen, 1) >= exports.PAGE_ADVANCE_MIN_TEXT_LENGTH_RATIO) {
|
|
212
|
+
return true;
|
|
213
|
+
}
|
|
214
|
+
return similarity < exports.PAGE_ADVANCE_MAX_HEAD_SIMILARITY;
|
|
215
|
+
}
|
|
149
216
|
/**
|
|
150
217
|
* Classifies one cascade attempt as `phantom` (Stagehand claimed success but
|
|
151
218
|
* pre/post shows zero network, zero URL change, and only trivial DOM
|
|
@@ -162,12 +229,19 @@ function classifyPhantomClick(attempt) {
|
|
|
162
229
|
if (!attempt.destinationPlausible)
|
|
163
230
|
return "phantom";
|
|
164
231
|
const urlChanged = hasOriginOrPathChanged(attempt.pre.url, attempt.post.url);
|
|
232
|
+
const midFlowSubmit = attempt.isMidFlowSubmitStep === true;
|
|
233
|
+
const submitShaped = attempt.isSubmitShapedStep === true || midFlowSubmit;
|
|
165
234
|
// Element-scoped selection flip; not on a submit-shaped step, which must
|
|
166
235
|
// prove itself via network/URL or the deep-submit escalation is defeated.
|
|
167
|
-
const elementStateChanged = !
|
|
236
|
+
const elementStateChanged = !submitShaped && attempt.elementStateChanged === true;
|
|
168
237
|
// A submit-shaped step likewise ignores a mere DOM-byte reflow.
|
|
169
|
-
const bytesChangedSignificantly = !
|
|
170
|
-
const
|
|
238
|
+
const bytesChangedSignificantly = !submitShaped && Math.abs(bytesDelta) >= exports.TRIVIAL_DOM_DELTA_BYTES;
|
|
239
|
+
const midFlowAdvanced = midFlowSubmit && hasPageAdvanced(attempt.pre, attempt.post);
|
|
240
|
+
const hasEffect = networkDelta !== 0 ||
|
|
241
|
+
urlChanged ||
|
|
242
|
+
elementStateChanged ||
|
|
243
|
+
bytesChangedSignificantly ||
|
|
244
|
+
midFlowAdvanced;
|
|
171
245
|
return hasEffect ? "effective" : "phantom";
|
|
172
246
|
}
|
|
173
247
|
//# sourceMappingURL=phantom-click.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"phantom-click.js","sourceRoot":"","sources":["../../src/scraper/phantom-click.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG
|
|
1
|
+
{"version":3,"file":"phantom-click.js","sourceRoot":"","sources":["../../src/scraper/phantom-click.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;;;;;;AAyDH,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,gCAAuC,MAAc,EAAE,OAAe;IACpE,OAAO,eAAe,CAAC,MAAM,CAAC,KAAK,eAAe,CAAC,OAAO,CAAC,CAAC;AAC9D,CAAC;AAED,uEAAuE;AAC1D,QAAA,gBAAgB,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAC;AAExE,uHAAuH;AACvH,SAAS,wBAAwB,CAAC,WAAmB;IACnD,OAAO,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAC/D,CAAC;AAED;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG;IACxB,OAAO;IACP,KAAK;IACL,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,SAAS;IACT,QAAQ;IACR,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,MAAM;IACN,QAAQ;IACR,UAAU;IACV,SAAS;IACT,OAAO;IACP,QAAQ;IACR,UAAU;CACX,CAAC;AAEF,MAAM,wBAAwB,GAAG,IAAI,MAAM,CAAC,SAAS,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAExF;;;;;;;;GAQG;AACH,MAAM,mBAAmB,GAAG,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAEtF;;;;;;;;;;;;GAYG;AACH,gCAAuC,WAAmB;IACxD,MAAM,UAAU,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;IACzD,MAAM,mBAAmB,GAAG,IAAI,MAAM,CAAC,SAAS,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtF,MAAM,aAAa,GAAG,iBAAiB,CAAC;IACxC,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE,CACtE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,aAAa,GAAG,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,aAAa;SAC1B,KAAK,CAAC,gBAAgB,CAAC;SACvB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;SAC3D,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACxF,OAAO,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,oCACE,eAAuB,EACvB,OAAe,EACf,MAAe;IAEf,2EAA2E;IAC3E,uEAAuE;IACvE,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAClF,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE;QACrB,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IACL,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,cAAc,GAAG,QAAA,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACpF,IAAI,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,aAAa,GAAG,sBAAsB,CAAC,eAAe,CAAC,CAAC;IAC9D,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAA,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAUD;;;;;;GAMG;AACU,QAAA,uBAAuB,GAAG,GAAG,CAAC;AAE3C,2FAA2F;AAC9E,QAAA,kCAAkC,GAAG,GAAG,CAAC;AAEtD,0FAA0F;AAC7E,QAAA,kCAAkC,GAAG,GAAG,CAAC;AAEtD,mFAAmF;AACtE,QAAA,2CAA2C,GAAG,EAAE,CAAC;AAE9D,gFAAgF;AACnE,QAAA,gCAAgC,GAAG,GAAG,CAAC;AAEpD,SAAS,YAAY,CAAC,SAA6B;IACjD,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB;IACtC,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACvE,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,IAAY;IACnD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;IACnD,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACpE,OAAO,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;AAC7B,CAAC;AAED;;;;;;;;;GASG;AACH,yBAAgC,GAAyB,EAAE,IAA0B;IACnF,MAAM,MAAM,GAAG,GAAG,CAAC,oBAAoB,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,oBAAoB,CAAC;IAC1C,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS,IAAI,MAAM,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC;IAC/C,IAAI,WAAW,IAAI,QAAA,kCAAkC;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvD,oEAAoE;IACpE,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,IACE,WAAW,IAAI,QAAA,2CAA2C;QAC1D,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,QAAA,kCAAkC,EACvE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,UAAU,GAAG,QAAA,gCAAgC,CAAC;AACvD,CAAC;AAED;;;;;;;GAOG;AACH,8BAAqC,OAA4B;IAC/D,IAAI,OAAO,CAAC,gBAAgB,KAAK,IAAI;QAAE,OAAO,YAAY,CAAC;IAE3D,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC1E,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC5E,IAAI,CAAC,OAAO,CAAC,oBAAoB;QAAE,OAAO,SAAS,CAAC;IAEpD,MAAM,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7E,MAAM,aAAa,GAAG,OAAO,CAAC,mBAAmB,KAAK,IAAI,CAAC;IAC3D,MAAM,YAAY,GAAG,OAAO,CAAC,kBAAkB,KAAK,IAAI,IAAI,aAAa,CAAC;IAC1E,yEAAyE;IACzE,0EAA0E;IAC1E,MAAM,mBAAmB,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,mBAAmB,KAAK,IAAI,CAAC;IAClF,gEAAgE;IAChE,MAAM,yBAAyB,GAC7B,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,QAAA,uBAAuB,CAAC;IAEnE,MAAM,eAAe,GAAG,aAAa,IAAI,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpF,MAAM,SAAS,GACb,YAAY,KAAK,CAAC;QAClB,UAAU;QACV,mBAAmB;QACnB,yBAAyB;QACzB,eAAe,CAAC;IAClB,OAAO,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export interface QuotedSpan {
|
|
2
|
+
text: string;
|
|
3
|
+
start: number;
|
|
4
|
+
end: number;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Hides apostrophes that belong to a quoted value rather than delimiting it, so
|
|
8
|
+
* a runtime value spliced between quotes is read back in full: contractions and
|
|
9
|
+
* possessives (alphanumeric on both sides) plus a leading or trailing
|
|
10
|
+
* apostrophe of the value itself (`''Smith'`, `'Smith''`). Open/closed state is
|
|
11
|
+
* tracked across the text; the output has the same length as the input.
|
|
12
|
+
*/
|
|
13
|
+
export declare function maskApostrophes(text: string): string;
|
|
14
|
+
/**
|
|
15
|
+
* Reverses `maskApostrophes` on text captured from a masked string.
|
|
16
|
+
*/
|
|
17
|
+
export declare function restoreApostrophes(text: string): string;
|
|
18
|
+
/**
|
|
19
|
+
* Finds real single-quoted spans in instruction copy. `start`/`end` index the
|
|
20
|
+
* ORIGINAL string (delimiters inclusive) and `text` has apostrophes restored.
|
|
21
|
+
*/
|
|
22
|
+
export declare function findQuotedSpans(text: string): QuotedSpan[];
|
|
23
|
+
export interface AnchoredMatch {
|
|
24
|
+
/** Full match followed by capture groups, apostrophes restored; unmatched groups are `undefined`. */
|
|
25
|
+
groups: (string | undefined)[];
|
|
26
|
+
start: number;
|
|
27
|
+
end: number;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Runs an anchored-verb regex over the apostrophe-masked instruction so every
|
|
31
|
+
* parser shares one tokenizer: a letter-flanked apostrophe never ends a quoted
|
|
32
|
+
* span. `start`/`end` index the ORIGINAL string (masking preserves length).
|
|
33
|
+
*/
|
|
34
|
+
export declare function matchAnchored(text: string, pattern: RegExp): AnchoredMatch | null;
|
|
35
|
+
//# sourceMappingURL=quoted-spans.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quoted-spans.d.ts","sourceRoot":"","sources":["../../src/scraper/quoted-spans.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAMD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA0BpD;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,CAM1D;AAED,MAAM,WAAW,aAAa;IAC5B,qGAAqG;IACrG,MAAM,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAWjF"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.maskApostrophes = maskApostrophes;
|
|
4
|
+
exports.restoreApostrophes = restoreApostrophes;
|
|
5
|
+
exports.findQuotedSpans = findQuotedSpans;
|
|
6
|
+
exports.matchAnchored = matchAnchored;
|
|
7
|
+
const APOSTROPHE_PLACEHOLDER = "\u0000";
|
|
8
|
+
const ALPHANUMERIC = /[A-Za-z0-9]/;
|
|
9
|
+
const SINGLE_QUOTED_SPAN = /'([^']+)'/g;
|
|
10
|
+
function isAlphanumeric(char) {
|
|
11
|
+
return char !== undefined && ALPHANUMERIC.test(char);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Hides apostrophes that belong to a quoted value rather than delimiting it, so
|
|
15
|
+
* a runtime value spliced between quotes is read back in full: contractions and
|
|
16
|
+
* possessives (alphanumeric on both sides) plus a leading or trailing
|
|
17
|
+
* apostrophe of the value itself (`''Smith'`, `'Smith''`). Open/closed state is
|
|
18
|
+
* tracked across the text; the output has the same length as the input.
|
|
19
|
+
*/
|
|
20
|
+
function maskApostrophes(text) {
|
|
21
|
+
const chars = [...text];
|
|
22
|
+
let inSpan = false;
|
|
23
|
+
let index = 0;
|
|
24
|
+
while (index < chars.length) {
|
|
25
|
+
if (chars[index] !== "'") {
|
|
26
|
+
index++;
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
let end = index;
|
|
30
|
+
while (chars[end] === "'")
|
|
31
|
+
end++;
|
|
32
|
+
const run = end - index;
|
|
33
|
+
const nextIsValue = isAlphanumeric(chars[end]);
|
|
34
|
+
const interior = run === 1 && isAlphanumeric(chars[index - 1]) && nextIsValue;
|
|
35
|
+
const opens = !inSpan && !interior;
|
|
36
|
+
const closes = !interior && (inSpan ? !(run > 1 && nextIsValue) : run > 1 && !nextIsValue);
|
|
37
|
+
// The opening delimiter is the first quote of the run and the closing one
|
|
38
|
+
// the last; every other quote in the run is value content.
|
|
39
|
+
for (let at = index; at < end; at++) {
|
|
40
|
+
const isDelimiter = (opens && at === index) || (closes && at === end - 1);
|
|
41
|
+
if (!isDelimiter)
|
|
42
|
+
chars[at] = APOSTROPHE_PLACEHOLDER;
|
|
43
|
+
}
|
|
44
|
+
if (opens !== closes)
|
|
45
|
+
inSpan = !inSpan;
|
|
46
|
+
index = end;
|
|
47
|
+
}
|
|
48
|
+
return chars.join("");
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Reverses `maskApostrophes` on text captured from a masked string.
|
|
52
|
+
*/
|
|
53
|
+
function restoreApostrophes(text) {
|
|
54
|
+
return text.replaceAll(APOSTROPHE_PLACEHOLDER, "'");
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Finds real single-quoted spans in instruction copy. `start`/`end` index the
|
|
58
|
+
* ORIGINAL string (delimiters inclusive) and `text` has apostrophes restored.
|
|
59
|
+
*/
|
|
60
|
+
function findQuotedSpans(text) {
|
|
61
|
+
return Array.from(maskApostrophes(text).matchAll(SINGLE_QUOTED_SPAN), (match) => ({
|
|
62
|
+
text: restoreApostrophes(match[1] ?? ""),
|
|
63
|
+
start: match.index ?? 0,
|
|
64
|
+
end: (match.index ?? 0) + match[0].length,
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Runs an anchored-verb regex over the apostrophe-masked instruction so every
|
|
69
|
+
* parser shares one tokenizer: a letter-flanked apostrophe never ends a quoted
|
|
70
|
+
* span. `start`/`end` index the ORIGINAL string (masking preserves length).
|
|
71
|
+
*/
|
|
72
|
+
function matchAnchored(text, pattern) {
|
|
73
|
+
const match = pattern.exec(maskApostrophes(text));
|
|
74
|
+
if (!match)
|
|
75
|
+
return null;
|
|
76
|
+
const start = match.index;
|
|
77
|
+
return {
|
|
78
|
+
groups: Array.from(match, (group) => group === undefined ? undefined : restoreApostrophes(group)),
|
|
79
|
+
start,
|
|
80
|
+
end: start + match[0].length,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=quoted-spans.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quoted-spans.js","sourceRoot":"","sources":["../../src/scraper/quoted-spans.ts"],"names":[],"mappings":";;;;;;AAAA,MAAM,sBAAsB,GAAG,QAAQ,CAAC;AACxC,MAAM,YAAY,GAAG,aAAa,CAAC;AACnC,MAAM,kBAAkB,GAAG,YAAY,CAAC;AAQxC,SAAS,cAAc,CAAC,IAAwB;IAC9C,OAAO,IAAI,KAAK,SAAS,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACvD,CAAC;AAED;;;;;;GAMG;AACH,yBAAgC,IAAY;IAC1C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YACzB,KAAK,EAAE,CAAC;YACR,SAAS;QACX,CAAC;QACD,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG;YAAE,GAAG,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC;QACxB,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC;QAC9E,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC;QACnC,MAAM,MAAM,GAAG,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3F,0EAA0E;QAC1E,2DAA2D;QAC3D,KAAK,IAAI,EAAE,GAAG,KAAK,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC;YAC1E,IAAI,CAAC,WAAW;gBAAE,KAAK,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC;QACvD,CAAC;QACD,IAAI,KAAK,KAAK,MAAM;YAAE,MAAM,GAAG,CAAC,MAAM,CAAC;QACvC,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,4BAAmC,IAAY;IAC7C,OAAO,IAAI,CAAC,UAAU,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,yBAAgC,IAAY;IAC1C,OAAO,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChF,IAAI,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,CAAC;QACvB,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;KAC1C,CAAC,CAAC,CAAC;AACN,CAAC;AASD;;;;GAIG;AACH,uBAA8B,IAAY,EAAE,OAAe;IACzD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IAClD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1B,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAClC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAC5D;QACD,KAAK;QACL,GAAG,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;KAC7B,CAAC;AACJ,CAAC"}
|
|
@@ -71,8 +71,10 @@ export declare function makeOutboundIpAccessor(resolve: () => Promise<string | n
|
|
|
71
71
|
* code path Stagehand validates first.
|
|
72
72
|
*
|
|
73
73
|
* Session lifecycle: Stagehand owns Browserbase session creation when
|
|
74
|
-
* `env: "BROWSERBASE"` is set
|
|
75
|
-
*
|
|
74
|
+
* `env: "BROWSERBASE"` is set, but because the session is created with
|
|
75
|
+
* `keepAlive: true`, `stagehand.close()` does NOT release it. `close()`
|
|
76
|
+
* therefore explicitly issues `sessions.update(..., REQUEST_RELEASE)` through
|
|
77
|
+
* the Browserbase SDK; a failed release is logged and never aborts teardown.
|
|
76
78
|
*
|
|
77
79
|
* Viewport rotation: forwarded via `fingerprint.screen.{min,max}{Width,Height}`
|
|
78
80
|
* pinned to the chosen viewport. The min/max bracket forces Browserbase's
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-browserbase.d.ts","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,0BAA0B,CAAC;AAOlC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAa9C;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,MAAM,GAAG;IAC/D,QAAQ,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC3C,gBAAgB,EAAE,MAAM,IAAI,CAAC;IAC7B,kBAAkB,EAAE,MAAM,MAAM,CAAC;CAClC,CA6BA;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,EACrC,IAAI,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,GACzB,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAS9B;AAmBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,+BAA+B,CACnD,IAAI,CAAC,EAAE,IAAI,CAAC,qBAAqB,EAAE,iBAAiB,GAAG,gCAAgC,CAAC,GACvF,OAAO,CAAC,cAAc,CAAC,CAqOzB"}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
exports.makeFilteredStagehandLogger = makeFilteredStagehandLogger;
|
|
4
7
|
exports.makeOutboundIpAccessor = makeOutboundIpAccessor;
|
|
5
8
|
exports.createBrowserbaseBrowserSession = createBrowserbaseBrowserSession;
|
|
9
|
+
const sdk_1 = __importDefault(require("@browserbasehq/sdk"));
|
|
6
10
|
const stagehand_1 = require("@browserbasehq/stagehand");
|
|
7
11
|
const config_1 = require("../config");
|
|
8
12
|
const bedrock_1 = require("../lib/bedrock");
|
|
@@ -102,6 +106,24 @@ function makeOutboundIpAccessor(resolve, opts) {
|
|
|
102
106
|
return inFlight;
|
|
103
107
|
};
|
|
104
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Releases the Browserbase session explicitly because `keepAlive: true` leaves
|
|
111
|
+
* teardown to us; failures are logged so they never mask the rest of close().
|
|
112
|
+
*/
|
|
113
|
+
async function releaseBrowserbaseSession(sessionId) {
|
|
114
|
+
if (sessionId === "unknown")
|
|
115
|
+
return;
|
|
116
|
+
try {
|
|
117
|
+
const client = new sdk_1.default({ apiKey: config_1.config.scraper.browserbaseApiKey });
|
|
118
|
+
await client.sessions.update(sessionId, {
|
|
119
|
+
projectId: config_1.config.scraper.browserbaseProjectId,
|
|
120
|
+
status: "REQUEST_RELEASE",
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
logger.warn(`browserbase release failed for session ${sessionId}: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
105
127
|
/**
|
|
106
128
|
* Spins up one Browserbase cloud session with the configured Stagehand LLM.
|
|
107
129
|
*
|
|
@@ -113,8 +135,10 @@ function makeOutboundIpAccessor(resolve, opts) {
|
|
|
113
135
|
* code path Stagehand validates first.
|
|
114
136
|
*
|
|
115
137
|
* Session lifecycle: Stagehand owns Browserbase session creation when
|
|
116
|
-
* `env: "BROWSERBASE"` is set
|
|
117
|
-
*
|
|
138
|
+
* `env: "BROWSERBASE"` is set, but because the session is created with
|
|
139
|
+
* `keepAlive: true`, `stagehand.close()` does NOT release it. `close()`
|
|
140
|
+
* therefore explicitly issues `sessions.update(..., REQUEST_RELEASE)` through
|
|
141
|
+
* the Browserbase SDK; a failed release is logged and never aborts teardown.
|
|
118
142
|
*
|
|
119
143
|
* Viewport rotation: forwarded via `fingerprint.screen.{min,max}{Width,Height}`
|
|
120
144
|
* pinned to the chosen viewport. The min/max bracket forces Browserbase's
|
|
@@ -221,7 +245,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
221
245
|
sessionTimeoutHit = classifyLifetime() ?? sessionTimeoutHit;
|
|
222
246
|
return sessionTimeoutHit;
|
|
223
247
|
};
|
|
224
|
-
const { watchLogLine: watchForTeardown, deathSignal, deathAbortSignal, } = (0, session_teardown_1.createSessionTeardownDetector)({ classifyDeath });
|
|
248
|
+
const { watchLogLine: watchForTeardown, signalTransportClosed, deathSignal, deathAbortSignal, } = (0, session_teardown_1.createSessionTeardownDetector)({ classifyDeath });
|
|
225
249
|
const stagehandLoggerCallback = (line) => {
|
|
226
250
|
filteredStagehandLoggerCallback(line);
|
|
227
251
|
watchForTeardown(line);
|
|
@@ -250,6 +274,8 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
250
274
|
...customSessionParams.browserSettings,
|
|
251
275
|
...(advancedStealth ? { advancedStealth: true, solveCaptchas: true } : {}),
|
|
252
276
|
fingerprint,
|
|
277
|
+
logSession: config_1.config.scraper.browserbaseSessionArtifacts,
|
|
278
|
+
recordSession: config_1.config.scraper.browserbaseSessionArtifacts,
|
|
253
279
|
},
|
|
254
280
|
},
|
|
255
281
|
model: config_1.config.scraper.useBedrock
|
|
@@ -261,7 +287,11 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
261
287
|
fetch: (0, session_shared_1.createTimeoutFetch)(config_1.config.scraper.anthropicTimeoutMs),
|
|
262
288
|
},
|
|
263
289
|
llmClient,
|
|
264
|
-
|
|
290
|
+
// Off: the cache keys on instruction text, and instructions can still
|
|
291
|
+
// carry request values the guard leaves literal (keys outside the
|
|
292
|
+
// plugin's tokenizeKeys, replayed observe actions), which a cache entry
|
|
293
|
+
// would persist on Browserbase's side.
|
|
294
|
+
serverCache: false,
|
|
265
295
|
selfHeal: false,
|
|
266
296
|
// Stagehand spawns an out-of-process crash supervisor whenever
|
|
267
297
|
// keepAlive isn't true, watching its own stdin lifeline pipe and
|
|
@@ -299,7 +329,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
299
329
|
completedStepCount,
|
|
300
330
|
toleranceSeconds: TOLERANCE_SECONDS,
|
|
301
331
|
});
|
|
302
|
-
|
|
332
|
+
signalTransportClosed(why);
|
|
303
333
|
logger.warn(classification.message);
|
|
304
334
|
});
|
|
305
335
|
const sessionId = stagehand.browserbaseSessionID ?? "unknown";
|
|
@@ -319,6 +349,7 @@ async function createBrowserbaseBrowserSession(opts) {
|
|
|
319
349
|
catch (err) {
|
|
320
350
|
logger.warn(`stagehand close failed for session ${sessionId}: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
321
351
|
}
|
|
352
|
+
await releaseBrowserbaseSession(sessionId);
|
|
322
353
|
await limiter.stop({ dropWaitingJobs: true });
|
|
323
354
|
reportSuppressedAisdkErrors();
|
|
324
355
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-browserbase.js","sourceRoot":"","sources":["../../src/scraper/session-browserbase.ts"],"names":[],"mappings":";;;;;;;;AAAA,6DAA6C;AAC7C,wDAA2F;AAE3F,qCAAkC;AAClC,2CAAmD;AACnD,yCAA8C;AAC9C,2CAA0C;AAC1C,2DAAqE;AACrE,6CAA2D;AAC3D,qDAAgE;AAChE,6DAKkC;AAClC,iEAIoC;AACpC,iDAA0D;AAG1D,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,CAAC,CAAC;AAElE;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAc7B;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qCAA4C,UAAkB;IAK5D,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAChD,IAAI,IAAI,CAAC,QAAQ,KAAK,aAAa,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,wBAAwB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,eAAe,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;QACH,CAAC;QACD,gEAAgE;QAChE,4DAA4D;QAC5D,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CACb,gCAAgC,eAAe,mFAAmF,CACnI,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,MAAM,kBAAkB,GAAG,GAAW,EAAE,CAAC,eAAe,CAAC;IACzD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,CAAC;AAC5D,CAAC;AAED;;;;;;;;;GASG;AACH,gCACE,OAAqC,EACrC,IAA0B;IAE1B,IAAI,QAA4C,CAAC;IACjD,OAAO,GAA2B,EAAE;QAClC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,yBAAyB,CAAC,SAAiB;IACxD,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,aAAW,CAAC,EAAE,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC7E,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE;YACtC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;YAC9C,MAAM,EAAE,iBAAiB;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,KAAK,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH;;;;;;;;;;;;;;;;;;;;GAoBG;AACI,KAAK,0CACV,IAAwF;IAExF,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAA,mCAAkB,GAAE,CAAC;IACtC,MAAM,mBAAmB,GAAG,eAAM,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC;IACrF,MAAM,YAAY,GAAG,eAAM,CAAC,OAAO,CAAC,YAAY,CAAC;IACjD,MAAM,eAAe,GAAG,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IACvD,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,GAAG,mBAAmB,EAAE,GAC3D,IAAI,EAAE,8BAA8B,IAAI,EAAE,CAAC;IAE7C,IAAI,eAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,uBAAuB,eAAM,CAAC,OAAO,CAAC,KAAK,cAAc,eAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,SAAS,GAAG,eAAM,CAAC,OAAO,CAAC,UAAU;QACzC,CAAC,CAAC,IAAI,uBAAW,CAAC,EAAE,KAAK,EAAE,IAAA,4BAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,CAAC,CAAC,SAAS,CAAC;IAEd,2EAA2E;IAC3E,uEAAuE;IACvE,4BAA4B;IAC5B,MAAM,eAAe,GAAG;QACtB,MAAM,EAAE;YACN,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,QAAQ,EAAE,QAAQ,CAAC,KAAK;YACxB,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM;SAC3B;KACF,CAAC;IACF,MAAM,WAAW,GAAG,eAAe;QACjC,CAAC,CAAC;YACE,GAAG,eAAe;YAClB,OAAO,EAAE,CAAC,SAAkB,CAAC;YAC7B,gBAAgB,EAAE,CAAC,SAAkB,CAAC;SACvC;QACH,CAAC,CAAC,eAAe,CAAC;IAEpB,MAAM,EACJ,QAAQ,EAAE,+BAA+B,EACzC,gBAAgB,EAAE,2BAA2B,EAC7C,kBAAkB,EAAE,qCAAqC,GAC1D,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACxC,wEAAwE;IACxE,sEAAsE;IACtE,0EAA0E;IAC1E,oEAAoE;IACpE,wEAAwE;IACxE,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,uBAA4D,CAAC;IACjE,MAAM,wBAAwB,GAC3B,mBAAmB,CAAC,OAA8B;QACnD,eAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC;IAClD,IAAI,iBAA2F,CAAC;IAChG,uEAAuE;IACvE,sBAAsB;IACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,gBAAgB,GAAG,IAAA,wCAAqB,EAAC;QAC7C,wBAAwB;QACxB,gBAAgB,EAAE,iBAAiB;QACnC,UAAU,EAAE,GAAG,EAAE,CAAC,gBAAgB;QAClC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC,GAAG,IAAI;QAC/D,qBAAqB,EAAE,GAAG,EAAE,CAAC,kBAAkB;KAChD,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,GAA6B,EAAE;QACnD,iBAAiB,GAAG,gBAAgB,EAAE,IAAI,iBAAiB,CAAC;QAC5D,OAAO,iBAAiB,CAAC;IAC3B,CAAC,CAAC;IACF,MAAM,EACJ,YAAY,EAAE,gBAAgB,EAC9B,qBAAqB,EACrB,WAAW,EACX,gBAAgB,GACjB,GAAG,IAAA,gDAA6B,EAAC,EAAE,aAAa,EAAE,CAAC,CAAC;IACrD,MAAM,uBAAuB,GAAG,CAAC,IAAsB,EAAQ,EAAE;QAC/D,+BAA+B,CAAC,IAAI,CAAC,CAAC;QACtC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC,CAAC;IAEF,IAAI,SAAgC,CAAC;IACrC,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,qBAAS,CAAC;YACxB,GAAG,EAAE,aAAa;YAClB,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,iBAAiB;YACxC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;YAC9C,8BAA8B,EAAE;gBAC9B,OAAO,EAAE,eAAM,CAAC,OAAO,CAAC,gCAAgC;gBACxD,GAAG,mBAAmB;gBACtB,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,oBAAoB;gBAC9C,OAAO,EAAE,YAAY;oBACnB,CAAC,CAAC;wBACE;4BACE,IAAI,EAAE,UAAmB;4BACzB,MAAM,EAAE,GAAG,YAAY,CAAC,QAAQ,MAAM,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE;4BAC9E,QAAQ,EAAE,YAAY,CAAC,QAAQ;4BAC/B,QAAQ,EAAE,YAAY,CAAC,QAAQ;yBAChC;qBACF;oBACH,CAAC,CAAC,mBAAmB;gBACvB,eAAe,EAAE;oBACf,GAAG,mBAAmB,CAAC,eAAe;oBACtC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1E,WAAW;oBACX,UAAU,EAAE,eAAM,CAAC,OAAO,CAAC,2BAA2B;oBACtD,aAAa,EAAE,eAAM,CAAC,OAAO,CAAC,2BAA2B;iBAC1D;aACF;YACD,KAAK,EAAE,eAAM,CAAC,OAAO,CAAC,UAAU;gBAC9B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAE;oBACC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,KAAK;oBAC/B,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,eAAe;oBACtC,gFAAgF;oBAChF,KAAK,EAAE,IAAA,mCAAkB,EAAC,eAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;iBACtC;YAC5B,SAAS;YACT,sEAAsE;YACtE,kEAAkE;YAClE,wEAAwE;YACxE,uCAAuC;YACvC,WAAW,EAAE,KAAK;YAClB,QAAQ,EAAE,KAAK;YACf,+DAA+D;YAC/D,iEAAiE;YACjE,sEAAsE;YACtE,oEAAoE;YACpE,mEAAmE;YACnE,6DAA6D;YAC7D,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,CAAC;YACV,wDAAwD;YACxD,mDAAmD;YACnD,MAAM,EAAE,uBAAuB;SAChC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,IAAI,SAAS;gBAAE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QACzC,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAA,uBAAc,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAG,IAAA,0CAA0B,EAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAEjF,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,GAAW,EAAE,EAAE;QACvD,IAAI,gBAAgB;YAAE,OAAO;QAC7B,uBAAuB,GAAG,IAAI,gCAAuB,CACnD,0DAA0D,GAAG,EAAE,CAChE,CAAC;QACF,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC,GAAG,IAAI,CAAC;QAC9D,MAAM,cAAc,GAAG,IAAA,0CAAuB,EAAC;YAC7C,wBAAwB;YACxB,cAAc;YACd,kBAAkB;YAClB,gBAAgB,EAAE,iBAAiB;SACpC,CAAC,CAAC;QACH,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,SAAS,CAAC,oBAAoB,IAAI,SAAS,CAAC;IAC9D,MAAM,CAAC,IAAI,CACT,+BAA+B,SAAS,aAAa,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,YAAY,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,mBAAmB,oBAAoB,eAAe,EAAE,CACvL,CAAC;IAEF,MAAM,OAAO,GAAG,IAAA,+BAAoB,GAAE,CAAC;IAEvC,MAAM,aAAa,GAAG,sBAAsB,CAC1C,GAAG,EAAE,CACH,IAAA,qCAAwB,EAAC,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE;QAC1D,OAAO,EAAE,eAAM,CAAC,OAAO,CAAC,gBAAgB;QACxC,SAAS,EAAE,eAAM,CAAC,OAAO,CAAC,kBAAkB;QAC5C,MAAM;KACP,CAAC,EACJ,EAAE,OAAO,EAAE,eAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAC7C,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,SAAS,CAAC,IAAI,EAAE,CAAC;QACjB,gBAAgB,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,sCAAsC,SAAS,KAAK,IAAA,uBAAc,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,MAAM,yBAAyB,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,2BAA2B,EAAE,CAAC;IAChC,CAAC,CAAC;IAEF,MAAM,0BAA0B,GAAG,GAAwC,EAAE,CAC3E,uBAAuB,CAAC;IAE1B,MAAM,oBAAoB,GAAG,GAEf,EAAE,CAAC,iBAAiB,CAAC;IAEnC,MAAM,mBAAmB,GAAG,GAAS,EAAE;QACrC,kBAAkB,IAAI,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,OAAO;QACL,SAAS;QACT,OAAO;QACP,SAAS;QACT,QAAQ,EAAE,aAAa;QACvB,mBAAmB;QACnB,KAAK;QACL,qCAAqC;QACrC,aAAa;QACb,WAAW;QACX,gBAAgB;QAChB,0BAA0B;QAC1B,oBAAoB;QACpB,YAAY;KACb,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-steel.d.ts","sourceRoot":"","sources":["../../src/scraper/session-steel.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EACL,KAAK,cAAc,EAGpB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"session-steel.d.ts","sourceRoot":"","sources":["../../src/scraper/session-steel.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EACL,KAAK,cAAc,EAGpB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAe9C;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,MAAM,GAAG,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAUjG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,yBAAyB,IAAI,OAAO,CAAC,cAAc,CAAC,CAkKzE"}
|
|
@@ -17,6 +17,8 @@ const session_shared_1 = require("./session-shared");
|
|
|
17
17
|
const session_teardown_1 = require("./session-teardown");
|
|
18
18
|
const throttle_1 = require("./throttle");
|
|
19
19
|
const logger = (0, logging_1.getLogger)({ name: "scraper/session-steel" });
|
|
20
|
+
/** Mirrors Browserbase: providers overshoot their configured lifetime, never undershoot. */
|
|
21
|
+
const STEEL_TIMEOUT_TOLERANCE_SECONDS = 15;
|
|
20
22
|
/** Formats a session proxy tuple as Steel's documented `proxyUrl`: `protocol://[user:pass@]host:port`. */
|
|
21
23
|
function formatSteelProxyUrl(proxy) {
|
|
22
24
|
const credentials = proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : "";
|
|
@@ -97,7 +99,9 @@ async function createSteelBrowserSession() {
|
|
|
97
99
|
// failure leaves a live session burning minutes until Steel's own timeout.
|
|
98
100
|
let stagehand;
|
|
99
101
|
let deathSignal;
|
|
102
|
+
let deathAbortSignal;
|
|
100
103
|
let heartbeat;
|
|
104
|
+
let weInitiatedClose = false;
|
|
101
105
|
try {
|
|
102
106
|
// Steel's websocketUrl is `wss://connect.steel.dev?sessionId=…`. Stagehand's
|
|
103
107
|
// V3 CDP connector requires the apiKey as a query parameter too — without it
|
|
@@ -113,9 +117,19 @@ async function createSteelBrowserSession() {
|
|
|
113
117
|
? new stagehand_1.AISdkClient({ model: (0, bedrock_1.createBedrockModel)(config_1.config.bedrock) })
|
|
114
118
|
: undefined;
|
|
115
119
|
const forwardStagehandLogLine = makeStagehandLoggerForwarder(logger);
|
|
116
|
-
const
|
|
120
|
+
const steelStartedAt = Date.now();
|
|
121
|
+
const teardownDetector = (0, session_teardown_1.createSessionTeardownDetector)({
|
|
122
|
+
classifyDeath: (0, session_teardown_1.createDeathClassifier)({
|
|
123
|
+
configuredTimeoutSeconds: config_1.config.scraper.steelSessionTimeoutMs / 1000,
|
|
124
|
+
toleranceSeconds: STEEL_TIMEOUT_TOLERANCE_SECONDS,
|
|
125
|
+
isOwnClose: () => weInitiatedClose,
|
|
126
|
+
getElapsedSeconds: () => (Date.now() - steelStartedAt) / 1000,
|
|
127
|
+
getCompletedStepCount: () => 0,
|
|
128
|
+
}),
|
|
129
|
+
});
|
|
117
130
|
const watchForTeardown = teardownDetector.watchLogLine;
|
|
118
131
|
deathSignal = teardownDetector.deathSignal;
|
|
132
|
+
deathAbortSignal = teardownDetector.deathAbortSignal;
|
|
119
133
|
const stagehandLoggerCallback = (line) => {
|
|
120
134
|
forwardStagehandLogLine(line);
|
|
121
135
|
watchForTeardown(line);
|
|
@@ -171,7 +185,6 @@ async function createSteelBrowserSession() {
|
|
|
171
185
|
// flipped at the top of `close()` below, before `stagehand.close()` runs;
|
|
172
186
|
// only a transport close observed while it's still false is an SDK-
|
|
173
187
|
// initiated teardown mid-operation, not a consequence of our own close.
|
|
174
|
-
let weInitiatedClose = false;
|
|
175
188
|
let cdpTransportClosedError;
|
|
176
189
|
stagehand.context.conn.onTransportClosed((why) => {
|
|
177
190
|
if (weInitiatedClose)
|
|
@@ -204,6 +217,7 @@ async function createSteelBrowserSession() {
|
|
|
204
217
|
provider: "steel",
|
|
205
218
|
close,
|
|
206
219
|
deathSignal,
|
|
220
|
+
deathAbortSignal,
|
|
207
221
|
getCdpTransportClosedError,
|
|
208
222
|
sessionProxy,
|
|
209
223
|
};
|