@ultimat3/scraping 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +194 -0
- package/package.json +38 -0
- package/src/actionability.ts +106 -0
- package/src/artifacts.ts +69 -0
- package/src/auth.ts +200 -0
- package/src/cdp-fake.ts +150 -0
- package/src/cdp-port.ts +75 -0
- package/src/cdp-snapshot.ts +63 -0
- package/src/cdp-target.ts +320 -0
- package/src/clock.ts +97 -0
- package/src/cookie-scope.ts +97 -0
- package/src/driver-cdp.ts +184 -0
- package/src/driver-fake.ts +119 -0
- package/src/driver-fixture.ts +65 -0
- package/src/driver.ts +88 -0
- package/src/error-throws.ts +258 -0
- package/src/errors.ts +180 -0
- package/src/events.ts +74 -0
- package/src/expect.ts +133 -0
- package/src/failures.ts +47 -0
- package/src/hosts.ts +56 -0
- package/src/html-query.ts +119 -0
- package/src/html-requests.ts +45 -0
- package/src/html-target.ts +229 -0
- package/src/http-recorded.ts +85 -0
- package/src/http.ts +158 -0
- package/src/index.ts +178 -0
- package/src/intercept.ts +41 -0
- package/src/offline-session.ts +67 -0
- package/src/page-over-target.ts +215 -0
- package/src/page.ts +103 -0
- package/src/rate.ts +23 -0
- package/src/recording.ts +68 -0
- package/src/recover.ts +51 -0
- package/src/rings.ts +73 -0
- package/src/robots.ts +144 -0
- package/src/scrape-run.ts +226 -0
- package/src/scrape.ts +151 -0
- package/src/secrets.ts +91 -0
- package/src/session-state.ts +181 -0
- package/src/target.ts +118 -0
- package/src/watchdog.ts +100 -0
package/src/cdp-fake.ts
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// A CDP-shaped browser with no browser behind it — the seam `driver-parity.test.ts` needs to run
|
|
2
|
+
// the SAME suite against the real driver's code path without Chrome.
|
|
3
|
+
//
|
|
4
|
+
// It exists for one reason, and it is the reason `mock.module('puppeteer-core')` is banned in this
|
|
5
|
+
// package: `mock.module` replaces a module for the whole run, `bun test` does not fully serialise
|
|
6
|
+
// test files, and a mock installed here has been observed leaking into a concurrently-running
|
|
7
|
+
// file's assertions. An INJECTED launcher is a value; a value cannot leak.
|
|
8
|
+
//
|
|
9
|
+
// Precedent: `packages/storage/src/driver-s3-fixture.ts` ships the same way.
|
|
10
|
+
|
|
11
|
+
import type { CdpBrowserLike, CdpFrameLike, CdpLauncherLike, CdpPageLike } from './cdp-port';
|
|
12
|
+
import { queryHtml } from './html-query';
|
|
13
|
+
import type { ElementSnapshot, ScrapeCookie } from './target';
|
|
14
|
+
|
|
15
|
+
/** The selector inside `snapshotExpression()`'s `document.querySelectorAll("…")`. */
|
|
16
|
+
const selectorOf = (expression: string): string | undefined => {
|
|
17
|
+
const match = /querySelectorAll\((".*?")\)/s.exec(expression);
|
|
18
|
+
if (match?.[1] === undefined) return undefined;
|
|
19
|
+
return JSON.parse(match[1]) as string;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export interface FakeCdpPageInit {
|
|
23
|
+
readonly url: string;
|
|
24
|
+
readonly html: string;
|
|
25
|
+
/** Selectors a click navigates from, and where to. */
|
|
26
|
+
readonly routes?: Readonly<Record<string, { readonly url: string; readonly html: string }>>;
|
|
27
|
+
readonly cookies?: readonly ScrapeCookie[];
|
|
28
|
+
readonly storage?: Readonly<Record<string, string>>;
|
|
29
|
+
readonly userAgent?: string;
|
|
30
|
+
/** Selectors whose element is covered at its centre — what only a layout engine can see. */
|
|
31
|
+
readonly covered?: readonly string[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type Handlers = Map<string, ((payload: unknown) => void)[]>;
|
|
35
|
+
|
|
36
|
+
export interface FakeCdpBrowser extends CdpBrowserLike {
|
|
37
|
+
/** Fire a request event, as a real browser would when the page fetches a subresource. */
|
|
38
|
+
emitRequest(url: string, resourceType: string): void;
|
|
39
|
+
readonly aborted: readonly string[];
|
|
40
|
+
readonly closed: boolean;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** A layout box every element gets, so the CDP path exercises the fields the fake target lacks. */
|
|
44
|
+
const boxed = (snapshot: ElementSnapshot, covered: boolean): ElementSnapshot => ({
|
|
45
|
+
...snapshot,
|
|
46
|
+
box: { x: 10, y: 10, width: 100, height: 20 },
|
|
47
|
+
hitTarget: !covered,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
|
|
51
|
+
const handlers: Handlers = new Map();
|
|
52
|
+
const aborted: string[] = [];
|
|
53
|
+
let url = init.url;
|
|
54
|
+
let html = init.html;
|
|
55
|
+
let closed = false;
|
|
56
|
+
const covered = new Set(init.covered ?? []);
|
|
57
|
+
const storage: Record<string, string> = { ...init.storage };
|
|
58
|
+
let cookies: readonly ScrapeCookie[] = init.cookies ?? [];
|
|
59
|
+
|
|
60
|
+
const evaluate = async (expression: string): Promise<unknown> => {
|
|
61
|
+
const selector = selectorOf(expression);
|
|
62
|
+
if (selector !== undefined) {
|
|
63
|
+
const found = await queryHtml(html, selector);
|
|
64
|
+
return JSON.stringify(found.map((element) => boxed(element, covered.has(selector))));
|
|
65
|
+
}
|
|
66
|
+
if (expression.includes('localStorage')) {
|
|
67
|
+
if (expression.includes('setItem')) return undefined;
|
|
68
|
+
return JSON.stringify(storage);
|
|
69
|
+
}
|
|
70
|
+
if (expression.includes('navigator.userAgent')) return init.userAgent ?? 'fake-agent';
|
|
71
|
+
return undefined;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
const page: CdpPageLike = {
|
|
75
|
+
url: () => url,
|
|
76
|
+
goto: (next: string) => {
|
|
77
|
+
url = next;
|
|
78
|
+
return Promise.resolve(undefined);
|
|
79
|
+
},
|
|
80
|
+
content: () => Promise.resolve(html),
|
|
81
|
+
evaluate,
|
|
82
|
+
click: (selector: string) => {
|
|
83
|
+
const route = init.routes?.[selector];
|
|
84
|
+
if (route !== undefined) {
|
|
85
|
+
url = route.url;
|
|
86
|
+
html = route.html;
|
|
87
|
+
}
|
|
88
|
+
return Promise.resolve();
|
|
89
|
+
},
|
|
90
|
+
type: () => Promise.resolve(),
|
|
91
|
+
select: () => Promise.resolve([]),
|
|
92
|
+
screenshot: () => Promise.resolve(new Uint8Array([1, 2, 3])),
|
|
93
|
+
pdf: () => Promise.resolve(new Uint8Array([4, 5])),
|
|
94
|
+
setRequestInterception: () => Promise.resolve(),
|
|
95
|
+
on: (event: string, handler: (payload: unknown) => void) => {
|
|
96
|
+
const listeners = handlers.get(event) ?? [];
|
|
97
|
+
listeners.push(handler);
|
|
98
|
+
handlers.set(event, listeners);
|
|
99
|
+
return undefined;
|
|
100
|
+
},
|
|
101
|
+
frames: (): readonly CdpFrameLike[] => [],
|
|
102
|
+
close: () => {
|
|
103
|
+
closed = true;
|
|
104
|
+
return Promise.resolve();
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
newPage: () => Promise.resolve(page),
|
|
110
|
+
cookies: () => Promise.resolve(cookies),
|
|
111
|
+
setCookie: (...next: readonly unknown[]) => {
|
|
112
|
+
cookies = next as readonly ScrapeCookie[];
|
|
113
|
+
return Promise.resolve();
|
|
114
|
+
},
|
|
115
|
+
close: () => {
|
|
116
|
+
closed = true;
|
|
117
|
+
return Promise.resolve();
|
|
118
|
+
},
|
|
119
|
+
process: () => null,
|
|
120
|
+
emitRequest(requestUrl: string, resourceType: string): void {
|
|
121
|
+
for (const handler of handlers.get('request') ?? []) {
|
|
122
|
+
handler({
|
|
123
|
+
url: () => requestUrl,
|
|
124
|
+
resourceType: () => resourceType,
|
|
125
|
+
abort: () => {
|
|
126
|
+
aborted.push(requestUrl);
|
|
127
|
+
return Promise.resolve();
|
|
128
|
+
},
|
|
129
|
+
continue: () => Promise.resolve(),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
aborted,
|
|
134
|
+
get closed(): boolean {
|
|
135
|
+
return closed;
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** A launcher over `fakeCdpBrowser`, for `localBrowser({ launcher })` in a test. */
|
|
141
|
+
export function fakeCdpLauncher(init: FakeCdpPageInit): CdpLauncherLike & {
|
|
142
|
+
readonly browser: FakeCdpBrowser;
|
|
143
|
+
} {
|
|
144
|
+
const browser = fakeCdpBrowser(init);
|
|
145
|
+
return {
|
|
146
|
+
browser,
|
|
147
|
+
launch: () => Promise.resolve(browser),
|
|
148
|
+
connect: () => Promise.resolve(browser),
|
|
149
|
+
};
|
|
150
|
+
}
|
package/src/cdp-port.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// The browser library's shape, declared STRUCTURALLY and imported from nowhere.
|
|
2
|
+
//
|
|
3
|
+
// `puppeteer-core` is the intended implementation and `docs/idea/18-build-vs-wrap.md` permits it
|
|
4
|
+
// at exactly this seam — a driver/transport boundary, never the vocabulary. Declaring the port
|
|
5
|
+
// instead of importing the types is what keeps that promise mechanical rather than aspirational:
|
|
6
|
+
// nothing in `@ultimat3/scraping` can name a `Page`, an `ElementHandle` or a `CDPSession`, so a
|
|
7
|
+
// puppeteer type cannot reach `ScrapePage` even by accident, and this package takes no runtime
|
|
8
|
+
// dependency at all. The app passes its own `puppeteer` in — the same way `s3Driver({ client })`
|
|
9
|
+
// takes a `S3ClientLike` rather than importing a cloud SDK.
|
|
10
|
+
//
|
|
11
|
+
// Verified against puppeteer-core 25.8.0 on Bun 1.3.14 (both `launch` and
|
|
12
|
+
// `connect({ browserWSEndpoint })`, headless Chrome 150): the WebSocket upgrade Playwright's
|
|
13
|
+
// `connectOverCDP` cannot do under Bun works here, which is why this is the intended library.
|
|
14
|
+
|
|
15
|
+
export interface CdpRequestLike {
|
|
16
|
+
url(): string;
|
|
17
|
+
resourceType(): string;
|
|
18
|
+
/**
|
|
19
|
+
* OPTIONAL, and read defensively: this is the shape of somebody else's event payload, so a
|
|
20
|
+
* launcher that predates the method (or a provider SDK that never had it) still satisfies the
|
|
21
|
+
* port and its requests are recorded as `GET` rather than crashing the interception handler.
|
|
22
|
+
*/
|
|
23
|
+
method?(): string;
|
|
24
|
+
abort(): Promise<void>;
|
|
25
|
+
continue(): Promise<void>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface CdpPageLike {
|
|
29
|
+
url(): string;
|
|
30
|
+
goto(url: string, options?: { readonly timeout?: number }): Promise<unknown>;
|
|
31
|
+
content(): Promise<string>;
|
|
32
|
+
/** The string form. Every read this package makes is an expression, never a closure. */
|
|
33
|
+
evaluate(expression: string): Promise<unknown>;
|
|
34
|
+
click(selector: string): Promise<void>;
|
|
35
|
+
type(selector: string, text: string): Promise<void>;
|
|
36
|
+
select(selector: string, ...values: string[]): Promise<string[]>;
|
|
37
|
+
screenshot(options: { readonly fullPage?: boolean }): Promise<Uint8Array | string>;
|
|
38
|
+
pdf(options?: Record<string, unknown>): Promise<Uint8Array>;
|
|
39
|
+
setRequestInterception(enabled: boolean): Promise<void>;
|
|
40
|
+
on(event: string, handler: (payload: unknown) => void): unknown;
|
|
41
|
+
frames(): readonly CdpFrameLike[];
|
|
42
|
+
close(): Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface CdpFrameLike {
|
|
46
|
+
name(): string;
|
|
47
|
+
url(): string;
|
|
48
|
+
content(): Promise<string>;
|
|
49
|
+
evaluate(expression: string): Promise<unknown>;
|
|
50
|
+
click(selector: string): Promise<void>;
|
|
51
|
+
type(selector: string, text: string): Promise<void>;
|
|
52
|
+
select(selector: string, ...values: string[]): Promise<string[]>;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface CdpBrowserLike {
|
|
56
|
+
newPage(): Promise<CdpPageLike>;
|
|
57
|
+
/** Present on a browser that owns a cookie jar. Absent ones answer `X_NOT_IMPLEMENTED`. */
|
|
58
|
+
cookies?(): Promise<unknown>;
|
|
59
|
+
setCookie?(...cookies: readonly unknown[]): Promise<void>;
|
|
60
|
+
/** Ends the browser. For an attached session this is what stops the REMOTE half. */
|
|
61
|
+
close(): Promise<void>;
|
|
62
|
+
/** Drops the local connection and leaves the remote browser running. */
|
|
63
|
+
disconnect?(): Promise<void>;
|
|
64
|
+
process?(): { readonly pid?: number | undefined; kill(signal?: number | string): void } | null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* What an app passes in: `puppeteer` itself. Both methods are optional so a launcher that can only
|
|
69
|
+
* attach (a provider SDK) still satisfies the port — `remoteBrowser()` refuses one with no
|
|
70
|
+
* `connect` at the call site, with a code, rather than at a property access.
|
|
71
|
+
*/
|
|
72
|
+
export interface CdpLauncherLike {
|
|
73
|
+
launch?(options: Record<string, unknown>): Promise<CdpBrowserLike>;
|
|
74
|
+
connect?(options: Record<string, unknown>): Promise<CdpBrowserLike>;
|
|
75
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// The one in-page expression this package runs, and the schema that reads its answer back.
|
|
2
|
+
//
|
|
3
|
+
// It computes exactly `ElementSnapshot` — including the two fields the offline drivers cannot
|
|
4
|
+
// have: the layout box, and whether the element is what a click at its own centre would hit. A
|
|
5
|
+
// cookie banner over the submit button is invisible to every DOM-only check ever written, and
|
|
6
|
+
// visible here.
|
|
7
|
+
|
|
8
|
+
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
9
|
+
import { parse, t } from '@ultimat3/schema';
|
|
10
|
+
import type { ElementSnapshot } from './target';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Returns JSON TEXT, not an object: a CDP round trip serialises the result anyway, and a string
|
|
14
|
+
* has one deserialiser here — `parse` against the schema below — instead of an implicit one in
|
|
15
|
+
* the library plus a cast.
|
|
16
|
+
*/
|
|
17
|
+
export const snapshotExpression = (selector: string): string => `(() => {
|
|
18
|
+
const out = [];
|
|
19
|
+
for (const el of document.querySelectorAll(${JSON.stringify(selector)})) {
|
|
20
|
+
const style = getComputedStyle(el);
|
|
21
|
+
const rect = el.getBoundingClientRect();
|
|
22
|
+
const attrs = {};
|
|
23
|
+
for (const attribute of el.attributes) attrs[attribute.name] = attribute.value;
|
|
24
|
+
const top = document.elementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
|
|
25
|
+
out.push({
|
|
26
|
+
tag: el.tagName.toLowerCase(),
|
|
27
|
+
attrs,
|
|
28
|
+
text: (el.textContent || '').replace(/\\s+/g, ' ').trim(),
|
|
29
|
+
value: typeof el.value === 'string' ? el.value : '',
|
|
30
|
+
visible: style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0',
|
|
31
|
+
enabled: el.disabled !== true && el.getAttribute('aria-disabled') !== 'true',
|
|
32
|
+
box: { x: rect.left, y: rect.top, width: rect.width, height: rect.height },
|
|
33
|
+
hitTarget: top !== null && (top === el || el.contains(top) || el.contains(top.parentNode)),
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
return JSON.stringify(out);
|
|
37
|
+
})()`;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `t.string` refuses an empty string, and an element's `text` and `value` are legitimately empty
|
|
41
|
+
* far more often than not — a `<div>` wrapper, an unfilled input. `.min(0)` says "a string, and
|
|
42
|
+
* empty is a real answer", which is the difference between parsing a page and refusing one.
|
|
43
|
+
*/
|
|
44
|
+
const anyString = t.string.min(0);
|
|
45
|
+
|
|
46
|
+
const snapshotSchema = t.array(
|
|
47
|
+
t.object({
|
|
48
|
+
tag: t.string,
|
|
49
|
+
attrs: t.record(anyString),
|
|
50
|
+
text: anyString,
|
|
51
|
+
value: anyString,
|
|
52
|
+
visible: t.boolean,
|
|
53
|
+
enabled: t.boolean,
|
|
54
|
+
box: t.object({ x: t.number, y: t.number, width: t.number, height: t.number }),
|
|
55
|
+
hitTarget: t.boolean,
|
|
56
|
+
}),
|
|
57
|
+
) as unknown as StandardSchemaV1<unknown, ElementSnapshot[]>;
|
|
58
|
+
|
|
59
|
+
/** The browser's answer is `unknown` and stays `unknown` until this parses it. Never a cast. */
|
|
60
|
+
export function parseSnapshots(raw: unknown): readonly ElementSnapshot[] {
|
|
61
|
+
if (typeof raw !== 'string') return parse(snapshotSchema, raw);
|
|
62
|
+
return parse(snapshotSchema, JSON.parse(raw) as unknown);
|
|
63
|
+
}
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
// `ScrapeTarget` over a real browser, through the structural CDP port. Everything driver-specific
|
|
2
|
+
// in this package lives here and in `driver-cdp.ts`; the vocabulary above it does not change.
|
|
3
|
+
|
|
4
|
+
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
5
|
+
import { parse, t } from '@ultimat3/schema';
|
|
6
|
+
import type { CdpBrowserLike, CdpFrameLike, CdpPageLike, CdpRequestLike } from './cdp-port';
|
|
7
|
+
import { parseSnapshots, snapshotExpression } from './cdp-snapshot';
|
|
8
|
+
import type { ScrapeClock } from './clock';
|
|
9
|
+
import { browserUnreachable, pageCrashed, scrapeNotImplemented } from './error-throws';
|
|
10
|
+
import type { InterceptRules } from './intercept';
|
|
11
|
+
import { interceptVerdict, refusalEntry } from './intercept';
|
|
12
|
+
import type { ConsoleLine, NetworkEntry, ResourceType } from './rings';
|
|
13
|
+
import { createRing, RESOURCE_TYPES } from './rings';
|
|
14
|
+
import type { SessionSnapshot } from './session-state';
|
|
15
|
+
import type {
|
|
16
|
+
CaptureOptions,
|
|
17
|
+
FrameRef,
|
|
18
|
+
GotoOptions,
|
|
19
|
+
ScrapeCookie,
|
|
20
|
+
ScrapeDownloadFile,
|
|
21
|
+
ScrapeTarget,
|
|
22
|
+
} from './target';
|
|
23
|
+
|
|
24
|
+
export const CDP_DRIVER = 'puppeteer';
|
|
25
|
+
|
|
26
|
+
// `.min(0)` for the same reason `cdp-snapshot.ts` needs it: a cookie's value is legitimately the
|
|
27
|
+
// empty string (a cleared session cookie), and refusing one would refuse the whole jar.
|
|
28
|
+
const anyString = t.string.min(0);
|
|
29
|
+
|
|
30
|
+
const cookieSchema = t.array(
|
|
31
|
+
t.object({
|
|
32
|
+
name: t.string,
|
|
33
|
+
value: anyString,
|
|
34
|
+
domain: anyString,
|
|
35
|
+
path: anyString,
|
|
36
|
+
httpOnly: t.boolean,
|
|
37
|
+
secure: t.boolean,
|
|
38
|
+
}),
|
|
39
|
+
) as unknown as StandardSchemaV1<unknown, ScrapeCookie[]>;
|
|
40
|
+
|
|
41
|
+
const storageSchema = t.record(anyString) as unknown as StandardSchemaV1<
|
|
42
|
+
unknown,
|
|
43
|
+
Record<string, string>
|
|
44
|
+
>;
|
|
45
|
+
|
|
46
|
+
const asResourceType = (raw: string): ResourceType =>
|
|
47
|
+
(RESOURCE_TYPES as readonly string[]).includes(raw) ? (raw as ResourceType) : 'other';
|
|
48
|
+
|
|
49
|
+
/** The library's event payloads are `unknown` here — read structurally, never cast. */
|
|
50
|
+
const asRequest = (payload: unknown): CdpRequestLike | undefined => {
|
|
51
|
+
if (typeof payload !== 'object' || payload === null) return undefined;
|
|
52
|
+
const candidate = payload as Partial<CdpRequestLike>;
|
|
53
|
+
return typeof candidate.url === 'function' && typeof candidate.abort === 'function'
|
|
54
|
+
? (candidate as CdpRequestLike)
|
|
55
|
+
: undefined;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* CDP's console levels, mapped onto this package's five. `warning` is the library's spelling of
|
|
60
|
+
* `warn`, `verbose` of `debug`, and everything structural (`table`, `startGroup`, `dir`) is a log
|
|
61
|
+
* line with a shape — never its own level, because `ConsoleLine.level` is what an author filters on.
|
|
62
|
+
*/
|
|
63
|
+
const CONSOLE_LEVELS: Readonly<Record<string, ConsoleLine['level']>> = {
|
|
64
|
+
error: 'error',
|
|
65
|
+
assert: 'error',
|
|
66
|
+
warning: 'warn',
|
|
67
|
+
warn: 'warn',
|
|
68
|
+
info: 'info',
|
|
69
|
+
debug: 'debug',
|
|
70
|
+
verbose: 'debug',
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Reads a string out of somebody else's event payload, calling an accessor THROUGH ITS OWNER.
|
|
75
|
+
*
|
|
76
|
+
* `HTTPRequest.method()` and `ConsoleMessage.type()`/`.text()` read `this` — they are methods on
|
|
77
|
+
* the library's own objects, not closures over a value. Handing the bare function to a helper
|
|
78
|
+
* (`readString(request.method)`) drops the receiver, so the accessor answers against `undefined`:
|
|
79
|
+
* on one build that throws inside the interception handler, on another it answers wrong.
|
|
80
|
+
*/
|
|
81
|
+
const readStringFrom = (owner: unknown, key: string): string | undefined => {
|
|
82
|
+
if (typeof owner !== 'object' || owner === null) return undefined;
|
|
83
|
+
const value = (owner as Record<string, unknown>)[key];
|
|
84
|
+
if (typeof value === 'string') return value;
|
|
85
|
+
if (typeof value !== 'function') return undefined;
|
|
86
|
+
const answer = (value as (this: unknown) => unknown).call(owner);
|
|
87
|
+
return typeof answer === 'string' ? answer : undefined;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
export interface CdpTargetInit {
|
|
91
|
+
readonly page: CdpPageLike;
|
|
92
|
+
readonly browser: CdpBrowserLike;
|
|
93
|
+
readonly rules: InterceptRules;
|
|
94
|
+
readonly clock: ScrapeClock;
|
|
95
|
+
readonly ringCapacity?: number | undefined;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Interception is armed BEFORE the first navigation and refuses at the request, not after the
|
|
100
|
+
* response — an `allowHosts` that reported afterwards would be a log line about bytes that
|
|
101
|
+
* already left the container.
|
|
102
|
+
*/
|
|
103
|
+
async function arm(
|
|
104
|
+
init: CdpTargetInit,
|
|
105
|
+
network: ReturnType<typeof createRing<NetworkEntry>>,
|
|
106
|
+
console_: ReturnType<typeof createRing<ConsoleLine>>,
|
|
107
|
+
crashed: { value: string | undefined },
|
|
108
|
+
): Promise<void> {
|
|
109
|
+
await init.page.setRequestInterception(true);
|
|
110
|
+
init.page.on('request', (payload) => {
|
|
111
|
+
const request = asRequest(payload);
|
|
112
|
+
if (request === undefined) return;
|
|
113
|
+
const url = request.url();
|
|
114
|
+
const type = asResourceType(request.resourceType());
|
|
115
|
+
// The METHOD the browser is actually sending. Recording every request as a GET made
|
|
116
|
+
// `page.network()` — which `X_SCRAPE_HTTP_FAILED`'s own fix line tells the reader to open —
|
|
117
|
+
// misreport every POST and PUT the page made.
|
|
118
|
+
const method = readStringFrom(request, 'method') ?? 'GET';
|
|
119
|
+
const verdict = interceptVerdict(url, type, init.rules);
|
|
120
|
+
const at = init.clock.now().getTime();
|
|
121
|
+
if (verdict === 'allow') {
|
|
122
|
+
network.push({ method, url, resourceType: type, at });
|
|
123
|
+
void request.continue();
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
network.push(refusalEntry(url, type, verdict, at, method));
|
|
127
|
+
void request.abort();
|
|
128
|
+
});
|
|
129
|
+
init.page.on('console', (payload) => {
|
|
130
|
+
console_.push({
|
|
131
|
+
level: CONSOLE_LEVELS[(readStringFrom(payload, 'type') ?? '').toLowerCase()] ?? 'log',
|
|
132
|
+
text: readStringFrom(payload, 'text') ?? '',
|
|
133
|
+
at: init.clock.now().getTime(),
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
// A renderer that dies must be a CODE, not a hang: every later call answers X_SCRAPE_PAGE_CRASHED
|
|
137
|
+
// instead of waiting out its own timeout against a tab that is gone.
|
|
138
|
+
init.page.on('error', (payload) => {
|
|
139
|
+
crashed.value = readStringFrom(payload, 'message') ?? 'renderer crashed';
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
144
|
+
const console_ = createRing<ConsoleLine>(init.ringCapacity);
|
|
145
|
+
const network = createRing<NetworkEntry>(init.ringCapacity);
|
|
146
|
+
const crashed: { value: string | undefined } = { value: undefined };
|
|
147
|
+
await arm(init, network, console_, crashed);
|
|
148
|
+
let pendingStorage: SessionSnapshot | undefined;
|
|
149
|
+
|
|
150
|
+
const originOf = (url: string): string => {
|
|
151
|
+
try {
|
|
152
|
+
return new URL(url).origin;
|
|
153
|
+
} catch {
|
|
154
|
+
return '';
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* `localStorage` is PER ORIGIN, and `restore()` runs before the first navigation — on
|
|
160
|
+
* `about:blank`, an opaque origin with no storage to write to. So the storage half waits for the
|
|
161
|
+
* navigation that reaches the origin the session belongs to, and lands there.
|
|
162
|
+
*
|
|
163
|
+
* The origin has to MATCH: applying it to whatever page loaded first would write the site's
|
|
164
|
+
* bearer token — `session-state.ts` says this is where most sites keep it — into a different
|
|
165
|
+
* site's storage. A session whose origin is never visited simply keeps its storage, which is
|
|
166
|
+
* the same answer a browser gives.
|
|
167
|
+
*/
|
|
168
|
+
const applyPendingStorage = async (): Promise<void> => {
|
|
169
|
+
const pending = pendingStorage;
|
|
170
|
+
if (pending === undefined) return;
|
|
171
|
+
if (originOf(init.page.url()) !== pending.origin || pending.origin === '') return;
|
|
172
|
+
pendingStorage = undefined;
|
|
173
|
+
await init.page.evaluate(
|
|
174
|
+
`(() => { const entries = ${JSON.stringify(pending.storage)}; for (const key of Object.keys(entries)) localStorage.setItem(key, entries[key]); })()`,
|
|
175
|
+
);
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const live = (): void => {
|
|
179
|
+
if (crashed.value !== undefined) throw pageCrashed(init.page.url());
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
const guard = async <T>(what: string, run: () => Promise<T>): Promise<T> => {
|
|
183
|
+
live();
|
|
184
|
+
try {
|
|
185
|
+
return await run();
|
|
186
|
+
} catch (thrown) {
|
|
187
|
+
live();
|
|
188
|
+
throw browserUnreachable(`${CDP_DRIVER} ${what}`, thrown);
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const frameTarget = (frame: CdpFrameLike, parent: ScrapeTarget): ScrapeTarget => ({
|
|
193
|
+
...parent,
|
|
194
|
+
url: () => frame.url(),
|
|
195
|
+
content: () => guard('content', () => frame.content()),
|
|
196
|
+
query: (selector) =>
|
|
197
|
+
guard('query', async () =>
|
|
198
|
+
parseSnapshots(await frame.evaluate(snapshotExpression(selector))),
|
|
199
|
+
),
|
|
200
|
+
click: (selector) => guard('click', () => frame.click(selector)),
|
|
201
|
+
type: (selector, text) => guard('type', () => frame.type(selector, text)),
|
|
202
|
+
select: (selector, values) =>
|
|
203
|
+
guard('select', async () => {
|
|
204
|
+
await frame.select(selector, ...values);
|
|
205
|
+
}),
|
|
206
|
+
evaluate: (expression) => guard('evaluate', () => frame.evaluate(expression)),
|
|
207
|
+
frames: () => Promise.resolve([]),
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
const target: ScrapeTarget = {
|
|
211
|
+
driver: CDP_DRIVER,
|
|
212
|
+
console: console_,
|
|
213
|
+
network,
|
|
214
|
+
url: () => init.page.url(),
|
|
215
|
+
goto: (url: string, options: GotoOptions) =>
|
|
216
|
+
guard('goto', async () => {
|
|
217
|
+
await init.page.goto(url, { timeout: options.timeoutMs });
|
|
218
|
+
await applyPendingStorage();
|
|
219
|
+
}),
|
|
220
|
+
content: () => guard('content', () => init.page.content()),
|
|
221
|
+
query: (selector) =>
|
|
222
|
+
guard('query', async () =>
|
|
223
|
+
parseSnapshots(await init.page.evaluate(snapshotExpression(selector))),
|
|
224
|
+
),
|
|
225
|
+
click: (selector) => guard('click', () => init.page.click(selector)),
|
|
226
|
+
type: (selector, text) => guard('type', () => init.page.type(selector, text)),
|
|
227
|
+
clear: (selector) =>
|
|
228
|
+
guard('clear', async () => {
|
|
229
|
+
await init.page.evaluate(
|
|
230
|
+
`(() => { const el = document.querySelector(${JSON.stringify(selector)}); if (el) { el.value = ''; el.dispatchEvent(new Event('input', { bubbles: true })); } })()`,
|
|
231
|
+
);
|
|
232
|
+
}),
|
|
233
|
+
select: (selector, values) =>
|
|
234
|
+
guard('select', async () => {
|
|
235
|
+
await init.page.select(selector, ...values);
|
|
236
|
+
}),
|
|
237
|
+
evaluate: (expression) => guard('evaluate', () => init.page.evaluate(expression)),
|
|
238
|
+
screenshot: (options: CaptureOptions) =>
|
|
239
|
+
guard('screenshot', async () => {
|
|
240
|
+
const shot = await init.page.screenshot({ fullPage: options.fullPage === true });
|
|
241
|
+
// Some builds answer base64 text, some answer bytes. `atob` is the one decoder both a
|
|
242
|
+
// browser and Bun agree on, and it keeps this file free of a Buffer import.
|
|
243
|
+
return typeof shot === 'string'
|
|
244
|
+
? Uint8Array.from(atob(shot), (character) => character.charCodeAt(0))
|
|
245
|
+
: shot;
|
|
246
|
+
}),
|
|
247
|
+
pdf: (_options: CaptureOptions) => guard('pdf', () => init.page.pdf()),
|
|
248
|
+
cookies: () =>
|
|
249
|
+
guard('cookies', async () => {
|
|
250
|
+
const source = init.browser as { cookies?: () => Promise<unknown> };
|
|
251
|
+
if (typeof source.cookies !== 'function') {
|
|
252
|
+
throw scrapeNotImplemented(
|
|
253
|
+
'cookies() on a CDP browser with no cookies() method',
|
|
254
|
+
'upgrade the launcher to a puppeteer-core that exposes browser.cookies(), or read them with page.evaluate("document.cookie")',
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
return parse(cookieSchema, await source.cookies());
|
|
258
|
+
}),
|
|
259
|
+
download: (_options): Promise<ScrapeDownloadFile> => {
|
|
260
|
+
// Honest stub, in the shape `packages/jobs/src/driver-redis.ts` uses. A real one needs
|
|
261
|
+
// `Browser.setDownloadBehavior` over a raw CDP session plus a directory watch, and a
|
|
262
|
+
// half-written version that silently returned empty bytes is worse than this line.
|
|
263
|
+
throw scrapeNotImplemented(
|
|
264
|
+
'download() on the puppeteer driver',
|
|
265
|
+
'fetch the file inside the page — page.evaluate("fetch(url).then(r => r.text())") — or run this scrape on fixtureBrowser(), whose download() is complete',
|
|
266
|
+
);
|
|
267
|
+
},
|
|
268
|
+
frames: () =>
|
|
269
|
+
guard('frames', () =>
|
|
270
|
+
Promise.resolve(
|
|
271
|
+
init.page.frames().map(
|
|
272
|
+
(frame): FrameRef => ({
|
|
273
|
+
name: frame.name(),
|
|
274
|
+
url: frame.url(),
|
|
275
|
+
target: frameTarget(frame, target),
|
|
276
|
+
}),
|
|
277
|
+
),
|
|
278
|
+
),
|
|
279
|
+
),
|
|
280
|
+
session: () =>
|
|
281
|
+
guard('session', async () => {
|
|
282
|
+
const storage = await init.page.evaluate(
|
|
283
|
+
'(() => JSON.stringify(Object.fromEntries(Object.entries(localStorage))))()',
|
|
284
|
+
);
|
|
285
|
+
const agent = await init.page.evaluate('navigator.userAgent');
|
|
286
|
+
let origin = '';
|
|
287
|
+
try {
|
|
288
|
+
origin = new URL(init.page.url()).origin;
|
|
289
|
+
} catch {
|
|
290
|
+
origin = '';
|
|
291
|
+
}
|
|
292
|
+
const source = init.browser as { cookies?: () => Promise<unknown> };
|
|
293
|
+
return {
|
|
294
|
+
cookies:
|
|
295
|
+
typeof source.cookies === 'function' ? parse(cookieSchema, await source.cookies()) : [],
|
|
296
|
+
headers: {},
|
|
297
|
+
storage: parse(storageSchema, JSON.parse(typeof storage === 'string' ? storage : '{}')),
|
|
298
|
+
userAgent: typeof agent === 'string' ? agent : '',
|
|
299
|
+
origin,
|
|
300
|
+
};
|
|
301
|
+
}),
|
|
302
|
+
restore: (session: SessionSnapshot) =>
|
|
303
|
+
guard('restore', async () => {
|
|
304
|
+
// Two halves, because they belong to two different moments: cookies are the browser's and
|
|
305
|
+
// can be put back now, storage is an ORIGIN's and cannot exist until one is loaded.
|
|
306
|
+
const source = init.browser as {
|
|
307
|
+
setCookie?: (...cookies: readonly unknown[]) => Promise<void>;
|
|
308
|
+
};
|
|
309
|
+
if (typeof source.setCookie === 'function' && session.cookies.length > 0) {
|
|
310
|
+
await source.setCookie(...session.cookies);
|
|
311
|
+
}
|
|
312
|
+
pendingStorage = Object.keys(session.storage).length > 0 ? session : undefined;
|
|
313
|
+
await applyPendingStorage();
|
|
314
|
+
}),
|
|
315
|
+
close: async (): Promise<void> => {
|
|
316
|
+
await init.page.close();
|
|
317
|
+
},
|
|
318
|
+
};
|
|
319
|
+
return target;
|
|
320
|
+
}
|