@ultimat3/scraping 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +15 -5
- package/src/cdp-port.ts +23 -1
- package/src/cdp-target.ts +12 -1
- package/src/clock.ts +0 -15
- package/src/driver-cdp.ts +13 -2
- package/src/error-throws-pdf.ts +16 -0
- package/src/error-throws.ts +1 -1
- package/src/errors.ts +6 -0
- package/src/html-target.ts +15 -1
- package/src/http.ts +3 -3
- package/src/index.ts +4 -0
- package/src/page-over-target.ts +25 -8
- package/src/page.ts +14 -2
- package/src/pdf-print.ts +61 -0
- package/src/pdf-request.ts +125 -0
- package/src/pinned-host.ts +11 -20
- package/src/robots-fetch.ts +3 -3
- package/src/target.ts +8 -1
- package/src/watchdog.ts +50 -29
- package/src/deadline-signal.ts +0 -46
package/README.md
CHANGED
|
@@ -4,6 +4,8 @@ Browser automation as a **job**. `scrape()` returns a `JobHandle` — one row of
|
|
|
4
4
|
`PRIMITIVE_FACTORIES` in `@ultimat3/core`, the derived list of every factory that ships. There is
|
|
5
5
|
no ninth primitive, and no ordinal here to go stale when the next factory lands.
|
|
6
6
|
|
|
7
|
+
`bun add @ultimat3/scraping` · guide: [Scraping](https://github.com/developerz-ai/ultimate/wiki/Scraping)
|
|
8
|
+
|
|
7
9
|
```ts
|
|
8
10
|
import { t } from '@ultimat3/schema';
|
|
9
11
|
import { scrape, storageSessionStore } from '@ultimat3/scraping';
|
|
@@ -327,6 +329,12 @@ bound threw away, so a count taken from one is never quietly a floor.
|
|
|
327
329
|
| `page.pageErrors()` / `page.pageErrorsDropped()` | `PageError[]` — what the page THREW and nobody caught, with the `stack` when the exception carried one |
|
|
328
330
|
| `page.network()` / `page.networkDropped()` | `NetworkEntry[]` — every request, refusals included |
|
|
329
331
|
|
|
332
|
+
None of the three reads is activity; every verb that reaches the browser is, `cookies()`,
|
|
333
|
+
`session()`, `screenshot()` and `pdf()` included. Work on a channel the page cannot see — the app's
|
|
334
|
+
own CDP session, a long warm-up — is reported with `page.activity()`: the wedge watchdog's idle
|
|
335
|
+
clock only, no usage counted. When the watchdog fires it closes the browser then, not at unwind,
|
|
336
|
+
so a call blocked on an attached browser's socket rejects as `X_SCRAPE_WEDGED`.
|
|
337
|
+
|
|
330
338
|
All three are **redacted by value** on the way out, the same pass `page.html()` makes: a login
|
|
331
339
|
endpoint fetched with the password in its query string, or a site that logs the credential it
|
|
332
340
|
rejected, would otherwise put the value in `page.network()`/`page.console()` verbatim and from
|
|
@@ -363,6 +371,17 @@ whole-viewport picture spends them on everything that is not the component under
|
|
|
363
371
|
| `clip` with no area, or entirely in negative coordinates | `X_SCRAPE_CAPTURE_INVALID` — a blank picture that reads as a successful capture is the failure this exists to remove |
|
|
364
372
|
| `clip` on `page.pdf()` | `X_SCRAPE_CAPTURE_INVALID` — a print engine paginates the document and has no crop |
|
|
365
373
|
|
|
374
|
+
### A PDF laid out as asked
|
|
375
|
+
|
|
376
|
+
`page.pdf(layout)` takes `PdfRequest`: `format` (`PDF_FORMATS` — `a4`, `letter`, …) **or**
|
|
377
|
+
`widthMm` + `heightMm`, `marginMm: { top, right, bottom, left }`, `printBackground`,
|
|
378
|
+
`preferCssPageSize` (the document's own `@page`), and `headerHtml` / `footerHtml` (Chrome's print
|
|
379
|
+
templates). `pdf-request.ts` checks it once, before any driver, beside the clip rule: two sizes,
|
|
380
|
+
half a size, a size beside `preferCssPageSize` or a non-finite measure is
|
|
381
|
+
`X_SCRAPE_PDF_OPTIONS_INVALID` (422, terminal). The CDP driver forwards the layout in puppeteer's
|
|
382
|
+
words (`pdf-print.ts`: `'210mm'`, `'A4'`, a blank template for the band the caller left empty); the
|
|
383
|
+
offline drivers stamp it into deterministic bytes, so a dropped field fails a test. `As of 2026-11`.
|
|
384
|
+
|
|
366
385
|
A rectangle **below the fold is accepted**. It is not checked against a viewport: this package
|
|
367
386
|
neither sets nor reads one, and a component under the fold is the case a component crop is for.
|
|
368
387
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/scraping",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "30.0.0",
|
|
4
4
|
"description": "Browser automation as a job: scrape() returns a JobHandle",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"ultimate",
|
|
7
|
+
"bun",
|
|
8
|
+
"framework",
|
|
9
|
+
"scraping",
|
|
10
|
+
"browser",
|
|
11
|
+
"automation",
|
|
12
|
+
"cdp"
|
|
13
|
+
],
|
|
14
|
+
"homepage": "https://github.com/developerz-ai/ultimate/wiki/Scraping",
|
|
5
15
|
"license": "MIT",
|
|
6
16
|
"type": "module",
|
|
7
17
|
"repository": {
|
|
@@ -31,9 +41,9 @@
|
|
|
31
41
|
"test": "bun test"
|
|
32
42
|
},
|
|
33
43
|
"dependencies": {
|
|
34
|
-
"@ultimat3/core": "
|
|
35
|
-
"@ultimat3/jobs": "
|
|
36
|
-
"@ultimat3/schema": "
|
|
37
|
-
"@ultimat3/storage": "
|
|
44
|
+
"@ultimat3/core": "30.0.0",
|
|
45
|
+
"@ultimat3/jobs": "30.0.0",
|
|
46
|
+
"@ultimat3/schema": "30.0.0",
|
|
47
|
+
"@ultimat3/storage": "30.0.0"
|
|
38
48
|
}
|
|
39
49
|
}
|
package/src/cdp-port.ts
CHANGED
|
@@ -31,6 +31,28 @@ export interface CdpScreenshotOptions {
|
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Puppeteer's `PDFOptions`, the part this package sends — restated for the screenshot options'
|
|
36
|
+
* reason. Lengths are CSS length strings (`'210mm'`); `format` is the library's own capitalised
|
|
37
|
+
* spelling (`'A4'`, `'Letter'`), which every version since 1.x accepts.
|
|
38
|
+
*/
|
|
39
|
+
export interface CdpPdfOptions {
|
|
40
|
+
readonly format?: string;
|
|
41
|
+
readonly width?: string;
|
|
42
|
+
readonly height?: string;
|
|
43
|
+
readonly margin?: {
|
|
44
|
+
readonly top?: string;
|
|
45
|
+
readonly right?: string;
|
|
46
|
+
readonly bottom?: string;
|
|
47
|
+
readonly left?: string;
|
|
48
|
+
};
|
|
49
|
+
readonly printBackground?: boolean;
|
|
50
|
+
readonly preferCSSPageSize?: boolean;
|
|
51
|
+
readonly displayHeaderFooter?: boolean;
|
|
52
|
+
readonly headerTemplate?: string;
|
|
53
|
+
readonly footerTemplate?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
34
56
|
export interface CdpRequestLike {
|
|
35
57
|
url(): string;
|
|
36
58
|
resourceType(): string;
|
|
@@ -77,7 +99,7 @@ export interface CdpPageLike {
|
|
|
77
99
|
type(selector: string, text: string): Promise<void>;
|
|
78
100
|
select(selector: string, ...values: string[]): Promise<string[]>;
|
|
79
101
|
screenshot(options: CdpScreenshotOptions): Promise<Uint8Array | string>;
|
|
80
|
-
pdf(options?:
|
|
102
|
+
pdf(options?: CdpPdfOptions): Promise<Uint8Array>;
|
|
81
103
|
setRequestInterception(enabled: boolean): Promise<void>;
|
|
82
104
|
/**
|
|
83
105
|
* The browser's own network condition — `Network.emulateNetworkConditions` under CDP.
|
package/src/cdp-target.ts
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
import { type ColorScheme, colorSchemeFeatures } from './color-scheme';
|
|
19
19
|
import { browserUnreachable, pageCrashed } from './error-throws';
|
|
20
20
|
import { parseKeyChord } from './key-chord';
|
|
21
|
+
import { printOptionsOf } from './pdf-print';
|
|
21
22
|
import type { ConsoleLine, NetworkEntry, PageError } from './rings';
|
|
22
23
|
import { boundedRing } from './rings';
|
|
23
24
|
import type { SessionSnapshot } from './session-state';
|
|
@@ -26,6 +27,7 @@ import type {
|
|
|
26
27
|
CaptureOptions,
|
|
27
28
|
FrameRef,
|
|
28
29
|
GotoOptions,
|
|
30
|
+
PdfOptions,
|
|
29
31
|
ScrapeCookie,
|
|
30
32
|
ScrapeDownloadFile,
|
|
31
33
|
ScrapeTarget,
|
|
@@ -91,6 +93,13 @@ const isStructuralRefusal = (thrown: unknown): boolean =>
|
|
|
91
93
|
*/
|
|
92
94
|
export interface CdpTargetInit extends CdpArmInit {
|
|
93
95
|
readonly browser: CdpBrowserLike;
|
|
96
|
+
/**
|
|
97
|
+
* Why the session was ended from OUTSIDE the call, if it was — the fired wedge guard's reason.
|
|
98
|
+
* A thunk, because the guard is built after the target. Read before a failure is re-labelled:
|
|
99
|
+
* the fire closes the socket, so a call pending on it rejects, and that rejection is the fire's
|
|
100
|
+
* consequence. Labelled `X_SCRAPE_BROWSER_UNREACHABLE` it hides the cause and its fix.
|
|
101
|
+
*/
|
|
102
|
+
readonly ended?: (() => unknown) | undefined;
|
|
94
103
|
}
|
|
95
104
|
|
|
96
105
|
export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
@@ -141,6 +150,8 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
|
141
150
|
try {
|
|
142
151
|
return await run();
|
|
143
152
|
} catch (thrown) {
|
|
153
|
+
const ended = init.ended?.();
|
|
154
|
+
if (ended !== undefined) throw ended;
|
|
144
155
|
live();
|
|
145
156
|
if (isStructuralRefusal(thrown)) throw thrown;
|
|
146
157
|
throw browserUnreachable(`${CDP_DRIVER} ${what}`, thrown);
|
|
@@ -336,7 +347,7 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
|
|
|
336
347
|
? Uint8Array.from(atob(shot), (character) => character.charCodeAt(0))
|
|
337
348
|
: shot;
|
|
338
349
|
}),
|
|
339
|
-
pdf: (
|
|
350
|
+
pdf: (options: PdfOptions) => guard('pdf', () => init.page.pdf(printOptionsOf(options))),
|
|
340
351
|
cookies: () =>
|
|
341
352
|
guard('cookies', async () => {
|
|
342
353
|
const source = init.browser as { cookies?: () => Promise<unknown> };
|
package/src/clock.ts
CHANGED
|
@@ -131,18 +131,3 @@ export function deadline(clock: Clock, totalMs: number, subject = 'a scrape wait
|
|
|
131
131
|
const remainingMs = (): number => Math.max(0, budgetMs - (clock.monotonic() - startedAt));
|
|
132
132
|
return { totalMs: budgetMs, remainingMs, expired: () => remainingMs() <= 0 };
|
|
133
133
|
}
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* The timer a request DEADLINE is armed on (`deadline-signal.ts`) — not a wait this package
|
|
137
|
-
* performs, but a timer all the same, so it lives here. Cleared when the request settles, where
|
|
138
|
-
* `AbortSignal.timeout` stayed armed for its whole span after the answer had landed.
|
|
139
|
-
*/
|
|
140
|
-
export interface DeadlineTimer {
|
|
141
|
-
set(fire: () => void, ms: number): unknown;
|
|
142
|
-
clear(handle: unknown): void;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
export const platformDeadlineTimer: DeadlineTimer = Object.freeze({
|
|
146
|
-
set: (fire: () => void, ms: number): unknown => setTimeout(fire, ms),
|
|
147
|
-
clear: (handle: unknown): void => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
148
|
-
});
|
package/src/driver-cdp.ts
CHANGED
|
@@ -88,6 +88,7 @@ async function opened(
|
|
|
88
88
|
browser: CdpBrowserLike,
|
|
89
89
|
init: SessionInit,
|
|
90
90
|
wiring: SessionWiring,
|
|
91
|
+
ended: () => unknown,
|
|
91
92
|
): Promise<ScrapeTarget> {
|
|
92
93
|
try {
|
|
93
94
|
const page = await browser.newPage();
|
|
@@ -105,7 +106,13 @@ async function opened(
|
|
|
105
106
|
const { username, password } = splitCredentials(wiring.proxy);
|
|
106
107
|
await page.authenticate({ username, password });
|
|
107
108
|
}
|
|
108
|
-
const target = await cdpTarget({
|
|
109
|
+
const target = await cdpTarget({
|
|
110
|
+
page,
|
|
111
|
+
browser,
|
|
112
|
+
rules: init.rules,
|
|
113
|
+
clock: init.clock,
|
|
114
|
+
ended,
|
|
115
|
+
});
|
|
109
116
|
if (init.restore !== undefined) await target.restore(init.restore);
|
|
110
117
|
return target;
|
|
111
118
|
} catch (thrown) {
|
|
@@ -140,7 +147,10 @@ async function sessionOver(
|
|
|
140
147
|
// `runScrape`'s `finally { session.close() }` never runs for a session `open()` did not return,
|
|
141
148
|
// so a tab limit, a refused interception or a restore that threw left a real Chrome process —
|
|
142
149
|
// or a remote session somebody is billing for — running per attempt, unattributed.
|
|
143
|
-
|
|
150
|
+
// Late-bound: the target is built before the guard that can end it. Until the guard exists no
|
|
151
|
+
// fire can have happened, so a failure inside `opened()` keeps its own label.
|
|
152
|
+
let ended = (): unknown => undefined;
|
|
153
|
+
const target = await opened(browser, init, wiring, () => ended());
|
|
144
154
|
const guard = createWedgeGuard({
|
|
145
155
|
clock: init.clock,
|
|
146
156
|
what: `scrape "${init.name}"`,
|
|
@@ -158,6 +168,7 @@ async function sessionOver(
|
|
|
158
168
|
child?.kill(9);
|
|
159
169
|
},
|
|
160
170
|
});
|
|
171
|
+
ended = () => (guard.fired ? guard.signal.reason : undefined);
|
|
161
172
|
const onActivity = (): void => {
|
|
162
173
|
guard.touch();
|
|
163
174
|
init.onActivity?.();
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// The failure mode of a PDF layout request. Split from `error-throws.ts`, which stands at its size
|
|
2
|
+
// ceiling; the code itself stays declared in `errors.ts` — one registry, one place.
|
|
3
|
+
|
|
4
|
+
import { ScrapeError } from './errors';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A layout no print engine lays out as asked. `reason` is built by `pdf-request.ts` from the
|
|
8
|
+
* caller's own numbers and literal format names — never from an unknown value.
|
|
9
|
+
*/
|
|
10
|
+
export const pdfOptionsInvalid = (reason: string, meta: { readonly field: string }): ScrapeError =>
|
|
11
|
+
new ScrapeError({
|
|
12
|
+
code: 'X_SCRAPE_PDF_OPTIONS_INVALID',
|
|
13
|
+
cause: `page.pdf() cannot lay this document out: ${reason}`,
|
|
14
|
+
fix: "page.pdf({ format: 'a4', marginMm: { top: 12, bottom: 12 } }) — exactly one size: a format, or widthMm with heightMm, or preferCssPageSize alone",
|
|
15
|
+
meta: { field: meta.field },
|
|
16
|
+
});
|
package/src/error-throws.ts
CHANGED
|
@@ -203,7 +203,7 @@ export const wedged = (what: string, idleMs: number): ScrapeError =>
|
|
|
203
203
|
new ScrapeError({
|
|
204
204
|
code: 'X_SCRAPE_WEDGED',
|
|
205
205
|
cause: `${what} produced no browser activity for ${String(idleMs)}ms, so the process was killed`,
|
|
206
|
-
fix: 'raise watchdog: { idleMs } on the scrape() definition if the site is genuinely this slow, otherwise re-run — the browser was
|
|
206
|
+
fix: 'raise watchdog: { idleMs } on the scrape() definition if the site is genuinely this slow, or call page.activity() from the channel the framework cannot see (your own CDP session, a warm-up), otherwise re-run — the browser was closed and its session is gone',
|
|
207
207
|
meta: { what, idleMs },
|
|
208
208
|
});
|
|
209
209
|
|
package/src/errors.ts
CHANGED
|
@@ -42,6 +42,7 @@ export const SCRAPE_OWNED_ERROR_CODES = [
|
|
|
42
42
|
'X_SCRAPE_EGRESS_UNSUPPORTED',
|
|
43
43
|
'X_SCRAPE_EGRESS_IN_PAYLOAD',
|
|
44
44
|
'X_SCRAPE_LAUNCH_ARGS_INVALID',
|
|
45
|
+
'X_SCRAPE_PDF_OPTIONS_INVALID',
|
|
45
46
|
] as const;
|
|
46
47
|
|
|
47
48
|
/**
|
|
@@ -98,6 +99,8 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
|
|
|
98
99
|
X_SCRAPE_EGRESS_IN_PAYLOAD: 'the run exit carries a credential that is also in the job payload',
|
|
99
100
|
X_SCRAPE_LAUNCH_ARGS_INVALID:
|
|
100
101
|
'the launch options carry arguments this driver cannot hand the browser',
|
|
102
|
+
X_SCRAPE_PDF_OPTIONS_INVALID:
|
|
103
|
+
'the PDF request names no single page size a print engine can lay out',
|
|
101
104
|
};
|
|
102
105
|
|
|
103
106
|
// One unconditional call, so a second package claiming one of these codes throws
|
|
@@ -208,6 +211,9 @@ export const SCRAPE_ERROR_RETRY = {
|
|
|
208
211
|
// The args are the deploy's own literal: attempt 2 launches with the same list and is refused
|
|
209
212
|
// the same way. Launching anyway is the retry that "works" from the worker's own address.
|
|
210
213
|
X_SCRAPE_LAUNCH_ARGS_INVALID: 'terminal',
|
|
214
|
+
// The request is the caller's literal, refused before any driver sees it: attempt 2 asks for the
|
|
215
|
+
// same two sizes and is refused identically.
|
|
216
|
+
X_SCRAPE_PDF_OPTIONS_INVALID: 'terminal',
|
|
211
217
|
} as const satisfies Readonly<Record<ScrapeOwnedErrorCode, 'retryable' | 'terminal'>>;
|
|
212
218
|
|
|
213
219
|
registerErrorRetry(SCRAPE_ERROR_RETRY);
|
package/src/html-target.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { markupRequests } from './html-requests';
|
|
|
15
15
|
import type { InterceptRules } from './intercept';
|
|
16
16
|
import { interceptVerdict, refusalEntry } from './intercept';
|
|
17
17
|
import { parseKeyChord } from './key-chord';
|
|
18
|
+
import { pdfLayoutText } from './pdf-request';
|
|
18
19
|
import type { PageRecording } from './recording';
|
|
19
20
|
import { splitDownload } from './recording';
|
|
20
21
|
import type { ConsoleLine, NetworkEntry, PageError } from './rings';
|
|
@@ -27,6 +28,7 @@ import type {
|
|
|
27
28
|
ElementSnapshot,
|
|
28
29
|
FrameRef,
|
|
29
30
|
GotoOptions,
|
|
31
|
+
PdfOptions,
|
|
30
32
|
ScrapeCookie,
|
|
31
33
|
ScrapeDownloadFile,
|
|
32
34
|
ScrapeTarget,
|
|
@@ -64,6 +66,18 @@ const framedPng = (clip: CaptureClip | undefined, scheme: ColorScheme | null): U
|
|
|
64
66
|
};
|
|
65
67
|
const FAKE_PDF = new TextEncoder().encode('%PDF-1.4 offline');
|
|
66
68
|
|
|
69
|
+
/**
|
|
70
|
+
* The PDF rule `framedPng` keeps for pictures: different deterministic bytes per layout, so a
|
|
71
|
+
* driver that dropped `format` or a margin fails a test instead of passing every one. No layout
|
|
72
|
+
* asked for is the old constant, so every digest asserted before this existed still holds.
|
|
73
|
+
*/
|
|
74
|
+
const laidOutPdf = (options: PdfOptions): Uint8Array => {
|
|
75
|
+
// The framing is not layout: a PDF has no clip (refused upstream) and is always the document.
|
|
76
|
+
const { fullPage: _fullPage, clip: _clip, ...layout } = options;
|
|
77
|
+
const notes = pdfLayoutText(layout);
|
|
78
|
+
return notes === '' ? FAKE_PDF : new TextEncoder().encode(`%PDF-1.4 offline ${notes}`);
|
|
79
|
+
};
|
|
80
|
+
|
|
67
81
|
export type RecordingLookup = (url: string) => Promise<PageRecording | undefined>;
|
|
68
82
|
|
|
69
83
|
export interface HtmlTargetInit {
|
|
@@ -421,7 +435,7 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
|
|
|
421
435
|
},
|
|
422
436
|
screenshot: (options: CaptureOptions): Promise<Uint8Array> =>
|
|
423
437
|
Promise.resolve(framedPng(options.clip, colorScheme)),
|
|
424
|
-
pdf: (
|
|
438
|
+
pdf: (options: PdfOptions): Promise<Uint8Array> => Promise.resolve(laidOutPdf(options)),
|
|
425
439
|
cookies: (): Promise<readonly ScrapeCookie[]> => Promise.resolve(session.cookies),
|
|
426
440
|
session: (): Promise<SessionSnapshot> => Promise.resolve(session),
|
|
427
441
|
restore: (next: SessionSnapshot): Promise<void> => {
|
package/src/http.ts
CHANGED
|
@@ -11,18 +11,18 @@
|
|
|
11
11
|
// anti-bot systems look for.
|
|
12
12
|
|
|
13
13
|
import { finiteCount, readWithinLimit } from '@ultimat3/core';
|
|
14
|
+
import { type DeadlineTimer, deadlineSignal, resolveDialHost } from '@ultimat3/core/outbound';
|
|
14
15
|
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
15
16
|
import { parse } from '@ultimat3/schema';
|
|
16
17
|
import type { ScrapeClock } from './clock';
|
|
17
18
|
import { cookieHeaderFor } from './cookie-scope';
|
|
18
|
-
import { type DeadlineTimer, deadlineSignal } from './deadline-signal';
|
|
19
19
|
import { bodyTooLarge, hostBlocked, httpFailed, redirectLoop, scrapeTimeout } from './error-throws';
|
|
20
20
|
import type { RedirectHop } from './http-redirect';
|
|
21
21
|
import { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
|
|
22
22
|
import type { InterceptRules } from './intercept';
|
|
23
23
|
import { interceptVerdict } from './intercept';
|
|
24
24
|
import type { DialTarget, HostResolve } from './pinned-host';
|
|
25
|
-
import { dialTarget
|
|
25
|
+
import { dialTarget } from './pinned-host';
|
|
26
26
|
import type { NetworkRing } from './rings';
|
|
27
27
|
import type { RobotsGate } from './robots';
|
|
28
28
|
import type { ScrapeSecrets } from './secrets';
|
|
@@ -279,7 +279,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
|
|
|
279
279
|
* The three gates, in one place, so the initial URL and hop seven are screened by the same code
|
|
280
280
|
* in the same order. A second copy for redirects is how the two drift.
|
|
281
281
|
*/
|
|
282
|
-
const resolve = init.resolve ??
|
|
282
|
+
const resolve = init.resolve ?? resolveDialHost;
|
|
283
283
|
const screen = async (target: string): Promise<DialTarget> => {
|
|
284
284
|
if (interceptVerdict(target, 'fetch', init.rules) !== 'allow') {
|
|
285
285
|
throw hostBlocked(target, init.rules.allowHosts);
|
package/src/index.ts
CHANGED
|
@@ -25,6 +25,7 @@ export type {
|
|
|
25
25
|
CdpKeyboardLike,
|
|
26
26
|
CdpLauncherLike,
|
|
27
27
|
CdpPageLike,
|
|
28
|
+
CdpPdfOptions,
|
|
28
29
|
CdpRequestLike,
|
|
29
30
|
CdpScreenshotOptions,
|
|
30
31
|
CdpSessionLike,
|
|
@@ -145,6 +146,8 @@ export type {
|
|
|
145
146
|
} from './page';
|
|
146
147
|
export type { PageContext } from './page-over-target';
|
|
147
148
|
export { pageOverTarget } from './page-over-target';
|
|
149
|
+
export type { PdfFormat, PdfLayout, PdfMarginMm, PdfRequest } from './pdf-request';
|
|
150
|
+
export { PDF_FORMATS } from './pdf-request';
|
|
148
151
|
export type { Pacer } from './rate';
|
|
149
152
|
export { scrapePacer } from './rate';
|
|
150
153
|
export type { HttpRecording, PageRecording } from './recording';
|
|
@@ -229,6 +232,7 @@ export type {
|
|
|
229
232
|
ElementSnapshot,
|
|
230
233
|
FrameRef,
|
|
231
234
|
GotoOptions,
|
|
235
|
+
PdfOptions,
|
|
232
236
|
QueryOptions,
|
|
233
237
|
ScrapeCookie,
|
|
234
238
|
ScrapeDownloadFile,
|
package/src/page-over-target.ts
CHANGED
|
@@ -22,6 +22,7 @@ import type {
|
|
|
22
22
|
WaitOptions,
|
|
23
23
|
} from './page';
|
|
24
24
|
import { DEFAULT_ACCESSIBILITY_MAX } from './page';
|
|
25
|
+
import { type PdfRequest, pdfLayoutOf } from './pdf-request';
|
|
25
26
|
import type { RobotsGate } from './robots';
|
|
26
27
|
import type { ScrapeSecrets } from './secrets';
|
|
27
28
|
import { safeConsole, safeHtml, safeNetwork, safePageErrors } from './secrets';
|
|
@@ -232,9 +233,16 @@ async function guardNavigation(url: string, ctx: PageContext): Promise<void> {
|
|
|
232
233
|
export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePage {
|
|
233
234
|
const state: PageState = { tainted: false };
|
|
234
235
|
const frame = frameOver(() => Promise.resolve(target), ctx, state, target.url());
|
|
236
|
+
// The page-level verbs hold their target rather than resolving it, so they skip `watched()` —
|
|
237
|
+
// and a minutes-long capture or cookie read then looked like a dead browser to the watchdog.
|
|
238
|
+
// Every one that reaches the browser goes through this instead.
|
|
239
|
+
const reported = (): ScrapeTarget => {
|
|
240
|
+
ctx.onActivity?.();
|
|
241
|
+
return target;
|
|
242
|
+
};
|
|
235
243
|
const capture = async (
|
|
236
244
|
kind: 'screenshot' | 'pdf',
|
|
237
|
-
options: CaptureRequest | undefined,
|
|
245
|
+
options: CaptureRequest | PdfRequest | undefined,
|
|
238
246
|
): Promise<Uint8Array> => {
|
|
239
247
|
// The leak nobody remembers: a screenshot of a filled login form IS the password, in pixels,
|
|
240
248
|
// in object storage, forever. Refused rather than masked — a mask over pixels is a guess
|
|
@@ -248,7 +256,10 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
|
|
|
248
256
|
fullPage: options?.fullPage,
|
|
249
257
|
...(options?.clip === undefined ? {} : { clip: options.clip }),
|
|
250
258
|
};
|
|
251
|
-
|
|
259
|
+
if (kind === 'screenshot') return reported().screenshot(request);
|
|
260
|
+
// The layout is checked here too, for the framing's reason: once, before any driver.
|
|
261
|
+
const layout = pdfLayoutOf(options ?? {});
|
|
262
|
+
return reported().pdf({ ...request, ...layout });
|
|
252
263
|
};
|
|
253
264
|
return {
|
|
254
265
|
...frame,
|
|
@@ -274,24 +285,30 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
|
|
|
274
285
|
// party implements, and a driver that THROWS from its promise-typed `download()` would escape
|
|
275
286
|
// past this page's caller `.catch()` if the forward were a bare arrow.
|
|
276
287
|
async download(options?: DownloadRequest): Promise<ScrapeDownloadFile> {
|
|
277
|
-
return await
|
|
288
|
+
return await reported().download({ timeoutMs: options?.timeout ?? ctx.defaultTimeoutMs });
|
|
278
289
|
},
|
|
279
290
|
// `async`, for `download()`'s reason: `ScrapeTarget` is the seam a third party implements, and
|
|
280
291
|
// a driver that throws synchronously from a promise-typed method would escape `.catch()`.
|
|
281
292
|
async offline(enabled: boolean): Promise<void> {
|
|
282
|
-
await
|
|
293
|
+
await reported().setOfflineMode(enabled);
|
|
283
294
|
},
|
|
284
295
|
// `async`, for `offline()`'s reason: a driver that throws synchronously from a promise-typed
|
|
285
296
|
// method would escape the caller's `.catch()`.
|
|
286
297
|
async colorScheme(scheme: ColorScheme): Promise<void> {
|
|
287
|
-
await
|
|
298
|
+
await reported().setColorScheme(scheme);
|
|
288
299
|
},
|
|
289
300
|
// `async`, for `offline()`'s reason.
|
|
290
301
|
async prepare(expression: string): Promise<void> {
|
|
291
|
-
await
|
|
302
|
+
await reported().prepare(expression);
|
|
303
|
+
},
|
|
304
|
+
cookies: (): Promise<readonly ScrapeCookie[]> => reported().cookies(),
|
|
305
|
+
session: () => reported().session(),
|
|
306
|
+
// The app's own statement that work is happening on a channel this page cannot see — its own
|
|
307
|
+
// CDP session, a long warm-up. The watchdog's clock only: no usage is counted, because no
|
|
308
|
+
// navigation or request was made through the framework.
|
|
309
|
+
activity: (): void => {
|
|
310
|
+
ctx.onActivity?.();
|
|
292
311
|
},
|
|
293
|
-
cookies: (): Promise<readonly ScrapeCookie[]> => target.cookies(),
|
|
294
|
-
session: () => target.session(),
|
|
295
312
|
// Redacted BY VALUE on the way out, the same pass `html()` makes. A console line and a request
|
|
296
313
|
// URL are two of the four surfaces `secrets.ts` names, and neither had a redaction caller: a
|
|
297
314
|
// login endpoint fetched with the password in its query string, or a site that logs the
|
package/src/page.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { Secret } from '@ultimat3/core';
|
|
|
11
11
|
import type { ActionabilityState } from './actionability';
|
|
12
12
|
import type { CaptureFraming } from './capture-clip';
|
|
13
13
|
import type { ColorScheme } from './color-scheme';
|
|
14
|
+
import type { PdfRequest } from './pdf-request';
|
|
14
15
|
import type { ConsoleLine, NetworkEntry, PageError } from './rings';
|
|
15
16
|
import type { SessionSnapshot } from './session-state';
|
|
16
17
|
import type { AxNode, ElementSnapshot, ScrapeCookie, ScrapeDownloadFile } from './target';
|
|
@@ -127,11 +128,22 @@ export interface ScrapePage extends ScrapeFrame {
|
|
|
127
128
|
goto(url: string, options?: { readonly timeout?: number | undefined }): Promise<void>;
|
|
128
129
|
/** PNG bytes. REFUSED once a secret has been typed into this page — see `secrets.ts`. */
|
|
129
130
|
screenshot(options?: CaptureRequest): Promise<Uint8Array>;
|
|
130
|
-
/**
|
|
131
|
-
|
|
131
|
+
/**
|
|
132
|
+
* PDF bytes, laid out as asked: paper (`format`, or `widthMm` + `heightMm`), `marginMm`,
|
|
133
|
+
* `printBackground`, `headerHtml`/`footerHtml`. Two sizes are `X_SCRAPE_PDF_OPTIONS_INVALID`.
|
|
134
|
+
* Refused once a secret was typed, for `screenshot()`'s reason.
|
|
135
|
+
*/
|
|
136
|
+
pdf(options?: PdfRequest): Promise<Uint8Array>;
|
|
132
137
|
/** The file the last click produced, or `X_SCRAPE_DOWNLOAD_TIMEOUT`. */
|
|
133
138
|
download(options?: DownloadRequest): Promise<ScrapeDownloadFile>;
|
|
134
139
|
cookies(): Promise<readonly ScrapeCookie[]>;
|
|
140
|
+
/**
|
|
141
|
+
* Tell the wedge watchdog the run is working on a channel this page cannot see — the app's own
|
|
142
|
+
* CDP session, a long warm-up. Every verb here already reports; this is for the work that does
|
|
143
|
+
* not go through one. Touches the watchdog's idle clock only and counts no usage. Present on
|
|
144
|
+
* every driver, so a run body never branches on which one it has.
|
|
145
|
+
*/
|
|
146
|
+
activity(): void;
|
|
135
147
|
/**
|
|
136
148
|
* Cut the BROWSER's network, or restore it — what a PWA's offline behaviour has to be tested
|
|
137
149
|
* against. Refused with `X_NOT_IMPLEMENTED` on a driver that has no browser: patching `fetch`
|
package/src/pdf-print.ts
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// A checked PDF layout in the CDP library's own words: millimetres become CSS lengths, a format its
|
|
2
|
+
// capitalised name, a header or footer turns the print engine's header band on. Only the fields
|
|
3
|
+
// the caller gave are sent — the engine's defaults stay the engine's.
|
|
4
|
+
|
|
5
|
+
import type { CdpPdfOptions } from './cdp-port';
|
|
6
|
+
import type { PdfFormat, PdfLayout } from './pdf-request';
|
|
7
|
+
|
|
8
|
+
/** A `Map`, never a record indexed by the caller's string: `format: 'constructor'` names nothing. */
|
|
9
|
+
const LIBRARY_FORMAT: ReadonlyMap<PdfFormat, string> = new Map([
|
|
10
|
+
['letter', 'Letter'],
|
|
11
|
+
['legal', 'Legal'],
|
|
12
|
+
['tabloid', 'Tabloid'],
|
|
13
|
+
['ledger', 'Ledger'],
|
|
14
|
+
['a0', 'A0'],
|
|
15
|
+
['a1', 'A1'],
|
|
16
|
+
['a2', 'A2'],
|
|
17
|
+
['a3', 'A3'],
|
|
18
|
+
['a4', 'A4'],
|
|
19
|
+
['a5', 'A5'],
|
|
20
|
+
['a6', 'A6'],
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
const mm = (value: number): string => `${String(value)}mm`;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* An empty template, never an absent one, for the band the caller left blank: with the band on and
|
|
27
|
+
* a template missing, Chrome prints its own date and URL there — an invoice footer nobody wrote.
|
|
28
|
+
*/
|
|
29
|
+
const BLANK_TEMPLATE = '<span></span>';
|
|
30
|
+
|
|
31
|
+
export function printOptionsOf(layout: PdfLayout): CdpPdfOptions {
|
|
32
|
+
const margin = layout.marginMm;
|
|
33
|
+
const format = layout.format === undefined ? undefined : LIBRARY_FORMAT.get(layout.format);
|
|
34
|
+
const band = layout.headerHtml !== undefined || layout.footerHtml !== undefined;
|
|
35
|
+
return {
|
|
36
|
+
...(format === undefined ? {} : { format }),
|
|
37
|
+
...(layout.widthMm === undefined ? {} : { width: mm(layout.widthMm) }),
|
|
38
|
+
...(layout.heightMm === undefined ? {} : { height: mm(layout.heightMm) }),
|
|
39
|
+
...(margin === undefined
|
|
40
|
+
? {}
|
|
41
|
+
: {
|
|
42
|
+
margin: {
|
|
43
|
+
...(margin.top === undefined ? {} : { top: mm(margin.top) }),
|
|
44
|
+
...(margin.right === undefined ? {} : { right: mm(margin.right) }),
|
|
45
|
+
...(margin.bottom === undefined ? {} : { bottom: mm(margin.bottom) }),
|
|
46
|
+
...(margin.left === undefined ? {} : { left: mm(margin.left) }),
|
|
47
|
+
},
|
|
48
|
+
}),
|
|
49
|
+
...(layout.printBackground === undefined ? {} : { printBackground: layout.printBackground }),
|
|
50
|
+
...(layout.preferCssPageSize === undefined
|
|
51
|
+
? {}
|
|
52
|
+
: { preferCSSPageSize: layout.preferCssPageSize }),
|
|
53
|
+
...(band
|
|
54
|
+
? {
|
|
55
|
+
displayHeaderFooter: true,
|
|
56
|
+
headerTemplate: layout.headerHtml ?? BLANK_TEMPLATE,
|
|
57
|
+
footerTemplate: layout.footerHtml ?? BLANK_TEMPLATE,
|
|
58
|
+
}
|
|
59
|
+
: {}),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// The print layout a PDF is asked for — paper, margins, background, header and footer — and the
|
|
2
|
+
// one check that refuses a request no print engine can honour. Decided ONCE, in `page-over-target.ts`,
|
|
3
|
+
// before any driver: two sizes resolve silently in Chrome (the format wins), which is an invoice
|
|
4
|
+
// laid out on paper nobody asked for.
|
|
5
|
+
|
|
6
|
+
import type { CaptureFraming } from './capture-clip';
|
|
7
|
+
import { pdfOptionsInvalid } from './error-throws-pdf';
|
|
8
|
+
|
|
9
|
+
/** The named paper sizes every print engine knows. Lowercase: the spelling is this package's. */
|
|
10
|
+
export const PDF_FORMATS = [
|
|
11
|
+
'letter',
|
|
12
|
+
'legal',
|
|
13
|
+
'tabloid',
|
|
14
|
+
'ledger',
|
|
15
|
+
'a0',
|
|
16
|
+
'a1',
|
|
17
|
+
'a2',
|
|
18
|
+
'a3',
|
|
19
|
+
'a4',
|
|
20
|
+
'a5',
|
|
21
|
+
'a6',
|
|
22
|
+
] as const;
|
|
23
|
+
|
|
24
|
+
export type PdfFormat = (typeof PDF_FORMATS)[number];
|
|
25
|
+
|
|
26
|
+
/** Millimetres on each side. An absent side is the engine's own default. */
|
|
27
|
+
export interface PdfMarginMm {
|
|
28
|
+
readonly top?: number | undefined;
|
|
29
|
+
readonly right?: number | undefined;
|
|
30
|
+
readonly bottom?: number | undefined;
|
|
31
|
+
readonly left?: number | undefined;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* What a print engine lays a document out on. Exactly one size: a `format`, or `widthMm` WITH
|
|
36
|
+
* `heightMm`, or `preferCssPageSize` (the document's own `@page`) — or none, the engine's letter.
|
|
37
|
+
* `headerHtml` / `footerHtml` are print templates: Chrome fills `.pageNumber`, `.totalPages`,
|
|
38
|
+
* `.date`, `.title` and `.url` inside them.
|
|
39
|
+
*/
|
|
40
|
+
export interface PdfLayout {
|
|
41
|
+
readonly format?: PdfFormat | undefined;
|
|
42
|
+
readonly widthMm?: number | undefined;
|
|
43
|
+
readonly heightMm?: number | undefined;
|
|
44
|
+
readonly marginMm?: PdfMarginMm | undefined;
|
|
45
|
+
readonly printBackground?: boolean | undefined;
|
|
46
|
+
readonly preferCssPageSize?: boolean | undefined;
|
|
47
|
+
readonly headerHtml?: string | undefined;
|
|
48
|
+
readonly footerHtml?: string | undefined;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** `page.pdf()`'s argument: the layout, plus the framing every capture shares (a clip is refused). */
|
|
52
|
+
export interface PdfRequest extends CaptureFraming, PdfLayout {}
|
|
53
|
+
|
|
54
|
+
const SIDES = ['top', 'right', 'bottom', 'left'] as const;
|
|
55
|
+
|
|
56
|
+
const isFormat = (value: string): value is PdfFormat =>
|
|
57
|
+
(PDF_FORMATS as readonly string[]).includes(value);
|
|
58
|
+
|
|
59
|
+
function measure(field: string, value: number | undefined, floor: 'positive' | 'zero'): void {
|
|
60
|
+
if (value === undefined) return;
|
|
61
|
+
const ok = Number.isFinite(value) && (floor === 'positive' ? value > 0 : value >= 0);
|
|
62
|
+
if (!ok) {
|
|
63
|
+
throw pdfOptionsInvalid(
|
|
64
|
+
`${field} is ${String(value)}, and a page measure is a finite number of millimetres${floor === 'positive' ? ' above zero' : ', zero or more'}`,
|
|
65
|
+
{ field },
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The layout half of a request, checked. Answers only the fields that were given, so a driver sees
|
|
72
|
+
* exactly what the caller asked for and supplies its own default for the rest.
|
|
73
|
+
*/
|
|
74
|
+
export function pdfLayoutOf(request: PdfRequest): PdfLayout {
|
|
75
|
+
const { format, widthMm, heightMm, preferCssPageSize } = request;
|
|
76
|
+
if (format !== undefined && !isFormat(format)) {
|
|
77
|
+
throw pdfOptionsInvalid(`format "${String(format)}" is not one of ${PDF_FORMATS.join(', ')}`, {
|
|
78
|
+
field: 'format',
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
measure('widthMm', widthMm, 'positive');
|
|
82
|
+
measure('heightMm', heightMm, 'positive');
|
|
83
|
+
for (const side of SIDES) measure(`marginMm.${side}`, request.marginMm?.[side], 'zero');
|
|
84
|
+
const sized = widthMm !== undefined || heightMm !== undefined;
|
|
85
|
+
if (format !== undefined && sized) {
|
|
86
|
+
throw pdfOptionsInvalid(
|
|
87
|
+
`the request names format "${format}" and widthMm/heightMm, and Chrome silently lays it out on the format`,
|
|
88
|
+
{ field: 'format' },
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
if (sized && (widthMm === undefined || heightMm === undefined)) {
|
|
92
|
+
throw pdfOptionsInvalid(
|
|
93
|
+
`the request names ${widthMm === undefined ? 'heightMm' : 'widthMm'} alone, which is half a page`,
|
|
94
|
+
{ field: widthMm === undefined ? 'heightMm' : 'widthMm' },
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
if (preferCssPageSize === true && (format !== undefined || sized)) {
|
|
98
|
+
throw pdfOptionsInvalid(
|
|
99
|
+
'the request names a size and preferCssPageSize: true, and the document @page rule then silently wins over it',
|
|
100
|
+
{ field: 'preferCssPageSize' },
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
...(format === undefined ? {} : { format }),
|
|
105
|
+
...(widthMm === undefined ? {} : { widthMm }),
|
|
106
|
+
...(heightMm === undefined ? {} : { heightMm }),
|
|
107
|
+
...(request.marginMm === undefined ? {} : { marginMm: { ...request.marginMm } }),
|
|
108
|
+
...(request.printBackground === undefined ? {} : { printBackground: request.printBackground }),
|
|
109
|
+
...(preferCssPageSize === undefined ? {} : { preferCssPageSize }),
|
|
110
|
+
...(request.headerHtml === undefined ? {} : { headerHtml: request.headerHtml }),
|
|
111
|
+
...(request.footerHtml === undefined ? {} : { footerHtml: request.footerHtml }),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** The same layout as one deterministic line — what an offline driver stamps into its bytes. */
|
|
116
|
+
export function pdfLayoutText(layout: PdfLayout): string {
|
|
117
|
+
// Sorted by hand: a replacer ARRAY would also filter `marginMm`'s own keys out of the line.
|
|
118
|
+
const margin = layout.marginMm;
|
|
119
|
+
const sides = margin === undefined ? '' : SIDES.map((side) => String(margin[side])).join('/');
|
|
120
|
+
return Object.entries({ ...layout, marginMm: sides === '' ? undefined : sides })
|
|
121
|
+
.filter(([, value]) => value !== undefined)
|
|
122
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
123
|
+
.map(([key, value]) => `${key}=${String(value)}`)
|
|
124
|
+
.join(' ');
|
|
125
|
+
}
|
package/src/pinned-host.ts
CHANGED
|
@@ -2,19 +2,17 @@
|
|
|
2
2
|
// robots read. Core's `hostDecision` is synchronous and judges NAMES: a wildcard never admits an
|
|
3
3
|
// address literal inside the network, but `allowHosts: ['*']` still admits a hostname that
|
|
4
4
|
// resolves inward. So a name admitted by a wildcard alone is resolved here, every address
|
|
5
|
-
// classified
|
|
6
|
-
//
|
|
5
|
+
// classified and the connection PINNED to the approved address by core's `screenDial` — the one
|
|
6
|
+
// screen `@ultimat3/jobs`' `webhook-target.ts` and `outbound()` call too. This keeps its own code.
|
|
7
7
|
|
|
8
8
|
import type { HostRule } from '@ultimat3/core';
|
|
9
9
|
import { classifyAddress, hostMatches } from '@ultimat3/core';
|
|
10
|
+
import type { DialResolve } from '@ultimat3/core/outbound';
|
|
11
|
+
import { screenDial } from '@ultimat3/core/outbound';
|
|
10
12
|
import { hostResolvesInward } from './error-throws-session';
|
|
11
13
|
|
|
12
14
|
/** Every address a hostname answers. Injected in tests: the network is sealed in this repo. */
|
|
13
|
-
export type HostResolve =
|
|
14
|
-
|
|
15
|
-
/** `Bun.dns.lookup` — the runtime's own resolver, every family, every address. */
|
|
16
|
-
export const resolveHost: HostResolve = async (hostname) =>
|
|
17
|
-
(await Bun.dns.lookup(hostname)).map((entry) => entry.address);
|
|
15
|
+
export type HostResolve = DialResolve;
|
|
18
16
|
|
|
19
17
|
/** What the transport connects to: the URL to dial, and the name it proves when pinned. */
|
|
20
18
|
export interface DialTarget {
|
|
@@ -49,17 +47,10 @@ export async function dialTarget(
|
|
|
49
47
|
const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
|
|
50
48
|
if (classifyAddress(hostname) !== undefined) return { url };
|
|
51
49
|
if (allowHosts.some((rule) => !isWildcard(rule) && hostMatches(hostname, rule))) return { url };
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if (
|
|
57
|
-
|
|
58
|
-
const target = new URL(parsed.href);
|
|
59
|
-
target.hostname = pinned.includes(':') ? `[${pinned}]` : pinned;
|
|
60
|
-
return {
|
|
61
|
-
url: target.href,
|
|
62
|
-
host: parsed.host,
|
|
63
|
-
...(parsed.protocol === 'https:' ? { serverName: hostname } : {}),
|
|
64
|
-
};
|
|
50
|
+
const verdict = await screenDial(parsed, { resolve });
|
|
51
|
+
if ('pin' in verdict) return verdict.pin;
|
|
52
|
+
// A resolver failure propagates as itself: nothing unscreened is dialled, and the same name
|
|
53
|
+
// would not have resolved for `fetch` either. No address at all is refused, as before.
|
|
54
|
+
if ('unresolved' in verdict && verdict.error !== undefined) throw verdict.error;
|
|
55
|
+
throw hostResolvesInward(url, hostname, 'refused' in verdict ? verdict.refused : 'unresolvable');
|
|
65
56
|
}
|
package/src/robots-fetch.ts
CHANGED
|
@@ -10,12 +10,12 @@
|
|
|
10
10
|
|
|
11
11
|
import type { HostRule } from '@ultimat3/core';
|
|
12
12
|
import { finiteCount, isUltimateError, readWithinLimit } from '@ultimat3/core';
|
|
13
|
-
import { type DeadlineTimer, deadlineSignal } from '
|
|
13
|
+
import { type DeadlineTimer, deadlineSignal, resolveDialHost } from '@ultimat3/core/outbound';
|
|
14
14
|
import type { ScrapeFetch } from './http';
|
|
15
15
|
import { redirectHop } from './http-redirect';
|
|
16
16
|
import { interceptVerdict } from './intercept';
|
|
17
17
|
import type { HostResolve } from './pinned-host';
|
|
18
|
-
import { dialTarget
|
|
18
|
+
import { dialTarget } from './pinned-host';
|
|
19
19
|
import type { RobotsAnswer, RobotsFetch } from './robots';
|
|
20
20
|
|
|
21
21
|
/**
|
|
@@ -116,7 +116,7 @@ const answerOf = async (response: Response, limit: number): Promise<RobotsAnswer
|
|
|
116
116
|
*/
|
|
117
117
|
export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
|
|
118
118
|
const call: ScrapeFetch = init.fetch ?? fetch;
|
|
119
|
-
const resolve = init.resolve ??
|
|
119
|
+
const resolve = init.resolve ?? resolveDialHost;
|
|
120
120
|
// Both bounds are screened HERE, at construction, and both floors are 1 — because a bad bound
|
|
121
121
|
// turns into a failed read on every origin, and a read that fails is never evidence of a rule. A
|
|
122
122
|
// `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
|
package/src/target.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import type { CaptureFraming } from './capture-clip';
|
|
7
7
|
import type { ColorScheme } from './color-scheme';
|
|
8
|
+
import type { PdfLayout } from './pdf-request';
|
|
8
9
|
import type { ConsoleRing, NetworkRing, PageErrorRing } from './rings';
|
|
9
10
|
import type { SessionSnapshot } from './session-state';
|
|
10
11
|
|
|
@@ -122,6 +123,12 @@ export interface GotoOptions {
|
|
|
122
123
|
*/
|
|
123
124
|
export type CaptureOptions = CaptureFraming;
|
|
124
125
|
|
|
126
|
+
/**
|
|
127
|
+
* A PDF capture: the framing plus the print layout, already checked by `pdfLayoutOf` — a driver
|
|
128
|
+
* forwards every field it is given and invents none.
|
|
129
|
+
*/
|
|
130
|
+
export type PdfOptions = CaptureOptions & PdfLayout;
|
|
131
|
+
|
|
125
132
|
/**
|
|
126
133
|
* The port. Every member is something a browser genuinely does and nothing a scraper's vocabulary
|
|
127
134
|
* should be re-deriving per driver. NO COUNT: this comment said "Twelve methods" while the
|
|
@@ -228,7 +235,7 @@ export interface ScrapeTarget {
|
|
|
228
235
|
*/
|
|
229
236
|
prepare(expression: string): Promise<void>;
|
|
230
237
|
screenshot(options: CaptureOptions): Promise<Uint8Array>;
|
|
231
|
-
pdf(options:
|
|
238
|
+
pdf(options: PdfOptions): Promise<Uint8Array>;
|
|
232
239
|
cookies(): Promise<readonly ScrapeCookie[]>;
|
|
233
240
|
/** Whatever the last click/navigation produced as a file, or a `X_SCRAPE_DOWNLOAD_TIMEOUT`. */
|
|
234
241
|
download(options: { readonly timeoutMs: number }): Promise<ScrapeDownloadFile>;
|
package/src/watchdog.ts
CHANGED
|
@@ -65,15 +65,45 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
|
|
|
65
65
|
const graceMs = finiteCount('the scrape watchdog', 'graceMs', init.graceMs ?? DEFAULT_GRACE_MS);
|
|
66
66
|
const controller = new AbortController();
|
|
67
67
|
let lastTouch = init.clock.monotonic();
|
|
68
|
-
// TWO latches, not one. `stopped` ends the watch loop; `
|
|
69
|
-
// were the same flag, so a fire — which sets `stopped` — made `shutdown()`
|
|
70
|
-
// line and NEVER call `quit()`. `localBrowser()` hid it, because the fire's
|
|
71
|
-
// reaches a pid; `remoteBrowser()` is `driver-cdp.ts`'s primary path,
|
|
72
|
-
// `null` there, and `browser.close()` is then the only thing that ends
|
|
68
|
+
// TWO latches, not one. `stopped` ends the watch loop; the memoised `teardown` below guards
|
|
69
|
+
// the quit. They were the same flag, so a fire — which sets `stopped` — made `shutdown()`
|
|
70
|
+
// return on its first line and NEVER call `quit()`. `localBrowser()` hid it, because the fire's
|
|
71
|
+
// `kill()` still reaches a pid; `remoteBrowser()` is `driver-cdp.ts`'s primary path,
|
|
72
|
+
// `browser.process()` is `null` there, and `browser.close()` is then the only thing that ends
|
|
73
|
+
// the paid remote session.
|
|
73
74
|
let stopped = false;
|
|
74
|
-
let shuttingDown = false;
|
|
75
75
|
let fired = false;
|
|
76
76
|
|
|
77
|
+
const ceilinged = async (): Promise<void> => {
|
|
78
|
+
let quit = false;
|
|
79
|
+
// A ceiling, not a hope: whichever finishes first wins, and if it is the clock the process
|
|
80
|
+
// is killed. `browser.close()` on a wedged renderer never returns.
|
|
81
|
+
await Promise.race([
|
|
82
|
+
init.quit().then(
|
|
83
|
+
() => {
|
|
84
|
+
quit = true;
|
|
85
|
+
},
|
|
86
|
+
() => {
|
|
87
|
+
quit = false;
|
|
88
|
+
},
|
|
89
|
+
),
|
|
90
|
+
// The ceiling's own sleep is caught for the same reason the quit is: this runs in
|
|
91
|
+
// `runScrape`'s `finally`, `close()` is documented never to throw, and a clock that
|
|
92
|
+
// rejected here would replace the run's real failure with a teardown one. A ceiling that
|
|
93
|
+
// cannot be timed has already expired, which lands on `kill()` — the safe direction.
|
|
94
|
+
init.clock.sleep(graceMs).catch(() => undefined),
|
|
95
|
+
]);
|
|
96
|
+
if (!quit) init.kill();
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
// ONE teardown, shared by the fire and `shutdown()`: the polite quit raced against the ceiling,
|
|
100
|
+
// then the kill. Memoised, so a fire followed by the run's own `close()` quits once and kills at
|
|
101
|
+
// most twice (the fire's, and the ceiling's) rather than starting a second race.
|
|
102
|
+
let teardown: Promise<void> | undefined;
|
|
103
|
+
const end = (): Promise<void> => {
|
|
104
|
+
teardown ??= ceilinged();
|
|
105
|
+
return teardown;
|
|
106
|
+
};
|
|
77
107
|
const watch = async (): Promise<void> => {
|
|
78
108
|
while (!stopped) {
|
|
79
109
|
await init.clock.sleep(WATCH_POLL_MS);
|
|
@@ -84,8 +114,19 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
|
|
|
84
114
|
// Kill FIRST, abort second. The abort is what the awaiting code sees; the kill is what
|
|
85
115
|
// makes the socket it is blocked on close. Reversing them leaves the run cancelled and the
|
|
86
116
|
// browser alive, which is the zombie half of the same incident.
|
|
87
|
-
|
|
88
|
-
|
|
117
|
+
// `finally`: a throwing `kill()` still ends the run — `stopped` is already set, so the loop's
|
|
118
|
+
// catch below would return early and leave the run unaborted behind a dead guard.
|
|
119
|
+
try {
|
|
120
|
+
init.kill();
|
|
121
|
+
} finally {
|
|
122
|
+
// And the QUIT, now, not at unwind. On an attached browser the kill above reaches nothing —
|
|
123
|
+
// `process()` is `null` — and a call blocked on the socket (`evaluate` takes no signal) never
|
|
124
|
+
// unwinds to `close()`. Closing the browser is what rejects it. Unawaited: the abort below
|
|
125
|
+
// must not wait out a hung quit; `shutdown()` awaits this same promise. Its only rejection
|
|
126
|
+
// is a throwing `kill()`, which `shutdown()` hands its caller exactly as before.
|
|
127
|
+
void end().catch(() => undefined);
|
|
128
|
+
controller.abort(wedged(init.what, idleMs));
|
|
129
|
+
}
|
|
89
130
|
}
|
|
90
131
|
};
|
|
91
132
|
// Never floating. `ScrapeClock` is a seam an app implements and `init.kill()` comes from a
|
|
@@ -109,28 +150,8 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
|
|
|
109
150
|
lastTouch = init.clock.monotonic();
|
|
110
151
|
},
|
|
111
152
|
async shutdown(): Promise<void> {
|
|
112
|
-
if (shuttingDown) return;
|
|
113
|
-
shuttingDown = true;
|
|
114
153
|
stopped = true;
|
|
115
|
-
|
|
116
|
-
// A ceiling, not a hope: whichever finishes first wins, and if it is the clock the process
|
|
117
|
-
// is killed. `browser.close()` on a wedged renderer never returns.
|
|
118
|
-
await Promise.race([
|
|
119
|
-
init.quit().then(
|
|
120
|
-
() => {
|
|
121
|
-
quit = true;
|
|
122
|
-
},
|
|
123
|
-
() => {
|
|
124
|
-
quit = false;
|
|
125
|
-
},
|
|
126
|
-
),
|
|
127
|
-
// The ceiling's own sleep is caught for the same reason the quit is: this runs in
|
|
128
|
-
// `runScrape`'s `finally`, `close()` is documented never to throw, and a clock that
|
|
129
|
-
// rejected here would replace the run's real failure with a teardown one. A ceiling that
|
|
130
|
-
// cannot be timed has already expired, which lands on `kill()` — the safe direction.
|
|
131
|
-
init.clock.sleep(graceMs).catch(() => undefined),
|
|
132
|
-
]);
|
|
133
|
-
if (!quit) init.kill();
|
|
154
|
+
await end();
|
|
134
155
|
},
|
|
135
156
|
};
|
|
136
157
|
}
|
package/src/deadline-signal.ts
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
// One request's signal: a deadline plus the caller's own signal, undone when the request settles.
|
|
2
|
-
// `AbortSignal.any([AbortSignal.timeout(ms), outer])` composed the same thing and could not be
|
|
3
|
-
// undone — every read hung one listener (and one composite) on a session-long `outer` and armed a
|
|
4
|
-
// timer that outlived the answer. The shape is `@ultimat3/jobs`' `createRunSignal`.
|
|
5
|
-
|
|
6
|
-
import { type DeadlineTimer, platformDeadlineTimer } from './clock';
|
|
7
|
-
|
|
8
|
-
export type { DeadlineTimer } from './clock';
|
|
9
|
-
|
|
10
|
-
export interface DeadlineSignal extends Disposable {
|
|
11
|
-
/** Aborts at the deadline (a `TimeoutError`, as `AbortSignal.timeout`'s) or with `outer`. */
|
|
12
|
-
readonly signal: AbortSignal;
|
|
13
|
-
/** Whether the DEADLINE fired — not the caller cancelling. What the timeout code is chosen by. */
|
|
14
|
-
timedOut(): boolean;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* `ms` is the caller's already-screened deadline (a positive finite count). `[Symbol.dispose]`
|
|
19
|
-
* clears the timer and detaches from `outer`; idempotent, and it never aborts.
|
|
20
|
-
*/
|
|
21
|
-
export function deadlineSignal(
|
|
22
|
-
ms: number,
|
|
23
|
-
outer?: AbortSignal,
|
|
24
|
-
timer: DeadlineTimer = platformDeadlineTimer,
|
|
25
|
-
): DeadlineSignal {
|
|
26
|
-
const controller = new AbortController();
|
|
27
|
-
let fired = false;
|
|
28
|
-
const handle = timer.set(() => {
|
|
29
|
-
fired = true;
|
|
30
|
-
controller.abort(new DOMException('The operation timed out.', 'TimeoutError'));
|
|
31
|
-
}, ms);
|
|
32
|
-
const forward = (): void => controller.abort(outer?.reason);
|
|
33
|
-
if (outer?.aborted === true) forward();
|
|
34
|
-
else outer?.addEventListener('abort', forward, { once: true });
|
|
35
|
-
let open = true;
|
|
36
|
-
return {
|
|
37
|
-
signal: controller.signal,
|
|
38
|
-
timedOut: () => fired,
|
|
39
|
-
[Symbol.dispose]: () => {
|
|
40
|
-
if (!open) return;
|
|
41
|
-
open = false;
|
|
42
|
-
timer.clear(handle);
|
|
43
|
-
outer?.removeEventListener('abort', forward);
|
|
44
|
-
},
|
|
45
|
-
};
|
|
46
|
-
}
|