@ultimat3/scraping 30.9.0 → 31.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -8
- package/package.json +9 -6
- package/src/actionability.ts +2 -2
- package/src/adapter.ts +22 -0
- package/src/auth.ts +4 -5
- package/src/browser-dial.ts +55 -0
- package/src/cdp-arm.ts +81 -22
- package/src/cdp-connection.ts +290 -0
- package/src/cdp-error-codes.ts +28 -0
- package/src/cdp-errors.ts +179 -0
- package/src/cdp-launch-attempt.ts +234 -0
- package/src/cdp-launch-candidates.ts +62 -0
- package/src/cdp-launch-probe.ts +75 -0
- package/src/cdp-launch-profile.ts +106 -0
- package/src/cdp-launch-reap.ts +68 -0
- package/src/cdp-launch-wire.ts +171 -0
- package/src/cdp-launch.ts +216 -0
- package/src/cdp-pipe.ts +77 -0
- package/src/cdp-wire-watch.ts +113 -0
- package/src/cdp.ts +46 -0
- package/src/clock.ts +4 -111
- package/src/driver-cdp.ts +3 -0
- package/src/driver-fake.ts +4 -5
- package/src/driver.ts +16 -11
- package/src/error-throws-pdf.ts +45 -0
- package/src/error-throws-session.ts +1 -1
- package/src/error-throws.ts +4 -4
- package/src/errors.ts +17 -1
- package/src/event-prompt.ts +10 -4
- package/src/events.ts +2 -3
- package/src/html-target.ts +2 -2
- package/src/http-recorded.ts +2 -2
- package/src/http.ts +2 -2
- package/src/index.ts +21 -75
- package/src/launch-args.ts +10 -1
- package/src/page-over-target.ts +2 -3
- package/src/pdf-action.ts +116 -0
- package/src/pdf-cdp-params.ts +65 -0
- package/src/pdf-fence.ts +86 -0
- package/src/pdf-render.ts +241 -0
- package/src/pdf-slots.ts +68 -0
- package/src/rate.ts +2 -2
- package/src/scrape-run.ts +3 -4
- package/src/scrape.ts +5 -5
- package/src/target.ts +1 -1
- package/src/usage.ts +2 -2
- package/src/watchdog.ts +3 -3
package/src/clock.ts
CHANGED
|
@@ -1,118 +1,11 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// `clock-discipline.test.ts` fails the build if a second file reaches for a timer directly.
|
|
1
|
+
// Elapsed-time budgets for every wait a scraper does — the actionability poll, the wedge watchdog,
|
|
2
|
+
// the rate limiter, the graceful-quit ceiling. The waiting itself is `@ultimat3/core`'s `SleepClock`
|
|
3
|
+
// (`ambientClock()` unless a definition pins one), so a suite that exercises a 30-second timeout
|
|
4
|
+
// finishes in microseconds. `clock-discipline.test.ts` fails the build if a file here arms a timer.
|
|
6
5
|
|
|
7
6
|
import type { Clock } from '@ultimat3/core';
|
|
8
7
|
import { finiteCount } from '@ultimat3/core';
|
|
9
8
|
|
|
10
|
-
export interface ScrapeClock extends Clock {
|
|
11
|
-
/**
|
|
12
|
-
* Resolve after `ms` of this clock's time, or reject with the signal's reason the moment it
|
|
13
|
-
* aborts. Every poll loop here composes a wait with a cancellation, so a clock whose sleep
|
|
14
|
-
* ignored the signal would leave a killed run still counting.
|
|
15
|
-
*/
|
|
16
|
-
sleep(ms: number, signal?: AbortSignal): Promise<void>;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export const systemScrapeClock: ScrapeClock = Object.freeze({
|
|
20
|
-
now(): Date {
|
|
21
|
-
return new Date();
|
|
22
|
-
},
|
|
23
|
-
monotonic(): number {
|
|
24
|
-
return performance.now();
|
|
25
|
-
},
|
|
26
|
-
async sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
27
|
-
if (signal === undefined) {
|
|
28
|
-
await Bun.sleep(ms);
|
|
29
|
-
return;
|
|
30
|
-
}
|
|
31
|
-
signal.throwIfAborted();
|
|
32
|
-
// A timer raced against the signal, never `Bun.sleep(ms).then(check)`: the watchdog aborts a
|
|
33
|
-
// wedged run precisely so nothing waits out the rest of a five-minute budget, and a sleep
|
|
34
|
-
// that only notices the abort when it expires would wait out every one of them.
|
|
35
|
-
await new Promise<void>((resolve, reject) => {
|
|
36
|
-
// The reason is rejected VERBATIM, never wrapped: whoever aborted put a `ScrapeError` with
|
|
37
|
-
// a code and a fix there, and re-wrapping it would replace an instruction with `Error`.
|
|
38
|
-
const onAbort = (): void => {
|
|
39
|
-
clearTimeout(timer);
|
|
40
|
-
reject(signal.reason);
|
|
41
|
-
};
|
|
42
|
-
const timer = setTimeout(() => {
|
|
43
|
-
signal.removeEventListener('abort', onAbort);
|
|
44
|
-
resolve();
|
|
45
|
-
}, ms);
|
|
46
|
-
signal.addEventListener('abort', onAbort, { once: true });
|
|
47
|
-
});
|
|
48
|
-
},
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* The system clock with the waiting taken out: `now()` and `monotonic()` are real, and every
|
|
53
|
-
* `sleep` is ONE turn of the event loop whatever was asked for. For a test whose run waits on
|
|
54
|
-
* something that really happens in the same process — an answer the test publishes, a job another
|
|
55
|
-
* worker holds — where `testScrapeClock()`'s "sleeping is advancing" would run a five-minute budget out
|
|
56
|
-
* in microtasks before the test's next line. A deadline is still a real one.
|
|
57
|
-
*/
|
|
58
|
-
export const noWaitClock: ScrapeClock = Object.freeze({
|
|
59
|
-
now: () => systemScrapeClock.now(),
|
|
60
|
-
monotonic: () => systemScrapeClock.monotonic(),
|
|
61
|
-
sleep: (_ms: number, signal?: AbortSignal) => systemScrapeClock.sleep(0, signal),
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
let ambient: ScrapeClock | undefined;
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* The clock every run in this process waits on, unless its definition pins one — the counterpart
|
|
68
|
-
* of `setScrapeDriver()`, and the seam a TEST takes: `setScrapeClock(noWaitClock)` in a
|
|
69
|
-
* `beforeEach`, and a declaration under test needs no `clock:` field written for the test's sake.
|
|
70
|
-
*/
|
|
71
|
-
export function setScrapeClock(clock: ScrapeClock): void {
|
|
72
|
-
ambient = clock;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/** The process's scrape clock: the installed one, or the system's. */
|
|
76
|
-
export function scrapeClock(): ScrapeClock {
|
|
77
|
-
return ambient ?? systemScrapeClock;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/** Test seam: back to the system clock, so the next file runs on real time again. */
|
|
81
|
-
export function resetScrapeClock(): void {
|
|
82
|
-
ambient = undefined;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
/**
|
|
86
|
-
* A clock in which sleeping IS advancing: `await clock.sleep(30_000)` returns on the next
|
|
87
|
-
* microtask with thirty seconds elapsed. This is what makes a poll-until-deadline test instant
|
|
88
|
-
* without the test knowing how many polls the loop makes — a count a test that asserted it would
|
|
89
|
-
* pin to the implementation rather than to the behaviour.
|
|
90
|
-
*/
|
|
91
|
-
export interface TestScrapeClock extends ScrapeClock {
|
|
92
|
-
advance(ms: number): void;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
export function testScrapeClock(at: Date | number = 0): TestScrapeClock {
|
|
96
|
-
let epochMs = at instanceof Date ? at.getTime() : at;
|
|
97
|
-
let mono = 0;
|
|
98
|
-
const advance = (ms: number): void => {
|
|
99
|
-
epochMs += ms;
|
|
100
|
-
mono += ms;
|
|
101
|
-
};
|
|
102
|
-
return {
|
|
103
|
-
now: () => new Date(epochMs),
|
|
104
|
-
monotonic: () => mono,
|
|
105
|
-
advance,
|
|
106
|
-
sleep: async (ms: number, signal?: AbortSignal) => {
|
|
107
|
-
advance(ms);
|
|
108
|
-
// Yield the microtask queue anyway: a loop that never awaits anything real starves whatever
|
|
109
|
-
// the test armed to cancel it, and the wedge tests arm exactly that.
|
|
110
|
-
await Promise.resolve();
|
|
111
|
-
signal?.throwIfAborted();
|
|
112
|
-
},
|
|
113
|
-
};
|
|
114
|
-
}
|
|
115
|
-
|
|
116
9
|
/** Elapsed-time budget, read from one clock. Built once per waiting call, never shared. */
|
|
117
10
|
export interface Deadline {
|
|
118
11
|
readonly totalMs: number;
|
package/src/driver-cdp.ts
CHANGED
|
@@ -111,6 +111,9 @@ async function opened(
|
|
|
111
111
|
browser,
|
|
112
112
|
rules: init.rules,
|
|
113
113
|
clock: init.clock,
|
|
114
|
+
// Through an exit the EXIT resolves names, so the browser leg's DNS screen stands down — the
|
|
115
|
+
// HTTP leg's rule, for the same reason (`browser-dial.ts`).
|
|
116
|
+
proxied: wiring.proxy !== undefined,
|
|
114
117
|
ended,
|
|
115
118
|
});
|
|
116
119
|
if (init.restore !== undefined) await target.restore(init.restore);
|
package/src/driver-fake.ts
CHANGED
|
@@ -5,9 +5,8 @@
|
|
|
5
5
|
// It covers BOTH legs. `fakeBrowser({ pages, http })` replays the browser walk and the JSON
|
|
6
6
|
// endpoints behind it from one declaration, so a hybrid scrape is tested the way it runs.
|
|
7
7
|
|
|
8
|
-
import {
|
|
9
|
-
import
|
|
10
|
-
import { systemScrapeClock } from './clock';
|
|
8
|
+
import type { SleepClock } from '@ultimat3/core';
|
|
9
|
+
import { ambientClock, finiteCount } from '@ultimat3/core';
|
|
11
10
|
import type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
|
|
12
11
|
import { htmlTarget } from './html-target';
|
|
13
12
|
import { httpRecordingsOf } from './http-recorded';
|
|
@@ -78,7 +77,7 @@ export function fakeBrowser(pages: FakePages, options: FakeBrowserOptions = {}):
|
|
|
78
77
|
|
|
79
78
|
export interface FakePageOptions {
|
|
80
79
|
readonly url?: string | undefined;
|
|
81
|
-
readonly clock?:
|
|
80
|
+
readonly clock?: SleepClock | undefined;
|
|
82
81
|
readonly allowHosts?: readonly string[] | undefined;
|
|
83
82
|
readonly timeoutMs?: number | undefined;
|
|
84
83
|
readonly context?: Partial<PageContext> | undefined;
|
|
@@ -97,7 +96,7 @@ export const FAKE_PAGE_URL = 'https://fake.test/';
|
|
|
97
96
|
export function fakePage(dom: string, options: FakePageOptions = {}): ScrapePage {
|
|
98
97
|
const url = options.url ?? FAKE_PAGE_URL;
|
|
99
98
|
const start: PageRecording = { url, html: dom };
|
|
100
|
-
const clock = options.clock ??
|
|
99
|
+
const clock = options.clock ?? ambientClock();
|
|
101
100
|
const allowHosts = options.allowHosts ?? ['*'];
|
|
102
101
|
const byUrl = new Map(
|
|
103
102
|
[start, ...recordingsOf(options.pages ?? [])].map((page) => [normalise(page.url), page]),
|
package/src/driver.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
// The browser contract. Every driver implements exactly this, so a scrape's body never names one
|
|
2
2
|
// — the same shape `packages/jobs/src/driver.ts` gives the queue, for the same reason: swapping
|
|
3
|
-
// the browser is `
|
|
3
|
+
// the browser is `scrapeDriver: other` in `apps/<app>/runtime.ts` and ZERO run-body change.
|
|
4
4
|
//
|
|
5
5
|
// Two methods. `open` hands back a session; `close` ends it. Everything else a scraper does goes
|
|
6
6
|
// through `ScrapePage`, which is driver-blind by construction.
|
|
7
7
|
|
|
8
|
-
import type { Logger } from '@ultimat3/core';
|
|
8
|
+
import type { Logger, SleepClock } from '@ultimat3/core';
|
|
9
|
+
import { adapterSlot } from '@ultimat3/core/adapter';
|
|
9
10
|
import type { MoneyValue } from '@ultimat3/schema';
|
|
10
|
-
import type { ScrapeClock } from './clock';
|
|
11
11
|
import type { ScrapeHttp } from './http';
|
|
12
12
|
import type { InterceptRules } from './intercept';
|
|
13
13
|
import type { ScrapePage } from './page';
|
|
@@ -20,7 +20,7 @@ export interface SessionInit {
|
|
|
20
20
|
/** The scrape's name — every error cause raised inside the session carries it. */
|
|
21
21
|
readonly name: string;
|
|
22
22
|
readonly rules: InterceptRules;
|
|
23
|
-
readonly clock:
|
|
23
|
+
readonly clock: SleepClock;
|
|
24
24
|
/**
|
|
25
25
|
* The run's logger — `scrapeLogger()`'s child, so every line carries the scrape and the run.
|
|
26
26
|
* Required, because `close()` never throws: a teardown that failed (a rented browser its
|
|
@@ -100,22 +100,27 @@ export interface ScrapeDriver {
|
|
|
100
100
|
open(init: SessionInit): Promise<ScrapeSession>;
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
-
|
|
103
|
+
const ambient = /* @__PURE__ */ adapterSlot<ScrapeDriver | undefined>(() => undefined);
|
|
104
104
|
|
|
105
|
-
/**
|
|
106
|
-
|
|
107
|
-
|
|
105
|
+
/**
|
|
106
|
+
* The boot's install, from `scrapeDriver` in `apps/<app>/runtime.ts` — exported from
|
|
107
|
+
* `@ultimat3/scraping/adapter` only, never the `.` barrel, so that file is the one place an app
|
|
108
|
+
* chooses a browser. A scrape's own `driver:` overrides it. Answers the release that removes THIS
|
|
109
|
+
* install and no other (core's `adapterSlot`), in any order, once.
|
|
110
|
+
*/
|
|
111
|
+
export function installScrapeDriver(driver: ScrapeDriver): () => void {
|
|
112
|
+
return ambient.install(driver);
|
|
108
113
|
}
|
|
109
114
|
|
|
110
115
|
export function scrapeDriver(): ScrapeDriver | undefined {
|
|
111
|
-
return ambient;
|
|
116
|
+
return ambient.current();
|
|
112
117
|
}
|
|
113
118
|
|
|
114
119
|
/**
|
|
115
|
-
* Test/CLI seam: forget the ambient driver. The counterpart to `resetJobDriver()` — a test that
|
|
120
|
+
* Test/CLI seam: forget the ambient driver. The counterpart to `resetJobDriver()` (`@ultimat3/jobs/adapter`) — a test that
|
|
116
121
|
* installs a browser has to be able to put the process back, or every later file in the same bun
|
|
117
122
|
* process runs against a driver it never asked for.
|
|
118
123
|
*/
|
|
119
124
|
export function resetScrapeDriver(): void {
|
|
120
|
-
ambient
|
|
125
|
+
ambient.reset();
|
|
121
126
|
}
|
package/src/error-throws-pdf.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// The failure mode of a PDF layout request. Split from `error-throws.ts`, which stands at its size
|
|
2
2
|
// ceiling; the code itself stays declared in `errors.ts` — one registry, one place.
|
|
3
3
|
|
|
4
|
+
import { isRetryableStatus, renderCauseValue, renderFixShellArg } from '@ultimat3/core';
|
|
4
5
|
import { ScrapeError } from './errors';
|
|
5
6
|
|
|
6
7
|
/**
|
|
@@ -14,3 +15,47 @@ export const pdfOptionsInvalid = (reason: string, meta: { readonly field: string
|
|
|
14
15
|
fix: "page.pdf({ format: 'a4', marginMm: { top: 12, bottom: 12 } }) — exactly one size: a format, or widthMm with heightMm, or preferCssPageSize alone",
|
|
15
16
|
meta: { field: meta.field },
|
|
16
17
|
});
|
|
18
|
+
|
|
19
|
+
/** No server to load the route from: `APP_URL` unset and no `origin` on the declaration. */
|
|
20
|
+
export const pdfOriginMissing = (): ScrapeError =>
|
|
21
|
+
new ScrapeError({
|
|
22
|
+
code: 'X_SCRAPE_PDF_ORIGIN_MISSING',
|
|
23
|
+
cause:
|
|
24
|
+
'pdf() has no origin to load its route from: APP_URL is unset in this process and the declaration names no origin',
|
|
25
|
+
fix: "export APP_URL=https://your.app # or pdf({ origin: () => 'http://127.0.0.1:3000' }) in the declaring file",
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
/** `route()` answered something the printing browser would load from another origin. */
|
|
29
|
+
export const pdfRouteInvalid = (route: string): ScrapeError =>
|
|
30
|
+
new ScrapeError({
|
|
31
|
+
code: 'X_SCRAPE_PDF_ROUTE_INVALID',
|
|
32
|
+
cause: `pdf() route() answered ${renderCauseValue(route)}, which is not a path on this app's origin — the printing browser loads nothing else`,
|
|
33
|
+
fix: "pdf({ route: ({ input }) => '/invoices/' + input.id + '/print' }) # route() answers a path on this app, never a URL",
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The page answered an error status. A 4xx that no retry can change (a missing record, a route the
|
|
38
|
+
* printing browser may not read) is terminal on the instance; a 5xx or 429 stays retryable.
|
|
39
|
+
*/
|
|
40
|
+
export const pdfRouteFailed = (url: string, status: number): ScrapeError => {
|
|
41
|
+
const path = new URL(url).pathname;
|
|
42
|
+
return new ScrapeError({
|
|
43
|
+
code: 'X_SCRAPE_PDF_ROUTE_FAILED',
|
|
44
|
+
cause: `the route pdf() printed, ${renderCauseValue(path)}, answered HTTP ${String(status)}: printing it would store an error page as the document`,
|
|
45
|
+
fix: `curl -i "$APP_URL"${renderFixShellArg(path, '/<the path the cause names>')} # a private route needs pdf({ headers }); a missing one, a route() naming a record that exists`,
|
|
46
|
+
meta: { status },
|
|
47
|
+
retry: status >= 400 && status < 500 && !isRetryableStatus(status) ? 'terminal' : 'retryable',
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/** Every render slot stayed busy for the whole queue deadline: a burst, not a broken page. */
|
|
52
|
+
export const pdfBusy = (input: {
|
|
53
|
+
readonly maxConcurrent: number;
|
|
54
|
+
readonly queueTimeoutMs: number;
|
|
55
|
+
}): ScrapeError =>
|
|
56
|
+
new ScrapeError({
|
|
57
|
+
code: 'X_SCRAPE_PDF_BUSY',
|
|
58
|
+
cause: `all ${String(input.maxConcurrent)} pdf() render slot(s) stayed busy for ${String(input.queueTimeoutMs)}ms — each holds a live Chrome, so the renderer bounds them rather than let a burst exhaust this process's memory`,
|
|
59
|
+
fix: `pdf({ renderer: chromePdfRenderer({ maxConcurrent: ${String(input.maxConcurrent * 2)} }) }) # only on a host with the memory for it; otherwise the caller retries later`,
|
|
60
|
+
meta: { maxConcurrent: input.maxConcurrent, queueTimeoutMs: input.queueTimeoutMs },
|
|
61
|
+
});
|
|
@@ -51,7 +51,7 @@ export const promptTimedOut = (input: {
|
|
|
51
51
|
new ScrapeError({
|
|
52
52
|
code: 'X_SCRAPE_PROMPT_UNANSWERED',
|
|
53
53
|
cause: `scrape "${input.scrape}" asked for "${input.label}" and no answer was published to ${input.event} within ${String(input.waitedMs)}ms`,
|
|
54
|
-
fix: 'call answerPrompt({ runId, index, answer }) with the run id and index the event name in the cause ends with, from a process on the same STORED event bus —
|
|
54
|
+
fix: 'call answerPrompt({ runId, index, answer }) with the run id and index the event name in the cause ends with, from a process on the same STORED event bus — eventBus: postgresEventBus({ executor }) in apps/<app>/runtime.ts — or raise eventPrompt({ timeout })',
|
|
55
55
|
meta: {
|
|
56
56
|
scrape: input.scrape,
|
|
57
57
|
label: input.label,
|
package/src/error-throws.ts
CHANGED
|
@@ -30,8 +30,8 @@ export const driverUnknown = (
|
|
|
30
30
|
cause:
|
|
31
31
|
scrape === undefined
|
|
32
32
|
? `no scrape driver named "${name ?? 'none'}" is installed; installed: ${installed.join(', ') || 'none'}`
|
|
33
|
-
: `scrape "${scrape}" has no browser driver:
|
|
34
|
-
fix: '
|
|
33
|
+
: `scrape "${scrape}" has no browser driver: apps/<app>/runtime.ts names no scrapeDriver and the definition declares no driver; installed: ${installed.join(', ') || 'none'}`,
|
|
34
|
+
fix: 'export const runtime = { scrapeDriver: localBrowser() } # apps/<app>/runtime.ts — or pass driver: fakeBrowser() on the scrape() definition',
|
|
35
35
|
meta: { driver: name ?? 'none', ...(scrape === undefined ? {} : { scrape }) },
|
|
36
36
|
});
|
|
37
37
|
|
|
@@ -217,7 +217,7 @@ export const wedged = (what: string, idleMs: number): ScrapeError =>
|
|
|
217
217
|
* opposite fixes are two codes, and the classifications differ with them: a wedge is retryable,
|
|
218
218
|
* this is terminal (`errors.ts`).
|
|
219
219
|
*
|
|
220
|
-
* Reachable through the `
|
|
220
|
+
* Reachable through the `SleepClock` seam and the driver's `kill()`, both of which a third party
|
|
221
221
|
* writes. Leaving the loop dead and quiet is the alternative, and that is incident #1 with no
|
|
222
222
|
* guard armed at all.
|
|
223
223
|
*/
|
|
@@ -225,7 +225,7 @@ export const watchdogStopped = (what: string, thrown: unknown): ScrapeError =>
|
|
|
225
225
|
new ScrapeError({
|
|
226
226
|
code: 'X_SCRAPE_WATCHDOG_STOPPED',
|
|
227
227
|
cause: `the wedge watchdog for ${what} stopped measuring browser activity: ${renderThrowable(thrown)}`,
|
|
228
|
-
fix:
|
|
228
|
+
fix: "drop the custom clock: from the scrape() definition and any clock: line in apps/<app>/runtime.ts — @ultimat3/core's systemClock is the only SleepClock whose sleep() cannot reject — then re-run",
|
|
229
229
|
meta: { what },
|
|
230
230
|
});
|
|
231
231
|
|
package/src/errors.ts
CHANGED
|
@@ -43,6 +43,10 @@ export const SCRAPE_OWNED_ERROR_CODES = [
|
|
|
43
43
|
'X_SCRAPE_EGRESS_IN_PAYLOAD',
|
|
44
44
|
'X_SCRAPE_LAUNCH_ARGS_INVALID',
|
|
45
45
|
'X_SCRAPE_PDF_OPTIONS_INVALID',
|
|
46
|
+
'X_SCRAPE_PDF_ORIGIN_MISSING',
|
|
47
|
+
'X_SCRAPE_PDF_ROUTE_INVALID',
|
|
48
|
+
'X_SCRAPE_PDF_ROUTE_FAILED',
|
|
49
|
+
'X_SCRAPE_PDF_BUSY',
|
|
46
50
|
] as const;
|
|
47
51
|
|
|
48
52
|
/**
|
|
@@ -101,6 +105,10 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
|
|
|
101
105
|
'the launch options carry arguments this driver cannot hand the browser',
|
|
102
106
|
X_SCRAPE_PDF_OPTIONS_INVALID:
|
|
103
107
|
'the PDF request names no single page size a print engine can lay out',
|
|
108
|
+
X_SCRAPE_PDF_ORIGIN_MISSING: 'pdf() has no origin to render the route on',
|
|
109
|
+
X_SCRAPE_PDF_ROUTE_INVALID: 'a pdf() route is not a path on this app',
|
|
110
|
+
X_SCRAPE_PDF_ROUTE_FAILED: 'the route pdf() printed answered an error status',
|
|
111
|
+
X_SCRAPE_PDF_BUSY: 'every pdf() render slot stayed busy past the queue deadline',
|
|
104
112
|
};
|
|
105
113
|
|
|
106
114
|
// One unconditional call, so a second package claiming one of these codes throws
|
|
@@ -181,7 +189,7 @@ export const SCRAPE_ERROR_RETRY = {
|
|
|
181
189
|
// TERMINAL where its sibling `X_SCRAPE_WEDGED` is retryable, and the difference IS the reason
|
|
182
190
|
// the two codes are separate. A wedge is the site or the browser being slow — the definition of
|
|
183
191
|
// "run it again and it may go differently". This is the guard's own loop dying on code the
|
|
184
|
-
// DEFINITION supplied: a `
|
|
192
|
+
// DEFINITION supplied: a `SleepClock` an app wrote, reached identically on attempt 2. Retrying
|
|
185
193
|
// launches a real browser five times to die at the first poll, and on an authenticated target
|
|
186
194
|
// that is five arrivals at a login for no chance of a different answer — the rule this whole
|
|
187
195
|
// table is written to, stated at the top of it. The fix is an edit, so a human decides.
|
|
@@ -214,6 +222,14 @@ export const SCRAPE_ERROR_RETRY = {
|
|
|
214
222
|
// The request is the caller's literal, refused before any driver sees it: attempt 2 asks for the
|
|
215
223
|
// same two sizes and is refused identically.
|
|
216
224
|
X_SCRAPE_PDF_OPTIONS_INVALID: 'terminal',
|
|
225
|
+
// The deploy's environment and the declaration's own `route()`: attempt 2 reads the same ones.
|
|
226
|
+
X_SCRAPE_PDF_ORIGIN_MISSING: 'terminal',
|
|
227
|
+
X_SCRAPE_PDF_ROUTE_INVALID: 'terminal',
|
|
228
|
+
// The page answered an error: a 503 mid-deploy is the moment, so a `.job()` run tries again.
|
|
229
|
+
// The throw site narrows a 4xx to terminal — a missing record is missing on attempt 2 too.
|
|
230
|
+
X_SCRAPE_PDF_ROUTE_FAILED: 'retryable',
|
|
231
|
+
// Every slot busy is the moment, never the request: the same render fits once one ends.
|
|
232
|
+
X_SCRAPE_PDF_BUSY: 'retryable',
|
|
217
233
|
} as const satisfies Readonly<Record<ScrapeOwnedErrorCode, 'retryable' | 'terminal'>>;
|
|
218
234
|
|
|
219
235
|
registerErrorRetry(SCRAPE_ERROR_RETRY);
|
package/src/event-prompt.ts
CHANGED
|
@@ -54,7 +54,7 @@ export interface EventPromptOptions {
|
|
|
54
54
|
* Only an answer published AFTER the prompt was asked is consumed. The run id is the same on
|
|
55
55
|
* every attempt, so a code left over from attempt 1 would otherwise be typed into the site by
|
|
56
56
|
* attempt 2 — and a stale code is a refused login. "After" is decided on ONE clock, the bus's
|
|
57
|
-
* (`bus.now()`); the timeout and the poll stay on the run's own.
|
|
57
|
+
* (`bus.mark()`, or `bus.now()`); the timeout and the poll stay on the run's own.
|
|
58
58
|
*/
|
|
59
59
|
export function eventPrompt(options: EventPromptOptions): PromptHandler {
|
|
60
60
|
// Floors of 1, both: a zero timeout is a prompt that can never be answered and a zero interval
|
|
@@ -73,18 +73,24 @@ export function eventPrompt(options: EventPromptOptions): PromptHandler {
|
|
|
73
73
|
driver: 'event bus',
|
|
74
74
|
cause:
|
|
75
75
|
'eventPrompt() was asked on the in-memory event bus — the answer is published by another process and can never reach this one',
|
|
76
|
-
fix: '
|
|
76
|
+
fix: 'export const runtime = { eventBus: postgresEventBus({ executor }) } # apps/<app>/runtime.ts — or delete that line: x dev and the served roles install it — or pass bus: postgresEventBus({ executor }) to eventPrompt()',
|
|
77
77
|
});
|
|
78
78
|
}
|
|
79
79
|
const event = promptEventName(request.runId, request.index);
|
|
80
80
|
// The BUS's clock, read once: an answer's `publishedAt` is stamped by it — the database, for
|
|
81
81
|
// the stored bus — and `request.clock` is this worker's. Comparing the two let a few seconds
|
|
82
82
|
// of skew between pods decide that an answer given after the ask had come before it.
|
|
83
|
-
|
|
83
|
+
// Read with the bus's MARK where it has one, in the same statement: an answer stamped at
|
|
84
|
+
// INSERT and committed after this instant carries a stamp before it, and only a search under
|
|
85
|
+
// the mark's horizon still sees it (`EventLookup.mark`).
|
|
86
|
+
const mark: { readonly at: number; readonly horizon?: string } = (await bus.mark?.()) ?? {
|
|
87
|
+
at: await bus.now(),
|
|
88
|
+
};
|
|
89
|
+
const askedAt = mark.at;
|
|
84
90
|
const budget = deadline(request.clock, timeoutMs, 'eventPrompt');
|
|
85
91
|
for (;;) {
|
|
86
92
|
request.signal?.throwIfAborted();
|
|
87
|
-
const found = await bus.find(event, undefined, askedAt);
|
|
93
|
+
const found = await bus.find(event, undefined, askedAt, undefined, mark.horizon, mark);
|
|
88
94
|
if (found !== undefined) {
|
|
89
95
|
if (!isSealed(found.payload)) {
|
|
90
96
|
throw promptAnswerInvalid({ scrape: request.scrape, label: request.label, event });
|
package/src/events.ts
CHANGED
|
@@ -11,8 +11,7 @@
|
|
|
11
11
|
// of a log line: `ScrapeEventFields` has no key to put one in, and core's logger redacts the
|
|
12
12
|
// remaining spellings by key and every `Secret` by value.
|
|
13
13
|
|
|
14
|
-
import type { Logger } from '@ultimat3/core';
|
|
15
|
-
import type { ScrapeClock } from './clock';
|
|
14
|
+
import type { Logger, SleepClock } from '@ultimat3/core';
|
|
16
15
|
import { errorCode } from './failures';
|
|
17
16
|
|
|
18
17
|
export interface ScrapeEventFields {
|
|
@@ -48,7 +47,7 @@ export const scrapeLogger = (logger: Logger, fields: ScrapeEventFields): Logger
|
|
|
48
47
|
export interface StepEvent {
|
|
49
48
|
readonly name: string;
|
|
50
49
|
readonly logger: Logger;
|
|
51
|
-
readonly clock:
|
|
50
|
+
readonly clock: SleepClock;
|
|
52
51
|
readonly attempt?: number | undefined;
|
|
53
52
|
}
|
|
54
53
|
|
package/src/html-target.ts
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
// through to the network would make a green offline suite that is secretly hitting production —
|
|
6
6
|
// the exact failure an offline driver exists to prevent.
|
|
7
7
|
|
|
8
|
+
import type { SleepClock } from '@ultimat3/core';
|
|
8
9
|
import { NotImplementedError } from '@ultimat3/core';
|
|
9
10
|
import type { CaptureClip } from './capture-clip';
|
|
10
|
-
import type { ScrapeClock } from './clock';
|
|
11
11
|
import type { ColorScheme } from './color-scheme';
|
|
12
12
|
import { browserUnreachable, downloadTimeout, fixtureMissing, fixtureStale } from './error-throws';
|
|
13
13
|
import { queryHtml } from './html-query';
|
|
@@ -84,7 +84,7 @@ export interface HtmlTargetInit {
|
|
|
84
84
|
readonly driver: string;
|
|
85
85
|
readonly lookup: RecordingLookup;
|
|
86
86
|
readonly rules: InterceptRules;
|
|
87
|
-
readonly clock:
|
|
87
|
+
readonly clock: SleepClock;
|
|
88
88
|
/** Where recordings come from, for the cause line: a directory, or `fakeBrowser()`. */
|
|
89
89
|
readonly source: string;
|
|
90
90
|
readonly start?: PageRecording | undefined;
|
package/src/http-recorded.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// real network would make the hybrid path — which is where the interesting code lives — the one
|
|
6
6
|
// path a test never really covers.
|
|
7
7
|
|
|
8
|
-
import type {
|
|
8
|
+
import type { SleepClock } from '@ultimat3/core';
|
|
9
9
|
import { fixtureMissing, fixtureStale, hostBlocked } from './error-throws';
|
|
10
10
|
import type { HttpRequestInit, ScrapeHttp, ScrapeResponse } from './http';
|
|
11
11
|
import { responseOver } from './http';
|
|
@@ -26,7 +26,7 @@ export interface RecordedHttpInit {
|
|
|
26
26
|
readonly lookup: HttpRecordingLookup;
|
|
27
27
|
readonly rules: InterceptRules;
|
|
28
28
|
readonly network: NetworkRing;
|
|
29
|
-
readonly clock:
|
|
29
|
+
readonly clock: SleepClock;
|
|
30
30
|
readonly source: string;
|
|
31
31
|
/**
|
|
32
32
|
* The SAME gate the page leg holds. Without it a hybrid scrape whose JSON endpoint is
|
package/src/http.ts
CHANGED
|
@@ -10,11 +10,11 @@
|
|
|
10
10
|
// guarantee the page vocabulary makes — and a different exit IP mid-session is exactly what
|
|
11
11
|
// anti-bot systems look for.
|
|
12
12
|
|
|
13
|
+
import type { SleepClock } from '@ultimat3/core';
|
|
13
14
|
import { finiteCount, readWithinLimit } from '@ultimat3/core';
|
|
14
15
|
import { type DeadlineTimer, deadlineSignal, resolveDialHost } from '@ultimat3/core/outbound';
|
|
15
16
|
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
16
17
|
import { parse } from '@ultimat3/schema';
|
|
17
|
-
import type { ScrapeClock } from './clock';
|
|
18
18
|
import { cookieHeaderFor } from './cookie-scope';
|
|
19
19
|
import { bodyTooLarge, hostBlocked, httpFailed, redirectLoop, scrapeTimeout } from './error-throws';
|
|
20
20
|
import type { RedirectHop } from './http-redirect';
|
|
@@ -96,7 +96,7 @@ export interface ScrapeHttp {
|
|
|
96
96
|
|
|
97
97
|
export interface HttpTransportInit {
|
|
98
98
|
readonly rules: InterceptRules;
|
|
99
|
-
readonly clock:
|
|
99
|
+
readonly clock: SleepClock;
|
|
100
100
|
readonly timeoutMs: number;
|
|
101
101
|
readonly network: NetworkRing;
|
|
102
102
|
/**
|