@ultimat3/scraping 30.9.0 → 31.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +73 -8
  2. package/package.json +9 -6
  3. package/src/actionability.ts +2 -2
  4. package/src/adapter.ts +22 -0
  5. package/src/auth.ts +4 -5
  6. package/src/browser-dial.ts +55 -0
  7. package/src/cdp-arm.ts +81 -22
  8. package/src/cdp-connection.ts +290 -0
  9. package/src/cdp-error-codes.ts +28 -0
  10. package/src/cdp-errors.ts +179 -0
  11. package/src/cdp-launch-attempt.ts +234 -0
  12. package/src/cdp-launch-candidates.ts +62 -0
  13. package/src/cdp-launch-probe.ts +75 -0
  14. package/src/cdp-launch-profile.ts +106 -0
  15. package/src/cdp-launch-reap.ts +68 -0
  16. package/src/cdp-launch-wire.ts +171 -0
  17. package/src/cdp-launch.ts +216 -0
  18. package/src/cdp-pipe.ts +77 -0
  19. package/src/cdp-wire-watch.ts +113 -0
  20. package/src/cdp.ts +46 -0
  21. package/src/clock.ts +4 -111
  22. package/src/driver-cdp.ts +3 -0
  23. package/src/driver-fake.ts +4 -5
  24. package/src/driver.ts +16 -11
  25. package/src/error-throws-pdf.ts +45 -0
  26. package/src/error-throws-session.ts +1 -1
  27. package/src/error-throws.ts +4 -4
  28. package/src/errors.ts +17 -1
  29. package/src/event-prompt.ts +10 -4
  30. package/src/events.ts +2 -3
  31. package/src/html-target.ts +2 -2
  32. package/src/http-recorded.ts +2 -2
  33. package/src/http.ts +2 -2
  34. package/src/index.ts +21 -75
  35. package/src/launch-args.ts +10 -1
  36. package/src/page-over-target.ts +2 -3
  37. package/src/pdf-action.ts +116 -0
  38. package/src/pdf-cdp-params.ts +65 -0
  39. package/src/pdf-fence.ts +86 -0
  40. package/src/pdf-render.ts +241 -0
  41. package/src/pdf-slots.ts +68 -0
  42. package/src/rate.ts +2 -2
  43. package/src/scrape-run.ts +3 -4
  44. package/src/scrape.ts +5 -5
  45. package/src/target.ts +1 -1
  46. package/src/usage.ts +2 -2
  47. package/src/watchdog.ts +3 -3
package/src/clock.ts CHANGED
@@ -1,118 +1,11 @@
1
- // The ONE file in this package allowed to make time pass. Every wait a scraper does — the
2
- // actionability poll, the wedge watchdog, the rate limiter, the graceful-quit ceiling — goes
3
- // through `ScrapeClock`, so a suite that exercises a 30-second timeout finishes in microseconds.
4
- //
5
- // `clock-discipline.test.ts` fails the build if a second file reaches for a timer directly.
1
+ // Elapsed-time budgets for every wait a scraper does — the actionability poll, the wedge watchdog,
2
+ // the rate limiter, the graceful-quit ceiling. The waiting itself is `@ultimat3/core`'s `SleepClock`
3
+ // (`ambientClock()` unless a definition pins one), so a suite that exercises a 30-second timeout
4
+ // finishes in microseconds. `clock-discipline.test.ts` fails the build if a file here arms a timer.
6
5
 
7
6
  import type { Clock } from '@ultimat3/core';
8
7
  import { finiteCount } from '@ultimat3/core';
9
8
 
10
- export interface ScrapeClock extends Clock {
11
- /**
12
- * Resolve after `ms` of this clock's time, or reject with the signal's reason the moment it
13
- * aborts. Every poll loop here composes a wait with a cancellation, so a clock whose sleep
14
- * ignored the signal would leave a killed run still counting.
15
- */
16
- sleep(ms: number, signal?: AbortSignal): Promise<void>;
17
- }
18
-
19
- export const systemScrapeClock: ScrapeClock = Object.freeze({
20
- now(): Date {
21
- return new Date();
22
- },
23
- monotonic(): number {
24
- return performance.now();
25
- },
26
- async sleep(ms: number, signal?: AbortSignal): Promise<void> {
27
- if (signal === undefined) {
28
- await Bun.sleep(ms);
29
- return;
30
- }
31
- signal.throwIfAborted();
32
- // A timer raced against the signal, never `Bun.sleep(ms).then(check)`: the watchdog aborts a
33
- // wedged run precisely so nothing waits out the rest of a five-minute budget, and a sleep
34
- // that only notices the abort when it expires would wait out every one of them.
35
- await new Promise<void>((resolve, reject) => {
36
- // The reason is rejected VERBATIM, never wrapped: whoever aborted put a `ScrapeError` with
37
- // a code and a fix there, and re-wrapping it would replace an instruction with `Error`.
38
- const onAbort = (): void => {
39
- clearTimeout(timer);
40
- reject(signal.reason);
41
- };
42
- const timer = setTimeout(() => {
43
- signal.removeEventListener('abort', onAbort);
44
- resolve();
45
- }, ms);
46
- signal.addEventListener('abort', onAbort, { once: true });
47
- });
48
- },
49
- });
50
-
51
- /**
52
- * The system clock with the waiting taken out: `now()` and `monotonic()` are real, and every
53
- * `sleep` is ONE turn of the event loop whatever was asked for. For a test whose run waits on
54
- * something that really happens in the same process — an answer the test publishes, a job another
55
- * worker holds — where `testScrapeClock()`'s "sleeping is advancing" would run a five-minute budget out
56
- * in microtasks before the test's next line. A deadline is still a real one.
57
- */
58
- export const noWaitClock: ScrapeClock = Object.freeze({
59
- now: () => systemScrapeClock.now(),
60
- monotonic: () => systemScrapeClock.monotonic(),
61
- sleep: (_ms: number, signal?: AbortSignal) => systemScrapeClock.sleep(0, signal),
62
- });
63
-
64
- let ambient: ScrapeClock | undefined;
65
-
66
- /**
67
- * The clock every run in this process waits on, unless its definition pins one — the counterpart
68
- * of `setScrapeDriver()`, and the seam a TEST takes: `setScrapeClock(noWaitClock)` in a
69
- * `beforeEach`, and a declaration under test needs no `clock:` field written for the test's sake.
70
- */
71
- export function setScrapeClock(clock: ScrapeClock): void {
72
- ambient = clock;
73
- }
74
-
75
- /** The process's scrape clock: the installed one, or the system's. */
76
- export function scrapeClock(): ScrapeClock {
77
- return ambient ?? systemScrapeClock;
78
- }
79
-
80
- /** Test seam: back to the system clock, so the next file runs on real time again. */
81
- export function resetScrapeClock(): void {
82
- ambient = undefined;
83
- }
84
-
85
- /**
86
- * A clock in which sleeping IS advancing: `await clock.sleep(30_000)` returns on the next
87
- * microtask with thirty seconds elapsed. This is what makes a poll-until-deadline test instant
88
- * without the test knowing how many polls the loop makes — a count a test that asserted it would
89
- * pin to the implementation rather than to the behaviour.
90
- */
91
- export interface TestScrapeClock extends ScrapeClock {
92
- advance(ms: number): void;
93
- }
94
-
95
- export function testScrapeClock(at: Date | number = 0): TestScrapeClock {
96
- let epochMs = at instanceof Date ? at.getTime() : at;
97
- let mono = 0;
98
- const advance = (ms: number): void => {
99
- epochMs += ms;
100
- mono += ms;
101
- };
102
- return {
103
- now: () => new Date(epochMs),
104
- monotonic: () => mono,
105
- advance,
106
- sleep: async (ms: number, signal?: AbortSignal) => {
107
- advance(ms);
108
- // Yield the microtask queue anyway: a loop that never awaits anything real starves whatever
109
- // the test armed to cancel it, and the wedge tests arm exactly that.
110
- await Promise.resolve();
111
- signal?.throwIfAborted();
112
- },
113
- };
114
- }
115
-
116
9
  /** Elapsed-time budget, read from one clock. Built once per waiting call, never shared. */
117
10
  export interface Deadline {
118
11
  readonly totalMs: number;
package/src/driver-cdp.ts CHANGED
@@ -111,6 +111,9 @@ async function opened(
111
111
  browser,
112
112
  rules: init.rules,
113
113
  clock: init.clock,
114
+ // Through an exit the EXIT resolves names, so the browser leg's DNS screen stands down — the
115
+ // HTTP leg's rule, for the same reason (`browser-dial.ts`).
116
+ proxied: wiring.proxy !== undefined,
114
117
  ended,
115
118
  });
116
119
  if (init.restore !== undefined) await target.restore(init.restore);
@@ -5,9 +5,8 @@
5
5
  // It covers BOTH legs. `fakeBrowser({ pages, http })` replays the browser walk and the JSON
6
6
  // endpoints behind it from one declaration, so a hybrid scrape is tested the way it runs.
7
7
 
8
- import { finiteCount } from '@ultimat3/core';
9
- import type { ScrapeClock } from './clock';
10
- import { systemScrapeClock } from './clock';
8
+ import type { SleepClock } from '@ultimat3/core';
9
+ import { ambientClock, finiteCount } from '@ultimat3/core';
11
10
  import type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
12
11
  import { htmlTarget } from './html-target';
13
12
  import { httpRecordingsOf } from './http-recorded';
@@ -78,7 +77,7 @@ export function fakeBrowser(pages: FakePages, options: FakeBrowserOptions = {}):
78
77
 
79
78
  export interface FakePageOptions {
80
79
  readonly url?: string | undefined;
81
- readonly clock?: ScrapeClock | undefined;
80
+ readonly clock?: SleepClock | undefined;
82
81
  readonly allowHosts?: readonly string[] | undefined;
83
82
  readonly timeoutMs?: number | undefined;
84
83
  readonly context?: Partial<PageContext> | undefined;
@@ -97,7 +96,7 @@ export const FAKE_PAGE_URL = 'https://fake.test/';
97
96
  export function fakePage(dom: string, options: FakePageOptions = {}): ScrapePage {
98
97
  const url = options.url ?? FAKE_PAGE_URL;
99
98
  const start: PageRecording = { url, html: dom };
100
- const clock = options.clock ?? systemScrapeClock;
99
+ const clock = options.clock ?? ambientClock();
101
100
  const allowHosts = options.allowHosts ?? ['*'];
102
101
  const byUrl = new Map(
103
102
  [start, ...recordingsOf(options.pages ?? [])].map((page) => [normalise(page.url), page]),
package/src/driver.ts CHANGED
@@ -1,13 +1,13 @@
1
1
  // The browser contract. Every driver implements exactly this, so a scrape's body never names one
2
2
  // — the same shape `packages/jobs/src/driver.ts` gives the queue, for the same reason: swapping
3
- // the browser is `setScrapeDriver(other)` and ZERO run-body change.
3
+ // the browser is `scrapeDriver: other` in `apps/<app>/runtime.ts` and ZERO run-body change.
4
4
  //
5
5
  // Two methods. `open` hands back a session; `close` ends it. Everything else a scraper does goes
6
6
  // through `ScrapePage`, which is driver-blind by construction.
7
7
 
8
- import type { Logger } from '@ultimat3/core';
8
+ import type { Logger, SleepClock } from '@ultimat3/core';
9
+ import { adapterSlot } from '@ultimat3/core/adapter';
9
10
  import type { MoneyValue } from '@ultimat3/schema';
10
- import type { ScrapeClock } from './clock';
11
11
  import type { ScrapeHttp } from './http';
12
12
  import type { InterceptRules } from './intercept';
13
13
  import type { ScrapePage } from './page';
@@ -20,7 +20,7 @@ export interface SessionInit {
20
20
  /** The scrape's name — every error cause raised inside the session carries it. */
21
21
  readonly name: string;
22
22
  readonly rules: InterceptRules;
23
- readonly clock: ScrapeClock;
23
+ readonly clock: SleepClock;
24
24
  /**
25
25
  * The run's logger — `scrapeLogger()`'s child, so every line carries the scrape and the run.
26
26
  * Required, because `close()` never throws: a teardown that failed (a rented browser its
@@ -100,22 +100,27 @@ export interface ScrapeDriver {
100
100
  open(init: SessionInit): Promise<ScrapeSession>;
101
101
  }
102
102
 
103
- let ambient: ScrapeDriver | undefined;
103
+ const ambient = /* @__PURE__ */ adapterSlot<ScrapeDriver | undefined>(() => undefined);
104
104
 
105
- /** Set once by the app at boot — no config key names it. A scrape's own `driver:` overrides it. */
106
- export function setScrapeDriver(driver: ScrapeDriver): void {
107
- ambient = driver;
105
+ /**
106
+ * The boot's install, from `scrapeDriver` in `apps/<app>/runtime.ts` — exported from
107
+ * `@ultimat3/scraping/adapter` only, never the `.` barrel, so that file is the one place an app
108
+ * chooses a browser. A scrape's own `driver:` overrides it. Answers the release that removes THIS
109
+ * install and no other (core's `adapterSlot`), in any order, once.
110
+ */
111
+ export function installScrapeDriver(driver: ScrapeDriver): () => void {
112
+ return ambient.install(driver);
108
113
  }
109
114
 
110
115
  export function scrapeDriver(): ScrapeDriver | undefined {
111
- return ambient;
116
+ return ambient.current();
112
117
  }
113
118
 
114
119
  /**
115
- * Test/CLI seam: forget the ambient driver. The counterpart to `resetJobDriver()` — a test that
120
+ * Test/CLI seam: forget the ambient driver. The counterpart to `resetJobDriver()` (`@ultimat3/jobs/adapter`) — a test that
116
121
  * installs a browser has to be able to put the process back, or every later file in the same bun
117
122
  * process runs against a driver it never asked for.
118
123
  */
119
124
  export function resetScrapeDriver(): void {
120
- ambient = undefined;
125
+ ambient.reset();
121
126
  }
@@ -1,6 +1,7 @@
1
1
  // The failure mode of a PDF layout request. Split from `error-throws.ts`, which stands at its size
2
2
  // ceiling; the code itself stays declared in `errors.ts` — one registry, one place.
3
3
 
4
+ import { isRetryableStatus, renderCauseValue, renderFixShellArg } from '@ultimat3/core';
4
5
  import { ScrapeError } from './errors';
5
6
 
6
7
  /**
@@ -14,3 +15,47 @@ export const pdfOptionsInvalid = (reason: string, meta: { readonly field: string
14
15
  fix: "page.pdf({ format: 'a4', marginMm: { top: 12, bottom: 12 } }) — exactly one size: a format, or widthMm with heightMm, or preferCssPageSize alone",
15
16
  meta: { field: meta.field },
16
17
  });
18
+
19
+ /** No server to load the route from: `APP_URL` unset and no `origin` on the declaration. */
20
+ export const pdfOriginMissing = (): ScrapeError =>
21
+ new ScrapeError({
22
+ code: 'X_SCRAPE_PDF_ORIGIN_MISSING',
23
+ cause:
24
+ 'pdf() has no origin to load its route from: APP_URL is unset in this process and the declaration names no origin',
25
+ fix: "export APP_URL=https://your.app # or pdf({ origin: () => 'http://127.0.0.1:3000' }) in the declaring file",
26
+ });
27
+
28
+ /** `route()` answered something the printing browser would load from another origin. */
29
+ export const pdfRouteInvalid = (route: string): ScrapeError =>
30
+ new ScrapeError({
31
+ code: 'X_SCRAPE_PDF_ROUTE_INVALID',
32
+ cause: `pdf() route() answered ${renderCauseValue(route)}, which is not a path on this app's origin — the printing browser loads nothing else`,
33
+ fix: "pdf({ route: ({ input }) => '/invoices/' + input.id + '/print' }) # route() answers a path on this app, never a URL",
34
+ });
35
+
36
+ /**
37
+ * The page answered an error status. A 4xx that no retry can change (a missing record, a route the
38
+ * printing browser may not read) is terminal on the instance; a 5xx or 429 stays retryable.
39
+ */
40
+ export const pdfRouteFailed = (url: string, status: number): ScrapeError => {
41
+ const path = new URL(url).pathname;
42
+ return new ScrapeError({
43
+ code: 'X_SCRAPE_PDF_ROUTE_FAILED',
44
+ cause: `the route pdf() printed, ${renderCauseValue(path)}, answered HTTP ${String(status)}: printing it would store an error page as the document`,
45
+ fix: `curl -i "$APP_URL"${renderFixShellArg(path, '/<the path the cause names>')} # a private route needs pdf({ headers }); a missing one, a route() naming a record that exists`,
46
+ meta: { status },
47
+ retry: status >= 400 && status < 500 && !isRetryableStatus(status) ? 'terminal' : 'retryable',
48
+ });
49
+ };
50
+
51
+ /** Every render slot stayed busy for the whole queue deadline: a burst, not a broken page. */
52
+ export const pdfBusy = (input: {
53
+ readonly maxConcurrent: number;
54
+ readonly queueTimeoutMs: number;
55
+ }): ScrapeError =>
56
+ new ScrapeError({
57
+ code: 'X_SCRAPE_PDF_BUSY',
58
+ cause: `all ${String(input.maxConcurrent)} pdf() render slot(s) stayed busy for ${String(input.queueTimeoutMs)}ms — each holds a live Chrome, so the renderer bounds them rather than let a burst exhaust this process's memory`,
59
+ fix: `pdf({ renderer: chromePdfRenderer({ maxConcurrent: ${String(input.maxConcurrent * 2)} }) }) # only on a host with the memory for it; otherwise the caller retries later`,
60
+ meta: { maxConcurrent: input.maxConcurrent, queueTimeoutMs: input.queueTimeoutMs },
61
+ });
@@ -51,7 +51,7 @@ export const promptTimedOut = (input: {
51
51
  new ScrapeError({
52
52
  code: 'X_SCRAPE_PROMPT_UNANSWERED',
53
53
  cause: `scrape "${input.scrape}" asked for "${input.label}" and no answer was published to ${input.event} within ${String(input.waitedMs)}ms`,
54
- fix: 'call answerPrompt({ runId, index, answer }) with the run id and index the event name in the cause ends with, from a process on the same STORED event bus — setEventBus(postgresEventBus({ executor })) — or raise eventPrompt({ timeout })',
54
+ fix: 'call answerPrompt({ runId, index, answer }) with the run id and index the event name in the cause ends with, from a process on the same STORED event bus — eventBus: postgresEventBus({ executor }) in apps/<app>/runtime.ts — or raise eventPrompt({ timeout })',
55
55
  meta: {
56
56
  scrape: input.scrape,
57
57
  label: input.label,
@@ -30,8 +30,8 @@ export const driverUnknown = (
30
30
  cause:
31
31
  scrape === undefined
32
32
  ? `no scrape driver named "${name ?? 'none'}" is installed; installed: ${installed.join(', ') || 'none'}`
33
- : `scrape "${scrape}" has no browser driver: nothing called setScrapeDriver() and the definition declares no driver; installed: ${installed.join(', ') || 'none'}`,
34
- fix: 'call setScrapeDriver(localBrowser()) at boot, or pass driver: fakeBrowser() on the scrape() definition',
33
+ : `scrape "${scrape}" has no browser driver: apps/<app>/runtime.ts names no scrapeDriver and the definition declares no driver; installed: ${installed.join(', ') || 'none'}`,
34
+ fix: 'export const runtime = { scrapeDriver: localBrowser() } # apps/<app>/runtime.ts — or pass driver: fakeBrowser() on the scrape() definition',
35
35
  meta: { driver: name ?? 'none', ...(scrape === undefined ? {} : { scrape }) },
36
36
  });
37
37
 
@@ -217,7 +217,7 @@ export const wedged = (what: string, idleMs: number): ScrapeError =>
217
217
  * opposite fixes are two codes, and the classifications differ with them: a wedge is retryable,
218
218
  * this is terminal (`errors.ts`).
219
219
  *
220
- * Reachable through the `ScrapeClock` seam and the driver's `kill()`, both of which a third party
220
+ * Reachable through the `SleepClock` seam and the driver's `kill()`, both of which a third party
221
221
  * writes. Leaving the loop dead and quiet is the alternative, and that is incident #1 with no
222
222
  * guard armed at all.
223
223
  */
@@ -225,7 +225,7 @@ export const watchdogStopped = (what: string, thrown: unknown): ScrapeError =>
225
225
  new ScrapeError({
226
226
  code: 'X_SCRAPE_WATCHDOG_STOPPED',
227
227
  cause: `the wedge watchdog for ${what} stopped measuring browser activity: ${renderThrowable(thrown)}`,
228
- fix: 'drop the custom clock: from the scrape() definition — systemScrapeClock is the only ScrapeClock whose sleep() cannot reject — then re-run',
228
+ fix: "drop the custom clock: from the scrape() definition and any clock: line in apps/<app>/runtime.ts — @ultimat3/core's systemClock is the only SleepClock whose sleep() cannot reject — then re-run",
229
229
  meta: { what },
230
230
  });
231
231
 
package/src/errors.ts CHANGED
@@ -43,6 +43,10 @@ export const SCRAPE_OWNED_ERROR_CODES = [
43
43
  'X_SCRAPE_EGRESS_IN_PAYLOAD',
44
44
  'X_SCRAPE_LAUNCH_ARGS_INVALID',
45
45
  'X_SCRAPE_PDF_OPTIONS_INVALID',
46
+ 'X_SCRAPE_PDF_ORIGIN_MISSING',
47
+ 'X_SCRAPE_PDF_ROUTE_INVALID',
48
+ 'X_SCRAPE_PDF_ROUTE_FAILED',
49
+ 'X_SCRAPE_PDF_BUSY',
46
50
  ] as const;
47
51
 
48
52
  /**
@@ -101,6 +105,10 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
101
105
  'the launch options carry arguments this driver cannot hand the browser',
102
106
  X_SCRAPE_PDF_OPTIONS_INVALID:
103
107
  'the PDF request names no single page size a print engine can lay out',
108
+ X_SCRAPE_PDF_ORIGIN_MISSING: 'pdf() has no origin to render the route on',
109
+ X_SCRAPE_PDF_ROUTE_INVALID: 'a pdf() route is not a path on this app',
110
+ X_SCRAPE_PDF_ROUTE_FAILED: 'the route pdf() printed answered an error status',
111
+ X_SCRAPE_PDF_BUSY: 'every pdf() render slot stayed busy past the queue deadline',
104
112
  };
105
113
 
106
114
  // One unconditional call, so a second package claiming one of these codes throws
@@ -181,7 +189,7 @@ export const SCRAPE_ERROR_RETRY = {
181
189
  // TERMINAL where its sibling `X_SCRAPE_WEDGED` is retryable, and the difference IS the reason
182
190
  // the two codes are separate. A wedge is the site or the browser being slow — the definition of
183
191
  // "run it again and it may go differently". This is the guard's own loop dying on code the
184
- // DEFINITION supplied: a `ScrapeClock` an app wrote, reached identically on attempt 2. Retrying
192
+ // DEFINITION supplied: a `SleepClock` an app wrote, reached identically on attempt 2. Retrying
185
193
  // launches a real browser five times to die at the first poll, and on an authenticated target
186
194
  // that is five arrivals at a login for no chance of a different answer — the rule this whole
187
195
  // table is written to, stated at the top of it. The fix is an edit, so a human decides.
@@ -214,6 +222,14 @@ export const SCRAPE_ERROR_RETRY = {
214
222
  // The request is the caller's literal, refused before any driver sees it: attempt 2 asks for the
215
223
  // same two sizes and is refused identically.
216
224
  X_SCRAPE_PDF_OPTIONS_INVALID: 'terminal',
225
+ // The deploy's environment and the declaration's own `route()`: attempt 2 reads the same ones.
226
+ X_SCRAPE_PDF_ORIGIN_MISSING: 'terminal',
227
+ X_SCRAPE_PDF_ROUTE_INVALID: 'terminal',
228
+ // The page answered an error: a 503 mid-deploy is the moment, so a `.job()` run tries again.
229
+ // The throw site narrows a 4xx to terminal — a missing record is missing on attempt 2 too.
230
+ X_SCRAPE_PDF_ROUTE_FAILED: 'retryable',
231
+ // Every slot busy is the moment, never the request: the same render fits once one ends.
232
+ X_SCRAPE_PDF_BUSY: 'retryable',
217
233
  } as const satisfies Readonly<Record<ScrapeOwnedErrorCode, 'retryable' | 'terminal'>>;
218
234
 
219
235
  registerErrorRetry(SCRAPE_ERROR_RETRY);
@@ -54,7 +54,7 @@ export interface EventPromptOptions {
54
54
  * Only an answer published AFTER the prompt was asked is consumed. The run id is the same on
55
55
  * every attempt, so a code left over from attempt 1 would otherwise be typed into the site by
56
56
  * attempt 2 — and a stale code is a refused login. "After" is decided on ONE clock, the bus's
57
- * (`bus.now()`); the timeout and the poll stay on the run's own.
57
+ * (`bus.mark()`, or `bus.now()`); the timeout and the poll stay on the run's own.
58
58
  */
59
59
  export function eventPrompt(options: EventPromptOptions): PromptHandler {
60
60
  // Floors of 1, both: a zero timeout is a prompt that can never be answered and a zero interval
@@ -73,18 +73,24 @@ export function eventPrompt(options: EventPromptOptions): PromptHandler {
73
73
  driver: 'event bus',
74
74
  cause:
75
75
  'eventPrompt() was asked on the in-memory event bus — the answer is published by another process and can never reach this one',
76
- fix: 'call setEventBus(postgresEventBus({ executor })) at boot — x dev and the served roles do — or pass bus: postgresEventBus({ executor }) to eventPrompt()',
76
+ fix: 'export const runtime = { eventBus: postgresEventBus({ executor }) } # apps/<app>/runtime.ts — or delete that line: x dev and the served roles install it — or pass bus: postgresEventBus({ executor }) to eventPrompt()',
77
77
  });
78
78
  }
79
79
  const event = promptEventName(request.runId, request.index);
80
80
  // The BUS's clock, read once: an answer's `publishedAt` is stamped by it — the database, for
81
81
  // the stored bus — and `request.clock` is this worker's. Comparing the two let a few seconds
82
82
  // of skew between pods decide that an answer given after the ask had come before it.
83
- const askedAt = await bus.now();
83
+ // Read with the bus's MARK where it has one, in the same statement: an answer stamped at
84
+ // INSERT and committed after this instant carries a stamp before it, and only a search under
85
+ // the mark's horizon still sees it (`EventLookup.mark`).
86
+ const mark: { readonly at: number; readonly horizon?: string } = (await bus.mark?.()) ?? {
87
+ at: await bus.now(),
88
+ };
89
+ const askedAt = mark.at;
84
90
  const budget = deadline(request.clock, timeoutMs, 'eventPrompt');
85
91
  for (;;) {
86
92
  request.signal?.throwIfAborted();
87
- const found = await bus.find(event, undefined, askedAt);
93
+ const found = await bus.find(event, undefined, askedAt, undefined, mark.horizon, mark);
88
94
  if (found !== undefined) {
89
95
  if (!isSealed(found.payload)) {
90
96
  throw promptAnswerInvalid({ scrape: request.scrape, label: request.label, event });
package/src/events.ts CHANGED
@@ -11,8 +11,7 @@
11
11
  // of a log line: `ScrapeEventFields` has no key to put one in, and core's logger redacts the
12
12
  // remaining spellings by key and every `Secret` by value.
13
13
 
14
- import type { Logger } from '@ultimat3/core';
15
- import type { ScrapeClock } from './clock';
14
+ import type { Logger, SleepClock } from '@ultimat3/core';
16
15
  import { errorCode } from './failures';
17
16
 
18
17
  export interface ScrapeEventFields {
@@ -48,7 +47,7 @@ export const scrapeLogger = (logger: Logger, fields: ScrapeEventFields): Logger
48
47
  export interface StepEvent {
49
48
  readonly name: string;
50
49
  readonly logger: Logger;
51
- readonly clock: ScrapeClock;
50
+ readonly clock: SleepClock;
52
51
  readonly attempt?: number | undefined;
53
52
  }
54
53
 
@@ -5,9 +5,9 @@
5
5
  // through to the network would make a green offline suite that is secretly hitting production —
6
6
  // the exact failure an offline driver exists to prevent.
7
7
 
8
+ import type { SleepClock } from '@ultimat3/core';
8
9
  import { NotImplementedError } from '@ultimat3/core';
9
10
  import type { CaptureClip } from './capture-clip';
10
- import type { ScrapeClock } from './clock';
11
11
  import type { ColorScheme } from './color-scheme';
12
12
  import { browserUnreachable, downloadTimeout, fixtureMissing, fixtureStale } from './error-throws';
13
13
  import { queryHtml } from './html-query';
@@ -84,7 +84,7 @@ export interface HtmlTargetInit {
84
84
  readonly driver: string;
85
85
  readonly lookup: RecordingLookup;
86
86
  readonly rules: InterceptRules;
87
- readonly clock: ScrapeClock;
87
+ readonly clock: SleepClock;
88
88
  /** Where recordings come from, for the cause line: a directory, or `fakeBrowser()`. */
89
89
  readonly source: string;
90
90
  readonly start?: PageRecording | undefined;
@@ -5,7 +5,7 @@
5
5
  // real network would make the hybrid path — which is where the interesting code lives — the one
6
6
  // path a test never really covers.
7
7
 
8
- import type { ScrapeClock } from './clock';
8
+ import type { SleepClock } from '@ultimat3/core';
9
9
  import { fixtureMissing, fixtureStale, hostBlocked } from './error-throws';
10
10
  import type { HttpRequestInit, ScrapeHttp, ScrapeResponse } from './http';
11
11
  import { responseOver } from './http';
@@ -26,7 +26,7 @@ export interface RecordedHttpInit {
26
26
  readonly lookup: HttpRecordingLookup;
27
27
  readonly rules: InterceptRules;
28
28
  readonly network: NetworkRing;
29
- readonly clock: ScrapeClock;
29
+ readonly clock: SleepClock;
30
30
  readonly source: string;
31
31
  /**
32
32
  * The SAME gate the page leg holds. Without it a hybrid scrape whose JSON endpoint is
package/src/http.ts CHANGED
@@ -10,11 +10,11 @@
10
10
  // guarantee the page vocabulary makes — and a different exit IP mid-session is exactly what
11
11
  // anti-bot systems look for.
12
12
 
13
+ import type { SleepClock } from '@ultimat3/core';
13
14
  import { finiteCount, readWithinLimit } from '@ultimat3/core';
14
15
  import { type DeadlineTimer, deadlineSignal, resolveDialHost } from '@ultimat3/core/outbound';
15
16
  import type { StandardSchemaV1 } from '@ultimat3/schema';
16
17
  import { parse } from '@ultimat3/schema';
17
- import type { ScrapeClock } from './clock';
18
18
  import { cookieHeaderFor } from './cookie-scope';
19
19
  import { bodyTooLarge, hostBlocked, httpFailed, redirectLoop, scrapeTimeout } from './error-throws';
20
20
  import type { RedirectHop } from './http-redirect';
@@ -96,7 +96,7 @@ export interface ScrapeHttp {
96
96
 
97
97
  export interface HttpTransportInit {
98
98
  readonly rules: InterceptRules;
99
- readonly clock: ScrapeClock;
99
+ readonly clock: SleepClock;
100
100
  readonly timeoutMs: number;
101
101
  readonly network: NetworkRing;
102
102
  /**