@ultimat3/scraping 16.0.0 → 18.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -93,6 +93,7 @@ gate, offline included.
93
93
  | `page.query(selector)` | `ElementSnapshot[]` — the same matches PLUS `visible`, `enabled`, and the layout `box`/`hitTarget` a driver with a layout engine can answer. The read for a decision ABOUT an element |
94
94
  | `page.frame(nameOrSelector)` | a `ScrapeFrame`, re-resolved on every call through it |
95
95
  | `page.offline(enabled)` | cuts the BROWSER's network, or restores it |
96
+ | `page.colorScheme(scheme)` | what the browser reports as the user's OS colour preference — `'light'`, `'dark'`, or `'no-preference'` to CLEAR the override |
96
97
 
97
98
  `query()` exists because there is exactly one definition of "visible" in this framework and it is
98
99
  the port's. A caller that had to compute its own wrote
@@ -111,6 +112,31 @@ NAVIGATE a frame: a recorded frame is one static document, so a click inside one
111
112
  browser's own requests, so a driver that quietly answered "done" would let "a like taken offline is
112
113
  queued" pass against an app that was online for the whole test.
113
114
 
115
+ **`page.colorScheme()` sets the INPUT to a theme decision, never its outcome.** An attribute on the
116
+ document — `data-theme`, `class="dark"` — is the outcome, and the component owns it: one that
117
+ resolves `'system'` itself overwrites or deletes the attribute on mount, so a harness that set it is
118
+ silently overruled. Measured on `examples/dummy`, `As of 2026-08`: `x shot --island` photographed
119
+ every state in both themes and the two files came back byte-identical, same md5, from two addresses
120
+ that really did serve different documents. Emulating the preference is what reaches a component that
121
+ decides for itself; set the attribute as well for one that reads a theme it does not own.
122
+
123
+ **`'no-preference'` CLEARS the override, and is not a third value.** CDP treats an explicit
124
+ `prefers-color-scheme: no-preference` as an override and an EMPTY feature list as a reset, so that
125
+ is what this sends. Measured on Chrome 150 headless, `As of 2026-08`: after either one,
126
+ `(prefers-color-scheme: dark)` is false and `(prefers-color-scheme: light)` is true — the same
127
+ answers an untouched page gives. They diverge on a browser that has a real preference, where the
128
+ override forces the light answer and the reset gives the machine's own back, which is what this
129
+ value promises. (`no-preference` left the `prefers-color-scheme` query itself in 2020, so nothing
130
+ matches it in any of those readings.)
131
+
132
+ Accepted and RECORDED on the offline drivers rather than refused, which is the opposite of
133
+ `offline()` — and the line between them is which side of a capture the verb is on. An offline
134
+ *assertion* is reachable on a driver that answers content, so a resolved `setOfflineMode` would let
135
+ it pass against an app that never went offline; a colour preference has no such assertion, and the
136
+ only thing it could be wrong about is a picture. So the offline drivers answer **different
137
+ deterministic bytes per scheme**, exactly as they already do per `clip` — a fake returning one
138
+ constant for both themes is precisely what let the defect above ship.
139
+
114
140
  ## What the page reports back, and the one thing a picture cannot say
115
141
 
116
142
  Three bounded rings, read off `ScrapePage`. Bounded because a ten-thousand-page run that kept
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/scraping",
3
- "version": "16.0.0",
3
+ "version": "18.0.0",
4
4
  "description": "Browser automation as a job: scrape() returns a JobHandle",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -23,16 +23,16 @@
23
23
  "LICENSE"
24
24
  ],
25
25
  "engines": {
26
- "bun": ">=1.3.0"
26
+ "bun": ">=1.4.0"
27
27
  },
28
28
  "scripts": {
29
29
  "typecheck": "tsc --noEmit -p tsconfig.json",
30
30
  "test": "bun test"
31
31
  },
32
32
  "dependencies": {
33
- "@ultimat3/core": "16.0.0",
34
- "@ultimat3/jobs": "16.0.0",
35
- "@ultimat3/schema": "16.0.0",
36
- "@ultimat3/storage": "16.0.0"
33
+ "@ultimat3/core": "18.0.0",
34
+ "@ultimat3/jobs": "18.0.0",
35
+ "@ultimat3/schema": "18.0.0",
36
+ "@ultimat3/storage": "18.0.0"
37
37
  }
38
38
  }
@@ -4,6 +4,7 @@
4
4
  // `waitForSelector('#aceptar:not([disabled])')`, which is this rule, spelled once, by hand, at
5
5
  // one call site out of forty.
6
6
 
7
+ import { finiteCount } from '@ultimat3/core';
7
8
  import type { ScrapeClock } from './clock';
8
9
  import { deadline } from './clock';
9
10
  import { notActionable, selectorMissing } from './error-throws';
@@ -85,8 +86,12 @@ export interface ActionabilityWait {
85
86
  * a scraper failure take an afternoon.
86
87
  */
87
88
  export async function awaitActionable(wait: ActionabilityWait): Promise<ElementSnapshot> {
88
- const budget = deadline(wait.clock, wait.timeoutMs);
89
- const pollMs = wait.pollMs ?? DEFAULT_POLL_MS;
89
+ const budget = deadline(wait.clock, wait.timeoutMs, 'page.waitFor');
90
+ // At least 1ms, and the floor is the point: `Math.min(0, remainingMs())` is 0, `clock.sleep(0)`
91
+ // returns on the next turn, and the poll is then one full round trip to the browser per
92
+ // event-loop turn for the whole budget. `NaN` is the same loop by a different route —
93
+ // `Math.min(NaN, x)` is `NaN` and `setTimeout(fn, NaN)` is `setTimeout(fn, 0)`.
94
+ const pollMs = finiteCount('page.waitFor', 'pollMs', wait.pollMs ?? DEFAULT_POLL_MS, 1);
90
95
  let previous: ElementSnapshot | undefined;
91
96
  let lastProblem: string | undefined;
92
97
  let everSeen = false;
package/src/auth.ts CHANGED
@@ -18,6 +18,7 @@
18
18
  // is written into the session record so the NEXT attempt fails before reaching the login form.
19
19
 
20
20
  import type { Logger } from '@ultimat3/core';
21
+ import { finiteCount } from '@ultimat3/core';
21
22
  import type { ScrapeClock } from './clock';
22
23
  import { authFailed, promptUnanswered, sessionExpired } from './error-throws';
23
24
  import type { ScrapePage } from './page';
@@ -121,8 +122,17 @@ export async function restorableSession<I>(
121
122
  if (plan.auth?.reuse === false) return undefined;
122
123
  const maxAge = plan.auth?.maxAge;
123
124
  if (maxAge !== undefined) {
125
+ // Screened, because `age > NaN` is false and false here means RESTORED: a `NaN` maxAge hands
126
+ // back a session of any age, with no re-login and nothing in the report. `0` is legal — it
127
+ // means "restore nothing stored before now" — so the floor stays there.
128
+ const limit = finiteCount('the scrape auth', 'maxAge', maxAge);
124
129
  const age = plan.clock.now().getTime() - new Date(found.savedAt).getTime();
125
- if (age > maxAge) return undefined;
130
+ // `!(age <= limit)` and not `age > limit`, which is the same test for every finite age and the
131
+ // OPPOSITE one for a `NaN`. `savedAt` is data, not configuration: it comes back off a bucket
132
+ // and `parseSessionState` only asks that it is a string, so an edited or half-written record
133
+ // produces a `NaN` age against a perfectly good `maxAge`. Failing closed costs one re-login;
134
+ // failing open acts as somebody else, indefinitely.
135
+ if (!(age <= limit)) return undefined;
126
136
  }
127
137
  plan.logger.debug('scrape.session.restored', sessionDigest(found));
128
138
  return found;
package/src/cdp-fake.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  // Precedent: `packages/storage/src/driver-s3-fixture.ts` ships the same way.
10
10
 
11
11
  import type { CdpBrowserLike, CdpFrameLike, CdpLauncherLike, CdpPageLike } from './cdp-port';
12
+ import { COLOR_SCHEME_FEATURE } from './color-scheme';
12
13
  import { queryHtml } from './html-query';
13
14
  import type { ElementSnapshot, ScrapeCookie } from './target';
14
15
 
@@ -70,6 +71,12 @@ export interface FakeCdpBrowser extends CdpBrowserLike {
70
71
  readonly closed: boolean;
71
72
  /** What `page.setOfflineMode()` last set, so a test asserts on the CONDITION, not on a call. */
72
73
  readonly offline: boolean;
74
+ /**
75
+ * What `page.emulateMediaFeatures()` last set `prefers-color-scheme` to, for `offline`'s reason
76
+ * — the condition the page is now in, never the fact that a method was called. `null` until
77
+ * something sets one, which is the launcher's own default and not a value this fake invents.
78
+ */
79
+ readonly colorScheme: string | null;
73
80
  }
74
81
 
75
82
  /** A layout box every element gets, so the CDP path exercises the fields the fake target lacks. */
@@ -113,6 +120,7 @@ export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
113
120
  let html = init.html;
114
121
  let closed = false;
115
122
  let offline = false;
123
+ let colorScheme: string | null = null;
116
124
  const covered = new Set(init.covered ?? []);
117
125
  const storage: Record<string, string> = { ...init.storage };
118
126
  let cookies: readonly ScrapeCookie[] = init.cookies ?? [];
@@ -202,6 +210,20 @@ export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
202
210
  offline = enabled;
203
211
  return Promise.resolve();
204
212
  },
213
+ // CDP's own two rules, both of them. An EMPTY list is a RESET — measured on Chrome 150, after
214
+ // `emulateMediaFeatures([])` the page answers exactly what an untouched one does — so it must
215
+ // clear here rather than be ignored, or `'no-preference'` records the previous scheme forever.
216
+ // And a non-empty list is read by NAME, never `features[0].value`: the port's shape is a list,
217
+ // and a caller setting `prefers-reduced-motion` beside the scheme must not overwrite it.
218
+ emulateMediaFeatures: (features: readonly { name: string; value: string }[]) => {
219
+ if (features.length === 0) {
220
+ colorScheme = null;
221
+ return Promise.resolve();
222
+ }
223
+ const found = features.find((feature) => feature.name === COLOR_SCHEME_FEATURE);
224
+ if (found !== undefined) colorScheme = found.value;
225
+ return Promise.resolve();
226
+ },
205
227
  on: (event: string, handler: (payload: unknown) => void) => {
206
228
  const listeners = handlers.get(event) ?? [];
207
229
  listeners.push(handler);
@@ -250,6 +272,9 @@ export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
250
272
  get offline(): boolean {
251
273
  return offline;
252
274
  },
275
+ get colorScheme(): string | null {
276
+ return colorScheme;
277
+ },
253
278
  };
254
279
  }
255
280
 
package/src/cdp-port.ts CHANGED
@@ -69,6 +69,22 @@ export interface CdpPageLike {
69
69
  * a type error for a capability the port cannot make them have.
70
70
  */
71
71
  setOfflineMode?(enabled: boolean): Promise<void>;
72
+ /**
73
+ * The browser's own media features — `Emulation.setEmulatedMedia` under CDP. `prefers-color-scheme`
74
+ * is the one this package sets; the array shape is the library's and is restated, not narrowed,
75
+ * because this file is the shape of somebody ELSE's object.
76
+ *
77
+ * OPTIONAL, read defensively, for `setOfflineMode`'s reason: a provider SDK or a launcher that
78
+ * predates the method must still satisfy the port. It does not go unwired by being optional —
79
+ * `cdp-target.ts` refuses BY NAME with `X_NOT_IMPLEMENTED` and a fix when a launcher lacks it.
80
+ *
81
+ * `name` stays a bare `string`. A union of the features this package sets would be this file
82
+ * naming somebody else's vocabulary, and a launcher accepting more would then fail to satisfy
83
+ * the port for no reason — the argument `on(event: string, …)` already makes.
84
+ */
85
+ emulateMediaFeatures?(
86
+ features: readonly { readonly name: string; readonly value: string }[],
87
+ ): Promise<void>;
72
88
  /**
73
89
  * `event` stays a bare `string` — a union of the four names this package subscribes to would be
74
90
  * this file naming somebody else's event vocabulary, which is the thing it exists not to do, and
package/src/cdp-target.ts CHANGED
@@ -8,6 +8,7 @@ import { browserRecord } from './browser-record';
8
8
  import type { CdpBrowserLike, CdpFrameLike, CdpPageLike, CdpRequestLike } from './cdp-port';
9
9
  import { clearExpression, parseSnapshots, snapshotExpression } from './cdp-snapshot';
10
10
  import type { ScrapeClock } from './clock';
11
+ import { type ColorScheme, colorSchemeFeatures } from './color-scheme';
11
12
  import { browserUnreachable, pageCrashed, scrapeNotImplemented } from './error-throws';
12
13
  import type { InterceptRules } from './intercept';
13
14
  import { interceptVerdict, refusalEntry } from './intercept';
@@ -361,6 +362,23 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
361
362
  }
362
363
  await source.setOfflineMode(enabled);
363
364
  }),
365
+ setColorScheme: (scheme: ColorScheme): Promise<void> =>
366
+ guard('setColorScheme', async () => {
367
+ // `CdpPageLike` already declares the member optional, so it is READ, never re-declared in
368
+ // a local cast: a second copy of somebody else's shape is a copy that can drift from the
369
+ // seam it is standing in for (axiom 2). `setOfflineMode` above still casts and is the
370
+ // older spelling.
371
+ const emulate = init.page.emulateMediaFeatures;
372
+ if (typeof emulate !== 'function') {
373
+ throw scrapeNotImplemented(
374
+ 'setColorScheme() on a CDP page with no emulateMediaFeatures() method',
375
+ 'upgrade the launcher to a puppeteer-core that exposes page.emulateMediaFeatures(), or have the page under test set its own theme',
376
+ );
377
+ }
378
+ // `.call`, because the member is read off the object: an unbound method loses `this` and
379
+ // puppeteer's own `Page` needs it.
380
+ await emulate.call(init.page, colorSchemeFeatures(scheme));
381
+ }),
364
382
  screenshot: (options: CaptureOptions) =>
365
383
  guard('screenshot', async () => {
366
384
  // `fullPage` is OMITTED when a clip is given rather than sent as `false`: the two are
package/src/clock.ts CHANGED
@@ -5,6 +5,7 @@
5
5
  // `clock-discipline.test.ts` fails the build if a second file reaches for a timer directly.
6
6
 
7
7
  import type { Clock } from '@ultimat3/core';
8
+ import { finiteCount } from '@ultimat3/core';
8
9
 
9
10
  export interface ScrapeClock extends Clock {
10
11
  /**
@@ -90,8 +91,14 @@ export interface Deadline {
90
91
  expired(): boolean;
91
92
  }
92
93
 
93
- export function deadline(clock: Clock, totalMs: number): Deadline {
94
+ export function deadline(clock: Clock, totalMs: number, subject = 'a scrape wait'): Deadline {
95
+ // Screened HERE because this is the one constructor of a budget, and every `for (;;)` in this
96
+ // package asks `expired()` to leave it. `Math.max(0, NaN - elapsed)` is `NaN` and `NaN <= 0` is
97
+ // false, so a budget that is not a number never expires: the loop in `awaitActionable` and the
98
+ // one in `page-over-target.ts`'s `frame()` both become unbounded, re-reading a live browser once
99
+ // per `Math.min(pollMs, NaN)` — which is `NaN`, which `setTimeout` reads as 0.
100
+ const budgetMs = finiteCount(subject, 'timeoutMs', totalMs);
94
101
  const startedAt = clock.monotonic();
95
- const remainingMs = (): number => Math.max(0, totalMs - (clock.monotonic() - startedAt));
96
- return { totalMs, remainingMs, expired: () => remainingMs() <= 0 };
102
+ const remainingMs = (): number => Math.max(0, budgetMs - (clock.monotonic() - startedAt));
103
+ return { totalMs: budgetMs, remainingMs, expired: () => remainingMs() <= 0 };
97
104
  }
@@ -0,0 +1,49 @@
1
+ // The OS-level colour preference a page is shown, and the CDP media feature that carries it.
2
+ //
3
+ // Its own module for `capture-clip.ts`'s reason: the vocabulary belongs to EVERY driver
4
+ // identically, and `target.ts` is a port declaration, which is not a place a decision lives.
5
+
6
+ /**
7
+ * What a browser is told the user prefers — the INPUT to a theme decision, never its outcome.
8
+ *
9
+ * Setting `data-theme` on the document models the outcome, and the component owns that: a
10
+ * component that resolves `'system'` itself deletes or overwrites the attribute on mount, so a
11
+ * harness that set it is silently overruled and both themes converge on one picture. Measured on
12
+ * `examples/dummy`'s settings island (issue #338): `<state>-light.png` and `<state>-dark.png` came
13
+ * back byte-identical, same md5, from two addresses that really did serve different documents.
14
+ *
15
+ * The set is closed at THREE. `'light'` and `'dark'` are CSS's own values; `'no-preference'` is
16
+ * this vocabulary's way of saying **clear the override**, and it earns its place because without
17
+ * it a preference, once set, is permanent for the session.
18
+ *
19
+ * It is a CLEAR and not a value, and the difference is measured. CDP's `Emulation.setEmulatedMedia`
20
+ * treats an explicit `prefers-color-scheme: no-preference` as an OVERRIDE and an EMPTY feature list
21
+ * as a reset, so `colorSchemeFeatures` sends the empty list. On Chrome 150 headless the two are
22
+ * observationally identical — after either, `(prefers-color-scheme: dark)` is false,
23
+ * `(prefers-color-scheme: light)` is true, and `(prefers-color-scheme: no-preference)` is false,
24
+ * which is also what an untouched page answers. They diverge on a browser that HAS a real
25
+ * preference: the override forces the light-equivalent answer, while the reset gives the machine's
26
+ * own back — and "the launcher's default" is what this value promises.
27
+ *
28
+ * `no-preference` was dropped from the `prefers-color-scheme` media query in 2020, which is why
29
+ * the third row above is false in every one of those readings. It is a control word here, never a
30
+ * query a stylesheet can match.
31
+ */
32
+ export const COLOR_SCHEMES = ['light', 'dark', 'no-preference'] as const;
33
+ export type ColorScheme = (typeof COLOR_SCHEMES)[number];
34
+
35
+ /** CSS's own feature name, and CDP's. One constant, because two spellings is a silent no-op. */
36
+ export const COLOR_SCHEME_FEATURE = 'prefers-color-scheme';
37
+
38
+ export function isColorScheme(value: unknown): value is ColorScheme {
39
+ return typeof value === 'string' && COLOR_SCHEMES.includes(value as ColorScheme);
40
+ }
41
+
42
+ /**
43
+ * The CDP feature list for one scheme — the ONE place the clear is spelled, so a driver and a fake
44
+ * cannot disagree about what `'no-preference'` means.
45
+ */
46
+ export const colorSchemeFeatures = (
47
+ scheme: ColorScheme,
48
+ ): readonly { readonly name: string; readonly value: string }[] =>
49
+ scheme === 'no-preference' ? [] : [{ name: COLOR_SCHEME_FEATURE, value: scheme }];
@@ -5,6 +5,7 @@
5
5
  // It covers BOTH legs. `fakeBrowser({ pages, http })` replays the browser walk and the JSON
6
6
  // endpoints behind it from one declaration, so a hybrid scrape is tested the way it runs.
7
7
 
8
+ import { finiteCount } from '@ultimat3/core';
8
9
  import type { ScrapeClock } from './clock';
9
10
  import { systemScrapeClock } from './clock';
10
11
  import type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
@@ -113,7 +114,10 @@ export function fakePage(dom: string, options: FakePageOptions = {}): ScrapePage
113
114
  return pageOverTarget(target, {
114
115
  clock,
115
116
  allowHosts,
116
- defaultTimeoutMs: options.timeoutMs ?? 1_000,
117
+ // The default budget for every wait AND every navigation on this page, so the floor is 1: a
118
+ // session default of 0 is already out of time everywhere, which a per-call `{ timeout: 0 }` —
119
+ // "is it there right now" — is not. Non-finite is the loop that never leaves.
120
+ defaultTimeoutMs: finiteCount('fakePage', 'timeoutMs', options.timeoutMs ?? 1_000, 1),
117
121
  ...options.context,
118
122
  });
119
123
  }
package/src/expect.ts CHANGED
@@ -3,6 +3,7 @@
3
3
  // where the data went. `expect` turns that into a red run: a yield under `minRows`, or under
4
4
  // `maxDrop` of what this scrape normally returns, throws.
5
5
 
6
+ import { finiteCount, finiteOption } from '@ultimat3/core';
6
7
  import { yieldCollapsed } from './error-throws';
7
8
  import type { ScrapeError } from './errors';
8
9
 
@@ -61,7 +62,23 @@ export interface YieldCheck {
61
62
  * fifty histories without a queue, a browser or a clock.
62
63
  */
63
64
  export function yieldProblem(check: YieldCheck): ScrapeError | undefined {
64
- const { minRows, maxDrop } = check.expect;
65
+ // Screened where the rule is, because this is the alarm and both directions are silent. A `NaN`
66
+ // minRows makes `rows < minRows` false for every yield, so the floor never fires and the scrape
67
+ // is green on zero rows forever — the exact failure this file exists to prevent. A `NaN` maxDrop
68
+ // makes `rows >= baseline * (1 - NaN)` false for every yield, which fires the alarm on every run
69
+ // instead, and an alarm that always fires is an alarm somebody turns off.
70
+ //
71
+ // `minRows: 0` is legal and stays legal: this file asks an author whose answer is legitimately
72
+ // sometimes zero to declare exactly that. `maxDrop` is a FRACTION of a median, so `finiteOption`
73
+ // and not `finiteCount` — `0.5` is the documented value.
74
+ const minRows =
75
+ check.expect.minRows === undefined
76
+ ? undefined
77
+ : finiteCount('the scrape expect', 'minRows', check.expect.minRows);
78
+ const maxDrop =
79
+ check.expect.maxDrop === undefined
80
+ ? undefined
81
+ : finiteOption('the scrape expect', 'maxDrop', check.expect.maxDrop);
65
82
  if (minRows !== undefined && check.rows < minRows) {
66
83
  return yieldCollapsed({ scrape: check.scrape, rows: check.rows, reason: 'min-rows', minRows });
67
84
  }
@@ -101,7 +118,15 @@ export async function guardYield(input: YieldGuardInput): Promise<void> {
101
118
  // `maxDrop` needs `MIN_BASELINE_RUNS` runs after it is declared before it can fire — a delay,
102
119
  // not a hole. `expect.test.ts` pins both halves.
103
120
  if (input.expect === undefined) return;
104
- const window = input.expect.window ?? DEFAULT_YIELD_WINDOW;
121
+ // At least 1: `[…].slice(-0)` is `slice(0)`, the WHOLE history, so a zero window is the largest
122
+ // baseline rather than no baseline — and the number is handed to an app's own `recent()`, where
123
+ // it is usually a SQL `limit`.
124
+ const window = finiteCount(
125
+ 'the scrape expect',
126
+ 'window',
127
+ input.expect.window ?? DEFAULT_YIELD_WINDOW,
128
+ 1,
129
+ );
105
130
  const history =
106
131
  input.history === undefined ? [] : await input.history.recent(input.scrape, window);
107
132
  const problem = yieldProblem({
@@ -7,6 +7,7 @@
7
7
 
8
8
  import type { CaptureClip } from './capture-clip';
9
9
  import type { ScrapeClock } from './clock';
10
+ import type { ColorScheme } from './color-scheme';
10
11
  import {
11
12
  browserUnreachable,
12
13
  downloadTimeout,
@@ -38,15 +39,27 @@ import type {
38
39
  const FAKE_PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
39
40
 
40
41
  /**
41
- * A clipped capture answers DIFFERENT deterministic bytes, one picture per rectangle. CI has no
42
- * Chrome, so the offline drivers are the only place a crop can be proved at all: a fake that
43
- * answered the same eight bytes whatever the rectangle would let a driver that silently drops the
44
- * clip pass every test there is. Unclipped bytes are unchanged, byte for byte.
42
+ * A capture answers DIFFERENT deterministic bytes for every framing fact that was set — one
43
+ * picture per rectangle, one per colour preference. CI has no Chrome, so the offline drivers are
44
+ * the only place a framing knob can be proved at all: a fake that answered the same eight bytes
45
+ * whatever the rectangle would let a driver that silently drops the clip pass every test there is.
46
+ *
47
+ * The colour scheme joined that rule on 2026-08-26, and it is the same defect one axis over. `x
48
+ * shot --island` photographed every state twice and delivered the second picture as a byte-for-
49
+ * byte COPY of the first (issue #338), and no test could see it because this fake answered one
50
+ * constant for both themes.
51
+ *
52
+ * Bytes with NO framing fact set are unchanged, and clip-only bytes are unchanged too — the notes
53
+ * append in a fixed order, so every digest asserted before this existed still holds.
45
54
  */
46
- const clippedPng = (clip: CaptureClip): Uint8Array => {
47
- const suffix = new TextEncoder().encode(
48
- ` clip ${String(clip.x)},${String(clip.y)},${String(clip.width)},${String(clip.height)}`,
49
- );
55
+ const framedPng = (clip: CaptureClip | undefined, scheme: ColorScheme | null): Uint8Array => {
56
+ const notes =
57
+ (clip === undefined
58
+ ? ''
59
+ : ` clip ${String(clip.x)},${String(clip.y)},${String(clip.width)},${String(clip.height)}`) +
60
+ (scheme === null ? '' : ` scheme ${scheme}`);
61
+ if (notes === '') return FAKE_PNG;
62
+ const suffix = new TextEncoder().encode(notes);
50
63
  const out = new Uint8Array(FAKE_PNG.length + suffix.length);
51
64
  out.set(FAKE_PNG);
52
65
  out.set(suffix, FAKE_PNG.length);
@@ -127,6 +140,9 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
127
140
  const frameOverlays = new Map<string, Map<string, string>>();
128
141
  let page: PageRecording = init.start ?? EMPTY;
129
142
  let armed: string | undefined;
143
+ // `null` and not `'no-preference'`: nothing set one, which is the launcher's own default and not
144
+ // a value this driver invents — and it is what keeps an unframed picture's bytes unchanged.
145
+ let colorScheme: ColorScheme | null = null;
130
146
  let closed = false;
131
147
  let session: SessionSnapshot = init.session ?? {
132
148
  ...EMPTY_SESSION,
@@ -333,8 +349,34 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
333
349
  'run this assertion on localBrowser()/remoteBrowser(), whose setOfflineMode() reaches a real browser — an offline driver has no network to cut, so it cannot prove an offline behaviour',
334
350
  );
335
351
  },
352
+ /**
353
+ * ACCEPTED and recorded, where `setOfflineMode` refuses — and the line between them is which
354
+ * side of a capture the verb is on, not whether this driver has a browser.
355
+ *
356
+ * `setOfflineMode` is refused because an offline ASSERTION is reachable here: this driver
357
+ * answers content, so `expect(page).toShow('queued')` would go green against an app that was
358
+ * online the whole time. A colour preference has no such assertion to pass. Nothing here
359
+ * evaluates CSS, and the only thing a preference could be wrong ABOUT is a picture — which on
360
+ * this driver is `FAKE_PNG`, a constant that claims nothing about a theme, exactly as `pdf()`
361
+ * answers `FAKE_PDF` rather than refusing.
362
+ *
363
+ * Refusing would cost the capability its unit test. `x shot --island` is proved end to end on
364
+ * a machine with no Chrome by driving this driver, so a refusal here makes the command
365
+ * untestable offline — and the picture it would be protecting does not exist.
366
+ *
367
+ * Observable through the PICTURE, which is the only place it could ever be wrong: `framedPng`
368
+ * answers different deterministic bytes per scheme, exactly as it already does per clip, so a
369
+ * driver that silently dropped the preference fails a test rather than passing every one.
370
+ */
371
+ setColorScheme(scheme: ColorScheme): Promise<void> {
372
+ // `'no-preference'` is a CLEAR on the real driver, so it is a clear here — and the picture
373
+ // goes back to the unframed bytes, which is the same fact stated in the one place a fake can
374
+ // state it.
375
+ colorScheme = scheme === 'no-preference' ? null : scheme;
376
+ return Promise.resolve();
377
+ },
336
378
  screenshot: (options: CaptureOptions): Promise<Uint8Array> =>
337
- Promise.resolve(options.clip === undefined ? FAKE_PNG : clippedPng(options.clip)),
379
+ Promise.resolve(framedPng(options.clip, colorScheme)),
338
380
  pdf: (_options: CaptureOptions): Promise<Uint8Array> => Promise.resolve(FAKE_PDF),
339
381
  cookies: (): Promise<readonly ScrapeCookie[]> => Promise.resolve(session.cookies),
340
382
  session: (): Promise<SessionSnapshot> => Promise.resolve(session),
package/src/http.ts CHANGED
@@ -10,7 +10,7 @@
10
10
  // guarantee the page vocabulary makes — and a different exit IP mid-session is exactly what
11
11
  // anti-bot systems look for.
12
12
 
13
- import { readWithinLimit } from '@ultimat3/core';
13
+ import { finiteCount, readWithinLimit } from '@ultimat3/core';
14
14
  import type { StandardSchemaV1 } from '@ultimat3/schema';
15
15
  import { parse } from '@ultimat3/schema';
16
16
  import type { ScrapeClock } from './clock';
@@ -172,6 +172,29 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
172
172
  const call: ScrapeFetch = init.fetch ?? fetch;
173
173
  return {
174
174
  async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
175
+ // Screened FIRST — before the activity touch, before the robots read this method performs
176
+ // and before a byte leaves. `AbortSignal.timeout(NaN)` THROWS, and it throws a bare
177
+ // `TypeError` ("Value NaN is outside the range [0, 9007199254740991]"), which is the one
178
+ // thing the deadline below exists to prevent: an unclassified platform error reaching a
179
+ // job's retry classifier instead of this package's own `X_SCRAPE_TIMEOUT`. And the cap is
180
+ // the only thing between a hostile stream and the worker's heap, so `readWithinLimit`'s own
181
+ // refusal is too late: it arrives once the request — a POST included — has been performed.
182
+ //
183
+ // Both floors are 1. A zero deadline aborts on the tick it is armed and a zero cap puts
184
+ // every response over, so either one makes every request on this leg fail; neither is a
185
+ // caller declining a feature the way `watchdog.graceMs: 0` is.
186
+ const timeoutMs = finiteCount(
187
+ 'http.request',
188
+ 'timeout',
189
+ request.timeout ?? init.timeoutMs,
190
+ 1,
191
+ );
192
+ const maxBytes = finiteCount(
193
+ 'http.request',
194
+ 'maxBytes',
195
+ request.maxBytes ?? DEFAULT_HTTP_MAX_BYTES,
196
+ 1,
197
+ );
175
198
  init.onActivity?.();
176
199
  if (interceptVerdict(url, 'fetch', init.rules) !== 'allow') {
177
200
  throw hostBlocked(url, init.rules.allowHosts);
@@ -180,7 +203,6 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
180
203
  await init.pace?.(init.signal);
181
204
  const session = await init.session();
182
205
  const cookies = cookieHeaderFor(session.cookies, url);
183
- const timeoutMs = request.timeout ?? init.timeoutMs;
184
206
  // `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
185
207
  // own fetch, not a wait this package performs — and under a test clock a slept deadline
186
208
  // would fire on the microtask after it was armed, cancelling every request instantly.
@@ -210,7 +232,6 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
210
232
  // Counted as it arrives rather than `.text()`, which materialises first and checks never:
211
233
  // a 30s stream at 50MB/s is a 1.5GB allocation the worker does not get back, and it takes
212
234
  // every other job on that worker with it. The same read `robots-fetch.ts` performs.
213
- const maxBytes = request.maxBytes ?? DEFAULT_HTTP_MAX_BYTES;
214
235
  const capped = await readWithinLimit(response.body, maxBytes);
215
236
  if ('over' in capped) throw bodyTooLarge(url, capped.over, maxBytes);
216
237
  const body = new TextDecoder().decode(capped.bytes);
package/src/index.ts CHANGED
@@ -34,6 +34,8 @@ export type { CdpTargetInit } from './cdp-target';
34
34
  export { CDP_DRIVER, cdpTarget } from './cdp-target';
35
35
  export type { Deadline, ScrapeClock, TestScrapeClock } from './clock';
36
36
  export { deadline, systemScrapeClock, testClock, throwIfAborted } from './clock';
37
+ export type { ColorScheme } from './color-scheme';
38
+ export { COLOR_SCHEME_FEATURE, COLOR_SCHEMES, isColorScheme } from './color-scheme';
37
39
  export {
38
40
  cookieDomainMatches,
39
41
  cookieHeaderFor,
@@ -10,6 +10,7 @@ import { awaitActionable } from './actionability';
10
10
  import { assertCaptureFraming } from './capture-clip';
11
11
  import type { ScrapeClock } from './clock';
12
12
  import { deadline } from './clock';
13
+ import type { ColorScheme } from './color-scheme';
13
14
  import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
14
15
  import { hostDecision } from './hosts';
15
16
  import type {
@@ -243,6 +244,11 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
243
244
  async offline(enabled: boolean): Promise<void> {
244
245
  await target.setOfflineMode(enabled);
245
246
  },
247
+ // `async`, for `offline()`'s reason: a driver that throws synchronously from a promise-typed
248
+ // method would escape the caller's `.catch()`.
249
+ async colorScheme(scheme: ColorScheme): Promise<void> {
250
+ await target.setColorScheme(scheme);
251
+ },
246
252
  cookies: (): Promise<readonly ScrapeCookie[]> => target.cookies(),
247
253
  session: () => target.session(),
248
254
  // Redacted BY VALUE on the way out, the same pass `html()` makes. A console line and a request
package/src/page.ts CHANGED
@@ -10,6 +10,7 @@
10
10
  import type { Secret } from '@ultimat3/core';
11
11
  import type { ActionabilityState } from './actionability';
12
12
  import type { CaptureFraming } from './capture-clip';
13
+ import type { ColorScheme } from './color-scheme';
13
14
  import type { ConsoleLine, NetworkEntry, PageError } from './rings';
14
15
  import type { SessionSnapshot } from './session-state';
15
16
  import type { ElementSnapshot, ScrapeCookie, ScrapeDownloadFile } from './target';
@@ -109,6 +110,21 @@ export interface ScrapePage extends ScrapeFrame {
109
110
  * "done" would let an offline assertion pass against an app that never went offline.
110
111
  */
111
112
  offline(enabled: boolean): Promise<void>;
113
+ /**
114
+ * Tell the browser what the user's OS colour preference is, so a component that resolves its own
115
+ * theme resolves to this one. `'no-preference'` CLEARS the override and gives the browser's own
116
+ * answer back — it is not a third value a stylesheet can match.
117
+ *
118
+ * The INPUT, never the outcome: an attribute on the document is the outcome of a theme decision
119
+ * and the component owns that — one honouring `'system'` overwrites or deletes it on mount, and
120
+ * the picture then shows the theme the component chose rather than the one that was asked for.
121
+ *
122
+ * ACCEPTED on every driver, unlike `offline()`: the offline drivers evaluate no CSS, but they
123
+ * answer different deterministic bytes per scheme, so the one thing a preference could be wrong
124
+ * about is still under test. `X_NOT_IMPLEMENTED` is reserved for a LAUNCHER that lacks the
125
+ * method, which is a fact about the build rather than about the driver.
126
+ */
127
+ colorScheme(scheme: ColorScheme): Promise<void>;
112
128
  /**
113
129
  * The handoff, made explicit: what the HTTP leg will send, as a value an author can inspect and
114
130
  * a fixture can assert on. `http` uses it automatically — this is for seeing what carried over.
@@ -8,7 +8,7 @@
8
8
  // The exit is a RESOLVER, not a string: `scrape-run.ts` builds this gate as an argument to
9
9
  // `driver.open()`, and the proxy is a driver option the session only reports on the way back out.
10
10
 
11
- import { readWithinLimit } from '@ultimat3/core';
11
+ import { finiteCount, readWithinLimit } from '@ultimat3/core';
12
12
  import type { ScrapeFetch } from './http';
13
13
  import type { RobotsFetch } from './robots';
14
14
 
@@ -56,11 +56,29 @@ export interface RobotsFetchInit {
56
56
  */
57
57
  export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
58
58
  const call: ScrapeFetch = init.fetch ?? fetch;
59
- const limit = init.maxBytes ?? DEFAULT_ROBOTS_MAX_BYTES;
59
+ // Both bounds are screened HERE, at construction, and both floors are 1 — because every way this
60
+ // read can fail is the same answer, `undefined`, which the gate reads as "no restrictions". A
61
+ // `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
62
+ // outside the range [0, 9007199254740991]") straight into the gate's own `.catch`, and a `NaN`
63
+ // cap makes `readWithinLimit` refuse after the request already left. A zero of either is the
64
+ // same outcome spelled deliberately: an expired deadline and a cap every file is over. Robots
65
+ // enforcement off, for the whole run, with nothing in the log.
66
+ const limit = finiteCount(
67
+ 'robotsFetcher',
68
+ 'maxBytes',
69
+ init.maxBytes ?? DEFAULT_ROBOTS_MAX_BYTES,
70
+ 1,
71
+ );
72
+ const timeoutMs = finiteCount(
73
+ 'robotsFetcher',
74
+ 'timeoutMs',
75
+ init.timeoutMs ?? DEFAULT_ROBOTS_TIMEOUT_MS,
76
+ 1,
77
+ );
60
78
  return async (robotsUrl: string): Promise<string | undefined> => {
61
79
  // Armed per read, not per gate: the gate is long-lived and reads once per origin, so a
62
80
  // deadline created alongside it would already have expired by the second origin.
63
- const deadline = AbortSignal.timeout(init.timeoutMs ?? DEFAULT_ROBOTS_TIMEOUT_MS);
81
+ const deadline = AbortSignal.timeout(timeoutMs);
64
82
  const signal = init.signal === undefined ? deadline : AbortSignal.any([deadline, init.signal]);
65
83
  // Resolved here, at the read, because the session that owns the exit did not exist when this
66
84
  // fetcher was built. An empty string is not an exit and is dropped with the absent one.
package/src/scrape-run.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  // A step record saying "logged in" would be a checkpoint asserting something about a session that
10
10
  // may have expired an hour ago.
11
11
 
12
+ import { finiteCount, finiteOption } from '@ultimat3/core';
12
13
  import type { JobRunArgs } from '@ultimat3/jobs';
13
14
  import { parse } from '@ultimat3/schema';
14
15
  import { createArtifactWriter } from './artifacts';
@@ -41,13 +42,23 @@ const orgOf = (ctx: unknown): string | undefined => {
41
42
  return typeof actor?.orgId === 'string' ? actor.orgId : undefined;
42
43
  };
43
44
 
44
- const toMillis = (value: string | number | undefined, fallback: number): number => {
45
+ /**
46
+ * `'30s'` | `30_000` | absent, as milliseconds — screened under the name the DEFINITION uses.
47
+ *
48
+ * The number branch is the one that needs it: the string branch can only ever produce digits, and
49
+ * a number a definition declares is whatever the app computed. What it lands on is the reason the
50
+ * refusal is here rather than downstream — this one value becomes the robots read's deadline
51
+ * (where a non-finite one turns robots enforcement off silently, because every failure of that
52
+ * read answers "no restrictions"), the session's `timeoutMs`, and through it every actionability
53
+ * budget in the run, where `NaN <= 0` is false so the poll loop never leaves.
54
+ */
55
+ const toMillis = (value: string | number | undefined, fallback: number, option: string): number => {
45
56
  if (value === undefined) return fallback;
46
- if (typeof value === 'number') return value;
57
+ if (typeof value === 'number') return finiteCount('the scrape definition', option, value, 1);
47
58
  const match = /^(\d+(?:\.\d+)?)(ms|s|m|h)?$/.exec(value.trim());
48
59
  if (match === null) return fallback;
49
60
  const scale = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000 }[match[2] ?? 'ms'] ?? 1;
50
- return Number(match[1]) * scale;
61
+ return finiteCount('the scrape definition', option, Number(match[1]) * scale, 1);
51
62
  };
52
63
 
53
64
  export const DEFAULT_PAGE_TIMEOUT_MS = 30_000;
@@ -69,7 +80,14 @@ export async function runScrape<I, Row>(
69
80
  });
70
81
  const secrets = createSecretBag(definition.secrets ?? []);
71
82
  const rules = { allowHosts: definition.allowHosts, block: definition.block };
72
- const pace = createPacer(definition.rate ?? DEFAULT_NAVIGATION_RATE, clock);
83
+ // Screened here as well as in `scrape()`, and the two are not one check written twice: that one
84
+ // refuses the DECLARATION and never sees a definition assembled by hand, which `runScrape` is
85
+ // exported to accept. `finiteOption` and not `finiteCount` — a rate of 0.5 is one navigation
86
+ // every two seconds, and `scrape()` owns the "greater than zero" half.
87
+ const pace = createPacer(
88
+ finiteOption('the scrape definition', 'rate', definition.rate ?? DEFAULT_NAVIGATION_RATE),
89
+ clock,
90
+ );
73
91
  const artifact = createArtifactWriter({
74
92
  storage: definition.artifacts?.storage,
75
93
  scrape: definition.name,
@@ -94,7 +112,7 @@ export async function runScrape<I, Row>(
94
112
  // Read BEFORE the browser opens: a refused credential must not reach a login form again, and
95
113
  // opening a session first would already have spent an identity on a run that cannot succeed.
96
114
  const restored = await restorableSession(plan);
97
- const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS);
115
+ const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS, 'pageTimeout');
98
116
  // The exit the session dials, readable only AFTER `driver.open()` — the proxy is a driver
99
117
  // option and the gate below is an argument to `open()`, so the gate asks for it per read
100
118
  // instead of being handed a value that cannot exist yet. Every read happens during a
package/src/target.ts CHANGED
@@ -4,6 +4,7 @@
4
4
  // fixture replayer and the fake alike.
5
5
 
6
6
  import type { CaptureFraming } from './capture-clip';
7
+ import type { ColorScheme } from './color-scheme';
7
8
  import type { ConsoleRing, NetworkRing, PageErrorRing } from './rings';
8
9
  import type { SessionSnapshot } from './session-state';
9
10
 
@@ -141,6 +142,28 @@ export interface ScrapeTarget {
141
142
  * green against an app that was online the whole time.
142
143
  */
143
144
  setOfflineMode(enabled: boolean): Promise<void>;
145
+ /**
146
+ * What the browser reports as the user's OS colour preference — the INPUT to a theme decision.
147
+ * `'no-preference'` CLEARS the override rather than setting one (`color-scheme.ts`).
148
+ *
149
+ * The INPUT and never the outcome. Setting `data-theme` on the document models the outcome, and
150
+ * the component owns that: one that resolves `'system'` itself overwrites or deletes the
151
+ * attribute on mount, so a harness that set it is silently overruled and every theme converges
152
+ * on one picture — measured, and byte-identical, in issue #338.
153
+ *
154
+ * REQUIRED here where `CdpPageLike.emulateMediaFeatures` is optional, for `setOfflineMode`'s
155
+ * reason: the asymmetry is the enforcement, and a driver author gets a type error naming this
156
+ * member rather than a silent no-op. A launcher without the method is `X_NOT_IMPLEMENTED`.
157
+ *
158
+ * A DRIVER WITH NO CSS ENGINE ACCEPTS IT, which is the one place this parts company with
159
+ * `setOfflineMode`, and the line between them is which side of a capture the verb is on. An
160
+ * offline ASSERTION is reachable on a driver that answers content, so a resolved
161
+ * `setOfflineMode` would let it pass against an app that never went offline. A colour preference
162
+ * has no such assertion: the only thing it could be wrong about is a picture, and the offline
163
+ * drivers answer different deterministic bytes per scheme — exactly as they do per `clip` —
164
+ * so a driver that dropped the preference fails a test rather than passing every one.
165
+ */
166
+ setColorScheme(scheme: ColorScheme): Promise<void>;
144
167
  screenshot(options: CaptureOptions): Promise<Uint8Array>;
145
168
  pdf(options: CaptureOptions): Promise<Uint8Array>;
146
169
  cookies(): Promise<readonly ScrapeCookie[]>;
package/src/watchdog.ts CHANGED
@@ -10,6 +10,7 @@
10
10
  // socket die, which is what turns an infinite await into a catchable `X_SCRAPE_WEDGED` — and a
11
11
  // graceful-quit CEILING on shutdown, past which the same kill runs.
12
12
 
13
+ import { finiteCount } from '@ultimat3/core';
13
14
  import type { ScrapeClock } from './clock';
14
15
  import { watchdogStopped, wedged } from './error-throws';
15
16
 
@@ -49,8 +50,19 @@ export interface WedgeGuard {
49
50
  }
50
51
 
51
52
  export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
52
- const idleMs = init.idleMs ?? DEFAULT_IDLE_MS;
53
- const graceMs = init.graceMs ?? DEFAULT_GRACE_MS;
53
+ // Screened before the watch loop starts, because both failures are silent in the expensive
54
+ // direction. `elapsed < NaN` is FALSE, so a `NaN` idleMs fires on the first 250ms poll and every
55
+ // run dies as `X_SCRAPE_WEDGED` against a browser that answered; `elapsed < Infinity` is always
56
+ // true, so the loop never fires and the guard is incident #1 with nothing armed. A `NaN` graceMs
57
+ // is `clock.sleep(NaN)`, which `setTimeout` reads as 0: `browser.close()` cannot win the race, so
58
+ // `kill()` runs instead — and on `remoteBrowser()`, where `process()` is `null`, that reaches
59
+ // nothing and the paid remote session survives the run.
60
+ //
61
+ // The floors differ because zero means two different things. `idleMs: 0` is "kill at the first
62
+ // poll", which is the NaN outcome spelled deliberately; `graceMs: 0` is "do not wait for the
63
+ // polite close", which is a policy a caller may hold.
64
+ const idleMs = finiteCount('the scrape watchdog', 'idleMs', init.idleMs ?? DEFAULT_IDLE_MS, 1);
65
+ const graceMs = finiteCount('the scrape watchdog', 'graceMs', init.graceMs ?? DEFAULT_GRACE_MS);
54
66
  const controller = new AbortController();
55
67
  let lastTouch = init.clock.monotonic();
56
68
  // TWO latches, not one. `stopped` ends the watch loop; `shuttingDown` guards the teardown. They