@ultimat3/scraping 16.0.0 → 17.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "16.0.0",
+  "version": "17.0.0",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -30,9 +30,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "16.0.0",
-    "@ultimat3/jobs": "16.0.0",
-    "@ultimat3/schema": "16.0.0",
-    "@ultimat3/storage": "16.0.0"
+    "@ultimat3/core": "17.0.0",
+    "@ultimat3/jobs": "17.0.0",
+    "@ultimat3/schema": "17.0.0",
+    "@ultimat3/storage": "17.0.0"
   }
 }

package/src/actionability.ts

@@ -4,6 +4,7 @@
 // `waitForSelector('#aceptar:not([disabled])')`, which is this rule, spelled once, by hand, at
 // one call site out of forty.
 
+import { finiteCount } from '@ultimat3/core';
 import type { ScrapeClock } from './clock';
 import { deadline } from './clock';
 import { notActionable, selectorMissing } from './error-throws';
@@ -85,8 +86,12 @@ export interface ActionabilityWait {
  * a scraper failure take an afternoon.
  */
 export async function awaitActionable(wait: ActionabilityWait): Promise<ElementSnapshot> {
-  const budget = deadline(wait.clock, wait.timeoutMs);
-  const pollMs = wait.pollMs ?? DEFAULT_POLL_MS;
+  const budget = deadline(wait.clock, wait.timeoutMs, 'page.waitFor');
+  // At least 1ms, and the floor is the point: `Math.min(0, remainingMs())` is 0, `clock.sleep(0)`
+  // returns on the next turn, and the poll is then one full round trip to the browser per
+  // event-loop turn for the whole budget. `NaN` is the same loop by a different route —
+  // `Math.min(NaN, x)` is `NaN` and `setTimeout(fn, NaN)` is `setTimeout(fn, 0)`.
+  const pollMs = finiteCount('page.waitFor', 'pollMs', wait.pollMs ?? DEFAULT_POLL_MS, 1);
   let previous: ElementSnapshot | undefined;
   let lastProblem: string | undefined;
   let everSeen = false;

package/src/auth.ts

@@ -18,6 +18,7 @@
 // is written into the session record so the NEXT attempt fails before reaching the login form.
 
 import type { Logger } from '@ultimat3/core';
+import { finiteCount } from '@ultimat3/core';
 import type { ScrapeClock } from './clock';
 import { authFailed, promptUnanswered, sessionExpired } from './error-throws';
 import type { ScrapePage } from './page';
@@ -121,8 +122,17 @@ export async function restorableSession<I>(
   if (plan.auth?.reuse === false) return undefined;
   const maxAge = plan.auth?.maxAge;
   if (maxAge !== undefined) {
+    // Screened, because `age > NaN` is false and false here means RESTORED: a `NaN` maxAge hands
+    // back a session of any age, with no re-login and nothing in the report. `0` is legal — it
+    // means "restore nothing stored before now" — so the floor stays there.
+    const limit = finiteCount('the scrape auth', 'maxAge', maxAge);
     const age = plan.clock.now().getTime() - new Date(found.savedAt).getTime();
-    if (age > maxAge) return undefined;
+    // `!(age <= limit)` and not `age > limit`, which is the same test for every finite age and the
+    // OPPOSITE one for a `NaN`. `savedAt` is data, not configuration: it comes back off a bucket
+    // and `parseSessionState` only asks that it is a string, so an edited or half-written record
+    // produces a `NaN` age against a perfectly good `maxAge`. Failing closed costs one re-login;
+    // failing open acts as somebody else, indefinitely.
+    if (!(age <= limit)) return undefined;
   }
   plan.logger.debug('scrape.session.restored', sessionDigest(found));
   return found;

package/src/clock.ts

@@ -5,6 +5,7 @@
 // `clock-discipline.test.ts` fails the build if a second file reaches for a timer directly.
 
 import type { Clock } from '@ultimat3/core';
+import { finiteCount } from '@ultimat3/core';
 
 export interface ScrapeClock extends Clock {
   /**
@@ -90,8 +91,14 @@ export interface Deadline {
   expired(): boolean;
 }
 
-export function deadline(clock: Clock, totalMs: number): Deadline {
+export function deadline(clock: Clock, totalMs: number, subject = 'a scrape wait'): Deadline {
+  // Screened HERE because this is the one constructor of a budget, and every `for (;;)` in this
+  // package asks `expired()` to leave it. `Math.max(0, NaN - elapsed)` is `NaN` and `NaN <= 0` is
+  // false, so a budget that is not a number never expires: the loop in `awaitActionable` and the
+  // one in `page-over-target.ts`'s `frame()` both become unbounded, re-reading a live browser once
+  // per `Math.min(pollMs, NaN)` — which is `NaN`, which `setTimeout` reads as 0.
+  const budgetMs = finiteCount(subject, 'timeoutMs', totalMs);
   const startedAt = clock.monotonic();
-  const remainingMs = (): number => Math.max(0, totalMs - (clock.monotonic() - startedAt));
-  return { totalMs, remainingMs, expired: () => remainingMs() <= 0 };
+  const remainingMs = (): number => Math.max(0, budgetMs - (clock.monotonic() - startedAt));
+  return { totalMs: budgetMs, remainingMs, expired: () => remainingMs() <= 0 };
 }

package/src/driver-fake.ts

@@ -5,6 +5,7 @@
 // It covers BOTH legs. `fakeBrowser({ pages, http })` replays the browser walk and the JSON
 // endpoints behind it from one declaration, so a hybrid scrape is tested the way it runs.
 
+import { finiteCount } from '@ultimat3/core';
 import type { ScrapeClock } from './clock';
 import { systemScrapeClock } from './clock';
 import type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
@@ -113,7 +114,10 @@ export function fakePage(dom: string, options: FakePageOptions = {}): ScrapePage
   return pageOverTarget(target, {
     clock,
     allowHosts,
-    defaultTimeoutMs: options.timeoutMs ?? 1_000,
+    // The default budget for every wait AND every navigation on this page, so the floor is 1: a
+    // session default of 0 is already out of time everywhere, which a per-call `{ timeout: 0 }` —
+    // "is it there right now" — is not. Non-finite is the loop that never leaves.
+    defaultTimeoutMs: finiteCount('fakePage', 'timeoutMs', options.timeoutMs ?? 1_000, 1),
     ...options.context,
   });
 }

package/src/expect.ts

@@ -3,6 +3,7 @@
 // where the data went. `expect` turns that into a red run: a yield under `minRows`, or under
 // `maxDrop` of what this scrape normally returns, throws.
 
+import { finiteCount, finiteOption } from '@ultimat3/core';
 import { yieldCollapsed } from './error-throws';
 import type { ScrapeError } from './errors';
 
@@ -61,7 +62,23 @@ export interface YieldCheck {
  * fifty histories without a queue, a browser or a clock.
  */
 export function yieldProblem(check: YieldCheck): ScrapeError | undefined {
-  const { minRows, maxDrop } = check.expect;
+  // Screened where the rule is, because this is the alarm and both directions are silent. A `NaN`
+  // minRows makes `rows < minRows` false for every yield, so the floor never fires and the scrape
+  // is green on zero rows forever — the exact failure this file exists to prevent. A `NaN` maxDrop
+  // makes `rows >= baseline * (1 - NaN)` false for every yield, which fires the alarm on every run
+  // instead, and an alarm that always fires is an alarm somebody turns off.
+  //
+  // `minRows: 0` is legal and stays legal: this file asks an author whose answer is legitimately
+  // sometimes zero to declare exactly that. `maxDrop` is a FRACTION of a median, so `finiteOption`
+  // and not `finiteCount` — `0.5` is the documented value.
+  const minRows =
+    check.expect.minRows === undefined
+      ? undefined
+      : finiteCount('the scrape expect', 'minRows', check.expect.minRows);
+  const maxDrop =
+    check.expect.maxDrop === undefined
+      ? undefined
+      : finiteOption('the scrape expect', 'maxDrop', check.expect.maxDrop);
   if (minRows !== undefined && check.rows < minRows) {
     return yieldCollapsed({ scrape: check.scrape, rows: check.rows, reason: 'min-rows', minRows });
   }
@@ -101,7 +118,15 @@ export async function guardYield(input: YieldGuardInput): Promise<void> {
   // `maxDrop` needs `MIN_BASELINE_RUNS` runs after it is declared before it can fire — a delay,
   // not a hole. `expect.test.ts` pins both halves.
   if (input.expect === undefined) return;
-  const window = input.expect.window ?? DEFAULT_YIELD_WINDOW;
+  // At least 1: `[…].slice(-0)` is `slice(0)`, the WHOLE history, so a zero window is the largest
+  // baseline rather than no baseline — and the number is handed to an app's own `recent()`, where
+  // it is usually a SQL `limit`.
+  const window = finiteCount(
+    'the scrape expect',
+    'window',
+    input.expect.window ?? DEFAULT_YIELD_WINDOW,
+    1,
+  );
   const history =
     input.history === undefined ? [] : await input.history.recent(input.scrape, window);
   const problem = yieldProblem({

package/src/http.ts

@@ -10,7 +10,7 @@
 // guarantee the page vocabulary makes — and a different exit IP mid-session is exactly what
 // anti-bot systems look for.
 
-import { readWithinLimit } from '@ultimat3/core';
+import { finiteCount, readWithinLimit } from '@ultimat3/core';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import { parse } from '@ultimat3/schema';
 import type { ScrapeClock } from './clock';
@@ -172,6 +172,29 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
   const call: ScrapeFetch = init.fetch ?? fetch;
   return {
     async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
+      // Screened FIRST — before the activity touch, before the robots read this method performs
+      // and before a byte leaves. `AbortSignal.timeout(NaN)` THROWS, and it throws a bare
+      // `TypeError` ("Value NaN is outside the range [0, 9007199254740991]"), which is the one
+      // thing the deadline below exists to prevent: an unclassified platform error reaching a
+      // job's retry classifier instead of this package's own `X_SCRAPE_TIMEOUT`. And the cap is
+      // the only thing between a hostile stream and the worker's heap, so `readWithinLimit`'s own
+      // refusal is too late: it arrives once the request — a POST included — has been performed.
+      //
+      // Both floors are 1. A zero deadline aborts on the tick it is armed and a zero cap puts
+      // every response over, so either one makes every request on this leg fail; neither is a
+      // caller declining a feature the way `watchdog.graceMs: 0` is.
+      const timeoutMs = finiteCount(
+        'http.request',
+        'timeout',
+        request.timeout ?? init.timeoutMs,
+        1,
+      );
+      const maxBytes = finiteCount(
+        'http.request',
+        'maxBytes',
+        request.maxBytes ?? DEFAULT_HTTP_MAX_BYTES,
+        1,
+      );
       init.onActivity?.();
       if (interceptVerdict(url, 'fetch', init.rules) !== 'allow') {
         throw hostBlocked(url, init.rules.allowHosts);
@@ -180,7 +203,6 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
       await init.pace?.(init.signal);
       const session = await init.session();
       const cookies = cookieHeaderFor(session.cookies, url);
-      const timeoutMs = request.timeout ?? init.timeoutMs;
       // `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
       // own fetch, not a wait this package performs — and under a test clock a slept deadline
       // would fire on the microtask after it was armed, cancelling every request instantly.
@@ -210,7 +232,6 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
         // Counted as it arrives rather than `.text()`, which materialises first and checks never:
         // a 30s stream at 50MB/s is a 1.5GB allocation the worker does not get back, and it takes
         // every other job on that worker with it. The same read `robots-fetch.ts` performs.
-        const maxBytes = request.maxBytes ?? DEFAULT_HTTP_MAX_BYTES;
         const capped = await readWithinLimit(response.body, maxBytes);
         if ('over' in capped) throw bodyTooLarge(url, capped.over, maxBytes);
         const body = new TextDecoder().decode(capped.bytes);

package/src/robots-fetch.ts

@@ -8,7 +8,7 @@
 // The exit is a RESOLVER, not a string: `scrape-run.ts` builds this gate as an argument to
 // `driver.open()`, and the proxy is a driver option the session only reports on the way back out.
 
-import { readWithinLimit } from '@ultimat3/core';
+import { finiteCount, readWithinLimit } from '@ultimat3/core';
 import type { ScrapeFetch } from './http';
 import type { RobotsFetch } from './robots';
 
@@ -56,11 +56,29 @@ export interface RobotsFetchInit {
  */
 export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
   const call: ScrapeFetch = init.fetch ?? fetch;
-  const limit = init.maxBytes ?? DEFAULT_ROBOTS_MAX_BYTES;
+  // Both bounds are screened HERE, at construction, and both floors are 1 — because every way this
+  // read can fail is the same answer, `undefined`, which the gate reads as "no restrictions". A
+  // `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
+  // outside the range [0, 9007199254740991]") straight into the gate's own `.catch`, and a `NaN`
+  // cap makes `readWithinLimit` refuse after the request already left. A zero of either is the
+  // same outcome spelled deliberately: an expired deadline and a cap every file is over. Robots
+  // enforcement off, for the whole run, with nothing in the log.
+  const limit = finiteCount(
+    'robotsFetcher',
+    'maxBytes',
+    init.maxBytes ?? DEFAULT_ROBOTS_MAX_BYTES,
+    1,
+  );
+  const timeoutMs = finiteCount(
+    'robotsFetcher',
+    'timeoutMs',
+    init.timeoutMs ?? DEFAULT_ROBOTS_TIMEOUT_MS,
+    1,
+  );
   return async (robotsUrl: string): Promise<string | undefined> => {
     // Armed per read, not per gate: the gate is long-lived and reads once per origin, so a
     // deadline created alongside it would already have expired by the second origin.
-    const deadline = AbortSignal.timeout(init.timeoutMs ?? DEFAULT_ROBOTS_TIMEOUT_MS);
+    const deadline = AbortSignal.timeout(timeoutMs);
     const signal = init.signal === undefined ? deadline : AbortSignal.any([deadline, init.signal]);
     // Resolved here, at the read, because the session that owns the exit did not exist when this
     // fetcher was built. An empty string is not an exit and is dropped with the absent one.

package/src/scrape-run.ts

@@ -9,6 +9,7 @@
 // A step record saying "logged in" would be a checkpoint asserting something about a session that
 // may have expired an hour ago.
 
+import { finiteCount, finiteOption } from '@ultimat3/core';
 import type { JobRunArgs } from '@ultimat3/jobs';
 import { parse } from '@ultimat3/schema';
 import { createArtifactWriter } from './artifacts';
@@ -41,13 +42,23 @@ const orgOf = (ctx: unknown): string | undefined => {
   return typeof actor?.orgId === 'string' ? actor.orgId : undefined;
 };
 
-const toMillis = (value: string | number | undefined, fallback: number): number => {
+/**
+ * `'30s'` | `30_000` | absent, as milliseconds — screened under the name the DEFINITION uses.
+ *
+ * The number branch is the one that needs it: the string branch can only ever produce digits, and
+ * a number a definition declares is whatever the app computed. What it lands on is the reason the
+ * refusal is here rather than downstream — this one value becomes the robots read's deadline
+ * (where a non-finite one turns robots enforcement off silently, because every failure of that
+ * read answers "no restrictions"), the session's `timeoutMs`, and through it every actionability
+ * budget in the run, where `NaN <= 0` is false so the poll loop never leaves.
+ */
+const toMillis = (value: string | number | undefined, fallback: number, option: string): number => {
   if (value === undefined) return fallback;
-  if (typeof value === 'number') return value;
+  if (typeof value === 'number') return finiteCount('the scrape definition', option, value, 1);
   const match = /^(\d+(?:\.\d+)?)(ms|s|m|h)?$/.exec(value.trim());
   if (match === null) return fallback;
   const scale = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000 }[match[2] ?? 'ms'] ?? 1;
-  return Number(match[1]) * scale;
+  return finiteCount('the scrape definition', option, Number(match[1]) * scale, 1);
 };
 
 export const DEFAULT_PAGE_TIMEOUT_MS = 30_000;
@@ -69,7 +80,14 @@ export async function runScrape<I, Row>(
   });
   const secrets = createSecretBag(definition.secrets ?? []);
   const rules = { allowHosts: definition.allowHosts, block: definition.block };
-  const pace = createPacer(definition.rate ?? DEFAULT_NAVIGATION_RATE, clock);
+  // Screened here as well as in `scrape()`, and the two are not one check written twice: that one
+  // refuses the DECLARATION and never sees a definition assembled by hand, which `runScrape` is
+  // exported to accept. `finiteOption` and not `finiteCount` — a rate of 0.5 is one navigation
+  // every two seconds, and `scrape()` owns the "greater than zero" half.
+  const pace = createPacer(
+    finiteOption('the scrape definition', 'rate', definition.rate ?? DEFAULT_NAVIGATION_RATE),
+    clock,
+  );
   const artifact = createArtifactWriter({
     storage: definition.artifacts?.storage,
     scrape: definition.name,
@@ -94,7 +112,7 @@ export async function runScrape<I, Row>(
   // Read BEFORE the browser opens: a refused credential must not reach a login form again, and
   // opening a session first would already have spent an identity on a run that cannot succeed.
   const restored = await restorableSession(plan);
-  const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS);
+  const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS, 'pageTimeout');
   // The exit the session dials, readable only AFTER `driver.open()` — the proxy is a driver
   // option and the gate below is an argument to `open()`, so the gate asks for it per read
   // instead of being handed a value that cannot exist yet. Every read happens during a

package/src/watchdog.ts

@@ -10,6 +10,7 @@
 // socket die, which is what turns an infinite await into a catchable `X_SCRAPE_WEDGED` — and a
 // graceful-quit CEILING on shutdown, past which the same kill runs.
 
+import { finiteCount } from '@ultimat3/core';
 import type { ScrapeClock } from './clock';
 import { watchdogStopped, wedged } from './error-throws';
 
@@ -49,8 +50,19 @@ export interface WedgeGuard {
 }
 
 export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
-  const idleMs = init.idleMs ?? DEFAULT_IDLE_MS;
-  const graceMs = init.graceMs ?? DEFAULT_GRACE_MS;
+  // Screened before the watch loop starts, because both failures are silent in the expensive
+  // direction. `elapsed < NaN` is FALSE, so a `NaN` idleMs fires on the first 250ms poll and every
+  // run dies as `X_SCRAPE_WEDGED` against a browser that answered; `elapsed < Infinity` is always
+  // true, so the loop never fires and the guard is incident #1 with nothing armed. A `NaN` graceMs
+  // is `clock.sleep(NaN)`, which `setTimeout` reads as 0: `browser.close()` cannot win the race, so
+  // `kill()` runs instead — and on `remoteBrowser()`, where `process()` is `null`, that reaches
+  // nothing and the paid remote session survives the run.
+  //
+  // The floors differ because zero means two different things. `idleMs: 0` is "kill at the first
+  // poll", which is the NaN outcome spelled deliberately; `graceMs: 0` is "do not wait for the
+  // polite close", which is a policy a caller may hold.
+  const idleMs = finiteCount('the scrape watchdog', 'idleMs', init.idleMs ?? DEFAULT_IDLE_MS, 1);
+  const graceMs = finiteCount('the scrape watchdog', 'graceMs', init.graceMs ?? DEFAULT_GRACE_MS);
   const controller = new AbortController();
   let lastTouch = init.clock.monotonic();
   // TWO latches, not one. `stopped` ends the watch loop; `shuttingDown` guards the teardown. They