@ultimat3/scraping 11.0.0 → 11.2.0

package/README.md

@@ -108,6 +108,29 @@ same reason — a third-party driver that could omit it would be silent about er
 Entries are built with `pageErrorEntry()`, which truncates at `MAX_PAGE_ERROR_CHARS`: the ring
 bounds the count, and one `Maximum call stack size exceeded` is thousands of frames.
 
+### A picture of ONE component
+
+`page.screenshot({ clip: { x, y, width, height } })` crops to a rectangle in the page's own
+coordinate space — the space `getBoundingClientRect()` answers in. It exists because the reader of
+a scrape's screenshot is increasingly a vision model, whose pixels are the scarce resource: a
+whole-viewport picture spends them on everything that is not the component under review.
+
+| Framing | Answer |
+|---|---|
+| neither | the viewport, exactly as before the clip existed — byte for byte |
+| `fullPage: true` | the whole document |
+| `clip` | that rectangle |
+| **both** | `X_SCRAPE_CAPTURE_INVALID` — a browser honours one of the two without saying which |
+| `clip` with no area, or entirely in negative coordinates | `X_SCRAPE_CAPTURE_INVALID` — a blank picture that reads as a successful capture is the failure this exists to remove |
+| `clip` on `page.pdf()` | `X_SCRAPE_CAPTURE_INVALID` — a print engine paginates the document and has no crop |
+
+A rectangle **below the fold is accepted**. It is not checked against a viewport: this package
+neither sets nor reads one, and a component under the fold is the case a component crop is for.
+
+The offline drivers honour it, so the crop is provable with no Chrome — `fakeBrowser()` answers
+different deterministic bytes per rectangle, which is what lets a component screenshot be tested
+on a machine with no browser — the case CI is.
+
 ## What it owns
 
 | Module | Owns |
@@ -121,6 +144,7 @@ bounds the count, and one `Maximum call stack size exceeded` is thousands of fra
 | `auth.ts` / `session-state.ts` | acquire → persist → reuse → validate → burn |
 | `expect.ts` | the silent-green alarm |
 | `watchdog.ts` | the wedge and zombie discipline |
+| `capture-clip.ts` | the one framing rule — crop, whole page, or a refusal — checked before any driver sees it |
 | `errors.ts` / `error-throws.ts` | this package's `X_*` codes and their retry classification |
 
 ## Extending it — there is no plugin API, and none is needed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "11.0.0",
+  "version": "11.2.0",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -30,9 +30,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "11.0.0",
-    "@ultimat3/jobs": "11.0.0",
-    "@ultimat3/schema": "11.0.0",
-    "@ultimat3/storage": "11.0.0"
+    "@ultimat3/core": "11.2.0",
+    "@ultimat3/jobs": "11.2.0",
+    "@ultimat3/schema": "11.2.0",
+    "@ultimat3/storage": "11.2.0"
   }
 }

package/src/capture-clip.ts

@@ -0,0 +1,57 @@
+// The crop rectangle a capture may name, and the four shapes of one that would answer a blank
+// picture instead of a refusal. Its own module because the rule belongs to EVERY driver
+// identically — a check per driver is three chances to disagree — and because `target.ts` is a
+// port declaration, which is not a place decisions live.
+
+import { captureClipEmpty, captureClipOnPdf, captureFramingConflict } from './error-throws';
+
+/**
+ * CSS pixels in the page's own coordinate space, top-left origin — the space
+ * `getBoundingClientRect()` answers in, which is where a caller's rectangle comes from.
+ *
+ * No `scale`. CDP's clip has one and nothing here would read it, which is this repo's most
+ * repeated defect (`scripts/config-readers.ts`); a device pixel ratio belongs to the launch, not
+ * to one capture.
+ */
+export interface CaptureClip {
+  readonly x: number;
+  readonly y: number;
+  readonly width: number;
+  readonly height: number;
+}
+
+/** What both capture verbs are framed by. `CaptureOptions` and `CaptureRequest` are this, named. */
+export interface CaptureFraming {
+  readonly fullPage?: boolean | undefined;
+  readonly clip?: CaptureClip | undefined;
+}
+
+const finite = (value: number): boolean => Number.isFinite(value);
+
+/**
+ * Refuse a framing no driver can honour, BEFORE any driver sees it.
+ *
+ * What is deliberately NOT checked: whether the rectangle is inside the viewport. This package
+ * neither sets nor reads a viewport — nothing in `src/` mentions one — so the answer would cost a
+ * round trip into the page, and it would be the WRONG answer: a component below the fold is
+ * exactly what a component crop is for, and every current build captures beyond the viewport. A
+ * rectangle entirely in negative space needs no browser to refuse and is the half that is real.
+ */
+export function assertCaptureFraming(kind: 'screenshot' | 'pdf', framing: CaptureFraming): void {
+  const clip = framing.clip;
+  if (clip === undefined) return;
+  // A PDF is paginated by the print engine; CDP's `Page.printToPDF` has no clip to forward one to.
+  // Refused rather than dropped: a whole-page PDF returned to a caller who asked for one component
+  // is the same silent wrong answer the fullPage/clip pair produces.
+  if (kind === 'pdf') throw captureClipOnPdf(clip);
+  // `=== true` and not truthiness: `fullPage: false` beside a clip is a caller spelling out the
+  // default, which is not a conflict. Only the pair that CDP resolves silently is refused.
+  if (framing.fullPage === true) throw captureFramingConflict(clip);
+  if (!finite(clip.x) || !finite(clip.y) || !finite(clip.width) || !finite(clip.height)) {
+    throw captureClipEmpty(clip, 'is not four finite numbers');
+  }
+  if (clip.width <= 0 || clip.height <= 0) throw captureClipEmpty(clip, 'has no area');
+  if (clip.x + clip.width <= 0 || clip.y + clip.height <= 0) {
+    throw captureClipEmpty(clip, 'lies entirely in negative coordinates, where no page content is');
+  }
+}

package/src/cdp-port.ts

@@ -12,6 +12,25 @@
 // `connect({ browserWSEndpoint })`, headless Chrome 150): the WebSocket upgrade Playwright's
 // `connectOverCDP` cannot do under Bun works here, which is why this is the intended library.
 
+/**
+ * The library's screenshot options, restated here rather than shared with `CaptureClip`.
+ *
+ * This file is the shape of somebody ELSE's object, and importing this package's own vocabulary
+ * into it would make a puppeteer call signature depend on a scraping type — the direction this
+ * seam exists to forbid. The two shapes are structurally identical, so a `CaptureClip` passes
+ * where this is expected and no cast is needed anywhere.
+ */
+export interface CdpScreenshotOptions {
+  readonly fullPage?: boolean;
+  /** CSS pixels, page coordinates. Set INSTEAD of `fullPage`, never beside it. */
+  readonly clip?: {
+    readonly x: number;
+    readonly y: number;
+    readonly width: number;
+    readonly height: number;
+  };
+}
+
 export interface CdpRequestLike {
   url(): string;
   resourceType(): string;
@@ -34,7 +53,7 @@ export interface CdpPageLike {
   click(selector: string): Promise<void>;
   type(selector: string, text: string): Promise<void>;
   select(selector: string, ...values: string[]): Promise<string[]>;
-  screenshot(options: { readonly fullPage?: boolean }): Promise<Uint8Array | string>;
+  screenshot(options: CdpScreenshotOptions): Promise<Uint8Array | string>;
   pdf(options?: Record<string, unknown>): Promise<Uint8Array>;
   setRequestInterception(enabled: boolean): Promise<void>;
   /**

package/src/cdp-target.ts

@@ -329,7 +329,14 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
     evaluate: (expression) => guard('evaluate', () => init.page.evaluate(expression)),
     screenshot: (options: CaptureOptions) =>
       guard('screenshot', async () => {
-        const shot = await init.page.screenshot({ fullPage: options.fullPage === true });
+        // `fullPage` is OMITTED when a clip is given rather than sent as `false`: the two are
+        // exclusive at the library too, and some builds refuse the pair on truthiness while others
+        // resolve it silently. Sending only what was asked for is the request neither can misread.
+        const shot = await init.page.screenshot(
+          options.clip === undefined
+            ? { fullPage: options.fullPage === true }
+            : { clip: options.clip },
+        );
         // Some builds answer base64 text, some answer bytes. `atob` is the one decoder both a
         // browser and Bun agree on, and it keeps this file free of a Buffer import.
         return typeof shot === 'string'

package/src/error-throws.ts

@@ -3,7 +3,8 @@
 // a browser is rendered through core's `renderThrowable`, which is the rule `bun run error-render`
 // enforces.
 
-import { renderThrowable, UltimateError } from '@ultimat3/core';
+import { isRetryableStatus, renderThrowable, UltimateError } from '@ultimat3/core';
+import type { CaptureClip } from './capture-clip';
 import { ScrapeError } from './errors';
 
 /**
@@ -230,14 +231,29 @@ export const secretExposed = (artifact: string, url: string): ScrapeError =>
     meta: { artifact, url },
   });
 
-/** A non-2xx from the HTTP leg. 4xx below 429 is terminal; everything else may be tried again. */
+/**
+ * A non-2xx from the HTTP leg. The per-instance override that `errors.ts` reserves for this code:
+ * the code is registered `retryable`, and only a PERMANENT status downgrades an instance.
+ *
+ * Which 4xx are permanent is `@ultimat3/core`'s `isRetryableStatus` and not this package's opinion.
+ * Scraping had the framework's fifth copy of that table and it had drifted: `status !== 429` alone
+ * made a 408, a 409 and a 425 terminal here while `cache`, `mail` and `ai` all called them
+ * retryable — and terminal is not documentation, it dead-letters the run on the attempt that threw
+ * it (`packages/jobs/src/retry-classification.ts`), so a scrape burned a five-attempt policy on one
+ * request timeout. Nothing about a SCRAPE argues for a different answer: the statuses that could,
+ * 401 and 403, are permanent in core's table too and stay terminal here.
+ *
+ * The 4xx window stays because it is the range this override speaks for. A sub-400 non-ok — a 304
+ * is the reachable one — keeps the code's registered `retryable`: an early dead-letter on a status
+ * no table in the framework classifies is the expensive way to guess.
+ */
 export const httpFailed = (url: string, status: number, body: string): ScrapeError =>
   new ScrapeError({
     code: 'X_SCRAPE_HTTP_FAILED',
     cause: `${url} answered ${String(status)}: ${body}`,
     fix: 'confirm the endpoint the browser leg calls is the one this request names — page.network() lists every URL the page actually fetched',
     meta: { url, status },
-    retry: status >= 400 && status < 500 && status !== 429 ? 'terminal' : 'retryable',
+    retry: status >= 400 && status < 500 && !isRetryableStatus(status) ? 'terminal' : 'retryable',
   });
 
 /**
@@ -306,3 +322,35 @@ export const scrapeNotImplemented = (feature: string, fix: string): UltimateErro
     fix,
     meta: { feature },
   });
+
+/**
+ * The crop rectangle's three refusals, one code. Each is a caller's DECLARATION being impossible
+ * rather than a site behaving badly, so each names the edit that fixes it — and the alternative in
+ * every case is a picture that looks like a successful capture and is not of what was asked for.
+ */
+const clipText = (clip: CaptureClip): string =>
+  `${String(clip.width)}x${String(clip.height)} at ${String(clip.x)},${String(clip.y)}`;
+
+export const captureFramingConflict = (clip: CaptureClip): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_CAPTURE_INVALID',
+    cause: `this capture names both fullPage: true and clip ${clipText(clip)}, and a browser silently honours one of them without saying which`,
+    fix: 'drop fullPage — a clip is already the whole request — or drop clip to photograph the whole page',
+    meta: { clip: { ...clip }, fullPage: true },
+  });
+
+export const captureClipEmpty = (clip: CaptureClip, reason: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_CAPTURE_INVALID',
+    cause: `the clip ${clipText(clip)} ${reason}, so the capture would answer a blank picture that reads as a success`,
+    fix: "measure the element first — page.query('<selector>') answers a box with a positive width and height — and pass that rectangle as clip",
+    meta: { clip: { ...clip } },
+  });
+
+export const captureClipOnPdf = (clip: CaptureClip): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_CAPTURE_INVALID',
+    cause: `page.pdf() was given the clip ${clipText(clip)} and a PDF has no crop rectangle — the print engine paginates the whole document`,
+    fix: 'call page.screenshot({ clip }) for one component, or page.pdf() with no clip for the document',
+    meta: { clip: { ...clip } },
+  });

package/src/errors.ts

@@ -29,6 +29,7 @@ export const SCRAPE_OWNED_ERROR_CODES = [
   'X_SCRAPE_REMOTE_REQUIRED',
   'X_SCRAPE_RECOVER_REFUSED',
   'X_SCRAPE_SECRET_EXPOSED',
+  'X_SCRAPE_CAPTURE_INVALID',
   'X_SCRAPE_HTTP_FAILED',
   'X_SCRAPE_BODY_TOO_LARGE',
   'X_SCRAPE_AUTH_FAILED',
@@ -77,6 +78,7 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
   X_SCRAPE_REMOTE_REQUIRED: 'this driver needs a cdpUrl and was given none',
   X_SCRAPE_RECOVER_REFUSED: 'the recovery hook declined to recover this failure',
   X_SCRAPE_SECRET_EXPOSED: 'an artifact would have carried a secret this run typed',
+  X_SCRAPE_CAPTURE_INVALID: 'the capture names a framing no picture can be taken with',
   X_SCRAPE_HTTP_FAILED: 'the site answered the HTTP leg with a non-2xx status',
   X_SCRAPE_BODY_TOO_LARGE: 'the HTTP response body passed its byte cap',
   X_SCRAPE_AUTH_FAILED: 'the credentials were rejected',
@@ -112,9 +114,12 @@ export const SCRAPE_ERROR_RETRY = {
   // persisted identity re-trips it every time: the flagged cookies are the thing being refused,
   // so the retry has to arrive as somebody else or it is arithmetic, not a retry.
   X_SCRAPE_BLOCKED: 'retryable',
-  // Non-2xx is transient far more often than not (429, 502, a deploy). A 4xx that is genuinely
-  // permanent is thrown with a per-instance `terminal` override, which `UltimateError` supports —
-  // one code, and the throw site decides, because the same status is both at different sites.
+  // Non-2xx is transient far more often than not (408, 429, 502, a deploy). A 4xx that is
+  // genuinely permanent is thrown with a per-instance `terminal` override, which `UltimateError`
+  // supports — one code, and the throw site decides, because the same status is both at different
+  // sites. WHICH 4xx are permanent is not decided here: `httpFailed` reads core's
+  // `isRetryableStatus`, the framework's one table, so this leg cannot drift from `cache`, `mail`
+  // and `ai` again — it had, on 408, 409 and 425.
   X_SCRAPE_HTTP_FAILED: 'retryable',
   // Everything below is terminal, and each one is listed rather than left to the default so that
   // deleting a line is a visible decision.
@@ -135,6 +140,11 @@ export const SCRAPE_ERROR_RETRY = {
   // A declaration error, raised by `scrape()` before any attempt exists — there is no run to
   // retry, and the same definition would refuse identically forever.
   X_SCRAPE_YIELD_HISTORY_MISSING: 'terminal',
+  // A declaration error too, and the early dead letter is the CORRECT answer here: the rectangle
+  // is the caller's own literal, attempt 2 passes the identical one, and a retry is a browser
+  // launch and a login for no chance of a different verdict. Refused before any driver runs, so
+  // there is no half-done capture to resume.
+  X_SCRAPE_CAPTURE_INVALID: 'terminal',
   // TERMINAL where its sibling `X_SCRAPE_WEDGED` is retryable, and the difference IS the reason
   // the two codes are separate. A wedge is the site or the browser being slow — the definition of
   // "run it again and it may go differently". This is the guard's own loop dying on code the

package/src/html-target.ts

@@ -5,6 +5,7 @@
 // through to the network would make a green offline suite that is secretly hitting production —
 // the exact failure an offline driver exists to prevent.
 
+import type { CaptureClip } from './capture-clip';
 import type { ScrapeClock } from './clock';
 import { browserUnreachable, downloadTimeout, fixtureMissing, fixtureStale } from './error-throws';
 import { queryHtml } from './html-query';
@@ -29,6 +30,22 @@ import type {
 
 /** Deterministic bytes, so an artifact test asserts on a stable digest. Not a real PNG render. */
 const FAKE_PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+/**
+ * A clipped capture answers DIFFERENT deterministic bytes, one picture per rectangle. CI has no
+ * Chrome, so the offline drivers are the only place a crop can be proved at all: a fake that
+ * answered the same eight bytes whatever the rectangle would let a driver that silently drops the
+ * clip pass every test there is. Unclipped bytes are unchanged, byte for byte.
+ */
+const clippedPng = (clip: CaptureClip): Uint8Array => {
+  const suffix = new TextEncoder().encode(
+    ` clip ${String(clip.x)},${String(clip.y)},${String(clip.width)},${String(clip.height)}`,
+  );
+  const out = new Uint8Array(FAKE_PNG.length + suffix.length);
+  out.set(FAKE_PNG);
+  out.set(suffix, FAKE_PNG.length);
+  return out;
+};
 const FAKE_PDF = new TextEncoder().encode('%PDF-1.4 offline');
 
 export type RecordingLookup = (url: string) => Promise<PageRecording | undefined>;
@@ -208,7 +225,8 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
         throw fixtureMissing(`${page.url} evaluate(${expression})`, init.source);
       return Promise.resolve(JSON.parse(answer) as unknown);
     },
-    screenshot: (_options: CaptureOptions): Promise<Uint8Array> => Promise.resolve(FAKE_PNG),
+    screenshot: (options: CaptureOptions): Promise<Uint8Array> =>
+      Promise.resolve(options.clip === undefined ? FAKE_PNG : clippedPng(options.clip)),
     pdf: (_options: CaptureOptions): Promise<Uint8Array> => Promise.resolve(FAKE_PDF),
     cookies: (): Promise<readonly ScrapeCookie[]> => Promise.resolve(session.cookies),
     session: (): Promise<SessionSnapshot> => Promise.resolve(session),

package/src/index.ts

@@ -18,12 +18,15 @@ export type {
   ScrapeAuth,
 } from './auth';
 export { burnSession, createPrompt, ensureAuthenticated, restorableSession } from './auth';
+export type { CaptureClip, CaptureFraming } from './capture-clip';
+export { assertCaptureFraming } from './capture-clip';
 export type {
   CdpBrowserLike,
   CdpFrameLike,
   CdpLauncherLike,
   CdpPageLike,
   CdpRequestLike,
+  CdpScreenshotOptions,
 } from './cdp-port';
 export { parseSnapshots, snapshotExpression } from './cdp-snapshot';
 export type { CdpTargetInit } from './cdp-target';

package/src/page-over-target.ts

@@ -7,6 +7,7 @@ import type { Secret } from '@ultimat3/core';
 import { isSecret, revealSecret } from '@ultimat3/core';
 import type { ActionabilityState } from './actionability';
 import { awaitActionable } from './actionability';
+import { assertCaptureFraming } from './capture-clip';
 import type { ScrapeClock } from './clock';
 import { deadline } from './clock';
 import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
@@ -22,7 +23,13 @@ import type {
 import type { RobotsGate } from './robots';
 import type { ScrapeSecrets } from './secrets';
 import { safeHtml } from './secrets';
-import type { ElementSnapshot, ScrapeCookie, ScrapeDownloadFile, ScrapeTarget } from './target';
+import type {
+  CaptureOptions,
+  ElementSnapshot,
+  ScrapeCookie,
+  ScrapeDownloadFile,
+  ScrapeTarget,
+} from './target';
 import { ROOT_SELECTOR } from './target';
 
 export interface PageContext {
@@ -187,7 +194,14 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
     // in object storage, forever. Refused rather than masked — a mask over pixels is a guess
     // about layout, and `page.html()` already gives a redacted artifact that is exact.
     if (state.tainted) throw secretExposed(kind, target.url());
-    const request = { fullPage: options?.fullPage };
+    // The ONE place a capture request is built, so the framing rule is checked once for every
+    // driver rather than three times with three chances to disagree. After the secret guard, which
+    // is the stronger refusal: a tainted page must not be told its rectangle is fine.
+    assertCaptureFraming(kind, options ?? {});
+    const request: CaptureOptions = {
+      fullPage: options?.fullPage,
+      ...(options?.clip === undefined ? {} : { clip: options.clip }),
+    };
     return kind === 'screenshot' ? target.screenshot(request) : target.pdf(request);
   };
   return {

package/src/page.ts

@@ -8,6 +8,7 @@
 
 import type { Secret } from '@ultimat3/core';
 import type { ActionabilityState } from './actionability';
+import type { CaptureFraming } from './capture-clip';
 import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 import type { SessionSnapshot } from './session-state';
 import type { ElementSnapshot, ScrapeCookie, ScrapeDownloadFile } from './target';
@@ -64,10 +65,13 @@ export interface ScrapeFrame {
   frame(nameOrSelector: string): ScrapeFrame;
 }
 
-/** `fullPage` only. `timeout` is gone with the port's — see `CaptureOptions` for why. */
-export interface CaptureRequest {
-  readonly fullPage?: boolean | undefined;
-}
+/**
+ * `fullPage`, or a `clip` — never both. `timeout` is gone with the port's; see `CaptureOptions`.
+ *
+ * A clip is what makes a capture reviewable by a vision model: the whole viewport spends the
+ * reader's scarce pixels on everything that is not the component under review.
+ */
+export type CaptureRequest = CaptureFraming;
 
 export interface DownloadRequest {
   readonly timeout?: number | undefined;

package/src/target.ts

@@ -3,6 +3,7 @@
 // lets `page-over-target.ts` be the ONE implementation of `ScrapePage` for the real browser, the
 // fixture replayer and the fake alike.
 
+import type { CaptureFraming } from './capture-clip';
 import type { ConsoleRing, NetworkRing, PageErrorRing } from './rings';
 import type { SessionSnapshot } from './session-state';
 
@@ -75,7 +76,12 @@ export interface GotoOptions {
 }
 
 /**
- * `fullPage` and nothing else. It carried a required `timeoutMs` until 2026-08 that NO driver
+ * `CaptureFraming`, under the port's own name: `fullPage` for the whole document, or a `clip` to
+ * crop to one rectangle, never both. ONE declaration for the port and the vocabulary — the two
+ * were separate copies of `{ fullPage }` and a field added to one would have reached neither the
+ * drivers nor the callers.
+ *
+ * It carried a required `timeoutMs` until 2026-08 that NO driver
  * honoured — `cdp-target.ts` read only `fullPage`, `html-target.ts` ignored the whole object —
  * so `page.screenshot({ timeout })` was a documented deadline that bounded nothing. Deleted
  * rather than implemented: the CDP port's own `screenshot({ fullPage })` has no timeout slot to
@@ -83,9 +89,7 @@ export interface GotoOptions {
  * `ScrapeClock.sleep`, which under `testClock` resolves on the first microtask and would time
  * out every capture in every test. A driver's own default is the honest bound.
  */
-export interface CaptureOptions {
-  readonly fullPage?: boolean | undefined;
-}
+export type CaptureOptions = CaptureFraming;
 
 /**
  * The port. Twelve methods, every one of them something a browser genuinely does and nothing a