@ultimat3/scraping 7.0.0 → 9.0.0

package/README.md

@@ -1,7 +1,8 @@
 # @ultimat3/scraping
 
-Browser automation as a **job**. `scrape()` returns a `JobHandle` — the rule's fourth instance
-after `llm()` (an action factory) and `backfill()` (a job factory). There is no ninth primitive.
+Browser automation as a **job**. `scrape()` returns a `JobHandle` — one row of
+`PRIMITIVE_FACTORIES` in `@ultimat3/core`, the derived list of every factory that ships. There is
+no ninth primitive, and no ordinal here to go stale when the next factory lands.
 
 ```ts
 import { t } from '@ultimat3/schema';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "7.0.0",
+  "version": "9.0.0",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -30,9 +30,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "7.0.0",
-    "@ultimat3/jobs": "7.0.0",
-    "@ultimat3/schema": "7.0.0",
-    "@ultimat3/storage": "7.0.0"
+    "@ultimat3/core": "9.0.0",
+    "@ultimat3/jobs": "9.0.0",
+    "@ultimat3/schema": "9.0.0",
+    "@ultimat3/storage": "9.0.0"
   }
 }

package/src/error-throws.ts

@@ -106,6 +106,28 @@ export const wedged = (what: string, idleMs: number): ScrapeError =>
     meta: { what, idleMs },
   });
 
+/**
+ * The watchdog's SECOND way of ending a run, and its OWN code rather than `X_SCRAPE_WEDGED`.
+ *
+ * An exact `cause:` cannot rescue a wrong title. A reader who hits this runs
+ * `x errors explain X_SCRAPE_WEDGED`, reads "the browser stopped answering and was killed", and
+ * goes to investigate a page that is fine — the browser never stopped answering, the guard's own
+ * loop died on code the DEFINITION supplied. Two events with two different subsystems and two
+ * opposite fixes are two codes, and the classifications differ with them: a wedge is retryable,
+ * this is terminal (`errors.ts`).
+ *
+ * Reachable through the `ScrapeClock` seam and the driver's `kill()`, both of which a third party
+ * writes. Leaving the loop dead and quiet is the alternative, and that is incident #1 with no
+ * guard armed at all.
+ */
+export const watchdogStopped = (what: string, thrown: unknown): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_WATCHDOG_STOPPED',
+    cause: `the wedge watchdog for ${what} stopped measuring browser activity: ${renderThrowable(thrown)}`,
+    fix: 'drop the custom clock: from the scrape() definition — systemScrapeClock is the only ScrapeClock whose sleep() cannot reject — then re-run',
+    meta: { what },
+  });
+
 export const pageCrashed = (url: string): ScrapeError =>
   new ScrapeError({
     code: 'X_SCRAPE_PAGE_CRASHED',

package/src/errors.ts

@@ -22,6 +22,7 @@ export const SCRAPE_OWNED_ERROR_CODES = [
   'X_SCRAPE_NOT_ACTIONABLE',
   'X_SCRAPE_TIMEOUT',
   'X_SCRAPE_WEDGED',
+  'X_SCRAPE_WATCHDOG_STOPPED',
   'X_SCRAPE_PAGE_CRASHED',
   'X_SCRAPE_OUTPUT_INVALID',
   'X_SCRAPE_YIELD_COLLAPSED',
@@ -68,6 +69,7 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
   X_SCRAPE_NOT_ACTIONABLE: 'the element is present and cannot be acted on',
   X_SCRAPE_TIMEOUT: 'the step exceeded its wall-clock budget',
   X_SCRAPE_WEDGED: 'the browser stopped answering and was killed',
+  X_SCRAPE_WATCHDOG_STOPPED: 'the wedge watchdog stopped measuring and the run was ended',
   X_SCRAPE_PAGE_CRASHED: 'the renderer process died',
   X_SCRAPE_OUTPUT_INVALID: 'the extracted rows do not match the extract schema',
   X_SCRAPE_YIELD_COLLAPSED: 'the run succeeded and returned far too little',
@@ -138,6 +140,14 @@ export const SCRAPE_ERROR_RETRY = {
   // A declaration error, raised by `scrape()` before any attempt exists — there is no run to
   // retry, and the same definition would refuse identically forever.
   X_SCRAPE_YIELD_HISTORY_MISSING: 'terminal',
+  // TERMINAL where its sibling `X_SCRAPE_WEDGED` is retryable, and the difference IS the reason
+  // the two codes are separate. A wedge is the site or the browser being slow — the definition of
+  // "run it again and it may go differently". This is the guard's own loop dying on code the
+  // DEFINITION supplied: a `ScrapeClock` an app wrote, reached identically on attempt 2. Retrying
+  // launches a real browser five times to die at the first poll, and on an authenticated target
+  // that is five arrivals at a login for no chance of a different answer — the rule this whole
+  // table is written to, stated at the top of it. The fix is an edit, so a human decides.
+  X_SCRAPE_WATCHDOG_STOPPED: 'terminal',
   X_SCRAPE_ROBOTS_DISALLOWED: 'terminal',
   X_SCRAPE_FIXTURE_MISSING: 'terminal',
   X_SCRAPE_FIXTURE_STALE: 'terminal',

package/src/page-over-target.ts

@@ -192,6 +192,12 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
   };
   return {
     ...frame,
+    // The DOCUMENT's URL is asked of the target, never of the frame's cached `lastUrl`. `ScrapeFrame`
+    // resolves its target asynchronously and `url()` is synchronous, so a child frame can only ever
+    // answer from a cache refreshed on its last wait — but the page HOLDS its target, so it has no
+    // such excuse. Spreading `...frame` without this override made `page.url()` answer the seed
+    // (`about:blank`) after every `goto`, which is what `x shot` reported as `finalUrl`.
+    url: () => target.url(),
     async goto(url, options): Promise<void> {
       await guardNavigation(url, ctx);
       await ctx.pace?.(ctx.signal);

package/src/scrape.ts

@@ -1,5 +1,6 @@
-// `scrape()` — a browser run, declared as a `job` and NOT as a ninth primitive. The rule's fourth
-// instance after `llm()` (an action factory) and `backfill()` (a job factory).
+// `scrape()` — a browser run, declared as a `job` and NOT as a ninth primitive. One row of
+// `PRIMITIVE_FACTORIES` in `@ultimat3/core`, which is the derived list of every factory that
+// ships; an ordinal written here would be wrong the moment the next one lands, and was.
 //
 // It is a job by every field of the definition, not by analogy: a scrape has an input schema, a
 // tenant, a retry policy, a timeout, a concurrency cap, a queue, and — decisively — a REQUIRED

package/src/watchdog.ts

@@ -11,7 +11,7 @@
 // graceful-quit CEILING on shutdown, past which the same kill runs.
 
 import type { ScrapeClock } from './clock';
-import { wedged } from './error-throws';
+import { watchdogStopped, wedged } from './error-throws';
 
 export const DEFAULT_IDLE_MS = 120_000;
 export const DEFAULT_GRACE_MS = 5_000;
@@ -41,7 +41,10 @@ export interface WedgeGuard {
   touch(): void;
   /** Graceful quit under the ceiling, then kill. Idempotent, and never throws. */
   shutdown(): Promise<void>;
-  /** True once the watchdog fired — a run that ends after this ended because of it. */
+  /**
+   * True once the watchdog ended the run — the idle budget passed, or the guard's own loop died
+   * and stopped measuring. Either way a run that ends after this ended because of the watchdog.
+   */
   readonly fired: boolean;
 }
 
@@ -50,7 +53,13 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
   const graceMs = init.graceMs ?? DEFAULT_GRACE_MS;
   const controller = new AbortController();
   let lastTouch = init.clock.monotonic();
+  // TWO latches, not one. `stopped` ends the watch loop; `shuttingDown` guards the teardown. They
+  // were the same flag, so a fire — which sets `stopped` — made `shutdown()` return on its first
+  // line and NEVER call `quit()`. `localBrowser()` hid it, because the fire's `kill()` still
+  // reaches a pid; `remoteBrowser()` is `driver-cdp.ts`'s primary path, `browser.process()` is
+  // `null` there, and `browser.close()` is then the only thing that ends the paid remote session.
   let stopped = false;
+  let shuttingDown = false;
   let fired = false;
 
   const watch = async (): Promise<void> => {
@@ -67,7 +76,17 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
       controller.abort(wedged(init.what, idleMs));
     }
   };
-  void watch();
+  // Never floating. `ScrapeClock` is a seam an app implements and `init.kill()` comes from a
+  // driver, so this loop can reject on somebody else's code — and a bare `void` turned that into
+  // an unhandled rejection, a process-level event belonging to no run, with the guard silently
+  // dead behind it. A guard that stopped measuring is incident #1 with nothing armed, so the run
+  // is ended with an instruction instead of left unwatched.
+  void watch().catch((thrown: unknown) => {
+    if (stopped) return;
+    stopped = true;
+    fired = true;
+    controller.abort(watchdogStopped(init.what, thrown));
+  });
 
   return {
     signal: controller.signal,
@@ -78,7 +97,8 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
       lastTouch = init.clock.monotonic();
     },
     async shutdown(): Promise<void> {
-      if (stopped) return;
+      if (shuttingDown) return;
+      shuttingDown = true;
       stopped = true;
       let quit = false;
       // A ceiling, not a hope: whichever finishes first wins, and if it is the clock the process
@@ -92,7 +112,11 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
             quit = false;
           },
         ),
-        init.clock.sleep(graceMs),
+        // The ceiling's own sleep is caught for the same reason the quit is: this runs in
+        // `runScrape`'s `finally`, `close()` is documented never to throw, and a clock that
+        // rejected here would replace the run's real failure with a teardown one. A ceiling that
+        // cannot be timed has already expired, which lands on `kill()` — the safe direction.
+        init.clock.sleep(graceMs).catch(() => undefined),
       ]);
       if (!quit) init.kill();
     },