@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/verify-migration/check-resource-items-count.js

@@ -0,0 +1,33 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #6: every legacy `resources` row is mirrored
+ * by one row in `resource_items`.
+ *
+ * `populateResourceItems` fills `resource_items` with the same PK as
+ * `resources.id`
+ * (see `populate-resource-items.ts`); the invariant is broken only if the
+ * populate loop skipped rows during URL / header-set / content-type
+ * resolution. The paired invariant `resource_ref_edges = "resources-referrers"`
+ * is left unchecked here because issue #194's spec list stops at the
+ * six enumerated checks, and `resource_ref_edges` populate is a straight
+ * `INSERT … SELECT` (no dedup, no resolution) with negligible failure surface.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export async function checkResourceItemsCount(trx) {
+    const resourceItemsRows = await trx('resource_items').count({
+        n: '*',
+    });
+    const resourcesRows = await trx('resources').count({ n: '*' });
+    const resourceItemsCount = Number(resourceItemsRows[0].n);
+    const resourcesCount = Number(resourcesRows[0].n);
+    if (resourceItemsCount !== resourcesCount) {
+        throw new MigrationVerificationError({
+            check: '#6 resource_items row count',
+            context: {
+                resource_items: resourceItemsCount,
+                resources: resourcesCount,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-url-round-trip.d.ts

@@ -0,0 +1,43 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #8: `content_items.url_id → url_refs.url`
+ * round-trips back to the original `pages.url` on a deterministic
+ * ≤ {@link SAMPLE_SIZE} sample.
+ *
+ * A mismatch means one of two things:
+ *
+ * - `populate-url-refs.ts` collapsed two distinct URLs into the same
+ *   `url_refs.id` (BINARY collation bug, whitespace normalisation, …).
+ * - `populate-content-items.ts` associated a page id with the wrong
+ *   `url_ref.id` (row-order dependency in the batch resolver).
+ *
+ * Additionally an FK gap (`content_items.url_id` pointing at a
+ * non-existent `url_refs.id`) or a missing `pages` row surfaces as a
+ * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a
+ * round-trip failure. LEFT JOINs are used deliberately so orphan rows
+ * are observable — an INNER JOIN would silently drop them and let the
+ * bug the invariant is meant to catch slip through.
+ *
+ * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉
+ * gives approximately {@link SAMPLE_SIZE} rows spread across the id
+ * range for large archives, and the full table for archives with
+ * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same
+ * archive always produces the same verdict; a random sample would let a
+ * failing archive pass on retry if the offending rows happen to fall
+ * outside the second draw.
+ *
+ * The check throws when the sampled row count is smaller than expected
+ * (LEFT JOINs may return `null` sides but never fewer rows than the
+ * driving `content_items` set — a shortfall means SQLite silently
+ * dropped rows we intended to inspect). "Expected" is a separate
+ * `COUNT(*)` against the same `(id % stride) = 0` predicate, not a
+ * `floor(total / stride)` estimate: `content_items.id` is copied
+ * verbatim from the legacy `pages.id`, which accumulates gaps over an
+ * archive's crawl history (redirect consolidation, deleted pages, …).
+ * A density estimate over-counts whenever the id range is sparser than
+ * `[1, total]`, which produced false-positive failures on real
+ * multi-thousand-page archives.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when at least one sampled URL does not round-trip.
+ */
+export declare function checkUrlRoundTrip(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-url-round-trip.js

@@ -0,0 +1,112 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Sample size for the URL round-trip smoke test — a 1,000-row sample per
+ * issue #194.
+ * Not statistical coverage — URL normalisation bugs tend to be systematic
+ * (all rows with a particular scheme or host), so a modest evenly-spread
+ * sample catches them reliably.
+ */
+const SAMPLE_SIZE = 1000;
+/**
+ * Verifies 0.13 invariant #8: `content_items.url_id → url_refs.url`
+ * round-trips back to the original `pages.url` on a deterministic
+ * ≤ {@link SAMPLE_SIZE} sample.
+ *
+ * A mismatch means one of two things:
+ *
+ * - `populate-url-refs.ts` collapsed two distinct URLs into the same
+ *   `url_refs.id` (BINARY collation bug, whitespace normalisation, …).
+ * - `populate-content-items.ts` associated a page id with the wrong
+ *   `url_ref.id` (row-order dependency in the batch resolver).
+ *
+ * Additionally an FK gap (`content_items.url_id` pointing at a
+ * non-existent `url_refs.id`) or a missing `pages` row surfaces as a
+ * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a
+ * round-trip failure. LEFT JOINs are used deliberately so orphan rows
+ * are observable — an INNER JOIN would silently drop them and let the
+ * bug the invariant is meant to catch slip through.
+ *
+ * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉
+ * gives approximately {@link SAMPLE_SIZE} rows spread across the id
+ * range for large archives, and the full table for archives with
+ * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same
+ * archive always produces the same verdict; a random sample would let a
+ * failing archive pass on retry if the offending rows happen to fall
+ * outside the second draw.
+ *
+ * The check throws when the sampled row count is smaller than expected
+ * (LEFT JOINs may return `null` sides but never fewer rows than the
+ * driving `content_items` set — a shortfall means SQLite silently
+ * dropped rows we intended to inspect). "Expected" is a separate
+ * `COUNT(*)` against the same `(id % stride) = 0` predicate, not a
+ * `floor(total / stride)` estimate: `content_items.id` is copied
+ * verbatim from the legacy `pages.id`, which accumulates gaps over an
+ * archive's crawl history (redirect consolidation, deleted pages, …).
+ * A density estimate over-counts whenever the id range is sparser than
+ * `[1, total]`, which produced false-positive failures on real
+ * multi-thousand-page archives.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when at least one sampled URL does not round-trip.
+ */
+export async function checkUrlRoundTrip(trx) {
+    const totalRows = await trx('content_items').count({ n: '*' });
+    const total = Number(totalRows[0].n);
+    if (total === 0) {
+        return;
+    }
+    // stride = ⌈total / SAMPLE_SIZE⌉ so `id % stride = 0` picks approximately
+    // SAMPLE_SIZE rows evenly across the id range for large archives, and the
+    // full table for archives ≤ SAMPLE_SIZE.
+    const stride = Math.max(1, Math.ceil(total / SAMPLE_SIZE));
+    const matchingRows = await trx('content_items')
+        .whereRaw('(id % ?) = 0', [stride])
+        .count({ n: '*' });
+    const expectedSize = Math.min(SAMPLE_SIZE, Number(matchingRows[0].n));
+    const sample = await trx('content_items as ci')
+        .leftJoin('pages as p', 'p.id', 'ci.id')
+        .leftJoin('url_refs as ur', 'ur.id', 'ci.url_id')
+        .select('ci.id as id', 'p.url as sourceUrl', 'ur.url as roundTripUrl')
+        .whereRaw('(ci.id % ?) = 0', [stride])
+        .orderBy('ci.id')
+        .limit(SAMPLE_SIZE);
+    if (sample.length < expectedSize) {
+        // LEFT JOINs preserve driving-table row counts (`p.id` / `url_refs.id`
+        // are PKs → 1:1 or 0:1 match). A shortfall means SQLite silently
+        // dropped rows the WHERE clause matched — a real integrity signal.
+        throw new MigrationVerificationError({
+            check: '#8 URL round-trip',
+            context: {
+                content_items_total: total,
+                stride,
+                expected_sample_size: expectedSize,
+                actual_sample_size: sample.length,
+                reason: 'sample query returned fewer rows than expected',
+            },
+        });
+    }
+    for (const row of sample) {
+        if (row.sourceUrl === null || row.roundTripUrl === null) {
+            throw new MigrationVerificationError({
+                check: '#8 URL round-trip',
+                context: {
+                    page_id: row.id,
+                    source_url: row.sourceUrl,
+                    round_trip_url: row.roundTripUrl,
+                    sample_size: sample.length,
+                    reason: 'orphan row — pages or url_refs join returned null (FK gap)',
+                },
+            });
+        }
+        if (row.sourceUrl !== row.roundTripUrl) {
+            throw new MigrationVerificationError({
+                check: '#8 URL round-trip',
+                context: {
+                    page_id: row.id,
+                    source_url: row.sourceUrl,
+                    round_trip_url: row.roundTripUrl,
+                    sample_size: sample.length,
+                },
+            });
+        }
+    }
+}

package/lib/archive/verify-migration/types.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Shared type definitions for 0.13 migration verification.
+ *
+ * These are the acceptance-invariant checks that `scripts/migrate-to-0.13.mjs`
+ * runs at the end of the migration to confirm that the 0.13 entity /
+ * edge populates produced a consistent archive. Each check is one file
+ * under `verify-migration/` and throws {@link MigrationVerificationError} on mismatch;
+ * the orchestrator {@link ./verify-migration.ts} chains them in the
+ * order defined by issue #194.
+ * @module
+ */
+/**
+ * Row-count snapshot collected by every successful verify run and returned
+ * by {@link import('./verify-migration.js').verifyMigration}.
+ * The migration script logs these on the successful path so operators can
+ * eyeball migration outcomes from stdout without re-opening the archive.
+ */
+export interface MigrationVerificationSummary {
+    /** `count(*) FROM content_items` — matches `count(pages)`. */
+    readonly contentItems: number;
+    /** `count(*) FROM page_meta` — matches `count(pages WHERE scraped=1)`. */
+    readonly pageMeta: number;
+    /** `count(*) FROM anchor_edges` — always ≤ `count(anchors)`. */
+    readonly anchorEdges: number;
+    /** `SUM(count) FROM anchor_edges` — matches `count(anchors)`. */
+    readonly anchorEdgesSum: number;
+    /** `count(*) FROM image_items` — matches `count(images)`. */
+    readonly imageItems: number;
+    /** `count(*) FROM resource_items` — matches `count(resources)`. */
+    readonly resourceItems: number;
+}
+/**
+ * Structured details of a single migration invariant that failed. Attached to
+ * {@link MigrationVerificationError} so operators can see which check tripped
+ * without parsing the error message.
+ */
+export interface MigrationVerificationErrorDetails {
+    /**
+     * Human-readable check identifier matching issue #194's numbered list
+     * (e.g. `'#3'` for the anchor-edge dedup range check). Included in the
+     * error message so migration logs point at the exact clause.
+     */
+    readonly check: string;
+    /**
+     * Optional structured payload — typically `expected` / `actual` counts or
+     * the offending page id from the URL round-trip sample. Rendered into the
+     * error message so operators can diagnose without re-running the check.
+     */
+    readonly context?: Readonly<Record<string, string | number | null>>;
+}
+/**
+ * Error thrown by any 0.13 check when an invariant does not hold. The
+ * migration script catches this at the top level and aborts with a non-zero
+ * exit code; the enclosing transaction rolls back so the archive returns to
+ * its pre-populate state (ref tables from ref populate stay populated but are
+ * additive).
+ *
+ * Prefer a single error class with a structured {@link details} payload over
+ * one subclass per check: eight subclasses would create eight nearly
+ * identical constructors, and callers (script + integration test) only need
+ * to distinguish "verification failed" from other error kinds.
+ */
+export declare class MigrationVerificationError extends Error {
+    readonly details: MigrationVerificationErrorDetails;
+    /**
+     * @param details - Structured description of which check failed and why.
+     *   Rendered into the error message on construction.
+     */
+    constructor(details: MigrationVerificationErrorDetails);
+}

package/lib/archive/verify-migration/types.js

@@ -0,0 +1,63 @@
+/**
+ * Shared type definitions for 0.13 migration verification.
+ *
+ * These are the acceptance-invariant checks that `scripts/migrate-to-0.13.mjs`
+ * runs at the end of the migration to confirm that the 0.13 entity /
+ * edge populates produced a consistent archive. Each check is one file
+ * under `verify-migration/` and throws {@link MigrationVerificationError} on mismatch;
+ * the orchestrator {@link ./verify-migration.ts} chains them in the
+ * order defined by issue #194.
+ * @module
+ */
+/**
+ * Error thrown by any 0.13 check when an invariant does not hold. The
+ * migration script catches this at the top level and aborts with a non-zero
+ * exit code; the enclosing transaction rolls back so the archive returns to
+ * its pre-populate state (ref tables from ref populate stay populated but are
+ * additive).
+ *
+ * Prefer a single error class with a structured {@link details} payload over
+ * one subclass per check: eight subclasses would create eight nearly
+ * identical constructors, and callers (script + integration test) only need
+ * to distinguish "verification failed" from other error kinds.
+ */
+export class MigrationVerificationError extends Error {
+    details;
+    /**
+     * @param details - Structured description of which check failed and why.
+     *   Rendered into the error message on construction.
+     */
+    constructor(details) {
+        const contextText = details.context === undefined
+            ? ''
+            : ' — ' +
+                Object.entries(details.context)
+                    .map(([key, value]) => 
+                // Render null as the parenthesised literal `(null)` so
+                // operators reading the log can distinguish "the sample
+                // query returned no rows" from a caller that passed the
+                // four-character string `"null"`. Strings get quoted for
+                // the same reason.
+                `${key}=${formatContextValue(value)}`)
+                    .join(', ');
+        super(`migration verification failed ${details.check}${contextText}`);
+        this.details = details;
+        this.name = 'MigrationVerificationError';
+    }
+}
+/**
+ * Renders one context value for {@link MigrationVerificationError}'s message.
+ * `null` becomes the literal `(null)`; strings are wrapped in double quotes;
+ * numbers pass through as their decimal representation. The intent is to
+ * make missing-vs-present distinguishable in operator logs.
+ * @param value - Value from `MigrationVerificationErrorDetails.context`.
+ */
+function formatContextValue(value) {
+    if (value === null) {
+        return '(null)';
+    }
+    if (typeof value === 'string') {
+        return JSON.stringify(value);
+    }
+    return String(value);
+}

package/lib/archive/verify-migration/verify-migration.d.ts

@@ -0,0 +1,41 @@
+import type { MigrationVerificationSummary } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Runs every 0.13 migration acceptance invariant against a populated archive and
+ * returns the verified row-count summary on success. The migration script
+ * (`scripts/migrate-to-0.13.mjs`) calls this **inside** the same
+ * `knex.transaction()` block that ran `populateEntityTables` so a thrown
+ * {@link MigrationVerificationError} rolls back the entire entity populate — ref
+ * tables from ref populate stay committed because they live in a separate, additive
+ * transaction and never lose facts on re-run.
+ *
+ * The returned {@link MigrationVerificationSummary} is echoed to stdout by the
+ * migration script so operators can eyeball migration outcomes (`did we lose
+ * rows silently within the invariant window?`) from batch pipeline logs
+ * without re-opening the archive. Callers must run this inside a `db.transaction()`
+ * for the counts and the JOIN-based checks (#7 / #8) to see a consistent
+ * snapshot of the archive.
+ *
+ * Checks run sequentially in issue #194's numbered order so error messages
+ * pinpoint the earliest broken invariant. Sequential (rather than parallel)
+ * execution keeps the failure diagnostics focused: fanning eight COUNTs out
+ * at once would return the first-completed rejection rather than the
+ * lowest-numbered one, which surprises operators reading migration logs.
+ * The individual COUNT queries are fast enough at any real archive scale
+ * for sequential wall-clock cost to be irrelevant.
+ *
+ * Non-{@link MigrationVerificationError} exceptions from a check (e.g. a
+ * transient libsql error or a schema-drift SqliteError) are wrapped in a
+ * {@link MigrationVerificationError} so the migration script's `catch` sees a
+ * uniform failure surface — otherwise operators grepping stderr for
+ * `migration verification failed` would miss driver-side errors.
+ * @param trx - Knex instance or transaction connected to the archive
+ *   **after** every 0.13 migration populate has run. Must be a transaction
+ *   in production so ref counts, JOIN samples, and error rollback see the
+ *   same snapshot; non-transactional callers (unit tests) work but do not
+ *   get snapshot isolation.
+ * @returns Row-count summary of the verified archive.
+ * @throws {MigrationVerificationError} on the first invariant that does not
+ *   hold; subsequent checks are skipped.
+ */
+export declare function verifyMigration(trx: Knex): Promise<MigrationVerificationSummary>;

package/lib/archive/verify-migration/verify-migration.js

@@ -0,0 +1,120 @@
+import { checkAnchorEdgesCount } from './check-anchor-edges-count.js';
+import { checkAnchorEdgesSum } from './check-anchor-edges-sum.js';
+import { checkContentItemsCount } from './check-content-items-count.js';
+import { checkContentTypePreservation } from './check-content-type-preservation.js';
+import { checkImageItemsCount } from './check-image-items-count.js';
+import { checkPageMetaCount } from './check-page-meta-count.js';
+import { checkReaderParity } from './check-reader-parity.js';
+import { checkResourceItemsCount } from './check-resource-items-count.js';
+import { checkUrlRoundTrip } from './check-url-round-trip.js';
+import { MigrationVerificationError } from './types.js';
+/**
+ * Runs every 0.13 migration acceptance invariant against a populated archive and
+ * returns the verified row-count summary on success. The migration script
+ * (`scripts/migrate-to-0.13.mjs`) calls this **inside** the same
+ * `knex.transaction()` block that ran `populateEntityTables` so a thrown
+ * {@link MigrationVerificationError} rolls back the entire entity populate — ref
+ * tables from ref populate stay committed because they live in a separate, additive
+ * transaction and never lose facts on re-run.
+ *
+ * The returned {@link MigrationVerificationSummary} is echoed to stdout by the
+ * migration script so operators can eyeball migration outcomes (`did we lose
+ * rows silently within the invariant window?`) from batch pipeline logs
+ * without re-opening the archive. Callers must run this inside a `db.transaction()`
+ * for the counts and the JOIN-based checks (#7 / #8) to see a consistent
+ * snapshot of the archive.
+ *
+ * Checks run sequentially in issue #194's numbered order so error messages
+ * pinpoint the earliest broken invariant. Sequential (rather than parallel)
+ * execution keeps the failure diagnostics focused: fanning eight COUNTs out
+ * at once would return the first-completed rejection rather than the
+ * lowest-numbered one, which surprises operators reading migration logs.
+ * The individual COUNT queries are fast enough at any real archive scale
+ * for sequential wall-clock cost to be irrelevant.
+ *
+ * Non-{@link MigrationVerificationError} exceptions from a check (e.g. a
+ * transient libsql error or a schema-drift SqliteError) are wrapped in a
+ * {@link MigrationVerificationError} so the migration script's `catch` sees a
+ * uniform failure surface — otherwise operators grepping stderr for
+ * `migration verification failed` would miss driver-side errors.
+ * @param trx - Knex instance or transaction connected to the archive
+ *   **after** every 0.13 migration populate has run. Must be a transaction
+ *   in production so ref counts, JOIN samples, and error rollback see the
+ *   same snapshot; non-transactional callers (unit tests) work but do not
+ *   get snapshot isolation.
+ * @returns Row-count summary of the verified archive.
+ * @throws {MigrationVerificationError} on the first invariant that does not
+ *   hold; subsequent checks are skipped.
+ */
+export async function verifyMigration(trx) {
+    try {
+        await checkContentItemsCount(trx);
+        await checkPageMetaCount(trx);
+        await checkAnchorEdgesCount(trx);
+        await checkAnchorEdgesSum(trx);
+        await checkImageItemsCount(trx);
+        await checkResourceItemsCount(trx);
+        await checkContentTypePreservation(trx);
+        await checkUrlRoundTrip(trx);
+        // Reader-level parity between the legacy tables and the
+        // new entity tables (see `checkReaderParity` for the eight totals
+        // this catches that the row-count checks above miss).
+        await checkReaderParity(trx);
+    }
+    catch (error) {
+        if (error instanceof MigrationVerificationError) {
+            throw error;
+        }
+        throw new MigrationVerificationError({
+            check: 'runtime',
+            context: {
+                underlying_error: error instanceof Error ? error.message : String(error),
+            },
+        });
+    }
+    return collectSummary(trx);
+}
+/**
+ * Runs the six row counts that populate {@link MigrationVerificationSummary}.
+ * Separate from the invariant checks so those stay side-effect-free asserts,
+ * and this runs once at the end when all invariants have already held.
+ * @param trx - Same Knex handle passed to {@link verifyMigration}.
+ */
+async function collectSummary(trx) {
+    const [contentItems, pageMeta, anchorEdges, anchorEdgesSum, imageItems, resourceItems] = await Promise.all([
+        countOf(trx, 'content_items'),
+        countOf(trx, 'page_meta'),
+        countOf(trx, 'anchor_edges'),
+        sumOf(trx, 'anchor_edges', 'count'),
+        countOf(trx, 'image_items'),
+        countOf(trx, 'resource_items'),
+    ]);
+    return {
+        contentItems,
+        pageMeta,
+        anchorEdges,
+        anchorEdgesSum,
+        imageItems,
+        resourceItems,
+    };
+}
+/**
+ * `SELECT count(*) FROM <table>` returning a plain number.
+ * @param trx - Knex handle.
+ * @param table - Table to count.
+ */
+async function countOf(trx, table) {
+    const rows = await trx(table).count({ n: '*' });
+    return Number(rows[0].n);
+}
+/**
+ * `SELECT SUM(<column>) FROM <table>` returning a plain number (`null` sum
+ * on an empty table collapses to zero).
+ * @param trx - Knex handle.
+ * @param table - Table to sum over.
+ * @param column - Column to sum.
+ */
+async function sumOf(trx, table, column) {
+    const rows = await trx(table).sum({ n: column });
+    return Number(rows[0].n ?? 0);
+}

package/lib/crawler/build-redirect-event.d.ts

@@ -14,7 +14,7 @@ import type { PageData } from '../utils/types/types.js';
  *
  * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
  * URL the redirect chain STARTS from (the page being scraped), not the
- * destination — see {@link Database.recordRedirect} JSDoc for why
+ * destination — see `Database.recordRedirect` JSDoc for why
  * lineage propagates from origin, not from the destination.
  * @param pageData - HEAD-resolved page data carrying the redirect chain.
  * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.

package/lib/crawler/build-redirect-event.js

@@ -13,7 +13,7 @@ import { derivePageSource } from './derive-page-source.js';
  *
  * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
  * URL the redirect chain STARTS from (the page being scraped), not the
- * destination — see {@link Database.recordRedirect} JSDoc for why
+ * destination — see `Database.recordRedirect` JSDoc for why
  * lineage propagates from origin, not from the destination.
  * @param pageData - HEAD-resolved page data carrying the redirect chain.
  * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.

package/lib/crawler/capture-image-dom-paths.d.ts

@@ -0,0 +1,33 @@
+import type { DomPathCandidate } from '../archive/populate-entity-tables/types.js';
+import type { Page as PuppeteerPage } from 'puppeteer';
+/**
+ * Captures every `<img>` in the rendered document — its `outerHTML` and
+ * its `dom_path` string — in document order, for `image_items.dom_path_text_id`
+ * resolution at write time.
+ *
+ * `@d-zero/beholder`'s image metadata carries the element's `outerHTML`
+ * (`sourceCode`) but no positional information, so the crawler runs this
+ * one extra `page.evaluate` after `scrapeStart` returns, while the page
+ * is still alive. The in-browser walk is
+ * {@link ./collect-image-dom-paths.ts} — a self-contained function whose
+ * source puppeteer serialises into the page, and whose spec pins its
+ * output against the Node-side
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} so a
+ * live-crawled archive and one whose dom paths were reconstructed from
+ * HTML snapshots by the migration script produce identical strings for
+ * identical DOM shapes.
+ *
+ * Returns `undefined` on any evaluation failure (page context died,
+ * navigation raced the call) — dom paths are best-effort enrichment and
+ * a capture failure must not fail the scrape. Callers fall back to the
+ * `unknown/<id>` synthetic marker per image via
+ * {@link ../archive/populate-entity-tables/match-images-to-dom-paths.ts}.
+ * @param page - The live puppeteer page, after the scrape completed and
+ *   before the browser closes.
+ * @returns Candidates in document order, or `undefined` when the page
+ *   could not be evaluated.
+ * @example
+ * const result = await scraper.scrapeStart(page, url, options);
+ * const imageDomPaths = await captureImageDomPaths(page);
+ */
+export declare function captureImageDomPaths(page: PuppeteerPage): Promise<DomPathCandidate[] | undefined>;

package/lib/crawler/capture-image-dom-paths.js

@@ -0,0 +1,39 @@
+import { collectImageDomPaths } from './collect-image-dom-paths.js';
+/**
+ * Captures every `<img>` in the rendered document — its `outerHTML` and
+ * its `dom_path` string — in document order, for `image_items.dom_path_text_id`
+ * resolution at write time.
+ *
+ * `@d-zero/beholder`'s image metadata carries the element's `outerHTML`
+ * (`sourceCode`) but no positional information, so the crawler runs this
+ * one extra `page.evaluate` after `scrapeStart` returns, while the page
+ * is still alive. The in-browser walk is
+ * {@link ./collect-image-dom-paths.ts} — a self-contained function whose
+ * source puppeteer serialises into the page, and whose spec pins its
+ * output against the Node-side
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} so a
+ * live-crawled archive and one whose dom paths were reconstructed from
+ * HTML snapshots by the migration script produce identical strings for
+ * identical DOM shapes.
+ *
+ * Returns `undefined` on any evaluation failure (page context died,
+ * navigation raced the call) — dom paths are best-effort enrichment and
+ * a capture failure must not fail the scrape. Callers fall back to the
+ * `unknown/<id>` synthetic marker per image via
+ * {@link ../archive/populate-entity-tables/match-images-to-dom-paths.ts}.
+ * @param page - The live puppeteer page, after the scrape completed and
+ *   before the browser closes.
+ * @returns Candidates in document order, or `undefined` when the page
+ *   could not be evaluated.
+ * @example
+ * const result = await scraper.scrapeStart(page, url, options);
+ * const imageDomPaths = await captureImageDomPaths(page);
+ */
+export async function captureImageDomPaths(page) {
+    try {
+        return await page.evaluate(collectImageDomPaths);
+    }
+    catch {
+        return undefined;
+    }
+}

package/lib/crawler/clear-dns-burned-host-cache.d.ts

@@ -1,6 +1,6 @@
 /**
  * Clears the DNS-burned host cache and resets the short-circuit counter.
- * Called between crawl sessions, alongside {@link clearDestinationCache}, to
+ * Called between crawl sessions, alongside `clearDestinationCache`, to
  * avoid leaking state from a previous session into a fresh one.
  */
 export declare function clearDnsBurnedHostCache(): void;

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -2,7 +2,7 @@ import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
 import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
 /**
  * Clears the DNS-burned host cache and resets the short-circuit counter.
- * Called between crawl sessions, alongside {@link clearDestinationCache}, to
+ * Called between crawl sessions, alongside `clearDestinationCache`, to
  * avoid leaking state from a previous session into a fresh one.
  */
 export function clearDnsBurnedHostCache() {

package/lib/crawler/collect-image-dom-paths.d.ts

@@ -0,0 +1,23 @@
+import type { DomPathCandidate } from '../archive/populate-entity-tables/types.js';
+/**
+ * Collects every `<img>` in a document — its `outerHTML` and its
+ * `dom_path` string — in document order.
+ *
+ * **Self-contained by contract.** This function is passed verbatim to
+ * puppeteer's `page.evaluate`, which serialises the function source and
+ * executes it inside the browser: it must not reference imports,
+ * module-scope bindings, or any closure state. That constraint is also
+ * what makes single-sourcing possible — the exact same function body
+ * runs in the browser during a live crawl AND in Node (against a jsdom
+ * document) in its spec, so the dom-path derivation cannot drift between
+ * the two runtimes. The spec additionally pins its output against
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} (the
+ * Node-side derivation the migration script uses) element-for-element.
+ * @param doc - The document to walk. Defaults to the global `document`,
+ *   which is how the in-browser `page.evaluate(collectImageDomPaths)`
+ *   call resolves it; Node callers (specs) pass a jsdom document.
+ * @returns Candidates in document order.
+ * @example
+ * const candidates = await page.evaluate(collectImageDomPaths);
+ */
+export declare function collectImageDomPaths(doc?: Document): DomPathCandidate[];