@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/crawler/collect-image-dom-paths.js

@@ -0,0 +1,64 @@
+/**
+ * Collects every `<img>` in a document — its `outerHTML` and its
+ * `dom_path` string — in document order.
+ *
+ * **Self-contained by contract.** This function is passed verbatim to
+ * puppeteer's `page.evaluate`, which serialises the function source and
+ * executes it inside the browser: it must not reference imports,
+ * module-scope bindings, or any closure state. That constraint is also
+ * what makes single-sourcing possible — the exact same function body
+ * runs in the browser during a live crawl AND in Node (against a jsdom
+ * document) in its spec, so the dom-path derivation cannot drift between
+ * the two runtimes. The spec additionally pins its output against
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} (the
+ * Node-side derivation the migration script uses) element-for-element.
+ * @param doc - The document to walk. Defaults to the global `document`,
+ *   which is how the in-browser `page.evaluate(collectImageDomPaths)`
+ *   call resolves it; Node callers (specs) pass a jsdom document.
+ * @returns Candidates in document order.
+ * @example
+ * const candidates = await page.evaluate(collectImageDomPaths);
+ */
+export function collectImageDomPaths(doc) {
+    const target = doc ?? document;
+    /**
+     * Counts earlier same-tag siblings, returning the 1-based ordinal.
+     * @param element - The element whose ordinal is being computed.
+     * @param tag - The element's lower-cased tag name.
+     * @returns 1-based ordinal among same-tag siblings.
+     */
+    function computeSiblingOrdinal(element, tag) {
+        let count = 1;
+        let sibling = element.previousElementSibling;
+        while (sibling !== null) {
+            if (sibling.tagName.toLowerCase() === tag) {
+                count += 1;
+            }
+            sibling = sibling.previousElementSibling;
+        }
+        return count;
+    }
+    /**
+     * Derives the slash-joined dom_path for one element.
+     * @param element - The element to derive the path for.
+     * @returns The dom_path string.
+     */
+    function deriveDomPath(element) {
+        const segments = [];
+        let current = element;
+        while (current !== null) {
+            const tag = current.tagName.toLowerCase();
+            if (tag === 'html') {
+                segments.unshift('html');
+                break;
+            }
+            segments.unshift(`${tag}[${computeSiblingOrdinal(current, tag)}]`);
+            current = current.parentElement;
+        }
+        return segments.join('/');
+    }
+    return Array.from(target.querySelectorAll('img'), (img) => ({
+        outerHTML: img.outerHTML,
+        path: deriveDomPath(img),
+    }));
+}

package/lib/crawler/crawler.d.ts

@@ -81,6 +81,25 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
     start(urls: ExURL[], opts?: {
         recursive?: boolean;
     }): void;
+    /**
+     * Launches a fresh Puppeteer browser, runs the beholder scraper, and cleans up.
+     *
+     * WHY per-URL browser: Each URL gets its own browser instance to ensure
+     * complete isolation (cookies, cache, service workers). The browser is always
+     * closed in the `finally` block, even on error.
+     *
+     * The cascade-guard contract for the puppeteer-fallback success / skipped
+     * branches can be exercised via `vi.spyOn(Crawler.prototype,
+     * '_launchBrowserAndScrape')` in unit tests. There is no production
+     * consumer outside this class.
+     * @internal
+     * @param url - Target URL to scrape
+     * @param update - Callback for progress messages
+     * @param isExternal - Whether the URL is external to the crawl scope
+     * @param metadataOnly - When true, only extract title metadata
+     * @param headCheckResult - Optional HEAD result to pass to the scraper, avoiding a redundant request
+     * @returns The scrape result from beholder
+     */
     private _launchBrowserAndScrape;
     /**
      * The default maximum number of concurrent scraping processes.

package/lib/crawler/crawler.js

@@ -13,6 +13,7 @@ import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
 import { buildRedirectEvent } from './build-redirect-event.js';
+import { captureImageDomPaths } from './capture-image-dom-paths.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
 import { derivePageSource } from './derive-page-source.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
@@ -469,31 +470,6 @@ class Crawler extends EventEmitter {
             }
         }
     }
-    /**
-     * Launches a fresh Puppeteer browser, runs the beholder scraper, and cleans up.
-     *
-     * WHY per-URL browser: Each URL gets its own browser instance to ensure
-     * complete isolation (cookies, cache, service workers). The browser is always
-     * closed in the `finally` block, even on error.
-     * @param url - Target URL to scrape
-     * @param update - Callback for progress messages
-     * @param isExternal - Whether the URL is external to the crawl scope
-     * @param metadataOnly - When true, only extract title metadata
-     * @param headCheckResult - Optional HEAD result to pass to the scraper, avoiding a redundant request
-     * @returns The scrape result from beholder
-     */
-    /**
-     * @param url
-     * @param update
-     * @param isExternal
-     * @param metadataOnly
-     * @param headCheckResult
-     * @internal
-     * cascade-guard contract for the puppeteer-fallback success / skipped
-     * branches can be exercised via `vi.spyOn(Crawler.prototype,
-     * '_launchBrowserAndScrape')` in unit tests. There is no production
-     * consumer outside this class.
-     */
     /**
      * Resolve the source label of the page being scraped so sub-resources
      * captured during its render can inherit the correct lineage label
@@ -1185,7 +1161,7 @@ class Crawler extends EventEmitter {
         // - the source is not committed via `setPage`/`updatePage` on this
         //   path (the redirect-edge handler in `#runDeal` only calls
         //   `linkList.done` + `emit('redirect', ...)` → `Archive.setRedirect`),
-        //   so `recordRedirect` → `#getIdByUrl` creates a NULL-status
+        //   so `recordRedirect` → `resolveContentItemId` creates a NULL-status
         //   placeholder row for the source if it did not already exist;
         // - `#linkRedirectSources` then stamps `status = 301
         //   statusText='Moved Permanently'` because NULL satisfies its
@@ -1312,6 +1288,25 @@ class Crawler extends EventEmitter {
             },
         });
     }
+    /**
+     * Launches a fresh Puppeteer browser, runs the beholder scraper, and cleans up.
+     *
+     * WHY per-URL browser: Each URL gets its own browser instance to ensure
+     * complete isolation (cookies, cache, service workers). The browser is always
+     * closed in the `finally` block, even on error.
+     *
+     * The cascade-guard contract for the puppeteer-fallback success / skipped
+     * branches can be exercised via `vi.spyOn(Crawler.prototype,
+     * '_launchBrowserAndScrape')` in unit tests. There is no production
+     * consumer outside this class.
+     * @internal
+     * @param url - Target URL to scrape
+     * @param update - Callback for progress messages
+     * @param isExternal - Whether the URL is external to the crawl scope
+     * @param metadataOnly - When true, only extract title metadata
+     * @param headCheckResult - Optional HEAD result to pass to the scraper, avoiding a redundant request
+     * @returns The scrape result from beholder
+     */
     // eslint-disable-next-line no-restricted-syntax -- intentional `private` (vs `#`) so tests can spyOn the prototype to drive the puppeteer-fallback cascade-guard branches without a full browser mock; see JSDoc above.
     async _launchBrowserAndScrape(url, update, isExternal, metadataOnly, headCheckResult) {
         update('Launching browser%dots%');
@@ -1405,6 +1400,25 @@ class Crawler extends EventEmitter {
                 retries: this.#options.retry,
                 headCheckResult,
             });
+            // Image dom-path capture runs here — after the scrape completed but
+            // while `page` is still alive — because beholder's image metadata
+            // carries each element's `outerHTML` with no positional
+            // information. The captured candidates ride on the page data into
+            // `image_items.dom_path_text_id` resolution at write time; a
+            // capture failure (or a page with no images) falls back to the
+            // synthetic `unknown/<n>` markers, so this stays best-effort.
+            if (result.type === 'success' &&
+                result.pageData &&
+                result.pageData.imageList.length > 0) {
+                const imageDomPaths = await captureImageDomPaths(page);
+                if (imageDomPaths !== undefined) {
+                    const withDomPaths = {
+                        ...result.pageData,
+                        imageDomPaths,
+                    };
+                    result.pageData = withDomPaths;
+                }
+            }
             update('Closing browser%dots%');
             // JS-redirect rescue capture: when `scrapeStart` catches a
             // `#fetchData` throw internally (e.g. `Page.goto returned null`

package/lib/crawler/dns-burned-host-cache.d.ts

@@ -3,15 +3,15 @@ import type { ErrorKind } from '../types.js';
  * In-memory set of hostnames known to be unreachable due to DNS errors
  * (e.g. `getaddrinfo ENOTFOUND`).
  *
- * Lives for one crawl session — cleared by {@link clearDnsBurnedHostCache} at
- * the same four orchestrator sites as {@link destinationCache}. Hosts are
+ * Lives for one crawl session — cleared by `clearDnsBurnedHostCache` at
+ * the same four orchestrator sites as `destinationCache`. Hosts are
  * marked in two ways:
  *
  * - **Session learning**: the `onGiveUp` callback of `#sendHeadRequest`
  *   classifies the final retry error and stores `'dns'` when the matcher
  *   fires. The next URL on the same hostname short-circuits before retry.
  * - **Session preload**: re-open paths (`append` / `inventory` / `retryFailed`
- *   / `resume`) call {@link Archive.listDnsBurnedHostCandidates} and seed the
+ *   / `resume`) call `Archive.listDnsBurnedHostCandidates` and seed the
  *   map from `crawl_errors`, so previously-burned hosts cost zero retries on
  *   the next crawl.
  *

package/lib/crawler/dns-burned-host-cache.js

@@ -2,15 +2,15 @@
  * In-memory set of hostnames known to be unreachable due to DNS errors
  * (e.g. `getaddrinfo ENOTFOUND`).
  *
- * Lives for one crawl session — cleared by {@link clearDnsBurnedHostCache} at
- * the same four orchestrator sites as {@link destinationCache}. Hosts are
+ * Lives for one crawl session — cleared by `clearDnsBurnedHostCache` at
+ * the same four orchestrator sites as `destinationCache`. Hosts are
  * marked in two ways:
  *
  * - **Session learning**: the `onGiveUp` callback of `#sendHeadRequest`
  *   classifies the final retry error and stores `'dns'` when the matcher
  *   fires. The next URL on the same hostname short-circuits before retry.
  * - **Session preload**: re-open paths (`append` / `inventory` / `retryFailed`
- *   / `resume`) call {@link Archive.listDnsBurnedHostCandidates} and seed the
+ *   / `resume`) call `Archive.listDnsBurnedHostCandidates` and seed the
  *   map from `crawl_errors`, so previously-burned hosts cost zero retries on
  *   the next crawl.
  *

package/lib/crawler/dns-burned-host-short-circuit-counter.d.ts

@@ -1,8 +1,8 @@
 /**
  * Session-scoped counter for how many URL fetches were short-circuited by the
- * {@link dnsBurnedHostCache}. The orchestrator reads this at crawl shutdown
+ * `dnsBurnedHostCache`. The orchestrator reads this at crawl shutdown
  * to emit a `[preload] Short-circuited N URL(s)` summary line, then
- * {@link clearDnsBurnedHostCache} zeroes it for the next session.
+ * `clearDnsBurnedHostCache` zeroes it for the next session.
  *
  * Exposed as a mutable object (not a plain `let`) so that the counter remains
  * a single shared reference across crawler / orchestrator imports — `let`

package/lib/crawler/dns-burned-host-short-circuit-counter.js

@@ -1,8 +1,8 @@
 /**
  * Session-scoped counter for how many URL fetches were short-circuited by the
- * {@link dnsBurnedHostCache}. The orchestrator reads this at crawl shutdown
+ * `dnsBurnedHostCache`. The orchestrator reads this at crawl shutdown
  * to emit a `[preload] Short-circuited N URL(s)` summary line, then
- * {@link clearDnsBurnedHostCache} zeroes it for the next session.
+ * `clearDnsBurnedHostCache` zeroes it for the next session.
  *
  * Exposed as a mutable object (not a plain `let`) so that the counter remains
  * a single shared reference across crawler / orchestrator imports — `let`

package/lib/crawler/inject-scope-auth.d.ts

@@ -3,7 +3,7 @@ import type { ExURL } from '@d-zero/shared/parse-url';
  * Copy `username` / `password` from a matched scope URL into the target URL.
  *
  * The matched scope is supplied by the caller (typically the result of a single
- * {@link findScopeEntry} call). This avoids the previous implementation's
+ * `findScopeEntry` call). This avoids the previous implementation's
  * redundant hostname lookup and re-search.
  *
  * Mutates the `url` parameter in place. Only non-empty credentials overwrite

package/lib/crawler/inject-scope-auth.js

@@ -2,7 +2,7 @@
  * Copy `username` / `password` from a matched scope URL into the target URL.
  *
  * The matched scope is supplied by the caller (typically the result of a single
- * {@link findScopeEntry} call). This avoids the previous implementation's
+ * `findScopeEntry` call). This avoids the previous implementation's
  * redundant hostname lookup and re-search.
  *
  * Mutates the `url` parameter in place. Only non-empty credentials overwrite

package/lib/crawler/normalize-content-type.d.ts

@@ -7,7 +7,7 @@
  * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
  * database. Storing the canonical (trimmed, lower-cased) form lets the exact
  * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
- * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * code-level `isHtmlContentType` check, which trims and lower-cases.
  * @param contentType - The raw media type, or `null` when unknown.
  * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
  */

package/lib/crawler/normalize-content-type.js

@@ -7,7 +7,7 @@
  * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
  * database. Storing the canonical (trimmed, lower-cased) form lets the exact
  * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
- * code-level {@link isHtmlContentType} check, which trims and lower-cases.
+ * code-level `isHtmlContentType` check, which trims and lower-cases.
  * @param contentType - The raw media type, or `null` when unknown.
  * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
  */

package/lib/crawler/types.d.ts

@@ -82,7 +82,7 @@ export type BrowserScrapeResult = ScrapeResult & {
      * exactly that.
      *
      * Consumers should also confirm the URL is meaningful via
-     * {@link deriveJsRedirectTarget} — `about:blank`, identity values,
+     * `deriveJsRedirectTarget` — `about:blank`, identity values,
      * case-only or trailing-slash variants are all filtered there, not
      * here.
      */
@@ -188,7 +188,7 @@ export type ResourceLookup = (urls: readonly string[]) => Promise<ResourceLookup
  * that has not been INSERTed yet) so the caller can fall through to its
  * default behaviour.
  *
- * Used by {@link Crawler} during sub-resource lineage propagation:
+ * Used by `Crawler` during sub-resource lineage propagation:
  * `#scrapePage` consults this once per page to resolve the parent's
  * lineage when the in-memory `inventoryMode` is unavailable (i.e. on
  * `--resume` / `--retry-failed` sessions where inventory state lives only
@@ -250,7 +250,7 @@ export interface CrawlerEventTypes {
     externalPage: {
         /** The scraped page data for the external page. */
         result: PageData;
-        /** Inventory provenance for new rows — see {@link CrawlerEventTypes.page.source}. */
+        /** Inventory provenance for new rows — see `CrawlerEventTypes.page.source`. */
         source?: PageSource;
     };
     /**

package/lib/crawler-orchestrator.d.ts

@@ -8,6 +8,9 @@ import Archive from './archive/archive.js';
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
  * but provide no useful crawl data.
+ * @example
+ * // Merged ahead of user-supplied excludeUrls when a crawl starts:
+ * const excludeUrls = [...DEFAULT_EXCLUDED_EXTERNAL_URLS, 'https://ads.example.com'];
  */
 export declare const DEFAULT_EXCLUDED_EXTERNAL_URLS: string[];
 /**
@@ -117,6 +120,12 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
     /**
      * Write the archive to its configured file path.
      *
+     * The crawler's write path inserts directly into the 0.13 entity
+     * tables (`content_items` / `page_meta` / `anchor_edges` / …) during
+     * `crawling` / `append` / `resume` / `retryFailed` / `inventory`, so by
+     * the time `write()` is called those tables are already populated.
+     * This method just tars.
+     *
      * Emits `writeFileStart` before writing and `writeFileEnd` after
      * the write completes successfully.
      */

package/lib/crawler-orchestrator.js

@@ -5,6 +5,7 @@ import { sortUrl } from '@d-zero/shared/sort-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import pkg from '../package.json' with { type: 'json' };
 import Archive from './archive/archive.js';
+import { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
 import { clearDestinationCache } from './crawler/clear-destination-cache.js';
 import { clearDnsBurnedHostCache } from './crawler/clear-dns-burned-host-cache.js';
 import Crawler from './crawler/crawler.js';
@@ -24,6 +25,9 @@ import { WriteQueue } from './write-queue.js';
  * Default list of external URL prefixes excluded from crawling.
  * Includes social media sharing endpoints that are commonly linked
  * but provide no useful crawl data.
+ * @example
+ * // Merged ahead of user-supplied excludeUrls when a crawl starts:
+ * const excludeUrls = [...DEFAULT_EXCLUDED_EXTERNAL_URLS, 'https://ads.example.com'];
  */
 export const DEFAULT_EXCLUDED_EXTERNAL_URLS = [
     'https://social-plugins.line.me',
@@ -128,8 +132,8 @@ export class CrawlerOrchestrator extends EventEmitter {
             // back to the DB DEFAULT `'crawled'` and lose their
             // `'inventory-discovered'` provenance.
             lookupPageSource: async (url) => this.#archive.getPageSourceByUrl(url),
-            // Inventory mode is opted into by `CrawlerOrchestrator.inventory`
-            // (see T3); the default crawl path stays in normal mode so new
+            // Inventory mode is opted into by `CrawlerOrchestrator.inventory`;
+            // the default crawl path stays in normal mode so new
             // rows continue to land in pages/resources with the DB DEFAULT
             // `'crawled'` provenance label.
             inventoryMode: options?.inventoryMode ?? null,
@@ -258,6 +262,12 @@ export class CrawlerOrchestrator extends EventEmitter {
     /**
      * Write the archive to its configured file path.
      *
+     * The crawler's write path inserts directly into the 0.13 entity
+     * tables (`content_items` / `page_meta` / `anchor_edges` / …) during
+     * `crawling` / `append` / `resume` / `retryFailed` / `inventory`, so by
+     * the time `write()` is called those tables are already populated.
+     * This method just tars.
+     *
      * Emits `writeFileStart` before writing and `writeFileEnd` after
      * the write completes successfully.
      */
@@ -299,7 +309,14 @@ export class CrawlerOrchestrator extends EventEmitter {
         // Each positional URL is both a starting point and a scope entry.
         const rootHrefs = list.map((u) => u.withoutHash);
         await archive.setConfig({
-            version: pkg.version,
+            // `version` is the archive-format version (see
+            // `assertCompatibleVersion`), NOT the npm package version. Decoupled
+            // because format-breaking changes and code-release cadence are
+            // different concerns — a patch release must not silently bump the
+            // format version and reject older archives, and a dev build of an
+            // unreleased breaking change must be able to produce archives the
+            // same build can read back.
+            version: REQUIRED_FORMAT_VERSION,
             name: fileName,
             baseUrl: rootHrefs[0],
             roots: rootHrefs,
@@ -520,13 +537,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // anchor-referenced, `scraped=0` rows. Predicted-discard leaks
                 // and external anomalies are filtered out at the reader, so a
                 // non-empty pending here means the previous session genuinely
-                // stopped with interrupted in-scope work. The original hard
-                // rejection blocked legitimate inventory runs in practice
-                // because leak rows polluted the count; with the strict
-                // reader those false positives are gone, so a warning is
+                // stopped with interrupted in-scope work. A hard rejection is
+                // still not warranted (with a looser reader it would block
+                // legitimate inventory runs whenever leak rows polluted the
+                // count), so a warning is
                 // enough — the inventory pass continues and the crawled-wins
                 // source priority keeps stale labels stable even if some of
                 // the strict-pending rows happen to land on inventory seeds.
+                // eslint-disable-next-line no-console -- operator-facing warning, must be visible regardless of DEBUG filters
                 console.warn(`inventory: archive has ${pending.length} pending URLs from a previous crawl. Proceeding — crawled-wins priority keeps their labels stable. Consider \`--resume\` first if you want the prior work finalized.`);
             }
             // Parse + scope-classify the candidate URLs. sortUrl drops
@@ -556,7 +574,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             }
             // Drop URLs that are already represented in the archive (either
             // as pages or resources). Comparison key is `withoutHashAndAuth`
-            // to mirror what `#getIdByUrl` / `insertResource` actually store.
+            // to mirror what `resolveContentItemId` / `insertResource` actually store.
             // Two independent reads — Promise.all halves the wait on large
             // archives where each `WHERE url IN (?)` chunk costs real I/O.
             const candidateUrls = inScope.map((u) => u.withoutHashAndAuth);
@@ -656,8 +674,8 @@ export class CrawlerOrchestrator extends EventEmitter {
                     htmlSeeds.push(url);
                 }
                 // Bulk-record non-HTML novel URLs in `resources` as
-                // `source='inventory-seed'` placeholders. The previous
-                // per-URL `await setResources(...)` loop spent minutes
+                // `source='inventory-seed'` placeholders. A
+                // per-URL `await setResources(...)` loop would spend minutes
                 // inside the `.bak`-protected window on large inventory
                 // lists; the chunked bulk path collapses N round-trips
                 // to N/500.
@@ -665,7 +683,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // Pre-insert HTML seeds as `scraped = 0`,
                 // `source = 'inventory-seed'` placeholders *before* the
                 // scrape phase, so a Ctrl+C between here and `setPage`
-                // no longer loses the URL. The strict-pending set picks
+                // cannot lose the URL. The strict-pending set picks
                 // these rows up on the next `--resume` via the
                 // `OR p.source != 'crawled'` clause.
                 await archive.insertInventorySeeds(htmlSeeds);
@@ -674,9 +692,9 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // hiccup or transient lock on the INSERT aborts the ingestion
                 // and the `.bak` restore wipes the pre-inserted seeds too,
                 // so "either the whole run took or none of it did" holds at
-                // the ingestion boundary. Past behaviour swallowed the
-                // failure post-scrape; the new boundary makes restore safe
-                // and useful, so the swallow is gone (see
+                // the ingestion boundary. Audit failures are deliberately
+                // NOT swallowed — inside the `.bak` window a restore is
+                // safe and useful (see
                 // {@link CrawlerOrchestrator.#writeInventoryRunRow}).
                 await CrawlerOrchestrator.#writeInventoryRunRow(archive, {
                     inventoryUrlsCount: inventoryUrls.length,
@@ -962,13 +980,12 @@ export class CrawlerOrchestrator extends EventEmitter {
      * static helper because the audit-row shape (timestamp stamping + label
      * auto-gen + the privacy-driven path elision documented below) is a
      * cohesive concern that benefits from staying outside the long
-     * `inventory()` body even though only one caller remains after the
-     * ingestion-phase consolidation.
+     * `inventory()` body even though it has a single caller.
      *
      * `ran_at` is stamped now (ingestion-completion timestamp; the scrape
      * phase that may follow is treated as separate). `list_label` is
-     * auto-generated from `ran_at` when the CLI did not pass one — Phase 1
-     * has no `--label` flag, so this is always the auto form.
+     * auto-generated from `ran_at` when the CLI did not pass one — there is
+     * no `--label` flag, so this is always the auto form.
      * `source_file_sha256` arrives pre-computed via
      * `aggregates.sourceFileSha256` (the CLI's `inventoryCrawl` ran
      * `computeFileSha256` against the input txt before the orchestrator
@@ -976,17 +993,17 @@ export class CrawlerOrchestrator extends EventEmitter {
      * the absolute path — see {@link InventoryRunAggregates} for the
      * privacy rationale.
      *
-     * **Audit-write failures abort the ingestion phase.** The earlier
-     * implementation swallowed them because the audit was the last write
-     * after* the scrape, so re-throwing would have wiped a completed crawl;
-     * with audit now lifted into the `.bak`-protected ingestion phase the
+     * **Audit-write failures abort the ingestion phase.** Swallowing them
+     * would only be justified if the audit were the last write after the
+     * scrape (re-throwing there would wipe a completed crawl); inside the
+     * `.bak`-protected ingestion phase the
      * trade-off flips. A failed audit row is restorable: the outer catch
      * copies `.bak` back over the archive and the operator reruns the
      * (short) ingestion from scratch. That keeps `inventory_runs` honest
      * (no "ran but unrecorded" rows) at the cost of one rerun.
      *
-     * Forward-compat: when Phase 2 introduces an explicit `--label` flag,
-     * thread `labelOverride` through {@link inventory} into the `aggregates`
+     * Forward-compat: if an explicit `--label` flag is ever added, thread
+     * `labelOverride` through {@link inventory} into the `aggregates`
      * shape so the auto-name can be overridden.
      * @param archive - The opened archive to write the audit row into.
      * @param aggregates - The counts captured during the inventory pass; see {@link InventoryRunAggregates}.
@@ -1005,10 +1022,9 @@ export class CrawlerOrchestrator extends EventEmitter {
     }
     /**
      * Tears down session-scoped crawler caches and prints a short-circuit
-     * summary if any URL fetches were skipped. Invoked at the four
-     * crawl-session boundaries (`crawling` / `append` / `inventory` /
-     * `retryFailed` / `resume`) where the previous `clearDestinationCache`
-     * call already lived.
+     * summary if any URL fetches were skipped. Invoked at every
+     * crawl-session boundary (`crawling` / `append` / `inventory` /
+     * `retryFailed` / `resume`).
      */
     static #finalizeCrawlSession() {
         const skipped = dnsBurnedHostShortCircuitCounter.count;

package/lib/crawler.d.ts

@@ -16,6 +16,7 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { ArchiveLockHolder } from './archive/peek-archive-lock.js';
 export type { FlatPageMetaColumns, JsonLdRow, JsonLdRowForInsert, TagRow, TagRowForInsert, JsonLdSummary, TagsSummary, TagInventoryEntry, PageDenormalizedColumns, } from './archive/meta/types.js';
@@ -26,3 +27,14 @@ export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
+export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
+export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';
+export { populateRefTables } from './archive/populate-ref-tables/populate-refs.js';
+export { populateContentTypeRefs } from './archive/populate-ref-tables/populate-content-type-refs.js';
+export { populateUrlRefs } from './archive/populate-ref-tables/populate-url-refs.js';
+export { populateTextRefs } from './archive/populate-ref-tables/populate-text-refs.js';
+export { populateJsonRefs } from './archive/populate-ref-tables/populate-json-refs.js';
+export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-refs.js';
+export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
+export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
+export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';

package/lib/crawler.js

@@ -17,6 +17,7 @@ export { default as ArchiveResource } from './archive/resource.js';
 export * from './archive/types.js';
 export { default as Archive } from './archive/archive.js';
 export { isArchiveCacheDisabled } from './archive/cache/is-archive-cache-disabled.js';
+export { acquireArchiveLock, ArchiveLockError } from './archive/archive-lock.js';
 export { peekArchiveLockHolder } from './archive/peek-archive-lock.js';
 export { IncompatibleArchiveError } from './archive/meta/types.js';
 export { REQUIRED_FORMAT_VERSION } from './archive/meta/assert-compatible-version.js';
@@ -26,3 +27,23 @@ export * from './types.js';
 export * from './crawler/types.js';
 export { classifyErrorKind } from './classify-error-kind.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
+// 0.13 ref-table population (issue #191, epic #103). Exposed as the
+// public seam that the migration script (`scripts/migrate-to-0.13.mjs`)
+// drives against an already-connected archive.
+// The individual sub-steps are also exported so the migration script can
+// resume mid-way if the caller decides to split the transaction.
+export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
+export { populateRefTables } from './archive/populate-ref-tables/populate-refs.js';
+export { populateContentTypeRefs } from './archive/populate-ref-tables/populate-content-type-refs.js';
+export { populateUrlRefs } from './archive/populate-ref-tables/populate-url-refs.js';
+export { populateTextRefs } from './archive/populate-ref-tables/populate-text-refs.js';
+export { populateJsonRefs } from './archive/populate-ref-tables/populate-json-refs.js';
+export { populateBlobRefs } from './archive/populate-ref-tables/populate-blob-refs.js';
+export { populateHeaderTables } from './archive/populate-ref-tables/populate-header-tables.js';
+// 0.13 read-side reconstruction primitives. Exported so downstream
+// readers (`@nitpicker/query`'s page-detail view) reconstruct
+// `responseHeaders` / json_refs payloads with the exact same merge and
+// decode semantics as the crawler's own read paths — one implementation,
+// no cross-package drift.
+export { loadResponseHeadersBySetIds } from './archive/db-ops/_shared/load-response-headers-by-set-ids.js';
+export { decodeJsonRef } from './archive/db-ops/_shared/decode-json-ref.js';

package/lib/permanent-error-kinds.d.ts

@@ -4,7 +4,7 @@ import type { ErrorKind } from './types.js';
  * (without changing the network, the certificate, the destination host, or the
  * browser's block-list state) will almost certainly produce the same failure.
  *
- * Used by {@link resetFailedPages} to exclude pages whose latest recorded error
+ * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
  * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every

package/lib/permanent-error-kinds.js

@@ -3,7 +3,7 @@
  * (without changing the network, the certificate, the destination host, or the
  * browser's block-list state) will almost certainly produce the same failure.
  *
- * Used by {@link resetFailedPages} to exclude pages whose latest recorded error
+ * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
  * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every

package/lib/types.d.ts

@@ -3,7 +3,7 @@ import type { CrawlerError, PageData } from './utils/types/types.js';
  * Aggregate counts captured during a `--inventory` invocation, forwarded to
  * `#writeInventoryRunRow` so the audit log row is consistent between the
  * HTML-seed branch and the non-HTML-only branch of
- * {@link CrawlerOrchestrator.inventory}.
+ * `CrawlerOrchestrator.inventory`.
  *
  * Spelled out here (not inlined at the call site) so a new field added to
  * the audit row has a single edit point and so each field's semantics are

package/lib/utils/compute-file-sha256.d.ts

@@ -2,10 +2,11 @@
  * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
  * O(1) regardless of file size.
  *
- * Used by {@link CrawlerOrchestrator.inventory} to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row.
- * Phase 3 (`--refresh`) will key dedupe on this column; Phase 1 just
- * records it for operator audit.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
+ * URL list `.txt` and store the digest on the `inventory_runs` row. The
+ * digest is the content-identity key for "was this exact list applied
+ * before" — recorded for operator audit; nothing dedupes on it at write
+ * time.
  *
  * Returns `null` instead of throwing when the file cannot be read (e.g.
  * vanished mid-run, permissions issue) so the inventory run's audit row