@nitpicker/crawler 0.20.0 → 0.22.0

package/lib/archive/db-ops/pages/reset/reset-pages-by-urls.d.ts

@@ -0,0 +1,78 @@
+import type { ResetPagesByUrlsResult } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Reset pages matching an operator-supplied URL list back to pending, so a
+ * follow-up crawl re-fetches them from scratch — the un-scrape half of
+ * `crawl --recrawl`.
+ *
+ * Unlike {@link resetFailedPages}, this function does not filter by prior
+ * failure kind: a URL named explicitly by the operator is reset regardless
+ * of what its last observation was, including a definitive `4xx`/`5xx` — the
+ * whole point of `--recrawl` is to catch a page that used to 404 and is now
+ * live. The only requirement on the raw candidate scan is `scraped = 1` (a
+ * page never scraped has nothing to reset; it is the caller's "novel URL"
+ * path instead).
+ *
+ * Three conservative guards then apply — matching content follows the same
+ * union used to skip `resetFailedPages`'s permanent-failure filter had none:
+ * a matched row is excluded (not reset) rather than silently promoted, and
+ * the exclusion is reported back to the caller by category:
+ *
+ * - **Redirect source** (`redirect_dest_id` is set): resetting it would
+ *   discard a real, previously-observed 3xx. If the operator wants the
+ *   redirect re-verified, that is `--retry-failed`'s job (which does NOT
+ *   exclude redirect sources from a permanent-failure reset — a different
+ *   operation with a different default).
+ * - **Intentionally skipped** (`is_skipped = 1`): the row was excluded by
+ *   `excludes`/`excludeUrls` at ingestion or fetch time. `--recrawl` does not
+ *   second-guess that configuration; re-running the crawl with a changed
+ *   exclude list is the supported path to un-skip a page.
+ * - **External** (`is_external = 1`): bringing a scope-external URL back
+ *   into scope is `--append`'s job (`repromoteExternalPages`), which also
+ *   updates the scope map. `--recrawl` only re-fetches in-scope pages.
+ *
+ * Deliberately does NOT exclude candidates whose URL shape already has a
+ * confirmed same-cluster trap recorded (`dedupe_cap_events.shape_key`) — the
+ * exclusion {@link resetFailedPages} applies for exactly that reason (see its
+ * own JSDoc). The two functions differ on what "the operator asked for this"
+ * means: `resetFailedPages` widens an automatic, unattended SQL scan
+ * (`--retry-failed`), so a known trap is safe to silently skip re-admitting.
+ * `resetPagesByUrls` resets URLs the operator named one-by-one in a file —
+ * the same reasoning that already exempts explicit URLs from the
+ * failure-kind filter above applies here too: an operator who names a
+ * specific trap-shaped URL is not "re-discovering" it by accident, and
+ * silently refusing to reset a row they explicitly listed would be a more
+ * surprising failure mode than honouring the request.
+ *
+ * A URL that matches no `content_items` row at all (not yet known to the
+ * archive) is silently absent from every array on the result — the caller
+ * treats it as a novel URL, the same "not yet known" path `--inventory`
+ * takes. A URL matching a row with `scraped = 0` is likewise absent from
+ * every array: it is already pending, so there is nothing to reset.
+ *
+ * On a match, `content_items` is UPDATEd exactly like
+ * {@link resetFailedPages} (`scraped`/`status`/`status_text`/
+ * `content_type_id`/`content_length`/`header_set_id` cleared,
+ * `first_crawled_at`/`last_crawled_at` preserved), and every derived row is
+ * cleared via {@link clearPageDerivedRows} plus `page_errors` and
+ * `analysis_violations` — the latter is not part of the shared helper
+ * (`repromoteExternalPages` never clears it) but a re-fetched page's old
+ * lint findings would otherwise report on HTML that no longer exists until
+ * the next `analyze` run overwrites the whole table. `analysis_text_refs` is
+ * a content-hash dictionary shared across pages and is not touched; an
+ * orphaned entry is a harmless, unreferenced row, the same trade-off already
+ * made for `page_html_blobs`.
+ *
+ * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to match against `content_items`, already in
+ *   `withoutHashAndAuth` form (mirrors every other URL-list comparison in
+ *   this package, e.g. `getExistingPageUrls`).
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to reset. Omit
+ *   for no reporting (the default; e.g. tests).
+ * @returns The reset URLs, plus the excluded URLs grouped by exclusion
+ *   reason — see {@link ResetPagesByUrlsResult}.
+ */
+export declare function resetPagesByUrls(knex: Knex, urls: readonly string[], onProgress?: (processed: number, total: number) => void): Promise<ResetPagesByUrlsResult>;

package/lib/archive/db-ops/pages/reset/reset-pages-by-urls.js

@@ -0,0 +1,146 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { dbLog } from '../../../debug.js';
+import { clearPageDerivedRows } from './clear-page-derived-rows.js';
+/**
+ * Reset pages matching an operator-supplied URL list back to pending, so a
+ * follow-up crawl re-fetches them from scratch — the un-scrape half of
+ * `crawl --recrawl`.
+ *
+ * Unlike {@link resetFailedPages}, this function does not filter by prior
+ * failure kind: a URL named explicitly by the operator is reset regardless
+ * of what its last observation was, including a definitive `4xx`/`5xx` — the
+ * whole point of `--recrawl` is to catch a page that used to 404 and is now
+ * live. The only requirement on the raw candidate scan is `scraped = 1` (a
+ * page never scraped has nothing to reset; it is the caller's "novel URL"
+ * path instead).
+ *
+ * Three conservative guards then apply — matching content follows the same
+ * union used to skip `resetFailedPages`'s permanent-failure filter had none:
+ * a matched row is excluded (not reset) rather than silently promoted, and
+ * the exclusion is reported back to the caller by category:
+ *
+ * - **Redirect source** (`redirect_dest_id` is set): resetting it would
+ *   discard a real, previously-observed 3xx. If the operator wants the
+ *   redirect re-verified, that is `--retry-failed`'s job (which does NOT
+ *   exclude redirect sources from a permanent-failure reset — a different
+ *   operation with a different default).
+ * - **Intentionally skipped** (`is_skipped = 1`): the row was excluded by
+ *   `excludes`/`excludeUrls` at ingestion or fetch time. `--recrawl` does not
+ *   second-guess that configuration; re-running the crawl with a changed
+ *   exclude list is the supported path to un-skip a page.
+ * - **External** (`is_external = 1`): bringing a scope-external URL back
+ *   into scope is `--append`'s job (`repromoteExternalPages`), which also
+ *   updates the scope map. `--recrawl` only re-fetches in-scope pages.
+ *
+ * Deliberately does NOT exclude candidates whose URL shape already has a
+ * confirmed same-cluster trap recorded (`dedupe_cap_events.shape_key`) — the
+ * exclusion {@link resetFailedPages} applies for exactly that reason (see its
+ * own JSDoc). The two functions differ on what "the operator asked for this"
+ * means: `resetFailedPages` widens an automatic, unattended SQL scan
+ * (`--retry-failed`), so a known trap is safe to silently skip re-admitting.
+ * `resetPagesByUrls` resets URLs the operator named one-by-one in a file —
+ * the same reasoning that already exempts explicit URLs from the
+ * failure-kind filter above applies here too: an operator who names a
+ * specific trap-shaped URL is not "re-discovering" it by accident, and
+ * silently refusing to reset a row they explicitly listed would be a more
+ * surprising failure mode than honouring the request.
+ *
+ * A URL that matches no `content_items` row at all (not yet known to the
+ * archive) is silently absent from every array on the result — the caller
+ * treats it as a novel URL, the same "not yet known" path `--inventory`
+ * takes. A URL matching a row with `scraped = 0` is likewise absent from
+ * every array: it is already pending, so there is nothing to reset.
+ *
+ * On a match, `content_items` is UPDATEd exactly like
+ * {@link resetFailedPages} (`scraped`/`status`/`status_text`/
+ * `content_type_id`/`content_length`/`header_set_id` cleared,
+ * `first_crawled_at`/`last_crawled_at` preserved), and every derived row is
+ * cleared via {@link clearPageDerivedRows} plus `page_errors` and
+ * `analysis_violations` — the latter is not part of the shared helper
+ * (`repromoteExternalPages` never clears it) but a re-fetched page's old
+ * lint findings would otherwise report on HTML that no longer exists until
+ * the next `analyze` run overwrites the whole table. `analysis_text_refs` is
+ * a content-hash dictionary shared across pages and is not touched; an
+ * orphaned entry is a harmless, unreferenced row, the same trade-off already
+ * made for `page_html_blobs`.
+ *
+ * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
+ * `SQLITE_LIMIT_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to match against `content_items`, already in
+ *   `withoutHashAndAuth` form (mirrors every other URL-list comparison in
+ *   this package, e.g. `getExistingPageUrls`).
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to reset. Omit
+ *   for no reporting (the default; e.g. tests).
+ * @returns The reset URLs, plus the excluded URLs grouped by exclusion
+ *   reason — see {@link ResetPagesByUrlsResult}.
+ */
+export async function resetPagesByUrls(knex, urls, onProgress) {
+    const empty = {
+        resetUrls: [],
+        excludedRedirects: [],
+        excludedSkipped: [],
+        excludedExternal: [],
+    };
+    if (urls.length === 0) {
+        return empty;
+    }
+    const candidates = [];
+    await eachSplitted([...urls], 500, async (chunk) => {
+        const rows = await knex('content_items')
+            .join('url_refs', 'url_refs.id', 'content_items.url_id')
+            .select('content_items.id as id', 'url_refs.url as url', 'content_items.redirect_dest_id as redirect_dest_id', 'content_items.is_skipped as is_skipped', 'content_items.is_external as is_external')
+            .where('content_items.scraped', 1)
+            .whereIn('url_refs.url', chunk);
+        candidates.push(...rows);
+    });
+    if (candidates.length === 0) {
+        return empty;
+    }
+    const resettable = [];
+    const excludedRedirects = [];
+    const excludedSkipped = [];
+    const excludedExternal = [];
+    for (const row of candidates) {
+        if (row.redirect_dest_id != null) {
+            excludedRedirects.push(row.url);
+            continue;
+        }
+        if (row.is_skipped) {
+            excludedSkipped.push(row.url);
+            continue;
+        }
+        if (row.is_external) {
+            excludedExternal.push(row.url);
+            continue;
+        }
+        resettable.push(row);
+    }
+    if (resettable.length === 0) {
+        return { resetUrls: [], excludedRedirects, excludedSkipped, excludedExternal };
+    }
+    const ids = resettable.map((row) => row.id);
+    const resetUrls = resettable.map((row) => row.url);
+    const chunkSize = 500;
+    for (let i = 0; i < ids.length; i += chunkSize) {
+        const chunk = ids.slice(i, i + chunkSize);
+        await knex('content_items').whereIn('id', chunk).update({
+            scraped: 0,
+            status: null,
+            status_text: null,
+            content_type_id: null,
+            content_length: null,
+            header_set_id: null,
+            // `first_crawled_at` / `last_crawled_at` are deliberately left
+            // untouched, matching `resetFailedPages` — the last-success
+            // timestamp records survive the demotion.
+        });
+        await knex('page_errors').whereIn('pageId', chunk).delete();
+        await knex('analysis_violations').whereIn('page_id', chunk).delete();
+        await clearPageDerivedRows(knex, chunk);
+        onProgress?.(Math.min(i + chunkSize, ids.length), ids.length);
+    }
+    dbLog('Reset %d page(s) matched by URL list back to pending', resetUrls.length);
+    return { resetUrls, excludedRedirects, excludedSkipped, excludedExternal };
+}

package/lib/archive/db-ops/pages/reset/retryable-image-scan-codes.d.ts

@@ -0,0 +1,24 @@
+/**
+ * `page_meta.image_scan_desktop` / `image_scan_mobile` codes that qualify a
+ * page for `--retry-failed`, mirroring `@d-zero/beholder`'s `IMAGE_SCAN_CODE`
+ * (kept as a plain numeric literal list here rather than importing that
+ * constant, since `@nitpicker/crawler`'s `page_meta` columns persist across
+ * a `@d-zero/beholder` version bump and must keep meaning the same numbers
+ * even if a future crawler build pins an older beholder).
+ *
+ * - `2` (`nav-unsettled`) and `3` (`frame-lost`) are transient failures the
+ *   scan itself could not recover from — a retry is likely to succeed.
+ * - `255` (`unknown`) is retried too, erring on the side of investigation,
+ *   consistent with {@link import('../../../../classify-error-kind.js').classifyErrorKind}'s
+ *   "unknown" kind never being treated as permanent.
+ *
+ * Deliberately excluded:
+ * - `0` (`ok`) and `1` (`degraded`) — the scan produced data.
+ * - `4` (`scroll-height-exceeded`) — a deterministic, page-shape-driven
+ *   outcome that a retry cannot change.
+ * @example
+ * ```ts
+ * qb.whereIn('page_meta.image_scan_desktop', RETRYABLE_IMAGE_SCAN_CODES);
+ * ```
+ */
+export declare const RETRYABLE_IMAGE_SCAN_CODES: readonly [2, 3, 255];

package/lib/archive/db-ops/pages/reset/retryable-image-scan-codes.js

@@ -0,0 +1,24 @@
+/**
+ * `page_meta.image_scan_desktop` / `image_scan_mobile` codes that qualify a
+ * page for `--retry-failed`, mirroring `@d-zero/beholder`'s `IMAGE_SCAN_CODE`
+ * (kept as a plain numeric literal list here rather than importing that
+ * constant, since `@nitpicker/crawler`'s `page_meta` columns persist across
+ * a `@d-zero/beholder` version bump and must keep meaning the same numbers
+ * even if a future crawler build pins an older beholder).
+ *
+ * - `2` (`nav-unsettled`) and `3` (`frame-lost`) are transient failures the
+ *   scan itself could not recover from — a retry is likely to succeed.
+ * - `255` (`unknown`) is retried too, erring on the side of investigation,
+ *   consistent with {@link import('../../../../classify-error-kind.js').classifyErrorKind}'s
+ *   "unknown" kind never being treated as permanent.
+ *
+ * Deliberately excluded:
+ * - `0` (`ok`) and `1` (`degraded`) — the scan produced data.
+ * - `4` (`scroll-height-exceeded`) — a deterministic, page-shape-driven
+ *   outcome that a retry cannot change.
+ * @example
+ * ```ts
+ * qb.whereIn('page_meta.image_scan_desktop', RETRYABLE_IMAGE_SCAN_CODES);
+ * ```
+ */
+export const RETRYABLE_IMAGE_SCAN_CODES = [2, 3, 255];

package/lib/archive/db-ops/pages/write/insert-inventory-content-items.js

@@ -45,7 +45,7 @@ export async function insertInventoryContentItems(params) {
         });
         await knex('content_items').insert(rows).onConflict('url_id').ignore();
         const inserted = (await knex
-            .select('ci.id', 'ci.source', 'ur.url')
+            .select('ci.id', 'ci.source', 'ci.is_metadata_only', 'ur.url')
             .from('content_items as ci')
             .join('url_refs as ur', 'ur.id', 'ci.url_id')
             .whereIn('ur.url', chunk));
@@ -53,6 +53,7 @@ export async function insertInventoryContentItems(params) {
             caches.contentItems.set(insertedRow.url, {
                 id: insertedRow.id,
                 source: insertedRow.source,
+                isMetadataOnly: insertedRow.is_metadata_only,
             });
         }
     });

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -65,10 +65,12 @@ import { upsertUrlRef } from '../../_shared/upsert-url-ref.js';
  */
 export async function insertPage(knex, caches, page, isTarget, trx, source) {
     const qb = trx ?? knex;
-    const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, undefined, source);
+    const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, {
+        source,
+    });
     const flat = deriveFlatFromMeta(page.meta, page.url.href);
     const denorm = computePageDenormalized(page.meta);
-    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight, page.mainContentCustomElements?.length);
+    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight, page.mainContentCustomElements?.length, page.imageScan);
     const extras = deriveMetaExtras(page.meta);
     const now = Date.now();
     // Canonicalize so the stored dictionary value matches the exact-string

package/lib/archive/db-ops/pages/write/link-redirect-sources.js

@@ -53,7 +53,9 @@ export async function linkRedirectSources(trx, caches, sources, destId, destUrlN
         // argument is `'crawled'`, matching the anchor-lineage propagation
         // contract — an existing inventory-* intermediate that is later
         // traversed by a `'crawled'` chain gets downgraded.
-        const redirectId = await resolveContentItemId(trx, caches, redirect, undefined, chainLineageSource);
+        const redirectId = await resolveContentItemId(trx, caches, redirect, {
+            source: chainLineageSource,
+        });
         await trx('content_items')
             .where('id', redirectId)
             .update({

package/lib/archive/db-ops/pages/write/record-redirect.js

@@ -62,7 +62,7 @@ export async function recordRedirect(knex, caches, page, source) {
             // inventory chain — without the pass-through, inventory lineage
             // would be laundered to `'crawled'` for js-redirect rescue / #73
             // convergence destinations that have not yet been rendered.
-            const destId = await resolveContentItemId(trx, caches, destUrlObject.withoutHashAndAuth, undefined, source);
+            const destId = await resolveContentItemId(trx, caches, destUrlObject.withoutHashAndAuth, { source });
             // Chain lineage propagates FROM the originating URL (`page.url`),
             // NOT from the destination. The originating URL is what initiated
             // the redirect chain, so its lineage is what every intermediate hop

package/lib/archive/db-ops/pages/write/set-skipped-page.js

@@ -10,7 +10,9 @@ import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
  * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
  */
 export async function setSkippedPage(knex, caches, url, reason, isExternal = false) {
-    const pageId = await resolveContentItemId(knex, caches, url, isExternal ? 1 : 0);
+    const pageId = await resolveContentItemId(knex, caches, url, {
+        isExternal: isExternal ? 1 : 0,
+    });
     await knex('content_items')
         .where('id', pageId)
         .update({

package/lib/archive/db-ops/pages/write/update-page.d.ts

@@ -27,6 +27,14 @@ import type { Knex } from 'knex';
  * @param bodyHash - Precomputed body hash for the page's HTML (see
  *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
  *   computing it from the HTML instead.
+ * @param recursive - The crawl session's `recursive` option, forwarded to
+ *   `replaceAnchorEdges` so it can compute each discovered anchor's
+ *   `is_metadata_only` value the same way `processAnchors`
+ *   (`handle-scrape-end.ts`) decides whether to queue it for a full or
+ *   metadata-only scrape (#369). Defaults to `true` (the historical,
+ *   pre-#369 behaviour: never persist an anchor as metadata-only) for
+ *   callers that do not track the option, such as `setExternalPage`, whose
+ *   anchor list is always empty anyway.
  * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
  */
-export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null): Promise<number>;
+export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null, recursive?: boolean): Promise<number>;

package/lib/archive/db-ops/pages/write/update-page.js

@@ -50,9 +50,17 @@ import { writePageHtmlBlob } from './write-page-html-blob.js';
  * @param bodyHash - Precomputed body hash for the page's HTML (see
  *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
  *   computing it from the HTML instead.
+ * @param recursive - The crawl session's `recursive` option, forwarded to
+ *   `replaceAnchorEdges` so it can compute each discovered anchor's
+ *   `is_metadata_only` value the same way `processAnchors`
+ *   (`handle-scrape-end.ts`) decides whether to queue it for a full or
+ *   metadata-only scrape (#369). Defaults to `true` (the historical,
+ *   pre-#369 behaviour: never persist an anchor as metadata-only) for
+ *   callers that do not track the option, such as `setExternalPage`, whose
+ *   anchor list is always empty anyway.
  * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
  */
-export async function updatePage(knex, caches, page, writeHtml, isTarget, source, bodyHash) {
+export async function updatePage(knex, caches, page, writeHtml, isTarget, source, bodyHash, recursive = true) {
     const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
     const destUrlObject = parseUrl(destUrl);
     if (!destUrlObject) {
@@ -60,7 +68,7 @@ export async function updatePage(knex, caches, page, writeHtml, isTarget, source
     }
     try {
         return await knex.transaction(async (trx) => {
-            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash);
+            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash, recursive);
         });
     }
     catch (error) {
@@ -88,9 +96,10 @@ export async function updatePage(knex, caches, page, writeHtml, isTarget, source
  * @param isTarget - See {@link updatePage}.
  * @param source - See {@link updatePage}.
  * @param bodyHash - See {@link updatePage}.
+ * @param recursive - See {@link updatePage}.
  * @returns The `content_items.id` of the inserted or updated row.
  */
-async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash) {
+async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash, recursive) {
     const pageId = await insertPage(knex, caches, {
         ...page,
         url: destUrlObject,
@@ -218,7 +227,7 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
         .from('content_items')
         .where('id', pageId));
     const anchorLineageSource = deriveLineageFromParent(parentRow?.source, 'crawled');
-    await replaceAnchorEdges(trx, caches, pageId, page, anchorLineageSource);
+    await replaceAnchorEdges(trx, caches, pageId, page, anchorLineageSource, recursive);
     await replaceImageItems(trx, caches, pageId, page);
     // Clear this page's resource_ref_edges unconditionally (no non-empty
     // guard, unlike anchors/images above): the crawler always emits this
@@ -279,14 +288,27 @@ async function readSourceByUrl(trx, caches, url) {
  * @param page - The scraped page payload.
  * @param anchorLineageSource - Lineage label for anchor-target
  *   placeholder rows (see the caller's lineage comment).
+ * @param recursive - The crawl session's `recursive` option. Combined with
+ *   each anchor's `isExternal` (already resolved by `processAnchors` before
+ *   this ever runs) to compute `is_metadata_only` the same way
+ *   `processAnchors` (`handle-scrape-end.ts`) decides whether to queue the
+ *   anchor for a full or metadata-only scrape: `!recursive || isExternal`.
+ *   Both `recursive` and a given URL's `isExternal` are constant for the
+ *   whole crawl session, so this is a pure recomputation, not a second
+ *   source of truth (#369).
  */
-async function replaceAnchorEdges(trx, caches, pageId, page, anchorLineageSource) {
+async function replaceAnchorEdges(trx, caches, pageId, page, anchorLineageSource, recursive) {
     if (page.anchorList.length === 0) {
         return;
     }
     const edges = new Map();
     for (const anchor of page.anchorList) {
-        const hrefId = await resolveContentItemId(trx, caches, anchor.href.withoutHashAndAuth, anchor.isExternal ? 1 : 0, anchorLineageSource);
+        const isExternal = anchor.isExternal ?? false;
+        const hrefId = await resolveContentItemId(trx, caches, anchor.href.withoutHashAndAuth, {
+            isExternal: isExternal ? 1 : 0,
+            source: anchorLineageSource,
+            isMetadataOnly: !recursive || isExternal ? 1 : 0,
+        });
         const existing = edges.get(hrefId);
         if (existing !== undefined) {
             existing.count += 1;

package/lib/archive/db-ops/resources/get-resource-url-list.d.ts

@@ -5,12 +5,9 @@ import type { Knex } from 'knex';
  * two tables and returns the resolved strings.
  *
  * Read in `resource_items.id`-keyset chunks rather than a single SELECT
- * (issue #294): on a resource-heavy archive (images/JS/CSS in the tens or
- * hundreds of thousands) this was one unbounded, multi-second-to-minutes
- * query with no way to report progress mid-scan. The accumulated result is
- * identical to the previous single-SELECT read — chunking exists purely to
- * make the scan observable, not to bound memory (the flat URL list is
- * already fully materialised for the caller either way).
+ * (issue #294) via `keysetPaginateById` — see that function's JSDoc for
+ * why (progress observability on a resource-heavy archive, not memory
+ * bounding).
  * @param knex - Knex query builder connected to the archive DB.
  * @param onProgress - Called after each chunk with the `resource_items.id`
  *   scanned up to so far and the max id. Omit for no reporting (the

package/lib/archive/db-ops/resources/get-resource-url-list.js

@@ -1,3 +1,4 @@
+import { keysetPaginateById } from '../_shared/keyset-paginate-by-id.js';
 /** Rows read per `resource_items.id` keyset chunk. */
 const READ_CHUNK_SIZE = 2000;
 /**
@@ -6,12 +7,9 @@ const READ_CHUNK_SIZE = 2000;
  * two tables and returns the resolved strings.
  *
  * Read in `resource_items.id`-keyset chunks rather than a single SELECT
- * (issue #294): on a resource-heavy archive (images/JS/CSS in the tens or
- * hundreds of thousands) this was one unbounded, multi-second-to-minutes
- * query with no way to report progress mid-scan. The accumulated result is
- * identical to the previous single-SELECT read — chunking exists purely to
- * make the scan observable, not to bound memory (the flat URL list is
- * already fully materialised for the caller either way).
+ * (issue #294) via `keysetPaginateById` — see that function's JSDoc for
+ * why (progress observability on a resource-heavy archive, not memory
+ * bounding).
  * @param knex - Knex query builder connected to the archive DB.
  * @param onProgress - Called after each chunk with the `resource_items.id`
  *   scanned up to so far and the max id. Omit for no reporting (the
@@ -19,33 +17,10 @@ const READ_CHUNK_SIZE = 2000;
  * @returns An array of resource URL strings.
  */
 export async function getResourceUrlList(knex, onProgress) {
-    // MAX() over the keyset column is an O(1) index-tail read; only fetched
-    // when someone is listening.
-    let maxId = 0;
-    if (onProgress) {
-        const [maxRow] = await knex('resource_items').max({
-            max: 'id',
-        });
-        maxId = maxRow?.max ?? 0;
-    }
-    const urls = [];
-    let lastId = 0;
-    for (;;) {
-        const rows = (await knex('resource_items')
-            .join('url_refs', 'url_refs.id', 'resource_items.url_id')
-            .where('resource_items.id', '>', lastId)
-            .orderBy('resource_items.id', 'asc')
-            .limit(READ_CHUNK_SIZE)
-            .select('resource_items.id as id', 'url_refs.url as url'));
-        if (rows.length === 0) {
-            onProgress?.(maxId, maxId);
-            break;
-        }
-        lastId = rows.at(-1).id;
-        for (const row of rows) {
-            urls.push(row.url);
-        }
-        onProgress?.(Math.min(lastId, maxId), maxId);
-    }
-    return urls;
+    return keysetPaginateById(knex, 'resource_items', (lastId) => knex('resource_items')
+        .join('url_refs', 'url_refs.id', 'resource_items.url_id')
+        .where('resource_items.id', '>', lastId)
+        .orderBy('resource_items.id', 'asc')
+        .limit(READ_CHUNK_SIZE)
+        .select('resource_items.id as id', 'url_refs.url as url'), (row) => row.url, onProgress);
 }

package/lib/archive/init-schema.d.ts

@@ -30,7 +30,7 @@ export declare function applyConnectionPragmas(instance: Knex): Promise<void>;
  *   writes during a crawl and every reader queries. Must run AFTER
  *   `createRefTables` because most entity tables reference ref-table PKs.
  * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
- *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `inventory_runs`,
+ *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `list_reconcile_runs`,
  *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
  *   `page_html_ref`. Must run AFTER `createEntityTables` because the
  *   page-scoped tables FK into `content_items(id)`.

package/lib/archive/init-schema.js

@@ -55,7 +55,7 @@ export async function applyConnectionPragmas(instance) {
  *   writes during a crawl and every reader queries. Must run AFTER
  *   `createRefTables` because most entity tables reference ref-table PKs.
  * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
- *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `inventory_runs`,
+ *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `list_reconcile_runs`,
  *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
  *   `page_html_ref`. Must run AFTER `createEntityTables` because the
  *   page-scoped tables FK into `content_items(id)`.
@@ -113,6 +113,7 @@ export async function initSchema(instance) {
             t.string('userAgent');
             t.boolean('ignoreRobots');
             t.string('mainContentSelector');
+            t.string('createdCwd');
         });
     }
     // 0.13 ref / header dictionary tables. DDL + column-level rationale
@@ -135,7 +136,7 @@ export async function initSchema(instance) {
     // Adjunct tables that FK into `content_items` (page_errors /
     // technology_signals / page_technologies / page_jsonld / analysis_* /
     // page_html_*) plus the standalone log tables
-    // (crawl_errors / inventory_runs). MUST run after
+    // (crawl_errors / list_reconcile_runs). MUST run after
     // {@link createEntityTables} so the FK targets exist. DDL +
     // column-level rationale lives in {@link createAdjunctTables}, which is
     // shared with `scripts/migrate-to-0.13.mjs` — a divergence between the

package/lib/archive/meta/compute-main-contents-denormalized.d.ts

@@ -1,4 +1,4 @@
-import type { MainContentsDenormalizedColumns } from './types.js';
+import type { ImageScanColumns, MainContentsDenormalizedColumns } from './types.js';
 import type { MainContentsData, ScrollHeightData } from '@d-zero/beholder';
 /**
  * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
@@ -23,11 +23,19 @@ import type { MainContentsData, ScrollHeightData } from '@d-zero/beholder';
  * `undefined`/capture-not-attempted and capture-failure both collapse to
  * `null` (unknown — NOT the same as "captured, zero found"), while any
  * number (including `0`) means capture succeeded.
+ * `imageScan` follows the same independent-of-`mainContents` shape as
+ * `scrollHeight` (both are per-viewport beholder scan outcomes, not derived
+ * from the main-content region) — it is typed as a plain `number | null`
+ * pair here rather than beholder's `ImageScanCode` so this function does not
+ * require a `@d-zero/beholder` version that exports it; the caller narrows.
  * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
  * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
  * @param customElementCount - Count of Web Components nitpicker captured in
  *   the main-content region, or `null`/`undefined` when capture was not
  *   attempted or failed.
- * @returns The eighteen denormalised columns.
+ * @param imageScan - Beholder's per-viewport `<img>` scan outcome codes, or
+ *   `null`/`undefined` when not attempted (page not fully rendered, or the
+ *   scraper version does not report it yet).
+ * @returns The twenty denormalised columns.
  */
-export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined, customElementCount?: number | null): MainContentsDenormalizedColumns;
+export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined, customElementCount?: number | null, imageScan?: ImageScanColumns | null): MainContentsDenormalizedColumns;

package/lib/archive/meta/compute-main-contents-denormalized.js

@@ -21,14 +21,22 @@
  * `undefined`/capture-not-attempted and capture-failure both collapse to
  * `null` (unknown — NOT the same as "captured, zero found"), while any
  * number (including `0`) means capture succeeded.
+ * `imageScan` follows the same independent-of-`mainContents` shape as
+ * `scrollHeight` (both are per-viewport beholder scan outcomes, not derived
+ * from the main-content region) — it is typed as a plain `number | null`
+ * pair here rather than beholder's `ImageScanCode` so this function does not
+ * require a `@d-zero/beholder` version that exports it; the caller narrows.
  * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
  * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
  * @param customElementCount - Count of Web Components nitpicker captured in
  *   the main-content region, or `null`/`undefined` when capture was not
  *   attempted or failed.
- * @returns The eighteen denormalised columns.
+ * @param imageScan - Beholder's per-viewport `<img>` scan outcome codes, or
+ *   `null`/`undefined` when not attempted (page not fully rendered, or the
+ *   scraper version does not report it yet).
+ * @returns The twenty denormalised columns.
  */
-export function computeMainContentsDenormalized(mainContents, scrollHeight, customElementCount) {
+export function computeMainContentsDenormalized(mainContents, scrollHeight, customElementCount, imageScan) {
     if (mainContents == null) {
         return {
             main_content_node_name: null,
@@ -49,6 +57,8 @@ export function computeMainContentsDenormalized(mainContents, scrollHeight, cust
             main_content_custom_element_count: null,
             scroll_height_desktop: null,
             scroll_height_mobile: null,
+            image_scan_desktop: null,
+            image_scan_mobile: null,
         };
     }
     return {
@@ -72,5 +82,7 @@ export function computeMainContentsDenormalized(mainContents, scrollHeight, cust
         main_content_custom_element_count: customElementCount ?? null,
         scroll_height_desktop: scrollHeight?.desktop ?? null,
         scroll_height_mobile: scrollHeight?.mobile ?? null,
+        image_scan_desktop: imageScan?.desktop ?? null,
+        image_scan_mobile: imageScan?.mobile ?? null,
     };
 }