@nitpicker/crawler 0.18.0 → 0.19.0

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -62,9 +62,14 @@ import { listNetworkOutages } from '../../outages/list-network-outages.js';
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to reset (issue
+ *   #294: a large `--retry-failed` can reset thousands of pages across 13
+ *   tables, running for seconds to minutes with no other signal it hasn't
+ *   hung). Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were reset to pending.
  */
-export async function resetFailedPages(knex) {
+export async function resetFailedPages(knex, onProgress) {
     const candidates = await knex('content_items')
         .join('url_refs', 'content_items.url_id', 'url_refs.id')
         .select('content_items.id as id', 'url_refs.url as url')
@@ -146,7 +151,8 @@ export async function resetFailedPages(knex) {
         await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
         await knex('page_errors').whereIn('pageId', chunk).delete();
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
-        await knex('page_tags').whereIn('pageId', chunk).delete();
+        await knex('technology_signals').whereIn('pageId', chunk).delete();
+        await knex('page_technologies').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
         await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
         await knex('page_main_content_images').whereIn('pageId', chunk).delete();
@@ -156,6 +162,8 @@ export async function resetFailedPages(knex) {
         await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
         await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
         await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_custom_elements').whereIn('pageId', chunk).delete();
+        onProgress?.(Math.min(i + chunkSize, ids.length), ids.length);
     }
     dbLog('Reset %d failed pages back to pending', urls.length);
     return urls;

package/lib/archive/db-ops/pages/write/insert-custom-elements.d.ts

@@ -0,0 +1,27 @@
+import type { MainContentCustomElementCandidate } from '../../../../crawler/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_custom_elements` rows with the
+ * freshly captured set. Called inside `updatePage`'s transaction, gated on
+ * `page.mainContentCustomElements !== undefined` by the caller — the caller
+ * must NOT call this with a defaulted `[]` when the capture itself failed
+ * (`undefined`), since this function cannot tell "capture found nothing"
+ * apart from "capture didn't run" once it receives a plain array.
+ *
+ * Unlike `insertButtons` / `insertHeadings` etc. (whose empty-array case is
+ * genuinely ambiguous — beholder's `MainContentsData` gives no way to tell
+ * "successfully found zero" from "degraded scrape" — this function always
+ * deletes existing rows once called, including for an empty array: because
+ * the caller already resolved that ambiguity via the `!== undefined` gate,
+ * an empty array here unambiguously means "capture succeeded, found none,"
+ * so stale rows from a previous crawl must be cleared.
+ *
+ * Unlike `insertButtons`, the input is nitpicker's own
+ * `MainContentCustomElementCandidate[]` (from `capture-custom-elements.ts`),
+ * not a slice of beholder's `MainContentsData` — there is no such category
+ * in `MainContentsData`.
+ * @param pageId - The owning `content_items.id`.
+ * @param customElements - The captured custom elements, in document order.
+ * @param trx - The active transaction.
+ */
+export declare function insertCustomElements(pageId: number, customElements: readonly MainContentCustomElementCandidate[], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-custom-elements.js

@@ -0,0 +1,40 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_custom_elements` rows with the
+ * freshly captured set. Called inside `updatePage`'s transaction, gated on
+ * `page.mainContentCustomElements !== undefined` by the caller — the caller
+ * must NOT call this with a defaulted `[]` when the capture itself failed
+ * (`undefined`), since this function cannot tell "capture found nothing"
+ * apart from "capture didn't run" once it receives a plain array.
+ *
+ * Unlike `insertButtons` / `insertHeadings` etc. (whose empty-array case is
+ * genuinely ambiguous — beholder's `MainContentsData` gives no way to tell
+ * "successfully found zero" from "degraded scrape" — this function always
+ * deletes existing rows once called, including for an empty array: because
+ * the caller already resolved that ambiguity via the `!== undefined` gate,
+ * an empty array here unambiguously means "capture succeeded, found none,"
+ * so stale rows from a previous crawl must be cleared.
+ *
+ * Unlike `insertButtons`, the input is nitpicker's own
+ * `MainContentCustomElementCandidate[]` (from `capture-custom-elements.ts`),
+ * not a slice of beholder's `MainContentsData` — there is no such category
+ * in `MainContentsData`.
+ * @param pageId - The owning `content_items.id`.
+ * @param customElements - The captured custom elements, in document order.
+ * @param trx - The active transaction.
+ */
+export async function insertCustomElements(pageId, customElements, trx) {
+    await trx('page_main_content_custom_elements').where('pageId', pageId).delete();
+    if (customElements.length === 0)
+        return;
+    const rows = customElements.map((el, order) => ({
+        pageId,
+        order,
+        nodeName: el.nodeName,
+        elementId: el.elementId,
+        classList: JSON.stringify(el.classList),
+    }));
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_custom_elements').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-page.d.ts

@@ -1,5 +1,4 @@
-import type { PageData } from '../../../../utils/types/types.js';
-import type { PageSource } from '../../../types.js';
+import type { PageDataWithDomPaths, PageSource } from '../../../types.js';
 import type { WriteRefCaches } from '../../_shared/types.js';
 import type { Knex } from 'knex';
 /**
@@ -15,6 +14,26 @@ import type { Knex } from 'knex';
  * anything reachable via the crawled chain must be labelled `'crawled'`
  * even if previously labelled `'inventory-*'`.
  *
+ * `is_external`, by contrast, IS overwritten on every call — but demotion is
+ * guarded. `updatePage` keys the row by the redirect DESTINATION url while
+ * passing the REQUESTING url's `isExternal`, so the value written here
+ * describes the requester, not necessarily this row. Inheriting it is
+ * deliberate when PROMOTING (an out-of-scope soft-404 page reached from an
+ * in-scope request counts as covered by the crawl, and the viewer relies on
+ * that — see `@nitpicker/query`'s `build-directory-tree-rows.ts`). It would be
+ * wrong when DEMOTING: an out-of-scope url redirecting to an in-scope page
+ * that was already taken on as a target must not flip that page to
+ * `is_external = 1` — no reading of the column justifies erasing a real
+ * observation with an inherited one. `crawler.ts`'s `#scrapedDestinations`
+ * blocks this within one run, but that is per-`#runDeal` memory, so a later
+ * `--append` / `--retry-failed` process starts blind to what the DB already
+ * knows — hence the CASE below, which checks the ROW's own prior state
+ * instead: once `scraped = 1 AND is_external = 0` is true, no later call can
+ * flip it back to `1`. Promotion (`0 → 1` before the row has been scraped, or
+ * `1 → 0` at any time) is untouched. This mirrors how `first_crawled_at`'s `COALESCE` below
+ * protects an established value — deliberately NOT by re-deriving scope from
+ * the destination url, which would also kill the wanted promoting case.
+ *
  * The page's response headers are decomposed and written into the
  * header dictionary tables here — per response, not deferred to
  * crawl-end — and the resulting `header_set_id` lands on the same
@@ -26,11 +45,12 @@ import type { Knex } from 'knex';
  * @param knex - Knex query builder connected to the archive DB. Used as the
  *   fallback when `trx` is not provided.
  * @param caches - The connection's write-side id caches.
- * @param page - The scraped page data.
+ * @param page - The scraped page data, optionally carrying nitpicker's own
+ *   in-browser custom-element capture (see {@link PageDataWithDomPaths}).
  * @param isTarget - Whether this page is a crawl target.
  * @param trx - Optional transaction all statements run through.
  * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE
  *   except through the priority lattice described above.
  * @returns The `content_items.id` of the inserted or updated row.
  */
-export declare function insertPage(knex: Knex, caches: WriteRefCaches, page: PageData, isTarget: boolean, trx?: Knex.Transaction, source?: PageSource): Promise<number>;
+export declare function insertPage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, isTarget: boolean, trx?: Knex.Transaction, source?: PageSource): Promise<number>;

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -24,6 +24,26 @@ import { upsertUrlRef } from '../../_shared/upsert-url-ref.js';
  * anything reachable via the crawled chain must be labelled `'crawled'`
  * even if previously labelled `'inventory-*'`.
  *
+ * `is_external`, by contrast, IS overwritten on every call — but demotion is
+ * guarded. `updatePage` keys the row by the redirect DESTINATION url while
+ * passing the REQUESTING url's `isExternal`, so the value written here
+ * describes the requester, not necessarily this row. Inheriting it is
+ * deliberate when PROMOTING (an out-of-scope soft-404 page reached from an
+ * in-scope request counts as covered by the crawl, and the viewer relies on
+ * that — see `@nitpicker/query`'s `build-directory-tree-rows.ts`). It would be
+ * wrong when DEMOTING: an out-of-scope url redirecting to an in-scope page
+ * that was already taken on as a target must not flip that page to
+ * `is_external = 1` — no reading of the column justifies erasing a real
+ * observation with an inherited one. `crawler.ts`'s `#scrapedDestinations`
+ * blocks this within one run, but that is per-`#runDeal` memory, so a later
+ * `--append` / `--retry-failed` process starts blind to what the DB already
+ * knows — hence the CASE below, which checks the ROW's own prior state
+ * instead: once `scraped = 1 AND is_external = 0` is true, no later call can
+ * flip it back to `1`. Promotion (`0 → 1` before the row has been scraped, or
+ * `1 → 0` at any time) is untouched. This mirrors how `first_crawled_at`'s `COALESCE` below
+ * protects an established value — deliberately NOT by re-deriving scope from
+ * the destination url, which would also kill the wanted promoting case.
+ *
  * The page's response headers are decomposed and written into the
  * header dictionary tables here — per response, not deferred to
  * crawl-end — and the resulting `header_set_id` lands on the same
@@ -35,7 +55,8 @@ import { upsertUrlRef } from '../../_shared/upsert-url-ref.js';
  * @param knex - Knex query builder connected to the archive DB. Used as the
  *   fallback when `trx` is not provided.
  * @param caches - The connection's write-side id caches.
- * @param page - The scraped page data.
+ * @param page - The scraped page data, optionally carrying nitpicker's own
+ *   in-browser custom-element capture (see {@link PageDataWithDomPaths}).
  * @param isTarget - Whether this page is a crawl target.
  * @param trx - Optional transaction all statements run through.
  * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE
@@ -47,7 +68,7 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
     const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, undefined, source);
     const flat = deriveFlatFromMeta(page.meta, page.url.href);
     const denorm = computePageDenormalized(page.meta);
-    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight);
+    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight, page.mainContentCustomElements?.length);
     const extras = deriveMetaExtras(page.meta);
     const now = Date.now();
     // Canonicalize so the stored dictionary value matches the exact-string
@@ -86,8 +107,23 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
         .where('id', pageId)
         .update({
         scraped: 1,
-        is_target: isTarget ? 1 : 0,
-        is_external: page.isExternal ? 1 : 0,
+        // Once a row has been scraped as a real crawl target, no later call
+        // may flip it back off — same inheritance-from-the-requester bug as
+        // `is_external` below (`setExternalPage` always passes `isTarget:
+        // false`), and the same fix: guard on the row's own prior state
+        // instead of per-run memory. Demoting is_target away from an
+        // established value would under-count `getScrapedHtmlPageCount`'s
+        // resume offset and silently break the "isTarget=1 means covered by
+        // the crawl" contract `accessor.getPages('page')` documents.
+        is_target: qb.raw('CASE WHEN scraped = 1 AND is_target = 1 THEN 1 ELSE ? END', [
+            isTarget ? 1 : 0,
+        ]),
+        // Once a row has been scraped as internal, no later call may flip it
+        // back to external — see this function's docs for why the write this
+        // guards against happens at all.
+        is_external: qb.raw('CASE WHEN scraped = 1 AND is_external = 0 THEN 0 ELSE ? END', [
+            page.isExternal ? 1 : 0,
+        ]),
         status: page.status,
         status_text: page.statusText,
         content_type_id: contentTypeId,

package/lib/archive/db-ops/pages/write/insert-technologies.d.ts

@@ -0,0 +1,25 @@
+import type { PageData } from '../../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `technology_signals` + `page_technologies` rows with
+ * the freshly extracted set. Called inside `updatePage`'s transaction
+ * unconditionally — the `wappalyzer` signal source draws on `<script src>` /
+ * `<iframe src>` / window globals / response headers, not the HTML body, so
+ * external pages that skip rendering still contribute technology signals
+ * (structural signals naturally find nothing against an empty `html`
+ * string, which is exactly what a skipped render leaves it as).
+ *
+ * Direct replacement for `insertTags` in the live crawl write path (that
+ * file is untouched — `scripts/migrate-to-0.10.mjs` still depends on it).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does
+ * not wipe prior rows on a degraded re-scrape. Both tables are always
+ * written together in this one call — see ARCHITECTURE.md's invariant
+ * that `technology_signals` and `page_technologies` are never updated
+ * independently.
+ * @param pageId
+ * @param html - The page's raw HTML string (empty for external/metadata-only scrapes).
+ * @param meta
+ * @param trx
+ */
+export declare function insertTechnologies(pageId: number, html: string, meta: PageData['meta'], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-technologies.js

@@ -0,0 +1,45 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { extractTechnologiesForArchive } from '../../../meta/technologies/extract-technologies-for-archive.js';
+/**
+ * Replaces the page's `technology_signals` + `page_technologies` rows with
+ * the freshly extracted set. Called inside `updatePage`'s transaction
+ * unconditionally — the `wappalyzer` signal source draws on `<script src>` /
+ * `<iframe src>` / window globals / response headers, not the HTML body, so
+ * external pages that skip rendering still contribute technology signals
+ * (structural signals naturally find nothing against an empty `html`
+ * string, which is exactly what a skipped render leaves it as).
+ *
+ * Direct replacement for `insertTags` in the live crawl write path (that
+ * file is untouched — `scripts/migrate-to-0.10.mjs` still depends on it).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does
+ * not wipe prior rows on a degraded re-scrape. Both tables are always
+ * written together in this one call — see ARCHITECTURE.md's invariant
+ * that `technology_signals` and `page_technologies` are never updated
+ * independently.
+ * @param pageId
+ * @param html - The page's raw HTML string (empty for external/metadata-only scrapes).
+ * @param meta
+ * @param trx
+ */
+export async function insertTechnologies(pageId, html, meta, trx) {
+    const { signals, technologies } = extractTechnologiesForArchive(html, meta);
+    if (signals.length === 0)
+        return;
+    await trx('technology_signals').where('pageId', pageId).delete();
+    await trx('page_technologies').where('pageId', pageId).delete();
+    await eachSplitted(signals, 100, async (chunk) => {
+        await trx('technology_signals').insert(chunk.map((s) => ({
+            pageId,
+            technology: s.technology,
+            signalType: s.signalType,
+            evidence: s.evidence,
+            weight: s.weight,
+        })));
+    });
+    if (technologies.length > 0) {
+        await eachSplitted(technologies, 100, async (chunk) => {
+            await trx('page_technologies').insert(chunk.map((t) => ({ pageId, ...t })));
+        });
+    }
+}

package/lib/archive/db-ops/pages/write/link-redirect-sources.d.ts

@@ -20,7 +20,13 @@ import type { Knex } from 'knex';
  * @param destId - `content_items.id` of the redirect destination page.
  * @param destUrlNormalized - Normalised destination URL, used to detect and
  *   skip self-redirects.
- * @param isExternal - Whether the sources are external to the crawl scope.
+ * @param isExternal - Whether the sources are external to the crawl scope, as
+ *   a single value applied to every hop in `sources` — the same
+ *   requester-describes-a-different-row shape `insertPage` guards against
+ *   (see its docs in `insert-page.ts`), since a hop can independently be a
+ *   real, already-scraped internal page in its own right (reached earlier as
+ *   a direct crawl target) before it is later observed as an intermediate hop
+ *   in some other chain. The UPDATE below carries the same CASE guard.
  * @param chainLineageSource - Lineage label propagated to each intermediate
  *   hop's row (passed through to {@link resolveContentItemId}). Derived by
  *   the caller from the **originating** page's source (`page.url`), not from

package/lib/archive/db-ops/pages/write/link-redirect-sources.js

@@ -19,7 +19,13 @@ import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
  * @param destId - `content_items.id` of the redirect destination page.
  * @param destUrlNormalized - Normalised destination URL, used to detect and
  *   skip self-redirects.
- * @param isExternal - Whether the sources are external to the crawl scope.
+ * @param isExternal - Whether the sources are external to the crawl scope, as
+ *   a single value applied to every hop in `sources` — the same
+ *   requester-describes-a-different-row shape `insertPage` guards against
+ *   (see its docs in `insert-page.ts`), since a hop can independently be a
+ *   real, already-scraped internal page in its own right (reached earlier as
+ *   a direct crawl target) before it is later observed as an intermediate hop
+ *   in some other chain. The UPDATE below carries the same CASE guard.
  * @param chainLineageSource - Lineage label propagated to each intermediate
  *   hop's row (passed through to {@link resolveContentItemId}). Derived by
  *   the caller from the **originating** page's source (`page.url`), not from
@@ -53,7 +59,7 @@ export async function linkRedirectSources(trx, caches, sources, destId, destUrlN
             .update({
             scraped: 1,
             redirect_dest_id: destId,
-            is_external: isExternal ? 1 : 0,
+            is_external: trx.raw('CASE WHEN scraped = 1 AND is_external = 0 THEN 0 ELSE ? END', [isExternal ? 1 : 0]),
         });
         // Conditional `301 Moved Permanently` stamp — applied ONLY when the
         // row carries no definitive status yet (NULL or the `-1`

package/lib/archive/db-ops/pages/write/update-page.d.ts

@@ -24,6 +24,9 @@ import type { Knex } from 'knex';
  *   inserted. Existing rows keep their original `source` (this is why a
  *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
  *   that was discovered earlier).
+ * @param bodyHash - Precomputed body hash for the page's HTML (see
+ *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+ *   computing it from the HTML instead.
  * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
  */
-export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource): Promise<number>;
+export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null): Promise<number>;

package/lib/archive/db-ops/pages/write/update-page.js

@@ -13,13 +13,14 @@ import { resolveUrlOrBlob } from '../../_shared/resolve-url-or-blob.js';
 import { insertAudios } from './insert-audios.js';
 import { insertButtons } from './insert-buttons.js';
 import { insertCanvases } from './insert-canvases.js';
+import { insertCustomElements } from './insert-custom-elements.js';
 import { insertHeadings } from './insert-headings.js';
 import { insertIframes } from './insert-iframes.js';
 import { insertJsonLd } from './insert-jsonld.js';
 import { insertMainContentImages } from './insert-main-content-images.js';
 import { insertMainContentTables } from './insert-main-content-tables.js';
 import { insertPage } from './insert-page.js';
-import { insertTags } from './insert-tags.js';
+import { insertTechnologies } from './insert-technologies.js';
 import { insertVideos } from './insert-videos.js';
 import { linkRedirectSources } from './link-redirect-sources.js';
 import { writePageHtmlBlob } from './write-page-html-blob.js';
@@ -46,9 +47,12 @@ import { writePageHtmlBlob } from './write-page-html-blob.js';
  *   inserted. Existing rows keep their original `source` (this is why a
  *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
  *   that was discovered earlier).
+ * @param bodyHash - Precomputed body hash for the page's HTML (see
+ *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+ *   computing it from the HTML instead.
  * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
  */
-export async function updatePage(knex, caches, page, writeHtml, isTarget, source) {
+export async function updatePage(knex, caches, page, writeHtml, isTarget, source, bodyHash) {
     const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
     const destUrlObject = parseUrl(destUrl);
     if (!destUrlObject) {
@@ -56,7 +60,7 @@ export async function updatePage(knex, caches, page, writeHtml, isTarget, source
     }
     try {
         return await knex.transaction(async (trx) => {
-            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source);
+            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash);
         });
     }
     catch (error) {
@@ -83,21 +87,24 @@ export async function updatePage(knex, caches, page, writeHtml, isTarget, source
  * @param writeHtml - See {@link updatePage}.
  * @param isTarget - See {@link updatePage}.
  * @param source - See {@link updatePage}.
+ * @param bodyHash - See {@link updatePage}.
  * @returns The `content_items.id` of the inserted or updated row.
  */
-async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source) {
+async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash) {
     const pageId = await insertPage(knex, caches, {
         ...page,
         url: destUrlObject,
     }, isTarget, trx, source);
-    // Wappalyzer tag detection is HTML-body independent (relies on
-    // `<script src>` / `<iframe src>` / window globals / response
-    // headers) so it runs for every page including external /
-    // metadata-only. JSON-LD on the other hand lives inside the
-    // rendered HTML body, so we only write it when there is HTML to
-    // scrape — see the same `writeHtml` gate as `writePageHtmlBlob`
-    // below.
-    await insertTags(pageId, page.meta, trx);
+    // Technology detection combines a Wappalyzer signal source (HTML-body
+    // independent — relies on `<script src>` / `<iframe src>` / window
+    // globals / response headers, so it fires for every page including
+    // external / metadata-only) with structural signals that scan
+    // `page.html` (naturally a no-op against the empty string an
+    // external/metadata-only scrape leaves it as). Both run unconditionally,
+    // unlike JSON-LD below, which lives inside the rendered HTML body and
+    // is only written when there is HTML to scrape — see the same
+    // `writeHtml` gate as `writePageHtmlBlob` below.
+    await insertTechnologies(pageId, page.html, page.meta, trx);
     if (writeHtml) {
         await insertJsonLd(pageId, page.meta, trx);
     }
@@ -113,6 +120,16 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
         await insertVideos(pageId, page.mainContents, trx);
         await insertAudios(pageId, page.mainContents, trx);
         await insertCanvases(pageId, page.mainContents, trx);
+        // Unlike the eight calls above, the source here is nitpicker's own
+        // `capture-custom-elements.ts` capture, not `page.mainContents` — see
+        // `insert-custom-elements.ts`. That capture can fail independently
+        // of `page.mainContents` (a separate `page.evaluate()` call), so
+        // `undefined` (capture failed) must skip the call entirely — passing
+        // a defaulted `[]` would look identical to "found zero" and, per
+        // `insertCustomElements`'s docs, wipe stale-but-still-good rows.
+        if (page.mainContentCustomElements !== undefined) {
+            await insertCustomElements(pageId, page.mainContentCustomElements, trx);
+        }
     }
     // Chain lineage propagates FROM the originating URL
     // (`page.url`), NOT from the destination. See the matching
@@ -142,12 +159,17 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     // content check alone expresses the intent without a redundant term.
     if (writeHtml && page.html.length > 0) {
         await writePageHtmlBlob(pageId, page.html, trx);
-        // Computed from the same `page.html` written above, in the same
-        // transaction, so `page_meta.body_hash` never observably lags the
-        // snapshot it was derived from.
+        // Prefer the caller's precomputed hash (the crawler already hashed
+        // this exact `page.html` once, before this write path ever runs) —
+        // falling back to computing it here only when the caller has not
+        // (direct callers such as tests, or a future write path that has not
+        // adopted the precompute). Either way this stays derived from the
+        // same `page.html` written above, in the same transaction, so
+        // `page_meta.body_hash` never observably lags the snapshot it was
+        // derived from.
         await trx('page_meta')
             .where('page_id', pageId)
-            .update({ body_hash: computeBodyHash(page.html) });
+            .update({ body_hash: bodyHash ?? computeBodyHash(page.html) });
     }
     else if (writeHtml &&
         page.contentType !== null &&

package/lib/archive/db-ops/resources/get-resource-url-list.d.ts

@@ -3,7 +3,18 @@ import type { Knex } from 'knex';
  * Retrieves a flat list of all resource URLs from the `resource_items`
  * table. URL text is normalised into `url_refs`, so the read joins the
  * two tables and returns the resolved strings.
+ *
+ * Read in `resource_items.id`-keyset chunks rather than a single SELECT
+ * (issue #294): on a resource-heavy archive (images/JS/CSS in the tens or
+ * hundreds of thousands) this was one unbounded, multi-second-to-minutes
+ * query with no way to report progress mid-scan. The accumulated result is
+ * identical to the previous single-SELECT read — chunking exists purely to
+ * make the scan observable, not to bound memory (the flat URL list is
+ * already fully materialised for the caller either way).
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk with the `resource_items.id`
+ *   scanned up to so far and the max id. Omit for no reporting (the
+ *   default; e.g. tests).
  * @returns An array of resource URL strings.
  */
-export declare function getResourceUrlList(knex: Knex): Promise<string[]>;
+export declare function getResourceUrlList(knex: Knex, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<string[]>;

package/lib/archive/db-ops/resources/get-resource-url-list.js

@@ -1,13 +1,51 @@
+/** Rows read per `resource_items.id` keyset chunk. */
+const READ_CHUNK_SIZE = 2000;
 /**
  * Retrieves a flat list of all resource URLs from the `resource_items`
  * table. URL text is normalised into `url_refs`, so the read joins the
  * two tables and returns the resolved strings.
+ *
+ * Read in `resource_items.id`-keyset chunks rather than a single SELECT
+ * (issue #294): on a resource-heavy archive (images/JS/CSS in the tens or
+ * hundreds of thousands) this was one unbounded, multi-second-to-minutes
+ * query with no way to report progress mid-scan. The accumulated result is
+ * identical to the previous single-SELECT read — chunking exists purely to
+ * make the scan observable, not to bound memory (the flat URL list is
+ * already fully materialised for the caller either way).
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk with the `resource_items.id`
+ *   scanned up to so far and the max id. Omit for no reporting (the
+ *   default; e.g. tests).
  * @returns An array of resource URL strings.
  */
-export async function getResourceUrlList(knex) {
-    const res = await knex('resource_items')
-        .join('url_refs', 'url_refs.id', 'resource_items.url_id')
-        .select('url_refs.url as url');
-    return res.map((r) => r.url);
+export async function getResourceUrlList(knex, onProgress) {
+    // MAX() over the keyset column is an O(1) index-tail read; only fetched
+    // when someone is listening.
+    let maxId = 0;
+    if (onProgress) {
+        const [maxRow] = await knex('resource_items').max({
+            max: 'id',
+        });
+        maxId = maxRow?.max ?? 0;
+    }
+    const urls = [];
+    let lastId = 0;
+    for (;;) {
+        const rows = (await knex('resource_items')
+            .join('url_refs', 'url_refs.id', 'resource_items.url_id')
+            .where('resource_items.id', '>', lastId)
+            .orderBy('resource_items.id', 'asc')
+            .limit(READ_CHUNK_SIZE)
+            .select('resource_items.id as id', 'url_refs.url as url'));
+        if (rows.length === 0) {
+            onProgress?.(maxId, maxId);
+            break;
+        }
+        lastId = rows.at(-1).id;
+        for (const row of rows) {
+            urls.push(row.url);
+        }
+        onProgress?.(Math.min(lastId, maxId), maxId);
+    }
+    return urls;
 }

package/lib/archive/filesystem/copy-file-with-progress.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Copies a file while reporting byte progress — `fs.copyFile` offers no
+ * observability, and a `.nitpicker` archive's `.bak` safety copy (taken
+ * before every mutating `viewer-build`/`crawl --append`/`--inventory`/
+ * `--retry-failed` run, and copied back on failure) can be 15 GB+, running
+ * for tens of seconds with nothing on screen (issue #294). Byte granularity
+ * is the read-stream chunk size (~64 KB); callers wanting coarser updates
+ * throttle in their own callback.
+ * @param src - The file to copy.
+ * @param dest - The destination path, overwritten if present.
+ * @param onProgress - Called as bytes are copied, with the bytes copied so
+ *   far and the source file's total size. Omit for a silent copy.
+ * @example
+ * ```ts
+ * await copyFileWithProgress(archivePath, `${archivePath}.bak`, (copied, total) => {
+ *   console.error(`${copied}/${total}`);
+ * });
+ * ```
+ */
+export declare function copyFileWithProgress(src: string, dest: string, onProgress?: (copiedBytes: number, totalBytes: number) => void): Promise<void>;

package/lib/archive/filesystem/copy-file-with-progress.js

@@ -0,0 +1,34 @@
+import { createReadStream, createWriteStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
+/**
+ * Copies a file while reporting byte progress — `fs.copyFile` offers no
+ * observability, and a `.nitpicker` archive's `.bak` safety copy (taken
+ * before every mutating `viewer-build`/`crawl --append`/`--inventory`/
+ * `--retry-failed` run, and copied back on failure) can be 15 GB+, running
+ * for tens of seconds with nothing on screen (issue #294). Byte granularity
+ * is the read-stream chunk size (~64 KB); callers wanting coarser updates
+ * throttle in their own callback.
+ * @param src - The file to copy.
+ * @param dest - The destination path, overwritten if present.
+ * @param onProgress - Called as bytes are copied, with the bytes copied so
+ *   far and the source file's total size. Omit for a silent copy.
+ * @example
+ * ```ts
+ * await copyFileWithProgress(archivePath, `${archivePath}.bak`, (copied, total) => {
+ *   console.error(`${copied}/${total}`);
+ * });
+ * ```
+ */
+export async function copyFileWithProgress(src, dest, onProgress) {
+    const { size: totalBytes } = await stat(src);
+    const source = createReadStream(src);
+    let copiedBytes = 0;
+    if (onProgress) {
+        source.on('data', (chunk) => {
+            copiedBytes += chunk.length;
+            onProgress(copiedBytes, totalBytes);
+        });
+    }
+    await pipeline(source, createWriteStream(dest));
+}

package/lib/archive/filesystem/parse-pax-path.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Parses a PAX extended header record block for the `path` key.
+ *
+ * Format: a sequence of `"<record-length> <key>=<value>\n"` records, where
+ * `<record-length>` is the decimal byte length of the ENTIRE record
+ * (including the length prefix itself and the trailing newline). `tar`
+ * (node-tar) emits one of these ahead of any entry whose name doesn't fit
+ * the ustar format's 100-byte field (UTF-8 multi-byte names, or names
+ * longer than 100 bytes) — verified empirically against this package's own
+ * `tar()`/`create()` output.
+ * @param data - The PAX header entry's raw data payload.
+ * @returns The `path` value if present, otherwise `null`.
+ * @example
+ * ```ts
+ * parsePaxPath(Buffer.from('20 path=日本語/\n', 'utf8')); // '日本語/'
+ * ```
+ */
+export declare function parsePaxPath(data: Buffer): string | null;