@nitpicker/crawler 0.18.1 → 0.19.0

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -62,9 +62,14 @@ import { listNetworkOutages } from '../../outages/list-network-outages.js';
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to reset (issue
+ *   #294: a large `--retry-failed` can reset thousands of pages across 13
+ *   tables, running for seconds to minutes with no other signal it hasn't
+ *   hung). Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were reset to pending.
  */
-export async function resetFailedPages(knex) {
+export async function resetFailedPages(knex, onProgress) {
     const candidates = await knex('content_items')
         .join('url_refs', 'content_items.url_id', 'url_refs.id')
         .select('content_items.id as id', 'url_refs.url as url')
@@ -146,7 +151,8 @@ export async function resetFailedPages(knex) {
         await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
         await knex('page_errors').whereIn('pageId', chunk).delete();
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
-        await knex('page_tags').whereIn('pageId', chunk).delete();
+        await knex('technology_signals').whereIn('pageId', chunk).delete();
+        await knex('page_technologies').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
         await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
         await knex('page_main_content_images').whereIn('pageId', chunk).delete();
@@ -156,6 +162,8 @@ export async function resetFailedPages(knex) {
         await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
         await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
         await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_custom_elements').whereIn('pageId', chunk).delete();
+        onProgress?.(Math.min(i + chunkSize, ids.length), ids.length);
     }
     dbLog('Reset %d failed pages back to pending', urls.length);
     return urls;

package/lib/archive/db-ops/pages/write/insert-custom-elements.d.ts

@@ -0,0 +1,27 @@
+import type { MainContentCustomElementCandidate } from '../../../../crawler/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `page_main_content_custom_elements` rows with the
+ * freshly captured set. Called inside `updatePage`'s transaction, gated on
+ * `page.mainContentCustomElements !== undefined` by the caller — the caller
+ * must NOT call this with a defaulted `[]` when the capture itself failed
+ * (`undefined`), since this function cannot tell "capture found nothing"
+ * apart from "capture didn't run" once it receives a plain array.
+ *
+ * Unlike `insertButtons` / `insertHeadings` etc. (whose empty-array case is
+ * genuinely ambiguous — beholder's `MainContentsData` gives no way to tell
+ * "successfully found zero" from "degraded scrape" — this function always
+ * deletes existing rows once called, including for an empty array: because
+ * the caller already resolved that ambiguity via the `!== undefined` gate,
+ * an empty array here unambiguously means "capture succeeded, found none,"
+ * so stale rows from a previous crawl must be cleared.
+ *
+ * Unlike `insertButtons`, the input is nitpicker's own
+ * `MainContentCustomElementCandidate[]` (from `capture-custom-elements.ts`),
+ * not a slice of beholder's `MainContentsData` — there is no such category
+ * in `MainContentsData`.
+ * @param pageId - The owning `content_items.id`.
+ * @param customElements - The captured custom elements, in document order.
+ * @param trx - The active transaction.
+ */
+export declare function insertCustomElements(pageId: number, customElements: readonly MainContentCustomElementCandidate[], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-custom-elements.js

@@ -0,0 +1,40 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Replaces the page's `page_main_content_custom_elements` rows with the
+ * freshly captured set. Called inside `updatePage`'s transaction, gated on
+ * `page.mainContentCustomElements !== undefined` by the caller — the caller
+ * must NOT call this with a defaulted `[]` when the capture itself failed
+ * (`undefined`), since this function cannot tell "capture found nothing"
+ * apart from "capture didn't run" once it receives a plain array.
+ *
+ * Unlike `insertButtons` / `insertHeadings` etc. (whose empty-array case is
+ * genuinely ambiguous — beholder's `MainContentsData` gives no way to tell
+ * "successfully found zero" from "degraded scrape" — this function always
+ * deletes existing rows once called, including for an empty array: because
+ * the caller already resolved that ambiguity via the `!== undefined` gate,
+ * an empty array here unambiguously means "capture succeeded, found none,"
+ * so stale rows from a previous crawl must be cleared.
+ *
+ * Unlike `insertButtons`, the input is nitpicker's own
+ * `MainContentCustomElementCandidate[]` (from `capture-custom-elements.ts`),
+ * not a slice of beholder's `MainContentsData` — there is no such category
+ * in `MainContentsData`.
+ * @param pageId - The owning `content_items.id`.
+ * @param customElements - The captured custom elements, in document order.
+ * @param trx - The active transaction.
+ */
+export async function insertCustomElements(pageId, customElements, trx) {
+    await trx('page_main_content_custom_elements').where('pageId', pageId).delete();
+    if (customElements.length === 0)
+        return;
+    const rows = customElements.map((el, order) => ({
+        pageId,
+        order,
+        nodeName: el.nodeName,
+        elementId: el.elementId,
+        classList: JSON.stringify(el.classList),
+    }));
+    await eachSplitted(rows, 100, async (chunk) => {
+        await trx('page_main_content_custom_elements').insert(chunk);
+    });
+}

package/lib/archive/db-ops/pages/write/insert-page.d.ts

@@ -1,5 +1,4 @@
-import type { PageData } from '../../../../utils/types/types.js';
-import type { PageSource } from '../../../types.js';
+import type { PageDataWithDomPaths, PageSource } from '../../../types.js';
 import type { WriteRefCaches } from '../../_shared/types.js';
 import type { Knex } from 'knex';
 /**
@@ -46,11 +45,12 @@ import type { Knex } from 'knex';
  * @param knex - Knex query builder connected to the archive DB. Used as the
  *   fallback when `trx` is not provided.
  * @param caches - The connection's write-side id caches.
- * @param page - The scraped page data.
+ * @param page - The scraped page data, optionally carrying nitpicker's own
+ *   in-browser custom-element capture (see {@link PageDataWithDomPaths}).
  * @param isTarget - Whether this page is a crawl target.
  * @param trx - Optional transaction all statements run through.
  * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE
  *   except through the priority lattice described above.
  * @returns The `content_items.id` of the inserted or updated row.
  */
-export declare function insertPage(knex: Knex, caches: WriteRefCaches, page: PageData, isTarget: boolean, trx?: Knex.Transaction, source?: PageSource): Promise<number>;
+export declare function insertPage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, isTarget: boolean, trx?: Knex.Transaction, source?: PageSource): Promise<number>;

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -55,7 +55,8 @@ import { upsertUrlRef } from '../../_shared/upsert-url-ref.js';
  * @param knex - Knex query builder connected to the archive DB. Used as the
  *   fallback when `trx` is not provided.
  * @param caches - The connection's write-side id caches.
- * @param page - The scraped page data.
+ * @param page - The scraped page data, optionally carrying nitpicker's own
+ *   in-browser custom-element capture (see {@link PageDataWithDomPaths}).
  * @param isTarget - Whether this page is a crawl target.
  * @param trx - Optional transaction all statements run through.
  * @param source - Inventory provenance for the INSERT path. Ignored on UPDATE
@@ -67,7 +68,7 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
     const pageId = await resolveContentItemId(qb, caches, page.url.withoutHashAndAuth, undefined, source);
     const flat = deriveFlatFromMeta(page.meta, page.url.href);
     const denorm = computePageDenormalized(page.meta);
-    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight);
+    const mainContentsDenorm = computeMainContentsDenormalized(page.mainContents, page.scrollHeight, page.mainContentCustomElements?.length);
     const extras = deriveMetaExtras(page.meta);
     const now = Date.now();
     // Canonicalize so the stored dictionary value matches the exact-string

package/lib/archive/db-ops/pages/write/insert-technologies.d.ts

@@ -0,0 +1,25 @@
+import type { PageData } from '../../../../utils/types/types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the page's `technology_signals` + `page_technologies` rows with
+ * the freshly extracted set. Called inside `updatePage`'s transaction
+ * unconditionally — the `wappalyzer` signal source draws on `<script src>` /
+ * `<iframe src>` / window globals / response headers, not the HTML body, so
+ * external pages that skip rendering still contribute technology signals
+ * (structural signals naturally find nothing against an empty `html`
+ * string, which is exactly what a skipped render leaves it as).
+ *
+ * Direct replacement for `insertTags` in the live crawl write path (that
+ * file is untouched — `scripts/migrate-to-0.10.mjs` still depends on it).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does
+ * not wipe prior rows on a degraded re-scrape. Both tables are always
+ * written together in this one call — see ARCHITECTURE.md's invariant
+ * that `technology_signals` and `page_technologies` are never updated
+ * independently.
+ * @param pageId
+ * @param html - The page's raw HTML string (empty for external/metadata-only scrapes).
+ * @param meta
+ * @param trx
+ */
+export declare function insertTechnologies(pageId: number, html: string, meta: PageData['meta'], trx: Knex.Transaction): Promise<void>;

package/lib/archive/db-ops/pages/write/insert-technologies.js

@@ -0,0 +1,45 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+import { extractTechnologiesForArchive } from '../../../meta/technologies/extract-technologies-for-archive.js';
+/**
+ * Replaces the page's `technology_signals` + `page_technologies` rows with
+ * the freshly extracted set. Called inside `updatePage`'s transaction
+ * unconditionally — the `wappalyzer` signal source draws on `<script src>` /
+ * `<iframe src>` / window globals / response headers, not the HTML body, so
+ * external pages that skip rendering still contribute technology signals
+ * (structural signals naturally find nothing against an empty `html`
+ * string, which is exactly what a skipped render leaves it as).
+ *
+ * Direct replacement for `insertTags` in the live crawl write path (that
+ * file is untouched — `scripts/migrate-to-0.10.mjs` still depends on it).
+ *
+ * Same empty-guard as `insertTags` / `insertJsonLd`: an empty array does
+ * not wipe prior rows on a degraded re-scrape. Both tables are always
+ * written together in this one call — see ARCHITECTURE.md's invariant
+ * that `technology_signals` and `page_technologies` are never updated
+ * independently.
+ * @param pageId
+ * @param html - The page's raw HTML string (empty for external/metadata-only scrapes).
+ * @param meta
+ * @param trx
+ */
+export async function insertTechnologies(pageId, html, meta, trx) {
+    const { signals, technologies } = extractTechnologiesForArchive(html, meta);
+    if (signals.length === 0)
+        return;
+    await trx('technology_signals').where('pageId', pageId).delete();
+    await trx('page_technologies').where('pageId', pageId).delete();
+    await eachSplitted(signals, 100, async (chunk) => {
+        await trx('technology_signals').insert(chunk.map((s) => ({
+            pageId,
+            technology: s.technology,
+            signalType: s.signalType,
+            evidence: s.evidence,
+            weight: s.weight,
+        })));
+    });
+    if (technologies.length > 0) {
+        await eachSplitted(technologies, 100, async (chunk) => {
+            await trx('page_technologies').insert(chunk.map((t) => ({ pageId, ...t })));
+        });
+    }
+}

package/lib/archive/db-ops/pages/write/update-page.d.ts

@@ -24,6 +24,9 @@ import type { Knex } from 'knex';
  *   inserted. Existing rows keep their original `source` (this is why a
  *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
  *   that was discovered earlier).
+ * @param bodyHash - Precomputed body hash for the page's HTML (see
+ *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+ *   computing it from the HTML instead.
  * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
  */
-export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource): Promise<number>;
+export declare function updatePage(knex: Knex, caches: WriteRefCaches, page: PageDataWithDomPaths, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null): Promise<number>;

package/lib/archive/db-ops/pages/write/update-page.js

@@ -13,13 +13,14 @@ import { resolveUrlOrBlob } from '../../_shared/resolve-url-or-blob.js';
 import { insertAudios } from './insert-audios.js';
 import { insertButtons } from './insert-buttons.js';
 import { insertCanvases } from './insert-canvases.js';
+import { insertCustomElements } from './insert-custom-elements.js';
 import { insertHeadings } from './insert-headings.js';
 import { insertIframes } from './insert-iframes.js';
 import { insertJsonLd } from './insert-jsonld.js';
 import { insertMainContentImages } from './insert-main-content-images.js';
 import { insertMainContentTables } from './insert-main-content-tables.js';
 import { insertPage } from './insert-page.js';
-import { insertTags } from './insert-tags.js';
+import { insertTechnologies } from './insert-technologies.js';
 import { insertVideos } from './insert-videos.js';
 import { linkRedirectSources } from './link-redirect-sources.js';
 import { writePageHtmlBlob } from './write-page-html-blob.js';
@@ -46,9 +47,12 @@ import { writePageHtmlBlob } from './write-page-html-blob.js';
  *   inserted. Existing rows keep their original `source` (this is why a
  *   second `crawl --inventory` does not "demote" an `'inventory-seed'` row
  *   that was discovered earlier).
+ * @param bodyHash - Precomputed body hash for the page's HTML (see
+ *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+ *   computing it from the HTML instead.
  * @returns The database `pageId` (`content_items.id`) of the inserted or updated row.
  */
-export async function updatePage(knex, caches, page, writeHtml, isTarget, source) {
+export async function updatePage(knex, caches, page, writeHtml, isTarget, source, bodyHash) {
     const { destUrl, sources } = resolveRedirectChain(page.url.withoutHashAndAuth, page.redirectPaths);
     const destUrlObject = parseUrl(destUrl);
     if (!destUrlObject) {
@@ -56,7 +60,7 @@ export async function updatePage(knex, caches, page, writeHtml, isTarget, source
     }
     try {
         return await knex.transaction(async (trx) => {
-            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source);
+            return await updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash);
         });
     }
     catch (error) {
@@ -83,21 +87,24 @@ export async function updatePage(knex, caches, page, writeHtml, isTarget, source
  * @param writeHtml - See {@link updatePage}.
  * @param isTarget - See {@link updatePage}.
  * @param source - See {@link updatePage}.
+ * @param bodyHash - See {@link updatePage}.
  * @returns The `content_items.id` of the inserted or updated row.
  */
-async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source) {
+async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, sources, writeHtml, isTarget, source, bodyHash) {
     const pageId = await insertPage(knex, caches, {
         ...page,
         url: destUrlObject,
     }, isTarget, trx, source);
-    // Wappalyzer tag detection is HTML-body independent (relies on
-    // `<script src>` / `<iframe src>` / window globals / response
-    // headers) so it runs for every page including external /
-    // metadata-only. JSON-LD on the other hand lives inside the
-    // rendered HTML body, so we only write it when there is HTML to
-    // scrape — see the same `writeHtml` gate as `writePageHtmlBlob`
-    // below.
-    await insertTags(pageId, page.meta, trx);
+    // Technology detection combines a Wappalyzer signal source (HTML-body
+    // independent — relies on `<script src>` / `<iframe src>` / window
+    // globals / response headers, so it fires for every page including
+    // external / metadata-only) with structural signals that scan
+    // `page.html` (naturally a no-op against the empty string an
+    // external/metadata-only scrape leaves it as). Both run unconditionally,
+    // unlike JSON-LD below, which lives inside the rendered HTML body and
+    // is only written when there is HTML to scrape — see the same
+    // `writeHtml` gate as `writePageHtmlBlob` below.
+    await insertTechnologies(pageId, page.html, page.meta, trx);
     if (writeHtml) {
         await insertJsonLd(pageId, page.meta, trx);
     }
@@ -113,6 +120,16 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
         await insertVideos(pageId, page.mainContents, trx);
         await insertAudios(pageId, page.mainContents, trx);
         await insertCanvases(pageId, page.mainContents, trx);
+        // Unlike the eight calls above, the source here is nitpicker's own
+        // `capture-custom-elements.ts` capture, not `page.mainContents` — see
+        // `insert-custom-elements.ts`. That capture can fail independently
+        // of `page.mainContents` (a separate `page.evaluate()` call), so
+        // `undefined` (capture failed) must skip the call entirely — passing
+        // a defaulted `[]` would look identical to "found zero" and, per
+        // `insertCustomElements`'s docs, wipe stale-but-still-good rows.
+        if (page.mainContentCustomElements !== undefined) {
+            await insertCustomElements(pageId, page.mainContentCustomElements, trx);
+        }
     }
     // Chain lineage propagates FROM the originating URL
     // (`page.url`), NOT from the destination. See the matching
@@ -142,12 +159,17 @@ async function updatePageInTransaction(trx, knex, caches, page, destUrlObject, s
     // content check alone expresses the intent without a redundant term.
     if (writeHtml && page.html.length > 0) {
         await writePageHtmlBlob(pageId, page.html, trx);
-        // Computed from the same `page.html` written above, in the same
-        // transaction, so `page_meta.body_hash` never observably lags the
-        // snapshot it was derived from.
+        // Prefer the caller's precomputed hash (the crawler already hashed
+        // this exact `page.html` once, before this write path ever runs) —
+        // falling back to computing it here only when the caller has not
+        // (direct callers such as tests, or a future write path that has not
+        // adopted the precompute). Either way this stays derived from the
+        // same `page.html` written above, in the same transaction, so
+        // `page_meta.body_hash` never observably lags the snapshot it was
+        // derived from.
         await trx('page_meta')
             .where('page_id', pageId)
-            .update({ body_hash: computeBodyHash(page.html) });
+            .update({ body_hash: bodyHash ?? computeBodyHash(page.html) });
     }
     else if (writeHtml &&
         page.contentType !== null &&

package/lib/archive/db-ops/resources/get-resource-url-list.d.ts

@@ -3,7 +3,18 @@ import type { Knex } from 'knex';
  * Retrieves a flat list of all resource URLs from the `resource_items`
  * table. URL text is normalised into `url_refs`, so the read joins the
  * two tables and returns the resolved strings.
+ *
+ * Read in `resource_items.id`-keyset chunks rather than a single SELECT
+ * (issue #294): on a resource-heavy archive (images/JS/CSS in the tens or
+ * hundreds of thousands) this was one unbounded, multi-second-to-minutes
+ * query with no way to report progress mid-scan. The accumulated result is
+ * identical to the previous single-SELECT read — chunking exists purely to
+ * make the scan observable, not to bound memory (the flat URL list is
+ * already fully materialised for the caller either way).
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk with the `resource_items.id`
+ *   scanned up to so far and the max id. Omit for no reporting (the
+ *   default; e.g. tests).
  * @returns An array of resource URL strings.
  */
-export declare function getResourceUrlList(knex: Knex): Promise<string[]>;
+export declare function getResourceUrlList(knex: Knex, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<string[]>;

package/lib/archive/db-ops/resources/get-resource-url-list.js

@@ -1,13 +1,51 @@
+/** Rows read per `resource_items.id` keyset chunk. */
+const READ_CHUNK_SIZE = 2000;
 /**
  * Retrieves a flat list of all resource URLs from the `resource_items`
  * table. URL text is normalised into `url_refs`, so the read joins the
  * two tables and returns the resolved strings.
+ *
+ * Read in `resource_items.id`-keyset chunks rather than a single SELECT
+ * (issue #294): on a resource-heavy archive (images/JS/CSS in the tens or
+ * hundreds of thousands) this was one unbounded, multi-second-to-minutes
+ * query with no way to report progress mid-scan. The accumulated result is
+ * identical to the previous single-SELECT read — chunking exists purely to
+ * make the scan observable, not to bound memory (the flat URL list is
+ * already fully materialised for the caller either way).
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk with the `resource_items.id`
+ *   scanned up to so far and the max id. Omit for no reporting (the
+ *   default; e.g. tests).
  * @returns An array of resource URL strings.
  */
-export async function getResourceUrlList(knex) {
-    const res = await knex('resource_items')
-        .join('url_refs', 'url_refs.id', 'resource_items.url_id')
-        .select('url_refs.url as url');
-    return res.map((r) => r.url);
+export async function getResourceUrlList(knex, onProgress) {
+    // MAX() over the keyset column is an O(1) index-tail read; only fetched
+    // when someone is listening.
+    let maxId = 0;
+    if (onProgress) {
+        const [maxRow] = await knex('resource_items').max({
+            max: 'id',
+        });
+        maxId = maxRow?.max ?? 0;
+    }
+    const urls = [];
+    let lastId = 0;
+    for (;;) {
+        const rows = (await knex('resource_items')
+            .join('url_refs', 'url_refs.id', 'resource_items.url_id')
+            .where('resource_items.id', '>', lastId)
+            .orderBy('resource_items.id', 'asc')
+            .limit(READ_CHUNK_SIZE)
+            .select('resource_items.id as id', 'url_refs.url as url'));
+        if (rows.length === 0) {
+            onProgress?.(maxId, maxId);
+            break;
+        }
+        lastId = rows.at(-1).id;
+        for (const row of rows) {
+            urls.push(row.url);
+        }
+        onProgress?.(Math.min(lastId, maxId), maxId);
+    }
+    return urls;
 }

package/lib/archive/filesystem/copy-file-with-progress.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Copies a file while reporting byte progress — `fs.copyFile` offers no
+ * observability, and a `.nitpicker` archive's `.bak` safety copy (taken
+ * before every mutating `viewer-build`/`crawl --append`/`--inventory`/
+ * `--retry-failed` run, and copied back on failure) can be 15 GB+, running
+ * for tens of seconds with nothing on screen (issue #294). Byte granularity
+ * is the read-stream chunk size (~64 KB); callers wanting coarser updates
+ * throttle in their own callback.
+ * @param src - The file to copy.
+ * @param dest - The destination path, overwritten if present.
+ * @param onProgress - Called as bytes are copied, with the bytes copied so
+ *   far and the source file's total size. Omit for a silent copy.
+ * @example
+ * ```ts
+ * await copyFileWithProgress(archivePath, `${archivePath}.bak`, (copied, total) => {
+ *   console.error(`${copied}/${total}`);
+ * });
+ * ```
+ */
+export declare function copyFileWithProgress(src: string, dest: string, onProgress?: (copiedBytes: number, totalBytes: number) => void): Promise<void>;

package/lib/archive/filesystem/copy-file-with-progress.js

@@ -0,0 +1,34 @@
+import { createReadStream, createWriteStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
+/**
+ * Copies a file while reporting byte progress — `fs.copyFile` offers no
+ * observability, and a `.nitpicker` archive's `.bak` safety copy (taken
+ * before every mutating `viewer-build`/`crawl --append`/`--inventory`/
+ * `--retry-failed` run, and copied back on failure) can be 15 GB+, running
+ * for tens of seconds with nothing on screen (issue #294). Byte granularity
+ * is the read-stream chunk size (~64 KB); callers wanting coarser updates
+ * throttle in their own callback.
+ * @param src - The file to copy.
+ * @param dest - The destination path, overwritten if present.
+ * @param onProgress - Called as bytes are copied, with the bytes copied so
+ *   far and the source file's total size. Omit for a silent copy.
+ * @example
+ * ```ts
+ * await copyFileWithProgress(archivePath, `${archivePath}.bak`, (copied, total) => {
+ *   console.error(`${copied}/${total}`);
+ * });
+ * ```
+ */
+export async function copyFileWithProgress(src, dest, onProgress) {
+    const { size: totalBytes } = await stat(src);
+    const source = createReadStream(src);
+    let copiedBytes = 0;
+    if (onProgress) {
+        source.on('data', (chunk) => {
+            copiedBytes += chunk.length;
+            onProgress(copiedBytes, totalBytes);
+        });
+    }
+    await pipeline(source, createWriteStream(dest));
+}

package/lib/archive/filesystem/parse-pax-path.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Parses a PAX extended header record block for the `path` key.
+ *
+ * Format: a sequence of `"<record-length> <key>=<value>\n"` records, where
+ * `<record-length>` is the decimal byte length of the ENTIRE record
+ * (including the length prefix itself and the trailing newline). `tar`
+ * (node-tar) emits one of these ahead of any entry whose name doesn't fit
+ * the ustar format's 100-byte field (UTF-8 multi-byte names, or names
+ * longer than 100 bytes) — verified empirically against this package's own
+ * `tar()`/`create()` output.
+ * @param data - The PAX header entry's raw data payload.
+ * @returns The `path` value if present, otherwise `null`.
+ * @example
+ * ```ts
+ * parsePaxPath(Buffer.from('20 path=日本語/\n', 'utf8')); // '日本語/'
+ * ```
+ */
+export declare function parsePaxPath(data: Buffer): string | null;

package/lib/archive/filesystem/parse-pax-path.js

@@ -0,0 +1,47 @@
+/**
+ * Parses a PAX extended header record block for the `path` key.
+ *
+ * Format: a sequence of `"<record-length> <key>=<value>\n"` records, where
+ * `<record-length>` is the decimal byte length of the ENTIRE record
+ * (including the length prefix itself and the trailing newline). `tar`
+ * (node-tar) emits one of these ahead of any entry whose name doesn't fit
+ * the ustar format's 100-byte field (UTF-8 multi-byte names, or names
+ * longer than 100 bytes) — verified empirically against this package's own
+ * `tar()`/`create()` output.
+ * @param data - The PAX header entry's raw data payload.
+ * @returns The `path` value if present, otherwise `null`.
+ * @example
+ * ```ts
+ * parsePaxPath(Buffer.from('20 path=日本語/\n', 'utf8')); // '日本語/'
+ * ```
+ */
+export function parsePaxPath(data) {
+    let offset = 0;
+    while (offset < data.length) {
+        const spaceIndex = data.indexOf(0x20 /* ' ' */, offset);
+        if (spaceIndex === -1) {
+            return null;
+        }
+        const lengthText = data.subarray(offset, spaceIndex).toString('latin1');
+        const recordLength = Number.parseInt(lengthText, 10);
+        if (!Number.isFinite(recordLength) || recordLength <= 0) {
+            return null;
+        }
+        const recordEnd = offset + recordLength;
+        if (recordEnd > data.length) {
+            return null;
+        }
+        // Decimal (not `0x3D`) to sidestep the Prettier/eslint hex-case
+        // conflict — see `parse-tar-size-field.ts` for the full explanation.
+        const equalsIndex = data.indexOf(61 /* '=' */, spaceIndex);
+        if (equalsIndex !== -1 && equalsIndex < recordEnd) {
+            const key = data.subarray(spaceIndex + 1, equalsIndex).toString('utf8');
+            if (key === 'path') {
+                // Value runs from after '=' to just before the trailing '\n'.
+                return data.subarray(equalsIndex + 1, recordEnd - 1).toString('utf8');
+            }
+        }
+        offset = recordEnd;
+    }
+    return null;
+}

package/lib/archive/filesystem/parse-tar-size-field.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Parses a tar header's size field (offset 124, 12 bytes), which is either
+ * a NUL/space-terminated octal ASCII string (standard) or, when the
+ * high bit of the first byte is set, a GNU base-256 big-endian binary
+ * encoding (used for sizes too large for the 11-digit octal field).
+ * @param field - The 12-byte size field.
+ * @returns The size in bytes, or `null` if the field cannot be parsed.
+ * @example
+ * ```ts
+ * parseTarSizeField(Buffer.from('00000000004\0', 'latin1')); // 4
+ * ```
+ */
+export declare function parseTarSizeField(field: Buffer): number | null;

package/lib/archive/filesystem/parse-tar-size-field.js

@@ -0,0 +1,40 @@
+/**
+ * Parses a tar header's size field (offset 124, 12 bytes), which is either
+ * a NUL/space-terminated octal ASCII string (standard) or, when the
+ * high bit of the first byte is set, a GNU base-256 big-endian binary
+ * encoding (used for sizes too large for the 11-digit octal field).
+ * @param field - The 12-byte size field.
+ * @returns The size in bytes, or `null` if the field cannot be parsed.
+ * @example
+ * ```ts
+ * parseTarSizeField(Buffer.from('00000000004\0', 'latin1')); // 4
+ * ```
+ */
+export function parseTarSizeField(field) {
+    const first = field[0];
+    if (first === undefined) {
+        return null;
+    }
+    if ((first & 0x80) !== 0) {
+        // GNU base-256: the remaining bits of the first byte plus all
+        // following bytes form a big-endian unsigned integer. `.nitpicker`
+        // tars never approach sizes needing this (single-digit-terabyte
+        // range before it would even matter), but a correct read here still
+        // costs nothing.
+        // `0b0111_1111` (not `0x7F`) to sidestep the Prettier/eslint hex-case
+        // conflict (Prettier always lowercases hex digits; this repo's
+        // `unicorn/number-literal-case` requires uppercase) — a binary literal
+        // has no letter digits to disagree about.
+        let value = BigInt(first & 0b0111_1111);
+        for (let i = 1; i < field.length; i++) {
+            value = (value << 8n) | BigInt(field[i] ?? 0);
+        }
+        return Number(value);
+    }
+    const text = field.toString('latin1').replaceAll('\0', ' ').trim();
+    if (text === '') {
+        return 0;
+    }
+    const parsed = Number.parseInt(text, 8);
+    return Number.isNaN(parsed) ? null : parsed;
+}

package/lib/archive/filesystem/peek-tar-top-dir.d.ts

@@ -12,9 +12,16 @@
  * directory it sees, so callers (`Archive.open`, the migration script) can
  * use the actual inner name regardless of what the outer file is called.
  *
- * Reads the tar's central directory only; does not extract any data. Runs
- * in O(number of entries) but stops at the first match, so a typical
- * archive (1–2 top-level entries) completes in a single read.
+ * Parses raw tar headers directly (ustar fixed fields, PAX extended headers,
+ * GNU longname/longlink) instead of using the `tar` package's `list()` —
+ * `list()` reads the entire archive stream to completion even after
+ * `onReadEntry` has found what it needs (no internal early-abort), so on a
+ * 15 GB+ archive it was a second full read on top of `untar()`'s own read
+ * (issue #294). The fast path here stops at the first top-level directory
+ * entry, typically reading well under 1 KB. Falls back to the original
+ * `list()`-based scan on anything it doesn't recognize (unknown typeflag
+ * sequence, truncated read, non-tar content) — safety over speed for
+ * archives this function has never been exercised against.
  * @param tarFilePath - Path to the `.nitpicker` tar file.
  * @returns The first top-level directory name found in the archive.
  * @throws {Error} When the tar contains no top-level directory entry