@nitpicker/archive 0.0.0 → 0.25.1

package/lib/filesystem/peek-tar-top-dir.js

@@ -0,0 +1,215 @@
+import { open } from 'node:fs/promises';
+import { list } from 'tar';
+import { parsePaxPath } from './parse-pax-path.js';
+import { parseTarSizeField } from './parse-tar-size-field.js';
+/** Bytes per tar header/data block (fixed by the tar format). */
+const BLOCK_SIZE = 512;
+/** Defensive cap on entries scanned before giving up and falling back to `list()`. */
+const MAX_ENTRIES = 100_000;
+/** Cap on PAX/GNU-longname payload size read into memory; larger is treated as malformed. */
+const MAX_EXTENDED_PAYLOAD_SIZE = 1_000_000;
+/**
+ * Reads the top-level directory name from a `.nitpicker` tar archive.
+ *
+ * Why: a `.nitpicker` is a tar that wraps `<basename>/db.sqlite` (and
+ * historically other files). `Archive.write()` names the inner directory
+ * from the file's basename at write time, but users routinely rename
+ * `.nitpicker` files after the fact (e.g. `mv X.migrated.nitpicker
+ * X.nitpicker`) — a perfectly reasonable filesystem operation that breaks
+ * any code that recomputes the inner-dir name from the outer filename.
+ *
+ * This helper scans the tar's entry list and returns the first top-level
+ * directory it sees, so callers (`Archive.open`, the migration script) can
+ * use the actual inner name regardless of what the outer file is called.
+ *
+ * Parses raw tar headers directly (ustar fixed fields, PAX extended headers,
+ * GNU longname/longlink) instead of using the `tar` package's `list()` —
+ * `list()` reads the entire archive stream to completion even after
+ * `onReadEntry` has found what it needs (no internal early-abort), so on a
+ * 15 GB+ archive it was a second full read on top of `untar()`'s own read
+ * (issue #294). The fast path here stops at the first top-level directory
+ * entry, typically reading well under 1 KB. Falls back to the original
+ * `list()`-based scan on anything it doesn't recognize (unknown typeflag
+ * sequence, truncated read, non-tar content) — safety over speed for
+ * archives this function has never been exercised against.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry
+ *   (e.g. file is empty, corrupted, or non-tar). The error message
+ *   identifies the path for diagnostics.
+ * @example
+ * // The user renamed `original.nitpicker` to `renamed.nitpicker`. The
+ * // inner directory was baked in at write time and is unaffected:
+ * await peekTarTopDir('renamed.nitpicker'); // → 'original'
+ */
+export async function peekTarTopDir(tarFilePath) {
+    const fast = await peekTarTopDirFast(tarFilePath);
+    if (fast !== null) {
+        return fast;
+    }
+    return await peekTarTopDirViaList(tarFilePath);
+}
+/**
+ * Returns `true` for a top-level directory name that should be skipped: an
+ * AppleDouble resource-fork sidecar (`._foo`, from macOS BSD tar) or a PAX
+ * header directory (`PaxHeaders.NNN`/`@PaxHeader`). Real `.nitpicker`
+ * archives never name their actual top dir with these prefixes. Shared by
+ * both the fast path and the `list()` fallback so the two can't drift into
+ * accepting different names for the same archive.
+ * @param name - Candidate top-level directory name.
+ */
+function isSkippableTopDirName(name) {
+    return (name.startsWith('._') ||
+        name.startsWith('PaxHeaders') ||
+        name.startsWith('@PaxHeader'));
+}
+/**
+ * Extracts the first path segment from a tar entry path, after stripping a
+ * leading `./` some tar producers prepend.
+ * @param entryPath - The raw path recorded in (or resolved for) a tar entry.
+ */
+function topSegmentOf(entryPath) {
+    return entryPath.replace(/^\.\//, '').split('/')[0] ?? '';
+}
+/**
+ * Reads a NUL-terminated (or full-width, if no NUL) string field out of a
+ * tar header block.
+ * @param header - The 512-byte header block.
+ * @param start - Field start offset.
+ * @param length - Field length in bytes.
+ */
+function readHeaderString(header, start, length) {
+    const field = header.subarray(start, start + length);
+    const nulIndex = field.indexOf(0);
+    const raw = nulIndex === -1 ? field : field.subarray(0, nulIndex);
+    return raw.toString('utf8');
+}
+/**
+ * Fast path for {@link peekTarTopDir}: reads raw 512-byte tar header blocks
+ * directly from the file, stopping at the first top-level directory entry.
+ * Understands ustar fixed headers, PAX extended headers (`path=`, the form
+ * `tar`/node-tar emits for UTF-8 or >100-byte names — verified empirically:
+ * see this function's test fixtures), and GNU longname (`typeflag 'L'`,
+ * treated the same as a PAX path override for the next entry — this
+ * codebase's own `tar()` never emits it, but a `.nitpicker` produced by a
+ * different tar implementation should still resolve correctly). GNU
+ * longlink (`typeflag 'K'`, a symlink target — irrelevant to a directory
+ * name lookup) is skipped without being applied to anything.
+ *
+ * Returns `null` — never throws — on anything unrecognized: truncated read,
+ * unparseable size field, entry count past {@link MAX_ENTRIES}, or reaching
+ * the end-of-archive marker without finding a directory. The caller falls
+ * back to the exhaustive `list()`-based scan in every `null` case.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name, or `null` to fall back.
+ */
+async function peekTarTopDirFast(tarFilePath) {
+    let fileHandle;
+    try {
+        fileHandle = await open(tarFilePath, 'r');
+        const header = Buffer.alloc(BLOCK_SIZE);
+        let position = 0;
+        // Set by a preceding PAX ('x') or GNU longname ('L') entry; applies
+        // to the very next entry only, then is cleared.
+        let pendingPath = null;
+        for (let entryIndex = 0; entryIndex < MAX_ENTRIES; entryIndex++) {
+            const { bytesRead } = await fileHandle.read(header, 0, BLOCK_SIZE, position);
+            if (bytesRead < BLOCK_SIZE) {
+                return null;
+            }
+            position += BLOCK_SIZE;
+            if (header.every((byte) => byte === 0)) {
+                // End-of-archive marker reached with no directory found —
+                // genuinely absent, not a parse failure, but the caller's
+                // error message is clearer coming from the exhaustive path.
+                return null;
+            }
+            const typeflag = String.fromCodePoint(header[156] ?? 0);
+            const size = parseTarSizeField(header.subarray(124, 136));
+            if (size === null) {
+                return null;
+            }
+            const dataBlockCount = Math.ceil(size / BLOCK_SIZE);
+            const dataStart = position;
+            position += dataBlockCount * BLOCK_SIZE;
+            if (typeflag === 'x' || typeflag === 'L') {
+                if (size <= 0 || size > MAX_EXTENDED_PAYLOAD_SIZE) {
+                    return null;
+                }
+                const payload = Buffer.alloc(size);
+                const { bytesRead: payloadBytesRead } = await fileHandle.read(payload, 0, size, dataStart);
+                if (payloadBytesRead !== size) {
+                    return null;
+                }
+                pendingPath =
+                    typeflag === 'x'
+                        ? parsePaxPath(payload)
+                        : payload.toString('utf8').replace(/\0.*$/, '');
+                continue;
+            }
+            if (typeflag === 'g' || typeflag === 'K') {
+                // PAX global header (applies archive-wide, never carries a
+                // per-entry path) / GNU longlink (symlink target, irrelevant
+                // to a directory name lookup) — skip without consuming
+                // `pendingPath`.
+                continue;
+            }
+            const magic = header.toString('latin1', 257, 263);
+            const nameField = readHeaderString(header, 0, 100);
+            const prefixField = magic.startsWith('ustar')
+                ? readHeaderString(header, 345, 155)
+                : '';
+            const entryPath = pendingPath ?? (prefixField ? `${prefixField}/${nameField}` : nameField);
+            pendingPath = null;
+            if (typeflag === '5') {
+                const top = topSegmentOf(entryPath);
+                if (top && !isSkippableTopDirName(top)) {
+                    return top;
+                }
+            }
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        await fileHandle?.close();
+    }
+}
+/**
+ * Exhaustive fallback for {@link peekTarTopDir}: scans the tar's entry list
+ * via the `tar` package. Reads the archive to completion regardless of when
+ * a match is found (no early-abort in the `tar` package's `list()` API),
+ * which is exactly the cost {@link peekTarTopDirFast} exists to avoid — this
+ * path only runs when the fast path can't make sense of the header stream.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry.
+ */
+async function peekTarTopDirViaList(tarFilePath) {
+    let found = null;
+    await list({
+        file: tarFilePath,
+        onReadEntry: (entry) => {
+            if (found !== null)
+                return;
+            // Only Directory entries count. Files at the tar root would
+            // otherwise win — and macOS BSD tar embeds AppleDouble (`._*`)
+            // sidecar files at the tar root for resource forks; those are
+            // File entries that BSD `tar -tf` hides but Node's `tar`
+            // surfaces verbatim, so without this filter the resource-fork
+            // of the real top-level dir would win instead of the dir itself.
+            if (entry.type !== 'Directory')
+                return;
+            const top = topSegmentOf(entry.path);
+            if (!top || isSkippableTopDirName(top))
+                return;
+            found = top;
+        },
+    });
+    if (found === null) {
+        throw new Error(`Tar contains no top-level directory entry: ${tarFilePath}`);
+    }
+    return found;
+}

package/lib/filesystem/read-json.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Reads and parses a JSON file from the specified path.
+ * @template T - The expected type of the parsed JSON content. Defaults to `unknown`.
+ * @param filePath - The absolute or relative path to the JSON file to read.
+ * @returns The parsed JSON content, cast to the specified generic type.
+ */
+export declare function readJSON<T = unknown>(filePath: string): Promise<T>;

package/lib/filesystem/read-json.js

@@ -0,0 +1,11 @@
+import { promises as fs } from 'node:fs';
+/**
+ * Reads and parses a JSON file from the specified path.
+ * @template T - The expected type of the parsed JSON content. Defaults to `unknown`.
+ * @param filePath - The absolute or relative path to the JSON file to read.
+ * @returns The parsed JSON content, cast to the specified generic type.
+ */
+export async function readJSON(filePath) {
+    const data = await fs.readFile(filePath, { encoding: 'utf8' });
+    return JSON.parse(data);
+}

package/lib/filesystem/read-text.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Reads the entire contents of a text file as a UTF-8 string.
+ * @param filePath - The absolute or relative path to the text file to read.
+ * @returns The text content of the file.
+ */
+export declare function readText(filePath: string): Promise<string>;

package/lib/filesystem/read-text.js

@@ -0,0 +1,10 @@
+import { promises as fs } from 'node:fs';
+/**
+ * Reads the entire contents of a text file as a UTF-8 string.
+ * @param filePath - The absolute or relative path to the text file to read.
+ * @returns The text content of the file.
+ */
+export async function readText(filePath) {
+    const data = await fs.readFile(filePath, { encoding: 'utf8' });
+    return data;
+}

package/lib/filesystem/readline.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Reads a file line by line and invokes the callback for each line.
+ *
+ * The callback may return a Promise for asynchronous processing.
+ * All callback results are collected and awaited via `Promise.all` before returning.
+ * @param filePath - The path to the file to read line by line.
+ * @param callback - A function invoked for each line of the file.
+ *   May return a Promise for asynchronous operations.
+ * @returns A promise that resolves when all line callbacks have completed.
+ */
+export declare function readline(filePath: string, callback: (line: string) => Promise<void> | void): Promise<void[]>;

package/lib/filesystem/readline.js

@@ -0,0 +1,26 @@
+import { createReadStream } from 'node:fs';
+import Readline from 'node:readline';
+/**
+ * Reads a file line by line and invokes the callback for each line.
+ *
+ * The callback may return a Promise for asynchronous processing.
+ * All callback results are collected and awaited via `Promise.all` before returning.
+ * @param filePath - The path to the file to read line by line.
+ * @param callback - A function invoked for each line of the file.
+ *   May return a Promise for asynchronous operations.
+ * @returns A promise that resolves when all line callbacks have completed.
+ */
+export async function readline(filePath, callback) {
+    const stream = createReadStream(filePath);
+    const rLine = Readline.createInterface(stream);
+    const promiseBuffer = [];
+    await new Promise((resolve) => {
+        rLine.on('line', (line) => {
+            promiseBuffer.push(Promise.resolve(callback(line)));
+        });
+        rLine.on('close', () => {
+            resolve();
+        });
+    });
+    return Promise.all(promiseBuffer);
+}

package/lib/filesystem/remove.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Recursively removes a file or directory at the specified path.
+ * @param dirPath - The path of the file or directory to remove.
+ */
+export declare function remove(dirPath: string): Promise<void>;

package/lib/filesystem/remove.js

@@ -0,0 +1,10 @@
+import { promises as fs } from 'node:fs';
+/**
+ * Recursively removes a file or directory at the specified path.
+ * @param dirPath - The path of the file or directory to remove.
+ */
+export async function remove(dirPath) {
+    await fs.rm(dirPath, {
+        recursive: true,
+    });
+}

package/lib/filesystem/rename.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Renames (moves) a file or directory from one path to another.
+ *
+ * If `override` is `true`, the destination is unconditionally removed
+ * before renaming. This avoids a TOCTOU race condition between
+ * checking existence and performing the removal.
+ *
+ * When `fs.rename` fails with `EPERM` (common on Windows due to file locks
+ * from antivirus or indexer processes) or `EXDEV` (cross-device move),
+ * falls back to a copy-then-remove strategy.
+ * @param oldPath - The current path of the file or directory.
+ * @param newPath - The new path for the file or directory.
+ * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
+ * @returns A promise that resolves when the rename (or fallback copy-then-remove) is complete.
+ */
+export declare function rename(oldPath: string, newPath: string, override?: boolean): Promise<void>;

package/lib/filesystem/rename.js

@@ -0,0 +1,48 @@
+import { promises as fs } from 'node:fs';
+import { remove } from './remove.js';
+/**
+ * Checks whether a value is a `NodeJS.ErrnoException`.
+ * @param error - The value to check.
+ * @returns `true` if the value has a `code` property of type `string`.
+ */
+function isNodeError(error) {
+    return (error instanceof Error && typeof error.code === 'string');
+}
+/**
+ * Renames (moves) a file or directory from one path to another.
+ *
+ * If `override` is `true`, the destination is unconditionally removed
+ * before renaming. This avoids a TOCTOU race condition between
+ * checking existence and performing the removal.
+ *
+ * When `fs.rename` fails with `EPERM` (common on Windows due to file locks
+ * from antivirus or indexer processes) or `EXDEV` (cross-device move),
+ * falls back to a copy-then-remove strategy.
+ * @param oldPath - The current path of the file or directory.
+ * @param newPath - The new path for the file or directory.
+ * @param override - Whether to overwrite the destination if it already exists. Defaults to `false`.
+ * @returns A promise that resolves when the rename (or fallback copy-then-remove) is complete.
+ */
+export async function rename(oldPath, newPath, override = false) {
+    if (override) {
+        await remove(newPath).catch(() => { });
+    }
+    try {
+        await fs.rename(oldPath, newPath);
+    }
+    catch (error) {
+        if (isNodeError(error) && (error.code === 'EPERM' || error.code === 'EXDEV')) {
+            try {
+                await fs.cp(oldPath, newPath, { recursive: true });
+            }
+            catch (cpError) {
+                await remove(newPath).catch(() => { });
+                throw cpError;
+            }
+            await remove(oldPath);
+        }
+        else {
+            throw error;
+        }
+    }
+}

package/lib/filesystem/tar.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Creates an uncompressed TAR archive from a directory.
+ *
+ * The archive preserves the relative directory structure.
+ * The `dir` parameter is resolved relative to its parent directory
+ * so only the target directory name appears in the archive.
+ *
+ * When `onProgress` is given, the tar is produced through a manual stream
+ * pipe instead of tar's own `file` mode so the written byte count can be
+ * observed — a `.nitpicker` tar is dominated by the single giant `db.sqlite`
+ * entry (15 GB+ on large crawls), so per-entry callbacks would fire once and
+ * report nothing for minutes (issue #294). `totalBytes` is an estimate (the
+ * sum of the directory's file sizes; tar adds per-entry headers and padding
+ * on top), so `writtenBytes` is clamped to it and a final
+ * `(totalBytes, totalBytes)` call marks completion.
+ * @param dir - The absolute path of the directory to archive.
+ * @param outputPath - The file path where the TAR archive will be written.
+ * @param onProgress - Called as archive bytes are written, with the bytes
+ *   written so far (clamped to the estimate) and the estimated total. Omit
+ *   for the original silent, tar-managed-file creation.
+ * @returns A promise that resolves when the TAR archive has been created.
+ */
+export declare function tar(dir: string, outputPath: string, onProgress?: (writtenBytes: number, totalBytes: number) => void): Promise<void>;

package/lib/filesystem/tar.js

@@ -0,0 +1,78 @@
+import { createWriteStream } from 'node:fs';
+import { readdir, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { create } from 'tar';
+/**
+ * Creates an uncompressed TAR archive from a directory.
+ *
+ * The archive preserves the relative directory structure.
+ * The `dir` parameter is resolved relative to its parent directory
+ * so only the target directory name appears in the archive.
+ *
+ * When `onProgress` is given, the tar is produced through a manual stream
+ * pipe instead of tar's own `file` mode so the written byte count can be
+ * observed — a `.nitpicker` tar is dominated by the single giant `db.sqlite`
+ * entry (15 GB+ on large crawls), so per-entry callbacks would fire once and
+ * report nothing for minutes (issue #294). `totalBytes` is an estimate (the
+ * sum of the directory's file sizes; tar adds per-entry headers and padding
+ * on top), so `writtenBytes` is clamped to it and a final
+ * `(totalBytes, totalBytes)` call marks completion.
+ * @param dir - The absolute path of the directory to archive.
+ * @param outputPath - The file path where the TAR archive will be written.
+ * @param onProgress - Called as archive bytes are written, with the bytes
+ *   written so far (clamped to the estimate) and the estimated total. Omit
+ *   for the original silent, tar-managed-file creation.
+ * @returns A promise that resolves when the TAR archive has been created.
+ */
+export async function tar(dir, outputPath, onProgress) {
+    const baseDir = path.dirname(dir);
+    const targetDir = path.relative(baseDir, dir);
+    if (!onProgress) {
+        await create({
+            gzip: false,
+            cwd: baseDir,
+            file: outputPath,
+            preservePaths: false,
+        }, [targetDir]);
+        return;
+    }
+    const entries = await readdir(dir, { recursive: true, withFileTypes: true });
+    let totalBytes = 0;
+    for (const entry of entries) {
+        if (!entry.isFile()) {
+            continue;
+        }
+        const { size } = await stat(path.join(entry.parentPath, entry.name));
+        totalBytes += size;
+    }
+    await new Promise((resolve, reject) => {
+        const source = create({
+            gzip: false,
+            cwd: baseDir,
+            preservePaths: false,
+        }, [targetDir]);
+        const sink = createWriteStream(outputPath);
+        let writtenBytes = 0;
+        source.on('data', (chunk) => {
+            writtenBytes += chunk.length;
+            onProgress(Math.min(writtenBytes, totalBytes), totalBytes);
+        });
+        // .pipe() does not destroy the other side on error, so an error on
+        // either stream would otherwise leave the write side dangling open.
+        source.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            sink.destroy(err);
+            reject(err);
+        });
+        sink.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            source.destroy(err);
+            reject(err);
+        });
+        sink.on('close', () => {
+            onProgress(totalBytes, totalBytes);
+            resolve();
+        });
+        source.pipe(sink);
+    });
+}

package/lib/filesystem/untar.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Extracts files from a TAR archive.
+ *
+ * Only files newer than existing files in the target directory are extracted
+ * (uses the `newer` option). Optionally restricts extraction to a specific
+ * working directory and/or a subset of files.
+ *
+ * When `onProgress` is given, the tar is read through a manual
+ * `createReadStream` pipe instead of tar's own `file` mode so the consumed
+ * byte count can be observed — a `.nitpicker` tar is dominated by the single
+ * giant `db.sqlite` entry (15 GB+ on large crawls), so per-entry callbacks
+ * would fire once and report nothing for minutes (issue #294). Byte
+ * granularity is the read-stream chunk size (~64 KB); callers wanting
+ * coarser updates (e.g. one per percent) throttle in their own callback.
+ * @param tarFilePath - The path to the TAR archive to extract.
+ * @param options - Optional extraction settings.
+ * @param options.cwd - The working directory to extract files into.
+ *   If omitted, the current working directory is used.
+ * @param options.fileList - An array of specific file paths within the archive
+ *   to extract. If omitted, all files in the archive are extracted.
+ * @param options.onProgress - Called as archive bytes are consumed, with the
+ *   bytes read so far and the archive's total size. Omit for the original
+ *   silent, tar-managed-file extraction.
+ * @returns A promise that resolves when extraction is complete.
+ */
+export declare function untar(tarFilePath: string, options?: {
+    /** The working directory to extract files into. */
+    cwd?: string;
+    /** An array of specific file paths within the archive to extract. */
+    fileList?: string[];
+    /** Byte-level progress callback — see the function docs. */
+    onProgress?: (readBytes: number, totalBytes: number) => void;
+}): Promise<void>;

package/lib/filesystem/untar.js

@@ -0,0 +1,84 @@
+import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
+import { extract } from 'tar';
+/**
+ * Extracts files from a TAR archive.
+ *
+ * Only files newer than existing files in the target directory are extracted
+ * (uses the `newer` option). Optionally restricts extraction to a specific
+ * working directory and/or a subset of files.
+ *
+ * When `onProgress` is given, the tar is read through a manual
+ * `createReadStream` pipe instead of tar's own `file` mode so the consumed
+ * byte count can be observed — a `.nitpicker` tar is dominated by the single
+ * giant `db.sqlite` entry (15 GB+ on large crawls), so per-entry callbacks
+ * would fire once and report nothing for minutes (issue #294). Byte
+ * granularity is the read-stream chunk size (~64 KB); callers wanting
+ * coarser updates (e.g. one per percent) throttle in their own callback.
+ * @param tarFilePath - The path to the TAR archive to extract.
+ * @param options - Optional extraction settings.
+ * @param options.cwd - The working directory to extract files into.
+ *   If omitted, the current working directory is used.
+ * @param options.fileList - An array of specific file paths within the archive
+ *   to extract. If omitted, all files in the archive are extracted.
+ * @param options.onProgress - Called as archive bytes are consumed, with the
+ *   bytes read so far and the archive's total size. Omit for the original
+ *   silent, tar-managed-file extraction.
+ * @returns A promise that resolves when extraction is complete.
+ */
+export async function untar(tarFilePath, options) {
+    const { cwd, fileList, onProgress } = options ?? {};
+    if (!onProgress) {
+        await extract({
+            file: tarFilePath,
+            newer: true,
+            cwd,
+            preservePaths: false,
+            noMtime: true,
+        }, fileList ?? []);
+        return;
+    }
+    const { size: totalBytes } = await stat(tarFilePath);
+    await new Promise((resolve, reject) => {
+        const source = createReadStream(tarFilePath);
+        let readBytes = 0;
+        source.on('data', (chunk) => {
+            readBytes += chunk.length;
+            onProgress(readBytes, totalBytes);
+        });
+        const sink = extract({
+            newer: true,
+            cwd,
+            preservePaths: false,
+            noMtime: true,
+        }, fileList ?? []);
+        // .pipe() does not destroy the other side on error, so an error on
+        // either stream would otherwise leave the write side dangling open.
+        // `Unpack` (`extends Parser extends EE`, a plain `EventEmitter`, NOT
+        // a Minipass/stream base class despite behaving like a writable
+        // stream) has no `destroy()` — `abort()` is its own equivalent:
+        // idempotent (guarded internally), and it emits its own 'error'
+        // (confirmed via `tar`'s `warnMethod` with `recoverable: false`),
+        // which re-enters the `sink.on('error', ...)` handler below —
+        // harmless, since `source.destroy()`/`reject()` are already
+        // idempotent no-ops on a second call.
+        source.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            sink.abort(err);
+            reject(err);
+        });
+        sink.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            source.destroy(err);
+            reject(err);
+        });
+        // node-tar's Unpack stream signals completion via 'close' (all entry
+        // writes flushed), not 'finish' (input fully consumed) — resolving on
+        // 'finish' could hand the caller a tmpDir whose db.sqlite is still
+        // being written.
+        sink.on('close', () => {
+            resolve();
+        });
+        source.pipe(sink);
+    });
+}

package/lib/get-failed-page-messages.d.ts

@@ -0,0 +1,56 @@
+import type { Knex } from 'knex';
+/**
+ * One resolved failure message, with the timestamp it was recorded at.
+ * `createdAt` is what lets a caller (`resetFailedPages`) decide whether the
+ * failure falls inside a `network_outages` window and should be treated as
+ * retryable regardless of its classified `ErrorKind` — see
+ * `is-within-outage-window.ts`.
+ */
+export interface FailedPageMessage {
+    message: string;
+    /** Epoch ms the message was recorded (`page_errors.createdAt` or `crawl_errors.createdAt`). */
+    createdAt: number;
+}
+/**
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. The `crawl_errors` table is created empty (by
+ * `initSchema` on fresh archives, by `scripts/migrate-to-0.13.mjs` on
+ * migrated ones) and historical lines from `error.log` are never
+ * back-filled, so an archive whose failures live only in `error.log`
+ * will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, FailedPageMessage>>;