@nitpicker/archive 0.0.0 → 0.25.0

package/lib/body-hash/compute-body-hash.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Computes a content hash of a page's `<body>`, after normalizing away the
+ * kinds of incidental variance that would otherwise make two structurally
+ * identical pages hash differently: `/index.{ext}` URL-suffix forms and
+ * embedded dynamic tokens (cache-busting hashes, session/order ids, per-build
+ * CSS-module suffixes).
+ *
+ * Only the resulting hash is persisted (`page_meta.body_hash`) — the masked
+ * intermediate string is never stored. The unmasked original HTML remains
+ * fully recoverable from `page_html_blobs`, so nothing is lost by discarding
+ * it here.
+ *
+ * The computation runs in the native addon (`@nitpicker/core`). The
+ * JavaScript stages next to this file (`extract-body.ts`,
+ * `normalize-url-like-strings.ts`, `mask-dynamic-ids.ts`) are not called
+ * from here and are excluded from the build: they exist only as the oracle
+ * for `compute-body-hash.parity.spec.ts`, which pins the addon to their
+ * output and so to the values existing archives already hold.
+ * @param html - A full HTML document (or fragment): a string, or its UTF-8
+ *   bytes — pass bytes when they are already at hand (a stored blob via
+ *   `decodeStoredBlobBytes`) to skip a decode/re-encode round-trip. Strings
+ *   are encoded as `Buffer.from(html, 'utf8')` does, lone surrogates as U+FFFD.
+ * @returns 32-byte SHA-256 hash of the masked `<body>` content, ready to
+ *   insert into a `BLOB` column.
+ * @example
+ * ```ts
+ * const hashA = computeBodyHash('<body><a href="/p/a1b2c3d4">x</a></body>');
+ * const hashB = computeBodyHash('<body><a href="/p/z9y8x7w6">x</a></body>');
+ * hashA.equals(hashB); // true — the differing token is masked before hashing
+ * ```
+ */
+export declare function computeBodyHash(html: string | Uint8Array): Buffer;

package/lib/body-hash/compute-body-hash.js

@@ -0,0 +1,35 @@
+import { computeBodyHash as computeBodyHashNative } from '@nitpicker/core/compute-body-hash';
+/**
+ * Computes a content hash of a page's `<body>`, after normalizing away the
+ * kinds of incidental variance that would otherwise make two structurally
+ * identical pages hash differently: `/index.{ext}` URL-suffix forms and
+ * embedded dynamic tokens (cache-busting hashes, session/order ids, per-build
+ * CSS-module suffixes).
+ *
+ * Only the resulting hash is persisted (`page_meta.body_hash`) — the masked
+ * intermediate string is never stored. The unmasked original HTML remains
+ * fully recoverable from `page_html_blobs`, so nothing is lost by discarding
+ * it here.
+ *
+ * The computation runs in the native addon (`@nitpicker/core`). The
+ * JavaScript stages next to this file (`extract-body.ts`,
+ * `normalize-url-like-strings.ts`, `mask-dynamic-ids.ts`) are not called
+ * from here and are excluded from the build: they exist only as the oracle
+ * for `compute-body-hash.parity.spec.ts`, which pins the addon to their
+ * output and so to the values existing archives already hold.
+ * @param html - A full HTML document (or fragment): a string, or its UTF-8
+ *   bytes — pass bytes when they are already at hand (a stored blob via
+ *   `decodeStoredBlobBytes`) to skip a decode/re-encode round-trip. Strings
+ *   are encoded as `Buffer.from(html, 'utf8')` does, lone surrogates as U+FFFD.
+ * @returns 32-byte SHA-256 hash of the masked `<body>` content, ready to
+ *   insert into a `BLOB` column.
+ * @example
+ * ```ts
+ * const hashA = computeBodyHash('<body><a href="/p/a1b2c3d4">x</a></body>');
+ * const hashB = computeBodyHash('<body><a href="/p/z9y8x7w6">x</a></body>');
+ * hashA.equals(hashB); // true — the differing token is masked before hashing
+ * ```
+ */
+export function computeBodyHash(html) {
+    return computeBodyHashNative(typeof html === 'string' ? Buffer.from(html, 'utf8') : html);
+}

package/lib/cache/clear-archive-cache-entry.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Remove a single archive cache entry (one directory previously resolved by
+ * {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}).
+ *
+ * Deliberately scoped to exactly `cacheDir` — it never touches sibling
+ * entries under the same cache root (in particular, other archives'
+ * tar-cache entries must survive a per-archive clear).
+ * @param cacheDir - Absolute path to the single cache entry to remove.
+ * @returns `true` if `cacheDir` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const cacheKey = await computeArchiveCacheKey(archivePath);
+ * const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath);
+ * const removed = await clearArchiveCacheEntry(cacheDir);
+ * ```
+ */
+export declare function clearArchiveCacheEntry(cacheDir: string): Promise<boolean>;

package/lib/cache/clear-archive-cache-entry.js

@@ -0,0 +1,24 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove a single archive cache entry (one directory previously resolved by
+ * {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}).
+ *
+ * Deliberately scoped to exactly `cacheDir` — it never touches sibling
+ * entries under the same cache root (in particular, other archives'
+ * tar-cache entries must survive a per-archive clear).
+ * @param cacheDir - Absolute path to the single cache entry to remove.
+ * @returns `true` if `cacheDir` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const cacheKey = await computeArchiveCacheKey(archivePath);
+ * const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath);
+ * const removed = await clearArchiveCacheEntry(cacheDir);
+ * ```
+ */
+export async function clearArchiveCacheEntry(cacheDir) {
+    const existedBefore = await pathExists(cacheDir);
+    await fs.rm(cacheDir, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/cache/clear-archive-cache-root.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. a leftover `table` scratch
+ * directory from an earlier version).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export declare function clearArchiveCacheRoot(cacheRoot: string): Promise<boolean>;

package/lib/cache/clear-archive-cache-root.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. a leftover `table` scratch
+ * directory from an earlier version).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export async function clearArchiveCacheRoot(cacheRoot) {
+    const existedBefore = await pathExists(cacheRoot);
+    await fs.rm(cacheRoot, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/cache/compute-archive-cache-key.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Derive a stable cache key from a `.nitpicker` file's inode metadata
+ * plus a head+tail content sample.
+ *
+ * Metadata fields:
+ *
+ * - `size` covers the bulk of accidental cache collision risk in O(1).
+ * - `mtime` (mod time) changes whenever the file's content is rewritten,
+ *   which is the common case (`crawl --append`, `crawl --retry-failed`,
+ *   `cp -f`, rsync).
+ * - `ctime` (inode change time) closes the `touch -m -t <past>` loophole:
+ *   even if a user resets `mtime` to fake "unchanged", the act of touching
+ *   bumps `ctime` on POSIX so the key still rolls.
+ *
+ * Head + tail digest:
+ *
+ * - On filesystems with low-resolution timestamps (FAT / exFAT / NFSv3 /
+ *   some Docker volume mounts where mtime/ctime are second-granular and
+ *   sometimes don't move on small appends), `size + mtime + ctime`
+ *   alone can stay identical across an in-place rewrite.
+ * - We hash the first {@link HEAD_TAIL_HASH_BYTES} bytes and the last
+ *   {@link HEAD_TAIL_HASH_BYTES} bytes to detect this. For tar archives
+ *   the head holds the first entry's header (which moves when the inner
+ *   directory's name changes) and the tail holds the last entry's data
+ *   (which always moves on `crawl --append` because the appended pages
+ *   land near the end of the tar stream).
+ * - Full-content sha256 was rejected: ~20-30 s on a 10 GB archive,
+ *   which is slower than the untar this cache is meant to avoid.
+ *
+ * The key is stable across symlinks (the caller resolves via
+ * `fs.realpath` upstream) but **not** across hardlinks pointing at a
+ * mutated inode — that is intentional, hardlinking is a power-user
+ * move and the user is expected to know the cache will share an entry.
+ * @param archivePath - Absolute path to the `.nitpicker` file.
+ * @returns A string of the form
+ *   `<size>-<mtime_ns>-<ctime_ns>-<headHex>-<tailHex>` suitable for use
+ *   as a directory-name component.
+ */
+export declare function computeArchiveCacheKey(archivePath: string): Promise<string>;

package/lib/cache/compute-archive-cache-key.js

@@ -0,0 +1,95 @@
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import fs from 'node:fs/promises';
+/**
+ * Number of bytes hashed from each end of the file to corner the case
+ * where two different archives share `size + mtime + ctime` by accident.
+ *
+ * 64 KiB is a sweet spot: on a 10 GB archive it reads 0.0006 % of the
+ * file (~1 ms on an SSD, ~5 ms on a spinning disk) while reliably
+ * sampling enough of both the tar leading header AND the tail (which
+ * for tar archives contains the last-written entry's data, so any real
+ * mutation moves at least one of the two windows).
+ */
+const HEAD_TAIL_HASH_BYTES = 64 * 1024;
+/**
+ * Cache-key segment substituted when a file is smaller than the
+ * head+tail sample window. In that case the head segment alone already
+ * spans the whole file (size column also rolls), so a separate tail
+ * digest is meaningless and we encode that explicitly rather than
+ * silently overlapping reads.
+ */
+const TAIL_NOT_NEEDED = 'short';
+/**
+ * Derive a stable cache key from a `.nitpicker` file's inode metadata
+ * plus a head+tail content sample.
+ *
+ * Metadata fields:
+ *
+ * - `size` covers the bulk of accidental cache collision risk in O(1).
+ * - `mtime` (mod time) changes whenever the file's content is rewritten,
+ *   which is the common case (`crawl --append`, `crawl --retry-failed`,
+ *   `cp -f`, rsync).
+ * - `ctime` (inode change time) closes the `touch -m -t <past>` loophole:
+ *   even if a user resets `mtime` to fake "unchanged", the act of touching
+ *   bumps `ctime` on POSIX so the key still rolls.
+ *
+ * Head + tail digest:
+ *
+ * - On filesystems with low-resolution timestamps (FAT / exFAT / NFSv3 /
+ *   some Docker volume mounts where mtime/ctime are second-granular and
+ *   sometimes don't move on small appends), `size + mtime + ctime`
+ *   alone can stay identical across an in-place rewrite.
+ * - We hash the first {@link HEAD_TAIL_HASH_BYTES} bytes and the last
+ *   {@link HEAD_TAIL_HASH_BYTES} bytes to detect this. For tar archives
+ *   the head holds the first entry's header (which moves when the inner
+ *   directory's name changes) and the tail holds the last entry's data
+ *   (which always moves on `crawl --append` because the appended pages
+ *   land near the end of the tar stream).
+ * - Full-content sha256 was rejected: ~20-30 s on a 10 GB archive,
+ *   which is slower than the untar this cache is meant to avoid.
+ *
+ * The key is stable across symlinks (the caller resolves via
+ * `fs.realpath` upstream) but **not** across hardlinks pointing at a
+ * mutated inode — that is intentional, hardlinking is a power-user
+ * move and the user is expected to know the cache will share an entry.
+ * @param archivePath - Absolute path to the `.nitpicker` file.
+ * @returns A string of the form
+ *   `<size>-<mtime_ns>-<ctime_ns>-<headHex>-<tailHex>` suitable for use
+ *   as a directory-name component.
+ */
+export async function computeArchiveCacheKey(archivePath) {
+    const stats = await fs.stat(archivePath, { bigint: true });
+    const fileSize = Number(stats.size);
+    const headHex = await sha256OfRange(archivePath, 0, Math.min(HEAD_TAIL_HASH_BYTES, fileSize));
+    const tailHex = fileSize > HEAD_TAIL_HASH_BYTES
+        ? await sha256OfRange(archivePath, fileSize - HEAD_TAIL_HASH_BYTES, HEAD_TAIL_HASH_BYTES)
+        : TAIL_NOT_NEEDED;
+    return `${stats.size}-${stats.mtimeNs}-${stats.ctimeNs}-${headHex}-${tailHex}`;
+}
+/**
+ * SHA-256 the requested byte range of a file via a single read stream.
+ * Truncated to 16 hex chars (64 bits) because the digest only needs to
+ * disambiguate within a `(size, mtime, ctime)` bucket — full 256-bit
+ * fingerprints would only waste filesystem path budget.
+ * @param filePath - Absolute path to the file.
+ * @param start - Inclusive byte offset to begin reading at.
+ * @param length - Number of bytes to read; 0 returns the empty-input digest.
+ * @returns A 16-character hex string.
+ */
+async function sha256OfRange(filePath, start, length) {
+    const hash = createHash('sha256');
+    if (length === 0) {
+        return hash.digest('hex').slice(0, 16);
+    }
+    await new Promise((resolve, reject) => {
+        const stream = createReadStream(filePath, {
+            start,
+            end: start + length - 1,
+        });
+        stream.on('data', (chunk) => hash.update(chunk));
+        stream.on('end', () => resolve());
+        stream.on('error', (error) => reject(error));
+    });
+    return hash.digest('hex').slice(0, 16);
+}

package/lib/cache/extract-archive-to-cache.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Untar a `.nitpicker` archive into the given cache directory.
+ *
+ * Concurrency / re-entry contract:
+ *
+ * - Two viewers opening the same archive race for the cache lock; the
+ *   loser waits and recheck-loops until the winner writes the ready
+ *   marker, then short-circuits.
+ * - Same-process concurrent callers dedupe through
+ *   `inFlightByCacheDir` so they share one extraction promise.
+ *
+ * Per-archive staging:
+ *
+ * - The tar's inner directory is extracted into `${cacheDir}.staging/`
+ *   first, then atomically renamed into `cacheDir`. Crucially, the
+ *   staging path is keyed off `cacheDir` (not the tar's inner directory
+ *   name) so two archives that happen to share an inner-dir name never
+ *   collide on the same staging slot.
+ *
+ * Integrity guard:
+ *
+ * - After rename, we run the writer-side `Database.connect` so all
+ *   migrations apply (column adds, new tables) and the version check
+ *   fires before the ready marker is written. A corrupt or incompatible
+ *   archive therefore poisons no cache entry — the cacheDir is removed
+ *   and the next caller retries from scratch.
+ *
+ * TOCTOU guard:
+ *
+ * - The cache key is recomputed from the file's stat AFTER the untar
+ *   completes. If the file changed mid-flight (a concurrent
+ *   `crawl --append` rewrote it), the freshly-landed cache contents do
+ *   not correspond to the original key, so we abort and remove the
+ *   cacheDir so the caller upstream sees the inconsistency rather than
+ *   serving misattributed data.
+ * @param archivePath - Absolute path to the source `.nitpicker` file.
+ * @param cacheRoot - Absolute path to the cache root directory.
+ * @param cacheDir - Absolute path the extracted contents should end up at.
+ * @param cacheKey - The cache key used to derive `cacheDir`. Recomputed
+ *   after extraction to detect concurrent writers; must match.
+ * @param onExtractProgress - Called during the untar step with bytes read
+ *   so far and the archive's total size (issue #294: a cold cache on a
+ *   large archive can take tens of seconds with no other signal it isn't
+ *   hung). Never called on a cache hit — {@link isCacheDirReady} short-
+ *   circuits before `untar` runs — so callers can treat "never invoked" as
+ *   the definition of a hit and skip printing a phase label until the
+ *   first call actually arrives. Only the first concurrent caller for a
+ *   given `cacheDir` sees callbacks; same-`cacheDir` callers deduped
+ *   through {@link inFlightByCacheDir} just await the shared promise.
+ * @param onLog - Forwarded to the cache-miss writer connection
+ *   (`runMigrationsOnCacheDir`'s `Database.connect`) so a legacy archive's
+ *   self-healing schema migrations do not fall back to a bare
+ *   `console.error` mid-redraw of a caller's display (issue #294). Omit for
+ *   that fallback — see `Archive.openCached`'s own `onLog` doc.
+ * @returns Resolves once `cacheDir` is ready to be opened read-only.
+ */
+export declare function extractArchiveToCache(archivePath: string, cacheRoot: string, cacheDir: string, cacheKey: string, onExtractProgress?: (readBytes: number, totalBytes: number) => void, onLog?: (message: string) => void): Promise<void>;