@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/archive/body-hash/mask-dynamic-ids.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Replaces mixed-alphanumeric tokens of 8+ characters with a fixed
+ * placeholder, so that two bodies differing only in an embedded dynamic
+ * value (a cache-busting hash, a per-build CSS-module suffix, a session or
+ * order id) hash the same.
+ *
+ * A token is masked only when it mixes letters and digits — a pure-digit
+ * match (phone numbers, product codes, dates) or a pure-alphabetic match (an
+ * ordinary English word) is left untouched, since those are far more likely
+ * to be stable content than a dynamic identifier.
+ *
+ * This is a heuristic, not a semantic classifier, and it is accepted as such:
+ * a mixed-alphanumeric SKU/order/product code that is itself the only
+ * distinguishing content between two otherwise-identical pages (e.g.
+ * `ABC12345` vs `XYZ98765`) also gets masked, collapsing two genuinely
+ * different pages into one `computeBodyHash` duplicate-detection group — a
+ * false positive. Narrowing the pattern to reduce that risk would also
+ * narrow what it catches (the whole point of this mask is to absorb
+ * cache-busting hashes and CSS-module suffixes whose shape is
+ * indistinguishable from such a code), so this trade-off is deliberate
+ * rather than something a stricter regex would cleanly resolve.
+ *
+ * The placeholder is a fixed constant rather than a per-match incrementing
+ * id: nothing needs to reverse the substitution back to the original value —
+ * the full original HTML is already preserved verbatim in `page_html_blobs`,
+ * so recovering it here would be redundant. All that matters for duplicate
+ * detection is that every masked token collapses to the same value.
+ * @param text - Text to mask (already URL-normalized by
+ *   `normalizeUrlLikeStrings`).
+ * @returns `text` with every mixed-alphanumeric 8+ character token replaced
+ *   by a fixed placeholder.
+ */
+export declare function maskDynamicIds(text: string): string;

package/lib/archive/body-hash/mask-dynamic-ids.js

@@ -0,0 +1,44 @@
+const DYNAMIC_ID_PATTERN = /[a-z0-9]{8,}/gi;
+const DIGITS_ONLY_PATTERN = /^\d+$/;
+const ALPHA_ONLY_PATTERN = /^[a-z]+$/i;
+const MASK_PLACEHOLDER = '__MASKED_ID__';
+/**
+ * Replaces mixed-alphanumeric tokens of 8+ characters with a fixed
+ * placeholder, so that two bodies differing only in an embedded dynamic
+ * value (a cache-busting hash, a per-build CSS-module suffix, a session or
+ * order id) hash the same.
+ *
+ * A token is masked only when it mixes letters and digits — a pure-digit
+ * match (phone numbers, product codes, dates) or a pure-alphabetic match (an
+ * ordinary English word) is left untouched, since those are far more likely
+ * to be stable content than a dynamic identifier.
+ *
+ * This is a heuristic, not a semantic classifier, and it is accepted as such:
+ * a mixed-alphanumeric SKU/order/product code that is itself the only
+ * distinguishing content between two otherwise-identical pages (e.g.
+ * `ABC12345` vs `XYZ98765`) also gets masked, collapsing two genuinely
+ * different pages into one `computeBodyHash` duplicate-detection group — a
+ * false positive. Narrowing the pattern to reduce that risk would also
+ * narrow what it catches (the whole point of this mask is to absorb
+ * cache-busting hashes and CSS-module suffixes whose shape is
+ * indistinguishable from such a code), so this trade-off is deliberate
+ * rather than something a stricter regex would cleanly resolve.
+ *
+ * The placeholder is a fixed constant rather than a per-match incrementing
+ * id: nothing needs to reverse the substitution back to the original value —
+ * the full original HTML is already preserved verbatim in `page_html_blobs`,
+ * so recovering it here would be redundant. All that matters for duplicate
+ * detection is that every masked token collapses to the same value.
+ * @param text - Text to mask (already URL-normalized by
+ *   `normalizeUrlLikeStrings`).
+ * @returns `text` with every mixed-alphanumeric 8+ character token replaced
+ *   by a fixed placeholder.
+ */
+export function maskDynamicIds(text) {
+    return text.replaceAll(DYNAMIC_ID_PATTERN, (match) => {
+        if (DIGITS_ONLY_PATTERN.test(match) || ALPHA_ONLY_PATTERN.test(match)) {
+            return match;
+        }
+        return MASK_PLACEHOLDER;
+    });
+}

package/lib/archive/body-hash/normalize-url-like-strings.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Normalizes `/index.{ext}` path suffixes (`/index.html`, `/index.php`, ...)
+ * to a bare trailing `/` throughout the given text, so that two otherwise
+ * identical bodies that differ only in which equivalent URL form a template
+ * happened to render (`/about/` vs `/about/index.html`) hash the same.
+ *
+ * Applied as a blanket string sweep over the whole body — not scoped to
+ * `href`/`src` attribute values — because the same trailing-suffix variance
+ * can appear anywhere a URL-shaped string is rendered as text (breadcrumbs,
+ * "print this page" widgets, JSON embedded in an inline `<script>`), and the
+ * body is treated as an opaque string for masking purposes (see
+ * `computeBodyHash`).
+ * @param body - The `<body>` inner HTML (or any text) to normalize.
+ * @returns `body` with every `/index.{ext}` suffix collapsed to `/`.
+ */
+export declare function normalizeUrlLikeStrings(body: string): string;

package/lib/archive/body-hash/normalize-url-like-strings.js

@@ -0,0 +1,19 @@
+const INDEX_SUFFIX_PATTERN = /\/index\.\w+/gi;
+/**
+ * Normalizes `/index.{ext}` path suffixes (`/index.html`, `/index.php`, ...)
+ * to a bare trailing `/` throughout the given text, so that two otherwise
+ * identical bodies that differ only in which equivalent URL form a template
+ * happened to render (`/about/` vs `/about/index.html`) hash the same.
+ *
+ * Applied as a blanket string sweep over the whole body — not scoped to
+ * `href`/`src` attribute values — because the same trailing-suffix variance
+ * can appear anywhere a URL-shaped string is rendered as text (breadcrumbs,
+ * "print this page" widgets, JSON embedded in an inline `<script>`), and the
+ * body is treated as an opaque string for masking purposes (see
+ * `computeBodyHash`).
+ * @param body - The `<body>` inner HTML (or any text) to normalize.
+ * @returns `body` with every `/index.{ext}` suffix collapsed to `/`.
+ */
+export function normalizeUrlLikeStrings(body) {
+    return body.replaceAll(INDEX_SUFFIX_PATTERN, '/');
+}

package/lib/archive/cache/clear-archive-cache-entry.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Remove a single archive cache entry (one directory previously resolved by
+ * {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}).
+ *
+ * Deliberately scoped to exactly `cacheDir` — it never touches sibling
+ * entries under the same cache root (in particular, the `@nitpicker/core`
+ * analyze `table` cache is never archive-scoped and must survive a
+ * per-archive clear).
+ * @param cacheDir - Absolute path to the single cache entry to remove.
+ * @returns `true` if `cacheDir` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const cacheKey = await computeArchiveCacheKey(archivePath);
+ * const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath);
+ * const removed = await clearArchiveCacheEntry(cacheDir);
+ * ```
+ */
+export declare function clearArchiveCacheEntry(cacheDir: string): Promise<boolean>;

package/lib/archive/cache/clear-archive-cache-entry.js

@@ -0,0 +1,25 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove a single archive cache entry (one directory previously resolved by
+ * {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}).
+ *
+ * Deliberately scoped to exactly `cacheDir` — it never touches sibling
+ * entries under the same cache root (in particular, the `@nitpicker/core`
+ * analyze `table` cache is never archive-scoped and must survive a
+ * per-archive clear).
+ * @param cacheDir - Absolute path to the single cache entry to remove.
+ * @returns `true` if `cacheDir` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const cacheKey = await computeArchiveCacheKey(archivePath);
+ * const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath);
+ * const removed = await clearArchiveCacheEntry(cacheDir);
+ * ```
+ */
+export async function clearArchiveCacheEntry(cacheDir) {
+    const existedBefore = await pathExists(cacheDir);
+    await fs.rm(cacheDir, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/archive/cache/clear-archive-cache-root.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. the `@nitpicker/core` analyze
+ * `table` cache, which is resolved as a child of the same root).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export declare function clearArchiveCacheRoot(cacheRoot: string): Promise<boolean>;

package/lib/archive/cache/clear-archive-cache-root.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs/promises';
+import { pathExists } from './path-exists.js';
+/**
+ * Remove an entire archive cache root, including every tar-cache entry and
+ * any sibling directory living under it (e.g. the `@nitpicker/core` analyze
+ * `table` cache, which is resolved as a child of the same root).
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself, so
+ * tests can point it at a throwaway directory instead of the real OS temp
+ * cache. Mirrors the "manual `rm -rf` is safe" contract already documented
+ * for the tar cache (ARCHITECTURE.md) — no confirmation, no lock check.
+ *
+ * Why not lock-aware: a concurrent `extractArchiveToCache` elsewhere may be
+ * holding `<cacheDir>.lock` / writing `<cacheDir>.staging` inside this root
+ * when it is removed, which can surface as an ENOENT in that extraction (or,
+ * rarely, a second extractor racing into a freshly recreated `.staging`
+ * path). This is the same exposure a manual `rm -rf` already has today —
+ * this function does not add new risk, it just makes that pre-existing,
+ * accepted risk reachable via a single explicit command.
+ * @param cacheRoot - Absolute path to the cache root to remove.
+ * @returns `true` if `cacheRoot` existed and was removed, `false` if it was
+ *   already absent.
+ * @example
+ * ```ts
+ * const removed = await clearArchiveCacheRoot(getArchiveCacheRoot());
+ * ```
+ */
+export async function clearArchiveCacheRoot(cacheRoot) {
+    const existedBefore = await pathExists(cacheRoot);
+    await fs.rm(cacheRoot, { recursive: true, force: true });
+    return existedBefore;
+}

package/lib/archive/cache/list-archive-cache-entries.d.ts

@@ -0,0 +1,23 @@
+import type { ArchiveCacheEntry } from './types.js';
+/**
+ * List the top-level entries found directly under an archive cache root,
+ * with a recursive size total and the most recent mtime found in each.
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
+ * production root), so tests can point it at a throwaway directory instead
+ * of the real OS temp cache.
+ *
+ * Symbolic links are reported but never followed, both at the top level and
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
+ * and crashing on dangling links.
+ * @param cacheRoot - Absolute path to the cache root to inspect.
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
+ *   order. Returns `[]` if `cacheRoot` does not exist.
+ * @example
+ * ```ts
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
+ * ```
+ */
+export declare function listArchiveCacheEntries(cacheRoot: string): Promise<ArchiveCacheEntry[]>;

package/lib/archive/cache/list-archive-cache-entries.js

@@ -0,0 +1,125 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/**
+ * List the top-level entries found directly under an archive cache root,
+ * with a recursive size total and the most recent mtime found in each.
+ *
+ * Pure: takes `cacheRoot` as a parameter and never resolves it itself (see
+ * {@link import('./get-archive-cache-root.js').getArchiveCacheRoot} for the
+ * production root), so tests can point it at a throwaway directory instead
+ * of the real OS temp cache.
+ *
+ * Symbolic links are reported but never followed, both at the top level and
+ * while walking a directory's contents — this avoids escaping `cacheRoot`
+ * and crashing on dangling links.
+ * @param cacheRoot - Absolute path to the cache root to inspect.
+ * @returns One entry per top-level child of `cacheRoot`, in `fs.readdir`
+ *   order. Returns `[]` if `cacheRoot` does not exist.
+ * @example
+ * ```ts
+ * const entries = await listArchiveCacheEntries(getArchiveCacheRoot());
+ * const totalBytes = entries.reduce((sum, e) => sum + e.sizeBytes, 0);
+ * ```
+ */
+export async function listArchiveCacheEntries(cacheRoot) {
+    let topLevel;
+    try {
+        topLevel = await fs.readdir(cacheRoot, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+    const entries = [];
+    for (const dirent of topLevel) {
+        const entryPath = path.join(cacheRoot, dirent.name);
+        if (dirent.isSymbolicLink()) {
+            entries.push({
+                kind: 'unknown',
+                name: dirent.name,
+                path: entryPath,
+                sizeBytes: 0,
+                mtimeMs: 0,
+            });
+            continue;
+        }
+        if (dirent.isDirectory()) {
+            const { sizeBytes, mtimeMs } = await computeDirectoryStats(entryPath);
+            entries.push({
+                kind: classifyEntryName(dirent.name),
+                name: dirent.name,
+                path: entryPath,
+                sizeBytes,
+                mtimeMs,
+            });
+            continue;
+        }
+        const stat = await fs.stat(entryPath);
+        entries.push({
+            kind: 'unknown',
+            name: dirent.name,
+            path: entryPath,
+            sizeBytes: stat.size,
+            mtimeMs: stat.mtimeMs,
+        });
+    }
+    return entries;
+}
+/**
+ * Matches the corrupt-quarantine suffix `extractArchiveToCache` appends
+ * (`${cacheDir}.corrupt.${pid}.${counter}`, both segments always numeric —
+ * see `quarantineHalfPopulatedCacheDir` in `extract-archive-to-cache.ts`).
+ * Anchored and numeric-specific rather than a loose `includes('.corrupt.')`
+ * so an archive whose own (sanitized) basename merely contains the
+ * substring `.corrupt.` — e.g. `my.corrupt.report.nitpicker` — is not
+ * misclassified as an orphan and offered up for deletion by `cache list`.
+ */
+const CORRUPT_QUARANTINE_SUFFIX = /\.corrupt\.\d+\.\d+$/;
+/**
+ * Classify a cache-root child by name pattern alone (no filesystem access).
+ *
+ * This is a heuristic: `.staging`/corrupt-quarantine suffixes are appended
+ * by `extractArchiveToCache` onto an existing tar-cache dir name, so an
+ * archive whose own sanitized basename happens to end in exactly one of
+ * these literal suffixes is indistinguishable from a real orphan by name
+ * alone. Disambiguating fully would require re-deriving each entry's
+ * expected name from its source archive, which `list`/`clear` (by design,
+ * see grill-me scope) never has access to. Accepted as a rare, low-cost
+ * misclassification.
+ * @param name - Base name of the top-level entry.
+ */
+function classifyEntryName(name) {
+    if (name === 'table') {
+        return 'table';
+    }
+    if (name.endsWith('.staging') || CORRUPT_QUARANTINE_SUFFIX.test(name)) {
+        return 'orphan';
+    }
+    return 'tar-cache';
+}
+/**
+ * Recursively sum file sizes and find the most recent mtime under a
+ * directory. Symbolic links are neither followed nor counted.
+ * @param dirPath - Absolute path to the directory to walk.
+ */
+async function computeDirectoryStats(dirPath) {
+    let sizeBytes = 0;
+    let mtimeMs = 0;
+    const children = await fs.readdir(dirPath, { recursive: true, withFileTypes: true });
+    for (const child of children) {
+        if (!child.isFile()) {
+            continue;
+        }
+        const childPath = path.join(child.parentPath, child.name);
+        const stat = await fs.stat(childPath);
+        sizeBytes += stat.size;
+        mtimeMs = Math.max(mtimeMs, stat.mtimeMs);
+    }
+    if (mtimeMs === 0) {
+        const dirStat = await fs.stat(dirPath);
+        mtimeMs = dirStat.mtimeMs;
+    }
+    return { sizeBytes, mtimeMs };
+}

package/lib/archive/cache/path-exists.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Async existence probe — avoids blocking the event loop on the common
+ * "check before remove" path shared by
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
+ * @param targetPath - Absolute path to probe.
+ * @returns `true` if the path is reachable via `fs.access`.
+ */
+export declare function pathExists(targetPath: string): Promise<boolean>;

package/lib/archive/cache/path-exists.js

@@ -0,0 +1,18 @@
+import fs from 'node:fs/promises';
+/**
+ * Async existence probe — avoids blocking the event loop on the common
+ * "check before remove" path shared by
+ * {@link import('./clear-archive-cache-root.js').clearArchiveCacheRoot} and
+ * {@link import('./clear-archive-cache-entry.js').clearArchiveCacheEntry}.
+ * @param targetPath - Absolute path to probe.
+ * @returns `true` if the path is reachable via `fs.access`.
+ */
+export async function pathExists(targetPath) {
+    try {
+        await fs.access(targetPath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/lib/archive/cache/types.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Classification of a top-level entry found directly under an archive cache
+ * root ({@link import('./get-archive-cache-root.js').getArchiveCacheRoot}).
+ *
+ * - `'tar-cache'`: a per-archive extraction produced by
+ *   {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}
+ *   (`<cacheKey>-<safeBasename>`).
+ * - `'table'`: the `@nitpicker/core` analyze-plugin scratch cache
+ *   (`getTableCacheRoot()`), a sibling directory literally named `table`.
+ * - `'orphan'`: a `.staging` or `.corrupt.<pid>.<n>` leftover from an
+ *   interrupted {@link import('./extract-archive-to-cache.js').extractArchiveToCache}
+ *   run (see that file's quarantine/staging logic).
+ * - `'unknown'`: anything else (a stray file, a foreign directory a user
+ *   dropped into the cache root).
+ *
+ * `'tar-cache'` and `'table'` cannot collide: `resolveArchiveCacheDir` always
+ * prefixes the cache key (digits and hyphens), so a tar-cache directory name
+ * is never the bare literal `table`.
+ */
+export type ArchiveCacheEntryKind = 'tar-cache' | 'table' | 'orphan' | 'unknown';
+/** A single top-level entry found under an archive cache root. */
+export interface ArchiveCacheEntry {
+    /** How this entry was classified by name pattern. */
+    readonly kind: ArchiveCacheEntryKind;
+    /** Base name of the entry (its final path segment). */
+    readonly name: string;
+    /** Absolute path to the entry. */
+    readonly path: string;
+    /** Total size in bytes of all regular files found under this entry (recursive for directories). */
+    readonly sizeBytes: number;
+    /** Most recent mtime (epoch ms) found among the entry's files, or the entry's own mtime if it has none. */
+    readonly mtimeMs: number;
+}

package/lib/archive/cache/types.js

@@ -0,0 +1 @@
+export {};

package/lib/archive/create-adjunct-tables.d.ts

@@ -9,11 +9,29 @@ import type { Knex } from 'knex';
  *   an external link that failed DNS, or null for a process-level error)
  * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
  * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
+ *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
+ *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
+ *   `content_items(id)`
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
  * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
  *   FK → `content_items(id)`
+ * - `page_templates` — DOM-structure template classification (`--templates`,
+ *   `@nitpicker/core`'s `template-classification/`), one row per classified
+ *   page, FK → `content_items(id)`
+ * - `page_template_clusters` — one row per distinct `page_templates.template_key`,
+ *   holding `@d-zero/page-cluster`'s cluster-selection evidence (no FK;
+ *   `template_key` is not a `page_templates` FK target, so consistency is
+ *   maintained by replacing both tables together, not by a foreign key)
  * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
  *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
  *
  * The DDL is shared between fresh-archive provisioning ({@link initSchema}
  * calls this right after `createEntityTables`) and the migration script