@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/cache/resolve-archive-cache-dir.js

@@ -0,0 +1,32 @@
+import path from 'node:path';
+/**
+ * Build the absolute path to a single archive's cache directory.
+ *
+ * The directory name is `<cacheKey>-<safeBasename>`:
+ *
+ * - `cacheKey` carries the freshness signal (size/mtime/ctime) and is
+ *   what actually drives cache hit vs miss.
+ * - `safeBasename` is appended purely so a human running `ls` on the
+ *   cache root can recognise which archive an entry belongs to. It is
+ *   NOT used to disambiguate keys — two archives with identical inode
+ *   metadata (e.g. an identical copy under a different name) intentionally
+ *   share an entry under the first basename that landed there.
+ *
+ * The basename is sanitised: anything outside `[A-Za-z0-9._-]` becomes
+ * `_`. This keeps the path portable across filesystems (no spaces,
+ * unicode normalisation surprises, Windows-reserved chars) and removes
+ * any chance that a crafted archive name could escape the cache root
+ * (e.g. via `..` or path separators), independent of the upstream
+ * `path.basename` call that already drops directory components.
+ * @param cacheRoot - Absolute path returned by `getArchiveCacheRoot()`.
+ * @param cacheKey - The freshness key from `computeArchiveCacheKey()`.
+ * @param archivePath - Absolute path to the source `.nitpicker` file;
+ *   only its basename contributes to the cache entry name.
+ * @returns Absolute path to the per-archive cache directory.
+ */
+export function resolveArchiveCacheDir(cacheRoot, cacheKey, archivePath) {
+    const rawBasename = path.basename(archivePath, path.extname(archivePath));
+    const safeBasename = rawBasename.replaceAll(/[^\w.-]+/g, '_').slice(0, 80);
+    const dirName = safeBasename.length > 0 ? `${cacheKey}-${safeBasename}` : cacheKey;
+    return path.resolve(cacheRoot, dirName);
+}

package/lib/cache/types.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Classification of a top-level entry found directly under an archive cache
+ * root ({@link import('./get-archive-cache-root.js').getArchiveCacheRoot}).
+ *
+ * - `'tar-cache'`: a per-archive extraction produced by
+ *   {@link import('./resolve-archive-cache-dir.js').resolveArchiveCacheDir}
+ *   (`<cacheKey>-<safeBasename>`).
+ * - `'orphan'`: a `.staging` or `.corrupt.<pid>.<n>` leftover from an
+ *   interrupted {@link import('./extract-archive-to-cache.js').extractArchiveToCache}
+ *   run (see that file's quarantine/staging logic), or a directory literally
+ *   named `table` — the scratch cache earlier versions kept beside the
+ *   tar-cache entries, which nothing reads or writes any more.
+ * - `'unknown'`: anything else (a stray file, a foreign directory a user
+ *   dropped into the cache root).
+ *
+ * `'tar-cache'` and the bare `table` orphan cannot collide:
+ * `resolveArchiveCacheDir` always prefixes the cache key (digits and hyphens),
+ * so a tar-cache directory name is never the bare literal `table`.
+ */
+export type ArchiveCacheEntryKind = 'tar-cache' | 'orphan' | 'unknown';
+/** A single top-level entry found under an archive cache root. */
+export interface ArchiveCacheEntry {
+    /** How this entry was classified by name pattern. */
+    readonly kind: ArchiveCacheEntryKind;
+    /** Base name of the entry (its final path segment). */
+    readonly name: string;
+    /** Absolute path to the entry. */
+    readonly path: string;
+    /** Total size in bytes of all regular files found under this entry (recursive for directories). */
+    readonly sizeBytes: number;
+    /** Most recent mtime (epoch ms) found among the entry's files, or the entry's own mtime if it has none. */
+    readonly mtimeMs: number;
+}

package/lib/cache/types.js

@@ -0,0 +1 @@
+export {};

package/lib/content-type/is-html-content-type.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045), and values captured from
+ * Puppeteer responses preserve the server's original casing, so the
+ * comparison must normalize case — `text/HTML` is HTML. Surrounding
+ * whitespace (e.g. `text/html ` left over after parameter stripping)
+ * is also tolerated.
+ *
+ * This is the single source of truth for HTML detection — `Page.isPage()`
+ * and the link list delegate here so the classification never diverges
+ * between code paths.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ */
+export declare function isHtmlContentType(contentType: string | null): boolean;

package/lib/content-type/is-html-content-type.js

@@ -0,0 +1,19 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045), and values captured from
+ * Puppeteer responses preserve the server's original casing, so the
+ * comparison must normalize case — `text/HTML` is HTML. Surrounding
+ * whitespace (e.g. `text/html ` left over after parameter stripping)
+ * is also tolerated.
+ *
+ * This is the single source of truth for HTML detection — `Page.isPage()`
+ * and the link list delegate here so the classification never diverges
+ * between code paths.
+ * @param contentType - The media type portion of a Content-Type header
+ *   (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ */
+export function isHtmlContentType(contentType) {
+    return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}

package/lib/content-type/normalize-content-type.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Canonicalizes a Content-Type media type for storage.
+ *
+ * MIME types are case-insensitive (RFC 2045) and may arrive with surrounding
+ * whitespace (e.g. `text/html ` left after stripping the `; charset=...`
+ * parameter). Responses are recorded verbatim (`header.split(';')[0]`) without
+ * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
+ * database. Storing the canonical (trimmed, lower-cased) form lets the exact
+ * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
+ * code-level `isHtmlContentType` check, which trims and lower-cases.
+ * @param contentType - The raw media type, or `null` when unknown.
+ * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
+ */
+export declare function normalizeContentType(contentType: string | null): string | null;

package/lib/content-type/normalize-content-type.js

@@ -0,0 +1,20 @@
+/**
+ * Canonicalizes a Content-Type media type for storage.
+ *
+ * MIME types are case-insensitive (RFC 2045) and may arrive with surrounding
+ * whitespace (e.g. `text/html ` left after stripping the `; charset=...`
+ * parameter). Responses are recorded verbatim (`header.split(';')[0]`) without
+ * normalization, so `Text/HTML` or `text/html ` can otherwise reach the
+ * database. Storing the canonical (trimmed, lower-cased) form lets the exact
+ * SQL page-ness predicate (`WHERE contentType = 'text/html'`) agree with the
+ * code-level `isHtmlContentType` check, which trims and lower-cases.
+ * @param contentType - The raw media type, or `null` when unknown.
+ * @returns The trimmed, lower-cased media type, or `null` when unknown/blank.
+ */
+export function normalizeContentType(contentType) {
+    if (contentType === null) {
+        return null;
+    }
+    const normalized = contentType.trim().toLowerCase();
+    return normalized === '' ? null : normalized;
+}

package/lib/create-adjunct-tables.d.ts

@@ -0,0 +1,66 @@
+import type { Knex } from 'knex';
+/**
+ * Creates the adjunct tables that hang off `content_items` (plus the two
+ * standalone log tables), guarded per-table so partially-provisioned
+ * archives converge to the full set:
+ *
+ * - `page_errors` — partial scrape failures, FK → `content_items(id)`
+ * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
+ *   an external link that failed DNS, or null for a process-level error)
+ * - `technology_signals` — one un-combined technology-detection signal
+ *   (Wappalyzer or structural) per row, FK → `content_items(id)`
+ * - `page_technologies` — confidence-combined roll-up of
+ *   `technology_signals`, one row per (page, technology), FK →
+ *   `content_items(id)`. Supersedes the removed `page_tags` (Wappalyzer-only)
+ * - `technology_js_scan_cache` — resource-scoped cache for the post-crawl
+ *   JS license-comment scan, FK → `resource_items(id)`
+ * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
+ *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
+ *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
+ *   `content_items(id)`
+ * - `page_main_content_custom_elements` — Web Components (custom elements)
+ *   found in the main-content region, one row per DOM element, FK →
+ *   `content_items(id)`. Unlike its eight siblings above, captured by
+ *   nitpicker itself (`@nitpicker/crawler`'s `crawler/capture-custom-elements.ts`), not beholder
+ * - `list_reconcile_runs` — `--inventory` / `--recrawl` audit log (no FK;
+ *   append-only)
+ * - `network_outages` — operator-network-outage journal (no FK; append-only
+ *   except `ended_at`, which is written once on recovery)
+ * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
+ *   FK; append-only except `rejected_count`, which is written once at
+ *   `crawlEnd`)
+ * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
+ *   snapshots, FK → `content_items(id)`
+ * - `console_log_items` — content-addressable dictionary of distinct
+ *   console messages / page errors (no FK; hash-deduplicated across every
+ *   page in the archive, mirroring `text_refs` / `json_refs`)
+ * - `page_console_logs` — one row per (page, console log) occurrence, FK →
+ *   `content_items(id)` and `console_log_items(id)`
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this right after `createEntityTables`) and the migration script
+ * (`scripts/migrate-to-0.13.mjs` calls it before retargeting FK
+ * declarations and dropping the legacy tables). Keeping the schema in one
+ * function guarantees both origin points produce identical tables — the
+ * pre-0.13 era kept per-table copies of this DDL in separate lazy-migration
+ * modules, and those copies drifted: they declared `REFERENCES pages(id)`
+ * while `initSchema` had moved on to `content_items(id)`, leaving migrated
+ * archives with stale FK targets that only `scripts/migrate-to-0.13.mjs`'s
+ * rename-copy-drop pass can now repair.
+ *
+ * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
+ * a single sentinel table), each table here is guarded individually because
+ * the migration-script caller sees archives where any subset may already
+ * exist (e.g. `page_jsonld` from the 0.10 migration but no `list_reconcile_runs`).
+ * Index creation stays inside each guard: an existing table keeps whatever
+ * indexes its creation path declared.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * // Must run after createEntityTables — the page-scoped tables FK into
+ * // content_items(id).
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await createAdjunctTables(db);
+ */
+export declare function createAdjunctTables(instance: Knex): Promise<void>;