@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/derive-lineage-from-parent.d.ts

@@ -0,0 +1,37 @@
+import type { PageSource } from './types.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `#getIdByUrl` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export declare function deriveLineageFromParent(parentSource: PageSource | undefined, fallback: PageSource | undefined): PageSource | undefined;

package/lib/archive/derive-lineage-from-parent.js

@@ -0,0 +1,42 @@
+import { isInventorySource } from './is-inventory-source.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `#getIdByUrl` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export function deriveLineageFromParent(parentSource, fallback) {
+    if (isInventorySource(parentSource)) {
+        return 'inventory-discovered';
+    }
+    return fallback;
+}

package/lib/archive/filesystem/peek-tar-top-dir.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Reads the top-level directory name from a `.nitpicker` tar archive.
+ *
+ * Why: a `.nitpicker` is a tar that wraps `<basename>/db.sqlite` (and
+ * historically other files). `Archive.write()` names the inner directory
+ * from the file's basename at write time, but users routinely rename
+ * `.nitpicker` files after the fact (e.g. `mv X.migrated.nitpicker
+ * X.nitpicker`) — a perfectly reasonable filesystem operation that breaks
+ * any code that recomputes the inner-dir name from the outer filename.
+ *
+ * This helper scans the tar's entry list and returns the first top-level
+ * directory it sees, so callers (`Archive.open`, the migration script) can
+ * use the actual inner name regardless of what the outer file is called.
+ *
+ * Reads the tar's central directory only; does not extract any data. Runs
+ * in O(number of entries) but stops at the first match, so a typical
+ * archive (1–2 top-level entries) completes in a single read.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry
+ *   (e.g. file is empty, corrupted, or non-tar). The error message
+ *   identifies the path for diagnostics.
+ * @example
+ * // The user renamed `original.nitpicker` to `renamed.nitpicker`. The
+ * // inner directory was baked in at write time and is unaffected:
+ * await peekTarTopDir('renamed.nitpicker'); // → 'original'
+ */
+export declare function peekTarTopDir(tarFilePath: string): Promise<string>;

package/lib/archive/filesystem/peek-tar-top-dir.js

@@ -0,0 +1,65 @@
+import { list } from 'tar';
+/**
+ * Reads the top-level directory name from a `.nitpicker` tar archive.
+ *
+ * Why: a `.nitpicker` is a tar that wraps `<basename>/db.sqlite` (and
+ * historically other files). `Archive.write()` names the inner directory
+ * from the file's basename at write time, but users routinely rename
+ * `.nitpicker` files after the fact (e.g. `mv X.migrated.nitpicker
+ * X.nitpicker`) — a perfectly reasonable filesystem operation that breaks
+ * any code that recomputes the inner-dir name from the outer filename.
+ *
+ * This helper scans the tar's entry list and returns the first top-level
+ * directory it sees, so callers (`Archive.open`, the migration script) can
+ * use the actual inner name regardless of what the outer file is called.
+ *
+ * Reads the tar's central directory only; does not extract any data. Runs
+ * in O(number of entries) but stops at the first match, so a typical
+ * archive (1–2 top-level entries) completes in a single read.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry
+ *   (e.g. file is empty, corrupted, or non-tar). The error message
+ *   identifies the path for diagnostics.
+ * @example
+ * // The user renamed `original.nitpicker` to `renamed.nitpicker`. The
+ * // inner directory was baked in at write time and is unaffected:
+ * await peekTarTopDir('renamed.nitpicker'); // → 'original'
+ */
+export async function peekTarTopDir(tarFilePath) {
+    let found = null;
+    await list({
+        file: tarFilePath,
+        onReadEntry: (entry) => {
+            if (found !== null)
+                return;
+            // Only Directory entries count. Files at the tar root would
+            // otherwise win — and macOS BSD tar embeds AppleDouble (`._*`)
+            // sidecar files at the tar root for resource forks; those are
+            // File entries that BSD `tar -tf` hides but Node's `tar`
+            // surfaces verbatim, so without this filter the resource-fork
+            // of the real top-level dir would win instead of the dir itself.
+            if (entry.type !== 'Directory')
+                return;
+            // `entry.path` is the path inside the tar. Strip any leading
+            // `./` (some tar producers prepend it) before splitting.
+            const cleaned = entry.path.replace(/^\.\//, '');
+            const top = cleaned.split('/')[0];
+            if (!top || top === '')
+                return;
+            // Defense in depth: skip AppleDouble directory entries
+            // (`._foo/`) and pax extended header entries
+            // (`PaxHeaders.NNN/...` / `@PaxHeader`). Real `.nitpicker`
+            // archives never name their top dir with these prefixes.
+            if (top.startsWith('._'))
+                return;
+            if (top.startsWith('PaxHeaders') || top.startsWith('@PaxHeader'))
+                return;
+            found = top;
+        },
+    });
+    if (found === null) {
+        throw new Error(`Tar contains no top-level directory entry: ${tarFilePath}`);
+    }
+    return found;
+}

package/lib/archive/get-failed-page-messages.d.ts

@@ -0,0 +1,43 @@
+import type { Knex } from 'knex';
+/**
+ * Bulk-resolve a raw error message for each given page id, using only sources
+ * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
+ * `pageId`, the most direct signal a scrape attempt recorded), then
+ * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
+ * that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. `migrateCrawlErrors` creates the `crawl_errors` table on
+ * writer connect but does NOT back-fill historical lines from `error.log`,
+ * so an archive that predates the `crawl_errors` schema and whose failures
+ * live only in `error.log` will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, message>` populated only for ids whose message was
+ *   found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, string>>;

package/lib/archive/get-failed-page-messages.js

@@ -0,0 +1,131 @@
+/** chunk size for `WHERE … IN (?)` SQLite parameter packing. */
+const CHUNK_SIZE = 500;
+/**
+ * Bulk-resolve a raw error message for each given page id, using only sources
+ * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
+ * `pageId`, the most direct signal a scrape attempt recorded), then
+ * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
+ * that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. `migrateCrawlErrors` creates the `crawl_errors` table on
+ * writer connect but does NOT back-fill historical lines from `error.log`,
+ * so an archive that predates the `crawl_errors` schema and whose failures
+ * live only in `error.log` will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, message>` populated only for ids whose message was
+ *   found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export async function getFailedPageMessages(instance, ids, urls) {
+    if (ids.length === 0) {
+        return new Map();
+    }
+    if (ids.length !== urls.length) {
+        throw new Error(`getFailedPageMessages: ids.length (${ids.length}) !== urls.length (${urls.length}) — must be 1:1`);
+    }
+    const messageByPageId = new Map();
+    if (await instance.schema.hasTable('page_errors')) {
+        for (let i = 0; i < ids.length; i += CHUNK_SIZE) {
+            const chunk = ids.slice(i, i + CHUNK_SIZE);
+            // `orderBy('id', 'asc')` makes the per-pageId "first row seen"
+            // behavior deterministic regardless of SQLite's natural ROWID
+            // scan order, which is otherwise implementation-defined under
+            // concurrent / migrated archives. Without the explicit order, a
+            // reset that classified a page as `unknown` once could classify
+            // it as `parse-error` on the next run when the rows happen to be
+            // returned in a different order.
+            const rows = (await instance('page_errors')
+                .select('pageId', 'message')
+                .whereIn('pageId', chunk)
+                .orderBy('id', 'asc'));
+            for (const row of rows) {
+                // Earliest-id wins. Schema permits multiple rows per pageId
+                // (the same scrape can record several phase errors); the
+                // first row inserted is usually the trigger cause, later
+                // rows are follow-on noise from the same failure cascade.
+                //
+                // An empty `message` is treated as "no signal" and ignored
+                // so the crawl_errors lookup can fill it in. Without this,
+                // a page_errors row with `message=''` (recorded by a
+                // scraper phase that fired its trigger but had no error
+                // text) would short-circuit and we'd lose access to the
+                // crawl_errors row that classifies the failure as
+                // `dns` / `tls` / `client-blocked` etc. — defeating
+                // `--retry-failed`'s permanent-kind exclusion.
+                if (row.message !== '' && !messageByPageId.has(row.pageId)) {
+                    messageByPageId.set(row.pageId, row.message);
+                }
+            }
+        }
+    }
+    const idsMissing = ids.filter((id) => !messageByPageId.has(id));
+    if (idsMissing.length === 0 || !(await instance.schema.hasTable('crawl_errors'))) {
+        // Early-exit short-circuits BOTH the idToUrl Map construction and
+        // the crawl_errors round-trip. On a 1M-page archive where every
+        // failed page already has a `page_errors` row, this avoids walking
+        // the candidate list a second time.
+        return messageByPageId;
+    }
+    const idToUrl = new Map();
+    for (const [i, id] of ids.entries()) {
+        const url = urls[i];
+        if (url !== undefined) {
+            idToUrl.set(id, url);
+        }
+    }
+    const missingUrls = [];
+    for (const id of idsMissing) {
+        const url = idToUrl.get(id);
+        if (url !== undefined) {
+            missingUrls.push(url);
+        }
+    }
+    const urlToMessage = new Map();
+    for (let i = 0; i < missingUrls.length; i += CHUNK_SIZE) {
+        const chunk = missingUrls.slice(i, i + CHUNK_SIZE);
+        const rows = (await instance('crawl_errors')
+            .select('url', 'message')
+            .whereIn('url', chunk));
+        for (const row of rows) {
+            if (row.url !== null && !urlToMessage.has(row.url)) {
+                urlToMessage.set(row.url, row.message);
+            }
+        }
+    }
+    for (const id of idsMissing) {
+        const url = idToUrl.get(id);
+        if (url === undefined) {
+            continue;
+        }
+        const message = urlToMessage.get(url);
+        if (message !== undefined) {
+            messageByPageId.set(id, message);
+        }
+    }
+    return messageByPageId;
+}

package/lib/archive/init-schema.d.ts

@@ -1,8 +1,59 @@
 import type { Knex } from 'knex';
+/**
+ * Applies the connection-level PRAGMAs that govern foreign-key enforcement
+ * and BLOB-read performance. These are **per-connection** settings (libsql
+ * resets them when a new connection is opened), so they must be reapplied
+ * every time `Database.connect` runs — not just on first-time schema
+ * initialization. Keeping them separate from `initSchema`'s one-shot path
+ * also lets `page_size` (which only takes effect against an empty DB)
+ * stay gated behind the existence check.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function applyConnectionPragmas(instance: Knex): Promise<void>;
 /**
  * Initializes the archive database schema if tables do not exist.
- * Enables WAL journal mode and foreign keys, then creates all tables
- * (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
+ *
+ * Schema notes:
+ *
+ * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
+ *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
+ *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
+ *   JSON column for everything not flattened. URL-shaped columns are
+ *   absolutised against the page URL before write (see
+ *   `archive/meta/derive-flat-from-meta.ts`).
+ * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
+ *   `tags_providers_csv`): computed at write time from `meta.tags` /
+ *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
+ *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
+ * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
+ *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
+ *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
+ *   leaves them alone so failure-reset does not erase the last-success
+ *   record.
+ * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
+ *   `categories`/`sources` JSON columns. Compound indexes
+ *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
+ *   Phase 2+ "find duplicate IDs across pages" and "list pages using
+ *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
+ * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
+ *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
+ *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
+ *   if cross-archive bulk export becomes a use case, add a `codec` column
+ *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
+ *   `list_pages_by_jsonld_type` JOINs.
+ * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
+ *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
+ *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
+ *   express it.
+ * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ *   `CREATE TABLE` because SQLite only honors `page_size` changes against
+ *   an empty database, and `journal_mode = WAL` is persistent. Other
+ *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects pre-0.10 archives with a friendly
+ * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
+ * policy allows breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export declare function initSchema(instance: Knex): Promise<void>;