@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/get-failed-page-messages.js

@@ -0,0 +1,132 @@
+/** chunk size for `WHERE … IN (?)` SQLite parameter packing. */
+const CHUNK_SIZE = 500;
+/**
+ * Bulk-resolve a raw error message for each given page id, using only sources
+ * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
+ * `pageId`, the most direct signal a scrape attempt recorded), then
+ * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
+ * that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. The `crawl_errors` table is created empty (by
+ * `initSchema` on fresh archives, by `scripts/migrate-to-0.13.mjs` on
+ * migrated ones) and historical lines from `error.log` are never
+ * back-filled, so an archive whose failures live only in `error.log`
+ * will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, message>` populated only for ids whose message was
+ *   found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export async function getFailedPageMessages(instance, ids, urls) {
+    if (ids.length === 0) {
+        return new Map();
+    }
+    if (ids.length !== urls.length) {
+        throw new Error(`getFailedPageMessages: ids.length (${ids.length}) !== urls.length (${urls.length}) — must be 1:1`);
+    }
+    const messageByPageId = new Map();
+    if (await instance.schema.hasTable('page_errors')) {
+        for (let i = 0; i < ids.length; i += CHUNK_SIZE) {
+            const chunk = ids.slice(i, i + CHUNK_SIZE);
+            // `orderBy('id', 'asc')` makes the per-pageId "first row seen"
+            // behavior deterministic regardless of SQLite's natural ROWID
+            // scan order, which is otherwise implementation-defined under
+            // concurrent / migrated archives. Without the explicit order, a
+            // reset that classified a page as `unknown` once could classify
+            // it as `parse-error` on the next run when the rows happen to be
+            // returned in a different order.
+            const rows = (await instance('page_errors')
+                .select('pageId', 'message')
+                .whereIn('pageId', chunk)
+                .orderBy('id', 'asc'));
+            for (const row of rows) {
+                // Earliest-id wins. Schema permits multiple rows per pageId
+                // (the same scrape can record several phase errors); the
+                // first row inserted is usually the trigger cause, later
+                // rows are follow-on noise from the same failure cascade.
+                //
+                // An empty `message` is treated as "no signal" and ignored
+                // so the crawl_errors lookup can fill it in. Without this,
+                // a page_errors row with `message=''` (recorded by a
+                // scraper phase that fired its trigger but had no error
+                // text) would short-circuit and we'd lose access to the
+                // crawl_errors row that classifies the failure as
+                // `dns` / `tls` / `client-blocked` etc. — defeating
+                // `--retry-failed`'s permanent-kind exclusion.
+                if (row.message !== '' && !messageByPageId.has(row.pageId)) {
+                    messageByPageId.set(row.pageId, row.message);
+                }
+            }
+        }
+    }
+    const idsMissing = ids.filter((id) => !messageByPageId.has(id));
+    if (idsMissing.length === 0 || !(await instance.schema.hasTable('crawl_errors'))) {
+        // Early-exit short-circuits BOTH the idToUrl Map construction and
+        // the crawl_errors round-trip. On a 1M-page archive where every
+        // failed page already has a `page_errors` row, this avoids walking
+        // the candidate list a second time.
+        return messageByPageId;
+    }
+    const idToUrl = new Map();
+    for (const [i, id] of ids.entries()) {
+        const url = urls[i];
+        if (url !== undefined) {
+            idToUrl.set(id, url);
+        }
+    }
+    const missingUrls = [];
+    for (const id of idsMissing) {
+        const url = idToUrl.get(id);
+        if (url !== undefined) {
+            missingUrls.push(url);
+        }
+    }
+    const urlToMessage = new Map();
+    for (let i = 0; i < missingUrls.length; i += CHUNK_SIZE) {
+        const chunk = missingUrls.slice(i, i + CHUNK_SIZE);
+        const rows = (await instance('crawl_errors')
+            .select('url', 'message')
+            .whereIn('url', chunk));
+        for (const row of rows) {
+            if (row.url !== null && !urlToMessage.has(row.url)) {
+                urlToMessage.set(row.url, row.message);
+            }
+        }
+    }
+    for (const id of idsMissing) {
+        const url = idToUrl.get(id);
+        if (url === undefined) {
+            continue;
+        }
+        const message = urlToMessage.get(url);
+        if (message !== undefined) {
+            messageByPageId.set(id, message);
+        }
+    }
+    return messageByPageId;
+}

package/lib/archive/init-schema.d.ts

@@ -13,47 +13,43 @@ export declare function applyConnectionPragmas(instance: Knex): Promise<void>;
 /**
  * Initializes the archive database schema if tables do not exist.
  *
- * Schema notes:
+ * The schema is composed of four groups, each owned by a dedicated DDL
+ * function so the migration script (`scripts/migrate-to-0.13.mjs`) can
+ * provision the exact same shapes on archives it upgrades:
  *
- * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
- *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
- *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
- *   JSON column for everything not flattened. URL-shaped columns are
- *   absolutised against the page URL before write (see
- *   `archive/meta/derive-flat-from-meta.ts`).
- * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
- *   `tags_providers_csv`): computed at write time from `meta.tags` /
- *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
- *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
- * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
- *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
- *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
- *   leaves them alone so failure-reset does not erase the last-success
- *   record.
- * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
- *   `categories`/`sources` JSON columns. Compound indexes
- *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
- *   Phase 2+ "find duplicate IDs across pages" and "list pages using
- *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
- * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
- *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
- *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
- *   if cross-archive bulk export becomes a use case, add a `codec` column
- *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
- *   `list_pages_by_jsonld_type` JOINs.
- * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
- *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
- *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
- *   express it.
- * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
- *   `CREATE TABLE` because SQLite only honors `page_size` changes against
- *   an empty database, and `journal_mode = WAL` is persistent. Other
- *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ * - **`info`** (inline below): single-row crawl configuration.
+ * - **Ref / header dictionary tables** ({@link createRefTables}):
+ *   `url_refs`, `content_type_refs`, `text_refs`, `json_refs`,
+ *   `blob_refs`, `header_name_refs`, `header_value_refs`, `header_sets`,
+ *   `header_set_entries`, `header_flags`. See that file for column-level
+ *   rationale and the reason `blob_refs` uses a regular rowid PK instead
+ *   of WITHOUT ROWID.
+ * - **Entity / edge tables** ({@link createEntityTables}):
+ *   `content_items`, `page_meta`, `resource_items`, `anchor_edges`,
+ *   `resource_ref_edges`, `image_items` — the write-model the crawler
+ *   writes during a crawl and every reader queries. Must run AFTER
+ *   `createRefTables` because most entity tables reference ref-table PKs.
+ * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
+ *   `crawl_errors`, `page_tags`, `page_jsonld`, `inventory_runs`,
+ *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
+ *   `page_html_ref`. Must run AFTER `createEntityTables` because the
+ *   page-scoped tables FK into `content_items(id)`.
  *
- * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
- * (called before `initSchema`) rejects pre-0.10 archives with a friendly
- * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
- * policy allows breaking changes.
+ * The legacy flat write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) are deliberately NOT created:
+ * they only exist inside pre-0.13 archives, where they serve as the
+ * populate source for `scripts/migrate-to-0.13.mjs` before that script
+ * drops them.
+ *
+ * **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ * `CREATE TABLE` because SQLite only honors `page_size` changes against
+ * an empty database, and `journal_mode = WAL` is persistent. Other
+ * per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.13 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects older archives with a friendly
+ * error naming the migration scripts to run; `v0.x` policy allows
+ * breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export declare function initSchema(instance: Knex): Promise<void>;