@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/create-adjunct-tables.d.ts

@@ -0,0 +1,43 @@
+import type { Knex } from 'knex';
+/**
+ * Creates the adjunct tables that hang off `content_items` (plus the two
+ * standalone log tables), guarded per-table so partially-provisioned
+ * archives converge to the full set:
+ *
+ * - `page_errors` — partial scrape failures, FK → `content_items(id)`
+ * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
+ *   an external link that failed DNS, or null for a process-level error)
+ * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
+ * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
+ *   FK → `content_items(id)`
+ * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
+ *   snapshots, FK → `content_items(id)`
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this right after `createEntityTables`) and the migration script
+ * (`scripts/migrate-to-0.13.mjs` calls it before retargeting FK
+ * declarations and dropping the legacy tables). Keeping the schema in one
+ * function guarantees both origin points produce identical tables — the
+ * pre-0.13 era kept per-table copies of this DDL in separate lazy-migration
+ * modules, and those copies drifted: they declared `REFERENCES pages(id)`
+ * while `initSchema` had moved on to `content_items(id)`, leaving migrated
+ * archives with stale FK targets that only `scripts/migrate-to-0.13.mjs`'s
+ * rename-copy-drop pass can now repair.
+ *
+ * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
+ * a single sentinel table), each table here is guarded individually because
+ * the migration-script caller sees archives where any subset may already
+ * exist (e.g. `page_tags` from the 0.10 migration but no `inventory_runs`).
+ * Index creation stays inside each guard: an existing table keeps whatever
+ * indexes its creation path declared.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * // Must run after createEntityTables — the page-scoped tables FK into
+ * // content_items(id).
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await createAdjunctTables(db);
+ */
+export declare function createAdjunctTables(instance: Knex): Promise<void>;

package/lib/archive/create-adjunct-tables.js

@@ -0,0 +1,213 @@
+/**
+ * Creates the adjunct tables that hang off `content_items` (plus the two
+ * standalone log tables), guarded per-table so partially-provisioned
+ * archives converge to the full set:
+ *
+ * - `page_errors` — partial scrape failures, FK → `content_items(id)`
+ * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
+ *   an external link that failed DNS, or null for a process-level error)
+ * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
+ * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
+ *   FK → `content_items(id)`
+ * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
+ *   snapshots, FK → `content_items(id)`
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this right after `createEntityTables`) and the migration script
+ * (`scripts/migrate-to-0.13.mjs` calls it before retargeting FK
+ * declarations and dropping the legacy tables). Keeping the schema in one
+ * function guarantees both origin points produce identical tables — the
+ * pre-0.13 era kept per-table copies of this DDL in separate lazy-migration
+ * modules, and those copies drifted: they declared `REFERENCES pages(id)`
+ * while `initSchema` had moved on to `content_items(id)`, leaving migrated
+ * archives with stale FK targets that only `scripts/migrate-to-0.13.mjs`'s
+ * rename-copy-drop pass can now repair.
+ *
+ * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
+ * a single sentinel table), each table here is guarded individually because
+ * the migration-script caller sees archives where any subset may already
+ * exist (e.g. `page_tags` from the 0.10 migration but no `inventory_runs`).
+ * Index creation stays inside each guard: an existing table keeps whatever
+ * indexes its creation path declared.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * // Must run after createEntityTables — the page-scoped tables FK into
+ * // content_items(id).
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await createAdjunctTables(db);
+ */
+export async function createAdjunctTables(instance) {
+    if (!(await instance.schema.hasTable('page_errors'))) {
+        await instance.schema.createTable('page_errors', (t) => {
+            // Records partial scrape failures (e.g. a viewport switch that
+            // detaches the frame and trips beholder's @retryable into the
+            // `retryExhausted` phase). A page can have zero or more rows here
+            // in addition to its normal `content_items` entry — the page
+            // itself is considered successfully scraped, but image capture or
+            // another secondary step failed for at least one device preset.
+            t.increments('id');
+            t.integer('pageId').notNullable().unsigned().references('content_items.id');
+            t.string('phase').notNullable();
+            t.text('message').notNullable();
+            t.integer('createdAt').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('crawl_errors'))) {
+        await instance.schema.createTable('crawl_errors', (t) => {
+            // Structured form of the crawler-level `error` channel that otherwise
+            // only lands in `error.log`. Unlike `page_errors` these are not tied to
+            // a scraped page (the URL may be an external link that failed DNS, or
+            // null for a process-level error), so there is no `pageId` FK and `url`
+            // is nullable. The cause is NOT stored — it is classified on read from
+            // `message` so older archives (which only have `error.log`) classify the
+            // same way.
+            t.increments('id');
+            t.string('url', 8190).nullable();
+            t.boolean('isExternal');
+            t.text('message').notNullable();
+            t.integer('createdAt').notNullable();
+        });
+    }
+    if (!(await instance.schema.hasTable('page_tags'))) {
+        await instance.schema.createTable('page_tags', (t) => {
+            // Wappalyzer-derived technology detection. One row per
+            // (provider × externalId) tuple per page. `category` is the first
+            // element of `categories`; the full list lives in the JSON
+            // `categories` column. `sources` records where the provider was
+            // detected (script-src / inline / iframe-src / window-global / …).
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.string('provider').notNullable();
+            t.string('category');
+            t.string('externalId');
+            t.string('version');
+            t.integer('confidence');
+            t.json('categories');
+            t.json('sources');
+            t.index('pageId');
+            t.index('provider');
+            t.index('externalId');
+        });
+        // Compound indexes for the "find duplicate IDs across pages" and
+        // "list pages using provider X" hot paths. Knex's schema builder
+        // can't express compound indexes inline in a way that round-trips
+        // through libsql consistently, so raw SQL is used.
+        await instance.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
+        await instance.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
+    }
+    if (!(await instance.schema.hasTable('page_jsonld'))) {
+        await instance.schema.createTable('page_jsonld', (t) => {
+            // JSON-LD and SpeculationRules entries captured from
+            // `<script type="application/ld+json">` and
+            // `<script type="speculationrules">`. `kind` discriminates; `type`
+            // is the top-level `@type` extracted by classify-jsonld-type for
+            // indexable filtering. `raw` is stored uncompressed; SQLite
+            // overflow pages handle multi-KB JSON bodies transparently.
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.string('kind').notNullable();
+            t.string('type');
+            t.text('raw').notNullable();
+            t.json('parsed');
+            t.text('parseError');
+            t.index('pageId');
+            t.index('type');
+        });
+        // Compound `(type, pageId)` accelerates streaming
+        // `list_pages_by_jsonld_type` JOINs.
+        await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
+    }
+    if (!(await instance.schema.hasTable('inventory_runs'))) {
+        await instance.schema.createTable('inventory_runs', (t) => {
+            // One row per successful `--inventory <list>` invocation. The
+            // archive's audit log of "when did we apply which deploy list
+            // at what scale". `.bak` is removed on success so this table
+            // is the only durable provenance record. Column semantics live
+            // on the `InventoryRunMeta` interface in `archive/types.ts`.
+            t.increments('id');
+            t.string('ran_at').notNullable();
+            t.string('list_label').nullable();
+            t.string('source_file_sha256', 64).nullable();
+            t.integer('total_lines').nullable();
+            t.integer('new_pages').nullable();
+            t.integer('new_resources').nullable();
+            t.integer('scope_skipped').nullable();
+            t.text('notes').nullable();
+            t.index('ran_at');
+        });
+    }
+    if (!(await instance.schema.hasTable('analysis_text_refs'))) {
+        await instance.raw(`
+			CREATE TABLE analysis_text_refs (
+				id integer primary key,
+				text text not null,
+				sha256 text not null,
+				unique(sha256, text)
+			)
+		`);
+    }
+    if (!(await instance.schema.hasTable('analysis_violations'))) {
+        await instance.raw(`
+			CREATE TABLE analysis_violations (
+				id integer primary key,
+				page_id integer not null references content_items(id),
+				validator text not null,
+				severity text not null,
+				rule text not null,
+				message_text_id integer not null references analysis_text_refs(id),
+				code_text_id integer references analysis_text_refs(id),
+				page_url_sort_key text not null,
+				message_sort_key text not null,
+				code_sort_key text not null,
+				line integer,
+				col integer
+			)
+		`);
+        await instance.raw('CREATE INDEX av_url_order ON analysis_violations(page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_filter_url ON analysis_violations(validator, severity, rule, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_validator_url ON analysis_violations(validator, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_severity_url ON analysis_violations(severity, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_rule_url ON analysis_violations(rule, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_message_order ON analysis_violations(message_sort_key, id)');
+        await instance.raw('CREATE INDEX av_code_order ON analysis_violations(code_sort_key, id)');
+        await instance.raw('CREATE INDEX av_page ON analysis_violations(page_id, id)');
+    }
+    // Content-addressable HTML blob storage. Knex's schema builder doesn't
+    // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
+    // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
+    // (no hidden rowid + secondary index pair), which matters for the blob
+    // table where a 32-byte hash PK + multi-KB body is the dominant row
+    // shape.
+    if (!(await instance.schema.hasTable('page_html_blobs'))) {
+        await instance.raw(`
+			CREATE TABLE page_html_blobs (
+				hash         BLOB PRIMARY KEY,
+				body         BLOB NOT NULL,
+				codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+				size_raw     INTEGER NOT NULL,
+				size_stored  INTEGER NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
+    if (!(await instance.schema.hasTable('page_html_ref'))) {
+        await instance.raw(`
+			CREATE TABLE page_html_ref (
+				page_id  INTEGER PRIMARY KEY REFERENCES content_items(id) ON DELETE CASCADE,
+				hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+			) WITHOUT ROWID
+		`);
+        await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
+    }
+}

package/lib/archive/create-entity-tables.d.ts

@@ -0,0 +1,173 @@
+import type { Knex } from 'knex';
+/**
+ * Creates the 6 core entity and edge tables of the 0.13 format (issue #192).
+ *
+ * These are the normalised entity / edge tables that replace the legacy
+ * write model (`pages`, `anchors`, `images`, `resources`, `resources-referrers`)
+ * in the 0.13 format. This migration is purely additive DDL — no existing table
+ * is modified and no row is inserted; the populate step
+ * (`populateEntityTables`) fills the tables from the legacy sources.
+ *
+ * Tables created:
+ *
+ * - `content_items` — replaces `pages` (core columns). The remaining
+ *   per-page metadata splits into `page_meta`.
+ * - `page_meta` — page-specific metadata (title, description, meta tags,
+ *   OpenGraph, Twitter, robots, denormalised aggregates, meta_extras JSON
+ *   pointer). Split from `pages` so that filter / cursor queries against
+ *   `content_items` scan a narrower row.
+ * - `resource_items` — replaces `resources`. Identity URL splits into
+ *   `url_id` (regular URLs) vs `url_blob_id` (large `data:` URIs, routed
+ *   via `blob_refs`) — a resource's own URL is captured verbatim from
+ *   the network layer and, unlike a page's URL, can legally be a large
+ *   inline `data:` URI (e.g. a CSS `background-image` sub-resource).
+ * - `anchor_edges` — replaces `anchors`. Deduped to distinct
+ *   `(page_id, href_page_id)` pairs; `count` records how many instances
+ *   were observed and `first_hash` / `first_text_id` capture the first
+ *   instance's identity so the most important observable fact per link is
+ *   preserved without every redundant row.
+ * - `resource_ref_edges` — replaces `resources-referrers`. Structural
+ *   rename with an added `count` column; `WITHOUT ROWID` because the
+ *   composite PK `(resource_id, page_id)` is the natural clustering key
+ *   and every column is small — the same reasoning as `page_html_blobs`.
+ * - `image_items` — replaces `images`. `src` / `currentSrc` split into
+ *   `*_url_id` (regular URLs) vs `*_blob_id` (large `data:` URIs, routed
+ *   via `blob_refs`); `sourceCode` is replaced by `dom_path_text_id`,
+ *   which the populate step derives from the `outerHTML` string.
+ *
+ * ### Key invariants
+ *
+ * **Same PK values as legacy.** `content_items.id`, `resource_items.id`,
+ * and `image_items.id` reuse the exact PK values from `pages.id`,
+ * `resources.id`, and `images.id` respectively. The populate step inserts
+ * rows with explicit IDs so every existing FK reference in `page_errors`,
+ * `page_tags`, `page_jsonld`, `page_html_ref`, and `resources-referrers`
+ * (before it becomes `resource_ref_edges`) survives the switch without
+ * any per-row UPDATE. `AUTOINCREMENT` is nevertheless applied on all four
+ * entity PKs (`content_items` / `resource_items` / `anchor_edges` /
+ * `image_items`) to match the legacy contract (`t.increments()` on
+ * `pages` / `resources` / `anchors` / `images`) — this prevents rowid
+ * reuse after DELETE, so external systems (analysis outputs, viewer
+ * bookmarks, Sheets exports) keyed on `content_item.id` cannot be
+ * silently reassigned to a different record.
+ *
+ * **UNIQUE(url_id) on content_items and resource_items.** The legacy
+ * `pages.url UNIQUE` / `resources.url UNIQUE` guaranteed one row per
+ * URL string. Under the new model the URL string is stored once in
+ * `url_refs` and each entity references `url_refs.id` via `url_id` —
+ * but `url_refs.url UNIQUE` only guarantees "one URL string → one
+ * ref id", it does NOT prevent two entity rows from claiming the same
+ * ref id. `UNIQUE(url_id)` on both `content_items` and `resource_items`
+ * re-establishes that guarantee at the entity level — without it a
+ * populate bug could attach two entity rows to the same URL, and every
+ * by-URL reader would silently return an arbitrary one. The UNIQUE
+ * constraint's auto-index also serves the by-URL seek path, so no
+ * separate `CREATE INDEX ... ON content_items(url_id)` is needed.
+ * `resource_items.url_id` is nullable (unlike `content_items.url_id`)
+ * because a resource's identity URL can itself be blob-routed — see the
+ * `resource_items url / blob mutual-exclusion CHECK` invariant below.
+ * `UNIQUE` on a nullable SQLite column only dedups the non-NULL values,
+ * so the guarantee still holds for every resource that does have a
+ * `url_id`.
+ *
+ * **`redirect_dest_id` self-reference is DEFERRABLE INITIALLY DEFERRED.**
+ * `content_items.redirect_dest_id` references `content_items(id)` (not
+ * `pages(id)`), mirroring the legacy `pages.redirectDestId → pages.id`
+ * self-reference. In the real archive shape, a redirect source can be
+ * discovered as an anchor (assigned a low `pages.id`) BEFORE its
+ * destination is crawled (higher `pages.id`), so a straight
+ * `ORDER BY id ASC` populate would violate the FK on the source row.
+ * Making the FK deferred lets the populate step commit all `content_items`
+ * inserts inside one transaction without any per-row ordering
+ * discipline — SQLite validates the constraint only at COMMIT time.
+ *
+ * **`page_meta.page_id` PK == FK ON DELETE CASCADE.** `page_meta` uses
+ * `page_id` as both the primary key and the FK back to `content_items(id)`.
+ * This is a 1:1 relation (every content_item has at most one meta row),
+ * so a separate autoincrement id would only bloat the row. The cascade
+ * mirrors the legacy `page_html_ref` / `page_tags` / `page_jsonld` FKs
+ * to `pages(id)` so a future cleanup pass can DELETE from
+ * `content_items` without leaving orphan meta rows.
+ *
+ * **`content_items.source` and `resource_items.source` DEFAULT 'crawled'.**
+ * The legacy `pages.source` / `resources.source` columns were
+ * `NOT NULL DEFAULT 'crawled'`, and multiple existing writer paths rely
+ * on the default (redirect-source insert, per-chunk resource insert, and
+ * `markBrowserScrape`). Preserving the default keeps those callsites
+ * working when the 0.13 write path re-points them at the new tables — dropping
+ * the default would turn a routine crawl into an immediate hard failure
+ * the first time a callsite omits `source`.
+ *
+ * **`anchor_edges` dedup shape.** `unique(page_id, href_page_id)` enforces
+ * the collapse from ≈ 13M legacy `anchors` rows to ≈ 9.7M `anchor_edges`
+ * rows (~25 % reduction on the reference archive).
+ * `first_hash` and `first_text_id` capture the anchor `hash` and
+ * `textContent` of the FIRST instance encountered (lowest `anchors.id`
+ * for the pair); duplicate observations are counted via `count` without
+ * retaining their bodies. The UNIQUE constraint's auto-index on
+ * `(page_id, href_page_id)` also serves any `WHERE page_id = ?` seek via
+ * SQLite's leading-prefix rule, so no separate `CREATE INDEX ... ON
+ * anchor_edges(page_id)` is needed; only the reverse-direction
+ * `idx_anchor_edges_href` on `href_page_id` alone is added.
+ *
+ * **`image_items.dom_path_text_id NOT NULL`.** Every image row must
+ * resolve to a `text_refs` entry describing its DOM position. The
+ * image-items populate derives the path from `images.sourceCode` (the
+ * stored `outerHTML`); an
+ * image whose `sourceCode` is empty falls back to the synthetic label
+ * `img[unknown]` (via the image-items populate) so the FK is never violated.
+ *
+ * **`image_items` src / blob mutual-exclusion CHECK.** `src_url_id` and
+ * `src_blob_id` (and their `current_src_*` mirror) are separate nullable
+ * columns because a `<img>` value can be either a plain URL or a large
+ * `data:` URI. A single legacy `images.src` TEXT column could not
+ * express the ambiguity; the new schema splits the two but adds a CHECK
+ * so at most one slot per (src, currentSrc) pair is non-null. Without
+ * the CHECK a populate bug or future writer could silently write both,
+ * and the viewer would arbitrarily pick one to render.
+ *
+ * **`resource_items` url / blob mutual-exclusion CHECK.** Same rationale
+ * as the image CHECK above, but stricter: `(url_id IS NULL) !=
+ * (url_blob_id IS NULL)` requires EXACTLY one slot to be non-null (not
+ * "at most one") because a resource — unlike an `<img>` that can
+ * legitimately have neither `src` nor `currentSrc` — is always
+ * identified by some URL. `resolveUrlOrBlob` (writer) /
+ * `resolveUrlOrBlobFromMaps` (populate) are shared with `image_items`'
+ * routing so the two entities never disagree on which values count as
+ * "large data URI".
+ *
+ * ### Index rationale
+ *
+ * Every index below reflects a legacy-baseline single-column index that
+ * an existing hot path already relies on. `content_items` gets six
+ * indexes: `url_id` (via `UNIQUE` auto-index), `is_external`, `scraped`,
+ * `redirect_dest_id` (for the Page Detail inbound-redirect list),
+ * `content_type_id`, `crawl_order`, and `source` — all present on
+ * legacy `pages`. `page_meta` gets `og_type` and `robots_noindex`
+ * because both were indexed on legacy `pages` for the noindex /
+ * og:type filter hot paths; after the split those readers land on
+ * `page_meta` and need the indexes to remain sub-second. `resource_items` gets `url_id`
+ * (via `UNIQUE` auto-index) and `source`. `anchor_edges` gets one
+ * reverse-direction index (`idx_anchor_edges_href`); the UNIQUE
+ * auto-index covers the forward direction. `resource_ref_edges` needs
+ * an explicit reverse-direction index on `page_id` because the
+ * composite PK `(resource_id, page_id)` only satisfies the `resource_id`-
+ * first prefix; the legacy `resources-referrers` table had exactly
+ * this reverse-direction index. `image_items` gets one index on
+ * `page_id`. No summary / listfilter compound indexes are created here —
+ * they belong to the viewer read model built directly against these
+ * tables.
+ *
+ * ### Idempotency
+ *
+ * All statements use `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT
+ * EXISTS` so the primitive is safe to call multiple times against the
+ * same DB. Both call sites (`initSchema` and
+ * `migrateEntityTables`) still guard on their own sentinels,
+ * but the function itself no longer depends on the sentinel being
+ * checked first — this closes the partial-corruption gap where, say,
+ * `content_items` exists but `page_meta` was dropped by an external
+ * repair pass.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function createEntityTables(instance: Knex): Promise<void>;