@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/init-schema.js

@@ -1,3 +1,6 @@
+import { createAdjunctTables } from './create-adjunct-tables.js';
+import { createEntityTables } from './create-entity-tables.js';
+import { createRefTables } from './create-ref-tables.js';
 /**
  * Applies the connection-level PRAGMAs that govern foreign-key enforcement
  * and BLOB-read performance. These are **per-connection** settings (libsql
@@ -34,470 +37,106 @@ export async function applyConnectionPragmas(instance) {
 /**
  * Initializes the archive database schema if tables do not exist.
  *
- * Schema notes:
+ * The schema is composed of four groups, each owned by a dedicated DDL
+ * function so the migration script (`scripts/migrate-to-0.13.mjs`) can
+ * provision the exact same shapes on archives it upgrades:
  *
- * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
- *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
- *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
- *   JSON column for everything not flattened. URL-shaped columns are
- *   absolutised against the page URL before write (see
- *   `archive/meta/derive-flat-from-meta.ts`).
- * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
- *   `tags_providers_csv`): computed at write time from `meta.tags` /
- *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
- *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
- * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
- *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
- *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
- *   leaves them alone so failure-reset does not erase the last-success
- *   record.
- * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
- *   `categories`/`sources` JSON columns. Compound indexes
- *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
- *   Phase 2+ "find duplicate IDs across pages" and "list pages using
- *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
- * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
- *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
- *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
- *   if cross-archive bulk export becomes a use case, add a `codec` column
- *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
- *   `list_pages_by_jsonld_type` JOINs.
- * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
- *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
- *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
- *   express it.
- * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
- *   `CREATE TABLE` because SQLite only honors `page_size` changes against
- *   an empty database, and `journal_mode = WAL` is persistent. Other
- *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ * - **`info`** (inline below): single-row crawl configuration.
+ * - **Ref / header dictionary tables** ({@link createRefTables}):
+ *   `url_refs`, `content_type_refs`, `text_refs`, `json_refs`,
+ *   `blob_refs`, `header_name_refs`, `header_value_refs`, `header_sets`,
+ *   `header_set_entries`, `header_flags`. See that file for column-level
+ *   rationale and the reason `blob_refs` uses a regular rowid PK instead
+ *   of WITHOUT ROWID.
+ * - **Entity / edge tables** ({@link createEntityTables}):
+ *   `content_items`, `page_meta`, `resource_items`, `anchor_edges`,
+ *   `resource_ref_edges`, `image_items` — the write-model the crawler
+ *   writes during a crawl and every reader queries. Must run AFTER
+ *   `createRefTables` because most entity tables reference ref-table PKs.
+ * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
+ *   `crawl_errors`, `page_tags`, `page_jsonld`, `inventory_runs`,
+ *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
+ *   `page_html_ref`. Must run AFTER `createEntityTables` because the
+ *   page-scoped tables FK into `content_items(id)`.
  *
- * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
- * (called before `initSchema`) rejects pre-0.10 archives with a friendly
- * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
- * policy allows breaking changes.
+ * The legacy flat write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) are deliberately NOT created:
+ * they only exist inside pre-0.13 archives, where they serve as the
+ * populate source for `scripts/migrate-to-0.13.mjs` before that script
+ * drops them.
+ *
+ * **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ * `CREATE TABLE` because SQLite only honors `page_size` changes against
+ * an empty database, and `journal_mode = WAL` is persistent. Other
+ * per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.13 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects older archives with a friendly
+ * error naming the migration scripts to run; `v0.x` policy allows
+ * breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export async function initSchema(instance) {
-    const isExists = await instance.schema.hasTable('info');
-    if (isExists) {
-        return;
+    // Only the one-shot work (PRAGMAs + `info` creation) is gated on the
+    // `info` table's existence. The three DDL groups below run on EVERY
+    // call: each is internally idempotent (sentinel / IF NOT EXISTS /
+    // per-table guards), and re-running them self-heals an archive whose
+    // provisioning crashed partway through — `info` created but a later
+    // group missing. With a single all-or-nothing gate, such a stub would
+    // short-circuit here forever and every subsequent write would die with
+    // `no such table` and no way back.
+    if (!(await instance.schema.hasTable('info'))) {
+        // Page size must be set on an empty database file; once any data is
+        // written, only VACUUM can change it. journal_mode is also one-shot
+        // (persistent) and so stays here.
+        await instance.raw('PRAGMA page_size = 16384');
+        await instance.raw('PRAGMA journal_mode = WAL');
+        await instance.schema.createTable('info', (t) => {
+            t.increments('id');
+            t.string('version');
+            t.string('name');
+            t.string('baseUrl');
+            t.json('roots');
+            t.boolean('recursive');
+            t.integer('interval');
+            t.boolean('image');
+            t.boolean('fetchExternal');
+            t.integer('parallels');
+            t.json('excludes');
+            t.json('excludeKeywords');
+            t.json('excludeUrls');
+            t.integer('maxExcludedDepth');
+            t.integer('retry');
+            t.boolean('fromList');
+            t.boolean('disableQueries');
+            t.string('userAgent');
+            t.boolean('ignoreRobots');
+        });
     }
-    // Page size must be set on an empty database file; once any data is
-    // written, only VACUUM can change it. journal_mode is also one-shot
-    // (persistent) and so stays here.
-    await instance.raw('PRAGMA page_size = 16384');
-    await instance.raw('PRAGMA journal_mode = WAL');
-    await instance.schema
-        .createTable('info', (t) => {
-        t.increments('id');
-        t.string('version');
-        t.string('name');
-        t.string('baseUrl');
-        t.json('roots');
-        t.boolean('recursive');
-        t.integer('interval');
-        t.boolean('image');
-        t.boolean('fetchExternal');
-        t.integer('parallels');
-        t.json('excludes');
-        t.json('excludeKeywords');
-        t.json('excludeUrls');
-        t.integer('maxExcludedDepth');
-        t.integer('retry');
-        t.boolean('fromList');
-        t.boolean('disableQueries');
-        t.string('userAgent');
-        t.boolean('ignoreRobots');
-    })
-        .createTable('pages', (t) => {
-        t.increments('id');
-        t.string('url', 8190).notNullable().unique();
-        t.integer('redirectDestId').unsigned().references('pages.id').defaultTo(null);
-        t.boolean('scraped').notNullable();
-        t.boolean('isTarget').notNullable();
-        t.boolean('isExternal');
-        t.integer('status');
-        t.string('statusText');
-        t.string('contentType').nullable();
-        t.integer('contentLength').unsigned().nullable();
-        t.json('responseHeaders').nullable();
-        // Document basics
-        t.string('lang');
-        t.string('dir');
-        t.string('charset');
-        t.string('baseHref');
-        t.text('viewport_raw');
-        t.string('themeColor');
-        t.string('applicationName');
-        t.string('author');
-        t.string('generator');
-        t.string('publisher');
-        // Title / description / keywords (top-level Meta fields)
-        t.string('title');
-        t.text('description');
-        t.text('keywords');
-        // Robots
-        t.text('robots_raw');
-        t.integer('robots_noindex');
-        t.integer('robots_nofollow');
-        t.integer('robots_noarchive');
-        t.integer('robots_noimageindex');
-        t.string('googlebot');
-        // Link (1:1 only — array shapes live in meta_extras)
-        t.string('canonical', 8190);
-        t.string('amphtml', 8190);
-        t.string('manifest', 8190);
-        t.string('icon_href', 8190);
-        t.string('appleTouchIcon_href', 8190);
-        // Open Graph
-        t.string('og_type');
-        t.string('og_title');
-        t.string('og_url', 8190);
-        t.string('og_site_name');
-        t.text('og_description');
-        t.string('og_image', 8190);
-        t.string('og_image_alt');
-        t.string('og_image_width');
-        t.string('og_image_height');
-        t.string('og_locale');
-        t.string('og_article_published_time');
-        t.string('og_article_modified_time');
-        // Twitter
-        t.string('twitter_card');
-        t.string('twitter_site');
-        t.string('twitter_creator');
-        t.string('twitter_title');
-        t.text('twitter_description');
-        t.string('twitter_image', 8190);
-        // One-offs
-        t.string('fb_app_id');
-        t.string('verification_google');
-        t.integer('formatDetection_telephone');
-        // Within-archive observation timestamps (UNIX ms)
-        t.integer('firstCrawledAt');
-        t.integer('lastCrawledAt');
-        // Denormalised aggregates (written at scrape time, see
-        // archive/meta/compute-page-denormalized.ts)
-        t.integer('tag_count');
-        t.integer('jsonld_count');
-        t.text('tags_providers_csv');
-        // Catch-all JSON for nested Meta sub-objects not flattened above
-        t.json('meta_extras');
-        // Crawl lifecycle
-        t.boolean('isSkipped');
-        t.string('skipReason');
-        t.integer('order').unsigned().nullable();
-        // Provenance: which channel inserted this row. Values:
-        //   'crawled'              — discovered via the recursive crawl from one of `info.roots`
-        //   'inventory-seed'       — supplied directly by `crawl --inventory` URL list
-        //   'inventory-discovered' — found by following links from an `inventory-seed` page
-        // Used by `listIsolatedPages` only for badge display; isolation
-        // itself is judged by `anchors.hrefId IS NULL`, not by source.
-        t.string('source').notNullable().defaultTo('crawled');
-        t.index('isExternal');
-        t.index('contentType');
-        t.index('scraped');
-        t.index('redirectDestId');
-        t.index('order');
-        // Phase 1: noindex filter (list_pages) and og:type filter
-        // (analytics) are the only new flat-column filters with enough
-        // selectivity to benefit from an index. `lang` has cardinality 1
-        // on mono-language sites (D-Zero's typical customer) so it is
-        // skipped.
-        t.index('robots_noindex');
-        t.index('og_type');
-        t.index('source');
-    })
-        .createTable('anchors', (t) => {
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.integer('hrefId').notNullable().unsigned().references('pages.id');
-        t.string('hash');
-        t.string('textContent').nullable();
-        t.index('pageId');
-        t.index('hrefId');
-    })
-        .createTable('images', (t) => {
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.string('src', 8190);
-        t.string('currentSrc', 8190);
-        t.string('alt');
-        t.float('width').unsigned().notNullable();
-        t.float('height').unsigned().notNullable();
-        t.integer('naturalWidth').unsigned().notNullable();
-        t.integer('naturalHeight').unsigned().notNullable();
-        t.boolean('isLazy');
-        t.integer('viewportWidth').unsigned().notNullable();
-        t.string('sourceCode');
-        t.index('pageId');
-    })
-        .createTable('resources', (t) => {
-        t.increments('id');
-        t.string('url', 8190).notNullable().unique();
-        t.boolean('isExternal');
-        t.integer('status');
-        t.string('statusText');
-        t.string('contentType').nullable();
-        t.integer('contentLength').unsigned().nullable();
-        t.string('compress').nullable();
-        t.string('cdn').nullable();
-        t.json('responseHeaders').nullable();
-        // See `pages.source` for the provenance taxonomy. `inventory-seed`
-        // rows here come from non-HTML URLs handed in by
-        // `crawl --inventory`; `inventory-discovered` rows are sub-resources
-        // pulled in while puppeteer rendered an inventory-seed page.
-        t.string('source').notNullable().defaultTo('crawled');
-        t.index('source');
-    })
-        .createTable('resources-referrers', (t) => {
-        t.increments('id');
-        t.integer('resourceId').notNullable().unsigned().references('resources.id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.unique(['resourceId', 'pageId']);
-        t.index('resourceId');
-        t.index('pageId');
-    })
-        .createTable('page_errors', (t) => {
-        // Records partial scrape failures (e.g. a viewport switch that
-        // detaches the frame and trips beholder's @retryable into the
-        // `retryExhausted` phase). A page can have zero or more rows here
-        // in addition to its normal `pages` entry — the page itself is
-        // considered successfully scraped, but image capture or another
-        // secondary step failed for at least one device preset.
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.string('phase').notNullable();
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-        t.index('pageId');
-    })
-        .createTable('crawl_errors', (t) => {
-        // Structured form of the crawler-level `error` channel that otherwise
-        // only lands in `error.log`. Unlike `page_errors` these are not tied to
-        // a scraped page (the URL may be an external link that failed DNS, or
-        // null for a process-level error), so there is no `pageId` FK and `url`
-        // is nullable. The cause is NOT stored — it is classified on read from
-        // `message` so older archives (which only have `error.log`) classify the
-        // same way.
-        t.increments('id');
-        t.string('url', 8190).nullable();
-        t.boolean('isExternal');
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-    })
-        .createTable('page_tags', (t) => {
-        // Wappalyzer-derived technology detection. One row per
-        // (provider × externalId) tuple per page. `category` is the first
-        // element of `categories`; the full list lives in the JSON
-        // `categories` column. `sources` records where the provider was
-        // detected (script-src / inline / iframe-src / window-global / …).
-        t.increments('id');
-        t.integer('pageId')
-            .notNullable()
-            .unsigned()
-            .references('pages.id')
-            .onDelete('CASCADE');
-        t.string('provider').notNullable();
-        t.string('category');
-        t.string('externalId');
-        t.string('version');
-        t.integer('confidence');
-        t.json('categories');
-        t.json('sources');
-        t.index('pageId');
-        t.index('provider');
-        t.index('externalId');
-    })
-        .createTable('page_jsonld', (t) => {
-        // JSON-LD and SpeculationRules entries captured from
-        // `<script type="application/ld+json">` and
-        // `<script type="speculationrules">`. `kind` discriminates; `type`
-        // is the top-level `@type` extracted by classify-jsonld-type for
-        // indexable filtering. `raw` is stored uncompressed; SQLite
-        // overflow pages handle multi-KB JSON bodies transparently.
-        t.increments('id');
-        t.integer('pageId')
-            .notNullable()
-            .unsigned()
-            .references('pages.id')
-            .onDelete('CASCADE');
-        t.string('kind').notNullable();
-        t.string('type');
-        t.text('raw').notNullable();
-        t.json('parsed');
-        t.text('parseError');
-        t.index('pageId');
-        t.index('type');
-    })
-        .createTable('inventory_runs', (t) => {
-        // One row per successful `--inventory <list>` invocation. The
-        // archive's audit log of "when did we apply which deploy list
-        // at what scale". `.bak` is removed on success so this table
-        // is the only durable provenance record. Schema rationale +
-        // non-goals live in {@link migrateInventoryRuns}.
-        t.increments('id');
-        t.string('ran_at').notNullable();
-        t.string('list_label').nullable();
-        t.string('source_file_sha256', 64).nullable();
-        t.integer('total_lines').nullable();
-        t.integer('new_pages').nullable();
-        t.integer('new_resources').nullable();
-        t.integer('scope_skipped').nullable();
-        t.text('notes').nullable();
-        t.index('ran_at');
-    });
-    // ON DELETE CASCADE and compound indexes for the new tables. Knex's
-    // schema builder can't express CASCADE / compound indexes inline in a
-    // way that round-trips through libsql consistently, so we use raw SQL
-    // to mirror the `page_html_ref` pattern.
-    await instance.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
-    await instance.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
-    await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
-    // Content-addressable HTML blob storage. Knex's schema builder doesn't
-    // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
-    // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
-    // (no hidden rowid + secondary index pair), which matters for the blob
-    // table where a 32-byte hash PK + multi-KB body is the dominant row
-    // shape.
-    await instance.raw(`
-		CREATE TABLE page_html_blobs (
-			hash         BLOB PRIMARY KEY,
-			body         BLOB NOT NULL,
-			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
-			size_raw     INTEGER NOT NULL,
-			size_stored  INTEGER NOT NULL
-		) WITHOUT ROWID
-	`);
-    await instance.raw(`
-		CREATE TABLE page_html_ref (
-			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
-			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
-		) WITHOUT ROWID
-	`);
-    await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
-    // Composite covering index for the default Pages-view filter + url-ordered
-    // scan. Without it, `listPages` on a 400k-row archive runs ~15s per page
-    // click (SCAN pages USING pages_scraped_index + TEMP B-TREE FOR ORDER BY);
-    // with it, the same query runs ~45ms (368x speedup, confirmed via
-    // `scripts/bench-partial-listfilter.mjs` against a real customer archive).
-    // The same index also serves `listIsolatedPages`, `listIsolatedClusters`,
-    // and `getSummary`'s HTML-page counts.
-    //
-    // **Column order: `(isExternal, scraped, redirectDestId, url, contentType)`.**
-    // The leading `isExternal` is critical: the Pages view's default
-    // "external excluded" filter adds `WHERE isExternal = 0` to both the
-    // SELECT and the paginate-query COUNT. A previous version of this index
-    // (`(scraped, redirectDestId, url, contentType)`) shipped without
-    // `isExternal`, and the SELECT picked it up (`ORDER BY url` forced the
-    // match) while the COUNT — having no `ORDER BY` — fell back to the
-    // single-column `pages_isexternal_index` + scan + per-row WHERE filter,
-    // costing ~8.7s for the COUNT alone on a 165k-internal-page archive.
-    // Putting `isExternal` first makes both shapes pick this index as a
-    // covering scan (~33ms COUNT, ~1ms SELECT warm).
-    //
-    // **DO NOT RUN `ANALYZE` ON .nitpicker ARCHIVES.** With ANALYZE statistics
-    // available, the planner switches the JOIN paths in `listLinks`,
-    // `getLinkGraph`, and `listPageLinks` to use this index for source/dest
-    // seeks (SCAN dest → SEARCH anchors → SEARCH source) instead of the
-    // existing `SCAN anchors → rowid seek` plan. That regression takes those
-    // queries from ~15s to ~500s (33x worse). The unanalyzed-table heuristic
-    // happens to pick the right plan for the joins while still picking the new
-    // index for `listPages` because the column order exactly matches the
-    // WHERE+ORDER predicates. If a future change adds `ANALYZE` anywhere in
-    // the crawler / viewer / MCP / migration paths, this index must be
-    // re-evaluated first.
-    await instance.raw('CREATE INDEX idx_pages_listfilter ON pages(isExternal, scraped, redirectDestId, url, contentType)');
-    // Covering index for `listUnusedResources`. Without it the query SCAN s
-    // `resources_url_unique` (every resource, including externals) then
-    // filters `isExternal = 0` row-by-row — ~66s on the bench archive. With
-    // the `(isExternal, url)` leading prefix, the planner serves the WHERE
-    // + ORDER BY url from one covering scan — ~7.5s (8.8x). Same
-    // no-ANALYZE invariant applies (see `idx_pages_listfilter` above);
-    // validated against the 4 regression sentinels in
-    // `scripts/bench-unused-images.mjs`.
-    await instance.raw('CREATE INDEX idx_resources_internal_url ON resources(isExternal, url)');
-    // Covering index for `listImages`. The default query joins `images` to
-    // `pages` and orders by `pages.url`. Without this index the planner
-    // scans `images` first, seeks `pages` by rowid, and pays a TEMP B-TREE
-    // FOR ORDER BY (~32s on the bench archive). With the index the plan
-    // flips to SCAN pages (via `pages_url_unique`, url-ordered already)
-    // → SEARCH images via the covering pageId index — no temp sort, ~16s
-    // (2.0x). The included columns (src, alt, dimensions, isLazy) make
-    // `idx_images_covering` covering for every `select` `listImages` does,
-    // so the SEARCH does not need to materialise the underlying row.
-    // Validated by `scripts/bench-unused-images.mjs`.
-    await instance.raw('CREATE INDEX idx_images_covering ON images(pageId, src, alt, width, height, naturalWidth, naturalHeight, isLazy)');
-    // Targets `getSummary` Q2 (metadata fulfilment) + Q3 (content-type
-    // histogram). With this index Q2 and Q3 both become covering
-    // (`SEARCH ... USING COVERING INDEX`) — the SELECT columns are
-    // contained inside the index entry, so no rowid lookup is needed.
-    // Q1 (status histogram) also picks this index for its seek but
-    // keeps `USE TEMP B-TREE FOR GROUP BY` because the index column
-    // order leads with `contentType, isExternal` while Q1's GROUP BY
-    // is `(isExternal, status)` — the residual ordering inside the
-    // `scraped=1 AND redirectDestId IS NULL` slice does not match.
-    // Empirically this still gives the largest net win because Q2 +
-    // Q3 dominate `getSummary` on archives whose `pages` table dwarfs
-    // the SQLite page cache (10 GB bench: 1157 ms → 717 ms, 38 %).
-    //
-    // **An additional candidate index `(scraped, redirectDestId,
-    // isExternal, status)` was empirically rejected**: in isolation it
-    // matches Q1's GROUP BY column order and would eliminate the temp
-    // B-tree there, but in combination with this one or with the
-    // `pages_scraped_index` fallback the planner shifted to plans that
-    // regressed `getSummary` to 4.6-10 s (PR #96 教訓 — bulk index
-    // addition without ANALYZE confuses the heuristic). The
-    // `idx_pages_summary_contenttype` form below is the only summary
-    // index that survived the matrix test in
-    // `scripts/bench-summary-configs.mjs`.
-    //
-    // Column order rationale:
-    //
-    // 1. `scraped` — leading seek key. All summary queries constrain
-    //    it to `=1`.
-    // 2. `redirectDestId` — post-seek filter, IS NULL folded into the
-    //    seek key by SQLite's index walk without needing ANALYZE
-    //    (per operator forum; the IS NULL leading column rule only
-    //    bites when the column is the LEADING one and there is no
-    //    other equality constraint).
-    // 3. `contentType` — the column Q3 groups by.
-    // 4. `isExternal` — Q3's second GROUP BY column AND Q2's WHERE
-    //    constraint (`isExternal=0`).
-    // 5. `isSkipped` — Q1/Q3's residual `(isSkipped=0 OR IS NULL)`
-    //    filter (`excludeSkippedPages`). Having it in the index lets
-    //    the residual filter use the index entry instead of a per-row
-    //    rowid lookup.
-    //
-    // `id` is implicitly included (every SQLite index entry carries
-    // the rowid), so the `count(id)` aggregates cover off-index.
-    //
-    // **No-ANALYZE invariant** identical to `idx_pages_listfilter`
-    // (PR #96): the column order matches the WHERE+GROUP BY predicates
-    // exactly, so SQLite's heuristic-only planner picks it without
-    // needing `sqlite_stat1`. Adding `ANALYZE` would risk planner
-    // shifts in this and other queries.
-    //
-    // **Regression check**: `listPages` / `listPages COUNT` /
-    // `listLinks broken` / `listPageLinks` plans were re-verified
-    // against this index — `idx_pages_listfilter` continues to win
-    // for all of them. See `scripts/bench-summary-configs.mjs`.
-    await instance.raw('CREATE INDEX idx_pages_summary_contenttype ON pages(scraped, redirectDestId, contentType, isExternal, isSkipped)');
-    // Targets `getSummary` Q4 (`failedPageIdRows`) — selects pages with
-    // `scraped=1 AND status=-1 AND redirectDestId IS NULL`. `status=-1`
-    // is highly selective (a few hundred rows on archives with
-    // ~400 k `scraped=1` pages), but without this index the planner
-    // seeks all `scraped=1` rows via `pages_scraped_index` and then
-    // row-by-row filters status, costing ~5 s. The 3-column form
-    // `(scraped, status, redirectDestId)` is fully covering for
-    // `SELECT id` and gives a 5113 ms → 14 ms (~365x) reduction
-    // verified by `scripts/prototype-summary-indexes.mjs`.
-    //
-    // Note the column order: `status` comes second so the `(scraped=1
-    // AND status=-1)` 2-column equality seek lands directly in the
-    // failed-page slice without scanning the 400 k+ healthy rows.
-    // Putting `redirectDestId` last keeps it as a 3rd-level seek
-    // constraint that the planner folds into the slice once the
-    // (scraped, status) pair is fixed.
-    await instance.raw('CREATE INDEX idx_pages_summary_failed ON pages(scraped, status, redirectDestId)');
+    // 0.13 ref / header dictionary tables. DDL + column-level rationale
+    // lives in {@link createRefTables}, shared with the migration script's
+    // schema catch-up — a divergence between the two paths would silently
+    // break the population step's UNIQUE / CHECK contract.
+    // `createRefTables` uses bare CREATE TABLE, so it needs a sentinel
+    // guard here (all 10 tables are created together — any one of them
+    // works as the sentinel).
+    if (!(await instance.schema.hasTable('url_refs'))) {
+        await createRefTables(instance);
+    }
+    // 0.13 core entity / edge tables — the write-model the crawler writes
+    // during a crawl and every reader queries. MUST run after
+    // {@link createRefTables} because `content_items`, `page_meta`,
+    // `resource_items`, `anchor_edges`, and `image_items` all reference
+    // the ref tables (`url_refs`, `content_type_refs`, `text_refs`,
+    // `json_refs`, `blob_refs`, `header_sets`) via FK clauses.
+    await createEntityTables(instance);
+    // Adjunct tables that FK into `content_items` (page_errors / page_tags /
+    // page_jsonld / analysis_* / page_html_*) plus the standalone log tables
+    // (crawl_errors / inventory_runs). MUST run after
+    // {@link createEntityTables} so the FK targets exist. DDL +
+    // column-level rationale lives in {@link createAdjunctTables}, which is
+    // shared with `scripts/migrate-to-0.13.mjs` — a divergence between the
+    // two paths is exactly how migrated archives ended up with stale
+    // `REFERENCES pages(id)` declarations in the pre-0.13 era.
+    await createAdjunctTables(instance);
 }

package/lib/archive/limited-page-ids.d.ts

@@ -1,7 +1,8 @@
 import type { Knex } from 'knex';
 /**
  * Returns a Knex subquery builder that selects page IDs with pagination,
- * ordered by the `order` column (nulls last), excluding redirected pages.
+ * ordered by `content_items.crawl_order` (nulls last), excluding
+ * redirected pages.
  * @param limit - The maximum number of page IDs to return.
  * @param offset - The number of page IDs to skip before returning results.
  */

package/lib/archive/limited-page-ids.js

@@ -1,6 +1,7 @@
 /**
  * Returns a Knex subquery builder that selects page IDs with pagination,
- * ordered by the `order` column (nulls last), excluding redirected pages.
+ * ordered by `content_items.crawl_order` (nulls last), excluding
+ * redirected pages.
  * @param limit - The maximum number of page IDs to return.
  * @param offset - The number of page IDs to skip before returning results.
  */
@@ -8,9 +9,9 @@ export function limitedPageIds(limit, offset) {
     return async (qb) => {
         await qb
             .select('id')
-            .from('pages')
-            .orderByRaw('`order` ASC NULLS LAST')
-            .whereNull('redirectDestId')
+            .from('content_items')
+            .orderByRaw('`crawl_order` ASC NULLS LAST')
+            .whereNull('redirect_dest_id')
             .limit(limit)
             .offset(offset);
     };

package/lib/archive/meta/assert-compatible-version.d.ts

@@ -1,18 +1,39 @@
 import type { Knex } from 'knex';
 /**
  * Minimum `info.version` this build accepts. Archives older than this must
- * be upgraded with `scripts/migrate-to-0.10.mjs` before they can be opened.
+ * be upgraded by running the matching migration script before they can be
+ * opened (see {@link IncompatibleArchiveError}'s message for the mapping
+ * from archive version to script).
+ *
+ * This value tracks the **archive format cut**, not the npm `pkg.version`:
+ * a format-breaking change bumps it ahead of the corresponding release,
+ * and releases that do not cut the format leave it unchanged. The crawler
+ * writes this exact value into every new archive's `info.version`.
  *
  * History:
  *
  * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
  *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
  *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
- * - **0.10.0**: this build. `page_html_blobs` BLOB storage (#75/#84) +
+ * - **0.10.0**: `page_html_blobs` BLOB storage (#75/#84) +
  *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
  *   `page_jsonld` tables, denormalised aggregates (#85).
+ * - **0.13.0**: this build. Write-model refactor (epic #103) —
+ *   `content_items` / `page_meta` / `resource_items` / `anchor_edges` /
+ *   `resource_ref_edges` / `image_items` entity tables plus `url_refs` /
+ *   `text_refs` / `content_type_refs` / `json_refs` / `blob_refs` /
+ *   `header_flags` ref tables. Readers query the new tables exclusively;
+ *   pre-0.13 archives must be upgraded with `scripts/migrate-to-0.13.mjs`
+ *   which bumps `info.version` to `0.13.0` on completion.
+ * @example
+ * import { REQUIRED_FORMAT_VERSION } from '@nitpicker/crawler';
+ *
+ * // Warn an operator before attempting to open an old archive:
+ * if (compareSemver(archiveInfo.version, REQUIRED_FORMAT_VERSION) < 0) {
+ *   console.error(`Archive needs migration to format ${REQUIRED_FORMAT_VERSION}`);
+ * }
  */
-export declare const REQUIRED_FORMAT_VERSION = "0.10.0";
+export declare const REQUIRED_FORMAT_VERSION = "0.13.0";
 /**
  * Verifies that the archive's on-disk format is compatible with this build.
  *