@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/init-schema.js

@@ -1,3 +1,6 @@
+import { createAdjunctTables } from './create-adjunct-tables.js';
+import { createEntityTables } from './create-entity-tables.js';
+import { createRefTables } from './create-ref-tables.js';
 /**
  * Applies the connection-level PRAGMAs that govern foreign-key enforcement
  * and BLOB-read performance. These are **per-connection** settings (libsql
@@ -15,337 +18,125 @@ export async function applyConnectionPragmas(instance) {
     await instance.raw('PRAGMA foreign_keys = ON');
     await instance.raw('PRAGMA wal_autocheckpoint = 1000');
     // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
+    //
+    // Empirically validated against larger values on a 10 GB archive:
+    // bumping to 512 MiB regressed `getSummary` (1.9s → 5.7s), `pages`
+    // (2.3s → 21s), and `images` (3.7s → 12s) — libsql's page eviction
+    // policy interacts poorly with a cache sized comparable to the
+    // host's page-cache window when the DB itself far exceeds RAM.
+    // 64 MiB stays the sweet spot.
     await instance.raw('PRAGMA cache_size = -65536');
     // 256 MiB mmap window. SQLite falls back to read() past this so the
-    // limit is a soft ceiling, not a hard one.
+    // limit is a soft ceiling, not a hard one. A 4 GiB window was
+    // catastrophic on a 10 GB archive on macOS (summary 1.9s → 43s,
+    // pages 2.3s → 21s) — the kernel's read-ahead policy and libsql's
+    // mmap path interact badly when the window can cover most of the
+    // DB. Keep this conservative.
     await instance.raw('PRAGMA mmap_size = 268435456');
 }
 /**
  * Initializes the archive database schema if tables do not exist.
  *
- * Schema notes:
+ * The schema is composed of four groups, each owned by a dedicated DDL
+ * function so the migration script (`scripts/migrate-to-0.13.mjs`) can
+ * provision the exact same shapes on archives it upgrades:
  *
- * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
- *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
- *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
- *   JSON column for everything not flattened. URL-shaped columns are
- *   absolutised against the page URL before write (see
- *   `archive/meta/derive-flat-from-meta.ts`).
- * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
- *   `tags_providers_csv`): computed at write time from `meta.tags` /
- *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
- *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
- * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
- *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
- *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
- *   leaves them alone so failure-reset does not erase the last-success
- *   record.
- * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
- *   `categories`/`sources` JSON columns. Compound indexes
- *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
- *   Phase 2+ "find duplicate IDs across pages" and "list pages using
- *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
- * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
- *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
- *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
- *   if cross-archive bulk export becomes a use case, add a `codec` column
- *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
- *   `list_pages_by_jsonld_type` JOINs.
- * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
- *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
- *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
- *   express it.
- * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
- *   `CREATE TABLE` because SQLite only honors `page_size` changes against
- *   an empty database, and `journal_mode = WAL` is persistent. Other
- *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ * - **`info`** (inline below): single-row crawl configuration.
+ * - **Ref / header dictionary tables** ({@link createRefTables}):
+ *   `url_refs`, `content_type_refs`, `text_refs`, `json_refs`,
+ *   `blob_refs`, `header_name_refs`, `header_value_refs`, `header_sets`,
+ *   `header_set_entries`, `header_flags`. See that file for column-level
+ *   rationale and the reason `blob_refs` uses a regular rowid PK instead
+ *   of WITHOUT ROWID.
+ * - **Entity / edge tables** ({@link createEntityTables}):
+ *   `content_items`, `page_meta`, `resource_items`, `anchor_edges`,
+ *   `resource_ref_edges`, `image_items` — the write-model the crawler
+ *   writes during a crawl and every reader queries. Must run AFTER
+ *   `createRefTables` because most entity tables reference ref-table PKs.
+ * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
+ *   `crawl_errors`, `page_tags`, `page_jsonld`, `inventory_runs`,
+ *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
+ *   `page_html_ref`. Must run AFTER `createEntityTables` because the
+ *   page-scoped tables FK into `content_items(id)`.
  *
- * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
- * (called before `initSchema`) rejects pre-0.10 archives with a friendly
- * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
- * policy allows breaking changes.
+ * The legacy flat write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) are deliberately NOT created:
+ * they only exist inside pre-0.13 archives, where they serve as the
+ * populate source for `scripts/migrate-to-0.13.mjs` before that script
+ * drops them.
+ *
+ * **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ * `CREATE TABLE` because SQLite only honors `page_size` changes against
+ * an empty database, and `journal_mode = WAL` is persistent. Other
+ * per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.13 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects older archives with a friendly
+ * error naming the migration scripts to run; `v0.x` policy allows
+ * breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export async function initSchema(instance) {
-    const isExists = await instance.schema.hasTable('info');
-    if (isExists) {
-        return;
+    // Only the one-shot work (PRAGMAs + `info` creation) is gated on the
+    // `info` table's existence. The three DDL groups below run on EVERY
+    // call: each is internally idempotent (sentinel / IF NOT EXISTS /
+    // per-table guards), and re-running them self-heals an archive whose
+    // provisioning crashed partway through — `info` created but a later
+    // group missing. With a single all-or-nothing gate, such a stub would
+    // short-circuit here forever and every subsequent write would die with
+    // `no such table` and no way back.
+    if (!(await instance.schema.hasTable('info'))) {
+        // Page size must be set on an empty database file; once any data is
+        // written, only VACUUM can change it. journal_mode is also one-shot
+        // (persistent) and so stays here.
+        await instance.raw('PRAGMA page_size = 16384');
+        await instance.raw('PRAGMA journal_mode = WAL');
+        await instance.schema.createTable('info', (t) => {
+            t.increments('id');
+            t.string('version');
+            t.string('name');
+            t.string('baseUrl');
+            t.json('roots');
+            t.boolean('recursive');
+            t.integer('interval');
+            t.boolean('image');
+            t.boolean('fetchExternal');
+            t.integer('parallels');
+            t.json('excludes');
+            t.json('excludeKeywords');
+            t.json('excludeUrls');
+            t.integer('maxExcludedDepth');
+            t.integer('retry');
+            t.boolean('fromList');
+            t.boolean('disableQueries');
+            t.string('userAgent');
+            t.boolean('ignoreRobots');
+        });
+    }
+    // 0.13 ref / header dictionary tables. DDL + column-level rationale
+    // lives in {@link createRefTables}, shared with the migration script's
+    // schema catch-up — a divergence between the two paths would silently
+    // break the population step's UNIQUE / CHECK contract.
+    // `createRefTables` uses bare CREATE TABLE, so it needs a sentinel
+    // guard here (all 10 tables are created together — any one of them
+    // works as the sentinel).
+    if (!(await instance.schema.hasTable('url_refs'))) {
+        await createRefTables(instance);
     }
-    // Page size must be set on an empty database file; once any data is
-    // written, only VACUUM can change it. journal_mode is also one-shot
-    // (persistent) and so stays here.
-    await instance.raw('PRAGMA page_size = 16384');
-    await instance.raw('PRAGMA journal_mode = WAL');
-    await instance.schema
-        .createTable('info', (t) => {
-        t.increments('id');
-        t.string('version');
-        t.string('name');
-        t.string('baseUrl');
-        t.json('roots');
-        t.boolean('recursive');
-        t.integer('interval');
-        t.boolean('image');
-        t.boolean('fetchExternal');
-        t.integer('parallels');
-        t.json('excludes');
-        t.json('excludeKeywords');
-        t.json('excludeUrls');
-        t.integer('maxExcludedDepth');
-        t.integer('retry');
-        t.boolean('fromList');
-        t.boolean('disableQueries');
-        t.string('userAgent');
-        t.boolean('ignoreRobots');
-    })
-        .createTable('pages', (t) => {
-        t.increments('id');
-        t.string('url', 8190).notNullable().unique();
-        t.integer('redirectDestId').unsigned().references('pages.id').defaultTo(null);
-        t.boolean('scraped').notNullable();
-        t.boolean('isTarget').notNullable();
-        t.boolean('isExternal');
-        t.integer('status');
-        t.string('statusText');
-        t.string('contentType').nullable();
-        t.integer('contentLength').unsigned().nullable();
-        t.json('responseHeaders').nullable();
-        // Document basics
-        t.string('lang');
-        t.string('dir');
-        t.string('charset');
-        t.string('baseHref');
-        t.text('viewport_raw');
-        t.string('themeColor');
-        t.string('applicationName');
-        t.string('author');
-        t.string('generator');
-        t.string('publisher');
-        // Title / description / keywords (top-level Meta fields)
-        t.string('title');
-        t.text('description');
-        t.text('keywords');
-        // Robots
-        t.text('robots_raw');
-        t.integer('robots_noindex');
-        t.integer('robots_nofollow');
-        t.integer('robots_noarchive');
-        t.integer('robots_noimageindex');
-        t.string('googlebot');
-        // Link (1:1 only — array shapes live in meta_extras)
-        t.string('canonical', 8190);
-        t.string('amphtml', 8190);
-        t.string('manifest', 8190);
-        t.string('icon_href', 8190);
-        t.string('appleTouchIcon_href', 8190);
-        // Open Graph
-        t.string('og_type');
-        t.string('og_title');
-        t.string('og_url', 8190);
-        t.string('og_site_name');
-        t.text('og_description');
-        t.string('og_image', 8190);
-        t.string('og_image_alt');
-        t.string('og_image_width');
-        t.string('og_image_height');
-        t.string('og_locale');
-        t.string('og_article_published_time');
-        t.string('og_article_modified_time');
-        // Twitter
-        t.string('twitter_card');
-        t.string('twitter_site');
-        t.string('twitter_creator');
-        t.string('twitter_title');
-        t.text('twitter_description');
-        t.string('twitter_image', 8190);
-        // One-offs
-        t.string('fb_app_id');
-        t.string('verification_google');
-        t.integer('formatDetection_telephone');
-        // Within-archive observation timestamps (UNIX ms)
-        t.integer('firstCrawledAt');
-        t.integer('lastCrawledAt');
-        // Denormalised aggregates (written at scrape time, see
-        // archive/meta/compute-page-denormalized.ts)
-        t.integer('tag_count');
-        t.integer('jsonld_count');
-        t.text('tags_providers_csv');
-        // Catch-all JSON for nested Meta sub-objects not flattened above
-        t.json('meta_extras');
-        // Crawl lifecycle
-        t.boolean('isSkipped');
-        t.string('skipReason');
-        t.integer('order').unsigned().nullable();
-        // Provenance: which channel inserted this row. Values:
-        //   'crawled'              — discovered via the recursive crawl from one of `info.roots`
-        //   'inventory-seed'       — supplied directly by `crawl --inventory` URL list
-        //   'inventory-discovered' — found by following links from an `inventory-seed` page
-        // Used by `listIsolatedPages` only for badge display; isolation
-        // itself is judged by `anchors.hrefId IS NULL`, not by source.
-        t.string('source').notNullable().defaultTo('crawled');
-        t.index('isExternal');
-        t.index('contentType');
-        t.index('scraped');
-        t.index('redirectDestId');
-        t.index('order');
-        // Phase 1: noindex filter (list_pages) and og:type filter
-        // (analytics) are the only new flat-column filters with enough
-        // selectivity to benefit from an index. `lang` has cardinality 1
-        // on mono-language sites (D-Zero's typical customer) so it is
-        // skipped.
-        t.index('robots_noindex');
-        t.index('og_type');
-        t.index('source');
-    })
-        .createTable('anchors', (t) => {
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.integer('hrefId').notNullable().unsigned().references('pages.id');
-        t.string('hash');
-        t.string('textContent').nullable();
-        t.index('pageId');
-        t.index('hrefId');
-    })
-        .createTable('images', (t) => {
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.string('src', 8190);
-        t.string('currentSrc', 8190);
-        t.string('alt');
-        t.float('width').unsigned().notNullable();
-        t.float('height').unsigned().notNullable();
-        t.integer('naturalWidth').unsigned().notNullable();
-        t.integer('naturalHeight').unsigned().notNullable();
-        t.boolean('isLazy');
-        t.integer('viewportWidth').unsigned().notNullable();
-        t.string('sourceCode');
-        t.index('pageId');
-    })
-        .createTable('resources', (t) => {
-        t.increments('id');
-        t.string('url', 8190).notNullable().unique();
-        t.boolean('isExternal');
-        t.integer('status');
-        t.string('statusText');
-        t.string('contentType').nullable();
-        t.integer('contentLength').unsigned().nullable();
-        t.string('compress').nullable();
-        t.string('cdn').nullable();
-        t.json('responseHeaders').nullable();
-        // See `pages.source` for the provenance taxonomy. `inventory-seed`
-        // rows here come from non-HTML URLs handed in by
-        // `crawl --inventory`; `inventory-discovered` rows are sub-resources
-        // pulled in while puppeteer rendered an inventory-seed page.
-        t.string('source').notNullable().defaultTo('crawled');
-        t.index('source');
-    })
-        .createTable('resources-referrers', (t) => {
-        t.increments('id');
-        t.integer('resourceId').notNullable().unsigned().references('resources.id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.unique(['resourceId', 'pageId']);
-        t.index('resourceId');
-        t.index('pageId');
-    })
-        .createTable('page_errors', (t) => {
-        // Records partial scrape failures (e.g. a viewport switch that
-        // detaches the frame and trips beholder's @retryable into the
-        // `retryExhausted` phase). A page can have zero or more rows here
-        // in addition to its normal `pages` entry — the page itself is
-        // considered successfully scraped, but image capture or another
-        // secondary step failed for at least one device preset.
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.string('phase').notNullable();
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-        t.index('pageId');
-    })
-        .createTable('crawl_errors', (t) => {
-        // Structured form of the crawler-level `error` channel that otherwise
-        // only lands in `error.log`. Unlike `page_errors` these are not tied to
-        // a scraped page (the URL may be an external link that failed DNS, or
-        // null for a process-level error), so there is no `pageId` FK and `url`
-        // is nullable. The cause is NOT stored — it is classified on read from
-        // `message` so older archives (which only have `error.log`) classify the
-        // same way.
-        t.increments('id');
-        t.string('url', 8190).nullable();
-        t.boolean('isExternal');
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-    })
-        .createTable('page_tags', (t) => {
-        // Wappalyzer-derived technology detection. One row per
-        // (provider × externalId) tuple per page. `category` is the first
-        // element of `categories`; the full list lives in the JSON
-        // `categories` column. `sources` records where the provider was
-        // detected (script-src / inline / iframe-src / window-global / …).
-        t.increments('id');
-        t.integer('pageId')
-            .notNullable()
-            .unsigned()
-            .references('pages.id')
-            .onDelete('CASCADE');
-        t.string('provider').notNullable();
-        t.string('category');
-        t.string('externalId');
-        t.string('version');
-        t.integer('confidence');
-        t.json('categories');
-        t.json('sources');
-        t.index('pageId');
-        t.index('provider');
-        t.index('externalId');
-    })
-        .createTable('page_jsonld', (t) => {
-        // JSON-LD and SpeculationRules entries captured from
-        // `<script type="application/ld+json">` and
-        // `<script type="speculationrules">`. `kind` discriminates; `type`
-        // is the top-level `@type` extracted by classify-jsonld-type for
-        // indexable filtering. `raw` is stored uncompressed; SQLite
-        // overflow pages handle multi-KB JSON bodies transparently.
-        t.increments('id');
-        t.integer('pageId')
-            .notNullable()
-            .unsigned()
-            .references('pages.id')
-            .onDelete('CASCADE');
-        t.string('kind').notNullable();
-        t.string('type');
-        t.text('raw').notNullable();
-        t.json('parsed');
-        t.text('parseError');
-        t.index('pageId');
-        t.index('type');
-    });
-    // ON DELETE CASCADE and compound indexes for the new tables. Knex's
-    // schema builder can't express CASCADE / compound indexes inline in a
-    // way that round-trips through libsql consistently, so we use raw SQL
-    // to mirror the `page_html_ref` pattern.
-    await instance.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
-    await instance.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
-    await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
-    // Content-addressable HTML blob storage. Knex's schema builder doesn't
-    // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
-    // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
-    // (no hidden rowid + secondary index pair), which matters for the blob
-    // table where a 32-byte hash PK + multi-KB body is the dominant row
-    // shape.
-    await instance.raw(`
-		CREATE TABLE page_html_blobs (
-			hash         BLOB PRIMARY KEY,
-			body         BLOB NOT NULL,
-			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
-			size_raw     INTEGER NOT NULL,
-			size_stored  INTEGER NOT NULL
-		) WITHOUT ROWID
-	`);
-    await instance.raw(`
-		CREATE TABLE page_html_ref (
-			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
-			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
-		) WITHOUT ROWID
-	`);
-    await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
+    // 0.13 core entity / edge tables — the write-model the crawler writes
+    // during a crawl and every reader queries. MUST run after
+    // {@link createRefTables} because `content_items`, `page_meta`,
+    // `resource_items`, `anchor_edges`, and `image_items` all reference
+    // the ref tables (`url_refs`, `content_type_refs`, `text_refs`,
+    // `json_refs`, `blob_refs`, `header_sets`) via FK clauses.
+    await createEntityTables(instance);
+    // Adjunct tables that FK into `content_items` (page_errors / page_tags /
+    // page_jsonld / analysis_* / page_html_*) plus the standalone log tables
+    // (crawl_errors / inventory_runs). MUST run after
+    // {@link createEntityTables} so the FK targets exist. DDL +
+    // column-level rationale lives in {@link createAdjunctTables}, which is
+    // shared with `scripts/migrate-to-0.13.mjs` — a divergence between the
+    // two paths is exactly how migrated archives ended up with stale
+    // `REFERENCES pages(id)` declarations in the pre-0.13 era.
+    await createAdjunctTables(instance);
 }

package/lib/archive/is-inventory-source.d.ts

@@ -0,0 +1,21 @@
+import type { PageSource } from './types.js';
+/**
+ * Predicate that returns `true` when the given page source value belongs to
+ * the inventory chain — i.e. it is one of the `'inventory-*'` variants of
+ * {@link PageSource}.
+ *
+ * Centralises the membership check that decides whether lineage
+ * propagation should fire. Three call sites used to inline
+ * `s === 'inventory-seed' || s === 'inventory-discovered'`, which is both
+ * a DRY violation AND a future-proofing trap: when a new inventory-family
+ * label gets added (e.g. `'inventory-promoted'`), every inlined check has
+ * to be located and updated by hand. Routing through this predicate
+ * keeps the membership rule in one place.
+ *
+ * Returns `false` for `undefined` so callers can pass the raw `source`
+ * column value (which is non-NULL in the DB schema but reads as
+ * `undefined` from a missing row in JS) without a separate null check.
+ * @param source - The source value to test, or `undefined` when no row matched.
+ * @returns `true` if the source is in the inventory chain, `false` otherwise.
+ */
+export declare function isInventorySource(source: PageSource | undefined): boolean;

package/lib/archive/is-inventory-source.js

@@ -0,0 +1,22 @@
+/**
+ * Predicate that returns `true` when the given page source value belongs to
+ * the inventory chain — i.e. it is one of the `'inventory-*'` variants of
+ * {@link PageSource}.
+ *
+ * Centralises the membership check that decides whether lineage
+ * propagation should fire. Three call sites used to inline
+ * `s === 'inventory-seed' || s === 'inventory-discovered'`, which is both
+ * a DRY violation AND a future-proofing trap: when a new inventory-family
+ * label gets added (e.g. `'inventory-promoted'`), every inlined check has
+ * to be located and updated by hand. Routing through this predicate
+ * keeps the membership rule in one place.
+ *
+ * Returns `false` for `undefined` so callers can pass the raw `source`
+ * column value (which is non-NULL in the DB schema but reads as
+ * `undefined` from a missing row in JS) without a separate null check.
+ * @param source - The source value to test, or `undefined` when no row matched.
+ * @returns `true` if the source is in the inventory chain, `false` otherwise.
+ */
+export function isInventorySource(source) {
+    return source === 'inventory-seed' || source === 'inventory-discovered';
+}

package/lib/archive/limited-page-ids.d.ts

@@ -1,7 +1,8 @@
 import type { Knex } from 'knex';
 /**
  * Returns a Knex subquery builder that selects page IDs with pagination,
- * ordered by the `order` column (nulls last), excluding redirected pages.
+ * ordered by `content_items.crawl_order` (nulls last), excluding
+ * redirected pages.
  * @param limit - The maximum number of page IDs to return.
  * @param offset - The number of page IDs to skip before returning results.
  */

package/lib/archive/limited-page-ids.js

@@ -1,6 +1,7 @@
 /**
  * Returns a Knex subquery builder that selects page IDs with pagination,
- * ordered by the `order` column (nulls last), excluding redirected pages.
+ * ordered by `content_items.crawl_order` (nulls last), excluding
+ * redirected pages.
  * @param limit - The maximum number of page IDs to return.
  * @param offset - The number of page IDs to skip before returning results.
  */
@@ -8,9 +9,9 @@ export function limitedPageIds(limit, offset) {
     return async (qb) => {
         await qb
             .select('id')
-            .from('pages')
-            .orderByRaw('`order` ASC NULLS LAST')
-            .whereNull('redirectDestId')
+            .from('content_items')
+            .orderByRaw('`crawl_order` ASC NULLS LAST')
+            .whereNull('redirect_dest_id')
             .limit(limit)
             .offset(offset);
     };

package/lib/archive/meta/assert-compatible-version.d.ts

@@ -1,18 +1,39 @@
 import type { Knex } from 'knex';
 /**
  * Minimum `info.version` this build accepts. Archives older than this must
- * be upgraded with `scripts/migrate-to-0.10.mjs` before they can be opened.
+ * be upgraded by running the matching migration script before they can be
+ * opened (see {@link IncompatibleArchiveError}'s message for the mapping
+ * from archive version to script).
+ *
+ * This value tracks the **archive format cut**, not the npm `pkg.version`:
+ * a format-breaking change bumps it ahead of the corresponding release,
+ * and releases that do not cut the format leave it unchanged. The crawler
+ * writes this exact value into every new archive's `info.version`.
  *
  * History:
  *
  * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
  *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
  *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
- * - **0.10.0**: this build. `page_html_blobs` BLOB storage (#75/#84) +
+ * - **0.10.0**: `page_html_blobs` BLOB storage (#75/#84) +
  *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
  *   `page_jsonld` tables, denormalised aggregates (#85).
+ * - **0.13.0**: this build. Write-model refactor (epic #103) —
+ *   `content_items` / `page_meta` / `resource_items` / `anchor_edges` /
+ *   `resource_ref_edges` / `image_items` entity tables plus `url_refs` /
+ *   `text_refs` / `content_type_refs` / `json_refs` / `blob_refs` /
+ *   `header_flags` ref tables. Readers query the new tables exclusively;
+ *   pre-0.13 archives must be upgraded with `scripts/migrate-to-0.13.mjs`
+ *   which bumps `info.version` to `0.13.0` on completion.
+ * @example
+ * import { REQUIRED_FORMAT_VERSION } from '@nitpicker/crawler';
+ *
+ * // Warn an operator before attempting to open an old archive:
+ * if (compareSemver(archiveInfo.version, REQUIRED_FORMAT_VERSION) < 0) {
+ *   console.error(`Archive needs migration to format ${REQUIRED_FORMAT_VERSION}`);
+ * }
  */
-export declare const REQUIRED_FORMAT_VERSION = "0.10.0";
+export declare const REQUIRED_FORMAT_VERSION = "0.13.0";
 /**
  * Verifies that the archive's on-disk format is compatible with this build.
  *

package/lib/archive/meta/assert-compatible-version.js

@@ -2,18 +2,39 @@ import { compareSemver } from './compare-semver.js';
 import { IncompatibleArchiveError } from './types.js';
 /**
  * Minimum `info.version` this build accepts. Archives older than this must
- * be upgraded with `scripts/migrate-to-0.10.mjs` before they can be opened.
+ * be upgraded by running the matching migration script before they can be
+ * opened (see {@link IncompatibleArchiveError}'s message for the mapping
+ * from archive version to script).
+ *
+ * This value tracks the **archive format cut**, not the npm `pkg.version`:
+ * a format-breaking change bumps it ahead of the corresponding release,
+ * and releases that do not cut the format leave it unchanged. The crawler
+ * writes this exact value into every new archive's `info.version`.
  *
  * History:
  *
  * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
  *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
  *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
- * - **0.10.0**: this build. `page_html_blobs` BLOB storage (#75/#84) +
+ * - **0.10.0**: `page_html_blobs` BLOB storage (#75/#84) +
  *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
  *   `page_jsonld` tables, denormalised aggregates (#85).
+ * - **0.13.0**: this build. Write-model refactor (epic #103) —
+ *   `content_items` / `page_meta` / `resource_items` / `anchor_edges` /
+ *   `resource_ref_edges` / `image_items` entity tables plus `url_refs` /
+ *   `text_refs` / `content_type_refs` / `json_refs` / `blob_refs` /
+ *   `header_flags` ref tables. Readers query the new tables exclusively;
+ *   pre-0.13 archives must be upgraded with `scripts/migrate-to-0.13.mjs`
+ *   which bumps `info.version` to `0.13.0` on completion.
+ * @example
+ * import { REQUIRED_FORMAT_VERSION } from '@nitpicker/crawler';
+ *
+ * // Warn an operator before attempting to open an old archive:
+ * if (compareSemver(archiveInfo.version, REQUIRED_FORMAT_VERSION) < 0) {
+ *   console.error(`Archive needs migration to format ${REQUIRED_FORMAT_VERSION}`);
+ * }
  */
-export const REQUIRED_FORMAT_VERSION = '0.10.0';
+export const REQUIRED_FORMAT_VERSION = '0.13.0';
 /**
  * Verifies that the archive's on-disk format is compatible with this build.
  *