@nitpicker/crawler 0.12.0 → 0.13.0

package/README.md

@@ -1,12 +1,14 @@
 # @nitpicker/crawler
 
-ヘッドレスブラウザによる Web クローラーエンジン。
+ヘッドレスブラウザでWebサイトをクロールし、`.nitpicker` アーカイブを生成・更新する内部パッケージです。
 
-## 概要
+通常は [@nitpicker/cli](../cli/README.md) の `crawl` コマンドから利用します。
 
-Puppeteer を使用して Web サイトをクロールし、各ページのメタデータ・リンク構造・ネットワークリソース・レンダリング後 HTML スナップショットを SQLite ベースのアーカイブ（`.nitpicker`）に保存します。
+## 関連リンク
 
-このパッケージは [Nitpicker](../../README.md) モノレポの内部パッケージです。単体での利用は想定していません。
+- [Nitpicker README](../../../README.md)
+- [CLI crawl docs](../cli/docs/crawl.md)
+- [ARCHITECTURE.md](../../../ARCHITECTURE.md)
 
 ## ライセンス
 

package/lib/archive/archive-accessor.d.ts

@@ -39,7 +39,7 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * tmpDir), where touching the filesystem would race with — or destroy —
      * the live crawler's working state.
      *
-     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * Subclasses that own the archive's lifecycle (notably `Archive`)
      * override this to add write/cleanup steps.
      *
      * **Idempotent and concurrent-safe**: the first invocation captures the
@@ -171,7 +171,7 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      * Retrieves a flat list of all resource URLs stored in the archive.
      * @returns An array of resource URL strings.
      */
-    getResourceUrlList(): Promise<any[]>;
+    getResourceUrlList(): Promise<string[]>;
     /**
      * Retrieves the Wappalyzer tag entries for the given page, parsed back
      * from the `page_tags` table.

package/lib/archive/archive-accessor.js

@@ -30,7 +30,7 @@ export class ArchiveAccessor extends EventEmitter {
      * Promise tracking an in-progress (or completed) close. `null` means the
      * accessor is open and idle; a settled promise means we are closed (the
      * accessor stays "closed" even if `db.destroy()` rejected, because there
-     * is nothing safe to retry — see {@link close}).
+     * is nothing safe to retry — see {@link ArchiveAccessor.close}).
      */
     #closeOnce = null;
     /** The SQLite database instance for querying archived data. */
@@ -91,7 +91,7 @@ export class ArchiveAccessor extends EventEmitter {
      * tmpDir), where touching the filesystem would race with — or destroy —
      * the live crawler's working state.
      *
-     * Subclasses that own the archive's lifecycle (notably {@link Archive})
+     * Subclasses that own the archive's lifecycle (notably `Archive`)
      * override this to add write/cleanup steps.
      *
      * **Idempotent and concurrent-safe**: the first invocation captures the

package/lib/archive/archive-lock.d.ts

@@ -34,5 +34,12 @@ export declare class ArchiveLockError extends Error {
  * @param tmpDir - Absolute path to the archive's temporary working directory.
  * @returns A release function to be called when the work is done.
  * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
+ * @example
+ * const releaseLock = await acquireArchiveLock(tmpDir);
+ * try {
+ *   // ... exclusive work against tmpDir ...
+ * } finally {
+ *   await releaseLock();
+ * }
  */
 export declare function acquireArchiveLock(tmpDir: string): Promise<() => Promise<void>>;

package/lib/archive/archive-lock.js

@@ -42,6 +42,13 @@ export class ArchiveLockError extends Error {
  * @param tmpDir - Absolute path to the archive's temporary working directory.
  * @returns A release function to be called when the work is done.
  * @throws {ArchiveLockError} When the lock cannot be acquired even after a stale-lock retry.
+ * @example
+ * const releaseLock = await acquireArchiveLock(tmpDir);
+ * try {
+ *   // ... exclusive work against tmpDir ...
+ * } finally {
+ *   await releaseLock();
+ * }
  */
 export async function acquireArchiveLock(tmpDir) {
     const lockPath = `${tmpDir}.lock`;

package/lib/archive/archive.d.ts

@@ -14,6 +14,15 @@ import { ArchiveAccessor } from './archive-accessor.js';
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
  * The constructor is private.
+ * @example
+ * const archive = await Archive.create({ filePath: '/path/to/site.nitpicker' });
+ * try {
+ *   await archive.setConfig(config);
+ *   const pageId = await archive.setPage(pageData);
+ * } finally {
+ *   // Writes the `.nitpicker` tar (if absent), removes tmpDir, releases the lock.
+ *   await archive.close();
+ * }
  */
 export default class Archive extends ArchiveAccessor {
     #private;
@@ -112,15 +121,14 @@ export default class Archive extends ArchiveAccessor {
      * Retrieves the base URL of the crawl session from the archive database.
      * @returns The base URL string.
      */
-    getUrl(): Promise<any>;
+    getUrl(): Promise<string>;
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
-     * {@link Archive.insertInventorySeeds}. Replaces the previous per-URL
-     * `setResources` loop in `CrawlerOrchestrator.inventory` so the
-     * ingestion phase commits all non-HTML rows in one chunked round-trip
-     * per 500 (a 50k-URL inventory list dropped from minutes-inside-`.bak`
-     * to seconds).
+     * {@link Archive.insertInventorySeeds}. Rows are committed in chunked
+     * bulk inserts (500 per round-trip) rather than per-URL awaits — a
+     * per-URL loop would keep a 50k-URL inventory list inside the `.bak`
+     * window for minutes instead of seconds.
      *
      * Thin facade over {@link Database.insertInventoryResources}.
      * `ExURL.href` is the storage key for `resources.url` (matches what
@@ -137,7 +145,7 @@ export default class Archive extends ArchiveAccessor {
      * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
      *
      * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
-     * key matches what `#getIdByUrl` writes for crawled rows, keeping the
+     * key matches what `resolveContentItemId` writes for crawled rows, keeping the
      * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
      * lookups consistent.
      * @param urls - HTML seed URLs to pre-insert. No-op when empty.
@@ -180,6 +188,24 @@ export default class Archive extends ArchiveAccessor {
      * are mutually exclusive (the first one called wins).
      */
     releaseHandle(): Promise<void>;
+    /**
+     * Replaces the archive's analysis violations with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replaceAnalysisViolations}; kept on
+     * `Archive` so the analyze pipeline can persist violations without
+     * reaching into the low-level database class directly.
+     * @param violations - Flat analyze violations.
+     */
+    replaceAnalysisViolations(violations: readonly {
+        validator: string;
+        severity: string;
+        rule: string;
+        code?: string | null;
+        message: string;
+        url: string;
+        line?: number | null;
+        col?: number | null;
+    }[]): Promise<void>;
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -287,23 +313,44 @@ export default class Archive extends ArchiveAccessor {
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX: string;
     /**
-     * Opens a read-only connection to an existing archive's database.
+     * Opens a connection to an existing archive's database, defaulting to
+     * read-only.
      *
-     * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive. The DB is opened
-     * in **read-only mode**: no schema migrations run, and the connection
+     * Returns an {@link ArchiveAccessor} that provides query methods. In the
+     * default read-only mode, no schema migrations run and the connection
      * refuses to resurrect a missing parent directory or db file (so a
      * TOCTOU window between source classification and this call cannot
-     * silently produce an empty phantom tmpDir).
+     * silently produce an empty phantom tmpDir); the returned accessor is
+     * also marked read-only so consumer-facing helpers (e.g.
+     * {@link ArchiveAccessor.getHtmlOfPage}) avoid any filesystem mutation
+     * on the user's tmpDir.
      *
-     * The returned accessor is also marked read-only so consumer-facing
-     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
-     * filesystem mutation on the user's tmpDir.
+     * `options.readOnly: false` is a narrow escape hatch for opening a
+     * second, writable connection to a `tmpDir` that {@link Archive.openCached}
+     * already extracted (and migrated) into an OS-temp cache directory —
+     * never the caller's live/interrupted crawl tmpDir, which must stay
+     * read-only. A read-only open (`Archive.openCached`/`ArchiveManager.open`)
+     * must never take this path itself — blocking or writing during what
+     * must be a read-only open is forbidden (issue #177). This escape
+     * hatch has no current production caller; any future
+     * one is responsible for its own cross-process coordination (see
+     * `acquireArchiveLock`) — this method does not acquire any lock itself.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
+     * @param options - Connection options.
+     * @param options.readOnly - Defaults to `true`. Pass `false` to obtain a
+     *   writable accessor against an already-extracted cache directory.
      * @returns An ArchiveAccessor instance for querying the archive data.
+     * @example
+     * // Default (read-only) — safe for stub mode and cache reads:
+     * const accessor = await Archive.connect(tmpDir);
+     * @example
+     * // Writable escape hatch — only against a tar-cache extraction:
+     * const writable = await Archive.connect(cacheDir, null, { readOnly: false });
      */
-    static connect(tmpDir: string, namespace?: string | null): Promise<ArchiveAccessor>;
+    static connect(tmpDir: string, namespace?: string | null, options?: {
+        readOnly?: boolean;
+    }): Promise<ArchiveAccessor>;
     /**
      * Open a `.nitpicker` archive through the read-only tar cache.
      *

package/lib/archive/archive.js

@@ -27,6 +27,15 @@ import { untar } from './filesystem/untar.js';
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
  * The constructor is private.
+ * @example
+ * const archive = await Archive.create({ filePath: '/path/to/site.nitpicker' });
+ * try {
+ *   await archive.setConfig(config);
+ *   const pageId = await archive.setPage(pageData);
+ * } finally {
+ *   // Writes the `.nitpicker` tar (if absent), removes tmpDir, releases the lock.
+ *   await archive.close();
+ * }
  */
 export default class Archive extends ArchiveAccessor {
     /**
@@ -180,11 +189,10 @@ export default class Archive extends ArchiveAccessor {
     /**
      * Pre-insert inventory non-HTML URLs as `source='inventory-seed'`
      * placeholders in the `resources` table — the non-HTML counterpart of
-     * {@link Archive.insertInventorySeeds}. Replaces the previous per-URL
-     * `setResources` loop in `CrawlerOrchestrator.inventory` so the
-     * ingestion phase commits all non-HTML rows in one chunked round-trip
-     * per 500 (a 50k-URL inventory list dropped from minutes-inside-`.bak`
-     * to seconds).
+     * {@link Archive.insertInventorySeeds}. Rows are committed in chunked
+     * bulk inserts (500 per round-trip) rather than per-URL awaits — a
+     * per-URL loop would keep a 50k-URL inventory list inside the `.bak`
+     * window for minutes instead of seconds.
      *
      * Thin facade over {@link Database.insertInventoryResources}.
      * `ExURL.href` is the storage key for `resources.url` (matches what
@@ -207,7 +215,7 @@ export default class Archive extends ArchiveAccessor {
      * Ctrl+C-tolerance rationale and the `getCrawlingState` interaction.
      *
      * `ExURL` inputs are normalised to `withoutHashAndAuth` here so the storage
-     * key matches what `#getIdByUrl` writes for crawled rows, keeping the
+     * key matches what `resolveContentItemId` writes for crawled rows, keeping the
      * crawled-wins downgrade and the existing-URL filter (`getExistingPageUrls`)
      * lookups consistent.
      * @param urls - HTML seed URLs to pre-insert. No-op when empty.
@@ -267,6 +275,17 @@ export default class Archive extends ArchiveAccessor {
         this.#closeOnce = this.#runReleaseHandle();
         return this.#closeOnce;
     }
+    /**
+     * Replaces the archive's analysis violations with a fresh SQL-backed set.
+     *
+     * Thin facade over {@link Database.replaceAnalysisViolations}; kept on
+     * `Archive` so the analyze pipeline can persist violations without
+     * reaching into the low-level database class directly.
+     * @param violations - Flat analyze violations.
+     */
+    async replaceAnalysisViolations(violations) {
+        await this.#db.replaceAnalysisViolations(violations);
+    }
     /**
      * Promote previously-external pages that now fall under the (possibly extended)
      * scope back to a pending state so that the crawler re-scrapes them as fully
@@ -452,25 +471,45 @@ export default class Archive extends ArchiveAccessor {
     /** The prefix used for temporary working directories during archive operations. */
     static TMP_DIR_PREFIX = '._nitpicker-';
     /**
-     * Opens a read-only connection to an existing archive's database.
+     * Opens a connection to an existing archive's database, defaulting to
+     * read-only.
      *
-     * Returns an {@link ArchiveAccessor} that provides query methods
-     * without the ability to modify or write the archive. The DB is opened
-     * in **read-only mode**: no schema migrations run, and the connection
+     * Returns an {@link ArchiveAccessor} that provides query methods. In the
+     * default read-only mode, no schema migrations run and the connection
      * refuses to resurrect a missing parent directory or db file (so a
      * TOCTOU window between source classification and this call cannot
-     * silently produce an empty phantom tmpDir).
+     * silently produce an empty phantom tmpDir); the returned accessor is
+     * also marked read-only so consumer-facing helpers (e.g.
+     * {@link ArchiveAccessor.getHtmlOfPage}) avoid any filesystem mutation
+     * on the user's tmpDir.
      *
-     * The returned accessor is also marked read-only so consumer-facing
-     * helpers (e.g. {@link ArchiveAccessor.getHtmlOfPage}) avoid any
-     * filesystem mutation on the user's tmpDir.
+     * `options.readOnly: false` is a narrow escape hatch for opening a
+     * second, writable connection to a `tmpDir` that {@link Archive.openCached}
+     * already extracted (and migrated) into an OS-temp cache directory —
+     * never the caller's live/interrupted crawl tmpDir, which must stay
+     * read-only. A read-only open (`Archive.openCached`/`ArchiveManager.open`)
+     * must never take this path itself — blocking or writing during what
+     * must be a read-only open is forbidden (issue #177). This escape
+     * hatch has no current production caller; any future
+     * one is responsible for its own cross-process coordination (see
+     * `acquireArchiveLock`) — this method does not acquire any lock itself.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
+     * @param options - Connection options.
+     * @param options.readOnly - Defaults to `true`. Pass `false` to obtain a
+     *   writable accessor against an already-extracted cache directory.
      * @returns An ArchiveAccessor instance for querying the archive data.
+     * @example
+     * // Default (read-only) — safe for stub mode and cache reads:
+     * const accessor = await Archive.connect(tmpDir);
+     * @example
+     * // Writable escape hatch — only against a tar-cache extraction:
+     * const writable = await Archive.connect(cacheDir, null, { readOnly: false });
      */
-    static async connect(tmpDir, namespace = null) {
-        const db = await Archive.#connectDB(tmpDir, { readOnly: true });
-        const archive = new ArchiveAccessor(tmpDir, db, namespace, { readOnly: true });
+    static async connect(tmpDir, namespace = null, options = {}) {
+        const readOnly = options.readOnly ?? true;
+        const db = await Archive.#connectDB(tmpDir, { readOnly });
+        const archive = new ArchiveAccessor(tmpDir, db, namespace, { readOnly });
         return archive;
     }
     /**

package/lib/archive/create-adjunct-tables.d.ts

@@ -0,0 +1,43 @@
+import type { Knex } from 'knex';
+/**
+ * Creates the adjunct tables that hang off `content_items` (plus the two
+ * standalone log tables), guarded per-table so partially-provisioned
+ * archives converge to the full set:
+ *
+ * - `page_errors` — partial scrape failures, FK → `content_items(id)`
+ * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
+ *   an external link that failed DNS, or null for a process-level error)
+ * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
+ * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
+ *   FK → `content_items(id)`
+ * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
+ *   snapshots, FK → `content_items(id)`
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this right after `createEntityTables`) and the migration script
+ * (`scripts/migrate-to-0.13.mjs` calls it before retargeting FK
+ * declarations and dropping the legacy tables). Keeping the schema in one
+ * function guarantees both origin points produce identical tables — the
+ * pre-0.13 era kept per-table copies of this DDL in separate lazy-migration
+ * modules, and those copies drifted: they declared `REFERENCES pages(id)`
+ * while `initSchema` had moved on to `content_items(id)`, leaving migrated
+ * archives with stale FK targets that only `scripts/migrate-to-0.13.mjs`'s
+ * rename-copy-drop pass can now repair.
+ *
+ * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
+ * a single sentinel table), each table here is guarded individually because
+ * the migration-script caller sees archives where any subset may already
+ * exist (e.g. `page_tags` from the 0.10 migration but no `inventory_runs`).
+ * Index creation stays inside each guard: an existing table keeps whatever
+ * indexes its creation path declared.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * // Must run after createEntityTables — the page-scoped tables FK into
+ * // content_items(id).
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await createAdjunctTables(db);
+ */
+export declare function createAdjunctTables(instance: Knex): Promise<void>;

package/lib/archive/create-adjunct-tables.js

@@ -0,0 +1,213 @@
+/**
+ * Creates the adjunct tables that hang off `content_items` (plus the two
+ * standalone log tables), guarded per-table so partially-provisioned
+ * archives converge to the full set:
+ *
+ * - `page_errors` — partial scrape failures, FK → `content_items(id)`
+ * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
+ *   an external link that failed DNS, or null for a process-level error)
+ * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
+ * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
+ * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `analysis_text_refs` + `analysis_violations` — analyze-phase findings,
+ *   FK → `content_items(id)`
+ * - `page_html_blobs` + `page_html_ref` — content-addressable HTML
+ *   snapshots, FK → `content_items(id)`
+ *
+ * The DDL is shared between fresh-archive provisioning ({@link initSchema}
+ * calls this right after `createEntityTables`) and the migration script
+ * (`scripts/migrate-to-0.13.mjs` calls it before retargeting FK
+ * declarations and dropping the legacy tables). Keeping the schema in one
+ * function guarantees both origin points produce identical tables — the
+ * pre-0.13 era kept per-table copies of this DDL in separate lazy-migration
+ * modules, and those copies drifted: they declared `REFERENCES pages(id)`
+ * while `initSchema` had moved on to `content_items(id)`, leaving migrated
+ * archives with stale FK targets that only `scripts/migrate-to-0.13.mjs`'s
+ * rename-copy-drop pass can now repair.
+ *
+ * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
+ * a single sentinel table), each table here is guarded individually because
+ * the migration-script caller sees archives where any subset may already
+ * exist (e.g. `page_tags` from the 0.10 migration but no `inventory_runs`).
+ * Index creation stays inside each guard: an existing table keeps whatever
+ * indexes its creation path declared.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * // Must run after createEntityTables — the page-scoped tables FK into
+ * // content_items(id).
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await createAdjunctTables(db);
+ */
+export async function createAdjunctTables(instance) {
+    if (!(await instance.schema.hasTable('page_errors'))) {
+        await instance.schema.createTable('page_errors', (t) => {
+            // Records partial scrape failures (e.g. a viewport switch that
+            // detaches the frame and trips beholder's @retryable into the
+            // `retryExhausted` phase). A page can have zero or more rows here
+            // in addition to its normal `content_items` entry — the page
+            // itself is considered successfully scraped, but image capture or
+            // another secondary step failed for at least one device preset.
+            t.increments('id');
+            t.integer('pageId').notNullable().unsigned().references('content_items.id');
+            t.string('phase').notNullable();
+            t.text('message').notNullable();
+            t.integer('createdAt').notNullable();
+            t.index('pageId');
+        });
+    }
+    if (!(await instance.schema.hasTable('crawl_errors'))) {
+        await instance.schema.createTable('crawl_errors', (t) => {
+            // Structured form of the crawler-level `error` channel that otherwise
+            // only lands in `error.log`. Unlike `page_errors` these are not tied to
+            // a scraped page (the URL may be an external link that failed DNS, or
+            // null for a process-level error), so there is no `pageId` FK and `url`
+            // is nullable. The cause is NOT stored — it is classified on read from
+            // `message` so older archives (which only have `error.log`) classify the
+            // same way.
+            t.increments('id');
+            t.string('url', 8190).nullable();
+            t.boolean('isExternal');
+            t.text('message').notNullable();
+            t.integer('createdAt').notNullable();
+        });
+    }
+    if (!(await instance.schema.hasTable('page_tags'))) {
+        await instance.schema.createTable('page_tags', (t) => {
+            // Wappalyzer-derived technology detection. One row per
+            // (provider × externalId) tuple per page. `category` is the first
+            // element of `categories`; the full list lives in the JSON
+            // `categories` column. `sources` records where the provider was
+            // detected (script-src / inline / iframe-src / window-global / …).
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.string('provider').notNullable();
+            t.string('category');
+            t.string('externalId');
+            t.string('version');
+            t.integer('confidence');
+            t.json('categories');
+            t.json('sources');
+            t.index('pageId');
+            t.index('provider');
+            t.index('externalId');
+        });
+        // Compound indexes for the "find duplicate IDs across pages" and
+        // "list pages using provider X" hot paths. Knex's schema builder
+        // can't express compound indexes inline in a way that round-trips
+        // through libsql consistently, so raw SQL is used.
+        await instance.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
+        await instance.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
+    }
+    if (!(await instance.schema.hasTable('page_jsonld'))) {
+        await instance.schema.createTable('page_jsonld', (t) => {
+            // JSON-LD and SpeculationRules entries captured from
+            // `<script type="application/ld+json">` and
+            // `<script type="speculationrules">`. `kind` discriminates; `type`
+            // is the top-level `@type` extracted by classify-jsonld-type for
+            // indexable filtering. `raw` is stored uncompressed; SQLite
+            // overflow pages handle multi-KB JSON bodies transparently.
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.string('kind').notNullable();
+            t.string('type');
+            t.text('raw').notNullable();
+            t.json('parsed');
+            t.text('parseError');
+            t.index('pageId');
+            t.index('type');
+        });
+        // Compound `(type, pageId)` accelerates streaming
+        // `list_pages_by_jsonld_type` JOINs.
+        await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
+    }
+    if (!(await instance.schema.hasTable('inventory_runs'))) {
+        await instance.schema.createTable('inventory_runs', (t) => {
+            // One row per successful `--inventory <list>` invocation. The
+            // archive's audit log of "when did we apply which deploy list
+            // at what scale". `.bak` is removed on success so this table
+            // is the only durable provenance record. Column semantics live
+            // on the `InventoryRunMeta` interface in `archive/types.ts`.
+            t.increments('id');
+            t.string('ran_at').notNullable();
+            t.string('list_label').nullable();
+            t.string('source_file_sha256', 64).nullable();
+            t.integer('total_lines').nullable();
+            t.integer('new_pages').nullable();
+            t.integer('new_resources').nullable();
+            t.integer('scope_skipped').nullable();
+            t.text('notes').nullable();
+            t.index('ran_at');
+        });
+    }
+    if (!(await instance.schema.hasTable('analysis_text_refs'))) {
+        await instance.raw(`
+			CREATE TABLE analysis_text_refs (
+				id integer primary key,
+				text text not null,
+				sha256 text not null,
+				unique(sha256, text)
+			)
+		`);
+    }
+    if (!(await instance.schema.hasTable('analysis_violations'))) {
+        await instance.raw(`
+			CREATE TABLE analysis_violations (
+				id integer primary key,
+				page_id integer not null references content_items(id),
+				validator text not null,
+				severity text not null,
+				rule text not null,
+				message_text_id integer not null references analysis_text_refs(id),
+				code_text_id integer references analysis_text_refs(id),
+				page_url_sort_key text not null,
+				message_sort_key text not null,
+				code_sort_key text not null,
+				line integer,
+				col integer
+			)
+		`);
+        await instance.raw('CREATE INDEX av_url_order ON analysis_violations(page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_filter_url ON analysis_violations(validator, severity, rule, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_validator_url ON analysis_violations(validator, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_severity_url ON analysis_violations(severity, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_rule_url ON analysis_violations(rule, page_url_sort_key, id)');
+        await instance.raw('CREATE INDEX av_message_order ON analysis_violations(message_sort_key, id)');
+        await instance.raw('CREATE INDEX av_code_order ON analysis_violations(code_sort_key, id)');
+        await instance.raw('CREATE INDEX av_page ON analysis_violations(page_id, id)');
+    }
+    // Content-addressable HTML blob storage. Knex's schema builder doesn't
+    // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
+    // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
+    // (no hidden rowid + secondary index pair), which matters for the blob
+    // table where a 32-byte hash PK + multi-KB body is the dominant row
+    // shape.
+    if (!(await instance.schema.hasTable('page_html_blobs'))) {
+        await instance.raw(`
+			CREATE TABLE page_html_blobs (
+				hash         BLOB PRIMARY KEY,
+				body         BLOB NOT NULL,
+				codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+				size_raw     INTEGER NOT NULL,
+				size_stored  INTEGER NOT NULL
+			) WITHOUT ROWID
+		`);
+    }
+    if (!(await instance.schema.hasTable('page_html_ref'))) {
+        await instance.raw(`
+			CREATE TABLE page_html_ref (
+				page_id  INTEGER PRIMARY KEY REFERENCES content_items(id) ON DELETE CASCADE,
+				hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+			) WITHOUT ROWID
+		`);
+        await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
+    }
+}