@nitpicker/crawler 0.9.0 → 0.11.0

package/lib/archive/filesystem/peek-tar-top-dir.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Reads the top-level directory name from a `.nitpicker` tar archive.
+ *
+ * Why: a `.nitpicker` is a tar that wraps `<basename>/db.sqlite` (and
+ * historically other files). `Archive.write()` names the inner directory
+ * from the file's basename at write time, but users routinely rename
+ * `.nitpicker` files after the fact (e.g. `mv X.migrated.nitpicker
+ * X.nitpicker`) — a perfectly reasonable filesystem operation that breaks
+ * any code that recomputes the inner-dir name from the outer filename.
+ *
+ * This helper scans the tar's entry list and returns the first top-level
+ * directory it sees, so callers (`Archive.open`, the migration script) can
+ * use the actual inner name regardless of what the outer file is called.
+ *
+ * Reads the tar's central directory only; does not extract any data. Runs
+ * in O(number of entries) but stops at the first match, so a typical
+ * archive (1–2 top-level entries) completes in a single read.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry
+ *   (e.g. file is empty, corrupted, or non-tar). The error message
+ *   identifies the path for diagnostics.
+ * @example
+ * // The user renamed `original.nitpicker` to `renamed.nitpicker`. The
+ * // inner directory was baked in at write time and is unaffected:
+ * await peekTarTopDir('renamed.nitpicker'); // → 'original'
+ */
+export declare function peekTarTopDir(tarFilePath: string): Promise<string>;

package/lib/archive/filesystem/peek-tar-top-dir.js

@@ -0,0 +1,65 @@
+import { list } from 'tar';
+/**
+ * Reads the top-level directory name from a `.nitpicker` tar archive.
+ *
+ * Why: a `.nitpicker` is a tar that wraps `<basename>/db.sqlite` (and
+ * historically other files). `Archive.write()` names the inner directory
+ * from the file's basename at write time, but users routinely rename
+ * `.nitpicker` files after the fact (e.g. `mv X.migrated.nitpicker
+ * X.nitpicker`) — a perfectly reasonable filesystem operation that breaks
+ * any code that recomputes the inner-dir name from the outer filename.
+ *
+ * This helper scans the tar's entry list and returns the first top-level
+ * directory it sees, so callers (`Archive.open`, the migration script) can
+ * use the actual inner name regardless of what the outer file is called.
+ *
+ * Reads the tar's central directory only; does not extract any data. Runs
+ * in O(number of entries) but stops at the first match, so a typical
+ * archive (1–2 top-level entries) completes in a single read.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry
+ *   (e.g. file is empty, corrupted, or non-tar). The error message
+ *   identifies the path for diagnostics.
+ * @example
+ * // The user renamed `original.nitpicker` to `renamed.nitpicker`. The
+ * // inner directory was baked in at write time and is unaffected:
+ * await peekTarTopDir('renamed.nitpicker'); // → 'original'
+ */
+export async function peekTarTopDir(tarFilePath) {
+    let found = null;
+    await list({
+        file: tarFilePath,
+        onReadEntry: (entry) => {
+            if (found !== null)
+                return;
+            // Only Directory entries count. Files at the tar root would
+            // otherwise win — and macOS BSD tar embeds AppleDouble (`._*`)
+            // sidecar files at the tar root for resource forks; those are
+            // File entries that BSD `tar -tf` hides but Node's `tar`
+            // surfaces verbatim, so without this filter the resource-fork
+            // of the real top-level dir would win instead of the dir itself.
+            if (entry.type !== 'Directory')
+                return;
+            // `entry.path` is the path inside the tar. Strip any leading
+            // `./` (some tar producers prepend it) before splitting.
+            const cleaned = entry.path.replace(/^\.\//, '');
+            const top = cleaned.split('/')[0];
+            if (!top || top === '')
+                return;
+            // Defense in depth: skip AppleDouble directory entries
+            // (`._foo/`) and pax extended header entries
+            // (`PaxHeaders.NNN/...` / `@PaxHeader`). Real `.nitpicker`
+            // archives never name their top dir with these prefixes.
+            if (top.startsWith('._'))
+                return;
+            if (top.startsWith('PaxHeaders') || top.startsWith('@PaxHeader'))
+                return;
+            found = top;
+        },
+    });
+    if (found === null) {
+        throw new Error(`Tar contains no top-level directory entry: ${tarFilePath}`);
+    }
+    return found;
+}

package/lib/archive/init-schema.d.ts

@@ -1,8 +1,59 @@
 import type { Knex } from 'knex';
+/**
+ * Applies the connection-level PRAGMAs that govern foreign-key enforcement
+ * and BLOB-read performance. These are **per-connection** settings (libsql
+ * resets them when a new connection is opened), so they must be reapplied
+ * every time `Database.connect` runs — not just on first-time schema
+ * initialization. Keeping them separate from `initSchema`'s one-shot path
+ * also lets `page_size` (which only takes effect against an empty DB)
+ * stay gated behind the existence check.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function applyConnectionPragmas(instance: Knex): Promise<void>;
 /**
  * Initializes the archive database schema if tables do not exist.
- * Enables WAL journal mode and foreign keys, then creates all tables
- * (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
+ *
+ * Schema notes:
+ *
+ * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
+ *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
+ *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
+ *   JSON column for everything not flattened. URL-shaped columns are
+ *   absolutised against the page URL before write (see
+ *   `archive/meta/derive-flat-from-meta.ts`).
+ * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
+ *   `tags_providers_csv`): computed at write time from `meta.tags` /
+ *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
+ *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
+ * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
+ *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
+ *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
+ *   leaves them alone so failure-reset does not erase the last-success
+ *   record.
+ * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
+ *   `categories`/`sources` JSON columns. Compound indexes
+ *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
+ *   Phase 2+ "find duplicate IDs across pages" and "list pages using
+ *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
+ * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
+ *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
+ *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
+ *   if cross-archive bulk export becomes a use case, add a `codec` column
+ *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
+ *   `list_pages_by_jsonld_type` JOINs.
+ * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
+ *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
+ *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
+ *   express it.
+ * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ *   `CREATE TABLE` because SQLite only honors `page_size` changes against
+ *   an empty database, and `journal_mode = WAL` is persistent. Other
+ *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects pre-0.10 archives with a friendly
+ * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
+ * policy allows breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export declare function initSchema(instance: Knex): Promise<void>;

package/lib/archive/init-schema.js

@@ -1,7 +1,69 @@
+/**
+ * Applies the connection-level PRAGMAs that govern foreign-key enforcement
+ * and BLOB-read performance. These are **per-connection** settings (libsql
+ * resets them when a new connection is opened), so they must be reapplied
+ * every time `Database.connect` runs — not just on first-time schema
+ * initialization. Keeping them separate from `initSchema`'s one-shot path
+ * also lets `page_size` (which only takes effect against an empty DB)
+ * stay gated behind the existence check.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function applyConnectionPragmas(instance) {
+    // Foreign-key enforcement defaults to OFF on every new SQLite
+    // connection. Required for ON DELETE CASCADE on `page_html_ref`,
+    // `page_tags`, and `page_jsonld` to fire.
+    await instance.raw('PRAGMA foreign_keys = ON');
+    await instance.raw('PRAGMA wal_autocheckpoint = 1000');
+    // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
+    await instance.raw('PRAGMA cache_size = -65536');
+    // 256 MiB mmap window. SQLite falls back to read() past this so the
+    // limit is a soft ceiling, not a hard one.
+    await instance.raw('PRAGMA mmap_size = 268435456');
+}
 /**
  * Initializes the archive database schema if tables do not exist.
- * Enables WAL journal mode and foreign keys, then creates all tables
- * (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
+ *
+ * Schema notes:
+ *
+ * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
+ *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
+ *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
+ *   JSON column for everything not flattened. URL-shaped columns are
+ *   absolutised against the page URL before write (see
+ *   `archive/meta/derive-flat-from-meta.ts`).
+ * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
+ *   `tags_providers_csv`): computed at write time from `meta.tags` /
+ *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
+ *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
+ * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
+ *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
+ *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
+ *   leaves them alone so failure-reset does not erase the last-success
+ *   record.
+ * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
+ *   `categories`/`sources` JSON columns. Compound indexes
+ *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
+ *   Phase 2+ "find duplicate IDs across pages" and "list pages using
+ *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
+ * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
+ *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
+ *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
+ *   if cross-archive bulk export becomes a use case, add a `codec` column
+ *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
+ *   `list_pages_by_jsonld_type` JOINs.
+ * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
+ *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
+ *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
+ *   express it.
+ * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ *   `CREATE TABLE` because SQLite only honors `page_size` changes against
+ *   an empty database, and `journal_mode = WAL` is persistent. Other
+ *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects pre-0.10 archives with a friendly
+ * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
+ * policy allows breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export async function initSchema(instance) {
@@ -9,9 +71,11 @@ export async function initSchema(instance) {
     if (isExists) {
         return;
     }
-    // Enable WAL mode and foreign keys for better performance and data integrity
+    // Page size must be set on an empty database file; once any data is
+    // written, only VACUUM can change it. journal_mode is also one-shot
+    // (persistent) and so stays here.
+    await instance.raw('PRAGMA page_size = 16384');
     await instance.raw('PRAGMA journal_mode = WAL');
-    await instance.raw('PRAGMA foreign_keys = ON');
     await instance.schema
         .createTable('info', (t) => {
         t.increments('id');
@@ -46,31 +110,92 @@ export async function initSchema(instance) {
         t.string('contentType').nullable();
         t.integer('contentLength').unsigned().nullable();
         t.json('responseHeaders').nullable();
+        // Document basics
         t.string('lang');
+        t.string('dir');
+        t.string('charset');
+        t.string('baseHref');
+        t.text('viewport_raw');
+        t.string('themeColor');
+        t.string('applicationName');
+        t.string('author');
+        t.string('generator');
+        t.string('publisher');
+        // Title / description / keywords (top-level Meta fields)
         t.string('title');
-        t.string('description');
-        t.string('keywords');
-        t.boolean('noindex');
-        t.boolean('nofollow');
-        t.boolean('noarchive');
-        t.string('canonical');
-        t.string('alternate');
+        t.text('description');
+        t.text('keywords');
+        // Robots
+        t.text('robots_raw');
+        t.integer('robots_noindex');
+        t.integer('robots_nofollow');
+        t.integer('robots_noarchive');
+        t.integer('robots_noimageindex');
+        t.string('googlebot');
+        // Link (1:1 only — array shapes live in meta_extras)
+        t.string('canonical', 8190);
+        t.string('amphtml', 8190);
+        t.string('manifest', 8190);
+        t.string('icon_href', 8190);
+        t.string('appleTouchIcon_href', 8190);
+        // Open Graph
         t.string('og_type');
         t.string('og_title');
+        t.string('og_url', 8190);
         t.string('og_site_name');
-        t.string('og_description');
-        t.string('og_url');
-        t.string('og_image');
+        t.text('og_description');
+        t.string('og_image', 8190);
+        t.string('og_image_alt');
+        t.string('og_image_width');
+        t.string('og_image_height');
+        t.string('og_locale');
+        t.string('og_article_published_time');
+        t.string('og_article_modified_time');
+        // Twitter
         t.string('twitter_card');
-        t.string('html');
+        t.string('twitter_site');
+        t.string('twitter_creator');
+        t.string('twitter_title');
+        t.text('twitter_description');
+        t.string('twitter_image', 8190);
+        // One-offs
+        t.string('fb_app_id');
+        t.string('verification_google');
+        t.integer('formatDetection_telephone');
+        // Within-archive observation timestamps (UNIX ms)
+        t.integer('firstCrawledAt');
+        t.integer('lastCrawledAt');
+        // Denormalised aggregates (written at scrape time, see
+        // archive/meta/compute-page-denormalized.ts)
+        t.integer('tag_count');
+        t.integer('jsonld_count');
+        t.text('tags_providers_csv');
+        // Catch-all JSON for nested Meta sub-objects not flattened above
+        t.json('meta_extras');
+        // Crawl lifecycle
         t.boolean('isSkipped');
         t.string('skipReason');
         t.integer('order').unsigned().nullable();
+        // Provenance: which channel inserted this row. Values:
+        //   'crawled'              — discovered via the recursive crawl from one of `info.roots`
+        //   'inventory-seed'       — supplied directly by `crawl --inventory` URL list
+        //   'inventory-discovered' — found by following links from an `inventory-seed` page
+        // Used by `listIsolatedPages` only for badge display; isolation
+        // itself is judged by `anchors.hrefId IS NULL`, not by source.
+        t.string('source').notNullable().defaultTo('crawled');
         t.index('isExternal');
         t.index('contentType');
         t.index('scraped');
         t.index('redirectDestId');
         t.index('order');
+        // Phase 1: noindex filter (list_pages) and og:type filter
+        // (analytics) are the only new flat-column filters with enough
+        // selectivity to benefit from an index. `lang` has cardinality 1
+        // on mono-language sites (D-Zero's typical customer) so it is
+        // skipped.
+        t.index('robots_noindex');
+        t.index('og_type');
+        t.index('source');
     })
         .createTable('anchors', (t) => {
         t.increments('id');
@@ -107,6 +232,12 @@ export async function initSchema(instance) {
         t.string('compress').nullable();
         t.string('cdn').nullable();
         t.json('responseHeaders').nullable();
+        // See `pages.source` for the provenance taxonomy. `inventory-seed`
+        // rows here come from non-HTML URLs handed in by
+        // `crawl --inventory`; `inventory-discovered` rows are sub-resources
+        // pulled in while puppeteer rendered an inventory-seed page.
+        t.string('source').notNullable().defaultTo('crawled');
+        t.index('source');
     })
         .createTable('resources-referrers', (t) => {
         t.increments('id');
@@ -115,5 +246,106 @@ export async function initSchema(instance) {
         t.unique(['resourceId', 'pageId']);
         t.index('resourceId');
         t.index('pageId');
+    })
+        .createTable('page_errors', (t) => {
+        // Records partial scrape failures (e.g. a viewport switch that
+        // detaches the frame and trips beholder's @retryable into the
+        // `retryExhausted` phase). A page can have zero or more rows here
+        // in addition to its normal `pages` entry — the page itself is
+        // considered successfully scraped, but image capture or another
+        // secondary step failed for at least one device preset.
+        t.increments('id');
+        t.integer('pageId').notNullable().unsigned().references('pages.id');
+        t.string('phase').notNullable();
+        t.text('message').notNullable();
+        t.integer('createdAt').notNullable();
+        t.index('pageId');
+    })
+        .createTable('crawl_errors', (t) => {
+        // Structured form of the crawler-level `error` channel that otherwise
+        // only lands in `error.log`. Unlike `page_errors` these are not tied to
+        // a scraped page (the URL may be an external link that failed DNS, or
+        // null for a process-level error), so there is no `pageId` FK and `url`
+        // is nullable. The cause is NOT stored — it is classified on read from
+        // `message` so older archives (which only have `error.log`) classify the
+        // same way.
+        t.increments('id');
+        t.string('url', 8190).nullable();
+        t.boolean('isExternal');
+        t.text('message').notNullable();
+        t.integer('createdAt').notNullable();
+    })
+        .createTable('page_tags', (t) => {
+        // Wappalyzer-derived technology detection. One row per
+        // (provider × externalId) tuple per page. `category` is the first
+        // element of `categories`; the full list lives in the JSON
+        // `categories` column. `sources` records where the provider was
+        // detected (script-src / inline / iframe-src / window-global / …).
+        t.increments('id');
+        t.integer('pageId')
+            .notNullable()
+            .unsigned()
+            .references('pages.id')
+            .onDelete('CASCADE');
+        t.string('provider').notNullable();
+        t.string('category');
+        t.string('externalId');
+        t.string('version');
+        t.integer('confidence');
+        t.json('categories');
+        t.json('sources');
+        t.index('pageId');
+        t.index('provider');
+        t.index('externalId');
+    })
+        .createTable('page_jsonld', (t) => {
+        // JSON-LD and SpeculationRules entries captured from
+        // `<script type="application/ld+json">` and
+        // `<script type="speculationrules">`. `kind` discriminates; `type`
+        // is the top-level `@type` extracted by classify-jsonld-type for
+        // indexable filtering. `raw` is stored uncompressed; SQLite
+        // overflow pages handle multi-KB JSON bodies transparently.
+        t.increments('id');
+        t.integer('pageId')
+            .notNullable()
+            .unsigned()
+            .references('pages.id')
+            .onDelete('CASCADE');
+        t.string('kind').notNullable();
+        t.string('type');
+        t.text('raw').notNullable();
+        t.json('parsed');
+        t.text('parseError');
+        t.index('pageId');
+        t.index('type');
     });
+    // ON DELETE CASCADE and compound indexes for the new tables. Knex's
+    // schema builder can't express CASCADE / compound indexes inline in a
+    // way that round-trips through libsql consistently, so we use raw SQL
+    // to mirror the `page_html_ref` pattern.
+    await instance.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
+    await instance.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
+    await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
+    // Content-addressable HTML blob storage. Knex's schema builder doesn't
+    // expose a WITHOUT ROWID toggle, so the BLOB tables are created via raw
+    // SQL. WITHOUT ROWID keeps the rows packed inside the b-tree leaves
+    // (no hidden rowid + secondary index pair), which matters for the blob
+    // table where a 32-byte hash PK + multi-KB body is the dominant row
+    // shape.
+    await instance.raw(`
+		CREATE TABLE page_html_blobs (
+			hash         BLOB PRIMARY KEY,
+			body         BLOB NOT NULL,
+			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw     INTEGER NOT NULL,
+			size_stored  INTEGER NOT NULL
+		) WITHOUT ROWID
+	`);
+    await instance.raw(`
+		CREATE TABLE page_html_ref (
+			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
+			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+		) WITHOUT ROWID
+	`);
+    await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
 }

package/lib/archive/meta/assert-compatible-version.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Minimum `info.version` this build accepts. Archives older than this must
+ * be upgraded with `scripts/migrate-to-0.10.mjs` before they can be opened.
+ *
+ * History:
+ *
+ * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
+ *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
+ *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
+ * - **0.10.0**: this build. `page_html_blobs` BLOB storage (#75/#84) +
+ *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
+ *   `page_jsonld` tables, denormalised aggregates (#85).
+ */
+export declare const REQUIRED_FORMAT_VERSION = "0.10.0";
+/**
+ * Verifies that the archive's on-disk format is compatible with this build.
+ *
+ * Compares the archive's `info.version` (a semver string written by
+ * `setConfig` at archive-create time, or by the migration script) against
+ * {@link REQUIRED_FORMAT_VERSION}. Older archives throw
+ * {@link IncompatibleArchiveError} pointing the operator at the migration
+ * script.
+ *
+ * Called from `Database.#init` for both writer and read-only (stub viewer)
+ * connections so old `._nitpicker-*` stubs surface the error too. New
+ * archives where the `info` table does not yet exist are tolerated —
+ * `initSchema` will fill them in next.
+ *
+ * The check is intentionally version-string-only, not schema-shape-based:
+ * `info.version` is the single declared source of truth, and a v0.10
+ * `migrate-to-0.10.mjs` run bumps it explicitly so the assertion passes
+ * once migration completes.
+ * @param instance - The Knex query builder for the archive's libsql connection.
+ * @throws {IncompatibleArchiveError} when `info.version` is older than
+ *   {@link REQUIRED_FORMAT_VERSION}, or missing entirely on a non-empty
+ *   archive.
+ */
+export declare function assertCompatibleVersion(instance: Knex): Promise<void>;

package/lib/archive/meta/assert-compatible-version.js

@@ -0,0 +1,72 @@
+import { compareSemver } from './compare-semver.js';
+import { IncompatibleArchiveError } from './types.js';
+/**
+ * Minimum `info.version` this build accepts. Archives older than this must
+ * be upgraded with `scripts/migrate-to-0.10.mjs` before they can be opened.
+ *
+ * History:
+ *
+ * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
+ *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
+ *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
+ * - **0.10.0**: this build. `page_html_blobs` BLOB storage (#75/#84) +
+ *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
+ *   `page_jsonld` tables, denormalised aggregates (#85).
+ */
+export const REQUIRED_FORMAT_VERSION = '0.10.0';
+/**
+ * Verifies that the archive's on-disk format is compatible with this build.
+ *
+ * Compares the archive's `info.version` (a semver string written by
+ * `setConfig` at archive-create time, or by the migration script) against
+ * {@link REQUIRED_FORMAT_VERSION}. Older archives throw
+ * {@link IncompatibleArchiveError} pointing the operator at the migration
+ * script.
+ *
+ * Called from `Database.#init` for both writer and read-only (stub viewer)
+ * connections so old `._nitpicker-*` stubs surface the error too. New
+ * archives where the `info` table does not yet exist are tolerated —
+ * `initSchema` will fill them in next.
+ *
+ * The check is intentionally version-string-only, not schema-shape-based:
+ * `info.version` is the single declared source of truth, and a v0.10
+ * `migrate-to-0.10.mjs` run bumps it explicitly so the assertion passes
+ * once migration completes.
+ * @param instance - The Knex query builder for the archive's libsql connection.
+ * @throws {IncompatibleArchiveError} when `info.version` is older than
+ *   {@link REQUIRED_FORMAT_VERSION}, or missing entirely on a non-empty
+ *   archive.
+ */
+export async function assertCompatibleVersion(instance) {
+    const hasInfo = await instance.schema.hasTable('info');
+    if (!hasInfo) {
+        // Brand-new archive — `initSchema` will create `info` and fill in
+        // the version next.
+        return;
+    }
+    const hasVersionColumn = await instance.schema.hasColumn('info', 'version');
+    if (!hasVersionColumn) {
+        // Pre-version-tracked archive (very old). The column did not exist
+        // before the version was added to the info schema; reject with
+        // `'unknown'` so the operator runs the migration script.
+        throw new IncompatibleArchiveError('unknown', REQUIRED_FORMAT_VERSION);
+    }
+    const row = await instance
+        .from('info')
+        .select('version')
+        .first();
+    if (row === undefined) {
+        // `Archive.create()` calls `Database.connect` (which runs
+        // `initSchema` to create the info table) BEFORE `setConfig` writes
+        // the initial row. The transient empty-info state is a normal step
+        // of archive creation, not a corrupted pre-0.10 archive.
+        return;
+    }
+    const archiveVersion = row.version ?? null;
+    if (archiveVersion === null || archiveVersion === '') {
+        throw new IncompatibleArchiveError('unknown', REQUIRED_FORMAT_VERSION);
+    }
+    if (compareSemver(archiveVersion, REQUIRED_FORMAT_VERSION) < 0) {
+        throw new IncompatibleArchiveError(archiveVersion, REQUIRED_FORMAT_VERSION);
+    }
+}

package/lib/archive/meta/classify-jsonld-type.d.ts

@@ -0,0 +1,23 @@
+import type { JsonLdEntry } from '@d-zero/beholder';
+/**
+ * Extracts the top-level `@type` from a parsed {@link JsonLdEntry}, normalised
+ * to a single string suitable for an indexed SQL column.
+ *
+ * JSON-LD `@type` has four shapes in the wild:
+ *
+ * - **String** — common single-type case (e.g. `"Product"`).
+ * - **Array** — multi-type entries (e.g. `["Product", "Offer"]`). The first
+ *   element is taken; the rest are recoverable from `parsed`.
+ * - **Missing** — top-level object without an `@type` (often a `@graph`
+ *   wrapper). Returns `null` so the column is filterable but does not over-
+ *   fit a single type.
+ * - **Parse error** — `parsed === undefined` because beholder set
+ *   `parseError`. Returns `null`.
+ *
+ * `@graph` entries deliberately return `null` rather than `'@graph'` — a
+ * graph is a container, not a type, and consumers grouping by `@type` would
+ * see noise.
+ * @param entry - One JSON-LD entry as captured by beholder.
+ * @returns The normalised `@type` string, or `null` when not extractable.
+ */
+export declare function classifyJsonLdType(entry: JsonLdEntry): string | null;

package/lib/archive/meta/classify-jsonld-type.js

@@ -0,0 +1,43 @@
+/**
+ * Extracts the top-level `@type` from a parsed {@link JsonLdEntry}, normalised
+ * to a single string suitable for an indexed SQL column.
+ *
+ * JSON-LD `@type` has four shapes in the wild:
+ *
+ * - **String** — common single-type case (e.g. `"Product"`).
+ * - **Array** — multi-type entries (e.g. `["Product", "Offer"]`). The first
+ *   element is taken; the rest are recoverable from `parsed`.
+ * - **Missing** — top-level object without an `@type` (often a `@graph`
+ *   wrapper). Returns `null` so the column is filterable but does not over-
+ *   fit a single type.
+ * - **Parse error** — `parsed === undefined` because beholder set
+ *   `parseError`. Returns `null`.
+ *
+ * `@graph` entries deliberately return `null` rather than `'@graph'` — a
+ * graph is a container, not a type, and consumers grouping by `@type` would
+ * see noise.
+ * @param entry - One JSON-LD entry as captured by beholder.
+ * @returns The normalised `@type` string, or `null` when not extractable.
+ */
+export function classifyJsonLdType(entry) {
+    if (entry.parsed === undefined || entry.parsed === null)
+        return null;
+    if (typeof entry.parsed !== 'object')
+        return null;
+    const obj = entry.parsed;
+    const rawType = obj['@type'];
+    if (typeof rawType === 'string') {
+        const trimmed = rawType.trim();
+        return trimmed === '' ? null : trimmed;
+    }
+    if (Array.isArray(rawType)) {
+        for (const t of rawType) {
+            if (typeof t === 'string') {
+                const trimmed = t.trim();
+                if (trimmed !== '')
+                    return trimmed;
+            }
+        }
+    }
+    return null;
+}