@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/meta/assert-compatible-version.js

@@ -0,0 +1,93 @@
+import { compareSemver } from './compare-semver.js';
+import { IncompatibleArchiveError } from './types.js';
+/**
+ * Minimum `info.version` this build accepts. Archives older than this must
+ * be upgraded by running the matching migration script before they can be
+ * opened (see {@link IncompatibleArchiveError}'s message for the mapping
+ * from archive version to script).
+ *
+ * This value tracks the **archive format cut**, not the npm `pkg.version`:
+ * a format-breaking change bumps it ahead of the corresponding release,
+ * and releases that do not cut the format leave it unchanged. The crawler
+ * writes this exact value into every new archive's `info.version`.
+ *
+ * History:
+ *
+ * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
+ *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
+ *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
+ * - **0.10.0**: `page_html_blobs` BLOB storage (#75/#84) +
+ *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
+ *   `page_jsonld` tables, denormalised aggregates (#85).
+ * - **0.13.0**: this build. Write-model refactor (epic #103) —
+ *   `content_items` / `page_meta` / `resource_items` / `anchor_edges` /
+ *   `resource_ref_edges` / `image_items` entity tables plus `url_refs` /
+ *   `text_refs` / `content_type_refs` / `json_refs` / `blob_refs` /
+ *   `header_flags` ref tables. Readers query the new tables exclusively;
+ *   pre-0.13 archives must be upgraded with `scripts/migrate-to-0.13.mjs`
+ *   which bumps `info.version` to `0.13.0` on completion.
+ * @example
+ * import { REQUIRED_FORMAT_VERSION } from '@nitpicker/archive/meta/assert-compatible-version';
+ *
+ * // Warn an operator before attempting to open an old archive:
+ * if (compareSemver(archiveInfo.version, REQUIRED_FORMAT_VERSION) < 0) {
+ *   console.error(`Archive needs migration to format ${REQUIRED_FORMAT_VERSION}`);
+ * }
+ */
+export const REQUIRED_FORMAT_VERSION = '0.13.0';
+/**
+ * Verifies that the archive's on-disk format is compatible with this build.
+ *
+ * Compares the archive's `info.version` (a semver string written by
+ * `setConfig` at archive-create time, or by the migration script) against
+ * {@link REQUIRED_FORMAT_VERSION}. Older archives throw
+ * {@link IncompatibleArchiveError} pointing the operator at the migration
+ * script.
+ *
+ * Called from `Database.#init` for both writer and read-only (stub viewer)
+ * connections so old `._nitpicker-*` stubs surface the error too. New
+ * archives where the `info` table does not yet exist are tolerated —
+ * `initSchema` will fill them in next.
+ *
+ * The check is intentionally version-string-only, not schema-shape-based:
+ * `info.version` is the single declared source of truth, and a v0.10
+ * `migrate-to-0.10.mjs` run bumps it explicitly so the assertion passes
+ * once migration completes.
+ * @param instance - The Knex query builder for the archive's libsql connection.
+ * @throws {IncompatibleArchiveError} when `info.version` is older than
+ *   {@link REQUIRED_FORMAT_VERSION}, or missing entirely on a non-empty
+ *   archive.
+ */
+export async function assertCompatibleVersion(instance) {
+    const hasInfo = await instance.schema.hasTable('info');
+    if (!hasInfo) {
+        // Brand-new archive — `initSchema` will create `info` and fill in
+        // the version next.
+        return;
+    }
+    const hasVersionColumn = await instance.schema.hasColumn('info', 'version');
+    if (!hasVersionColumn) {
+        // Pre-version-tracked archive (very old). The column did not exist
+        // before the version was added to the info schema; reject with
+        // `'unknown'` so the operator runs the migration script.
+        throw new IncompatibleArchiveError('unknown', REQUIRED_FORMAT_VERSION);
+    }
+    const row = await instance
+        .from('info')
+        .select('version')
+        .first();
+    if (row === undefined) {
+        // `Archive.create()` calls `Database.connect` (which runs
+        // `initSchema` to create the info table) BEFORE `setConfig` writes
+        // the initial row. The transient empty-info state is a normal step
+        // of archive creation, not a corrupted pre-0.10 archive.
+        return;
+    }
+    const archiveVersion = row.version ?? null;
+    if (archiveVersion === null || archiveVersion === '') {
+        throw new IncompatibleArchiveError('unknown', REQUIRED_FORMAT_VERSION);
+    }
+    if (compareSemver(archiveVersion, REQUIRED_FORMAT_VERSION) < 0) {
+        throw new IncompatibleArchiveError(archiveVersion, REQUIRED_FORMAT_VERSION);
+    }
+}

package/lib/meta/classify-jsonld-type.d.ts

@@ -0,0 +1,23 @@
+import type { JsonLdEntry } from './types.js';
+/**
+ * Extracts the top-level `@type` from a parsed {@link JsonLdEntry}, normalised
+ * to a single string suitable for an indexed SQL column.
+ *
+ * JSON-LD `@type` has four shapes in the wild:
+ *
+ * - **String** — common single-type case (e.g. `"Product"`).
+ * - **Array** — multi-type entries (e.g. `["Product", "Offer"]`). The first
+ *   element is taken; the rest are recoverable from `parsed`.
+ * - **Missing** — top-level object without an `@type` (often a `@graph`
+ *   wrapper). Returns `null` so the column is filterable but does not over-
+ *   fit a single type.
+ * - **Parse error** — `parsed === undefined` because beholder set
+ *   `parseError`. Returns `null`.
+ *
+ * `@graph` entries deliberately return `null` rather than `'@graph'` — a
+ * graph is a container, not a type, and consumers grouping by `@type` would
+ * see noise.
+ * @param entry - One JSON-LD entry as captured by beholder.
+ * @returns The normalised `@type` string, or `null` when not extractable.
+ */
+export declare function classifyJsonLdType(entry: JsonLdEntry): string | null;

package/lib/meta/classify-jsonld-type.js

@@ -0,0 +1,43 @@
+/**
+ * Extracts the top-level `@type` from a parsed {@link JsonLdEntry}, normalised
+ * to a single string suitable for an indexed SQL column.
+ *
+ * JSON-LD `@type` has four shapes in the wild:
+ *
+ * - **String** — common single-type case (e.g. `"Product"`).
+ * - **Array** — multi-type entries (e.g. `["Product", "Offer"]`). The first
+ *   element is taken; the rest are recoverable from `parsed`.
+ * - **Missing** — top-level object without an `@type` (often a `@graph`
+ *   wrapper). Returns `null` so the column is filterable but does not over-
+ *   fit a single type.
+ * - **Parse error** — `parsed === undefined` because beholder set
+ *   `parseError`. Returns `null`.
+ *
+ * `@graph` entries deliberately return `null` rather than `'@graph'` — a
+ * graph is a container, not a type, and consumers grouping by `@type` would
+ * see noise.
+ * @param entry - One JSON-LD entry as captured by beholder.
+ * @returns The normalised `@type` string, or `null` when not extractable.
+ */
+export function classifyJsonLdType(entry) {
+    if (entry.parsed === undefined || entry.parsed === null)
+        return null;
+    if (typeof entry.parsed !== 'object')
+        return null;
+    const obj = entry.parsed;
+    const rawType = obj['@type'];
+    if (typeof rawType === 'string') {
+        const trimmed = rawType.trim();
+        return trimmed === '' ? null : trimmed;
+    }
+    if (Array.isArray(rawType)) {
+        for (const t of rawType) {
+            if (typeof t === 'string') {
+                const trimmed = t.trim();
+                if (trimmed !== '')
+                    return trimmed;
+            }
+        }
+    }
+    return null;
+}

package/lib/meta/compare-semver.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Compares two semver-like version strings.
+ *
+ * Only the major / minor / patch numeric components are considered;
+ * pre-release tags (`-alpha.1`) and build metadata (`+sha.abc`) are
+ * dropped before comparison so `"0.10.0-alpha.1"` compares equal to
+ * `"0.10.0"`. Missing components default to `0`, so `"0.10"` compares
+ * equal to `"0.10.0"`.
+ *
+ * Designed for `assertCompatibleVersion`'s
+ * "is the archive's `info.version` at least the required format
+ * version?" check — we do not need full semver semantics there, and
+ * pulling in the `semver` package would be overkill for one numeric
+ * comparison.
+ * @param a - The left-hand version string.
+ * @param b - The right-hand version string.
+ * @returns Negative when `a < b`, positive when `a > b`, zero when equal.
+ * @example
+ * compareSemver('0.9.0', '0.10.0')          // → -1
+ * compareSemver('0.10.0', '0.10.0')         // → 0
+ * compareSemver('0.10.5-alpha.1', '0.10.0') // → +5
+ */
+export declare function compareSemver(a: string, b: string): number;

package/lib/meta/compare-semver.js

@@ -0,0 +1,51 @@
+/**
+ * Compares two semver-like version strings.
+ *
+ * Only the major / minor / patch numeric components are considered;
+ * pre-release tags (`-alpha.1`) and build metadata (`+sha.abc`) are
+ * dropped before comparison so `"0.10.0-alpha.1"` compares equal to
+ * `"0.10.0"`. Missing components default to `0`, so `"0.10"` compares
+ * equal to `"0.10.0"`.
+ *
+ * Designed for `assertCompatibleVersion`'s
+ * "is the archive's `info.version` at least the required format
+ * version?" check — we do not need full semver semantics there, and
+ * pulling in the `semver` package would be overkill for one numeric
+ * comparison.
+ * @param a - The left-hand version string.
+ * @param b - The right-hand version string.
+ * @returns Negative when `a < b`, positive when `a > b`, zero when equal.
+ * @example
+ * compareSemver('0.9.0', '0.10.0')          // → -1
+ * compareSemver('0.10.0', '0.10.0')         // → 0
+ * compareSemver('0.10.5-alpha.1', '0.10.0') // → +5
+ */
+export function compareSemver(a, b) {
+    const pa = parseComponents(a);
+    const pb = parseComponents(b);
+    for (let i = 0; i < 3; i++) {
+        const diff = (pa[i] ?? 0) - (pb[i] ?? 0);
+        if (diff !== 0)
+            return diff;
+    }
+    return 0;
+}
+/**
+ * Strips pre-release / build metadata and parses the first three numeric
+ * components. Missing components default to `0`. Non-numeric components
+ * collapse to `0` rather than throwing — `assertCompatibleVersion` needs
+ * defensive behavior against hand-edited `info.version` strings.
+ * @param version
+ */
+function parseComponents(version) {
+    const core = version.split(/[-+]/)[0] ?? '';
+    const parts = core.split('.');
+    const major = Number.parseInt(parts[0] ?? '0', 10);
+    const minor = Number.parseInt(parts[1] ?? '0', 10);
+    const patch = Number.parseInt(parts[2] ?? '0', 10);
+    return [
+        Number.isFinite(major) ? major : 0,
+        Number.isFinite(minor) ? minor : 0,
+        Number.isFinite(patch) ? patch : 0,
+    ];
+}

package/lib/meta/compute-main-contents-denormalized.d.ts

@@ -0,0 +1,41 @@
+import type { ImageScanColumns, MainContentsDenormalizedColumns } from './types.js';
+import type { MainContentsData, ScrollHeightData } from '../utils/types/types.js';
+/**
+ * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
+ * aggregate columns from beholder's `MainContentsData` / `ScrollHeightData`.
+ *
+ * `mainContents` is `null` for pages that were not fully rendered (external,
+ * non-HTML, or metadata-only scrapes) — in that case every column is `null`.
+ * This mirrors {@link import('./compute-page-denormalized.js').computePageDenormalized}'s
+ * write-once-at-scrape-time pattern so list / detail reads never re-derive
+ * counts from the `page_main_content_*` child tables.
+ *
+ * Accepts `undefined` as well as `null`: `PageData` declares `mainContents`
+ * as required, but test fixtures across the codebase predate this field and
+ * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
+ * uncaught at compile time) — `== null` tolerates both without forcing every
+ * fixture to be updated.
+ *
+ * `customElementCount` is a separate parameter, not read off `mainContents`,
+ * because it is not one of beholder's `MainContentsData` categories —
+ * nitpicker captures Web Components itself (`@nitpicker/crawler`'s `crawler/capture-custom-elements.ts`)
+ * after `scrapeStart` returns. It carries three distinct states:
+ * `undefined`/capture-not-attempted and capture-failure both collapse to
+ * `null` (unknown — NOT the same as "captured, zero found"), while any
+ * number (including `0`) means capture succeeded.
+ * `imageScan` follows the same independent-of-`mainContents` shape as
+ * `scrollHeight` (both are per-viewport beholder scan outcomes, not derived
+ * from the main-content region) — it is typed as a plain `number | null`
+ * pair here rather than beholder's `ImageScanCode` so this function does not
+ * require a `@d-zero/beholder` version that exports it; the caller narrows.
+ * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
+ * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
+ * @param customElementCount - Count of Web Components nitpicker captured in
+ *   the main-content region, or `null`/`undefined` when capture was not
+ *   attempted or failed.
+ * @param imageScan - Beholder's per-viewport `<img>` scan outcome codes, or
+ *   `null`/`undefined` when not attempted (page not fully rendered, or the
+ *   scraper version does not report it yet).
+ * @returns The twenty denormalised columns.
+ */
+export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined, customElementCount?: number | null, imageScan?: ImageScanColumns | null): MainContentsDenormalizedColumns;

package/lib/meta/compute-main-contents-denormalized.js

@@ -0,0 +1,88 @@
+/**
+ * Computes the `page_meta.main_content_*` / `scroll_height_*` denormalised
+ * aggregate columns from beholder's `MainContentsData` / `ScrollHeightData`.
+ *
+ * `mainContents` is `null` for pages that were not fully rendered (external,
+ * non-HTML, or metadata-only scrapes) — in that case every column is `null`.
+ * This mirrors {@link import('./compute-page-denormalized.js').computePageDenormalized}'s
+ * write-once-at-scrape-time pattern so list / detail reads never re-derive
+ * counts from the `page_main_content_*` child tables.
+ *
+ * Accepts `undefined` as well as `null`: `PageData` declares `mainContents`
+ * as required, but test fixtures across the codebase predate this field and
+ * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
+ * uncaught at compile time) — `== null` tolerates both without forcing every
+ * fixture to be updated.
+ *
+ * `customElementCount` is a separate parameter, not read off `mainContents`,
+ * because it is not one of beholder's `MainContentsData` categories —
+ * nitpicker captures Web Components itself (`@nitpicker/crawler`'s `crawler/capture-custom-elements.ts`)
+ * after `scrapeStart` returns. It carries three distinct states:
+ * `undefined`/capture-not-attempted and capture-failure both collapse to
+ * `null` (unknown — NOT the same as "captured, zero found"), while any
+ * number (including `0`) means capture succeeded.
+ * `imageScan` follows the same independent-of-`mainContents` shape as
+ * `scrollHeight` (both are per-viewport beholder scan outcomes, not derived
+ * from the main-content region) — it is typed as a plain `number | null`
+ * pair here rather than beholder's `ImageScanCode` so this function does not
+ * require a `@d-zero/beholder` version that exports it; the caller narrows.
+ * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
+ * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
+ * @param customElementCount - Count of Web Components nitpicker captured in
+ *   the main-content region, or `null`/`undefined` when capture was not
+ *   attempted or failed.
+ * @param imageScan - Beholder's per-viewport `<img>` scan outcome codes, or
+ *   `null`/`undefined` when not attempted (page not fully rendered, or the
+ *   scraper version does not report it yet).
+ * @returns The twenty denormalised columns.
+ */
+export function computeMainContentsDenormalized(mainContents, scrollHeight, customElementCount, imageScan) {
+    if (mainContents == null) {
+        return {
+            main_content_node_name: null,
+            main_content_id: null,
+            main_content_role: null,
+            main_content_selector: null,
+            main_content_class_list: null,
+            main_content_word_count: null,
+            main_content_body_word_count: null,
+            main_content_heading_count: null,
+            main_content_image_count: null,
+            main_content_table_count: null,
+            main_content_button_count: null,
+            main_content_iframe_count: null,
+            main_content_video_count: null,
+            main_content_audio_count: null,
+            main_content_canvas_count: null,
+            main_content_custom_element_count: null,
+            scroll_height_desktop: null,
+            scroll_height_mobile: null,
+            image_scan_desktop: null,
+            image_scan_mobile: null,
+        };
+    }
+    return {
+        main_content_node_name: mainContents.main?.nodeName ?? null,
+        main_content_id: mainContents.main?.id ?? null,
+        main_content_role: mainContents.main?.role ?? null,
+        main_content_selector: mainContents.main?.selector ?? null,
+        main_content_class_list: mainContents.main
+            ? JSON.stringify(mainContents.main.classList)
+            : null,
+        main_content_word_count: mainContents.wordCount,
+        main_content_body_word_count: mainContents.bodyWordCount,
+        main_content_heading_count: mainContents.headings.length,
+        main_content_image_count: mainContents.images.length,
+        main_content_table_count: mainContents.tables.length,
+        main_content_button_count: mainContents.buttons.length,
+        main_content_iframe_count: mainContents.iframes.length,
+        main_content_video_count: mainContents.videos.length,
+        main_content_audio_count: mainContents.audios.length,
+        main_content_canvas_count: mainContents.canvases.length,
+        main_content_custom_element_count: customElementCount ?? null,
+        scroll_height_desktop: scrollHeight?.desktop ?? null,
+        scroll_height_mobile: scrollHeight?.mobile ?? null,
+        image_scan_desktop: imageScan?.desktop ?? null,
+        image_scan_mobile: imageScan?.mobile ?? null,
+    };
+}

package/lib/meta/compute-page-denormalized.d.ts

@@ -0,0 +1,20 @@
+import type { PageDenormalizedColumns, Meta } from './types.js';
+/**
+ * Computes the denormalised aggregate columns stored on the `pages` table at
+ * write time: `tag_count`, `jsonld_count`, `tags_providers_csv`.
+ *
+ * Why denormalise: the Sheets report and the `get-page-detail` summary need
+ * "how many JSON-LD entries?" and "which Wappalyzer providers?" for every
+ * page row. Computing those at read time forces either a per-page sub-query
+ * (N+1) or a site-wide `GROUP BY` (heavy on 1M-page archives). Writing them
+ * once at scrape time keeps the read path to a single `pages` projection.
+ *
+ * `tags_providers_csv` uses comma as separator (no escaping). Wappalyzer
+ * provider names do not contain commas in practice; if that ever changes,
+ * switch to a JSON array column or use `\x1f` as separator. The CSV form
+ * exists because Sheets / Google Spreadsheet renders comma-separated lists
+ * natively without unwrapping JSON.
+ * @param meta - Beholder-derived metadata for the page.
+ * @returns The three denormalised columns.
+ */
+export declare function computePageDenormalized(meta: Meta): PageDenormalizedColumns;

package/lib/meta/compute-page-denormalized.js

@@ -0,0 +1,35 @@
+/**
+ * Computes the denormalised aggregate columns stored on the `pages` table at
+ * write time: `tag_count`, `jsonld_count`, `tags_providers_csv`.
+ *
+ * Why denormalise: the Sheets report and the `get-page-detail` summary need
+ * "how many JSON-LD entries?" and "which Wappalyzer providers?" for every
+ * page row. Computing those at read time forces either a per-page sub-query
+ * (N+1) or a site-wide `GROUP BY` (heavy on 1M-page archives). Writing them
+ * once at scrape time keeps the read path to a single `pages` projection.
+ *
+ * `tags_providers_csv` uses comma as separator (no escaping). Wappalyzer
+ * provider names do not contain commas in practice; if that ever changes,
+ * switch to a JSON array column or use `\x1f` as separator. The CSV form
+ * exists because Sheets / Google Spreadsheet renders comma-separated lists
+ * natively without unwrapping JSON.
+ * @param meta - Beholder-derived metadata for the page.
+ * @returns The three denormalised columns.
+ */
+export function computePageDenormalized(meta) {
+    // `??` guards tolerate the legacy "minimal meta" shape (`{ title: 'X' }`)
+    // produced by older test fixtures. Real beholder 3.0.0 always populates
+    // these required fields; the guards only matter for test fixtures that
+    // pre-date the v2 schema.
+    const tagEntries = meta.tags?.entries ?? [];
+    const jsonLd = meta.jsonLd ?? [];
+    const speculationRules = meta.speculationRules ?? [];
+    const tag_count = tagEntries.length;
+    const jsonld_count = jsonLd.length + speculationRules.length;
+    const providers = new Set();
+    for (const entry of tagEntries) {
+        providers.add(entry.provider);
+    }
+    const tags_providers_csv = [...providers].toSorted().join(',');
+    return { tag_count, jsonld_count, tags_providers_csv };
+}

package/lib/meta/derive-flat-from-meta.d.ts

@@ -0,0 +1,34 @@
+import type { FlatPageMetaColumns, Meta } from './types.js';
+/**
+ * Derives the {@link FlatPageMetaColumns} (`pages` table flat columns) from a
+ * beholder {@link Meta} object.
+ *
+ * Handles three concerns the database layer should not re-implement:
+ *
+ * 1. **Null-safe traversal** of the new nested Meta shape
+ *    (`meta.og?.title`, `meta.robots?.noindex`, `meta.link?.canonical`, etc.).
+ *    All optional sub-objects are guarded so an absent meta tag becomes
+ *    `null` rather than a runtime crash.
+ *
+ * 2. **URL absolutisation** for every URL-shaped column. beholder 3.0.0
+ *    extracts link/og href values via `link.getAttribute('href')`, which
+ *    preserves relative URLs as-written. `find-mismatches` compares
+ *    `canonical != url` directly, so we must resolve everything against the
+ *    page URL (or `<base href>` if present) before persistence. Malformed
+ *    URLs become `null` rather than blowing up the entire insert.
+ *
+ * 3. **Boolean → integer coercion** for SQLite-style boolean columns
+ *    (`robots_*`, `formatDetection_telephone`). `undefined` and unparseable
+ *    values map to `null`.
+ *
+ * URL-shaped columns: `canonical`, `og_url`, `og_image`, `amphtml`,
+ * `manifest`, `icon_href`, `appleTouchIcon_href`, `twitter_image`.
+ * @param meta - Beholder-derived metadata for the page.
+ * @param pageUrl - Absolute URL of the page being persisted. Used as the
+ *   fallback base when `meta.baseHref` is absent or relative.
+ * @returns A flat object whose keys map 1:1 to `pages` table columns.
+ * @example
+ * deriveFlatFromMeta(meta, 'https://example.com/about')
+ *   // → { canonical: 'https://example.com/about', og_title: '...', ... }
+ */
+export declare function deriveFlatFromMeta(meta: Meta, pageUrl: string): FlatPageMetaColumns;

package/lib/meta/derive-flat-from-meta.js

@@ -0,0 +1,158 @@
+/**
+ * Derives the {@link FlatPageMetaColumns} (`pages` table flat columns) from a
+ * beholder {@link Meta} object.
+ *
+ * Handles three concerns the database layer should not re-implement:
+ *
+ * 1. **Null-safe traversal** of the new nested Meta shape
+ *    (`meta.og?.title`, `meta.robots?.noindex`, `meta.link?.canonical`, etc.).
+ *    All optional sub-objects are guarded so an absent meta tag becomes
+ *    `null` rather than a runtime crash.
+ *
+ * 2. **URL absolutisation** for every URL-shaped column. beholder 3.0.0
+ *    extracts link/og href values via `link.getAttribute('href')`, which
+ *    preserves relative URLs as-written. `find-mismatches` compares
+ *    `canonical != url` directly, so we must resolve everything against the
+ *    page URL (or `<base href>` if present) before persistence. Malformed
+ *    URLs become `null` rather than blowing up the entire insert.
+ *
+ * 3. **Boolean → integer coercion** for SQLite-style boolean columns
+ *    (`robots_*`, `formatDetection_telephone`). `undefined` and unparseable
+ *    values map to `null`.
+ *
+ * URL-shaped columns: `canonical`, `og_url`, `og_image`, `amphtml`,
+ * `manifest`, `icon_href`, `appleTouchIcon_href`, `twitter_image`.
+ * @param meta - Beholder-derived metadata for the page.
+ * @param pageUrl - Absolute URL of the page being persisted. Used as the
+ *   fallback base when `meta.baseHref` is absent or relative.
+ * @returns A flat object whose keys map 1:1 to `pages` table columns.
+ * @example
+ * deriveFlatFromMeta(meta, 'https://example.com/about')
+ *   // → { canonical: 'https://example.com/about', og_title: '...', ... }
+ */
+export function deriveFlatFromMeta(meta, pageUrl) {
+    const base = resolveBase(meta.baseHref, pageUrl);
+    const robots = meta.robots;
+    const og = meta.og;
+    const ogArticle = og?.article;
+    const twitter = meta.twitter;
+    const link = meta.link;
+    const formatDetection = meta.formatDetection;
+    return {
+        // Document basics
+        lang: nullableString(meta.lang),
+        dir: nullableString(meta.dir),
+        charset: nullableString(meta.charset),
+        baseHref: absolutizeUrl(meta.baseHref, pageUrl),
+        viewport_raw: nullableString(meta.viewport?.raw),
+        themeColor: nullableString(meta.themeColor),
+        applicationName: nullableString(meta.applicationName),
+        author: nullableString(meta.author),
+        generator: nullableString(meta.generator),
+        publisher: nullableString(meta.publisher),
+        // Robots
+        robots_raw: nullableString(robots?.raw),
+        robots_noindex: booleanToInt(robots?.noindex),
+        robots_nofollow: booleanToInt(robots?.nofollow),
+        robots_noarchive: booleanToInt(robots?.noarchive),
+        robots_noimageindex: booleanToInt(robots?.noimageindex),
+        googlebot: nullableString(meta.googlebot),
+        // Link (1:1 only)
+        canonical: absolutizeUrl(link?.canonical, base),
+        amphtml: absolutizeUrl(link?.amphtml, base),
+        manifest: absolutizeUrl(link?.manifest, base),
+        icon_href: absolutizeUrl(link?.icon?.href, base),
+        appleTouchIcon_href: absolutizeUrl(link?.appleTouchIcon?.href, base),
+        // Open Graph
+        og_type: nullableString(og?.type),
+        og_title: nullableString(og?.title),
+        og_url: absolutizeUrl(og?.url, base),
+        og_site_name: nullableString(og?.siteName),
+        og_description: nullableString(og?.description),
+        og_image: absolutizeUrl(og?.imageUrl ?? og?.image?.[0], base),
+        og_image_alt: nullableString(og?.imageAlt),
+        og_image_width: nullableString(og?.imageWidth),
+        og_image_height: nullableString(og?.imageHeight),
+        og_locale: nullableString(og?.locale),
+        og_article_published_time: nullableString(ogArticle?.publishedTime),
+        og_article_modified_time: nullableString(ogArticle?.modifiedTime),
+        // Twitter
+        twitter_card: nullableString(twitter?.card),
+        twitter_site: nullableString(twitter?.site),
+        twitter_creator: nullableString(twitter?.creator),
+        twitter_title: nullableString(twitter?.title),
+        twitter_description: nullableString(twitter?.description),
+        twitter_image: absolutizeUrl(twitter?.image ?? twitter?.imageSrc, base),
+        // One-offs
+        fb_app_id: nullableString(meta.fb?.appId),
+        verification_google: nullableString(meta.verification?.google),
+        formatDetection_telephone: booleanToInt(formatDetection?.telephone),
+        // Title / description / keywords (top-level Meta fields)
+        title: nullableString(meta.title),
+        description: nullableString(meta.description),
+        keywords: nullableString(meta.keywords),
+    };
+}
+/**
+ * Resolves the effective base URL for relative-URL absolutisation.
+ *
+ * Falls back to `pageUrl` when `baseHref` is absent. When `baseHref` itself
+ * is a relative URL (rare but spec-permitted), resolves it against
+ * `pageUrl` first so subsequent `new URL(value, base)` calls see an
+ * absolute base.
+ * @param baseHref
+ * @param pageUrl
+ */
+function resolveBase(baseHref, pageUrl) {
+    if (!baseHref)
+        return pageUrl;
+    try {
+        return new URL(baseHref, pageUrl).href;
+    }
+    catch {
+        return pageUrl;
+    }
+}
+/**
+ * Resolves a possibly-relative URL against a base.
+ *
+ * Returns `null` for `undefined` / empty string / unparseable input so a
+ * malformed page does not abort the entire `#insertPage` call.
+ * @param value
+ * @param base
+ */
+function absolutizeUrl(value, base) {
+    if (value === undefined || value === null || value === '')
+        return null;
+    try {
+        return new URL(value, base).href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Normalises optional / empty strings to `null` so SQL `IS NULL` queries
+ * behave consistently. Trim is intentional — leading/trailing whitespace in
+ * `<meta content=>` is meaningless for downstream consumers.
+ * @param value
+ */
+function nullableString(value) {
+    if (value === undefined || value === null)
+        return null;
+    const trimmed = value.trim();
+    return trimmed === '' ? null : trimmed;
+}
+/**
+ * Converts an optional boolean to SQLite's 0/1 integer representation. Any
+ * non-boolean (`undefined`, non-boolean string from `boolean-true` keys)
+ * becomes `null`.
+ * @param value
+ */
+function booleanToInt(value) {
+    if (value === true)
+        return 1;
+    if (value === false)
+        return 0;
+    return null;
+}

package/lib/meta/derive-meta-extras.d.ts

@@ -0,0 +1,20 @@
+import type { Meta } from './types.js';
+/**
+ * Derives the `meta_extras` JSON payload from a beholder {@link Meta} object.
+ *
+ * Strategy: pass-through everything except data that lives in dedicated
+ * tables (`page_jsonld`, `page_tags`) or that is debug-only
+ * (`_raw`). Flat columns are intentionally **not** stripped — the modest
+ * storage duplication is preferred over a fragile field-by-field denylist
+ * that breaks every time beholder grows new Meta fields.
+ *
+ * The output is JSON-stringified by the caller (`#insertPage`) and stored in
+ * `pages.meta_extras`. Consumers read it back via `get-page-detail` and the
+ * Page wrapper's `metaExtras` getter.
+ *
+ * Plan note: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先" — this is exactly that
+ * trade-off. Future Meta fields are auto-captured without code change.
+ * @param meta - Beholder-derived metadata for the page.
+ * @returns Plain object suitable for `JSON.stringify`.
+ */
+export declare function deriveMetaExtras(meta: Meta): Record<string, unknown>;

package/lib/meta/derive-meta-extras.js

@@ -0,0 +1,23 @@
+/**
+ * Derives the `meta_extras` JSON payload from a beholder {@link Meta} object.
+ *
+ * Strategy: pass-through everything except data that lives in dedicated
+ * tables (`page_jsonld`, `page_tags`) or that is debug-only
+ * (`_raw`). Flat columns are intentionally **not** stripped — the modest
+ * storage duplication is preferred over a fragile field-by-field denylist
+ * that breaks every time beholder grows new Meta fields.
+ *
+ * The output is JSON-stringified by the caller (`#insertPage`) and stored in
+ * `pages.meta_extras`. Consumers read it back via `get-page-detail` and the
+ * Page wrapper's `metaExtras` getter.
+ *
+ * Plan note: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先" — this is exactly that
+ * trade-off. Future Meta fields are auto-captured without code change.
+ * @param meta - Beholder-derived metadata for the page.
+ * @returns Plain object suitable for `JSON.stringify`.
+ */
+export function deriveMetaExtras(meta) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars -- we destructure to omit these from the spread
+    const { jsonLd, speculationRules, tags, _raw, ...rest } = meta;
+    return rest;
+}