@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/migrate-page-meta-custom-element-count.js

@@ -0,0 +1,51 @@
+/**
+ * Adds the `page_meta.main_content_custom_element_count` column to archives
+ * created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same catch-up
+ * shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
+ *
+ * Deliberately its own file rather than an addition to
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}:
+ * that migration early-returns once its sentinel column
+ * (`main_content_word_count`) already exists, so an archive that predates
+ * only this one column (but already has the other sixteen main-content
+ * columns) would never reach an appended `ALTER TABLE` inside it.
+ *
+ * No separate backfill: a page that has never been re-scraped since this
+ * feature shipped stays `NULL` (unknown), which is correct — it is not
+ * conflated with `0` (capture succeeded, zero custom elements found). See
+ * {@link import('./meta/compute-main-contents-denormalized.js').computeMainContentsDenormalized}.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export async function migratePageMetaCustomElementCount(instance, onLog) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'main_content_custom_element_count');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.integer('main_content_custom_element_count');
+        });
+        const message = '[migrate] page_meta.main_content_custom_element_count column added';
+        if (onLog) {
+            onLog(message);
+        }
+        else {
+            // eslint-disable-next-line no-console
+            console.error(message);
+        }
+    }
+}

package/lib/migrate-page-meta-image-scan.d.ts

@@ -0,0 +1,27 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.image_scan_desktop` / `page_meta.image_scan_mobile`
+ * columns to archives created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same catch-up
+ * shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
+ *
+ * Both columns are added in the same migration since they are always
+ * written together (one `@d-zero/beholder` `imageScan` result per page).
+ * There is no backfill: a page that has never been re-scraped since this
+ * feature shipped has no recorded outcome, so `NULL` (not attempted) is
+ * already the correct value.
+ *
+ * Idempotent: adding a column that already exists is a no-op.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export declare function migratePageMetaImageScan(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/migrate-page-meta-image-scan.js

@@ -0,0 +1,52 @@
+/**
+ * Adds the `page_meta.image_scan_desktop` / `page_meta.image_scan_mobile`
+ * columns to archives created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same catch-up
+ * shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
+ *
+ * Both columns are added in the same migration since they are always
+ * written together (one `@d-zero/beholder` `imageScan` result per page).
+ * There is no backfill: a page that has never been re-scraped since this
+ * feature shipped has no recorded outcome, so `NULL` (not attempted) is
+ * already the correct value.
+ *
+ * Idempotent: adding a column that already exists is a no-op.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export async function migratePageMetaImageScan(instance, onLog) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasDesktopColumn = await instance.schema.hasColumn('page_meta', 'image_scan_desktop');
+    const hasMobileColumn = await instance.schema.hasColumn('page_meta', 'image_scan_mobile');
+    if (hasDesktopColumn && hasMobileColumn) {
+        return;
+    }
+    await instance.schema.table('page_meta', (t) => {
+        if (!hasDesktopColumn) {
+            t.integer('image_scan_desktop');
+        }
+        if (!hasMobileColumn) {
+            t.integer('image_scan_mobile');
+        }
+    });
+    const message = '[migrate] page_meta.image_scan_desktop/image_scan_mobile columns added';
+    if (onLog) {
+        onLog(message);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(message);
+    }
+}

package/lib/migrate-page-tags-to-page-technologies.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Converts every `page_tags` row (the removed Wappalyzer-only detection
+ * table) into `technology_signals` + `page_technologies` rows, then drops
+ * `page_tags` — for archives crawled before this feature shipped, which
+ * still carry the old table.
+ *
+ * Unlike every other `migrate*` function in this directory (which add a
+ * nullable column and never touch existing tables), this one both writes
+ * to two NEW tables and DROPS an old one — because `page_technologies`
+ * supersedes `page_tags` outright rather than extending it. The conversion
+ * itself is shared with `retarget-legacy-fk-tables.ts`'s special-cased
+ * `page_tags` handling (0.10→0.13 upgrade path) via
+ * `convertLegacyPageTagsToInserts` — a `page_tags` row converts identically
+ * regardless of which migration path produced it.
+ *
+ * No separate backfill step is needed (v0.x policy: read-time migration
+ * only, no retroactive re-crawl): only the `wappalyzer` signal survives the
+ * conversion — the structural/`js-license-comment` signals this feature
+ * also detects require re-crawling to populate, which is the same
+ * "existing archives need a re-crawl for new signal types" trade-off
+ * already accepted for this feature.
+ *
+ * Idempotent: a no-op once `page_tags` is gone (either because this
+ * function already ran, or because the archive was crawled after this
+ * feature shipped and never had `page_tags` at all).
+ *
+ * Logs a row count before starting (issue #294): on a large legacy archive
+ * this SELECT-all-convert-drop runs as one transaction with no per-chunk
+ * progress, so without an upfront count it looks indistinguishable from a
+ * hang for however long the conversion actually takes.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` for both the start and
+ *   end notices (issue #294: a bare `console.error` here can fire while a
+ *   `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export declare function migratePageTagsToPageTechnologies(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/migrate-page-tags-to-page-technologies.js

@@ -0,0 +1,79 @@
+import { convertLegacyPageTagsToInserts } from './meta/technologies/convert-legacy-page-tags-to-inserts.js';
+import { eachSplitted } from './utils/array/each-splitted.js';
+/**
+ * Converts every `page_tags` row (the removed Wappalyzer-only detection
+ * table) into `technology_signals` + `page_technologies` rows, then drops
+ * `page_tags` — for archives crawled before this feature shipped, which
+ * still carry the old table.
+ *
+ * Unlike every other `migrate*` function in this directory (which add a
+ * nullable column and never touch existing tables), this one both writes
+ * to two NEW tables and DROPS an old one — because `page_technologies`
+ * supersedes `page_tags` outright rather than extending it. The conversion
+ * itself is shared with `retarget-legacy-fk-tables.ts`'s special-cased
+ * `page_tags` handling (0.10→0.13 upgrade path) via
+ * `convertLegacyPageTagsToInserts` — a `page_tags` row converts identically
+ * regardless of which migration path produced it.
+ *
+ * No separate backfill step is needed (v0.x policy: read-time migration
+ * only, no retroactive re-crawl): only the `wappalyzer` signal survives the
+ * conversion — the structural/`js-license-comment` signals this feature
+ * also detects require re-crawling to populate, which is the same
+ * "existing archives need a re-crawl for new signal types" trade-off
+ * already accepted for this feature.
+ *
+ * Idempotent: a no-op once `page_tags` is gone (either because this
+ * function already ran, or because the archive was crawled after this
+ * feature shipped and never had `page_tags` at all).
+ *
+ * Logs a row count before starting (issue #294): on a large legacy archive
+ * this SELECT-all-convert-drop runs as one transaction with no per-chunk
+ * progress, so without an upfront count it looks indistinguishable from a
+ * hang for however long the conversion actually takes.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` for both the start and
+ *   end notices (issue #294: a bare `console.error` here can fire while a
+ *   `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export async function migratePageTagsToPageTechnologies(instance, onLog) {
+    const hasPageTags = await instance.schema.hasTable('page_tags');
+    if (!hasPageTags)
+        return;
+    const [row] = await instance('page_tags').count({ total: '*' });
+    const total = row?.total ?? 0;
+    const startMessage = `[migrate] converting ${total} page_tags row(s) to technology_signals/page_technologies…`;
+    if (onLog) {
+        onLog(startMessage);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(startMessage);
+    }
+    await instance.transaction(async (trx) => {
+        const rows = await trx
+            .select('pageId', 'provider', 'version', 'confidence', 'categories')
+            .from('page_tags');
+        const { signalInserts, technologyInserts } = convertLegacyPageTagsToInserts(rows);
+        if (signalInserts.length > 0) {
+            await eachSplitted(signalInserts, 100, async (chunk) => {
+                await trx('technology_signals').insert(chunk);
+            });
+        }
+        if (technologyInserts.length > 0) {
+            await eachSplitted(technologyInserts, 100, async (chunk) => {
+                await trx('page_technologies').insert(chunk);
+            });
+        }
+        await trx.schema.dropTable('page_tags');
+    });
+    const endMessage = '[migrate] page_tags converted to technology_signals/page_technologies and dropped';
+    if (onLog) {
+        onLog(endMessage);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(endMessage);
+    }
+}

package/lib/migrate-ref-tables.d.ts

@@ -0,0 +1,25 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the 0.13 dictionary tables (issue #190) to archives created before
+ * the 0.13 format.
+ *
+ * These are the 10 ref / header dictionary tables that form the
+ * durable write-model of the 0.13 format. Fresh archives get them via
+ * `initSchema`; this migration is called only by
+ * `scripts/migrate-to-0.13.mjs` as its schema catch-up step for pre-0.13
+ * input archives. The archive-open path (`db-ops/lifecycle/init.ts`) does
+ * not run it — `assertCompatibleVersion` guarantees every openable
+ * archive already has the full table set.
+ *
+ * The migration is intentionally table-only. It does not back-fill any data
+ * from `pages` / `resources` / `anchors` / `images` into the ref tables — that
+ * is the job of the 0.13 population step, which reads from the legacy
+ * tables and writes the ref rows in a single WAL transaction with `.bak`
+ * protection. This migration only guarantees the empty tables exist so
+ * consumers don't crash with `SQLITE_ERROR: no such table: url_refs`.
+ *
+ * Idempotent: presence of `url_refs` is used as the sentinel (all 10
+ * tables are created together, so any one of them can serve as the sentinel).
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateRefTables(instance: Knex): Promise<void>;

package/lib/migrate-ref-tables.js

@@ -0,0 +1,38 @@
+import { createRefTables } from './create-ref-tables.js';
+/**
+ * Adds the 0.13 dictionary tables (issue #190) to archives created before
+ * the 0.13 format.
+ *
+ * These are the 10 ref / header dictionary tables that form the
+ * durable write-model of the 0.13 format. Fresh archives get them via
+ * `initSchema`; this migration is called only by
+ * `scripts/migrate-to-0.13.mjs` as its schema catch-up step for pre-0.13
+ * input archives. The archive-open path (`db-ops/lifecycle/init.ts`) does
+ * not run it — `assertCompatibleVersion` guarantees every openable
+ * archive already has the full table set.
+ *
+ * The migration is intentionally table-only. It does not back-fill any data
+ * from `pages` / `resources` / `anchors` / `images` into the ref tables — that
+ * is the job of the 0.13 population step, which reads from the legacy
+ * tables and writes the ref rows in a single WAL transaction with `.bak`
+ * protection. This migration only guarantees the empty tables exist so
+ * consumers don't crash with `SQLITE_ERROR: no such table: url_refs`.
+ *
+ * Idempotent: presence of `url_refs` is used as the sentinel (all 10
+ * tables are created together, so any one of them can serve as the sentinel).
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateRefTables(instance) {
+    const hasUrlRefs = await instance.schema.hasTable('url_refs');
+    if (hasUrlRefs) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; the regular initSchema path will create the tables.
+        return;
+    }
+    await createRefTables(instance);
+    // eslint-disable-next-line no-console
+    console.error('[migrate] 0.13 ref/header tables created');
+}

package/lib/native-executor/check-native-executor-config.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Checks the native thread pool's environment settings
+ * (`NITPICKER_NATIVE_THREADS`, `NITPICKER_NATIVE_CONCURRENCY`) before any work
+ * starts. The settings, their defaults and when they are read are documented
+ * on `@nitpicker/core`'s `checkNativeExecutorConfig`.
+ *
+ * Why here and not a direct import of core: the CLI and the MCP server reach
+ * the addon only through this package (as `query` does for the snapshot
+ * scans), so the dependency direction stays `archive ← cli / mcp-server`.
+ * @throws {Error} With code `InvalidArg`, naming the variable and its value,
+ *   when a setting is not a positive integer.
+ * @example
+ * ```ts
+ * checkNativeExecutorConfig(); // throws before the server starts if the env is wrong
+ * ```
+ */
+export declare function checkNativeExecutorConfig(): void;

package/lib/native-executor/check-native-executor-config.js

@@ -0,0 +1,20 @@
+import { checkNativeExecutorConfig as checkNativeExecutorConfigNatively } from '@nitpicker/core/check-native-executor-config';
+/**
+ * Checks the native thread pool's environment settings
+ * (`NITPICKER_NATIVE_THREADS`, `NITPICKER_NATIVE_CONCURRENCY`) before any work
+ * starts. The settings, their defaults and when they are read are documented
+ * on `@nitpicker/core`'s `checkNativeExecutorConfig`.
+ *
+ * Why here and not a direct import of core: the CLI and the MCP server reach
+ * the addon only through this package (as `query` does for the snapshot
+ * scans), so the dependency direction stays `archive ← cli / mcp-server`.
+ * @throws {Error} With code `InvalidArg`, naming the variable and its value,
+ *   when a setting is not a positive integer.
+ * @example
+ * ```ts
+ * checkNativeExecutorConfig(); // throws before the server starts if the env is wrong
+ * ```
+ */
+export function checkNativeExecutorConfig() {
+    checkNativeExecutorConfigNatively();
+}

package/lib/page-meta-column-maps.d.ts

@@ -0,0 +1,32 @@
+import type { FlatPageMetaColumns } from './meta/types.js';
+/**
+ * Flat-meta-field → `page_meta` FK-column mapping, shared by the two
+ * writers that materialise `page_meta` rows: the live-crawl upsert
+ * (`db-ops/pages/write/insert-page.ts`) and the archive-migration
+ * populate (`populate-entity-tables/populate-page-meta.ts`).
+ *
+ * The two writers MUST agree on this mapping — a fork would make
+ * live-crawled archives and migrated archives disagree on which flat
+ * meta field feeds which `page_meta` column for structurally identical
+ * inputs. Keeping the single definition here is what enforces that;
+ * adding, renaming, or removing a flat meta column is a one-place edit.
+ *
+ * - `text`: text-shaped columns mapped 1:1 to `<name>_text_id` FKs into
+ *   `text_refs`.
+ * - `url`: URL-shaped columns mapped 1:1 to `<name>_url_id` FKs into
+ *   `url_refs`.
+ * @example
+ * for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+ *   row[target] = textIds.get(flat[source]) ?? null;
+ * }
+ */
+export declare const PAGE_META_COLUMN_MAPS: {
+    readonly text: readonly {
+        source: keyof FlatPageMetaColumns;
+        target: string;
+    }[];
+    readonly url: readonly {
+        source: keyof FlatPageMetaColumns;
+        target: string;
+    }[];
+};

package/lib/page-meta-column-maps.js

@@ -0,0 +1,43 @@
+/**
+ * Flat-meta-field → `page_meta` FK-column mapping, shared by the two
+ * writers that materialise `page_meta` rows: the live-crawl upsert
+ * (`db-ops/pages/write/insert-page.ts`) and the archive-migration
+ * populate (`populate-entity-tables/populate-page-meta.ts`).
+ *
+ * The two writers MUST agree on this mapping — a fork would make
+ * live-crawled archives and migrated archives disagree on which flat
+ * meta field feeds which `page_meta` column for structurally identical
+ * inputs. Keeping the single definition here is what enforces that;
+ * adding, renaming, or removing a flat meta column is a one-place edit.
+ *
+ * - `text`: text-shaped columns mapped 1:1 to `<name>_text_id` FKs into
+ *   `text_refs`.
+ * - `url`: URL-shaped columns mapped 1:1 to `<name>_url_id` FKs into
+ *   `url_refs`.
+ * @example
+ * for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+ *   row[target] = textIds.get(flat[source]) ?? null;
+ * }
+ */
+export const PAGE_META_COLUMN_MAPS = {
+    text: [
+        { source: 'title', target: 'title_text_id' },
+        { source: 'description', target: 'description_text_id' },
+        { source: 'keywords', target: 'keywords_text_id' },
+        { source: 'robots_raw', target: 'robots_raw_text_id' },
+        { source: 'og_title', target: 'og_title_text_id' },
+        { source: 'og_description', target: 'og_description_text_id' },
+        { source: 'twitter_title', target: 'twitter_title_text_id' },
+        { source: 'twitter_description', target: 'twitter_description_text_id' },
+    ],
+    url: [
+        { source: 'canonical', target: 'canonical_url_id' },
+        { source: 'amphtml', target: 'amphtml_url_id' },
+        { source: 'manifest', target: 'manifest_url_id' },
+        { source: 'icon_href', target: 'icon_url_id' },
+        { source: 'appleTouchIcon_href', target: 'apple_touch_icon_url_id' },
+        { source: 'og_url', target: 'og_url_id' },
+        { source: 'og_image', target: 'og_image_url_id' },
+        { source: 'twitter_image', target: 'twitter_image_url_id' },
+    ],
+};