@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/archive/meta/types.d.ts

@@ -0,0 +1,207 @@
+/**
+ * Shared type definitions for archive-side meta processing helpers under {@link ./}.
+ *
+ * Mirrors the shape of {@link import('@d-zero/beholder').Meta} after archive-side
+ * derivation: flat columns persisted to the `pages` table, denormalised
+ * aggregates, per-row shapes for `page_jsonld` / `page_tags`, and the summary
+ * objects returned by `get-page-detail` consumers.
+ * @module
+ */
+/**
+ * Flat columns of the `pages` table derived from {@link import('@d-zero/beholder').Meta}.
+ *
+ * Each field maps to a single SQL column (string / number / boolean / null).
+ * URL-shaped columns are absolutised by the deriver before persistence so
+ * downstream consumers (e.g. `find-mismatches`) can compare against the
+ * absolute page URL directly.
+ * @see derive-flat-from-meta.ts
+ */
+export interface FlatPageMetaColumns {
+    lang: string | null;
+    dir: string | null;
+    charset: string | null;
+    baseHref: string | null;
+    viewport_raw: string | null;
+    themeColor: string | null;
+    applicationName: string | null;
+    author: string | null;
+    generator: string | null;
+    publisher: string | null;
+    robots_raw: string | null;
+    robots_noindex: number | null;
+    robots_nofollow: number | null;
+    robots_noarchive: number | null;
+    robots_noimageindex: number | null;
+    googlebot: string | null;
+    canonical: string | null;
+    amphtml: string | null;
+    manifest: string | null;
+    icon_href: string | null;
+    appleTouchIcon_href: string | null;
+    og_type: string | null;
+    og_title: string | null;
+    og_url: string | null;
+    og_site_name: string | null;
+    og_description: string | null;
+    og_image: string | null;
+    og_image_alt: string | null;
+    og_image_width: string | null;
+    og_image_height: string | null;
+    og_locale: string | null;
+    og_article_published_time: string | null;
+    og_article_modified_time: string | null;
+    twitter_card: string | null;
+    twitter_site: string | null;
+    twitter_creator: string | null;
+    twitter_title: string | null;
+    twitter_description: string | null;
+    twitter_image: string | null;
+    fb_app_id: string | null;
+    verification_google: string | null;
+    formatDetection_telephone: number | null;
+    title: string | null;
+    description: string | null;
+    keywords: string | null;
+}
+/**
+ * Denormalised aggregates computed at write time from `meta.tags` / `meta.jsonLd`.
+ *
+ * Stored on the `pages` table so list / report read paths can avoid joining
+ * `page_tags` / `page_jsonld` for the common "how many?" and "which
+ * providers?" questions.
+ * @see compute-page-denormalized.ts
+ */
+export interface PageDenormalizedColumns {
+    /** Total Wappalyzer tag entries for the page. */
+    tag_count: number;
+    /** `meta.jsonLd.length + meta.speculationRules.length`. */
+    jsonld_count: number;
+    /** Sorted unique providers, comma-separated (empty string when no tags). */
+    tags_providers_csv: string;
+}
+/**
+ * One row in the `page_jsonld` table.
+ *
+ * Captures both `<script type="application/ld+json">` (`kind = 'ld+json'`) and
+ * `<script type="speculationrules">` (`kind = 'speculationrules'`) entries.
+ * @see extract-tags-for-archive.ts (sibling for tags) and the table definition
+ * in `archive/init-schema.ts`.
+ */
+export interface JsonLdRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `pages.id`. */
+    pageId: number;
+    /** `'ld+json'` for `application/ld+json` scripts, `'speculationrules'` for speculation rules. */
+    kind: 'ld+json' | 'speculationrules';
+    /** Top-level `@type` extracted from `parsed`, normalised to a single string. `null` when missing / unparseable. */
+    type: string | null;
+    /** Original script text content (uncompressed; SQLite overflow pages handle large rows). */
+    raw: string;
+    /** Parsed JSON object (`null` when `parseError` is set). */
+    parsed: unknown | null;
+    /** Parse error message preserved from beholder; `null` when the entry parsed cleanly. */
+    parseError: string | null;
+}
+/**
+ * Insert shape for {@link JsonLdRow}.
+ *
+ * Mirrors the row shape minus the auto-increment `id`. `parsed` is the raw
+ * JSON value (the database layer JSON-stringifies it before write).
+ */
+export type JsonLdRowForInsert = Omit<JsonLdRow, 'id'>;
+/**
+ * One row in the `page_tags` table.
+ *
+ * Each row represents one detected Wappalyzer provider × external-id tuple for
+ * one page. A page typically has 1–10 rows.
+ */
+export interface TagRow {
+    /** Auto-increment primary key. */
+    id: number;
+    /** FK to `pages.id`. */
+    pageId: number;
+    /** Wappalyzer provider name (e.g. `'Google Tag Manager'`). */
+    provider: string;
+    /** First entry of `categories`. `null` when Wappalyzer did not report a category. Convenient projection only; canonical list is `categories`. */
+    category: string | null;
+    /** Real external identifier extracted by `meta/id-extractors` (e.g. `GTM-XXXX`, `G-XXXX`). `null` when none. */
+    externalId: string | null;
+    /** Wappalyzer-reported version, when available. */
+    version: string | null;
+    /** Wappalyzer-reported confidence 0–100, when available. */
+    confidence: number | null;
+    /** Full `categories` array preserved as JSON. */
+    categories: readonly string[];
+    /** `TagSource[]` preserved as JSON; describes where the provider was detected (script-src / inline / iframe-src / window-global / etc.). */
+    sources: ReadonlyArray<{
+        type: 'script-src' | 'inline' | 'iframe-src' | 'window-global' | 'img-src' | 'header' | 'meta' | 'html';
+        src?: string;
+        location?: 'head' | 'body' | 'noscript';
+        globalName?: string;
+    }>;
+}
+/**
+ * Insert shape for {@link TagRow}.
+ *
+ * Mirrors the row shape minus the auto-increment `id`. `categories` and
+ * `sources` are passed as plain JS arrays (the database layer JSON-stringifies
+ * them before write).
+ */
+export type TagRowForInsert = Omit<TagRow, 'id'>;
+/**
+ * Summary of one page's JSON-LD entries returned by `get-page-detail`.
+ *
+ * Keeps the response token-bounded for MCP / LLM consumers; the full `raw`
+ * payload is fetched separately via `get-page-jsonld(url)`.
+ * @see summarize-jsonld.ts
+ */
+export interface JsonLdSummary {
+    /** Total entries across `ld+json` and `speculationrules`. */
+    count: number;
+    /** Unique `@type` values (sorted). `null` slots are emitted as the string `'(unknown)'`. */
+    types: readonly string[];
+    /** Number of entries that failed to parse (i.e. have a non-null `parseError`). */
+    parseErrorCount: number;
+}
+/**
+ * Summary of one page's Wappalyzer tags returned by `get-page-detail`.
+ * @see summarize-tags.ts
+ */
+export interface TagsSummary {
+    /** Total tag rows for the page. */
+    count: number;
+    /** Provider → list of external IDs (unique, sorted). Providers with no IDs map to `[]`. */
+    providerIds: Readonly<Record<string, readonly string[]>>;
+}
+/**
+ * One row of {@link import('@d-zero/beholder').Meta.tags.detected} after
+ * archive-side flattening.
+ *
+ * Used by `get-tag-inventory` to return per-provider page counts across the
+ * whole site.
+ */
+export interface TagInventoryEntry {
+    /** Wappalyzer provider name. */
+    provider: string;
+    /** Number of distinct pages where the provider was detected. */
+    pageCount: number;
+}
+/**
+ * Error thrown by `assert-compatible-version` when the archive's
+ * `info.version` is older than the format version this build accepts.
+ *
+ * Catch this at CLI / viewer boundaries to print a friendly message; do not
+ * confuse with generic `Error` thrown by `Database.connect` (lockfile / I/O).
+ */
+export declare class IncompatibleArchiveError extends Error {
+    readonly archiveVersion: string;
+    readonly requiredVersion: string;
+    /**
+     * @param archiveVersion - The `info.version` value read from the archive
+     *   (or `'unknown'` when the column is missing / null).
+     * @param requiredVersion - The minimum format version this build accepts
+     *   (semver string, e.g. `'0.10.0'`).
+     */
+    constructor(archiveVersion: string, requiredVersion: string);
+}

package/lib/archive/meta/types.js

@@ -0,0 +1,33 @@
+/**
+ * Shared type definitions for archive-side meta processing helpers under {@link ./}.
+ *
+ * Mirrors the shape of {@link import('@d-zero/beholder').Meta} after archive-side
+ * derivation: flat columns persisted to the `pages` table, denormalised
+ * aggregates, per-row shapes for `page_jsonld` / `page_tags`, and the summary
+ * objects returned by `get-page-detail` consumers.
+ * @module
+ */
+/**
+ * Error thrown by `assert-compatible-version` when the archive's
+ * `info.version` is older than the format version this build accepts.
+ *
+ * Catch this at CLI / viewer boundaries to print a friendly message; do not
+ * confuse with generic `Error` thrown by `Database.connect` (lockfile / I/O).
+ */
+export class IncompatibleArchiveError extends Error {
+    archiveVersion;
+    requiredVersion;
+    /**
+     * @param archiveVersion - The `info.version` value read from the archive
+     *   (or `'unknown'` when the column is missing / null).
+     * @param requiredVersion - The minimum format version this build accepts
+     *   (semver string, e.g. `'0.10.0'`).
+     */
+    constructor(archiveVersion, requiredVersion) {
+        super(`Archive uses Nitpicker ${archiveVersion}; this build requires ${requiredVersion} or newer. ` +
+            `Run \`node scripts/migrate-to-0.10.mjs <path>\` to produce an upgraded copy next to it.`);
+        this.archiveVersion = archiveVersion;
+        this.requiredVersion = requiredVersion;
+        this.name = 'IncompatibleArchiveError';
+    }
+}

package/lib/archive/migrate-crawl-errors.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `crawl_errors` table to archives created before structured capture
+ * of the crawler-level `error` channel landed.
+ *
+ * `crawl_errors` records errors that previously only reached `error.log` (DNS
+ * failures, connection resets, TLS problems, process-level errors), in a
+ * queryable form so the `error-kinds` analysis can classify them without
+ * parsing the text log. Older `.nitpicker` files predate this table; this
+ * migration creates it idempotently. Existing archives stay empty until they
+ * are crawled again (resume / append / retry), at which point new errors are
+ * written here — analysis of pre-existing data falls back to parsing
+ * `error.log`.
+ *
+ * Idempotent: when the table already exists, the function exits without
+ * touching the schema or writing to stderr. It also skips empty archives (no
+ * `pages` table), which the regular initSchema path will fully provision.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateCrawlErrors(instance: Knex): Promise<void>;

package/lib/archive/migrate-crawl-errors.js

@@ -0,0 +1,38 @@
+/**
+ * Adds the `crawl_errors` table to archives created before structured capture
+ * of the crawler-level `error` channel landed.
+ *
+ * `crawl_errors` records errors that previously only reached `error.log` (DNS
+ * failures, connection resets, TLS problems, process-level errors), in a
+ * queryable form so the `error-kinds` analysis can classify them without
+ * parsing the text log. Older `.nitpicker` files predate this table; this
+ * migration creates it idempotently. Existing archives stay empty until they
+ * are crawled again (resume / append / retry), at which point new errors are
+ * written here — analysis of pre-existing data falls back to parsing
+ * `error.log`.
+ *
+ * Idempotent: when the table already exists, the function exits without
+ * touching the schema or writing to stderr. It also skips empty archives (no
+ * `pages` table), which the regular initSchema path will fully provision.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateCrawlErrors(instance) {
+    const hasTable = await instance.schema.hasTable('crawl_errors');
+    if (hasTable) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; the regular initSchema path will create the table.
+        return;
+    }
+    await instance.schema.createTable('crawl_errors', (t) => {
+        t.increments('id');
+        t.string('url', 8190).nullable();
+        t.boolean('isExternal');
+        t.text('message').notNullable();
+        t.integer('createdAt').notNullable();
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] crawl_errors table created');
+}

package/lib/archive/migrate-html-blob-tables.d.ts

@@ -0,0 +1,24 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the BLOB-backed HTML snapshot tables (`page_html_blobs` +
+ * `page_html_ref`) to archives created before #75.
+ *
+ * A pre-#75 `.nitpicker` already has the `info` table, so `initSchema`'s
+ * early-return path skips the freshly-introduced `CREATE TABLE`s. This
+ * migration brings the schema forward idempotently so the read API stops
+ * raising raw "no such table" errors against legacy archives. It does NOT
+ * touch the `pages.html` column or backfill the new tables from the
+ * archive's `snapshot-html.zip`: that data migration belongs to the
+ * `scripts/migrate-to-0.10.mjs` one-shot, which can run whenever the
+ * user is ready to commit the (multi-hour) CPU cost.
+ *
+ * Outcome on a legacy archive that has NOT yet been data-migrated:
+ *   - new tables exist but are empty,
+ *   - `getHtmlOfPageById` returns `null` for every page (no row in `page_html_ref`),
+ *   - viewer / MCP / analyze plugins see "snapshot unavailable" instead of crashing.
+ *
+ * The migration prints a one-line notice when it runs so the user knows
+ * the archive was upgraded in place.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateHtmlBlobTables(instance: Knex): Promise<void>;

package/lib/archive/migrate-html-blob-tables.js

@@ -0,0 +1,53 @@
+/**
+ * Adds the BLOB-backed HTML snapshot tables (`page_html_blobs` +
+ * `page_html_ref`) to archives created before #75.
+ *
+ * A pre-#75 `.nitpicker` already has the `info` table, so `initSchema`'s
+ * early-return path skips the freshly-introduced `CREATE TABLE`s. This
+ * migration brings the schema forward idempotently so the read API stops
+ * raising raw "no such table" errors against legacy archives. It does NOT
+ * touch the `pages.html` column or backfill the new tables from the
+ * archive's `snapshot-html.zip`: that data migration belongs to the
+ * `scripts/migrate-to-0.10.mjs` one-shot, which can run whenever the
+ * user is ready to commit the (multi-hour) CPU cost.
+ *
+ * Outcome on a legacy archive that has NOT yet been data-migrated:
+ *   - new tables exist but are empty,
+ *   - `getHtmlOfPageById` returns `null` for every page (no row in `page_html_ref`),
+ *   - viewer / MCP / analyze plugins see "snapshot unavailable" instead of crashing.
+ *
+ * The migration prints a one-line notice when it runs so the user knows
+ * the archive was upgraded in place.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateHtmlBlobTables(instance) {
+    const hasBlobs = await instance.schema.hasTable('page_html_blobs');
+    if (hasBlobs) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; initSchema will create both tables on the
+        // fresh-DB path. Nothing to migrate.
+        return;
+    }
+    await instance.raw(`
+		CREATE TABLE page_html_blobs (
+			hash         BLOB PRIMARY KEY,
+			body         BLOB NOT NULL,
+			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw     INTEGER NOT NULL,
+			size_stored  INTEGER NOT NULL
+		) WITHOUT ROWID
+	`);
+    await instance.raw(`
+		CREATE TABLE page_html_ref (
+			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
+			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+		) WITHOUT ROWID
+	`);
+    await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
+    // eslint-disable-next-line no-console
+    console.error('[migrate] page_html_blobs / page_html_ref tables created. ' +
+        'HTML snapshots are empty until `node scripts/migrate-to-0.10.mjs` is run on the original archive.');
+}

package/lib/archive/migrate-inventory-runs.d.ts

@@ -0,0 +1,29 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs` audit-log table to archives created before
+ * Phase 1 of inventory run tracking shipped.
+ *
+ * `inventory_runs` records one row per successful `--inventory <list>`
+ * invocation, capturing which deploy list was applied when and what
+ * scale it operated at. The motivation is operational: client /
+ * director conversations repeatedly ask "did you apply last month's
+ * list" / "we didn't double-apply, right" — the archive itself had no
+ * trace of inventory passes (`.bak` is unlinked on success), so this
+ * table is the durable provenance record.
+ *
+ * Schema details (column semantics, NULL policy, index) live in
+ * {@link initSchema} — this migration only re-creates the table shape
+ * on legacy archives so the rest of the codebase can treat the table
+ * as always-present once a writer connection has opened the file.
+ *
+ * Idempotent: when the table already exists, the function exits
+ * silently — the `[migrate] inventory_runs table created` stderr line
+ * fires **only** on the first run against a legacy archive, matching
+ * the established pattern of `migrate-page-errors.ts` /
+ * `migrate-crawl-errors.ts`. Operators can rely on the log line as a
+ * stable "first time this archive saw Phase 1 schema" event marker.
+ * Empty archives (no `pages` table) are skipped entirely; the regular
+ * `initSchema` path provisions them at first crawl.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRuns(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs.js

@@ -0,0 +1,52 @@
+/**
+ * Adds the `inventory_runs` audit-log table to archives created before
+ * Phase 1 of inventory run tracking shipped.
+ *
+ * `inventory_runs` records one row per successful `--inventory <list>`
+ * invocation, capturing which deploy list was applied when and what
+ * scale it operated at. The motivation is operational: client /
+ * director conversations repeatedly ask "did you apply last month's
+ * list" / "we didn't double-apply, right" — the archive itself had no
+ * trace of inventory passes (`.bak` is unlinked on success), so this
+ * table is the durable provenance record.
+ *
+ * Schema details (column semantics, NULL policy, index) live in
+ * {@link initSchema} — this migration only re-creates the table shape
+ * on legacy archives so the rest of the codebase can treat the table
+ * as always-present once a writer connection has opened the file.
+ *
+ * Idempotent: when the table already exists, the function exits
+ * silently — the `[migrate] inventory_runs table created` stderr line
+ * fires **only** on the first run against a legacy archive, matching
+ * the established pattern of `migrate-page-errors.ts` /
+ * `migrate-crawl-errors.ts`. Operators can rely on the log line as a
+ * stable "first time this archive saw Phase 1 schema" event marker.
+ * Empty archives (no `pages` table) are skipped entirely; the regular
+ * `initSchema` path provisions them at first crawl.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRuns(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (hasTable) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; the regular initSchema path will create the table.
+        return;
+    }
+    await instance.schema.createTable('inventory_runs', (t) => {
+        t.increments('id');
+        t.string('ran_at').notNullable();
+        t.string('list_label').nullable();
+        t.string('source_file_sha256', 64).nullable();
+        t.integer('total_lines').nullable();
+        t.integer('new_pages').nullable();
+        t.integer('new_resources').nullable();
+        t.integer('scope_skipped').nullable();
+        t.text('notes').nullable();
+        t.index('ran_at');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs table created');
+}

package/lib/archive/migrate-page-errors.d.ts

@@ -0,0 +1,16 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_errors` table to archives created before partial-failure
+ * recording landed.
+ *
+ * `page_errors` captures secondary scrape failures (e.g. a viewport switch
+ * that detaches the frame and surfaces `retryExhausted`) so they show up in
+ * the archive instead of being lost to stdout logs. Older `.nitpicker` files
+ * predate this table; this migration creates it idempotently.
+ *
+ * Idempotent: when the table already exists, the function exits without
+ * touching the schema or writing to stderr. When it has to run, a single
+ * notice is written so the user knows the file was upgraded.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageErrors(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-errors.js

@@ -0,0 +1,35 @@
+/**
+ * Adds the `page_errors` table to archives created before partial-failure
+ * recording landed.
+ *
+ * `page_errors` captures secondary scrape failures (e.g. a viewport switch
+ * that detaches the frame and surfaces `retryExhausted`) so they show up in
+ * the archive instead of being lost to stdout logs. Older `.nitpicker` files
+ * predate this table; this migration creates it idempotently.
+ *
+ * Idempotent: when the table already exists, the function exits without
+ * touching the schema or writing to stderr. When it has to run, a single
+ * notice is written so the user knows the file was upgraded.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageErrors(instance) {
+    const hasTable = await instance.schema.hasTable('page_errors');
+    if (hasTable) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; the regular initSchema path will create both tables.
+        return;
+    }
+    await instance.schema.createTable('page_errors', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable().unsigned().references('pages.id');
+        t.string('phase').notNullable();
+        t.text('message').notNullable();
+        t.integer('createdAt').notNullable();
+        t.index('pageId');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] page_errors table created');
+}

package/lib/archive/migrate-pages-resources-source.d.ts

@@ -0,0 +1,16 @@
+import type { Knex } from 'knex';
+/**
+ * Add `pages.source` and `resources.source` columns (provenance taxonomy:
+ * `crawled` / `inventory-seed` / `inventory-discovered`) and their indexes
+ * to archives that pre-date the `crawl --inventory` feature.
+ *
+ * Idempotent: a no-op when both columns already exist. SQLite's
+ * `ALTER TABLE ADD COLUMN` with a NOT NULL DEFAULT applies the default to
+ * every existing row at column-add time, so no explicit `UPDATE` is needed
+ * to backfill — pre-existing rows become `'crawled'` automatically.
+ *
+ * Runs only on writer-side {@link Database.connect}; read-only viewer
+ * attaches skip this so the user's tmpDir is never rewritten.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePagesResourcesSource(instance: Knex): Promise<void>;

package/lib/archive/migrate-pages-resources-source.js

@@ -0,0 +1,46 @@
+/**
+ * Add `pages.source` and `resources.source` columns (provenance taxonomy:
+ * `crawled` / `inventory-seed` / `inventory-discovered`) and their indexes
+ * to archives that pre-date the `crawl --inventory` feature.
+ *
+ * Idempotent: a no-op when both columns already exist. SQLite's
+ * `ALTER TABLE ADD COLUMN` with a NOT NULL DEFAULT applies the default to
+ * every existing row at column-add time, so no explicit `UPDATE` is needed
+ * to backfill — pre-existing rows become `'crawled'` automatically.
+ *
+ * Runs only on writer-side {@link Database.connect}; read-only viewer
+ * attaches skip this so the user's tmpDir is never rewritten.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePagesResourcesSource(instance) {
+    const hasPages = await instance.schema.hasTable('pages');
+    const hasResources = await instance.schema.hasTable('resources');
+    if (!hasPages && !hasResources) {
+        return;
+    }
+    const hasPagesSource = hasPages && (await instance.schema.hasColumn('pages', 'source'));
+    const hasResourcesSource = hasResources && (await instance.schema.hasColumn('resources', 'source'));
+    if (hasPagesSource && hasResourcesSource) {
+        return;
+    }
+    const changes = [];
+    if (hasPages && !hasPagesSource) {
+        await instance.schema.table('pages', (t) => {
+            t.string('source').notNullable().defaultTo('crawled');
+            t.index('source');
+        });
+        changes.push('pages.source added');
+    }
+    if (hasResources && !hasResourcesSource) {
+        await instance.schema.table('resources', (t) => {
+            t.string('source').notNullable().defaultTo('crawled');
+            t.index('source');
+        });
+        changes.push('resources.source added');
+    }
+    if (changes.length === 0) {
+        return;
+    }
+    // eslint-disable-next-line no-console
+    console.error(`[migrate] ${changes.join(', ')}`);
+}