@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/archive/db-ops/pages/write/write-page-html-blob.js

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto';
-import { zstdCompressSync } from 'node:zlib';
+import { compressPayload } from '../../_shared/compress-payload.js';
 /**
  * Encodes, dedups, and persists a page's HTML snapshot.
  *
@@ -20,14 +20,14 @@ import { zstdCompressSync } from 'node:zlib';
 export async function writePageHtmlBlob(pageId, html, trx) {
     const rawBytes = Buffer.from(html, 'utf8');
     const hash = createHash('sha256').update(rawBytes).digest();
-    const compressed = zstdCompressSync(rawBytes);
+    const { body, codec, sizeRaw, sizeStored } = compressPayload(rawBytes);
     await trx('page_html_blobs')
         .insert({
         hash,
-        body: compressed,
-        codec: 'zstd',
-        size_raw: rawBytes.byteLength,
-        size_stored: compressed.byteLength,
+        body,
+        codec,
+        size_raw: sizeRaw,
+        size_stored: sizeStored,
     })
         .onConflict('hash')
         .ignore();

package/lib/archive/filesystem/output-binary.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Writes raw bytes to a file at the specified path, creating parent
+ * directories if needed.
+ *
+ * Unlike {@link outputText}, the buffer is written verbatim with no UTF-8
+ * re-encoding, so callers that need byte-for-byte fidelity (e.g. archiving
+ * a source file of unknown or mixed encoding for audit purposes) are not
+ * exposed to lossy round-tripping through a JS string.
+ * @param filePath - The absolute or relative path to the file to write.
+ * @param data - The raw bytes to write.
+ */
+export declare function outputBinary(filePath: string, data: Buffer): Promise<void>;

package/lib/archive/filesystem/output-binary.js

@@ -0,0 +1,17 @@
+import { promises as fs } from 'node:fs';
+import { mkdir } from './mkdir.js';
+/**
+ * Writes raw bytes to a file at the specified path, creating parent
+ * directories if needed.
+ *
+ * Unlike {@link outputText}, the buffer is written verbatim with no UTF-8
+ * re-encoding, so callers that need byte-for-byte fidelity (e.g. archiving
+ * a source file of unknown or mixed encoding for audit purposes) are not
+ * exposed to lossy round-tripping through a JS string.
+ * @param filePath - The absolute or relative path to the file to write.
+ * @param data - The raw bytes to write.
+ */
+export async function outputBinary(filePath, data) {
+    mkdir(filePath);
+    await fs.writeFile(filePath, data);
+}

package/lib/archive/get-failed-page-messages.d.ts

@@ -1,10 +1,22 @@
 import type { Knex } from 'knex';
 /**
- * Bulk-resolve a raw error message for each given page id, using only sources
- * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
- * `pageId`, the most direct signal a scrape attempt recorded), then
- * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
- * that happened before a page row was scraped).
+ * One resolved failure message, with the timestamp it was recorded at.
+ * `createdAt` is what lets a caller (`resetFailedPages`) decide whether the
+ * failure falls inside a `network_outages` window and should be treated as
+ * retryable regardless of its classified `ErrorKind` — see
+ * `is-within-outage-window.ts`.
+ */
+export interface FailedPageMessage {
+    message: string;
+    /** Epoch ms the message was recorded (`page_errors.createdAt` or `crawl_errors.createdAt`). */
+    createdAt: number;
+}
+/**
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
  *
  * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
  * read `error.log`. The `crawl_errors` table is created empty (by
@@ -30,8 +42,8 @@ import type { Knex } from 'knex';
  * @param urls - The corresponding `pages.url` values, in the same order as
  *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
  *   reconstructed without a second `pages` round-trip.
- * @returns `Map<pageId, message>` populated only for ids whose message was
- *   found in one of the consulted tables.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
  * @example
  * ```ts
  * const messages = await getFailedPageMessages(
@@ -41,4 +53,4 @@ import type { Knex } from 'knex';
  * );
  * ```
  */
-export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, string>>;
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, FailedPageMessage>>;

package/lib/archive/get-failed-page-messages.js

@@ -1,11 +1,11 @@
 /** chunk size for `WHERE … IN (?)` SQLite parameter packing. */
 const CHUNK_SIZE = 500;
 /**
- * Bulk-resolve a raw error message for each given page id, using only sources
- * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
- * `pageId`, the most direct signal a scrape attempt recorded), then
- * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
- * that happened before a page row was scraped).
+ * Bulk-resolve a raw error message (plus its timestamp) for each given page
+ * id, using only sources reachable from a {@link Knex} handle. Read order:
+ * `page_errors` (keyed by `pageId`, the most direct signal a scrape attempt
+ * recorded), then `crawl_errors` (keyed by `url`, the crawler-channel
+ * record for failures that happened before a page row was scraped).
  *
  * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
  * read `error.log`. The `crawl_errors` table is created empty (by
@@ -31,8 +31,8 @@ const CHUNK_SIZE = 500;
  * @param urls - The corresponding `pages.url` values, in the same order as
  *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
  *   reconstructed without a second `pages` round-trip.
- * @returns `Map<pageId, message>` populated only for ids whose message was
- *   found in one of the consulted tables.
+ * @returns `Map<pageId, FailedPageMessage>` populated only for ids whose
+ *   message was found in one of the consulted tables.
  * @example
  * ```ts
  * const messages = await getFailedPageMessages(
@@ -61,7 +61,7 @@ export async function getFailedPageMessages(instance, ids, urls) {
             // it as `parse-error` on the next run when the rows happen to be
             // returned in a different order.
             const rows = (await instance('page_errors')
-                .select('pageId', 'message')
+                .select('pageId', 'message', 'createdAt')
                 .whereIn('pageId', chunk)
                 .orderBy('id', 'asc'));
             for (const row of rows) {
@@ -79,7 +79,10 @@ export async function getFailedPageMessages(instance, ids, urls) {
                 // `dns` / `tls` / `client-blocked` etc. — defeating
                 // `--retry-failed`'s permanent-kind exclusion.
                 if (row.message !== '' && !messageByPageId.has(row.pageId)) {
-                    messageByPageId.set(row.pageId, row.message);
+                    messageByPageId.set(row.pageId, {
+                        message: row.message,
+                        createdAt: row.createdAt,
+                    });
                 }
             }
         }
@@ -110,11 +113,22 @@ export async function getFailedPageMessages(instance, ids, urls) {
     for (let i = 0; i < missingUrls.length; i += CHUNK_SIZE) {
         const chunk = missingUrls.slice(i, i + CHUNK_SIZE);
         const rows = (await instance('crawl_errors')
-            .select('url', 'message')
+            .select('url', 'message', 'createdAt')
             .whereIn('url', chunk));
         for (const row of rows) {
-            if (row.url !== null && !urlToMessage.has(row.url)) {
-                urlToMessage.set(row.url, row.message);
+            if (row.url === null) {
+                continue;
+            }
+            // Latest-createdAt wins (fixes a previously-undefined
+            // selection among duplicate URLs — SQLite's natural scan
+            // order is implementation-defined). The most recent message
+            // is the most relevant one for both classification and outage
+            // attribution: an old NXDOMAIN followed by a network-outage
+            // blip should resolve to the outage-era message, not whichever
+            // happened to be inserted first.
+            const existing = urlToMessage.get(row.url);
+            if (existing === undefined || row.createdAt > existing.createdAt) {
+                urlToMessage.set(row.url, { message: row.message, createdAt: row.createdAt });
             }
         }
     }
@@ -123,9 +137,9 @@ export async function getFailedPageMessages(instance, ids, urls) {
         if (url === undefined) {
             continue;
         }
-        const message = urlToMessage.get(url);
-        if (message !== undefined) {
-            messageByPageId.set(id, message);
+        const resolved = urlToMessage.get(url);
+        if (resolved !== undefined) {
+            messageByPageId.set(id, resolved);
         }
     }
     return messageByPageId;

package/lib/archive/migrate-content-items-alias-of-id.d.ts

@@ -0,0 +1,47 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `content_items.alias_of_id` column to archives created before
+ * this feature, then ensures its index exists.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit — the same
+ * self-referencing shape as `redirect_dest_id` (see
+ * `create-entity-tables.ts`'s JSDoc on why that FK must be deferred: a
+ * lower-id row can reference a higher-id row within the same write).
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `alias_of_id` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `alias_of_id` at that point; an unconditional
+ * `CREATE INDEX ... alias_of_id` there would fail with `no such column`
+ * before this migration ever runs. This function is the one place
+ * guaranteed to run only after the column is confirmed present, for both
+ * archive kinds.
+ *
+ * Only adds the column and its index — it does not compute values for
+ * existing rows (they stay `NULL`). That computation runs separately, from
+ * `backfillAliasOfId` during a viewer-read-model build, since it requires
+ * comparing every page's title and (for the trailing-slash tier) its
+ * `body_hash` against every other page.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `content_items`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateContentItemsAliasOfId(instance: Knex): Promise<void>;

package/lib/archive/migrate-content-items-alias-of-id.js

@@ -0,0 +1,58 @@
+/**
+ * Adds the `content_items.alias_of_id` column to archives created before
+ * this feature, then ensures its index exists.
+ *
+ * `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `content_items` predates this change. This
+ * mirrors {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * Uses a raw `ALTER TABLE` (not the knex schema builder) so the retrofitted
+ * column's `REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED`
+ * constraint matches the fresh-archive DDL bit-for-bit — the same
+ * self-referencing shape as `redirect_dest_id` (see
+ * `create-entity-tables.ts`'s JSDoc on why that FK must be deferred: a
+ * lower-id row can reference a higher-id row within the same write).
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `alias_of_id` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `alias_of_id` at that point; an unconditional
+ * `CREATE INDEX ... alias_of_id` there would fail with `no such column`
+ * before this migration ever runs. This function is the one place
+ * guaranteed to run only after the column is confirmed present, for both
+ * archive kinds.
+ *
+ * Only adds the column and its index — it does not compute values for
+ * existing rows (they stay `NULL`). That computation runs separately, from
+ * `backfillAliasOfId` during a viewer-read-model build, since it requires
+ * comparing every page's title and (for the trailing-slash tier) its
+ * `body_hash` against every other page.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `content_items`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateContentItemsAliasOfId(instance) {
+    const hasContentItems = await instance.schema.hasTable('content_items');
+    if (!hasContentItems) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('content_items', 'alias_of_id');
+    if (!hasColumn) {
+        await instance.raw('ALTER TABLE content_items ADD COLUMN alias_of_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED');
+        // eslint-disable-next-line no-console
+        console.error('[migrate] content_items.alias_of_id column added');
+    }
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_alias_of_id ON content_items(alias_of_id)');
+}

package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `inventory_runs.invalid_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInventoryRunsInvalidSkipped(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs-invalid-skipped.js

@@ -0,0 +1,24 @@
+/**
+ * Adds the `inventory_runs.invalid_skipped` column to archives created
+ * before it existed. `CREATE TABLE IF NOT EXISTS` (used for `inventory_runs`
+ * itself) cannot retrofit a new column onto an already-existing table, so
+ * this lightweight, `hasColumn`-guarded `ALTER TABLE` runs on every
+ * `initSchema` call — idempotent, and self-healing for archives whose
+ * provisioning crashed partway through.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInventoryRunsInvalidSkipped(instance) {
+    const hasTable = await instance.schema.hasTable('inventory_runs');
+    if (!hasTable) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('inventory_runs', 'invalid_skipped');
+    if (hasColumn) {
+        return;
+    }
+    await instance.schema.table('inventory_runs', (t) => {
+        t.integer('invalid_skipped');
+    });
+    // eslint-disable-next-line no-console
+    console.error('[migrate] inventory_runs.invalid_skipped column added');
+}

package/lib/archive/migrate-page-meta-body-hash.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.body_hash` column to archives created before this
+ * feature, then ensures its index exists.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `body_hash` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `body_hash` at that point; an unconditional
+ * `CREATE INDEX ... body_hash` there would fail with `no such column` before
+ * this migration ever runs. This function is the one place guaranteed to
+ * run only after the column is confirmed present, for both archive kinds.
+ *
+ * Only adds the column and its index — it does not backfill values for
+ * existing rows (they stay `NULL`). That backfill runs separately, from
+ * `backfillBodyHashFromHtmlBlobs` during a viewer-read-model build, since it
+ * requires decompressing every page's stored HTML and is too heavy to run on
+ * every archive open.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `page_meta`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageMetaBodyHash(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-meta-body-hash.js

@@ -0,0 +1,52 @@
+/**
+ * Adds the `page_meta.body_hash` column to archives created before this
+ * feature, then ensures its index exists.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. This mirrors
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}'s
+ * catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
+ * `CREATE TABLE IF NOT EXISTS` cannot retrofit.
+ *
+ * The index is created here — unconditionally, after the column-add guard,
+ * not inside `createEntityTables`'s DDL — for both a fresh archive (where
+ * `body_hash` already exists from the DDL, so only the index still needs
+ * creating) and a legacy archive (where the column is added just above,
+ * then the index follows in the same call). `createEntityTables` runs
+ * unconditionally on every archive open, including legacy archives that
+ * still lack `body_hash` at that point; an unconditional
+ * `CREATE INDEX ... body_hash` there would fail with `no such column` before
+ * this migration ever runs. This function is the one place guaranteed to
+ * run only after the column is confirmed present, for both archive kinds.
+ *
+ * Only adds the column and its index — it does not backfill values for
+ * existing rows (they stay `NULL`). That backfill runs separately, from
+ * `backfillBodyHashFromHtmlBlobs` during a viewer-read-model build, since it
+ * requires decompressing every page's stored HTML and is too heavy to run on
+ * every archive open.
+ *
+ * Idempotent: adding the column is a no-op once it exists (the index
+ * creation always runs, but `IF NOT EXISTS` makes repeat runs a no-op too).
+ * Guards on `page_meta`'s existence defensively, though by the time this
+ * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
+ * pre-0.13 archives) the table is always present.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageMetaBodyHash(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'body_hash');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.binary('body_hash');
+        });
+        // eslint-disable-next-line no-console
+        console.error('[migrate] page_meta.body_hash column added');
+    }
+    await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_body_hash ON page_meta(body_hash)');
+}

package/lib/archive/migrate-page-meta-console-error-count.d.ts

@@ -0,0 +1,22 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.console_error_count` column to archives created
+ * before this feature (issue #228).
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same
+ * catch-up shape as {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}.
+ *
+ * Unlike `body_hash`, there is no separate backfill step: a page that has
+ * never been re-scraped since this feature shipped has no
+ * `page_console_logs` rows either, so `0` (the value `replaceConsoleLogs`
+ * writes for a page with no error/pageerror entries) is already the
+ * correct value, not a placeholder pending backfill.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migratePageMetaConsoleErrorCount(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-meta-console-error-count.js

@@ -0,0 +1,34 @@
+/**
+ * Adds the `page_meta.console_error_count` column to archives created
+ * before this feature (issue #228).
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same
+ * catch-up shape as {@link import('./migrate-page-meta-body-hash.js').migratePageMetaBodyHash}.
+ *
+ * Unlike `body_hash`, there is no separate backfill step: a page that has
+ * never been re-scraped since this feature shipped has no
+ * `page_console_logs` rows either, so `0` (the value `replaceConsoleLogs`
+ * writes for a page with no error/pageerror entries) is already the
+ * correct value, not a placeholder pending backfill.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migratePageMetaConsoleErrorCount(instance) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'console_error_count');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.integer('console_error_count');
+        });
+        // eslint-disable-next-line no-console
+        console.error('[migrate] page_meta.console_error_count column added');
+    }
+}

package/lib/archive/page.d.ts

@@ -406,6 +406,7 @@ export default class Page {
         isInternalPage: boolean;
         getAnchors: Anchor[];
         getReferrers: Referrer[];
+        dir: string | number | null;
         title: string | number | null;
         main_content_word_count: string | number | null;
         main_content_node_name: string | number | null;
@@ -425,7 +426,6 @@ export default class Page {
         scroll_height_desktop: string | number | null;
         scroll_height_mobile: string | number | null;
         lang: string | number | null;
-        dir: string | number | null;
         charset: string | number | null;
         baseHref: string | number | null;
         viewport_raw: string | number | null;

package/lib/archive/types.d.ts

@@ -128,9 +128,9 @@ export interface InventoryRunMeta {
     ran_at: string;
     /** Human-readable identifier (e.g. `'prod-2026-06-21'`). `null` when the caller did not supply one. */
     list_label?: string | null;
-    /** SHA-256 hex digest of the source file. `null` if hashing failed (e.g. file vanished mid-run). */
+    /** SHA-256 hex digest of the source file. `null` for programmatic callers that built the URL list in-memory (no source file to hash). */
     source_file_sha256?: string | null;
-    /** Number of non-empty lines in the input list (= URL count before scope filtering). */
+    /** Number of valid URLs in the input list, after the CLI has warned-and-dropped unparseable-URL lines but before scope filtering. */
     total_lines?: number | null;
     /** Number of new HTML seeds inserted as `content_items` rows by this run. */
     new_pages?: number | null;
@@ -138,9 +138,54 @@ export interface InventoryRunMeta {
     new_resources?: number | null;
     /** Number of input URLs dropped because they fell outside the archived scope. */
     scope_skipped?: number | null;
+    /** Number of source-file lines dropped by the CLI for failing URL validation, before this row's `total_lines` was counted. `null` for programmatic callers that built the URL list in-memory (no source file to have invalid lines). */
+    invalid_skipped?: number | null;
     /** Free-form text for backfill annotations or operator notes. */
     notes?: string | null;
 }
+/**
+ * A row in `network_outages` — one detected operator-network outage.
+ *
+ * Append-only except `ended_at`: it is written once, `NULL`, when the
+ * outage is first detected, and updated exactly once when a recovery probe
+ * succeeds. A row can also be left `ended_at = NULL` forever if the crawl
+ * process is killed mid-outage — readers must resolve this via a clamp
+ * (see `is-within-outage-window.ts` and the `db-ops/outages/` writer that
+ * closes stale-open rows on the next writer session), never by treating
+ * `NULL` as an unbounded window.
+ */
+export interface NetworkOutageRow {
+    id: number;
+    /** Epoch ms, backdated to the earliest error still inside the detector's sliding window at trigger time. */
+    started_at: number;
+    /** Epoch ms the sliding window actually crossed both thresholds. */
+    detected_at: number;
+    /** Epoch ms the recovery probe first succeeded, or `null` while still open / if the session crashed before recovery. */
+    ended_at: number | null;
+    /** Hostname the recovery probe targeted, or `null` if none was available (see `choose-probe-host.ts`). */
+    probe_host: string | null;
+    /** Error count in the detector's window at trigger time. */
+    trigger_error_count: number;
+    /** Distinct host count in the detector's window at trigger time. */
+    trigger_host_count: number;
+}
+/**
+ * Fields required to record a newly-detected outage via
+ * `Database.insertNetworkOutage`. camelCase (unlike {@link NetworkOutageRow}
+ * / {@link InventoryRunMeta}) because callers build this directly from
+ * `NetworkOutageDetector`'s camelCase `OutageSuspect` plus a probe host —
+ * the db-op does the camelCase → snake_case column mapping on write.
+ */
+export interface InsertNetworkOutageParams {
+    /** Backdated to the earliest error still inside the detector's window at trigger time. */
+    startedAt: number;
+    /** When the sliding window actually crossed both thresholds. */
+    detectedAt: number;
+    /** Hostname the recovery probe will target, or `null` if none was available. */
+    probeHost: string | null;
+    triggerErrorCount: number;
+    triggerHostCount: number;
+}
 /**
  * Filter type for querying pages from the database.
  *

package/lib/archive/url-alias/compute-tier-a-alias-key.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive), unlike
+ * `compareUrlSortKeys`'s pairwise "roughly equal" comparison (see
+ * ARCHITECTURE.md's "URL natural-sort comparator は推移律を保証しない" —
+ * that non-transitivity problem is inherent to pairwise comparators, and
+ * does not apply to a canonical-key partition like this one).
+ *
+ * Deliberately does not touch the query string or fragment: a query-string
+ * difference is not addressed by this feature (out of scope — see the
+ * `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates), and fragments are
+ * client-side only and never affect server-rendered content.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export declare function computeTierAAliasKey(url: string): string | null;

package/lib/archive/url-alias/compute-tier-a-alias-key.js

@@ -0,0 +1,36 @@
+import { formatAliasKey } from './format-alias-key.js';
+import { parseAliasKeyParts } from './parse-alias-key-parts.js';
+/**
+ * Computes the Tier A alias key for `url`: two URLs share a Tier A key when
+ * they differ only by scheme (`http` vs `https` are folded together),
+ * host letter-casing, or a `/index.{ext}` path suffix — differences that
+ * are safe to treat as "the same resource" from the URL string alone, with
+ * no need to compare rendered content.
+ *
+ * This is a canonical-key function, not a pairwise comparator: grouping
+ * rows by exact string equality of this function's output is definitionally
+ * an equivalence relation (reflexive, symmetric, transitive), unlike
+ * `compareUrlSortKeys`'s pairwise "roughly equal" comparison (see
+ * ARCHITECTURE.md's "URL natural-sort comparator は推移律を保証しない" —
+ * that non-transitivity problem is inherent to pairwise comparators, and
+ * does not apply to a canonical-key partition like this one).
+ *
+ * Deliberately does not touch the query string or fragment: a query-string
+ * difference is not addressed by this feature (out of scope — see the
+ * `body_hash` masking, a separate content-hash-based mechanism, for
+ * detecting meaningless-parameter duplicates), and fragments are
+ * client-side only and never affect server-rendered content.
+ * @param url - The URL string to compute a key for.
+ * @returns The Tier A key, or `null` if `url` is not a parseable http(s) URL.
+ * @example
+ * ```ts
+ * computeTierAAliasKey('http://Example.com/about/index.html');
+ * // 'example.com/about/'
+ * computeTierAAliasKey('https://example.com/about/');
+ * // 'example.com/about/' -- same key: scheme and /index.html are folded
+ * ```
+ */
+export function computeTierAAliasKey(url) {
+    const parts = parseAliasKeyParts(url);
+    return parts ? formatAliasKey(parts) : null;
+}