@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/populate-ref-tables/volatile-header-names.js

@@ -0,0 +1,33 @@
+/**
+ * Header names — always lower-cased — that are EXCLUDED from `stable_hash`.
+ * These change on every response (timestamps / request IDs / cache
+ * telemetry) and would defeat the dedup goal if hashed alongside the
+ * stable set.
+ *
+ * `set-cookie` sits here because per-session cookies rotate constantly on
+ * authenticated sites — it is deliberately classified as volatile even
+ * though it is technically a stable security signal.
+ *
+ * Any header not in this set is
+ * treated as **stable** by {@link ./header-stability.ts:isVolatileHeader}
+ * — the safer default because a genuinely volatile header misclassified
+ * as stable only weakens dedup (its per-response values fork
+ * otherwise-identical stable profiles into separate `stable_hash`
+ * clusters), while a stable header misclassified as volatile would drop
+ * a meaningful signal from `stable_hash` and falsely cluster distinct
+ * header configurations together.
+ */
+export const VOLATILE_HEADER_NAMES = new Set([
+    'date',
+    'expires',
+    'last-modified',
+    'etag',
+    'age',
+    'via',
+    'x-cache',
+    'cf-ray',
+    'x-request-id',
+    'set-cookie',
+    'server-timing',
+    'x-amz-request-id',
+]);

package/lib/archive/redirect-table.d.ts

@@ -1,7 +1,9 @@
 import type { Knex } from 'knex';
 /**
- * Returns a Knex subquery builder that joins pages with their redirect destinations.
- * When `includeNull` is true, also includes pages without redirects (self-referencing).
+ * Returns a Knex subquery builder that joins `content_items` with their
+ * redirect destinations (resolved via `url_refs` for both sides). When
+ * `includeNull` is true, also includes pages without redirects
+ * (self-referencing).
  * @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
  */
 export declare function redirectTable(includeNull?: boolean): (qb: Knex.QueryBuilder<Record<string, unknown>, unknown>) => Promise<void>;

package/lib/archive/redirect-table.js

@@ -1,22 +1,27 @@
 /**
- * Returns a Knex subquery builder that joins pages with their redirect destinations.
- * When `includeNull` is true, also includes pages without redirects (self-referencing).
+ * Returns a Knex subquery builder that joins `content_items` with their
+ * redirect destinations (resolved via `url_refs` for both sides). When
+ * `includeNull` is true, also includes pages without redirects
+ * (self-referencing).
  * @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
  */
 export function redirectTable(includeNull = true) {
     return async (qb) => {
         const list = qb
-            .select('A.id as fromId', 'A.url as from', 'B.url as to', 'B.id as toId')
-            .from('pages as A')
-            .join('pages as B', (j) => {
-            j.on('A.redirectDestId', '=', 'B.id').andOnNotNull('A.redirectDestId');
-        });
+            .select('A.id as fromId', 'A_url.url as from', 'B_url.url as to', 'B.id as toId')
+            .from('content_items as A')
+            .join('url_refs as A_url', 'A.url_id', 'A_url.id')
+            .join('content_items as B', (j) => {
+            j.on('A.redirect_dest_id', '=', 'B.id').andOnNotNull('A.redirect_dest_id');
+        })
+            .join('url_refs as B_url', 'B.url_id', 'B_url.id');
         if (includeNull) {
             await list.union(async (qb) => {
                 await qb
-                    .select('A.id as fromId', 'A.url as from', 'A.url as to', 'A.id as toId')
-                    .from('pages as A')
-                    .whereNull('A.redirectDestId');
+                    .select('A.id as fromId', 'A_url.url as from', 'A_url.url as to', 'A.id as toId')
+                    .from('content_items as A')
+                    .join('url_refs as A_url', 'A.url_id', 'A_url.id')
+                    .whereNull('A.redirect_dest_id');
             });
         }
     };

package/lib/archive/resolve-redirect-chain.d.ts

@@ -5,7 +5,7 @@ export interface RedirectChain {
     /** The final destination URL the chain lands on (normalised). */
     destUrl: string;
     /**
-     * The source URLs that redirect to {@link destUrl}: the originally requested
+     * The source URLs that redirect to {@link RedirectChain.destUrl}: the originally requested
      * URL followed by every intermediate hop. Empty when the page was not
      * redirected at all.
      */
@@ -22,8 +22,8 @@ export interface RedirectChain {
  * every intermediate hop are the sources. When the page was not redirected,
  * the destination is the page URL itself and there are no sources.
  *
- * Used by both {@link Database.updatePage} (which renders and stores the
- * destination) and {@link Database.recordRedirect} (the #73 convergence
+ * Used by both `Database.updatePage` (which renders and stores the
+ * destination) and `Database.recordRedirect` (the #73 convergence
  * optimisation, which records the edge for a destination already rendered
  * elsewhere without touching its content).
  * @param pageUrl - The originally requested URL (normalised, without hash/auth).

package/lib/archive/resolve-redirect-chain.js

@@ -9,8 +9,8 @@
  * every intermediate hop are the sources. When the page was not redirected,
  * the destination is the page URL itself and there are no sources.
  *
- * Used by both {@link Database.updatePage} (which renders and stores the
- * destination) and {@link Database.recordRedirect} (the #73 convergence
+ * Used by both `Database.updatePage` (which renders and stores the
+ * destination) and `Database.recordRedirect` (the #73 convergence
  * optimisation, which records the edge for a destination already rendered
  * elsewhere without touching its content).
  * @param pageUrl - The originally requested URL (normalised, without hash/auth).

package/lib/archive/resource.d.ts

@@ -31,7 +31,7 @@ export default class Resource {
     /**
      * The URL of the resource.
      */
-    get url(): string;
+    get url(): string | null;
     /**
      * Creates a new Resource instance.
      * @param archive - The ArchiveAccessor used for querying referrer data.

package/lib/archive/retarget-legacy-fk-tables.d.ts

@@ -0,0 +1,47 @@
+import type { Knex } from 'knex';
+/**
+ * Rewrites the FK declarations of the adjunct tables from the legacy
+ * `pages(id)` to `content_items(id)` by rebuilding each table. SQLite has
+ * no `ALTER TABLE … DROP CONSTRAINT`, so the only way to change an FK
+ * target is to recreate the table:
+ *
+ * 1. Stage each table's rows into a constraint-free `<table>__retarget`
+ *    copy (`CREATE TABLE … AS SELECT *`) and drop the original, freeing
+ *    its index names.
+ * 2. Recreate every table through {@link createAdjunctTables} — the same
+ *    DDL fresh archives get, so migrated archives end up identical in
+ *    shape and index naming. Rebuilding the DDL inline here instead
+ *    would recreate the exact drift this function repairs.
+ * 3. Copy the staged rows back using the recreated table's own column
+ *    list (read via `pragma_table_info`). A column the canonical DDL has
+ *    but the input archive lacks aborts the INSERT with a clear error —
+ *    never a silent data drop.
+ *
+ * A rename-based recipe (`ALTER TABLE x RENAME TO x__retarget`, skip
+ * copy #1) is deliberately NOT used: SQLite keeps the renamed table's
+ * indexes attached under their original names, which collide with the
+ * index names {@link createAdjunctTables} declares — and the old index
+ * names vary by the era of the archive's original provisioning, so
+ * dropping them first would require enumerating unknown names. The
+ * copy-out is cheap at these tables' scale (a handful of rows per page).
+ *
+ * Adjunct tables that do not exist on the input archive (e.g. a 0.10
+ * archive that never ran `analyze`, so `analysis_violations` was never
+ * lazily provisioned) are simply created empty by step 2.
+ *
+ * The migrated-archive PK-preservation contract makes the copy safe:
+ * `content_items.id` reuses the legacy `pages.id` values verbatim, so
+ * every staged `pageId` / `page_id` resolves to the same row under the
+ * new FK target. Run inside a transaction with `PRAGMA foreign_keys = ON`
+ * so the copy-back doubles as a row-level FK validation — a stale id
+ * aborts the transaction here rather than surfacing later as a
+ * `foreign_key_check` finding.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @example
+ * await db.transaction(async (trx) => {
+ *   await retargetLegacyFkTables(trx);
+ * });
+ * // pragma_foreign_key_list('page_errors') now reports content_items.
+ */
+export declare function retargetLegacyFkTables(trx: Knex): Promise<void>;

package/lib/archive/retarget-legacy-fk-tables.js

@@ -0,0 +1,107 @@
+import { createAdjunctTables } from './create-adjunct-tables.js';
+/**
+ * The adjunct tables whose FK declarations may still point at the legacy
+ * `pages(id)` on migrated archives. Only names are listed — the column
+ * set is derived at runtime from the canonical table recreated by
+ * {@link createAdjunctTables}, so a future column addition there cannot
+ * silently drift from a hardcoded list (a missing column in the staged
+ * copy fails the copy-back INSERT loudly instead of dropping data).
+ */
+const RETARGET_TABLES = [
+    'page_html_ref',
+    'page_tags',
+    'page_jsonld',
+    'page_errors',
+    'analysis_violations',
+];
+/**
+ * Canonical columns that a staged (pre-migration) table may legitimately
+ * lack because {@link createAdjunctTables}'s DDL gained them after the
+ * table was first lazily provisioned on the input archive (`analysis_violations`
+ * predates its `line`/`col` columns — issue #225). Missing columns listed
+ * here are copied back as `NULL` instead of aborting the migration; this is
+ * safe only because the column did not exist when the staged data was
+ * written, so `NULL` is the only value it could ever have had. Any other
+ * canonical column absent from the staged table still aborts the copy (see
+ * {@link retargetLegacyFkTables} JSDoc) — that case must stay loud, since a
+ * genuine rename/drop cannot be told apart from benign schema growth by
+ * column existence alone.
+ */
+const NULLABLE_ON_RETARGET = {
+    analysis_violations: ['line', 'col'],
+};
+/**
+ * Rewrites the FK declarations of the adjunct tables from the legacy
+ * `pages(id)` to `content_items(id)` by rebuilding each table. SQLite has
+ * no `ALTER TABLE … DROP CONSTRAINT`, so the only way to change an FK
+ * target is to recreate the table:
+ *
+ * 1. Stage each table's rows into a constraint-free `<table>__retarget`
+ *    copy (`CREATE TABLE … AS SELECT *`) and drop the original, freeing
+ *    its index names.
+ * 2. Recreate every table through {@link createAdjunctTables} — the same
+ *    DDL fresh archives get, so migrated archives end up identical in
+ *    shape and index naming. Rebuilding the DDL inline here instead
+ *    would recreate the exact drift this function repairs.
+ * 3. Copy the staged rows back using the recreated table's own column
+ *    list (read via `pragma_table_info`). A column the canonical DDL has
+ *    but the input archive lacks aborts the INSERT with a clear error —
+ *    never a silent data drop.
+ *
+ * A rename-based recipe (`ALTER TABLE x RENAME TO x__retarget`, skip
+ * copy #1) is deliberately NOT used: SQLite keeps the renamed table's
+ * indexes attached under their original names, which collide with the
+ * index names {@link createAdjunctTables} declares — and the old index
+ * names vary by the era of the archive's original provisioning, so
+ * dropping them first would require enumerating unknown names. The
+ * copy-out is cheap at these tables' scale (a handful of rows per page).
+ *
+ * Adjunct tables that do not exist on the input archive (e.g. a 0.10
+ * archive that never ran `analyze`, so `analysis_violations` was never
+ * lazily provisioned) are simply created empty by step 2.
+ *
+ * The migrated-archive PK-preservation contract makes the copy safe:
+ * `content_items.id` reuses the legacy `pages.id` values verbatim, so
+ * every staged `pageId` / `page_id` resolves to the same row under the
+ * new FK target. Run inside a transaction with `PRAGMA foreign_keys = ON`
+ * so the copy-back doubles as a row-level FK validation — a stale id
+ * aborts the transaction here rather than surfacing later as a
+ * `foreign_key_check` finding.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @example
+ * await db.transaction(async (trx) => {
+ *   await retargetLegacyFkTables(trx);
+ * });
+ * // pragma_foreign_key_list('page_errors') now reports content_items.
+ */
+export async function retargetLegacyFkTables(trx) {
+    const staged = [];
+    for (const table of RETARGET_TABLES) {
+        if (!(await trx.schema.hasTable(table))) {
+            continue;
+        }
+        await trx.raw(`CREATE TABLE "${table}__retarget" AS SELECT * FROM "${table}"`);
+        await trx.raw(`DROP TABLE "${table}"`);
+        staged.push(table);
+    }
+    await createAdjunctTables(trx);
+    for (const table of staged) {
+        const columns = await trx
+            .select('name')
+            .from(trx.raw('pragma_table_info(?)', [table]));
+        const stagedColumns = await trx
+            .select('name')
+            .from(trx.raw('pragma_table_info(?)', [`${table}__retarget`]));
+        const stagedColumnNames = new Set(stagedColumns.map((column) => column.name));
+        const nullableOnRetarget = NULLABLE_ON_RETARGET[table] ?? [];
+        const insertColumnList = columns.map((column) => `"${column.name}"`).join(', ');
+        const selectColumnList = columns
+            .map((column) => !stagedColumnNames.has(column.name) && nullableOnRetarget.includes(column.name)
+            ? `NULL AS "${column.name}"`
+            : `"${column.name}"`)
+            .join(', ');
+        await trx.raw(`INSERT INTO "${table}" (${insertColumnList}) SELECT ${selectColumnList} FROM "${table}__retarget"`);
+        await trx.raw(`DROP TABLE "${table}__retarget"`);
+    }
+}

package/lib/archive/test-utils/fk-parent-tables.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Returns the set of parent table names referenced by `table`'s FK
+ * clauses, read via SQLite's `pragma_foreign_key_list` table-valued
+ * function. Shared by the specs that assert FK declarations were
+ * retargeted from the legacy `pages(id)` to `content_items(id)`.
+ * @param db - Knex connected to the test DB.
+ * @param table - Table whose FK declarations to inspect.
+ * @returns Parent table names, deduplicated.
+ * @example
+ * const parents = await fkParentTables(db, 'page_errors');
+ * expect(parents.has('content_items')).toBe(true);
+ * expect(parents.has('pages')).toBe(false);
+ */
+export declare function fkParentTables(db: Knex, table: string): Promise<Set<string>>;

package/lib/archive/test-utils/fk-parent-tables.js

@@ -0,0 +1,19 @@
+/**
+ * Returns the set of parent table names referenced by `table`'s FK
+ * clauses, read via SQLite's `pragma_foreign_key_list` table-valued
+ * function. Shared by the specs that assert FK declarations were
+ * retargeted from the legacy `pages(id)` to `content_items(id)`.
+ * @param db - Knex connected to the test DB.
+ * @param table - Table whose FK declarations to inspect.
+ * @returns Parent table names, deduplicated.
+ * @example
+ * const parents = await fkParentTables(db, 'page_errors');
+ * expect(parents.has('content_items')).toBe(true);
+ * expect(parents.has('pages')).toBe(false);
+ */
+export async function fkParentTables(db, table) {
+    const rows = await db
+        .select('table')
+        .from(db.raw('pragma_foreign_key_list(?)', [table]));
+    return new Set(rows.map((row) => row.table));
+}

package/lib/archive/test-utils/seed-content-item.d.ts

@@ -0,0 +1,35 @@
+import type { PageSource } from '../types.js';
+import type { Knex } from 'knex';
+/**
+ * Optional column overrides for {@link seedContentItem}. Defaults model a
+ * scraped, internal, targeted page with no content type.
+ */
+interface SeedContentItemOptions {
+    /** `content_items.scraped` (default 1). */
+    scraped?: 0 | 1;
+    /** `content_items.is_target` (default 1). */
+    isTarget?: 0 | 1;
+    /** `content_items.is_external` (default 0). */
+    isExternal?: 0 | 1;
+    /** `content_items.source` (default: schema default `'crawled'`). */
+    source?: PageSource;
+    /** Registers the value in `content_type_refs` and links it (default: none). */
+    contentType?: string;
+}
+/**
+ * Inserts one `url_refs` + `content_items` pair — the minimal seeding
+ * contract for specs that exercise entity-table readers. Shared so a
+ * schema change to the pair (e.g. a new NOT NULL column) is fixed in one
+ * place instead of drifting across per-spec copies.
+ * @param db - Knex connected to the test DB.
+ * @param url - URL string to register in `url_refs`.
+ * @param options - Column overrides; see {@link SeedContentItemOptions}.
+ * @returns The inserted `content_items.id`.
+ * @example
+ * const pageId = await seedContentItem(db, 'https://example.com/');
+ * const pdfId = await seedContentItem(db, 'https://example.com/doc.pdf', {
+ *   contentType: 'application/pdf',
+ * });
+ */
+export declare function seedContentItem(db: Knex, url: string, options?: SeedContentItemOptions): Promise<number>;
+export {};

package/lib/archive/test-utils/seed-content-item.js

@@ -0,0 +1,42 @@
+/**
+ * Inserts one `url_refs` + `content_items` pair — the minimal seeding
+ * contract for specs that exercise entity-table readers. Shared so a
+ * schema change to the pair (e.g. a new NOT NULL column) is fixed in one
+ * place instead of drifting across per-spec copies.
+ * @param db - Knex connected to the test DB.
+ * @param url - URL string to register in `url_refs`.
+ * @param options - Column overrides; see {@link SeedContentItemOptions}.
+ * @returns The inserted `content_items.id`.
+ * @example
+ * const pageId = await seedContentItem(db, 'https://example.com/');
+ * const pdfId = await seedContentItem(db, 'https://example.com/doc.pdf', {
+ *   contentType: 'application/pdf',
+ * });
+ */
+export async function seedContentItem(db, url, options = {}) {
+    const [urlRef] = await db('url_refs').insert({ url }).returning('id');
+    let contentTypeId = null;
+    if (options.contentType !== undefined) {
+        const [ctRef] = await db('content_type_refs')
+            .insert({
+            raw: options.contentType,
+            normalized: options.contentType,
+            category: 'other',
+        })
+            .onConflict('raw')
+            .merge({ raw: options.contentType })
+            .returning('id');
+        contentTypeId = ctRef.id;
+    }
+    const [item] = await db('content_items')
+        .insert({
+        url_id: urlRef.id,
+        scraped: options.scraped ?? 1,
+        is_target: options.isTarget ?? 1,
+        is_external: options.isExternal ?? 0,
+        content_type_id: contentTypeId,
+        ...(options.source === undefined ? {} : { source: options.source }),
+    })
+        .returning('id');
+    return item.id;
+}

package/lib/archive/test-utils/setup-legacy-fk-db.d.ts

@@ -0,0 +1,33 @@
+import type { Knex } from 'knex';
+/**
+ * Provisions the pre-0.13 legacy table shape on a test database:
+ * the five legacy write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) plus the five adjunct tables in
+ * their old form whose FK columns still point at `pages(id)`
+ * (`page_html_ref` / `page_tags` / `page_jsonld` / `page_errors` /
+ * `analysis_violations`, with their companion tables `page_html_blobs` /
+ * `analysis_text_refs`).
+ *
+ * `initSchema` no longer creates any of these shapes — fresh archives get
+ * neither the legacy write-model tables nor `pages(id)`-referencing
+ * adjunct tables — so specs that exercise the migration input format
+ * (`scripts/migrate-to-0.13.mjs` integration, `retargetLegacyFkTables`,
+ * `dropLegacyTables`) build it through this helper instead. The DDL is a
+ * verbatim copy of what pre-0.13 archives actually contain: the legacy
+ * write-model tables match the pre-0.13 `initSchema`, and the adjunct
+ * tables match the `scripts/migrate-to-0.10.mjs` / lazy-runtime-migration
+ * output of that era.
+ *
+ * Any current-form adjunct tables already present (e.g. created by
+ * `Archive.create()` on the current schema) are dropped first so the
+ * old-FK versions can take their place — callers get a deterministic
+ * "genuine pre-0.13 archive" shape regardless of how the underlying DB
+ * was provisioned.
+ * @param db - Knex connected to the test DB (or a transaction).
+ * @example
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await setupLegacyFkDb(db);
+ * await db('pages').insert({ url: 'https://example.com/', scraped: 1, isTarget: 1 });
+ */
+export declare function setupLegacyFkDb(db: Knex): Promise<void>;