@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/resources/get-resource-url-list.js

@@ -0,0 +1,13 @@
+/**
+ * Retrieves a flat list of all resource URLs from the `resource_items`
+ * table. URL text is normalised into `url_refs`, so the read joins the
+ * two tables and returns the resolved strings.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An array of resource URL strings.
+ */
+export async function getResourceUrlList(knex) {
+    const res = await knex('resource_items')
+        .join('url_refs', 'url_refs.id', 'resource_items.url_id')
+        .select('url_refs.url as url');
+    return res.map((r) => r.url);
+}

package/lib/archive/db-ops/resources/get-resources.d.ts

@@ -0,0 +1,8 @@
+import type { DB_Resource } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all sub-resources.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An array of reconstructed {@link DB_Resource} rows.
+ */
+export declare function getResources(knex: Knex): Promise<DB_Resource[]>;

package/lib/archive/db-ops/resources/get-resources.js

@@ -0,0 +1,11 @@
+import { buildResourceQuery } from './build-resource-query.js';
+import { reconstructResourceRows } from './reconstruct-resource-rows.js';
+/**
+ * Retrieves all sub-resources.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An array of reconstructed {@link DB_Resource} rows.
+ */
+export async function getResources(knex) {
+    const rows = await buildResourceQuery(knex);
+    return reconstructResourceRows(knex, rows);
+}

package/lib/archive/db-ops/resources/insert-inventory-resources.d.ts

@@ -0,0 +1,24 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Pre-insert inventory non-HTML URLs into `resource_items` as placeholder
+ * rows with `source = 'inventory-seed'` and all metadata columns NULL —
+ * the non-HTML counterpart of `insertInventorySeeds`. Used by
+ * `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
+ * its non-HTML URLs in one chunked round-trip per 500 instead of N
+ * sequential `insertResource` awaits — a per-URL loop would spend
+ * minutes inside the `.bak`-protected window on a 50k-URL inventory
+ * list, where the bulk path finishes in seconds.
+ *
+ * Idempotent: both the `url_refs` and `resource_items` inserts are
+ * `ON CONFLICT ... IGNORE`, leaving existing rows untouched (the
+ * orchestrator's `getExistingResourceUrls` filter is what keeps a
+ * crawled-lineage `resource_items` row from being downgraded to the
+ * inventory label here).
+ *
+ * Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches
+ * @param urls - URL strings (already in `withoutHashAndAuth` form).
+ */
+export declare function insertInventoryResources(knex: Knex, caches: WriteRefCaches, urls: readonly string[]): Promise<void>;

package/lib/archive/db-ops/resources/insert-inventory-resources.js

@@ -0,0 +1,64 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+import { resolveUrlRefs } from '../../populate-entity-tables/resolve-url-refs.js';
+import { decomposeUrl } from '../../populate-ref-tables/decompose-url.js';
+/**
+ * Pre-insert inventory non-HTML URLs into `resource_items` as placeholder
+ * rows with `source = 'inventory-seed'` and all metadata columns NULL —
+ * the non-HTML counterpart of `insertInventorySeeds`. Used by
+ * `CrawlerOrchestrator.inventory` so the ingestion phase commits all of
+ * its non-HTML URLs in one chunked round-trip per 500 instead of N
+ * sequential `insertResource` awaits — a per-URL loop would spend
+ * minutes inside the `.bak`-protected window on a 50k-URL inventory
+ * list, where the bulk path finishes in seconds.
+ *
+ * Idempotent: both the `url_refs` and `resource_items` inserts are
+ * `ON CONFLICT ... IGNORE`, leaving existing rows untouched (the
+ * orchestrator's `getExistingResourceUrls` filter is what keeps a
+ * crawled-lineage `resource_items` row from being downgraded to the
+ * inventory label here).
+ *
+ * Chunked at 500 to stay well under SQLite's `SQLITE_MAX_VARIABLE_NUMBER`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches
+ * @param urls - URL strings (already in `withoutHashAndAuth` form).
+ */
+export async function insertInventoryResources(knex, caches, urls) {
+    if (urls.length === 0) {
+        return;
+    }
+    await eachSplitted([...urls], 500, async (chunk) => {
+        await knex('url_refs')
+            .insert(chunk.map((url) => ({ url, ...decomposeUrl(url) })))
+            .onConflict('url')
+            .ignore();
+        const urlIds = await resolveUrlRefs(knex, chunk);
+        const rows = chunk.map((url) => {
+            const urlId = urlIds.get(url);
+            if (urlId === undefined) {
+                throw new Error(`insertInventoryResources: url_refs.id not resolved for ${url}`);
+            }
+            caches.urlIds.set(url, urlId);
+            return {
+                url_id: urlId,
+                is_external: 0,
+                status: null,
+                status_text: null,
+                content_type_id: null,
+                content_length: null,
+                header_set_id: null,
+                compress: 0,
+                cdn: 0,
+                source: 'inventory-seed',
+            };
+        });
+        await knex('resource_items').insert(rows).onConflict('url_id').ignore();
+        const inserted = (await knex
+            .select('ri.id', 'ur.url')
+            .from('resource_items as ri')
+            .join('url_refs as ur', 'ur.id', 'ri.url_id')
+            .whereIn('ur.url', chunk));
+        for (const row of inserted) {
+            caches.resourceIds.set(row.url, row.id);
+        }
+    });
+}

package/lib/archive/db-ops/resources/insert-resource-referrers.d.ts

@@ -0,0 +1,15 @@
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Inserts a referrer relationship between a resource and a page into the
+ * `resource_ref_edges` table. Silently skips if the resource is not found.
+ *
+ * A repeat observation of the same `(resource, page)` pair is ignored
+ * (`ON CONFLICT ... IGNORE`), leaving `count` at its DEFAULT `1` — one
+ * row per pair, matching the legacy uniqueness contract.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param src - The URL of the resource.
+ * @param pageUrl - The URL of the page that references the resource.
+ */
+export declare function insertResourceReferrers(knex: Knex, caches: WriteRefCaches, src: string, pageUrl: string): Promise<void>;

package/lib/archive/db-ops/resources/insert-resource-referrers.js

@@ -0,0 +1,54 @@
+import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
+/**
+ * Inserts a referrer relationship between a resource and a page into the
+ * `resource_ref_edges` table. Silently skips if the resource is not found.
+ *
+ * A repeat observation of the same `(resource, page)` pair is ignored
+ * (`ON CONFLICT ... IGNORE`), leaving `count` at its DEFAULT `1` — one
+ * row per pair, matching the legacy uniqueness contract.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param src - The URL of the resource.
+ * @param pageUrl - The URL of the page that references the resource.
+ */
+export async function insertResourceReferrers(knex, caches, src, pageUrl) {
+    const resourceId = await findResourceIdByUrl(knex, caches, src);
+    if (resourceId === undefined) {
+        // Ignore when the resource is not found
+        return;
+    }
+    const pageId = await resolveContentItemId(knex, caches, pageUrl);
+    await knex('resource_ref_edges')
+        .insert({
+        resource_id: resourceId,
+        page_id: pageId,
+    })
+        .onConflict(['resource_id', 'page_id'])
+        .ignore();
+}
+/**
+ * Cache-first lookup of `resource_items.id` by the resource URL. The
+ * cache only ever gains entries for URLs that exist in the table
+ * (resource identities are never deleted during a crawl), so a hit is
+ * always valid; a miss falls through to a `url_refs` join.
+ * @param knex - Knex query builder.
+ * @param caches - The connection's write-side id caches.
+ * @param url - The resource URL, verbatim as stored.
+ * @returns The resource id, or `undefined` when the URL is unknown.
+ */
+async function findResourceIdByUrl(knex, caches, url) {
+    const cached = caches.resourceIds.get(url);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const [row] = (await knex
+        .select('ri.id')
+        .from('resource_items as ri')
+        .join('url_refs as ur', 'ur.id', 'ri.url_id')
+        .where('ur.url', url));
+    if (row === undefined) {
+        return undefined;
+    }
+    caches.resourceIds.set(url, row.id);
+    return row.id;
+}

package/lib/archive/db-ops/resources/insert-resource.d.ts

@@ -0,0 +1,34 @@
+import type { Resource } from '../../../utils/types/types.js';
+import type { PageSource } from '../../types.js';
+import type { WriteRefCaches } from '../_shared/types.js';
+import type { Knex } from 'knex';
+/**
+ * Inserts a sub-resource into the `resource_items` table, interning its
+ * URL / content type / response headers into the ref tables first. A
+ * resource's identity URL routes through {@link resolveUrlOrBlob} like an
+ * image `src` — a large inline `data:` URI (e.g. a CSS
+ * `background-image` sub-resource) lands in `blob_refs` instead of
+ * `url_refs`.
+ *
+ * A conflict on the identity column (`url_id` or `url_blob_id`) is an
+ * **upgrade-only merge**, not first-write-wins: the existing row's
+ * response fields (`status` / `status_text` / `content_type_id` /
+ * `content_length` / `header_set_id` / `compress` / `cdn`) are replaced
+ * only when the existing row currently has `status IS NULL` (beholder's
+ * signal for "the request never got a response" — see the `Resource`
+ * type) AND the new observation has a non-null `status`. A resource that
+ * failed on first sight and later succeeds (the exact case
+ * `--retry-failed` re-renders a page for) is therefore healed instead of
+ * staying frozen at its failure; a resource that already has a real
+ * response never gets overwritten by a later transient failure, and two
+ * failures in a row are a no-op — neither carries new information.
+ *
+ * The `source` provenance label is written ONLY on insert; a conflict
+ * (upgrade or no-op alike) leaves an existing row's source untouched
+ * (this is what makes a second `crawl --inventory` non-destructive).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param resource - The resource data to insert.
+ * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
+ */
+export declare function insertResource(knex: Knex, caches: WriteRefCaches, resource: Resource, source?: PageSource): Promise<void>;

package/lib/archive/db-ops/resources/insert-resource.js

@@ -0,0 +1,73 @@
+import { normalizeContentType } from '../../../crawler/normalize-content-type.js';
+import { resolveUrlOrBlob } from '../_shared/resolve-url-or-blob.js';
+import { upsertContentTypeRef } from '../_shared/upsert-content-type-ref.js';
+import { upsertResponseHeaders } from '../_shared/upsert-response-headers.js';
+/**
+ * Inserts a sub-resource into the `resource_items` table, interning its
+ * URL / content type / response headers into the ref tables first. A
+ * resource's identity URL routes through {@link resolveUrlOrBlob} like an
+ * image `src` — a large inline `data:` URI (e.g. a CSS
+ * `background-image` sub-resource) lands in `blob_refs` instead of
+ * `url_refs`.
+ *
+ * A conflict on the identity column (`url_id` or `url_blob_id`) is an
+ * **upgrade-only merge**, not first-write-wins: the existing row's
+ * response fields (`status` / `status_text` / `content_type_id` /
+ * `content_length` / `header_set_id` / `compress` / `cdn`) are replaced
+ * only when the existing row currently has `status IS NULL` (beholder's
+ * signal for "the request never got a response" — see the `Resource`
+ * type) AND the new observation has a non-null `status`. A resource that
+ * failed on first sight and later succeeds (the exact case
+ * `--retry-failed` re-renders a page for) is therefore healed instead of
+ * staying frozen at its failure; a resource that already has a real
+ * response never gets overwritten by a later transient failure, and two
+ * failures in a row are a no-op — neither carries new information.
+ *
+ * The `source` provenance label is written ONLY on insert; a conflict
+ * (upgrade or no-op alike) leaves an existing row's source untouched
+ * (this is what makes a second `crawl --inventory` non-destructive).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param caches - The connection's write-side id caches.
+ * @param resource - The resource data to insert.
+ * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
+ */
+export async function insertResource(knex, caches, resource, source) {
+    const slot = await resolveUrlOrBlob(knex, caches, resource.url.href);
+    // Canonicalize like the page path (see `insertPage`) so resource
+    // content-type filters / dedupe keys are case- and whitespace-stable.
+    const contentType = normalizeContentType(resource.contentType);
+    const contentTypeId = contentType == null || contentType === ''
+        ? null
+        : await upsertContentTypeRef(knex, caches, contentType);
+    const headerSetId = await upsertResponseHeaders(knex, caches, resource.headers);
+    const identityColumn = slot.blob === null ? 'url_id' : 'url_blob_id';
+    const identityValue = slot.blob === null ? slot.url : slot.blob;
+    const responseFields = {
+        is_external: resource.isExternal ? 1 : 0,
+        status: resource.status,
+        status_text: resource.statusText,
+        content_type_id: contentTypeId,
+        content_length: resource.contentLength,
+        header_set_id: headerSetId,
+        compress: resource.compress || 0,
+        cdn: resource.cdn || 0,
+    };
+    const existing = await knex('resource_items')
+        .select('id', 'status')
+        .where(identityColumn, identityValue)
+        .first();
+    if (existing === undefined) {
+        await knex('resource_items').insert({
+            url_id: slot.url,
+            url_blob_id: slot.blob,
+            ...responseFields,
+            ...(source === undefined ? {} : { source }),
+        });
+        return;
+    }
+    if (existing.status === null && resource.status !== null) {
+        await knex('resource_items')
+            .where('id', existing.id)
+            .update(responseFields);
+    }
+}

package/lib/archive/db-ops/resources/reconstruct-resource-rows.d.ts

@@ -0,0 +1,26 @@
+import type { DB_Resource } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Raw row shape produced by {@link ../resources/build-resource-query.js}
+ * before `responseHeaders` reconstruction.
+ */
+interface RawResourceRow extends Omit<DB_Resource, 'responseHeaders'> {
+    /** `resource_items.header_set_id`, or null when no headers were recorded. */
+    headerSetId: number | null;
+}
+/**
+ * Reconstructs `responseHeaders` (JSON string) for a batch of raw resource
+ * rows. Headers load through
+ * {@link ../_shared/load-response-headers-by-set-ids.js} (chunked batch
+ * lookup, never N+1) — the same reconstruction the page read path uses,
+ * so pages and resources always agree on how a given `header_set_id`
+ * reads back.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param rows - Raw rows from {@link ../resources/build-resource-query.js}.
+ * @returns Fully reconstructed `DB_Resource` rows, in the same order as `rows`.
+ * @example
+ * const raw = await buildResourceQuery(knex).whereIn('ur.url', urls);
+ * const resources = await reconstructResourceRows(knex, raw);
+ */
+export declare function reconstructResourceRows(knex: Knex, rows: readonly RawResourceRow[]): Promise<DB_Resource[]>;
+export {};

package/lib/archive/db-ops/resources/reconstruct-resource-rows.js

@@ -0,0 +1,30 @@
+import { loadResponseHeadersBySetIds } from '../_shared/load-response-headers-by-set-ids.js';
+/**
+ * Reconstructs `responseHeaders` (JSON string) for a batch of raw resource
+ * rows. Headers load through
+ * {@link ../_shared/load-response-headers-by-set-ids.js} (chunked batch
+ * lookup, never N+1) — the same reconstruction the page read path uses,
+ * so pages and resources always agree on how a given `header_set_id`
+ * reads back.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param rows - Raw rows from {@link ../resources/build-resource-query.js}.
+ * @returns Fully reconstructed `DB_Resource` rows, in the same order as `rows`.
+ * @example
+ * const raw = await buildResourceQuery(knex).whereIn('ur.url', urls);
+ * const resources = await reconstructResourceRows(knex, raw);
+ */
+export async function reconstructResourceRows(knex, rows) {
+    const headerSetIds = [
+        ...new Set(rows.map((r) => r.headerSetId).filter((id) => id != null)),
+    ];
+    const headersBySetId = await loadResponseHeadersBySetIds(knex, headerSetIds);
+    return rows.map((row) => {
+        const { headerSetId, ...rest } = row;
+        return {
+            ...rest,
+            responseHeaders: headerSetId == null
+                ? null
+                : JSON.stringify(headersBySetId.get(headerSetId) ?? {}),
+        };
+    });
+}

package/lib/archive/decode-html-blob.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Decodes a stored HTML body BLOB according to its codec marker. The codec
+ * column on `page_html_blobs` records which encoding was applied at write
+ * time so individual rows can be migrated to a different compression without
+ * a table-wide rewrite.
+ *
+ * Shared by {@link ./database.ts#Database.getHtmlOfPageById} (writer read
+ * path) and {@link ./populate-entity-tables/populate-image-items.ts} (0.13
+ * migration, which reads HTML *inside* its writer transaction and therefore
+ * cannot go through `Database.getHtmlOfPageById` — that method uses the
+ * non-transactional Knex instance, and re-entering the pool from within an
+ * outer writer trx serialises on libsql's single writer connection).
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns The decoded HTML string.
+ * @throws {Error} If the codec is not recognised.
+ */
+export declare function decodeStoredBlob(body: Uint8Array, codec: string): string;

package/lib/archive/decode-html-blob.js

@@ -0,0 +1,31 @@
+import { zstdDecompressSync } from 'node:zlib';
+/**
+ * Decodes a stored HTML body BLOB according to its codec marker. The codec
+ * column on `page_html_blobs` records which encoding was applied at write
+ * time so individual rows can be migrated to a different compression without
+ * a table-wide rewrite.
+ *
+ * Shared by {@link ./database.ts#Database.getHtmlOfPageById} (writer read
+ * path) and {@link ./populate-entity-tables/populate-image-items.ts} (0.13
+ * migration, which reads HTML *inside* its writer transaction and therefore
+ * cannot go through `Database.getHtmlOfPageById` — that method uses the
+ * non-transactional Knex instance, and re-entering the pool from within an
+ * outer writer trx serialises on libsql's single writer connection).
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns The decoded HTML string.
+ * @throws {Error} If the codec is not recognised.
+ */
+export function decodeStoredBlob(body, codec) {
+    // `Buffer.from(buffer)` accepts Uint8Array, Buffer, and array-like
+    // shapes uniformly; libsql may hand back any of these for a BLOB
+    // column depending on the row encoding.
+    const buffer = Buffer.from(body);
+    if (codec === 'zstd') {
+        return zstdDecompressSync(buffer).toString('utf8');
+    }
+    if (codec === 'none') {
+        return buffer.toString('utf8');
+    }
+    throw new Error(`Unknown page_html_blobs.codec: ${codec}`);
+}

package/lib/archive/derive-lineage-from-parent.d.ts

@@ -0,0 +1,37 @@
+import type { PageSource } from './types.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `resolveContentItemId` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export declare function deriveLineageFromParent(parentSource: PageSource | undefined, fallback: PageSource | undefined): PageSource | undefined;

package/lib/archive/derive-lineage-from-parent.js

@@ -0,0 +1,42 @@
+import { isInventorySource } from './is-inventory-source.js';
+/**
+ * Decide which {@link PageSource} label a child row reached through the
+ * crawl graph (anchor placeholder, sub-resource, redirect chain
+ * intermediate) should inherit from its parent.
+ *
+ * Two simple rules, expressed once so anchor / redirect / sub-resource
+ * call sites stay in lockstep:
+ *
+ * 1. If the parent is in the inventory chain
+ *    ({@link isInventorySource}) → propagate
+ *    `'inventory-discovered'`. The child is itself a transitively
+ *    reached node in the inventory chain; it is NOT a new seed (the
+ *    seed label is reserved for URLs the operator listed in
+ *    `--inventory ./list.txt`).
+ *
+ * 2. Otherwise → return `fallback`. The two production fallbacks differ
+ *    by call site:
+ *
+ *    - Anchor lineage passes `'crawled'` explicitly so the crawled-wins
+ *      downgrade inside `resolveContentItemId` fires when the anchor reaches an
+ *      existing `'inventory-*'` row.
+ *    - Sub-resource emit passes `undefined` so the DB DEFAULT
+ *      `'crawled'` lands on the freshly INSERTed `resources` row (the
+ *      `setResources` path is INSERT-only with `onConflict.ignore()`, so
+ *      no downgrade is needed).
+ *    - Redirect chain intermediate uses `'crawled'` (same reason as
+ *      anchor): an existing inventory-* intermediate reached by a
+ *      crawled redirect chain must be downgraded.
+ *
+ * Pure function — keeps the lineage decision testable in isolation from
+ * the database transaction / event-emitter wiring that consumes it.
+ * @param parentSource - The parent page's stored `source` column (or `undefined` when no parent row exists).
+ * @param fallback - The label to return when the parent is NOT in the inventory chain. Pass `'crawled'` to enable the crawled-wins downgrade, or `undefined` to let the DB DEFAULT apply.
+ * @returns The lineage label to attach to the child row.
+ */
+export function deriveLineageFromParent(parentSource, fallback) {
+    if (isInventorySource(parentSource)) {
+        return 'inventory-discovered';
+    }
+    return fallback;
+}

package/lib/archive/drop-legacy-tables.d.ts

@@ -0,0 +1,45 @@
+import type { Knex } from 'knex';
+/**
+ * Drops the five legacy write-model tables from a migrated archive. Reader
+ * and writer both operate exclusively on the entity / ref tables, so once
+ * the 0.13 populate has copied everything across (and
+ * `retargetLegacyFkTables` has repointed the adjunct FKs at
+ * `content_items`), the legacy tables are pure dead weight — dropping them
+ * reclaims the duplicated storage and removes the last consumers of the
+ * `pages(id)` id-space.
+ *
+ * **Requires `PRAGMA foreign_keys = OFF`** (asserted at runtime, because a
+ * caller that forgets the toggle would fail nondeterministically instead
+ * of loudly). Two reasons for the requirement:
+ *
+ * 1. `pages.redirectDestId` is a self-referential FK. With enforcement ON,
+ *    `DROP TABLE` performs an implicit `DELETE FROM`, and deleting a
+ *    redirect-destination row before its source row trips the constraint —
+ *    whether the drop survives depends on row order, so archives with
+ *    redirect chains can fail nondeterministically.
+ * 2. With enforcement OFF the implicit `DELETE FROM` is skipped entirely
+ *    and SQLite reclaims each table as a whole b-tree — O(pages) row
+ *    deletions become O(1) truncations, which matters at the several
+ *    hundred-thousand-row scale real archives reach.
+ *
+ * The drops still run children-first (`resources-referrers` / `anchors` /
+ * `images` reference `pages` / `resources`) as defence in depth. Missing
+ * tables are skipped so the function is idempotent.
+ *
+ * Data-integrity of the surviving tables is asserted separately by
+ * `checkForeignKeyIntegrity` (`PRAGMA foreign_key_check` ignores the
+ * `foreign_keys` setting), which the migration script runs right after
+ * this function.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @throws {Error} when the connection still has `PRAGMA foreign_keys = ON`
+ *   (`foreign_keys` cannot change inside a transaction, so the caller must
+ *   toggle it off before opening the transaction).
+ * @example
+ * await db.raw('PRAGMA foreign_keys = OFF');
+ * await db.transaction(async (trx) => {
+ *   await dropLegacyTables(trx);
+ * });
+ * await db.raw('PRAGMA foreign_keys = ON');
+ */
+export declare function dropLegacyTables(trx: Knex): Promise<void>;

package/lib/archive/drop-legacy-tables.js

@@ -0,0 +1,56 @@
+/**
+ * Drops the five legacy write-model tables from a migrated archive. Reader
+ * and writer both operate exclusively on the entity / ref tables, so once
+ * the 0.13 populate has copied everything across (and
+ * `retargetLegacyFkTables` has repointed the adjunct FKs at
+ * `content_items`), the legacy tables are pure dead weight — dropping them
+ * reclaims the duplicated storage and removes the last consumers of the
+ * `pages(id)` id-space.
+ *
+ * **Requires `PRAGMA foreign_keys = OFF`** (asserted at runtime, because a
+ * caller that forgets the toggle would fail nondeterministically instead
+ * of loudly). Two reasons for the requirement:
+ *
+ * 1. `pages.redirectDestId` is a self-referential FK. With enforcement ON,
+ *    `DROP TABLE` performs an implicit `DELETE FROM`, and deleting a
+ *    redirect-destination row before its source row trips the constraint —
+ *    whether the drop survives depends on row order, so archives with
+ *    redirect chains can fail nondeterministically.
+ * 2. With enforcement OFF the implicit `DELETE FROM` is skipped entirely
+ *    and SQLite reclaims each table as a whole b-tree — O(pages) row
+ *    deletions become O(1) truncations, which matters at the several
+ *    hundred-thousand-row scale real archives reach.
+ *
+ * The drops still run children-first (`resources-referrers` / `anchors` /
+ * `images` reference `pages` / `resources`) as defence in depth. Missing
+ * tables are skipped so the function is idempotent.
+ *
+ * Data-integrity of the surviving tables is asserted separately by
+ * `checkForeignKeyIntegrity` (`PRAGMA foreign_key_check` ignores the
+ * `foreign_keys` setting), which the migration script runs right after
+ * this function.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @throws {Error} when the connection still has `PRAGMA foreign_keys = ON`
+ *   (`foreign_keys` cannot change inside a transaction, so the caller must
+ *   toggle it off before opening the transaction).
+ * @example
+ * await db.raw('PRAGMA foreign_keys = OFF');
+ * await db.transaction(async (trx) => {
+ *   await dropLegacyTables(trx);
+ * });
+ * await db.raw('PRAGMA foreign_keys = ON');
+ */
+export async function dropLegacyTables(trx) {
+    const pragma = await trx.raw('PRAGMA foreign_keys');
+    if (pragma[0]?.foreign_keys === 1) {
+        throw new Error('dropLegacyTables: PRAGMA foreign_keys is ON — turn it OFF before the transaction. ' +
+            'With enforcement ON, dropping `pages` runs an implicit DELETE whose row order ' +
+            'can trip the redirectDestId self-FK nondeterministically.');
+    }
+    await trx.raw('DROP TABLE IF EXISTS "resources-referrers"');
+    await trx.raw('DROP TABLE IF EXISTS "anchors"');
+    await trx.raw('DROP TABLE IF EXISTS "images"');
+    await trx.raw('DROP TABLE IF EXISTS "resources"');
+    await trx.raw('DROP TABLE IF EXISTS "pages"');
+}

package/lib/archive/filesystem/rename.js

@@ -1,7 +1,7 @@
 import { promises as fs } from 'node:fs';
 import { remove } from './remove.js';
 /**
- * Checks whether a value is a {@link NodeJS.ErrnoException}.
+ * Checks whether a value is a `NodeJS.ErrnoException`.
  * @param error - The value to check.
  * @returns `true` if the value has a `code` property of type `string`.
  */

package/lib/archive/get-failed-page-messages.d.ts

@@ -0,0 +1,44 @@
+import type { Knex } from 'knex';
+/**
+ * Bulk-resolve a raw error message for each given page id, using only sources
+ * reachable from a {@link Knex} handle. Read order: `page_errors` (keyed by
+ * `pageId`, the most direct signal a scrape attempt recorded), then
+ * `crawl_errors` (keyed by `url`, the crawler-channel record for failures
+ * that happened before a page row was scraped).
+ *
+ * **Known limitation — pre-`crawl_errors` archives**: This helper does NOT
+ * read `error.log`. The `crawl_errors` table is created empty (by
+ * `initSchema` on fresh archives, by `scripts/migrate-to-0.13.mjs` on
+ * migrated ones) and historical lines from `error.log` are never
+ * back-filled, so an archive whose failures live only in `error.log`
+ * will resolve every id to "no message" here. The
+ * downstream `Database.resetFailedPages` treats absence as `unknown` (still
+ * retryable), so legacy archives lose the permanent-kind exclusion until a
+ * fresh crawl run populates `crawl_errors` / `page_errors`. The trade-off
+ * (no error.log parsing in the writer path) keeps the writer dependency
+ * surface narrow and avoids re-implementing the parser already living in
+ * `@nitpicker/query`'s `resolveFailedPageMessages` — which the crawler
+ * package cannot import (reverse-direction dependency). When this matters
+ * in practice, run the archive through one fresh `crawl --retry-failed`
+ * pass first to populate the structured tables.
+ *
+ * Pages with no message in any consulted source are simply absent from the
+ * returned map; callers treat the absence as "unclassifiable, keep retrying"
+ * (i.e. `unknown`).
+ * @param instance - The {@link Knex} handle.
+ * @param ids - Candidate `pages.id` values.
+ * @param urls - The corresponding `pages.url` values, in the same order as
+ *   `ids`. Length and indexing MUST match `ids` so the page → url join can be
+ *   reconstructed without a second `pages` round-trip.
+ * @returns `Map<pageId, message>` populated only for ids whose message was
+ *   found in one of the consulted tables.
+ * @example
+ * ```ts
+ * const messages = await getFailedPageMessages(
+ *   instance,
+ *   candidates.map(c => c.id),
+ *   candidates.map(c => c.url),
+ * );
+ * ```
+ */
+export declare function getFailedPageMessages(instance: Knex, ids: readonly number[], urls: readonly string[]): Promise<Map<number, string>>;