@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/db-ops/meta/get-jsonld-of-page.d.ts

@@ -0,0 +1,13 @@
+import type { JsonLdRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+ * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+ *
+ * Read-side counterpart to `insertJsonLd`. Returns rows in insertion order
+ * by `id` so the order observed by `get-page-jsonld` matches the order the
+ * scraper saw them.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getJsonLdOfPage(knex: Knex, pageId: number): Promise<JsonLdRow[]>;

package/lib/archive/db-ops/meta/get-jsonld-of-page.js

@@ -0,0 +1,27 @@
+import { safeParseJson } from '../_shared/safe-parse-json.js';
+/**
+ * Retrieves all `page_jsonld` rows for the given page id, parsed back into
+ * {@link JsonLdRow} shape (with `parsed` deserialised from its JSON column).
+ *
+ * Read-side counterpart to `insertJsonLd`. Returns rows in insertion order
+ * by `id` so the order observed by `get-page-jsonld` matches the order the
+ * scraper saw them.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getJsonLdOfPage(knex, pageId) {
+    const rows = await knex
+        .select('id', 'pageId', 'kind', 'type', 'raw', 'parsed', 'parseError')
+        .from('page_jsonld')
+        .where('pageId', pageId)
+        .orderBy('id', 'asc');
+    return rows.map((r) => ({
+        id: r.id,
+        pageId: r.pageId,
+        kind: r.kind === 'speculationrules' ? 'speculationrules' : 'ld+json',
+        type: r.type,
+        raw: r.raw,
+        parsed: r.parsed === null ? null : safeParseJson(r.parsed),
+        parseError: r.parseError,
+    }));
+}

package/lib/archive/db-ops/meta/get-tags-of-page.d.ts

@@ -0,0 +1,12 @@
+import type { TagRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_tags` rows for the given page id, parsed back into
+ * {@link TagRow} shape (with `categories` and `sources` JSON columns
+ * deserialised).
+ *
+ * Read-side counterpart to `insertTags`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getTagsOfPage(knex: Knex, pageId: number): Promise<TagRow[]>;

package/lib/archive/db-ops/meta/get-tags-of-page.js

@@ -0,0 +1,28 @@
+import { safeParseJson } from '../_shared/safe-parse-json.js';
+/**
+ * Retrieves all `page_tags` rows for the given page id, parsed back into
+ * {@link TagRow} shape (with `categories` and `sources` JSON columns
+ * deserialised).
+ *
+ * Read-side counterpart to `insertTags`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getTagsOfPage(knex, pageId) {
+    const rows = await knex
+        .select('id', 'pageId', 'provider', 'category', 'externalId', 'version', 'confidence', 'categories', 'sources')
+        .from('page_tags')
+        .where('pageId', pageId)
+        .orderBy('id', 'asc');
+    return rows.map((r) => ({
+        id: r.id,
+        pageId: r.pageId,
+        provider: r.provider,
+        category: r.category,
+        externalId: r.externalId,
+        version: r.version,
+        confidence: r.confidence,
+        categories: r.categories === null ? [] : (safeParseJson(r.categories) ?? []),
+        sources: r.sources === null ? [] : (safeParseJson(r.sources) ?? []),
+    }));
+}

package/lib/archive/db-ops/pages/order/set-url-order.d.ts

@@ -0,0 +1,8 @@
+import type { Knex } from 'knex';
+/**
+ * Assigns natural URL sort order values (`content_items.crawl_order`) to
+ * all internal pages. Pages are sorted using {@link pathComparator} and
+ * assigned sequential order numbers.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function setUrlOrder(knex: Knex): Promise<void>;

package/lib/archive/db-ops/pages/order/set-url-order.js

@@ -0,0 +1,32 @@
+import { pathComparator } from '@d-zero/shared/sort/path';
+import { dbLog } from '../../../debug.js';
+/**
+ * Assigns natural URL sort order values (`content_items.crawl_order`) to
+ * all internal pages. Pages are sorted using {@link pathComparator} and
+ * assigned sequential order numbers.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function setUrlOrder(knex) {
+    dbLog('Set URL Order');
+    const res = (await knex
+        .select('ci.id', 'ur.url')
+        .from('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .where('ci.is_external', '=', 0));
+    const sorted = res.toSorted((a, b) => pathComparator(a.url, b.url));
+    // Batch update using chunked CASE statements to avoid N+1 queries
+    const BATCH_SIZE = 500;
+    for (let i = 0; i < sorted.length; i += BATCH_SIZE) {
+        const batch = sorted.slice(i, i + BATCH_SIZE);
+        const ids = batch.map((row) => row.id);
+        const bindings = [];
+        const cases = batch
+            .map((row, j) => {
+            bindings.push(row.id, i + j + 1);
+            return 'WHEN ? THEN ?';
+        })
+            .join(' ');
+        const placeholders = ids.map(() => '?').join(',');
+        await knex.raw(`UPDATE content_items SET crawl_order = CASE id ${cases} END WHERE id IN (${placeholders})`, [...bindings, ...ids]);
+    }
+}

package/lib/archive/db-ops/pages/read/build-page-query.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Builds a `content_items` query joined against `page_meta` and every ref
+ * table needed to reconstruct a flat, legacy-shaped page row. Selected
+ * columns are aliased to the `DB_Page` field names; `responseHeaders` and
+ * `meta_extras` are NOT reconstructed here (they need a second, batched
+ * pass — see {@link ../read/reconstruct-page-rows.js}) so this query stays
+ * a single-pass join with no N+1 subqueries. `headerSetId` and the
+ * `extras_body` / `extras_codec` pair are selected as intermediates for
+ * that second pass.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns A query builder pre-configured with every join `get-pages.ts` /
+ *   `get-pages-with-rels.ts` need; callers add `.where()` / `.limit()` /
+ *   `.offset()` on top.
+ * @example
+ * const rows = await buildPageQuery(knex).where('ci.is_target', 1);
+ */
+export declare function buildPageQuery(knex: Knex): Knex.QueryBuilder;

package/lib/archive/db-ops/pages/read/build-page-query.js

@@ -0,0 +1,40 @@
+/**
+ * Builds a `content_items` query joined against `page_meta` and every ref
+ * table needed to reconstruct a flat, legacy-shaped page row. Selected
+ * columns are aliased to the `DB_Page` field names; `responseHeaders` and
+ * `meta_extras` are NOT reconstructed here (they need a second, batched
+ * pass — see {@link ../read/reconstruct-page-rows.js}) so this query stays
+ * a single-pass join with no N+1 subqueries. `headerSetId` and the
+ * `extras_body` / `extras_codec` pair are selected as intermediates for
+ * that second pass.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns A query builder pre-configured with every join `get-pages.ts` /
+ *   `get-pages-with-rels.ts` need; callers add `.where()` / `.limit()` /
+ *   `.offset()` on top.
+ * @example
+ * const rows = await buildPageQuery(knex).where('ci.is_target', 1);
+ */
+export function buildPageQuery(knex) {
+    return knex('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .leftJoin('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+        .leftJoin('page_meta as pm', 'pm.page_id', 'ci.id')
+        .leftJoin('text_refs as title_ref', 'title_ref.id', 'pm.title_text_id')
+        .leftJoin('text_refs as description_ref', 'description_ref.id', 'pm.description_text_id')
+        .leftJoin('text_refs as keywords_ref', 'keywords_ref.id', 'pm.keywords_text_id')
+        .leftJoin('text_refs as robots_raw_ref', 'robots_raw_ref.id', 'pm.robots_raw_text_id')
+        .leftJoin('text_refs as og_title_ref', 'og_title_ref.id', 'pm.og_title_text_id')
+        .leftJoin('text_refs as og_description_ref', 'og_description_ref.id', 'pm.og_description_text_id')
+        .leftJoin('text_refs as twitter_title_ref', 'twitter_title_ref.id', 'pm.twitter_title_text_id')
+        .leftJoin('text_refs as twitter_description_ref', 'twitter_description_ref.id', 'pm.twitter_description_text_id')
+        .leftJoin('url_refs as canonical_ur', 'canonical_ur.id', 'pm.canonical_url_id')
+        .leftJoin('url_refs as amphtml_ur', 'amphtml_ur.id', 'pm.amphtml_url_id')
+        .leftJoin('url_refs as manifest_ur', 'manifest_ur.id', 'pm.manifest_url_id')
+        .leftJoin('url_refs as icon_ur', 'icon_ur.id', 'pm.icon_url_id')
+        .leftJoin('url_refs as apple_ur', 'apple_ur.id', 'pm.apple_touch_icon_url_id')
+        .leftJoin('url_refs as og_url_ur', 'og_url_ur.id', 'pm.og_url_id')
+        .leftJoin('url_refs as og_image_ur', 'og_image_ur.id', 'pm.og_image_url_id')
+        .leftJoin('url_refs as twitter_image_ur', 'twitter_image_ur.id', 'pm.twitter_image_url_id')
+        .leftJoin('json_refs as extras_ref', 'extras_ref.id', 'pm.meta_extras_json_id')
+        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
+}

package/lib/archive/db-ops/pages/read/get-crawling-state.d.ts

@@ -0,0 +1,70 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the current crawling state by listing scraped and pending URLs.
+ *
+ * `scraped` is straightforward: every page row whose `scraped` flag is `1`
+ * — that is, every URL the crawl reached a terminal state on, including
+ * setSkippedPage / setExternalPage / outright setPage success or failure.
+ *
+ * `pending` is intentionally STRICT — not "every `scraped = 0` row".
+ * Three filters apply:
+ *
+ * 1. `scraped = 0` — work still incomplete.
+ * 2. `is_external = 0` — only in-scope work. External URLs go through a
+ *    HEAD-only path that always lands on `scraped = 1` (either setPage or
+ *    setExternalPage). A row with `is_external = 1 AND scraped = 0` is
+ *    therefore a data anomaly, and resume / inventory / append have no
+ *    business retrying it on the next session.
+ * 3. `EXISTS (anchor_edges with href_page_id = content_items.id) OR source
+ *    != 'crawled'` — the row was either discovered as an anchor destination
+ *    during a previous scrape OR was explicitly tagged with a non-default
+ *    source label (`'inventory-seed'`, `'inventory-discovered'`, …). Both
+ *    halves of the OR represent "deliberately enqueued, expected to be
+ *    processed", which is exactly what `resume` should pick up.
+ *
+ *    The orphan filter targets the **predicted-discard leak** in
+ *    `crawler.ts` where `shouldDiscardPredicted` returns true but no
+ *    `emit('skip')` follows. Such placeholders are inserted with the
+ *    DB DEFAULT `source = 'crawled'` (no caller explicitly labels
+ *    them) AND have no anchor referrer (predicted URLs are
+ *    synthesised from pagination patterns, never anchored from a
+ *    rendered page) — both halves of the OR are therefore false and
+ *    the leak is excluded.
+ *
+ *    The `source != 'crawled'` clause specifically saves the
+ *    `--inventory` × `--retry-failed` interaction: an inventory-seed
+ *    URL came from the operator's URL list (no anchor referrer) and
+ *    `resetFailedPages` puts it back at `scraped = 0`. Without this
+ *    clause those legitimate retries would be dropped on resume.
+ *
+ * The defensive shape is on purpose: the data source can drift into
+ * anomalous states under interruption, but the reader must never throw
+ * or feed garbage back into the dealer. A real in-scope URL that was
+ * truly interrupted mid-crawl will always have at least one anchor
+ * referrer (otherwise the dealer would not have queued it), so the
+ * strict filter loses no legitimate pending work.
+ *
+ * Seeds passed directly to `Crawler.start()` are NOT in the strict
+ * pending set when they were never picked by the dealer — they have no
+ * DB row at all in that case (`linkList.add` is purely in-memory until
+ * `setPage` runs). A Ctrl-C between dealer pick and `setPage` likewise
+ * leaves no row to recover. Recovery of un-picked seeds is the
+ * responsibility of the caller (e.g. re-running `--inventory ./list.txt`
+ * with the same URL list).
+ *
+ * The query uses an explicit `ci` alias on the `content_items` table so the
+ * correlated `EXISTS` subquery can join via `whereRaw('anchor_edges.href_page_id
+ * = ci.id')`. A future refactor that renames the alias must update both
+ * sites — the raw string in the subquery cannot be grep-resolved
+ * automatically. Read-only / stub viewer connections never call this
+ * method (they do not need to know about pending state), so the EXISTS
+ * shape is safe to use without the `migrate*` guards that other writer
+ * methods carry.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An object with `scraped` (completed URLs) and `pending` (the
+ *   strict set of in-scope, anchor-referenced, unfinished URLs).
+ */
+export declare function getCrawlingState(knex: Knex): Promise<{
+    scraped: string[];
+    pending: string[];
+}>;

package/lib/archive/db-ops/pages/read/get-crawling-state.js

@@ -0,0 +1,98 @@
+/**
+ * Retrieves the current crawling state by listing scraped and pending URLs.
+ *
+ * `scraped` is straightforward: every page row whose `scraped` flag is `1`
+ * — that is, every URL the crawl reached a terminal state on, including
+ * setSkippedPage / setExternalPage / outright setPage success or failure.
+ *
+ * `pending` is intentionally STRICT — not "every `scraped = 0` row".
+ * Three filters apply:
+ *
+ * 1. `scraped = 0` — work still incomplete.
+ * 2. `is_external = 0` — only in-scope work. External URLs go through a
+ *    HEAD-only path that always lands on `scraped = 1` (either setPage or
+ *    setExternalPage). A row with `is_external = 1 AND scraped = 0` is
+ *    therefore a data anomaly, and resume / inventory / append have no
+ *    business retrying it on the next session.
+ * 3. `EXISTS (anchor_edges with href_page_id = content_items.id) OR source
+ *    != 'crawled'` — the row was either discovered as an anchor destination
+ *    during a previous scrape OR was explicitly tagged with a non-default
+ *    source label (`'inventory-seed'`, `'inventory-discovered'`, …). Both
+ *    halves of the OR represent "deliberately enqueued, expected to be
+ *    processed", which is exactly what `resume` should pick up.
+ *
+ *    The orphan filter targets the **predicted-discard leak** in
+ *    `crawler.ts` where `shouldDiscardPredicted` returns true but no
+ *    `emit('skip')` follows. Such placeholders are inserted with the
+ *    DB DEFAULT `source = 'crawled'` (no caller explicitly labels
+ *    them) AND have no anchor referrer (predicted URLs are
+ *    synthesised from pagination patterns, never anchored from a
+ *    rendered page) — both halves of the OR are therefore false and
+ *    the leak is excluded.
+ *
+ *    The `source != 'crawled'` clause specifically saves the
+ *    `--inventory` × `--retry-failed` interaction: an inventory-seed
+ *    URL came from the operator's URL list (no anchor referrer) and
+ *    `resetFailedPages` puts it back at `scraped = 0`. Without this
+ *    clause those legitimate retries would be dropped on resume.
+ *
+ * The defensive shape is on purpose: the data source can drift into
+ * anomalous states under interruption, but the reader must never throw
+ * or feed garbage back into the dealer. A real in-scope URL that was
+ * truly interrupted mid-crawl will always have at least one anchor
+ * referrer (otherwise the dealer would not have queued it), so the
+ * strict filter loses no legitimate pending work.
+ *
+ * Seeds passed directly to `Crawler.start()` are NOT in the strict
+ * pending set when they were never picked by the dealer — they have no
+ * DB row at all in that case (`linkList.add` is purely in-memory until
+ * `setPage` runs). A Ctrl-C between dealer pick and `setPage` likewise
+ * leaves no row to recover. Recovery of un-picked seeds is the
+ * responsibility of the caller (e.g. re-running `--inventory ./list.txt`
+ * with the same URL list).
+ *
+ * The query uses an explicit `ci` alias on the `content_items` table so the
+ * correlated `EXISTS` subquery can join via `whereRaw('anchor_edges.href_page_id
+ * = ci.id')`. A future refactor that renames the alias must update both
+ * sites — the raw string in the subquery cannot be grep-resolved
+ * automatically. Read-only / stub viewer connections never call this
+ * method (they do not need to know about pending state), so the EXISTS
+ * shape is safe to use without the `migrate*` guards that other writer
+ * methods carry.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An object with `scraped` (completed URLs) and `pending` (the
+ *   strict set of in-scope, anchor-referenced, unfinished URLs).
+ */
+export async function getCrawlingState(knex) {
+    const ex = (r) => r.url;
+    const $scraped = await knex('content_items')
+        .join('url_refs', 'url_refs.id', 'content_items.url_id')
+        .select('url_refs.url as url')
+        .where('content_items.scraped', 1);
+    const scraped = $scraped.map(ex);
+    const $pending = await knex
+        .select('ur.url as url')
+        .from({ ci: 'content_items' })
+        .join({ ur: 'url_refs' }, 'ur.id', 'ci.url_id')
+        .where('ci.scraped', 0)
+        .where('ci.is_external', 0)
+        .where((qb) => {
+        // "Anchored OR explicitly labelled". Either side is evidence
+        // that the row was deliberately enqueued for processing —
+        // only the predicted-discard leak (DEFAULT 'crawled' + no
+        // anchor) fails both halves. The `whereExists` callback
+        // uses `select('*')` since the column list is irrelevant
+        // inside an EXISTS check; calling through `client.raw(...)`
+        // would reach a private builder field.
+        qb.whereExists(function () {
+            this.select('*')
+                .from('anchor_edges')
+                .whereRaw('anchor_edges.href_page_id = ci.id');
+        }).orWhereNot('ci.source', 'crawled');
+    });
+    const pending = $pending.map(ex);
+    return {
+        scraped,
+        pending,
+    };
+}

package/lib/archive/db-ops/pages/read/get-existing-page-urls.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Return the subset of `urls` that already exist as `content_items` rows.
+ * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+ * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+ * inventory list contains tens of thousands of URLs.
+ *
+ * Read-only — no transaction, no lock contention with the crawler write
+ * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+ * crawl is started).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+ * @returns URLs found among `content_items`. Order is not preserved.
+ */
+export declare function getExistingPageUrls(knex: Knex, urls: readonly string[]): Promise<string[]>;

package/lib/archive/db-ops/pages/read/get-existing-page-urls.js

@@ -0,0 +1,30 @@
+import { eachSplitted } from '../../../../utils/array/each-splitted.js';
+/**
+ * Return the subset of `urls` that already exist as `content_items` rows.
+ * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+ * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+ * inventory list contains tens of thousands of URLs.
+ *
+ * Read-only — no transaction, no lock contention with the crawler write
+ * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+ * crawl is started).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+ * @returns URLs found among `content_items`. Order is not preserved.
+ */
+export async function getExistingPageUrls(knex, urls) {
+    if (urls.length === 0) {
+        return [];
+    }
+    const found = [];
+    await eachSplitted([...urls], 500, async (chunk) => {
+        const rows = await knex('content_items')
+            .join('url_refs', 'url_refs.id', 'content_items.url_id')
+            .select('url_refs.url as url')
+            .whereIn('url_refs.url', chunk);
+        for (const row of rows) {
+            found.push(row.url);
+        }
+    });
+    return found;
+}

package/lib/archive/db-ops/pages/read/get-page-count.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Counts the total number of pages in the database.
+ *
+ * Pages live in `content_items` (resources are a physically separate
+ * `resource_items` table), so a bare count needs no type discriminator
+ * and no join.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The total page count.
+ * @throws {Error} If the count query fails.
+ */
+export declare function getPageCount(knex: Knex): Promise<number>;

package/lib/archive/db-ops/pages/read/get-page-count.js

@@ -0,0 +1,21 @@
+import { dbLog } from '../../../debug.js';
+/**
+ * Counts the total number of pages in the database.
+ *
+ * Pages live in `content_items` (resources are a physically separate
+ * `resource_items` table), so a bare count needs no type discriminator
+ * and no join.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The total page count.
+ * @throws {Error} If the count query fails.
+ */
+export async function getPageCount(knex) {
+    const selected = await knex.count('id').from('content_items');
+    if (!selected[0]) {
+        throw new Error('No count');
+    }
+    // @ts-expect-error
+    const count = selected[0]['count(`id`)'];
+    dbLog('Number of pages: %d', count);
+    return count;
+}

package/lib/archive/db-ops/pages/read/get-page-source-by-url.d.ts

@@ -0,0 +1,24 @@
+import type { PageSource } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Look up the `source` column of a single page by its URL key. Used by
+ * the orchestrator's `PageSourceLookup` injection so the Crawler can
+ * resolve a parent page's lineage on `--resume` / `--retry-failed`
+ * sessions, where the in-memory `inventoryMode` is no longer
+ * available but the DB still remembers what label was last persisted.
+ *
+ * Returns `undefined` when the URL has no `content_items` row (e.g. a
+ * brand-new URL that has not been seen yet) so the caller can fall
+ * through to its default behaviour without distinguishing "row absent"
+ * from "row present with NULL source" — the schema's
+ * `NOT NULL DEFAULT 'crawled'` makes a NULL value impossible in practice.
+ *
+ * Read-only — no transaction. URL text lives in `url_refs` (a UNIQUE
+ * column), so the lookup is a single indexed seek plus one PK join to
+ * `content_items`. The Crawler calls this at most once per page render,
+ * NOT per sub-resource, so the N+1 risk does not apply.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - URL key in `url.withoutHashAndAuth` form.
+ * @returns The recorded `source`, or `undefined` when no row exists.
+ */
+export declare function getPageSourceByUrl(knex: Knex, url: string): Promise<PageSource | undefined>;

package/lib/archive/db-ops/pages/read/get-page-source-by-url.js

@@ -0,0 +1,28 @@
+/**
+ * Look up the `source` column of a single page by its URL key. Used by
+ * the orchestrator's `PageSourceLookup` injection so the Crawler can
+ * resolve a parent page's lineage on `--resume` / `--retry-failed`
+ * sessions, where the in-memory `inventoryMode` is no longer
+ * available but the DB still remembers what label was last persisted.
+ *
+ * Returns `undefined` when the URL has no `content_items` row (e.g. a
+ * brand-new URL that has not been seen yet) so the caller can fall
+ * through to its default behaviour without distinguishing "row absent"
+ * from "row present with NULL source" — the schema's
+ * `NOT NULL DEFAULT 'crawled'` makes a NULL value impossible in practice.
+ *
+ * Read-only — no transaction. URL text lives in `url_refs` (a UNIQUE
+ * column), so the lookup is a single indexed seek plus one PK join to
+ * `content_items`. The Crawler calls this at most once per page render,
+ * NOT per sub-resource, so the N+1 risk does not apply.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - URL key in `url.withoutHashAndAuth` form.
+ * @returns The recorded `source`, or `undefined` when no row exists.
+ */
+export async function getPageSourceByUrl(knex, url) {
+    const [row] = await knex('content_items')
+        .join('url_refs', 'url_refs.id', 'content_items.url_id')
+        .select('content_items.source as source')
+        .where('url_refs.url', url);
+    return row?.source;
+}

package/lib/archive/db-ops/pages/read/get-pages-with-rels.d.ts

@@ -0,0 +1,38 @@
+import type { DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves pages along with their related redirect, anchor, and referrer
+ * data. Results are ordered by the natural URL sort order (materialised
+ * through {@link setUrlOrder} into `content_items.crawl_order` before the
+ * join queries run). Only non-redirected pages are returned in the
+ * top-level `pages` array; redirect sources are surfaced only through the
+ * `redirects` array so downstream reports (Sheets export) can attribute
+ * every source without double-counting the destination.
+ *
+ * **Redirect resolution stays a read-time join.** `anchor_edges.href_page_id`
+ * (like the legacy `anchors.hrefId` it replaces) is resolved by URL only at
+ * write time (`resolveContentItemId`), with no redirect-chain following —
+ * an anchor's target may itself become a redirect source discovered later
+ * in the crawl. `redirectTable()` joins the single `redirect_dest_id` hop
+ * at read time so `anchors[].url` reflects the final destination while
+ * `anchors[].href` keeps the immediate (pre-redirect) target URL; one hop
+ * suffices only because `linkRedirectSources` pre-flattens every hop's
+ * `redirect_dest_id` to the final destination at write time.
+ *
+ * **Deduped anchor identity.** `anchor_edges` collapses repeated
+ * `(page_id, href_page_id)` instances into one row with a `count`; `hash` /
+ * `textContent` below come from `first_hash` / `first_text_id` (the first
+ * observed instance), not every instance — an intentional behavior change
+ * from the legacy `anchors` table's one-row-per-instance model.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param offset - The number of rows to skip.
+ * @param limit - The maximum number of pages to return.
+ * @returns An object containing `pages`, `redirects`, `anchors`, and
+ *   `referrers` arrays.
+ */
+export declare function getPagesWithRels(knex: Knex, offset: number, limit: number): Promise<{
+    pages: DB_Page[];
+    redirects: DB_Redirect[];
+    anchors: DB_Anchor[];
+    referrers: DB_Referrer[];
+}>;

package/lib/archive/db-ops/pages/read/get-pages-with-rels.js

@@ -0,0 +1,107 @@
+import { dbLog } from '../../../debug.js';
+import { limitedPageIds } from '../../../limited-page-ids.js';
+import { redirectTable } from '../../../redirect-table.js';
+import { setUrlOrder } from '../order/set-url-order.js';
+import { buildPageQuery } from './build-page-query.js';
+import { reconstructPageRows } from './reconstruct-page-rows.js';
+/**
+ * Retrieves pages along with their related redirect, anchor, and referrer
+ * data. Results are ordered by the natural URL sort order (materialised
+ * through {@link setUrlOrder} into `content_items.crawl_order` before the
+ * join queries run). Only non-redirected pages are returned in the
+ * top-level `pages` array; redirect sources are surfaced only through the
+ * `redirects` array so downstream reports (Sheets export) can attribute
+ * every source without double-counting the destination.
+ *
+ * **Redirect resolution stays a read-time join.** `anchor_edges.href_page_id`
+ * (like the legacy `anchors.hrefId` it replaces) is resolved by URL only at
+ * write time (`resolveContentItemId`), with no redirect-chain following —
+ * an anchor's target may itself become a redirect source discovered later
+ * in the crawl. `redirectTable()` joins the single `redirect_dest_id` hop
+ * at read time so `anchors[].url` reflects the final destination while
+ * `anchors[].href` keeps the immediate (pre-redirect) target URL; one hop
+ * suffices only because `linkRedirectSources` pre-flattens every hop's
+ * `redirect_dest_id` to the final destination at write time.
+ *
+ * **Deduped anchor identity.** `anchor_edges` collapses repeated
+ * `(page_id, href_page_id)` instances into one row with a `count`; `hash` /
+ * `textContent` below come from `first_hash` / `first_text_id` (the first
+ * observed instance), not every instance — an intentional behavior change
+ * from the legacy `anchors` table's one-row-per-instance model.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param offset - The number of rows to skip.
+ * @param limit - The maximum number of pages to return.
+ * @returns An object containing `pages`, `redirects`, `anchors`, and
+ *   `referrers` arrays.
+ */
+export async function getPagesWithRels(knex, offset, limit) {
+    await setUrlOrder(knex);
+    dbLog('Get Pages');
+    const rawRows = await buildPageQuery(knex)
+        .whereNull('ci.redirect_dest_id')
+        .orderByRaw('`ci`.`crawl_order` ASC NULLS LAST')
+        .limit(limit)
+        .offset(offset);
+    const pages = await reconstructPageRows(knex, rawRows);
+    // When empty
+    if (pages.length === 0) {
+        return {
+            pages: [],
+            redirects: [],
+            referrers: [],
+            anchors: [],
+        };
+    }
+    dbLog('Get Pages: Redirects');
+    const redirects = await knex
+        .with('limitedPages', limitedPageIds(limit, offset))
+        .with('redirect', redirectTable(false))
+        .select('id as pageId', 'from', 'fromId')
+        .from('redirect')
+        // Filter
+        .join('limitedPages', 'redirect.toId', '=', 'limitedPages.id')
+        // Sort
+        .orderBy('id', 'asc');
+    dbLog('Get Pages: Anchors');
+    const anchors = await knex
+        .with('limitedPages', limitedPageIds(limit, offset))
+        .with('redirect', redirectTable())
+        .select('limitedPages.id as pageId', 'href_url.url as url', 'redirect.from as href', 'href.is_external as isExternal', 'href_title.text as title', 'href.status as status', 'href.status_text as statusText', 'href_ctr.raw as contentType', 'anchor_edges.first_hash as hash', 'anchor_text.text as textContent')
+        .from('anchor_edges')
+        // Filters
+        .join('limitedPages', 'anchor_edges.page_id', '=', 'limitedPages.id')
+        // Resolves redirect
+        .join('redirect', 'anchor_edges.href_page_id', '=', 'redirect.fromId')
+        // Target
+        .join('content_items as href', 'redirect.toId', '=', 'href.id')
+        .join('url_refs as href_url', 'href.url_id', '=', 'href_url.id')
+        .leftJoin('content_type_refs as href_ctr', 'href.content_type_id', '=', 'href_ctr.id')
+        .leftJoin('page_meta as href_pm', 'href.id', '=', 'href_pm.page_id')
+        .leftJoin('text_refs as href_title', 'href_pm.title_text_id', '=', 'href_title.id')
+        .leftJoin('text_refs as anchor_text', 'anchor_edges.first_text_id', '=', 'anchor_text.id')
+        // Sort
+        .orderBy('anchor_edges.id', 'asc');
+    dbLog('Get Pages: Referrers');
+    const referrers = await knex
+        .with('limitedPages', limitedPageIds(limit, offset))
+        .with('redirect', redirectTable())
+        .select('redirect.toId as pageId', 'referrer_url.url as url', 'redirect.from as through', 'redirect.fromId as throughId', 'anchor_edges.first_hash as hash', 'anchor_text.text as textContent')
+        .from('anchor_edges')
+        // Resolves redirect
+        .join('redirect', 'anchor_edges.href_page_id', '=', 'redirect.fromId')
+        // Referrer
+        .join('content_items as referrer', 'anchor_edges.page_id', '=', 'referrer.id')
+        .join('url_refs as referrer_url', 'referrer.url_id', '=', 'referrer_url.id')
+        // Filters
+        .join('limitedPages', 'redirect.toId', '=', 'limitedPages.id')
+        .leftJoin('text_refs as anchor_text', 'anchor_edges.first_text_id', '=', 'anchor_text.id')
+        // Sort
+        .orderBy('anchor_edges.id', 'asc');
+    dbLog('Get Pages: Done');
+    return {
+        pages,
+        redirects,
+        anchors,
+        referrers,
+    };
+}