@nitpicker/archive 0.0.0 → 0.25.0

package/lib/db-ops/pages/read/get-crawling-state.js

@@ -0,0 +1,161 @@
+import { computeShapeKey } from '../../../url-pattern/compute-shape-key.js';
+import { listDedupeCapShapeKeys } from '../../dedupe-cap/list-dedupe-cap-shape-keys.js';
+/**
+ * Retrieves the current crawling state by listing scraped and pending URLs.
+ *
+ * `scraped` is straightforward: every page row whose `scraped` flag is `1`
+ * — that is, every URL the crawl reached a terminal state on, including
+ * setSkippedPage / setExternalPage / outright setPage success or failure.
+ *
+ * `pending` is intentionally STRICT — not "every `scraped = 0` row".
+ * Four filters apply:
+ *
+ * 1. `scraped = 0` — work still incomplete.
+ * 2. `is_external = 0` — only in-scope work. External URLs go through a
+ *    HEAD-only path that always lands on `scraped = 1` (either setPage or
+ *    setExternalPage). A row with `is_external = 1 AND scraped = 0` is
+ *    therefore a data anomaly, and resume / inventory / append have no
+ *    business retrying it on the next session.
+ * 3. `EXISTS (anchor_edges with href_page_id = content_items.id) OR source
+ *    != 'crawled'` — the row was either discovered as an anchor destination
+ *    during a previous scrape OR was explicitly tagged with a non-default
+ *    source label (`'inventory-seed'`, `'inventory-discovered'`, …). Both
+ *    halves of the OR represent "deliberately enqueued, expected to be
+ *    processed", which is exactly what `resume` should pick up.
+ *
+ *    The orphan filter targets the **predicted-discard leak** in
+ *    `@nitpicker/crawler`'s `crawler/crawler.ts` where `shouldDiscardPredicted` returns true but no
+ *    `emit('skip')` follows. Such placeholders are inserted with the
+ *    DB DEFAULT `source = 'crawled'` (no caller explicitly labels
+ *    them) AND have no anchor referrer (predicted URLs are
+ *    synthesised from pagination patterns, never anchored from a
+ *    rendered page) — both halves of the OR are therefore false and
+ *    the leak is excluded.
+ *
+ *    The `source != 'crawled'` clause specifically saves the
+ *    `--inventory` × `--retry-failed` interaction: an inventory-seed
+ *    URL came from the operator's URL list (no anchor referrer) and
+ *    `resetFailedPages` puts it back at `scraped = 0`. Without this
+ *    clause those legitimate retries would be dropped on resume.
+ * 4. **Confirmed same-cluster trap exclusion** (issue #350): a row whose
+ *    URL shape (`computeShapeKey`) matches a `dedupe_cap_events.shape_key`
+ *    already recorded in this archive is dropped, UNLESS its `source` is
+ *    `'inventory-seed'` (an operator-listed URL, not an anchor-discovered
+ *    one — the same "explicit URL survives cap" carve-out `resetPagesByUrls`
+ *    gives `--recrawl`). NOT the same as `resetFailedPages`'s own
+ *    same-shape exclusion, which drops a matching candidate regardless of
+ *    `source` — a `scraped=1, status=-1` inventory-seed row of a
+ *    confirmed-capped shape is therefore excluded there too and never
+ *    resets to `scraped=0` for `--retry-failed` to pick back up. That
+ *    asymmetry pre-dates this filter and is unrelated to it (out of scope
+ *    here): this filter only decides what `getCrawlingState` reports as
+ *    `pending`, never whether a `scraped=1` row gets reset in the first
+ *    place.
+ *
+ *    Why this is needed: `--dedupe-cap`'s enqueue gate (`@nitpicker/crawler`'s `crawler/crawler.ts`
+ *    `addUrl` closure) only stops a capped shape's anchor from being
+ *    ADDED TO THE QUEUE in memory — it does not stop `replaceAnchorEdges`
+ *    (`update-page.ts`) from writing a `scraped = 0` row for that same
+ *    anchor (post-hoc marking, `content_items.dedupe_cap_event_id`,
+ *    deliberately needs that row to exist). Before this filter, such a
+ *    row still satisfied filter 3 (it has an anchor referrer) and entered
+ *    `pending`, so `CrawlerOrchestrator#crawlUntilPendingClears`'s
+ *    auto-retry loop (and any `--resume`/`--retry-failed`/`--append`) would
+ *    requeue it — `Crawler#resume`'s `LinkList#resume` has no cap gate of
+ *    its own — actually fetch the trap page, and its anchors would create
+ *    a fresh batch of same-shape `scraped = 0` rows. Pending never shrank,
+ *    so the loop ended in `PendingUrlsRemainError('no-progress')` even
+ *    though nothing was actually wrong with the target site.
+ *
+ *    Why not skip the row instead of merely omitting it from `pending`:
+ *    this function is read-only — leaving `is_skipped`/`skip_reason`
+ *    unset keeps `dedupe_cap_event_id` backfill (`viewer-build`) as the
+ *    single place that judges a row "capped", matching the archive's
+ *    existing dedupe-cap post-hoc-marking pattern rather than adding a
+ *    second one here.
+ *
+ *    A row whose shape cannot be computed (`computeShapeKey` returns
+ *    `null`) stays in `pending` — no signal either way, same
+ *    err-on-the-side-of-retrying choice `resetFailedPages` makes.
+ *
+ * The defensive shape is on purpose: the data source can drift into
+ * anomalous states under interruption, but the reader must never throw
+ * or feed garbage back into the dealer. A real in-scope URL that was
+ * truly interrupted mid-crawl will always have at least one anchor
+ * referrer (otherwise the dealer would not have queued it), so the
+ * strict filter loses no legitimate pending work.
+ *
+ * Seeds passed directly to `Crawler.start()` are NOT in the strict
+ * pending set when they were never picked by the dealer — they have no
+ * DB row at all in that case (`linkList.add` is purely in-memory until
+ * `setPage` runs). A Ctrl-C between dealer pick and `setPage` likewise
+ * leaves no row to recover. Recovery of un-picked seeds is the
+ * responsibility of the caller (e.g. re-running `--inventory ./list.txt`
+ * with the same URL list).
+ *
+ * The query uses an explicit `ci` alias on the `content_items` table so the
+ * correlated `EXISTS` subquery can join via `whereRaw('anchor_edges.href_page_id
+ * = ci.id')`. A future refactor that renames the alias must update both
+ * sites — the raw string in the subquery cannot be grep-resolved
+ * automatically. Read-only / stub viewer connections never call this
+ * method (they do not need to know about pending state), so the EXISTS
+ * shape is safe to use without the `migrate*` guards that other writer
+ * methods carry.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An object with `scraped` (completed URLs), `pending` (the
+ *   strict set of in-scope, anchor-referenced, unfinished, non-capped
+ *   URLs), and `pendingMetadataOnly` (the subset of `pending` whose
+ *   `content_items.is_metadata_only` was persisted as `1` — see
+ *   `replaceAnchorEdges`/`resolveContentItemId`). Callers that resume a
+ *   crawl (`Crawler#resume` → `LinkList#resume`) pass this subset through
+ *   so a metadata-only anchor discovered before an interruption is not
+ *   silently promoted to a full-scrape target on resume (#369).
+ */
+export async function getCrawlingState(knex) {
+    const ex = (r) => r.url;
+    const $scraped = await knex('content_items')
+        .join('url_refs', 'url_refs.id', 'content_items.url_id')
+        .select('url_refs.url as url')
+        .where('content_items.scraped', 1);
+    const scraped = $scraped.map(ex);
+    const $pending = await knex
+        .select('ur.url as url', 'ci.is_metadata_only as isMetadataOnly', 'ci.source as source')
+        .from({ ci: 'content_items' })
+        .join({ ur: 'url_refs' }, 'ur.id', 'ci.url_id')
+        .where('ci.scraped', 0)
+        .where('ci.is_external', 0)
+        .where((qb) => {
+        // "Anchored OR explicitly labelled". Either side is evidence
+        // that the row was deliberately enqueued for processing —
+        // only the predicted-discard leak (DEFAULT 'crawled' + no
+        // anchor) fails both halves. The `whereExists` callback
+        // uses `select('*')` since the column list is irrelevant
+        // inside an EXISTS check; calling through `client.raw(...)`
+        // would reach a private builder field.
+        qb.whereExists(function () {
+            this.select('*')
+                .from('anchor_edges')
+                .whereRaw('anchor_edges.href_page_id = ci.id');
+        }).orWhereNot('ci.source', 'crawled');
+    });
+    // Confirmed same-cluster trap exclusion (filter 4, see this function's
+    // JSDoc) — an archive with no recorded cap events behaves exactly as
+    // before this filter existed (`cappedShapeKeys.size === 0` short-circuits
+    // to a no-op, same guard `resetFailedPages` uses).
+    const cappedShapeKeys = new Set(await listDedupeCapShapeKeys(knex));
+    const $notCapped = cappedShapeKeys.size === 0
+        ? $pending
+        : $pending.filter((row) => {
+            if (row.source === 'inventory-seed')
+                return true;
+            const shapeKey = computeShapeKey(row.url);
+            return shapeKey === null || !cappedShapeKeys.has(shapeKey);
+        });
+    const pending = $notCapped.map(ex);
+    const pendingMetadataOnly = $notCapped.filter((r) => r.isMetadataOnly === 1).map(ex);
+    return {
+        scraped,
+        pending,
+        pendingMetadataOnly,
+    };
+}

package/lib/db-ops/pages/read/get-existing-page-urls.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Return the subset of `urls` that already exist as `content_items` rows.
+ * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+ * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+ * inventory list contains tens of thousands of URLs.
+ *
+ * Read-only — no transaction, no lock contention with the crawler write
+ * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+ * crawl is started).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+ * @returns URLs found among `content_items`. Order is not preserved.
+ */
+export declare function getExistingPageUrls(knex: Knex, urls: readonly string[]): Promise<string[]>;

package/lib/db-ops/pages/read/get-existing-page-urls.js

@@ -0,0 +1,30 @@
+import { eachSplitted } from '../../../utils/array/each-splitted.js';
+/**
+ * Return the subset of `urls` that already exist as `content_items` rows.
+ * Chunked into batches so SQLite's `IN (?, ?, …)` parameter limit
+ * (`SQLITE_MAX_VARIABLE_NUMBER`, default 999) cannot be hit even when the
+ * inventory list contains tens of thousands of URLs.
+ *
+ * Read-only — no transaction, no lock contention with the crawler write
+ * pipeline (callers run this BEFORE the `<archive>.bak` is taken and the
+ * crawl is started).
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param urls - URL strings to probe (already in `withoutHashAndAuth` form).
+ * @returns URLs found among `content_items`. Order is not preserved.
+ */
+export async function getExistingPageUrls(knex, urls) {
+    if (urls.length === 0) {
+        return [];
+    }
+    const found = [];
+    await eachSplitted([...urls], 500, async (chunk) => {
+        const rows = await knex('content_items')
+            .join('url_refs', 'url_refs.id', 'content_items.url_id')
+            .select('url_refs.url as url')
+            .whereIn('url_refs.url', chunk);
+        for (const row of rows) {
+            found.push(row.url);
+        }
+    });
+    return found;
+}

package/lib/db-ops/pages/read/get-page-count.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Counts the total number of pages in the database.
+ *
+ * Pages live in `content_items` (resources are a physically separate
+ * `resource_items` table), so a bare count needs no type discriminator
+ * and no join.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The total page count.
+ * @throws {Error} If the count query fails.
+ */
+export declare function getPageCount(knex: Knex): Promise<number>;

package/lib/db-ops/pages/read/get-page-count.js

@@ -0,0 +1,21 @@
+import { dbLog } from '../../../debug.js';
+/**
+ * Counts the total number of pages in the database.
+ *
+ * Pages live in `content_items` (resources are a physically separate
+ * `resource_items` table), so a bare count needs no type discriminator
+ * and no join.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The total page count.
+ * @throws {Error} If the count query fails.
+ */
+export async function getPageCount(knex) {
+    const selected = await knex.count('id').from('content_items');
+    if (!selected[0]) {
+        throw new Error('No count');
+    }
+    // @ts-expect-error
+    const count = selected[0]['count(`id`)'];
+    dbLog('Number of pages: %d', count);
+    return count;
+}

package/lib/db-ops/pages/read/get-page-source-by-url.d.ts

@@ -0,0 +1,24 @@
+import type { PageSource } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Look up the `source` column of a single page by its URL key. Used by
+ * the orchestrator's `PageSourceLookup` injection so the Crawler can
+ * resolve a parent page's lineage on `--resume` / `--retry-failed`
+ * sessions, where the in-memory `inventoryMode` is no longer
+ * available but the DB still remembers what label was last persisted.
+ *
+ * Returns `undefined` when the URL has no `content_items` row (e.g. a
+ * brand-new URL that has not been seen yet) so the caller can fall
+ * through to its default behaviour without distinguishing "row absent"
+ * from "row present with NULL source" — the schema's
+ * `NOT NULL DEFAULT 'crawled'` makes a NULL value impossible in practice.
+ *
+ * Read-only — no transaction. URL text lives in `url_refs` (a UNIQUE
+ * column), so the lookup is a single indexed seek plus one PK join to
+ * `content_items`. The Crawler calls this at most once per page render,
+ * NOT per sub-resource, so the N+1 risk does not apply.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - URL key in `url.withoutHashAndAuth` form.
+ * @returns The recorded `source`, or `undefined` when no row exists.
+ */
+export declare function getPageSourceByUrl(knex: Knex, url: string): Promise<PageSource | undefined>;

package/lib/db-ops/pages/read/get-page-source-by-url.js

@@ -0,0 +1,28 @@
+/**
+ * Look up the `source` column of a single page by its URL key. Used by
+ * the orchestrator's `PageSourceLookup` injection so the Crawler can
+ * resolve a parent page's lineage on `--resume` / `--retry-failed`
+ * sessions, where the in-memory `inventoryMode` is no longer
+ * available but the DB still remembers what label was last persisted.
+ *
+ * Returns `undefined` when the URL has no `content_items` row (e.g. a
+ * brand-new URL that has not been seen yet) so the caller can fall
+ * through to its default behaviour without distinguishing "row absent"
+ * from "row present with NULL source" — the schema's
+ * `NOT NULL DEFAULT 'crawled'` makes a NULL value impossible in practice.
+ *
+ * Read-only — no transaction. URL text lives in `url_refs` (a UNIQUE
+ * column), so the lookup is a single indexed seek plus one PK join to
+ * `content_items`. The Crawler calls this at most once per page render,
+ * NOT per sub-resource, so the N+1 risk does not apply.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param url - URL key in `url.withoutHashAndAuth` form.
+ * @returns The recorded `source`, or `undefined` when no row exists.
+ */
+export async function getPageSourceByUrl(knex, url) {
+    const [row] = await knex('content_items')
+        .join('url_refs', 'url_refs.id', 'content_items.url_id')
+        .select('content_items.source as source')
+        .where('url_refs.url', url);
+    return row?.source;
+}

package/lib/db-ops/pages/read/get-pages-with-rels.d.ts

@@ -0,0 +1,38 @@
+import type { DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves pages along with their related redirect, anchor, and referrer
+ * data. Results are ordered by the natural URL sort order (materialised
+ * through {@link setUrlOrder} into `content_items.crawl_order` before the
+ * join queries run). Only non-redirected pages are returned in the
+ * top-level `pages` array; redirect sources are surfaced only through the
+ * `redirects` array so downstream reports (Sheets export) can attribute
+ * every source without double-counting the destination.
+ *
+ * **Redirect resolution stays a read-time join.** `anchor_edges.href_page_id`
+ * (like the legacy `anchors.hrefId` it replaces) is resolved by URL only at
+ * write time (`resolveContentItemId`), with no redirect-chain following —
+ * an anchor's target may itself become a redirect source discovered later
+ * in the crawl. `redirectTable()` joins the single `redirect_dest_id` hop
+ * at read time so `anchors[].url` reflects the final destination while
+ * `anchors[].href` keeps the immediate (pre-redirect) target URL; one hop
+ * suffices only because `linkRedirectSources` pre-flattens every hop's
+ * `redirect_dest_id` to the final destination at write time.
+ *
+ * **Deduped anchor identity.** `anchor_edges` collapses repeated
+ * `(page_id, href_page_id)` instances into one row with a `count`; `hash` /
+ * `textContent` below come from `first_hash` / `first_text_id` (the first
+ * observed instance), not every instance — an intentional behavior change
+ * from the legacy `anchors` table's one-row-per-instance model.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param offset - The number of rows to skip.
+ * @param limit - The maximum number of pages to return.
+ * @returns An object containing `pages`, `redirects`, `anchors`, and
+ *   `referrers` arrays.
+ */
+export declare function getPagesWithRels(knex: Knex, offset: number, limit: number): Promise<{
+    pages: DB_Page[];
+    redirects: DB_Redirect[];
+    anchors: DB_Anchor[];
+    referrers: DB_Referrer[];
+}>;

package/lib/db-ops/pages/read/get-pages-with-rels.js

@@ -0,0 +1,107 @@
+import { dbLog } from '../../../debug.js';
+import { limitedPageIds } from '../../../limited-page-ids.js';
+import { redirectTable } from '../../../redirect-table.js';
+import { setUrlOrder } from '../order/set-url-order.js';
+import { buildPageQuery } from './build-page-query.js';
+import { reconstructPageRows } from './reconstruct-page-rows.js';
+/**
+ * Retrieves pages along with their related redirect, anchor, and referrer
+ * data. Results are ordered by the natural URL sort order (materialised
+ * through {@link setUrlOrder} into `content_items.crawl_order` before the
+ * join queries run). Only non-redirected pages are returned in the
+ * top-level `pages` array; redirect sources are surfaced only through the
+ * `redirects` array so downstream reports (Sheets export) can attribute
+ * every source without double-counting the destination.
+ *
+ * **Redirect resolution stays a read-time join.** `anchor_edges.href_page_id`
+ * (like the legacy `anchors.hrefId` it replaces) is resolved by URL only at
+ * write time (`resolveContentItemId`), with no redirect-chain following —
+ * an anchor's target may itself become a redirect source discovered later
+ * in the crawl. `redirectTable()` joins the single `redirect_dest_id` hop
+ * at read time so `anchors[].url` reflects the final destination while
+ * `anchors[].href` keeps the immediate (pre-redirect) target URL; one hop
+ * suffices only because `linkRedirectSources` pre-flattens every hop's
+ * `redirect_dest_id` to the final destination at write time.
+ *
+ * **Deduped anchor identity.** `anchor_edges` collapses repeated
+ * `(page_id, href_page_id)` instances into one row with a `count`; `hash` /
+ * `textContent` below come from `first_hash` / `first_text_id` (the first
+ * observed instance), not every instance — an intentional behavior change
+ * from the legacy `anchors` table's one-row-per-instance model.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param offset - The number of rows to skip.
+ * @param limit - The maximum number of pages to return.
+ * @returns An object containing `pages`, `redirects`, `anchors`, and
+ *   `referrers` arrays.
+ */
+export async function getPagesWithRels(knex, offset, limit) {
+    await setUrlOrder(knex);
+    dbLog('Get Pages');
+    const rawRows = await buildPageQuery(knex)
+        .whereNull('ci.redirect_dest_id')
+        .orderByRaw('`ci`.`crawl_order` ASC NULLS LAST')
+        .limit(limit)
+        .offset(offset);
+    const pages = await reconstructPageRows(knex, rawRows);
+    // When empty
+    if (pages.length === 0) {
+        return {
+            pages: [],
+            redirects: [],
+            referrers: [],
+            anchors: [],
+        };
+    }
+    dbLog('Get Pages: Redirects');
+    const redirects = await knex
+        .with('limitedPages', limitedPageIds(limit, offset))
+        .with('redirect', redirectTable(false))
+        .select('id as pageId', 'from', 'fromId')
+        .from('redirect')
+        // Filter
+        .join('limitedPages', 'redirect.toId', '=', 'limitedPages.id')
+        // Sort
+        .orderBy('id', 'asc');
+    dbLog('Get Pages: Anchors');
+    const anchors = await knex
+        .with('limitedPages', limitedPageIds(limit, offset))
+        .with('redirect', redirectTable())
+        .select('limitedPages.id as pageId', 'href_url.url as url', 'redirect.from as href', 'href.is_external as isExternal', 'href_title.text as title', 'href.status as status', 'href.status_text as statusText', 'href_ctr.raw as contentType', 'anchor_edges.first_hash as hash', 'anchor_text.text as textContent')
+        .from('anchor_edges')
+        // Filters
+        .join('limitedPages', 'anchor_edges.page_id', '=', 'limitedPages.id')
+        // Resolves redirect
+        .join('redirect', 'anchor_edges.href_page_id', '=', 'redirect.fromId')
+        // Target
+        .join('content_items as href', 'redirect.toId', '=', 'href.id')
+        .join('url_refs as href_url', 'href.url_id', '=', 'href_url.id')
+        .leftJoin('content_type_refs as href_ctr', 'href.content_type_id', '=', 'href_ctr.id')
+        .leftJoin('page_meta as href_pm', 'href.id', '=', 'href_pm.page_id')
+        .leftJoin('text_refs as href_title', 'href_pm.title_text_id', '=', 'href_title.id')
+        .leftJoin('text_refs as anchor_text', 'anchor_edges.first_text_id', '=', 'anchor_text.id')
+        // Sort
+        .orderBy('anchor_edges.id', 'asc');
+    dbLog('Get Pages: Referrers');
+    const referrers = await knex
+        .with('limitedPages', limitedPageIds(limit, offset))
+        .with('redirect', redirectTable())
+        .select('redirect.toId as pageId', 'referrer_url.url as url', 'redirect.from as through', 'redirect.fromId as throughId', 'anchor_edges.first_hash as hash', 'anchor_text.text as textContent')
+        .from('anchor_edges')
+        // Resolves redirect
+        .join('redirect', 'anchor_edges.href_page_id', '=', 'redirect.fromId')
+        // Referrer
+        .join('content_items as referrer', 'anchor_edges.page_id', '=', 'referrer.id')
+        .join('url_refs as referrer_url', 'referrer.url_id', '=', 'referrer_url.id')
+        // Filters
+        .join('limitedPages', 'redirect.toId', '=', 'limitedPages.id')
+        .leftJoin('text_refs as anchor_text', 'anchor_edges.first_text_id', '=', 'anchor_text.id')
+        // Sort
+        .orderBy('anchor_edges.id', 'asc');
+    dbLog('Get Pages: Done');
+    return {
+        pages,
+        redirects,
+        anchors,
+        referrers,
+    };
+}

package/lib/db-ops/pages/read/get-pages.d.ts

@@ -0,0 +1,11 @@
+import type { DB_Page, PageFilter } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves pages from the database with optional filtering, pagination via offset and limit.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param filter - An optional {@link PageFilter} to narrow results by content type and origin.
+ * @param offset - The number of rows to skip. Defaults to `0`.
+ * @param limit - The maximum number of rows to return. Defaults to `100000`.
+ * @returns An array of reconstructed {@link DB_Page} rows.
+ */
+export declare function getPages(knex: Knex, filter?: PageFilter, offset?: number, limit?: number): Promise<DB_Page[]>;

package/lib/db-ops/pages/read/get-pages.js

@@ -0,0 +1,51 @@
+import { buildPageQuery } from './build-page-query.js';
+import { reconstructPageRows } from './reconstruct-page-rows.js';
+/**
+ * Retrieves pages from the database with optional filtering, pagination via offset and limit.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param filter - An optional {@link PageFilter} to narrow results by content type and origin.
+ * @param offset - The number of rows to skip. Defaults to `0`.
+ * @param limit - The maximum number of rows to return. Defaults to `100000`.
+ * @returns An array of reconstructed {@link DB_Page} rows.
+ */
+export async function getPages(knex, filter, offset = 0, limit = 100_000) {
+    const q = buildPageQuery(knex);
+    switch (filter) {
+        case 'page': {
+            q.where('ctr.raw', 'text/html').andWhere('ci.is_target', 1);
+            break;
+        }
+        case 'page-included-no-target': {
+            q.where('ctr.raw', 'text/html');
+            break;
+        }
+        case 'external-page': {
+            q.where('ctr.raw', 'text/html').andWhere('ci.is_external', 1);
+            break;
+        }
+        case 'internal-page': {
+            q.where('ctr.raw', 'text/html').andWhere('ci.is_external', 0);
+            break;
+        }
+        case 'no-page': {
+            q.where((qb) => {
+                qb.whereNull('ctr.raw').orWhereNot('ctr.raw', 'text/html');
+            });
+            break;
+        }
+        case 'external-no-page': {
+            q.where((qb) => {
+                qb.whereNull('ctr.raw').orWhereNot('ctr.raw', 'text/html');
+            }).andWhere('ci.is_external', 1);
+            break;
+        }
+        case 'internal-no-page': {
+            q.where((qb) => {
+                qb.whereNull('ctr.raw').orWhereNot('ctr.raw', 'text/html');
+            }).andWhere('ci.is_external', 0);
+            break;
+        }
+    }
+    const rows = await q.limit(limit).offset(offset);
+    return reconstructPageRows(knex, rows);
+}

package/lib/db-ops/pages/read/get-scraped-html-page-count.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Counts pages that were scraped as crawl targets (full HTML render).
+ *
+ * Used by the crawler to seed its `pagesScraped` counter on resume so the
+ * progress display reflects all browser-rendered HTML pages across sessions,
+ * not just the current one.
+ *
+ * "HTML page" is guaranteed by `content_type_refs.raw = 'text/html'`, NOT by
+ * `is_target` alone: `is_target` means "in-scope crawl target" and is set
+ * for in-scope non-HTML resources too (e.g. a PDF reached via the HEAD
+ * pre-flight is `is_target = 1`). Counting those would over-report the HTML
+ * page total, so page-ness is asserted at the read layer here rather than by
+ * trusting `is_target`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The number of `text/html` rows with `is_target = 1` and `scraped = 1`.
+ */
+export declare function getScrapedHtmlPageCount(knex: Knex): Promise<number>;

package/lib/db-ops/pages/read/get-scraped-html-page-count.js

@@ -0,0 +1,25 @@
+/**
+ * Counts pages that were scraped as crawl targets (full HTML render).
+ *
+ * Used by the crawler to seed its `pagesScraped` counter on resume so the
+ * progress display reflects all browser-rendered HTML pages across sessions,
+ * not just the current one.
+ *
+ * "HTML page" is guaranteed by `content_type_refs.raw = 'text/html'`, NOT by
+ * `is_target` alone: `is_target` means "in-scope crawl target" and is set
+ * for in-scope non-HTML resources too (e.g. a PDF reached via the HEAD
+ * pre-flight is `is_target = 1`). Counting those would over-report the HTML
+ * page total, so page-ness is asserted at the read layer here rather than by
+ * trusting `is_target`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The number of `text/html` rows with `is_target = 1` and `scraped = 1`.
+ */
+export async function getScrapedHtmlPageCount(knex) {
+    const [row] = await knex('content_items')
+        .join('content_type_refs', 'content_type_refs.id', 'content_items.content_type_id')
+        .where('content_items.is_target', 1)
+        .andWhere('content_items.scraped', 1)
+        .andWhere('content_type_refs.raw', 'text/html')
+        .count('* as count');
+    return row ? Number(row.count) : 0;
+}

package/lib/db-ops/pages/read/reconstruct-page-rows.d.ts

@@ -0,0 +1,31 @@
+import type { DB_Page } from '../../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Raw row shape produced by {@link ../read/build-page-query.js} before
+ * `responseHeaders` / `meta_extras` reconstruction.
+ */
+interface RawPageRow extends Omit<DB_Page, 'responseHeaders' | 'meta_extras' | 'networkLogs'> {
+    /** `content_items.header_set_id`, or null when no headers were recorded. */
+    headerSetId: number | null;
+    /** `json_refs.json_text` for `meta_extras`, or null when absent. */
+    extras_body: Buffer | string | null;
+    /** `json_refs.codec` for `meta_extras` (`'zstd'` or `'none'`), or null when absent. */
+    extras_codec: 'zstd' | 'none' | null;
+}
+/**
+ * Reconstructs `responseHeaders` (JSON string) and `meta_extras` (JSON
+ * string) for a batch of raw page rows. Headers load through
+ * {@link ../../_shared/load-response-headers-by-set-ids.js} (chunked
+ * batch lookup, never N+1); `meta_extras` decodes through
+ * {@link ../../_shared/decode-json-ref.js}. `networkLogs` has no 0.13
+ * equivalent (it is a legacy-only field nothing ever wrote past the
+ * pre-0.13 write path either) and is always `null`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param rows - Raw rows from {@link ../read/build-page-query.js}.
+ * @returns Fully reconstructed `DB_Page` rows, in the same order as `rows`.
+ * @example
+ * const raw = await buildPageQuery(knex).where('ci.is_target', 1);
+ * const pages = await reconstructPageRows(knex, raw);
+ */
+export declare function reconstructPageRows(knex: Knex, rows: readonly RawPageRow[]): Promise<DB_Page[]>;
+export {};

package/lib/db-ops/pages/read/reconstruct-page-rows.js

@@ -0,0 +1,32 @@
+import { decodeJsonRef } from '../../_shared/decode-json-ref.js';
+import { loadResponseHeadersBySetIds } from '../../_shared/load-response-headers-by-set-ids.js';
+/**
+ * Reconstructs `responseHeaders` (JSON string) and `meta_extras` (JSON
+ * string) for a batch of raw page rows. Headers load through
+ * {@link ../../_shared/load-response-headers-by-set-ids.js} (chunked
+ * batch lookup, never N+1); `meta_extras` decodes through
+ * {@link ../../_shared/decode-json-ref.js}. `networkLogs` has no 0.13
+ * equivalent (it is a legacy-only field nothing ever wrote past the
+ * pre-0.13 write path either) and is always `null`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param rows - Raw rows from {@link ../read/build-page-query.js}.
+ * @returns Fully reconstructed `DB_Page` rows, in the same order as `rows`.
+ * @example
+ * const raw = await buildPageQuery(knex).where('ci.is_target', 1);
+ * const pages = await reconstructPageRows(knex, raw);
+ */
+export async function reconstructPageRows(knex, rows) {
+    const headerSetIds = [
+        ...new Set(rows.map((r) => r.headerSetId).filter((id) => id != null)),
+    ];
+    const headersBySetId = await loadResponseHeadersBySetIds(knex, headerSetIds);
+    return rows.map((row) => {
+        const { headerSetId, extras_body, extras_codec, ...rest } = row;
+        return {
+            ...rest,
+            responseHeaders: JSON.stringify(headerSetId == null ? {} : (headersBySetId.get(headerSetId) ?? {})),
+            meta_extras: decodeJsonRef(extras_body, extras_codec),
+            networkLogs: null,
+        };
+    });
+}

package/lib/db-ops/pages/reset/clear-page-derived-rows.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Deletes every Scoped-Replace derived row for the given page ids, across the
+ * 17 tables a page's re-scrape needs to start clean.
+ *
+ * Shared by {@link resetFailedPages} and {@link repromoteExternalPages} —
+ * both un-scrape a page back to `scraped = 0` and need the same "delete this
+ * page's derived rows so the re-scrape can re-insert fresh data without
+ * duplicates" sweep. `page_errors` is deliberately NOT included here:
+ * `repromoteExternalPages` never cleared it (an external page's prior error
+ * history stays visible after being promoted back into scope), and folding
+ * it into this shared helper would silently change that existing behaviour.
+ * Callers that do want `page_errors` cleared (`resetFailedPages`) delete it
+ * themselves alongside this call.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageIds - `content_items.id` values whose derived rows should be
+ *   deleted. Expected to already be chunked below
+ *   `SQLITE_LIMIT_VARIABLE_NUMBER` by the caller.
+ */
+export declare function clearPageDerivedRows(knex: Knex, pageIds: readonly number[]): Promise<void>;