@nitpicker/crawler 0.20.0 → 0.22.0

package/lib/archive/db-ops/_shared/keyset-paginate-by-id.d.ts

@@ -0,0 +1,49 @@
+import type { Knex } from 'knex';
+/**
+ * Reads every row a caller-supplied query matches, in ascending-`id`
+ * keyset-paginated chunks rather than one unbounded SELECT (issue #294): on
+ * a large archive, a single multi-second-to-minutes scan gives the caller
+ * no way to report progress mid-read. The accumulated result is identical
+ * to a single un-chunked SELECT — chunking exists purely to make the scan
+ * observable, not to bound memory (the mapped output is already fully
+ * materialised for the caller either way).
+ *
+ * Extracted from `getResourceUrlList` / `listDedupeCapObservations`, which
+ * had grown byte-for-byte identical pagination boilerplate (the `MAX(id)`
+ * probe, the `for (;;)` chunk loop, the `onProgress` shape) around two
+ * otherwise-unrelated queries. Callers own everything query-shaped
+ * (joins, `WHERE`, `SELECT`, chunk size via `.limit()` inside
+ * `buildQuery`) — this helper only owns the keyset-chunking mechanics
+ * common to both.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param idTable - The table `id` keys off, used for the O(1) `MAX(id)`
+ *   probe that seeds `onProgress`'s total (only queried when `onProgress`
+ *   is given).
+ * @param buildQuery - Builds one chunk's query given the last-seen `id`
+ *   (`0` on the first call). Must filter on `id > lastId`, order by `id`
+ *   ascending, and cap the row count (typically via `.limit()`) — this
+ *   helper does not add any of those clauses itself. The resolved rows
+ *   must each carry an `id` field so the next chunk's cursor can advance.
+ * @param mapRow - Transforms one raw row into the caller's output shape.
+ * @param onProgress - Called after each chunk, with the highest `id`
+ *   scanned so far and the max `id` in `idTable`. Omit for no reporting.
+ * @returns Every matched row, mapped via `mapRow`, in `id` order.
+ * @example
+ * ```ts
+ * const urls = await keysetPaginateById(
+ * 	knex,
+ * 	'resource_items',
+ * 	(lastId) =>
+ * 		knex('resource_items')
+ * 			.join('url_refs', 'url_refs.id', 'resource_items.url_id')
+ * 			.where('resource_items.id', '>', lastId)
+ * 			.orderBy('resource_items.id', 'asc')
+ * 			.limit(2000)
+ * 			.select('resource_items.id as id', 'url_refs.url as url'),
+ * 	(row) => row.url,
+ * );
+ * ```
+ */
+export declare function keysetPaginateById<Row extends {
+    id: number;
+}, Out>(knex: Knex, idTable: string, buildQuery: (lastId: number) => Knex.QueryBuilder, mapRow: (row: Row) => Out, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<Out[]>;

package/lib/archive/db-ops/_shared/keyset-paginate-by-id.js

@@ -0,0 +1,69 @@
+/**
+ * Reads every row a caller-supplied query matches, in ascending-`id`
+ * keyset-paginated chunks rather than one unbounded SELECT (issue #294): on
+ * a large archive, a single multi-second-to-minutes scan gives the caller
+ * no way to report progress mid-read. The accumulated result is identical
+ * to a single un-chunked SELECT — chunking exists purely to make the scan
+ * observable, not to bound memory (the mapped output is already fully
+ * materialised for the caller either way).
+ *
+ * Extracted from `getResourceUrlList` / `listDedupeCapObservations`, which
+ * had grown byte-for-byte identical pagination boilerplate (the `MAX(id)`
+ * probe, the `for (;;)` chunk loop, the `onProgress` shape) around two
+ * otherwise-unrelated queries. Callers own everything query-shaped
+ * (joins, `WHERE`, `SELECT`, chunk size via `.limit()` inside
+ * `buildQuery`) — this helper only owns the keyset-chunking mechanics
+ * common to both.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param idTable - The table `id` keys off, used for the O(1) `MAX(id)`
+ *   probe that seeds `onProgress`'s total (only queried when `onProgress`
+ *   is given).
+ * @param buildQuery - Builds one chunk's query given the last-seen `id`
+ *   (`0` on the first call). Must filter on `id > lastId`, order by `id`
+ *   ascending, and cap the row count (typically via `.limit()`) — this
+ *   helper does not add any of those clauses itself. The resolved rows
+ *   must each carry an `id` field so the next chunk's cursor can advance.
+ * @param mapRow - Transforms one raw row into the caller's output shape.
+ * @param onProgress - Called after each chunk, with the highest `id`
+ *   scanned so far and the max `id` in `idTable`. Omit for no reporting.
+ * @returns Every matched row, mapped via `mapRow`, in `id` order.
+ * @example
+ * ```ts
+ * const urls = await keysetPaginateById(
+ * 	knex,
+ * 	'resource_items',
+ * 	(lastId) =>
+ * 		knex('resource_items')
+ * 			.join('url_refs', 'url_refs.id', 'resource_items.url_id')
+ * 			.where('resource_items.id', '>', lastId)
+ * 			.orderBy('resource_items.id', 'asc')
+ * 			.limit(2000)
+ * 			.select('resource_items.id as id', 'url_refs.url as url'),
+ * 	(row) => row.url,
+ * );
+ * ```
+ */
+export async function keysetPaginateById(knex, idTable, buildQuery, mapRow, onProgress) {
+    // MAX() over the keyset column is an O(1) index-tail read; only fetched
+    // when someone is listening.
+    let maxId = 0;
+    if (onProgress) {
+        const [maxRow] = await knex(idTable).max({ max: 'id' });
+        maxId = maxRow?.max ?? 0;
+    }
+    const results = [];
+    let lastId = 0;
+    for (;;) {
+        const rows = (await buildQuery(lastId));
+        if (rows.length === 0) {
+            onProgress?.(maxId, maxId);
+            break;
+        }
+        lastId = rows.at(-1).id;
+        for (const row of rows) {
+            results.push(mapRow(row));
+        }
+        onProgress?.(Math.min(lastId, maxId), maxId);
+    }
+    return results;
+}

package/lib/archive/db-ops/_shared/resolve-content-item-id.d.ts

@@ -1,5 +1,4 @@
-import type { WriteRefCaches } from './types.js';
-import type { PageSource } from '../../types.js';
+import type { ResolveContentItemIdOptions, WriteRefCaches } from './types.js';
 import type { Knex } from 'knex';
 /**
  * Returns the `content_items.id` for `url`, inserting a placeholder row
@@ -23,6 +22,11 @@ import type { Knex } from 'knex';
  * inventory label. The cached `source` is updated in the same step so a
  * later hit does not re-issue the UPDATE.
  *
+ * **`is_metadata_only` promotion** (issue #369) is evaluated in the same
+ * pass as the crawled-wins downgrade, and both diffs are folded into a
+ * single `UPDATE` statement (not two) when either column actually
+ * changes — see {@link applyExistingRowUpdates}.
+ *
  * **Cache poisoning on rollback.** Entries cached inside a transaction
  * that later rolls back would point at ids that no longer exist. Every
  * write op that opens a multi-statement transaction around this function
@@ -44,18 +48,15 @@ import type { Knex } from 'knex';
  * @param caches - The connection's write-side id caches; mutated in place.
  * @param url - The URL to look up or insert (normalised
  *   `withoutHashAndAuth` form, matching the legacy identity contract).
- * @param isExternal - Optional; recorded on new inserts only. `1` marks
- *   the row as an external URL that will never be scraped as a target.
- *   Defaults to `0` (in-scope) on insert, mirroring the legacy column
- *   default.
- * @param source - Optional provenance label put on a newly-inserted row.
- *   Omit to let the `content_items.source` DEFAULT (`'crawled'`) apply.
- *   Pass `'crawled'` to arm the crawled-wins downgrade on existing
- *   inventory-labelled rows.
+ * @param options - See {@link ResolveContentItemIdOptions}.
  * @returns The `content_items.id` of the existing or newly inserted row.
  * @throws {Error} When the upsert's `RETURNING` yields no row — should
  *   not happen, so it surfaces as a hard error.
  * @example
- * const pageId = await resolveContentItemId(trx, caches, anchor.href, 1, 'crawled');
+ * const pageId = await resolveContentItemId(trx, caches, anchor.href, {
+ *   isExternal: 1,
+ *   source: 'crawled',
+ *   isMetadataOnly: 1,
+ * });
  */
-export declare function resolveContentItemId(qb: Knex | Knex.Transaction, caches: WriteRefCaches, url: string, isExternal?: 0 | 1, source?: PageSource): Promise<number>;
+export declare function resolveContentItemId(qb: Knex | Knex.Transaction, caches: WriteRefCaches, url: string, options?: ResolveContentItemIdOptions): Promise<number>;

package/lib/archive/db-ops/_shared/resolve-content-item-id.js

@@ -21,6 +21,11 @@ import { upsertUrlRef } from './upsert-url-ref.js';
  * inventory label. The cached `source` is updated in the same step so a
  * later hit does not re-issue the UPDATE.
  *
+ * **`is_metadata_only` promotion** (issue #369) is evaluated in the same
+ * pass as the crawled-wins downgrade, and both diffs are folded into a
+ * single `UPDATE` statement (not two) when either column actually
+ * changes — see {@link applyExistingRowUpdates}.
+ *
  * **Cache poisoning on rollback.** Entries cached inside a transaction
  * that later rolls back would point at ids that no longer exist. Every
  * write op that opens a multi-statement transaction around this function
@@ -42,41 +47,45 @@ import { upsertUrlRef } from './upsert-url-ref.js';
  * @param caches - The connection's write-side id caches; mutated in place.
  * @param url - The URL to look up or insert (normalised
  *   `withoutHashAndAuth` form, matching the legacy identity contract).
- * @param isExternal - Optional; recorded on new inserts only. `1` marks
- *   the row as an external URL that will never be scraped as a target.
- *   Defaults to `0` (in-scope) on insert, mirroring the legacy column
- *   default.
- * @param source - Optional provenance label put on a newly-inserted row.
- *   Omit to let the `content_items.source` DEFAULT (`'crawled'`) apply.
- *   Pass `'crawled'` to arm the crawled-wins downgrade on existing
- *   inventory-labelled rows.
+ * @param options - See {@link ResolveContentItemIdOptions}.
  * @returns The `content_items.id` of the existing or newly inserted row.
  * @throws {Error} When the upsert's `RETURNING` yields no row — should
  *   not happen, so it surfaces as a hard error.
  * @example
- * const pageId = await resolveContentItemId(trx, caches, anchor.href, 1, 'crawled');
+ * const pageId = await resolveContentItemId(trx, caches, anchor.href, {
+ *   isExternal: 1,
+ *   source: 'crawled',
+ *   isMetadataOnly: 1,
+ * });
  */
-export async function resolveContentItemId(qb, caches, url, isExternal, source) {
+export async function resolveContentItemId(qb, caches, url, options) {
+    const { isExternal, source, isMetadataOnly } = options ?? {};
     const cached = caches.contentItems.get(url);
     if (cached !== undefined) {
-        await applyCrawledWinsDowngrade(qb, cached, source);
+        await applyExistingRowUpdates(qb, cached, source, isMetadataOnly);
         return cached.id;
     }
     const urlId = await upsertUrlRef(qb, caches, url);
     const [record] = (await qb
-        .select('id', 'source')
+        .select('id', 'source', 'is_metadata_only')
         .from('content_items')
         .where('url_id', urlId));
     if (record !== undefined) {
-        const entry = { id: record.id, source: record.source };
-        await applyCrawledWinsDowngrade(qb, entry, source);
+        const entry = {
+            id: record.id,
+            source: record.source,
+            isMetadataOnly: record.is_metadata_only,
+        };
+        await applyExistingRowUpdates(qb, entry, source, isMetadataOnly);
         caches.contentItems.set(url, entry);
         return entry.id;
     }
-    const insertedRows = await qb.raw(`INSERT INTO content_items (url_id, scraped, is_target, is_external${source === undefined ? '' : ', source'})
-		 VALUES (?, 0, 0, ?${source === undefined ? '' : ', ?'})
-		 ON CONFLICT(url_id) DO UPDATE SET url_id = url_id
-		 RETURNING id, source`, source === undefined ? [urlId, isExternal ?? 0] : [urlId, isExternal ?? 0, source]);
+    const insertedRows = await qb.raw(`INSERT INTO content_items (url_id, scraped, is_target, is_external, is_metadata_only${source === undefined ? '' : ', source'})
+			 VALUES (?, 0, 0, ?, ?${source === undefined ? '' : ', ?'})
+			 ON CONFLICT(url_id) DO UPDATE SET url_id = url_id
+			 RETURNING id, source, is_metadata_only`, source === undefined
+        ? [urlId, isExternal ?? 0, isMetadataOnly ?? 0]
+        : [urlId, isExternal ?? 0, isMetadataOnly ?? 0, source]);
     const inserted = insertedRows[0];
     if (inserted === undefined) {
         throw new Error(`Failed to insert a new content item: ${url}`);
@@ -84,28 +93,51 @@ export async function resolveContentItemId(qb, caches, url, isExternal, source)
     const insertedEntry = {
         id: inserted.id,
         source: inserted.source,
+        isMetadataOnly: inserted.is_metadata_only,
     };
     // A conflict means a concurrent writer created the row between this
-    // function's SELECT miss and the INSERT — the returned `source` is that
-    // row's value, so the downgrade must be evaluated exactly as on the
-    // SELECT-hit path.
-    await applyCrawledWinsDowngrade(qb, insertedEntry, source);
+    // function's SELECT miss and the INSERT — the returned `source` /
+    // `is_metadata_only` are that row's values, so both follow-ups must be
+    // evaluated exactly as on the SELECT-hit path.
+    await applyExistingRowUpdates(qb, insertedEntry, source, isMetadataOnly);
     caches.contentItems.set(url, insertedEntry);
     return insertedEntry.id;
 }
 /**
- * Fires the crawled-wins downgrade when a `'crawled'`-lineage resolution
- * lands on a row whose last-known `source` is an inventory label, and
- * keeps the cache entry in sync so the UPDATE runs at most once per
- * (connection, row).
+ * Applies both the crawled-wins `source` downgrade and the `is_metadata_only`
+ * promotion (issue #369) to an already-resolved row in a single `UPDATE`,
+ * and keeps the cache entry in sync so a repeat call with the same values
+ * does not re-issue any write.
+ *
+ * Folded into one function (and one statement) rather than two independent
+ * ones: `resolveContentItemId` runs once per anchor on every scraped page
+ * across a whole crawl, so a page whose anchors trip both conditions would
+ * otherwise pay two round trips instead of one.
  * @param qb - Knex instance or transaction.
  * @param entry - The cached identity to check and mutate.
- * @param source - The resolution's lineage label.
+ * @param source - The resolution's lineage label, if any — see
+ *   {@link ResolveContentItemIdOptions.source}.
+ * @param isMetadataOnly - The resolution's explicit scrape-depth opinion, if
+ *   any — see {@link ResolveContentItemIdOptions.isMetadataOnly}. Omitted by
+ *   every caller except `replaceAnchorEdges`, which always has an opinion,
+ *   so this is a no-op for the rest.
  */
-async function applyCrawledWinsDowngrade(qb, entry, source) {
-    if (source !== 'crawled' || entry.source === 'crawled') {
+async function applyExistingRowUpdates(qb, entry, source, isMetadataOnly) {
+    const updates = {};
+    if (source === 'crawled' && entry.source !== 'crawled') {
+        updates.source = 'crawled';
+    }
+    if (isMetadataOnly !== undefined && entry.isMetadataOnly !== isMetadataOnly) {
+        updates.is_metadata_only = isMetadataOnly;
+    }
+    if (Object.keys(updates).length === 0) {
         return;
     }
-    await qb('content_items').where('id', entry.id).update({ source: 'crawled' });
-    entry.source = 'crawled';
+    await qb('content_items').where('id', entry.id).update(updates);
+    if (updates.source !== undefined) {
+        entry.source = updates.source;
+    }
+    if (updates.is_metadata_only !== undefined) {
+        entry.isMetadataOnly = updates.is_metadata_only;
+    }
 }

package/lib/archive/db-ops/_shared/types.d.ts

@@ -12,6 +12,12 @@ export interface ContentItemCacheEntry {
     id: number;
     /** `content_items.source` as last observed / written by this process. */
     source: PageSource;
+    /**
+     * `content_items.is_metadata_only` as last observed / written by this
+     * process. Lets {@link ../_shared/resolve-content-item-id.ts} skip a
+     * redundant `UPDATE` when a later resolution recomputes the same value.
+     */
+    isMetadataOnly: 0 | 1;
 }
 /**
  * In-process id caches shared across one archive connection's write path.
@@ -53,3 +59,36 @@ export interface WriteRefCaches {
      */
     headers: HeaderTableCaches | null;
 }
+/**
+ * Optional per-call flags for {@link ../_shared/resolve-content-item-id.ts}.
+ */
+export interface ResolveContentItemIdOptions {
+    /**
+     * `1` marks the row as an external URL that will never be scraped as a
+     * target. Recorded on new inserts only. Defaults to `0` (in-scope) on
+     * insert, mirroring the legacy column default.
+     */
+    isExternal?: 0 | 1;
+    /**
+     * Provenance label put on a newly-inserted row. Omit to let the
+     * `content_items.source` DEFAULT (`'crawled'`) apply. Pass `'crawled'`
+     * to arm the crawled-wins downgrade on existing inventory-labelled rows.
+     */
+    source?: PageSource;
+    /**
+     * `1` marks the row as fated for a metadata-only (title-only) scrape
+     * rather than a full one, `0` explicitly marks it as a full-scrape
+     * target. Omit (the default for every caller except
+     * `replaceAnchorEdges`) to leave the column untouched — those callers
+     * (redirects, resources, errors, skipped pages, console logs) have no
+     * opinion on scrape depth and must not clobber a value an
+     * anchor-discovery call already established.
+     * `!options.recursive || anchor.isExternal` is a pure function of the
+     * URL within one crawl session (`recursive` is session-constant,
+     * `isExternal` depends only on scope matching), so every anchor-path
+     * call for the same URL always recomputes the same value — there is no
+     * real conflict to arbitrate between two *opinionated* calls, only
+     * between an opinionated call and the unopinionated majority.
+     */
+    isMetadataOnly?: 0 | 1;
+}

package/lib/archive/db-ops/analysis/replace-analysis-violations.js

@@ -33,10 +33,13 @@ function parseLegacyCorruptedUrl(url) {
  * version-gated migration script (see `scripts/migrate-to-0.13.mjs`). This
  * function relies on the exception documented in ARCHITECTURE.md's
  * invariants list: a nullable, additive column on a table with a single
- * write path may self-heal here without a version bump, because
- * `replaceAnalysisViolations` is that single write path for
- * `analysis_violations`. Runs before the transaction below so the DDL is
- * not mixed with the DML rewrite.
+ * INSERT path may self-heal here without a version bump, because
+ * `replaceAnalysisViolations` is that single INSERT path for
+ * `analysis_violations` — `resetPagesByUrls` (`db-ops/pages/reset/`) also
+ * writes to this table, but only ever DELETEs by `page_id`, so it never
+ * needs to know about `line`/`col` and cannot violate this self-heal
+ * argument. Runs before the transaction below so the DDL is not mixed with
+ * the DML rewrite.
  * @param knex - Knex query builder connected to the archive DB.
  */
 async function ensureLineColColumns(knex) {

package/lib/archive/db-ops/config/get-created-cwd.d.ts

@@ -0,0 +1,11 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the working directory recorded when this stub's crawl session
+ * was created, so {@link import('../../archive.js').default.resume} can
+ * reconstruct the completed archive's output path independent of the
+ * directory `crawl --resume` happens to be invoked from.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The recorded cwd, or `null` for a stub created before this column
+ *   existed (or created by a code path that never stamps it, i.e. `resume`).
+ */
+export declare function getCreatedCwd(knex: Knex): Promise<string | null>;

package/lib/archive/db-ops/config/get-created-cwd.js

@@ -0,0 +1,13 @@
+/**
+ * Retrieves the working directory recorded when this stub's crawl session
+ * was created, so {@link import('../../archive.js').default.resume} can
+ * reconstruct the completed archive's output path independent of the
+ * directory `crawl --resume` happens to be invoked from.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The recorded cwd, or `null` for a stub created before this column
+ *   existed (or created by a code path that never stamps it, i.e. `resume`).
+ */
+export async function getCreatedCwd(knex) {
+    const [row] = await knex.select('createdCwd').from('info');
+    return row?.createdCwd ?? null;
+}

package/lib/archive/db-ops/config/info-column-allowlist.js

@@ -24,4 +24,5 @@ export const INFO_COLUMN_ALLOWLIST = new Set([
     'userAgent',
     'ignoreRobots',
     'mainContentSelector',
+    'createdCwd',
 ]);

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts

@@ -0,0 +1,57 @@
+import type { DedupeCapObservationRow } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Reads back every previously-scraped internal page's raw fields in the
+ * exact population a live crawl's `DedupeCapTracker#observe` call site
+ * would have fed it, so `CrawlerOrchestrator`'s five resuming-session
+ * methods (`append`/`inventory`/`recrawl`/`retryFailed`/`resume`) can
+ * replay a prior session's Misra-Gries observations into a fresh tracker
+ * instance (see `buildDedupeCapObservation`) instead of restarting every
+ * not-yet-capped shape's counter at 0.
+ *
+ * The WHERE clause reproduces the live observation gate
+ * (`!isExternal && !isMetadataOnly && html.length > 0`) using columns that
+ * survive a process restart:
+ *
+ * - `is_external = 0` — external pages carry no useful signal (same
+ *   exclusion `computeMetaSignature`'s design already assumes).
+ * - `is_target = 1` — the archived equivalent of "not metadata-only": a
+ *   metadata-only fetch never invokes the browser and is written with
+ *   `is_target = 0` (see `Crawler`'s metadata-only branch), matching the
+ *   live gate's `!isMetadataOnly` exactly.
+ * - `redirect_dest_id IS NULL` — a redirect source's own body was never
+ *   rendered; only the destination page (a separate row) was.
+ * - `is_skipped IS NULL OR is_skipped = 0` — a skipped placeholder row
+ *   (`source = 'inventory-seed'` etc.) has no rendered body either.
+ * - `pm.body_hash IS NOT NULL` — the archived proxy for `html.length > 0`:
+ *   `update-page.ts` only ever writes a non-null `body_hash` on the same
+ *   `writeHtml && html.length > 0` gate the live site's `precomputedBodyHash`
+ *   computation uses, so this column faithfully reconstructs that
+ *   condition. Also excludes pre-`body_hash`-migration legacy archives'
+ *   un-backfilled rows (`migrate-page-meta-body-hash.ts` adds the column on
+ *   every writer open but does not compute values for existing rows) —
+ *   understating the replayed count is safe (never over-counts), so no
+ *   column-presence guard is needed here.
+ *
+ * Deliberately does NOT exclude rows already marked
+ * `content_items.dedupe_cap_event_id` (the post-hoc marking column) —
+ * unlike `body_hash`, that column is only backfilled during a
+ * `viewer-build`, not written during a live crawl, so filtering on it here
+ * would silently diverge from what the live crawl-time gate actually saw.
+ * `DedupeCapTracker#observe` is an O(1) no-op for a shape already in its
+ * sticky set regardless, so replaying an already-capped shape's rows costs
+ * a discarded observation, never an incorrect one.
+ *
+ * Rows are read in `ci.id` order (insertion / discovery order) via
+ * `keysetPaginateById` — Misra-Gries counting is order-dependent, so
+ * replaying in the archive's original discovery order reproduces what the
+ * tracker's state would look like had the process never restarted, subject
+ * to the same `mapCap` LRU eviction.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk, with the highest `ci.id`
+ *   scanned so far and the max `ci.id` in the table — mirrors
+ *   `getResourceUrlList`'s progress shape (`SetupProgressCallbacks.onChunkProgress`).
+ *   Omit for no reporting (tests, and callers that don't need it).
+ * @returns Every qualifying page's raw fields, in `ci.id` order.
+ */
+export declare function listDedupeCapObservations(knex: Knex, onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<DedupeCapObservationRow[]>;

package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-observations.js

@@ -0,0 +1,87 @@
+import { keysetPaginateById } from '../_shared/keyset-paginate-by-id.js';
+const READ_CHUNK_SIZE = 2000;
+/**
+ * Reads back every previously-scraped internal page's raw fields in the
+ * exact population a live crawl's `DedupeCapTracker#observe` call site
+ * would have fed it, so `CrawlerOrchestrator`'s five resuming-session
+ * methods (`append`/`inventory`/`recrawl`/`retryFailed`/`resume`) can
+ * replay a prior session's Misra-Gries observations into a fresh tracker
+ * instance (see `buildDedupeCapObservation`) instead of restarting every
+ * not-yet-capped shape's counter at 0.
+ *
+ * The WHERE clause reproduces the live observation gate
+ * (`!isExternal && !isMetadataOnly && html.length > 0`) using columns that
+ * survive a process restart:
+ *
+ * - `is_external = 0` — external pages carry no useful signal (same
+ *   exclusion `computeMetaSignature`'s design already assumes).
+ * - `is_target = 1` — the archived equivalent of "not metadata-only": a
+ *   metadata-only fetch never invokes the browser and is written with
+ *   `is_target = 0` (see `Crawler`'s metadata-only branch), matching the
+ *   live gate's `!isMetadataOnly` exactly.
+ * - `redirect_dest_id IS NULL` — a redirect source's own body was never
+ *   rendered; only the destination page (a separate row) was.
+ * - `is_skipped IS NULL OR is_skipped = 0` — a skipped placeholder row
+ *   (`source = 'inventory-seed'` etc.) has no rendered body either.
+ * - `pm.body_hash IS NOT NULL` — the archived proxy for `html.length > 0`:
+ *   `update-page.ts` only ever writes a non-null `body_hash` on the same
+ *   `writeHtml && html.length > 0` gate the live site's `precomputedBodyHash`
+ *   computation uses, so this column faithfully reconstructs that
+ *   condition. Also excludes pre-`body_hash`-migration legacy archives'
+ *   un-backfilled rows (`migrate-page-meta-body-hash.ts` adds the column on
+ *   every writer open but does not compute values for existing rows) —
+ *   understating the replayed count is safe (never over-counts), so no
+ *   column-presence guard is needed here.
+ *
+ * Deliberately does NOT exclude rows already marked
+ * `content_items.dedupe_cap_event_id` (the post-hoc marking column) —
+ * unlike `body_hash`, that column is only backfilled during a
+ * `viewer-build`, not written during a live crawl, so filtering on it here
+ * would silently diverge from what the live crawl-time gate actually saw.
+ * `DedupeCapTracker#observe` is an O(1) no-op for a shape already in its
+ * sticky set regardless, so replaying an already-capped shape's rows costs
+ * a discarded observation, never an incorrect one.
+ *
+ * Rows are read in `ci.id` order (insertion / discovery order) via
+ * `keysetPaginateById` — Misra-Gries counting is order-dependent, so
+ * replaying in the archive's original discovery order reproduces what the
+ * tracker's state would look like had the process never restarted, subject
+ * to the same `mapCap` LRU eviction.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk, with the highest `ci.id`
+ *   scanned so far and the max `ci.id` in the table — mirrors
+ *   `getResourceUrlList`'s progress shape (`SetupProgressCallbacks.onChunkProgress`).
+ *   Omit for no reporting (tests, and callers that don't need it).
+ * @returns Every qualifying page's raw fields, in `ci.id` order.
+ */
+export async function listDedupeCapObservations(knex, onProgress) {
+    return keysetPaginateById(knex, 'content_items', (lastId) => knex('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .join('page_meta as pm', 'pm.page_id', 'ci.id')
+        .leftJoin('text_refs as title_ref', 'title_ref.id', 'pm.title_text_id')
+        .leftJoin('text_refs as description_ref', 'description_ref.id', 'pm.description_text_id')
+        .leftJoin('text_refs as og_title_ref', 'og_title_ref.id', 'pm.og_title_text_id')
+        .leftJoin('url_refs as og_url_ur', 'og_url_ur.id', 'pm.og_url_id')
+        .where('ci.id', '>', lastId)
+        .andWhere('ci.scraped', 1)
+        .andWhere('ci.is_external', 0)
+        .andWhere('ci.is_target', 1)
+        .whereNull('ci.redirect_dest_id')
+        .where((qb) => {
+        qb.whereNull('ci.is_skipped').orWhere('ci.is_skipped', 0);
+    })
+        .whereNotNull('pm.body_hash')
+        .orderBy('ci.id', 'asc')
+        .limit(READ_CHUNK_SIZE)
+        .select('ci.id as id', 'ur.url as url', 'title_ref.text as title', 'description_ref.text as description', 'og_title_ref.text as ogTitle', 'og_url_ur.url as ogUrl', 'pm.body_hash as bodyHash'), (row) => ({
+        url: row.url,
+        title: row.title,
+        description: row.description,
+        ogTitle: row.ogTitle,
+        ogUrl: row.ogUrl,
+        // libsql returns BLOB columns as `Uint8Array`, not `Buffer` —
+        // `DedupeCapTracker#observe` compares hashes via `Buffer#equals`,
+        // so this must be a real `Buffer`.
+        bodyHash: Buffer.from(row.bodyHash),
+    }), onProgress);
+}

package/lib/archive/db-ops/errors/insert-page-error.js

@@ -18,7 +18,9 @@ import { resolveContentItemId } from '../_shared/resolve-content-item-id.js';
  * @param isExternal - Whether the URL is external. Defaults to `false`.
  */
 export async function insertPageError(knex, caches, url, phase, message, isExternal = false) {
-    const pageId = await resolveContentItemId(knex, caches, url, isExternal ? 1 : 0);
+    const pageId = await resolveContentItemId(knex, caches, url, {
+        isExternal: isExternal ? 1 : 0,
+    });
     await knex('page_errors').insert({
         pageId,
         phase,

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -2,8 +2,8 @@ import type { Knex } from 'knex';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`,
- * `inventory_runs.exclude_skipped`).
+ * `page_meta.main_content_*`, `list_reconcile_runs.invalid_skipped`,
+ * `list_reconcile_runs.exclude_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -18,9 +18,10 @@ import type { Knex } from 'knex';
  * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
- * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migrateInfoCreatedCwd`, `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
  * `migratePageMetaConsoleErrorCount`, `migratePageMetaCustomElementCount`,
- * `migrateContentItemsAliasOfId`,
+ * `migratePageMetaImageScan`, `migrateContentItemsAliasOfId`,
+ * `migrateContentItemsIsMetadataOnly`,
  *
  * `migratePageTagsToPageTechnologies` is the one exception to "column adds
  * only": it converts `page_tags` (removed) rows into `technology_signals`/
@@ -28,8 +29,14 @@ import type { Knex } from 'knex';
  * not a column add, but it belongs in this same boot phase for the same
  * reason (self-healing an old archive's schema before any reader runs).
  *
- * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
- * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
+ * `migrateContentItemsDedupeCapEventId`, `migrateListReconcileRunsInvalidSkipped`,
+ * `migrateListReconcileRunsExcludeSkipped`) rather than a DDL-string change alone.
+ *
+ * `migrateInventoryRunsToListReconcileRuns` runs BEFORE `initSchema`
+ * (unlike every other migration here) — see its own JSDoc for why: a
+ * plain rename must land before `createAdjunctTables`'s
+ * `IF NOT EXISTS` guard would otherwise create an empty `list_reconcile_runs`
+ * and strand the old table's rows.
  *
  * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
  * change) but belongs at this same boot phase for the same reason the