@nitpicker/crawler 0.20.0 → 0.22.0

package/lib/append-retry-failed-common-setup-phases.d.ts

@@ -11,5 +11,5 @@
  */
 export declare const APPEND_RETRY_FAILED_COMMON_SETUP_PHASES: {
     readonly prefix: readonly ["Extracting archive", "Loading archive config", "Backing up archive"];
-    readonly suffix: readonly ["Loading dedupe-cap shape keys", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];
+    readonly suffix: readonly ["Loading dedupe-cap state", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];
 };

package/lib/append-retry-failed-common-setup-phases.js

@@ -12,7 +12,11 @@
 export const APPEND_RETRY_FAILED_COMMON_SETUP_PHASES = {
     prefix: ['Extracting archive', 'Loading archive config', 'Backing up archive'],
     suffix: [
-        'Loading dedupe-cap shape keys',
+        // Renamed from 'Loading dedupe-cap shape keys' — this phase now
+        // covers both the confirmed-capped shape-key read AND the
+        // not-yet-capped shape observation replay (`archive.listDedupeCapObservations`
+        // / `#preloadDedupeCapObservations`), not just the former.
+        'Loading dedupe-cap state',
         'Loading crawl state',
         'Loading resource list',
         'Loading scraped page count',

package/lib/append-setup-phases.d.ts

@@ -14,4 +14,4 @@
  * a `SETUP_RECOVERY_PHASE_LABELS` entry (`'Restoring archive from backup'`)
  * is announced instead of continuing this sequence.
  */
-export declare const APPEND_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Backing up archive", "Repromoting external pages", "Loading dedupe-cap shape keys", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];
+export declare const APPEND_SETUP_PHASES: readonly ["Extracting archive", "Loading archive config", "Backing up archive", "Repromoting external pages", "Loading dedupe-cap state", "Loading crawl state", "Loading resource list", "Loading scraped page count", "Restoring crawl state"];

package/lib/archive/archive.d.ts

@@ -1,5 +1,5 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
-import type { Config, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageSource } from './types.js';
+import type { Config, DedupeCapObservationRow, InsertDedupeCapEventParams, InsertNetworkOutageParams, ListReconcileRunMeta, PageSource } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ConsoleLogEntry } from '@d-zero/beholder';
@@ -101,12 +101,19 @@ export default class Archive extends ArchiveAccessor {
     finalizeDedupeCapEvent(id: number, rejectedCount: number): Promise<void>;
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
-     * @returns An object with `scraped` and `pending` URL arrays.
+     * @returns An object with `scraped`, `pending`, and `pendingMetadataOnly` URL arrays.
      */
     getCrawlingState(): Promise<{
         scraped: string[];
         pending: string[];
+        pendingMetadataOnly: string[];
     }>;
+    /**
+     * Retrieves the `info.createdCwd` value stamped when this stub was
+     * created — see {@link Archive.resume} for how it is used.
+     * @returns The recorded cwd, or `null` if never stamped.
+     */
+    getCreatedCwd(): Promise<string | null>;
     /**
      * Return the subset of `urls` that already exist as `pages.url`. Used by
      * `CrawlerOrchestrator.inventory` to filter the user-supplied URL list
@@ -210,11 +217,26 @@ export default class Archive extends ArchiveAccessor {
      * Appends one open row to the `network_outages` journal.
      *
      * Thin facade over {@link Database.insertNetworkOutage} — see
-     * {@link recordInventoryRun}'s docstring for why this indirection exists.
+     * {@link recordListReconcileRun}'s docstring for why this indirection exists.
      * @param params - The confirmed-outage fields to record.
      * @returns The autoincremented `id` of the inserted row.
      */
     insertNetworkOutage(params: InsertNetworkOutageParams): Promise<number>;
+    /**
+     * Every previously-scraped internal page's raw fields, in the shape
+     * `buildDedupeCapObservation` needs to replay this archive's prior
+     * `DedupeCapTracker` observations (not just its confirmed-capped
+     * shapes — see {@link listDedupeCapShapeKeys}) into a fresh tracker
+     * instance on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory` / `--recrawl`. Without this, every not-yet-capped
+     * shape's Misra-Gries counter restarts at 0 each session, even one
+     * that was one observation away from confirming a trap.
+     * @param onProgress - Called after each internal chunk read, with the
+     *   highest `content_items.id` scanned so far and the max id in the
+     *   table. Omit for no reporting.
+     * @returns Every qualifying page's raw fields, in discovery order.
+     */
+    listDedupeCapObservations(onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<DedupeCapObservationRow[]>;
     /**
      * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
      * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
@@ -246,16 +268,16 @@ export default class Archive extends ArchiveAccessor {
      */
     listNetworkOutages(): Promise<OutageWindow[]>;
     /**
-     * Appends one row to the `inventory_runs` audit log.
+     * Appends one row to the `list_reconcile_runs` audit log.
      *
-     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * Thin facade over {@link Database.recordListReconcileRun} — keeps the
      * orchestrator decoupled from the knex layer and gives a single
      * write entry point that future Archive-level concerns (locking,
      * mirror sync, etc.) can hook into without touching every caller.
      * @param meta - The run metadata. Only `ran_at` is required.
      * @returns The autoincremented `id` of the inserted row.
      */
-    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
+    recordListReconcileRun(meta: ListReconcileRunMeta): Promise<number>;
     /**
      * Releases the SQLite handle and the advisory lock **without** writing
      * the archive or removing `tmpDir`.
@@ -322,6 +344,19 @@ export default class Archive extends ArchiveAccessor {
      * @returns The URLs of the pages that were reset to pending.
      */
     resetFailedPages(onProgress?: (processed: number, total: number) => void): Promise<string[]>;
+    /**
+     * Reset pages matching an operator-supplied URL list back to pending so a
+     * follow-up crawl re-fetches them from scratch.
+     *
+     * Delegates to {@link Database.resetPagesByUrls}. See that method for the
+     * conservative exclusion rules (redirect sources / intentionally-skipped /
+     * external pages are matched but not reset).
+     * @param urls - URL strings to match, already in `withoutHashAndAuth` form.
+     * @param onProgress - Forwarded to {@link Database.resetPagesByUrls} — see
+     *   that method's docs.
+     * @returns The reset URLs plus the excluded URLs grouped by reason.
+     */
+    resetPagesByUrls(urls: readonly string[], onProgress?: (processed: number, total: number) => void): Promise<import("./types.js").ResetPagesByUrlsResult>;
     /**
      * Persists the raw bytes of an `--inventory` source URL list into the
      * archive's tar payload, at `inventory/<sha256>.txt`.
@@ -368,8 +403,15 @@ export default class Archive extends ArchiveAccessor {
      * content-type), never a rendered body.
      * @param pageInfo - The page data to store.
      * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
+     * @param recursive - The crawl session's `recursive` option, forwarded so
+     *   `is_metadata_only` on any newly-discovered anchor is computed
+     *   correctly (#369). Defaults to `true` (never persist an anchor as
+     *   metadata-only) since an external page's `anchorList` is always
+     *   empty — this only matters for callers that pass one anyway.
+     * @example
+     * await archive.setExternalPage(externalPageData, undefined, options.recursive);
      */
-    setExternalPage(pageInfo: PageData, source?: PageSource): Promise<void>;
+    setExternalPage(pageInfo: PageData, source?: PageSource, recursive?: boolean): Promise<void>;
     /**
      * Stores a crawled page's data in the archive database, persisting the
      * rendered HTML body as a zstd-compressed BLOB inside the same SQLite
@@ -380,9 +422,16 @@ export default class Archive extends ArchiveAccessor {
      * @param bodyHash - Precomputed body hash for the page's HTML (see
      *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
      *   computing it from the HTML instead.
+     * @param recursive - The crawl session's `recursive` option, forwarded so
+     *   any anchor discovered on this page gets the correct
+     *   `is_metadata_only` value (#369). Defaults to `true` (never persist
+     *   an anchor as metadata-only) for callers that do not track the
+     *   option.
      * @returns The database ID of the stored page.
+     * @example
+     * const pageId = await archive.setPage(pageData, undefined, undefined, options.recursive);
      */
-    setPage(pageInfo: PageData, source?: PageSource, bodyHash?: Buffer | null): Promise<number>;
+    setPage(pageInfo: PageData, source?: PageSource, bodyHash?: Buffer | null, recursive?: boolean): Promise<number>;
     /**
      * Records a redirect edge without re-storing the destination's content.
      *

package/lib/archive/archive.js

@@ -162,11 +162,19 @@ export default class Archive extends ArchiveAccessor {
     }
     /**
      * Retrieves the current crawling state, including lists of scraped and pending URLs.
-     * @returns An object with `scraped` and `pending` URL arrays.
+     * @returns An object with `scraped`, `pending`, and `pendingMetadataOnly` URL arrays.
      */
     async getCrawlingState() {
         return this.#db.getCrawlingState();
     }
+    /**
+     * Retrieves the `info.createdCwd` value stamped when this stub was
+     * created — see {@link Archive.resume} for how it is used.
+     * @returns The recorded cwd, or `null` if never stamped.
+     */
+    async getCreatedCwd() {
+        return this.#db.getCreatedCwd();
+    }
     /**
      * Return the subset of `urls` that already exist as `pages.url`. Used by
      * `CrawlerOrchestrator.inventory` to filter the user-supplied URL list
@@ -303,7 +311,7 @@ export default class Archive extends ArchiveAccessor {
      * Appends one open row to the `network_outages` journal.
      *
      * Thin facade over {@link Database.insertNetworkOutage} — see
-     * {@link recordInventoryRun}'s docstring for why this indirection exists.
+     * {@link recordListReconcileRun}'s docstring for why this indirection exists.
      * @param params - The confirmed-outage fields to record.
      * @returns The autoincremented `id` of the inserted row.
      */
@@ -311,6 +319,23 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Insert network outage: startedAt=%d probeHost=%s', params.startedAt, params.probeHost);
         return await this.#db.insertNetworkOutage(params);
     }
+    /**
+     * Every previously-scraped internal page's raw fields, in the shape
+     * `buildDedupeCapObservation` needs to replay this archive's prior
+     * `DedupeCapTracker` observations (not just its confirmed-capped
+     * shapes — see {@link listDedupeCapShapeKeys}) into a fresh tracker
+     * instance on `--resume` / `--append` / `--retry-failed` /
+     * `--inventory` / `--recrawl`. Without this, every not-yet-capped
+     * shape's Misra-Gries counter restarts at 0 each session, even one
+     * that was one observation away from confirming a trap.
+     * @param onProgress - Called after each internal chunk read, with the
+     *   highest `content_items.id` scanned so far and the max id in the
+     *   table. Omit for no reporting.
+     * @returns Every qualifying page's raw fields, in discovery order.
+     */
+    async listDedupeCapObservations(onProgress) {
+        return this.#db.listDedupeCapObservations(onProgress);
+    }
     /**
      * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
      * Consumed by `CrawlerOrchestrator` to preload `DedupeCapTracker`'s
@@ -348,18 +373,18 @@ export default class Archive extends ArchiveAccessor {
         return await this.#db.listNetworkOutages();
     }
     /**
-     * Appends one row to the `inventory_runs` audit log.
+     * Appends one row to the `list_reconcile_runs` audit log.
      *
-     * Thin facade over {@link Database.recordInventoryRun} — keeps the
+     * Thin facade over {@link Database.recordListReconcileRun} — keeps the
      * orchestrator decoupled from the knex layer and gives a single
      * write entry point that future Archive-level concerns (locking,
      * mirror sync, etc.) can hook into without touching every caller.
      * @param meta - The run metadata. Only `ran_at` is required.
      * @returns The autoincremented `id` of the inserted row.
      */
-    async recordInventoryRun(meta) {
-        dbLog('Record inventory run: %s', meta.list_label ?? meta.ran_at);
-        return await this.#db.recordInventoryRun(meta);
+    async recordListReconcileRun(meta) {
+        dbLog('Record list reconcile run: %s', meta.list_label ?? meta.ran_at);
+        return await this.#db.recordListReconcileRun(meta);
     }
     /**
      * Releases the SQLite handle and the advisory lock **without** writing
@@ -434,6 +459,22 @@ export default class Archive extends ArchiveAccessor {
         dbLog('Reset failed pages back to pending');
         return this.#db.resetFailedPages(onProgress);
     }
+    /**
+     * Reset pages matching an operator-supplied URL list back to pending so a
+     * follow-up crawl re-fetches them from scratch.
+     *
+     * Delegates to {@link Database.resetPagesByUrls}. See that method for the
+     * conservative exclusion rules (redirect sources / intentionally-skipped /
+     * external pages are matched but not reset).
+     * @param urls - URL strings to match, already in `withoutHashAndAuth` form.
+     * @param onProgress - Forwarded to {@link Database.resetPagesByUrls} — see
+     *   that method's docs.
+     * @returns The reset URLs plus the excluded URLs grouped by reason.
+     */
+    async resetPagesByUrls(urls, onProgress) {
+        dbLog('Reset %d URL-list-matched page(s) back to pending', urls.length);
+        return this.#db.resetPagesByUrls(urls, onProgress);
+    }
     /**
      * Persists the raw bytes of an `--inventory` source URL list into the
      * archive's tar payload, at `inventory/<sha256>.txt`.
@@ -489,10 +530,17 @@ export default class Archive extends ArchiveAccessor {
      * content-type), never a rendered body.
      * @param pageInfo - The page data to store.
      * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
+     * @param recursive - The crawl session's `recursive` option, forwarded so
+     *   `is_metadata_only` on any newly-discovered anchor is computed
+     *   correctly (#369). Defaults to `true` (never persist an anchor as
+     *   metadata-only) since an external page's `anchorList` is always
+     *   empty — this only matters for callers that pass one anyway.
+     * @example
+     * await archive.setExternalPage(externalPageData, undefined, options.recursive);
      */
-    async setExternalPage(pageInfo, source) {
+    async setExternalPage(pageInfo, source, recursive) {
         dbLog('Set external page: %s', pageInfo.url.href);
-        await this.#db.updatePage(pageInfo, false, false, source);
+        await this.#db.updatePage(pageInfo, false, false, source, undefined, recursive);
     }
     /**
      * Stores a crawled page's data in the archive database, persisting the
@@ -504,11 +552,18 @@ export default class Archive extends ArchiveAccessor {
      * @param bodyHash - Precomputed body hash for the page's HTML (see
      *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
      *   computing it from the HTML instead.
+     * @param recursive - The crawl session's `recursive` option, forwarded so
+     *   any anchor discovered on this page gets the correct
+     *   `is_metadata_only` value (#369). Defaults to `true` (never persist
+     *   an anchor as metadata-only) for callers that do not track the
+     *   option.
      * @returns The database ID of the stored page.
+     * @example
+     * const pageId = await archive.setPage(pageData, undefined, undefined, options.recursive);
      */
-    async setPage(pageInfo, source, bodyHash) {
+    async setPage(pageInfo, source, bodyHash, recursive) {
         dbLog('Set page: %s', pageInfo.url.href);
-        return await this.#db.updatePage(pageInfo, true, pageInfo.isTarget, source, bodyHash);
+        return await this.#db.updatePage(pageInfo, true, pageInfo.isTarget, source, bodyHash, recursive);
     }
     /**
      * Records a redirect edge without re-storing the destination's content.
@@ -610,6 +665,15 @@ export default class Archive extends ArchiveAccessor {
      */
     async write(options) {
         saveLog('Starts: %s', this.#filePath);
+        // `.nitpicker` files are routinely shared between users — scrub the
+        // stub-local `createdCwd` (see `Config.createdCwd`'s JSDoc) before it
+        // gets folded into `db.sqlite` and tarred, so a packaged archive never
+        // carries another user's local absolute path. Bundled into the
+        // `checkpoint` step below (a single UPDATE ahead of the WAL fold-back)
+        // rather than its own `onStep` phase — both are near-instant and
+        // reporting them separately would add a phase label for no visible
+        // wait.
+        await this.#db.updateConfig({ createdCwd: null });
         options?.onStep?.('checkpoint');
         await this.#db.checkpoint();
         const filePathWithoutExt = path.resolve(path.dirname(this.#filePath), path.basename(this.#filePath, path.extname(this.#filePath)));
@@ -876,7 +940,14 @@ export default class Archive extends ArchiveAccessor {
                 const db = await Archive.#connectDB(tmpDir, { onLog });
                 const name = (await db.getName()) ||
                     path.basename(targetPath).replace(Archive.TMP_DIR_PREFIX, '');
-                const filePath = path.resolve(process.cwd(), name + '.' + Archive.FILE_EXTENSION);
+                // Reconstruct the output path from the cwd the interrupted
+                // session was originally started from (see `Config.createdCwd`),
+                // not this invocation's own `process.cwd()` — `crawl --resume
+                // <stub>` is routinely run from a different directory than the
+                // original `crawl`/`--append`/etc. call. Falls back to this
+                // invocation's cwd for a stub that predates this column.
+                const createdCwd = await db.getCreatedCwd();
+                const filePath = path.resolve(createdCwd ?? process.cwd(), name + '.' + Archive.FILE_EXTENSION);
                 return new Archive(filePath, tmpDir, db, releaseLock);
             }
             catch (error) {

package/lib/archive/create-adjunct-tables.d.ts

@@ -23,7 +23,8 @@ import type { Knex } from 'knex';
  *   found in the main-content region, one row per DOM element, FK →
  *   `content_items(id)`. Unlike its eight siblings above, captured by
  *   nitpicker itself (`crawler/capture-custom-elements.ts`), not beholder
- * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `list_reconcile_runs` — `--inventory` / `--recrawl` audit log (no FK;
+ *   append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
  * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
@@ -60,7 +61,7 @@ import type { Knex } from 'knex';
  * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
  * a single sentinel table), each table here is guarded individually because
  * the migration-script caller sees archives where any subset may already
- * exist (e.g. `page_jsonld` from the 0.10 migration but no `inventory_runs`).
+ * exist (e.g. `page_jsonld` from the 0.10 migration but no `list_reconcile_runs`).
  * Index creation stays inside each guard: an existing table keeps whatever
  * indexes its creation path declared.
  * @param instance - The Knex query builder instance connected to the database.

package/lib/archive/create-adjunct-tables.js

@@ -22,7 +22,8 @@
  *   found in the main-content region, one row per DOM element, FK →
  *   `content_items(id)`. Unlike its eight siblings above, captured by
  *   nitpicker itself (`crawler/capture-custom-elements.ts`), not beholder
- * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
+ * - `list_reconcile_runs` — `--inventory` / `--recrawl` audit log (no FK;
+ *   append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
  * - `dedupe_cap_events` — `--dedupe-cap` same-cluster soft-cap audit log (no
@@ -59,7 +60,7 @@
  * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
  * a single sentinel table), each table here is guarded individually because
  * the migration-script caller sees archives where any subset may already
- * exist (e.g. `page_jsonld` from the 0.10 migration but no `inventory_runs`).
+ * exist (e.g. `page_jsonld` from the 0.10 migration but no `list_reconcile_runs`).
  * Index creation stays inside each guard: an existing table keeps whatever
  * indexes its creation path declared.
  * @param instance - The Knex query builder instance connected to the database.
@@ -340,13 +341,15 @@ export async function createAdjunctTables(instance) {
             t.index('pageId');
         });
     }
-    if (!(await instance.schema.hasTable('inventory_runs'))) {
-        await instance.schema.createTable('inventory_runs', (t) => {
-            // One row per successful `--inventory <list>` invocation. The
-            // archive's audit log of "when did we apply which deploy list
-            // at what scale". `.bak` is removed on success so this table
-            // is the only durable provenance record. Column semantics live
-            // on the `InventoryRunMeta` interface in `archive/types.ts`.
+    if (!(await instance.schema.hasTable('list_reconcile_runs'))) {
+        await instance.schema.createTable('list_reconcile_runs', (t) => {
+            // One row per successful `--inventory <list>` or `--recrawl
+            // <list>` invocation — both reconcile a URL list against the
+            // archive via the same ingestion path. The archive's audit log
+            // of "when did we apply which list at what scale". `.bak` is
+            // removed on success so this table is the only durable
+            // provenance record. Column semantics live on the
+            // `ListReconcileRunMeta` interface in `archive/types.ts`.
             t.increments('id');
             t.string('ran_at').notNullable();
             t.string('list_label').nullable();

package/lib/archive/create-entity-tables.js

@@ -195,6 +195,7 @@ export async function createEntityTables(instance) {
 			is_external      INTEGER NOT NULL,
 			scraped          INTEGER NOT NULL,
 			is_target        INTEGER NOT NULL,
+			is_metadata_only INTEGER NOT NULL DEFAULT 0,
 			status           INTEGER,
 			status_text      TEXT,
 			content_type_id  INTEGER REFERENCES content_type_refs(id),
@@ -305,6 +306,8 @@ export async function createEntityTables(instance) {
 			main_content_custom_element_count INTEGER,
 			scroll_height_desktop       INTEGER,
 			scroll_height_mobile        INTEGER,
+			image_scan_desktop          INTEGER,
+			image_scan_mobile           INTEGER,
 			body_hash                   BLOB,
 			console_error_count         INTEGER
 		)

package/lib/archive/database.d.ts

@@ -1,6 +1,6 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
 import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentCustomElementRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, PageTechnologyRow, TechnologySignalRow } from './meta/types.js';
-import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
+import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, DedupeCapObservationRow, InsertDedupeCapEventParams, InsertNetworkOutageParams, ListReconcileRunMeta, PageFilter, PageSource, ResetPagesByUrlsResult } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
 import type { ConsoleLogEntry } from '@d-zero/beholder';
@@ -108,13 +108,22 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * Retrieves the current crawling state by listing scraped and pending URLs.
      * Delegates to {@link getCrawlingStateOp} — see the op for the strict
      * pending-set rationale.
-     * @returns An object with `scraped` (completed URLs) and `pending` (the
-     *   strict set of in-scope, anchor-referenced, unfinished URLs).
+     * @returns An object with `scraped` (completed URLs), `pending` (the
+     *   strict set of in-scope, anchor-referenced, unfinished URLs), and
+     *   `pendingMetadataOnly` (the subset of `pending` fated for a
+     *   metadata-only scrape — see the op's doc).
      */
     getCrawlingState(): Promise<{
         scraped: string[];
         pending: string[];
+        pendingMetadataOnly: string[];
     }>;
+    /**
+     * Retrieves the `info.createdCwd` value stamped at stub-creation time.
+     * Delegates to {@link getCreatedCwdOp}.
+     * @returns The recorded cwd, or `null` if never stamped.
+     */
+    getCreatedCwd(): Promise<string | null>;
     /**
      * Retrieves all `page_main_content_custom_elements` rows for the given
      * page id. Delegates to {@link getCustomElementsOfPageOp}.
@@ -370,6 +379,17 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param pageUrl - The URL of the page that references the resource.
      */
     insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Every previously-scraped internal page's raw fields, in the shape
+     * `buildDedupeCapObservation` needs to replay this archive's prior
+     * `DedupeCapTracker` observations into a fresh tracker instance.
+     * Delegates to {@link listDedupeCapObservationsOp} — see that function
+     * for the row selection criteria.
+     * @param onProgress - Forwarded to {@link listDedupeCapObservationsOp} —
+     *   see that function's docs.
+     * @returns Every qualifying page's raw fields, in `content_items.id` order.
+     */
+    listDedupeCapObservations(onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<DedupeCapObservationRow[]>;
     /**
      * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
      * Delegates to {@link listDedupeCapShapeKeysOp}.
@@ -392,12 +412,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      */
     listNetworkOutages(): Promise<OutageWindow[]>;
     /**
-     * Appends one row to the `inventory_runs` audit log.
-     * Delegates to {@link recordInventoryRunOp}.
+     * Appends one row to the `list_reconcile_runs` audit log.
+     * Delegates to {@link recordListReconcileRunOp}.
      * @param meta - The run metadata to record. Only `ran_at` is required.
      * @returns The autoincremented `id` of the newly-inserted row.
      */
-    recordInventoryRun(meta: InventoryRunMeta): Promise<number>;
+    recordListReconcileRun(meta: ListReconcileRunMeta): Promise<number>;
     /**
      * Records a redirect edge (source → destination) **without** re-storing the
      * destination's content. Delegates to {@link recordRedirectOp}.
@@ -461,6 +481,17 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @returns The URLs of the pages that were reset to pending.
      */
     resetFailedPages(onProgress?: (processed: number, total: number) => void): Promise<string[]>;
+    /**
+     * Reset pages matching an operator-supplied URL list back to pending so a
+     * follow-up crawl re-fetches them from scratch. Delegates to
+     * {@link resetPagesByUrlsOp} — see the op for the conservative exclusion
+     * rationale (redirect sources / intentionally-skipped / external pages).
+     * @param urls - URL strings to match, already in `withoutHashAndAuth` form.
+     * @param onProgress - Forwarded to {@link resetPagesByUrlsOp} — see that
+     *   function's docs.
+     * @returns The reset URLs plus the excluded URLs grouped by reason.
+     */
+    resetPagesByUrls(urls: readonly string[], onProgress?: (processed: number, total: number) => void): Promise<ResetPagesByUrlsResult>;
     /**
      * Stores the crawl configuration in the `info` table.
      * Delegates to {@link setConfigOp}.
@@ -503,9 +534,10 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param bodyHash - Precomputed body hash for the page's HTML (see
      *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
      *   computing it from the HTML instead.
+     * @param recursive - See {@link updatePageOp}'s `recursive` doc.
      * @returns The database `pageId` of the inserted/updated row.
      */
-    updatePage(page: PageData, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null): Promise<number>;
+    updatePage(page: PageData, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null, recursive?: boolean): Promise<number>;
     /**
      * Creates and initializes a new Database instance.
      *

package/lib/archive/database.js

@@ -12,6 +12,7 @@ import { replacePageTemplates as replacePageTemplatesOp } from './db-ops/analysi
 import { getAnchorsOnPage as getAnchorsOnPageOp } from './db-ops/anchors/get-anchors-on-page.js';
 import { getBaseUrl as getBaseUrlOp } from './db-ops/config/get-base-url.js';
 import { getConfig as getConfigOp } from './db-ops/config/get-config.js';
+import { getCreatedCwd as getCreatedCwdOp } from './db-ops/config/get-created-cwd.js';
 import { getName as getNameOp } from './db-ops/config/get-name.js';
 import { setConfig as setConfigOp } from './db-ops/config/set-config.js';
 import { updateConfig as updateConfigOp } from './db-ops/config/update-config.js';
@@ -19,15 +20,16 @@ import { replaceConsoleLogs as replaceConsoleLogsOp } from './db-ops/console-log
 import { accumulateDedupeCapRejectedCount as accumulateDedupeCapRejectedCountOp } from './db-ops/dedupe-cap/accumulate-dedupe-cap-rejected-count.js';
 import { finalizeDedupeCapEvent as finalizeDedupeCapEventOp } from './db-ops/dedupe-cap/finalize-dedupe-cap-event.js';
 import { insertDedupeCapEvent as insertDedupeCapEventOp } from './db-ops/dedupe-cap/insert-dedupe-cap-event.js';
+import { listDedupeCapObservations as listDedupeCapObservationsOp } from './db-ops/dedupe-cap/list-dedupe-cap-observations.js';
 import { listDedupeCapShapeKeys as listDedupeCapShapeKeysOp } from './db-ops/dedupe-cap/list-dedupe-cap-shape-keys.js';
 import { insertCrawlError as insertCrawlErrorOp } from './db-ops/errors/insert-crawl-error.js';
 import { insertPageError as insertPageErrorOp } from './db-ops/errors/insert-page-error.js';
 import { listDnsBurnedHostCandidates as listDnsBurnedHostCandidatesOp } from './db-ops/errors/list-dns-burned-host-candidates.js';
 import { getHtmlOfPageById as getHtmlOfPageByIdOp } from './db-ops/html/get-html-of-page-by-id.js';
-import { recordInventoryRun as recordInventoryRunOp } from './db-ops/inventory/record-inventory-run.js';
 import { checkpoint as checkpointOp } from './db-ops/lifecycle/checkpoint.js';
 import { destroy as destroyOp } from './db-ops/lifecycle/destroy.js';
 import { init as initOp } from './db-ops/lifecycle/init.js';
+import { recordListReconcileRun as recordListReconcileRunOp } from './db-ops/list-reconcile/record-list-reconcile-run.js';
 import { getAudiosOfPage as getAudiosOfPageOp } from './db-ops/meta/get-audios-of-page.js';
 import { getButtonsOfPage as getButtonsOfPageOp } from './db-ops/meta/get-buttons-of-page.js';
 import { getCanvasesOfPage as getCanvasesOfPageOp } from './db-ops/meta/get-canvases-of-page.js';
@@ -53,6 +55,7 @@ import { getPages as getPagesOp } from './db-ops/pages/read/get-pages.js';
 import { getScrapedHtmlPageCount as getScrapedHtmlPageCountOp } from './db-ops/pages/read/get-scraped-html-page-count.js';
 import { repromoteExternalPages as repromoteExternalPagesOp } from './db-ops/pages/reset/repromote-external-pages.js';
 import { resetFailedPages as resetFailedPagesOp } from './db-ops/pages/reset/reset-failed-pages.js';
+import { resetPagesByUrls as resetPagesByUrlsOp } from './db-ops/pages/reset/reset-pages-by-urls.js';
 import { insertInventorySeeds as insertInventorySeedsOp } from './db-ops/pages/write/insert-inventory-seeds.js';
 import { insertInventorySkippedPages as insertInventorySkippedPagesOp } from './db-ops/pages/write/insert-inventory-skipped-pages.js';
 import { recordRedirect as recordRedirectOp } from './db-ops/pages/write/record-redirect.js';
@@ -223,12 +226,22 @@ export class Database extends EventEmitter {
      * Retrieves the current crawling state by listing scraped and pending URLs.
      * Delegates to {@link getCrawlingStateOp} — see the op for the strict
      * pending-set rationale.
-     * @returns An object with `scraped` (completed URLs) and `pending` (the
-     *   strict set of in-scope, anchor-referenced, unfinished URLs).
+     * @returns An object with `scraped` (completed URLs), `pending` (the
+     *   strict set of in-scope, anchor-referenced, unfinished URLs), and
+     *   `pendingMetadataOnly` (the subset of `pending` fated for a
+     *   metadata-only scrape — see the op's doc).
      */
     async getCrawlingState() {
         return emitErrorAndRetry(this, 'Database.getCrawlingState', async () => await getCrawlingStateOp(this.#instance), retrySetting);
     }
+    /**
+     * Retrieves the `info.createdCwd` value stamped at stub-creation time.
+     * Delegates to {@link getCreatedCwdOp}.
+     * @returns The recorded cwd, or `null` if never stamped.
+     */
+    async getCreatedCwd() {
+        return emitErrorAndRetry(this, 'Database.getCreatedCwd', async () => await getCreatedCwdOp(this.#instance), retrySetting);
+    }
     /**
      * Retrieves all `page_main_content_custom_elements` rows for the given
      * page id. Delegates to {@link getCustomElementsOfPageOp}.
@@ -557,6 +570,19 @@ export class Database extends EventEmitter {
     async insertResourceReferrers(src, pageUrl) {
         return emitErrorAndRetry(this, 'Database.insertResourceReferrers', async () => await insertResourceReferrersOp(this.#instance, this.#writeRefCaches, src, pageUrl), retrySetting);
     }
+    /**
+     * Every previously-scraped internal page's raw fields, in the shape
+     * `buildDedupeCapObservation` needs to replay this archive's prior
+     * `DedupeCapTracker` observations into a fresh tracker instance.
+     * Delegates to {@link listDedupeCapObservationsOp} — see that function
+     * for the row selection criteria.
+     * @param onProgress - Forwarded to {@link listDedupeCapObservationsOp} —
+     *   see that function's docs.
+     * @returns Every qualifying page's raw fields, in `content_items.id` order.
+     */
+    async listDedupeCapObservations(onProgress) {
+        return emitErrorAndRetry(this, 'Database.listDedupeCapObservations', async () => await listDedupeCapObservationsOp(this.#instance, onProgress), retrySetting);
+    }
     /**
      * Every distinct `dedupe_cap_events.shape_key` recorded in this archive.
      * Delegates to {@link listDedupeCapShapeKeysOp}.
@@ -585,13 +611,13 @@ export class Database extends EventEmitter {
         return emitErrorAndRetry(this, 'Database.listNetworkOutages', async () => await listNetworkOutagesOp(this.#instance), retrySetting);
     }
     /**
-     * Appends one row to the `inventory_runs` audit log.
-     * Delegates to {@link recordInventoryRunOp}.
+     * Appends one row to the `list_reconcile_runs` audit log.
+     * Delegates to {@link recordListReconcileRunOp}.
      * @param meta - The run metadata to record. Only `ran_at` is required.
      * @returns The autoincremented `id` of the newly-inserted row.
      */
-    async recordInventoryRun(meta) {
-        return emitErrorAndRetry(this, 'Database.recordInventoryRun', async () => await recordInventoryRunOp(this.#instance, meta), retrySetting);
+    async recordListReconcileRun(meta) {
+        return emitErrorAndRetry(this, 'Database.recordListReconcileRun', async () => await recordListReconcileRunOp(this.#instance, meta), retrySetting);
     }
     /**
      * Records a redirect edge (source → destination) **without** re-storing the
@@ -662,6 +688,19 @@ export class Database extends EventEmitter {
     async resetFailedPages(onProgress) {
         return emitErrorAndRetry(this, 'Database.resetFailedPages', async () => await resetFailedPagesOp(this.#instance, onProgress), retrySetting);
     }
+    /**
+     * Reset pages matching an operator-supplied URL list back to pending so a
+     * follow-up crawl re-fetches them from scratch. Delegates to
+     * {@link resetPagesByUrlsOp} — see the op for the conservative exclusion
+     * rationale (redirect sources / intentionally-skipped / external pages).
+     * @param urls - URL strings to match, already in `withoutHashAndAuth` form.
+     * @param onProgress - Forwarded to {@link resetPagesByUrlsOp} — see that
+     *   function's docs.
+     * @returns The reset URLs plus the excluded URLs grouped by reason.
+     */
+    async resetPagesByUrls(urls, onProgress) {
+        return emitErrorAndRetry(this, 'Database.resetPagesByUrls', async () => await resetPagesByUrlsOp(this.#instance, urls, onProgress), retrySetting);
+    }
     /**
      * Stores the crawl configuration in the `info` table.
      * Delegates to {@link setConfigOp}.
@@ -712,10 +751,11 @@ export class Database extends EventEmitter {
      * @param bodyHash - Precomputed body hash for the page's HTML (see
      *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
      *   computing it from the HTML instead.
+     * @param recursive - See {@link updatePageOp}'s `recursive` doc.
      * @returns The database `pageId` of the inserted/updated row.
      */
-    async updatePage(page, writeHtml, isTarget, source, bodyHash) {
-        return emitErrorAndRetry(this, 'Database.updatePage', async () => await updatePageOp(this.#instance, this.#writeRefCaches, page, writeHtml, isTarget, source, bodyHash), retrySetting);
+    async updatePage(page, writeHtml, isTarget, source, bodyHash, recursive) {
+        return emitErrorAndRetry(this, 'Database.updatePage', async () => await updatePageOp(this.#instance, this.#writeRefCaches, page, writeHtml, isTarget, source, bodyHash, recursive), retrySetting);
     }
     /**
      * Initializes the database schema if tables do not exist, then runs