@nitpicker/archive 0.0.0 → 0.25.0

package/lib/db-ops/outages/close-network-outage.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export declare function closeNetworkOutage(knex: Knex, id: number, endedAt: number): Promise<void>;

package/lib/db-ops/outages/close-network-outage.js

@@ -0,0 +1,15 @@
+/**
+ * Close an outage row by stamping `ended_at` — but ONLY if it is still
+ * open. The `whereNull('ended_at')` guard is what makes this idempotent: a
+ * second call (e.g. a duplicate recovery-probe success racing the first)
+ * matches zero rows and is a silent no-op, rather than overwriting an
+ * already-recorded `ended_at` with a later timestamp.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param id - The `network_outages.id` to close.
+ * @param endedAt - Epoch ms the outage is considered to have ended.
+ */
+export async function closeNetworkOutage(knex, id, endedAt) {
+    await knex('network_outages').where({ id }).whereNull('ended_at').update({
+        ended_at: endedAt,
+    });
+}

package/lib/db-ops/outages/close-stale-open-network-outages.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export declare function closeStaleOpenNetworkOutages(knex: Knex): Promise<void>;

package/lib/db-ops/outages/close-stale-open-network-outages.js

@@ -0,0 +1,42 @@
+import { closeNetworkOutage } from './close-network-outage.js';
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * Durably close every `network_outages` row still `ended_at = NULL` at the
+ * start of a writer session.
+ *
+ * A row is left open only when the crawl process was killed (Ctrl-C / OOM
+ * / SIGKILL) mid-outage, before a recovery probe could close it. Called
+ * once from `db-ops/lifecycle/init.ts` — right after `initSchema` (which
+ * guarantees the table exists) and before anything else touches
+ * `network_outages` — so that by the time `resetFailedPages` /
+ * `listDnsBurnedHostCandidates` / any other reader runs, no row can still
+ * be open from a PRIOR session. (A row opened by THIS session cannot exist
+ * yet at this point in the boot sequence — the sliding-window detector
+ * only starts once the crawl loop begins.)
+ *
+ * This complements, but does not replace, `list-network-outages.ts`'s
+ * on-the-fly clamp: that read-side resolution stays in place as a
+ * defensive fallback, but after this runs there should be nothing left for
+ * it to resolve.
+ *
+ * Idempotent: a row closed by a previous call (or by a normal
+ * recovery-probe success) is simply absent from the `whereNull('ended_at')`
+ * scan, so re-running this on every writer open is always safe.
+ * @param knex - Knex query builder connected to the archive DB.
+ */
+export async function closeStaleOpenNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return;
+    }
+    const openRows = (await knex('network_outages')
+        .whereNull('ended_at')
+        .select('id', 'started_at'));
+    if (openRows.length === 0) {
+        return;
+    }
+    const clamp = await computeOutageClampTimestamp(knex);
+    for (const row of openRows) {
+        await closeNetworkOutage(knex, row.id, Math.max(clamp, row.started_at));
+    }
+}

package/lib/db-ops/outages/compute-outage-clamp-timestamp.d.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export declare function computeOutageClampTimestamp(knex: Knex): Promise<number>;

package/lib/db-ops/outages/compute-outage-clamp-timestamp.js

@@ -0,0 +1,31 @@
+/**
+ * Compute the timestamp used to resolve an outage row whose `ended_at` is
+ * still `NULL` (the crawl process was killed before a recovery probe could
+ * close it) into a concrete, bounded window.
+ *
+ * The archive's own most-recent observations — the latest `crawl_errors`
+ * timestamp and the latest `content_items.last_crawled_at` — are the only
+ * evidence available for "when did activity in this archive last happen",
+ * so the later of the two stands in for "the outage cannot have lasted
+ * past this point, because the archive shows activity here". Using the
+ * LARGER of the two (not just `crawl_errors`) matters because a session
+ * that crashed mid-outage may have its last successful page write be more
+ * recent than its last recorded error, or vice versa.
+ *
+ * Both source tables are guaranteed to exist by the time this runs — this
+ * helper is only ever called from writer-context code (see
+ * `list-network-outages.ts` / the boot-time stale-open finalizer), and
+ * `initSchema` creates both `crawl_errors` and `content_items` before any
+ * writer session's first query.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns The larger of `MAX(crawl_errors.createdAt)` and
+ *   `MAX(content_items.last_crawled_at)`, or `0` if the archive has neither
+ *   (a brand-new archive with no activity yet).
+ */
+export async function computeOutageClampTimestamp(knex) {
+    const [crawlErrorsRow] = (await knex('crawl_errors').max('createdAt as maxCreatedAt'));
+    const [contentItemsRow] = (await knex('content_items').max('last_crawled_at as maxLastCrawledAt'));
+    const latestError = crawlErrorsRow?.maxCreatedAt ?? 0;
+    const latestCrawl = contentItemsRow?.maxLastCrawledAt ?? 0;
+    return Math.max(latestError, latestCrawl);
+}

package/lib/db-ops/outages/insert-network-outage.d.ts

@@ -0,0 +1,14 @@
+import type { InsertNetworkOutageParams } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function insertNetworkOutage(knex: Knex, params: InsertNetworkOutageParams): Promise<number>;

package/lib/db-ops/outages/insert-network-outage.js

@@ -0,0 +1,29 @@
+/**
+ * Appends one row to the `network_outages` journal, with `ended_at` left
+ * `NULL` — the row starts life as an open outage.
+ *
+ * Called the moment a recovery probe CONFIRMS a suspect outage (i.e. the
+ * probe itself failed, not merely the sliding-window threshold trip). See
+ * `close-network-outage.ts` for how the row is later closed.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - The confirmed-outage fields to record.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export async function insertNetworkOutage(knex, params) {
+    const inserted = await knex
+        .from('network_outages')
+        .insert({
+        started_at: params.startedAt,
+        detected_at: params.detectedAt,
+        ended_at: null,
+        probe_host: params.probeHost,
+        trigger_error_count: params.triggerErrorCount,
+        trigger_host_count: params.triggerHostCount,
+    })
+        .returning('id');
+    const id = inserted[0]?.id;
+    if (typeof id !== 'number') {
+        throw new TypeError('insertNetworkOutage: INSERT returned no row id');
+    }
+    return id;
+}

package/lib/db-ops/outages/list-network-outages.d.ts

@@ -0,0 +1,28 @@
+import type { OutageWindow } from '../../is-within-outage-window.js';
+import type { Knex } from 'knex';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export declare function listNetworkOutages(knex: Knex): Promise<OutageWindow[]>;

package/lib/db-ops/outages/list-network-outages.js

@@ -0,0 +1,42 @@
+import { computeOutageClampTimestamp } from './compute-outage-clamp-timestamp.js';
+/**
+ * List every recorded outage as a resolved (closed) {@link OutageWindow},
+ * suitable for `isWithinOutageWindow`.
+ *
+ * This is the crawler-internal counterpart to `@nitpicker/query`'s richer
+ * `listNetworkOutages` (full row shape, for CLI/MCP display) — this one
+ * exists purely to feed the three write-path consumers
+ * (`resetFailedPages`, `listDnsBurnedHostCandidates`, the gate itself) that
+ * only need "was this timestamp inside an outage", never the row's other
+ * columns.
+ *
+ * Any row whose `ended_at` is still `NULL` (the crawl process was killed
+ * before recovery) is resolved on the fly via
+ * {@link computeOutageClampTimestamp} rather than ever being returned as an
+ * unbounded window — see `is-within-outage-window.ts`'s `OutageWindow`
+ * docstring for why an open-ended window would be a correctness bug (every
+ * later error would retroactively read as network-caused, forever). This
+ * on-the-fly resolution does NOT persist to the row — it is a defensive
+ * fallback independent of whichever boot-time finalizer durably closes
+ * stale-open rows.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns Resolved outage windows. Empty when the archive predates the
+ *   `network_outages` table (self-healed on next writer open, so this is
+ *   never a permanent state) or has recorded no outages.
+ */
+export async function listNetworkOutages(knex) {
+    const hasTable = await knex.schema.hasTable('network_outages');
+    if (!hasTable) {
+        return [];
+    }
+    const rows = (await knex('network_outages').select('started_at', 'ended_at'));
+    if (rows.length === 0) {
+        return [];
+    }
+    const hasOpenRow = rows.some((row) => row.ended_at === null);
+    const clamp = hasOpenRow ? await computeOutageClampTimestamp(knex) : 0;
+    return rows.map((row) => ({
+        startedAt: row.started_at,
+        endedAt: row.ended_at ?? Math.max(clamp, row.started_at),
+    }));
+}

package/lib/db-ops/pages/order/set-url-order.d.ts

@@ -0,0 +1,13 @@
+import type { Knex } from 'knex';
+/**
+ * Assigns natural URL sort order values (`content_items.crawl_order`) to
+ * all internal pages. Pages are sorted using {@link pathComparator} and
+ * assigned sequential order numbers.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's UPDATE completes, with
+ *   pages ordered so far and the total (issue #294: a large archive's page
+ *   count can make this a multi-second-to-minutes pass — a single SELECT
+ *   plus a JS sort followed by chunked UPDATEs — with no other signal it
+ *   hasn't hung). Omit for no reporting (the default; e.g. tests).
+ */
+export declare function setUrlOrder(knex: Knex, onProgress?: (processed: number, total: number) => void): Promise<void>;

package/lib/db-ops/pages/order/set-url-order.js

@@ -0,0 +1,38 @@
+import { pathComparator } from '@d-zero/shared/sort/path';
+import { dbLog } from '../../../debug.js';
+/**
+ * Assigns natural URL sort order values (`content_items.crawl_order`) to
+ * all internal pages. Pages are sorted using {@link pathComparator} and
+ * assigned sequential order numbers.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's UPDATE completes, with
+ *   pages ordered so far and the total (issue #294: a large archive's page
+ *   count can make this a multi-second-to-minutes pass — a single SELECT
+ *   plus a JS sort followed by chunked UPDATEs — with no other signal it
+ *   hasn't hung). Omit for no reporting (the default; e.g. tests).
+ */
+export async function setUrlOrder(knex, onProgress) {
+    dbLog('Set URL Order');
+    const res = (await knex
+        .select('ci.id', 'ur.url')
+        .from('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .where('ci.is_external', '=', 0));
+    const sorted = res.toSorted((a, b) => pathComparator(a.url, b.url));
+    // Batch update using chunked CASE statements to avoid N+1 queries
+    const BATCH_SIZE = 500;
+    for (let i = 0; i < sorted.length; i += BATCH_SIZE) {
+        const batch = sorted.slice(i, i + BATCH_SIZE);
+        const ids = batch.map((row) => row.id);
+        const bindings = [];
+        const cases = batch
+            .map((row, j) => {
+            bindings.push(row.id, i + j + 1);
+            return 'WHEN ? THEN ?';
+        })
+            .join(' ');
+        const placeholders = ids.map(() => '?').join(',');
+        await knex.raw(`UPDATE content_items SET crawl_order = CASE id ${cases} END WHERE id IN (${placeholders})`, [...bindings, ...ids]);
+        onProgress?.(Math.min(i + BATCH_SIZE, sorted.length), sorted.length);
+    }
+}

package/lib/db-ops/pages/read/build-page-query.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Builds a `content_items` query joined against `page_meta` and every ref
+ * table needed to reconstruct a flat, legacy-shaped page row. Selected
+ * columns are aliased to the `DB_Page` field names; `responseHeaders` and
+ * `meta_extras` are NOT reconstructed here (they need a second, batched
+ * pass — see {@link ../read/reconstruct-page-rows.js}) so this query stays
+ * a single-pass join with no N+1 subqueries. `headerSetId` and the
+ * `extras_body` / `extras_codec` pair are selected as intermediates for
+ * that second pass.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns A query builder pre-configured with every join `get-pages.ts` /
+ *   `get-pages-with-rels.ts` need; callers add `.where()` / `.limit()` /
+ *   `.offset()` on top.
+ * @example
+ * const rows = await buildPageQuery(knex).where('ci.is_target', 1);
+ */
+export declare function buildPageQuery(knex: Knex): Knex.QueryBuilder;

package/lib/db-ops/pages/read/build-page-query.js

@@ -0,0 +1,40 @@
+/**
+ * Builds a `content_items` query joined against `page_meta` and every ref
+ * table needed to reconstruct a flat, legacy-shaped page row. Selected
+ * columns are aliased to the `DB_Page` field names; `responseHeaders` and
+ * `meta_extras` are NOT reconstructed here (they need a second, batched
+ * pass — see {@link ../read/reconstruct-page-rows.js}) so this query stays
+ * a single-pass join with no N+1 subqueries. `headerSetId` and the
+ * `extras_body` / `extras_codec` pair are selected as intermediates for
+ * that second pass.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns A query builder pre-configured with every join `get-pages.ts` /
+ *   `get-pages-with-rels.ts` need; callers add `.where()` / `.limit()` /
+ *   `.offset()` on top.
+ * @example
+ * const rows = await buildPageQuery(knex).where('ci.is_target', 1);
+ */
+export function buildPageQuery(knex) {
+    return knex('content_items as ci')
+        .join('url_refs as ur', 'ur.id', 'ci.url_id')
+        .leftJoin('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+        .leftJoin('page_meta as pm', 'pm.page_id', 'ci.id')
+        .leftJoin('text_refs as title_ref', 'title_ref.id', 'pm.title_text_id')
+        .leftJoin('text_refs as description_ref', 'description_ref.id', 'pm.description_text_id')
+        .leftJoin('text_refs as keywords_ref', 'keywords_ref.id', 'pm.keywords_text_id')
+        .leftJoin('text_refs as robots_raw_ref', 'robots_raw_ref.id', 'pm.robots_raw_text_id')
+        .leftJoin('text_refs as og_title_ref', 'og_title_ref.id', 'pm.og_title_text_id')
+        .leftJoin('text_refs as og_description_ref', 'og_description_ref.id', 'pm.og_description_text_id')
+        .leftJoin('text_refs as twitter_title_ref', 'twitter_title_ref.id', 'pm.twitter_title_text_id')
+        .leftJoin('text_refs as twitter_description_ref', 'twitter_description_ref.id', 'pm.twitter_description_text_id')
+        .leftJoin('url_refs as canonical_ur', 'canonical_ur.id', 'pm.canonical_url_id')
+        .leftJoin('url_refs as amphtml_ur', 'amphtml_ur.id', 'pm.amphtml_url_id')
+        .leftJoin('url_refs as manifest_ur', 'manifest_ur.id', 'pm.manifest_url_id')
+        .leftJoin('url_refs as icon_ur', 'icon_ur.id', 'pm.icon_url_id')
+        .leftJoin('url_refs as apple_ur', 'apple_ur.id', 'pm.apple_touch_icon_url_id')
+        .leftJoin('url_refs as og_url_ur', 'og_url_ur.id', 'pm.og_url_id')
+        .leftJoin('url_refs as og_image_ur', 'og_image_ur.id', 'pm.og_image_url_id')
+        .leftJoin('url_refs as twitter_image_ur', 'twitter_image_ur.id', 'pm.twitter_image_url_id')
+        .leftJoin('json_refs as extras_ref', 'extras_ref.id', 'pm.meta_extras_json_id')
+        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.main_content_custom_element_count as main_content_custom_element_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'pm.image_scan_desktop as image_scan_desktop', 'pm.image_scan_mobile as image_scan_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
+}

package/lib/db-ops/pages/read/get-crawling-state.d.ts

@@ -0,0 +1,117 @@
+import type { Knex } from 'knex';
+/**
+ * Retrieves the current crawling state by listing scraped and pending URLs.
+ *
+ * `scraped` is straightforward: every page row whose `scraped` flag is `1`
+ * — that is, every URL the crawl reached a terminal state on, including
+ * setSkippedPage / setExternalPage / outright setPage success or failure.
+ *
+ * `pending` is intentionally STRICT — not "every `scraped = 0` row".
+ * Four filters apply:
+ *
+ * 1. `scraped = 0` — work still incomplete.
+ * 2. `is_external = 0` — only in-scope work. External URLs go through a
+ *    HEAD-only path that always lands on `scraped = 1` (either setPage or
+ *    setExternalPage). A row with `is_external = 1 AND scraped = 0` is
+ *    therefore a data anomaly, and resume / inventory / append have no
+ *    business retrying it on the next session.
+ * 3. `EXISTS (anchor_edges with href_page_id = content_items.id) OR source
+ *    != 'crawled'` — the row was either discovered as an anchor destination
+ *    during a previous scrape OR was explicitly tagged with a non-default
+ *    source label (`'inventory-seed'`, `'inventory-discovered'`, …). Both
+ *    halves of the OR represent "deliberately enqueued, expected to be
+ *    processed", which is exactly what `resume` should pick up.
+ *
+ *    The orphan filter targets the **predicted-discard leak** in
+ *    `@nitpicker/crawler`'s `crawler/crawler.ts` where `shouldDiscardPredicted` returns true but no
+ *    `emit('skip')` follows. Such placeholders are inserted with the
+ *    DB DEFAULT `source = 'crawled'` (no caller explicitly labels
+ *    them) AND have no anchor referrer (predicted URLs are
+ *    synthesised from pagination patterns, never anchored from a
+ *    rendered page) — both halves of the OR are therefore false and
+ *    the leak is excluded.
+ *
+ *    The `source != 'crawled'` clause specifically saves the
+ *    `--inventory` × `--retry-failed` interaction: an inventory-seed
+ *    URL came from the operator's URL list (no anchor referrer) and
+ *    `resetFailedPages` puts it back at `scraped = 0`. Without this
+ *    clause those legitimate retries would be dropped on resume.
+ * 4. **Confirmed same-cluster trap exclusion** (issue #350): a row whose
+ *    URL shape (`computeShapeKey`) matches a `dedupe_cap_events.shape_key`
+ *    already recorded in this archive is dropped, UNLESS its `source` is
+ *    `'inventory-seed'` (an operator-listed URL, not an anchor-discovered
+ *    one — the same "explicit URL survives cap" carve-out `resetPagesByUrls`
+ *    gives `--recrawl`). NOT the same as `resetFailedPages`'s own
+ *    same-shape exclusion, which drops a matching candidate regardless of
+ *    `source` — a `scraped=1, status=-1` inventory-seed row of a
+ *    confirmed-capped shape is therefore excluded there too and never
+ *    resets to `scraped=0` for `--retry-failed` to pick back up. That
+ *    asymmetry pre-dates this filter and is unrelated to it (out of scope
+ *    here): this filter only decides what `getCrawlingState` reports as
+ *    `pending`, never whether a `scraped=1` row gets reset in the first
+ *    place.
+ *
+ *    Why this is needed: `--dedupe-cap`'s enqueue gate (`@nitpicker/crawler`'s `crawler/crawler.ts`
+ *    `addUrl` closure) only stops a capped shape's anchor from being
+ *    ADDED TO THE QUEUE in memory — it does not stop `replaceAnchorEdges`
+ *    (`update-page.ts`) from writing a `scraped = 0` row for that same
+ *    anchor (post-hoc marking, `content_items.dedupe_cap_event_id`,
+ *    deliberately needs that row to exist). Before this filter, such a
+ *    row still satisfied filter 3 (it has an anchor referrer) and entered
+ *    `pending`, so `CrawlerOrchestrator#crawlUntilPendingClears`'s
+ *    auto-retry loop (and any `--resume`/`--retry-failed`/`--append`) would
+ *    requeue it — `Crawler#resume`'s `LinkList#resume` has no cap gate of
+ *    its own — actually fetch the trap page, and its anchors would create
+ *    a fresh batch of same-shape `scraped = 0` rows. Pending never shrank,
+ *    so the loop ended in `PendingUrlsRemainError('no-progress')` even
+ *    though nothing was actually wrong with the target site.
+ *
+ *    Why not skip the row instead of merely omitting it from `pending`:
+ *    this function is read-only — leaving `is_skipped`/`skip_reason`
+ *    unset keeps `dedupe_cap_event_id` backfill (`viewer-build`) as the
+ *    single place that judges a row "capped", matching the archive's
+ *    existing dedupe-cap post-hoc-marking pattern rather than adding a
+ *    second one here.
+ *
+ *    A row whose shape cannot be computed (`computeShapeKey` returns
+ *    `null`) stays in `pending` — no signal either way, same
+ *    err-on-the-side-of-retrying choice `resetFailedPages` makes.
+ *
+ * The defensive shape is on purpose: the data source can drift into
+ * anomalous states under interruption, but the reader must never throw
+ * or feed garbage back into the dealer. A real in-scope URL that was
+ * truly interrupted mid-crawl will always have at least one anchor
+ * referrer (otherwise the dealer would not have queued it), so the
+ * strict filter loses no legitimate pending work.
+ *
+ * Seeds passed directly to `Crawler.start()` are NOT in the strict
+ * pending set when they were never picked by the dealer — they have no
+ * DB row at all in that case (`linkList.add` is purely in-memory until
+ * `setPage` runs). A Ctrl-C between dealer pick and `setPage` likewise
+ * leaves no row to recover. Recovery of un-picked seeds is the
+ * responsibility of the caller (e.g. re-running `--inventory ./list.txt`
+ * with the same URL list).
+ *
+ * The query uses an explicit `ci` alias on the `content_items` table so the
+ * correlated `EXISTS` subquery can join via `whereRaw('anchor_edges.href_page_id
+ * = ci.id')`. A future refactor that renames the alias must update both
+ * sites — the raw string in the subquery cannot be grep-resolved
+ * automatically. Read-only / stub viewer connections never call this
+ * method (they do not need to know about pending state), so the EXISTS
+ * shape is safe to use without the `migrate*` guards that other writer
+ * methods carry.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @returns An object with `scraped` (completed URLs), `pending` (the
+ *   strict set of in-scope, anchor-referenced, unfinished, non-capped
+ *   URLs), and `pendingMetadataOnly` (the subset of `pending` whose
+ *   `content_items.is_metadata_only` was persisted as `1` — see
+ *   `replaceAnchorEdges`/`resolveContentItemId`). Callers that resume a
+ *   crawl (`Crawler#resume` → `LinkList#resume`) pass this subset through
+ *   so a metadata-only anchor discovered before an interruption is not
+ *   silently promoted to a full-scrape target on resume (#369).
+ */
+export declare function getCrawlingState(knex: Knex): Promise<{
+    scraped: string[];
+    pending: string[];
+    pendingMetadataOnly: string[];
+}>;