@nitpicker/crawler 0.20.0 → 0.22.0

package/lib/archive/db-ops/lifecycle/init.js

@@ -2,21 +2,25 @@ import { applyConnectionPragmas, initSchema } from '../../init-schema.js';
 import { assertCompatibleVersion } from '../../meta/assert-compatible-version.js';
 import { migrateContentItemsAliasOfId } from '../../migrate-content-items-alias-of-id.js';
 import { migrateContentItemsDedupeCapEventId } from '../../migrate-content-items-dedupe-cap-event-id.js';
+import { migrateContentItemsIsMetadataOnly } from '../../migrate-content-items-is-metadata-only.js';
+import { migrateInfoCreatedCwd } from '../../migrate-info-created-cwd.js';
 import { migrateInfoMainContentSelector } from '../../migrate-info-main-content-selector.js';
 import { migrateInfoRoots } from '../../migrate-info-roots.js';
-import { migrateInventoryRunsExcludeSkipped } from '../../migrate-inventory-runs-exclude-skipped.js';
-import { migrateInventoryRunsInvalidSkipped } from '../../migrate-inventory-runs-invalid-skipped.js';
+import { migrateInventoryRunsToListReconcileRuns } from '../../migrate-inventory-runs-to-list-reconcile-runs.js';
+import { migrateListReconcileRunsExcludeSkipped } from '../../migrate-list-reconcile-runs-exclude-skipped.js';
+import { migrateListReconcileRunsInvalidSkipped } from '../../migrate-list-reconcile-runs-invalid-skipped.js';
 import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
 import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
 import { migratePageMetaConsoleErrorCount } from '../../migrate-page-meta-console-error-count.js';
 import { migratePageMetaCustomElementCount } from '../../migrate-page-meta-custom-element-count.js';
+import { migratePageMetaImageScan } from '../../migrate-page-meta-image-scan.js';
 import { migratePageTagsToPageTechnologies } from '../../migrate-page-tags-to-page-technologies.js';
 import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-network-outages.js';
 /**
  * Initializes the database schema if tables do not exist, then runs the
  * remaining lightweight migrations (`info.roots`, `info.mainContentSelector`,
- * `page_meta.main_content_*`, `inventory_runs.invalid_skipped`,
- * `inventory_runs.exclude_skipped`).
+ * `page_meta.main_content_*`, `list_reconcile_runs.invalid_skipped`,
+ * `list_reconcile_runs.exclude_skipped`).
  *
  * There is deliberately no per-table *table-creation* migration chain here:
  * `assertCompatibleVersion` (called below, before any schema work) rejects
@@ -31,9 +35,10 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
  * `CREATE TABLE IF NOT EXISTS` is a no-op once the table is present. Column
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
- * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
+ * `migrateInfoCreatedCwd`, `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
  * `migratePageMetaConsoleErrorCount`, `migratePageMetaCustomElementCount`,
- * `migrateContentItemsAliasOfId`,
+ * `migratePageMetaImageScan`, `migrateContentItemsAliasOfId`,
+ * `migrateContentItemsIsMetadataOnly`,
  *
  * `migratePageTagsToPageTechnologies` is the one exception to "column adds
  * only": it converts `page_tags` (removed) rows into `technology_signals`/
@@ -41,8 +46,14 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
  * not a column add, but it belongs in this same boot phase for the same
  * reason (self-healing an old archive's schema before any reader runs).
  *
- * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
- * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
+ * `migrateContentItemsDedupeCapEventId`, `migrateListReconcileRunsInvalidSkipped`,
+ * `migrateListReconcileRunsExcludeSkipped`) rather than a DDL-string change alone.
+ *
+ * `migrateInventoryRunsToListReconcileRuns` runs BEFORE `initSchema`
+ * (unlike every other migration here) — see its own JSDoc for why: a
+ * plain rename must land before `createAdjunctTables`'s
+ * `IF NOT EXISTS` guard would otherwise create an empty `list_reconcile_runs`
+ * and strand the old table's rows.
  *
  * `closeStaleOpenNetworkOutages` is not a schema migration (no columns
  * change) but belongs at this same boot phase for the same reason the
@@ -77,13 +88,19 @@ export async function init(knex, readOnly, onLog) {
     if (readOnly) {
         return;
     }
+    // Must run before `initSchema`: see the function's own JSDoc for why a
+    // plain table rename has to land before `createAdjunctTables`'s
+    // `IF NOT EXISTS` guard would otherwise create an empty replacement.
+    await migrateInventoryRunsToListReconcileRuns(knex, onLog);
     await initSchema(knex);
     await migrateInfoRoots(knex, onLog);
     await migrateInfoMainContentSelector(knex, onLog);
+    await migrateInfoCreatedCwd(knex, onLog);
     await migrateMainContentsColumns(knex, onLog);
     await migratePageMetaBodyHash(knex, onLog);
     await migratePageMetaConsoleErrorCount(knex, onLog);
     await migratePageMetaCustomElementCount(knex, onLog);
+    await migratePageMetaImageScan(knex, onLog);
     // Table-level migration (converts + drops page_tags), not a column
     // add — see its own JSDoc for why it still belongs in this boot phase.
     await migratePageTagsToPageTechnologies(knex, onLog);
@@ -93,7 +110,10 @@ export async function init(knex, readOnly, onLog) {
     // column's `REFERENCES dedupe_cap_events(id)` target always exists by
     // this point, for both fresh and legacy archives.
     await migrateContentItemsDedupeCapEventId(knex, onLog);
-    await migrateInventoryRunsInvalidSkipped(knex, onLog);
-    await migrateInventoryRunsExcludeSkipped(knex, onLog);
+    // A plain `hasColumn`-guarded ADD COLUMN, same as `migrateContentItemsAliasOfId`
+    // above — no adjunct-table dependency, so ordering relative to it doesn't matter.
+    await migrateContentItemsIsMetadataOnly(knex, onLog);
+    await migrateListReconcileRunsInvalidSkipped(knex, onLog);
+    await migrateListReconcileRunsExcludeSkipped(knex, onLog);
     await closeStaleOpenNetworkOutages(knex);
 }

package/lib/archive/db-ops/list-reconcile/record-list-reconcile-run.d.ts

@@ -0,0 +1,22 @@
+import type { ListReconcileRunMeta } from '../../types.js';
+import type { Knex } from 'knex';
+/**
+ * Appends one row to the `list_reconcile_runs` audit log.
+ *
+ * Called by `CrawlerOrchestrator.inventory` and `CrawlerOrchestrator.recrawl`
+ * on every successful `--inventory <list>` / `--recrawl <list>` invocation
+ * so the archive carries a durable record of which list was applied when
+ * and at what scale — the operational question "did we apply last month's
+ * list" the archive itself can answer without consulting external
+ * bookkeeping.
+ *
+ * Append-only. There is intentionally no UPDATE path and no UNIQUE
+ * constraint on `source_file_sha256`; two applies of the same list
+ * each get their own row. Duplicate detection is left to readers —
+ * the hash is recorded as the content-identity key they would use.
+ * Field-level NULL semantics live on {@link ListReconcileRunMeta}.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param meta - The run metadata to record. Only `ran_at` is required.
+ * @returns The autoincremented `id` of the newly-inserted row.
+ */
+export declare function recordListReconcileRun(knex: Knex, meta: ListReconcileRunMeta): Promise<number>;

package/lib/archive/db-ops/{inventory/record-inventory-run.js → list-reconcile/record-list-reconcile-run.js}

@@ -1,24 +1,25 @@
 /**
- * Appends one row to the `inventory_runs` audit log.
+ * Appends one row to the `list_reconcile_runs` audit log.
  *
- * Called by `CrawlerOrchestrator.inventory` on every successful
- * `--inventory <list>` invocation so the archive carries a durable
- * record of which deploy list was applied when and at what scale —
- * the operational question "did we apply last month's list" the
- * archive itself can answer without consulting external bookkeeping.
+ * Called by `CrawlerOrchestrator.inventory` and `CrawlerOrchestrator.recrawl`
+ * on every successful `--inventory <list>` / `--recrawl <list>` invocation
+ * so the archive carries a durable record of which list was applied when
+ * and at what scale — the operational question "did we apply last month's
+ * list" the archive itself can answer without consulting external
+ * bookkeeping.
  *
  * Append-only. There is intentionally no UPDATE path and no UNIQUE
  * constraint on `source_file_sha256`; two applies of the same list
  * each get their own row. Duplicate detection is left to readers —
  * the hash is recorded as the content-identity key they would use.
- * Field-level NULL semantics live on {@link InventoryRunMeta}.
+ * Field-level NULL semantics live on {@link ListReconcileRunMeta}.
  * @param knex - Knex query builder connected to the archive DB.
  * @param meta - The run metadata to record. Only `ran_at` is required.
  * @returns The autoincremented `id` of the newly-inserted row.
  */
-export async function recordInventoryRun(knex, meta) {
+export async function recordListReconcileRun(knex, meta) {
     const inserted = await knex
-        .from('inventory_runs')
+        .from('list_reconcile_runs')
         .insert({
         ran_at: meta.ran_at,
         list_label: meta.list_label ?? null,
@@ -34,7 +35,7 @@ export async function recordInventoryRun(knex, meta) {
         .returning('id');
     const id = inserted[0]?.id;
     if (typeof id !== 'number') {
-        throw new TypeError('recordInventoryRun: INSERT returned no row id');
+        throw new TypeError('recordListReconcileRun: INSERT returned no row id');
     }
     return id;
 }

package/lib/archive/db-ops/pages/read/build-page-query.js

@@ -36,5 +36,5 @@ export function buildPageQuery(knex) {
         .leftJoin('url_refs as og_image_ur', 'og_image_ur.id', 'pm.og_image_url_id')
         .leftJoin('url_refs as twitter_image_ur', 'twitter_image_ur.id', 'pm.twitter_image_url_id')
         .leftJoin('json_refs as extras_ref', 'extras_ref.id', 'pm.meta_extras_json_id')
-        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.main_content_custom_element_count as main_content_custom_element_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
+        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.main_content_custom_element_count as main_content_custom_element_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'pm.image_scan_desktop as image_scan_desktop', 'pm.image_scan_mobile as image_scan_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
 }

package/lib/archive/db-ops/pages/read/get-crawling-state.d.ts

@@ -61,10 +61,17 @@ import type { Knex } from 'knex';
  * shape is safe to use without the `migrate*` guards that other writer
  * methods carry.
  * @param knex - Knex query builder connected to the archive DB.
- * @returns An object with `scraped` (completed URLs) and `pending` (the
- *   strict set of in-scope, anchor-referenced, unfinished URLs).
+ * @returns An object with `scraped` (completed URLs), `pending` (the
+ *   strict set of in-scope, anchor-referenced, unfinished URLs), and
+ *   `pendingMetadataOnly` (the subset of `pending` whose
+ *   `content_items.is_metadata_only` was persisted as `1` — see
+ *   `replaceAnchorEdges`/`resolveContentItemId`). Callers that resume a
+ *   crawl (`Crawler#resume` → `LinkList#resume`) pass this subset through
+ *   so a metadata-only anchor discovered before an interruption is not
+ *   silently promoted to a full-scrape target on resume (#369).
  */
 export declare function getCrawlingState(knex: Knex): Promise<{
     scraped: string[];
     pending: string[];
+    pendingMetadataOnly: string[];
 }>;

package/lib/archive/db-ops/pages/read/get-crawling-state.js

@@ -60,8 +60,14 @@
  * shape is safe to use without the `migrate*` guards that other writer
  * methods carry.
  * @param knex - Knex query builder connected to the archive DB.
- * @returns An object with `scraped` (completed URLs) and `pending` (the
- *   strict set of in-scope, anchor-referenced, unfinished URLs).
+ * @returns An object with `scraped` (completed URLs), `pending` (the
+ *   strict set of in-scope, anchor-referenced, unfinished URLs), and
+ *   `pendingMetadataOnly` (the subset of `pending` whose
+ *   `content_items.is_metadata_only` was persisted as `1` — see
+ *   `replaceAnchorEdges`/`resolveContentItemId`). Callers that resume a
+ *   crawl (`Crawler#resume` → `LinkList#resume`) pass this subset through
+ *   so a metadata-only anchor discovered before an interruption is not
+ *   silently promoted to a full-scrape target on resume (#369).
  */
 export async function getCrawlingState(knex) {
     const ex = (r) => r.url;
@@ -71,7 +77,7 @@ export async function getCrawlingState(knex) {
         .where('content_items.scraped', 1);
     const scraped = $scraped.map(ex);
     const $pending = await knex
-        .select('ur.url as url')
+        .select('ur.url as url', 'ci.is_metadata_only as isMetadataOnly')
         .from({ ci: 'content_items' })
         .join({ ur: 'url_refs' }, 'ur.id', 'ci.url_id')
         .where('ci.scraped', 0)
@@ -91,8 +97,10 @@ export async function getCrawlingState(knex) {
         }).orWhereNot('ci.source', 'crawled');
     });
     const pending = $pending.map(ex);
+    const pendingMetadataOnly = $pending.filter((r) => r.isMetadataOnly === 1).map(ex);
     return {
         scraped,
         pending,
+        pendingMetadataOnly,
     };
 }

package/lib/archive/db-ops/pages/reset/clear-page-derived-rows.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Deletes every Scoped-Replace derived row for the given page ids, across the
+ * 17 tables a page's re-scrape needs to start clean.
+ *
+ * Shared by {@link resetFailedPages} and {@link repromoteExternalPages} —
+ * both un-scrape a page back to `scraped = 0` and need the same "delete this
+ * page's derived rows so the re-scrape can re-insert fresh data without
+ * duplicates" sweep. `page_errors` is deliberately NOT included here:
+ * `repromoteExternalPages` never cleared it (an external page's prior error
+ * history stays visible after being promoted back into scope), and folding
+ * it into this shared helper would silently change that existing behaviour.
+ * Callers that do want `page_errors` cleared (`resetFailedPages`) delete it
+ * themselves alongside this call.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageIds - `content_items.id` values whose derived rows should be
+ *   deleted. Expected to already be chunked below
+ *   `SQLITE_LIMIT_VARIABLE_NUMBER` by the caller.
+ */
+export declare function clearPageDerivedRows(knex: Knex, pageIds: readonly number[]): Promise<void>;

package/lib/archive/db-ops/pages/reset/clear-page-derived-rows.js

@@ -0,0 +1,37 @@
+/**
+ * Deletes every Scoped-Replace derived row for the given page ids, across the
+ * 17 tables a page's re-scrape needs to start clean.
+ *
+ * Shared by {@link resetFailedPages} and {@link repromoteExternalPages} —
+ * both un-scrape a page back to `scraped = 0` and need the same "delete this
+ * page's derived rows so the re-scrape can re-insert fresh data without
+ * duplicates" sweep. `page_errors` is deliberately NOT included here:
+ * `repromoteExternalPages` never cleared it (an external page's prior error
+ * history stays visible after being promoted back into scope), and folding
+ * it into this shared helper would silently change that existing behaviour.
+ * Callers that do want `page_errors` cleared (`resetFailedPages`) delete it
+ * themselves alongside this call.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageIds - `content_items.id` values whose derived rows should be
+ *   deleted. Expected to already be chunked below
+ *   `SQLITE_LIMIT_VARIABLE_NUMBER` by the caller.
+ */
+export async function clearPageDerivedRows(knex, pageIds) {
+    await knex('page_meta').whereIn('page_id', pageIds).delete();
+    await knex('anchor_edges').whereIn('page_id', pageIds).delete();
+    await knex('image_items').whereIn('page_id', pageIds).delete();
+    await knex('resource_ref_edges').whereIn('page_id', pageIds).delete();
+    await knex('page_html_ref').whereIn('page_id', pageIds).delete();
+    await knex('technology_signals').whereIn('pageId', pageIds).delete();
+    await knex('page_technologies').whereIn('pageId', pageIds).delete();
+    await knex('page_jsonld').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_headings').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_images').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_tables').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_buttons').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_iframes').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_videos').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_audios').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_canvases').whereIn('pageId', pageIds).delete();
+    await knex('page_main_content_custom_elements').whereIn('pageId', pageIds).delete();
+}

package/lib/archive/db-ops/pages/reset/repromote-external-pages.d.ts

@@ -22,8 +22,9 @@ import type { Knex } from 'knex';
  * @param onProgress - Called after each chunk's DELETE/UPDATE statements
  *   complete, with the pages processed so far and the total to promote
  *   (issue #294: a large `--append` can promote thousands of pages across
- *   14 tables, running for seconds to minutes with no other signal it
- *   hasn't hung). Omit for no reporting (the default; e.g. tests).
+ *   `content_items` plus the 17 tables {@link clearPageDerivedRows} clears,
+ *   running for seconds to minutes with no other signal it hasn't hung).
+ *   Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were promoted.
  */
 export declare function repromoteExternalPages(knex: Knex, scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions, onProgress?: (processed: number, total: number) => void): Promise<string[]>;

package/lib/archive/db-ops/pages/reset/repromote-external-pages.js

@@ -1,6 +1,7 @@
 import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { findScopeEntry } from '../../../../crawler/find-scope-entry.js';
 import { dbLog } from '../../../debug.js';
+import { clearPageDerivedRows } from './clear-page-derived-rows.js';
 /**
  * Promote previously-external pages whose URL falls under any of the new scope
  * entries back to a "needs scraping" state so that the next crawl picks them up
@@ -23,8 +24,9 @@ import { dbLog } from '../../../debug.js';
  * @param onProgress - Called after each chunk's DELETE/UPDATE statements
  *   complete, with the pages processed so far and the total to promote
  *   (issue #294: a large `--append` can promote thousands of pages across
- *   14 tables, running for seconds to minutes with no other signal it
- *   hasn't hung). Omit for no reporting (the default; e.g. tests).
+ *   `content_items` plus the 17 tables {@link clearPageDerivedRows} clears,
+ *   running for seconds to minutes with no other signal it hasn't hung).
+ *   Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were promoted.
  */
 export async function repromoteExternalPages(knex, scopes, options, onProgress) {
@@ -69,41 +71,19 @@ export async function repromoteExternalPages(knex, scopes, options, onProgress)
             // left untouched — the last-success timestamp survives the
             // demotion.
         });
-        // Clear the prior crawl's data for the repromoted pages. `updatePage`
-        // also replaces anchor_edges/image_items/tags/jsonld/page_main_content_*
-        // when it re-scrapes them, but only when the new scrape is non-empty —
-        // so this pre-clear is still load-bearing for pages that get
-        // repromoted but then re-scrape to nothing (or are never reached
-        // again), and it is the only place `resource_ref_edges` is cleared.
-        // Deleting the `page_meta` row (rather than nulling every column)
-        // clears title / description / og:* / twitter:* / meta_extras /
-        // main_content_* in one statement; a re-scrape re-inserts it via
-        // `ON CONFLICT(page_id) DO UPDATE`. `technology_signals` /
-        // `page_technologies` / `page_jsonld` /
-        // `page_main_content_*` are cleared explicitly even though all of
-        // them also carry ON DELETE CASCADE — we keep the existing pattern
-        // of explicit chunked DELETEs rather than relying on CASCADE
-        // indirectly (and would not cascade anyway: the parent
-        // `content_items` row is updated, not
-        // deleted). Orphan blobs in `page_html_blobs` are left behind; #23
-        // will add GC.
-        await knex('page_meta').whereIn('page_id', chunk).delete();
-        await knex('anchor_edges').whereIn('page_id', chunk).delete();
-        await knex('image_items').whereIn('page_id', chunk).delete();
-        await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
-        await knex('page_html_ref').whereIn('page_id', chunk).delete();
-        await knex('technology_signals').whereIn('pageId', chunk).delete();
-        await knex('page_technologies').whereIn('pageId', chunk).delete();
-        await knex('page_jsonld').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_images').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_tables').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_buttons').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_iframes').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_custom_elements').whereIn('pageId', chunk).delete();
+        // Clear the prior crawl's data for the repromoted pages via the
+        // shared sweep (also used by `resetFailedPages` / `resetPagesByUrls`).
+        // `updatePage` also replaces anchor_edges/image_items/tags/jsonld/
+        // page_main_content_* when it re-scrapes them, but only when the new
+        // scrape is non-empty — so this pre-clear is still load-bearing for
+        // pages that get repromoted but then re-scrape to nothing (or are
+        // never reached again), and it is the only place `resource_ref_edges`
+        // is cleared for this operation. Deleting the `page_meta` row (rather
+        // than nulling every column) clears title / description / og:* /
+        // twitter:* / meta_extras / main_content_* in one statement; a
+        // re-scrape re-inserts it via `ON CONFLICT(page_id) DO UPDATE`.
+        // Orphan blobs in `page_html_blobs` are left behind; #23 will add GC.
+        await clearPageDerivedRows(knex, chunk);
         onProgress?.(Math.min(i + chunkSize, promotedIds.length), promotedIds.length);
     }
     dbLog('Repromoted %d external pages back to pending', promotedUrls.length);

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -13,7 +13,13 @@ import type { Knex } from 'knex';
  * - `status IS NULL` — no status was ever stored for the row;
  * - the row has no `content_type_refs` link — the content type could not be
  *   determined;
- * - `status` is in the `5xx` range — a (frequently transient) server error.
+ * - `status` is in the `5xx` range — a (frequently transient) server error;
+ * - `page_meta.image_scan_desktop` or `image_scan_mobile` is one of
+ *   {@link RETRYABLE_IMAGE_SCAN_CODES} — `@d-zero/beholder` abandoned that
+ *   viewport's `<img>` scan for a transient reason (navigation never
+ *   settled, or the frame/session was lost mid-scan), independent of the
+ *   page's own HTTP `status` (a page can score `status = 200` and still
+ *   have failed its mobile image scan).
  *
  * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
  * almost always yields the same answer.
@@ -42,6 +48,21 @@ import type { Knex } from 'knex';
  * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
  * exactly as before this override existed.
  *
+ * **Confirmed same-cluster trap exclusion**: a third pass drops any
+ * remaining candidate whose URL shape (`computeShapeKey`) matches a
+ * `dedupe_cap_events.shape_key` already recorded in this archive (see
+ * `DedupeCapTracker`). A trap page frequently fails outright (timeout / 5xx)
+ * rather than rendering a comparable body, so it never reaches
+ * `DedupeCapTracker#observe` during the crawl that hit it — without this
+ * exclusion, `--retry-failed` would keep re-queueing the very pages the cap
+ * already exists to suppress, re-inflating the pending count on every pass
+ * for a shape this archive has already confirmed is not worth the cost of
+ * re-discovering. Left as a failed row rather than rewritten to a skip —
+ * post-hoc marking (`content_items.dedupe_cap_event_id`, computed at
+ * `viewer-build`) is what surfaces these pages as capped, not this reset
+ * path. An archive with no recorded cap events (`listDedupeCapShapeKeys`
+ * returns `[]`) behaves exactly as before this exclusion existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).

package/lib/archive/db-ops/pages/reset/reset-failed-pages.js

@@ -1,9 +1,13 @@
 import { classifyErrorKind } from '../../../../classify-error-kind.js';
+import { computeShapeKey } from '../../../../crawler/dedupe/compute-shape-key.js';
 import { isWithinOutageWindow } from '../../../../is-within-outage-window.js';
 import { PERMANENT_ERROR_KINDS } from '../../../../permanent-error-kinds.js';
 import { dbLog } from '../../../debug.js';
 import { getFailedPageMessages } from '../../../get-failed-page-messages.js';
+import { listDedupeCapShapeKeys } from '../../dedupe-cap/list-dedupe-cap-shape-keys.js';
 import { listNetworkOutages } from '../../outages/list-network-outages.js';
+import { clearPageDerivedRows } from './clear-page-derived-rows.js';
+import { RETRYABLE_IMAGE_SCAN_CODES } from './retryable-image-scan-codes.js';
 /**
  * Reset previously-attempted pages that ended in a recoverable failure so a
  * follow-up crawl can re-fetch them from scratch.
@@ -18,7 +22,13 @@ import { listNetworkOutages } from '../../outages/list-network-outages.js';
  * - `status IS NULL` — no status was ever stored for the row;
  * - the row has no `content_type_refs` link — the content type could not be
  *   determined;
- * - `status` is in the `5xx` range — a (frequently transient) server error.
+ * - `status` is in the `5xx` range — a (frequently transient) server error;
+ * - `page_meta.image_scan_desktop` or `image_scan_mobile` is one of
+ *   {@link RETRYABLE_IMAGE_SCAN_CODES} — `@d-zero/beholder` abandoned that
+ *   viewport's `<img>` scan for a transient reason (navigation never
+ *   settled, or the frame/session was lost mid-scan), independent of the
+ *   page's own HTTP `status` (a page can score `status = 200` and still
+ *   have failed its mobile image scan).
  *
  * Definitive `4xx` responses are intentionally excluded: re-fetching a 404
  * almost always yields the same answer.
@@ -47,6 +57,21 @@ import { listNetworkOutages } from '../../outages/list-network-outages.js';
  * with no recorded outages (`listNetworkOutages` returns `[]`) behaves
  * exactly as before this override existed.
  *
+ * **Confirmed same-cluster trap exclusion**: a third pass drops any
+ * remaining candidate whose URL shape (`computeShapeKey`) matches a
+ * `dedupe_cap_events.shape_key` already recorded in this archive (see
+ * `DedupeCapTracker`). A trap page frequently fails outright (timeout / 5xx)
+ * rather than rendering a comparable body, so it never reaches
+ * `DedupeCapTracker#observe` during the crawl that hit it — without this
+ * exclusion, `--retry-failed` would keep re-queueing the very pages the cap
+ * already exists to suppress, re-inflating the pending count on every pass
+ * for a shape this archive has already confirmed is not worth the cost of
+ * re-discovering. Left as a failed row rather than rewritten to a skip —
+ * post-hoc marking (`content_items.dedupe_cap_event_id`, computed at
+ * `viewer-build`) is what surfaces these pages as capped, not this reset
+ * path. An archive with no recorded cap events (`listDedupeCapShapeKeys`
+ * returns `[]`) behaves exactly as before this exclusion existed.
+ *
  * Matching rows — internal and external alike — are demoted back to pending
  * (`scraped = 0`) and have their stale scrape metadata cleared (the
  * `page_meta` row is deleted outright rather than nulled column-by-column).
@@ -72,7 +97,8 @@ import { listNetworkOutages } from '../../outages/list-network-outages.js';
 export async function resetFailedPages(knex, onProgress) {
     const candidates = await knex('content_items')
         .join('url_refs', 'content_items.url_id', 'url_refs.id')
-        .select('content_items.id as id', 'url_refs.url as url')
+        .leftJoin('page_meta', 'content_items.id', 'page_meta.page_id')
+        .select('content_items.id as id', 'url_refs.url as url', 'content_items.status as status', 'content_items.content_type_id as contentTypeId')
         .where('content_items.scraped', 1)
         .whereNull('content_items.redirect_dest_id')
         .where((qb) => {
@@ -82,19 +108,37 @@ export async function resetFailedPages(knex, onProgress) {
         qb.whereNull('content_items.status')
             .orWhere('content_items.status', -1)
             .orWhereNull('content_items.content_type_id')
-            .orWhereBetween('content_items.status', [500, 599]);
+            .orWhereBetween('content_items.status', [500, 599])
+            .orWhereIn('page_meta.image_scan_desktop', RETRYABLE_IMAGE_SCAN_CODES)
+            .orWhereIn('page_meta.image_scan_mobile', RETRYABLE_IMAGE_SCAN_CODES);
     });
     if (candidates.length === 0) {
         return [];
     }
+    // Diagnostic only: how many candidates matched solely because of an
+    // image-scan outcome, not the pre-existing status-based conditions —
+    // i.e. a page whose HTTP status looks fine but whose mobile/desktop
+    // `<img>` scan was abandoned for a transient reason.
+    const imageScanOnlyCount = candidates.filter((row) => {
+        const statusQualifies = row.status === null ||
+            row.status === -1 ||
+            row.contentTypeId === null ||
+            (row.status >= 500 && row.status <= 599);
+        return !statusQualifies;
+    }).length;
+    if (imageScanOnlyCount > 0) {
+        dbLog('%d of %d retry candidate(s) matched via image-scan degradation only', imageScanOnlyCount, candidates.length);
+    }
     const candidateIds = candidates.map((row) => row.id);
     const candidateUrls = candidates.map((row) => row.url);
-    // Unrelated tables (page_errors/crawl_errors vs network_outages), no data
-    // dependency between them — run concurrently instead of paying two
-    // sequential round-trips on every `--retry-failed` pass.
-    const [messages, outageWindows] = await Promise.all([
+    // Three unrelated reads (page_errors/crawl_errors, network_outages,
+    // dedupe_cap_events) with no data dependency between them — run
+    // concurrently instead of paying three sequential round-trips on every
+    // `--retry-failed` pass.
+    const [messages, outageWindows, cappedShapeKeys] = await Promise.all([
         getFailedPageMessages(knex, candidateIds, candidateUrls),
         listNetworkOutages(knex),
+        listDedupeCapShapeKeys(knex).then((shapeKeys) => new Set(shapeKeys)),
     ]);
     // Drop candidates whose latest recorded message classifies as permanent —
     // UNLESS that message's timestamp falls inside a recorded network outage,
@@ -117,11 +161,26 @@ export async function resetFailedPages(knex, onProgress) {
     if (excludedCount > 0) {
         dbLog('Excluded %d page(s) from retry — permanent failure kinds (dns/tls/client-blocked/parse-error/connection-refused)', excludedCount);
     }
-    if (retryable.length === 0) {
+    // Drop candidates whose URL shape already has a confirmed same-cluster
+    // trap recorded (see the "Confirmed same-cluster trap exclusion" section
+    // of this function's docstring). A row whose shape cannot be computed
+    // (`computeShapeKey` returns `null`) stays in the retry pool — no signal
+    // either way, so err on the side of retrying it.
+    const notCapped = cappedShapeKeys.size === 0
+        ? retryable
+        : retryable.filter((row) => {
+            const shapeKey = computeShapeKey(row.url);
+            return shapeKey === null || !cappedShapeKeys.has(shapeKey);
+        });
+    const excludedCappedCount = retryable.length - notCapped.length;
+    if (excludedCappedCount > 0) {
+        dbLog('Excluded %d page(s) from retry — confirmed same-cluster trap shape', excludedCappedCount);
+    }
+    if (notCapped.length === 0) {
         return [];
     }
-    const ids = retryable.map((row) => row.id);
-    const urls = retryable.map((row) => row.url);
+    const ids = notCapped.map((row) => row.id);
+    const urls = notCapped.map((row) => row.url);
     const chunkSize = 500;
     for (let i = 0; i < ids.length; i += chunkSize) {
         const chunk = ids.slice(i, i + chunkSize);
@@ -136,33 +195,20 @@ export async function resetFailedPages(knex, onProgress) {
             // untouched so the last-success timestamp records survive the
             // demotion (the within-archive observation axis for #11/#17/#19).
         });
-        // Clear the prior crawl's per-page data so the re-scrape starts clean.
-        // `updatePage` only replaces anchor_edges/image_items/tags/jsonld when
-        // the new scrape is non-empty, so this pre-clear is load-bearing for
-        // pages that reset but then fail again (or are never reached), and it
-        // is the only place `resource_ref_edges` and `page_errors` are
-        // cleared. Deleting the `page_meta` row (rather than nulling every
+        // Clear the prior crawl's per-page data so the re-scrape starts clean,
+        // via the shared sweep (also used by `repromoteExternalPages` /
+        // `resetPagesByUrls`) plus `page_errors`, which that sweep
+        // deliberately excludes (see its JSDoc). `updatePage` only replaces
+        // anchor_edges/image_items/tags/jsonld when the new scrape is
+        // non-empty, so this pre-clear is load-bearing for pages that reset
+        // but then fail again (or are never reached), and it is the only
+        // place `resource_ref_edges` and `page_errors` are cleared for this
+        // operation. Deleting the `page_meta` row (rather than nulling every
         // column) clears title / description / og:* / twitter:* /
         // meta_extras in one statement; a re-scrape re-inserts it via
         // `ON CONFLICT(page_id) DO UPDATE`.
-        await knex('page_meta').whereIn('page_id', chunk).delete();
-        await knex('anchor_edges').whereIn('page_id', chunk).delete();
-        await knex('image_items').whereIn('page_id', chunk).delete();
-        await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
         await knex('page_errors').whereIn('pageId', chunk).delete();
-        await knex('page_html_ref').whereIn('page_id', chunk).delete();
-        await knex('technology_signals').whereIn('pageId', chunk).delete();
-        await knex('page_technologies').whereIn('pageId', chunk).delete();
-        await knex('page_jsonld').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_images').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_tables').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_buttons').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_iframes').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
-        await knex('page_main_content_custom_elements').whereIn('pageId', chunk).delete();
+        await clearPageDerivedRows(knex, chunk);
         onProgress?.(Math.min(i + chunkSize, ids.length), ids.length);
     }
     dbLog('Reset %d failed pages back to pending', urls.length);