@nitpicker/crawler 0.18.0 → 0.19.0

package/lib/archive/database.d.ts

@@ -1,5 +1,5 @@
 import type { TemplateClusterReason } from './db-ops/analysis/types.js';
-import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, TagRow } from './meta/types.js';
+import type { JsonLdRow, MainContentAudioRow, MainContentButtonRow, MainContentCanvasRow, MainContentCustomElementRow, MainContentHeadingRow, MainContentIframeRow, MainContentImageRow, MainContentTableRow, MainContentVideoRow, PageTechnologyRow, TechnologySignalRow } from './meta/types.js';
 import type { Config, DatabaseOption, DB_Redirect, DB_Resource, DatabaseEvent, InsertDedupeCapEventParams, InsertNetworkOutageParams, InventoryRunMeta, PageFilter, PageSource } from './types.js';
 import type { OutageWindow } from '../is-within-outage-window.js';
 import type { PageData, Resource } from '../utils/types/types.js';
@@ -115,6 +115,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         scraped: string[];
         pending: string[];
     }>;
+    /**
+     * Retrieves all `page_main_content_custom_elements` rows for the given
+     * page id. Delegates to {@link getCustomElementsOfPageOp}.
+     * @param pageId
+     */
+    getCustomElementsOfPage(pageId: number): Promise<MainContentCustomElementRow[]>;
     /**
      * Return the subset of `urls` that already exist in the `pages` table.
      * Delegates to {@link getExistingPageUrlsOp}.
@@ -217,6 +223,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
         anchors: import("./types.js").DB_Anchor[];
         referrers: import("./types.js").DB_Referrer[];
     }>;
+    /**
+     * Retrieves all `page_technologies` rows for the given page id.
+     * Delegates to {@link getPageTechnologiesOfPageOp}.
+     * @param pageId
+     */
+    getPageTechnologiesOfPage(pageId: number): Promise<PageTechnologyRow[]>;
     /**
      * Retrieves redirect sources for the given page IDs in bulk.
      * Delegates to {@link getRedirectsForPagesOp}.
@@ -260,9 +272,18 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     /**
      * Retrieves a flat list of all resource URLs from the `resources` table.
      * Delegates to {@link getResourceUrlListOp}.
+     *
+     * `onProgress` is clamped to a high-water mark across retries: a
+     * transient failure partway through the keyset scan restarts
+     * {@link getResourceUrlListOp} from `id = 0` (its own `lastId` is a local
+     * closure variable, reset on every `fn` invocation `retryCall` retries),
+     * which would otherwise visibly rewind the reported progress backwards
+     * before catching back up.
+     * @param onProgress - Forwarded to {@link getResourceUrlListOp} — see
+     *   that function's docs.
      * @returns An array of resource URL strings.
      */
-    getResourceUrlList(): Promise<string[]>;
+    getResourceUrlList(onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<string[]>;
     /**
      * Counts pages that were scraped as crawl targets (full HTML render).
      * Delegates to {@link getScrapedHtmlPageCountOp}.
@@ -270,11 +291,11 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      */
     getScrapedHtmlPageCount(): Promise<number>;
     /**
-     * Retrieves all `page_tags` rows for the given page id, parsed back into
-     * {@link TagRow} shape. Delegates to {@link getTagsOfPageOp}.
+     * Retrieves all `technology_signals` rows for the given page id.
+     * Delegates to {@link getTechnologySignalsOfPageOp}.
      * @param pageId
      */
-    getTagsOfPage(pageId: number): Promise<TagRow[]>;
+    getTechnologySignalsOfPage(pageId: number): Promise<TechnologySignalRow[]>;
     /**
      * Retrieves all `page_main_content_videos` rows for the given page id.
      * Delegates to {@link getVideosOfPageOp}.
@@ -425,17 +446,21 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * Delegates to {@link repromoteExternalPagesOp}.
      * @param scopes - The hostname-indexed scope map after the new roots are merged.
      * @param options - URL parsing options forwarded to the scope matcher.
+     * @param onProgress - Forwarded to {@link repromoteExternalPagesOp} — see
+     *   that function's docs.
      * @returns The URLs of the pages that were promoted.
      */
-    repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
+    repromoteExternalPages(scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions, onProgress?: (processed: number, total: number) => void): Promise<string[]>;
     /**
      * Reset previously-attempted pages that ended in a recoverable failure so a
      * follow-up crawl can re-fetch them from scratch. Delegates to
      * {@link resetFailedPagesOp} — see the op for the permanent-failure
      * exclusion rationale.
+     * @param onProgress - Forwarded to {@link resetFailedPagesOp} — see that
+     *   function's docs.
      * @returns The URLs of the pages that were reset to pending.
      */
-    resetFailedPages(): Promise<string[]>;
+    resetFailedPages(onProgress?: (processed: number, total: number) => void): Promise<string[]>;
     /**
      * Stores the crawl configuration in the `info` table.
      * Delegates to {@link setConfigOp}.
@@ -453,8 +478,10 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
     /**
      * Assigns natural URL sort order values to all internal pages.
      * Delegates to {@link setUrlOrderOp}.
+     * @param onProgress - Forwarded to {@link setUrlOrderOp} — see that
+     *   function's docs.
      */
-    setUrlOrder(): Promise<void>;
+    setUrlOrder(onProgress?: (processed: number, total: number) => void): Promise<void>;
     /**
      * Update the single row in the `info` table with a partial config patch.
      * Delegates to {@link updateConfigOp}.
@@ -473,9 +500,12 @@ export declare class Database extends EventEmitter<DatabaseEvent> {
      * @param isTarget - Whether this page is a crawl target.
      * @param source - Provenance label written ONLY when the row is freshly
      *   inserted. Existing rows keep their original `source`.
+     * @param bodyHash - Precomputed body hash for the page's HTML (see
+     *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+     *   computing it from the HTML instead.
      * @returns The database `pageId` of the inserted/updated row.
      */
-    updatePage(page: PageData, writeHtml: boolean, isTarget: boolean, source?: PageSource): Promise<number>;
+    updatePage(page: PageData, writeHtml: boolean, isTarget: boolean, source?: PageSource, bodyHash?: Buffer | null): Promise<number>;
     /**
      * Creates and initializes a new Database instance.
      *

package/lib/archive/database.js

@@ -31,12 +31,14 @@ import { init as initOp } from './db-ops/lifecycle/init.js';
 import { getAudiosOfPage as getAudiosOfPageOp } from './db-ops/meta/get-audios-of-page.js';
 import { getButtonsOfPage as getButtonsOfPageOp } from './db-ops/meta/get-buttons-of-page.js';
 import { getCanvasesOfPage as getCanvasesOfPageOp } from './db-ops/meta/get-canvases-of-page.js';
+import { getCustomElementsOfPage as getCustomElementsOfPageOp } from './db-ops/meta/get-custom-elements-of-page.js';
 import { getHeadingsOfPage as getHeadingsOfPageOp } from './db-ops/meta/get-headings-of-page.js';
 import { getIframesOfPage as getIframesOfPageOp } from './db-ops/meta/get-iframes-of-page.js';
 import { getJsonLdOfPage as getJsonLdOfPageOp } from './db-ops/meta/get-jsonld-of-page.js';
 import { getMainContentImagesOfPage as getMainContentImagesOfPageOp } from './db-ops/meta/get-main-content-images-of-page.js';
 import { getMainContentTablesOfPage as getMainContentTablesOfPageOp } from './db-ops/meta/get-main-content-tables-of-page.js';
-import { getTagsOfPage as getTagsOfPageOp } from './db-ops/meta/get-tags-of-page.js';
+import { getPageTechnologiesOfPage as getPageTechnologiesOfPageOp } from './db-ops/meta/get-page-technologies-of-page.js';
+import { getTechnologySignalsOfPage as getTechnologySignalsOfPageOp } from './db-ops/meta/get-technology-signals-of-page.js';
 import { getVideosOfPage as getVideosOfPageOp } from './db-ops/meta/get-videos-of-page.js';
 import { closeNetworkOutage as closeNetworkOutageOp } from './db-ops/outages/close-network-outage.js';
 import { insertNetworkOutage as insertNetworkOutageOp } from './db-ops/outages/insert-network-outage.js';
@@ -227,6 +229,14 @@ export class Database extends EventEmitter {
     async getCrawlingState() {
         return emitErrorAndRetry(this, 'Database.getCrawlingState', async () => await getCrawlingStateOp(this.#instance), retrySetting);
     }
+    /**
+     * Retrieves all `page_main_content_custom_elements` rows for the given
+     * page id. Delegates to {@link getCustomElementsOfPageOp}.
+     * @param pageId
+     */
+    async getCustomElementsOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getCustomElementsOfPage', async () => await getCustomElementsOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Return the subset of `urls` that already exist in the `pages` table.
      * Delegates to {@link getExistingPageUrlsOp}.
@@ -352,6 +362,14 @@ export class Database extends EventEmitter {
     async getPagesWithRels(offset, limit) {
         return emitErrorAndRetry(this, 'Database.getPagesWithRels', async () => await getPagesWithRelsOp(this.#instance, offset, limit), retrySetting);
     }
+    /**
+     * Retrieves all `page_technologies` rows for the given page id.
+     * Delegates to {@link getPageTechnologiesOfPageOp}.
+     * @param pageId
+     */
+    async getPageTechnologiesOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getPageTechnologiesOfPage', async () => await getPageTechnologiesOfPageOp(this.#instance, pageId), retrySetting);
+    }
     /**
      * Retrieves redirect sources for the given page IDs in bulk.
      * Delegates to {@link getRedirectsForPagesOp}.
@@ -408,10 +426,26 @@ export class Database extends EventEmitter {
     /**
      * Retrieves a flat list of all resource URLs from the `resources` table.
      * Delegates to {@link getResourceUrlListOp}.
+     *
+     * `onProgress` is clamped to a high-water mark across retries: a
+     * transient failure partway through the keyset scan restarts
+     * {@link getResourceUrlListOp} from `id = 0` (its own `lastId` is a local
+     * closure variable, reset on every `fn` invocation `retryCall` retries),
+     * which would otherwise visibly rewind the reported progress backwards
+     * before catching back up.
+     * @param onProgress - Forwarded to {@link getResourceUrlListOp} — see
+     *   that function's docs.
      * @returns An array of resource URL strings.
      */
-    async getResourceUrlList() {
-        return emitErrorAndRetry(this, 'Database.getResourceUrlList', async () => await getResourceUrlListOp(this.#instance), retrySetting);
+    async getResourceUrlList(onProgress) {
+        let highWaterMark = 0;
+        const monotonicProgress = onProgress
+            ? (scannedUpToId, maxId) => {
+                highWaterMark = Math.max(highWaterMark, scannedUpToId);
+                onProgress(highWaterMark, maxId);
+            }
+            : undefined;
+        return emitErrorAndRetry(this, 'Database.getResourceUrlList', async () => await getResourceUrlListOp(this.#instance, monotonicProgress), retrySetting);
     }
     /**
      * Counts pages that were scraped as crawl targets (full HTML render).
@@ -422,12 +456,12 @@ export class Database extends EventEmitter {
         return emitErrorAndRetry(this, 'Database.getScrapedHtmlPageCount', async () => await getScrapedHtmlPageCountOp(this.#instance), retrySetting);
     }
     /**
-     * Retrieves all `page_tags` rows for the given page id, parsed back into
-     * {@link TagRow} shape. Delegates to {@link getTagsOfPageOp}.
+     * Retrieves all `technology_signals` rows for the given page id.
+     * Delegates to {@link getTechnologySignalsOfPageOp}.
      * @param pageId
      */
-    async getTagsOfPage(pageId) {
-        return emitErrorAndRetry(this, 'Database.getTagsOfPage', async () => await getTagsOfPageOp(this.#instance, pageId), retrySetting);
+    async getTechnologySignalsOfPage(pageId) {
+        return emitErrorAndRetry(this, 'Database.getTechnologySignalsOfPage', async () => await getTechnologySignalsOfPageOp(this.#instance, pageId), retrySetting);
     }
     /**
      * Retrieves all `page_main_content_videos` rows for the given page id.
@@ -609,20 +643,24 @@ export class Database extends EventEmitter {
      * Delegates to {@link repromoteExternalPagesOp}.
      * @param scopes - The hostname-indexed scope map after the new roots are merged.
      * @param options - URL parsing options forwarded to the scope matcher.
+     * @param onProgress - Forwarded to {@link repromoteExternalPagesOp} — see
+     *   that function's docs.
      * @returns The URLs of the pages that were promoted.
      */
-    async repromoteExternalPages(scopes, options) {
-        return emitErrorAndRetry(this, 'Database.repromoteExternalPages', async () => await repromoteExternalPagesOp(this.#instance, scopes, options), retrySetting);
+    async repromoteExternalPages(scopes, options, onProgress) {
+        return emitErrorAndRetry(this, 'Database.repromoteExternalPages', async () => await repromoteExternalPagesOp(this.#instance, scopes, options, onProgress), retrySetting);
     }
     /**
      * Reset previously-attempted pages that ended in a recoverable failure so a
      * follow-up crawl can re-fetch them from scratch. Delegates to
      * {@link resetFailedPagesOp} — see the op for the permanent-failure
      * exclusion rationale.
+     * @param onProgress - Forwarded to {@link resetFailedPagesOp} — see that
+     *   function's docs.
      * @returns The URLs of the pages that were reset to pending.
      */
-    async resetFailedPages() {
-        return emitErrorAndRetry(this, 'Database.resetFailedPages', async () => await resetFailedPagesOp(this.#instance), retrySetting);
+    async resetFailedPages(onProgress) {
+        return emitErrorAndRetry(this, 'Database.resetFailedPages', async () => await resetFailedPagesOp(this.#instance, onProgress), retrySetting);
     }
     /**
      * Stores the crawl configuration in the `info` table.
@@ -645,9 +683,11 @@ export class Database extends EventEmitter {
     /**
      * Assigns natural URL sort order values to all internal pages.
      * Delegates to {@link setUrlOrderOp}.
+     * @param onProgress - Forwarded to {@link setUrlOrderOp} — see that
+     *   function's docs.
      */
-    async setUrlOrder() {
-        await setUrlOrderOp(this.#instance);
+    async setUrlOrder(onProgress) {
+        await setUrlOrderOp(this.#instance, onProgress);
     }
     /**
      * Update the single row in the `info` table with a partial config patch.
@@ -669,19 +709,24 @@ export class Database extends EventEmitter {
      * @param isTarget - Whether this page is a crawl target.
      * @param source - Provenance label written ONLY when the row is freshly
      *   inserted. Existing rows keep their original `source`.
+     * @param bodyHash - Precomputed body hash for the page's HTML (see
+     *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+     *   computing it from the HTML instead.
      * @returns The database `pageId` of the inserted/updated row.
      */
-    async updatePage(page, writeHtml, isTarget, source) {
-        return emitErrorAndRetry(this, 'Database.updatePage', async () => await updatePageOp(this.#instance, this.#writeRefCaches, page, writeHtml, isTarget, source), retrySetting);
+    async updatePage(page, writeHtml, isTarget, source, bodyHash) {
+        return emitErrorAndRetry(this, 'Database.updatePage', async () => await updatePageOp(this.#instance, this.#writeRefCaches, page, writeHtml, isTarget, source, bodyHash), retrySetting);
     }
     /**
      * Initializes the database schema if tables do not exist, then runs
      * lightweight migrations; in read-only mode both are skipped.
      * Delegates to {@link initOp}.
      * @param readOnly - When true, skip schema init + migrations.
+     * @param onLog - Forwarded to {@link initOp} — see
+     *   {@link DatabaseOption.onLog}'s docs.
      */
-    async #init(readOnly) {
-        await initOp(this.#instance, readOnly);
+    async #init(readOnly, onLog) {
+        await initOp(this.#instance, readOnly, onLog);
     }
     /**
      * Creates and initializes a new Database instance.
@@ -714,7 +759,7 @@ export class Database extends EventEmitter {
             mkdir(options.filename);
         }
         const db = new Database(options);
-        await db.#init(options.readOnly ?? false);
+        await db.#init(options.readOnly ?? false, options.onLog);
         return db;
     }
 }

package/lib/archive/db-ops/lifecycle/init.d.ts

@@ -19,7 +19,15 @@ import type { Knex } from 'knex';
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
  * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
- * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
+ * `migratePageMetaConsoleErrorCount`, `migratePageMetaCustomElementCount`,
+ * `migrateContentItemsAliasOfId`,
+ *
+ * `migratePageTagsToPageTechnologies` is the one exception to "column adds
+ * only": it converts `page_tags` (removed) rows into `technology_signals`/
+ * `page_technologies` and DROPS the old table — a one-time table-level ETL,
+ * not a column add, but it belongs in this same boot phase for the same
+ * reason (self-healing an old archive's schema before any reader runs).
+ *
  * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
  * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
  *
@@ -35,5 +43,9 @@ import type { Knex } from 'knex';
  * crawl without rewriting the user's tmpDir.
  * @param knex - Knex query builder connected to the archive DB.
  * @param readOnly - When true, skip schema init + migrations.
+ * @param onLog - Forwarded to every `migrate*` call below (issue #294) —
+ *   see {@link import('../../migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ *   docs for why a bare `console.error` at this point is unsafe (it can
+ *   fire while a caller's `Lanes`/`TaskList` display is mid-redraw).
  */
-export declare function init(knex: Knex, readOnly: boolean): Promise<void>;
+export declare function init(knex: Knex, readOnly: boolean, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/db-ops/lifecycle/init.js

@@ -9,6 +9,8 @@ import { migrateInventoryRunsInvalidSkipped } from '../../migrate-inventory-runs
 import { migrateMainContentsColumns } from '../../migrate-main-contents-columns.js';
 import { migratePageMetaBodyHash } from '../../migrate-page-meta-body-hash.js';
 import { migratePageMetaConsoleErrorCount } from '../../migrate-page-meta-console-error-count.js';
+import { migratePageMetaCustomElementCount } from '../../migrate-page-meta-custom-element-count.js';
+import { migratePageTagsToPageTechnologies } from '../../migrate-page-tags-to-page-technologies.js';
 import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-network-outages.js';
 /**
  * Initializes the database schema if tables do not exist, then runs the
@@ -30,7 +32,15 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
  * additions to an existing 0.13 table are therefore the one case that still
  * needs an explicit `hasColumn`-guarded `ALTER TABLE` here (`migrateInfoRoots`,
  * `migrateMainContentsColumns`, `migratePageMetaBodyHash`,
- * `migratePageMetaConsoleErrorCount`, `migrateContentItemsAliasOfId`,
+ * `migratePageMetaConsoleErrorCount`, `migratePageMetaCustomElementCount`,
+ * `migrateContentItemsAliasOfId`,
+ *
+ * `migratePageTagsToPageTechnologies` is the one exception to "column adds
+ * only": it converts `page_tags` (removed) rows into `technology_signals`/
+ * `page_technologies` and DROPS the old table — a one-time table-level ETL,
+ * not a column add, but it belongs in this same boot phase for the same
+ * reason (self-healing an old archive's schema before any reader runs).
+ *
  * `migrateContentItemsDedupeCapEventId`, `migrateInventoryRunsInvalidSkipped`,
  * `migrateInventoryRunsExcludeSkipped`) rather than a DDL-string change alone.
  *
@@ -46,8 +56,12 @@ import { closeStaleOpenNetworkOutages } from '../outages/close-stale-open-networ
  * crawl without rewriting the user's tmpDir.
  * @param knex - Knex query builder connected to the archive DB.
  * @param readOnly - When true, skip schema init + migrations.
+ * @param onLog - Forwarded to every `migrate*` call below (issue #294) —
+ *   see {@link import('../../migrate-page-meta-body-hash.js').migratePageMetaBodyHash}'s
+ *   docs for why a bare `console.error` at this point is unsafe (it can
+ *   fire while a caller's `Lanes`/`TaskList` display is mid-redraw).
  */
-export async function init(knex, readOnly) {
+export async function init(knex, readOnly, onLog) {
     // Connection-level PRAGMAs (foreign_keys, mmap_size, …) must be
     // reapplied on every connect — they are not persisted across opens.
     // They are safe in read-only mode because they don't write to the
@@ -64,18 +78,22 @@ export async function init(knex, readOnly) {
         return;
     }
     await initSchema(knex);
-    await migrateInfoRoots(knex);
-    await migrateInfoMainContentSelector(knex);
-    await migrateMainContentsColumns(knex);
-    await migratePageMetaBodyHash(knex);
-    await migratePageMetaConsoleErrorCount(knex);
-    await migrateContentItemsAliasOfId(knex);
+    await migrateInfoRoots(knex, onLog);
+    await migrateInfoMainContentSelector(knex, onLog);
+    await migrateMainContentsColumns(knex, onLog);
+    await migratePageMetaBodyHash(knex, onLog);
+    await migratePageMetaConsoleErrorCount(knex, onLog);
+    await migratePageMetaCustomElementCount(knex, onLog);
+    // Table-level migration (converts + drops page_tags), not a column
+    // add — see its own JSDoc for why it still belongs in this boot phase.
+    await migratePageTagsToPageTechnologies(knex, onLog);
+    await migrateContentItemsAliasOfId(knex, onLog);
     // Runs after `initSchema` above, which already created
     // `dedupe_cap_events` (an adjunct table) unconditionally — so the new
     // column's `REFERENCES dedupe_cap_events(id)` target always exists by
     // this point, for both fresh and legacy archives.
-    await migrateContentItemsDedupeCapEventId(knex);
-    await migrateInventoryRunsInvalidSkipped(knex);
-    await migrateInventoryRunsExcludeSkipped(knex);
+    await migrateContentItemsDedupeCapEventId(knex, onLog);
+    await migrateInventoryRunsInvalidSkipped(knex, onLog);
+    await migrateInventoryRunsExcludeSkipped(knex, onLog);
     await closeStaleOpenNetworkOutages(knex);
 }

package/lib/archive/db-ops/meta/get-custom-elements-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { MainContentCustomElementRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_main_content_custom_elements` rows for the given page
+ * id, in DOM traversal order.
+ *
+ * Read-side counterpart to `insertCustomElements`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getCustomElementsOfPage(knex: Knex, pageId: number): Promise<MainContentCustomElementRow[]>;

package/lib/archive/db-ops/meta/get-custom-elements-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_main_content_custom_elements` rows for the given page
+ * id, in DOM traversal order.
+ *
+ * Read-side counterpart to `insertCustomElements`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getCustomElementsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'order', 'nodeName', 'elementId', 'classList')
+        .from('page_main_content_custom_elements')
+        .where('pageId', pageId)
+        .orderBy('order', 'asc');
+}

package/lib/archive/db-ops/meta/get-page-technologies-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { PageTechnologyRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `page_technologies` rows for the given page id — the
+ * confidence-combined roll-up (one row per detected technology).
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getPageTechnologiesOfPage(knex: Knex, pageId: number): Promise<PageTechnologyRow[]>;

package/lib/archive/db-ops/meta/get-page-technologies-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `page_technologies` rows for the given page id — the
+ * confidence-combined roll-up (one row per detected technology).
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getPageTechnologiesOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'technology', 'category', 'version', 'confidence', 'signalCount')
+        .from('page_technologies')
+        .where('pageId', pageId)
+        .orderBy('confidence', 'desc');
+}

package/lib/archive/db-ops/meta/get-technology-signals-of-page.d.ts

@@ -0,0 +1,11 @@
+import type { TechnologySignalRow } from '../../meta/types.js';
+import type { Knex } from 'knex';
+/**
+ * Retrieves all `technology_signals` rows for the given page id — the raw,
+ * per-signal evidence behind its `page_technologies` roll-up.
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export declare function getTechnologySignalsOfPage(knex: Knex, pageId: number): Promise<TechnologySignalRow[]>;

package/lib/archive/db-ops/meta/get-technology-signals-of-page.js

@@ -0,0 +1,15 @@
+/**
+ * Retrieves all `technology_signals` rows for the given page id — the raw,
+ * per-signal evidence behind its `page_technologies` roll-up.
+ *
+ * Read-side counterpart to `insertTechnologies`.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param pageId
+ */
+export async function getTechnologySignalsOfPage(knex, pageId) {
+    return knex
+        .select('id', 'pageId', 'technology', 'signalType', 'evidence', 'weight')
+        .from('technology_signals')
+        .where('pageId', pageId)
+        .orderBy('id', 'asc');
+}

package/lib/archive/db-ops/pages/order/set-url-order.d.ts

@@ -4,5 +4,10 @@ import type { Knex } from 'knex';
  * all internal pages. Pages are sorted using {@link pathComparator} and
  * assigned sequential order numbers.
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's UPDATE completes, with
+ *   pages ordered so far and the total (issue #294: a large archive's page
+ *   count can make this a multi-second-to-minutes pass — a single SELECT
+ *   plus a JS sort followed by chunked UPDATEs — with no other signal it
+ *   hasn't hung). Omit for no reporting (the default; e.g. tests).
  */
-export declare function setUrlOrder(knex: Knex): Promise<void>;
+export declare function setUrlOrder(knex: Knex, onProgress?: (processed: number, total: number) => void): Promise<void>;

package/lib/archive/db-ops/pages/order/set-url-order.js

@@ -5,8 +5,13 @@ import { dbLog } from '../../../debug.js';
  * all internal pages. Pages are sorted using {@link pathComparator} and
  * assigned sequential order numbers.
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's UPDATE completes, with
+ *   pages ordered so far and the total (issue #294: a large archive's page
+ *   count can make this a multi-second-to-minutes pass — a single SELECT
+ *   plus a JS sort followed by chunked UPDATEs — with no other signal it
+ *   hasn't hung). Omit for no reporting (the default; e.g. tests).
  */
-export async function setUrlOrder(knex) {
+export async function setUrlOrder(knex, onProgress) {
     dbLog('Set URL Order');
     const res = (await knex
         .select('ci.id', 'ur.url')
@@ -28,5 +33,6 @@ export async function setUrlOrder(knex) {
             .join(' ');
         const placeholders = ids.map(() => '?').join(',');
         await knex.raw(`UPDATE content_items SET crawl_order = CASE id ${cases} END WHERE id IN (${placeholders})`, [...bindings, ...ids]);
+        onProgress?.(Math.min(i + BATCH_SIZE, sorted.length), sorted.length);
     }
 }

package/lib/archive/db-ops/pages/read/build-page-query.js

@@ -36,5 +36,5 @@ export function buildPageQuery(knex) {
         .leftJoin('url_refs as og_image_ur', 'og_image_ur.id', 'pm.og_image_url_id')
         .leftJoin('url_refs as twitter_image_ur', 'twitter_image_ur.id', 'pm.twitter_image_url_id')
         .leftJoin('json_refs as extras_ref', 'extras_ref.id', 'pm.meta_extras_json_id')
-        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
+        .select('ci.id as id', 'ur.url as url', 'ci.redirect_dest_id as redirectDestId', 'ci.scraped as scraped', 'ci.is_target as isTarget', 'ci.is_external as isExternal', 'ci.status as status', 'ci.status_text as statusText', 'ctr.raw as contentType', 'ci.content_length as contentLength', 'ci.header_set_id as headerSetId', 'pm.lang as lang', 'pm.dir as dir', 'pm.charset as charset', 'pm.base_href as baseHref', 'pm.viewport_raw as viewport_raw', 'pm.theme_color as themeColor', 'pm.application_name as applicationName', 'pm.author as author', 'pm.generator as generator', 'pm.publisher as publisher', 'title_ref.text as title', 'description_ref.text as description', 'keywords_ref.text as keywords', 'robots_raw_ref.text as robots_raw', 'pm.robots_noindex as robots_noindex', 'pm.robots_nofollow as robots_nofollow', 'pm.robots_noarchive as robots_noarchive', 'pm.robots_noimageindex as robots_noimageindex', 'pm.googlebot as googlebot', 'canonical_ur.url as canonical', 'amphtml_ur.url as amphtml', 'manifest_ur.url as manifest', 'icon_ur.url as icon_href', 'apple_ur.url as appleTouchIcon_href', 'pm.og_type as og_type', 'og_title_ref.text as og_title', 'og_url_ur.url as og_url', 'pm.og_site_name as og_site_name', 'og_description_ref.text as og_description', 'og_image_ur.url as og_image', 'pm.og_image_alt as og_image_alt', 'pm.og_image_width as og_image_width', 'pm.og_image_height as og_image_height', 'pm.og_locale as og_locale', 'pm.og_article_published_time as og_article_published_time', 'pm.og_article_modified_time as og_article_modified_time', 'pm.twitter_card as twitter_card', 'pm.twitter_site as twitter_site', 'pm.twitter_creator as twitter_creator', 'twitter_title_ref.text as twitter_title', 'twitter_description_ref.text as twitter_description', 'twitter_image_ur.url as twitter_image', 'pm.fb_app_id as fb_app_id', 'pm.verification_google as verification_google', 'pm.format_detection_telephone as formatDetection_telephone', 'ci.first_crawled_at as firstCrawledAt', 'ci.last_crawled_at as lastCrawledAt', 'pm.tag_count as tag_count', 'pm.jsonld_count as jsonld_count', 'pm.tags_providers_csv as tags_providers_csv', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_selector as main_content_selector', 'pm.main_content_class_list as main_content_class_list', 'pm.main_content_word_count as main_content_word_count', 'pm.main_content_body_word_count as main_content_body_word_count', 'pm.main_content_heading_count as main_content_heading_count', 'pm.main_content_image_count as main_content_image_count', 'pm.main_content_table_count as main_content_table_count', 'pm.main_content_button_count as main_content_button_count', 'pm.main_content_iframe_count as main_content_iframe_count', 'pm.main_content_video_count as main_content_video_count', 'pm.main_content_audio_count as main_content_audio_count', 'pm.main_content_canvas_count as main_content_canvas_count', 'pm.main_content_custom_element_count as main_content_custom_element_count', 'pm.scroll_height_desktop as scroll_height_desktop', 'pm.scroll_height_mobile as scroll_height_mobile', 'extras_ref.json_text as extras_body', 'extras_ref.codec as extras_codec', 'ci.is_skipped as isSkipped', 'ci.skip_reason as skipReason', 'ci.crawl_order as order', 'ci.source as source');
 }

package/lib/archive/db-ops/pages/reset/repromote-external-pages.d.ts

@@ -19,6 +19,11 @@ import type { Knex } from 'knex';
  * @param knex - Knex query builder connected to the archive DB.
  * @param scopes - The hostname-indexed scope map after the new roots are merged.
  * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to promote
+ *   (issue #294: a large `--append` can promote thousands of pages across
+ *   14 tables, running for seconds to minutes with no other signal it
+ *   hasn't hung). Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were promoted.
  */
-export declare function repromoteExternalPages(knex: Knex, scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<string[]>;
+export declare function repromoteExternalPages(knex: Knex, scopes: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions, onProgress?: (processed: number, total: number) => void): Promise<string[]>;

package/lib/archive/db-ops/pages/reset/repromote-external-pages.js

@@ -20,9 +20,14 @@ import { dbLog } from '../../../debug.js';
  * @param knex - Knex query builder connected to the archive DB.
  * @param scopes - The hostname-indexed scope map after the new roots are merged.
  * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to promote
+ *   (issue #294: a large `--append` can promote thousands of pages across
+ *   14 tables, running for seconds to minutes with no other signal it
+ *   hasn't hung). Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were promoted.
  */
-export async function repromoteExternalPages(knex, scopes, options) {
+export async function repromoteExternalPages(knex, scopes, options, onProgress) {
     if (scopes.size === 0) {
         return [];
     }
@@ -73,7 +78,8 @@ export async function repromoteExternalPages(knex, scopes, options) {
         // Deleting the `page_meta` row (rather than nulling every column)
         // clears title / description / og:* / twitter:* / meta_extras /
         // main_content_* in one statement; a re-scrape re-inserts it via
-        // `ON CONFLICT(page_id) DO UPDATE`. `page_tags` / `page_jsonld` /
+        // `ON CONFLICT(page_id) DO UPDATE`. `technology_signals` /
+        // `page_technologies` / `page_jsonld` /
         // `page_main_content_*` are cleared explicitly even though all of
         // them also carry ON DELETE CASCADE — we keep the existing pattern
         // of explicit chunked DELETEs rather than relying on CASCADE
@@ -86,7 +92,8 @@ export async function repromoteExternalPages(knex, scopes, options) {
         await knex('image_items').whereIn('page_id', chunk).delete();
         await knex('resource_ref_edges').whereIn('page_id', chunk).delete();
         await knex('page_html_ref').whereIn('page_id', chunk).delete();
-        await knex('page_tags').whereIn('pageId', chunk).delete();
+        await knex('technology_signals').whereIn('pageId', chunk).delete();
+        await knex('page_technologies').whereIn('pageId', chunk).delete();
         await knex('page_jsonld').whereIn('pageId', chunk).delete();
         await knex('page_main_content_headings').whereIn('pageId', chunk).delete();
         await knex('page_main_content_images').whereIn('pageId', chunk).delete();
@@ -96,6 +103,8 @@ export async function repromoteExternalPages(knex, scopes, options) {
         await knex('page_main_content_videos').whereIn('pageId', chunk).delete();
         await knex('page_main_content_audios').whereIn('pageId', chunk).delete();
         await knex('page_main_content_canvases').whereIn('pageId', chunk).delete();
+        await knex('page_main_content_custom_elements').whereIn('pageId', chunk).delete();
+        onProgress?.(Math.min(i + chunkSize, promotedIds.length), promotedIds.length);
     }
     dbLog('Repromoted %d external pages back to pending', promotedUrls.length);
     return promotedUrls;

package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts

@@ -57,6 +57,11 @@ import type { Knex } from 'knex';
  * SELECT and UPDATE/DELETE statements are chunked to stay below SQLite's
  * `SQLITE_LIMIT_VARIABLE_NUMBER`.
  * @param knex - Knex query builder connected to the archive DB.
+ * @param onProgress - Called after each chunk's DELETE/UPDATE statements
+ *   complete, with the pages processed so far and the total to reset (issue
+ *   #294: a large `--retry-failed` can reset thousands of pages across 13
+ *   tables, running for seconds to minutes with no other signal it hasn't
+ *   hung). Omit for no reporting (the default; e.g. tests).
  * @returns The URLs of the pages that were reset to pending.
  */
-export declare function resetFailedPages(knex: Knex): Promise<string[]>;
+export declare function resetFailedPages(knex: Knex, onProgress?: (processed: number, total: number) => void): Promise<string[]>;