@nitpicker/crawler 0.18.1 → 0.20.0

package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts

@@ -7,5 +7,10 @@ import type { Knex } from 'knex';
  * `initSchema` call — idempotent, and self-healing for archives whose
  * provisioning crashed partway through.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export declare function migrateInventoryRunsInvalidSkipped(instance: Knex): Promise<void>;
+export declare function migrateInventoryRunsInvalidSkipped(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-inventory-runs-invalid-skipped.js

@@ -6,8 +6,13 @@
  * `initSchema` call — idempotent, and self-healing for archives whose
  * provisioning crashed partway through.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export async function migrateInventoryRunsInvalidSkipped(instance) {
+export async function migrateInventoryRunsInvalidSkipped(instance, onLog) {
     const hasTable = await instance.schema.hasTable('inventory_runs');
     if (!hasTable) {
         return;
@@ -19,6 +24,12 @@ export async function migrateInventoryRunsInvalidSkipped(instance) {
     await instance.schema.table('inventory_runs', (t) => {
         t.integer('invalid_skipped');
     });
-    // eslint-disable-next-line no-console
-    console.error('[migrate] inventory_runs.invalid_skipped column added');
+    const message = '[migrate] inventory_runs.invalid_skipped column added';
+    if (onLog) {
+        onLog(message);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(message);
+    }
 }

package/lib/archive/migrate-main-contents-columns.d.ts

@@ -17,5 +17,10 @@ import type { Knex } from 'knex';
  * itself after `assertCompatibleVersion` rejects pre-0.13 archives) the
  * table is always present.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export declare function migrateMainContentsColumns(instance: Knex): Promise<void>;
+export declare function migrateMainContentsColumns(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-main-contents-columns.js

@@ -16,8 +16,13 @@
  * itself after `assertCompatibleVersion` rejects pre-0.13 archives) the
  * table is always present.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export async function migrateMainContentsColumns(instance) {
+export async function migrateMainContentsColumns(instance, onLog) {
     const hasPageMeta = await instance.schema.hasTable('page_meta');
     if (!hasPageMeta) {
         return;
@@ -45,6 +50,12 @@ export async function migrateMainContentsColumns(instance) {
         t.integer('scroll_height_desktop');
         t.integer('scroll_height_mobile');
     });
-    // eslint-disable-next-line no-console
-    console.error('[migrate] page_meta.main_content_* / scroll_height_* columns added');
+    const message = '[migrate] page_meta.main_content_* / scroll_height_* columns added';
+    if (onLog) {
+        onLog(message);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(message);
+    }
 }

package/lib/archive/migrate-page-meta-body-hash.d.ts

@@ -35,5 +35,10 @@ import type { Knex } from 'knex';
  * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
  * pre-0.13 archives) the table is always present.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export declare function migratePageMetaBodyHash(instance: Knex): Promise<void>;
+export declare function migratePageMetaBodyHash(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-page-meta-body-hash.js

@@ -34,8 +34,13 @@
  * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
  * pre-0.13 archives) the table is always present.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export async function migratePageMetaBodyHash(instance) {
+export async function migratePageMetaBodyHash(instance, onLog) {
     const hasPageMeta = await instance.schema.hasTable('page_meta');
     if (!hasPageMeta) {
         return;
@@ -45,8 +50,14 @@ export async function migratePageMetaBodyHash(instance) {
         await instance.schema.table('page_meta', (t) => {
             t.binary('body_hash');
         });
-        // eslint-disable-next-line no-console
-        console.error('[migrate] page_meta.body_hash column added');
+        const message = '[migrate] page_meta.body_hash column added';
+        if (onLog) {
+            onLog(message);
+        }
+        else {
+            // eslint-disable-next-line no-console
+            console.error(message);
+        }
     }
     await instance.raw('CREATE INDEX IF NOT EXISTS idx_page_meta_body_hash ON page_meta(body_hash)');
 }

package/lib/archive/migrate-page-meta-console-error-count.d.ts

@@ -18,5 +18,10 @@ import type { Knex } from 'knex';
  *
  * Idempotent: adding the column is a no-op once it exists.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export declare function migratePageMetaConsoleErrorCount(instance: Knex): Promise<void>;
+export declare function migratePageMetaConsoleErrorCount(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-page-meta-console-error-count.js

@@ -17,8 +17,13 @@
  *
  * Idempotent: adding the column is a no-op once it exists.
  * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
  */
-export async function migratePageMetaConsoleErrorCount(instance) {
+export async function migratePageMetaConsoleErrorCount(instance, onLog) {
     const hasPageMeta = await instance.schema.hasTable('page_meta');
     if (!hasPageMeta) {
         return;
@@ -28,7 +33,13 @@ export async function migratePageMetaConsoleErrorCount(instance) {
         await instance.schema.table('page_meta', (t) => {
             t.integer('console_error_count');
         });
-        // eslint-disable-next-line no-console
-        console.error('[migrate] page_meta.console_error_count column added');
+        const message = '[migrate] page_meta.console_error_count column added';
+        if (onLog) {
+            onLog(message);
+        }
+        else {
+            // eslint-disable-next-line no-console
+            console.error(message);
+        }
     }
 }

package/lib/archive/migrate-page-meta-custom-element-count.d.ts

@@ -0,0 +1,33 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the `page_meta.main_content_custom_element_count` column to archives
+ * created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same catch-up
+ * shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
+ *
+ * Deliberately its own file rather than an addition to
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}:
+ * that migration early-returns once its sentinel column
+ * (`main_content_word_count`) already exists, so an archive that predates
+ * only this one column (but already has the other sixteen main-content
+ * columns) would never reach an appended `ALTER TABLE` inside it.
+ *
+ * No separate backfill: a page that has never been re-scraped since this
+ * feature shipped stays `NULL` (unknown), which is correct — it is not
+ * conflated with `0` (capture succeeded, zero custom elements found). See
+ * {@link import('./meta/compute-main-contents-denormalized.js').computeMainContentsDenormalized}.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export declare function migratePageMetaCustomElementCount(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-page-meta-custom-element-count.js

@@ -0,0 +1,51 @@
+/**
+ * Adds the `page_meta.main_content_custom_element_count` column to archives
+ * created before this feature.
+ *
+ * `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
+ * {@link import('./create-entity-tables.js').createEntityTables}, which
+ * self-heals a *missing table* on every `initSchema` call but is a no-op
+ * against an *existing* table — adding a column to the DDL string never
+ * reaches an archive whose `page_meta` predates this change. Same catch-up
+ * shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
+ *
+ * Deliberately its own file rather than an addition to
+ * {@link import('./migrate-main-contents-columns.js').migrateMainContentsColumns}:
+ * that migration early-returns once its sentinel column
+ * (`main_content_word_count`) already exists, so an archive that predates
+ * only this one column (but already has the other sixteen main-content
+ * columns) would never reach an appended `ALTER TABLE` inside it.
+ *
+ * No separate backfill: a page that has never been re-scraped since this
+ * feature shipped stays `NULL` (unknown), which is correct — it is not
+ * conflated with `0` (capture succeeded, zero custom elements found). See
+ * {@link import('./meta/compute-main-contents-denormalized.js').computeMainContentsDenormalized}.
+ *
+ * Idempotent: adding the column is a no-op once it exists.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` when this migration
+ *   actually applies (issue #294: a bare `console.error` here can fire
+ *   while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export async function migratePageMetaCustomElementCount(instance, onLog) {
+    const hasPageMeta = await instance.schema.hasTable('page_meta');
+    if (!hasPageMeta) {
+        return;
+    }
+    const hasColumn = await instance.schema.hasColumn('page_meta', 'main_content_custom_element_count');
+    if (!hasColumn) {
+        await instance.schema.table('page_meta', (t) => {
+            t.integer('main_content_custom_element_count');
+        });
+        const message = '[migrate] page_meta.main_content_custom_element_count column added';
+        if (onLog) {
+            onLog(message);
+        }
+        else {
+            // eslint-disable-next-line no-console
+            console.error(message);
+        }
+    }
+}

package/lib/archive/migrate-page-tags-to-page-technologies.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Converts every `page_tags` row (the removed Wappalyzer-only detection
+ * table) into `technology_signals` + `page_technologies` rows, then drops
+ * `page_tags` — for archives crawled before this feature shipped, which
+ * still carry the old table.
+ *
+ * Unlike every other `migrate*` function in this directory (which add a
+ * nullable column and never touch existing tables), this one both writes
+ * to two NEW tables and DROPS an old one — because `page_technologies`
+ * supersedes `page_tags` outright rather than extending it. The conversion
+ * itself is shared with `retarget-legacy-fk-tables.ts`'s special-cased
+ * `page_tags` handling (0.10→0.13 upgrade path) via
+ * `convertLegacyPageTagsToInserts` — a `page_tags` row converts identically
+ * regardless of which migration path produced it.
+ *
+ * No separate backfill step is needed (v0.x policy: read-time migration
+ * only, no retroactive re-crawl): only the `wappalyzer` signal survives the
+ * conversion — the structural/`js-license-comment` signals this feature
+ * also detects require re-crawling to populate, which is the same
+ * "existing archives need a re-crawl for new signal types" trade-off
+ * already accepted for this feature.
+ *
+ * Idempotent: a no-op once `page_tags` is gone (either because this
+ * function already ran, or because the archive was crawled after this
+ * feature shipped and never had `page_tags` at all).
+ *
+ * Logs a row count before starting (issue #294): on a large legacy archive
+ * this SELECT-all-convert-drop runs as one transaction with no per-chunk
+ * progress, so without an upfront count it looks indistinguishable from a
+ * hang for however long the conversion actually takes.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` for both the start and
+ *   end notices (issue #294: a bare `console.error` here can fire while a
+ *   `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export declare function migratePageTagsToPageTechnologies(instance: Knex, onLog?: (message: string) => void): Promise<void>;

package/lib/archive/migrate-page-tags-to-page-technologies.js

@@ -0,0 +1,79 @@
+import { eachSplitted } from '../utils/array/each-splitted.js';
+import { convertLegacyPageTagsToInserts } from './meta/technologies/convert-legacy-page-tags-to-inserts.js';
+/**
+ * Converts every `page_tags` row (the removed Wappalyzer-only detection
+ * table) into `technology_signals` + `page_technologies` rows, then drops
+ * `page_tags` — for archives crawled before this feature shipped, which
+ * still carry the old table.
+ *
+ * Unlike every other `migrate*` function in this directory (which add a
+ * nullable column and never touch existing tables), this one both writes
+ * to two NEW tables and DROPS an old one — because `page_technologies`
+ * supersedes `page_tags` outright rather than extending it. The conversion
+ * itself is shared with `retarget-legacy-fk-tables.ts`'s special-cased
+ * `page_tags` handling (0.10→0.13 upgrade path) via
+ * `convertLegacyPageTagsToInserts` — a `page_tags` row converts identically
+ * regardless of which migration path produced it.
+ *
+ * No separate backfill step is needed (v0.x policy: read-time migration
+ * only, no retroactive re-crawl): only the `wappalyzer` signal survives the
+ * conversion — the structural/`js-license-comment` signals this feature
+ * also detects require re-crawling to populate, which is the same
+ * "existing archives need a re-crawl for new signal types" trade-off
+ * already accepted for this feature.
+ *
+ * Idempotent: a no-op once `page_tags` is gone (either because this
+ * function already ran, or because the archive was crawled after this
+ * feature shipped and never had `page_tags` at all).
+ *
+ * Logs a row count before starting (issue #294): on a large legacy archive
+ * this SELECT-all-convert-drop runs as one transaction with no per-chunk
+ * progress, so without an upfront count it looks indistinguishable from a
+ * hang for however long the conversion actually takes.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @param onLog - Called instead of `console.error` for both the start and
+ *   end notices (issue #294: a bare `console.error` here can fire while a
+ *   `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
+ *   `Archive.open`, corrupting its cursor tracking). Falls back to
+ *   `console.error` when omitted (direct/test callers).
+ */
+export async function migratePageTagsToPageTechnologies(instance, onLog) {
+    const hasPageTags = await instance.schema.hasTable('page_tags');
+    if (!hasPageTags)
+        return;
+    const [row] = await instance('page_tags').count({ total: '*' });
+    const total = row?.total ?? 0;
+    const startMessage = `[migrate] converting ${total} page_tags row(s) to technology_signals/page_technologies…`;
+    if (onLog) {
+        onLog(startMessage);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(startMessage);
+    }
+    await instance.transaction(async (trx) => {
+        const rows = await trx
+            .select('pageId', 'provider', 'version', 'confidence', 'categories')
+            .from('page_tags');
+        const { signalInserts, technologyInserts } = convertLegacyPageTagsToInserts(rows);
+        if (signalInserts.length > 0) {
+            await eachSplitted(signalInserts, 100, async (chunk) => {
+                await trx('technology_signals').insert(chunk);
+            });
+        }
+        if (technologyInserts.length > 0) {
+            await eachSplitted(technologyInserts, 100, async (chunk) => {
+                await trx('page_technologies').insert(chunk);
+            });
+        }
+        await trx.schema.dropTable('page_tags');
+    });
+    const endMessage = '[migrate] page_tags converted to technology_signals/page_technologies and dropped';
+    if (onLog) {
+        onLog(endMessage);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error(endMessage);
+    }
+}

package/lib/archive/page.d.ts

@@ -1,5 +1,5 @@
 import type { ArchiveAccessor } from './archive-accessor.js';
-import type { JsonLdRow, JsonLdSummary, TagRow, TagsSummary } from './meta/types.js';
+import type { JsonLdRow, JsonLdSummary, PageTechnologyRow, TechnologySignalRow } from './meta/types.js';
 import type { Anchor, Redirect, Referrer, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer } from './types.js';
 /**
  * Subset of {@link DB_Page} that maps to the flat meta columns derived from
@@ -18,9 +18,10 @@ declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref",
  * iterable view over all ~47 flat meta columns and {@link Page.metaExtras} for
  * the JSON catch-all of nested sub-objects not flattened to columns.
  *
- * JSON-LD entries and Wappalyzer tag rows live in dedicated tables and are
- * fetched on demand via {@link Page.getJsonLd} / {@link Page.getTags} (lazy reads, same
- * pattern as {@link Page.getAnchors}).
+ * JSON-LD entries and detected technology signals live in dedicated tables
+ * and are fetched on demand via {@link Page.getJsonLd} /
+ * {@link Page.getTechnologySignals} / {@link Page.getPageTechnologies} (lazy
+ * reads, same pattern as {@link Page.getAnchors}).
  *
  * Instances are created by {@link ArchiveAccessor.getPages} or
  * {@link ArchiveAccessor.getPagesWithRefs}.
@@ -155,6 +156,12 @@ export default class Page {
      * Number of canvases within the main region (denormalised aggregate), or null.
      */
     get mainContentCanvasCount(): number | null;
+    /**
+     * Number of Web Components (custom elements) within the main region
+     * (denormalised aggregate), `null` when unknown (page not rendered, or
+     * nitpicker's own capture failed — see `compute-main-contents-denormalized.ts`).
+     */
+    get mainContentCustomElementCount(): number | null;
     /**
      * Iterable view over every flat meta column (~47 fields). Returns a frozen
      * record so consumers can pick fields by name without re-enumerating
@@ -251,8 +258,9 @@ export default class Page {
     get tagCount(): number | null;
     /**
      * Sorted unique Wappalyzer provider names, comma-separated, empty string
-     * when no tags. Denormalised aggregate; for the structured form fetch
-     * {@link Page.getTags} (lazy).
+     * when no tags. Denormalised aggregate; for the structured form (with
+     * confidence, category, and every raw signal) fetch
+     * {@link Page.getTechnologySignals} / {@link Page.getPageTechnologies} (lazy).
      */
     get tagsProvidersCsv(): string;
     /**
@@ -304,6 +312,13 @@ export default class Page {
      * @returns Ordered canvas rows.
      */
     getCanvases(): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
+    /**
+     * Retrieves the Web Components (custom elements) within this page's
+     * detected main content region from `page_main_content_custom_elements`.
+     * Lazy — runs a single SELECT per call.
+     * @returns Ordered custom-element rows.
+     */
+    getCustomElements(): Promise<import("./meta/types.js").MainContentCustomElementRow[]>;
     /**
      * Retrieves the headings within this page's detected main content region
      * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
@@ -349,6 +364,12 @@ export default class Page {
      * @returns Ordered table rows.
      */
     getMainContentTables(): Promise<import("./meta/types.js").MainContentTableRow[]>;
+    /**
+     * Retrieves the confidence-combined technology roll-up for this page
+     * from `page_technologies`. Lazy — runs a single SELECT per call.
+     * @returns Technology rows, highest confidence first.
+     */
+    getPageTechnologies(): Promise<readonly PageTechnologyRow[]>;
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -362,11 +383,11 @@ export default class Page {
      */
     getRequests(): Promise<Referrer[]>;
     /**
-     * Retrieves the Wappalyzer tag rows for this page from `page_tags`.
-     * Lazy — runs a single SELECT per call.
-     * @returns Ordered tag rows.
+     * Retrieves the raw technology-detection signals for this page from
+     * `technology_signals`. Lazy — runs a single SELECT per call.
+     * @returns Ordered signal rows.
      */
-    getTags(): Promise<readonly TagRow[]>;
+    getTechnologySignals(): Promise<readonly TechnologySignalRow[]>;
     /**
      * Retrieves the videos within this page's detected main content region
      * from `page_main_content_videos`. Lazy — runs a single SELECT per call.
@@ -385,8 +406,11 @@ export default class Page {
     isPage(): boolean;
     /**
      * Serializes the page data to a plain JSON object including the full flat
-     * meta column set, the `meta_extras` catch-all, and **summaries** of the
-     * JSON-LD and tag rows.
+     * meta column set, the `meta_extras` catch-all, a **summary** of the
+     * JSON-LD rows, and the confidence-combined technology roll-up
+     * (`page_technologies` — already compact, one row per detected
+     * technology, so no further summarization is needed the way `page_tags`'
+     * per-`externalId` granularity used to require).
      *
      * Summaries (not raw entries) are inlined so a Page detail payload stays
      * token-bounded for MCP / LLM consumers — the full `raw` JSON-LD payload
@@ -400,7 +424,7 @@ export default class Page {
     toJSON(): Promise<{
         metaExtras: Record<string, unknown>;
         jsonLd: JsonLdSummary;
-        tags: TagsSummary;
+        technologies: readonly PageTechnologyRow[];
         redirectFrom: Redirect[];
         isPage: boolean;
         isInternalPage: boolean;

package/lib/archive/page.js

@@ -2,7 +2,6 @@ import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
 import { isHtmlContentType } from '../crawler/is-html-content-type.js';
 import { parseResponseHeaders } from '../utils/object/parse-response-headers.js';
 import { summarizeJsonLd } from './meta/summarize-jsonld.js';
-import { summarizeTags } from './meta/summarize-tags.js';
 /**
  * Subset of {@link DB_Page} that maps to the flat meta columns derived from
  * beholder's nested Meta. Used by {@link Page.metaFlat} so consumers can
@@ -86,9 +85,10 @@ const FLAT_META_COLUMNS = [
  * iterable view over all ~47 flat meta columns and {@link Page.metaExtras} for
  * the JSON catch-all of nested sub-objects not flattened to columns.
  *
- * JSON-LD entries and Wappalyzer tag rows live in dedicated tables and are
- * fetched on demand via {@link Page.getJsonLd} / {@link Page.getTags} (lazy reads, same
- * pattern as {@link Page.getAnchors}).
+ * JSON-LD entries and detected technology signals live in dedicated tables
+ * and are fetched on demand via {@link Page.getJsonLd} /
+ * {@link Page.getTechnologySignals} / {@link Page.getPageTechnologies} (lazy
+ * reads, same pattern as {@link Page.getAnchors}).
  *
  * Instances are created by {@link ArchiveAccessor.getPages} or
  * {@link ArchiveAccessor.getPagesWithRefs}.
@@ -286,6 +286,14 @@ export default class Page {
     get mainContentCanvasCount() {
         return this.#raw.main_content_canvas_count;
     }
+    /**
+     * Number of Web Components (custom elements) within the main region
+     * (denormalised aggregate), `null` when unknown (page not rendered, or
+     * nitpicker's own capture failed — see `compute-main-contents-denormalized.ts`).
+     */
+    get mainContentCustomElementCount() {
+        return this.#raw.main_content_custom_element_count;
+    }
     /**
      * Iterable view over every flat meta column (~47 fields). Returns a frozen
      * record so consumers can pick fields by name without re-enumerating
@@ -436,8 +444,9 @@ export default class Page {
     }
     /**
      * Sorted unique Wappalyzer provider names, comma-separated, empty string
-     * when no tags. Denormalised aggregate; for the structured form fetch
-     * {@link Page.getTags} (lazy).
+     * when no tags. Denormalised aggregate; for the structured form (with
+     * confidence, category, and every raw signal) fetch
+     * {@link Page.getTechnologySignals} / {@link Page.getPageTechnologies} (lazy).
      */
     get tagsProvidersCsv() {
         return this.#raw.tags_providers_csv ?? '';
@@ -530,6 +539,15 @@ export default class Page {
     async getCanvases() {
         return this.#archive.getCanvasesOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the Web Components (custom elements) within this page's
+     * detected main content region from `page_main_content_custom_elements`.
+     * Lazy — runs a single SELECT per call.
+     * @returns Ordered custom-element rows.
+     */
+    async getCustomElements() {
+        return this.#archive.getCustomElementsOfPage(this.#raw.id);
+    }
     /**
      * Retrieves the headings within this page's detected main content region
      * from `page_main_content_headings`. Lazy — runs a single SELECT per call.
@@ -587,6 +605,14 @@ export default class Page {
     async getMainContentTables() {
         return this.#archive.getMainContentTablesOfPage(this.#raw.id);
     }
+    /**
+     * Retrieves the confidence-combined technology roll-up for this page
+     * from `page_technologies`. Lazy — runs a single SELECT per call.
+     * @returns Technology rows, highest confidence first.
+     */
+    async getPageTechnologies() {
+        return this.#archive.getPageTechnologiesOfPage(this.#raw.id);
+    }
     /**
      * Retrieves the referrers (incoming links) pointing to this page.
      * Uses pre-loaded data if available, otherwise queries the database.
@@ -627,12 +653,12 @@ export default class Page {
         }));
     }
     /**
-     * Retrieves the Wappalyzer tag rows for this page from `page_tags`.
-     * Lazy — runs a single SELECT per call.
-     * @returns Ordered tag rows.
+     * Retrieves the raw technology-detection signals for this page from
+     * `technology_signals`. Lazy — runs a single SELECT per call.
+     * @returns Ordered signal rows.
      */
-    async getTags() {
-        return this.#archive.getTagsOfPage(this.#raw.id);
+    async getTechnologySignals() {
+        return this.#archive.getTechnologySignalsOfPage(this.#raw.id);
     }
     /**
      * Retrieves the videos within this page's detected main content region
@@ -658,8 +684,11 @@ export default class Page {
     }
     /**
      * Serializes the page data to a plain JSON object including the full flat
-     * meta column set, the `meta_extras` catch-all, and **summaries** of the
-     * JSON-LD and tag rows.
+     * meta column set, the `meta_extras` catch-all, a **summary** of the
+     * JSON-LD rows, and the confidence-combined technology roll-up
+     * (`page_technologies` — already compact, one row per detected
+     * technology, so no further summarization is needed the way `page_tags`'
+     * per-`externalId` granularity used to require).
      *
      * Summaries (not raw entries) are inlined so a Page detail payload stays
      * token-bounded for MCP / LLM consumers — the full `raw` JSON-LD payload
@@ -671,14 +700,13 @@ export default class Page {
      * @returns A plain object containing all page metadata and relationships.
      */
     async toJSON() {
-        const [anchors, referrers, jsonLdRows, tagRows] = await Promise.all([
+        const [anchors, referrers, jsonLdRows, technologies] = await Promise.all([
             this.getAnchors(),
             this.getReferrers(),
             this.getJsonLd(),
-            this.getTags(),
+            this.getPageTechnologies(),
         ]);
         const jsonLdSummary = summarizeJsonLd(jsonLdRows);
-        const tagsSummary = summarizeTags(tagRows);
         return {
             url: this.url.href,
             status: this.status,
@@ -695,7 +723,7 @@ export default class Page {
             ...this.metaFlat,
             metaExtras: this.metaExtras,
             jsonLd: jsonLdSummary,
-            tags: tagsSummary,
+            technologies,
             redirectFrom: this.redirectFrom,
             isPage: this.isPage(),
             isInternalPage: this.isInternalPage(),