@nitpicker/crawler 0.18.1 → 0.20.0

package/lib/archive/filesystem/peek-tar-top-dir.js

@@ -1,4 +1,13 @@
+import { open } from 'node:fs/promises';
 import { list } from 'tar';
+import { parsePaxPath } from './parse-pax-path.js';
+import { parseTarSizeField } from './parse-tar-size-field.js';
+/** Bytes per tar header/data block (fixed by the tar format). */
+const BLOCK_SIZE = 512;
+/** Defensive cap on entries scanned before giving up and falling back to `list()`. */
+const MAX_ENTRIES = 100_000;
+/** Cap on PAX/GNU-longname payload size read into memory; larger is treated as malformed. */
+const MAX_EXTENDED_PAYLOAD_SIZE = 1_000_000;
 /**
  * Reads the top-level directory name from a `.nitpicker` tar archive.
  *
@@ -13,9 +22,16 @@ import { list } from 'tar';
  * directory it sees, so callers (`Archive.open`, the migration script) can
  * use the actual inner name regardless of what the outer file is called.
  *
- * Reads the tar's central directory only; does not extract any data. Runs
- * in O(number of entries) but stops at the first match, so a typical
- * archive (1–2 top-level entries) completes in a single read.
+ * Parses raw tar headers directly (ustar fixed fields, PAX extended headers,
+ * GNU longname/longlink) instead of using the `tar` package's `list()` —
+ * `list()` reads the entire archive stream to completion even after
+ * `onReadEntry` has found what it needs (no internal early-abort), so on a
+ * 15 GB+ archive it was a second full read on top of `untar()`'s own read
+ * (issue #294). The fast path here stops at the first top-level directory
+ * entry, typically reading well under 1 KB. Falls back to the original
+ * `list()`-based scan on anything it doesn't recognize (unknown typeflag
+ * sequence, truncated read, non-tar content) — safety over speed for
+ * archives this function has never been exercised against.
  * @param tarFilePath - Path to the `.nitpicker` tar file.
  * @returns The first top-level directory name found in the archive.
  * @throws {Error} When the tar contains no top-level directory entry
@@ -27,6 +43,151 @@ import { list } from 'tar';
  * await peekTarTopDir('renamed.nitpicker'); // → 'original'
  */
 export async function peekTarTopDir(tarFilePath) {
+    const fast = await peekTarTopDirFast(tarFilePath);
+    if (fast !== null) {
+        return fast;
+    }
+    return await peekTarTopDirViaList(tarFilePath);
+}
+/**
+ * Returns `true` for a top-level directory name that should be skipped: an
+ * AppleDouble resource-fork sidecar (`._foo`, from macOS BSD tar) or a PAX
+ * header directory (`PaxHeaders.NNN`/`@PaxHeader`). Real `.nitpicker`
+ * archives never name their actual top dir with these prefixes. Shared by
+ * both the fast path and the `list()` fallback so the two can't drift into
+ * accepting different names for the same archive.
+ * @param name - Candidate top-level directory name.
+ */
+function isSkippableTopDirName(name) {
+    return (name.startsWith('._') ||
+        name.startsWith('PaxHeaders') ||
+        name.startsWith('@PaxHeader'));
+}
+/**
+ * Extracts the first path segment from a tar entry path, after stripping a
+ * leading `./` some tar producers prepend.
+ * @param entryPath - The raw path recorded in (or resolved for) a tar entry.
+ */
+function topSegmentOf(entryPath) {
+    return entryPath.replace(/^\.\//, '').split('/')[0] ?? '';
+}
+/**
+ * Reads a NUL-terminated (or full-width, if no NUL) string field out of a
+ * tar header block.
+ * @param header - The 512-byte header block.
+ * @param start - Field start offset.
+ * @param length - Field length in bytes.
+ */
+function readHeaderString(header, start, length) {
+    const field = header.subarray(start, start + length);
+    const nulIndex = field.indexOf(0);
+    const raw = nulIndex === -1 ? field : field.subarray(0, nulIndex);
+    return raw.toString('utf8');
+}
+/**
+ * Fast path for {@link peekTarTopDir}: reads raw 512-byte tar header blocks
+ * directly from the file, stopping at the first top-level directory entry.
+ * Understands ustar fixed headers, PAX extended headers (`path=`, the form
+ * `tar`/node-tar emits for UTF-8 or >100-byte names — verified empirically:
+ * see this function's test fixtures), and GNU longname (`typeflag 'L'`,
+ * treated the same as a PAX path override for the next entry — this
+ * codebase's own `tar()` never emits it, but a `.nitpicker` produced by a
+ * different tar implementation should still resolve correctly). GNU
+ * longlink (`typeflag 'K'`, a symlink target — irrelevant to a directory
+ * name lookup) is skipped without being applied to anything.
+ *
+ * Returns `null` — never throws — on anything unrecognized: truncated read,
+ * unparseable size field, entry count past {@link MAX_ENTRIES}, or reaching
+ * the end-of-archive marker without finding a directory. The caller falls
+ * back to the exhaustive `list()`-based scan in every `null` case.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name, or `null` to fall back.
+ */
+async function peekTarTopDirFast(tarFilePath) {
+    let fileHandle;
+    try {
+        fileHandle = await open(tarFilePath, 'r');
+        const header = Buffer.alloc(BLOCK_SIZE);
+        let position = 0;
+        // Set by a preceding PAX ('x') or GNU longname ('L') entry; applies
+        // to the very next entry only, then is cleared.
+        let pendingPath = null;
+        for (let entryIndex = 0; entryIndex < MAX_ENTRIES; entryIndex++) {
+            const { bytesRead } = await fileHandle.read(header, 0, BLOCK_SIZE, position);
+            if (bytesRead < BLOCK_SIZE) {
+                return null;
+            }
+            position += BLOCK_SIZE;
+            if (header.every((byte) => byte === 0)) {
+                // End-of-archive marker reached with no directory found —
+                // genuinely absent, not a parse failure, but the caller's
+                // error message is clearer coming from the exhaustive path.
+                return null;
+            }
+            const typeflag = String.fromCodePoint(header[156] ?? 0);
+            const size = parseTarSizeField(header.subarray(124, 136));
+            if (size === null) {
+                return null;
+            }
+            const dataBlockCount = Math.ceil(size / BLOCK_SIZE);
+            const dataStart = position;
+            position += dataBlockCount * BLOCK_SIZE;
+            if (typeflag === 'x' || typeflag === 'L') {
+                if (size <= 0 || size > MAX_EXTENDED_PAYLOAD_SIZE) {
+                    return null;
+                }
+                const payload = Buffer.alloc(size);
+                const { bytesRead: payloadBytesRead } = await fileHandle.read(payload, 0, size, dataStart);
+                if (payloadBytesRead !== size) {
+                    return null;
+                }
+                pendingPath =
+                    typeflag === 'x'
+                        ? parsePaxPath(payload)
+                        : payload.toString('utf8').replace(/\0.*$/, '');
+                continue;
+            }
+            if (typeflag === 'g' || typeflag === 'K') {
+                // PAX global header (applies archive-wide, never carries a
+                // per-entry path) / GNU longlink (symlink target, irrelevant
+                // to a directory name lookup) — skip without consuming
+                // `pendingPath`.
+                continue;
+            }
+            const magic = header.toString('latin1', 257, 263);
+            const nameField = readHeaderString(header, 0, 100);
+            const prefixField = magic.startsWith('ustar')
+                ? readHeaderString(header, 345, 155)
+                : '';
+            const entryPath = pendingPath ?? (prefixField ? `${prefixField}/${nameField}` : nameField);
+            pendingPath = null;
+            if (typeflag === '5') {
+                const top = topSegmentOf(entryPath);
+                if (top && !isSkippableTopDirName(top)) {
+                    return top;
+                }
+            }
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        await fileHandle?.close();
+    }
+}
+/**
+ * Exhaustive fallback for {@link peekTarTopDir}: scans the tar's entry list
+ * via the `tar` package. Reads the archive to completion regardless of when
+ * a match is found (no early-abort in the `tar` package's `list()` API),
+ * which is exactly the cost {@link peekTarTopDirFast} exists to avoid — this
+ * path only runs when the fast path can't make sense of the header stream.
+ * @param tarFilePath - Path to the `.nitpicker` tar file.
+ * @returns The first top-level directory name found in the archive.
+ * @throws {Error} When the tar contains no top-level directory entry.
+ */
+async function peekTarTopDirViaList(tarFilePath) {
     let found = null;
     await list({
         file: tarFilePath,
@@ -41,19 +202,8 @@ export async function peekTarTopDir(tarFilePath) {
             // of the real top-level dir would win instead of the dir itself.
             if (entry.type !== 'Directory')
                 return;
-            // `entry.path` is the path inside the tar. Strip any leading
-            // `./` (some tar producers prepend it) before splitting.
-            const cleaned = entry.path.replace(/^\.\//, '');
-            const top = cleaned.split('/')[0];
-            if (!top || top === '')
-                return;
-            // Defense in depth: skip AppleDouble directory entries
-            // (`._foo/`) and pax extended header entries
-            // (`PaxHeaders.NNN/...` / `@PaxHeader`). Real `.nitpicker`
-            // archives never name their top dir with these prefixes.
-            if (top.startsWith('._'))
-                return;
-            if (top.startsWith('PaxHeaders') || top.startsWith('@PaxHeader'))
+            const top = topSegmentOf(entry.path);
+            if (!top || isSkippableTopDirName(top))
                 return;
             found = top;
         },

package/lib/archive/filesystem/tar.d.ts

@@ -4,8 +4,20 @@
  * The archive preserves the relative directory structure.
  * The `dir` parameter is resolved relative to its parent directory
  * so only the target directory name appears in the archive.
+ *
+ * When `onProgress` is given, the tar is produced through a manual stream
+ * pipe instead of tar's own `file` mode so the written byte count can be
+ * observed — a `.nitpicker` tar is dominated by the single giant `db.sqlite`
+ * entry (15 GB+ on large crawls), so per-entry callbacks would fire once and
+ * report nothing for minutes (issue #294). `totalBytes` is an estimate (the
+ * sum of the directory's file sizes; tar adds per-entry headers and padding
+ * on top), so `writtenBytes` is clamped to it and a final
+ * `(totalBytes, totalBytes)` call marks completion.
  * @param dir - The absolute path of the directory to archive.
  * @param outputPath - The file path where the TAR archive will be written.
+ * @param onProgress - Called as archive bytes are written, with the bytes
+ *   written so far (clamped to the estimate) and the estimated total. Omit
+ *   for the original silent, tar-managed-file creation.
  * @returns A promise that resolves when the TAR archive has been created.
  */
-export declare function tar(dir: string, outputPath: string): Promise<void>;
+export declare function tar(dir: string, outputPath: string, onProgress?: (writtenBytes: number, totalBytes: number) => void): Promise<void>;

package/lib/archive/filesystem/tar.js

@@ -1,3 +1,5 @@
+import { createWriteStream } from 'node:fs';
+import { readdir, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { create } from 'tar';
 /**
@@ -6,17 +8,71 @@ import { create } from 'tar';
  * The archive preserves the relative directory structure.
  * The `dir` parameter is resolved relative to its parent directory
  * so only the target directory name appears in the archive.
+ *
+ * When `onProgress` is given, the tar is produced through a manual stream
+ * pipe instead of tar's own `file` mode so the written byte count can be
+ * observed — a `.nitpicker` tar is dominated by the single giant `db.sqlite`
+ * entry (15 GB+ on large crawls), so per-entry callbacks would fire once and
+ * report nothing for minutes (issue #294). `totalBytes` is an estimate (the
+ * sum of the directory's file sizes; tar adds per-entry headers and padding
+ * on top), so `writtenBytes` is clamped to it and a final
+ * `(totalBytes, totalBytes)` call marks completion.
  * @param dir - The absolute path of the directory to archive.
  * @param outputPath - The file path where the TAR archive will be written.
+ * @param onProgress - Called as archive bytes are written, with the bytes
+ *   written so far (clamped to the estimate) and the estimated total. Omit
+ *   for the original silent, tar-managed-file creation.
  * @returns A promise that resolves when the TAR archive has been created.
  */
-export function tar(dir, outputPath) {
+export async function tar(dir, outputPath, onProgress) {
     const baseDir = path.dirname(dir);
     const targetDir = path.relative(baseDir, dir);
-    return create({
-        gzip: false,
-        cwd: baseDir,
-        file: outputPath,
-        preservePaths: false,
-    }, [targetDir]);
+    if (!onProgress) {
+        await create({
+            gzip: false,
+            cwd: baseDir,
+            file: outputPath,
+            preservePaths: false,
+        }, [targetDir]);
+        return;
+    }
+    const entries = await readdir(dir, { recursive: true, withFileTypes: true });
+    let totalBytes = 0;
+    for (const entry of entries) {
+        if (!entry.isFile()) {
+            continue;
+        }
+        const { size } = await stat(path.join(entry.parentPath, entry.name));
+        totalBytes += size;
+    }
+    await new Promise((resolve, reject) => {
+        const source = create({
+            gzip: false,
+            cwd: baseDir,
+            preservePaths: false,
+        }, [targetDir]);
+        const sink = createWriteStream(outputPath);
+        let writtenBytes = 0;
+        source.on('data', (chunk) => {
+            writtenBytes += chunk.length;
+            onProgress(Math.min(writtenBytes, totalBytes), totalBytes);
+        });
+        // .pipe() does not destroy the other side on error, so an error on
+        // either stream would otherwise leave the write side dangling open.
+        source.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            sink.destroy(err);
+            reject(err);
+        });
+        sink.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            source.destroy(err);
+            reject(err);
+        });
+        sink.on('close', () => {
+            onProgress(totalBytes, totalBytes);
+            resolve();
+        });
+        source.pipe(sink);
+    });
 }

package/lib/archive/filesystem/untar.d.ts

@@ -4,12 +4,23 @@
  * Only files newer than existing files in the target directory are extracted
  * (uses the `newer` option). Optionally restricts extraction to a specific
  * working directory and/or a subset of files.
+ *
+ * When `onProgress` is given, the tar is read through a manual
+ * `createReadStream` pipe instead of tar's own `file` mode so the consumed
+ * byte count can be observed — a `.nitpicker` tar is dominated by the single
+ * giant `db.sqlite` entry (15 GB+ on large crawls), so per-entry callbacks
+ * would fire once and report nothing for minutes (issue #294). Byte
+ * granularity is the read-stream chunk size (~64 KB); callers wanting
+ * coarser updates (e.g. one per percent) throttle in their own callback.
  * @param tarFilePath - The path to the TAR archive to extract.
  * @param options - Optional extraction settings.
  * @param options.cwd - The working directory to extract files into.
  *   If omitted, the current working directory is used.
  * @param options.fileList - An array of specific file paths within the archive
  *   to extract. If omitted, all files in the archive are extracted.
+ * @param options.onProgress - Called as archive bytes are consumed, with the
+ *   bytes read so far and the archive's total size. Omit for the original
+ *   silent, tar-managed-file extraction.
  * @returns A promise that resolves when extraction is complete.
  */
 export declare function untar(tarFilePath: string, options?: {
@@ -17,4 +28,6 @@ export declare function untar(tarFilePath: string, options?: {
     cwd?: string;
     /** An array of specific file paths within the archive to extract. */
     fileList?: string[];
+    /** Byte-level progress callback — see the function docs. */
+    onProgress?: (readBytes: number, totalBytes: number) => void;
 }): Promise<void>;

package/lib/archive/filesystem/untar.js

@@ -1,3 +1,5 @@
+import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
 import { extract } from 'tar';
 /**
  * Extracts files from a TAR archive.
@@ -5,20 +7,78 @@ import { extract } from 'tar';
  * Only files newer than existing files in the target directory are extracted
  * (uses the `newer` option). Optionally restricts extraction to a specific
  * working directory and/or a subset of files.
+ *
+ * When `onProgress` is given, the tar is read through a manual
+ * `createReadStream` pipe instead of tar's own `file` mode so the consumed
+ * byte count can be observed — a `.nitpicker` tar is dominated by the single
+ * giant `db.sqlite` entry (15 GB+ on large crawls), so per-entry callbacks
+ * would fire once and report nothing for minutes (issue #294). Byte
+ * granularity is the read-stream chunk size (~64 KB); callers wanting
+ * coarser updates (e.g. one per percent) throttle in their own callback.
  * @param tarFilePath - The path to the TAR archive to extract.
  * @param options - Optional extraction settings.
  * @param options.cwd - The working directory to extract files into.
  *   If omitted, the current working directory is used.
  * @param options.fileList - An array of specific file paths within the archive
  *   to extract. If omitted, all files in the archive are extracted.
+ * @param options.onProgress - Called as archive bytes are consumed, with the
+ *   bytes read so far and the archive's total size. Omit for the original
+ *   silent, tar-managed-file extraction.
  * @returns A promise that resolves when extraction is complete.
  */
-export function untar(tarFilePath, options) {
-    return extract({
-        file: tarFilePath,
-        newer: true,
-        cwd: options?.cwd,
-        preservePaths: false,
-        noMtime: true,
-    }, options?.fileList ?? []);
+export async function untar(tarFilePath, options) {
+    const { cwd, fileList, onProgress } = options ?? {};
+    if (!onProgress) {
+        await extract({
+            file: tarFilePath,
+            newer: true,
+            cwd,
+            preservePaths: false,
+            noMtime: true,
+        }, fileList ?? []);
+        return;
+    }
+    const { size: totalBytes } = await stat(tarFilePath);
+    await new Promise((resolve, reject) => {
+        const source = createReadStream(tarFilePath);
+        let readBytes = 0;
+        source.on('data', (chunk) => {
+            readBytes += chunk.length;
+            onProgress(readBytes, totalBytes);
+        });
+        const sink = extract({
+            newer: true,
+            cwd,
+            preservePaths: false,
+            noMtime: true,
+        }, fileList ?? []);
+        // .pipe() does not destroy the other side on error, so an error on
+        // either stream would otherwise leave the write side dangling open.
+        // `Unpack` (`extends Parser extends EE`, a plain `EventEmitter`, NOT
+        // a Minipass/stream base class despite behaving like a writable
+        // stream) has no `destroy()` — `abort()` is its own equivalent:
+        // idempotent (guarded internally), and it emits its own 'error'
+        // (confirmed via `tar`'s `warnMethod` with `recoverable: false`),
+        // which re-enters the `sink.on('error', ...)` handler below —
+        // harmless, since `source.destroy()`/`reject()` are already
+        // idempotent no-ops on a second call.
+        source.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            sink.abort(err);
+            reject(err);
+        });
+        sink.on('error', (error) => {
+            const err = error instanceof Error ? error : new Error(String(error));
+            source.destroy(err);
+            reject(err);
+        });
+        // node-tar's Unpack stream signals completion via 'close' (all entry
+        // writes flushed), not 'finish' (input fully consumed) — resolving on
+        // 'finish' could hand the caller a tmpDir whose db.sqlite is still
+        // being written.
+        sink.on('close', () => {
+            resolve();
+        });
+        source.pipe(sink);
+    });
 }

package/lib/archive/init-schema.d.ts

@@ -30,7 +30,7 @@ export declare function applyConnectionPragmas(instance: Knex): Promise<void>;
  *   writes during a crawl and every reader queries. Must run AFTER
  *   `createRefTables` because most entity tables reference ref-table PKs.
  * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
- *   `crawl_errors`, `page_tags`, `page_jsonld`, `inventory_runs`,
+ *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `inventory_runs`,
  *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
  *   `page_html_ref`. Must run AFTER `createEntityTables` because the
  *   page-scoped tables FK into `content_items(id)`.

package/lib/archive/init-schema.js

@@ -14,7 +14,8 @@ import { createRefTables } from './create-ref-tables.js';
 export async function applyConnectionPragmas(instance) {
     // Foreign-key enforcement defaults to OFF on every new SQLite
     // connection. Required for ON DELETE CASCADE on `page_html_ref`,
-    // `page_tags`, `page_jsonld`, and the `page_main_content_*` tables to fire.
+    // `technology_signals`, `page_technologies`, `page_jsonld`, and the
+    // `page_main_content_*` tables to fire.
     await instance.raw('PRAGMA foreign_keys = ON');
     await instance.raw('PRAGMA wal_autocheckpoint = 1000');
     // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
@@ -54,7 +55,7 @@ export async function applyConnectionPragmas(instance) {
  *   writes during a crawl and every reader queries. Must run AFTER
  *   `createRefTables` because most entity tables reference ref-table PKs.
  * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
- *   `crawl_errors`, `page_tags`, `page_jsonld`, `inventory_runs`,
+ *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `inventory_runs`,
  *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
  *   `page_html_ref`. Must run AFTER `createEntityTables` because the
  *   page-scoped tables FK into `content_items(id)`.
@@ -131,8 +132,9 @@ export async function initSchema(instance) {
     // the ref tables (`url_refs`, `content_type_refs`, `text_refs`,
     // `json_refs`, `blob_refs`, `header_sets`) via FK clauses.
     await createEntityTables(instance);
-    // Adjunct tables that FK into `content_items` (page_errors / page_tags /
-    // page_jsonld / analysis_* / page_html_*) plus the standalone log tables
+    // Adjunct tables that FK into `content_items` (page_errors /
+    // technology_signals / page_technologies / page_jsonld / analysis_* /
+    // page_html_*) plus the standalone log tables
     // (crawl_errors / inventory_runs). MUST run after
     // {@link createEntityTables} so the FK targets exist. DDL +
     // column-level rationale lives in {@link createAdjunctTables}, which is

package/lib/archive/meta/compute-main-contents-denormalized.d.ts

@@ -15,8 +15,19 @@ import type { MainContentsData, ScrollHeightData } from '@d-zero/beholder';
  * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
  * uncaught at compile time) — `== null` tolerates both without forcing every
  * fixture to be updated.
+ *
+ * `customElementCount` is a separate parameter, not read off `mainContents`,
+ * because it is not one of beholder's `MainContentsData` categories —
+ * nitpicker captures Web Components itself (`crawler/capture-custom-elements.ts`)
+ * after `scrapeStart` returns. It carries three distinct states:
+ * `undefined`/capture-not-attempted and capture-failure both collapse to
+ * `null` (unknown — NOT the same as "captured, zero found"), while any
+ * number (including `0`) means capture succeeded.
  * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
  * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
- * @returns The seventeen denormalised columns.
+ * @param customElementCount - Count of Web Components nitpicker captured in
+ *   the main-content region, or `null`/`undefined` when capture was not
+ *   attempted or failed.
+ * @returns The eighteen denormalised columns.
  */
-export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined): MainContentsDenormalizedColumns;
+export declare function computeMainContentsDenormalized(mainContents: MainContentsData | null | undefined, scrollHeight: ScrollHeightData | null | undefined, customElementCount?: number | null): MainContentsDenormalizedColumns;

package/lib/archive/meta/compute-main-contents-denormalized.js

@@ -13,11 +13,22 @@
  * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
  * uncaught at compile time) — `== null` tolerates both without forcing every
  * fixture to be updated.
+ *
+ * `customElementCount` is a separate parameter, not read off `mainContents`,
+ * because it is not one of beholder's `MainContentsData` categories —
+ * nitpicker captures Web Components itself (`crawler/capture-custom-elements.ts`)
+ * after `scrapeStart` returns. It carries three distinct states:
+ * `undefined`/capture-not-attempted and capture-failure both collapse to
+ * `null` (unknown — NOT the same as "captured, zero found"), while any
+ * number (including `0`) means capture succeeded.
  * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
  * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
- * @returns The seventeen denormalised columns.
+ * @param customElementCount - Count of Web Components nitpicker captured in
+ *   the main-content region, or `null`/`undefined` when capture was not
+ *   attempted or failed.
+ * @returns The eighteen denormalised columns.
  */
-export function computeMainContentsDenormalized(mainContents, scrollHeight) {
+export function computeMainContentsDenormalized(mainContents, scrollHeight, customElementCount) {
     if (mainContents == null) {
         return {
             main_content_node_name: null,
@@ -35,6 +46,7 @@ export function computeMainContentsDenormalized(mainContents, scrollHeight) {
             main_content_video_count: null,
             main_content_audio_count: null,
             main_content_canvas_count: null,
+            main_content_custom_element_count: null,
             scroll_height_desktop: null,
             scroll_height_mobile: null,
         };
@@ -57,6 +69,7 @@ export function computeMainContentsDenormalized(mainContents, scrollHeight) {
         main_content_video_count: mainContents.videos.length,
         main_content_audio_count: mainContents.audios.length,
         main_content_canvas_count: mainContents.canvases.length,
+        main_content_custom_element_count: customElementCount ?? null,
         scroll_height_desktop: scrollHeight?.desktop ?? null,
         scroll_height_mobile: scrollHeight?.mobile ?? null,
     };

package/lib/archive/meta/technologies/combine-technology-confidence.d.ts

@@ -0,0 +1,46 @@
+import type { PageTechnologyPartial, TechnologySignalPartial } from './types.js';
+/**
+ * Options for {@link combineTechnologyConfidence}.
+ */
+export interface CombineTechnologyConfidenceOptions {
+    /**
+     * Skip the `TECHNOLOGY_CONFIDENCE_THRESHOLD` drop. Set by the `page_tags`
+     * legacy-migration callers (`migrate-page-tags-to-page-technologies.ts`,
+     * `retarget-legacy-fk-tables.ts`): the threshold exists to filter noisy
+     * NEW detections at crawl time, but migration is converting Wappalyzer
+     * detections the old `page_tags`-based API already returned
+     * unconditionally (including low-confidence ones) — dropping them here
+     * would silently and permanently lose data with no recovery path (v0.x
+     * policy is read-time migration only, no retroactive re-crawl).
+     */
+    skipThreshold?: boolean;
+}
+/**
+ * Combines every signal detected for one technology on one page into a
+ * single confidence score via noisy-OR:
+ *
+ * ```
+ * confidence = round(100 * (1 - Π (1 - weight/100)))
+ * ```
+ *
+ * over each DISTINCT `signalType` present (same signal type appearing
+ * twice — e.g. two different `url-pattern` definitions both matching —
+ * counts once, at its highest weight; compounding only happens ACROSS
+ * signal types, which is what makes this "multiple independent signals
+ * agree" rather than "one noisy signal repeated"). A single signal's
+ * result equals its own weight exactly (`1 - (1 - w/100) = w/100`).
+ *
+ * Technologies whose combined confidence falls below
+ * `TECHNOLOGY_CONFIDENCE_THRESHOLD` are dropped entirely — a lone weak
+ * signal (e.g. `id="app"`, weight 15) is noise, not a detection, and
+ * `page_technologies` has no `isDetected` column to hide it behind; the
+ * row simply is not written. Pass `options.skipThreshold` to keep every
+ * technology regardless of confidence (see its docs for why).
+ * @param signals - Every signal detected for a single page, across all
+ *   technologies (this function groups by `technology` internally).
+ * @param options - See {@link CombineTechnologyConfidenceOptions}.
+ * @returns One {@link PageTechnologyPartial} per technology (filtered by
+ *   the confidence threshold unless `options.skipThreshold` is set), in no
+ *   particular order.
+ */
+export declare function combineTechnologyConfidence(signals: readonly TechnologySignalPartial[], options?: CombineTechnologyConfidenceOptions): PageTechnologyPartial[];