@nitpicker/crawler 0.18.0 → 0.19.0

package/lib/archive/archive.js

@@ -31,16 +31,15 @@ import { safePath } from './safe-path.js';
  *
  * Use the static factory methods ({@link Archive.create}, {@link Archive.open},
  * {@link Archive.resume}, {@link Archive.connect}) to obtain instances.
- * The constructor is private.
+ * The constructor is private. Implements `Symbol.asyncDispose` (inherited
+ * from {@link ArchiveAccessor}) so callers can use `await using` instead of
+ * a manual `try`/`finally` around {@link close}.
  * @example
- * const archive = await Archive.create({ filePath: '/path/to/site.nitpicker' });
- * try {
- *   await archive.setConfig(config);
- *   const pageId = await archive.setPage(pageData);
- * } finally {
- *   // Writes the `.nitpicker` tar (if absent), removes tmpDir, releases the lock.
- *   await archive.close();
- * }
+ * await using archive = await Archive.create({ filePath: '/path/to/site.nitpicker' });
+ * await archive.setConfig(config);
+ * const pageId = await archive.setPage(pageData);
+ * // Writes the `.nitpicker` tar (if absent), removes tmpDir, releases the
+ * // lock — all on scope exit, whether by fallthrough or thrown error.
  */
 export default class Archive extends ArchiveAccessor {
     /**
@@ -414,22 +413,26 @@ export default class Archive extends ArchiveAccessor {
      * internal pages on the next pass.
      * @param scopes - Hostname-indexed scope map representing the new scope.
      * @param options - URL parsing options forwarded to the scope-entry lookup.
+     * @param onProgress - Forwarded to {@link Database.repromoteExternalPages}
+     *   — see that method's docs.
      * @returns The URLs that were repromoted.
      */
-    async repromoteExternalPages(scopes, options) {
+    async repromoteExternalPages(scopes, options, onProgress) {
         dbLog('Repromote external pages with %d hostnames in scope', scopes.size);
-        return this.#db.repromoteExternalPages(scopes, options);
+        return this.#db.repromoteExternalPages(scopes, options, onProgress);
     }
     /**
      * Reset previously-failed pages back to pending so a follow-up crawl re-fetches them.
      *
      * Delegates to {@link Database.resetFailedPages}. See that method for the
      * exact failure criteria (missing status / content type, or a 5xx status).
+     * @param onProgress - Forwarded to {@link Database.resetFailedPages} —
+     *   see that method's docs.
      * @returns The URLs of the pages that were reset to pending.
      */
-    async resetFailedPages() {
+    async resetFailedPages(onProgress) {
         dbLog('Reset failed pages back to pending');
-        return this.#db.resetFailedPages();
+        return this.#db.resetFailedPages(onProgress);
     }
     /**
      * Persists the raw bytes of an `--inventory` source URL list into the
@@ -498,11 +501,14 @@ export default class Archive extends ArchiveAccessor {
      * pages share a single `page_html_blobs` row.
      * @param pageInfo - The page data to store.
      * @param source - Provenance label for new rows. `undefined` leaves the DB DEFAULT (`'crawled'`).
+     * @param bodyHash - Precomputed body hash for the page's HTML (see
+     *   `CrawlerEventTypes.page.bodyHash`). `undefined`/`null` falls back to
+     *   computing it from the HTML instead.
      * @returns The database ID of the stored page.
      */
-    async setPage(pageInfo, source) {
+    async setPage(pageInfo, source, bodyHash) {
         dbLog('Set page: %s', pageInfo.url.href);
-        return await this.#db.updatePage(pageInfo, true, pageInfo.isTarget, source);
+        return await this.#db.updatePage(pageInfo, true, pageInfo.isTarget, source, bodyHash);
     }
     /**
      * Records a redirect edge without re-storing the destination's content.
@@ -554,10 +560,12 @@ export default class Archive extends ArchiveAccessor {
     /**
      * Assigns natural URL sort order values to all pages in the database
      * that do not yet have an `order` field set.
+     * @param onProgress - Forwarded to {@link Database.setUrlOrder} — see that
+     *   method's docs.
      */
-    async setUrlOrder() {
+    async setUrlOrder(onProgress) {
         dbLog("Pages didn't have `order` field. So set URL order.");
-        await this.#db.setUrlOrder();
+        await this.#db.setUrlOrder(onProgress);
     }
     /**
      * Updates a subset of fields on the archive's `info` row. Used by the append
@@ -584,16 +592,35 @@ export default class Archive extends ArchiveAccessor {
      * `db.sqlite`, so a re-crawl (`append` / `inventory` / `retryFailed`)
      * opened without it would tar back a tmpDir missing those extra files,
      * silently dropping them from the rewritten archive.
-     */
-    async write() {
+     * @param options - Optional write settings.
+     * @param options.onTarProgress - Called as archive bytes are written
+     *   during the tar step, with the bytes written so far and the estimated
+     *   total (issue #294: tarring a large archive takes minutes, and
+     *   without this the CLI shows nothing until `write` returns). Omit for
+     *   a silent write (the default).
+     * @param options.onStep - Called once at the start of each of this
+     *   method's four steps (issue #294): `checkpoint` (WAL fold-back —
+     *   single synchronous PRAGMA, no countable progress) and `remove`
+     *   (deleting the tarred-away tmpDir) have no progress signal of their
+     *   own, so without this a large archive's write looks frozen between
+     *   the `tar` step's byte updates and completion. `rename` is nearly
+     *   instant (same-filesystem directory move) but included for
+     *   completeness — a caller displaying phase labels shouldn't have a
+     *   gap where the operation is silently between named steps.
+     */
+    async write(options) {
         saveLog('Starts: %s', this.#filePath);
+        options?.onStep?.('checkpoint');
         await this.#db.checkpoint();
         const filePathWithoutExt = path.resolve(path.dirname(this.#filePath), path.basename(this.#filePath, path.extname(this.#filePath)));
         saveLog('Rename temporary dir: %s to %s', this.#tmpDir, filePathWithoutExt);
+        options?.onStep?.('rename');
         await rename(this.#tmpDir, filePathWithoutExt, true);
         saveLog('Tar temporary dir to file: %s to %s', filePathWithoutExt, this.#filePath);
-        await tar(filePathWithoutExt, this.#filePath);
+        options?.onStep?.('tar');
+        await tar(filePathWithoutExt, this.#filePath, options?.onTarProgress);
         saveLog('Remove temporary dir: %s', filePathWithoutExt);
+        options?.onStep?.('remove');
         await remove(filePathWithoutExt);
         saveLog('Done: %s', this.#filePath);
     }
@@ -602,19 +629,25 @@ export default class Archive extends ArchiveAccessor {
      * (write or remove), drops the DB handle via the base class, then
      * releases the lock in a `finally` so the lock never leaks even on
      * partial failure.
+     * @param options - See {@link close}.
+     * @param options.timeoutMs
+     * @param options.onRecoveryStart
+     * @param options.onTarProgress
+     * @param options.onStep
      */
-    async #runFullClose() {
+    async #runFullClose(options) {
         log('Closing');
         try {
             if (!exists(this.#filePath)) {
                 log("Save the file because it doesn't exist");
-                await this.write();
+                options?.onRecoveryStart?.();
+                await this.write(options);
             }
             else if (exists(this.#tmpDir)) {
                 log('Remove temporary dir');
                 await remove(this.#tmpDir);
             }
-            await super.close();
+            await super.close({ timeoutMs: options?.timeoutMs });
         }
         finally {
             await this.#releaseLock();
@@ -654,27 +687,38 @@ export default class Archive extends ArchiveAccessor {
      * on the user's tmpDir.
      *
      * `options.readOnly: false` is a narrow escape hatch for opening a
-     * second, writable connection to a `tmpDir` that {@link Archive.openCached}
-     * already extracted (and migrated) into an OS-temp cache directory —
-     * never the caller's live/interrupted crawl tmpDir, which must stay
-     * read-only. A read-only open (`Archive.openCached`/`ArchiveManager.open`)
-     * must never take this path itself — blocking or writing during what
-     * must be a read-only open is forbidden (issue #177). This escape
-     * hatch has no current production caller; any future
-     * one is responsible for its own cross-process coordination (see
-     * `acquireArchiveLock`) — this method does not acquire any lock itself.
+     * second, writable connection to a `tmpDir` the caller's own process
+     * already owns and extracted itself. The one production caller is the
+     * viewer-read-model worker thread (`@nitpicker/query`'s
+     * `viewer-read-model-worker-entry.ts`, issue #294): the parent thread
+     * holds the archive via `Archive.open` (lock included — worker threads
+     * share the parent's PID, so the PID-based `acquireArchiveLock` guard
+     * stays valid), sits idle awaiting the worker, and re-tars the tmpDir
+     * afterward. What this hatch must NEVER target is a live/interrupted
+     * crawl tmpDir owned by a *different* process (the stub-mode
+     * `ArchiveManager.open` path attaches to exactly such directories, and
+     * must stay read-only): writable connects run the self-healing
+     * migrations, and mutating a directory out from under its owner is how
+     * archives corrupt. A read-only open (`Archive.openCached`/
+     * `ArchiveManager.open`) must never take this path itself — blocking or
+     * writing during what must be a read-only open is forbidden (issue
+     * #177). Any new caller is responsible for its own coordination with
+     * the tmpDir's owner (see `acquireArchiveLock` for the cross-process
+     * case) — this method does not acquire any lock itself.
      * @param tmpDir - The path to the temporary directory containing the database.
      * @param namespace - An optional namespace for scoping data access within the archive.
      * @param options - Connection options.
      * @param options.readOnly - Defaults to `true`. Pass `false` to obtain a
-     *   writable accessor against an already-extracted cache directory.
+     *   writable accessor against a tmpDir the calling process itself owns.
      * @returns An ArchiveAccessor instance for querying the archive data.
      * @example
      * // Default (read-only) — safe for stub mode and cache reads:
      * const accessor = await Archive.connect(tmpDir);
      * @example
-     * // Writable escape hatch — only against a tar-cache extraction:
-     * const writable = await Archive.connect(cacheDir, null, { readOnly: false });
+     * // Writable escape hatch — only against a tmpDir this process owns
+     * // (e.g. the viewer-read-model worker thread reconnecting to the
+     * // parent's Archive.open extraction):
+     * const writable = await Archive.connect(ownTmpDir, null, { readOnly: false });
      */
     static async connect(tmpDir, namespace = null, options = {}) {
         const readOnly = options.readOnly ?? true;
@@ -715,23 +759,22 @@ export default class Archive extends ArchiveAccessor {
      * {@link Archive.open}.
      * @param filePath - Absolute path to the `.nitpicker` file.
      * @param namespace - Optional namespace forwarded to {@link ArchiveAccessor}.
+     * @param onExtractProgress - Forwarded to {@link extractArchiveToCache} —
+     *   see that function's docs for the cache-hit/miss contract.
      * @returns A read-only {@link ArchiveAccessor} backed by the cache directory.
      * @example
      * ```ts
-     * const accessor = await Archive.openCached('/path/to/site.nitpicker');
-     * try {
-     *   const summary = await getSummary(accessor);
-     * } finally {
-     *   await accessor.close(); // tears down DB handle, cacheDir persists.
-     * }
+     * await using accessor = await Archive.openCached('/path/to/site.nitpicker');
+     * const summary = await getSummary(accessor);
+     * // tears down DB handle on scope exit; cacheDir persists.
      * ```
      */
-    static async openCached(filePath, namespace = null) {
+    static async openCached(filePath, namespace = null, onExtractProgress) {
         const cacheRoot = getArchiveCacheRoot();
         const cacheKey = await computeArchiveCacheKey(filePath);
         const cacheDir = resolveArchiveCacheDir(cacheRoot, cacheKey, filePath);
         log('Open cached: %s (cacheDir=%s)', filePath, cacheDir);
-        await extractArchiveToCache(filePath, cacheRoot, cacheDir, cacheKey);
+        await extractArchiveToCache(filePath, cacheRoot, cacheDir, cacheKey, onExtractProgress);
         return await Archive.connect(cacheDir, namespace);
     }
     /**
@@ -773,7 +816,7 @@ export default class Archive extends ArchiveAccessor {
      * @returns An Archive instance with the extracted data loaded.
      */
     static async open(options) {
-        const { filePath, openPluginData } = options;
+        const { filePath, openPluginData, onExtractProgress, onLog } = options;
         const cwd = options.cwd ?? process.cwd();
         log('Open: %O', {
             filePath,
@@ -802,11 +845,12 @@ export default class Archive extends ArchiveAccessor {
             await untar(filePath, {
                 cwd,
                 fileList: openFiles.length > 0 ? openFiles : undefined,
+                onProgress: onExtractProgress,
             });
             const extractedDir = path.resolve(cwd, innerDirName);
             log('Move directory: %s to %s', extractedDir, tmpDir);
             await rename(extractedDir, tmpDir, true);
-            return await Archive.#init(filePath, tmpDir, releaseLock);
+            return await Archive.#init(filePath, tmpDir, releaseLock, onLog);
         }
         catch (error) {
             await releaseLock();
@@ -817,16 +861,19 @@ export default class Archive extends ArchiveAccessor {
      * Resumes an archive from an existing temporary directory
      * (e.g., after an interrupted crawl session).
      * @param targetPath - The path to the temporary directory to resume from.
+     * @param onLog - Forwarded to {@link Database.connect} — see
+     *   {@link ArchiveOpenOptions.onLog}'s docs (this writable reconnect
+     *   runs the same self-healing migrations `Archive.open` does).
      * @returns An Archive instance reconnected to the existing data.
      * @throws {Error} If the specified path is not a directory.
      */
-    static async resume(targetPath) {
+    static async resume(targetPath, onLog) {
         log('Resume: %s', targetPath);
         if (await isDir(targetPath)) {
             const tmpDir = targetPath;
             const releaseLock = await acquireArchiveLock(tmpDir);
             try {
-                const db = await Archive.#connectDB(tmpDir);
+                const db = await Archive.#connectDB(tmpDir, { onLog });
                 const name = (await db.getName()) ||
                     path.basename(targetPath).replace(Archive.TMP_DIR_PREFIX, '');
                 const filePath = path.resolve(process.cwd(), name + '.' + Archive.FILE_EXTENSION);
@@ -865,6 +912,7 @@ export default class Archive extends ArchiveAccessor {
      *   `readOnly: true` so no migrations run and a missing tmpDir is not
      *   resurrected.
      * @param options.readOnly
+     * @param options.onLog
      */
     static async #connectDB(tmpDir, options) {
         const dbPath = path.resolve(tmpDir, Archive.SQLITE_DB_FILE_NAME);
@@ -872,6 +920,7 @@ export default class Archive extends ArchiveAccessor {
         return await Database.connect({
             filename: dbPath,
             readOnly: options?.readOnly,
+            onLog: options?.onLog,
         });
     }
     /**
@@ -883,9 +932,11 @@ export default class Archive extends ArchiveAccessor {
      * @param filePath - Output `.nitpicker` file path
      * @param tmpDir - Temporary working directory path
      * @param releaseLock - Function returned by {@link acquireArchiveLock}.
+     * @param onLog - Forwarded to {@link Database.connect} — see
+     *   {@link ArchiveOpenOptions.onLog}'s docs.
      */
-    static async #init(filePath, tmpDir, releaseLock) {
-        const db = await Archive.#connectDB(tmpDir);
+    static async #init(filePath, tmpDir, releaseLock, onLog) {
+        const db = await Archive.#connectDB(tmpDir, { onLog });
         const archive = new Archive(filePath, tmpDir, db, releaseLock);
         return archive;
     }
@@ -909,12 +960,27 @@ export default class Archive extends ArchiveAccessor {
      * {@link ArchiveAccessor} (not an `Archive`), so `close()` resolves to
      * the safe base implementation — no `write()`, no `remove()`, no lock
      * release — leaving the tmpDir intact for the live crawler.
-     */
-    async close() {
+     * @param options - Optional close settings. `timeoutMs` is accepted for
+     *   compatibility with {@link ArchiveAccessor.close}'s signature (forwarded
+     *   to the base `super.close()` call below); the rest are progress
+     *   callbacks (issue #294) forwarded to {@link write} when this call ends
+     *   up taking the recovery-write branch (the archive file doesn't exist
+     *   yet).
+     * @param options.timeoutMs - See {@link ArchiveAccessor.close}.
+     * @param options.onRecoveryStart - Called once, only when this `close()`
+     *   is about to write the archive because the file doesn't exist on
+     *   disk yet — e.g. a caller's own explicit `write()` threw before
+     *   finishing, or was never called at all. Without this, a listener
+     *   that already tore down its display after that earlier failure has
+     *   no way to know a second, recovery write is happening.
+     * @param options.onTarProgress - See {@link write}.
+     * @param options.onStep - See {@link write}.
+     */
+    async close(options) {
         if (this.#closeOnce) {
             return this.#closeOnce;
         }
-        this.#closeOnce = this.#runFullClose();
+        this.#closeOnce = this.#runFullClose(options);
         return this.#closeOnce;
     }
     /**

package/lib/archive/cache/extract-archive-to-cache.d.ts

@@ -38,6 +38,15 @@
  * @param cacheDir - Absolute path the extracted contents should end up at.
  * @param cacheKey - The cache key used to derive `cacheDir`. Recomputed
  *   after extraction to detect concurrent writers; must match.
+ * @param onExtractProgress - Called during the untar step with bytes read
+ *   so far and the archive's total size (issue #294: a cold cache on a
+ *   large archive can take tens of seconds with no other signal it isn't
+ *   hung). Never called on a cache hit — {@link isCacheDirReady} short-
+ *   circuits before `untar` runs — so callers can treat "never invoked" as
+ *   the definition of a hit and skip printing a phase label until the
+ *   first call actually arrives. Only the first concurrent caller for a
+ *   given `cacheDir` sees callbacks; same-`cacheDir` callers deduped
+ *   through {@link inFlightByCacheDir} just await the shared promise.
  * @returns Resolves once `cacheDir` is ready to be opened read-only.
  */
-export declare function extractArchiveToCache(archivePath: string, cacheRoot: string, cacheDir: string, cacheKey: string): Promise<void>;
+export declare function extractArchiveToCache(archivePath: string, cacheRoot: string, cacheDir: string, cacheKey: string, onExtractProgress?: (readBytes: number, totalBytes: number) => void): Promise<void>;

package/lib/archive/cache/extract-archive-to-cache.js

@@ -86,9 +86,18 @@ const inFlightByCacheDir = new Map();
  * @param cacheDir - Absolute path the extracted contents should end up at.
  * @param cacheKey - The cache key used to derive `cacheDir`. Recomputed
  *   after extraction to detect concurrent writers; must match.
+ * @param onExtractProgress - Called during the untar step with bytes read
+ *   so far and the archive's total size (issue #294: a cold cache on a
+ *   large archive can take tens of seconds with no other signal it isn't
+ *   hung). Never called on a cache hit — {@link isCacheDirReady} short-
+ *   circuits before `untar` runs — so callers can treat "never invoked" as
+ *   the definition of a hit and skip printing a phase label until the
+ *   first call actually arrives. Only the first concurrent caller for a
+ *   given `cacheDir` sees callbacks; same-`cacheDir` callers deduped
+ *   through {@link inFlightByCacheDir} just await the shared promise.
  * @returns Resolves once `cacheDir` is ready to be opened read-only.
  */
-export async function extractArchiveToCache(archivePath, cacheRoot, cacheDir, cacheKey) {
+export async function extractArchiveToCache(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress) {
     if (await isCacheDirReady(cacheDir)) {
         return;
     }
@@ -96,7 +105,7 @@ export async function extractArchiveToCache(archivePath, cacheRoot, cacheDir, ca
     if (existing) {
         return existing;
     }
-    const promise = runExtraction(archivePath, cacheRoot, cacheDir, cacheKey).finally(() => {
+    const promise = runExtraction(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress).finally(() => {
         inFlightByCacheDir.delete(cacheDir);
     });
     inFlightByCacheDir.set(cacheDir, promise);
@@ -110,8 +119,9 @@ export async function extractArchiveToCache(archivePath, cacheRoot, cacheDir, ca
  * @param cacheRoot - Absolute path to the cache root directory.
  * @param cacheDir - Absolute path the extracted contents should end up at.
  * @param cacheKey - Pre-extraction cache key, re-verified post-extraction.
+ * @param onExtractProgress - See {@link extractArchiveToCache}.
  */
-async function runExtraction(archivePath, cacheRoot, cacheDir, cacheKey) {
+async function runExtraction(archivePath, cacheRoot, cacheDir, cacheKey, onExtractProgress) {
     await fs.mkdir(cacheRoot, { recursive: true });
     const releaseLock = await acquireLockWithPeerWait(cacheDir);
     try {
@@ -131,7 +141,7 @@ async function runExtraction(archivePath, cacheRoot, cacheDir, cacheKey) {
         // it (with `newer:true` cherrypicking entries) so wipe first.
         await fs.rm(stagingDir, { recursive: true, force: true });
         await fs.mkdir(stagingDir, { recursive: true });
-        await untar(archivePath, { cwd: stagingDir });
+        await untar(archivePath, { cwd: stagingDir, onProgress: onExtractProgress });
         // Concurrent-writer detection: if the source archive changed
         // during our untar, the contents we just landed do NOT match the
         // `cacheKey` that named the directory. Drop the staging dir and

package/lib/archive/create-adjunct-tables.d.ts

@@ -7,12 +7,22 @@ import type { Knex } from 'knex';
  * - `page_errors` — partial scrape failures, FK → `content_items(id)`
  * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
  *   an external link that failed DNS, or null for a process-level error)
- * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
+ * - `technology_signals` — one un-combined technology-detection signal
+ *   (Wappalyzer or structural) per row, FK → `content_items(id)`
+ * - `page_technologies` — confidence-combined roll-up of
+ *   `technology_signals`, one row per (page, technology), FK →
+ *   `content_items(id)`. Supersedes the removed `page_tags` (Wappalyzer-only)
+ * - `technology_js_scan_cache` — resource-scoped cache for the post-crawl
+ *   JS license-comment scan, FK → `resource_items(id)`
  * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
  * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
  *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
  *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
  *   `content_items(id)`
+ * - `page_main_content_custom_elements` — Web Components (custom elements)
+ *   found in the main-content region, one row per DOM element, FK →
+ *   `content_items(id)`. Unlike its eight siblings above, captured by
+ *   nitpicker itself (`crawler/capture-custom-elements.ts`), not beholder
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
@@ -50,7 +60,7 @@ import type { Knex } from 'knex';
  * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
  * a single sentinel table), each table here is guarded individually because
  * the migration-script caller sees archives where any subset may already
- * exist (e.g. `page_tags` from the 0.10 migration but no `inventory_runs`).
+ * exist (e.g. `page_jsonld` from the 0.10 migration but no `inventory_runs`).
  * Index creation stays inside each guard: an existing table keeps whatever
  * indexes its creation path declared.
  * @param instance - The Knex query builder instance connected to the database.

package/lib/archive/create-adjunct-tables.js

@@ -6,12 +6,22 @@
  * - `page_errors` — partial scrape failures, FK → `content_items(id)`
  * - `crawl_errors` — crawler-level error channel (no FK; the URL may be
  *   an external link that failed DNS, or null for a process-level error)
- * - `page_tags` — Wappalyzer detections, FK → `content_items(id)`
+ * - `technology_signals` — one un-combined technology-detection signal
+ *   (Wappalyzer or structural) per row, FK → `content_items(id)`
+ * - `page_technologies` — confidence-combined roll-up of
+ *   `technology_signals`, one row per (page, technology), FK →
+ *   `content_items(id)`. Supersedes the removed `page_tags` (Wappalyzer-only)
+ * - `technology_js_scan_cache` — resource-scoped cache for the post-crawl
+ *   JS license-comment scan, FK → `resource_items(id)`
  * - `page_jsonld` — JSON-LD / SpeculationRules, FK → `content_items(id)`
  * - `page_main_content_headings` / `_images` / `_tables` / `_buttons` /
  *   `_iframes` / `_videos` / `_audios` / `_canvases` — beholder
  *   `MainContentsData` sub-entity arrays, one row per DOM element, FK →
  *   `content_items(id)`
+ * - `page_main_content_custom_elements` — Web Components (custom elements)
+ *   found in the main-content region, one row per DOM element, FK →
+ *   `content_items(id)`. Unlike its eight siblings above, captured by
+ *   nitpicker itself (`crawler/capture-custom-elements.ts`), not beholder
  * - `inventory_runs` — `--inventory` audit log (no FK; append-only)
  * - `network_outages` — operator-network-outage journal (no FK; append-only
  *   except `ended_at`, which is written once on recovery)
@@ -49,7 +59,7 @@
  * Unlike `createRefTables` / `createEntityTables` (whose callers guard with
  * a single sentinel table), each table here is guarded individually because
  * the migration-script caller sees archives where any subset may already
- * exist (e.g. `page_tags` from the 0.10 migration but no `inventory_runs`).
+ * exist (e.g. `page_jsonld` from the 0.10 migration but no `inventory_runs`).
  * Index creation stays inside each guard: an existing table keeps whatever
  * indexes its creation path declared.
  * @param instance - The Knex query builder instance connected to the database.
@@ -93,36 +103,69 @@ export async function createAdjunctTables(instance) {
             t.integer('createdAt').notNullable();
         });
     }
-    if (!(await instance.schema.hasTable('page_tags'))) {
-        await instance.schema.createTable('page_tags', (t) => {
-            // Wappalyzer-derived technology detection. One row per
-            // (provider × externalId) tuple per page. `category` is the first
-            // element of `categories`; the full list lives in the JSON
-            // `categories` column. `sources` records where the provider was
-            // detected (script-src / inline / iframe-src / window-global / …).
+    if (!(await instance.schema.hasTable('technology_signals'))) {
+        await instance.schema.createTable('technology_signals', (t) => {
+            // One un-combined signal for one technology on one page — the
+            // source-of-truth granular evidence `page_technologies` rolls up.
+            // Supersedes `page_tags` (Wappalyzer-only; removed): `signalType`
+            // covers Wappalyzer AND nitpicker's own structural detections
+            // (URL patterns, HTML markers, scoped attributes, meta generator,
+            // JS license comments) in one unified model. See
+            // `archive/meta/technologies/`.
             t.increments('id');
             t.integer('pageId')
                 .notNullable()
                 .unsigned()
                 .references('content_items.id')
                 .onDelete('CASCADE');
-            t.string('provider').notNullable();
+            t.string('technology').notNullable();
+            t.string('signalType').notNullable();
+            t.text('evidence');
+            t.integer('weight').notNullable();
+            t.index('pageId');
+        });
+        await instance.raw('CREATE INDEX technology_signals_tech_type ON technology_signals(technology, signalType)');
+    }
+    if (!(await instance.schema.hasTable('page_technologies'))) {
+        await instance.schema.createTable('page_technologies', (t) => {
+            // Confidence-combined roll-up of `technology_signals`, one row
+            // per (page, technology) — the read-optimised projection every
+            // query/viewer/MCP consumer reads instead of re-combining
+            // `technology_signals` on every request. Always written in the
+            // same transaction as its `technology_signals` rows; see
+            // ARCHITECTURE.md's invariant that the two tables are updated
+            // as a pair.
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.string('technology').notNullable();
             t.string('category');
-            t.string('externalId');
             t.string('version');
-            t.integer('confidence');
-            t.json('categories');
-            t.json('sources');
-            t.index('pageId');
-            t.index('provider');
-            t.index('externalId');
+            t.integer('confidence').notNullable();
+            t.integer('signalCount').notNullable();
+            t.unique(['pageId', 'technology']);
+            t.index('technology');
+        });
+    }
+    if (!(await instance.schema.hasTable('technology_js_scan_cache'))) {
+        await instance.schema.createTable('technology_js_scan_cache', (t) => {
+            // Resource-scoped cache preventing "JSスキャン・エンリッチメント"
+            // (post-crawl network enrichment, see ARCHITECTURE.md) from
+            // re-fetching the same JS resource on `--append`/`--retry-failed`.
+            // `resourceId` is the PK (one scan outcome per resource, ever);
+            // `technology`/`evidence` are `null` on a non-match.
+            t.integer('resourceId')
+                .primary()
+                .unsigned()
+                .references('resource_items.id')
+                .onDelete('CASCADE');
+            t.integer('scannedAt').notNullable();
+            t.string('technology');
+            t.text('evidence');
         });
-        // Compound indexes for the "find duplicate IDs across pages" and
-        // "list pages using provider X" hot paths. Knex's schema builder
-        // can't express compound indexes inline in a way that round-trips
-        // through libsql consistently, so raw SQL is used.
-        await instance.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
-        await instance.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
     }
     if (!(await instance.schema.hasTable('page_jsonld'))) {
         await instance.schema.createTable('page_jsonld', (t) => {
@@ -151,12 +194,16 @@ export async function createAdjunctTables(instance) {
         await instance.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
     }
     // Beholder `MainContentsData` sub-entities, one adjunct table per array
-    // (headings/images/tables/buttons/iframes/videos/audios/canvases). Same
-    // shape as `page_tags` / `page_jsonld`: `pageId` FK → `content_items(id)`
-    // ON DELETE CASCADE, individually guarded so any subset can pre-exist.
-    // `order` preserves the DOM traversal order beholder returns the array
-    // in (0-based); it is not itself an index target since these tables are
-    // always read whole-page via `WHERE pageId = ? ORDER BY "order"`.
+    // (headings/images/tables/buttons/iframes/videos/audios/canvases), plus
+    // `page_main_content_custom_elements` (Web Components) — the sole
+    // exception, captured by nitpicker itself rather than beholder (see
+    // `create-entity-tables.ts`'s `main_content_custom_element_count` JSDoc
+    // for why). Same shape as `page_tags` / `page_jsonld`: `pageId` FK →
+    // `content_items(id)` ON DELETE CASCADE, individually guarded so any
+    // subset can pre-exist. `order` preserves the DOM traversal order the
+    // source array is captured in (0-based); it is not itself an index
+    // target since these tables are always read whole-page via
+    // `WHERE pageId = ? ORDER BY "order"`.
     if (!(await instance.schema.hasTable('page_main_content_headings'))) {
         await instance.schema.createTable('page_main_content_headings', (t) => {
             t.increments('id');
@@ -278,6 +325,21 @@ export async function createAdjunctTables(instance) {
             t.index('pageId');
         });
     }
+    if (!(await instance.schema.hasTable('page_main_content_custom_elements'))) {
+        await instance.schema.createTable('page_main_content_custom_elements', (t) => {
+            t.increments('id');
+            t.integer('pageId')
+                .notNullable()
+                .unsigned()
+                .references('content_items.id')
+                .onDelete('CASCADE');
+            t.integer('order').notNullable();
+            t.string('nodeName').notNullable();
+            t.string('elementId');
+            t.text('classList');
+            t.index('pageId');
+        });
+    }
     if (!(await instance.schema.hasTable('inventory_runs'))) {
         await instance.schema.createTable('inventory_runs', (t) => {
             // One row per successful `--inventory <list>` invocation. The

package/lib/archive/create-entity-tables.d.ts

@@ -148,7 +148,11 @@ import type { Knex } from 'knex';
  * through `text_refs` — the values are page-specific diagnostics with low
  * cross-page reuse, so the ref-table dedup machinery would add write-path
  * cost without a corresponding storage win. `main_content_class_list` holds
- * a JSON-encoded string array.
+ * a JSON-encoded string array. `main_content_custom_element_count` is the
+ * one exception to "derived from beholder's `MainContentsData`" — beholder
+ * has no `customElements` category, so nitpicker captures Web Components
+ * itself (`crawler/capture-custom-elements.ts`) after `scrapeStart`
+ * returns, the same way it captures image dom-paths.
  *
  * ### Index rationale
  *

package/lib/archive/create-entity-tables.js

@@ -147,7 +147,11 @@
  * through `text_refs` — the values are page-specific diagnostics with low
  * cross-page reuse, so the ref-table dedup machinery would add write-path
  * cost without a corresponding storage win. `main_content_class_list` holds
- * a JSON-encoded string array.
+ * a JSON-encoded string array. `main_content_custom_element_count` is the
+ * one exception to "derived from beholder's `MainContentsData`" — beholder
+ * has no `customElements` category, so nitpicker captures Web Components
+ * itself (`crawler/capture-custom-elements.ts`) after `scrapeStart`
+ * returns, the same way it captures image dom-paths.
  *
  * ### Index rationale
  *
@@ -298,6 +302,7 @@ export async function createEntityTables(instance) {
 			main_content_video_count    INTEGER,
 			main_content_audio_count    INTEGER,
 			main_content_canvas_count   INTEGER,
+			main_content_custom_element_count INTEGER,
 			scroll_height_desktop       INTEGER,
 			scroll_height_mobile        INTEGER,
 			body_hash                   BLOB,