@nitpicker/crawler 0.18.0 → 0.19.0

package/lib/crawler/scan-js-resources-for-technology-signals.js

@@ -0,0 +1,196 @@
+import { combineTechnologyConfidence } from '../archive/meta/technologies/combine-technology-confidence.js';
+import { scanJsResourceForLicenseComment } from './scan-js-resource-for-license-comment.js';
+/**
+ * Content-Type strings this scan treats as JavaScript. Deliberately
+ * duplicated (not imported) from `@nitpicker/query`'s `content-type-rules.ts`
+ * `'javascript'` category rule: the crawler package must not depend back on
+ * query (see `ensure-viewer-read-model-quietly.ts`'s docs for the same
+ * boundary), and this list is small and stable enough that duplication is
+ * cheaper than introducing a shared third package for it.
+ */
+const JS_CONTENT_TYPES = [
+    'text/javascript',
+    'application/javascript',
+    'application/x-javascript',
+    'application/ecmascript',
+];
+const DEFAULT_CONCURRENCY = 4;
+/**
+ * Runs `worker` over `items` with at most `concurrency` calls in flight at
+ * once. A minimal worker-pool, not a chunk-then-parallelize helper like
+ * `eachSplitted` (which parallelizes across chunks, not within one) — this
+ * module needs a true cap on simultaneous outbound HTTP connections.
+ * @param items - The items to process.
+ * @param concurrency - Maximum simultaneous `worker` calls.
+ * @param worker - Called once per item; errors propagate to the caller.
+ */
+async function runWithConcurrency(items, concurrency, worker) {
+    let nextIndex = 0;
+    /**
+     *
+     */
+    async function runNext() {
+        const index = nextIndex++;
+        if (index >= items.length)
+            return;
+        await worker(items[index]);
+        await runNext();
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => runNext()));
+}
+/**
+ * Re-derives one page's `technology_signals` + `page_technologies` rows from
+ * its existing persisted signals plus newly-discovered `js-license-comment`
+ * signals, then scoped-replaces both tables — the same full-per-page-replace
+ * invariant `insertTechnologies` follows (see its docs: the two tables are
+ * never updated independently).
+ *
+ * `technology_signals` does not persist `category`/`version` (only
+ * `page_technologies` does — see `create-adjunct-tables.ts`), so a technology
+ * whose category/version came from a signal not re-derivable from the
+ * persisted rows (a `wappalyzer` or `meta-generator` signal, both computed at
+ * crawl time from data this function does not have) would otherwise regress
+ * to `null` here. The existing `page_technologies` row is read first and its
+ * `category`/`version` fall back in wherever the freshly combined result has
+ * none.
+ * @param knex - The archive's Knex instance.
+ * @param pageId - The page to update.
+ * @param newSignals - Newly-discovered signals for this page (from JS
+ *   resources it references).
+ */
+async function applyNewSignalsToPage(knex, pageId, newSignals) {
+    await knex.transaction(async (trx) => {
+        const existingSignalRows = await trx('technology_signals')
+            .where('pageId', pageId)
+            .select('technology', 'signalType', 'evidence', 'weight');
+        const existingTechnologyRows = await trx('page_technologies')
+            .where('pageId', pageId)
+            .select('technology', 'category', 'version');
+        const existingMetaByTechnology = new Map(existingTechnologyRows.map((row) => [row.technology, row]));
+        const allSignals = [...existingSignalRows, ...newSignals];
+        const technologies = combineTechnologyConfidence(allSignals).map((t) => {
+            const previous = existingMetaByTechnology.get(t.technology);
+            return {
+                ...t,
+                category: t.category ?? previous?.category ?? null,
+                version: t.version ?? previous?.version ?? null,
+            };
+        });
+        await trx('technology_signals').where('pageId', pageId).delete();
+        await trx('page_technologies').where('pageId', pageId).delete();
+        await trx('technology_signals').insert(allSignals.map((s) => ({
+            pageId,
+            technology: s.technology,
+            signalType: s.signalType,
+            evidence: s.evidence,
+            weight: s.weight,
+        })));
+        if (technologies.length > 0) {
+            await trx('page_technologies').insert(technologies.map((t) => ({ pageId, ...t })));
+        }
+    });
+}
+/**
+ * Post-crawl network enrichment (distinct from crawl-time extraction and
+ * from read-model-time backfill — see ARCHITECTURE.md): re-fetches the
+ * leading bytes of every not-yet-scanned internal JS resource, tests them
+ * for a known technology's license comment, and folds any match into the
+ * referencing pages' `technology_signals` / `page_technologies`.
+ *
+ * Each resource is scanned at most once ever, across the archive's whole
+ * lifetime — outcomes (including non-matches) are recorded in
+ * `technology_js_scan_cache` keyed by `resourceId`, so a later
+ * `--append`/`--retry-failed` run only pays the network cost for resources
+ * discovered since the last run.
+ *
+ * A single resource can be referenced by many pages (a shared bundle); a
+ * match is applied to every one of them independently. Network scanning
+ * runs at bounded concurrency; the per-page DB recombination that follows
+ * runs after every scan has settled, never concurrently for the same page,
+ * so two resources that both resolve to the same page cannot race each
+ * other's read-modify-write.
+ *
+ * Best-effort like `scanJsResourceForLicenseComment`: an unreachable
+ * resource is recorded as a non-match (cached as scanned, `technology:
+ * null`) rather than retried or surfaced as an error — a flaky CDN must not
+ * block the rest of the archive's enrichment, and the resource will not be
+ * retried until `technology_js_scan_cache` itself is cleared.
+ * @param accessor - The archive to enrich.
+ * @param options - Concurrency, byte-cap, timeout, and progress overrides.
+ * @returns Counters describing what was scanned, matched, and updated.
+ * @example
+ * const result = await scanJsResourcesForTechnologySignals(archive);
+ * // { candidateCount: 42, scannedCount: 42, matchedCount: 3, pagesUpdatedCount: 57 }
+ */
+export async function scanJsResourcesForTechnologySignals(accessor, options = {}) {
+    const knex = accessor.getKnex();
+    const concurrency = Math.max(options.concurrency ?? DEFAULT_CONCURRENCY, 1);
+    const candidates = await knex('resource_items as ri')
+        .join('url_refs as ur', 'ur.id', 'ri.url_id')
+        .leftJoin('content_type_refs as ctr', 'ctr.id', 'ri.content_type_id')
+        .leftJoin('technology_js_scan_cache as cache', 'cache.resourceId', 'ri.id')
+        .where('ri.is_external', 0)
+        .whereNull('cache.resourceId')
+        .where((qb) => {
+        qb.whereIn('ctr.raw', JS_CONTENT_TYPES)
+            .orWhere('ur.url', 'like', '%.js')
+            .orWhere('ur.url', 'like', '%.js?%')
+            .orWhere('ur.url', 'like', '%.mjs')
+            .orWhere('ur.url', 'like', '%.mjs?%');
+    })
+        .select('ri.id as resourceId', 'ur.url as url');
+    let scannedCount = 0;
+    const matchesByResourceId = new Map();
+    await runWithConcurrency(candidates, concurrency, async (candidate) => {
+        const signal = await scanJsResourceForLicenseComment(candidate.url, {
+            byteLimit: options.byteLimit,
+            timeout: options.timeout,
+            userAgent: options.userAgent,
+        });
+        scannedCount++;
+        options.onProgress?.(scannedCount, candidates.length);
+        await knex('technology_js_scan_cache').insert({
+            resourceId: candidate.resourceId,
+            scannedAt: Date.now(),
+            technology: signal?.technology ?? null,
+            evidence: signal?.evidence ?? null,
+        });
+        if (signal) {
+            matchesByResourceId.set(candidate.resourceId, signal);
+        }
+    });
+    if (matchesByResourceId.size === 0) {
+        return {
+            candidateCount: candidates.length,
+            scannedCount,
+            matchedCount: 0,
+            pagesUpdatedCount: 0,
+        };
+    }
+    const edges = await knex('resource_ref_edges')
+        .whereIn('resource_id', [...matchesByResourceId.keys()])
+        .select('resource_id', 'page_id');
+    const newSignalsByPageId = new Map();
+    for (const edge of edges) {
+        const signal = matchesByResourceId.get(edge.resource_id);
+        if (!signal)
+            continue;
+        const list = newSignalsByPageId.get(edge.page_id);
+        if (list) {
+            list.push(signal);
+        }
+        else {
+            newSignalsByPageId.set(edge.page_id, [signal]);
+        }
+    }
+    const affectedPageIds = [...newSignalsByPageId.keys()];
+    await runWithConcurrency(affectedPageIds, concurrency, async (pageId) => {
+        await applyNewSignalsToPage(knex, pageId, newSignalsByPageId.get(pageId));
+    });
+    return {
+        candidateCount: candidates.length,
+        scannedCount,
+        matchedCount: matchesByResourceId.size,
+        pagesUpdatedCount: affectedPageIds.length,
+    };
+}

package/lib/crawler/types.d.ts

@@ -301,6 +301,14 @@ export interface CrawlerEventTypes {
          * common case outside `crawl --inventory`. See {@link PageSource}.
          */
         source?: PageSource;
+        /**
+         * This page's body hash, precomputed by the crawler from `result.html`
+         * (non-null whenever `result.html.length > 0`, since this event is only
+         * emitted for internal pages). Forwarded through `Archive.setPage` to
+         * `update-page.ts` so `page_meta.body_hash` is written from this value
+         * instead of hashing the same html a second time.
+         */
+        bodyHash?: Buffer | null;
     };
     /**
      * Emitted when an external page (outside the crawl scope) has been scraped.
@@ -475,6 +483,21 @@ export interface CrawlerEventTypes {
         observedCount: number;
     };
 }
+/**
+ * One Web Component (custom element) found inside a page's main-content
+ * region by {@link ./collect-custom-elements.ts}. Unlike beholder's eight
+ * `MainContentsData` categories (headings/images/tables/buttons/iframes/
+ * videos/audios/canvases), this is captured independently by nitpicker
+ * itself — see {@link ./capture-custom-elements.ts} for why.
+ */
+export interface MainContentCustomElementCandidate {
+    /** The element's `nodeName` (always upper-cased in an HTML document, e.g. `MY-WIDGET`). */
+    nodeName: string;
+    /** The element's `id` attribute, or `null` when absent. */
+    elementId: string | null;
+    /** The element's class list, in DOM order. */
+    classList: string[];
+}
 /**
  * Tunables for `NetworkOutageDetector`.
  */

package/lib/crawler-orchestrator.d.ts

@@ -1,7 +1,7 @@
 import type { Config } from './archive/types.js';
 import type { NetworkProbe } from './crawler/probe-network.js';
 import type { InventoryMode } from './crawler/types.js';
-import type { CrawlEvent } from './types.js';
+import type { CrawlEvent, SetupProgressCallbacks } from './types.js';
 import type { ExURL } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import Archive from './archive/archive.js';
@@ -111,10 +111,13 @@ interface InventorySource {
  * writes the final archive file. It emits events defined by {@link CrawlEvent}.
  *
  * Instances are created via the static factory methods {@link CrawlerOrchestrator.crawling}
- * or {@link CrawlerOrchestrator.resume}; the constructor is private.
+ * or {@link CrawlerOrchestrator.resume}; the constructor is private. Implements
+ * `Symbol.asyncDispose` so callers can use `await using` to close the archive
+ * and reap zombie Chromium processes on scope exit instead of a manual
+ * `try`/`finally` around `archive.close()` + `garbageCollect()`.
  * @example
  * ```ts
- * const orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
+ * await using orchestrator = await CrawlerOrchestrator.crawling(['https://example.com'], { recursive: true });
  * await orchestrator.write();
  * ```
  */
@@ -125,6 +128,23 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      */
     get archive(): Archive;
     private constructor();
+    /**
+     * Enables `await using orchestrator = ...`. Closes the archive (write
+     * or remove tmpDir + release the lock, per {@link Archive.close}) and
+     * then reaps any zombie Chromium processes via {@link garbageCollect} —
+     * the same two-step teardown every CLI crawl command previously
+     * repeated by hand in a `finally` block.
+     *
+     * Relays `Archive.close()`'s recovery-write progress (issue #294) as
+     * `recoveringArchiveWrite`/`writeStep`/`writeTarProgress` — the same
+     * events `write()` emits — for the rare case where the file doesn't
+     * exist on disk yet at dispose time (e.g. an explicit `write()` call
+     * threw partway through). A CLI listener whose display is still open at
+     * that point (it hadn't yet seen `writeFileEnd`) picks these up for
+     * free; one that already tore down after the earlier failure silently
+     * drops them, same as any other post-close display update.
+     */
+    [Symbol.asyncDispose](): Promise<void>;
     /**
      * Abort the current crawl operation.
      *
@@ -173,8 +193,11 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * the time `write()` is called those tables are already populated.
      * This method just tars.
      *
-     * Emits `writeFileStart` before writing and `writeFileEnd` after
-     * the write completes successfully.
+     * Emits `writeFileStart` before writing and `writeFileEnd` after the
+     * write completes successfully. Also relays `Archive.write()`'s
+     * per-step (`writeStep`) and tar-byte (`writeTarProgress`) progress
+     * (issue #294) — tarring a 15 GB+ archive can take minutes, and without
+     * these events a CLI listener has no way to show it isn't hung.
      */
     write(): Promise<void>;
     /**
@@ -211,10 +234,14 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param newUrls - New root URLs to add and crawl.
      * @param options - Optional config overrides applied on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, repromote, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the append crawl completes.
      * @throws {Error} When `newUrls` is empty, the archive is in list mode, or it cannot be parsed.
      */
-    static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static append(archivePath: string, newUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Inventory mode: cross-reference a user-supplied URL list against an
      * existing `.nitpicker` archive and import ONLY the URLs that are not yet
@@ -292,10 +319,15 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   URLs. Pass `null` for programmatic callers that built
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, scope classification, bulk inserts, state rebuild) that runs
+     *   before `initializedCallback` — see {@link SetupProgressCallbacks} for
+     *   why this can't go through the orchestrator's event emitter (issue
+     *   #294).
      * @returns The orchestrator instance after a successful inventory pass.
      * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
-    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
+    static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Re-fetch previously-failed pages in an existing `.nitpicker` archive.
      *
@@ -326,10 +358,14 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param archivePath - Absolute or relative path to the existing `.nitpicker`.
      * @param options - Optional config overrides applied on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (untar, `.bak` copy, reset, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
      * @returns The orchestrator instance after the retry crawl completes.
      * @throws {Error} When the archive is in list mode or has no parseable roots.
      */
-    static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static retryFailed(archivePath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
     /**
      * Resume a previously interrupted crawl from an existing archive file.
      *
@@ -339,9 +375,17 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * @param stubPath - Path to the existing archive file to resume from.
      * @param options - Optional configuration overrides to apply on top of the archived config.
      * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
+     * @param setupProgress - Optional progress callbacks for the setup phase
+     *   (self-healing migrations, state rebuild) that runs before
+     *   `initializedCallback` — see {@link SetupProgressCallbacks} for why
+     *   this can't go through the orchestrator's event emitter (issue #294).
+     *   No `onExtractProgress`/`onCopyProgress`: unlike `append`/`inventory`/
+     *   `retryFailed`, `resume` reconnects to an existing tmpDir (no untar)
+     *   and takes no `.bak` (nothing to restore — the interrupted crawl's
+     *   tmpDir IS the source of truth).
      * @returns A promise that resolves to the CrawlerOrchestrator instance after crawling completes.
      * @throws {Error} If the archived URL is invalid.
      */
-    static resume(stubPath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback): Promise<CrawlerOrchestrator>;
+    static resume(stubPath: string, options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, setupProgress?: SetupProgressCallbacks): Promise<CrawlerOrchestrator>;
 }
 export {};