@nitpicker/crawler 0.18.0 → 0.19.0

package/lib/archive/meta/compute-main-contents-denormalized.js

@@ -13,11 +13,22 @@
  * omit it (`.spec.ts` files are excluded from the `tsc` build, so this goes
  * uncaught at compile time) — `== null` tolerates both without forcing every
  * fixture to be updated.
+ *
+ * `customElementCount` is a separate parameter, not read off `mainContents`,
+ * because it is not one of beholder's `MainContentsData` categories —
+ * nitpicker captures Web Components itself (`crawler/capture-custom-elements.ts`)
+ * after `scrapeStart` returns. It carries three distinct states:
+ * `undefined`/capture-not-attempted and capture-failure both collapse to
+ * `null` (unknown — NOT the same as "captured, zero found"), while any
+ * number (including `0`) means capture succeeded.
  * @param mainContents - Beholder's per-page main-content metrics, or `null`/`undefined`.
  * @param scrollHeight - Beholder's per-page scroll-height measurements, or `null`/`undefined`.
- * @returns The seventeen denormalised columns.
+ * @param customElementCount - Count of Web Components nitpicker captured in
+ *   the main-content region, or `null`/`undefined` when capture was not
+ *   attempted or failed.
+ * @returns The eighteen denormalised columns.
  */
-export function computeMainContentsDenormalized(mainContents, scrollHeight) {
+export function computeMainContentsDenormalized(mainContents, scrollHeight, customElementCount) {
     if (mainContents == null) {
         return {
             main_content_node_name: null,
@@ -35,6 +46,7 @@ export function computeMainContentsDenormalized(mainContents, scrollHeight) {
             main_content_video_count: null,
             main_content_audio_count: null,
             main_content_canvas_count: null,
+            main_content_custom_element_count: null,
             scroll_height_desktop: null,
             scroll_height_mobile: null,
         };
@@ -57,6 +69,7 @@ export function computeMainContentsDenormalized(mainContents, scrollHeight) {
         main_content_video_count: mainContents.videos.length,
         main_content_audio_count: mainContents.audios.length,
         main_content_canvas_count: mainContents.canvases.length,
+        main_content_custom_element_count: customElementCount ?? null,
         scroll_height_desktop: scrollHeight?.desktop ?? null,
         scroll_height_mobile: scrollHeight?.mobile ?? null,
     };

package/lib/archive/meta/technologies/combine-technology-confidence.d.ts

@@ -0,0 +1,46 @@
+import type { PageTechnologyPartial, TechnologySignalPartial } from './types.js';
+/**
+ * Options for {@link combineTechnologyConfidence}.
+ */
+export interface CombineTechnologyConfidenceOptions {
+    /**
+     * Skip the `TECHNOLOGY_CONFIDENCE_THRESHOLD` drop. Set by the `page_tags`
+     * legacy-migration callers (`migrate-page-tags-to-page-technologies.ts`,
+     * `retarget-legacy-fk-tables.ts`): the threshold exists to filter noisy
+     * NEW detections at crawl time, but migration is converting Wappalyzer
+     * detections the old `page_tags`-based API already returned
+     * unconditionally (including low-confidence ones) — dropping them here
+     * would silently and permanently lose data with no recovery path (v0.x
+     * policy is read-time migration only, no retroactive re-crawl).
+     */
+    skipThreshold?: boolean;
+}
+/**
+ * Combines every signal detected for one technology on one page into a
+ * single confidence score via noisy-OR:
+ *
+ * ```
+ * confidence = round(100 * (1 - Π (1 - weight/100)))
+ * ```
+ *
+ * over each DISTINCT `signalType` present (same signal type appearing
+ * twice — e.g. two different `url-pattern` definitions both matching —
+ * counts once, at its highest weight; compounding only happens ACROSS
+ * signal types, which is what makes this "multiple independent signals
+ * agree" rather than "one noisy signal repeated"). A single signal's
+ * result equals its own weight exactly (`1 - (1 - w/100) = w/100`).
+ *
+ * Technologies whose combined confidence falls below
+ * `TECHNOLOGY_CONFIDENCE_THRESHOLD` are dropped entirely — a lone weak
+ * signal (e.g. `id="app"`, weight 15) is noise, not a detection, and
+ * `page_technologies` has no `isDetected` column to hide it behind; the
+ * row simply is not written. Pass `options.skipThreshold` to keep every
+ * technology regardless of confidence (see its docs for why).
+ * @param signals - Every signal detected for a single page, across all
+ *   technologies (this function groups by `technology` internally).
+ * @param options - See {@link CombineTechnologyConfidenceOptions}.
+ * @returns One {@link PageTechnologyPartial} per technology (filtered by
+ *   the confidence threshold unless `options.skipThreshold` is set), in no
+ *   particular order.
+ */
+export declare function combineTechnologyConfidence(signals: readonly TechnologySignalPartial[], options?: CombineTechnologyConfidenceOptions): PageTechnologyPartial[];

package/lib/archive/meta/technologies/combine-technology-confidence.js

@@ -0,0 +1,93 @@
+import { TECHNOLOGY_CONFIDENCE_THRESHOLD } from './technology-signal-definitions.js';
+/**
+ * Priority order for picking which signal's `category`/`version` wins when
+ * more than one signal for the same technology supplies one. `wappalyzer`
+ * wins first — it is the only source with a real Wappalyzer-reported
+ * `version`; `meta-generator` second — also carries a real version
+ * (`<meta name="generator" content="Astro v4.2.0">`); the rest never
+ * carry a version and are listed only so the `.find()` below terminates
+ * deterministically regardless of definition order.
+ */
+const CATEGORY_VERSION_SOURCE_PRIORITY = [
+    'wappalyzer',
+    'meta-generator',
+    'html-marker',
+    'url-pattern',
+    'scoped-attr',
+    'js-license-comment',
+    'weak-marker',
+];
+/**
+ * Combines every signal detected for one technology on one page into a
+ * single confidence score via noisy-OR:
+ *
+ * ```
+ * confidence = round(100 * (1 - Π (1 - weight/100)))
+ * ```
+ *
+ * over each DISTINCT `signalType` present (same signal type appearing
+ * twice — e.g. two different `url-pattern` definitions both matching —
+ * counts once, at its highest weight; compounding only happens ACROSS
+ * signal types, which is what makes this "multiple independent signals
+ * agree" rather than "one noisy signal repeated"). A single signal's
+ * result equals its own weight exactly (`1 - (1 - w/100) = w/100`).
+ *
+ * Technologies whose combined confidence falls below
+ * `TECHNOLOGY_CONFIDENCE_THRESHOLD` are dropped entirely — a lone weak
+ * signal (e.g. `id="app"`, weight 15) is noise, not a detection, and
+ * `page_technologies` has no `isDetected` column to hide it behind; the
+ * row simply is not written. Pass `options.skipThreshold` to keep every
+ * technology regardless of confidence (see its docs for why).
+ * @param signals - Every signal detected for a single page, across all
+ *   technologies (this function groups by `technology` internally).
+ * @param options - See {@link CombineTechnologyConfidenceOptions}.
+ * @returns One {@link PageTechnologyPartial} per technology (filtered by
+ *   the confidence threshold unless `options.skipThreshold` is set), in no
+ *   particular order.
+ */
+export function combineTechnologyConfidence(signals, options = {}) {
+    const byTechnology = new Map();
+    for (const signal of signals) {
+        const group = byTechnology.get(signal.technology);
+        if (group) {
+            group.push(signal);
+        }
+        else {
+            byTechnology.set(signal.technology, [signal]);
+        }
+    }
+    const results = [];
+    for (const [technology, group] of byTechnology) {
+        const maxWeightByType = new Map();
+        for (const signal of group) {
+            const current = maxWeightByType.get(signal.signalType);
+            if (current === undefined || signal.weight > current) {
+                maxWeightByType.set(signal.signalType, signal.weight);
+            }
+        }
+        const product = [...maxWeightByType.values()].reduce((acc, weight) => acc * (1 - weight / 100), 1);
+        const confidence = Math.round(100 * (1 - product));
+        if (!options.skipThreshold && confidence < TECHNOLOGY_CONFIDENCE_THRESHOLD)
+            continue;
+        let categorySource;
+        let versionSource;
+        for (const type of CATEGORY_VERSION_SOURCE_PRIORITY) {
+            if (!categorySource) {
+                categorySource = group.find((s) => s.signalType === type && s.category != null);
+            }
+            if (!versionSource) {
+                versionSource = group.find((s) => s.signalType === type && s.version != null);
+            }
+            if (categorySource && versionSource)
+                break;
+        }
+        results.push({
+            technology,
+            category: categorySource?.category ?? null,
+            version: versionSource?.version ?? null,
+            confidence,
+            signalCount: maxWeightByType.size,
+        });
+    }
+    return results;
+}

package/lib/archive/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts

@@ -0,0 +1,45 @@
+import type { LegacyTagRowLike } from './convert-tag-row-to-wappalyzer-signal.js';
+/** One `technology_signals` row ready for insertion, `pageId` already resolved. */
+export interface LegacyTechnologySignalInsert {
+    pageId: number;
+    technology: string;
+    signalType: string;
+    evidence: string | null;
+    weight: number;
+}
+/** One `page_technologies` row ready for insertion, `pageId` already resolved. */
+export interface LegacyPageTechnologyInsert {
+    pageId: number;
+    technology: string;
+    category: string | null;
+    version: string | null;
+    confidence: number;
+    signalCount: number;
+}
+/** Result of {@link convertLegacyPageTagsToInserts}. */
+export interface LegacyPageTagsConversion {
+    signalInserts: LegacyTechnologySignalInsert[];
+    technologyInserts: LegacyPageTechnologyInsert[];
+}
+/**
+ * Converts every persisted (legacy) `page_tags` row into the `pageId`-keyed
+ * `technology_signals` + `page_technologies` insert rows for that page's
+ * whole tag set — shared by both `page_tags` migration paths
+ * (`migrate-page-tags-to-page-technologies.ts` for archives already on the
+ * current schema, `retarget-legacy-fk-tables.ts` for the 0.10→0.13 upgrade)
+ * so a `page_tags` row converts identically regardless of which path
+ * produced it, and a future fix to the conversion only needs to change one
+ * place.
+ *
+ * Uses `combineTechnologyConfidence(signals, { skipThreshold: true })` —
+ * migration preserves every historical Wappalyzer detection the old
+ * `page_tags`-based API returned unconditionally, rather than applying the
+ * confidence-noise filter meant for fresh crawl-time detections (see that
+ * option's docs).
+ * @param rows - Every `page_tags` row, in any order (grouped internally by
+ *   `pageId`).
+ * @returns The two insert-ready row sets, empty arrays if `rows` is empty.
+ */
+export declare function convertLegacyPageTagsToInserts(rows: readonly (LegacyTagRowLike & {
+    pageId: number;
+})[]): LegacyPageTagsConversion;

package/lib/archive/meta/technologies/convert-legacy-page-tags-to-inserts.js

@@ -0,0 +1,53 @@
+import { combineTechnologyConfidence } from './combine-technology-confidence.js';
+import { convertTagRowToWappalyzerSignal } from './convert-tag-row-to-wappalyzer-signal.js';
+/**
+ * Converts every persisted (legacy) `page_tags` row into the `pageId`-keyed
+ * `technology_signals` + `page_technologies` insert rows for that page's
+ * whole tag set — shared by both `page_tags` migration paths
+ * (`migrate-page-tags-to-page-technologies.ts` for archives already on the
+ * current schema, `retarget-legacy-fk-tables.ts` for the 0.10→0.13 upgrade)
+ * so a `page_tags` row converts identically regardless of which path
+ * produced it, and a future fix to the conversion only needs to change one
+ * place.
+ *
+ * Uses `combineTechnologyConfidence(signals, { skipThreshold: true })` —
+ * migration preserves every historical Wappalyzer detection the old
+ * `page_tags`-based API returned unconditionally, rather than applying the
+ * confidence-noise filter meant for fresh crawl-time detections (see that
+ * option's docs).
+ * @param rows - Every `page_tags` row, in any order (grouped internally by
+ *   `pageId`).
+ * @returns The two insert-ready row sets, empty arrays if `rows` is empty.
+ */
+export function convertLegacyPageTagsToInserts(rows) {
+    const signalsByPage = new Map();
+    for (const row of rows) {
+        const signal = convertTagRowToWappalyzerSignal(row);
+        const group = signalsByPage.get(row.pageId);
+        if (group) {
+            group.push(signal);
+        }
+        else {
+            signalsByPage.set(row.pageId, [signal]);
+        }
+    }
+    const signalInserts = [];
+    const technologyInserts = [];
+    for (const [pageId, signals] of signalsByPage) {
+        for (const signal of signals) {
+            signalInserts.push({
+                pageId,
+                technology: signal.technology,
+                signalType: signal.signalType,
+                evidence: signal.evidence,
+                weight: signal.weight,
+            });
+        }
+        for (const technology of combineTechnologyConfidence(signals, {
+            skipThreshold: true,
+        })) {
+            technologyInserts.push({ pageId, ...technology });
+        }
+    }
+    return { signalInserts, technologyInserts };
+}

package/lib/archive/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts

@@ -0,0 +1,33 @@
+import type { TechnologySignalPartial } from './types.js';
+/**
+ * Minimal shape this function needs from a persisted (legacy) `page_tags`
+ * row. Deliberately narrower than the old `TagRow` — only the columns a
+ * `wappalyzer` signal can use. `categories` accepts `unknown` because
+ * callers read it via two different paths that disagree on whether the
+ * driver already parsed the `json()` column: raw `SELECT` (still a JSON
+ * string) vs. knex's typed row mapping (already an array on some drivers).
+ */
+export interface LegacyTagRowLike {
+    provider: string;
+    version: string | null;
+    confidence: number | null;
+    categories: unknown;
+}
+/**
+ * Converts one persisted (legacy) `page_tags` row into a `wappalyzer`
+ * technology signal — the shared core `migrate-page-tags-to-page-technologies.ts`
+ * (read-time migration for archives already on the current schema) and
+ * `retarget-legacy-fk-tables.ts` (0.10→0.13 upgrade path) both call, so a
+ * `page_tags` row converts identically regardless of which migration path
+ * produced it.
+ *
+ * Mirrors `normalize-wappalyzer-entries.ts`'s live-beholder conversion, but
+ * reads from the OLD persisted row shape (`provider`/`version`/`confidence`/
+ * `categories`) instead of beholder's live `TagEntry` (`provider`/`version`/
+ * `confidence`/`categories`/`id`/`sources`) — structurally similar, but
+ * `page_tags` has no `id` (externalId) to use as `evidence`, so the provider
+ * name is used instead.
+ * @param row - The persisted `page_tags` row (or an equivalent shape).
+ * @returns The equivalent `wappalyzer` signal.
+ */
+export declare function convertTagRowToWappalyzerSignal(row: LegacyTagRowLike): TechnologySignalPartial;

package/lib/archive/meta/technologies/convert-tag-row-to-wappalyzer-signal.js

@@ -0,0 +1,46 @@
+import { WAPPALYZER_PROVIDER_TO_TECHNOLOGY, WAPPALYZER_SIGNAL_WEIGHT, } from './technology-signal-definitions.js';
+/**
+ * Converts one persisted (legacy) `page_tags` row into a `wappalyzer`
+ * technology signal — the shared core `migrate-page-tags-to-page-technologies.ts`
+ * (read-time migration for archives already on the current schema) and
+ * `retarget-legacy-fk-tables.ts` (0.10→0.13 upgrade path) both call, so a
+ * `page_tags` row converts identically regardless of which migration path
+ * produced it.
+ *
+ * Mirrors `normalize-wappalyzer-entries.ts`'s live-beholder conversion, but
+ * reads from the OLD persisted row shape (`provider`/`version`/`confidence`/
+ * `categories`) instead of beholder's live `TagEntry` (`provider`/`version`/
+ * `confidence`/`categories`/`id`/`sources`) — structurally similar, but
+ * `page_tags` has no `id` (externalId) to use as `evidence`, so the provider
+ * name is used instead.
+ * @param row - The persisted `page_tags` row (or an equivalent shape).
+ * @returns The equivalent `wappalyzer` signal.
+ */
+export function convertTagRowToWappalyzerSignal(row) {
+    const categories = parseCategories(row.categories);
+    return {
+        technology: WAPPALYZER_PROVIDER_TO_TECHNOLOGY[row.provider] ?? row.provider,
+        signalType: 'wappalyzer',
+        evidence: row.provider,
+        weight: row.confidence ?? WAPPALYZER_SIGNAL_WEIGHT,
+        category: categories[0] ?? null,
+        version: row.version ?? null,
+    };
+}
+/**
+ * @param raw
+ */
+function parseCategories(raw) {
+    if (Array.isArray(raw))
+        return raw;
+    if (typeof raw === 'string' && raw.length > 0) {
+        try {
+            const parsed = JSON.parse(raw);
+            return Array.isArray(parsed) ? parsed : [];
+        }
+        catch {
+            return [];
+        }
+    }
+    return [];
+}

package/lib/archive/meta/technologies/extract-technologies-for-archive.d.ts

@@ -0,0 +1,30 @@
+import type { PageTechnologyPartial, TechnologySignalPartial } from './types.js';
+import type { PageData } from '../../../utils/types/types.js';
+/**
+ * The result of one page's technology-signal extraction: every raw signal
+ * (for `technology_signals`, MCP/query signal-level lookups, and per-page
+ * "why was this detected" drill-down) plus the confidence-combined roll-up
+ * (for `page_technologies`).
+ */
+export interface ExtractedTechnologies {
+    signals: TechnologySignalPartial[];
+    technologies: PageTechnologyPartial[];
+}
+/**
+ * Extracts every technology signal for one page — structural (HTML markers,
+ * URL path fragments, scoped attributes, `<meta name="generator">`) plus
+ * Wappalyzer (beholder's `meta.tags.entries`) — and combines them into
+ * confidence-scored technology rows.
+ *
+ * Crawl-time, synchronous, zero extra network cost — the direct replacement
+ * for `extract-tags-for-archive.ts` in the live crawl write path (that file
+ * itself is untouched; `scripts/migrate-to-0.10.mjs` still depends on it).
+ * JS-license-comment signals are NOT produced here — they require an extra
+ * network fetch per JS resource and are added afterward, once per crawl, by
+ * `scan-js-resources-for-technology-signals.ts` ("JSスキャン・エンリッチメント",
+ * a distinct post-crawl category — see ARCHITECTURE.md).
+ * @param html - The page's raw HTML string.
+ * @param meta - `PageData.meta` (for `generator` and `tags.entries`).
+ * @returns Raw signals and their confidence-combined roll-up.
+ */
+export declare function extractTechnologiesForArchive(html: string, meta: PageData['meta']): ExtractedTechnologies;

package/lib/archive/meta/technologies/extract-technologies-for-archive.js

@@ -0,0 +1,27 @@
+import { combineTechnologyConfidence } from './combine-technology-confidence.js';
+import { matchStructuralTechnologySignals } from './match-structural-technology-signals.js';
+import { normalizeWappalyzerEntries } from './normalize-wappalyzer-entries.js';
+/**
+ * Extracts every technology signal for one page — structural (HTML markers,
+ * URL path fragments, scoped attributes, `<meta name="generator">`) plus
+ * Wappalyzer (beholder's `meta.tags.entries`) — and combines them into
+ * confidence-scored technology rows.
+ *
+ * Crawl-time, synchronous, zero extra network cost — the direct replacement
+ * for `extract-tags-for-archive.ts` in the live crawl write path (that file
+ * itself is untouched; `scripts/migrate-to-0.10.mjs` still depends on it).
+ * JS-license-comment signals are NOT produced here — they require an extra
+ * network fetch per JS resource and are added afterward, once per crawl, by
+ * `scan-js-resources-for-technology-signals.ts` ("JSスキャン・エンリッチメント",
+ * a distinct post-crawl category — see ARCHITECTURE.md).
+ * @param html - The page's raw HTML string.
+ * @param meta - `PageData.meta` (for `generator` and `tags.entries`).
+ * @returns Raw signals and their confidence-combined roll-up.
+ */
+export function extractTechnologiesForArchive(html, meta) {
+    const structuralSignals = matchStructuralTechnologySignals(html, meta.generator);
+    const wappalyzerSignals = normalizeWappalyzerEntries(meta.tags?.entries);
+    const signals = [...structuralSignals, ...wappalyzerSignals];
+    const technologies = combineTechnologyConfidence(signals);
+    return { signals, technologies };
+}

package/lib/archive/meta/technologies/match-structural-technology-signals.d.ts

@@ -0,0 +1,13 @@
+import type { TechnologySignalPartial } from './types.js';
+/**
+ * Detects every {@link TECHNOLOGY_SIGNAL_DEFINITIONS} pattern against a
+ * page's raw HTML string, plus `<meta name="generator">` prefix matches
+ * against the already-extracted `generator` field. Pure and synchronous —
+ * no network I/O, no Wappalyzer data (see `normalize-wappalyzer-entries.ts`
+ * for that source).
+ * @param html - The page's raw HTML string.
+ * @param generator - `page_meta.generator` (already extracted by
+ *   `derive-flat-from-meta.ts` from beholder's `Meta`), or `null`/`undefined`.
+ * @returns Un-combined structural signals, one per matched definition.
+ */
+export declare function matchStructuralTechnologySignals(html: string, generator: string | null | undefined): TechnologySignalPartial[];

package/lib/archive/meta/technologies/match-structural-technology-signals.js

@@ -0,0 +1,44 @@
+import { GENERATOR_TECHNOLOGY_MATCHERS, TECHNOLOGY_SIGNAL_DEFINITIONS, } from './technology-signal-definitions.js';
+const EVIDENCE_MAX_LENGTH = 200;
+/**
+ * Detects every {@link TECHNOLOGY_SIGNAL_DEFINITIONS} pattern against a
+ * page's raw HTML string, plus `<meta name="generator">` prefix matches
+ * against the already-extracted `generator` field. Pure and synchronous —
+ * no network I/O, no Wappalyzer data (see `normalize-wappalyzer-entries.ts`
+ * for that source).
+ * @param html - The page's raw HTML string.
+ * @param generator - `page_meta.generator` (already extracted by
+ *   `derive-flat-from-meta.ts` from beholder's `Meta`), or `null`/`undefined`.
+ * @returns Un-combined structural signals, one per matched definition.
+ */
+export function matchStructuralTechnologySignals(html, generator) {
+    const signals = [];
+    for (const def of TECHNOLOGY_SIGNAL_DEFINITIONS) {
+        const match = def.pattern.exec(html);
+        if (match) {
+            signals.push({
+                technology: def.technology,
+                signalType: def.signalType,
+                evidence: match[0].slice(0, EVIDENCE_MAX_LENGTH),
+                weight: def.weight,
+                category: def.category ?? null,
+            });
+        }
+    }
+    if (generator) {
+        const lowerGenerator = generator.toLowerCase();
+        for (const matcher of GENERATOR_TECHNOLOGY_MATCHERS) {
+            if (lowerGenerator.startsWith(matcher.prefix)) {
+                signals.push({
+                    technology: matcher.technology,
+                    signalType: 'meta-generator',
+                    evidence: generator.slice(0, EVIDENCE_MAX_LENGTH),
+                    weight: matcher.weight,
+                    category: matcher.category,
+                    version: generator.slice(matcher.prefix.length).trim() || null,
+                });
+            }
+        }
+    }
+    return signals;
+}

package/lib/archive/meta/technologies/normalize-wappalyzer-entries.d.ts

@@ -0,0 +1,25 @@
+import type { TechnologySignalPartial } from './types.js';
+import type { TagEntry } from '@d-zero/beholder';
+/**
+ * Converts beholder's `TagsMeta.entries` (per-page Wappalyzer detections,
+ * one row per provider × external-id tuple) into `wappalyzer` technology
+ * signals.
+ *
+ * Every entry becomes a signal — Wappalyzer is not limited to the curated
+ * frameworks {@link TECHNOLOGY_SIGNAL_DEFINITIONS} tracks; analytics tools,
+ * CMSes, and everything else Wappalyzer's fingerprint DB knows about pass
+ * through as their own technology (provider name verbatim). Only the
+ * curated frameworks get their provider name FOLDED onto the same
+ * technology name their structural signals use, via
+ * {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY} — so e.g. Wappalyzer's `'Vue.js'`
+ * and a `data-v-*` structural match both contribute to the `'Vue'` row,
+ * not two separate rows.
+ *
+ * This is the direct replacement for the old `extract-tags-for-archive.ts`
+ * (which still exists, unmodified, for `scripts/migrate-to-0.10.mjs`'s
+ * independent use — see that file's JSDoc) in the LIVE crawl write path.
+ * @param entries - `meta.tags.entries` from beholder, or `undefined`
+ *   (legacy minimal-`Meta` test fixtures predate the required field).
+ * @returns One signal per entry; empty array when no tags were detected.
+ */
+export declare function normalizeWappalyzerEntries(entries: readonly TagEntry[] | undefined): TechnologySignalPartial[];

package/lib/archive/meta/technologies/normalize-wappalyzer-entries.js

@@ -0,0 +1,35 @@
+import { WAPPALYZER_PROVIDER_TO_TECHNOLOGY, WAPPALYZER_SIGNAL_WEIGHT, } from './technology-signal-definitions.js';
+/**
+ * Converts beholder's `TagsMeta.entries` (per-page Wappalyzer detections,
+ * one row per provider × external-id tuple) into `wappalyzer` technology
+ * signals.
+ *
+ * Every entry becomes a signal — Wappalyzer is not limited to the curated
+ * frameworks {@link TECHNOLOGY_SIGNAL_DEFINITIONS} tracks; analytics tools,
+ * CMSes, and everything else Wappalyzer's fingerprint DB knows about pass
+ * through as their own technology (provider name verbatim). Only the
+ * curated frameworks get their provider name FOLDED onto the same
+ * technology name their structural signals use, via
+ * {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY} — so e.g. Wappalyzer's `'Vue.js'`
+ * and a `data-v-*` structural match both contribute to the `'Vue'` row,
+ * not two separate rows.
+ *
+ * This is the direct replacement for the old `extract-tags-for-archive.ts`
+ * (which still exists, unmodified, for `scripts/migrate-to-0.10.mjs`'s
+ * independent use — see that file's JSDoc) in the LIVE crawl write path.
+ * @param entries - `meta.tags.entries` from beholder, or `undefined`
+ *   (legacy minimal-`Meta` test fixtures predate the required field).
+ * @returns One signal per entry; empty array when no tags were detected.
+ */
+export function normalizeWappalyzerEntries(entries) {
+    if (!entries || entries.length === 0)
+        return [];
+    return entries.map((entry) => ({
+        technology: WAPPALYZER_PROVIDER_TO_TECHNOLOGY[entry.provider] ?? entry.provider,
+        signalType: 'wappalyzer',
+        evidence: entry.id ?? entry.provider,
+        weight: entry.confidence ?? WAPPALYZER_SIGNAL_WEIGHT,
+        category: entry.categories[0] ?? null,
+        version: entry.version ?? null,
+    }));
+}

package/lib/archive/meta/technologies/technology-signal-definitions.d.ts

@@ -0,0 +1,43 @@
+import type { TechnologySignalDefinition } from './types.js';
+/**
+ * Fixed fingerprint list for the frontend frameworks this module detects via
+ * structural signals (URL path fragments, HTML markers, scoped-attribute
+ * conventions). Deliberately a TypeScript array, not an external config
+ * file (per the accepted design: precision/coverage is refined by editing
+ * this file, not by runtime configuration).
+ *
+ * Every framework here is ALSO detectable via {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY}
+ * when beholder's Wappalyzer pass independently reports it — the two signal
+ * sources compound in `combineTechnologyConfidence` rather than one
+ * replacing the other. Wappalyzer detects many more technologies than the
+ * frameworks listed here (analytics, CMSes, ...); those pass through with
+ * the provider name verbatim as the technology name, uncurated.
+ * @module
+ */
+export declare const TECHNOLOGY_SIGNAL_DEFINITIONS: readonly TechnologySignalDefinition[];
+/**
+ * `<meta name="generator">` prefix matchers, checked case-insensitively
+ * against `page_meta.generator` (already extracted — no new HTML scan
+ * needed). Distinct from {@link TECHNOLOGY_SIGNAL_DEFINITIONS} because the
+ * match strategy is prefix-on-a-known-field, not regex-over-HTML.
+ */
+export declare const GENERATOR_TECHNOLOGY_MATCHERS: ReadonlyArray<{
+    technology: string;
+    prefix: string;
+    weight: number;
+    category: string;
+}>;
+/**
+ * Wappalyzer provider name → normalized technology name, for the curated
+ * frameworks that also have {@link TECHNOLOGY_SIGNAL_DEFINITIONS}. Providers
+ * NOT in this map still become `page_technologies` rows — the technology
+ * name is just the provider name verbatim (see `normalize-wappalyzer-entries.ts`)
+ * — this map exists only to fold curated frameworks' Wappalyzer detections
+ * onto the same technology name their structural signals use, so the two
+ * sources compound instead of appearing as two separate rows.
+ */
+export declare const WAPPALYZER_PROVIDER_TO_TECHNOLOGY: Readonly<Record<string, string>>;
+/** Confidence a lone `wappalyzer` signal contributes, before compounding with any structural signal. */
+export declare const WAPPALYZER_SIGNAL_WEIGHT = 60;
+/** Confidence threshold (`combineTechnologyConfidence`'s output) at or above which a `page_technologies` row is written at all. Below this, a single weak/ambiguous signal is treated as noise, not a detection. */
+export declare const TECHNOLOGY_CONFIDENCE_THRESHOLD = 30;