@nitpicker/archive 0.0.0 → 0.25.0

package/lib/meta/technologies/match-structural-technology-signals.js

@@ -0,0 +1,48 @@
+import { matchStructuralTechnologySignals as matchStructuralTechnologySignalsNative } from '@nitpicker/core/match-structural-technology-signals';
+import { GENERATOR_TECHNOLOGY_MATCHERS } from './technology-signal-definitions.js';
+const EVIDENCE_MAX_LENGTH = 200;
+/**
+ * Detects the structural fingerprint list (`@nitpicker/core`'s
+ * `crates/nitpicker_tech_detect`) against a page's raw HTML, plus
+ * `<meta name="generator">` prefix matches against the already-extracted
+ * `generator` field. Pure and synchronous — no network I/O, no Wappalyzer
+ * data (see `normalize-wappalyzer-entries.ts` for that source).
+ *
+ * The HTML match runs in the native addon, which scans the page once for
+ * every fingerprint. The JavaScript regular expressions it was ported from
+ * stay in `structural-signal-definitions.ts`, excluded from the build, as
+ * the oracle for `match-structural-technology-signals.parity.spec.ts`.
+ * @param html - The page's raw HTML string (empty for external or
+ *   metadata-only scrapes, which then match nothing).
+ * @param generator - `page_meta.generator` (already extracted by
+ *   `derive-flat-from-meta.ts` from beholder's `Meta`), or `null`/`undefined`.
+ * @returns Un-combined structural signals, one per matched fingerprint in
+ *   the list's order, then one per matched generator prefix.
+ * @example
+ * ```ts
+ * matchStructuralTechnologySignals('<div data-v-1a2b3c4d></div>', 'Nuxt 3.8');
+ * // [{ technology: 'Vue', signalType: 'scoped-attr', ... },
+ * //  { technology: 'Nuxt', signalType: 'meta-generator', version: '3.8', ... }]
+ * ```
+ */
+export function matchStructuralTechnologySignals(html, generator) {
+    const signals = html.length > 0
+        ? matchStructuralTechnologySignalsNative(Buffer.from(html, 'utf8'))
+        : [];
+    if (generator) {
+        const lowerGenerator = generator.toLowerCase();
+        for (const matcher of GENERATOR_TECHNOLOGY_MATCHERS) {
+            if (lowerGenerator.startsWith(matcher.prefix)) {
+                signals.push({
+                    technology: matcher.technology,
+                    signalType: 'meta-generator',
+                    evidence: generator.slice(0, EVIDENCE_MAX_LENGTH),
+                    weight: matcher.weight,
+                    category: matcher.category,
+                    version: generator.slice(matcher.prefix.length).trim() || null,
+                });
+            }
+        }
+    }
+    return signals;
+}

package/lib/meta/technologies/normalize-wappalyzer-entries.d.ts

@@ -0,0 +1,25 @@
+import type { TechnologySignalPartial } from './types.js';
+import type { TagEntry } from '../types.js';
+/**
+ * Converts beholder's `TagsMeta.entries` (per-page Wappalyzer detections,
+ * one row per provider × external-id tuple) into `wappalyzer` technology
+ * signals.
+ *
+ * Every entry becomes a signal — Wappalyzer is not limited to the curated
+ * frameworks the structural fingerprint list tracks; analytics tools,
+ * CMSes, and everything else Wappalyzer's fingerprint DB knows about pass
+ * through as their own technology (provider name verbatim). Only the
+ * curated frameworks get their provider name FOLDED onto the same
+ * technology name their structural signals use, via
+ * {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY} — so e.g. Wappalyzer's `'Vue.js'`
+ * and a `data-v-*` structural match both contribute to the `'Vue'` row,
+ * not two separate rows.
+ *
+ * This is the direct replacement for the old `extract-tags-for-archive.ts`
+ * (which still exists, unmodified, for `scripts/migrate-to-0.10.mjs`'s
+ * independent use — see that file's JSDoc) in the LIVE crawl write path.
+ * @param entries - `meta.tags.entries` from beholder, or `undefined`
+ *   (legacy minimal-`Meta` test fixtures predate the required field).
+ * @returns One signal per entry; empty array when no tags were detected.
+ */
+export declare function normalizeWappalyzerEntries(entries: readonly TagEntry[] | undefined): TechnologySignalPartial[];

package/lib/meta/technologies/normalize-wappalyzer-entries.js

@@ -0,0 +1,35 @@
+import { WAPPALYZER_PROVIDER_TO_TECHNOLOGY, WAPPALYZER_SIGNAL_WEIGHT, } from './technology-signal-definitions.js';
+/**
+ * Converts beholder's `TagsMeta.entries` (per-page Wappalyzer detections,
+ * one row per provider × external-id tuple) into `wappalyzer` technology
+ * signals.
+ *
+ * Every entry becomes a signal — Wappalyzer is not limited to the curated
+ * frameworks the structural fingerprint list tracks; analytics tools,
+ * CMSes, and everything else Wappalyzer's fingerprint DB knows about pass
+ * through as their own technology (provider name verbatim). Only the
+ * curated frameworks get their provider name FOLDED onto the same
+ * technology name their structural signals use, via
+ * {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY} — so e.g. Wappalyzer's `'Vue.js'`
+ * and a `data-v-*` structural match both contribute to the `'Vue'` row,
+ * not two separate rows.
+ *
+ * This is the direct replacement for the old `extract-tags-for-archive.ts`
+ * (which still exists, unmodified, for `scripts/migrate-to-0.10.mjs`'s
+ * independent use — see that file's JSDoc) in the LIVE crawl write path.
+ * @param entries - `meta.tags.entries` from beholder, or `undefined`
+ *   (legacy minimal-`Meta` test fixtures predate the required field).
+ * @returns One signal per entry; empty array when no tags were detected.
+ */
+export function normalizeWappalyzerEntries(entries) {
+    if (!entries || entries.length === 0)
+        return [];
+    return entries.map((entry) => ({
+        technology: WAPPALYZER_PROVIDER_TO_TECHNOLOGY[entry.provider] ?? entry.provider,
+        signalType: 'wappalyzer',
+        evidence: entry.id ?? entry.provider,
+        weight: entry.confidence ?? WAPPALYZER_SIGNAL_WEIGHT,
+        category: entry.categories[0] ?? null,
+        version: entry.version ?? null,
+    }));
+}

package/lib/meta/technologies/technology-signal-definitions.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Fixed tables for technology detection that run in TypeScript: the
+ * `<meta name="generator">` matchers, the Wappalyzer name map and the
+ * confidence constants. The structural fingerprint list (URL path
+ * fragments, HTML markers, scoped-attribute conventions) is not here: it
+ * lives with its matcher in `@nitpicker/core`'s
+ * `crates/nitpicker_tech_detect`, and precision is refined by editing it
+ * there, not by runtime configuration.
+ *
+ * Every framework that list covers is ALSO detectable via
+ * {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY} when beholder's Wappalyzer pass
+ * independently reports it — the two signal sources compound in
+ * `combineTechnologyConfidence` rather than one replacing the other.
+ * Wappalyzer detects many more technologies than those frameworks
+ * (analytics, CMSes, ...); those pass through with the provider name
+ * verbatim as the technology name, uncurated.
+ * @module
+ */
+/**
+ * `<meta name="generator">` prefix matchers, checked case-insensitively
+ * against `page_meta.generator` (already extracted — no new HTML scan
+ * needed). Distinct from the structural fingerprint list because the match
+ * strategy is prefix-on-a-known-field, not regex-over-HTML.
+ */
+export declare const GENERATOR_TECHNOLOGY_MATCHERS: ReadonlyArray<{
+    technology: string;
+    prefix: string;
+    weight: number;
+    category: string;
+}>;
+/**
+ * Wappalyzer provider name → normalized technology name, for the curated
+ * frameworks that also have structural fingerprints. Providers
+ * NOT in this map still become `page_technologies` rows — the technology
+ * name is just the provider name verbatim (see `normalize-wappalyzer-entries.ts`)
+ * — this map exists only to fold curated frameworks' Wappalyzer detections
+ * onto the same technology name their structural signals use, so the two
+ * sources compound instead of appearing as two separate rows.
+ */
+export declare const WAPPALYZER_PROVIDER_TO_TECHNOLOGY: Readonly<Record<string, string>>;
+/** Confidence a lone `wappalyzer` signal contributes, before compounding with any structural signal. */
+export declare const WAPPALYZER_SIGNAL_WEIGHT = 60;
+/** Confidence threshold (`combineTechnologyConfidence`'s output) at or above which a `page_technologies` row is written at all. Below this, a single weak/ambiguous signal is treated as noise, not a detection. */
+export declare const TECHNOLOGY_CONFIDENCE_THRESHOLD = 30;

package/lib/meta/technologies/technology-signal-definitions.js

@@ -0,0 +1,60 @@
+/**
+ * Fixed tables for technology detection that run in TypeScript: the
+ * `<meta name="generator">` matchers, the Wappalyzer name map and the
+ * confidence constants. The structural fingerprint list (URL path
+ * fragments, HTML markers, scoped-attribute conventions) is not here: it
+ * lives with its matcher in `@nitpicker/core`'s
+ * `crates/nitpicker_tech_detect`, and precision is refined by editing it
+ * there, not by runtime configuration.
+ *
+ * Every framework that list covers is ALSO detectable via
+ * {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY} when beholder's Wappalyzer pass
+ * independently reports it — the two signal sources compound in
+ * `combineTechnologyConfidence` rather than one replacing the other.
+ * Wappalyzer detects many more technologies than those frameworks
+ * (analytics, CMSes, ...); those pass through with the provider name
+ * verbatim as the technology name, uncurated.
+ * @module
+ */
+/**
+ * `<meta name="generator">` prefix matchers, checked case-insensitively
+ * against `page_meta.generator` (already extracted — no new HTML scan
+ * needed). Distinct from the structural fingerprint list because the match
+ * strategy is prefix-on-a-known-field, not regex-over-HTML.
+ */
+export const GENERATOR_TECHNOLOGY_MATCHERS = [
+    { technology: 'Astro', prefix: 'astro', weight: 85, category: 'Static site generator' },
+    { technology: 'Nuxt', prefix: 'nuxt', weight: 85, category: 'JavaScript frameworks' },
+    {
+        technology: 'Gatsby',
+        prefix: 'gatsby',
+        weight: 85,
+        category: 'Static site generator',
+    },
+];
+/**
+ * Wappalyzer provider name → normalized technology name, for the curated
+ * frameworks that also have structural fingerprints. Providers
+ * NOT in this map still become `page_technologies` rows — the technology
+ * name is just the provider name verbatim (see `normalize-wappalyzer-entries.ts`)
+ * — this map exists only to fold curated frameworks' Wappalyzer detections
+ * onto the same technology name their structural signals use, so the two
+ * sources compound instead of appearing as two separate rows.
+ */
+export const WAPPALYZER_PROVIDER_TO_TECHNOLOGY = {
+    'Next.js': 'Next.js',
+    Astro: 'Astro',
+    'Vue.js': 'Vue',
+    'Nuxt.js': 'Nuxt',
+    Nuxt: 'Nuxt',
+    Svelte: 'Svelte',
+    SvelteKit: 'SvelteKit',
+    Remix: 'Remix',
+    Gatsby: 'Gatsby',
+    Angular: 'Angular',
+    AngularJS: 'Angular',
+};
+/** Confidence a lone `wappalyzer` signal contributes, before compounding with any structural signal. */
+export const WAPPALYZER_SIGNAL_WEIGHT = 60;
+/** Confidence threshold (`combineTechnologyConfidence`'s output) at or above which a `page_technologies` row is written at all. Below this, a single weak/ambiguous signal is treated as noise, not a detection. */
+export const TECHNOLOGY_CONFIDENCE_THRESHOLD = 30;

package/lib/meta/technologies/types.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Domain types for technology-signal extraction (crawl time) and confidence
+ * roll-up. Kept in this dedicated `technologies/types.ts` rather than the
+ * parent `meta/types.ts` because these types are internal to the extraction
+ * pipeline (`match-structural-technology-signals.ts` →
+ * `combine-technology-confidence.ts` → `extract-technologies-for-archive.ts`)
+ * and never cross the `archive/meta/types.ts` boundary that the persisted
+ * `technology_signals` / `page_technologies` row shapes (`TechnologySignalRow`
+ * / `PageTechnologyRow`) live in — see the repo-wide `型は types.ts に集約`
+ * rule, applied per-module rather than globally so this module's internal
+ * vocabulary does not leak into the db-ops layer's public row types.
+ */
+import type { StructuralTechnologySignalType } from '@nitpicker/core/types';
+/**
+ * How a technology signal was detected. `wappalyzer` is beholder's
+ * `simple-wappalyzer`-based detection (imprecise on its own for frameworks,
+ * but authoritative for the long tail of non-framework technologies —
+ * analytics, CMSes, etc. — this module does not otherwise attempt to
+ * detect). The rest are structural signals this module detects itself.
+ */
+export type TechnologySignalType = 'wappalyzer' | 'meta-generator' | StructuralTechnologySignalType;
+/**
+ * One un-combined signal for one technology on one page. The unit that
+ * `combineTechnologyConfidence` groups (by `technology`) and reduces via
+ * noisy-OR into a single {@link PageTechnologyPartial} per technology.
+ */
+export interface TechnologySignalPartial {
+    /** Normalized technology name (e.g. `'Next.js'`, `'Google Analytics'`). */
+    technology: string;
+    signalType: TechnologySignalType;
+    /** Matched fragment or raw value, truncated to ~200 chars by the caller. `null` when the signal type carries no useful excerpt. */
+    evidence: string | null;
+    /** This signal's confidence in isolation, 0-100. */
+    weight: number;
+    /** Wappalyzer/generator-reported category, when this signal type can supply one. */
+    category?: string | null;
+    /** Wappalyzer/generator-reported version, when this signal type can supply one. */
+    version?: string | null;
+}
+/**
+ * One technology's combined roll-up for one page, before the `pageId` is
+ * known (added by the caller at insert time — same "id injected at write
+ * time" pattern as {@link import('../types.js').TagRowForInsert}).
+ */
+export interface PageTechnologyPartial {
+    technology: string;
+    category: string | null;
+    version: string | null;
+    /** `combineTechnologyConfidence`'s noisy-OR result, 0-100. */
+    confidence: number;
+    /** Count of distinct `signalType`s that contributed to `confidence`. */
+    signalCount: number;
+}
+/**
+ * A structural fingerprint definition as a JavaScript regular expression:
+ * one technology matched by one signal type, at one isolated weight. The
+ * shape of the parity oracle (`structural-signal-definitions.ts`, excluded
+ * from the build) only — detection runs the Rust port in `@nitpicker/core`'s
+ * `crates/nitpicker_tech_detect`, which owns the list.
+ */
+export interface TechnologySignalDefinition {
+    technology: string;
+    signalType: StructuralTechnologySignalType;
+    /** The expression the stored `technology_signals` rows were computed with. */
+    pattern: RegExp;
+    /** This definition's confidence in isolation, 0-100, when `pattern` matches. */
+    weight: number;
+    /** Category to attach to the technology when only this signal fired. */
+    category?: string;
+}

package/lib/meta/technologies/types.js

@@ -0,0 +1,13 @@
+/**
+ * Domain types for technology-signal extraction (crawl time) and confidence
+ * roll-up. Kept in this dedicated `technologies/types.ts` rather than the
+ * parent `meta/types.ts` because these types are internal to the extraction
+ * pipeline (`match-structural-technology-signals.ts` →
+ * `combine-technology-confidence.ts` → `extract-technologies-for-archive.ts`)
+ * and never cross the `archive/meta/types.ts` boundary that the persisted
+ * `technology_signals` / `page_technologies` row shapes (`TechnologySignalRow`
+ * / `PageTechnologyRow`) live in — see the repo-wide `型は types.ts に集約`
+ * rule, applied per-module rather than globally so this module's internal
+ * vocabulary does not leak into the db-ops layer's public row types.
+ */
+export {};