npm - @d-zero/beholder - Versions diffs - 2.1.6 → 3.1.0 - Mend

@d-zero/beholder 2.1.6 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (45) hide show

package/CHANGELOG.md +44 -0
package/README.md +26 -0
package/dist/dom-evaluation.d.ts +72 -24
package/dist/dom-evaluation.js +310 -84
package/dist/extract-meta.d.ts +98 -0
package/dist/extract-meta.js +75 -0
package/dist/index.d.ts +3 -1
package/dist/index.js +1 -0
package/dist/meta/classify.d.ts +52 -0
package/dist/meta/classify.js +731 -0
package/dist/meta/collect-head.d.ts +63 -0
package/dist/meta/collect-head.js +223 -0
package/dist/meta/id-extractors.d.ts +40 -0
package/dist/meta/id-extractors.js +196 -0
package/dist/meta/keys.d.ts +41 -0
package/dist/meta/keys.js +507 -0
package/dist/meta/parsers.d.ts +74 -0
package/dist/meta/parsers.js +293 -0
package/dist/meta/tag-detection.d.ts +59 -0
package/dist/meta/tag-detection.js +120 -0
package/dist/meta/types.d.ts +874 -0
package/dist/meta/types.js +12 -0
package/dist/scraper.js +15 -13
package/dist/types.d.ts +3 -38
package/package.json +8 -5
package/src/dom-evaluation.spec.ts +301 -73
package/src/dom-evaluation.ts +417 -88
package/src/extract-meta.spec.ts +247 -0
package/src/extract-meta.ts +121 -0
package/src/index.ts +45 -0
package/src/meta/classify.spec.ts +281 -0
package/src/meta/classify.ts +810 -0
package/src/meta/collect-head.ts +247 -0
package/src/meta/id-extractors.spec.ts +69 -0
package/src/meta/id-extractors.ts +206 -0
package/src/meta/keys.ts +568 -0
package/src/meta/parsers.spec.ts +178 -0
package/src/meta/parsers.ts +304 -0
package/src/meta/simple-wappalyzer.d.ts +37 -0
package/src/meta/tag-detection.spec.ts +134 -0
package/src/meta/tag-detection.ts +161 -0
package/src/meta/types.ts +949 -0
package/src/scraper.ts +19 -13
package/src/types.ts +49 -55
package/tsconfig.tsbuildinfo +1 -1

package/src/meta/collect-head.ts ADDED Viewed

@@ -0,0 +1,247 @@
+/**
+ * DOM-side raw `<head>` collector.
+ *
+ * `collectHeadFromDocument` walks a `Document` (Puppeteer page realm or jsdom realm
+ * alike) and produces a serializable {@link RawHeadEntry}[] that
+ * {@link ../meta/classify.ts | classify} can turn into a typed `Meta`.
+ *
+ * WHY this function is realm-agnostic:
+ *
+ * - The Puppeteer path stringifies this function via `Function.prototype.toString`
+ *   and runs it as a `page.evaluate(string)` expression, so any closure over
+ *   module-scope bindings would resolve to `undefined` in the browser realm.
+ * - The jsdom (Node) path calls it directly with the jsdom `Window`. Because
+ *   `HTMLLinkElement` (etc.) in jsdom is a *different class instance* from the
+ *   one in the page realm, `instanceof` only works when the constructor is read
+ *   from the *passed* `window` rather than from bare globals.
+ *
+ * Together those constraints dictate that the function MUST:
+ *
+ * 1. Reference no module-level variables — only its own parameters and inner locals.
+ * 2. Take every HTML class constructor (`HTMLBaseElement`, …) from the passed
+ *    `window` via destructuring instead of relying on ambient globals.
+ * 3. Stay in plain ES syntax (no TS-only constructs that need helper imports).
+ * @module
+ */
+import type { RawHeadEntry } from './types.js';
+/**
+ * Curated list of `window` globals whose presence indicates that a third-party
+ * tag library has been loaded on the page. Surfaced as a single
+ * `kind: 'window-global'` entry so that downstream consumers (e.g. tag-detection)
+ * can cross-reference the script/iframe signals.
+ *
+ * Kept here (rather than in `dom-evaluation.ts`) so the Puppeteer path and the
+ * jsdom path share one source of truth.
+ */
+export const WINDOW_GLOBALS_TO_CHECK: readonly string[] = [
+	'dataLayer',
+	'gtag',
+	'ga',
+	'_gaq',
+	'fbq',
+	'_fbq',
+	'clarity',
+	'_hjSettings',
+	'_hjid',
+	'twq',
+	'ttq',
+	'_linkedin_partner_id',
+	'pintrk',
+	'amplitude',
+	'mixpanel',
+	'analytics',
+	'heap',
+	'posthog',
+	'plausible',
+	'fathom',
+	'_paq',
+	's_account',
+	's',
+	'ym',
+	'UET',
+	'optimizely',
+	'_hsq',
+	'Sentry',
+	'Intercom',
+	'intercomSettings',
+	'drift',
+	'Tawk_API',
+	'zE',
+	'OneTrust',
+	'Cookiebot',
+	'Stripe',
+	'grecaptcha',
+];
+/**
+ * Walks the given window's `Document` and returns a serializable list of raw
+ * head entries.
+ *
+ * Two realms are supported:
+ *
+ * - Browser realm (Puppeteer): the function source is `.toString()`'d and run
+ *   inside the page via `page.evaluate(string)`. Inside the page, `window`
+ *   resolves to the page's global object, so destructured class constructors
+ *   match `instanceof` checks against elements returned from `querySelectorAll`.
+ * - Node realm (jsdom et al.): the caller passes `dom.window` directly. jsdom's
+ *   HTML element prototypes are distinct from the host Node's bare globals, so
+ *   reading the constructors off the passed `window` is what makes `instanceof`
+ *   succeed.
+ *
+ * The function MUST NOT close over any module-scope binding — all data it needs
+ * is reached through its two parameters.
+ * @param window - The window object whose `document` will be inspected. Provides
+ *                 both the DOM tree and the HTML element constructors used for
+ *                 `instanceof` narrowing.
+ * @param knownGlobals - Names of `window` properties that, when present,
+ *                       indicate a third-party tag library is loaded. Required
+ *                       (no default) so the Puppeteer-side string-eval path
+ *                       does not have to inline a default value list.
+ * @returns Serializable list of raw head entries for {@link ../meta/classify.ts | classify}.
+ */
+export function collectHeadFromDocument(
+	window: Window,
+	knownGlobals: readonly string[],
+): RawHeadEntry[] {
+	const document = window.document;
+	// TypeScript's `Window` interface in lib.dom does not directly expose the
+	// HTML element constructors (`HTMLLinkElement`, `HTMLScriptElement`, …)
+	// even though every real window object — browser realm AND jsdom realm —
+	// carries them at runtime. Widening the type here lets us destructure them
+	// uniformly; the runtime values come straight from the passed window, so
+	// the cast is purely cosmetic for TS and erased at compile time.
+	const w = window as Window & {
+		HTMLBaseElement: typeof globalThis.HTMLBaseElement;
+		HTMLMetaElement: typeof globalThis.HTMLMetaElement;
+		HTMLLinkElement: typeof globalThis.HTMLLinkElement;
+		HTMLScriptElement: typeof globalThis.HTMLScriptElement;
+		HTMLIFrameElement: typeof globalThis.HTMLIFrameElement;
+	};
+	const {
+		HTMLBaseElement,
+		HTMLMetaElement,
+		HTMLLinkElement,
+		HTMLScriptElement,
+		HTMLIFrameElement,
+	} = w;
+	const entries: RawHeadEntry[] = [];
+	const html = document.documentElement;
+	entries.push(
+		{
+			kind: 'html',
+			lang: html.lang || undefined,
+			dir: html.dir || undefined,
+			xmlns: html.getAttribute('xmlns') ?? undefined,
+			prefix: html.getAttribute('prefix') ?? undefined,
+			vocab: html.getAttribute('vocab') ?? undefined,
+			typeOf: html.getAttribute('typeof') ?? undefined,
+			itemscope: html.hasAttribute('itemscope') || undefined,
+			itemtype: html.getAttribute('itemtype') ?? undefined,
+			amp: html.hasAttribute('amp') || undefined,
+			lightning: html.hasAttribute('⚡') || undefined,
+		},
+		{ kind: 'title', content: document.title },
+	);
+	for (const base of document.querySelectorAll('base')) {
+		if (!(base instanceof HTMLBaseElement)) continue;
+		entries.push({
+			kind: 'base',
+			href: base.getAttribute('href') ?? undefined,
+			target: base.getAttribute('target') ?? undefined,
+		});
+	}
+	for (const meta of document.querySelectorAll('meta')) {
+		if (!(meta instanceof HTMLMetaElement)) continue;
+		const name = meta.getAttribute('name');
+		const property = meta.getAttribute('property');
+		const httpEquiv = meta.getAttribute('http-equiv');
+		const itemprop = meta.getAttribute('itemprop');
+		const charset = meta.getAttribute('charset');
+		const content = meta.getAttribute('content');
+		const media = meta.getAttribute('media');
+		entries.push({
+			kind: 'meta',
+			name: name ? name.toLowerCase() : undefined,
+			property: property ? property.toLowerCase() : undefined,
+			httpEquiv: httpEquiv ? httpEquiv.toLowerCase() : undefined,
+			itemprop: itemprop ?? undefined,
+			charset: charset ?? undefined,
+			content: content ?? undefined,
+			media: media ?? undefined,
+		});
+	}
+	for (const link of document.querySelectorAll('link[href]')) {
+		if (!(link instanceof HTMLLinkElement)) continue;
+		const relRaw = link.getAttribute('rel') ?? '';
+		const rel = relRaw.toLowerCase().split(/\s+/u).filter(Boolean);
+		entries.push({
+			kind: 'link',
+			rel,
+			href: link.getAttribute('href') ?? '',
+			type: link.getAttribute('type') ?? undefined,
+			media: link.getAttribute('media') ?? undefined,
+			sizes: link.getAttribute('sizes') ?? undefined,
+			title: link.getAttribute('title') ?? undefined,
+			hreflang: link.getAttribute('hreflang') ?? undefined,
+			as: link.getAttribute('as') ?? undefined,
+			crossorigin: link.getAttribute('crossorigin') ?? undefined,
+			color: link.getAttribute('color') ?? undefined,
+			blocking: link.getAttribute('blocking') ?? undefined,
+			imagesrcset: link.getAttribute('imagesrcset') ?? undefined,
+		});
+	}
+	const STRUCTURED_TYPES = new Set([
+		'application/ld+json',
+		'speculationrules',
+		'application/json+oembed',
+		'application/xml+oembed',
+	]);
+	for (const script of document.querySelectorAll('script[type]')) {
+		if (!(script instanceof HTMLScriptElement)) continue;
+		const scriptType = (script.getAttribute('type') ?? '').toLowerCase();
+		if (!STRUCTURED_TYPES.has(scriptType)) continue;
+		const src = script.getAttribute('src') ?? undefined;
+		const text = script.textContent ?? '';
+		const inHead = !!script.closest('head');
+		const inNoscript = !!script.closest('noscript');
+		const location = inHead ? 'head' : inNoscript ? 'noscript' : 'body';
+		entries.push({
+			kind: 'script',
+			scriptType,
+			content: text || undefined,
+			src,
+			location,
+		});
+	}
+	for (const iframe of document.querySelectorAll('iframe[src]')) {
+		if (!(iframe instanceof HTMLIFrameElement)) continue;
+		const src = iframe.getAttribute('src') ?? '';
+		if (!src) continue;
+		const inHead = !!iframe.closest('head');
+		const inNoscript = !!iframe.closest('noscript');
+		const location = inHead ? 'head' : inNoscript ? 'noscript' : 'body';
+		entries.push({ kind: 'iframe', src, location });
+	}
+	const win = window as unknown as Record<string, unknown>;
+	const presentGlobals: string[] = [];
+	for (const name of knownGlobals) {
+		if (win[name] !== undefined) {
+			presentGlobals.push(name);
+		}
+	}
+	if (presentGlobals.length > 0) {
+		entries.push({ kind: 'window-global', names: presentGlobals });
+	}
+	return entries;
+}

package/src/meta/id-extractors.spec.ts ADDED Viewed

@@ -0,0 +1,69 @@
+import { describe, expect, it } from 'vitest';
+import { extractIds } from './id-extractors.js';
+describe('extractIds', () => {
+	it('returns [] for unknown provider', () => {
+		expect(extractIds('NonExistentProvider', '<html></html>')).toEqual([]);
+	});
+	it('extracts GA4 measurement ID from gtag config', () => {
+		const html = `<script>gtag('config', 'G-ABCD1234XY')</script>`;
+		expect(extractIds('Google Analytics', html)).toContain('G-ABCD1234XY');
+	});
+	it('extracts GA4 measurement ID from script src', () => {
+		const html = `<script src="https://www.googletagmanager.com/gtag/js?id=G-XYZW9876AB"></script>`;
+		expect(extractIds('Google Analytics', html)).toContain('G-XYZW9876AB');
+	});
+	it('extracts UA tracking ID', () => {
+		const html = `<script>ga('create', 'UA-12345678-1', 'auto');</script>`;
+		expect(extractIds('Google Analytics', html)).toContain('UA-12345678-1');
+	});
+	it('extracts GTM container ID from src and inline', () => {
+		const html = `
+			<script src="https://www.googletagmanager.com/gtm.js?id=GTM-ABCD123"></script>
+			<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-ABCD123"></iframe></noscript>
+		`;
+		const ids = extractIds('Google Tag Manager', html);
+		expect(ids).toContain('GTM-ABCD123');
+		expect(ids.length).toBe(1);
+	});
+	it('extracts Facebook Pixel ID from fbq init', () => {
+		const html = `<script>fbq('init', '123456789012345');</script>`;
+		expect(extractIds('Facebook Pixel', html)).toContain('123456789012345');
+	});
+	it('extracts Hotjar site ID from inline', () => {
+		const html = `<script>(function(h,o,t,j,a,r){h.hj=h.hj||function(){};h._hjSettings={hjid:1234567,hjsv:6};})(window,document)</script>`;
+		expect(extractIds('Hotjar', html)).toContain('1234567');
+	});
+	it('extracts Microsoft Clarity project ID from src', () => {
+		const html = `<script src="https://www.clarity.ms/tag/abc123xyz"></script>`;
+		expect(extractIds('Microsoft Clarity', html)).toContain('abc123xyz');
+	});
+	it('extracts TikTok pixel ID from ttq.load', () => {
+		const html = `<script>ttq.load('ABCDEFGH12345678')</script>`;
+		expect(extractIds('TikTok Pixel', html)).toContain('ABCDEFGH12345678');
+	});
+	it('deduplicates IDs across multiple patterns', () => {
+		const html = `
+			<script src="https://www.googletagmanager.com/gtag/js?id=G-DUP12345A"></script>
+			<script>gtag('config', 'G-DUP12345A');</script>
+		`;
+		const ids = extractIds('Google Analytics', html);
+		const dupCount = ids.filter((id) => id === 'G-DUP12345A').length;
+		expect(dupCount).toBe(1);
+	});
+	it('extracts Yandex Metrica counter ID from ym init', () => {
+		const html = `<script>ym(12345678, 'init', { clickmap:true });</script>`;
+		expect(extractIds('Yandex Metrica', html)).toContain('12345678');
+	});
+});

package/src/meta/id-extractors.ts ADDED Viewed

@@ -0,0 +1,206 @@
+/**
+ * Provider-specific real-ID extraction rules.
+ *
+ * `simple-wappalyzer` identifies the *technology* (e.g., "Google Analytics") but
+ * does not surface the actual account/measurement ID. We layer real-ID
+ * extraction on top: for each detected provider, apply the registered regex
+ * over the page HTML and surface what we find.
+ *
+ * Provider keys must match the names produced by `simple-wappalyzer` exactly;
+ * these in turn track `wappalyzer-core@6` (the MIT-licensed fingerprint set).
+ *
+ * Keep the table **manually maintained**, not generated from Wappalyzer data.
+ * @module
+ */
+export type IdExtractor = {
+	/**
+	 * Each regex MUST contain at most one capturing group; the captured text
+	 * becomes the ID. Patterns without a capturing group fall back to
+	 * `match[0]`.
+	 */
+	readonly patterns: readonly RegExp[];
+};
+/**
+ * Lookup table keyed by Wappalyzer provider name.
+ *
+ * When extending: keep regexes anchored on stable, high-signal substrings
+ * (the surrounding API call, not just the bare ID character class). Otherwise
+ * the same regex will hit unrelated strings on pages that happen to share the
+ * shape (e.g., AWS ARNs containing `GA-...`).
+ */
+export const ID_EXTRACTORS: Record<string, IdExtractor> = {
+	'Google Analytics': {
+		patterns: [
+			/gtag\(\s*['"]config['"]\s*,\s*['"](G-[A-Z0-9]{4,20})['"]/g,
+			/googletagmanager\.com\/gtag\/js\?id=(G-[A-Z0-9]{4,20})/g,
+			/\bga\(\s*['"]create['"]\s*,\s*['"](UA-\d{4,10}-\d{1,4})['"]/g,
+			/['"](UA-\d{4,10}-\d{1,4})['"]/g,
+		],
+	},
+	'Google Tag Manager': {
+		patterns: [
+			/googletagmanager\.com\/(?:gtm|ns)\.[a-z]+\?id=(GTM-[A-Z0-9]{4,12})/g,
+			/['"](GTM-[A-Z0-9]{4,12})['"]/g,
+		],
+	},
+	'Google Ads': {
+		patterns: [/['"](AW-\d{4,12})['"]/g],
+	},
+	'Facebook Pixel': {
+		patterns: [
+			/fbq\(\s*['"]init['"]\s*,\s*['"](\d{6,20})['"]/g,
+			/connect\.facebook\.net\/[^"']+\/fbevents\.js\D*(\d{6,20})/g,
+		],
+	},
+	Hotjar: {
+		patterns: [
+			/hjid\s*[:=]\s*(\d{4,10})/g,
+			/static\.hotjar\.com\/c\/hotjar-(\d{4,10})\.js/g,
+		],
+	},
+	'Microsoft Clarity': {
+		patterns: [
+			/clarity\.ms\/tag\/([a-z0-9]{6,20})/g,
+			/clarity\(\s*['"]start['"]\s*,\s*['"]([a-z0-9]{6,20})['"]/gi,
+		],
+	},
+	Mixpanel: {
+		patterns: [/mixpanel\.init\(\s*['"]([a-f0-9]{16,40})['"]/g],
+	},
+	Segment: {
+		patterns: [
+			/analytics\.load\(\s*['"]([a-zA-Z0-9]{8,40})['"]/g,
+			/cdn\.segment\.com\/analytics\.js\/v1\/([a-zA-Z0-9]{8,40})/g,
+		],
+	},
+	Amplitude: {
+		patterns: [
+			/amplitude\.init\(\s*['"]([a-f0-9]{16,40})['"]/g,
+			/getInstance\(\)\.init\(\s*['"]([a-f0-9]{16,40})['"]/g,
+		],
+	},
+	Heap: {
+		patterns: [
+			/heap\.load\(\s*['"](\d{6,20})['"]/g,
+			/heap\.appid\s*=\s*['"](\d{6,20})['"]/g,
+		],
+	},
+	PostHog: {
+		patterns: [/posthog\.init\(\s*['"]([\w-]{16,80})['"]/g],
+	},
+	Plausible: {
+		patterns: [/plausible\.io\/js\/script\.js[?&]domain=([a-zA-Z0-9.,-]+)/g],
+	},
+	Matomo: {
+		patterns: [
+			/_paq\.push\(\s*\[\s*['"]setSiteId['"]\s*,\s*['"]?(\d{1,6})['"]?\s*\]/g,
+			/matomo\.php\?siteId=(\d{1,6})/g,
+		],
+	},
+	'Adobe Analytics': {
+		patterns: [
+			/s_account\s*=\s*['"]([a-z0-9,]{3,50})['"]/gi,
+			/s\.account\s*=\s*['"]([a-z0-9,]{3,50})['"]/gi,
+		],
+	},
+	'Yandex Metrica': {
+		patterns: [/ym\(\s*(\d{6,12})\s*,\s*['"]init['"]/g],
+	},
+	'LinkedIn Insight Tag': {
+		patterns: [/_linkedin_partner_id\s*=\s*['"](\d{4,10})['"]/g],
+	},
+	'Twitter Ads': {
+		patterns: [/twq\(\s*['"]config['"]\s*,\s*['"]([a-z0-9]{4,12})['"]/g],
+	},
+	'TikTok Pixel': {
+		patterns: [
+			/ttq\.load\(\s*['"]([A-Z0-9]{12,30})['"]/g,
+			/tiktok\.com\/i18n\/pixel\/events\.js\?sdkid=([A-Z0-9]{12,30})/g,
+		],
+	},
+	'Pinterest Tag': {
+		patterns: [/pintrk\(\s*['"]load['"]\s*,\s*['"](\d{12,20})['"]/g],
+	},
+	'Bing Universal Event Tracking': {
+		patterns: [
+			/setAttribute\(\s*['"]data-tag['"]\s*,\s*['"](\d{6,20})['"]/g,
+			/UET\(\{\s*ti:\s*['"](\d{6,20})['"]/g,
+		],
+	},
+	Optimizely: {
+		patterns: [/cdn\.optimizely\.com\/js\/(\d{6,20})\.js/g],
+	},
+	HubSpot: {
+		patterns: [
+			/js\.hs-?scripts\.com\/(\d{4,12})\.js/g,
+			/js\.hubspot\.com\/web-interactives\/v1\/embeds\/(\d{4,12})/g,
+		],
+	},
+	Sentry: {
+		patterns: [
+			/(https:\/\/[a-f0-9]+@[a-zA-Z0-9.-]+\.ingest\.sentry\.io\/\d+)/g,
+			/(https:\/\/[a-f0-9]+@[a-zA-Z0-9.-]+\.sentry\.io\/\d+)/g,
+		],
+	},
+	Intercom: {
+		patterns: [
+			/intercomSettings\s*=\s*\{[^}]*?app_id:\s*['"]([a-z0-9]{4,10})['"]/g,
+			/Intercom\(\s*['"]boot['"]\s*,\s*\{[^}]*?app_id:\s*['"]([a-z0-9]{4,10})['"]/g,
+		],
+	},
+	Drift: {
+		patterns: [/drift\.load\(\s*['"]([a-z0-9]{6,30})['"]/g],
+	},
+	'Tawk.to': {
+		patterns: [/embed\.tawk\.to\/([a-f0-9]{16,40})/g],
+	},
+	'Zendesk Chat': {
+		patterns: [/static\.zdassets\.com\/ekr\/snippet\.js\?key=([a-f0-9-]{16,40})/g],
+	},
+	Cookiebot: {
+		patterns: [/consent\.cookiebot\.com\/uc\.js[^"']*?cbid=([a-f0-9-]{16,40})/g],
+	},
+	OneTrust: {
+		patterns: [/dataDomain['"=]\s*['"]?([a-z0-9-]{16,80})['"]?/gi],
+	},
+	Stripe: {
+		patterns: [/js\.stripe\.com\/v\d+\//g],
+	},
+	'Google reCAPTCHA': {
+		patterns: [/google\.com\/recaptcha\/api\.js[^"']*?(?:render=)?([\w-]{20,60})/g],
+	},
+	'Facebook for WordPress': {
+		patterns: [/fbq\(\s*['"]init['"]\s*,\s*['"](\d{6,20})['"]/g],
+	},
+};
+/**
+ * Extracts real IDs for `provider` from the page HTML.
+ *
+ * Returns a de-duplicated, insertion-ordered list of IDs. Returns `[]` for
+ * unknown providers (so callers can compose freely).
+ * @param provider
+ * @param html
+ */
+export function extractIds(provider: string, html: string): string[] {
+	const extractor = ID_EXTRACTORS[provider];
+	if (!extractor) return [];
+	const seen = new Set<string>();
+	const result: string[] = [];
+	for (const pattern of extractor.patterns) {
+		// Patterns must be `g`-flagged for `matchAll` to work without re-creating.
+		const safe = pattern.flags.includes('g')
+			? pattern
+			: new RegExp(pattern.source, pattern.flags + 'g');
+		for (const match of html.matchAll(safe)) {
+			const id = match[1] ?? match[0];
+			if (id && !seen.has(id)) {
+				seen.add(id);
+				result.push(id);
+			}
+		}
+	}
+	return result;
+}