@d-zero/beholder 2.1.6 → 3.1.0

package/src/meta/parsers.spec.ts

@@ -0,0 +1,178 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+	capJsonLdContent,
+	JSON_LD_PER_ENTRY_LIMIT,
+	normalizeValue,
+	parseFormatDetection,
+	parseJsonLd,
+	parseRefresh,
+	parseReferrer,
+	parseRobots,
+	parseViewport,
+} from './parsers.js';
+
+describe('parseViewport', () => {
+	it('parses width=device-width and initial-scale', () => {
+		const result = parseViewport('width=device-width, initial-scale=1.0');
+		expect(result.raw).toBe('width=device-width, initial-scale=1.0');
+		expect(result.width).toBe('device-width');
+		expect(result.initialScale).toBe(1);
+	});
+
+	it('parses user-scalable=no as boolean false', () => {
+		const result = parseViewport('user-scalable=no');
+		expect(result.userScalable).toBe(false);
+	});
+
+	it('parses minimum-scale/maximum-scale as numbers', () => {
+		const result = parseViewport('minimum-scale=0.5, maximum-scale=2');
+		expect(result.minimumScale).toBe(0.5);
+		expect(result.maximumScale).toBe(2);
+	});
+
+	it('preserves viewport-fit and interactive-widget literally', () => {
+		const result = parseViewport('viewport-fit=cover, interactive-widget=resizes-visual');
+		expect(result.viewportFit).toBe('cover');
+		expect(result.interactiveWidget).toBe('resizes-visual');
+	});
+
+	it('keeps raw on unrecognizable input', () => {
+		const result = parseViewport('garbage');
+		expect(result.raw).toBe('garbage');
+		expect(result.width).toBeUndefined();
+	});
+});
+
+describe('parseRobots', () => {
+	it('flags noindex/nofollow/noarchive', () => {
+		const result = parseRobots('noindex, NOFOLLOW, noarchive');
+		expect(result.noindex).toBe(true);
+		expect(result.nofollow).toBe(true);
+		expect(result.noarchive).toBe(true);
+	});
+
+	it('extracts max-snippet, max-image-preview, max-video-preview', () => {
+		const result = parseRobots(
+			'max-snippet:50, max-image-preview:large, max-video-preview:120',
+		);
+		expect(result.maxSnippet).toBe(50);
+		expect(result.maxImagePreview).toBe('large');
+		expect(result.maxVideoPreview).toBe(120);
+	});
+
+	it('extracts unavailable_after', () => {
+		const result = parseRobots('unavailable_after:2026-12-31');
+		expect(result.unavailableAfter).toBe('2026-12-31');
+	});
+
+	it('flags index/follow positives', () => {
+		const result = parseRobots('index, follow');
+		expect(result.index).toBe(true);
+		expect(result.follow).toBe(true);
+	});
+});
+
+describe('parseReferrer', () => {
+	it('flags strict-origin-when-cross-origin', () => {
+		const result = parseReferrer('strict-origin-when-cross-origin');
+		expect(result.strictOriginWhenCrossOrigin).toBe(true);
+	});
+
+	it('flags no-referrer', () => {
+		const result = parseReferrer('no-referrer');
+		expect(result.noReferrer).toBe(true);
+	});
+});
+
+describe('parseFormatDetection', () => {
+	it('parses telephone=no, address=no', () => {
+		const result = parseFormatDetection('telephone=no, address=no');
+		expect(result.telephone).toBe(false);
+		expect(result.address).toBe(false);
+	});
+
+	it('parses date=no via semicolon separator', () => {
+		const result = parseFormatDetection('telephone=no; date=no');
+		expect(result.telephone).toBe(false);
+		expect(result.date).toBe(false);
+	});
+});
+
+describe('parseRefresh', () => {
+	it('parses seconds and url', () => {
+		const result = parseRefresh('5; url=https://example.com/');
+		expect(result.seconds).toBe(5);
+		expect(result.url).toBe('https://example.com/');
+	});
+
+	it('handles missing url', () => {
+		const result = parseRefresh('30');
+		expect(result.seconds).toBe(30);
+		expect(result.url).toBeUndefined();
+	});
+
+	it('strips surrounding quotes in url', () => {
+		const result = parseRefresh(`0; url='https://example.com/'`);
+		expect(result.url).toBe('https://example.com/');
+	});
+});
+
+describe('parseJsonLd', () => {
+	it('returns parsed object on valid JSON', () => {
+		const entry = parseJsonLd('{"@type":"WebSite","name":"Site"}');
+		expect(entry.parsed).toEqual({ '@type': 'WebSite', name: 'Site' });
+		expect(entry.parseError).toBeUndefined();
+	});
+
+	it('records parseError on invalid JSON', () => {
+		const entry = parseJsonLd('{ not valid }');
+		expect(entry.parsed).toBeUndefined();
+		expect(entry.parseError).toBeDefined();
+	});
+});
+
+describe('normalizeValue', () => {
+	it('passes through string by default', () => {
+		expect(normalizeValue('hello')).toBe('hello');
+		expect(normalizeValue('hello', 'string')).toBe('hello');
+	});
+
+	it('boolean-yes maps yes/no', () => {
+		expect(normalizeValue('yes', 'boolean-yes')).toBe(true);
+		expect(normalizeValue('no', 'boolean-yes')).toBe(false);
+		expect(normalizeValue('unknown', 'boolean-yes')).toBe('unknown');
+	});
+
+	it('boolean-on maps on/off/true/false/1/0', () => {
+		expect(normalizeValue('on', 'boolean-on')).toBe(true);
+		expect(normalizeValue('off', 'boolean-on')).toBe(false);
+		expect(normalizeValue('true', 'boolean-on')).toBe(true);
+		expect(normalizeValue('0', 'boolean-on')).toBe(false);
+	});
+
+	it('boolean-true maps true/false only', () => {
+		expect(normalizeValue('true', 'boolean-true')).toBe(true);
+		expect(normalizeValue('false', 'boolean-true')).toBe(false);
+		expect(normalizeValue('1', 'boolean-true')).toBe('1');
+	});
+
+	it('number parses floats and falls back to raw', () => {
+		expect(normalizeValue('3.14', 'number')).toBe(3.14);
+		expect(normalizeValue('NaN-ish', 'number')).toBe('NaN-ish');
+	});
+});
+
+describe('capJsonLdContent', () => {
+	it('returns content unchanged when under the limit', () => {
+		const result = capJsonLdContent('{}');
+		expect(result).toEqual({ content: '{}', truncated: false });
+	});
+
+	it('truncates content over the per-entry limit', () => {
+		const big = 'a'.repeat(JSON_LD_PER_ENTRY_LIMIT + 100);
+		const result = capJsonLdContent(big);
+		expect(result.truncated).toBe(true);
+		expect(result.content.length).toBe(JSON_LD_PER_ENTRY_LIMIT);
+	});
+});

package/src/meta/parsers.ts

@@ -0,0 +1,304 @@
+/**
+ * Value normalizers used by `classify()` to turn raw `content` strings into
+ * structured objects (viewport, robots, format-detection, etc.).
+ *
+ * Each parser is a pure function that takes the raw `content` string and
+ * returns a normalized structure. They never throw; on unrecognizable input
+ * they fall back to keeping the `raw` field only.
+ * @module
+ */
+
+import type { KeyTransform } from './keys.js';
+import type {
+	FormatDetectionMeta,
+	HttpEquivRefresh,
+	JsonLdEntry,
+	ReferrerMeta,
+	RobotsMeta,
+	ViewportMeta,
+} from './types.js';
+
+/**
+ * Parses `<meta name="viewport">` content into a structured `ViewportMeta`.
+ * @param raw
+ * @example parseViewport('width=device-width, initial-scale=1.0')
+ *   → { raw: '...', width: 'device-width', initialScale: 1 }
+ */
+export function parseViewport(raw: string): ViewportMeta {
+	const meta: ViewportMeta = { raw };
+	for (const part of raw.split(',')) {
+		const split = part.split('=');
+		const keyRaw = split[0] ?? '';
+		const valueRaw = split[1] ?? '';
+		const key = keyRaw.trim().toLowerCase();
+		const value = valueRaw.trim();
+		if (!key) continue;
+		switch (key) {
+			case 'width': {
+				meta.width = value;
+				break;
+			}
+			case 'height': {
+				meta.height = value;
+				break;
+			}
+			case 'initial-scale': {
+				const n = Number.parseFloat(value);
+				if (!Number.isNaN(n)) meta.initialScale = n;
+				break;
+			}
+			case 'minimum-scale': {
+				const n = Number.parseFloat(value);
+				if (!Number.isNaN(n)) meta.minimumScale = n;
+				break;
+			}
+			case 'maximum-scale': {
+				const n = Number.parseFloat(value);
+				if (!Number.isNaN(n)) meta.maximumScale = n;
+				break;
+			}
+			case 'user-scalable': {
+				const lower = value.toLowerCase();
+				if (lower === 'no' || lower === '0') meta.userScalable = false;
+				else if (lower === 'yes' || lower === '1') meta.userScalable = true;
+				else meta.userScalable = value;
+				break;
+			}
+			case 'viewport-fit': {
+				meta.viewportFit = value;
+				break;
+			}
+			case 'interactive-widget': {
+				meta.interactiveWidget = value;
+				break;
+			}
+		}
+	}
+	return meta;
+}
+
+const ROBOTS_BOOLEAN_FLAGS = new Set<keyof RobotsMeta>([
+	'index',
+	'noindex',
+	'follow',
+	'nofollow',
+	'none',
+	'all',
+	'noarchive',
+	'nosnippet',
+	'noimageindex',
+	'nocache',
+	'notranslate',
+	'noodp',
+	'noydir',
+	'indexifembedded',
+]);
+
+/**
+ * Parses `<meta name="robots">` content into a structured `RobotsMeta`.
+ * @param raw
+ * @example parseRobots('noindex, max-snippet:50, unavailable_after:2026-01-01')
+ *   → { raw: '...', noindex: true, maxSnippet: 50, unavailableAfter: '2026-01-01' }
+ */
+export function parseRobots(raw: string): RobotsMeta {
+	const meta: RobotsMeta = { raw };
+	for (const token of raw.split(',')) {
+		const trimmed = token.trim().toLowerCase();
+		if (!trimmed) continue;
+
+		if (ROBOTS_BOOLEAN_FLAGS.has(trimmed as keyof RobotsMeta)) {
+			(meta as Record<string, unknown>)[trimmed] = true;
+			continue;
+		}
+
+		const colonIndex = trimmed.indexOf(':');
+		if (colonIndex === -1) {
+			continue;
+		}
+		const key = trimmed.slice(0, colonIndex).trim();
+		const value = token.slice(token.indexOf(':') + 1).trim();
+		switch (key) {
+			case 'max-snippet': {
+				const n = Number.parseInt(value, 10);
+				if (!Number.isNaN(n)) meta.maxSnippet = n;
+				break;
+			}
+			case 'max-image-preview': {
+				meta.maxImagePreview = value;
+				break;
+			}
+			case 'max-video-preview': {
+				const n = Number.parseInt(value, 10);
+				if (!Number.isNaN(n)) meta.maxVideoPreview = n;
+				break;
+			}
+			case 'unavailable_after':
+			case 'unavailable-after': {
+				meta.unavailableAfter = value;
+				break;
+			}
+		}
+	}
+	return meta;
+}
+
+const REFERRER_POLICY_KEYS: Record<string, keyof ReferrerMeta> = {
+	'no-referrer': 'noReferrer',
+	origin: 'origin',
+	'origin-when-cross-origin': 'originWhenCrossOrigin',
+	'strict-origin': 'strictOrigin',
+	'strict-origin-when-cross-origin': 'strictOriginWhenCrossOrigin',
+	'unsafe-url': 'unsafeUrl',
+	'same-origin': 'sameOrigin',
+	'no-referrer-when-downgrade': 'noReferrerWhenDowngrade',
+};
+
+/**
+ * Parses `<meta name="referrer">` content into a structured `ReferrerMeta`.
+ * @param raw
+ */
+export function parseReferrer(raw: string): ReferrerMeta {
+	const meta: ReferrerMeta = { raw };
+	const key = REFERRER_POLICY_KEYS[raw.trim().toLowerCase()];
+	if (key) {
+		(meta as Record<string, unknown>)[key] = true;
+	}
+	return meta;
+}
+
+/**
+ * Parses `<meta name="format-detection">` content (e.g. `'telephone=no, address=no'`).
+ * @param raw
+ */
+export function parseFormatDetection(raw: string): FormatDetectionMeta {
+	const meta: FormatDetectionMeta = { raw };
+	for (const part of raw.split(/[,;]/)) {
+		const split = part.split('=');
+		const keyRaw = split[0] ?? '';
+		const valueRaw = split[1] ?? '';
+		const key = keyRaw.trim().toLowerCase();
+		const value = valueRaw.trim().toLowerCase();
+		if (!key) continue;
+		const enabled = value !== 'no' && value !== 'false' && value !== '0';
+		switch (key) {
+			case 'telephone': {
+				meta.telephone = enabled;
+				break;
+			}
+			case 'email': {
+				meta.email = enabled;
+				break;
+			}
+			case 'address': {
+				meta.address = enabled;
+				break;
+			}
+			case 'date': {
+				meta.date = enabled;
+				break;
+			}
+		}
+	}
+	return meta;
+}
+
+/**
+ * Parses `<meta http-equiv="refresh">` content (e.g. `'5; url=https://...'`).
+ * @param raw
+ */
+export function parseRefresh(raw: string): HttpEquivRefresh {
+	const refresh: HttpEquivRefresh = { raw };
+	const split = raw.split(';');
+	const secondsRaw = split[0] ?? '';
+	const rest = split.slice(1).join(';');
+	const seconds = Number.parseFloat(secondsRaw.trim());
+	if (!Number.isNaN(seconds)) {
+		refresh.seconds = seconds;
+	}
+	const urlMatch = /url\s*=\s*(.+)/i.exec(rest);
+	if (urlMatch?.[1]) {
+		refresh.url = urlMatch[1].trim().replaceAll(/^['"]|['"]$/g, '');
+	}
+	return refresh;
+}
+
+/**
+ * Parses a `<script type="application/ld+json">` (or speculationrules) body
+ * into a {@link JsonLdEntry}. On parse failure, the entry preserves the `raw`
+ * text and records the error message in `parseError`.
+ * @param content
+ */
+export function parseJsonLd(content: string): JsonLdEntry {
+	const raw = content;
+	try {
+		const parsed: unknown = JSON.parse(content);
+		return { raw, parsed };
+	} catch (error) {
+		const parseError = error instanceof Error ? error.message : String(error);
+		return { raw, parseError };
+	}
+}
+
+/**
+ * Normalizes a string value according to a {@link KeyTransform}.
+ *
+ * `'boolean-yes'`: `'yes'` → `true`, `'no'` → `false`, anything else → raw string
+ * `'boolean-on'`: `'on'`/`'true'`/`'1'` → `true`, `'off'`/`'false'`/`'0'` → `false`, else raw
+ * `'boolean-true'`: `'true'` → `true`, `'false'` → `false`, else raw
+ * `'number'`: parsed via `Number.parseFloat`, falls back to raw on NaN
+ * `'string'` (default): returns the value unchanged
+ * @param value
+ * @param transform
+ */
+export function normalizeValue(
+	value: string,
+	transform: KeyTransform | undefined,
+): string | number | boolean {
+	if (!transform || transform === 'string') {
+		return value;
+	}
+	const lower = value.trim().toLowerCase();
+	switch (transform) {
+		case 'boolean-yes': {
+			if (lower === 'yes') return true;
+			if (lower === 'no') return false;
+			return value;
+		}
+		case 'boolean-on': {
+			if (lower === 'on' || lower === 'true' || lower === '1') return true;
+			if (lower === 'off' || lower === 'false' || lower === '0') return false;
+			return value;
+		}
+		case 'boolean-true': {
+			if (lower === 'true') return true;
+			if (lower === 'false') return false;
+			return value;
+		}
+		case 'number': {
+			const n = Number.parseFloat(value);
+			return Number.isNaN(n) ? value : n;
+		}
+	}
+}
+
+/**
+ * JSON-LD / speculationrules content size caps (bytes). Above these sizes the
+ * content is truncated and a `truncated` marker is emitted via `parseError`.
+ */
+export const JSON_LD_PER_ENTRY_LIMIT = 200_000;
+export const JSON_LD_TOTAL_LIMIT = 1_000_000;
+
+/**
+ * Caps a single JSON-LD entry's raw content to {@link JSON_LD_PER_ENTRY_LIMIT}.
+ * Returns the (possibly truncated) entry and a `truncated` flag.
+ * @param content
+ */
+export function capJsonLdContent(content: string): {
+	content: string;
+	truncated: boolean;
+} {
+	if (content.length <= JSON_LD_PER_ENTRY_LIMIT) {
+		return { content, truncated: false };
+	}
+	return { content: content.slice(0, JSON_LD_PER_ENTRY_LIMIT), truncated: true };
+}

package/src/meta/simple-wappalyzer.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Ambient type declarations for `simple-wappalyzer` (no upstream types).
+ *
+ * Mirrors the runtime shape verified against the installed
+ * `simple-wappalyzer@1.1.99` (`node_modules/simple-wappalyzer/src/index.js`):
+ * the module exports a single async function taking `{ url, headers, html }`
+ * and returning the resolved Wappalyzer technology list.
+ *
+ * Only the subset of fields actually consumed in {@link tag-detection.ts} is
+ * declared; the runtime value may have more keys (icon, website, etc.) which
+ * we ignore.
+ */
+declare module 'simple-wappalyzer' {
+	export type WappalyzerCategory = {
+		readonly id?: number;
+		readonly slug?: string;
+		readonly name?: string;
+	};
+
+	export type WappalyzerDetection = {
+		readonly name: string;
+		readonly version?: string;
+		readonly confidence?: number;
+		readonly icon?: string;
+		readonly website?: string;
+		readonly categories?: readonly WappalyzerCategory[];
+	};
+
+	export type WappalyzerInput = {
+		readonly url: string;
+		readonly html: string;
+		readonly headers?: Record<string, string>;
+	};
+
+	const wappalyzer: (input: WappalyzerInput) => Promise<WappalyzerDetection[]>;
+	export default wappalyzer;
+}

package/src/meta/tag-detection.spec.ts

@@ -0,0 +1,134 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { assembleTagsMeta, detectTags, EMPTY_TAGS_META } from './tag-detection.js';
+
+vi.mock('simple-wappalyzer', () => ({
+	default: vi.fn(),
+}));
+
+const importedModule = await import('simple-wappalyzer');
+const wappalyzerMock = importedModule.default as unknown as ReturnType<typeof vi.fn>;
+
+describe('assembleTagsMeta', () => {
+	it('groups detections under their Wappalyzer categories', () => {
+		const tags = assembleTagsMeta(
+			[
+				{
+					name: 'Google Analytics',
+					version: 'GA4',
+					confidence: 100,
+					categories: [{ name: 'Analytics' }],
+				},
+				{
+					name: 'Google Tag Manager',
+					confidence: 100,
+					categories: [{ name: 'Tag Managers' }, { name: 'Analytics' }],
+				},
+			],
+			'',
+		);
+		expect(tags.detected.Analytics).toBeDefined();
+		expect(tags.detected.Analytics?.['Google Analytics']?.version).toBe('GA4');
+		expect(tags.detected['Tag Managers']?.['Google Tag Manager']).toBeDefined();
+		expect(tags.detected.Analytics?.['Google Tag Manager']).toBeDefined();
+	});
+
+	it('attaches extracted IDs to detail and emits one entry per ID', () => {
+		const html = `<script>gtag('config', 'G-XYZ123')</script><script>gtag('config', 'G-AAA999')</script>`;
+		const tags = assembleTagsMeta(
+			[
+				{
+					name: 'Google Analytics',
+					categories: [{ name: 'Analytics' }],
+				},
+			],
+			html,
+		);
+		expect(tags.detected.Analytics?.['Google Analytics']?.ids).toEqual([
+			'G-XYZ123',
+			'G-AAA999',
+		]);
+		const providerEntries = tags.entries.filter((e) => e.provider === 'Google Analytics');
+		expect(providerEntries.map((e) => e.id)).toEqual(['G-XYZ123', 'G-AAA999']);
+	});
+
+	it('emits one entry without id when no IDs are extracted', () => {
+		const tags = assembleTagsMeta(
+			[
+				{
+					name: 'jQuery',
+					version: '3.6.0',
+					categories: [{ name: 'JavaScript Libraries' }],
+				},
+			],
+			'<html></html>',
+		);
+		expect(tags.entries).toHaveLength(1);
+		expect(tags.entries[0]?.id).toBeUndefined();
+		expect(tags.entries[0]?.version).toBe('3.6.0');
+	});
+
+	it('falls back to "Other" category when no categories are present', () => {
+		const tags = assembleTagsMeta([{ name: 'Unknown', categories: [] }], '<html></html>');
+		expect(tags.detected['Other']?.['Unknown']).toBeDefined();
+	});
+
+	it('skips detections without a name', () => {
+		const tags = assembleTagsMeta(
+			[{ name: '', categories: [{ name: 'Analytics' }] }],
+			'<html></html>',
+		);
+		expect(tags.entries).toHaveLength(0);
+		expect(Object.keys(tags.detected)).toHaveLength(0);
+	});
+});
+
+describe('detectTags', () => {
+	it('falls back to empty TagsMeta when simple-wappalyzer throws', async () => {
+		wappalyzerMock.mockRejectedValueOnce(new Error('wappalyzer boom'));
+		const result = await detectTags({
+			url: 'https://example.com/',
+			html: '<html></html>',
+		});
+		expect(result).toEqual(EMPTY_TAGS_META);
+	});
+
+	it('falls back to empty TagsMeta when simple-wappalyzer returns non-array', async () => {
+		wappalyzerMock.mockResolvedValueOnce(null as unknown as never);
+		const result = await detectTags({
+			url: 'https://example.com/',
+			html: '<html></html>',
+		});
+		expect(result).toEqual(EMPTY_TAGS_META);
+	});
+
+	it('passes detections through assembleTagsMeta', async () => {
+		wappalyzerMock.mockResolvedValueOnce([
+			{
+				name: 'Google Analytics',
+				version: 'GA4',
+				categories: [{ name: 'Analytics' }],
+			},
+		] as never);
+		const result = await detectTags({
+			url: 'https://example.com/',
+			html: `<script>gtag('config', 'G-XYZ123')</script>`,
+		});
+		expect(result.entries).toHaveLength(1);
+		expect(result.entries[0]?.id).toBe('G-XYZ123');
+	});
+
+	it('normalizes headers to lowercase before calling wappalyzer', async () => {
+		wappalyzerMock.mockResolvedValueOnce([] as never);
+		await detectTags({
+			url: 'https://example.com/',
+			html: '<html></html>',
+			headers: { 'Content-Type': 'text/html', 'X-Custom': ['a', 'b'] },
+		});
+		const arg = wappalyzerMock.mock.calls.at(-1)?.[0] as {
+			headers?: Record<string, string>;
+		};
+		expect(arg?.headers?.['content-type']).toBe('text/html');
+		expect(arg?.headers?.['x-custom']).toBe('a, b');
+	});
+});