npm - @d-zero/beholder - Versions diffs - 3.0.0 → 3.1.0 - Mend

@d-zero/beholder 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (16) hide show

package/CHANGELOG.md +6 -0
package/README.md +26 -0
package/dist/dom-evaluation.js +20 -152
package/dist/extract-meta.d.ts +98 -0
package/dist/extract-meta.js +75 -0
package/dist/index.d.ts +2 -0
package/dist/index.js +1 -0
package/dist/meta/collect-head.d.ts +63 -0
package/dist/meta/collect-head.js +223 -0
package/package.json +5 -3
package/src/dom-evaluation.ts +20 -161
package/src/extract-meta.spec.ts +247 -0
package/src/extract-meta.ts +121 -0
package/src/index.ts +2 -0
package/src/meta/collect-head.ts +247 -0
package/tsconfig.tsbuildinfo +1 -1

package/CHANGELOG.md CHANGED Viewed

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+# [3.1.0](https://github.com/d-zero-dev/tools/compare/@d-zero/beholder@3.0.0...@d-zero/beholder@3.1.0) (2026-06-17)
+### Features
+- **beholder:** expose extractMetaFromDocument for jsdom-backed meta extraction ([a56e21c](https://github.com/d-zero-dev/tools/commit/a56e21c17dcc1e542595a596074c5d8e659c1168))
 # [3.0.0](https://github.com/d-zero-dev/tools/compare/@d-zero/beholder@2.1.6...@d-zero/beholder@3.0.0) (2026-06-16)
 ### Bug Fixes

package/README.md CHANGED Viewed

@@ -32,3 +32,29 @@ if (result.type === 'success') {
 ```
 設計判断（イベントではなく戻り値で返す理由、`page` のライフサイクル責務、リトライ機構など）は `src/scraper.ts` の JSDoc を参照。
+## DOM 文字列からメタ抽出（Puppeteer なし）
+HTML 文字列を jsdom などでパースしてから `Meta` を取り出したい場合、`extractMetaFromDocument` を使う。`Scraper` が内部で呼ぶ `collectHead → detectTags → classify` パイプラインと同じ実装を再利用するため、戻り値の `Meta` 形状は `scrapeStart` と同一。DOM ライブラリ（jsdom 等）はユーザランドの責務。
+```ts
+import { extractMetaFromDocument } from '@d-zero/beholder';
+import { JSDOM } from 'jsdom';
+const url = 'https://example.com/';
+const html = await (await fetch(url)).text();
+const dom = new JSDOM(html, { url });
+// `as unknown as Window` は jsdom の `DOMWindow` 型が lib.dom の `Window` と
+// 構造的に完全一致しないための型キャスト。ランタイムでは互換。
+const meta = await extractMetaFromDocument(dom.window as unknown as Window, {
+	url,
+	html,
+});
+console.log(meta.title);
+console.log(meta.og?.image);
+console.log(meta.tags.entries);
+```
+`context.html` を省略すると `window.document.documentElement.outerHTML` がフォールバックされる。ただし Wappalyzer の HTML パターンはスクリプト実行前の生 HTML に合わせて作られているので、可能なら取得直後の HTML 文字列を明示的に渡す方が検出が安定する。

package/dist/dom-evaluation.js CHANGED Viewed

@@ -16,6 +16,7 @@
 import { raceWithTimeout } from '@d-zero/shared/race-with-timeout';
 import { domDetailsLog, domLog } from './debug.js';
 import { classify, emptyMeta } from './meta/classify.js';
+import { WINDOW_GLOBALS_TO_CHECK, collectHeadFromDocument } from './meta/collect-head.js';
 import { detectTags } from './meta/tag-detection.js';
 import { parseUrl } from './parse-url.js';
 const pid = `${process.pid}`;
@@ -345,45 +346,6 @@ async function resolveAnchor($anchor, client, nameByBackendId, options, timeout)
         return null;
     }
 }
-const WINDOW_GLOBALS_TO_CHECK = [
-    'dataLayer',
-    'gtag',
-    'ga',
-    '_gaq',
-    'fbq',
-    '_fbq',
-    'clarity',
-    '_hjSettings',
-    '_hjid',
-    'twq',
-    'ttq',
-    '_linkedin_partner_id',
-    'pintrk',
-    'amplitude',
-    'mixpanel',
-    'analytics',
-    'heap',
-    'posthog',
-    'plausible',
-    'fathom',
-    '_paq',
-    's_account',
-    's',
-    'ym',
-    'UET',
-    'optimizely',
-    '_hsq',
-    'Sentry',
-    'Intercom',
-    'intercomSettings',
-    'drift',
-    'Tawk_API',
-    'zE',
-    'OneTrust',
-    'Cookiebot',
-    'Stripe',
-    'grecaptcha',
-];
 /**
  * Extracts comprehensive metadata from the page.
  *
@@ -456,121 +418,27 @@ async function runGetMeta(page, context) {
     }
 }
 /**
+ * Collects raw `<head>` entries from a Puppeteer page by injecting
+ * {@link collectHeadFromDocument} into the page realm.
  *
- * @param page
+ * WHY string-eval instead of `page.evaluate(fn, args)`: the shared
+ * implementation lives in this module (`collectHeadFromDocument`), and a
+ * `page.evaluate(() => collectHeadFromDocument(window, …))` wrapper cannot
+ * reach that module-scope binding inside the page realm — only the wrapper's
+ * own source crosses the CDP boundary. Serializing the implementation via
+ * `Function.prototype.toString` and invoking it through
+ * `page.evaluate(string)` is what keeps the Puppeteer path and the
+ * jsdom path on one source of truth.
+ *
+ * The same {@link collectHeadFromDocument} function is also exposed via
+ * {@link ../extract-meta.ts | extractMetaFromDocument} for jsdom/Node callers,
+ * so the two paths cannot drift apart.
+ * @param page - The Puppeteer page whose document will be inspected.
  */
 async function collectHeadOnPage(page) {
-    const raw = await page
-        .evaluate((knownGlobals) => {
-        const entries = [];
-        const html = document.documentElement;
-        entries.push({
-            kind: 'html',
-            lang: html.lang || undefined,
-            dir: html.dir || undefined,
-            xmlns: html.getAttribute('xmlns') ?? undefined,
-            prefix: html.getAttribute('prefix') ?? undefined,
-            vocab: html.getAttribute('vocab') ?? undefined,
-            typeOf: html.getAttribute('typeof') ?? undefined,
-            itemscope: html.hasAttribute('itemscope') || undefined,
-            itemtype: html.getAttribute('itemtype') ?? undefined,
-            amp: html.hasAttribute('amp') || undefined,
-            lightning: html.hasAttribute('⚡') || undefined,
-        }, { kind: 'title', content: document.title });
-        for (const base of document.querySelectorAll('base')) {
-            if (!(base instanceof HTMLBaseElement))
-                continue;
-            entries.push({
-                kind: 'base',
-                href: base.getAttribute('href') ?? undefined,
-                target: base.getAttribute('target') ?? undefined,
-            });
-        }
-        for (const meta of document.querySelectorAll('meta')) {
-            if (!(meta instanceof HTMLMetaElement))
-                continue;
-            const name = meta.getAttribute('name');
-            const property = meta.getAttribute('property');
-            const httpEquiv = meta.getAttribute('http-equiv');
-            const itemprop = meta.getAttribute('itemprop');
-            const charset = meta.getAttribute('charset');
-            const content = meta.getAttribute('content');
-            const media = meta.getAttribute('media');
-            entries.push({
-                kind: 'meta',
-                name: name ? name.toLowerCase() : undefined,
-                property: property ? property.toLowerCase() : undefined,
-                httpEquiv: httpEquiv ? httpEquiv.toLowerCase() : undefined,
-                itemprop: itemprop ?? undefined,
-                charset: charset ?? undefined,
-                content: content ?? undefined,
-                media: media ?? undefined,
-            });
-        }
-        for (const link of document.querySelectorAll('link[href]')) {
-            if (!(link instanceof HTMLLinkElement))
-                continue;
-            const relRaw = link.getAttribute('rel') ?? '';
-            const rel = relRaw.toLowerCase().split(/\s+/u).filter(Boolean);
-            entries.push({
-                kind: 'link',
-                rel,
-                href: link.getAttribute('href') ?? '',
-                type: link.getAttribute('type') ?? undefined,
-                media: link.getAttribute('media') ?? undefined,
-                sizes: link.getAttribute('sizes') ?? undefined,
-                title: link.getAttribute('title') ?? undefined,
-                hreflang: link.getAttribute('hreflang') ?? undefined,
-                as: link.getAttribute('as') ?? undefined,
-                crossorigin: link.getAttribute('crossorigin') ?? undefined,
-                color: link.getAttribute('color') ?? undefined,
-                blocking: link.getAttribute('blocking') ?? undefined,
-                imagesrcset: link.getAttribute('imagesrcset') ?? undefined,
-            });
-        }
-        const STRUCTURED_TYPES = new Set([
-            'application/ld+json',
-            'speculationrules',
-            'application/json+oembed',
-            'application/xml+oembed',
-        ]);
-        for (const script of document.querySelectorAll('script[type]')) {
-            if (!(script instanceof HTMLScriptElement))
-                continue;
-            const scriptType = (script.getAttribute('type') ?? '').toLowerCase();
-            if (!STRUCTURED_TYPES.has(scriptType))
-                continue;
-            const src = script.getAttribute('src') ?? undefined;
-            const text = script.textContent ?? '';
-            const inHead = !!script.closest('head');
-            const inNoscript = !!script.closest('noscript');
-            const location = inHead ? 'head' : inNoscript ? 'noscript' : 'body';
-            entries.push({
-                kind: 'script',
-                scriptType,
-                content: text || undefined,
-                src,
-                location,
-            });
-        }
-        for (const iframe of document.querySelectorAll('iframe[src]')) {
-            if (!(iframe instanceof HTMLIFrameElement))
-                continue;
-            const src = iframe.getAttribute('src') ?? '';
-            if (!src)
-                continue;
-            const inHead = !!iframe.closest('head');
-            const inNoscript = !!iframe.closest('noscript');
-            const location = inHead ? 'head' : inNoscript ? 'noscript' : 'body';
-            entries.push({ kind: 'iframe', src, location });
-        }
-        const win = window;
-        const presentGlobals = knownGlobals.filter((name) => win[name] !== undefined);
-        if (presentGlobals.length > 0) {
-            entries.push({ kind: 'window-global', names: presentGlobals });
-        }
-        return entries;
-    }, WINDOW_GLOBALS_TO_CHECK)
-        .catch(() => []);
+    const fnSource = collectHeadFromDocument.toString();
+    const globalsLiteral = JSON.stringify(WINDOW_GLOBALS_TO_CHECK);
+    const expr = `(${fnSource})(window, ${globalsLiteral})`;
+    const raw = await page.evaluate(expr).catch(() => []);
     return raw;
 }

package/dist/extract-meta.d.ts ADDED Viewed

@@ -0,0 +1,98 @@
+/**
+ * Public, Puppeteer-free entry point for extracting {@link Meta} from an
+ * already-parsed DOM (e.g. jsdom).
+ *
+ * WHY this exists alongside `Scraper.scrapeStart()` / `getMeta(page, …)`:
+ * callers who already have an HTML string (from `fetch`, a fixture, an
+ * archive) should not be forced to spin up Chromium just to read a few `<meta>`
+ * tags. This module reuses the same `collectHead → detectTags → classify`
+ * pipeline as the Puppeteer path — the `Meta` shape returned here is
+ * identical to what `Scraper` produces, so downstream consumers do not branch
+ * on the source.
+ *
+ * See {@link extractMetaFromDocument} for the usage example.
+ * @module
+ */
+import type { Meta } from './types.js';
+/**
+ * Inputs for {@link extractMetaFromDocument}.
+ *
+ * `url`/`statusCode`/`headers` mirror the inputs to the underlying
+ * `simple-wappalyzer` driver. They are not consumed by the DOM-walk side of
+ * the pipeline.
+ *
+ * `html` is optional: when omitted, `document.documentElement.outerHTML` is
+ * read off the passed window — matching the fallback `getMeta(page, …)` does
+ * via `page.content()`.
+ */
+export type ExtractMetaContext = {
+    /** The fully resolved URL of the page (used by Wappalyzer + AMP fields). */
+    readonly url: string;
+    /**
+     * Rendered HTML used for technology detection. Defaults to
+     * `window.document.documentElement.outerHTML` when omitted.
+     *
+     * WHY allow override: callers that fetched the raw HTML string from the
+     * network already have the *pre-script-execution* markup, which is what
+     * Wappalyzer's HTML patterns are tuned for. The serialized DOM from
+     * `outerHTML` reflects whatever scripts have already mutated; provide the
+     * raw string to get more stable detections.
+     */
+    readonly html?: string;
+    /** HTTP status code, surfaced to the Wappalyzer driver. */
+    readonly statusCode?: number;
+    /**
+     * Response headers; case is preserved by the caller, lowercased internally
+     * by `detectTags`.
+     */
+    readonly headers?: Record<string, string | string[] | undefined>;
+    /**
+     * When `true`, the returned `Meta` includes `_raw: RawHeadEntry[]` for
+     * debugging. Default `false` to keep the serialized payload small.
+     */
+    readonly includeRaw?: boolean;
+};
+/**
+ * Extracts a `Meta` object from a DOM provided by the caller.
+ *
+ * Pipeline:
+ *
+ * 1. {@link collectHeadFromDocument} walks `window.document` and returns a
+ *    serializable `RawHeadEntry[]`.
+ * 2. {@link detectTags} runs `simple-wappalyzer` over the HTML + headers to
+ *    detect third-party technologies.
+ * 3. {@link classify} folds the two signals together into a typed `Meta`.
+ *
+ * Step (1) is synchronous and runs first; step (2) is awaited next. The two
+ * are independent in principle, but the current shape is sequential — keeping
+ * it that way avoids forcing the synchronous DOM walk into a microtask just to
+ * gain a few milliseconds of overlap with the Wappalyzer call.
+ * @param window - The window whose `document` will be walked. jsdom's
+ *                 `dom.window` works; pass any object satisfying the `Window`
+ *                 type. The function never mutates the document.
+ * @param context - URL / HTML / headers / status code context. See
+ *                  {@link ExtractMetaContext}.
+ * @returns The extracted `Meta` (always defined; empty fields stay empty).
+ * @example
+ * ```ts
+ * import { JSDOM } from 'jsdom';
+ * import { extractMetaFromDocument } from '@d-zero/beholder';
+ *
+ * const url = 'https://example.com/';
+ * const html = await (await fetch(url)).text();
+ * const dom = new JSDOM(html, { url });
+ *
+ * // The `as unknown as Window` cast is needed because jsdom's `DOMWindow` is
+ * // not structurally identical to lib.dom's `Window` (a few rare globals
+ * // differ), but the runtime shape is compatible for this function's needs.
+ * const meta = await extractMetaFromDocument(dom.window as unknown as Window, {
+ *   url,
+ *   html,
+ * });
+ *
+ * meta.title;         // <title>
+ * meta.og?.image;     // og:image[]
+ * meta.tags.entries;  // Wappalyzer detections + extracted IDs
+ * ```
+ */
+export declare function extractMetaFromDocument(window: Window, context: ExtractMetaContext): Promise<Meta>;

package/dist/extract-meta.js ADDED Viewed

@@ -0,0 +1,75 @@
+/**
+ * Public, Puppeteer-free entry point for extracting {@link Meta} from an
+ * already-parsed DOM (e.g. jsdom).
+ *
+ * WHY this exists alongside `Scraper.scrapeStart()` / `getMeta(page, …)`:
+ * callers who already have an HTML string (from `fetch`, a fixture, an
+ * archive) should not be forced to spin up Chromium just to read a few `<meta>`
+ * tags. This module reuses the same `collectHead → detectTags → classify`
+ * pipeline as the Puppeteer path — the `Meta` shape returned here is
+ * identical to what `Scraper` produces, so downstream consumers do not branch
+ * on the source.
+ *
+ * See {@link extractMetaFromDocument} for the usage example.
+ * @module
+ */
+import { classify } from './meta/classify.js';
+import { collectHeadFromDocument, WINDOW_GLOBALS_TO_CHECK } from './meta/collect-head.js';
+import { detectTags } from './meta/tag-detection.js';
+/**
+ * Extracts a `Meta` object from a DOM provided by the caller.
+ *
+ * Pipeline:
+ *
+ * 1. {@link collectHeadFromDocument} walks `window.document` and returns a
+ *    serializable `RawHeadEntry[]`.
+ * 2. {@link detectTags} runs `simple-wappalyzer` over the HTML + headers to
+ *    detect third-party technologies.
+ * 3. {@link classify} folds the two signals together into a typed `Meta`.
+ *
+ * Step (1) is synchronous and runs first; step (2) is awaited next. The two
+ * are independent in principle, but the current shape is sequential — keeping
+ * it that way avoids forcing the synchronous DOM walk into a microtask just to
+ * gain a few milliseconds of overlap with the Wappalyzer call.
+ * @param window - The window whose `document` will be walked. jsdom's
+ *                 `dom.window` works; pass any object satisfying the `Window`
+ *                 type. The function never mutates the document.
+ * @param context - URL / HTML / headers / status code context. See
+ *                  {@link ExtractMetaContext}.
+ * @returns The extracted `Meta` (always defined; empty fields stay empty).
+ * @example
+ * ```ts
+ * import { JSDOM } from 'jsdom';
+ * import { extractMetaFromDocument } from '@d-zero/beholder';
+ *
+ * const url = 'https://example.com/';
+ * const html = await (await fetch(url)).text();
+ * const dom = new JSDOM(html, { url });
+ *
+ * // The `as unknown as Window` cast is needed because jsdom's `DOMWindow` is
+ * // not structurally identical to lib.dom's `Window` (a few rare globals
+ * // differ), but the runtime shape is compatible for this function's needs.
+ * const meta = await extractMetaFromDocument(dom.window as unknown as Window, {
+ *   url,
+ *   html,
+ * });
+ *
+ * meta.title;         // <title>
+ * meta.og?.image;     // og:image[]
+ * meta.tags.entries;  // Wappalyzer detections + extracted IDs
+ * ```
+ */
+export async function extractMetaFromDocument(window, context) {
+    const raw = collectHeadFromDocument(window, WINDOW_GLOBALS_TO_CHECK);
+    const html = context.html ?? window.document.documentElement.outerHTML;
+    const tags = await detectTags({
+        url: context.url,
+        html,
+        ...(context.statusCode === undefined ? {} : { statusCode: context.statusCode }),
+        ...(context.headers === undefined ? {} : { headers: context.headers }),
+    });
+    return classify(raw, {
+        tags,
+        ...(context.includeRaw ? { includeRaw: true } : {}),
+    });
+}

package/dist/index.d.ts CHANGED Viewed

@@ -12,6 +12,8 @@
  */
 export { default as default } from './scraper.js';
 export { isError } from './is-error.js';
+export { extractMetaFromDocument } from './extract-meta.js';
+export type { ExtractMetaContext } from './extract-meta.js';
 export { detectCompress } from '@d-zero/shared/detect-compress';
 export type { CompressType } from '@d-zero/shared/detect-compress';
 export { detectCDN } from '@d-zero/shared/detect-cdn';

package/dist/index.js CHANGED Viewed

@@ -12,5 +12,6 @@
  */
 export { default as default } from './scraper.js';
 export { isError } from './is-error.js';
+export { extractMetaFromDocument } from './extract-meta.js';
 export { detectCompress } from '@d-zero/shared/detect-compress';
 export { detectCDN } from '@d-zero/shared/detect-cdn';

package/dist/meta/collect-head.d.ts ADDED Viewed

@@ -0,0 +1,63 @@
+/**
+ * DOM-side raw `<head>` collector.
+ *
+ * `collectHeadFromDocument` walks a `Document` (Puppeteer page realm or jsdom realm
+ * alike) and produces a serializable {@link RawHeadEntry}[] that
+ * {@link ../meta/classify.ts | classify} can turn into a typed `Meta`.
+ *
+ * WHY this function is realm-agnostic:
+ *
+ * - The Puppeteer path stringifies this function via `Function.prototype.toString`
+ *   and runs it as a `page.evaluate(string)` expression, so any closure over
+ *   module-scope bindings would resolve to `undefined` in the browser realm.
+ * - The jsdom (Node) path calls it directly with the jsdom `Window`. Because
+ *   `HTMLLinkElement` (etc.) in jsdom is a *different class instance* from the
+ *   one in the page realm, `instanceof` only works when the constructor is read
+ *   from the *passed* `window` rather than from bare globals.
+ *
+ * Together those constraints dictate that the function MUST:
+ *
+ * 1. Reference no module-level variables — only its own parameters and inner locals.
+ * 2. Take every HTML class constructor (`HTMLBaseElement`, …) from the passed
+ *    `window` via destructuring instead of relying on ambient globals.
+ * 3. Stay in plain ES syntax (no TS-only constructs that need helper imports).
+ * @module
+ */
+import type { RawHeadEntry } from './types.js';
+/**
+ * Curated list of `window` globals whose presence indicates that a third-party
+ * tag library has been loaded on the page. Surfaced as a single
+ * `kind: 'window-global'` entry so that downstream consumers (e.g. tag-detection)
+ * can cross-reference the script/iframe signals.
+ *
+ * Kept here (rather than in `dom-evaluation.ts`) so the Puppeteer path and the
+ * jsdom path share one source of truth.
+ */
+export declare const WINDOW_GLOBALS_TO_CHECK: readonly string[];
+/**
+ * Walks the given window's `Document` and returns a serializable list of raw
+ * head entries.
+ *
+ * Two realms are supported:
+ *
+ * - Browser realm (Puppeteer): the function source is `.toString()`'d and run
+ *   inside the page via `page.evaluate(string)`. Inside the page, `window`
+ *   resolves to the page's global object, so destructured class constructors
+ *   match `instanceof` checks against elements returned from `querySelectorAll`.
+ * - Node realm (jsdom et al.): the caller passes `dom.window` directly. jsdom's
+ *   HTML element prototypes are distinct from the host Node's bare globals, so
+ *   reading the constructors off the passed `window` is what makes `instanceof`
+ *   succeed.
+ *
+ * The function MUST NOT close over any module-scope binding — all data it needs
+ * is reached through its two parameters.
+ * @param window - The window object whose `document` will be inspected. Provides
+ *                 both the DOM tree and the HTML element constructors used for
+ *                 `instanceof` narrowing.
+ * @param knownGlobals - Names of `window` properties that, when present,
+ *                       indicate a third-party tag library is loaded. Required
+ *                       (no default) so the Puppeteer-side string-eval path
+ *                       does not have to inline a default value list.
+ * @returns Serializable list of raw head entries for {@link ../meta/classify.ts | classify}.
+ */
+export declare function collectHeadFromDocument(window: Window, knownGlobals: readonly string[]): RawHeadEntry[];