@mcpmake/core 0.2.0

package/dist/analyzer/selector-builder.js

@@ -0,0 +1,200 @@
+/**
+ * Builds multi-strategy selector sets for DOM elements.
+ *
+ * Generates selectors in priority order (most stable first):
+ * data-testid > id > aria-label > name > role > CSS path > XPath
+ */
+/**
+ * Build a SelectorSet for a given element on the page.
+ * Tries multiple strategies and picks the most stable one as primary.
+ */
+export async function buildSelectorSet(page, element) {
+    const candidates = [];
+    const attrs = await element.evaluate((node) => {
+        const el = node;
+        return {
+            tagName: el.tagName.toLowerCase(),
+            id: el.id || null,
+            name: el.getAttribute('name'),
+            type: el.getAttribute('type'),
+            testId: el.getAttribute('data-testid') || el.getAttribute('data-test-id'),
+            ariaLabel: el.getAttribute('aria-label'),
+            role: el.getAttribute('role') || null,
+            placeholder: el.getAttribute('placeholder'),
+            textContent: el.textContent?.trim().slice(0, 80) || null,
+            className: el.className || null,
+        };
+    });
+    // 1. data-testid (most stable — intentionally placed for testing)
+    if (attrs.testId) {
+        candidates.push({
+            selector: `[data-testid="${cssEscape(attrs.testId)}"]`,
+            strategy: 'data-testid',
+            confidence: 0.95,
+        });
+    }
+    // 2. id (stable if present, but may be auto-generated)
+    if (attrs.id && !looksAutoGenerated(attrs.id)) {
+        candidates.push({
+            selector: `#${cssEscape(attrs.id)}`,
+            strategy: 'id',
+            confidence: 0.9,
+        });
+    }
+    // 3. aria-label (stable for accessible sites)
+    if (attrs.ariaLabel) {
+        candidates.push({
+            selector: `${attrs.tagName}[aria-label="${cssEscape(attrs.ariaLabel)}"]`,
+            strategy: 'aria-label',
+            confidence: 0.85,
+        });
+    }
+    // 4. name attribute (stable for form elements)
+    if (attrs.name) {
+        const typeQualifier = attrs.type ? `[type="${attrs.type}"]` : '';
+        candidates.push({
+            selector: `${attrs.tagName}[name="${cssEscape(attrs.name)}"]${typeQualifier}`,
+            strategy: 'name',
+            confidence: 0.85,
+        });
+    }
+    // 5. role + text (for buttons/links with visible text)
+    if (attrs.role && attrs.textContent) {
+        const shortText = attrs.textContent.slice(0, 40);
+        candidates.push({
+            selector: `${attrs.tagName}[role="${attrs.role}"]:has-text("${cssEscape(shortText)}")`,
+            strategy: 'role',
+            confidence: 0.7,
+        });
+    }
+    // 6. CSS path (always available but fragile)
+    const cssPath = await buildCssPath(element);
+    if (cssPath) {
+        candidates.push({
+            selector: cssPath,
+            strategy: 'css-path',
+            confidence: 0.4,
+        });
+    }
+    // Sort by confidence descending
+    candidates.sort((a, b) => b.confidence - a.confidence);
+    if (candidates.length === 0) {
+        // Absolute fallback: XPath
+        const xpath = await buildXPath(element);
+        return {
+            primary: xpath,
+            fallbacks: [],
+            strategy: 'xpath',
+            confidence: 0.2,
+        };
+    }
+    const primary = candidates[0];
+    const fallbacks = candidates.slice(1).map((c) => c.selector);
+    return {
+        primary: primary.selector,
+        fallbacks,
+        strategy: primary.strategy,
+        confidence: primary.confidence,
+    };
+}
+/**
+ * Validate that a selector still resolves to an element on the page.
+ * Tries the primary selector first, then fallbacks.
+ * Returns the first working selector or null if all fail.
+ */
+export async function validateSelector(page, selectorSet) {
+    const allSelectors = [selectorSet.primary, ...selectorSet.fallbacks];
+    for (const selector of allSelectors) {
+        try {
+            const el = await page.$(selector);
+            if (el)
+                return selector;
+        }
+        catch {
+            // Selector syntax may be invalid — skip
+        }
+    }
+    return null;
+}
+// ─── Internal helpers ───────────────────────────────────────────────
+/** Build a CSS path from element to a unique ancestor. */
+async function buildCssPath(element) {
+    try {
+        return await element.evaluate((node) => {
+            const parts = [];
+            let current = node;
+            const maxDepth = 5;
+            let depth = 0;
+            while (current && current !== document.body && depth < maxDepth) {
+                let part = current.tagName.toLowerCase();
+                if (current.id) {
+                    // CSS.escape keeps the selector valid (and injection-safe) for ids
+                    // containing quotes/brackets/etc. from untrusted crawled pages.
+                    part = `#${CSS.escape(current.id)}`;
+                    parts.unshift(part);
+                    break;
+                }
+                const parent = current.parentElement;
+                if (parent) {
+                    const siblings = Array.from(parent.children).filter((c) => c.tagName === current.tagName);
+                    if (siblings.length > 1) {
+                        const index = siblings.indexOf(current) + 1;
+                        part += `:nth-of-type(${index})`;
+                    }
+                }
+                parts.unshift(part);
+                current = parent;
+                depth++;
+            }
+            return parts.join(' > ');
+        });
+    }
+    catch {
+        return null;
+    }
+}
+/** Build an XPath for the element (absolute fallback). */
+async function buildXPath(element) {
+    try {
+        return await element.evaluate((node) => {
+            const parts = [];
+            let current = node;
+            while (current && current.nodeType === Node.ELEMENT_NODE) {
+                let index = 1;
+                let sibling = current.previousElementSibling;
+                while (sibling) {
+                    if (sibling.tagName === current.tagName)
+                        index++;
+                    sibling = sibling.previousElementSibling;
+                }
+                parts.unshift(`${current.tagName.toLowerCase()}[${index}]`);
+                current = current.parentElement;
+            }
+            return '/' + parts.join('/');
+        });
+    }
+    catch {
+        return '//body';
+    }
+}
+/** Check if an ID looks auto-generated (random hash, UUID-like, etc.) */
+function looksAutoGenerated(id) {
+    // Random hex strings (8+ chars of only hex)
+    if (/^[a-f0-9]{8,}$/i.test(id))
+        return true;
+    // React-style IDs like :r0:, :r1a:
+    if (/^:r[a-z0-9]+:$/.test(id))
+        return true;
+    // Contains UUID-like segments
+    if (/[a-f0-9]{8}-[a-f0-9]{4}/.test(id))
+        return true;
+    // Very long IDs are usually auto-generated
+    if (id.length > 40)
+        return true;
+    return false;
+}
+/** Escape a string for use in CSS selectors. */
+function cssEscape(str) {
+    return str.replace(/["'\\[\](){}:.,!#$%^&*+=|~<>?/;@]/g, '\\$&');
+}
+//# sourceMappingURL=selector-builder.js.map

package/dist/analyzer/selector-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"selector-builder.js","sourceRoot":"","sources":["../../src/analyzer/selector-builder.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU,EAAE,OAAsB;IACvE,MAAM,UAAU,GAAwB,EAAE,CAAC;IAE3C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,IAAe,CAAC;QAC3B,OAAO;YACL,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE;YACjC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,IAAI;YACjB,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;YAC7B,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC;YACzE,SAAS,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC;YACxC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,IAAI;YACrC,WAAW,EAAE,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC;YAC3C,WAAW,EAAE,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI;YACxD,SAAS,EAAE,EAAE,CAAC,SAAS,IAAI,IAAI;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,kEAAkE;IAClE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,UAAU,CAAC,IAAI,CAAC;YACd,QAAQ,EAAE,iBAAiB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI;YACtD,QAAQ,EAAE,aAAa;YACvB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9C,UAAU,CAAC,IAAI,CAAC;YACd,QAAQ,EAAE,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE;YACnC,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,8CAA8C;IAC9C,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC;YACd,QAAQ,EAAE,GAAG,KAAK,CAAC,OAAO,gBAAgB,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI;YACxE,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,+CAA+C;IAC/C,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,UAAU,CAAC,IAAI,CAAC;YACd,QAAQ,EAAE,GAAG,KAAK,CAAC,OAAO,UAAU,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,aAAa,EAAE;YAC7E,QAAQ,EAAE,MAAM;YAChB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACjD,UAAU,CAAC,IAAI,CAAC;YACd,QAAQ,EAAE,GAAG,KAAK,CAAC,OAAO,UAAU,KAAK,CAAC,IAAI,gBAAgB,SAAS,CAAC,SAAS,CAAC,IAAI;YACtF,QAAQ,EAAE,MAAM;YAChB,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,6CAA6C;IAC7C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,OAAO,EAAE,CAAC;QACZ,UAAU,CAAC,IAAI,CAAC;YACd,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE,UAAU;YACpB,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,gCAAgC;IAChC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEvD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,2BAA2B;QAC3B,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO;YACL,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,EAAE;YACb,QAAQ,EAAE,OAAO;YACjB,UAAU,EAAE,GAAG;SAChB,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE7D,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,QAAQ;QACzB,SAAS;QACT,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAU,EACV,WAAwB;IAExB,MAAM,YAAY,GAAG,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IAErE,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,EAAE;gBAAE,OAAO,QAAQ,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AAEvE,0DAA0D;AAC1D,KAAK,UAAU,YAAY,CAAC,OAAsB;IAChD,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE;YACrC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,OAAO,GAAG,IAAsB,CAAC;YACrC,MAAM,QAAQ,GAAG,CAAC,CAAC;YACnB,IAAI,KAAK,GAAG,CAAC,CAAC;YAEd,OAAO,OAAO,IAAI,OAAO,KAAK,QAAQ,CAAC,IAAI,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;gBAChE,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACzC,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;oBACf,mEAAmE;oBACnE,gEAAgE;oBAChE,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;oBACpC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBACpB,MAAM;gBACR,CAAC;gBACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;gBACrC,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAQ,CAAC,OAAO,CACtC,CAAC;oBACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC5C,IAAI,IAAI,gBAAgB,KAAK,GAAG,CAAC;oBACnC,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACpB,OAAO,GAAG,MAAM,CAAC;gBACjB,KAAK,EAAE,CAAC;YACV,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,KAAK,UAAU,UAAU,CAAC,OAAsB;IAC9C,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE;YACrC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,OAAO,GAAG,IAAsB,CAAC;YAErC,OAAO,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzD,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,OAAO,GAAG,OAAO,CAAC,sBAAsB,CAAC;gBAC7C,OAAO,OAAO,EAAE,CAAC;oBACf,IAAI,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO;wBAAE,KAAK,EAAE,CAAC;oBACjD,OAAO,GAAG,OAAO,CAAC,sBAAsB,CAAC;gBAC3C,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,KAAK,GAAG,CAAC,CAAC;gBAC5D,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC;YAClC,CAAC;YACD,OAAO,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,SAAS,kBAAkB,CAAC,EAAU;IACpC,4CAA4C;IAC5C,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,mCAAmC;IACnC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,8BAA8B;IAC9B,IAAI,yBAAyB,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,2CAA2C;IAC3C,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IAChC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,GAAG,CAAC,OAAO,CAAC,oCAAoC,EAAE,MAAM,CAAC,CAAC;AACnE,CAAC"}

package/dist/analyzer/semantic-analyzer.d.ts

@@ -0,0 +1,14 @@
+/**
+ * LLM-powered semantic analysis of crawled pages.
+ *
+ * Sends page structure to Claude (Haiku for cost efficiency) and infers
+ * human-readable semantic names and descriptions for forms, buttons, and links.
+ */
+import type { PageDescriptor } from '../types/site.js';
+/**
+ * Enrich PageDescriptors with LLM-inferred semantic names and descriptions.
+ *
+ * When ANTHROPIC_API_KEY is not set, returns pages unchanged with a warning.
+ */
+export declare function analyzeSemantics(pages: PageDescriptor[], _screenshots?: Map<string, Buffer>): Promise<PageDescriptor[]>;
+//# sourceMappingURL=semantic-analyzer.d.ts.map

package/dist/analyzer/semantic-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-analyzer.d.ts","sourceRoot":"","sources":["../../src/analyzer/semantic-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AA6DvD;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,cAAc,EAAE,EACvB,YAAY,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC,OAAO,CAAC,cAAc,EAAE,CAAC,CAgJ3B"}

package/dist/analyzer/semantic-analyzer.js

@@ -0,0 +1,146 @@
+/**
+ * LLM-powered semantic analysis of crawled pages.
+ *
+ * Sends page structure to Claude (Haiku for cost efficiency) and infers
+ * human-readable semantic names and descriptions for forms, buttons, and links.
+ */
+import Anthropic from '@anthropic-ai/sdk';
+import { logger } from '../utils/logger.js';
+const MODEL = 'claude-haiku-4-5-20251001';
+const MAX_TOKENS = 4096;
+/**
+ * Enrich PageDescriptors with LLM-inferred semantic names and descriptions.
+ *
+ * When ANTHROPIC_API_KEY is not set, returns pages unchanged with a warning.
+ */
+export async function analyzeSemantics(pages, _screenshots) {
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey) {
+        logger.warn('ANTHROPIC_API_KEY not set — skipping semantic analysis');
+        return pages;
+    }
+    const client = new Anthropic({ apiKey });
+    // Build compact page summaries for the prompt
+    const summaries = pages.map((page, pageIndex) => ({
+        pageIndex,
+        url: page.url,
+        title: page.title,
+        forms: page.forms.map((form, formIndex) => ({
+            formIndex,
+            formId: form.formId,
+            method: form.method,
+            action: form.action,
+            fieldNames: form.fields.map((f) => f.name),
+            fieldTypes: form.fields.map((f) => f.fieldType),
+            labels: form.fields.map((f) => f.label ?? f.name),
+            submitText: form.submitButton?.humanLabel,
+        })),
+        buttons: page.buttons.map((btn, buttonIndex) => ({
+            buttonIndex,
+            buttonId: btn.buttonId,
+            text: btn.text,
+            ariaLabel: btn.ariaLabel,
+            type: btn.type,
+            href: btn.href,
+        })),
+        links: page.links.map((link, linkIndex) => ({
+            linkIndex,
+            linkId: link.linkId,
+            text: link.text,
+            href: link.href,
+            isNavigation: link.isNavigation,
+        })),
+    }));
+    const prompt = `You are analyzing a website's interactive elements to generate MCP tool names. Given the page structure below, infer semantic names and descriptions for each element.
+
+Rules:
+- Form semanticName: snake_case, e.g. "login_form", "search_form", "checkout_form"
+- Button semanticAction: snake_case verb, e.g. "add_to_cart", "submit_order", "toggle_menu"
+- Link semanticAction: snake_case, e.g. "navigate_to_checkout", "view_product_details"
+- Page semanticName: snake_case, e.g. "login_page", "product_listing"
+- Descriptions: one sentence explaining what the element does
+- Only output valid JSON matching the schema, no explanation
+
+Schema:
+{
+  "pages": [{
+    "pageIndex": number,
+    "semanticName": string,
+    "description": string,
+    "forms": [{ "formIndex": number, "semanticName": string, "description": string }],
+    "buttons": [{ "buttonIndex": number, "semanticAction": string, "description": string }],
+    "links": [{ "linkIndex": number, "semanticAction": string }]
+  }]
+}
+
+Pages:
+${JSON.stringify(summaries, null, 2)}`;
+    try {
+        logger.info('Analyzing page semantics with LLM...');
+        const message = await client.messages.create({
+            model: MODEL,
+            max_tokens: MAX_TOKENS,
+            messages: [{ role: 'user', content: prompt }],
+        });
+        const content = message.content[0];
+        if (content.type !== 'text')
+            return pages;
+        let json = content.text.trim();
+        if (json.startsWith('```')) {
+            json = json.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
+        }
+        const result = JSON.parse(json);
+        // Apply inferred names back to the page descriptors
+        const enriched = pages.map((page, pageIndex) => {
+            const pageResult = result.pages.find((p) => p.pageIndex === pageIndex);
+            if (!pageResult)
+                return page;
+            const updatedPage = {
+                ...page,
+                semanticName: pageResult.semanticName ?? page.semanticName,
+                description: pageResult.description ?? page.description,
+            };
+            updatedPage.forms = page.forms.map((form, formIndex) => {
+                const formResult = pageResult.forms.find((f) => f.formIndex === formIndex);
+                if (!formResult)
+                    return form;
+                return {
+                    ...form,
+                    semanticName: formResult.semanticName,
+                    description: formResult.description,
+                };
+            });
+            updatedPage.buttons = page.buttons.map((btn, buttonIndex) => {
+                const btnResult = pageResult.buttons.find((b) => b.buttonIndex === buttonIndex);
+                if (!btnResult)
+                    return btn;
+                return {
+                    ...btn,
+                    semanticAction: btnResult.semanticAction,
+                    description: btnResult.description,
+                };
+            });
+            updatedPage.links = page.links.map((link, linkIndex) => {
+                const linkResult = pageResult.links.find((l) => l.linkIndex === linkIndex);
+                if (!linkResult)
+                    return link;
+                return {
+                    ...link,
+                    semanticAction: linkResult.semanticAction,
+                };
+            });
+            return updatedPage;
+        });
+        const totalNamed = enriched.reduce((n, p) => n +
+            p.forms.filter((f) => f.semanticName).length +
+            p.buttons.filter((b) => b.semanticAction).length +
+            p.links.filter((l) => l.semanticAction).length, 0);
+        logger.info(`Semantic analysis complete: ${totalNamed} elements named`);
+        return enriched;
+    }
+    catch (err) {
+        logger.warn(`Semantic analysis failed, using defaults: ${err instanceof Error ? err.message : err}`);
+        return pages;
+    }
+}
+//# sourceMappingURL=semantic-analyzer.js.map

package/dist/analyzer/semantic-analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-analyzer.js","sourceRoot":"","sources":["../../src/analyzer/semantic-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAyD5C,MAAM,KAAK,GAAG,2BAA2B,CAAC;AAC1C,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAuB,EACvB,YAAkC;IAElC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACtE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAEzC,8CAA8C;IAC9C,MAAM,SAAS,GAAkB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/D,SAAS;QACT,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC1C,SAAS;YACT,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC1C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC;YACjD,UAAU,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU;SAC1C,CAAC,CAAC;QACH,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;YAC/C,WAAW;YACX,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC,CAAC;QACH,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC1C,SAAS;YACT,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC;KACJ,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;EAuBf,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,KAAK,CAAC;QAE1C,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,MAAM,GAAmB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEhD,oDAAoD;QACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;YAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,UAAU;gBAAE,OAAO,IAAI,CAAC;YAE7B,MAAM,WAAW,GAAmB;gBAClC,GAAG,IAAI;gBACP,YAAY,EAAE,UAAU,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;gBAC1D,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;aACxD,CAAC;YAEF,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;gBACrD,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;gBAC3E,IAAI,CAAC,UAAU;oBAAE,OAAO,IAAI,CAAC;gBAC7B,OAAO;oBACL,GAAG,IAAI;oBACP,YAAY,EAAE,UAAU,CAAC,YAAY;oBACrC,WAAW,EAAE,UAAU,CAAC,WAAW;iBACpC,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,WAAW,EAAE,EAAE;gBAC1D,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC;gBAChF,IAAI,CAAC,SAAS;oBAAE,OAAO,GAAG,CAAC;gBAC3B,OAAO;oBACL,GAAG,GAAG;oBACN,cAAc,EAAE,SAAS,CAAC,cAAc;oBACxC,WAAW,EAAE,SAAS,CAAC,WAAW;iBACnC,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;gBACrD,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;gBAC3E,IAAI,CAAC,UAAU;oBAAE,OAAO,IAAI,CAAC;gBAC7B,OAAO;oBACL,GAAG,IAAI;oBACP,cAAc,EAAE,UAAU,CAAC,cAAc;iBAC1C,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC;YACD,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM;YAC5C,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM;YAChD,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,EAChD,CAAC,CACF,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,+BAA+B,UAAU,iBAAiB,CAAC,CAAC;QAExE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CACT,6CAA6C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CACxF,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/analyzer/site-crawler.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Crawls a website using Playwright, visiting pages breadth-first
+ * and extracting interactive elements from each page.
+ */
+import type { Entry } from 'har-format';
+import type { SiteDescriptor } from '../types/site.js';
+export interface CrawlOptions {
+    /** Target URL to start crawling from */
+    url: string;
+    /** Maximum crawl depth (default: 2) */
+    depth?: number;
+    /** Maximum number of pages to visit (default: 20) */
+    maxPages?: number;
+    /** Idle timeout in ms before auto-closing (default: 5 min) */
+    timeout?: number;
+    /** Run browser in headless mode (default: false) */
+    headless?: boolean;
+    /** Viewport dimensions */
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    /** Whether to capture screenshots during analysis (default: true) */
+    captureScreenshots?: boolean;
+    /** Whether to capture HAR entries during crawl (for hybrid mode) */
+    captureHar?: boolean;
+}
+export interface CrawlResult {
+    siteDescriptor: SiteDescriptor;
+    /** Screenshots keyed by pageId */
+    screenshots: Map<string, Buffer>;
+    /** HAR entries captured during the crawl (only when captureHar is true) */
+    harEntries?: Entry[];
+}
+/**
+ * Crawl a website and return a SiteDescriptor with all discovered pages.
+ */
+export declare function crawlSite(options: CrawlOptions): Promise<CrawlResult>;
+//# sourceMappingURL=site-crawler.d.ts.map

package/dist/analyzer/site-crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"site-crawler.d.ts","sourceRoot":"","sources":["../../src/analyzer/site-crawler.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAU,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAMvE,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,oEAAoE;IACpE,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,cAAc,EAAE,cAAc,CAAC;IAC/B,kCAAkC;IAClC,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,2EAA2E;IAC3E,UAAU,CAAC,EAAE,KAAK,EAAE,CAAC;CACtB;AAOD;;GAEG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA2I3E"}

package/dist/analyzer/site-crawler.js

@@ -0,0 +1,236 @@
+/**
+ * Crawls a website using Playwright, visiting pages breadth-first
+ * and extracting interactive elements from each page.
+ */
+import { chromium } from 'playwright';
+import { parsePage } from './dom-parser.js';
+import { captureViewportScreenshot } from './screenshot-capture.js';
+import { logger } from '../utils/logger.js';
+import crypto from 'node:crypto';
+const DEFAULT_DEPTH = 2;
+const DEFAULT_MAX_PAGES = 20;
+const DEFAULT_TIMEOUT = 5 * 60 * 1000;
+const DEFAULT_VIEWPORT = { width: 1280, height: 720 };
+/**
+ * Crawl a website and return a SiteDescriptor with all discovered pages.
+ */
+export async function crawlSite(options) {
+    const depth = options.depth ?? DEFAULT_DEPTH;
+    const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+    const timeout = options.timeout ?? DEFAULT_TIMEOUT;
+    const viewport = options.viewport ?? DEFAULT_VIEWPORT;
+    const captureScreenshots = options.captureScreenshots ?? true;
+    const parsedUrl = new URL(options.url);
+    if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+        throw new Error('Only http/https URLs are supported');
+    }
+    const baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
+    const captureHar = options.captureHar ?? false;
+    const pages = [];
+    const screenshots = new Map();
+    const harEntries = [];
+    const pendingRequests = new Map();
+    const visited = new Set();
+    const queue = [
+        { url: options.url, currentDepth: 0 },
+    ];
+    let browser;
+    try {
+        browser = await chromium.launch({ headless: options.headless ?? false });
+        const context = await browser.newContext({ viewport });
+        const page = await context.newPage();
+        // Optionally capture network requests as HAR entries during crawl
+        if (captureHar) {
+            page.on('request', (request) => {
+                pendingRequests.set(request, { startTime: Date.now() });
+            });
+            page.on('response', async (response) => {
+                const request = response.request();
+                const pending = pendingRequests.get(request);
+                if (!pending)
+                    return;
+                pendingRequests.delete(request);
+                try {
+                    const entry = await buildCrawlHarEntry(request, response, pending.startTime);
+                    harEntries.push(entry);
+                }
+                catch {
+                    // Some responses can't be read (redirects, aborted)
+                }
+            });
+        }
+        logger.info(`Crawling site: ${options.url} (depth: ${depth}, max pages: ${maxPages})`);
+        let lastActivityTime = Date.now();
+        while (queue.length > 0 && pages.length < maxPages) {
+            const item = queue.shift();
+            const normalizedUrl = normalizeUrl(item.url);
+            // Skip if already visited or different origin
+            if (visited.has(normalizedUrl))
+                continue;
+            if (!item.url.startsWith(baseUrl))
+                continue;
+            visited.add(normalizedUrl);
+            // Check idle timeout
+            if (Date.now() - lastActivityTime > timeout) {
+                logger.warn('Idle timeout reached during crawl');
+                break;
+            }
+            try {
+                logger.info(`[${pages.length + 1}/${maxPages}] Visiting: ${item.url}`);
+                await page.goto(item.url, {
+                    waitUntil: 'domcontentloaded',
+                    timeout: 15_000,
+                });
+                // Wait briefly for dynamic content to render
+                await page.waitForTimeout(1000);
+                lastActivityTime = Date.now();
+                // Parse the page DOM
+                const pageDescriptor = await parsePage(page);
+                pageDescriptor.url = item.url;
+                // Capture screenshot if enabled
+                if (captureScreenshots) {
+                    const screenshot = await captureViewportScreenshot(page);
+                    pageDescriptor.screenshotHash = screenshot.hash;
+                    screenshots.set(pageDescriptor.pageId, screenshot.data);
+                }
+                pages.push(pageDescriptor);
+                // Queue navigation links for further crawling
+                if (item.currentDepth < depth) {
+                    for (const link of pageDescriptor.links) {
+                        if (link.isNavigation && !visited.has(normalizeUrl(link.href))) {
+                            queue.push({ url: link.href, currentDepth: item.currentDepth + 1 });
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                logger.warn(`Failed to crawl ${item.url}: ${err}`);
+            }
+        }
+        // Close browser
+        await browser.close().catch(() => { });
+        browser = undefined;
+    }
+    catch (error) {
+        if (browser)
+            await browser.close().catch(() => { });
+        throw error;
+    }
+    logger.info(`Crawl complete: ${pages.length} pages discovered`);
+    // Extract site metadata from the first page
+    const metadata = {
+        title: pages[0]?.title,
+        description: undefined,
+        favicon: undefined,
+    };
+    const siteDescriptor = {
+        siteId: generateSiteId(baseUrl),
+        baseUrl,
+        pages,
+        analyzedAt: new Date().toISOString(),
+        version: 1,
+        crawlDepth: depth,
+        metadata,
+    };
+    return {
+        siteDescriptor,
+        screenshots,
+        ...(captureHar ? { harEntries } : {}),
+    };
+}
+// ─── HAR Capture (for hybrid mode) ─────────────────────────────────
+const MAX_RESPONSE_BODY_BYTES = 5 * 1024 * 1024;
+const SKIP_BODY_MIME_TYPES = ['image/', 'video/', 'audio/', 'font/', 'application/octet-stream'];
+async function buildCrawlHarEntry(request, response, startTime) {
+    const elapsed = Date.now() - startTime;
+    const url = request.url();
+    const parsedUrl = new URL(url);
+    const requestHeaders = Object.entries(request.headers()).map(([name, value]) => ({
+        name,
+        value,
+    }));
+    const queryString = [...parsedUrl.searchParams.entries()].map(([name, value]) => ({
+        name,
+        value,
+    }));
+    const postData = request.postData();
+    const contentTypeHeader = request.headers()['content-type'] ?? '';
+    const responseHeaders = Object.entries(response.headers()).map(([name, value]) => ({
+        name,
+        value,
+    }));
+    let responseText;
+    const responseMimeType = response.headers()['content-type'] ?? '';
+    const skipBody = SKIP_BODY_MIME_TYPES.some((m) => responseMimeType.startsWith(m));
+    const contentLength = parseInt(response.headers()['content-length'] ?? '0', 10);
+    if (!skipBody && contentLength <= MAX_RESPONSE_BODY_BYTES) {
+        try {
+            const body = await response.body();
+            if (body.length <= MAX_RESPONSE_BODY_BYTES) {
+                responseText = body.toString('utf-8');
+            }
+        }
+        catch {
+            // Body may not be available
+        }
+    }
+    return {
+        startedDateTime: new Date(startTime).toISOString(),
+        time: elapsed,
+        request: {
+            method: request.method(),
+            url,
+            httpVersion: 'HTTP/1.1',
+            headers: requestHeaders,
+            queryString,
+            cookies: [],
+            headersSize: -1,
+            bodySize: postData ? Buffer.byteLength(postData) : 0,
+            ...(postData
+                ? {
+                    postData: {
+                        mimeType: contentTypeHeader.split(';')[0].trim() || 'application/octet-stream',
+                        text: postData,
+                    },
+                }
+                : {}),
+        },
+        response: {
+            status: response.status(),
+            statusText: response.statusText(),
+            httpVersion: 'HTTP/1.1',
+            headers: responseHeaders,
+            cookies: [],
+            content: {
+                size: responseText ? Buffer.byteLength(responseText) : 0,
+                mimeType: responseMimeType.split(';')[0].trim() || 'application/octet-stream',
+                ...(responseText ? { text: responseText } : {}),
+            },
+            redirectURL: '',
+            headersSize: -1,
+            bodySize: responseText ? Buffer.byteLength(responseText) : 0,
+        },
+        cache: {},
+        timings: {
+            send: 1,
+            wait: Math.max(1, elapsed - 2),
+            receive: 1,
+        },
+    };
+}
+// ─── Helpers ────────────────────────────────────────────────────────
+function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        // Remove trailing slash, fragment, and normalize
+        return `${parsed.origin}${parsed.pathname.replace(/\/$/, '')}${parsed.search}`;
+    }
+    catch {
+        return url;
+    }
+}
+function generateSiteId(baseUrl) {
+    const hash = crypto.createHash('sha256').update(baseUrl).digest('hex').slice(0, 12);
+    return `site_${hash}`;
+}
+//# sourceMappingURL=site-crawler.js.map