mcpmake 0.1.1 → 0.2.1

package/dist/analyzer/selector-builder.d.ts

@@ -1,19 +0,0 @@
-/**
- * Builds multi-strategy selector sets for DOM elements.
- *
- * Generates selectors in priority order (most stable first):
- * data-testid > id > aria-label > name > role > CSS path > XPath
- */
-import type { Page, ElementHandle } from 'playwright';
-import type { SelectorSet } from '../types/site.js';
-/**
- * Build a SelectorSet for a given element on the page.
- * Tries multiple strategies and picks the most stable one as primary.
- */
-export declare function buildSelectorSet(page: Page, element: ElementHandle): Promise<SelectorSet>;
-/**
- * Validate that a selector still resolves to an element on the page.
- * Tries the primary selector first, then fallbacks.
- * Returns the first working selector or null if all fail.
- */
-export declare function validateSelector(page: Page, selectorSet: SelectorSet): Promise<string | null>;

package/dist/analyzer/selector-builder.js

@@ -1,199 +0,0 @@
-/**
- * Builds multi-strategy selector sets for DOM elements.
- *
- * Generates selectors in priority order (most stable first):
- * data-testid > id > aria-label > name > role > CSS path > XPath
- */
-/**
- * Build a SelectorSet for a given element on the page.
- * Tries multiple strategies and picks the most stable one as primary.
- */
-export async function buildSelectorSet(page, element) {
-    const candidates = [];
-    const attrs = await element.evaluate((node) => {
-        const el = node;
-        return {
-            tagName: el.tagName.toLowerCase(),
-            id: el.id || null,
-            name: el.getAttribute('name'),
-            type: el.getAttribute('type'),
-            testId: el.getAttribute('data-testid') || el.getAttribute('data-test-id'),
-            ariaLabel: el.getAttribute('aria-label'),
-            role: el.getAttribute('role') || null,
-            placeholder: el.getAttribute('placeholder'),
-            textContent: el.textContent?.trim().slice(0, 80) || null,
-            className: el.className || null,
-        };
-    });
-    // 1. data-testid (most stable — intentionally placed for testing)
-    if (attrs.testId) {
-        candidates.push({
-            selector: `[data-testid="${cssEscape(attrs.testId)}"]`,
-            strategy: 'data-testid',
-            confidence: 0.95,
-        });
-    }
-    // 2. id (stable if present, but may be auto-generated)
-    if (attrs.id && !looksAutoGenerated(attrs.id)) {
-        candidates.push({
-            selector: `#${cssEscape(attrs.id)}`,
-            strategy: 'id',
-            confidence: 0.9,
-        });
-    }
-    // 3. aria-label (stable for accessible sites)
-    if (attrs.ariaLabel) {
-        candidates.push({
-            selector: `${attrs.tagName}[aria-label="${cssEscape(attrs.ariaLabel)}"]`,
-            strategy: 'aria-label',
-            confidence: 0.85,
-        });
-    }
-    // 4. name attribute (stable for form elements)
-    if (attrs.name) {
-        const typeQualifier = attrs.type ? `[type="${attrs.type}"]` : '';
-        candidates.push({
-            selector: `${attrs.tagName}[name="${cssEscape(attrs.name)}"]${typeQualifier}`,
-            strategy: 'name',
-            confidence: 0.85,
-        });
-    }
-    // 5. role + text (for buttons/links with visible text)
-    if (attrs.role && attrs.textContent) {
-        const shortText = attrs.textContent.slice(0, 40);
-        candidates.push({
-            selector: `${attrs.tagName}[role="${attrs.role}"]:has-text("${cssEscape(shortText)}")`,
-            strategy: 'role',
-            confidence: 0.7,
-        });
-    }
-    // 6. CSS path (always available but fragile)
-    const cssPath = await buildCssPath(element);
-    if (cssPath) {
-        candidates.push({
-            selector: cssPath,
-            strategy: 'css-path',
-            confidence: 0.4,
-        });
-    }
-    // Sort by confidence descending
-    candidates.sort((a, b) => b.confidence - a.confidence);
-    if (candidates.length === 0) {
-        // Absolute fallback: XPath
-        const xpath = await buildXPath(element);
-        return {
-            primary: xpath,
-            fallbacks: [],
-            strategy: 'xpath',
-            confidence: 0.2,
-        };
-    }
-    const primary = candidates[0];
-    const fallbacks = candidates.slice(1).map((c) => c.selector);
-    return {
-        primary: primary.selector,
-        fallbacks,
-        strategy: primary.strategy,
-        confidence: primary.confidence,
-    };
-}
-/**
- * Validate that a selector still resolves to an element on the page.
- * Tries the primary selector first, then fallbacks.
- * Returns the first working selector or null if all fail.
- */
-export async function validateSelector(page, selectorSet) {
-    const allSelectors = [selectorSet.primary, ...selectorSet.fallbacks];
-    for (const selector of allSelectors) {
-        try {
-            const el = await page.$(selector);
-            if (el)
-                return selector;
-        }
-        catch {
-            // Selector syntax may be invalid — skip
-        }
-    }
-    return null;
-}
-// ─── Internal helpers ───────────────────────────────────────────────
-/** Build a CSS path from element to a unique ancestor. */
-async function buildCssPath(element) {
-    try {
-        return await element.evaluate((node) => {
-            const parts = [];
-            let current = node;
-            const maxDepth = 5;
-            let depth = 0;
-            while (current && current !== document.body && depth < maxDepth) {
-                let part = current.tagName.toLowerCase();
-                if (current.id) {
-                    // CSS.escape keeps the selector valid (and injection-safe) for ids
-                    // containing quotes/brackets/etc. from untrusted crawled pages.
-                    part = `#${CSS.escape(current.id)}`;
-                    parts.unshift(part);
-                    break;
-                }
-                const parent = current.parentElement;
-                if (parent) {
-                    const siblings = Array.from(parent.children).filter((c) => c.tagName === current.tagName);
-                    if (siblings.length > 1) {
-                        const index = siblings.indexOf(current) + 1;
-                        part += `:nth-of-type(${index})`;
-                    }
-                }
-                parts.unshift(part);
-                current = parent;
-                depth++;
-            }
-            return parts.join(' > ');
-        });
-    }
-    catch {
-        return null;
-    }
-}
-/** Build an XPath for the element (absolute fallback). */
-async function buildXPath(element) {
-    try {
-        return await element.evaluate((node) => {
-            const parts = [];
-            let current = node;
-            while (current && current.nodeType === Node.ELEMENT_NODE) {
-                let index = 1;
-                let sibling = current.previousElementSibling;
-                while (sibling) {
-                    if (sibling.tagName === current.tagName)
-                        index++;
-                    sibling = sibling.previousElementSibling;
-                }
-                parts.unshift(`${current.tagName.toLowerCase()}[${index}]`);
-                current = current.parentElement;
-            }
-            return '/' + parts.join('/');
-        });
-    }
-    catch {
-        return '//body';
-    }
-}
-/** Check if an ID looks auto-generated (random hash, UUID-like, etc.) */
-function looksAutoGenerated(id) {
-    // Random hex strings (8+ chars of only hex)
-    if (/^[a-f0-9]{8,}$/i.test(id))
-        return true;
-    // React-style IDs like :r0:, :r1a:
-    if (/^:r[a-z0-9]+:$/.test(id))
-        return true;
-    // Contains UUID-like segments
-    if (/[a-f0-9]{8}-[a-f0-9]{4}/.test(id))
-        return true;
-    // Very long IDs are usually auto-generated
-    if (id.length > 40)
-        return true;
-    return false;
-}
-/** Escape a string for use in CSS selectors. */
-function cssEscape(str) {
-    return str.replace(/["'\\[\](){}:.,!#$%^&*+=|~<>?/;@]/g, '\\$&');
-}

package/dist/analyzer/semantic-analyzer.d.ts

@@ -1,13 +0,0 @@
-/**
- * LLM-powered semantic analysis of crawled pages.
- *
- * Sends page structure to Claude (Haiku for cost efficiency) and infers
- * human-readable semantic names and descriptions for forms, buttons, and links.
- */
-import type { PageDescriptor } from '../types/site.js';
-/**
- * Enrich PageDescriptors with LLM-inferred semantic names and descriptions.
- *
- * When ANTHROPIC_API_KEY is not set, returns pages unchanged with a warning.
- */
-export declare function analyzeSemantics(pages: PageDescriptor[], _screenshots?: Map<string, Buffer>): Promise<PageDescriptor[]>;

package/dist/analyzer/semantic-analyzer.js

@@ -1,145 +0,0 @@
-/**
- * LLM-powered semantic analysis of crawled pages.
- *
- * Sends page structure to Claude (Haiku for cost efficiency) and infers
- * human-readable semantic names and descriptions for forms, buttons, and links.
- */
-import Anthropic from '@anthropic-ai/sdk';
-import { logger } from '../utils/logger.js';
-const MODEL = 'claude-haiku-4-5-20251001';
-const MAX_TOKENS = 4096;
-/**
- * Enrich PageDescriptors with LLM-inferred semantic names and descriptions.
- *
- * When ANTHROPIC_API_KEY is not set, returns pages unchanged with a warning.
- */
-export async function analyzeSemantics(pages, _screenshots) {
-    const apiKey = process.env.ANTHROPIC_API_KEY;
-    if (!apiKey) {
-        logger.warn('ANTHROPIC_API_KEY not set — skipping semantic analysis');
-        return pages;
-    }
-    const client = new Anthropic({ apiKey });
-    // Build compact page summaries for the prompt
-    const summaries = pages.map((page, pageIndex) => ({
-        pageIndex,
-        url: page.url,
-        title: page.title,
-        forms: page.forms.map((form, formIndex) => ({
-            formIndex,
-            formId: form.formId,
-            method: form.method,
-            action: form.action,
-            fieldNames: form.fields.map((f) => f.name),
-            fieldTypes: form.fields.map((f) => f.fieldType),
-            labels: form.fields.map((f) => f.label ?? f.name),
-            submitText: form.submitButton?.humanLabel,
-        })),
-        buttons: page.buttons.map((btn, buttonIndex) => ({
-            buttonIndex,
-            buttonId: btn.buttonId,
-            text: btn.text,
-            ariaLabel: btn.ariaLabel,
-            type: btn.type,
-            href: btn.href,
-        })),
-        links: page.links.map((link, linkIndex) => ({
-            linkIndex,
-            linkId: link.linkId,
-            text: link.text,
-            href: link.href,
-            isNavigation: link.isNavigation,
-        })),
-    }));
-    const prompt = `You are analyzing a website's interactive elements to generate MCP tool names. Given the page structure below, infer semantic names and descriptions for each element.
-
-Rules:
-- Form semanticName: snake_case, e.g. "login_form", "search_form", "checkout_form"
-- Button semanticAction: snake_case verb, e.g. "add_to_cart", "submit_order", "toggle_menu"
-- Link semanticAction: snake_case, e.g. "navigate_to_checkout", "view_product_details"
-- Page semanticName: snake_case, e.g. "login_page", "product_listing"
-- Descriptions: one sentence explaining what the element does
-- Only output valid JSON matching the schema, no explanation
-
-Schema:
-{
-  "pages": [{
-    "pageIndex": number,
-    "semanticName": string,
-    "description": string,
-    "forms": [{ "formIndex": number, "semanticName": string, "description": string }],
-    "buttons": [{ "buttonIndex": number, "semanticAction": string, "description": string }],
-    "links": [{ "linkIndex": number, "semanticAction": string }]
-  }]
-}
-
-Pages:
-${JSON.stringify(summaries, null, 2)}`;
-    try {
-        logger.info('Analyzing page semantics with LLM...');
-        const message = await client.messages.create({
-            model: MODEL,
-            max_tokens: MAX_TOKENS,
-            messages: [{ role: 'user', content: prompt }],
-        });
-        const content = message.content[0];
-        if (content.type !== 'text')
-            return pages;
-        let json = content.text.trim();
-        if (json.startsWith('```')) {
-            json = json.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
-        }
-        const result = JSON.parse(json);
-        // Apply inferred names back to the page descriptors
-        const enriched = pages.map((page, pageIndex) => {
-            const pageResult = result.pages.find((p) => p.pageIndex === pageIndex);
-            if (!pageResult)
-                return page;
-            const updatedPage = {
-                ...page,
-                semanticName: pageResult.semanticName ?? page.semanticName,
-                description: pageResult.description ?? page.description,
-            };
-            updatedPage.forms = page.forms.map((form, formIndex) => {
-                const formResult = pageResult.forms.find((f) => f.formIndex === formIndex);
-                if (!formResult)
-                    return form;
-                return {
-                    ...form,
-                    semanticName: formResult.semanticName,
-                    description: formResult.description,
-                };
-            });
-            updatedPage.buttons = page.buttons.map((btn, buttonIndex) => {
-                const btnResult = pageResult.buttons.find((b) => b.buttonIndex === buttonIndex);
-                if (!btnResult)
-                    return btn;
-                return {
-                    ...btn,
-                    semanticAction: btnResult.semanticAction,
-                    description: btnResult.description,
-                };
-            });
-            updatedPage.links = page.links.map((link, linkIndex) => {
-                const linkResult = pageResult.links.find((l) => l.linkIndex === linkIndex);
-                if (!linkResult)
-                    return link;
-                return {
-                    ...link,
-                    semanticAction: linkResult.semanticAction,
-                };
-            });
-            return updatedPage;
-        });
-        const totalNamed = enriched.reduce((n, p) => n +
-            p.forms.filter((f) => f.semanticName).length +
-            p.buttons.filter((b) => b.semanticAction).length +
-            p.links.filter((l) => l.semanticAction).length, 0);
-        logger.info(`Semantic analysis complete: ${totalNamed} elements named`);
-        return enriched;
-    }
-    catch (err) {
-        logger.warn(`Semantic analysis failed, using defaults: ${err instanceof Error ? err.message : err}`);
-        return pages;
-    }
-}

package/dist/analyzer/site-crawler.d.ts

@@ -1,38 +0,0 @@
-/**
- * Crawls a website using Playwright, visiting pages breadth-first
- * and extracting interactive elements from each page.
- */
-import type { Entry } from 'har-format';
-import type { SiteDescriptor } from '../types/site.js';
-export interface CrawlOptions {
-    /** Target URL to start crawling from */
-    url: string;
-    /** Maximum crawl depth (default: 2) */
-    depth?: number;
-    /** Maximum number of pages to visit (default: 20) */
-    maxPages?: number;
-    /** Idle timeout in ms before auto-closing (default: 5 min) */
-    timeout?: number;
-    /** Run browser in headless mode (default: false) */
-    headless?: boolean;
-    /** Viewport dimensions */
-    viewport?: {
-        width: number;
-        height: number;
-    };
-    /** Whether to capture screenshots during analysis (default: true) */
-    captureScreenshots?: boolean;
-    /** Whether to capture HAR entries during crawl (for hybrid mode) */
-    captureHar?: boolean;
-}
-export interface CrawlResult {
-    siteDescriptor: SiteDescriptor;
-    /** Screenshots keyed by pageId */
-    screenshots: Map<string, Buffer>;
-    /** HAR entries captured during the crawl (only when captureHar is true) */
-    harEntries?: Entry[];
-}
-/**
- * Crawl a website and return a SiteDescriptor with all discovered pages.
- */
-export declare function crawlSite(options: CrawlOptions): Promise<CrawlResult>;

package/dist/analyzer/site-crawler.js

@@ -1,235 +0,0 @@
-/**
- * Crawls a website using Playwright, visiting pages breadth-first
- * and extracting interactive elements from each page.
- */
-import { chromium } from 'playwright';
-import { parsePage } from './dom-parser.js';
-import { captureViewportScreenshot } from './screenshot-capture.js';
-import { logger } from '../utils/logger.js';
-import crypto from 'node:crypto';
-const DEFAULT_DEPTH = 2;
-const DEFAULT_MAX_PAGES = 20;
-const DEFAULT_TIMEOUT = 5 * 60 * 1000;
-const DEFAULT_VIEWPORT = { width: 1280, height: 720 };
-/**
- * Crawl a website and return a SiteDescriptor with all discovered pages.
- */
-export async function crawlSite(options) {
-    const depth = options.depth ?? DEFAULT_DEPTH;
-    const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
-    const timeout = options.timeout ?? DEFAULT_TIMEOUT;
-    const viewport = options.viewport ?? DEFAULT_VIEWPORT;
-    const captureScreenshots = options.captureScreenshots ?? true;
-    const parsedUrl = new URL(options.url);
-    if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
-        throw new Error('Only http/https URLs are supported');
-    }
-    const baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
-    const captureHar = options.captureHar ?? false;
-    const pages = [];
-    const screenshots = new Map();
-    const harEntries = [];
-    const pendingRequests = new Map();
-    const visited = new Set();
-    const queue = [
-        { url: options.url, currentDepth: 0 },
-    ];
-    let browser;
-    try {
-        browser = await chromium.launch({ headless: options.headless ?? false });
-        const context = await browser.newContext({ viewport });
-        const page = await context.newPage();
-        // Optionally capture network requests as HAR entries during crawl
-        if (captureHar) {
-            page.on('request', (request) => {
-                pendingRequests.set(request, { startTime: Date.now() });
-            });
-            page.on('response', async (response) => {
-                const request = response.request();
-                const pending = pendingRequests.get(request);
-                if (!pending)
-                    return;
-                pendingRequests.delete(request);
-                try {
-                    const entry = await buildCrawlHarEntry(request, response, pending.startTime);
-                    harEntries.push(entry);
-                }
-                catch {
-                    // Some responses can't be read (redirects, aborted)
-                }
-            });
-        }
-        logger.info(`Crawling site: ${options.url} (depth: ${depth}, max pages: ${maxPages})`);
-        let lastActivityTime = Date.now();
-        while (queue.length > 0 && pages.length < maxPages) {
-            const item = queue.shift();
-            const normalizedUrl = normalizeUrl(item.url);
-            // Skip if already visited or different origin
-            if (visited.has(normalizedUrl))
-                continue;
-            if (!item.url.startsWith(baseUrl))
-                continue;
-            visited.add(normalizedUrl);
-            // Check idle timeout
-            if (Date.now() - lastActivityTime > timeout) {
-                logger.warn('Idle timeout reached during crawl');
-                break;
-            }
-            try {
-                logger.info(`[${pages.length + 1}/${maxPages}] Visiting: ${item.url}`);
-                await page.goto(item.url, {
-                    waitUntil: 'domcontentloaded',
-                    timeout: 15_000,
-                });
-                // Wait briefly for dynamic content to render
-                await page.waitForTimeout(1000);
-                lastActivityTime = Date.now();
-                // Parse the page DOM
-                const pageDescriptor = await parsePage(page);
-                pageDescriptor.url = item.url;
-                // Capture screenshot if enabled
-                if (captureScreenshots) {
-                    const screenshot = await captureViewportScreenshot(page);
-                    pageDescriptor.screenshotHash = screenshot.hash;
-                    screenshots.set(pageDescriptor.pageId, screenshot.data);
-                }
-                pages.push(pageDescriptor);
-                // Queue navigation links for further crawling
-                if (item.currentDepth < depth) {
-                    for (const link of pageDescriptor.links) {
-                        if (link.isNavigation && !visited.has(normalizeUrl(link.href))) {
-                            queue.push({ url: link.href, currentDepth: item.currentDepth + 1 });
-                        }
-                    }
-                }
-            }
-            catch (err) {
-                logger.warn(`Failed to crawl ${item.url}: ${err}`);
-            }
-        }
-        // Close browser
-        await browser.close().catch(() => { });
-        browser = undefined;
-    }
-    catch (error) {
-        if (browser)
-            await browser.close().catch(() => { });
-        throw error;
-    }
-    logger.info(`Crawl complete: ${pages.length} pages discovered`);
-    // Extract site metadata from the first page
-    const metadata = {
-        title: pages[0]?.title,
-        description: undefined,
-        favicon: undefined,
-    };
-    const siteDescriptor = {
-        siteId: generateSiteId(baseUrl),
-        baseUrl,
-        pages,
-        analyzedAt: new Date().toISOString(),
-        version: 1,
-        crawlDepth: depth,
-        metadata,
-    };
-    return {
-        siteDescriptor,
-        screenshots,
-        ...(captureHar ? { harEntries } : {}),
-    };
-}
-// ─── HAR Capture (for hybrid mode) ─────────────────────────────────
-const MAX_RESPONSE_BODY_BYTES = 5 * 1024 * 1024;
-const SKIP_BODY_MIME_TYPES = ['image/', 'video/', 'audio/', 'font/', 'application/octet-stream'];
-async function buildCrawlHarEntry(request, response, startTime) {
-    const elapsed = Date.now() - startTime;
-    const url = request.url();
-    const parsedUrl = new URL(url);
-    const requestHeaders = Object.entries(request.headers()).map(([name, value]) => ({
-        name,
-        value,
-    }));
-    const queryString = [...parsedUrl.searchParams.entries()].map(([name, value]) => ({
-        name,
-        value,
-    }));
-    const postData = request.postData();
-    const contentTypeHeader = request.headers()['content-type'] ?? '';
-    const responseHeaders = Object.entries(response.headers()).map(([name, value]) => ({
-        name,
-        value,
-    }));
-    let responseText;
-    const responseMimeType = response.headers()['content-type'] ?? '';
-    const skipBody = SKIP_BODY_MIME_TYPES.some((m) => responseMimeType.startsWith(m));
-    const contentLength = parseInt(response.headers()['content-length'] ?? '0', 10);
-    if (!skipBody && contentLength <= MAX_RESPONSE_BODY_BYTES) {
-        try {
-            const body = await response.body();
-            if (body.length <= MAX_RESPONSE_BODY_BYTES) {
-                responseText = body.toString('utf-8');
-            }
-        }
-        catch {
-            // Body may not be available
-        }
-    }
-    return {
-        startedDateTime: new Date(startTime).toISOString(),
-        time: elapsed,
-        request: {
-            method: request.method(),
-            url,
-            httpVersion: 'HTTP/1.1',
-            headers: requestHeaders,
-            queryString,
-            cookies: [],
-            headersSize: -1,
-            bodySize: postData ? Buffer.byteLength(postData) : 0,
-            ...(postData
-                ? {
-                    postData: {
-                        mimeType: contentTypeHeader.split(';')[0].trim() || 'application/octet-stream',
-                        text: postData,
-                    },
-                }
-                : {}),
-        },
-        response: {
-            status: response.status(),
-            statusText: response.statusText(),
-            httpVersion: 'HTTP/1.1',
-            headers: responseHeaders,
-            cookies: [],
-            content: {
-                size: responseText ? Buffer.byteLength(responseText) : 0,
-                mimeType: responseMimeType.split(';')[0].trim() || 'application/octet-stream',
-                ...(responseText ? { text: responseText } : {}),
-            },
-            redirectURL: '',
-            headersSize: -1,
-            bodySize: responseText ? Buffer.byteLength(responseText) : 0,
-        },
-        cache: {},
-        timings: {
-            send: 1,
-            wait: Math.max(1, elapsed - 2),
-            receive: 1,
-        },
-    };
-}
-// ─── Helpers ────────────────────────────────────────────────────────
-function normalizeUrl(url) {
-    try {
-        const parsed = new URL(url);
-        // Remove trailing slash, fragment, and normalize
-        return `${parsed.origin}${parsed.pathname.replace(/\/$/, '')}${parsed.search}`;
-    }
-    catch {
-        return url;
-    }
-}
-function generateSiteId(baseUrl) {
-    const hash = crypto.createHash('sha256').update(baseUrl).digest('hex').slice(0, 12);
-    return `site_${hash}`;
-}