@detiq/crawler 1.1.9 → 1.1.10

package/dist/live-view.js

@@ -1,118 +1,101 @@
-import { chromium } from 'playwright';
-import { randomBytes } from 'node:crypto';
-import { logger } from './logger.js';
-const sessions = new Map();
-const SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min
+import { chromium } from "playwright";
+import { randomBytes } from "node:crypto";
+import { logger } from "./logger.js";
+const sessions = /* @__PURE__ */ new Map();
+const SESSION_TTL_MS = 15 * 60 * 1e3;
 function generateId() {
-    return randomBytes(16).toString('hex');
+  return randomBytes(16).toString("hex");
 }
-/**
- * Start a live view session: launches a headless browser, navigates to appUrl,
- * begins capturing frames. Returns the session ID.
- */
-export async function startLiveView(appUrl, storageState) {
-    const id = generateId();
-    const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === 'linux';
-    const browser = await chromium.launch({
-        headless: true,
-        args: isDocker ? ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] : [],
-    });
-    const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
-    // Restore session if provided
-    if (storageState) {
-        try {
-            const state = JSON.parse(storageState);
-            if (state.cookies?.length)
-                await context.addCookies(state.cookies);
-        }
-        catch { /* ignore */ }
+async function startLiveView(appUrl, storageState) {
+  const id = generateId();
+  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === "linux";
+  const browser = await chromium.launch({
+    headless: true,
+    args: isDocker ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu"] : []
+  });
+  const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+  if (storageState) {
+    try {
+      const state = JSON.parse(storageState);
+      if (state.cookies?.length) await context.addCookies(state.cookies);
+    } catch {
     }
-    const page = await context.newPage();
-    await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
-    const session = {
-        id, browser, context, page, appUrl,
-        lastFrame: null, streaming: true, frameInterval: null,
-        createdAt: Date.now(),
-    };
-    // Start capturing frames (every 500ms)
-    session.frameInterval = setInterval(async () => {
-        if (!session.streaming)
-            return;
-        try {
-            const buf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false });
-            session.lastFrame = buf.toString('base64');
-        }
-        catch {
-            session.streaming = false;
-        }
-    }, 500);
-    sessions.set(id, session);
-    // Auto-cleanup after TTL
-    setTimeout(() => closeLiveView(id).catch(() => { }), SESSION_TTL_MS);
-    logger.info({ id, appUrl }, '[live-view] Session started');
-    return id;
+  }
+  const page = await context.newPage();
+  await page.goto(appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
+  const session = {
+    id,
+    browser,
+    context,
+    page,
+    appUrl,
+    lastFrame: null,
+    streaming: true,
+    frameInterval: null,
+    createdAt: Date.now()
+  };
+  session.frameInterval = setInterval(async () => {
+    if (!session.streaming) return;
+    try {
+      const buf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false });
+      session.lastFrame = buf.toString("base64");
+    } catch {
+      session.streaming = false;
+    }
+  }, 500);
+  sessions.set(id, session);
+  setTimeout(() => closeLiveView(id).catch(() => {
+  }), SESSION_TTL_MS);
+  logger.info({ id, appUrl }, "[live-view] Session started");
+  return id;
 }
-/**
- * Get the latest frame for a session (base64 JPEG).
- */
-export function getFrame(id) {
-    return sessions.get(id)?.lastFrame ?? null;
+function getFrame(id) {
+  return sessions.get(id)?.lastFrame ?? null;
 }
-/**
- * Navigate the live view browser to a URL.
- */
-export async function liveViewNavigate(id, url) {
-    const session = sessions.get(id);
-    if (!session)
-        throw new Error(`Live view session ${id} not found`);
-    await session.page.goto(url, { waitUntil: 'domcontentloaded' });
+async function liveViewNavigate(id, url) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  await session.page.goto(url, { waitUntil: "domcontentloaded" });
 }
-/**
- * Capture storageState from the live view session, close the browser, return JSON string.
- */
-export async function completeLiveView(id) {
-    const session = sessions.get(id);
-    if (!session)
-        throw new Error(`Live view session ${id} not found or expired`);
-    const state = await session.context.storageState();
-    const json = JSON.stringify(state);
-    await closeLiveView(id);
-    return json;
+async function completeLiveView(id) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found or expired`);
+  const state = await session.context.storageState();
+  const json = JSON.stringify(state);
+  await closeLiveView(id);
+  return json;
 }
-/**
- * Send a mouse or keyboard input event to the live view browser.
- */
-export async function sendInput(id, event) {
-    const session = sessions.get(id);
-    if (!session)
-        throw new Error(`Live view session ${id} not found`);
-    if (event.type === 'click' && event.x !== undefined && event.y !== undefined) {
-        await session.page.mouse.click(event.x, event.y, { button: event.button ?? 'left' });
-    }
-    else if (event.type === 'type' && event.text) {
-        await session.page.keyboard.type(event.text);
-    }
-    else if (event.type === 'key' && event.key) {
-        await session.page.keyboard.press(event.key);
-    }
+async function sendInput(id, event) {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  if (event.type === "click" && event.x !== void 0 && event.y !== void 0) {
+    await session.page.mouse.click(event.x, event.y, { button: event.button ?? "left" });
+  } else if (event.type === "type" && event.text) {
+    await session.page.keyboard.type(event.text);
+  } else if (event.type === "key" && event.key) {
+    await session.page.keyboard.press(event.key);
+  }
 }
-/**
- * Close a live view session without capturing.
- */
-export async function closeLiveView(id) {
-    const session = sessions.get(id);
-    if (!session)
-        return;
-    session.streaming = false;
-    if (session.frameInterval)
-        clearInterval(session.frameInterval);
-    sessions.delete(id);
-    await session.browser.close().catch(() => { });
-    logger.info({ id }, '[live-view] Session closed');
+async function closeLiveView(id) {
+  const session = sessions.get(id);
+  if (!session) return;
+  session.streaming = false;
+  if (session.frameInterval) clearInterval(session.frameInterval);
+  sessions.delete(id);
+  await session.browser.close().catch(() => {
+  });
+  logger.info({ id }, "[live-view] Session closed");
 }
-/**
- * List all active session IDs (for health checks / cleanup).
- */
-export function listLiveViews() {
-    return [...sessions.keys()];
+function listLiveViews() {
+  return [...sessions.keys()];
 }
+export {
+  closeLiveView,
+  completeLiveView,
+  getFrame,
+  listLiveViews,
+  liveViewNavigate,
+  sendInput,
+  startLiveView
+};
+//# sourceMappingURL=live-view.js.map

package/dist/locator-extractor.js

@@ -1,199 +1,163 @@
-/**
- * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
- * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
- */
-/** Parse the notes JSON string stored on Element records. */
 function parseNotes(notes) {
-    if (!notes)
-        return {};
-    try {
-        const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
-        if (parsed && typeof parsed === 'object')
-            return parsed;
-    }
-    catch { }
-    return {};
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === "string" ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === "object") return parsed;
+  } catch {
+  }
+  return {};
 }
-/** Map crawler role strings to Playwright role values. */
 function mapRole(role) {
-    if (!role)
-        return null;
-    const r = role.toLowerCase().trim();
-    if (r === 'button' || r === 'submit' || r === 'reset')
-        return 'button';
-    if (r === 'link' || r === 'a' || r === 'anchor')
-        return 'link';
-    if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search')
-        return 'textbox';
-    if (r === 'combobox' || r === 'select' || r === 'dropdown')
-        return 'combobox';
-    if (r === 'checkbox')
-        return 'checkbox';
-    if (r === 'radio')
-        return 'radio';
-    if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6')
-        return 'heading';
-    if (r === 'img' || r === 'image')
-        return 'img';
-    if (r === 'nav' || r === 'navigation')
-        return 'navigation';
-    return r;
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === "button" || r === "submit" || r === "reset") return "button";
+  if (r === "link" || r === "a" || r === "anchor") return "link";
+  if (r === "textbox" || r === "input" || r === "text" || r === "email" || r === "password" || r === "tel" || r === "number" || r === "search") return "textbox";
+  if (r === "combobox" || r === "select" || r === "dropdown") return "combobox";
+  if (r === "checkbox") return "checkbox";
+  if (r === "radio") return "radio";
+  if (r === "heading" || r === "h1" || r === "h2" || r === "h3" || r === "h4" || r === "h5" || r === "h6") return "heading";
+  if (r === "img" || r === "image") return "img";
+  if (r === "nav" || r === "navigation") return "navigation";
+  return r;
 }
-/** Convert "Submit Form Button" → "submitFormButton" */
 function toKey(text) {
-    return text
-        .replace(/[^a-zA-Z0-9\s]/g, ' ')
-        .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
-        .replace(/^(.)/, (_, c) => c.toLowerCase())
-        .replace(/\s/g, '')
-        .slice(0, 40)
-        || 'element';
+  return text.replace(/[^a-zA-Z0-9\s]/g, " ").replace(/\s+(.)/g, (_, c) => c.toUpperCase()).replace(/^(.)/, (_, c) => c.toLowerCase()).replace(/\s/g, "").slice(0, 40) || "element";
 }
-/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
-export function elementToPlaywrightLocator(element) {
-    const attrs = parseNotes(element.notes);
-    const testId = attrs['data-testid'];
-    const ariaLabel = attrs['aria-label'] || element.meaning;
-    const id = attrs['id'];
-    const placeholder = attrs['placeholder'];
-    const name = attrs['name'];
-    const text = (element.text ?? '').trim();
-    const meaning = (element.meaning ?? '').trim();
-    const role = mapRole(element.role);
-    const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
-    const key = toKey(label);
-    // Priority 1: data-testid
-    if (testId) {
-        return {
-            key,
-            expression: `page.getByTestId(${JSON.stringify(testId)})`,
-            elementType: role ?? 'element',
-            confidence: 'high',
-            source: 'testid',
-            description: meaning || testId,
-        };
-    }
-    // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
-    if (role === 'textbox' || role === 'combobox') {
-        if (ariaLabel) {
-            return {
-                key,
-                expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
-                elementType: 'input',
-                confidence: 'high',
-                source: 'label',
-                description: ariaLabel,
-            };
-        }
-        if (placeholder) {
-            return {
-                key,
-                expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
-                elementType: 'input',
-                confidence: 'high',
-                source: 'placeholder',
-                description: placeholder,
-            };
-        }
-    }
-    // Priority 3: checkbox/radio → getByLabel
-    if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
-        return {
-            key,
-            expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
-            elementType: role,
-            confidence: 'high',
-            source: 'label',
-            description: ariaLabel,
-        };
-    }
-    // Priority 4: button/link with meaningful name → getByRole
-    if ((role === 'button' || role === 'link') && meaning) {
-        return {
-            key,
-            expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
-            elementType: role,
-            confidence: 'high',
-            source: 'role-name',
-            description: meaning,
-        };
-    }
-    // Priority 5: heading → getByRole heading
-    if (role === 'heading' && (meaning || text)) {
-        const name2 = meaning || text;
-        return {
-            key,
-            expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
-            elementType: 'heading',
-            confidence: 'medium',
-            source: 'heading',
-            description: name2,
-        };
-    }
-    // Priority 6: existing CSS selector from crawler
-    if (element.selector && element.selector.length > 0) {
-        return {
-            key,
-            expression: `page.locator(${JSON.stringify(element.selector)})`,
-            elementType: role ?? 'element',
-            confidence: 'medium',
-            source: 'id',
-            description: meaning || text || element.selector,
-        };
-    }
-    // Priority 7: id from attrs
-    if (id) {
-        return {
-            key,
-            expression: `page.locator(${JSON.stringify('#' + id)})`,
-            elementType: role ?? 'element',
-            confidence: 'medium',
-            source: 'id',
-            description: meaning || id,
-        };
+function elementToPlaywrightLocator(element) {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs["data-testid"];
+  const ariaLabel = attrs["aria-label"] || element.meaning;
+  const id = attrs["id"];
+  const placeholder = attrs["placeholder"];
+  const name = attrs["name"];
+  const text = (element.text ?? "").trim();
+  const meaning = (element.meaning ?? "").trim();
+  const role = mapRole(element.role);
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || "element";
+  const key = toKey(label);
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? "element",
+      confidence: "high",
+      source: "testid",
+      description: meaning || testId
+    };
+  }
+  if (role === "textbox" || role === "combobox") {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "label",
+        description: ariaLabel
+      };
     }
-    // Priority 8: fallback to text content
-    if (meaning || text) {
-        const t = meaning || text;
-        return {
-            key,
-            expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
-            elementType: role ?? 'text',
-            confidence: 'low',
-            source: 'text',
-            description: t,
-        };
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: "input",
+        confidence: "high",
+        source: "placeholder",
+        description: placeholder
+      };
     }
-    return null;
+  }
+  if ((role === "checkbox" || role === "radio") && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: "high",
+      source: "label",
+      description: ariaLabel
+    };
+  }
+  if ((role === "button" || role === "link") && meaning) {
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+      elementType: role,
+      confidence: "high",
+      source: "role-name",
+      description: meaning
+    };
+  }
+  if (role === "heading" && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: "heading",
+      confidence: "medium",
+      source: "heading",
+      description: name2
+    };
+  }
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || text || element.selector
+    };
+  }
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify("#" + id)})`,
+      elementType: role ?? "element",
+      confidence: "medium",
+      source: "id",
+      description: meaning || id
+    };
+  }
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? "text",
+      confidence: "low",
+      source: "text",
+      description: t
+    };
+  }
+  return null;
 }
-/**
- * Convert an array of DB Element records to deduplicated Playwright locators.
- * Deduplicates by key — higher confidence wins.
- */
-export function elementsToPlaywrightLocators(elements) {
-    const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
-    const byKey = new Map();
-    for (const el of elements) {
-        const loc = elementToPlaywrightLocator(el);
-        if (!loc)
-            continue;
-        const existing = byKey.get(loc.key);
-        if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
-            byKey.set(loc.key, loc);
-        }
+function elementsToPlaywrightLocators(elements) {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = /* @__PURE__ */ new Map();
+  for (const el of elements) {
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
     }
-    return Array.from(byKey.values());
+  }
+  return Array.from(byKey.values());
 }
-/**
- * Format locators as a block for injection into LLM script generation prompts.
- * Returns empty string if no locators.
- */
-export function formatLocatorsForPrompt(locators) {
-    if (locators.length === 0)
-        return '';
-    const lines = locators.map(l => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`);
-    return [
-        'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
-        ...lines,
-    ].join('\n');
+function formatLocatorsForPrompt(locators) {
+  if (locators.length === 0) return "";
+  const lines = locators.map(
+    (l) => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    "EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):",
+    ...lines
+  ].join("\n");
 }
+export {
+  elementToPlaywrightLocator,
+  elementsToPlaywrightLocators,
+  formatLocatorsForPrompt
+};
+//# sourceMappingURL=locator-extractor.js.map

package/dist/logger.js

@@ -1,8 +1,10 @@
-import pino from 'pino';
-export const logger = pino({
-    level: process.env.LOG_LEVEL ?? 'info',
-    ...(process.env.NODE_ENV === 'development'
-        ? { transport: { target: 'pino-pretty', options: { colorize: true } } }
-        : {}),
-    base: { service: 'crawler' },
+import pino from "pino";
+const logger = pino({
+  level: process.env.LOG_LEVEL ?? "info",
+  ...process.env.NODE_ENV === "development" ? { transport: { target: "pino-pretty", options: { colorize: true } } } : {},
+  base: { service: "crawler" }
 });
+export {
+  logger
+};
+//# sourceMappingURL=logger.js.map