@detiq/crawler 1.1.8 → 1.1.9

package/dist/live-view.js

@@ -1,101 +1,118 @@
-import { chromium } from "playwright";
-import { randomBytes } from "node:crypto";
-import { logger } from "./logger.js";
-const sessions = /* @__PURE__ */ new Map();
-const SESSION_TTL_MS = 15 * 60 * 1e3;
+import { chromium } from 'playwright';
+import { randomBytes } from 'node:crypto';
+import { logger } from './logger.js';
+const sessions = new Map();
+const SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min
 function generateId() {
-  return randomBytes(16).toString("hex");
+    return randomBytes(16).toString('hex');
 }
-async function startLiveView(appUrl, storageState) {
-  const id = generateId();
-  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === "linux";
-  const browser = await chromium.launch({
-    headless: true,
-    args: isDocker ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu"] : []
-  });
-  const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
-  if (storageState) {
-    try {
-      const state = JSON.parse(storageState);
-      if (state.cookies?.length) await context.addCookies(state.cookies);
-    } catch {
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export async function startLiveView(appUrl, storageState) {
+    const id = generateId();
+    const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === 'linux';
+    const browser = await chromium.launch({
+        headless: true,
+        args: isDocker ? ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] : [],
+    });
+    const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+    // Restore session if provided
+    if (storageState) {
+        try {
+            const state = JSON.parse(storageState);
+            if (state.cookies?.length)
+                await context.addCookies(state.cookies);
+        }
+        catch { /* ignore */ }
     }
-  }
-  const page = await context.newPage();
-  await page.goto(appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
-  const session = {
-    id,
-    browser,
-    context,
-    page,
-    appUrl,
-    lastFrame: null,
-    streaming: true,
-    frameInterval: null,
-    createdAt: Date.now()
-  };
-  session.frameInterval = setInterval(async () => {
-    if (!session.streaming) return;
-    try {
-      const buf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false });
-      session.lastFrame = buf.toString("base64");
-    } catch {
-      session.streaming = false;
-    }
-  }, 500);
-  sessions.set(id, session);
-  setTimeout(() => closeLiveView(id).catch(() => {
-  }), SESSION_TTL_MS);
-  logger.info({ id, appUrl }, "[live-view] Session started");
-  return id;
+    const page = await context.newPage();
+    await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
+    const session = {
+        id, browser, context, page, appUrl,
+        lastFrame: null, streaming: true, frameInterval: null,
+        createdAt: Date.now(),
+    };
+    // Start capturing frames (every 500ms)
+    session.frameInterval = setInterval(async () => {
+        if (!session.streaming)
+            return;
+        try {
+            const buf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false });
+            session.lastFrame = buf.toString('base64');
+        }
+        catch {
+            session.streaming = false;
+        }
+    }, 500);
+    sessions.set(id, session);
+    // Auto-cleanup after TTL
+    setTimeout(() => closeLiveView(id).catch(() => { }), SESSION_TTL_MS);
+    logger.info({ id, appUrl }, '[live-view] Session started');
+    return id;
 }
-function getFrame(id) {
-  return sessions.get(id)?.lastFrame ?? null;
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export function getFrame(id) {
+    return sessions.get(id)?.lastFrame ?? null;
 }
-async function liveViewNavigate(id, url) {
-  const session = sessions.get(id);
-  if (!session) throw new Error(`Live view session ${id} not found`);
-  await session.page.goto(url, { waitUntil: "domcontentloaded" });
+/**
+ * Navigate the live view browser to a URL.
+ */
+export async function liveViewNavigate(id, url) {
+    const session = sessions.get(id);
+    if (!session)
+        throw new Error(`Live view session ${id} not found`);
+    await session.page.goto(url, { waitUntil: 'domcontentloaded' });
 }
-async function completeLiveView(id) {
-  const session = sessions.get(id);
-  if (!session) throw new Error(`Live view session ${id} not found or expired`);
-  const state = await session.context.storageState();
-  const json = JSON.stringify(state);
-  await closeLiveView(id);
-  return json;
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export async function completeLiveView(id) {
+    const session = sessions.get(id);
+    if (!session)
+        throw new Error(`Live view session ${id} not found or expired`);
+    const state = await session.context.storageState();
+    const json = JSON.stringify(state);
+    await closeLiveView(id);
+    return json;
 }
-async function sendInput(id, event) {
-  const session = sessions.get(id);
-  if (!session) throw new Error(`Live view session ${id} not found`);
-  if (event.type === "click" && event.x !== void 0 && event.y !== void 0) {
-    await session.page.mouse.click(event.x, event.y, { button: event.button ?? "left" });
-  } else if (event.type === "type" && event.text) {
-    await session.page.keyboard.type(event.text);
-  } else if (event.type === "key" && event.key) {
-    await session.page.keyboard.press(event.key);
-  }
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export async function sendInput(id, event) {
+    const session = sessions.get(id);
+    if (!session)
+        throw new Error(`Live view session ${id} not found`);
+    if (event.type === 'click' && event.x !== undefined && event.y !== undefined) {
+        await session.page.mouse.click(event.x, event.y, { button: event.button ?? 'left' });
+    }
+    else if (event.type === 'type' && event.text) {
+        await session.page.keyboard.type(event.text);
+    }
+    else if (event.type === 'key' && event.key) {
+        await session.page.keyboard.press(event.key);
+    }
 }
-async function closeLiveView(id) {
-  const session = sessions.get(id);
-  if (!session) return;
-  session.streaming = false;
-  if (session.frameInterval) clearInterval(session.frameInterval);
-  sessions.delete(id);
-  await session.browser.close().catch(() => {
-  });
-  logger.info({ id }, "[live-view] Session closed");
+/**
+ * Close a live view session without capturing.
+ */
+export async function closeLiveView(id) {
+    const session = sessions.get(id);
+    if (!session)
+        return;
+    session.streaming = false;
+    if (session.frameInterval)
+        clearInterval(session.frameInterval);
+    sessions.delete(id);
+    await session.browser.close().catch(() => { });
+    logger.info({ id }, '[live-view] Session closed');
 }
-function listLiveViews() {
-  return [...sessions.keys()];
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export function listLiveViews() {
+    return [...sessions.keys()];
 }
-export {
-  closeLiveView,
-  completeLiveView,
-  getFrame,
-  listLiveViews,
-  liveViewNavigate,
-  sendInput,
-  startLiveView
-};
-//# sourceMappingURL=live-view.js.map

package/dist/locator-extractor.js

@@ -1,163 +1,199 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+/** Parse the notes JSON string stored on Element records. */
 function parseNotes(notes) {
-  if (!notes) return {};
-  try {
-    const parsed = typeof notes === "string" ? JSON.parse(notes) : notes;
-    if (parsed && typeof parsed === "object") return parsed;
-  } catch {
-  }
-  return {};
+    if (!notes)
+        return {};
+    try {
+        const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
+        if (parsed && typeof parsed === 'object')
+            return parsed;
+    }
+    catch { }
+    return {};
 }
+/** Map crawler role strings to Playwright role values. */
 function mapRole(role) {
-  if (!role) return null;
-  const r = role.toLowerCase().trim();
-  if (r === "button" || r === "submit" || r === "reset") return "button";
-  if (r === "link" || r === "a" || r === "anchor") return "link";
-  if (r === "textbox" || r === "input" || r === "text" || r === "email" || r === "password" || r === "tel" || r === "number" || r === "search") return "textbox";
-  if (r === "combobox" || r === "select" || r === "dropdown") return "combobox";
-  if (r === "checkbox") return "checkbox";
-  if (r === "radio") return "radio";
-  if (r === "heading" || r === "h1" || r === "h2" || r === "h3" || r === "h4" || r === "h5" || r === "h6") return "heading";
-  if (r === "img" || r === "image") return "img";
-  if (r === "nav" || r === "navigation") return "navigation";
-  return r;
+    if (!role)
+        return null;
+    const r = role.toLowerCase().trim();
+    if (r === 'button' || r === 'submit' || r === 'reset')
+        return 'button';
+    if (r === 'link' || r === 'a' || r === 'anchor')
+        return 'link';
+    if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search')
+        return 'textbox';
+    if (r === 'combobox' || r === 'select' || r === 'dropdown')
+        return 'combobox';
+    if (r === 'checkbox')
+        return 'checkbox';
+    if (r === 'radio')
+        return 'radio';
+    if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6')
+        return 'heading';
+    if (r === 'img' || r === 'image')
+        return 'img';
+    if (r === 'nav' || r === 'navigation')
+        return 'navigation';
+    return r;
 }
+/** Convert "Submit Form Button" → "submitFormButton" */
 function toKey(text) {
-  return text.replace(/[^a-zA-Z0-9\s]/g, " ").replace(/\s+(.)/g, (_, c) => c.toUpperCase()).replace(/^(.)/, (_, c) => c.toLowerCase()).replace(/\s/g, "").slice(0, 40) || "element";
+    return text
+        .replace(/[^a-zA-Z0-9\s]/g, ' ')
+        .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
+        .replace(/^(.)/, (_, c) => c.toLowerCase())
+        .replace(/\s/g, '')
+        .slice(0, 40)
+        || 'element';
 }
-function elementToPlaywrightLocator(element) {
-  const attrs = parseNotes(element.notes);
-  const testId = attrs["data-testid"];
-  const ariaLabel = attrs["aria-label"] || element.meaning;
-  const id = attrs["id"];
-  const placeholder = attrs["placeholder"];
-  const name = attrs["name"];
-  const text = (element.text ?? "").trim();
-  const meaning = (element.meaning ?? "").trim();
-  const role = mapRole(element.role);
-  const label = meaning || text || testId || ariaLabel || placeholder || id || name || "element";
-  const key = toKey(label);
-  if (testId) {
-    return {
-      key,
-      expression: `page.getByTestId(${JSON.stringify(testId)})`,
-      elementType: role ?? "element",
-      confidence: "high",
-      source: "testid",
-      description: meaning || testId
-    };
-  }
-  if (role === "textbox" || role === "combobox") {
-    if (ariaLabel) {
-      return {
-        key,
-        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
-        elementType: "input",
-        confidence: "high",
-        source: "label",
-        description: ariaLabel
-      };
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export function elementToPlaywrightLocator(element) {
+    const attrs = parseNotes(element.notes);
+    const testId = attrs['data-testid'];
+    const ariaLabel = attrs['aria-label'] || element.meaning;
+    const id = attrs['id'];
+    const placeholder = attrs['placeholder'];
+    const name = attrs['name'];
+    const text = (element.text ?? '').trim();
+    const meaning = (element.meaning ?? '').trim();
+    const role = mapRole(element.role);
+    const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
+    const key = toKey(label);
+    // Priority 1: data-testid
+    if (testId) {
+        return {
+            key,
+            expression: `page.getByTestId(${JSON.stringify(testId)})`,
+            elementType: role ?? 'element',
+            confidence: 'high',
+            source: 'testid',
+            description: meaning || testId,
+        };
+    }
+    // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
+    if (role === 'textbox' || role === 'combobox') {
+        if (ariaLabel) {
+            return {
+                key,
+                expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+                elementType: 'input',
+                confidence: 'high',
+                source: 'label',
+                description: ariaLabel,
+            };
+        }
+        if (placeholder) {
+            return {
+                key,
+                expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+                elementType: 'input',
+                confidence: 'high',
+                source: 'placeholder',
+                description: placeholder,
+            };
+        }
+    }
+    // Priority 3: checkbox/radio → getByLabel
+    if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
+        return {
+            key,
+            expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+            elementType: role,
+            confidence: 'high',
+            source: 'label',
+            description: ariaLabel,
+        };
+    }
+    // Priority 4: button/link with meaningful name → getByRole
+    if ((role === 'button' || role === 'link') && meaning) {
+        return {
+            key,
+            expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+            elementType: role,
+            confidence: 'high',
+            source: 'role-name',
+            description: meaning,
+        };
+    }
+    // Priority 5: heading → getByRole heading
+    if (role === 'heading' && (meaning || text)) {
+        const name2 = meaning || text;
+        return {
+            key,
+            expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+            elementType: 'heading',
+            confidence: 'medium',
+            source: 'heading',
+            description: name2,
+        };
+    }
+    // Priority 6: existing CSS selector from crawler
+    if (element.selector && element.selector.length > 0) {
+        return {
+            key,
+            expression: `page.locator(${JSON.stringify(element.selector)})`,
+            elementType: role ?? 'element',
+            confidence: 'medium',
+            source: 'id',
+            description: meaning || text || element.selector,
+        };
+    }
+    // Priority 7: id from attrs
+    if (id) {
+        return {
+            key,
+            expression: `page.locator(${JSON.stringify('#' + id)})`,
+            elementType: role ?? 'element',
+            confidence: 'medium',
+            source: 'id',
+            description: meaning || id,
+        };
     }
-    if (placeholder) {
-      return {
-        key,
-        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
-        elementType: "input",
-        confidence: "high",
-        source: "placeholder",
-        description: placeholder
-      };
+    // Priority 8: fallback to text content
+    if (meaning || text) {
+        const t = meaning || text;
+        return {
+            key,
+            expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+            elementType: role ?? 'text',
+            confidence: 'low',
+            source: 'text',
+            description: t,
+        };
     }
-  }
-  if ((role === "checkbox" || role === "radio") && ariaLabel) {
-    return {
-      key,
-      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
-      elementType: role,
-      confidence: "high",
-      source: "label",
-      description: ariaLabel
-    };
-  }
-  if ((role === "button" || role === "link") && meaning) {
-    return {
-      key,
-      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
-      elementType: role,
-      confidence: "high",
-      source: "role-name",
-      description: meaning
-    };
-  }
-  if (role === "heading" && (meaning || text)) {
-    const name2 = meaning || text;
-    return {
-      key,
-      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
-      elementType: "heading",
-      confidence: "medium",
-      source: "heading",
-      description: name2
-    };
-  }
-  if (element.selector && element.selector.length > 0) {
-    return {
-      key,
-      expression: `page.locator(${JSON.stringify(element.selector)})`,
-      elementType: role ?? "element",
-      confidence: "medium",
-      source: "id",
-      description: meaning || text || element.selector
-    };
-  }
-  if (id) {
-    return {
-      key,
-      expression: `page.locator(${JSON.stringify("#" + id)})`,
-      elementType: role ?? "element",
-      confidence: "medium",
-      source: "id",
-      description: meaning || id
-    };
-  }
-  if (meaning || text) {
-    const t = meaning || text;
-    return {
-      key,
-      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
-      elementType: role ?? "text",
-      confidence: "low",
-      source: "text",
-      description: t
-    };
-  }
-  return null;
+    return null;
 }
-function elementsToPlaywrightLocators(elements) {
-  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
-  const byKey = /* @__PURE__ */ new Map();
-  for (const el of elements) {
-    const loc = elementToPlaywrightLocator(el);
-    if (!loc) continue;
-    const existing = byKey.get(loc.key);
-    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
-      byKey.set(loc.key, loc);
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export function elementsToPlaywrightLocators(elements) {
+    const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+    const byKey = new Map();
+    for (const el of elements) {
+        const loc = elementToPlaywrightLocator(el);
+        if (!loc)
+            continue;
+        const existing = byKey.get(loc.key);
+        if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+            byKey.set(loc.key, loc);
+        }
     }
-  }
-  return Array.from(byKey.values());
+    return Array.from(byKey.values());
 }
-function formatLocatorsForPrompt(locators) {
-  if (locators.length === 0) return "";
-  const lines = locators.map(
-    (l) => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
-  );
-  return [
-    "EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):",
-    ...lines
-  ].join("\n");
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export function formatLocatorsForPrompt(locators) {
+    if (locators.length === 0)
+        return '';
+    const lines = locators.map(l => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`);
+    return [
+        'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
+        ...lines,
+    ].join('\n');
 }
-export {
-  elementToPlaywrightLocator,
-  elementsToPlaywrightLocators,
-  formatLocatorsForPrompt
-};
-//# sourceMappingURL=locator-extractor.js.map

package/dist/logger.js

@@ -1,10 +1,8 @@
-import pino from "pino";
-const logger = pino({
-  level: process.env.LOG_LEVEL ?? "info",
-  ...process.env.NODE_ENV === "development" ? { transport: { target: "pino-pretty", options: { colorize: true } } } : {},
-  base: { service: "crawler" }
+import pino from 'pino';
+export const logger = pino({
+    level: process.env.LOG_LEVEL ?? 'info',
+    ...(process.env.NODE_ENV === 'development'
+        ? { transport: { target: 'pino-pretty', options: { colorize: true } } }
+        : {}),
+    base: { service: 'crawler' },
 });
-export {
-  logger
-};
-//# sourceMappingURL=logger.js.map