@detiq/crawler 1.1.7 → 1.1.9

package/dist/live-view.js

@@ -1,101 +1,118 @@
-import { chromium } from "playwright";
-import { randomBytes } from "node:crypto";
-import { logger } from "./logger.js";
-const sessions = /* @__PURE__ */ new Map();
-const SESSION_TTL_MS = 15 * 60 * 1e3;
+import { chromium } from 'playwright';
+import { randomBytes } from 'node:crypto';
+import { logger } from './logger.js';
+const sessions = new Map();
+const SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min
 function generateId() {
-  return randomBytes(16).toString("hex");
+    return randomBytes(16).toString('hex');
 }
-async function startLiveView(appUrl, storageState) {
-  const id = generateId();
-  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === "linux";
-  const browser = await chromium.launch({
-    headless: true,
-    args: isDocker ? ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu"] : []
-  });
-  const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
-  if (storageState) {
-    try {
-      const state = JSON.parse(storageState);
-      if (state.cookies?.length) await context.addCookies(state.cookies);
-    } catch {
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export async function startLiveView(appUrl, storageState) {
+    const id = generateId();
+    const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === 'linux';
+    const browser = await chromium.launch({
+        headless: true,
+        args: isDocker ? ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] : [],
+    });
+    const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+    // Restore session if provided
+    if (storageState) {
+        try {
+            const state = JSON.parse(storageState);
+            if (state.cookies?.length)
+                await context.addCookies(state.cookies);
+        }
+        catch { /* ignore */ }
     }
-  }
-  const page = await context.newPage();
-  await page.goto(appUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
-  const session = {
-    id,
-    browser,
-    context,
-    page,
-    appUrl,
-    lastFrame: null,
-    streaming: true,
-    frameInterval: null,
-    createdAt: Date.now()
-  };
-  session.frameInterval = setInterval(async () => {
-    if (!session.streaming) return;
-    try {
-      const buf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false });
-      session.lastFrame = buf.toString("base64");
-    } catch {
-      session.streaming = false;
-    }
-  }, 500);
-  sessions.set(id, session);
-  setTimeout(() => closeLiveView(id).catch(() => {
-  }), SESSION_TTL_MS);
-  logger.info({ id, appUrl }, "[live-view] Session started");
-  return id;
+    const page = await context.newPage();
+    await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
+    const session = {
+        id, browser, context, page, appUrl,
+        lastFrame: null, streaming: true, frameInterval: null,
+        createdAt: Date.now(),
+    };
+    // Start capturing frames (every 500ms)
+    session.frameInterval = setInterval(async () => {
+        if (!session.streaming)
+            return;
+        try {
+            const buf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false });
+            session.lastFrame = buf.toString('base64');
+        }
+        catch {
+            session.streaming = false;
+        }
+    }, 500);
+    sessions.set(id, session);
+    // Auto-cleanup after TTL
+    setTimeout(() => closeLiveView(id).catch(() => { }), SESSION_TTL_MS);
+    logger.info({ id, appUrl }, '[live-view] Session started');
+    return id;
 }
-function getFrame(id) {
-  return sessions.get(id)?.lastFrame ?? null;
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export function getFrame(id) {
+    return sessions.get(id)?.lastFrame ?? null;
 }
-async function liveViewNavigate(id, url) {
-  const session = sessions.get(id);
-  if (!session) throw new Error(`Live view session ${id} not found`);
-  await session.page.goto(url, { waitUntil: "domcontentloaded" });
+/**
+ * Navigate the live view browser to a URL.
+ */
+export async function liveViewNavigate(id, url) {
+    const session = sessions.get(id);
+    if (!session)
+        throw new Error(`Live view session ${id} not found`);
+    await session.page.goto(url, { waitUntil: 'domcontentloaded' });
 }
-async function completeLiveView(id) {
-  const session = sessions.get(id);
-  if (!session) throw new Error(`Live view session ${id} not found or expired`);
-  const state = await session.context.storageState();
-  const json = JSON.stringify(state);
-  await closeLiveView(id);
-  return json;
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export async function completeLiveView(id) {
+    const session = sessions.get(id);
+    if (!session)
+        throw new Error(`Live view session ${id} not found or expired`);
+    const state = await session.context.storageState();
+    const json = JSON.stringify(state);
+    await closeLiveView(id);
+    return json;
 }
-async function sendInput(id, event) {
-  const session = sessions.get(id);
-  if (!session) throw new Error(`Live view session ${id} not found`);
-  if (event.type === "click" && event.x !== void 0 && event.y !== void 0) {
-    await session.page.mouse.click(event.x, event.y, { button: event.button ?? "left" });
-  } else if (event.type === "type" && event.text) {
-    await session.page.keyboard.type(event.text);
-  } else if (event.type === "key" && event.key) {
-    await session.page.keyboard.press(event.key);
-  }
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export async function sendInput(id, event) {
+    const session = sessions.get(id);
+    if (!session)
+        throw new Error(`Live view session ${id} not found`);
+    if (event.type === 'click' && event.x !== undefined && event.y !== undefined) {
+        await session.page.mouse.click(event.x, event.y, { button: event.button ?? 'left' });
+    }
+    else if (event.type === 'type' && event.text) {
+        await session.page.keyboard.type(event.text);
+    }
+    else if (event.type === 'key' && event.key) {
+        await session.page.keyboard.press(event.key);
+    }
 }
-async function closeLiveView(id) {
-  const session = sessions.get(id);
-  if (!session) return;
-  session.streaming = false;
-  if (session.frameInterval) clearInterval(session.frameInterval);
-  sessions.delete(id);
-  await session.browser.close().catch(() => {
-  });
-  logger.info({ id }, "[live-view] Session closed");
+/**
+ * Close a live view session without capturing.
+ */
+export async function closeLiveView(id) {
+    const session = sessions.get(id);
+    if (!session)
+        return;
+    session.streaming = false;
+    if (session.frameInterval)
+        clearInterval(session.frameInterval);
+    sessions.delete(id);
+    await session.browser.close().catch(() => { });
+    logger.info({ id }, '[live-view] Session closed');
 }
-function listLiveViews() {
-  return [...sessions.keys()];
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export function listLiveViews() {
+    return [...sessions.keys()];
 }
-export {
-  closeLiveView,
-  completeLiveView,
-  getFrame,
-  listLiveViews,
-  liveViewNavigate,
-  sendInput,
-  startLiveView
-};
-//# sourceMappingURL=live-view.js.map

package/dist/locator-extractor.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+export interface PlaywrightLocator {
+    key: string;
+    expression: string;
+    elementType: string;
+    confidence: 'high' | 'medium' | 'low';
+    source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+    description: string;
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export declare function elementToPlaywrightLocator(element: {
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+}): PlaywrightLocator | null;
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export declare function elementsToPlaywrightLocators(elements: Array<{
+    role?: string | null;
+    meaning?: string | null;
+    text?: string | null;
+    notes?: unknown;
+    selector?: string | null;
+    isInteractive?: boolean | null;
+}>): PlaywrightLocator[];
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export declare function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string;

package/dist/locator-extractor.js

@@ -0,0 +1,199 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+/** Parse the notes JSON string stored on Element records. */
+function parseNotes(notes) {
+    if (!notes)
+        return {};
+    try {
+        const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
+        if (parsed && typeof parsed === 'object')
+            return parsed;
+    }
+    catch { }
+    return {};
+}
+/** Map crawler role strings to Playwright role values. */
+function mapRole(role) {
+    if (!role)
+        return null;
+    const r = role.toLowerCase().trim();
+    if (r === 'button' || r === 'submit' || r === 'reset')
+        return 'button';
+    if (r === 'link' || r === 'a' || r === 'anchor')
+        return 'link';
+    if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search')
+        return 'textbox';
+    if (r === 'combobox' || r === 'select' || r === 'dropdown')
+        return 'combobox';
+    if (r === 'checkbox')
+        return 'checkbox';
+    if (r === 'radio')
+        return 'radio';
+    if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6')
+        return 'heading';
+    if (r === 'img' || r === 'image')
+        return 'img';
+    if (r === 'nav' || r === 'navigation')
+        return 'navigation';
+    return r;
+}
+/** Convert "Submit Form Button" → "submitFormButton" */
+function toKey(text) {
+    return text
+        .replace(/[^a-zA-Z0-9\s]/g, ' ')
+        .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
+        .replace(/^(.)/, (_, c) => c.toLowerCase())
+        .replace(/\s/g, '')
+        .slice(0, 40)
+        || 'element';
+}
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export function elementToPlaywrightLocator(element) {
+    const attrs = parseNotes(element.notes);
+    const testId = attrs['data-testid'];
+    const ariaLabel = attrs['aria-label'] || element.meaning;
+    const id = attrs['id'];
+    const placeholder = attrs['placeholder'];
+    const name = attrs['name'];
+    const text = (element.text ?? '').trim();
+    const meaning = (element.meaning ?? '').trim();
+    const role = mapRole(element.role);
+    const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
+    const key = toKey(label);
+    // Priority 1: data-testid
+    if (testId) {
+        return {
+            key,
+            expression: `page.getByTestId(${JSON.stringify(testId)})`,
+            elementType: role ?? 'element',
+            confidence: 'high',
+            source: 'testid',
+            description: meaning || testId,
+        };
+    }
+    // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
+    if (role === 'textbox' || role === 'combobox') {
+        if (ariaLabel) {
+            return {
+                key,
+                expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+                elementType: 'input',
+                confidence: 'high',
+                source: 'label',
+                description: ariaLabel,
+            };
+        }
+        if (placeholder) {
+            return {
+                key,
+                expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+                elementType: 'input',
+                confidence: 'high',
+                source: 'placeholder',
+                description: placeholder,
+            };
+        }
+    }
+    // Priority 3: checkbox/radio → getByLabel
+    if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
+        return {
+            key,
+            expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+            elementType: role,
+            confidence: 'high',
+            source: 'label',
+            description: ariaLabel,
+        };
+    }
+    // Priority 4: button/link with meaningful name → getByRole
+    if ((role === 'button' || role === 'link') && meaning) {
+        return {
+            key,
+            expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,
+            elementType: role,
+            confidence: 'high',
+            source: 'role-name',
+            description: meaning,
+        };
+    }
+    // Priority 5: heading → getByRole heading
+    if (role === 'heading' && (meaning || text)) {
+        const name2 = meaning || text;
+        return {
+            key,
+            expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+            elementType: 'heading',
+            confidence: 'medium',
+            source: 'heading',
+            description: name2,
+        };
+    }
+    // Priority 6: existing CSS selector from crawler
+    if (element.selector && element.selector.length > 0) {
+        return {
+            key,
+            expression: `page.locator(${JSON.stringify(element.selector)})`,
+            elementType: role ?? 'element',
+            confidence: 'medium',
+            source: 'id',
+            description: meaning || text || element.selector,
+        };
+    }
+    // Priority 7: id from attrs
+    if (id) {
+        return {
+            key,
+            expression: `page.locator(${JSON.stringify('#' + id)})`,
+            elementType: role ?? 'element',
+            confidence: 'medium',
+            source: 'id',
+            description: meaning || id,
+        };
+    }
+    // Priority 8: fallback to text content
+    if (meaning || text) {
+        const t = meaning || text;
+        return {
+            key,
+            expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+            elementType: role ?? 'text',
+            confidence: 'low',
+            source: 'text',
+            description: t,
+        };
+    }
+    return null;
+}
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ */
+export function elementsToPlaywrightLocators(elements) {
+    const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+    const byKey = new Map();
+    for (const el of elements) {
+        const loc = elementToPlaywrightLocator(el);
+        if (!loc)
+            continue;
+        const existing = byKey.get(loc.key);
+        if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+            byKey.set(loc.key, loc);
+        }
+    }
+    return Array.from(byKey.values());
+}
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export function formatLocatorsForPrompt(locators) {
+    if (locators.length === 0)
+        return '';
+    const lines = locators.map(l => `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`);
+    return [
+        'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
+        ...lines,
+    ].join('\n');
+}

package/dist/locator-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/locator-extractor.ts"],
+  "sourcesContent": ["/**\n * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.\n * Priority: data-testid \u2192 aria-label/placeholder (inputs) \u2192 role+name (buttons/links) \u2192 heading \u2192 text \u2192 id\n */\n\nexport interface PlaywrightLocator {\n  key: string;           // camelCase property name for page object\n  expression: string;    // e.g. \"page.getByRole('button', { name: 'Submit' })\"\n  elementType: string;   // button | input | link | heading | text | element\n  confidence: 'high' | 'medium' | 'low';\n  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';\n  description: string;   // human-readable label\n}\n\n/** Parse the notes JSON string stored on Element records. */\nfunction parseNotes(notes: unknown): Record<string, string> {\n  if (!notes) return {};\n  try {\n    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;\n    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;\n  } catch {}\n  return {};\n}\n\n/** Map crawler role strings to Playwright role values. */\nfunction mapRole(role?: string | null): string | null {\n  if (!role) return null;\n  const r = role.toLowerCase().trim();\n  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';\n  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';\n  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';\n  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';\n  if (r === 'checkbox') return 'checkbox';\n  if (r === 'radio') return 'radio';\n  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';\n  if (r === 'img' || r === 'image') return 'img';\n  if (r === 'nav' || r === 'navigation') return 'navigation';\n  return r;\n}\n\n/** Convert \"Submit Form Button\" \u2192 \"submitFormButton\" */\nfunction toKey(text: string): string {\n  return text\n    .replace(/[^a-zA-Z0-9\\s]/g, ' ')\n    .replace(/\\s+(.)/g, (_, c) => c.toUpperCase())\n    .replace(/^(.)/, (_, c) => c.toLowerCase())\n    .replace(/\\s/g, '')\n    .slice(0, 40)\n    || 'element';\n}\n\n/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */\nexport function elementToPlaywrightLocator(element: {\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n}): PlaywrightLocator | null {\n  const attrs = parseNotes(element.notes);\n  const testId = attrs['data-testid'];\n  const ariaLabel = attrs['aria-label'] || element.meaning;\n  const id = attrs['id'];\n  const placeholder = attrs['placeholder'];\n  const name = attrs['name'];\n  const text = (element.text ?? '').trim();\n  const meaning = (element.meaning ?? '').trim();\n  const role = mapRole(element.role);\n\n  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';\n  const key = toKey(label);\n\n  // Priority 1: data-testid\n  if (testId) {\n    return {\n      key,\n      expression: `page.getByTestId(${JSON.stringify(testId)})`,\n      elementType: role ?? 'element',\n      confidence: 'high',\n      source: 'testid',\n      description: meaning || testId,\n    };\n  }\n\n  // Priority 2: input/combobox/textarea with label \u2192 getByLabel; with placeholder \u2192 getByPlaceholder\n  if (role === 'textbox' || role === 'combobox') {\n    if (ariaLabel) {\n      return {\n        key,\n        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'label',\n        description: ariaLabel,\n      };\n    }\n    if (placeholder) {\n      return {\n        key,\n        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,\n        elementType: 'input',\n        confidence: 'high',\n        source: 'placeholder',\n        description: placeholder,\n      };\n    }\n  }\n\n  // Priority 3: checkbox/radio \u2192 getByLabel\n  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {\n    return {\n      key,\n      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,\n      elementType: role,\n      confidence: 'high',\n      source: 'label',\n      description: ariaLabel,\n    };\n  }\n\n  // Priority 4: button/link with meaningful name \u2192 getByRole\n  if ((role === 'button' || role === 'link') && meaning) {\n    return {\n      key,\n      expression: `page.getByRole(${JSON.stringify(role)}, { name: ${JSON.stringify(meaning)} })`,\n      elementType: role,\n      confidence: 'high',\n      source: 'role-name',\n      description: meaning,\n    };\n  }\n\n  // Priority 5: heading \u2192 getByRole heading\n  if (role === 'heading' && (meaning || text)) {\n    const name2 = meaning || text;\n    return {\n      key,\n      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,\n      elementType: 'heading',\n      confidence: 'medium',\n      source: 'heading',\n      description: name2,\n    };\n  }\n\n  // Priority 6: existing CSS selector from crawler\n  if (element.selector && element.selector.length > 0) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify(element.selector)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || text || element.selector,\n    };\n  }\n\n  // Priority 7: id from attrs\n  if (id) {\n    return {\n      key,\n      expression: `page.locator(${JSON.stringify('#' + id)})`,\n      elementType: role ?? 'element',\n      confidence: 'medium',\n      source: 'id',\n      description: meaning || id,\n    };\n  }\n\n  // Priority 8: fallback to text content\n  if (meaning || text) {\n    const t = meaning || text;\n    return {\n      key,\n      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,\n      elementType: role ?? 'text',\n      confidence: 'low',\n      source: 'text',\n      description: t,\n    };\n  }\n\n  return null;\n}\n\n/**\n * Convert an array of DB Element records to deduplicated Playwright locators.\n * Deduplicates by key \u2014 higher confidence wins.\n */\nexport function elementsToPlaywrightLocators(elements: Array<{\n  role?: string | null;\n  meaning?: string | null;\n  text?: string | null;\n  notes?: unknown;\n  selector?: string | null;\n  isInteractive?: boolean | null;\n}>): PlaywrightLocator[] {\n  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };\n  const byKey = new Map<string, PlaywrightLocator>();\n\n  for (const el of elements) {\n    const loc = elementToPlaywrightLocator(el);\n    if (!loc) continue;\n    const existing = byKey.get(loc.key);\n    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {\n      byKey.set(loc.key, loc);\n    }\n  }\n\n  return Array.from(byKey.values());\n}\n\n/**\n * Format locators as a block for injection into LLM script generation prompts.\n * Returns empty string if no locators.\n */\nexport function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {\n  if (locators.length === 0) return '';\n  const lines = locators.map(l =>\n    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`\n  );\n  return [\n    'EXTRACTED PAGE LOCATORS (use these EXACTLY \u2014 do not invent other selectors; these were verified against the live page):',\n    ...lines,\n  ].join('\\n');\n}\n"],
+  "mappings": "AAeA,SAAS,WAAW,OAAwC;AAC1D,MAAI,CAAC,MAAO,QAAO,CAAC;AACpB,MAAI;AACF,UAAM,SAAS,OAAO,UAAU,WAAW,KAAK,MAAM,KAAK,IAAI;AAC/D,QAAI,UAAU,OAAO,WAAW,SAAU,QAAO;AAAA,EACnD,QAAQ;AAAA,EAAC;AACT,SAAO,CAAC;AACV;AAGA,SAAS,QAAQ,MAAqC;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,IAAI,KAAK,YAAY,EAAE,KAAK;AAClC,MAAI,MAAM,YAAY,MAAM,YAAY,MAAM,QAAS,QAAO;AAC9D,MAAI,MAAM,UAAU,MAAM,OAAO,MAAM,SAAU,QAAO;AACxD,MAAI,MAAM,aAAa,MAAM,WAAW,MAAM,UAAU,MAAM,WAAW,MAAM,cAAc,MAAM,SAAS,MAAM,YAAY,MAAM,SAAU,QAAO;AACrJ,MAAI,MAAM,cAAc,MAAM,YAAY,MAAM,WAAY,QAAO;AACnE,MAAI,MAAM,WAAY,QAAO;AAC7B,MAAI,MAAM,QAAS,QAAO;AAC1B,MAAI,MAAM,aAAa,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,QAAQ,MAAM,KAAM,QAAO;AAChH,MAAI,MAAM,SAAS,MAAM,QAAS,QAAO;AACzC,MAAI,MAAM,SAAS,MAAM,aAAc,QAAO;AAC9C,SAAO;AACT;AAGA,SAAS,MAAM,MAAsB;AACnC,SAAO,KACJ,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,WAAW,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EAC5C,QAAQ,QAAQ,CAAC,GAAG,MAAM,EAAE,YAAY,CAAC,EACzC,QAAQ,OAAO,EAAE,EACjB,MAAM,GAAG,EAAE,KACT;AACP;AAGO,SAAS,2BAA2B,SAMd;AAC3B,QAAM,QAAQ,WAAW,QAAQ,KAAK;AACtC,QAAM,SAAS,MAAM,aAAa;AAClC,QAAM,YAAY,MAAM,YAAY,KAAK,QAAQ;AACjD,QAAM,KAAK,MAAM,IAAI;AACrB,QAAM,cAAc,MAAM,aAAa;AACvC,QAAM,OAAO,MAAM,MAAM;AACzB,QAAM,QAAQ,QAAQ,QAAQ,IAAI,KAAK;AACvC,QAAM,WAAW,QAAQ,WAAW,IAAI,KAAK;AAC7C,QAAM,OAAO,QAAQ,QAAQ,IAAI;AAEjC,QAAM,QAAQ,WAAW,QAAQ,UAAU,aAAa,eAAe,MAAM,QAAQ;AACrF,QAAM,MAAM,MAAM,KAAK;AAGvB,MAAI,QAAQ;AACV,WAAO;AAAA,MACL;AAAA,MACA,YAAY,oBAAoB,KAAK,UAAU,MAAM,CAAC;AAAA,MACtD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,SAAS,aAAa,SAAS,YAAY;AAC7C,QAAI,WAAW;AACb,aAAO;AAAA,QACL;AAAA,QACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,QACxD,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AACA,QAAI,aAAa;AACf,aAAO;AAAA,QACL;AAAA,QACA,YAAY,yBAAyB,KAAK,UAAU,WAAW,CAAC;AAAA,QAChE,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,aAAa;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAGA,OAAK,SAAS,cAAc,SAAS,YAAY,WAAW;AAC1D,WAAO;AAAA,MACL;AAAA,MACA,YAAY,mBAAmB,KAAK,UAAU,SAAS,CAAC;AAAA,MACxD,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,OAAK,SAAS,YAAY,SAAS,WAAW,SAAS;AACrD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,IAAI,CAAC,aAAa,KAAK,UAAU,OAAO,CAAC;AAAA,MACtF,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,SAAS,cAAc,WAAW,OAAO;AAC3C,UAAM,QAAQ,WAAW;AACzB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,qCAAqC,KAAK,UAAU,KAAK,CAAC;AAAA,MACtE,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,QAAQ,QAAQ,CAAC;AAAA,MAC5D,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW,QAAQ,QAAQ;AAAA,IAC1C;AAAA,EACF;AAGA,MAAI,IAAI;AACN,WAAO;AAAA,MACL;AAAA,MACA,YAAY,gBAAgB,KAAK,UAAU,MAAM,EAAE,CAAC;AAAA,MACpD,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa,WAAW;AAAA,IAC1B;AAAA,EACF;AAGA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,WAAO;AAAA,MACL;AAAA,MACA,YAAY,kBAAkB,KAAK,UAAU,CAAC,CAAC;AAAA,MAC/C,aAAa,QAAQ;AAAA,MACrB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AAAA,EACF;AAEA,SAAO;AACT;AAMO,SAAS,6BAA6B,UAOpB;AACvB,QAAM,kBAAkB,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,EAAE;AACrD,QAAM,QAAQ,oBAAI,IAA+B;AAEjD,aAAW,MAAM,UAAU;AACzB,UAAM,MAAM,2BAA2B,EAAE;AACzC,QAAI,CAAC,IAAK;AACV,UAAM,WAAW,MAAM,IAAI,IAAI,GAAG;AAClC,QAAI,CAAC,YAAY,gBAAgB,IAAI,UAAU,IAAI,gBAAgB,SAAS,UAAU,GAAG;AACvF,YAAM,IAAI,IAAI,KAAK,GAAG;AAAA,IACxB;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,MAAM,OAAO,CAAC;AAClC;AAMO,SAAS,wBAAwB,UAAuC;AAC7E,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,QAAQ,SAAS;AAAA,IAAI,OACzB,KAAK,EAAE,GAAG,KAAK,EAAE,UAAU,QAAQ,EAAE,WAAW,KAAK,EAAE,UAAU;AAAA,EACnE;AACA,SAAO;AAAA,IACL;AAAA,IACA,GAAG;AAAA,EACL,EAAE,KAAK,IAAI;AACb;",
+  "names": []
+}

package/dist/logger.js

@@ -1,10 +1,8 @@
-import pino from "pino";
-const logger = pino({
-  level: process.env.LOG_LEVEL ?? "info",
-  ...process.env.NODE_ENV === "development" ? { transport: { target: "pino-pretty", options: { colorize: true } } } : {},
-  base: { service: "crawler" }
+import pino from 'pino';
+export const logger = pino({
+    level: process.env.LOG_LEVEL ?? 'info',
+    ...(process.env.NODE_ENV === 'development'
+        ? { transport: { target: 'pino-pretty', options: { colorize: true } } }
+        : {}),
+    base: { service: 'crawler' },
 });
-export {
-  logger
-};
-//# sourceMappingURL=logger.js.map

package/dist/markdown-extract.js

@@ -1,54 +1,67 @@
-import { parseHTML } from "linkedom";
-import { Readability } from "@mozilla/readability";
-import TurndownService from "turndown";
-const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
-turndown.remove(["script", "style", "noscript", "svg", "iframe"]);
+/**
+ * HTML → Markdown extraction for crawled pages — matches crawl4AI's markdown
+ * output capability. Two-tier, mirroring crawl4AI's "markdown" vs "fit markdown":
+ *
+ *  - "fit" markdown: Readability's best guess at the primary content (title +
+ *    article body), stripped of nav/ads/chrome. Ideal for content/marketing
+ *    pages and RAG chunking. null when Readability finds no article (most
+ *    app UI screens — dashboards, forms — legitimately have none).
+ *  - "full" markdown: the whole visible page converted straight to Markdown,
+ *    always present. The fallback for app screens where there's no "article".
+ *
+ * Runs entirely on an already-captured `page.content()` string — no extra
+ * network fetch, no extra page navigation.
+ */
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+turndown.remove(['script', 'style', 'noscript', 'svg', 'iframe']);
 function stripNoiseElements(document) {
-  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
-  noisy.forEach((el) => el.remove());
+    const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+    noisy.forEach((el) => el.remove());
 }
-function extractMarkdown(html, url) {
-  try {
-    const { document } = parseHTML(html);
-    stripNoiseElements(document);
-    let readabilityResult = null;
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export function extractMarkdown(html, url) {
     try {
-      const { document: readerDoc } = parseHTML(html);
-      stripNoiseElements(readerDoc);
-      readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
-    } catch {
-      readabilityResult = null;
+        const { document } = parseHTML(html);
+        stripNoiseElements(document);
+        let readabilityResult = null;
+        try {
+            // Readability mutates the DOM it's given — clone via re-parsing so the
+            // full-page fallback below still has an intact document.
+            const { document: readerDoc } = parseHTML(html);
+            stripNoiseElements(readerDoc);
+            readabilityResult = new Readability(readerDoc, { charThreshold: 50 }).parse();
+        }
+        catch {
+            readabilityResult = null;
+        }
+        if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+            const markdown = turndown.turndown(readabilityResult.content);
+            return {
+                markdown: `# ${readabilityResult.title || document.title || url}\n\n${markdown}`,
+                method: 'readability',
+                title: readabilityResult.title || document.title || null,
+                wordCount: readabilityResult.textContent.trim().split(/\s+/).length,
+            };
+        }
+        const body = document.body;
+        if (!body)
+            return null;
+        const markdown = turndown.turndown(body.innerHTML || '');
+        const text = markdown.trim();
+        if (!text)
+            return null;
+        const title = document.title || null;
+        return {
+            markdown: title ? `# ${title}\n\n${markdown}` : markdown,
+            method: 'full-page',
+            title,
+            wordCount: text.split(/\s+/).length,
+        };
     }
-    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
-      const markdown2 = turndown.turndown(readabilityResult.content);
-      return {
-        markdown: `# ${readabilityResult.title || document.title || url}
-
-${markdown2}`,
-        method: "readability",
-        title: readabilityResult.title || document.title || null,
-        wordCount: readabilityResult.textContent.trim().split(/\s+/).length
-      };
+    catch {
+        return null;
     }
-    const body = document.body;
-    if (!body) return null;
-    const markdown = turndown.turndown(body.innerHTML || "");
-    const text = markdown.trim();
-    if (!text) return null;
-    const title = document.title || null;
-    return {
-      markdown: title ? `# ${title}
-
-${markdown}` : markdown,
-      method: "full-page",
-      title,
-      wordCount: text.split(/\s+/).length
-    };
-  } catch {
-    return null;
-  }
 }
-export {
-  extractMarkdown
-};
-//# sourceMappingURL=markdown-extract.js.map