@taodigital/crawler 0.0.0-stage → 1.2.0

package/src/locator-extractor.ts

@@ -0,0 +1,309 @@
+/**
+ * Converts raw DB Element records (role, meaning, notes JSON) into Playwright-idiomatic locator strings.
+ * Priority: data-testid → aria-label/placeholder (inputs) → role+name (buttons/links) → heading → text → id
+ */
+
+export interface PlaywrightLocator {
+  key: string;           // camelCase property name for page object
+  expression: string;    // e.g. "page.getByRole('button', { name: 'Submit' })"
+  elementType: string;   // button | input | link | heading | text | element
+  confidence: 'high' | 'medium' | 'low';
+  source: 'testid' | 'label' | 'placeholder' | 'role-name' | 'heading' | 'text' | 'id';
+  description: string;   // human-readable label
+}
+
+/** Parse the notes JSON string stored on Element records. */
+function parseNotes(notes: unknown): Record<string, string> {
+  if (!notes) return {};
+  try {
+    const parsed = typeof notes === 'string' ? JSON.parse(notes) : notes;
+    if (parsed && typeof parsed === 'object') return parsed as Record<string, string>;
+  } catch {}
+  return {};
+}
+
+/** Map crawler role strings to Playwright role values. */
+function mapRole(role?: string | null): string | null {
+  if (!role) return null;
+  const r = role.toLowerCase().trim();
+  if (r === 'button' || r === 'submit' || r === 'reset') return 'button';
+  if (r === 'link' || r === 'a' || r === 'anchor') return 'link';
+  if (r === 'textbox' || r === 'input' || r === 'text' || r === 'email' || r === 'password' || r === 'tel' || r === 'number' || r === 'search') return 'textbox';
+  if (r === 'combobox' || r === 'select' || r === 'dropdown') return 'combobox';
+  if (r === 'checkbox') return 'checkbox';
+  if (r === 'radio') return 'radio';
+  if (r === 'heading' || r === 'h1' || r === 'h2' || r === 'h3' || r === 'h4' || r === 'h5' || r === 'h6') return 'heading';
+  if (r === 'img' || r === 'image') return 'img';
+  if (r === 'nav' || r === 'navigation') return 'navigation';
+  return r;
+}
+
+/** Convert "Submit Form Button" → "submitFormButton" */
+function toKey(text: string): string {
+  return text
+    .replace(/[^a-zA-Z0-9\s]/g, ' ')
+    .replace(/\s+(.)/g, (_, c) => c.toUpperCase())
+    .replace(/^(.)/, (_, c) => c.toLowerCase())
+    .replace(/\s/g, '')
+    .slice(0, 40)
+    || 'element';
+}
+
+/** Convert a single DB Element record to a Playwright locator. Returns null if insufficient data. */
+export function elementToPlaywrightLocator(element: {
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+  ariaState?: Record<string, unknown> | null;
+}): PlaywrightLocator | null {
+  const attrs = parseNotes(element.notes);
+  const testId = attrs['data-testid'];
+  const ariaLabel = attrs['aria-label'] || element.meaning;
+  const id = attrs['id'];
+  const placeholder = attrs['placeholder'];
+  const name = attrs['name'];
+  const text = (element.text ?? '').trim();
+  const meaning = (element.meaning ?? '').trim();
+  const role = mapRole(element.role);
+  const ariaState: Record<string, string> = {};
+  if (element.ariaState && typeof element.ariaState === 'object') {
+    for (const [k, v] of Object.entries(element.ariaState)) {
+      if (typeof v === 'string') ariaState[k] = v;
+    }
+  }
+
+  const label = meaning || text || testId || ariaLabel || placeholder || id || name || 'element';
+  const key = toKey(label);
+
+  // Priority 1: data-testid
+  if (testId) {
+    return {
+      key,
+      expression: `page.getByTestId(${JSON.stringify(testId)})`,
+      elementType: role ?? 'element',
+      confidence: 'high',
+      source: 'testid',
+      description: meaning || testId,
+    };
+  }
+
+  // Priority 2: input/combobox/textarea with label → getByLabel; with placeholder → getByPlaceholder
+  if (role === 'textbox' || role === 'combobox') {
+    if (ariaLabel) {
+      return {
+        key,
+        expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'label',
+        description: ariaLabel,
+      };
+    }
+    if (placeholder) {
+      return {
+        key,
+        expression: `page.getByPlaceholder(${JSON.stringify(placeholder)})`,
+        elementType: 'input',
+        confidence: 'high',
+        source: 'placeholder',
+        description: placeholder,
+      };
+    }
+  }
+
+  // Priority 3: checkbox/radio → getByRole with optional checked state from ariaState
+  if ((role === 'checkbox' || role === 'radio') && meaning) {
+    const isChecked = ariaState['checked'];
+    const stateOpt = isChecked === 'true' ? `, { name: ${JSON.stringify(meaning)}, checked: true }` : isChecked === 'false' ? `, { name: ${JSON.stringify(meaning)}, checked: false }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+  if ((role === 'checkbox' || role === 'radio') && ariaLabel) {
+    return {
+      key,
+      expression: `page.getByLabel(${JSON.stringify(ariaLabel)})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'label',
+      description: ariaLabel,
+    };
+  }
+
+  // Priority 3b: tab/option/treeitem → getByRole with optional selected state
+  const tabLikeRoles = ['tab', 'option', 'treeitem', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'gridcell'];
+  if (tabLikeRoles.includes(role ?? '') && meaning) {
+    const isSelected = ariaState['selected'];
+    const isCurrent = ariaState['current'];
+    const stateOpt = isSelected === 'true' ? `, { name: ${JSON.stringify(meaning)}, selected: true }` : isCurrent ? `, { name: ${JSON.stringify(meaning)} }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role ?? 'element',
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+
+  // Priority 4: button/link with meaningful name → getByRole; include pressed state for toggles
+  if ((role === 'button' || role === 'link') && meaning) {
+    const isPressed = ariaState['pressed'];
+    const stateOpt = isPressed === 'true' ? `, { name: ${JSON.stringify(meaning)}, pressed: true }` : isPressed === 'false' ? `, { name: ${JSON.stringify(meaning)}, pressed: false }` : `, { name: ${JSON.stringify(meaning)} }`;
+    return {
+      key,
+      expression: `page.getByRole(${JSON.stringify(role)}${stateOpt})`,
+      elementType: role,
+      confidence: 'high',
+      source: 'role-name',
+      description: meaning,
+    };
+  }
+
+  // Priority 5: heading → getByRole heading
+  if (role === 'heading' && (meaning || text)) {
+    const name2 = meaning || text;
+    return {
+      key,
+      expression: `page.getByRole('heading', { name: ${JSON.stringify(name2)} })`,
+      elementType: 'heading',
+      confidence: 'medium',
+      source: 'heading',
+      description: name2,
+    };
+  }
+
+  // Priority 6: existing CSS selector from crawler
+  if (element.selector && element.selector.length > 0) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify(element.selector)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || text || element.selector,
+    };
+  }
+
+  // Priority 7: id from attrs
+  if (id) {
+    return {
+      key,
+      expression: `page.locator(${JSON.stringify('#' + id)})`,
+      elementType: role ?? 'element',
+      confidence: 'medium',
+      source: 'id',
+      description: meaning || id,
+    };
+  }
+
+  // Priority 8: fallback to text content
+  if (meaning || text) {
+    const t = meaning || text;
+    return {
+      key,
+      expression: `page.getByText(${JSON.stringify(t)}, { exact: false })`,
+      elementType: role ?? 'text',
+      confidence: 'low',
+      source: 'text',
+      description: t,
+    };
+  }
+
+  return null;
+}
+
+// Patterns that indicate crawl-analysis meta-text stored as element records (not real UI elements).
+// These come from accessibility-auditor, screen-reader analysis, or CSS inspector annotations.
+const META_ANNOTATION_RE = /^(css animations|scroll effects detected|design tokens?\s*:|font imports?:|contrast ratio|aria-label audit|landmark|wcag|color contrast|background.*foreground.*card|--[\w-]+,\s*--)/i;
+
+// Auth-wall element patterns — if >50% of locators match these, the page was an auth wall.
+const AUTH_WALL_TEXT_RE = /sign\s*in|log\s*in|platform access required|please sign in|access required|authentication required|session expired|unauthorized/i;
+
+/**
+ * Returns true if an element's text content is crawl-analysis meta-text, not a real UI element.
+ * These are annotations stored by the accessibility auditor, not actual DOM elements.
+ */
+function isMetaAnnotation(el: { role?: string | null; meaning?: string | null; text?: string | null }): boolean {
+  const t = (el.meaning ?? el.text ?? '').trim();
+  if (!t) return false;
+  // Filter crawl-analysis annotations
+  if (META_ANNOTATION_RE.test(t)) return true;
+  // Filter very long text strings with no interactive role — these are paragraph/body copy,
+  // not useful UI locators. Threshold: 120 chars with no testId/label/role
+  const interactiveRoles = new Set(['button', 'link', 'textbox', 'combobox', 'checkbox', 'radio', 'tab', 'option', 'menuitem']);
+  if (t.length > 120 && !interactiveRoles.has((el.role ?? '').toLowerCase())) return true;
+  // Filter CSS variable dumps (contain multiple --var-name patterns)
+  if ((t.match(/--[\w-]+/g) ?? []).length > 3) return true;
+  return false;
+}
+
+/**
+ * Convert an array of DB Element records to deduplicated Playwright locators.
+ * Deduplicates by key — higher confidence wins.
+ * Filters out crawl-analysis meta-annotations and produces an auth-wall warning when applicable.
+ */
+export function elementsToPlaywrightLocators(elements: Array<{
+  role?: string | null;
+  meaning?: string | null;
+  text?: string | null;
+  notes?: unknown;
+  selector?: string | null;
+  isInteractive?: boolean | null;
+  ariaState?: Record<string, unknown> | null;
+}>): PlaywrightLocator[] {
+  const CONFIDENCE_RANK = { high: 0, medium: 1, low: 2 };
+  const byKey = new Map<string, PlaywrightLocator>();
+
+  for (const el of elements) {
+    // Skip crawl-analysis meta-annotations — these are not real page elements
+    if (isMetaAnnotation(el)) continue;
+
+    const loc = elementToPlaywrightLocator(el);
+    if (!loc) continue;
+    const existing = byKey.get(loc.key);
+    if (!existing || CONFIDENCE_RANK[loc.confidence] < CONFIDENCE_RANK[existing.confidence]) {
+      byKey.set(loc.key, loc);
+    }
+  }
+
+  return Array.from(byKey.values());
+}
+
+/**
+ * Detect whether a set of locators was extracted from an auth-wall page.
+ * Returns a warning message if auth-wall detected, null otherwise.
+ * Heuristic: >50% of locators are low-confidence text matches AND at least one matches auth-wall pattern.
+ */
+export function detectAuthWallLocators(locators: PlaywrightLocator[]): string | null {
+  if (locators.length === 0) return null;
+  const lowCount = locators.filter(l => l.confidence === 'low').length;
+  const hasAuthText = locators.some(l => AUTH_WALL_TEXT_RE.test(l.description));
+  const highCount = locators.filter(l => l.confidence === 'high').length;
+  if (hasAuthText && highCount === 0 && lowCount / locators.length > 0.5) {
+    return 'These locators appear to be from a login/access wall, not the target page. The page requires authentication. Add credentials in Project Settings and re-crawl to get accurate locators for the protected content.';
+  }
+  return null;
+}
+
+/**
+ * Format locators as a block for injection into LLM script generation prompts.
+ * Returns empty string if no locators.
+ */
+export function formatLocatorsForPrompt(locators: PlaywrightLocator[]): string {
+  if (locators.length === 0) return '';
+  const lines = locators.map(l =>
+    `  ${l.key}: ${l.expression}  // ${l.description} [${l.confidence}]`
+  );
+  return [
+    'EXTRACTED PAGE LOCATORS (use these EXACTLY — do not invent other selectors; these were verified against the live page):',
+    ...lines,
+  ].join('\n');
+}

package/src/logger.ts

@@ -0,0 +1,9 @@
+import pino from 'pino';
+
+export const logger = pino({
+  level: process.env.LOG_LEVEL ?? 'info',
+  ...(process.env.NODE_ENV === 'development'
+    ? { transport: { target: 'pino-pretty', options: { colorize: true } } }
+    : {}),
+  base: { service: 'crawler' },
+});

package/src/markdown-extract.test.ts

@@ -0,0 +1,73 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { extractMarkdown } from './markdown-extract.js';
+
+const ARTICLE_HTML = `<!doctype html><html><head><title>How Widgets Work</title></head>
+<body>
+<nav><a href="/">Home</a><a href="/about">About</a></nav>
+<header><div class="logo">WidgetCo</div></header>
+<article>
+<h1>How Widgets Work</h1>
+<p>${'Widgets are small mechanical devices used in countless everyday applications. '.repeat(10)}</p>
+<p>${'This second paragraph explains the history of widget manufacturing in great detail. '.repeat(10)}</p>
+<h2>Manufacturing Process</h2>
+<p>${'The manufacturing process involves several precise steps performed by skilled technicians. '.repeat(10)}</p>
+</article>
+<footer><p>Copyright 2026</p></footer>
+<script>console.log('tracking pixel');</script>
+</body></html>`;
+
+const APP_DASHBOARD_HTML = `<!doctype html><html><head><title>Dashboard</title></head>
+<body>
+<div id="app">
+<nav class="sidebar"><a href="/home">Home</a><a href="/settings">Settings</a></nav>
+<main>
+<h1>Dashboard</h1>
+<div class="stat-card"><span>Revenue</span><strong>$12,400</strong></div>
+<div class="stat-card"><span>Users</span><strong>842</strong></div>
+<button>Export CSV</button>
+</main>
+</div>
+<script>window.__STATE__ = {};</script>
+</body></html>`;
+
+const EMPTY_HTML = `<!doctype html><html><head><title></title></head><body></body></html>`;
+
+test('extractMarkdown: article page uses Readability and strips nav/footer chrome', () => {
+  const result = extractMarkdown(ARTICLE_HTML, 'https://example.com/blog/widgets');
+  assert.ok(result);
+  assert.equal(result!.method, 'readability');
+  assert.match(result!.markdown, /How Widgets Work/);
+  assert.match(result!.markdown, /Manufacturing Process/);
+  assert.doesNotMatch(result!.markdown, /Copyright 2026/);
+  assert.doesNotMatch(result!.markdown, /tracking pixel/);
+  assert.ok(result!.wordCount > 50);
+});
+
+test('extractMarkdown: app dashboard (no article) falls back to full-page conversion', () => {
+  const result = extractMarkdown(APP_DASHBOARD_HTML, 'https://app.example.com/dashboard');
+  assert.ok(result);
+  assert.equal(result!.method, 'full-page');
+  assert.match(result!.markdown, /Dashboard/);
+  assert.match(result!.markdown, /Revenue/);
+  assert.match(result!.markdown, /Export CSV/);
+  assert.doesNotMatch(result!.markdown, /__STATE__/);
+});
+
+test('extractMarkdown: empty page returns null instead of an empty markdown string', () => {
+  const result = extractMarkdown(EMPTY_HTML, 'https://example.com/empty');
+  assert.equal(result, null);
+});
+
+test('extractMarkdown: malformed HTML never throws', () => {
+  assert.doesNotThrow(() => extractMarkdown('<html><body><div>unclosed', 'https://example.com/x'));
+});
+
+test('extractMarkdown: script/style content is never leaked into markdown output', () => {
+  const html = `<!doctype html><html><head><title>Page</title><style>.x{color:red}</style></head>
+    <body><script>alert('xss-in-markdown')</script><h1>Real Content Heading For Testing</h1><p>${'Some real paragraph text that is long enough to be meaningful content here. '.repeat(6)}</p></body></html>`;
+  const result = extractMarkdown(html, 'https://example.com/page');
+  assert.ok(result);
+  assert.doesNotMatch(result!.markdown, /alert\(/);
+  assert.doesNotMatch(result!.markdown, /color:red/);
+});

package/src/markdown-extract.ts

@@ -0,0 +1,76 @@
+/**
+ * HTML → Markdown extraction for crawled pages — matches crawl4AI's markdown
+ * output capability. Two-tier, mirroring crawl4AI's "markdown" vs "fit markdown":
+ *
+ *  - "fit" markdown: Readability's best guess at the primary content (title +
+ *    article body), stripped of nav/ads/chrome. Ideal for content/marketing
+ *    pages and RAG chunking. null when Readability finds no article (most
+ *    app UI screens — dashboards, forms — legitimately have none).
+ *  - "full" markdown: the whole visible page converted straight to Markdown,
+ *    always present. The fallback for app screens where there's no "article".
+ *
+ * Runs entirely on an already-captured `page.content()` string — no extra
+ * network fetch, no extra page navigation.
+ */
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+turndown.remove(['script', 'style', 'noscript', 'svg', 'iframe'] as unknown as (keyof HTMLElementTagNameMap)[]);
+
+export interface ExtractedMarkdown {
+  markdown: string;
+  method: 'readability' | 'full-page';
+  title: string | null;
+  wordCount: number;
+}
+
+function stripNoiseElements(document: Document): void {
+  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+  noisy.forEach((el: any) => el.remove());
+}
+
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export function extractMarkdown(html: string, url: string): ExtractedMarkdown | null {
+  try {
+    const { document } = parseHTML(html);
+    stripNoiseElements(document);
+
+    let readabilityResult: { title: string; content: string; textContent: string } | null = null;
+    try {
+      // Readability mutates the DOM it's given — clone via re-parsing so the
+      // full-page fallback below still has an intact document.
+      const { document: readerDoc } = parseHTML(html);
+      stripNoiseElements(readerDoc);
+      readabilityResult = new Readability(readerDoc as any, { charThreshold: 50 }).parse() as any;
+    } catch {
+      readabilityResult = null;
+    }
+
+    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+      const markdown = turndown.turndown(readabilityResult.content);
+      return {
+        markdown: `# ${readabilityResult.title || document.title || url}\n\n${markdown}`,
+        method: 'readability',
+        title: readabilityResult.title || document.title || null,
+        wordCount: readabilityResult.textContent.trim().split(/\s+/).length,
+      };
+    }
+
+    const body = document.body;
+    if (!body) return null;
+    const markdown = turndown.turndown(body.innerHTML || '');
+    const text = markdown.trim();
+    if (!text) return null;
+    const title = document.title || null;
+    return {
+      markdown: title ? `# ${title}\n\n${markdown}` : markdown,
+      method: 'full-page',
+      title,
+      wordCount: text.split(/\s+/).length,
+    };
+  } catch {
+    return null;
+  }
+}