@detiq/crawler 0.1.3

package/src/js-coverage.ts

@@ -0,0 +1,87 @@
+/**
+ * JS/CSS coverage collector via Playwright's built-in coverage API.
+ *
+ * For each crawled page, records what percentage of loaded JS/CSS was actually
+ * executed/applied. High unused code % → performance opportunity (code splitting, lazy loading).
+ *
+ * Uses Playwright's page.coverage API (wraps CDP Coverage domain).
+ */
+
+import type { Page } from 'playwright';
+
+export interface CoverageEntry {
+  url: string;
+  type: 'js' | 'css';
+  totalBytes: number;
+  usedBytes: number;
+  unusedPercent: number;
+}
+
+export interface CoverageReport {
+  pageUrl: string;
+  entries: CoverageEntry[];
+  totalJsBytes: number;
+  usedJsBytes: number;
+  totalCssBytes: number;
+  usedCssBytes: number;
+  jsUnusedPercent: number;
+  cssUnusedPercent: number;
+  collectedAt: string;
+}
+
+export async function startCoverage(page: Page): Promise<void> {
+  try {
+    await Promise.all([
+      page.coverage.startJSCoverage({ resetOnNavigation: false }),
+      page.coverage.startCSSCoverage({ resetOnNavigation: false }),
+    ]);
+  } catch { /* coverage not available in all environments */ }
+}
+
+export async function stopAndCollectCoverage(page: Page, pageUrl: string): Promise<CoverageReport | null> {
+  try {
+    const [jsCoverage, cssCoverage] = await Promise.all([
+      page.coverage.stopJSCoverage().catch(() => [] as any[]),
+      page.coverage.stopCSSCoverage().catch(() => [] as any[]),
+    ]);
+
+    const entries: CoverageEntry[] = [];
+
+    for (const entry of jsCoverage) {
+      const totalBytes = entry.text?.length ?? 0;
+      const usedBytes = (entry.ranges ?? []).reduce((sum: number, r: any) => sum + (r.end - r.start), 0);
+      if (totalBytes > 0) {
+        entries.push({ url: entry.url, type: 'js', totalBytes, usedBytes, unusedPercent: Math.round((1 - usedBytes / totalBytes) * 100) });
+      }
+    }
+
+    for (const entry of cssCoverage) {
+      const totalBytes = entry.text?.length ?? 0;
+      const usedBytes = (entry.ranges ?? []).reduce((sum: number, r: any) => sum + (r.end - r.start), 0);
+      if (totalBytes > 0) {
+        entries.push({ url: entry.url, type: 'css', totalBytes, usedBytes, unusedPercent: Math.round((1 - usedBytes / totalBytes) * 100) });
+      }
+    }
+
+    const jsEntries = entries.filter(e => e.type === 'js');
+    const cssEntries = entries.filter(e => e.type === 'css');
+    const totalJsBytes = jsEntries.reduce((s, e) => s + e.totalBytes, 0);
+    const usedJsBytes = jsEntries.reduce((s, e) => s + e.usedBytes, 0);
+    const totalCssBytes = cssEntries.reduce((s, e) => s + e.totalBytes, 0);
+    const usedCssBytes = cssEntries.reduce((s, e) => s + e.usedBytes, 0);
+
+    return {
+      pageUrl,
+      entries,
+      totalJsBytes,
+      usedJsBytes,
+      totalCssBytes,
+      usedCssBytes,
+      jsUnusedPercent: totalJsBytes > 0 ? Math.round((1 - usedJsBytes / totalJsBytes) * 100) : 0,
+      cssUnusedPercent: totalCssBytes > 0 ? Math.round((1 - usedCssBytes / totalCssBytes) * 100) : 0,
+      collectedAt: new Date().toISOString(),
+    };
+  } catch {
+    return null;
+  }
+}

package/src/js-router-detector.ts

@@ -0,0 +1,112 @@
+/**
+ * P2: JS Router Detection
+ *
+ * Scans the loaded page for client-side route tables declared by React Router v6,
+ * Vue Router, Angular Router, or custom window globals. Supplements the existing
+ * history.pushState interception (which only captures runtime navigation) with a
+ * static analysis of routes declared in the JS bundle.
+ *
+ * Called once per page after goto(). Runs entirely in-browser via page.evaluate().
+ * Adds discovered routes to the BFS frontier so the crawler visits them.
+ */
+
+import { logger as rootLogger } from './logger.js';
+
+const logger = rootLogger.child({ module: 'js-router-detector' });
+
+/**
+ * Extract absolute page URLs from the current page's JS router config.
+ * Returns deduplicated absolute URLs (same origin). Dynamic segments (:id, *)
+ * are skipped — the crawler can't resolve them without knowing valid IDs.
+ */
+export async function detectJsRoutes(page: any, origin: string): Promise<string[]> {
+  try {
+    const raw: string[] = await page.evaluate(() => {
+      const found: string[] = [];
+
+      // 1. Check common framework globals ─────────────────────────────────────
+      const windowGlobals = [
+        '__routes__', '__ROUTER__', '__vue_router__', '__router__',
+        'router', '_router', '__reactRouter', '__NEXT_DATA__',
+      ];
+      for (const g of windowGlobals) {
+        try {
+          const val = (window as any)[g];
+          if (!val || typeof val !== 'object') continue;
+
+          // Next.js page manifest
+          if (g === '__NEXT_DATA__' && Array.isArray(val.buildManifest?.pages)) {
+            for (const p of val.buildManifest.pages) if (typeof p === 'string') found.push(p);
+            continue;
+          }
+
+          // Router objects: options.routes, routes, getRoutes()
+          const routeArr =
+            (Array.isArray(val.options?.routes) ? val.options.routes : null) ??
+            (Array.isArray(val.routes) ? val.routes : null) ??
+            (typeof val.getRoutes === 'function' ? val.getRoutes() : null);
+
+          if (!Array.isArray(routeArr)) continue;
+
+          const flatRoutes = (arr: any[], prefix = ''): void => {
+            for (const r of arr) {
+              if (!r || typeof r !== 'object') continue;
+              const path: string = r.path ?? r.href ?? r.url ?? '';
+              const full = path.startsWith('/') ? path : `${prefix}/${path}`.replace(/\/\//g, '/');
+              if (full && full !== '/') found.push(full);
+              if (Array.isArray(r.children)) flatRoutes(r.children, full);
+            }
+          };
+          flatRoutes(routeArr);
+        } catch { /* skip corrupt globals */ }
+      }
+
+      // 2. Scan inline <script> content for route patterns ───────────────────
+      // Two-pass: regex first, if >3 found trust them. Below 3 → skip (too noisy).
+      const routePatterns = [
+        /\bpath\s*:\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /\broute\s*\(\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /<Route[^>]+path\s*=\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+        /to\s*:\s*['"](\/?[\w\-\/\.~@]+)['"]/g,
+      ];
+      const scripts = Array.from(document.querySelectorAll('script:not([src])'));
+      for (const el of scripts.slice(0, 15)) {
+        const src = el.textContent ?? '';
+        if (src.length < 100 || src.length > 800_000) continue;
+        for (const pat of routePatterns) {
+          const hits: string[] = [];
+          let m: RegExpExecArray | null;
+          pat.lastIndex = 0;
+          while ((m = pat.exec(src)) !== null && hits.length < 100) {
+            if (m[1] && m[1].startsWith('/') && m[1].length > 1) hits.push(m[1]);
+          }
+          // Only trust if we found >2 hits (reduces false positives)
+          if (hits.length > 2) for (const h of hits) found.push(h);
+        }
+      }
+
+      return found;
+    });
+
+    // Post-process: resolve to absolute, dedup, drop dynamic segments
+    const DYNAMIC = /[*:?]/;
+    const ASSET = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|map|json|xml)$/i;
+    const seen = new Set<string>();
+    const result: string[] = [];
+    for (const r of raw) {
+      if (!r || typeof r !== 'string' || DYNAMIC.test(r) || ASSET.test(r)) continue;
+      const abs = r.startsWith('http') ? r : `${origin}${r.startsWith('/') ? '' : '/'}${r}`;
+      try {
+        const parsed = new URL(abs);
+        if (parsed.origin !== origin) continue;
+      } catch { continue; }
+      if (!seen.has(abs)) { seen.add(abs); result.push(abs); }
+    }
+
+    if (result.length > 0) logger.info({ count: result.length }, '[P2] JS router: routes detected');
+    return result;
+  } catch (err) {
+    logger.warn({ err: String(err) }, '[P2] JS router detection failed — non-fatal');
+    return [];
+  }
+}

package/src/language-detector.ts

@@ -0,0 +1,82 @@
+/**
+ * Language detector — identify page language from HTML attributes and content analysis.
+ *
+ * Sources (in priority order):
+ * 1. <html lang="..."> attribute
+ * 2. <meta http-equiv="Content-Language"> header
+ * 3. hreflang alternate links
+ * 4. Content-based n-gram fingerprint (detects top 20 languages)
+ *
+ * Used to: route pages to the correct markdown processor, group screens by locale,
+ * and detect missing hreflang implementations.
+ */
+
+import type { Page } from 'playwright';
+
+// Frequency signatures for n-gram based detection
+// Common trigrams by language (simplified — not production-grade)
+const LANG_SIGNATURES: Record<string, RegExp> = {
+  en: /\b(the|and|for|are|but|not|you|all|can|had|her|was|one|our|out|day|get|has|him|his|how)\b/gi,
+  es: /\b(que|con|una|por|del|las|los|para|como|pero|más|este|esto|sus|ser|hay|sobre|entre|también)\b/gi,
+  fr: /\b(les|des|une|est|dans|pour|qui|par|sur|avec|que|pas|son|tout|mais|bien|comme|vous)\b/gi,
+  de: /\b(die|der|und|den|das|ein|ist|von|mit|dem|auf|sich|des|aber|auch|bei|aus|nach)\b/gi,
+  pt: /\b(que|não|uma|com|por|dos|para|como|mais|mas|seu|sua|nas|nos|ele|ela|são)\b/gi,
+  it: /\b(che|con|del|per|una|nel|non|sono|gli|dei|alla|delle|questa|questo|come|più)\b/gi,
+  nl: /\b(van|het|een|zijn|maar|ook|dan|door|op|uit|aan|bij|om|al|veel|haar|hun)\b/gi,
+  ja: /[぀-ゟ゠-ヿ]/g,
+  zh: /[一-鿿]/g,
+  ar: /[؀-ۿ]/g,
+  hi: /[ऀ-ॿ]/g,
+  ko: /[가-힯ᄀ-ᇿ]/g,
+};
+
+function detectFromContent(text: string): string {
+  if (!text || text.length < 50) return 'en';
+  let best = 'en';
+  let bestScore = 0;
+  for (const [lang, re] of Object.entries(LANG_SIGNATURES)) {
+    const matches = (text.match(re) ?? []).length;
+    const score = matches / Math.max(text.length / 100, 1);
+    if (score > bestScore) { bestScore = score; best = lang; }
+  }
+  return best;
+}
+
+export interface LanguageDetectionResult {
+  url: string;
+  declaredLang: string;   // from <html lang> or meta
+  detectedLang: string;   // from content analysis
+  confidence: 'high' | 'medium' | 'low';
+  alternates: Record<string, string>; // lang → url from hreflang
+  isMultilingual: boolean;
+}
+
+export async function detectLanguage(page: Page, url: string): Promise<LanguageDetectionResult> {
+  const data = await page.evaluate(() => {
+    const htmlLang = document.documentElement.lang?.toLowerCase().split('-')[0] ?? '';
+    const metaLang = (document.querySelector('meta[http-equiv="Content-Language"]') as any)?.content?.toLowerCase().split('-')[0] ?? '';
+    const alternates: Record<string, string> = {};
+    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el: any) => {
+      const lang = el.getAttribute('hreflang')?.toLowerCase();
+      const href = el.getAttribute('href') ?? '';
+      if (lang && lang !== 'x-default') alternates[lang] = href;
+    });
+    const bodyText = (document.body?.innerText ?? '').slice(0, 2000);
+    return { htmlLang, metaLang, alternates, bodyText };
+  }).catch(() => ({ htmlLang: '', metaLang: '', alternates: {}, bodyText: '' }));
+
+  const declaredLang = data.htmlLang || data.metaLang || '';
+  const detectedLang = detectFromContent(data.bodyText);
+
+  const confidence: LanguageDetectionResult['confidence'] =
+    declaredLang ? 'high' : data.bodyText.length > 200 ? 'medium' : 'low';
+
+  return {
+    url,
+    declaredLang,
+    detectedLang: declaredLang || detectedLang,
+    confidence,
+    alternates: data.alternates,
+    isMultilingual: Object.keys(data.alternates).length > 1,
+  };
+}

package/src/link-classifier.ts

@@ -0,0 +1,65 @@
+/**
+ * Link classifier — assigns priority and category to discovered URLs.
+ *
+ * Categories:
+ * - 'nav'       : site navigation (homepage, top-level pages) — highest priority
+ * - 'auth'      : login/register/password flows — high priority
+ * - 'app'       : authenticated app screens — high priority
+ * - 'content'   : blog/docs/articles — medium priority
+ * - 'api'       : REST/GraphQL API endpoints — detected, not crawled as screens
+ * - 'resource'  : static assets (JS/CSS/img/fonts) — skip
+ * - 'external'  : different domain — skip unless cross-domain crawl enabled
+ * - 'low-value' : pagination, sort, filter query strings — deprioritize
+ */
+
+export type LinkCategory = 'nav' | 'auth' | 'app' | 'content' | 'api' | 'resource' | 'external' | 'low-value';
+
+export interface ClassifiedLink {
+  url: string;
+  category: LinkCategory;
+  priority: number; // 0-100, higher = crawl sooner
+  reason: string;
+}
+
+const RESOURCE_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|mp4|mp3|pdf|zip|gz|tar|exe|dmg|apk)(\?|$)/i;
+const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i;
+const AUTH_PATTERN = /\/(login|signin|sign-in|register|signup|sign-up|auth|oauth|sso|password|forgot|reset|logout|signout)\b/i;
+const LOW_VALUE_PATTERN = /[?&](page|p|offset|sort|order|filter|q|search|tag|category|color|size|brand)=/i;
+const PAGINATION_PATTERN = /\/page\/\d+|\/p\/\d+|\bpage=\d+/i;
+
+export function classifyLink(url: string, baseUrl: string): ClassifiedLink {
+  let parsed: URL;
+  let baseHost: string;
+  try {
+    parsed = new URL(url);
+    baseHost = new URL(baseUrl).hostname;
+  } catch {
+    return { url, category: 'low-value', priority: 0, reason: 'invalid URL' };
+  }
+
+  const path = parsed.pathname + parsed.search;
+  const isExternal = parsed.hostname !== baseHost;
+
+  if (isExternal) return { url, category: 'external', priority: 0, reason: 'different domain' };
+  if (RESOURCE_EXT.test(path)) return { url, category: 'resource', priority: 0, reason: 'static asset extension' };
+  if (API_PATTERN.test(path)) return { url, category: 'api', priority: 10, reason: 'API path pattern' };
+  if (AUTH_PATTERN.test(path)) return { url, category: 'auth', priority: 90, reason: 'auth flow pattern' };
+  if (PAGINATION_PATTERN.test(path)) return { url, category: 'low-value', priority: 15, reason: 'pagination' };
+  if (LOW_VALUE_PATTERN.test(path)) return { url, category: 'low-value', priority: 20, reason: 'filter/sort query param' };
+
+  const segments = parsed.pathname.split('/').filter(Boolean);
+  if (segments.length === 0) return { url, category: 'nav', priority: 100, reason: 'homepage' };
+  if (segments.length === 1) return { url, category: 'nav', priority: 80, reason: 'top-level nav' };
+  if (segments.length <= 3) return { url, category: 'app', priority: 60, reason: 'app screen' };
+
+  return { url, category: 'content', priority: 40, reason: 'deep content page' };
+}
+
+export function classifyLinks(urls: string[], baseUrl: string): ClassifiedLink[] {
+  return urls.map(u => classifyLink(u, baseUrl))
+    .sort((a, b) => b.priority - a.priority);
+}
+
+export function filterCrawlable(classified: ClassifiedLink[]): ClassifiedLink[] {
+  return classified.filter(l => l.category !== 'resource' && l.category !== 'external' && l.category !== 'api');
+}

package/src/live-view.ts

@@ -0,0 +1,141 @@
+import { chromium, type Browser, type BrowserContext, type Page } from 'playwright';
+import { logger } from './logger.js';
+
+interface LiveViewSession {
+  id: string;
+  browser: Browser;
+  context: BrowserContext;
+  page: Page;
+  appUrl: string;
+  lastFrame: string | null; // base64 JPEG
+  streaming: boolean;
+  frameInterval: ReturnType<typeof setInterval> | null;
+  createdAt: number;
+}
+
+const sessions = new Map<string, LiveViewSession>();
+const SESSION_TTL_MS = 15 * 60 * 1_000; // 15 min
+
+function generateId(): string {
+  return Math.random().toString(36).slice(2) + Date.now().toString(36);
+}
+
+/**
+ * Start a live view session: launches a headless browser, navigates to appUrl,
+ * begins capturing frames. Returns the session ID.
+ */
+export async function startLiveView(appUrl: string, storageState?: string): Promise<string> {
+  const id = generateId();
+  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.platform === 'linux';
+
+  const browser = await chromium.launch({
+    headless: true,
+    args: isDocker ? ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] : [],
+  });
+
+  const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+
+  // Restore session if provided
+  if (storageState) {
+    try {
+      const state = JSON.parse(storageState);
+      if (state.cookies?.length) await context.addCookies(state.cookies);
+    } catch { /* ignore */ }
+  }
+
+  const page = await context.newPage();
+  await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
+
+  const session: LiveViewSession = {
+    id, browser, context, page, appUrl,
+    lastFrame: null, streaming: true, frameInterval: null,
+    createdAt: Date.now(),
+  };
+
+  // Start capturing frames (every 500ms)
+  session.frameInterval = setInterval(async () => {
+    if (!session.streaming) return;
+    try {
+      const buf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false });
+      session.lastFrame = buf.toString('base64');
+    } catch { session.streaming = false; }
+  }, 500);
+
+  sessions.set(id, session);
+
+  // Auto-cleanup after TTL
+  setTimeout(() => closeLiveView(id).catch(() => {}), SESSION_TTL_MS);
+
+  logger.info({ id, appUrl }, '[live-view] Session started');
+  return id;
+}
+
+/**
+ * Get the latest frame for a session (base64 JPEG).
+ */
+export function getFrame(id: string): string | null {
+  return sessions.get(id)?.lastFrame ?? null;
+}
+
+/**
+ * Navigate the live view browser to a URL.
+ */
+export async function liveViewNavigate(id: string, url: string): Promise<void> {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  await session.page.goto(url, { waitUntil: 'domcontentloaded' });
+}
+
+/**
+ * Capture storageState from the live view session, close the browser, return JSON string.
+ */
+export async function completeLiveView(id: string): Promise<string> {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found or expired`);
+
+  const state = await session.context.storageState();
+  const json = JSON.stringify(state);
+  await closeLiveView(id);
+  return json;
+}
+
+/**
+ * Send a mouse or keyboard input event to the live view browser.
+ */
+export async function sendInput(id: string, event: {
+  type: 'click' | 'type' | 'key';
+  x?: number; y?: number;
+  button?: 'left' | 'right' | 'middle';
+  text?: string;
+  key?: string;
+}): Promise<void> {
+  const session = sessions.get(id);
+  if (!session) throw new Error(`Live view session ${id} not found`);
+  if (event.type === 'click' && event.x !== undefined && event.y !== undefined) {
+    await session.page.mouse.click(event.x, event.y, { button: event.button ?? 'left' });
+  } else if (event.type === 'type' && event.text) {
+    await session.page.keyboard.type(event.text);
+  } else if (event.type === 'key' && event.key) {
+    await session.page.keyboard.press(event.key);
+  }
+}
+
+/**
+ * Close a live view session without capturing.
+ */
+export async function closeLiveView(id: string): Promise<void> {
+  const session = sessions.get(id);
+  if (!session) return;
+  session.streaming = false;
+  if (session.frameInterval) clearInterval(session.frameInterval);
+  sessions.delete(id);
+  await session.browser.close().catch(() => {});
+  logger.info({ id }, '[live-view] Session closed');
+}
+
+/**
+ * List all active session IDs (for health checks / cleanup).
+ */
+export function listLiveViews(): string[] {
+  return [...sessions.keys()];
+}

package/src/logger.ts

@@ -0,0 +1,9 @@
+import pino from 'pino';
+
+export const logger = pino({
+  level: process.env.LOG_LEVEL ?? 'info',
+  ...(process.env.NODE_ENV === 'development'
+    ? { transport: { target: 'pino-pretty', options: { colorize: true } } }
+    : {}),
+  base: { service: 'crawler' },
+});

package/src/markdown-extract.test.ts

@@ -0,0 +1,73 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { extractMarkdown } from './markdown-extract.js';
+
+const ARTICLE_HTML = `<!doctype html><html><head><title>How Widgets Work</title></head>
+<body>
+<nav><a href="/">Home</a><a href="/about">About</a></nav>
+<header><div class="logo">WidgetCo</div></header>
+<article>
+<h1>How Widgets Work</h1>
+<p>${'Widgets are small mechanical devices used in countless everyday applications. '.repeat(10)}</p>
+<p>${'This second paragraph explains the history of widget manufacturing in great detail. '.repeat(10)}</p>
+<h2>Manufacturing Process</h2>
+<p>${'The manufacturing process involves several precise steps performed by skilled technicians. '.repeat(10)}</p>
+</article>
+<footer><p>Copyright 2026</p></footer>
+<script>console.log('tracking pixel');</script>
+</body></html>`;
+
+const APP_DASHBOARD_HTML = `<!doctype html><html><head><title>Dashboard</title></head>
+<body>
+<div id="app">
+<nav class="sidebar"><a href="/home">Home</a><a href="/settings">Settings</a></nav>
+<main>
+<h1>Dashboard</h1>
+<div class="stat-card"><span>Revenue</span><strong>$12,400</strong></div>
+<div class="stat-card"><span>Users</span><strong>842</strong></div>
+<button>Export CSV</button>
+</main>
+</div>
+<script>window.__STATE__ = {};</script>
+</body></html>`;
+
+const EMPTY_HTML = `<!doctype html><html><head><title></title></head><body></body></html>`;
+
+test('extractMarkdown: article page uses Readability and strips nav/footer chrome', () => {
+  const result = extractMarkdown(ARTICLE_HTML, 'https://example.com/blog/widgets');
+  assert.ok(result);
+  assert.equal(result!.method, 'readability');
+  assert.match(result!.markdown, /How Widgets Work/);
+  assert.match(result!.markdown, /Manufacturing Process/);
+  assert.doesNotMatch(result!.markdown, /Copyright 2026/);
+  assert.doesNotMatch(result!.markdown, /tracking pixel/);
+  assert.ok(result!.wordCount > 50);
+});
+
+test('extractMarkdown: app dashboard (no article) falls back to full-page conversion', () => {
+  const result = extractMarkdown(APP_DASHBOARD_HTML, 'https://app.example.com/dashboard');
+  assert.ok(result);
+  assert.equal(result!.method, 'full-page');
+  assert.match(result!.markdown, /Dashboard/);
+  assert.match(result!.markdown, /Revenue/);
+  assert.match(result!.markdown, /Export CSV/);
+  assert.doesNotMatch(result!.markdown, /__STATE__/);
+});
+
+test('extractMarkdown: empty page returns null instead of an empty markdown string', () => {
+  const result = extractMarkdown(EMPTY_HTML, 'https://example.com/empty');
+  assert.equal(result, null);
+});
+
+test('extractMarkdown: malformed HTML never throws', () => {
+  assert.doesNotThrow(() => extractMarkdown('<html><body><div>unclosed', 'https://example.com/x'));
+});
+
+test('extractMarkdown: script/style content is never leaked into markdown output', () => {
+  const html = `<!doctype html><html><head><title>Page</title><style>.x{color:red}</style></head>
+    <body><script>alert('xss-in-markdown')</script><h1>Real Content Heading For Testing</h1><p>${'Some real paragraph text that is long enough to be meaningful content here. '.repeat(6)}</p></body></html>`;
+  const result = extractMarkdown(html, 'https://example.com/page');
+  assert.ok(result);
+  assert.doesNotMatch(result!.markdown, /alert\(/);
+  assert.doesNotMatch(result!.markdown, /color:red/);
+});

package/src/markdown-extract.ts

@@ -0,0 +1,76 @@
+/**
+ * HTML → Markdown extraction for crawled pages — matches crawl4AI's markdown
+ * output capability. Two-tier, mirroring crawl4AI's "markdown" vs "fit markdown":
+ *
+ *  - "fit" markdown: Readability's best guess at the primary content (title +
+ *    article body), stripped of nav/ads/chrome. Ideal for content/marketing
+ *    pages and RAG chunking. null when Readability finds no article (most
+ *    app UI screens — dashboards, forms — legitimately have none).
+ *  - "full" markdown: the whole visible page converted straight to Markdown,
+ *    always present. The fallback for app screens where there's no "article".
+ *
+ * Runs entirely on an already-captured `page.content()` string — no extra
+ * network fetch, no extra page navigation.
+ */
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+turndown.remove(['script', 'style', 'noscript', 'svg', 'iframe'] as unknown as (keyof HTMLElementTagNameMap)[]);
+
+export interface ExtractedMarkdown {
+  markdown: string;
+  method: 'readability' | 'full-page';
+  title: string | null;
+  wordCount: number;
+}
+
+function stripNoiseElements(document: Document): void {
+  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
+  noisy.forEach((el: any) => el.remove());
+}
+
+/** Best-effort HTML→Markdown for one crawled page. Never throws. */
+export function extractMarkdown(html: string, url: string): ExtractedMarkdown | null {
+  try {
+    const { document } = parseHTML(html);
+    stripNoiseElements(document);
+
+    let readabilityResult: { title: string; content: string; textContent: string } | null = null;
+    try {
+      // Readability mutates the DOM it's given — clone via re-parsing so the
+      // full-page fallback below still has an intact document.
+      const { document: readerDoc } = parseHTML(html);
+      stripNoiseElements(readerDoc);
+      readabilityResult = new Readability(readerDoc as any, { charThreshold: 50 }).parse() as any;
+    } catch {
+      readabilityResult = null;
+    }
+
+    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
+      const markdown = turndown.turndown(readabilityResult.content);
+      return {
+        markdown: `# ${readabilityResult.title || document.title || url}\n\n${markdown}`,
+        method: 'readability',
+        title: readabilityResult.title || document.title || null,
+        wordCount: readabilityResult.textContent.trim().split(/\s+/).length,
+      };
+    }
+
+    const body = document.body;
+    if (!body) return null;
+    const markdown = turndown.turndown(body.innerHTML || '');
+    const text = markdown.trim();
+    if (!text) return null;
+    const title = document.title || null;
+    return {
+      markdown: title ? `# ${title}\n\n${markdown}` : markdown,
+      method: 'full-page',
+      title,
+      wordCount: text.split(/\s+/).length,
+    };
+  } catch {
+    return null;
+  }
+}