@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/engine-router.ts

@@ -0,0 +1,141 @@
+import type { DomainProfile } from './profiles/domain-profile.js';
+
+export type CrawlEngineName =
+  | 'http'
+  | 'browser'
+  | 'browser-stealth'
+  | 'stagehand'
+  | 'vision'
+  | 'recorded-session'
+  | 'native';
+
+export interface CrawlRouteDecisionInput {
+  appUrl: string;
+  appType?: string | null;
+  hasCredentials: boolean;
+  hasStorageState: boolean;
+  hasAuthToken: boolean;
+  hasSkyvern: boolean;
+  hasProxy: boolean;
+  profile: DomainProfile;
+  previousFailureReason?: string;
+}
+
+export interface CrawlRouteDecision {
+  engines: CrawlEngineName[];
+  reasons: string[];
+  confidence: number;
+}
+
+/**
+ * Per-domain rendering predictor — learns whether a domain requires a full browser
+ * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
+ *
+ * Strategy:
+ * 1. On first encounter of a domain, classify the HTTP response body.
+ * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
+ * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
+ */
+export class RenderingTypePredictor {
+  private cache = new Map<string, 'http' | 'browser'>();
+
+  /** Return cached decision if available, undefined if unknown. */
+  getCached(url: string): 'http' | 'browser' | undefined {
+    try {
+      return this.cache.get(new URL(url).hostname);
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * Analyze an HTTP response body to predict if JS rendering is required.
+   * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+   */
+  predict(url: string, htmlBody: string, statusCode: number): 'http' | 'browser' {
+    let decision: 'http' | 'browser' = 'http';
+    try {
+      if (statusCode >= 400) {
+        decision = 'browser'; // let browser handle error pages
+      } else {
+        const bodyLen = htmlBody.length;
+        // Strip tags to get raw text
+        const textOnly = htmlBody.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+        const wordCount = textOnly.split(/\s+/).filter(w => w.length > 2).length;
+
+        // SPA signals: very short text, presence of React/Vue/Angular root markers
+        const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody)
+          || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
+        const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
+        const tinyBody = bodyLen < 3000 && wordCount < 50;
+
+        if ((hasSpaRoot && wordCount < 100) || (tinyBody && !hasNoscriptFallback)) {
+          decision = 'browser';
+        }
+      }
+    } catch { /* default http */ }
+
+    try {
+      this.cache.set(new URL(url).hostname, decision);
+    } catch { /* ignore */ }
+    return decision;
+  }
+
+  /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+  set(url: string, engine: 'http' | 'browser'): void {
+    try { this.cache.set(new URL(url).hostname, engine); } catch { /* ignore */ }
+  }
+
+  stats(): { cached: number; browserDomains: string[]; httpDomains: string[] } {
+    const browserDomains: string[] = [];
+    const httpDomains: string[] = [];
+    for (const [domain, engine] of this.cache) {
+      if (engine === 'browser') browserDomains.push(domain);
+      else httpDomains.push(domain);
+    }
+    return { cached: this.cache.size, browserDomains, httpDomains };
+  }
+}
+
+/** Singleton predictor — shared across crawl jobs in the same process. */
+export const renderingPredictor = new RenderingTypePredictor();
+
+export function chooseCrawlEngines(input: CrawlRouteDecisionInput): CrawlRouteDecision {
+  const engines: CrawlEngineName[] = [];
+  const reasons: string[] = [];
+
+  const appType = input.appType?.toLowerCase();
+  if (appType === 'android' || appType === 'ios' || appType === 'windows' || appType === 'macos') {
+    return { engines: ['native'], reasons: [`appType=${input.appType}`], confidence: 0.95 };
+  }
+
+  if (input.hasStorageState || input.hasAuthToken) {
+    engines.push('recorded-session');
+    reasons.push('session/token auth available');
+  }
+
+  engines.push(input.hasProxy ? 'browser-stealth' : 'browser');
+  reasons.push(input.hasProxy ? 'proxy configured; prefer stealth browser path' : 'default browser path');
+
+  if (input.hasCredentials || input.profile.industry !== 'unknown') {
+    engines.push('stagehand');
+    reasons.push(input.hasCredentials ? 'credentials available for AI login/navigation' : `domain profile=${input.profile.industry}`);
+  }
+
+  const hardUi = input.profile.industry === 'legacy-enterprise' || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? '');
+  if (input.hasSkyvern && hardUi) {
+    engines.push('vision');
+    reasons.push('hard UI or previous block detected; vision fallback enabled');
+  }
+
+  if (engines.length === 1) {
+    engines.unshift('http');
+    reasons.push('HTTP/static extraction can be attempted before browser escalation');
+  }
+
+  return {
+    engines: Array.from(new Set(engines)),
+    reasons,
+    confidence: input.profile.confidence,
+  };
+}

package/src/export/bulk-export.ts

@@ -0,0 +1,85 @@
+/**
+ * Bulk export — export crawled screens to CSV / JSON / Markdown digest.
+ *
+ * Outputs:
+ * - CSV: url, name, wordCount, screenshotPath, markdownPath, createdAt — for spreadsheet analysis
+ * - JSON: full screen records array — for API integration
+ * - Markdown digest: concatenated markdown of all screens — for LLM context windows
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+export interface ExportScreen {
+  id: string;
+  name: string;
+  url: string;
+  projectId: string;
+  wordCount?: number;
+  screenshotPath?: string | null;
+  markdownPath?: string | null;
+  markdown?: string | null;
+  createdAt?: Date;
+  updatedAt?: Date;
+}
+
+function escapeCsv(val: string): string {
+  if (val.includes(',') || val.includes('"') || val.includes('\n')) {
+    return `"${val.replace(/"/g, '""')}"`;
+  }
+  return val;
+}
+
+export async function exportToCsv(screens: ExportScreen[], outputPath: string): Promise<void> {
+  const headers = ['id', 'name', 'url', 'projectId', 'wordCount', 'screenshotPath', 'markdownPath', 'createdAt'];
+  const rows = screens.map(s => [
+    s.id, s.name, s.url, s.projectId,
+    String(s.wordCount ?? 0),
+    s.screenshotPath ?? '',
+    s.markdownPath ?? '',
+    s.createdAt?.toISOString() ?? '',
+  ].map(escapeCsv).join(','));
+
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  await fs.writeFile(outputPath, [headers.join(','), ...rows].join('\n'), 'utf8');
+}
+
+export async function exportToJson(screens: ExportScreen[], outputPath: string): Promise<void> {
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+  const data = screens.map(s => ({
+    id: s.id,
+    name: s.name,
+    url: s.url,
+    projectId: s.projectId,
+    wordCount: s.wordCount,
+    screenshotPath: s.screenshotPath,
+    markdownPath: s.markdownPath,
+    createdAt: s.createdAt?.toISOString(),
+    updatedAt: s.updatedAt?.toISOString(),
+  }));
+  await fs.writeFile(outputPath, JSON.stringify(data, null, 2), 'utf8');
+}
+
+export async function exportToMarkdownDigest(
+  screens: ExportScreen[],
+  outputPath: string,
+  opts: { readMarkdownFromPath?: boolean; maxScreens?: number } = {},
+): Promise<void> {
+  const { readMarkdownFromPath = true, maxScreens = 200 } = opts;
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+
+  const handle = await fs.open(outputPath, 'w');
+  try {
+    await handle.write(`# ZeTa Crawl Digest\n\nExported: ${new Date().toISOString()}\nScreens: ${Math.min(screens.length, maxScreens)}\n\n---\n\n`);
+
+    for (const screen of screens.slice(0, maxScreens)) {
+      let md = screen.markdown ?? '';
+      if (!md && readMarkdownFromPath && screen.markdownPath) {
+        md = await fs.readFile(screen.markdownPath, 'utf8').catch(() => '');
+      }
+      await handle.write(`## ${screen.name}\n\n**URL:** ${screen.url}\n\n${md.trim()}\n\n---\n\n`);
+    }
+  } finally {
+    await handle.close();
+  }
+}

package/src/export/vector-export.ts

@@ -0,0 +1,160 @@
+/**
+ * Vector DB export — convert crawled screens to embedding-ready chunks.
+ *
+ * Produces a JSONL file where each line is one chunk ready for upsert into
+ * Pinecone, Qdrant, Weaviate, or any vector store. The embedding itself is
+ * NOT generated here (requires an embedding model at the user's end) — we
+ * produce the text chunks + metadata, and the caller embeds them.
+ *
+ * Format per line (JSONL):
+ * {
+ *   "id": "<screenId>-<chunkIndex>",
+ *   "text": "<chunk text>",
+ *   "metadata": {
+ *     "screenId": "...",
+ *     "projectId": "...",
+ *     "tenantId": "...",
+ *     "url": "...",
+ *     "screenName": "...",
+ *     "chunkIndex": 0,
+ *     "totalChunks": 3,
+ *     "crawledAt": "2026-...",
+ *     "wordCount": 142
+ *   }
+ * }
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+export interface VectorChunk {
+  id: string;
+  text: string;
+  metadata: {
+    screenId: string;
+    projectId: string;
+    tenantId: string;
+    url: string;
+    screenName: string;
+    chunkIndex: number;
+    totalChunks: number;
+    crawledAt: string;
+    wordCount: number;
+  };
+}
+
+export interface ScreenRecord {
+  id: string;
+  name: string;
+  url: string;
+  projectId: string;
+  tenantId: string;
+  markdown?: string | null;
+  markdownPath?: string | null;
+  updatedAt?: Date;
+}
+
+function splitIntoChunks(text: string, maxChunkWords = 300): string[] {
+  const paragraphs = text.split(/\n{2,}/).filter(p => p.trim().length > 0);
+  const chunks: string[] = [];
+  let current = '';
+  let wordCount = 0;
+
+  for (const para of paragraphs) {
+    const words = para.split(/\s+/).length;
+    if (wordCount + words > maxChunkWords && current.length > 0) {
+      chunks.push(current.trim());
+      current = '';
+      wordCount = 0;
+    }
+    current += (current ? '\n\n' : '') + para;
+    wordCount += words;
+  }
+  if (current.trim()) chunks.push(current.trim());
+  return chunks.length > 0 ? chunks : [text.slice(0, 2000)];
+}
+
+export async function exportScreensToVectorJsonl(
+  screens: ScreenRecord[],
+  outputPath: string,
+  opts: { maxChunkWords?: number; readMarkdownFromPath?: boolean } = {},
+): Promise<{ totalChunks: number; skipped: number }> {
+  const { maxChunkWords = 300, readMarkdownFromPath = true } = opts;
+  await fs.mkdir(path.dirname(outputPath), { recursive: true });
+
+  const handle = await fs.open(outputPath, 'w');
+  let totalChunks = 0;
+  let skipped = 0;
+
+  try {
+    for (const screen of screens) {
+      let markdown = screen.markdown ?? '';
+
+      if (!markdown && readMarkdownFromPath && screen.markdownPath) {
+        try {
+          markdown = await fs.readFile(screen.markdownPath, 'utf8');
+        } catch { /* file missing — skip */ }
+      }
+
+      if (!markdown || markdown.trim().length < 20) {
+        skipped++;
+        continue;
+      }
+
+      const textChunks = splitIntoChunks(markdown, maxChunkWords);
+      for (let i = 0; i < textChunks.length; i++) {
+        const chunk: VectorChunk = {
+          id: `${screen.id}-${i}`,
+          text: textChunks[i],
+          metadata: {
+            screenId: screen.id,
+            projectId: screen.projectId,
+            tenantId: screen.tenantId,
+            url: screen.url,
+            screenName: screen.name,
+            chunkIndex: i,
+            totalChunks: textChunks.length,
+            crawledAt: screen.updatedAt?.toISOString() ?? new Date().toISOString(),
+            wordCount: textChunks[i].split(/\s+/).length,
+          },
+        };
+        await handle.write(JSON.stringify(chunk) + '\n');
+        totalChunks++;
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+
+  return { totalChunks, skipped };
+}
+
+/**
+ * Pinecone-compatible upsert batch format.
+ * Call this after generating embeddings to produce the final upsert payload.
+ */
+export function toPineconeUpsert(
+  chunks: VectorChunk[],
+  embeddings: number[][],
+  namespace = 'zeta-crawl',
+): Array<{ id: string; values: number[]; metadata: VectorChunk['metadata'] }> {
+  return chunks.map((chunk, i) => ({
+    id: chunk.id,
+    values: embeddings[i],
+    metadata: chunk.metadata,
+  }));
+}
+
+/**
+ * Qdrant-compatible point format.
+ */
+export function toQdrantPoints(
+  chunks: VectorChunk[],
+  embeddings: number[][],
+): Array<{ id: string; vector: number[]; payload: VectorChunk['metadata'] & { text: string } }> {
+  return chunks.map((chunk, i) => ({
+    id: chunk.id,
+    vector: embeddings[i],
+    payload: { ...chunk.metadata, text: chunk.text },
+  }));
+}

package/src/extractors/advanced.ts

@@ -0,0 +1,309 @@
+import { parseHTML } from 'linkedom';
+
+export interface ExtractionResult {
+  matches: string[];
+  count: number;
+  method: string;
+}
+
+export type TableRow = Record<string, string>;
+
+export interface PageMetadata {
+  title: string | null;
+  description: string | null;
+  keywords: string | null;
+  ogTitle: string | null;
+  ogDescription: string | null;
+  ogImage: string | null;
+  ogUrl: string | null;
+  ogType: string | null;
+  ogSiteName: string | null;
+  ogLocale: string | null;
+  twitterCard: string | null;
+  twitterTitle: string | null;
+  twitterDescription: string | null;
+  twitterImage: string | null;
+  twitterSite: string | null;
+  canonical: string | null;
+  robots: string | null;
+  hreflang: Array<{ lang: string; href: string }>;
+  themeColor: string | null;
+  favicon: string | null;
+  jsonLd: object[];
+  schemaTypes: string[];
+  lang: string | null;
+}
+
+export function extractByCssSelector(html: string, selector: string, attr?: string): ExtractionResult {
+  try {
+    const { document } = parseHTML(html);
+    const nodes = Array.from(document.querySelectorAll(selector));
+    const matches = nodes.map((n: any) => {
+      if (attr) return (n as any).getAttribute(attr) ?? '';
+      return (n as any).textContent?.trim() ?? '';
+    }).filter(Boolean);
+    return { matches, count: matches.length, method: 'css-selector' };
+  } catch (e) {
+    return { matches: [], count: 0, method: 'css-selector-error' };
+  }
+}
+
+export function extractByXPath(html: string, xpath: string): ExtractionResult {
+  try {
+    const { document, XPathResult } = parseHTML(html) as any;
+    const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);
+    const matches: string[] = [];
+    let node = result.iterateNext?.();
+    while (node) {
+      const text = (node as any).textContent?.trim() ?? (node as any).nodeValue?.trim() ?? '';
+      if (text) matches.push(text);
+      node = result.iterateNext?.();
+    }
+    return { matches, count: matches.length, method: 'xpath' };
+  } catch (e) {
+    return { matches: [], count: 0, method: 'xpath-error' };
+  }
+}
+
+export function extractByRegex(text: string, pattern: string, flags?: string): ExtractionResult {
+  try {
+    const re = new RegExp(pattern, (flags ?? 'g').includes('g') ? flags : (flags ?? '') + 'g');
+    const matches: string[] = [];
+    let m: RegExpExecArray | null;
+    while ((m = re.exec(text)) !== null) {
+      matches.push(m[1] ?? m[0]);
+      if (!re.global) break;
+    }
+    return { matches, count: matches.length, method: 'regex' };
+  } catch {
+    return { matches: [], count: 0, method: 'regex-error' };
+  }
+}
+
+export function extractTables(html: string): TableRow[][] {
+  try {
+    const { document } = parseHTML(html);
+    const tables = Array.from(document.querySelectorAll('table'));
+    return tables.map((table: any) => {
+      const headers: string[] = [];
+      const rows: TableRow[] = [];
+      const ths = [...table.querySelectorAll('thead th, thead td, tr:first-child th')];
+      if (ths.length > 0) {
+        ths.forEach((th: any, i: number) => headers.push(th.textContent?.trim() || `col${i + 1}`));
+      }
+      const trs = [...table.querySelectorAll('tbody tr, tr')];
+      for (const tr of trs) {
+        const cells = [...(tr as any).querySelectorAll('td, th')];
+        if (cells.length === 0) continue;
+        if (headers.length === 0) {
+          cells.forEach((_: any, i: number) => headers.push(`col${i + 1}`));
+        }
+        const row: TableRow = {};
+        cells.forEach((cell: any, i: number) => {
+          const key = headers[i] || `col${i + 1}`;
+          row[key] = cell.textContent?.trim() ?? '';
+        });
+        if (Object.values(row).some(v => v)) rows.push(row);
+      }
+      return rows;
+    }).filter(t => t.length > 0);
+  } catch {
+    return [];
+  }
+}
+
+export function extractMetadata(html: string, url: string): PageMetadata {
+  const meta: PageMetadata = {
+    title: null, description: null, keywords: null,
+    ogTitle: null, ogDescription: null, ogImage: null, ogUrl: null, ogType: null,
+    ogSiteName: null, ogLocale: null,
+    twitterCard: null, twitterTitle: null, twitterDescription: null,
+    twitterImage: null, twitterSite: null,
+    canonical: null, robots: null,
+    hreflang: [], themeColor: null, favicon: null,
+    jsonLd: [], schemaTypes: [], lang: null,
+  };
+  try {
+    const { document } = parseHTML(html);
+    meta.title = (document as any).title ?? (document as any).querySelector('title')?.textContent?.trim() ?? null;
+    meta.lang = (document as any).documentElement?.getAttribute('lang') ?? null;
+
+    const metas = [...(document as any).querySelectorAll('meta')];
+    for (const m of metas) {
+      const name = (m as any).getAttribute('name')?.toLowerCase() ?? '';
+      const prop = (m as any).getAttribute('property')?.toLowerCase() ?? '';
+      const content = (m as any).getAttribute('content') ?? '';
+      if (name === 'description') meta.description = content;
+      if (name === 'keywords') meta.keywords = content;
+      if (name === 'robots') meta.robots = content;
+      if (name === 'twitter:card') meta.twitterCard = content;
+      if (name === 'twitter:title') meta.twitterTitle = content;
+      if (name === 'twitter:description') meta.twitterDescription = content;
+      if (name === 'twitter:image') meta.twitterImage = content;
+      if (name === 'twitter:site') meta.twitterSite = content;
+      if (name === 'theme-color') meta.themeColor = content;
+      if (prop === 'og:title') meta.ogTitle = content;
+      if (prop === 'og:description') meta.ogDescription = content;
+      if (prop === 'og:image') meta.ogImage = content;
+      if (prop === 'og:url') meta.ogUrl = content;
+      if (prop === 'og:type') meta.ogType = content;
+      if (prop === 'og:site_name') meta.ogSiteName = content;
+      if (prop === 'og:locale') meta.ogLocale = content;
+    }
+
+    const canonical = (document as any).querySelector('link[rel="canonical"]');
+    if (canonical) meta.canonical = canonical.getAttribute('href') ?? null;
+
+    const faviconEl = (document as any).querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+    if (faviconEl) meta.favicon = faviconEl.getAttribute('href') ?? null;
+
+    const hreflangEls = [...(document as any).querySelectorAll('link[rel="alternate"][hreflang]')];
+    for (const el of hreflangEls) {
+      const lang = (el as any).getAttribute('hreflang') ?? '';
+      const href = (el as any).getAttribute('href') ?? '';
+      if (lang && href) meta.hreflang.push({ lang, href });
+    }
+
+    const jsonLdScripts = [...(document as any).querySelectorAll('script[type="application/ld+json"]')];
+    for (const s of jsonLdScripts) {
+      try {
+        const parsed = JSON.parse((s as any).textContent ?? '{}');
+        meta.jsonLd.push(parsed);
+        const types = Array.isArray(parsed) ? parsed.map((p: any) => p['@type']).flat() : [parsed['@type']];
+        meta.schemaTypes.push(...types.filter(Boolean));
+      } catch { /* skip invalid JSON-LD */ }
+    }
+  } catch { /* non-fatal */ }
+  return meta;
+}
+
+export interface ProductData {
+  name: string | null;
+  description: string | null;
+  brand: string | null;
+  sku: string | null;
+  price: number | null;
+  priceCurrency: string | null;
+  availability: string | null;
+  rating: number | null;
+  reviewCount: number | null;
+  image: string | null;
+  url: string | null;
+}
+
+export function extractStructuredProduct(jsonLd: object[]): ProductData | null {
+  for (const item of jsonLd) {
+    try {
+      const type = (item as any)['@type'];
+      const isProduct = Array.isArray(type)
+        ? type.some((t: any) => typeof t === 'string' && t.toLowerCase() === 'product')
+        : typeof type === 'string' && type.toLowerCase() === 'product';
+      if (!isProduct) continue;
+
+      const i = item as any;
+      const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;
+
+      const rawPrice = offers?.price;
+      const price = rawPrice != null ? parseFloat(rawPrice) : NaN;
+
+      const rawRating = i.aggregateRating?.ratingValue;
+      const rating = rawRating != null ? parseFloat(rawRating) : NaN;
+
+      const rawReviewCount = i.aggregateRating?.reviewCount;
+      const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;
+
+      const rawAvailability: string | undefined = offers?.availability;
+      const availability = rawAvailability
+        ? rawAvailability.replace(/^https?:\/\/schema\.org\//, '')
+        : null;
+
+      const rawImage = i.image;
+      const image = Array.isArray(rawImage) ? (rawImage[0] ?? null) : (rawImage ?? null);
+
+      const brand = i.brand?.name ?? i.brand ?? null;
+
+      return {
+        name: i.name ?? null,
+        description: i.description ?? null,
+        brand: typeof brand === 'string' ? brand : null,
+        sku: i.sku ?? null,
+        price: isNaN(price) ? null : price,
+        priceCurrency: offers?.priceCurrency ?? null,
+        availability,
+        rating: isNaN(rating) ? null : rating,
+        reviewCount: isNaN(reviewCount) ? null : reviewCount,
+        image,
+        url: i.url ?? null,
+      };
+    } catch { /* skip malformed item */ }
+  }
+  return null;
+}
+
+export function extractImages(html: string, baseUrl: string): Array<{ src: string; alt: string; width?: number; height?: number }> {
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    return Array.from((document as any).querySelectorAll('img'))
+      .map((img: any) => {
+        const src = img.getAttribute('src') ?? img.getAttribute('data-src') ?? '';
+        if (!src) return null;
+        try {
+          return {
+            src: new URL(src, base).href,
+            alt: (img.getAttribute('alt') ?? '') as string,
+            width: img.getAttribute('width') ? parseInt(img.getAttribute('width')) : undefined,
+            height: img.getAttribute('height') ? parseInt(img.getAttribute('height')) : undefined,
+          } as { src: string; alt: string; width?: number; height?: number };
+        } catch { return null; }
+      })
+      .filter((x): x is { src: string; alt: string; width?: number; height?: number } => x !== null);
+  } catch {
+    return [];
+  }
+}
+
+export function extractMediaFiles(html: string, baseUrl: string): Array<{ url: string; type: 'image' | 'video' | 'audio' | 'document'; tag: string }> {
+  const IMAGE_EXT = /\.(png|jpe?g|gif|webp|svg|ico|avif)(\?.*)?$/i;
+  const VIDEO_EXT = /\.(mp4|webm|ogv|mov|avi)(\?.*)?$/i;
+  const AUDIO_EXT = /\.(mp3|ogg|wav|flac|aac)(\?.*)?$/i;
+  const DOC_EXT = /\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\?.*)?$/i;
+
+  function classify(url: string, tag: string): 'image' | 'video' | 'audio' | 'document' {
+    if (tag === 'img' || IMAGE_EXT.test(url)) return 'image';
+    if (tag === 'video' || VIDEO_EXT.test(url)) return 'video';
+    if (tag === 'audio' || AUDIO_EXT.test(url)) return 'audio';
+    if (DOC_EXT.test(url)) return 'document';
+    return 'document';
+  }
+
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    const results: Array<{ url: string; type: 'image' | 'video' | 'audio' | 'document'; tag: string }> = [];
+    const seen = new Set<string>();
+
+    const selectors: [string, string] = ['img[src],video[src],audio[src],source[src]', 'src'];
+    const attrPairs: Array<[string, string]> = [
+      ['img', 'src'], ['video', 'src'], ['audio', 'src'], ['source', 'src'],
+      ['a', 'href'],
+    ];
+
+    for (const [sel, attr] of attrPairs) {
+      for (const el of [...(document as any).querySelectorAll(sel)]) {
+        const raw = (el as any).getAttribute(attr) ?? '';
+        if (!raw) continue;
+        try {
+          const resolved = new URL(raw, base).href;
+          if (seen.has(resolved)) continue;
+          if (sel === 'a' && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved)) continue;
+          seen.add(resolved);
+          results.push({ url: resolved, type: classify(resolved, sel), tag: sel });
+        } catch { /* skip */ }
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}