@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/extractors/advanced.js

@@ -0,0 +1,265 @@
+import { parseHTML } from "linkedom";
+function extractByCssSelector(html, selector, attr) {
+  try {
+    const { document } = parseHTML(html);
+    const nodes = Array.from(document.querySelectorAll(selector));
+    const matches = nodes.map((n) => {
+      if (attr) return n.getAttribute(attr) ?? "";
+      return n.textContent?.trim() ?? "";
+    }).filter(Boolean);
+    return { matches, count: matches.length, method: "css-selector" };
+  } catch (e) {
+    return { matches: [], count: 0, method: "css-selector-error" };
+  }
+}
+function extractByXPath(html, xpath) {
+  try {
+    const { document, XPathResult } = parseHTML(html);
+    const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);
+    const matches = [];
+    let node = result.iterateNext?.();
+    while (node) {
+      const text = node.textContent?.trim() ?? node.nodeValue?.trim() ?? "";
+      if (text) matches.push(text);
+      node = result.iterateNext?.();
+    }
+    return { matches, count: matches.length, method: "xpath" };
+  } catch (e) {
+    return { matches: [], count: 0, method: "xpath-error" };
+  }
+}
+function extractByRegex(text, pattern, flags) {
+  try {
+    const re = new RegExp(pattern, (flags ?? "g").includes("g") ? flags : (flags ?? "") + "g");
+    const matches = [];
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      matches.push(m[1] ?? m[0]);
+      if (!re.global) break;
+    }
+    return { matches, count: matches.length, method: "regex" };
+  } catch {
+    return { matches: [], count: 0, method: "regex-error" };
+  }
+}
+function extractTables(html) {
+  try {
+    const { document } = parseHTML(html);
+    const tables = Array.from(document.querySelectorAll("table"));
+    return tables.map((table) => {
+      const headers = [];
+      const rows = [];
+      const ths = [...table.querySelectorAll("thead th, thead td, tr:first-child th")];
+      if (ths.length > 0) {
+        ths.forEach((th, i) => headers.push(th.textContent?.trim() || `col${i + 1}`));
+      }
+      const trs = [...table.querySelectorAll("tbody tr, tr")];
+      for (const tr of trs) {
+        const cells = [...tr.querySelectorAll("td, th")];
+        if (cells.length === 0) continue;
+        if (headers.length === 0) {
+          cells.forEach((_, i) => headers.push(`col${i + 1}`));
+        }
+        const row = {};
+        cells.forEach((cell, i) => {
+          const key = headers[i] || `col${i + 1}`;
+          row[key] = cell.textContent?.trim() ?? "";
+        });
+        if (Object.values(row).some((v) => v)) rows.push(row);
+      }
+      return rows;
+    }).filter((t) => t.length > 0);
+  } catch {
+    return [];
+  }
+}
+function extractMetadata(html, url) {
+  const meta = {
+    title: null,
+    description: null,
+    keywords: null,
+    ogTitle: null,
+    ogDescription: null,
+    ogImage: null,
+    ogUrl: null,
+    ogType: null,
+    ogSiteName: null,
+    ogLocale: null,
+    twitterCard: null,
+    twitterTitle: null,
+    twitterDescription: null,
+    twitterImage: null,
+    twitterSite: null,
+    canonical: null,
+    robots: null,
+    hreflang: [],
+    themeColor: null,
+    favicon: null,
+    jsonLd: [],
+    schemaTypes: [],
+    lang: null
+  };
+  try {
+    const { document } = parseHTML(html);
+    meta.title = document.title ?? document.querySelector("title")?.textContent?.trim() ?? null;
+    meta.lang = document.documentElement?.getAttribute("lang") ?? null;
+    const metas = [...document.querySelectorAll("meta")];
+    for (const m of metas) {
+      const name = m.getAttribute("name")?.toLowerCase() ?? "";
+      const prop = m.getAttribute("property")?.toLowerCase() ?? "";
+      const content = m.getAttribute("content") ?? "";
+      if (name === "description") meta.description = content;
+      if (name === "keywords") meta.keywords = content;
+      if (name === "robots") meta.robots = content;
+      if (name === "twitter:card") meta.twitterCard = content;
+      if (name === "twitter:title") meta.twitterTitle = content;
+      if (name === "twitter:description") meta.twitterDescription = content;
+      if (name === "twitter:image") meta.twitterImage = content;
+      if (name === "twitter:site") meta.twitterSite = content;
+      if (name === "theme-color") meta.themeColor = content;
+      if (prop === "og:title") meta.ogTitle = content;
+      if (prop === "og:description") meta.ogDescription = content;
+      if (prop === "og:image") meta.ogImage = content;
+      if (prop === "og:url") meta.ogUrl = content;
+      if (prop === "og:type") meta.ogType = content;
+      if (prop === "og:site_name") meta.ogSiteName = content;
+      if (prop === "og:locale") meta.ogLocale = content;
+    }
+    const canonical = document.querySelector('link[rel="canonical"]');
+    if (canonical) meta.canonical = canonical.getAttribute("href") ?? null;
+    const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+    if (faviconEl) meta.favicon = faviconEl.getAttribute("href") ?? null;
+    const hreflangEls = [...document.querySelectorAll('link[rel="alternate"][hreflang]')];
+    for (const el of hreflangEls) {
+      const lang = el.getAttribute("hreflang") ?? "";
+      const href = el.getAttribute("href") ?? "";
+      if (lang && href) meta.hreflang.push({ lang, href });
+    }
+    const jsonLdScripts = [...document.querySelectorAll('script[type="application/ld+json"]')];
+    for (const s of jsonLdScripts) {
+      try {
+        const parsed = JSON.parse(s.textContent ?? "{}");
+        meta.jsonLd.push(parsed);
+        const types = Array.isArray(parsed) ? parsed.map((p) => p["@type"]).flat() : [parsed["@type"]];
+        meta.schemaTypes.push(...types.filter(Boolean));
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return meta;
+}
+function extractStructuredProduct(jsonLd) {
+  for (const item of jsonLd) {
+    try {
+      const type = item["@type"];
+      const isProduct = Array.isArray(type) ? type.some((t) => typeof t === "string" && t.toLowerCase() === "product") : typeof type === "string" && type.toLowerCase() === "product";
+      if (!isProduct) continue;
+      const i = item;
+      const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;
+      const rawPrice = offers?.price;
+      const price = rawPrice != null ? parseFloat(rawPrice) : NaN;
+      const rawRating = i.aggregateRating?.ratingValue;
+      const rating = rawRating != null ? parseFloat(rawRating) : NaN;
+      const rawReviewCount = i.aggregateRating?.reviewCount;
+      const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;
+      const rawAvailability = offers?.availability;
+      const availability = rawAvailability ? rawAvailability.replace(/^https?:\/\/schema\.org\//, "") : null;
+      const rawImage = i.image;
+      const image = Array.isArray(rawImage) ? rawImage[0] ?? null : rawImage ?? null;
+      const brand = i.brand?.name ?? i.brand ?? null;
+      return {
+        name: i.name ?? null,
+        description: i.description ?? null,
+        brand: typeof brand === "string" ? brand : null,
+        sku: i.sku ?? null,
+        price: isNaN(price) ? null : price,
+        priceCurrency: offers?.priceCurrency ?? null,
+        availability,
+        rating: isNaN(rating) ? null : rating,
+        reviewCount: isNaN(reviewCount) ? null : reviewCount,
+        image,
+        url: i.url ?? null
+      };
+    } catch {
+    }
+  }
+  return null;
+}
+function extractImages(html, baseUrl) {
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    return Array.from(document.querySelectorAll("img")).map((img) => {
+      const src = img.getAttribute("src") ?? img.getAttribute("data-src") ?? "";
+      if (!src) return null;
+      try {
+        return {
+          src: new URL(src, base).href,
+          alt: img.getAttribute("alt") ?? "",
+          width: img.getAttribute("width") ? parseInt(img.getAttribute("width")) : void 0,
+          height: img.getAttribute("height") ? parseInt(img.getAttribute("height")) : void 0
+        };
+      } catch {
+        return null;
+      }
+    }).filter((x) => x !== null);
+  } catch {
+    return [];
+  }
+}
+function extractMediaFiles(html, baseUrl) {
+  const IMAGE_EXT = /\.(png|jpe?g|gif|webp|svg|ico|avif)(\?.*)?$/i;
+  const VIDEO_EXT = /\.(mp4|webm|ogv|mov|avi)(\?.*)?$/i;
+  const AUDIO_EXT = /\.(mp3|ogg|wav|flac|aac)(\?.*)?$/i;
+  const DOC_EXT = /\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\?.*)?$/i;
+  function classify(url, tag) {
+    if (tag === "img" || IMAGE_EXT.test(url)) return "image";
+    if (tag === "video" || VIDEO_EXT.test(url)) return "video";
+    if (tag === "audio" || AUDIO_EXT.test(url)) return "audio";
+    if (DOC_EXT.test(url)) return "document";
+    return "document";
+  }
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    const results = [];
+    const seen = /* @__PURE__ */ new Set();
+    const selectors = ["img[src],video[src],audio[src],source[src]", "src"];
+    const attrPairs = [
+      ["img", "src"],
+      ["video", "src"],
+      ["audio", "src"],
+      ["source", "src"],
+      ["a", "href"]
+    ];
+    for (const [sel, attr] of attrPairs) {
+      for (const el of [...document.querySelectorAll(sel)]) {
+        const raw = el.getAttribute(attr) ?? "";
+        if (!raw) continue;
+        try {
+          const resolved = new URL(raw, base).href;
+          if (seen.has(resolved)) continue;
+          if (sel === "a" && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved)) continue;
+          seen.add(resolved);
+          results.push({ url: resolved, type: classify(resolved, sel), tag: sel });
+        } catch {
+        }
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
+export {
+  extractByCssSelector,
+  extractByRegex,
+  extractByXPath,
+  extractImages,
+  extractMediaFiles,
+  extractMetadata,
+  extractStructuredProduct,
+  extractTables
+};
+//# sourceMappingURL=advanced.js.map

package/dist/extractors/advanced.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/advanced.ts"],
+  "sourcesContent": ["import { parseHTML } from 'linkedom';\n\nexport interface ExtractionResult {\n  matches: string[];\n  count: number;\n  method: string;\n}\n\nexport type TableRow = Record<string, string>;\n\nexport interface PageMetadata {\n  title: string | null;\n  description: string | null;\n  keywords: string | null;\n  ogTitle: string | null;\n  ogDescription: string | null;\n  ogImage: string | null;\n  ogUrl: string | null;\n  ogType: string | null;\n  ogSiteName: string | null;\n  ogLocale: string | null;\n  twitterCard: string | null;\n  twitterTitle: string | null;\n  twitterDescription: string | null;\n  twitterImage: string | null;\n  twitterSite: string | null;\n  canonical: string | null;\n  robots: string | null;\n  hreflang: Array<{ lang: string; href: string }>;\n  themeColor: string | null;\n  favicon: string | null;\n  jsonLd: object[];\n  schemaTypes: string[];\n  lang: string | null;\n}\n\nexport function extractByCssSelector(html: string, selector: string, attr?: string): ExtractionResult {\n  try {\n    const { document } = parseHTML(html);\n    const nodes = Array.from(document.querySelectorAll(selector));\n    const matches = nodes.map((n: any) => {\n      if (attr) return (n as any).getAttribute(attr) ?? '';\n      return (n as any).textContent?.trim() ?? '';\n    }).filter(Boolean);\n    return { matches, count: matches.length, method: 'css-selector' };\n  } catch (e) {\n    return { matches: [], count: 0, method: 'css-selector-error' };\n  }\n}\n\nexport function extractByXPath(html: string, xpath: string): ExtractionResult {\n  try {\n    const { document, XPathResult } = parseHTML(html) as any;\n    const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);\n    const matches: string[] = [];\n    let node = result.iterateNext?.();\n    while (node) {\n      const text = (node as any).textContent?.trim() ?? (node as any).nodeValue?.trim() ?? '';\n      if (text) matches.push(text);\n      node = result.iterateNext?.();\n    }\n    return { matches, count: matches.length, method: 'xpath' };\n  } catch (e) {\n    return { matches: [], count: 0, method: 'xpath-error' };\n  }\n}\n\nexport function extractByRegex(text: string, pattern: string, flags?: string): ExtractionResult {\n  try {\n    const re = new RegExp(pattern, (flags ?? 'g').includes('g') ? flags : (flags ?? '') + 'g');\n    const matches: string[] = [];\n    let m: RegExpExecArray | null;\n    while ((m = re.exec(text)) !== null) {\n      matches.push(m[1] ?? m[0]);\n      if (!re.global) break;\n    }\n    return { matches, count: matches.length, method: 'regex' };\n  } catch {\n    return { matches: [], count: 0, method: 'regex-error' };\n  }\n}\n\nexport function extractTables(html: string): TableRow[][] {\n  try {\n    const { document } = parseHTML(html);\n    const tables = Array.from(document.querySelectorAll('table'));\n    return tables.map((table: any) => {\n      const headers: string[] = [];\n      const rows: TableRow[] = [];\n      const ths = [...table.querySelectorAll('thead th, thead td, tr:first-child th')];\n      if (ths.length > 0) {\n        ths.forEach((th: any, i: number) => headers.push(th.textContent?.trim() || `col${i + 1}`));\n      }\n      const trs = [...table.querySelectorAll('tbody tr, tr')];\n      for (const tr of trs) {\n        const cells = [...(tr as any).querySelectorAll('td, th')];\n        if (cells.length === 0) continue;\n        if (headers.length === 0) {\n          cells.forEach((_: any, i: number) => headers.push(`col${i + 1}`));\n        }\n        const row: TableRow = {};\n        cells.forEach((cell: any, i: number) => {\n          const key = headers[i] || `col${i + 1}`;\n          row[key] = cell.textContent?.trim() ?? '';\n        });\n        if (Object.values(row).some(v => v)) rows.push(row);\n      }\n      return rows;\n    }).filter(t => t.length > 0);\n  } catch {\n    return [];\n  }\n}\n\nexport function extractMetadata(html: string, url: string): PageMetadata {\n  const meta: PageMetadata = {\n    title: null, description: null, keywords: null,\n    ogTitle: null, ogDescription: null, ogImage: null, ogUrl: null, ogType: null,\n    ogSiteName: null, ogLocale: null,\n    twitterCard: null, twitterTitle: null, twitterDescription: null,\n    twitterImage: null, twitterSite: null,\n    canonical: null, robots: null,\n    hreflang: [], themeColor: null, favicon: null,\n    jsonLd: [], schemaTypes: [], lang: null,\n  };\n  try {\n    const { document } = parseHTML(html);\n    meta.title = (document as any).title ?? (document as any).querySelector('title')?.textContent?.trim() ?? null;\n    meta.lang = (document as any).documentElement?.getAttribute('lang') ?? null;\n\n    const metas = [...(document as any).querySelectorAll('meta')];\n    for (const m of metas) {\n      const name = (m as any).getAttribute('name')?.toLowerCase() ?? '';\n      const prop = (m as any).getAttribute('property')?.toLowerCase() ?? '';\n      const content = (m as any).getAttribute('content') ?? '';\n      if (name === 'description') meta.description = content;\n      if (name === 'keywords') meta.keywords = content;\n      if (name === 'robots') meta.robots = content;\n      if (name === 'twitter:card') meta.twitterCard = content;\n      if (name === 'twitter:title') meta.twitterTitle = content;\n      if (name === 'twitter:description') meta.twitterDescription = content;\n      if (name === 'twitter:image') meta.twitterImage = content;\n      if (name === 'twitter:site') meta.twitterSite = content;\n      if (name === 'theme-color') meta.themeColor = content;\n      if (prop === 'og:title') meta.ogTitle = content;\n      if (prop === 'og:description') meta.ogDescription = content;\n      if (prop === 'og:image') meta.ogImage = content;\n      if (prop === 'og:url') meta.ogUrl = content;\n      if (prop === 'og:type') meta.ogType = content;\n      if (prop === 'og:site_name') meta.ogSiteName = content;\n      if (prop === 'og:locale') meta.ogLocale = content;\n    }\n\n    const canonical = (document as any).querySelector('link[rel=\"canonical\"]');\n    if (canonical) meta.canonical = canonical.getAttribute('href') ?? null;\n\n    const faviconEl = (document as any).querySelector('link[rel=\"icon\"], link[rel=\"shortcut icon\"]');\n    if (faviconEl) meta.favicon = faviconEl.getAttribute('href') ?? null;\n\n    const hreflangEls = [...(document as any).querySelectorAll('link[rel=\"alternate\"][hreflang]')];\n    for (const el of hreflangEls) {\n      const lang = (el as any).getAttribute('hreflang') ?? '';\n      const href = (el as any).getAttribute('href') ?? '';\n      if (lang && href) meta.hreflang.push({ lang, href });\n    }\n\n    const jsonLdScripts = [...(document as any).querySelectorAll('script[type=\"application/ld+json\"]')];\n    for (const s of jsonLdScripts) {\n      try {\n        const parsed = JSON.parse((s as any).textContent ?? '{}');\n        meta.jsonLd.push(parsed);\n        const types = Array.isArray(parsed) ? parsed.map((p: any) => p['@type']).flat() : [parsed['@type']];\n        meta.schemaTypes.push(...types.filter(Boolean));\n      } catch { /* skip invalid JSON-LD */ }\n    }\n  } catch { /* non-fatal */ }\n  return meta;\n}\n\nexport interface ProductData {\n  name: string | null;\n  description: string | null;\n  brand: string | null;\n  sku: string | null;\n  price: number | null;\n  priceCurrency: string | null;\n  availability: string | null;\n  rating: number | null;\n  reviewCount: number | null;\n  image: string | null;\n  url: string | null;\n}\n\nexport function extractStructuredProduct(jsonLd: object[]): ProductData | null {\n  for (const item of jsonLd) {\n    try {\n      const type = (item as any)['@type'];\n      const isProduct = Array.isArray(type)\n        ? type.some((t: any) => typeof t === 'string' && t.toLowerCase() === 'product')\n        : typeof type === 'string' && type.toLowerCase() === 'product';\n      if (!isProduct) continue;\n\n      const i = item as any;\n      const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;\n\n      const rawPrice = offers?.price;\n      const price = rawPrice != null ? parseFloat(rawPrice) : NaN;\n\n      const rawRating = i.aggregateRating?.ratingValue;\n      const rating = rawRating != null ? parseFloat(rawRating) : NaN;\n\n      const rawReviewCount = i.aggregateRating?.reviewCount;\n      const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;\n\n      const rawAvailability: string | undefined = offers?.availability;\n      const availability = rawAvailability\n        ? rawAvailability.replace(/^https?:\\/\\/schema\\.org\\//, '')\n        : null;\n\n      const rawImage = i.image;\n      const image = Array.isArray(rawImage) ? (rawImage[0] ?? null) : (rawImage ?? null);\n\n      const brand = i.brand?.name ?? i.brand ?? null;\n\n      return {\n        name: i.name ?? null,\n        description: i.description ?? null,\n        brand: typeof brand === 'string' ? brand : null,\n        sku: i.sku ?? null,\n        price: isNaN(price) ? null : price,\n        priceCurrency: offers?.priceCurrency ?? null,\n        availability,\n        rating: isNaN(rating) ? null : rating,\n        reviewCount: isNaN(reviewCount) ? null : reviewCount,\n        image,\n        url: i.url ?? null,\n      };\n    } catch { /* skip malformed item */ }\n  }\n  return null;\n}\n\nexport function extractImages(html: string, baseUrl: string): Array<{ src: string; alt: string; width?: number; height?: number }> {\n  try {\n    const { document } = parseHTML(html);\n    const base = new URL(baseUrl);\n    return Array.from((document as any).querySelectorAll('img'))\n      .map((img: any) => {\n        const src = img.getAttribute('src') ?? img.getAttribute('data-src') ?? '';\n        if (!src) return null;\n        try {\n          return {\n            src: new URL(src, base).href,\n            alt: (img.getAttribute('alt') ?? '') as string,\n            width: img.getAttribute('width') ? parseInt(img.getAttribute('width')) : undefined,\n            height: img.getAttribute('height') ? parseInt(img.getAttribute('height')) : undefined,\n          } as { src: string; alt: string; width?: number; height?: number };\n        } catch { return null; }\n      })\n      .filter((x): x is { src: string; alt: string; width?: number; height?: number } => x !== null);\n  } catch {\n    return [];\n  }\n}\n\nexport function extractMediaFiles(html: string, baseUrl: string): Array<{ url: string; type: 'image' | 'video' | 'audio' | 'document'; tag: string }> {\n  const IMAGE_EXT = /\\.(png|jpe?g|gif|webp|svg|ico|avif)(\\?.*)?$/i;\n  const VIDEO_EXT = /\\.(mp4|webm|ogv|mov|avi)(\\?.*)?$/i;\n  const AUDIO_EXT = /\\.(mp3|ogg|wav|flac|aac)(\\?.*)?$/i;\n  const DOC_EXT = /\\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\\?.*)?$/i;\n\n  function classify(url: string, tag: string): 'image' | 'video' | 'audio' | 'document' {\n    if (tag === 'img' || IMAGE_EXT.test(url)) return 'image';\n    if (tag === 'video' || VIDEO_EXT.test(url)) return 'video';\n    if (tag === 'audio' || AUDIO_EXT.test(url)) return 'audio';\n    if (DOC_EXT.test(url)) return 'document';\n    return 'document';\n  }\n\n  try {\n    const { document } = parseHTML(html);\n    const base = new URL(baseUrl);\n    const results: Array<{ url: string; type: 'image' | 'video' | 'audio' | 'document'; tag: string }> = [];\n    const seen = new Set<string>();\n\n    const selectors: [string, string] = ['img[src],video[src],audio[src],source[src]', 'src'];\n    const attrPairs: Array<[string, string]> = [\n      ['img', 'src'], ['video', 'src'], ['audio', 'src'], ['source', 'src'],\n      ['a', 'href'],\n    ];\n\n    for (const [sel, attr] of attrPairs) {\n      for (const el of [...(document as any).querySelectorAll(sel)]) {\n        const raw = (el as any).getAttribute(attr) ?? '';\n        if (!raw) continue;\n        try {\n          const resolved = new URL(raw, base).href;\n          if (seen.has(resolved)) continue;\n          if (sel === 'a' && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved)) continue;\n          seen.add(resolved);\n          results.push({ url: resolved, type: classify(resolved, sel), tag: sel });\n        } catch { /* skip */ }\n      }\n    }\n    return results;\n  } catch {\n    return [];\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,iBAAiB;AAoCnB,SAAS,qBAAqB,MAAc,UAAkB,MAAiC;AACpG,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,QAAQ,MAAM,KAAK,SAAS,iBAAiB,QAAQ,CAAC;AAC5D,UAAM,UAAU,MAAM,IAAI,CAAC,MAAW;AACpC,UAAI,KAAM,QAAQ,EAAU,aAAa,IAAI,KAAK;AAClD,aAAQ,EAAU,aAAa,KAAK,KAAK;AAAA,IAC3C,CAAC,EAAE,OAAO,OAAO;AACjB,WAAO,EAAE,SAAS,OAAO,QAAQ,QAAQ,QAAQ,eAAe;AAAA,EAClE,SAAS,GAAG;AACV,WAAO,EAAE,SAAS,CAAC,GAAG,OAAO,GAAG,QAAQ,qBAAqB;AAAA,EAC/D;AACF;AAEO,SAAS,eAAe,MAAc,OAAiC;AAC5E,MAAI;AACF,UAAM,EAAE,UAAU,YAAY,IAAI,UAAU,IAAI;AAChD,UAAM,SAAS,SAAS,SAAS,OAAO,UAAU,MAAM,aAAa,YAAY,GAAG,IAAI;AACxF,UAAM,UAAoB,CAAC;AAC3B,QAAI,OAAO,OAAO,cAAc;AAChC,WAAO,MAAM;AACX,YAAM,OAAQ,KAAa,aAAa,KAAK,KAAM,KAAa,WAAW,KAAK,KAAK;AACrF,UAAI,KAAM,SAAQ,KAAK,IAAI;AAC3B,aAAO,OAAO,cAAc;AAAA,IAC9B;AACA,WAAO,EAAE,SAAS,OAAO,QAAQ,QAAQ,QAAQ,QAAQ;AAAA,EAC3D,SAAS,GAAG;AACV,WAAO,EAAE,SAAS,CAAC,GAAG,OAAO,GAAG,QAAQ,cAAc;AAAA,EACxD;AACF;AAEO,SAAS,eAAe,MAAc,SAAiB,OAAkC;AAC9F,MAAI;AACF,UAAM,KAAK,IAAI,OAAO,UAAU,SAAS,KAAK,SAAS,GAAG,IAAI,SAAS,SAAS,MAAM,GAAG;AACzF,UAAM,UAAoB,CAAC;AAC3B,QAAI;AACJ,YAAQ,IAAI,GAAG,KAAK,IAAI,OAAO,MAAM;AACnC,cAAQ,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;AACzB,UAAI,CAAC,GAAG,OAAQ;AAAA,IAClB;AACA,WAAO,EAAE,SAAS,OAAO,QAAQ,QAAQ,QAAQ,QAAQ;AAAA,EAC3D,QAAQ;AACN,WAAO,EAAE,SAAS,CAAC,GAAG,OAAO,GAAG,QAAQ,cAAc;AAAA,EACxD;AACF;AAEO,SAAS,cAAc,MAA4B;AACxD,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,SAAS,MAAM,KAAK,SAAS,iBAAiB,OAAO,CAAC;AAC5D,WAAO,OAAO,IAAI,CAAC,UAAe;AAChC,YAAM,UAAoB,CAAC;AAC3B,YAAM,OAAmB,CAAC;AAC1B,YAAM,MAAM,CAAC,GAAG,MAAM,iBAAiB,uCAAuC,CAAC;AAC/E,UAAI,IAAI,SAAS,GAAG;AAClB,YAAI,QAAQ,CAAC,IAAS,MAAc,QAAQ,KAAK,GAAG,aAAa,KAAK,KAAK,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,MAC3F;AACA,YAAM,MAAM,CAAC,GAAG,MAAM,iBAAiB,cAAc,CAAC;AACtD,iBAAW,MAAM,KAAK;AACpB,cAAM,QAAQ,CAAC,GAAI,GAAW,iBAAiB,QAAQ,CAAC;AACxD,YAAI,MAAM,WAAW,EAAG;AACxB,YAAI,QAAQ,WAAW,GAAG;AACxB,gBAAM,QAAQ,CAAC,GAAQ,MAAc,QAAQ,KAAK,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,QAClE;AACA,cAAM,MAAgB,CAAC;AACvB,cAAM,QAAQ,CAAC,MAAW,MAAc;AACtC,gBAAM,MAAM,QAAQ,CAAC,KAAK,MAAM,IAAI,CAAC;AACrC,cAAI,GAAG,IAAI,KAAK,aAAa,KAAK,KAAK;AAAA,QACzC,CAAC;AACD,YAAI,OAAO,OAAO,GAAG,EAAE,KAAK,OAAK,CAAC,EAAG,MAAK,KAAK,GAAG;AAAA,MACpD;AACA,aAAO;AAAA,IACT,CAAC,EAAE,OAAO,OAAK,EAAE,SAAS,CAAC;AAAA,EAC7B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEO,SAAS,gBAAgB,MAAc,KAA2B;AACvE,QAAM,OAAqB;AAAA,IACzB,OAAO;AAAA,IAAM,aAAa;AAAA,IAAM,UAAU;AAAA,IAC1C,SAAS;AAAA,IAAM,eAAe;AAAA,IAAM,SAAS;AAAA,IAAM,OAAO;AAAA,IAAM,QAAQ;AAAA,IACxE,YAAY;AAAA,IAAM,UAAU;AAAA,IAC5B,aAAa;AAAA,IAAM,cAAc;AAAA,IAAM,oBAAoB;AAAA,IAC3D,cAAc;AAAA,IAAM,aAAa;AAAA,IACjC,WAAW;AAAA,IAAM,QAAQ;AAAA,IACzB,UAAU,CAAC;AAAA,IAAG,YAAY;AAAA,IAAM,SAAS;AAAA,IACzC,QAAQ,CAAC;AAAA,IAAG,aAAa,CAAC;AAAA,IAAG,MAAM;AAAA,EACrC;AACA,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,SAAK,QAAS,SAAiB,SAAU,SAAiB,cAAc,OAAO,GAAG,aAAa,KAAK,KAAK;AACzG,SAAK,OAAQ,SAAiB,iBAAiB,aAAa,MAAM,KAAK;AAEvE,UAAM,QAAQ,CAAC,GAAI,SAAiB,iBAAiB,MAAM,CAAC;AAC5D,eAAW,KAAK,OAAO;AACrB,YAAM,OAAQ,EAAU,aAAa,MAAM,GAAG,YAAY,KAAK;AAC/D,YAAM,OAAQ,EAAU,aAAa,UAAU,GAAG,YAAY,KAAK;AACnE,YAAM,UAAW,EAAU,aAAa,SAAS,KAAK;AACtD,UAAI,SAAS,cAAe,MAAK,cAAc;AAC/C,UAAI,SAAS,WAAY,MAAK,WAAW;AACzC,UAAI,SAAS,SAAU,MAAK,SAAS;AACrC,UAAI,SAAS,eAAgB,MAAK,cAAc;AAChD,UAAI,SAAS,gBAAiB,MAAK,eAAe;AAClD,UAAI,SAAS,sBAAuB,MAAK,qBAAqB;AAC9D,UAAI,SAAS,gBAAiB,MAAK,eAAe;AAClD,UAAI,SAAS,eAAgB,MAAK,cAAc;AAChD,UAAI,SAAS,cAAe,MAAK,aAAa;AAC9C,UAAI,SAAS,WAAY,MAAK,UAAU;AACxC,UAAI,SAAS,iBAAkB,MAAK,gBAAgB;AACpD,UAAI,SAAS,WAAY,MAAK,UAAU;AACxC,UAAI,SAAS,SAAU,MAAK,QAAQ;AACpC,UAAI,SAAS,UAAW,MAAK,SAAS;AACtC,UAAI,SAAS,eAAgB,MAAK,aAAa;AAC/C,UAAI,SAAS,YAAa,MAAK,WAAW;AAAA,IAC5C;AAEA,UAAM,YAAa,SAAiB,cAAc,uBAAuB;AACzE,QAAI,UAAW,MAAK,YAAY,UAAU,aAAa,MAAM,KAAK;AAElE,UAAM,YAAa,SAAiB,cAAc,6CAA6C;AAC/F,QAAI,UAAW,MAAK,UAAU,UAAU,aAAa,MAAM,KAAK;AAEhE,UAAM,cAAc,CAAC,GAAI,SAAiB,iBAAiB,iCAAiC,CAAC;AAC7F,eAAW,MAAM,aAAa;AAC5B,YAAM,OAAQ,GAAW,aAAa,UAAU,KAAK;AACrD,YAAM,OAAQ,GAAW,aAAa,MAAM,KAAK;AACjD,UAAI,QAAQ,KAAM,MAAK,SAAS,KAAK,EAAE,MAAM,KAAK,CAAC;AAAA,IACrD;AAEA,UAAM,gBAAgB,CAAC,GAAI,SAAiB,iBAAiB,oCAAoC,CAAC;AAClG,eAAW,KAAK,eAAe;AAC7B,UAAI;AACF,cAAM,SAAS,KAAK,MAAO,EAAU,eAAe,IAAI;AACxD,aAAK,OAAO,KAAK,MAAM;AACvB,cAAM,QAAQ,MAAM,QAAQ,MAAM,IAAI,OAAO,IAAI,CAAC,MAAW,EAAE,OAAO,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,OAAO,CAAC;AAClG,aAAK,YAAY,KAAK,GAAG,MAAM,OAAO,OAAO,CAAC;AAAA,MAChD,QAAQ;AAAA,MAA6B;AAAA,IACvC;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAgBO,SAAS,yBAAyB,QAAsC;AAC7E,aAAW,QAAQ,QAAQ;AACzB,QAAI;AACF,YAAM,OAAQ,KAAa,OAAO;AAClC,YAAM,YAAY,MAAM,QAAQ,IAAI,IAChC,KAAK,KAAK,CAAC,MAAW,OAAO,MAAM,YAAY,EAAE,YAAY,MAAM,SAAS,IAC5E,OAAO,SAAS,YAAY,KAAK,YAAY,MAAM;AACvD,UAAI,CAAC,UAAW;AAEhB,YAAM,IAAI;AACV,YAAM,SAAS,MAAM,QAAQ,EAAE,MAAM,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;AAEzD,YAAM,WAAW,QAAQ;AACzB,YAAM,QAAQ,YAAY,OAAO,WAAW,QAAQ,IAAI;AAExD,YAAM,YAAY,EAAE,iBAAiB;AACrC,YAAM,SAAS,aAAa,OAAO,WAAW,SAAS,IAAI;AAE3D,YAAM,iBAAiB,EAAE,iBAAiB;AAC1C,YAAM,cAAc,kBAAkB,OAAO,SAAS,gBAAgB,EAAE,IAAI;AAE5E,YAAM,kBAAsC,QAAQ;AACpD,YAAM,eAAe,kBACjB,gBAAgB,QAAQ,6BAA6B,EAAE,IACvD;AAEJ,YAAM,WAAW,EAAE;AACnB,YAAM,QAAQ,MAAM,QAAQ,QAAQ,IAAK,SAAS,CAAC,KAAK,OAAS,YAAY;AAE7E,YAAM,QAAQ,EAAE,OAAO,QAAQ,EAAE,SAAS;AAE1C,aAAO;AAAA,QACL,MAAM,EAAE,QAAQ;AAAA,QAChB,aAAa,EAAE,eAAe;AAAA,QAC9B,OAAO,OAAO,UAAU,WAAW,QAAQ;AAAA,QAC3C,KAAK,EAAE,OAAO;AAAA,QACd,OAAO,MAAM,KAAK,IAAI,OAAO;AAAA,QAC7B,eAAe,QAAQ,iBAAiB;AAAA,QACxC;AAAA,QACA,QAAQ,MAAM,MAAM,IAAI,OAAO;AAAA,QAC/B,aAAa,MAAM,WAAW,IAAI,OAAO;AAAA,QACzC;AAAA,QACA,KAAK,EAAE,OAAO;AAAA,MAChB;AAAA,IACF,QAAQ;AAAA,IAA4B;AAAA,EACtC;AACA,SAAO;AACT;AAEO,SAAS,cAAc,MAAc,SAAuF;AACjI,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,WAAO,MAAM,KAAM,SAAiB,iBAAiB,KAAK,CAAC,EACxD,IAAI,CAAC,QAAa;AACjB,YAAM,MAAM,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,UAAU,KAAK;AACvE,UAAI,CAAC,IAAK,QAAO;AACjB,UAAI;AACF,eAAO;AAAA,UACL,KAAK,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,UACxB,KAAM,IAAI,aAAa,KAAK,KAAK;AAAA,UACjC,OAAO,IAAI,aAAa,OAAO,IAAI,SAAS,IAAI,aAAa,OAAO,CAAC,IAAI;AAAA,UACzE,QAAQ,IAAI,aAAa,QAAQ,IAAI,SAAS,IAAI,aAAa,QAAQ,CAAC,IAAI;AAAA,QAC9E;AAAA,MACF,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IACzB,CAAC,EACA,OAAO,CAAC,MAA0E,MAAM,IAAI;AAAA,EACjG,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEO,SAAS,kBAAkB,MAAc,SAAsG;AACpJ,QAAM,YAAY;AAClB,QAAM,YAAY;AAClB,QAAM,YAAY;AAClB,QAAM,UAAU;AAEhB,WAAS,SAAS,KAAa,KAAuD;AACpF,QAAI,QAAQ,SAAS,UAAU,KAAK,GAAG,EAAG,QAAO;AACjD,QAAI,QAAQ,WAAW,UAAU,KAAK,GAAG,EAAG,QAAO;AACnD,QAAI,QAAQ,WAAW,UAAU,KAAK,GAAG,EAAG,QAAO;AACnD,QAAI,QAAQ,KAAK,GAAG,EAAG,QAAO;AAC9B,WAAO;AAAA,EACT;AAEA,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,UAA+F,CAAC;AACtG,UAAM,OAAO,oBAAI,IAAY;AAE7B,UAAM,YAA8B,CAAC,8CAA8C,KAAK;AACxF,UAAM,YAAqC;AAAA,MACzC,CAAC,OAAO,KAAK;AAAA,MAAG,CAAC,SAAS,KAAK;AAAA,MAAG,CAAC,SAAS,KAAK;AAAA,MAAG,CAAC,UAAU,KAAK;AAAA,MACpE,CAAC,KAAK,MAAM;AAAA,IACd;AAEA,eAAW,CAAC,KAAK,IAAI,KAAK,WAAW;AACnC,iBAAW,MAAM,CAAC,GAAI,SAAiB,iBAAiB,GAAG,CAAC,GAAG;AAC7D,cAAM,MAAO,GAAW,aAAa,IAAI,KAAK;AAC9C,YAAI,CAAC,IAAK;AACV,YAAI;AACF,gBAAM,WAAW,IAAI,IAAI,KAAK,IAAI,EAAE;AACpC,cAAI,KAAK,IAAI,QAAQ,EAAG;AACxB,cAAI,QAAQ,OAAO,CAAC,QAAQ,KAAK,QAAQ,KAAK,CAAC,UAAU,KAAK,QAAQ,KAAK,CAAC,UAAU,KAAK,QAAQ,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAG;AACnI,eAAK,IAAI,QAAQ;AACjB,kBAAQ,KAAK,EAAE,KAAK,UAAU,MAAM,SAAS,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC;AAAA,QACzE,QAAQ;AAAA,QAAa;AAAA,MACvB;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/extractors/artifact-envelope.d.ts

@@ -0,0 +1,63 @@
+import type { RawElement } from '../screen-reader.js';
+export interface CrawlEvidenceFact {
+    type: string;
+    payload: unknown;
+}
+export interface CrawlArtifactEnvelope {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';
+    outputs: {
+        screenshot?: string;
+        html?: string;
+        markdown?: string;
+        fitMarkdown?: string;
+        accessibilityTree?: string;
+        structuredJson?: unknown;
+        networkTrace?: string;
+        consoleLog?: string;
+    };
+    fingerprints: {
+        textHash?: string;
+        domHash?: string;
+        accessibilityHash?: string;
+        urlCanonicalHash: string;
+    };
+    evidence: CrawlEvidenceFact[];
+}
+export declare function sha256(input: string): string;
+export declare function buildFitMarkdown(markdown: string, keywords?: string[]): string;
+export declare function summarizeStructuredElements(elements: RawElement[]): {
+    elementCount: number;
+    formFieldCount: number;
+    actionCount: number;
+    fields: {
+        label: string;
+        role: string;
+        expectedData: string | undefined;
+    }[];
+    actions: {
+        label: string;
+        role: string;
+    }[];
+};
+export declare function buildCrawlArtifactEnvelope(input: {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    screenshotPath?: string;
+    domPath?: string;
+    domContent?: string;
+    markdown?: string;
+    fitMarkdown?: string;
+    accessibilityElements?: RawElement[];
+    structuredJson?: unknown;
+    evidence?: CrawlEvidenceFact[];
+}): CrawlArtifactEnvelope;

package/dist/extractors/artifact-envelope.js

@@ -0,0 +1,63 @@
+import crypto from "node:crypto";
+function sha256(input) {
+  return crypto.createHash("sha256").update(input).digest("hex");
+}
+function buildFitMarkdown(markdown, keywords = []) {
+  const lines = markdown.split("\n").map((line) => line.trimEnd()).filter((line) => line.trim().length > 0);
+  const lowerKeywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+  const filtered = lines.filter((line) => {
+    const plain = line.replace(/[#*_`>\-[\]()]/g, "").trim();
+    if (plain.length < 3) return false;
+    if (/^(cookie|privacy|terms|copyright|all rights reserved|subscribe|newsletter)$/i.test(plain)) return false;
+    if (lowerKeywords.length === 0) return true;
+    const lower = plain.toLowerCase();
+    return lowerKeywords.some((kw) => lower.includes(kw)) || /^#{1,4}\s/.test(line);
+  });
+  const candidate = filtered.join("\n");
+  return candidate.length >= 200 ? candidate : markdown;
+}
+function summarizeStructuredElements(elements) {
+  const forms = elements.filter((e) => ["email", "password", "text", "tel", "date", "number", "select", "textarea"].includes(e.role));
+  const actions = elements.filter((e) => e.role === "button" || e.role === "submit" || e.role === "link" || e.role === "a");
+  return {
+    elementCount: elements.length,
+    formFieldCount: forms.length,
+    actionCount: actions.length,
+    fields: forms.slice(0, 50).map((e) => ({ label: e.label, role: e.role, expectedData: e.expectedData })),
+    actions: actions.slice(0, 50).map((e) => ({ label: e.label, role: e.role }))
+  };
+}
+function buildCrawlArtifactEnvelope(input) {
+  const accessibilityTree = input.accessibilityElements ? JSON.stringify(input.accessibilityElements) : void 0;
+  return {
+    tenantId: input.tenantId,
+    projectId: input.projectId,
+    jobId: input.jobId,
+    url: input.url,
+    canonicalUrl: input.canonicalUrl,
+    screenId: input.screenId,
+    status: "ok",
+    outputs: {
+      screenshot: input.screenshotPath,
+      html: input.domPath,
+      markdown: input.markdown,
+      fitMarkdown: input.fitMarkdown,
+      accessibilityTree,
+      structuredJson: input.structuredJson
+    },
+    fingerprints: {
+      textHash: input.markdown ? sha256(input.markdown) : void 0,
+      domHash: input.domContent ? sha256(input.domContent) : void 0,
+      accessibilityHash: accessibilityTree ? sha256(accessibilityTree) : void 0,
+      urlCanonicalHash: sha256(input.canonicalUrl)
+    },
+    evidence: input.evidence ?? []
+  };
+}
+export {
+  buildCrawlArtifactEnvelope,
+  buildFitMarkdown,
+  sha256,
+  summarizeStructuredElements
+};
+//# sourceMappingURL=artifact-envelope.js.map

package/dist/extractors/artifact-envelope.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/artifact-envelope.ts"],
+  "sourcesContent": ["import crypto from 'node:crypto';\nimport type { RawElement } from '../screen-reader.js';\n\nexport interface CrawlEvidenceFact {\n  type: string;\n  payload: unknown;\n}\n\nexport interface CrawlArtifactEnvelope {\n  tenantId: string;\n  projectId: string;\n  jobId?: string;\n  url: string;\n  canonicalUrl: string;\n  screenId?: string;\n  status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';\n  outputs: {\n    screenshot?: string;\n    html?: string;\n    markdown?: string;\n    fitMarkdown?: string;\n    accessibilityTree?: string;\n    structuredJson?: unknown;\n    networkTrace?: string;\n    consoleLog?: string;\n  };\n  fingerprints: {\n    textHash?: string;\n    domHash?: string;\n    accessibilityHash?: string;\n    urlCanonicalHash: string;\n  };\n  evidence: CrawlEvidenceFact[];\n}\n\nexport function sha256(input: string): string {\n  return crypto.createHash('sha256').update(input).digest('hex');\n}\n\nexport function buildFitMarkdown(markdown: string, keywords: string[] = []): string {\n  const lines = markdown\n    .split('\\n')\n    .map((line) => line.trimEnd())\n    .filter((line) => line.trim().length > 0);\n  const lowerKeywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n  const filtered = lines.filter((line) => {\n    const plain = line.replace(/[#*_`>\\-[\\]()]/g, '').trim();\n    if (plain.length < 3) return false;\n    if (/^(cookie|privacy|terms|copyright|all rights reserved|subscribe|newsletter)$/i.test(plain)) return false;\n    if (lowerKeywords.length === 0) return true;\n    const lower = plain.toLowerCase();\n    return lowerKeywords.some((kw) => lower.includes(kw)) || /^#{1,4}\\s/.test(line);\n  });\n  const candidate = filtered.join('\\n');\n  return candidate.length >= 200 ? candidate : markdown;\n}\n\nexport function summarizeStructuredElements(elements: RawElement[]) {\n  const forms = elements.filter((e) => ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role));\n  const actions = elements.filter((e) => e.role === 'button' || e.role === 'submit' || e.role === 'link' || e.role === 'a');\n  return {\n    elementCount: elements.length,\n    formFieldCount: forms.length,\n    actionCount: actions.length,\n    fields: forms.slice(0, 50).map((e) => ({ label: e.label, role: e.role, expectedData: e.expectedData })),\n    actions: actions.slice(0, 50).map((e) => ({ label: e.label, role: e.role })),\n  };\n}\n\nexport function buildCrawlArtifactEnvelope(input: {\n  tenantId: string;\n  projectId: string;\n  jobId?: string;\n  url: string;\n  canonicalUrl: string;\n  screenId?: string;\n  screenshotPath?: string;\n  domPath?: string;\n  domContent?: string;\n  markdown?: string;\n  fitMarkdown?: string;\n  accessibilityElements?: RawElement[];\n  structuredJson?: unknown;\n  evidence?: CrawlEvidenceFact[];\n}): CrawlArtifactEnvelope {\n  const accessibilityTree = input.accessibilityElements ? JSON.stringify(input.accessibilityElements) : undefined;\n  return {\n    tenantId: input.tenantId,\n    projectId: input.projectId,\n    jobId: input.jobId,\n    url: input.url,\n    canonicalUrl: input.canonicalUrl,\n    screenId: input.screenId,\n    status: 'ok',\n    outputs: {\n      screenshot: input.screenshotPath,\n      html: input.domPath,\n      markdown: input.markdown,\n      fitMarkdown: input.fitMarkdown,\n      accessibilityTree,\n      structuredJson: input.structuredJson,\n    },\n    fingerprints: {\n      textHash: input.markdown ? sha256(input.markdown) : undefined,\n      domHash: input.domContent ? sha256(input.domContent) : undefined,\n      accessibilityHash: accessibilityTree ? sha256(accessibilityTree) : undefined,\n      urlCanonicalHash: sha256(input.canonicalUrl),\n    },\n    evidence: input.evidence ?? [],\n  };\n}\n"],
+  "mappings": "AAAA,OAAO,YAAY;AAmCZ,SAAS,OAAO,OAAuB;AAC5C,SAAO,OAAO,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AAC/D;AAEO,SAAS,iBAAiB,UAAkB,WAAqB,CAAC,GAAW;AAClF,QAAM,QAAQ,SACX,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,EAC5B,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,SAAS,CAAC;AAC1C,QAAM,gBAAgB,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACzE,QAAM,WAAW,MAAM,OAAO,CAAC,SAAS;AACtC,UAAM,QAAQ,KAAK,QAAQ,mBAAmB,EAAE,EAAE,KAAK;AACvD,QAAI,MAAM,SAAS,EAAG,QAAO;AAC7B,QAAI,+EAA+E,KAAK,KAAK,EAAG,QAAO;AACvG,QAAI,cAAc,WAAW,EAAG,QAAO;AACvC,UAAM,QAAQ,MAAM,YAAY;AAChC,WAAO,cAAc,KAAK,CAAC,OAAO,MAAM,SAAS,EAAE,CAAC,KAAK,YAAY,KAAK,IAAI;AAAA,EAChF,CAAC;AACD,QAAM,YAAY,SAAS,KAAK,IAAI;AACpC,SAAO,UAAU,UAAU,MAAM,YAAY;AAC/C;AAEO,SAAS,4BAA4B,UAAwB;AAClE,QAAM,QAAQ,SAAS,OAAO,CAAC,MAAM,CAAC,SAAS,YAAY,QAAQ,OAAO,QAAQ,UAAU,UAAU,UAAU,EAAE,SAAS,EAAE,IAAI,CAAC;AAClI,QAAM,UAAU,SAAS,OAAO,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,SAAS,YAAY,EAAE,SAAS,UAAU,EAAE,SAAS,GAAG;AACxH,SAAO;AAAA,IACL,cAAc,SAAS;AAAA,IACvB,gBAAgB,MAAM;AAAA,IACtB,aAAa,QAAQ;AAAA,IACrB,QAAQ,MAAM,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,MAAM,EAAE,MAAM,cAAc,EAAE,aAAa,EAAE;AAAA,IACtG,SAAS,QAAQ,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,MAAM,EAAE,KAAK,EAAE;AAAA,EAC7E;AACF;AAEO,SAAS,2BAA2B,OAejB;AACxB,QAAM,oBAAoB,MAAM,wBAAwB,KAAK,UAAU,MAAM,qBAAqB,IAAI;AACtG,SAAO;AAAA,IACL,UAAU,MAAM;AAAA,IAChB,WAAW,MAAM;AAAA,IACjB,OAAO,MAAM;AAAA,IACb,KAAK,MAAM;AAAA,IACX,cAAc,MAAM;AAAA,IACpB,UAAU,MAAM;AAAA,IAChB,QAAQ;AAAA,IACR,SAAS;AAAA,MACP,YAAY,MAAM;AAAA,MAClB,MAAM,MAAM;AAAA,MACZ,UAAU,MAAM;AAAA,MAChB,aAAa,MAAM;AAAA,MACnB;AAAA,MACA,gBAAgB,MAAM;AAAA,IACxB;AAAA,IACA,cAAc;AAAA,MACZ,UAAU,MAAM,WAAW,OAAO,MAAM,QAAQ,IAAI;AAAA,MACpD,SAAS,MAAM,aAAa,OAAO,MAAM,UAAU,IAAI;AAAA,MACvD,mBAAmB,oBAAoB,OAAO,iBAAiB,IAAI;AAAA,MACnE,kBAAkB,OAAO,MAAM,YAAY;AAAA,IAC7C;AAAA,IACA,UAAU,MAAM,YAAY,CAAC;AAAA,EAC/B;AACF;",
+  "names": []
+}

package/dist/extractors/branding-extractor.d.ts

@@ -0,0 +1,46 @@
+import type { Page } from 'playwright';
+export interface BrandingColors {
+    primary?: string;
+    accent?: string;
+    background?: string;
+    textPrimary?: string;
+    link?: string;
+}
+export interface BrandingFont {
+    family: string;
+    role: 'heading' | 'body';
+}
+export interface BrandingImages {
+    logo?: string;
+    favicon?: string;
+    ogImage?: string;
+}
+export interface BrandingPersonality {
+    tone?: string;
+    energy?: 'high' | 'medium' | 'low';
+    targetAudience?: string;
+}
+export interface BrandingResult {
+    colorScheme: 'light' | 'dark';
+    colors: BrandingColors;
+    fonts: BrandingFont[];
+    images: BrandingImages;
+    personality?: BrandingPersonality;
+    designSystem?: {
+        framework?: string;
+        componentLibrary?: string;
+    };
+    metadata?: {
+        title?: string;
+        description?: string;
+        themeColor?: string;
+        language?: string;
+    };
+    confidence: {
+        colors: number;
+        fonts: number;
+        images: number;
+        overall: number;
+    };
+}
+export declare function extractBranding(page: Page): Promise<BrandingResult>;

package/dist/extractors/branding-extractor.js

@@ -0,0 +1,104 @@
+async function extractBranding(page) {
+  try {
+    const result = await page.evaluate(() => {
+      const bodyStyle = window.getComputedStyle(document.body);
+      const isDark = window.matchMedia("(prefers-color-scheme: dark)").matches || document.documentElement.getAttribute("data-theme") === "dark" || document.body.classList.contains("dark");
+      const bg = bodyStyle.backgroundColor;
+      const text = bodyStyle.color;
+      const primaryBtn = document.querySelector(
+        'button[class*="primary"], .btn-primary, [class*="btn-primary"], [class*="button-primary"]'
+      );
+      const primaryColor = primaryBtn ? window.getComputedStyle(primaryBtn).backgroundColor : void 0;
+      const firstLink = document.querySelector("a");
+      const linkColor = firstLink ? window.getComputedStyle(firstLink).color : void 0;
+      const headingEl = document.querySelector("h1, h2");
+      const bodyEl = document.body;
+      const headingFont = headingEl ? window.getComputedStyle(headingEl).fontFamily : bodyStyle.fontFamily;
+      const bodyFont = bodyStyle.fontFamily;
+      const logoEl = document.querySelector(
+        'header img, img[alt*="logo" i], img[class*="logo" i], [class*="logo"] img, [id*="logo"] img'
+      );
+      const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+      const ogImageEl = document.querySelector('meta[property="og:image"]');
+      const titleEl = document.querySelector("title");
+      const descEl = document.querySelector('meta[name="description"]');
+      const themeColorEl = document.querySelector('meta[name="theme-color"]');
+      const hasBootstrap = !!document.querySelector('[class*="btn btn-"], .container-fluid, .navbar-expand');
+      const hasTailwind = !!document.querySelector('[class*="flex "], [class*="text-"], [class*="bg-"]');
+      const hasChakra = !!document.__chakra_factory__ || !!document.querySelector('[class*="chakra-"]');
+      const hasMUI = !!document.querySelector(".MuiButton-root, .MuiTypography-root");
+      const framework = hasBootstrap ? "Bootstrap" : hasTailwind ? "Tailwind CSS" : hasChakra ? "Chakra UI" : hasMUI ? "Material UI" : null;
+      return {
+        isDark,
+        bg,
+        text,
+        primaryColor,
+        linkColor,
+        headingFont,
+        bodyFont,
+        logo: logoEl?.src || logoEl?.getAttribute("src") || void 0,
+        favicon: faviconEl?.href || void 0,
+        ogImage: ogImageEl?.content || void 0,
+        title: titleEl?.textContent || void 0,
+        description: descEl?.content || void 0,
+        themeColor: themeColorEl?.content || void 0,
+        language: document.documentElement.lang || void 0,
+        framework
+      };
+    });
+    const fonts = [];
+    if (result.headingFont && result.headingFont !== result.bodyFont) {
+      fonts.push({ family: result.headingFont.split(",")[0].replace(/['"]/g, "").trim(), role: "heading" });
+    }
+    if (result.bodyFont) {
+      fonts.push({ family: result.bodyFont.split(",")[0].replace(/['"]/g, "").trim(), role: "body" });
+    }
+    const colorConfidence = result.bg || result.text || result.primaryColor ? 0.8 : 0.3;
+    const fontConfidence = fonts.length > 0 ? 0.7 : 0.2;
+    const imageConfidence = result.logo || result.ogImage ? 0.9 : 0.2;
+    return {
+      colorScheme: result.isDark ? "dark" : "light",
+      colors: {
+        background: result.bg || void 0,
+        textPrimary: result.text || void 0,
+        primary: result.primaryColor || void 0,
+        link: result.linkColor || void 0
+      },
+      fonts,
+      images: {
+        logo: result.logo,
+        favicon: result.favicon,
+        ogImage: result.ogImage
+      },
+      designSystem: {
+        framework: result.framework || void 0,
+        componentLibrary: void 0
+      },
+      metadata: {
+        title: result.title,
+        description: result.description,
+        themeColor: result.themeColor,
+        language: result.language
+      },
+      confidence: {
+        colors: colorConfidence,
+        fonts: fontConfidence,
+        images: imageConfidence,
+        overall: (colorConfidence + fontConfidence + imageConfidence) / 3
+      }
+    };
+  } catch (err) {
+    console.warn("[branding-extractor] Extraction failed:", err);
+    return {
+      colorScheme: "light",
+      colors: {},
+      fonts: [],
+      images: {},
+      confidence: { colors: 0, fonts: 0, images: 0, overall: 0 }
+    };
+  }
+}
+export {
+  extractBranding
+};
+//# sourceMappingURL=branding-extractor.js.map

package/dist/extractors/branding-extractor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/branding-extractor.ts"],
+  "sourcesContent": ["import type { Page } from 'playwright';\n\nexport interface BrandingColors {\n  primary?: string;\n  accent?: string;\n  background?: string;\n  textPrimary?: string;\n  link?: string;\n}\n\nexport interface BrandingFont {\n  family: string;\n  role: 'heading' | 'body';\n}\n\nexport interface BrandingImages {\n  logo?: string;\n  favicon?: string;\n  ogImage?: string;\n}\n\nexport interface BrandingPersonality {\n  tone?: string;\n  energy?: 'high' | 'medium' | 'low';\n  targetAudience?: string;\n}\n\nexport interface BrandingResult {\n  colorScheme: 'light' | 'dark';\n  colors: BrandingColors;\n  fonts: BrandingFont[];\n  images: BrandingImages;\n  personality?: BrandingPersonality;\n  designSystem?: { framework?: string; componentLibrary?: string };\n  metadata?: { title?: string; description?: string; themeColor?: string; language?: string };\n  confidence: { colors: number; fonts: number; images: number; overall: number };\n}\n\nexport async function extractBranding(page: Page): Promise<BrandingResult> {\n  try {\n    const result = await page.evaluate(() => {\n      const bodyStyle = window.getComputedStyle(document.body);\n      const isDark = window.matchMedia('(prefers-color-scheme: dark)').matches ||\n        document.documentElement.getAttribute('data-theme') === 'dark' ||\n        document.body.classList.contains('dark');\n\n      // Extract colors from computed styles\n      const bg = bodyStyle.backgroundColor;\n      const text = bodyStyle.color;\n\n      // Find primary button color\n      const primaryBtn = document.querySelector(\n        'button[class*=\"primary\"], .btn-primary, [class*=\"btn-primary\"], [class*=\"button-primary\"]'\n      );\n      const primaryColor = primaryBtn ? window.getComputedStyle(primaryBtn).backgroundColor : undefined;\n\n      // Find link color\n      const firstLink = document.querySelector('a');\n      const linkColor = firstLink ? window.getComputedStyle(firstLink).color : undefined;\n\n      // Extract font families\n      const headingEl = document.querySelector('h1, h2');\n      const bodyEl = document.body;\n      const headingFont = headingEl ? window.getComputedStyle(headingEl).fontFamily : bodyStyle.fontFamily;\n      const bodyFont = bodyStyle.fontFamily;\n\n      // Extract images\n      const logoEl = document.querySelector(\n        'header img, img[alt*=\"logo\" i], img[class*=\"logo\" i], [class*=\"logo\"] img, [id*=\"logo\"] img'\n      ) as HTMLImageElement | null;\n      const faviconEl = document.querySelector('link[rel=\"icon\"], link[rel=\"shortcut icon\"]') as HTMLLinkElement | null;\n      const ogImageEl = document.querySelector('meta[property=\"og:image\"]') as HTMLMetaElement | null;\n\n      // Extract metadata\n      const titleEl = document.querySelector('title');\n      const descEl = document.querySelector('meta[name=\"description\"]') as HTMLMetaElement | null;\n      const themeColorEl = document.querySelector('meta[name=\"theme-color\"]') as HTMLMetaElement | null;\n\n      // Detect design framework\n      const hasBootstrap = !!document.querySelector('[class*=\"btn btn-\"], .container-fluid, .navbar-expand');\n      const hasTailwind = !!document.querySelector('[class*=\"flex \"], [class*=\"text-\"], [class*=\"bg-\"]');\n      const hasChakra = !!(document as any).__chakra_factory__ || !!document.querySelector('[class*=\"chakra-\"]');\n      const hasMUI = !!document.querySelector('.MuiButton-root, .MuiTypography-root');\n\n      const framework = hasBootstrap ? 'Bootstrap' : hasTailwind ? 'Tailwind CSS' : hasChakra ? 'Chakra UI' : hasMUI ? 'Material UI' : null;\n\n      return {\n        isDark,\n        bg,\n        text,\n        primaryColor,\n        linkColor,\n        headingFont,\n        bodyFont,\n        logo: logoEl?.src || logoEl?.getAttribute('src') || undefined,\n        favicon: faviconEl?.href || undefined,\n        ogImage: ogImageEl?.content || undefined,\n        title: titleEl?.textContent || undefined,\n        description: descEl?.content || undefined,\n        themeColor: themeColorEl?.content || undefined,\n        language: document.documentElement.lang || undefined,\n        framework,\n      };\n    });\n\n    const fonts: BrandingFont[] = [];\n    if (result.headingFont && result.headingFont !== result.bodyFont) {\n      fonts.push({ family: result.headingFont.split(',')[0].replace(/['\"]/g, '').trim(), role: 'heading' });\n    }\n    if (result.bodyFont) {\n      fonts.push({ family: result.bodyFont.split(',')[0].replace(/['\"]/g, '').trim(), role: 'body' });\n    }\n\n    const colorConfidence = (result.bg || result.text || result.primaryColor) ? 0.8 : 0.3;\n    const fontConfidence = fonts.length > 0 ? 0.7 : 0.2;\n    const imageConfidence = (result.logo || result.ogImage) ? 0.9 : 0.2;\n\n    return {\n      colorScheme: result.isDark ? 'dark' : 'light',\n      colors: {\n        background: result.bg || undefined,\n        textPrimary: result.text || undefined,\n        primary: result.primaryColor || undefined,\n        link: result.linkColor || undefined,\n      },\n      fonts,\n      images: {\n        logo: result.logo,\n        favicon: result.favicon,\n        ogImage: result.ogImage,\n      },\n      designSystem: {\n        framework: result.framework || undefined,\n        componentLibrary: undefined,\n      },\n      metadata: {\n        title: result.title,\n        description: result.description,\n        themeColor: result.themeColor,\n        language: result.language,\n      },\n      confidence: {\n        colors: colorConfidence,\n        fonts: fontConfidence,\n        images: imageConfidence,\n        overall: (colorConfidence + fontConfidence + imageConfidence) / 3,\n      },\n    };\n  } catch (err) {\n    console.warn('[branding-extractor] Extraction failed:', err);\n    return {\n      colorScheme: 'light',\n      colors: {},\n      fonts: [],\n      images: {},\n      confidence: { colors: 0, fonts: 0, images: 0, overall: 0 },\n    };\n  }\n}\n"],
+  "mappings": "AAsCA,eAAsB,gBAAgB,MAAqC;AACzE,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,MAAM;AACvC,YAAM,YAAY,OAAO,iBAAiB,SAAS,IAAI;AACvD,YAAM,SAAS,OAAO,WAAW,8BAA8B,EAAE,WAC/D,SAAS,gBAAgB,aAAa,YAAY,MAAM,UACxD,SAAS,KAAK,UAAU,SAAS,MAAM;AAGzC,YAAM,KAAK,UAAU;AACrB,YAAM,OAAO,UAAU;AAGvB,YAAM,aAAa,SAAS;AAAA,QAC1B;AAAA,MACF;AACA,YAAM,eAAe,aAAa,OAAO,iBAAiB,UAAU,EAAE,kBAAkB;AAGxF,YAAM,YAAY,SAAS,cAAc,GAAG;AAC5C,YAAM,YAAY,YAAY,OAAO,iBAAiB,SAAS,EAAE,QAAQ;AAGzE,YAAM,YAAY,SAAS,cAAc,QAAQ;AACjD,YAAM,SAAS,SAAS;AACxB,YAAM,cAAc,YAAY,OAAO,iBAAiB,SAAS,EAAE,aAAa,UAAU;AAC1F,YAAM,WAAW,UAAU;AAG3B,YAAM,SAAS,SAAS;AAAA,QACtB;AAAA,MACF;AACA,YAAM,YAAY,SAAS,cAAc,6CAA6C;AACtF,YAAM,YAAY,SAAS,cAAc,2BAA2B;AAGpE,YAAM,UAAU,SAAS,cAAc,OAAO;AAC9C,YAAM,SAAS,SAAS,cAAc,0BAA0B;AAChE,YAAM,eAAe,SAAS,cAAc,0BAA0B;AAGtE,YAAM,eAAe,CAAC,CAAC,SAAS,cAAc,uDAAuD;AACrG,YAAM,cAAc,CAAC,CAAC,SAAS,cAAc,oDAAoD;AACjG,YAAM,YAAY,CAAC,CAAE,SAAiB,sBAAsB,CAAC,CAAC,SAAS,cAAc,oBAAoB;AACzG,YAAM,SAAS,CAAC,CAAC,SAAS,cAAc,sCAAsC;AAE9E,YAAM,YAAY,eAAe,cAAc,cAAc,iBAAiB,YAAY,cAAc,SAAS,gBAAgB;AAEjI,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM,QAAQ,OAAO,QAAQ,aAAa,KAAK,KAAK;AAAA,QACpD,SAAS,WAAW,QAAQ;AAAA,QAC5B,SAAS,WAAW,WAAW;AAAA,QAC/B,OAAO,SAAS,eAAe;AAAA,QAC/B,aAAa,QAAQ,WAAW;AAAA,QAChC,YAAY,cAAc,WAAW;AAAA,QACrC,UAAU,SAAS,gBAAgB,QAAQ;AAAA,QAC3C;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,QAAwB,CAAC;AAC/B,QAAI,OAAO,eAAe,OAAO,gBAAgB,OAAO,UAAU;AAChE,YAAM,KAAK,EAAE,QAAQ,OAAO,YAAY,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,SAAS,EAAE,EAAE,KAAK,GAAG,MAAM,UAAU,CAAC;AAAA,IACtG;AACA,QAAI,OAAO,UAAU;AACnB,YAAM,KAAK,EAAE,QAAQ,OAAO,SAAS,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,SAAS,EAAE,EAAE,KAAK,GAAG,MAAM,OAAO,CAAC;AAAA,IAChG;AAEA,UAAM,kBAAmB,OAAO,MAAM,OAAO,QAAQ,OAAO,eAAgB,MAAM;AAClF,UAAM,iBAAiB,MAAM,SAAS,IAAI,MAAM;AAChD,UAAM,kBAAmB,OAAO,QAAQ,OAAO,UAAW,MAAM;AAEhE,WAAO;AAAA,MACL,aAAa,OAAO,SAAS,SAAS;AAAA,MACtC,QAAQ;AAAA,QACN,YAAY,OAAO,MAAM;AAAA,QACzB,aAAa,OAAO,QAAQ;AAAA,QAC5B,SAAS,OAAO,gBAAgB;AAAA,QAChC,MAAM,OAAO,aAAa;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,QACN,MAAM,OAAO;AAAA,QACb,SAAS,OAAO;AAAA,QAChB,SAAS,OAAO;AAAA,MAClB;AAAA,MACA,cAAc;AAAA,QACZ,WAAW,OAAO,aAAa;AAAA,QAC/B,kBAAkB;AAAA,MACpB;AAAA,MACA,UAAU;AAAA,QACR,OAAO,OAAO;AAAA,QACd,aAAa,OAAO;AAAA,QACpB,YAAY,OAAO;AAAA,QACnB,UAAU,OAAO;AAAA,MACnB;AAAA,MACA,YAAY;AAAA,QACV,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,UAAU,kBAAkB,iBAAiB,mBAAmB;AAAA,MAClE;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,KAAK,2CAA2C,GAAG;AAC3D,WAAO;AAAA,MACL,aAAa;AAAA,MACb,QAAQ,CAAC;AAAA,MACT,OAAO,CAAC;AAAA,MACR,QAAQ,CAAC;AAAA,MACT,YAAY,EAAE,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,EAAE;AAAA,IAC3D;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/extractors/summary-extractor.d.ts

@@ -0,0 +1,10 @@
+export interface SummaryResult {
+    summary: string;
+    wordCount: number;
+    method: 'llm' | 'truncated';
+    model?: string;
+}
+export declare function extractSummary(markdown: string, options?: {
+    model?: string;
+    maxTokens?: number;
+}): Promise<SummaryResult>;