@detiq/crawler 0.1.3

package/dist/extractors/advanced.js

@@ -0,0 +1,245 @@
+import { parseHTML } from "linkedom";
+function extractByCssSelector(html, selector, attr) {
+  try {
+    const { document } = parseHTML(html);
+    const nodes = Array.from(document.querySelectorAll(selector));
+    const matches = nodes.map((n) => {
+      if (attr) return n.getAttribute(attr) ?? "";
+      return n.textContent?.trim() ?? "";
+    }).filter(Boolean);
+    return { matches, count: matches.length, method: "css-selector" };
+  } catch (e) {
+    return { matches: [], count: 0, method: "css-selector-error" };
+  }
+}
+function extractByXPath(html, xpath) {
+  try {
+    const { document, XPathResult } = parseHTML(html);
+    const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);
+    const matches = [];
+    let node = result.iterateNext?.();
+    while (node) {
+      const text = node.textContent?.trim() ?? node.nodeValue?.trim() ?? "";
+      if (text) matches.push(text);
+      node = result.iterateNext?.();
+    }
+    return { matches, count: matches.length, method: "xpath" };
+  } catch (e) {
+    return { matches: [], count: 0, method: "xpath-error" };
+  }
+}
+function extractByRegex(text, pattern, flags) {
+  try {
+    const re = new RegExp(pattern, (flags ?? "g").includes("g") ? flags : (flags ?? "") + "g");
+    const matches = [];
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      matches.push(m[1] ?? m[0]);
+      if (!re.global) break;
+    }
+    return { matches, count: matches.length, method: "regex" };
+  } catch {
+    return { matches: [], count: 0, method: "regex-error" };
+  }
+}
+function extractTables(html) {
+  try {
+    const { document } = parseHTML(html);
+    const tables = Array.from(document.querySelectorAll("table"));
+    return tables.map((table) => {
+      const headers = [];
+      const rows = [];
+      const ths = [...table.querySelectorAll("thead th, thead td, tr:first-child th")];
+      if (ths.length > 0) {
+        ths.forEach((th, i) => headers.push(th.textContent?.trim() || `col${i + 1}`));
+      }
+      const trs = [...table.querySelectorAll("tbody tr, tr")];
+      for (const tr of trs) {
+        const cells = [...tr.querySelectorAll("td, th")];
+        if (cells.length === 0) continue;
+        if (headers.length === 0) {
+          cells.forEach((_, i) => headers.push(`col${i + 1}`));
+        }
+        const row = {};
+        cells.forEach((cell, i) => {
+          const key = headers[i] || `col${i + 1}`;
+          row[key] = cell.textContent?.trim() ?? "";
+        });
+        if (Object.values(row).some((v) => v)) rows.push(row);
+      }
+      return rows;
+    }).filter((t) => t.length > 0);
+  } catch {
+    return [];
+  }
+}
+function extractMetadata(html, url) {
+  const meta = {
+    title: null,
+    description: null,
+    keywords: null,
+    ogTitle: null,
+    ogDescription: null,
+    ogImage: null,
+    ogUrl: null,
+    ogType: null,
+    twitterCard: null,
+    twitterTitle: null,
+    twitterDescription: null,
+    canonical: null,
+    robots: null,
+    jsonLd: [],
+    schemaTypes: [],
+    lang: null
+  };
+  try {
+    const { document } = parseHTML(html);
+    meta.title = document.title ?? document.querySelector("title")?.textContent?.trim() ?? null;
+    meta.lang = document.documentElement?.getAttribute("lang") ?? null;
+    const metas = [...document.querySelectorAll("meta")];
+    for (const m of metas) {
+      const name = m.getAttribute("name")?.toLowerCase() ?? "";
+      const prop = m.getAttribute("property")?.toLowerCase() ?? "";
+      const content = m.getAttribute("content") ?? "";
+      if (name === "description") meta.description = content;
+      if (name === "keywords") meta.keywords = content;
+      if (name === "robots") meta.robots = content;
+      if (name === "twitter:card") meta.twitterCard = content;
+      if (name === "twitter:title") meta.twitterTitle = content;
+      if (name === "twitter:description") meta.twitterDescription = content;
+      if (prop === "og:title") meta.ogTitle = content;
+      if (prop === "og:description") meta.ogDescription = content;
+      if (prop === "og:image") meta.ogImage = content;
+      if (prop === "og:url") meta.ogUrl = content;
+      if (prop === "og:type") meta.ogType = content;
+    }
+    const canonical = document.querySelector('link[rel="canonical"]');
+    if (canonical) meta.canonical = canonical.getAttribute("href") ?? null;
+    const jsonLdScripts = [...document.querySelectorAll('script[type="application/ld+json"]')];
+    for (const s of jsonLdScripts) {
+      try {
+        const parsed = JSON.parse(s.textContent ?? "{}");
+        meta.jsonLd.push(parsed);
+        const types = Array.isArray(parsed) ? parsed.map((p) => p["@type"]).flat() : [parsed["@type"]];
+        meta.schemaTypes.push(...types.filter(Boolean));
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return meta;
+}
+function extractStructuredProduct(jsonLd) {
+  for (const item of jsonLd) {
+    try {
+      const type = item["@type"];
+      const isProduct = Array.isArray(type) ? type.some((t) => typeof t === "string" && t.toLowerCase() === "product") : typeof type === "string" && type.toLowerCase() === "product";
+      if (!isProduct) continue;
+      const i = item;
+      const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;
+      const rawPrice = offers?.price;
+      const price = rawPrice != null ? parseFloat(rawPrice) : NaN;
+      const rawRating = i.aggregateRating?.ratingValue;
+      const rating = rawRating != null ? parseFloat(rawRating) : NaN;
+      const rawReviewCount = i.aggregateRating?.reviewCount;
+      const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;
+      const rawAvailability = offers?.availability;
+      const availability = rawAvailability ? rawAvailability.replace(/^https?:\/\/schema\.org\//, "") : null;
+      const rawImage = i.image;
+      const image = Array.isArray(rawImage) ? rawImage[0] ?? null : rawImage ?? null;
+      const brand = i.brand?.name ?? i.brand ?? null;
+      return {
+        name: i.name ?? null,
+        description: i.description ?? null,
+        brand: typeof brand === "string" ? brand : null,
+        sku: i.sku ?? null,
+        price: isNaN(price) ? null : price,
+        priceCurrency: offers?.priceCurrency ?? null,
+        availability,
+        rating: isNaN(rating) ? null : rating,
+        reviewCount: isNaN(reviewCount) ? null : reviewCount,
+        image,
+        url: i.url ?? null
+      };
+    } catch {
+    }
+  }
+  return null;
+}
+function extractImages(html, baseUrl) {
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    return Array.from(document.querySelectorAll("img")).map((img) => {
+      const src = img.getAttribute("src") ?? img.getAttribute("data-src") ?? "";
+      if (!src) return null;
+      try {
+        return {
+          src: new URL(src, base).href,
+          alt: img.getAttribute("alt") ?? "",
+          width: img.getAttribute("width") ? parseInt(img.getAttribute("width")) : void 0,
+          height: img.getAttribute("height") ? parseInt(img.getAttribute("height")) : void 0
+        };
+      } catch {
+        return null;
+      }
+    }).filter((x) => x !== null);
+  } catch {
+    return [];
+  }
+}
+function extractMediaFiles(html, baseUrl) {
+  const IMAGE_EXT = /\.(png|jpe?g|gif|webp|svg|ico|avif)(\?.*)?$/i;
+  const VIDEO_EXT = /\.(mp4|webm|ogv|mov|avi)(\?.*)?$/i;
+  const AUDIO_EXT = /\.(mp3|ogg|wav|flac|aac)(\?.*)?$/i;
+  const DOC_EXT = /\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\?.*)?$/i;
+  function classify(url, tag) {
+    if (tag === "img" || IMAGE_EXT.test(url)) return "image";
+    if (tag === "video" || VIDEO_EXT.test(url)) return "video";
+    if (tag === "audio" || AUDIO_EXT.test(url)) return "audio";
+    if (DOC_EXT.test(url)) return "document";
+    return "document";
+  }
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    const results = [];
+    const seen = /* @__PURE__ */ new Set();
+    const selectors = ["img[src],video[src],audio[src],source[src]", "src"];
+    const attrPairs = [
+      ["img", "src"],
+      ["video", "src"],
+      ["audio", "src"],
+      ["source", "src"],
+      ["a", "href"]
+    ];
+    for (const [sel, attr] of attrPairs) {
+      for (const el of [...document.querySelectorAll(sel)]) {
+        const raw = el.getAttribute(attr) ?? "";
+        if (!raw) continue;
+        try {
+          const resolved = new URL(raw, base).href;
+          if (seen.has(resolved)) continue;
+          if (sel === "a" && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved)) continue;
+          seen.add(resolved);
+          results.push({ url: resolved, type: classify(resolved, sel), tag: sel });
+        } catch {
+        }
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
+export {
+  extractByCssSelector,
+  extractByRegex,
+  extractByXPath,
+  extractImages,
+  extractMediaFiles,
+  extractMetadata,
+  extractStructuredProduct,
+  extractTables
+};
+//# sourceMappingURL=advanced.js.map

package/dist/extractors/advanced.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/advanced.ts"],
+  "sourcesContent": ["import { parseHTML } from 'linkedom';\n\nexport interface ExtractionResult {\n  matches: string[];\n  count: number;\n  method: string;\n}\n\nexport type TableRow = Record<string, string>;\n\nexport interface PageMetadata {\n  title: string | null;\n  description: string | null;\n  keywords: string | null;\n  ogTitle: string | null;\n  ogDescription: string | null;\n  ogImage: string | null;\n  ogUrl: string | null;\n  ogType: string | null;\n  twitterCard: string | null;\n  twitterTitle: string | null;\n  twitterDescription: string | null;\n  canonical: string | null;\n  robots: string | null;\n  jsonLd: object[];\n  schemaTypes: string[];\n  lang: string | null;\n}\n\nexport function extractByCssSelector(html: string, selector: string, attr?: string): ExtractionResult {\n  try {\n    const { document } = parseHTML(html);\n    const nodes = Array.from(document.querySelectorAll(selector));\n    const matches = nodes.map((n: any) => {\n      if (attr) return (n as any).getAttribute(attr) ?? '';\n      return (n as any).textContent?.trim() ?? '';\n    }).filter(Boolean);\n    return { matches, count: matches.length, method: 'css-selector' };\n  } catch (e) {\n    return { matches: [], count: 0, method: 'css-selector-error' };\n  }\n}\n\nexport function extractByXPath(html: string, xpath: string): ExtractionResult {\n  try {\n    const { document, XPathResult } = parseHTML(html) as any;\n    const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);\n    const matches: string[] = [];\n    let node = result.iterateNext?.();\n    while (node) {\n      const text = (node as any).textContent?.trim() ?? (node as any).nodeValue?.trim() ?? '';\n      if (text) matches.push(text);\n      node = result.iterateNext?.();\n    }\n    return { matches, count: matches.length, method: 'xpath' };\n  } catch (e) {\n    return { matches: [], count: 0, method: 'xpath-error' };\n  }\n}\n\nexport function extractByRegex(text: string, pattern: string, flags?: string): ExtractionResult {\n  try {\n    const re = new RegExp(pattern, (flags ?? 'g').includes('g') ? flags : (flags ?? '') + 'g');\n    const matches: string[] = [];\n    let m: RegExpExecArray | null;\n    while ((m = re.exec(text)) !== null) {\n      matches.push(m[1] ?? m[0]);\n      if (!re.global) break;\n    }\n    return { matches, count: matches.length, method: 'regex' };\n  } catch {\n    return { matches: [], count: 0, method: 'regex-error' };\n  }\n}\n\nexport function extractTables(html: string): TableRow[][] {\n  try {\n    const { document } = parseHTML(html);\n    const tables = Array.from(document.querySelectorAll('table'));\n    return tables.map((table: any) => {\n      const headers: string[] = [];\n      const rows: TableRow[] = [];\n      const ths = [...table.querySelectorAll('thead th, thead td, tr:first-child th')];\n      if (ths.length > 0) {\n        ths.forEach((th: any, i: number) => headers.push(th.textContent?.trim() || `col${i + 1}`));\n      }\n      const trs = [...table.querySelectorAll('tbody tr, tr')];\n      for (const tr of trs) {\n        const cells = [...(tr as any).querySelectorAll('td, th')];\n        if (cells.length === 0) continue;\n        if (headers.length === 0) {\n          cells.forEach((_: any, i: number) => headers.push(`col${i + 1}`));\n        }\n        const row: TableRow = {};\n        cells.forEach((cell: any, i: number) => {\n          const key = headers[i] || `col${i + 1}`;\n          row[key] = cell.textContent?.trim() ?? '';\n        });\n        if (Object.values(row).some(v => v)) rows.push(row);\n      }\n      return rows;\n    }).filter(t => t.length > 0);\n  } catch {\n    return [];\n  }\n}\n\nexport function extractMetadata(html: string, url: string): PageMetadata {\n  const meta: PageMetadata = {\n    title: null, description: null, keywords: null,\n    ogTitle: null, ogDescription: null, ogImage: null, ogUrl: null, ogType: null,\n    twitterCard: null, twitterTitle: null, twitterDescription: null,\n    canonical: null, robots: null, jsonLd: [], schemaTypes: [], lang: null,\n  };\n  try {\n    const { document } = parseHTML(html);\n    meta.title = (document as any).title ?? (document as any).querySelector('title')?.textContent?.trim() ?? null;\n    meta.lang = (document as any).documentElement?.getAttribute('lang') ?? null;\n\n    const metas = [...(document as any).querySelectorAll('meta')];\n    for (const m of metas) {\n      const name = (m as any).getAttribute('name')?.toLowerCase() ?? '';\n      const prop = (m as any).getAttribute('property')?.toLowerCase() ?? '';\n      const content = (m as any).getAttribute('content') ?? '';\n      if (name === 'description') meta.description = content;\n      if (name === 'keywords') meta.keywords = content;\n      if (name === 'robots') meta.robots = content;\n      if (name === 'twitter:card') meta.twitterCard = content;\n      if (name === 'twitter:title') meta.twitterTitle = content;\n      if (name === 'twitter:description') meta.twitterDescription = content;\n      if (prop === 'og:title') meta.ogTitle = content;\n      if (prop === 'og:description') meta.ogDescription = content;\n      if (prop === 'og:image') meta.ogImage = content;\n      if (prop === 'og:url') meta.ogUrl = content;\n      if (prop === 'og:type') meta.ogType = content;\n    }\n\n    const canonical = (document as any).querySelector('link[rel=\"canonical\"]');\n    if (canonical) meta.canonical = canonical.getAttribute('href') ?? null;\n\n    const jsonLdScripts = [...(document as any).querySelectorAll('script[type=\"application/ld+json\"]')];\n    for (const s of jsonLdScripts) {\n      try {\n        const parsed = JSON.parse((s as any).textContent ?? '{}');\n        meta.jsonLd.push(parsed);\n        const types = Array.isArray(parsed) ? parsed.map((p: any) => p['@type']).flat() : [parsed['@type']];\n        meta.schemaTypes.push(...types.filter(Boolean));\n      } catch { /* skip invalid JSON-LD */ }\n    }\n  } catch { /* non-fatal */ }\n  return meta;\n}\n\nexport interface ProductData {\n  name: string | null;\n  description: string | null;\n  brand: string | null;\n  sku: string | null;\n  price: number | null;\n  priceCurrency: string | null;\n  availability: string | null;\n  rating: number | null;\n  reviewCount: number | null;\n  image: string | null;\n  url: string | null;\n}\n\nexport function extractStructuredProduct(jsonLd: object[]): ProductData | null {\n  for (const item of jsonLd) {\n    try {\n      const type = (item as any)['@type'];\n      const isProduct = Array.isArray(type)\n        ? type.some((t: any) => typeof t === 'string' && t.toLowerCase() === 'product')\n        : typeof type === 'string' && type.toLowerCase() === 'product';\n      if (!isProduct) continue;\n\n      const i = item as any;\n      const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;\n\n      const rawPrice = offers?.price;\n      const price = rawPrice != null ? parseFloat(rawPrice) : NaN;\n\n      const rawRating = i.aggregateRating?.ratingValue;\n      const rating = rawRating != null ? parseFloat(rawRating) : NaN;\n\n      const rawReviewCount = i.aggregateRating?.reviewCount;\n      const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;\n\n      const rawAvailability: string | undefined = offers?.availability;\n      const availability = rawAvailability\n        ? rawAvailability.replace(/^https?:\\/\\/schema\\.org\\//, '')\n        : null;\n\n      const rawImage = i.image;\n      const image = Array.isArray(rawImage) ? (rawImage[0] ?? null) : (rawImage ?? null);\n\n      const brand = i.brand?.name ?? i.brand ?? null;\n\n      return {\n        name: i.name ?? null,\n        description: i.description ?? null,\n        brand: typeof brand === 'string' ? brand : null,\n        sku: i.sku ?? null,\n        price: isNaN(price) ? null : price,\n        priceCurrency: offers?.priceCurrency ?? null,\n        availability,\n        rating: isNaN(rating) ? null : rating,\n        reviewCount: isNaN(reviewCount) ? null : reviewCount,\n        image,\n        url: i.url ?? null,\n      };\n    } catch { /* skip malformed item */ }\n  }\n  return null;\n}\n\nexport function extractImages(html: string, baseUrl: string): Array<{ src: string; alt: string; width?: number; height?: number }> {\n  try {\n    const { document } = parseHTML(html);\n    const base = new URL(baseUrl);\n    return Array.from((document as any).querySelectorAll('img'))\n      .map((img: any) => {\n        const src = img.getAttribute('src') ?? img.getAttribute('data-src') ?? '';\n        if (!src) return null;\n        try {\n          return {\n            src: new URL(src, base).href,\n            alt: (img.getAttribute('alt') ?? '') as string,\n            width: img.getAttribute('width') ? parseInt(img.getAttribute('width')) : undefined,\n            height: img.getAttribute('height') ? parseInt(img.getAttribute('height')) : undefined,\n          } as { src: string; alt: string; width?: number; height?: number };\n        } catch { return null; }\n      })\n      .filter((x): x is { src: string; alt: string; width?: number; height?: number } => x !== null);\n  } catch {\n    return [];\n  }\n}\n\nexport function extractMediaFiles(html: string, baseUrl: string): Array<{ url: string; type: 'image' | 'video' | 'audio' | 'document'; tag: string }> {\n  const IMAGE_EXT = /\\.(png|jpe?g|gif|webp|svg|ico|avif)(\\?.*)?$/i;\n  const VIDEO_EXT = /\\.(mp4|webm|ogv|mov|avi)(\\?.*)?$/i;\n  const AUDIO_EXT = /\\.(mp3|ogg|wav|flac|aac)(\\?.*)?$/i;\n  const DOC_EXT = /\\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\\?.*)?$/i;\n\n  function classify(url: string, tag: string): 'image' | 'video' | 'audio' | 'document' {\n    if (tag === 'img' || IMAGE_EXT.test(url)) return 'image';\n    if (tag === 'video' || VIDEO_EXT.test(url)) return 'video';\n    if (tag === 'audio' || AUDIO_EXT.test(url)) return 'audio';\n    if (DOC_EXT.test(url)) return 'document';\n    return 'document';\n  }\n\n  try {\n    const { document } = parseHTML(html);\n    const base = new URL(baseUrl);\n    const results: Array<{ url: string; type: 'image' | 'video' | 'audio' | 'document'; tag: string }> = [];\n    const seen = new Set<string>();\n\n    const selectors: [string, string] = ['img[src],video[src],audio[src],source[src]', 'src'];\n    const attrPairs: Array<[string, string]> = [\n      ['img', 'src'], ['video', 'src'], ['audio', 'src'], ['source', 'src'],\n      ['a', 'href'],\n    ];\n\n    for (const [sel, attr] of attrPairs) {\n      for (const el of [...(document as any).querySelectorAll(sel)]) {\n        const raw = (el as any).getAttribute(attr) ?? '';\n        if (!raw) continue;\n        try {\n          const resolved = new URL(raw, base).href;\n          if (seen.has(resolved)) continue;\n          if (sel === 'a' && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved)) continue;\n          seen.add(resolved);\n          results.push({ url: resolved, type: classify(resolved, sel), tag: sel });\n        } catch { /* skip */ }\n      }\n    }\n    return results;\n  } catch {\n    return [];\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,iBAAiB;AA6BnB,SAAS,qBAAqB,MAAc,UAAkB,MAAiC;AACpG,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,QAAQ,MAAM,KAAK,SAAS,iBAAiB,QAAQ,CAAC;AAC5D,UAAM,UAAU,MAAM,IAAI,CAAC,MAAW;AACpC,UAAI,KAAM,QAAQ,EAAU,aAAa,IAAI,KAAK;AAClD,aAAQ,EAAU,aAAa,KAAK,KAAK;AAAA,IAC3C,CAAC,EAAE,OAAO,OAAO;AACjB,WAAO,EAAE,SAAS,OAAO,QAAQ,QAAQ,QAAQ,eAAe;AAAA,EAClE,SAAS,GAAG;AACV,WAAO,EAAE,SAAS,CAAC,GAAG,OAAO,GAAG,QAAQ,qBAAqB;AAAA,EAC/D;AACF;AAEO,SAAS,eAAe,MAAc,OAAiC;AAC5E,MAAI;AACF,UAAM,EAAE,UAAU,YAAY,IAAI,UAAU,IAAI;AAChD,UAAM,SAAS,SAAS,SAAS,OAAO,UAAU,MAAM,aAAa,YAAY,GAAG,IAAI;AACxF,UAAM,UAAoB,CAAC;AAC3B,QAAI,OAAO,OAAO,cAAc;AAChC,WAAO,MAAM;AACX,YAAM,OAAQ,KAAa,aAAa,KAAK,KAAM,KAAa,WAAW,KAAK,KAAK;AACrF,UAAI,KAAM,SAAQ,KAAK,IAAI;AAC3B,aAAO,OAAO,cAAc;AAAA,IAC9B;AACA,WAAO,EAAE,SAAS,OAAO,QAAQ,QAAQ,QAAQ,QAAQ;AAAA,EAC3D,SAAS,GAAG;AACV,WAAO,EAAE,SAAS,CAAC,GAAG,OAAO,GAAG,QAAQ,cAAc;AAAA,EACxD;AACF;AAEO,SAAS,eAAe,MAAc,SAAiB,OAAkC;AAC9F,MAAI;AACF,UAAM,KAAK,IAAI,OAAO,UAAU,SAAS,KAAK,SAAS,GAAG,IAAI,SAAS,SAAS,MAAM,GAAG;AACzF,UAAM,UAAoB,CAAC;AAC3B,QAAI;AACJ,YAAQ,IAAI,GAAG,KAAK,IAAI,OAAO,MAAM;AACnC,cAAQ,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;AACzB,UAAI,CAAC,GAAG,OAAQ;AAAA,IAClB;AACA,WAAO,EAAE,SAAS,OAAO,QAAQ,QAAQ,QAAQ,QAAQ;AAAA,EAC3D,QAAQ;AACN,WAAO,EAAE,SAAS,CAAC,GAAG,OAAO,GAAG,QAAQ,cAAc;AAAA,EACxD;AACF;AAEO,SAAS,cAAc,MAA4B;AACxD,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,SAAS,MAAM,KAAK,SAAS,iBAAiB,OAAO,CAAC;AAC5D,WAAO,OAAO,IAAI,CAAC,UAAe;AAChC,YAAM,UAAoB,CAAC;AAC3B,YAAM,OAAmB,CAAC;AAC1B,YAAM,MAAM,CAAC,GAAG,MAAM,iBAAiB,uCAAuC,CAAC;AAC/E,UAAI,IAAI,SAAS,GAAG;AAClB,YAAI,QAAQ,CAAC,IAAS,MAAc,QAAQ,KAAK,GAAG,aAAa,KAAK,KAAK,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,MAC3F;AACA,YAAM,MAAM,CAAC,GAAG,MAAM,iBAAiB,cAAc,CAAC;AACtD,iBAAW,MAAM,KAAK;AACpB,cAAM,QAAQ,CAAC,GAAI,GAAW,iBAAiB,QAAQ,CAAC;AACxD,YAAI,MAAM,WAAW,EAAG;AACxB,YAAI,QAAQ,WAAW,GAAG;AACxB,gBAAM,QAAQ,CAAC,GAAQ,MAAc,QAAQ,KAAK,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,QAClE;AACA,cAAM,MAAgB,CAAC;AACvB,cAAM,QAAQ,CAAC,MAAW,MAAc;AACtC,gBAAM,MAAM,QAAQ,CAAC,KAAK,MAAM,IAAI,CAAC;AACrC,cAAI,GAAG,IAAI,KAAK,aAAa,KAAK,KAAK;AAAA,QACzC,CAAC;AACD,YAAI,OAAO,OAAO,GAAG,EAAE,KAAK,OAAK,CAAC,EAAG,MAAK,KAAK,GAAG;AAAA,MACpD;AACA,aAAO;AAAA,IACT,CAAC,EAAE,OAAO,OAAK,EAAE,SAAS,CAAC;AAAA,EAC7B,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEO,SAAS,gBAAgB,MAAc,KAA2B;AACvE,QAAM,OAAqB;AAAA,IACzB,OAAO;AAAA,IAAM,aAAa;AAAA,IAAM,UAAU;AAAA,IAC1C,SAAS;AAAA,IAAM,eAAe;AAAA,IAAM,SAAS;AAAA,IAAM,OAAO;AAAA,IAAM,QAAQ;AAAA,IACxE,aAAa;AAAA,IAAM,cAAc;AAAA,IAAM,oBAAoB;AAAA,IAC3D,WAAW;AAAA,IAAM,QAAQ;AAAA,IAAM,QAAQ,CAAC;AAAA,IAAG,aAAa,CAAC;AAAA,IAAG,MAAM;AAAA,EACpE;AACA,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,SAAK,QAAS,SAAiB,SAAU,SAAiB,cAAc,OAAO,GAAG,aAAa,KAAK,KAAK;AACzG,SAAK,OAAQ,SAAiB,iBAAiB,aAAa,MAAM,KAAK;AAEvE,UAAM,QAAQ,CAAC,GAAI,SAAiB,iBAAiB,MAAM,CAAC;AAC5D,eAAW,KAAK,OAAO;AACrB,YAAM,OAAQ,EAAU,aAAa,MAAM,GAAG,YAAY,KAAK;AAC/D,YAAM,OAAQ,EAAU,aAAa,UAAU,GAAG,YAAY,KAAK;AACnE,YAAM,UAAW,EAAU,aAAa,SAAS,KAAK;AACtD,UAAI,SAAS,cAAe,MAAK,cAAc;AAC/C,UAAI,SAAS,WAAY,MAAK,WAAW;AACzC,UAAI,SAAS,SAAU,MAAK,SAAS;AACrC,UAAI,SAAS,eAAgB,MAAK,cAAc;AAChD,UAAI,SAAS,gBAAiB,MAAK,eAAe;AAClD,UAAI,SAAS,sBAAuB,MAAK,qBAAqB;AAC9D,UAAI,SAAS,WAAY,MAAK,UAAU;AACxC,UAAI,SAAS,iBAAkB,MAAK,gBAAgB;AACpD,UAAI,SAAS,WAAY,MAAK,UAAU;AACxC,UAAI,SAAS,SAAU,MAAK,QAAQ;AACpC,UAAI,SAAS,UAAW,MAAK,SAAS;AAAA,IACxC;AAEA,UAAM,YAAa,SAAiB,cAAc,uBAAuB;AACzE,QAAI,UAAW,MAAK,YAAY,UAAU,aAAa,MAAM,KAAK;AAElE,UAAM,gBAAgB,CAAC,GAAI,SAAiB,iBAAiB,oCAAoC,CAAC;AAClG,eAAW,KAAK,eAAe;AAC7B,UAAI;AACF,cAAM,SAAS,KAAK,MAAO,EAAU,eAAe,IAAI;AACxD,aAAK,OAAO,KAAK,MAAM;AACvB,cAAM,QAAQ,MAAM,QAAQ,MAAM,IAAI,OAAO,IAAI,CAAC,MAAW,EAAE,OAAO,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,OAAO,CAAC;AAClG,aAAK,YAAY,KAAK,GAAG,MAAM,OAAO,OAAO,CAAC;AAAA,MAChD,QAAQ;AAAA,MAA6B;AAAA,IACvC;AAAA,EACF,QAAQ;AAAA,EAAkB;AAC1B,SAAO;AACT;AAgBO,SAAS,yBAAyB,QAAsC;AAC7E,aAAW,QAAQ,QAAQ;AACzB,QAAI;AACF,YAAM,OAAQ,KAAa,OAAO;AAClC,YAAM,YAAY,MAAM,QAAQ,IAAI,IAChC,KAAK,KAAK,CAAC,MAAW,OAAO,MAAM,YAAY,EAAE,YAAY,MAAM,SAAS,IAC5E,OAAO,SAAS,YAAY,KAAK,YAAY,MAAM;AACvD,UAAI,CAAC,UAAW;AAEhB,YAAM,IAAI;AACV,YAAM,SAAS,MAAM,QAAQ,EAAE,MAAM,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;AAEzD,YAAM,WAAW,QAAQ;AACzB,YAAM,QAAQ,YAAY,OAAO,WAAW,QAAQ,IAAI;AAExD,YAAM,YAAY,EAAE,iBAAiB;AACrC,YAAM,SAAS,aAAa,OAAO,WAAW,SAAS,IAAI;AAE3D,YAAM,iBAAiB,EAAE,iBAAiB;AAC1C,YAAM,cAAc,kBAAkB,OAAO,SAAS,gBAAgB,EAAE,IAAI;AAE5E,YAAM,kBAAsC,QAAQ;AACpD,YAAM,eAAe,kBACjB,gBAAgB,QAAQ,6BAA6B,EAAE,IACvD;AAEJ,YAAM,WAAW,EAAE;AACnB,YAAM,QAAQ,MAAM,QAAQ,QAAQ,IAAK,SAAS,CAAC,KAAK,OAAS,YAAY;AAE7E,YAAM,QAAQ,EAAE,OAAO,QAAQ,EAAE,SAAS;AAE1C,aAAO;AAAA,QACL,MAAM,EAAE,QAAQ;AAAA,QAChB,aAAa,EAAE,eAAe;AAAA,QAC9B,OAAO,OAAO,UAAU,WAAW,QAAQ;AAAA,QAC3C,KAAK,EAAE,OAAO;AAAA,QACd,OAAO,MAAM,KAAK,IAAI,OAAO;AAAA,QAC7B,eAAe,QAAQ,iBAAiB;AAAA,QACxC;AAAA,QACA,QAAQ,MAAM,MAAM,IAAI,OAAO;AAAA,QAC/B,aAAa,MAAM,WAAW,IAAI,OAAO;AAAA,QACzC;AAAA,QACA,KAAK,EAAE,OAAO;AAAA,MAChB;AAAA,IACF,QAAQ;AAAA,IAA4B;AAAA,EACtC;AACA,SAAO;AACT;AAEO,SAAS,cAAc,MAAc,SAAuF;AACjI,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,WAAO,MAAM,KAAM,SAAiB,iBAAiB,KAAK,CAAC,EACxD,IAAI,CAAC,QAAa;AACjB,YAAM,MAAM,IAAI,aAAa,KAAK,KAAK,IAAI,aAAa,UAAU,KAAK;AACvE,UAAI,CAAC,IAAK,QAAO;AACjB,UAAI;AACF,eAAO;AAAA,UACL,KAAK,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,UACxB,KAAM,IAAI,aAAa,KAAK,KAAK;AAAA,UACjC,OAAO,IAAI,aAAa,OAAO,IAAI,SAAS,IAAI,aAAa,OAAO,CAAC,IAAI;AAAA,UACzE,QAAQ,IAAI,aAAa,QAAQ,IAAI,SAAS,IAAI,aAAa,QAAQ,CAAC,IAAI;AAAA,QAC9E;AAAA,MACF,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IACzB,CAAC,EACA,OAAO,CAAC,MAA0E,MAAM,IAAI;AAAA,EACjG,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEO,SAAS,kBAAkB,MAAc,SAAsG;AACpJ,QAAM,YAAY;AAClB,QAAM,YAAY;AAClB,QAAM,YAAY;AAClB,QAAM,UAAU;AAEhB,WAAS,SAAS,KAAa,KAAuD;AACpF,QAAI,QAAQ,SAAS,UAAU,KAAK,GAAG,EAAG,QAAO;AACjD,QAAI,QAAQ,WAAW,UAAU,KAAK,GAAG,EAAG,QAAO;AACnD,QAAI,QAAQ,WAAW,UAAU,KAAK,GAAG,EAAG,QAAO;AACnD,QAAI,QAAQ,KAAK,GAAG,EAAG,QAAO;AAC9B,WAAO;AAAA,EACT;AAEA,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,UAAM,OAAO,IAAI,IAAI,OAAO;AAC5B,UAAM,UAA+F,CAAC;AACtG,UAAM,OAAO,oBAAI,IAAY;AAE7B,UAAM,YAA8B,CAAC,8CAA8C,KAAK;AACxF,UAAM,YAAqC;AAAA,MACzC,CAAC,OAAO,KAAK;AAAA,MAAG,CAAC,SAAS,KAAK;AAAA,MAAG,CAAC,SAAS,KAAK;AAAA,MAAG,CAAC,UAAU,KAAK;AAAA,MACpE,CAAC,KAAK,MAAM;AAAA,IACd;AAEA,eAAW,CAAC,KAAK,IAAI,KAAK,WAAW;AACnC,iBAAW,MAAM,CAAC,GAAI,SAAiB,iBAAiB,GAAG,CAAC,GAAG;AAC7D,cAAM,MAAO,GAAW,aAAa,IAAI,KAAK;AAC9C,YAAI,CAAC,IAAK;AACV,YAAI;AACF,gBAAM,WAAW,IAAI,IAAI,KAAK,IAAI,EAAE;AACpC,cAAI,KAAK,IAAI,QAAQ,EAAG;AACxB,cAAI,QAAQ,OAAO,CAAC,QAAQ,KAAK,QAAQ,KAAK,CAAC,UAAU,KAAK,QAAQ,KAAK,CAAC,UAAU,KAAK,QAAQ,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAG;AACnI,eAAK,IAAI,QAAQ;AACjB,kBAAQ,KAAK,EAAE,KAAK,UAAU,MAAM,SAAS,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC;AAAA,QACzE,QAAQ;AAAA,QAAa;AAAA,MACvB;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/extractors/artifact-envelope.d.ts

@@ -0,0 +1,63 @@
+import type { RawElement } from '../screen-reader.js';
+export interface CrawlEvidenceFact {
+    type: string;
+    payload: unknown;
+}
+export interface CrawlArtifactEnvelope {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';
+    outputs: {
+        screenshot?: string;
+        html?: string;
+        markdown?: string;
+        fitMarkdown?: string;
+        accessibilityTree?: string;
+        structuredJson?: unknown;
+        networkTrace?: string;
+        consoleLog?: string;
+    };
+    fingerprints: {
+        textHash?: string;
+        domHash?: string;
+        accessibilityHash?: string;
+        urlCanonicalHash: string;
+    };
+    evidence: CrawlEvidenceFact[];
+}
+export declare function sha256(input: string): string;
+export declare function buildFitMarkdown(markdown: string, keywords?: string[]): string;
+export declare function summarizeStructuredElements(elements: RawElement[]): {
+    elementCount: number;
+    formFieldCount: number;
+    actionCount: number;
+    fields: {
+        label: string;
+        role: string;
+        expectedData: string | undefined;
+    }[];
+    actions: {
+        label: string;
+        role: string;
+    }[];
+};
+export declare function buildCrawlArtifactEnvelope(input: {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    screenshotPath?: string;
+    domPath?: string;
+    domContent?: string;
+    markdown?: string;
+    fitMarkdown?: string;
+    accessibilityElements?: RawElement[];
+    structuredJson?: unknown;
+    evidence?: CrawlEvidenceFact[];
+}): CrawlArtifactEnvelope;

package/dist/extractors/artifact-envelope.js

@@ -0,0 +1,63 @@
+import crypto from "node:crypto";
+function sha256(input) {
+  return crypto.createHash("sha256").update(input).digest("hex");
+}
+function buildFitMarkdown(markdown, keywords = []) {
+  const lines = markdown.split("\n").map((line) => line.trimEnd()).filter((line) => line.trim().length > 0);
+  const lowerKeywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+  const filtered = lines.filter((line) => {
+    const plain = line.replace(/[#*_`>\-[\]()]/g, "").trim();
+    if (plain.length < 3) return false;
+    if (/^(cookie|privacy|terms|copyright|all rights reserved|subscribe|newsletter)$/i.test(plain)) return false;
+    if (lowerKeywords.length === 0) return true;
+    const lower = plain.toLowerCase();
+    return lowerKeywords.some((kw) => lower.includes(kw)) || /^#{1,4}\s/.test(line);
+  });
+  const candidate = filtered.join("\n");
+  return candidate.length >= 200 ? candidate : markdown;
+}
+function summarizeStructuredElements(elements) {
+  const forms = elements.filter((e) => ["email", "password", "text", "tel", "date", "number", "select", "textarea"].includes(e.role));
+  const actions = elements.filter((e) => e.role === "button" || e.role === "submit" || e.role === "link" || e.role === "a");
+  return {
+    elementCount: elements.length,
+    formFieldCount: forms.length,
+    actionCount: actions.length,
+    fields: forms.slice(0, 50).map((e) => ({ label: e.label, role: e.role, expectedData: e.expectedData })),
+    actions: actions.slice(0, 50).map((e) => ({ label: e.label, role: e.role }))
+  };
+}
+function buildCrawlArtifactEnvelope(input) {
+  const accessibilityTree = input.accessibilityElements ? JSON.stringify(input.accessibilityElements) : void 0;
+  return {
+    tenantId: input.tenantId,
+    projectId: input.projectId,
+    jobId: input.jobId,
+    url: input.url,
+    canonicalUrl: input.canonicalUrl,
+    screenId: input.screenId,
+    status: "ok",
+    outputs: {
+      screenshot: input.screenshotPath,
+      html: input.domPath,
+      markdown: input.markdown,
+      fitMarkdown: input.fitMarkdown,
+      accessibilityTree,
+      structuredJson: input.structuredJson
+    },
+    fingerprints: {
+      textHash: input.markdown ? sha256(input.markdown) : void 0,
+      domHash: input.domContent ? sha256(input.domContent) : void 0,
+      accessibilityHash: accessibilityTree ? sha256(accessibilityTree) : void 0,
+      urlCanonicalHash: sha256(input.canonicalUrl)
+    },
+    evidence: input.evidence ?? []
+  };
+}
+export {
+  buildCrawlArtifactEnvelope,
+  buildFitMarkdown,
+  sha256,
+  summarizeStructuredElements
+};
+//# sourceMappingURL=artifact-envelope.js.map

package/dist/extractors/artifact-envelope.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extractors/artifact-envelope.ts"],
+  "sourcesContent": ["import crypto from 'node:crypto';\nimport type { RawElement } from '../screen-reader.js';\n\nexport interface CrawlEvidenceFact {\n  type: string;\n  payload: unknown;\n}\n\nexport interface CrawlArtifactEnvelope {\n  tenantId: string;\n  projectId: string;\n  jobId?: string;\n  url: string;\n  canonicalUrl: string;\n  screenId?: string;\n  status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';\n  outputs: {\n    screenshot?: string;\n    html?: string;\n    markdown?: string;\n    fitMarkdown?: string;\n    accessibilityTree?: string;\n    structuredJson?: unknown;\n    networkTrace?: string;\n    consoleLog?: string;\n  };\n  fingerprints: {\n    textHash?: string;\n    domHash?: string;\n    accessibilityHash?: string;\n    urlCanonicalHash: string;\n  };\n  evidence: CrawlEvidenceFact[];\n}\n\nexport function sha256(input: string): string {\n  return crypto.createHash('sha256').update(input).digest('hex');\n}\n\nexport function buildFitMarkdown(markdown: string, keywords: string[] = []): string {\n  const lines = markdown\n    .split('\\n')\n    .map((line) => line.trimEnd())\n    .filter((line) => line.trim().length > 0);\n  const lowerKeywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n  const filtered = lines.filter((line) => {\n    const plain = line.replace(/[#*_`>\\-[\\]()]/g, '').trim();\n    if (plain.length < 3) return false;\n    if (/^(cookie|privacy|terms|copyright|all rights reserved|subscribe|newsletter)$/i.test(plain)) return false;\n    if (lowerKeywords.length === 0) return true;\n    const lower = plain.toLowerCase();\n    return lowerKeywords.some((kw) => lower.includes(kw)) || /^#{1,4}\\s/.test(line);\n  });\n  const candidate = filtered.join('\\n');\n  return candidate.length >= 200 ? candidate : markdown;\n}\n\nexport function summarizeStructuredElements(elements: RawElement[]) {\n  const forms = elements.filter((e) => ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role));\n  const actions = elements.filter((e) => e.role === 'button' || e.role === 'submit' || e.role === 'link' || e.role === 'a');\n  return {\n    elementCount: elements.length,\n    formFieldCount: forms.length,\n    actionCount: actions.length,\n    fields: forms.slice(0, 50).map((e) => ({ label: e.label, role: e.role, expectedData: e.expectedData })),\n    actions: actions.slice(0, 50).map((e) => ({ label: e.label, role: e.role })),\n  };\n}\n\nexport function buildCrawlArtifactEnvelope(input: {\n  tenantId: string;\n  projectId: string;\n  jobId?: string;\n  url: string;\n  canonicalUrl: string;\n  screenId?: string;\n  screenshotPath?: string;\n  domPath?: string;\n  domContent?: string;\n  markdown?: string;\n  fitMarkdown?: string;\n  accessibilityElements?: RawElement[];\n  structuredJson?: unknown;\n  evidence?: CrawlEvidenceFact[];\n}): CrawlArtifactEnvelope {\n  const accessibilityTree = input.accessibilityElements ? JSON.stringify(input.accessibilityElements) : undefined;\n  return {\n    tenantId: input.tenantId,\n    projectId: input.projectId,\n    jobId: input.jobId,\n    url: input.url,\n    canonicalUrl: input.canonicalUrl,\n    screenId: input.screenId,\n    status: 'ok',\n    outputs: {\n      screenshot: input.screenshotPath,\n      html: input.domPath,\n      markdown: input.markdown,\n      fitMarkdown: input.fitMarkdown,\n      accessibilityTree,\n      structuredJson: input.structuredJson,\n    },\n    fingerprints: {\n      textHash: input.markdown ? sha256(input.markdown) : undefined,\n      domHash: input.domContent ? sha256(input.domContent) : undefined,\n      accessibilityHash: accessibilityTree ? sha256(accessibilityTree) : undefined,\n      urlCanonicalHash: sha256(input.canonicalUrl),\n    },\n    evidence: input.evidence ?? [],\n  };\n}\n"],
+  "mappings": "AAAA,OAAO,YAAY;AAmCZ,SAAS,OAAO,OAAuB;AAC5C,SAAO,OAAO,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AAC/D;AAEO,SAAS,iBAAiB,UAAkB,WAAqB,CAAC,GAAW;AAClF,QAAM,QAAQ,SACX,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,EAC5B,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,SAAS,CAAC;AAC1C,QAAM,gBAAgB,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACzE,QAAM,WAAW,MAAM,OAAO,CAAC,SAAS;AACtC,UAAM,QAAQ,KAAK,QAAQ,mBAAmB,EAAE,EAAE,KAAK;AACvD,QAAI,MAAM,SAAS,EAAG,QAAO;AAC7B,QAAI,+EAA+E,KAAK,KAAK,EAAG,QAAO;AACvG,QAAI,cAAc,WAAW,EAAG,QAAO;AACvC,UAAM,QAAQ,MAAM,YAAY;AAChC,WAAO,cAAc,KAAK,CAAC,OAAO,MAAM,SAAS,EAAE,CAAC,KAAK,YAAY,KAAK,IAAI;AAAA,EAChF,CAAC;AACD,QAAM,YAAY,SAAS,KAAK,IAAI;AACpC,SAAO,UAAU,UAAU,MAAM,YAAY;AAC/C;AAEO,SAAS,4BAA4B,UAAwB;AAClE,QAAM,QAAQ,SAAS,OAAO,CAAC,MAAM,CAAC,SAAS,YAAY,QAAQ,OAAO,QAAQ,UAAU,UAAU,UAAU,EAAE,SAAS,EAAE,IAAI,CAAC;AAClI,QAAM,UAAU,SAAS,OAAO,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,SAAS,YAAY,EAAE,SAAS,UAAU,EAAE,SAAS,GAAG;AACxH,SAAO;AAAA,IACL,cAAc,SAAS;AAAA,IACvB,gBAAgB,MAAM;AAAA,IACtB,aAAa,QAAQ;AAAA,IACrB,QAAQ,MAAM,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,MAAM,EAAE,MAAM,cAAc,EAAE,aAAa,EAAE;AAAA,IACtG,SAAS,QAAQ,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,MAAM,EAAE,KAAK,EAAE;AAAA,EAC7E;AACF;AAEO,SAAS,2BAA2B,OAejB;AACxB,QAAM,oBAAoB,MAAM,wBAAwB,KAAK,UAAU,MAAM,qBAAqB,IAAI;AACtG,SAAO;AAAA,IACL,UAAU,MAAM;AAAA,IAChB,WAAW,MAAM;AAAA,IACjB,OAAO,MAAM;AAAA,IACb,KAAK,MAAM;AAAA,IACX,cAAc,MAAM;AAAA,IACpB,UAAU,MAAM;AAAA,IAChB,QAAQ;AAAA,IACR,SAAS;AAAA,MACP,YAAY,MAAM;AAAA,MAClB,MAAM,MAAM;AAAA,MACZ,UAAU,MAAM;AAAA,MAChB,aAAa,MAAM;AAAA,MACnB;AAAA,MACA,gBAAgB,MAAM;AAAA,IACxB;AAAA,IACA,cAAc;AAAA,MACZ,UAAU,MAAM,WAAW,OAAO,MAAM,QAAQ,IAAI;AAAA,MACpD,SAAS,MAAM,aAAa,OAAO,MAAM,UAAU,IAAI;AAAA,MACvD,mBAAmB,oBAAoB,OAAO,iBAAiB,IAAI;AAAA,MACnE,kBAAkB,OAAO,MAAM,YAAY;AAAA,IAC7C;AAAA,IACA,UAAU,MAAM,YAAY,CAAC;AAAA,EAC/B;AACF;",
+  "names": []
+}

package/dist/failure-analysis-agent.d.ts

@@ -0,0 +1,16 @@
+export interface FailureContext {
+    tenantId?: string;
+    error: string;
+    stepDescription?: string;
+    consoleLogs?: Array<{
+        type: string;
+        text: string;
+    }>;
+    networkErrors?: Array<{
+        url: string;
+        status: number;
+        method: string;
+    }>;
+    screenshotPath?: string;
+}
+export declare function analyzeFailure(ctx: FailureContext): Promise<string | null>;

package/dist/failure-analysis-agent.js

@@ -0,0 +1,40 @@
+import { readFileSync } from "node:fs";
+import { askTrackedWithImages, askTracked } from "@qip/agents";
+import { AppBrain } from "@qip/app-brain";
+async function analyzeFailure(ctx) {
+  try {
+    let llmCfg;
+    if (ctx.tenantId) {
+      llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, "testRunner").catch(() => void 0) ?? void 0;
+    }
+    if (!llmCfg) return null;
+    const consoleSection = ctx.consoleLogs?.length ? `
+
+Browser console (last ${ctx.consoleLogs.length} entries):
+${ctx.consoleLogs.map((l) => `[${l.type}] ${l.text}`).join("\n")}` : "";
+    const networkSection = ctx.networkErrors?.length ? `
+
+Failed network requests:
+${ctx.networkErrors.map((n) => `${n.method} ${n.url} \u2192 ${n.status}`).join("\n")}` : "";
+    const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific \u2014 name the failing element, API endpoint, or error type. Do not suggest fixes.
+
+Test step: ${ctx.stepDescription ?? "unknown"}
+Error: ${ctx.error}${consoleSection}${networkSection}`;
+    if (ctx.screenshotPath) {
+      try {
+        const img = readFileSync(ctx.screenshotPath);
+        const result2 = await askTrackedWithImages("", userText, [{ base64: img.toString("base64"), mediaType: "image/png" }], llmCfg);
+        return result2.text || null;
+      } catch {
+      }
+    }
+    const result = await askTracked("", userText, llmCfg, 256);
+    return result.text || null;
+  } catch {
+    return null;
+  }
+}
+export {
+  analyzeFailure
+};
+//# sourceMappingURL=failure-analysis-agent.js.map

package/dist/failure-analysis-agent.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/failure-analysis-agent.ts"],
+  "sourcesContent": ["import { readFileSync } from 'node:fs';\nimport { askTrackedWithImages, askTracked } from '@qip/agents';\nimport type { LLMConfig } from '@qip/agents';\nimport { AppBrain } from '@qip/app-brain';\n\nexport interface FailureContext {\n  tenantId?: string;\n  error: string;\n  stepDescription?: string;\n  consoleLogs?: Array<{ type: string; text: string }>;\n  networkErrors?: Array<{ url: string; status: number; method: string }>;\n  screenshotPath?: string;\n}\n\nexport async function analyzeFailure(ctx: FailureContext): Promise<string | null> {\n  try {\n    let llmCfg: LLMConfig | undefined;\n    if (ctx.tenantId) {\n      llmCfg = await AppBrain.getTaskLLMConfig(ctx.tenantId, 'testRunner').catch(() => undefined) as LLMConfig | undefined ?? undefined;\n    }\n    if (!llmCfg) return null;\n\n    const consoleSection = ctx.consoleLogs?.length\n      ? `\\n\\nBrowser console (last ${ctx.consoleLogs.length} entries):\\n${ctx.consoleLogs.map(l => `[${l.type}] ${l.text}`).join('\\n')}`\n      : '';\n\n    const networkSection = ctx.networkErrors?.length\n      ? `\\n\\nFailed network requests:\\n${ctx.networkErrors.map(n => `${n.method} ${n.url} \u2192 ${n.status}`).join('\\n')}`\n      : '';\n\n    const userText = `A Playwright automated test failed. Identify the root cause in 2-3 sentences. Be specific \u2014 name the failing element, API endpoint, or error type. Do not suggest fixes.\n\nTest step: ${ctx.stepDescription ?? 'unknown'}\nError: ${ctx.error}${consoleSection}${networkSection}`;\n\n    if (ctx.screenshotPath) {\n      try {\n        const img = readFileSync(ctx.screenshotPath);\n        const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg);\n        return result.text || null;\n      } catch { /* fall through to text-only */ }\n    }\n\n    const result = await askTracked('', userText, llmCfg, 256);\n    return result.text || null;\n  } catch {\n    return null;\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,oBAAoB;AAC7B,SAAS,sBAAsB,kBAAkB;AAEjD,SAAS,gBAAgB;AAWzB,eAAsB,eAAe,KAA6C;AAChF,MAAI;AACF,QAAI;AACJ,QAAI,IAAI,UAAU;AAChB,eAAS,MAAM,SAAS,iBAAiB,IAAI,UAAU,YAAY,EAAE,MAAM,MAAM,MAAS,KAA8B;AAAA,IAC1H;AACA,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,iBAAiB,IAAI,aAAa,SACpC;AAAA;AAAA,wBAA6B,IAAI,YAAY,MAAM;AAAA,EAAe,IAAI,YAAY,IAAI,OAAK,IAAI,EAAE,IAAI,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,IAAI,CAAC,KAC9H;AAEJ,UAAM,iBAAiB,IAAI,eAAe,SACtC;AAAA;AAAA;AAAA,EAAiC,IAAI,cAAc,IAAI,OAAK,GAAG,EAAE,MAAM,IAAI,EAAE,GAAG,WAAM,EAAE,MAAM,EAAE,EAAE,KAAK,IAAI,CAAC,KAC5G;AAEJ,UAAM,WAAW;AAAA;AAAA,aAER,IAAI,mBAAmB,SAAS;AAAA,SACpC,IAAI,KAAK,GAAG,cAAc,GAAG,cAAc;AAEhD,QAAI,IAAI,gBAAgB;AACtB,UAAI;AACF,cAAM,MAAM,aAAa,IAAI,cAAc;AAC3C,cAAMA,UAAS,MAAM,qBAAqB,IAAI,UAAU,CAAC,EAAE,QAAQ,IAAI,SAAS,QAAQ,GAAG,WAAW,YAAY,CAAC,GAAG,MAAM;AAC5H,eAAOA,QAAO,QAAQ;AAAA,MACxB,QAAQ;AAAA,MAAkC;AAAA,IAC5C;AAEA,UAAM,SAAS,MAAM,WAAW,IAAI,UAAU,QAAQ,GAAG;AACzD,WAAO,OAAO,QAAQ;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": ["result"]
+}

package/dist/feature-flag-detector.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Feature flag + A/B test detector.
+ *
+ * Detects popular feature flag / experiment SDKs loaded on the page:
+ * LaunchDarkly, Optimizely, Split.io, Unleash, GrowthBook, Statsig, Flagsmith, Amplitude Experiment.
+ *
+ * Also detects common A/B test patterns: data-testid attributes, gtm.js experiment layers,
+ * Optimizely variation IDs, VWO, etc.
+ *
+ * Why: ZeTa needs to know when a page shows different content to different users
+ * so test generation can account for variants and not generate flaky tests.
+ */
+import type { Page } from 'playwright';
+export interface FeatureFlagInfo {
+    sdk: string;
+    confidence: 'high' | 'medium' | 'low';
+    details?: string;
+}
+export interface AbTestInfo {
+    tool: string;
+    experiments?: string[];
+}
+export interface FeatureFlagDetectionResult {
+    url: string;
+    flagSdks: FeatureFlagInfo[];
+    abTools: AbTestInfo[];
+    hasAnyFlags: boolean;
+    detectedAt: string;
+}
+export declare function detectFeatureFlags(page: Page, url: string): Promise<FeatureFlagDetectionResult>;

package/dist/feature-flag-detector.js

@@ -0,0 +1,55 @@
+const FLAG_SDK_CHECKS = [
+  { sdk: "LaunchDarkly", windowKeys: ["LDClient", "LaunchDarkly", "ldClient"], scriptPattern: /launchdarkly/i },
+  { sdk: "Optimizely", windowKeys: ["optimizely", "optimizelySdk"], scriptPattern: /optimizely/i },
+  { sdk: "Split.io", windowKeys: ["splitio", "SplitFactory"], scriptPattern: /split\.io/i },
+  { sdk: "GrowthBook", windowKeys: ["growthbook", "GrowthBook"], scriptPattern: /growthbook/i },
+  { sdk: "Statsig", windowKeys: ["statsig", "StatsigClient"], scriptPattern: /statsig/i },
+  { sdk: "Flagsmith", windowKeys: ["flagsmith"], scriptPattern: /flagsmith/i },
+  { sdk: "Unleash", windowKeys: ["unleash", "UnleashClient"], scriptPattern: /unleash/i },
+  { sdk: "VWO", windowKeys: ["vwo", "_vwo"], scriptPattern: /vwo\.com/i },
+  { sdk: "Google Optimize", windowKeys: ["gaData", "google_optimize"], scriptPattern: /googleoptimize/i },
+  { sdk: "AB Tasty", windowKeys: ["ABTasty"], scriptPattern: /abtasty/i }
+];
+async function detectFeatureFlags(page, url) {
+  const detectedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const result = await page.evaluate((checks) => {
+    const flagSdks = [];
+    const scripts = Array.from(document.querySelectorAll("script[src]")).map((s) => s.src ?? "");
+    for (const check of checks) {
+      let found = false;
+      let confidence = "low";
+      if (check.windowKeys?.some((k) => k in window)) {
+        found = true;
+        confidence = "high";
+      }
+      if (!found && check.scriptPattern) {
+        const re = new RegExp(check.scriptPattern.source, "i");
+        const matchingSrc = scripts.find((s) => re.test(s));
+        if (matchingSrc) {
+          found = true;
+          confidence = "medium";
+        }
+      }
+      if (found) flagSdks.push({ sdk: check.sdk, confidence, details: void 0 });
+    }
+    const abTools = [];
+    if ("_vwo_code" in window || "_vis_opt_queue" in window) abTools.push({ tool: "VWO", experiments: [] });
+    if ("dataLayer" in window) {
+      const dl = window.dataLayer ?? [];
+      const expEvents = dl.filter((e) => e?.event?.includes("experiment") || e?.experimentId);
+      if (expEvents.length > 0) abTools.push({ tool: "Google Tag Manager Experiments", experiments: expEvents.slice(0, 5).map((e) => e.experimentId ?? "") });
+    }
+    return { flagSdks, abTools };
+  }, FLAG_SDK_CHECKS).catch(() => ({ flagSdks: [], abTools: [] }));
+  return {
+    url,
+    flagSdks: result.flagSdks,
+    abTools: result.abTools,
+    hasAnyFlags: result.flagSdks.length > 0 || result.abTools.length > 0,
+    detectedAt
+  };
+}
+export {
+  detectFeatureFlags
+};
+//# sourceMappingURL=feature-flag-detector.js.map

package/dist/feature-flag-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/feature-flag-detector.ts"],
+  "sourcesContent": ["/**\n * Feature flag + A/B test detector.\n *\n * Detects popular feature flag / experiment SDKs loaded on the page:\n * LaunchDarkly, Optimizely, Split.io, Unleash, GrowthBook, Statsig, Flagsmith, Amplitude Experiment.\n *\n * Also detects common A/B test patterns: data-testid attributes, gtm.js experiment layers,\n * Optimizely variation IDs, VWO, etc.\n *\n * Why: ZeTa needs to know when a page shows different content to different users\n * so test generation can account for variants and not generate flaky tests.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface FeatureFlagInfo {\n  sdk: string;\n  confidence: 'high' | 'medium' | 'low';\n  details?: string;\n}\n\nexport interface AbTestInfo {\n  tool: string;\n  experiments?: string[];\n}\n\nexport interface FeatureFlagDetectionResult {\n  url: string;\n  flagSdks: FeatureFlagInfo[];\n  abTools: AbTestInfo[];\n  hasAnyFlags: boolean;\n  detectedAt: string;\n}\n\n// SDK detectors \u2014 checked against window object + script URLs\nconst FLAG_SDK_CHECKS: Array<{\n  sdk: string;\n  windowKeys?: string[];\n  scriptPattern?: RegExp;\n}> = [\n  { sdk: 'LaunchDarkly', windowKeys: ['LDClient', 'LaunchDarkly', 'ldClient'], scriptPattern: /launchdarkly/i },\n  { sdk: 'Optimizely', windowKeys: ['optimizely', 'optimizelySdk'], scriptPattern: /optimizely/i },\n  { sdk: 'Split.io', windowKeys: ['splitio', 'SplitFactory'], scriptPattern: /split\\.io/i },\n  { sdk: 'GrowthBook', windowKeys: ['growthbook', 'GrowthBook'], scriptPattern: /growthbook/i },\n  { sdk: 'Statsig', windowKeys: ['statsig', 'StatsigClient'], scriptPattern: /statsig/i },\n  { sdk: 'Flagsmith', windowKeys: ['flagsmith'], scriptPattern: /flagsmith/i },\n  { sdk: 'Unleash', windowKeys: ['unleash', 'UnleashClient'], scriptPattern: /unleash/i },\n  { sdk: 'VWO', windowKeys: ['vwo', '_vwo'], scriptPattern: /vwo\\.com/i },\n  { sdk: 'Google Optimize', windowKeys: ['gaData', 'google_optimize'], scriptPattern: /googleoptimize/i },\n  { sdk: 'AB Tasty', windowKeys: ['ABTasty'], scriptPattern: /abtasty/i },\n];\n\nexport async function detectFeatureFlags(page: Page, url: string): Promise<FeatureFlagDetectionResult> {\n  const detectedAt = new Date().toISOString();\n\n  const result = await page.evaluate((checks) => {\n    const flagSdks: any[] = [];\n    const scripts = Array.from(document.querySelectorAll('script[src]')).map((s: any) => s.src ?? '');\n\n    for (const check of checks) {\n      let found = false;\n      let confidence = 'low';\n\n      // Check window keys\n      if (check.windowKeys?.some((k: string) => k in (window as any))) {\n        found = true;\n        confidence = 'high';\n      }\n      // Check script URLs\n      if (!found && check.scriptPattern) {\n        const re = new RegExp(check.scriptPattern.source, 'i');\n        const matchingSrc = scripts.find(s => re.test(s));\n        if (matchingSrc) { found = true; confidence = 'medium'; }\n      }\n      if (found) flagSdks.push({ sdk: check.sdk, confidence, details: undefined });\n    }\n\n    // A/B tools\n    const abTools: any[] = [];\n    if ('_vwo_code' in window || '_vis_opt_queue' in (window as any)) abTools.push({ tool: 'VWO', experiments: [] });\n    if ('dataLayer' in window) {\n      const dl = (window as any).dataLayer ?? [];\n      const expEvents = dl.filter((e: any) => e?.event?.includes('experiment') || e?.experimentId);\n      if (expEvents.length > 0) abTools.push({ tool: 'Google Tag Manager Experiments', experiments: expEvents.slice(0, 5).map((e: any) => e.experimentId ?? '') });\n    }\n\n    return { flagSdks, abTools };\n  }, FLAG_SDK_CHECKS as any).catch(() => ({ flagSdks: [], abTools: [] }));\n\n  return {\n    url,\n    flagSdks: result.flagSdks,\n    abTools: result.abTools,\n    hasAnyFlags: result.flagSdks.length > 0 || result.abTools.length > 0,\n    detectedAt,\n  };\n}\n"],
+  "mappings": "AAmCA,MAAM,kBAID;AAAA,EACH,EAAE,KAAK,gBAAgB,YAAY,CAAC,YAAY,gBAAgB,UAAU,GAAG,eAAe,gBAAgB;AAAA,EAC5G,EAAE,KAAK,cAAc,YAAY,CAAC,cAAc,eAAe,GAAG,eAAe,cAAc;AAAA,EAC/F,EAAE,KAAK,YAAY,YAAY,CAAC,WAAW,cAAc,GAAG,eAAe,aAAa;AAAA,EACxF,EAAE,KAAK,cAAc,YAAY,CAAC,cAAc,YAAY,GAAG,eAAe,cAAc;AAAA,EAC5F,EAAE,KAAK,WAAW,YAAY,CAAC,WAAW,eAAe,GAAG,eAAe,WAAW;AAAA,EACtF,EAAE,KAAK,aAAa,YAAY,CAAC,WAAW,GAAG,eAAe,aAAa;AAAA,EAC3E,EAAE,KAAK,WAAW,YAAY,CAAC,WAAW,eAAe,GAAG,eAAe,WAAW;AAAA,EACtF,EAAE,KAAK,OAAO,YAAY,CAAC,OAAO,MAAM,GAAG,eAAe,YAAY;AAAA,EACtE,EAAE,KAAK,mBAAmB,YAAY,CAAC,UAAU,iBAAiB,GAAG,eAAe,kBAAkB;AAAA,EACtG,EAAE,KAAK,YAAY,YAAY,CAAC,SAAS,GAAG,eAAe,WAAW;AACxE;AAEA,eAAsB,mBAAmB,MAAY,KAAkD;AACrG,QAAM,cAAa,oBAAI,KAAK,GAAE,YAAY;AAE1C,QAAM,SAAS,MAAM,KAAK,SAAS,CAAC,WAAW;AAC7C,UAAM,WAAkB,CAAC;AACzB,UAAM,UAAU,MAAM,KAAK,SAAS,iBAAiB,aAAa,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,OAAO,EAAE;AAEhG,eAAW,SAAS,QAAQ;AAC1B,UAAI,QAAQ;AACZ,UAAI,aAAa;AAGjB,UAAI,MAAM,YAAY,KAAK,CAAC,MAAc,KAAM,MAAc,GAAG;AAC/D,gBAAQ;AACR,qBAAa;AAAA,MACf;AAEA,UAAI,CAAC,SAAS,MAAM,eAAe;AACjC,cAAM,KAAK,IAAI,OAAO,MAAM,cAAc,QAAQ,GAAG;AACrD,cAAM,cAAc,QAAQ,KAAK,OAAK,GAAG,KAAK,CAAC,CAAC;AAChD,YAAI,aAAa;AAAE,kBAAQ;AAAM,uBAAa;AAAA,QAAU;AAAA,MAC1D;AACA,UAAI,MAAO,UAAS,KAAK,EAAE,KAAK,MAAM,KAAK,YAAY,SAAS,OAAU,CAAC;AAAA,IAC7E;AAGA,UAAM,UAAiB,CAAC;AACxB,QAAI,eAAe,UAAU,oBAAqB,OAAgB,SAAQ,KAAK,EAAE,MAAM,OAAO,aAAa,CAAC,EAAE,CAAC;AAC/G,QAAI,eAAe,QAAQ;AACzB,YAAM,KAAM,OAAe,aAAa,CAAC;AACzC,YAAM,YAAY,GAAG,OAAO,CAAC,MAAW,GAAG,OAAO,SAAS,YAAY,KAAK,GAAG,YAAY;AAC3F,UAAI,UAAU,SAAS,EAAG,SAAQ,KAAK,EAAE,MAAM,kCAAkC,aAAa,UAAU,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,gBAAgB,EAAE,EAAE,CAAC;AAAA,IAC7J;AAEA,WAAO,EAAE,UAAU,QAAQ;AAAA,EAC7B,GAAG,eAAsB,EAAE,MAAM,OAAO,EAAE,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE,EAAE;AAEtE,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO;AAAA,IAChB,aAAa,OAAO,SAAS,SAAS,KAAK,OAAO,QAAQ,SAAS;AAAA,IACnE;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/form-detector.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Form detector — extract form metadata from crawled pages.
+ *
+ * For each form found on a page, captures: action URL, method, all input fields
+ * with their types/names/required/validation constraints, and submit button text.
+ *
+ * This data feeds ZeTa's test generator to create form validation test cases
+ * automatically (happy path + boundary/error cases per field type).
+ */
+import type { Page } from 'playwright';
+export interface FormField {
+    name: string;
+    type: string;
+    label?: string;
+    required: boolean;
+    placeholder?: string;
+    minLength?: number;
+    maxLength?: number;
+    min?: string;
+    max?: string;
+    pattern?: string;
+    options?: string[];
+    autocomplete?: string;
+}
+export interface DetectedForm {
+    index: number;
+    action?: string;
+    method: string;
+    fields: FormField[];
+    submitText?: string;
+    hasFileUpload: boolean;
+    purpose: 'login' | 'search' | 'registration' | 'contact' | 'checkout' | 'settings' | 'unknown';
+}
+export declare function detectForms(page: Page): Promise<DetectedForm[]>;