@detiq/crawler 1.1.8 → 1.1.9

package/dist/color-contrast.js

@@ -1,78 +1,90 @@
-async function auditColorContrast(page, url) {
-  const auditedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const violations = await page.evaluate(() => {
-    function parseRgb(color) {
-      const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
-      return m ? [+m[1], +m[2], +m[3], m[4] !== void 0 ? +m[4] : 1] : null;
-    }
-    function luminance([r, g, b]) {
-      const toLinear = (c) => {
-        const n = c / 255;
-        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
-      };
-      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
-    }
-    function contrastRatio(fg, bg) {
-      const l1 = luminance(fg);
-      const l2 = luminance(bg);
-      const lighter = Math.max(l1, l2);
-      const darker = Math.min(l1, l2);
-      return Math.round((lighter + 0.05) / (darker + 0.05) * 100) / 100;
-    }
-    const violations2 = [];
-    const elements = Array.from(document.querySelectorAll("p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button"));
-    let checked = 0;
-    for (const el of elements) {
-      if (checked >= 100) break;
-      const text = (el.textContent ?? "").trim().slice(0, 60);
-      if (!text) continue;
-      const style = window.getComputedStyle(el);
-      if (style.display === "none" || style.visibility === "hidden") continue;
-      const fg = parseRgb(style.color);
-      const bg = parseRgb(style.backgroundColor);
-      if (!fg || !bg || bg[3] === 0) continue;
-      const fontSize = parseFloat(style.fontSize);
-      const fontWeight = style.fontWeight;
-      const isLargeText = fontSize >= 18 || fontSize >= 14 && (fontWeight === "bold" || +fontWeight >= 700);
-      const ratio = contrastRatio(fg, bg);
-      const aaRequired = isLargeText ? 3 : 4.5;
-      const aaaRequired = isLargeText ? 4.5 : 7;
-      if (ratio < aaRequired) {
-        violations2.push({
-          selector: el.tagName.toLowerCase(),
-          text: text.slice(0, 50),
-          foreground: style.color,
-          background: style.backgroundColor,
-          ratio,
-          required: aaRequired,
-          level: "AA",
-          isLargeText,
-          severity: "fail"
-        });
-      } else if (ratio < aaaRequired) {
-        violations2.push({
-          selector: el.tagName.toLowerCase(),
-          text: text.slice(0, 50),
-          foreground: style.color,
-          background: style.backgroundColor,
-          ratio,
-          required: aaaRequired,
-          level: "AAA",
-          isLargeText,
-          severity: "warn"
-        });
-      } else {
-        checked++;
-      }
-    }
-    return violations2;
-  }).catch(() => []);
-  const passCount = Math.max(0, 100 - violations.length);
-  const criticals = violations.filter((v) => v.severity === "fail").length;
-  const score = Math.max(0, 100 - criticals * 15 - violations.filter((v) => v.severity === "warn").length * 3);
-  return { url, violations, passCount, score, auditedAt };
+/**
+ * WCAG color contrast checker.
+ *
+ * Evaluates text contrast ratios against backgrounds for all visible text elements.
+ * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)
+ * or AAA (7:1 for normal, 4.5:1 for large text) requirements.
+ *
+ * Uses computedStyle to get actual rendered colors — accounts for CSS variables
+ * and inheritance.
+ */
+export async function auditColorContrast(page, url) {
+    const auditedAt = new Date().toISOString();
+    const violations = await page.evaluate(() => {
+        function parseRgb(color) {
+            const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
+            return m ? [+m[1], +m[2], +m[3], m[4] !== undefined ? +m[4] : 1] : null;
+        }
+        function luminance([r, g, b]) {
+            const toLinear = (c) => {
+                const n = c / 255;
+                return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
+            };
+            return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
+        }
+        function contrastRatio(fg, bg) {
+            const l1 = luminance(fg);
+            const l2 = luminance(bg);
+            const lighter = Math.max(l1, l2);
+            const darker = Math.min(l1, l2);
+            return Math.round(((lighter + 0.05) / (darker + 0.05)) * 100) / 100;
+        }
+        const violations = [];
+        const elements = Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button'));
+        let checked = 0;
+        for (const el of elements) {
+            if (checked >= 100)
+                break; // limit to avoid perf hit
+            const text = (el.textContent ?? '').trim().slice(0, 60);
+            if (!text)
+                continue;
+            const style = window.getComputedStyle(el);
+            if (style.display === 'none' || style.visibility === 'hidden')
+                continue;
+            const fg = parseRgb(style.color);
+            const bg = parseRgb(style.backgroundColor);
+            if (!fg || !bg || bg[3] === 0)
+                continue; // transparent bg — can't check
+            const fontSize = parseFloat(style.fontSize);
+            const fontWeight = style.fontWeight;
+            const isLargeText = fontSize >= 18 || (fontSize >= 14 && (fontWeight === 'bold' || +fontWeight >= 700));
+            const ratio = contrastRatio(fg, bg);
+            const aaRequired = isLargeText ? 3.0 : 4.5;
+            const aaaRequired = isLargeText ? 4.5 : 7.0;
+            if (ratio < aaRequired) {
+                violations.push({
+                    selector: el.tagName.toLowerCase(),
+                    text: text.slice(0, 50),
+                    foreground: style.color,
+                    background: style.backgroundColor,
+                    ratio,
+                    required: aaRequired,
+                    level: 'AA',
+                    isLargeText,
+                    severity: 'fail',
+                });
+            }
+            else if (ratio < aaaRequired) {
+                violations.push({
+                    selector: el.tagName.toLowerCase(),
+                    text: text.slice(0, 50),
+                    foreground: style.color,
+                    background: style.backgroundColor,
+                    ratio,
+                    required: aaaRequired,
+                    level: 'AAA',
+                    isLargeText,
+                    severity: 'warn',
+                });
+            }
+            else {
+                checked++;
+            }
+        }
+        return violations;
+    }).catch(() => []);
+    const passCount = Math.max(0, 100 - violations.length);
+    const criticals = violations.filter(v => v.severity === 'fail').length;
+    const score = Math.max(0, 100 - criticals * 15 - violations.filter(v => v.severity === 'warn').length * 3);
+    return { url, violations, passCount, score, auditedAt };
 }
-export {
-  auditColorContrast
-};
-//# sourceMappingURL=color-contrast.js.map

package/dist/content-chunker.js

@@ -1,152 +1,152 @@
-function estimateTokens(text) {
-  return Math.ceil(text.length / 4);
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
 }
-function chunkContent(text, opts) {
-  const size = opts.chunkSize ?? 2e3;
-  const charSize = size * 4;
-  const overlap = opts.overlap ?? 200;
-  const overlapChars = overlap * 4;
-  switch (opts.strategy) {
-    case "fixed":
-      return fixedChunk(text, charSize);
-    case "overlap":
-      return overlapChunk(text, charSize, overlapChars);
-    case "sentence":
-      return sentenceChunk(text, charSize);
-    case "paragraph":
-      return paragraphChunk(text, charSize);
-    case "markdown-heading":
-      return markdownHeadingChunk(text, charSize);
-    case "semantic":
-      return semanticChunk(text, charSize);
-    case "recursive":
-      return recursiveChunk(text, charSize, opts.separators ?? ["\n\n", "\n", ". ", " "]);
-    default:
-      return fixedChunk(text, charSize);
-  }
+export function chunkContent(text, opts) {
+    const size = opts.chunkSize ?? 2000;
+    const charSize = size * 4;
+    const overlap = opts.overlap ?? 200;
+    const overlapChars = overlap * 4;
+    switch (opts.strategy) {
+        case 'fixed': return fixedChunk(text, charSize);
+        case 'overlap': return overlapChunk(text, charSize, overlapChars);
+        case 'sentence': return sentenceChunk(text, charSize);
+        case 'paragraph': return paragraphChunk(text, charSize);
+        case 'markdown-heading': return markdownHeadingChunk(text, charSize);
+        case 'semantic': return semanticChunk(text, charSize);
+        case 'recursive': return recursiveChunk(text, charSize, opts.separators ?? ['\n\n', '\n', '. ', ' ']);
+        default: return fixedChunk(text, charSize);
+    }
 }
 function makeChunk(text, index, startChar, heading) {
-  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
+    return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
 }
 function fixedChunk(text, charSize) {
-  const chunks = [];
-  let i = 0;
-  while (i < text.length) {
-    const slice = text.slice(i, i + charSize);
-    chunks.push(makeChunk(slice, chunks.length, i));
-    i += charSize;
-  }
-  return chunks;
+    const chunks = [];
+    let i = 0;
+    while (i < text.length) {
+        const slice = text.slice(i, i + charSize);
+        chunks.push(makeChunk(slice, chunks.length, i));
+        i += charSize;
+    }
+    return chunks;
 }
 function overlapChunk(text, charSize, overlapChars) {
-  const chunks = [];
-  let i = 0;
-  while (i < text.length) {
-    const slice = text.slice(i, i + charSize);
-    chunks.push(makeChunk(slice, chunks.length, i));
-    i += charSize - overlapChars;
-    if (i <= 0) break;
-  }
-  return chunks;
+    const chunks = [];
+    let i = 0;
+    while (i < text.length) {
+        const slice = text.slice(i, i + charSize);
+        chunks.push(makeChunk(slice, chunks.length, i));
+        i += charSize - overlapChars;
+        if (i <= 0)
+            break;
+    }
+    return chunks;
 }
 function sentenceChunk(text, charSize) {
-  const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
-  return groupIntoChunks(sentences, charSize);
+    const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
+    return groupIntoChunks(sentences, charSize);
 }
 function paragraphChunk(text, charSize) {
-  const paras = text.split(/\n\n+/).filter((p) => p.trim());
-  return groupIntoChunks(paras, charSize);
+    const paras = text.split(/\n\n+/).filter(p => p.trim());
+    return groupIntoChunks(paras, charSize);
 }
 function markdownHeadingChunk(text, charSize) {
-  const chunks = [];
-  const headingRe = /^(#{1,3}\s.+)$/m;
-  const sections = text.split(/(?=^#{1,3}\s)/m).filter((s) => s.trim());
-  let pos = 0;
-  for (const section of sections) {
-    const headingMatch = section.match(headingRe);
-    const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, "").trim() : void 0;
-    if (section.length <= charSize) {
-      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
-    } else {
-      const sub = paragraphChunk(section, charSize);
-      for (const s of sub) {
-        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
-      }
+    const chunks = [];
+    const headingRe = /^(#{1,3}\s.+)$/m;
+    const sections = text.split(/(?=^#{1,3}\s)/m).filter(s => s.trim());
+    let pos = 0;
+    for (const section of sections) {
+        const headingMatch = section.match(headingRe);
+        const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, '').trim() : undefined;
+        if (section.length <= charSize) {
+            chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
+        }
+        else {
+            // Section too large — sub-chunk by paragraph
+            const sub = paragraphChunk(section, charSize);
+            for (const s of sub) {
+                chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
+            }
+        }
+        pos += section.length;
     }
-    pos += section.length;
-  }
-  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
+    return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
 }
 function semanticChunk(text, charSize) {
-  const paras = text.split(/\n\n+/).filter((p) => p.trim());
-  const merged = [];
-  let current = "";
-  for (const para of paras) {
-    if (current.length + para.length + 2 <= charSize) {
-      current = current ? current + "\n\n" + para : para;
-    } else {
-      if (current) merged.push(current);
-      current = para;
+    // Like paragraph but merges short paragraphs with the next
+    const paras = text.split(/\n\n+/).filter(p => p.trim());
+    const merged = [];
+    let current = '';
+    for (const para of paras) {
+        if (current.length + para.length + 2 <= charSize) {
+            current = current ? current + '\n\n' + para : para;
+        }
+        else {
+            if (current)
+                merged.push(current);
+            current = para;
+        }
     }
-  }
-  if (current) merged.push(current);
-  return groupIntoChunks(merged, charSize);
+    if (current)
+        merged.push(current);
+    return groupIntoChunks(merged, charSize);
 }
 function recursiveChunk(text, charSize, separators) {
-  if (text.length <= charSize) return [makeChunk(text, 0, 0)];
-  const [sep, ...rest] = separators;
-  if (!sep) return fixedChunk(text, charSize);
-  const parts = text.split(sep).filter((p) => p);
-  const chunks = [];
-  let pos = 0;
-  for (const part of parts) {
-    if (part.length <= charSize) {
-      chunks.push(makeChunk(part, chunks.length, pos));
-    } else {
-      const sub = recursiveChunk(part, charSize, rest);
-      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
+    if (text.length <= charSize)
+        return [makeChunk(text, 0, 0)];
+    const [sep, ...rest] = separators;
+    if (!sep)
+        return fixedChunk(text, charSize);
+    const parts = text.split(sep).filter(p => p);
+    const chunks = [];
+    let pos = 0;
+    for (const part of parts) {
+        if (part.length <= charSize) {
+            chunks.push(makeChunk(part, chunks.length, pos));
+        }
+        else {
+            const sub = recursiveChunk(part, charSize, rest);
+            for (const s of sub)
+                chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
+        }
+        pos += part.length + sep.length;
     }
-    pos += part.length + sep.length;
-  }
-  return chunks;
+    return chunks;
 }
 function groupIntoChunks(parts, charSize) {
-  const chunks = [];
-  let current = "";
-  let startChar = 0;
-  let pos = 0;
-  for (const part of parts) {
-    if (current.length + part.length + 1 > charSize && current) {
-      chunks.push(makeChunk(current.trim(), chunks.length, startChar));
-      startChar = pos;
-      current = part;
-    } else {
-      current = current ? current + " " + part : part;
+    const chunks = [];
+    let current = '';
+    let startChar = 0;
+    let pos = 0;
+    for (const part of parts) {
+        if (current.length + part.length + 1 > charSize && current) {
+            chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+            startChar = pos;
+            current = part;
+        }
+        else {
+            current = current ? current + ' ' + part : part;
+        }
+        pos += part.length + 1;
     }
-    pos += part.length + 1;
-  }
-  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));
-  return chunks;
+    if (current.trim())
+        chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+    return chunks;
 }
-function mergeSmallChunks(chunks, minTokens) {
-  const minChars = minTokens * 4;
-  const result = [];
-  let i = 0;
-  while (i < chunks.length) {
-    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
-      const merged = chunks[i].text + "\n\n" + chunks[i + 1].text;
-      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
-      i += 2;
-    } else {
-      result.push({ ...chunks[i], index: result.length });
-      i++;
+export function mergeSmallChunks(chunks, minTokens) {
+    const minChars = minTokens * 4;
+    const result = [];
+    let i = 0;
+    while (i < chunks.length) {
+        if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
+            const merged = chunks[i].text + '\n\n' + chunks[i + 1].text;
+            result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
+            i += 2;
+        }
+        else {
+            result.push({ ...chunks[i], index: result.length });
+            i++;
+        }
     }
-  }
-  return result;
+    return result;
 }
-export {
-  chunkContent,
-  estimateTokens,
-  mergeSmallChunks
-};
-//# sourceMappingURL=content-chunker.js.map

package/dist/content-dedup.js

@@ -1,60 +1,77 @@
+/**
+ * Content deduplication — detect near-duplicate pages during crawl.
+ *
+ * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
+ * we use 8 structural fingerprints extracted from the markdown to detect pages that
+ * share the same layout/template even if the data differs.
+ *
+ * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
+ * These waste crawl budget and generate redundant test cases.
+ */
+/** Extract structural fingerprints from markdown — layout-sensitive, data-insensitive */
 function extractFingerprints(markdown) {
-  const lines = markdown.split("\n").filter((l) => l.trim().length > 0);
-  const fps = [];
-  const headings = lines.filter((l) => /^#{1,6} /.test(l));
-  const hCounts = [0, 0, 0, 0, 0, 0];
-  for (const h of headings) {
-    const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
-    hCounts[lvl - 1]++;
-  }
-  fps.push("h:" + hCounts.join(","));
-  const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
-  fps.push("lk:" + Math.floor(linkCount / 5));
-  const listCount = lines.filter((l) => /^[\-\*\+] |^\d+\. /.test(l)).length;
-  fps.push("li:" + Math.floor(listCount / 3));
-  fps.push("code:" + (markdown.includes("```") ? "1" : "0"));
-  fps.push("tbl:" + (markdown.includes("|") ? "1" : "0"));
-  const paraCount = markdown.split(/\n{2,}/).filter((p) => p.trim().length > 30).length;
-  fps.push("p:" + Math.floor(paraCount / 2));
-  const firstH = headings[0]?.replace(/^#{1,6} /, "").toLowerCase().replace(/[^a-z0-9]/g, "") ?? "";
-  fps.push("fh:" + firstH.slice(0, 20));
-  const wordCount = markdown.split(/\s+/).length;
-  fps.push("wc:" + Math.floor(wordCount / 50));
-  return fps;
+    const lines = markdown.split('\n').filter(l => l.trim().length > 0);
+    const fps = [];
+    // FP1: heading structure (H1-H6 count distribution)
+    const headings = lines.filter(l => /^#{1,6} /.test(l));
+    const hCounts = [0, 0, 0, 0, 0, 0];
+    for (const h of headings) {
+        const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
+        hCounts[lvl - 1]++;
+    }
+    fps.push('h:' + hCounts.join(','));
+    // FP2: link count (grouped into buckets)
+    const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
+    fps.push('lk:' + Math.floor(linkCount / 5));
+    // FP3: list item count (grouped)
+    const listCount = lines.filter(l => /^[\-\*\+] |^\d+\. /.test(l)).length;
+    fps.push('li:' + Math.floor(listCount / 3));
+    // FP4: code block presence
+    fps.push('code:' + (markdown.includes('```') ? '1' : '0'));
+    // FP5: table presence
+    fps.push('tbl:' + (markdown.includes('|') ? '1' : '0'));
+    // FP6: approximate paragraph count
+    const paraCount = markdown.split(/\n{2,}/).filter(p => p.trim().length > 30).length;
+    fps.push('p:' + Math.floor(paraCount / 2));
+    // FP7: first heading text (normalized)
+    const firstH = headings[0]?.replace(/^#{1,6} /, '').toLowerCase().replace(/[^a-z0-9]/g, '') ?? '';
+    fps.push('fh:' + firstH.slice(0, 20));
+    // FP8: word count bucket
+    const wordCount = markdown.split(/\s+/).length;
+    fps.push('wc:' + Math.floor(wordCount / 50));
+    return fps;
 }
 function similarity(a, b) {
-  if (a.length !== b.length) return 0;
-  let matches = 0;
-  for (let i = 0; i < a.length; i++) {
-    if (a[i] === b[i]) matches++;
-  }
-  return matches / a.length;
+    if (a.length !== b.length)
+        return 0;
+    let matches = 0;
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] === b[i])
+            matches++;
+    }
+    return matches / a.length;
 }
-class ContentDeduplicator {
-  seen = /* @__PURE__ */ new Map();
-  threshold;
-  constructor(threshold = 0.85) {
-    this.threshold = threshold;
-  }
-  check(url, markdown) {
-    const fps = extractFingerprints(markdown);
-    for (const [key, entry] of this.seen) {
-      const score = similarity(fps, entry.fps);
-      if (score >= this.threshold) {
-        return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
-      }
+export class ContentDeduplicator {
+    seen = new Map();
+    threshold;
+    constructor(threshold = 0.85) {
+        this.threshold = threshold;
+    }
+    check(url, markdown) {
+        const fps = extractFingerprints(markdown);
+        for (const [key, entry] of this.seen) {
+            const score = similarity(fps, entry.fps);
+            if (score >= this.threshold) {
+                return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
+            }
+        }
+        this.seen.set(url, { fps, url });
+        return { isDuplicate: false, similarityScore: 1.0 };
+    }
+    size() {
+        return this.seen.size;
+    }
+    reset() {
+        this.seen.clear();
     }
-    this.seen.set(url, { fps, url });
-    return { isDuplicate: false, similarityScore: 1 };
-  }
-  size() {
-    return this.seen.size;
-  }
-  reset() {
-    this.seen.clear();
-  }
 }
-export {
-  ContentDeduplicator
-};
-//# sourceMappingURL=content-dedup.js.map