@detiq/crawler 1.1.9 → 1.1.10

package/dist/color-contrast.js

@@ -1,90 +1,78 @@
-/**
- * WCAG color contrast checker.
- *
- * Evaluates text contrast ratios against backgrounds for all visible text elements.
- * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)
- * or AAA (7:1 for normal, 4.5:1 for large text) requirements.
- *
- * Uses computedStyle to get actual rendered colors — accounts for CSS variables
- * and inheritance.
- */
-export async function auditColorContrast(page, url) {
-    const auditedAt = new Date().toISOString();
-    const violations = await page.evaluate(() => {
-        function parseRgb(color) {
-            const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
-            return m ? [+m[1], +m[2], +m[3], m[4] !== undefined ? +m[4] : 1] : null;
-        }
-        function luminance([r, g, b]) {
-            const toLinear = (c) => {
-                const n = c / 255;
-                return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
-            };
-            return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
-        }
-        function contrastRatio(fg, bg) {
-            const l1 = luminance(fg);
-            const l2 = luminance(bg);
-            const lighter = Math.max(l1, l2);
-            const darker = Math.min(l1, l2);
-            return Math.round(((lighter + 0.05) / (darker + 0.05)) * 100) / 100;
-        }
-        const violations = [];
-        const elements = Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button'));
-        let checked = 0;
-        for (const el of elements) {
-            if (checked >= 100)
-                break; // limit to avoid perf hit
-            const text = (el.textContent ?? '').trim().slice(0, 60);
-            if (!text)
-                continue;
-            const style = window.getComputedStyle(el);
-            if (style.display === 'none' || style.visibility === 'hidden')
-                continue;
-            const fg = parseRgb(style.color);
-            const bg = parseRgb(style.backgroundColor);
-            if (!fg || !bg || bg[3] === 0)
-                continue; // transparent bg — can't check
-            const fontSize = parseFloat(style.fontSize);
-            const fontWeight = style.fontWeight;
-            const isLargeText = fontSize >= 18 || (fontSize >= 14 && (fontWeight === 'bold' || +fontWeight >= 700));
-            const ratio = contrastRatio(fg, bg);
-            const aaRequired = isLargeText ? 3.0 : 4.5;
-            const aaaRequired = isLargeText ? 4.5 : 7.0;
-            if (ratio < aaRequired) {
-                violations.push({
-                    selector: el.tagName.toLowerCase(),
-                    text: text.slice(0, 50),
-                    foreground: style.color,
-                    background: style.backgroundColor,
-                    ratio,
-                    required: aaRequired,
-                    level: 'AA',
-                    isLargeText,
-                    severity: 'fail',
-                });
-            }
-            else if (ratio < aaaRequired) {
-                violations.push({
-                    selector: el.tagName.toLowerCase(),
-                    text: text.slice(0, 50),
-                    foreground: style.color,
-                    background: style.backgroundColor,
-                    ratio,
-                    required: aaaRequired,
-                    level: 'AAA',
-                    isLargeText,
-                    severity: 'warn',
-                });
-            }
-            else {
-                checked++;
-            }
-        }
-        return violations;
-    }).catch(() => []);
-    const passCount = Math.max(0, 100 - violations.length);
-    const criticals = violations.filter(v => v.severity === 'fail').length;
-    const score = Math.max(0, 100 - criticals * 15 - violations.filter(v => v.severity === 'warn').length * 3);
-    return { url, violations, passCount, score, auditedAt };
+async function auditColorContrast(page, url) {
+  const auditedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const violations = await page.evaluate(() => {
+    function parseRgb(color) {
+      const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
+      return m ? [+m[1], +m[2], +m[3], m[4] !== void 0 ? +m[4] : 1] : null;
+    }
+    function luminance([r, g, b]) {
+      const toLinear = (c) => {
+        const n = c / 255;
+        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
+      };
+      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
+    }
+    function contrastRatio(fg, bg) {
+      const l1 = luminance(fg);
+      const l2 = luminance(bg);
+      const lighter = Math.max(l1, l2);
+      const darker = Math.min(l1, l2);
+      return Math.round((lighter + 0.05) / (darker + 0.05) * 100) / 100;
+    }
+    const violations2 = [];
+    const elements = Array.from(document.querySelectorAll("p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button"));
+    let checked = 0;
+    for (const el of elements) {
+      if (checked >= 100) break;
+      const text = (el.textContent ?? "").trim().slice(0, 60);
+      if (!text) continue;
+      const style = window.getComputedStyle(el);
+      if (style.display === "none" || style.visibility === "hidden") continue;
+      const fg = parseRgb(style.color);
+      const bg = parseRgb(style.backgroundColor);
+      if (!fg || !bg || bg[3] === 0) continue;
+      const fontSize = parseFloat(style.fontSize);
+      const fontWeight = style.fontWeight;
+      const isLargeText = fontSize >= 18 || fontSize >= 14 && (fontWeight === "bold" || +fontWeight >= 700);
+      const ratio = contrastRatio(fg, bg);
+      const aaRequired = isLargeText ? 3 : 4.5;
+      const aaaRequired = isLargeText ? 4.5 : 7;
+      if (ratio < aaRequired) {
+        violations2.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaRequired,
+          level: "AA",
+          isLargeText,
+          severity: "fail"
+        });
+      } else if (ratio < aaaRequired) {
+        violations2.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaaRequired,
+          level: "AAA",
+          isLargeText,
+          severity: "warn"
+        });
+      } else {
+        checked++;
+      }
+    }
+    return violations2;
+  }).catch(() => []);
+  const passCount = Math.max(0, 100 - violations.length);
+  const criticals = violations.filter((v) => v.severity === "fail").length;
+  const score = Math.max(0, 100 - criticals * 15 - violations.filter((v) => v.severity === "warn").length * 3);
+  return { url, violations, passCount, score, auditedAt };
 }
+export {
+  auditColorContrast
+};
+//# sourceMappingURL=color-contrast.js.map

package/dist/content-chunker.js

@@ -1,152 +1,152 @@
-export function estimateTokens(text) {
-    return Math.ceil(text.length / 4);
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
 }
-export function chunkContent(text, opts) {
-    const size = opts.chunkSize ?? 2000;
-    const charSize = size * 4;
-    const overlap = opts.overlap ?? 200;
-    const overlapChars = overlap * 4;
-    switch (opts.strategy) {
-        case 'fixed': return fixedChunk(text, charSize);
-        case 'overlap': return overlapChunk(text, charSize, overlapChars);
-        case 'sentence': return sentenceChunk(text, charSize);
-        case 'paragraph': return paragraphChunk(text, charSize);
-        case 'markdown-heading': return markdownHeadingChunk(text, charSize);
-        case 'semantic': return semanticChunk(text, charSize);
-        case 'recursive': return recursiveChunk(text, charSize, opts.separators ?? ['\n\n', '\n', '. ', ' ']);
-        default: return fixedChunk(text, charSize);
-    }
+function chunkContent(text, opts) {
+  const size = opts.chunkSize ?? 2e3;
+  const charSize = size * 4;
+  const overlap = opts.overlap ?? 200;
+  const overlapChars = overlap * 4;
+  switch (opts.strategy) {
+    case "fixed":
+      return fixedChunk(text, charSize);
+    case "overlap":
+      return overlapChunk(text, charSize, overlapChars);
+    case "sentence":
+      return sentenceChunk(text, charSize);
+    case "paragraph":
+      return paragraphChunk(text, charSize);
+    case "markdown-heading":
+      return markdownHeadingChunk(text, charSize);
+    case "semantic":
+      return semanticChunk(text, charSize);
+    case "recursive":
+      return recursiveChunk(text, charSize, opts.separators ?? ["\n\n", "\n", ". ", " "]);
+    default:
+      return fixedChunk(text, charSize);
+  }
 }
 function makeChunk(text, index, startChar, heading) {
-    return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
+  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
 }
 function fixedChunk(text, charSize) {
-    const chunks = [];
-    let i = 0;
-    while (i < text.length) {
-        const slice = text.slice(i, i + charSize);
-        chunks.push(makeChunk(slice, chunks.length, i));
-        i += charSize;
-    }
-    return chunks;
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize;
+  }
+  return chunks;
 }
 function overlapChunk(text, charSize, overlapChars) {
-    const chunks = [];
-    let i = 0;
-    while (i < text.length) {
-        const slice = text.slice(i, i + charSize);
-        chunks.push(makeChunk(slice, chunks.length, i));
-        i += charSize - overlapChars;
-        if (i <= 0)
-            break;
-    }
-    return chunks;
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize - overlapChars;
+    if (i <= 0) break;
+  }
+  return chunks;
 }
 function sentenceChunk(text, charSize) {
-    const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
-    return groupIntoChunks(sentences, charSize);
+  const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
+  return groupIntoChunks(sentences, charSize);
 }
 function paragraphChunk(text, charSize) {
-    const paras = text.split(/\n\n+/).filter(p => p.trim());
-    return groupIntoChunks(paras, charSize);
+  const paras = text.split(/\n\n+/).filter((p) => p.trim());
+  return groupIntoChunks(paras, charSize);
 }
 function markdownHeadingChunk(text, charSize) {
-    const chunks = [];
-    const headingRe = /^(#{1,3}\s.+)$/m;
-    const sections = text.split(/(?=^#{1,3}\s)/m).filter(s => s.trim());
-    let pos = 0;
-    for (const section of sections) {
-        const headingMatch = section.match(headingRe);
-        const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, '').trim() : undefined;
-        if (section.length <= charSize) {
-            chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
-        }
-        else {
-            // Section too large — sub-chunk by paragraph
-            const sub = paragraphChunk(section, charSize);
-            for (const s of sub) {
-                chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
-            }
-        }
-        pos += section.length;
+  const chunks = [];
+  const headingRe = /^(#{1,3}\s.+)$/m;
+  const sections = text.split(/(?=^#{1,3}\s)/m).filter((s) => s.trim());
+  let pos = 0;
+  for (const section of sections) {
+    const headingMatch = section.match(headingRe);
+    const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, "").trim() : void 0;
+    if (section.length <= charSize) {
+      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
+    } else {
+      const sub = paragraphChunk(section, charSize);
+      for (const s of sub) {
+        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
+      }
     }
-    return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
+    pos += section.length;
+  }
+  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
 }
 function semanticChunk(text, charSize) {
-    // Like paragraph but merges short paragraphs with the next
-    const paras = text.split(/\n\n+/).filter(p => p.trim());
-    const merged = [];
-    let current = '';
-    for (const para of paras) {
-        if (current.length + para.length + 2 <= charSize) {
-            current = current ? current + '\n\n' + para : para;
-        }
-        else {
-            if (current)
-                merged.push(current);
-            current = para;
-        }
+  const paras = text.split(/\n\n+/).filter((p) => p.trim());
+  const merged = [];
+  let current = "";
+  for (const para of paras) {
+    if (current.length + para.length + 2 <= charSize) {
+      current = current ? current + "\n\n" + para : para;
+    } else {
+      if (current) merged.push(current);
+      current = para;
     }
-    if (current)
-        merged.push(current);
-    return groupIntoChunks(merged, charSize);
+  }
+  if (current) merged.push(current);
+  return groupIntoChunks(merged, charSize);
 }
 function recursiveChunk(text, charSize, separators) {
-    if (text.length <= charSize)
-        return [makeChunk(text, 0, 0)];
-    const [sep, ...rest] = separators;
-    if (!sep)
-        return fixedChunk(text, charSize);
-    const parts = text.split(sep).filter(p => p);
-    const chunks = [];
-    let pos = 0;
-    for (const part of parts) {
-        if (part.length <= charSize) {
-            chunks.push(makeChunk(part, chunks.length, pos));
-        }
-        else {
-            const sub = recursiveChunk(part, charSize, rest);
-            for (const s of sub)
-                chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
-        }
-        pos += part.length + sep.length;
+  if (text.length <= charSize) return [makeChunk(text, 0, 0)];
+  const [sep, ...rest] = separators;
+  if (!sep) return fixedChunk(text, charSize);
+  const parts = text.split(sep).filter((p) => p);
+  const chunks = [];
+  let pos = 0;
+  for (const part of parts) {
+    if (part.length <= charSize) {
+      chunks.push(makeChunk(part, chunks.length, pos));
+    } else {
+      const sub = recursiveChunk(part, charSize, rest);
+      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
     }
-    return chunks;
+    pos += part.length + sep.length;
+  }
+  return chunks;
 }
 function groupIntoChunks(parts, charSize) {
-    const chunks = [];
-    let current = '';
-    let startChar = 0;
-    let pos = 0;
-    for (const part of parts) {
-        if (current.length + part.length + 1 > charSize && current) {
-            chunks.push(makeChunk(current.trim(), chunks.length, startChar));
-            startChar = pos;
-            current = part;
-        }
-        else {
-            current = current ? current + ' ' + part : part;
-        }
-        pos += part.length + 1;
+  const chunks = [];
+  let current = "";
+  let startChar = 0;
+  let pos = 0;
+  for (const part of parts) {
+    if (current.length + part.length + 1 > charSize && current) {
+      chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+      startChar = pos;
+      current = part;
+    } else {
+      current = current ? current + " " + part : part;
     }
-    if (current.trim())
-        chunks.push(makeChunk(current.trim(), chunks.length, startChar));
-    return chunks;
+    pos += part.length + 1;
+  }
+  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+  return chunks;
 }
-export function mergeSmallChunks(chunks, minTokens) {
-    const minChars = minTokens * 4;
-    const result = [];
-    let i = 0;
-    while (i < chunks.length) {
-        if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
-            const merged = chunks[i].text + '\n\n' + chunks[i + 1].text;
-            result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
-            i += 2;
-        }
-        else {
-            result.push({ ...chunks[i], index: result.length });
-            i++;
-        }
+function mergeSmallChunks(chunks, minTokens) {
+  const minChars = minTokens * 4;
+  const result = [];
+  let i = 0;
+  while (i < chunks.length) {
+    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
+      const merged = chunks[i].text + "\n\n" + chunks[i + 1].text;
+      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
+      i += 2;
+    } else {
+      result.push({ ...chunks[i], index: result.length });
+      i++;
     }
-    return result;
+  }
+  return result;
 }
+export {
+  chunkContent,
+  estimateTokens,
+  mergeSmallChunks
+};
+//# sourceMappingURL=content-chunker.js.map

package/dist/content-dedup.js

@@ -1,77 +1,60 @@
-/**
- * Content deduplication — detect near-duplicate pages during crawl.
- *
- * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
- * we use 8 structural fingerprints extracted from the markdown to detect pages that
- * share the same layout/template even if the data differs.
- *
- * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
- * These waste crawl budget and generate redundant test cases.
- */
-/** Extract structural fingerprints from markdown — layout-sensitive, data-insensitive */
 function extractFingerprints(markdown) {
-    const lines = markdown.split('\n').filter(l => l.trim().length > 0);
-    const fps = [];
-    // FP1: heading structure (H1-H6 count distribution)
-    const headings = lines.filter(l => /^#{1,6} /.test(l));
-    const hCounts = [0, 0, 0, 0, 0, 0];
-    for (const h of headings) {
-        const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
-        hCounts[lvl - 1]++;
-    }
-    fps.push('h:' + hCounts.join(','));
-    // FP2: link count (grouped into buckets)
-    const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
-    fps.push('lk:' + Math.floor(linkCount / 5));
-    // FP3: list item count (grouped)
-    const listCount = lines.filter(l => /^[\-\*\+] |^\d+\. /.test(l)).length;
-    fps.push('li:' + Math.floor(listCount / 3));
-    // FP4: code block presence
-    fps.push('code:' + (markdown.includes('```') ? '1' : '0'));
-    // FP5: table presence
-    fps.push('tbl:' + (markdown.includes('|') ? '1' : '0'));
-    // FP6: approximate paragraph count
-    const paraCount = markdown.split(/\n{2,}/).filter(p => p.trim().length > 30).length;
-    fps.push('p:' + Math.floor(paraCount / 2));
-    // FP7: first heading text (normalized)
-    const firstH = headings[0]?.replace(/^#{1,6} /, '').toLowerCase().replace(/[^a-z0-9]/g, '') ?? '';
-    fps.push('fh:' + firstH.slice(0, 20));
-    // FP8: word count bucket
-    const wordCount = markdown.split(/\s+/).length;
-    fps.push('wc:' + Math.floor(wordCount / 50));
-    return fps;
+  const lines = markdown.split("\n").filter((l) => l.trim().length > 0);
+  const fps = [];
+  const headings = lines.filter((l) => /^#{1,6} /.test(l));
+  const hCounts = [0, 0, 0, 0, 0, 0];
+  for (const h of headings) {
+    const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
+    hCounts[lvl - 1]++;
+  }
+  fps.push("h:" + hCounts.join(","));
+  const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
+  fps.push("lk:" + Math.floor(linkCount / 5));
+  const listCount = lines.filter((l) => /^[\-\*\+] |^\d+\. /.test(l)).length;
+  fps.push("li:" + Math.floor(listCount / 3));
+  fps.push("code:" + (markdown.includes("```") ? "1" : "0"));
+  fps.push("tbl:" + (markdown.includes("|") ? "1" : "0"));
+  const paraCount = markdown.split(/\n{2,}/).filter((p) => p.trim().length > 30).length;
+  fps.push("p:" + Math.floor(paraCount / 2));
+  const firstH = headings[0]?.replace(/^#{1,6} /, "").toLowerCase().replace(/[^a-z0-9]/g, "") ?? "";
+  fps.push("fh:" + firstH.slice(0, 20));
+  const wordCount = markdown.split(/\s+/).length;
+  fps.push("wc:" + Math.floor(wordCount / 50));
+  return fps;
 }
 function similarity(a, b) {
-    if (a.length !== b.length)
-        return 0;
-    let matches = 0;
-    for (let i = 0; i < a.length; i++) {
-        if (a[i] === b[i])
-            matches++;
-    }
-    return matches / a.length;
+  if (a.length !== b.length) return 0;
+  let matches = 0;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] === b[i]) matches++;
+  }
+  return matches / a.length;
 }
-export class ContentDeduplicator {
-    seen = new Map();
-    threshold;
-    constructor(threshold = 0.85) {
-        this.threshold = threshold;
-    }
-    check(url, markdown) {
-        const fps = extractFingerprints(markdown);
-        for (const [key, entry] of this.seen) {
-            const score = similarity(fps, entry.fps);
-            if (score >= this.threshold) {
-                return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
-            }
-        }
-        this.seen.set(url, { fps, url });
-        return { isDuplicate: false, similarityScore: 1.0 };
-    }
-    size() {
-        return this.seen.size;
-    }
-    reset() {
-        this.seen.clear();
+class ContentDeduplicator {
+  seen = /* @__PURE__ */ new Map();
+  threshold;
+  constructor(threshold = 0.85) {
+    this.threshold = threshold;
+  }
+  check(url, markdown) {
+    const fps = extractFingerprints(markdown);
+    for (const [key, entry] of this.seen) {
+      const score = similarity(fps, entry.fps);
+      if (score >= this.threshold) {
+        return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
+      }
     }
+    this.seen.set(url, { fps, url });
+    return { isDuplicate: false, similarityScore: 1 };
+  }
+  size() {
+    return this.seen.size;
+  }
+  reset() {
+    this.seen.clear();
+  }
 }
+export {
+  ContentDeduplicator
+};
+//# sourceMappingURL=content-dedup.js.map