@detiq/crawler 1.1.9 → 1.1.11

package/dist/incremental-diff.js

@@ -1,94 +1,61 @@
-/**
- * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
- *
- * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
- * On the next crawl of the same project, pages with the same fingerprint are
- * marked SKIPPED instead of re-processed, drastically reducing crawl time and
- * LLM token spend for sites with mostly-static content.
- *
- * Design: "same content hash" means the meaningful text hasn't changed.
- * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
- */
-/**
- * Compute a stable 64-bit-ish hash from text content.
- * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
- */
-export function hashContent(text) {
-    let h1 = 5381;
-    let h2 = 52711;
-    for (let i = 0; i < text.length; i++) {
-        const c = text.charCodeAt(i);
-        h1 = (Math.imul(h1, 33) ^ c) >>> 0;
-        h2 = (Math.imul(h2, 31) ^ c) >>> 0;
-    }
-    return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
+function hashContent(text) {
+  let h1 = 5381;
+  let h2 = 52711;
+  for (let i = 0; i < text.length; i++) {
+    const c = text.charCodeAt(i);
+    h1 = (Math.imul(h1, 33) ^ c) >>> 0;
+    h2 = (Math.imul(h2, 31) ^ c) >>> 0;
+  }
+  return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
 }
-/**
- * Normalize text before hashing — strips whitespace-only lines, trims each line,
- * collapses runs of blank lines. Makes hash stable across minor formatting changes.
- */
-export function normalizeText(text) {
-    return text
-        .split('\n')
-        .map(l => l.trim())
-        .filter(l => l.length > 0)
-        .join('\n');
+function normalizeText(text) {
+  return text.split("\n").map((l) => l.trim()).filter((l) => l.length > 0).join("\n");
 }
-export function computeContentHash(url, markdownText) {
-    const normalized = normalizeText(markdownText);
-    return {
-        url,
-        hash: hashContent(normalized),
-        textLength: normalized.length,
-        computedAt: new Date().toISOString(),
-    };
+function computeContentHash(url, markdownText) {
+  const normalized = normalizeText(markdownText);
+  return {
+    url,
+    hash: hashContent(normalized),
+    textLength: normalized.length,
+    computedAt: (/* @__PURE__ */ new Date()).toISOString()
+  };
 }
-/**
- * Compare a page's current content against prior hashes.
- * Returns { changed: false } for unchanged pages — caller should skip re-processing.
- */
-export function diffContent(url, currentMarkdown, priorHashes) {
-    const current = computeContentHash(url, currentMarkdown);
-    const prior = priorHashes.get(url);
-    if (!prior) {
-        return { url, changed: true, currentHash: current.hash, reason: 'new' };
-    }
-    if (prior === current.hash) {
-        return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };
-    }
-    return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };
+function diffContent(url, currentMarkdown, priorHashes) {
+  const current = computeContentHash(url, currentMarkdown);
+  const prior = priorHashes.get(url);
+  if (!prior) {
+    return { url, changed: true, currentHash: current.hash, reason: "new" };
+  }
+  if (prior === current.hash) {
+    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: "unchanged" };
+  }
+  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: "changed" };
 }
-/**
- * Build a hash map from prior screen records (from DB query result).
- * screenRecords: Array<{ url: string; elementHash?: string | null }>
- */
-export function buildPriorHashMap(screenRecords) {
-    const map = new Map();
-    for (const s of screenRecords) {
-        if (s.url && s.elementHash)
-            map.set(s.url, s.elementHash);
-    }
-    return map;
+function buildPriorHashMap(screenRecords) {
+  const map = /* @__PURE__ */ new Map();
+  for (const s of screenRecords) {
+    if (s.url && s.elementHash) map.set(s.url, s.elementHash);
+  }
+  return map;
 }
-/**
- * Filter a batch of discovered URLs against prior hashes.
- * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
- *
- * Note: filtering at URL level only works if the prior crawl already visited the URL.
- * For new URLs, they're always in toProcess.
- */
-export function partitionByChange(urls, priorHashes) {
-    const toProcess = [];
-    const toSkip = [];
-    for (const url of urls) {
-        if (priorHashes.has(url)) {
-            // Can't know if changed without fetching — still needs fetch, but hash after fetch
-            // will determine skip. Include all — hash check happens post-fetch.
-            toProcess.push(url);
-        }
-        else {
-            toProcess.push(url);
-        }
+function partitionByChange(urls, priorHashes) {
+  const toProcess = [];
+  const toSkip = [];
+  for (const url of urls) {
+    if (priorHashes.has(url)) {
+      toProcess.push(url);
+    } else {
+      toProcess.push(url);
     }
-    return { toProcess, toSkip };
+  }
+  return { toProcess, toSkip };
 }
+export {
+  buildPriorHashMap,
+  computeContentHash,
+  diffContent,
+  hashContent,
+  normalizeText,
+  partitionByChange
+};
+//# sourceMappingURL=incremental-diff.js.map

package/dist/index.js

@@ -1,28 +1,67 @@
-export * from './crawler.js';
-export * from './visual-diff-verdict.js';
-export * from './auth.js';
-export * from './screen-reader.js';
-export * from './job-queue.js';
-export { startSqsWorker } from './sqs-queue.js';
-export * from './session-capture.js';
-export * from './session-export.js';
-export * from './test-executor.js';
-export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
-export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
-export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
-export * from './engine-router.js';
-export * from './profiles/domain-profile.js';
-export * from './policies/crawl-safety.js';
-export * from './extractors/artifact-envelope.js';
-export * from './network-observer.js';
-export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
-export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
-export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
-export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
-export { httpCrawl } from './http-engine.js';
-export { extractMarkdown } from './markdown-extract.js';
-export { TieredProxyManager } from './proxy.js';
-export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
-export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
-export { analyzeFailure } from './failure-analysis-agent.js';
-export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';
+export * from "./crawler.js";
+export * from "./visual-diff-verdict.js";
+export * from "./auth.js";
+export * from "./screen-reader.js";
+export * from "./job-queue.js";
+import { startSqsWorker } from "./sqs-queue.js";
+export * from "./session-capture.js";
+export * from "./session-export.js";
+export * from "./test-executor.js";
+import { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from "./live-view.js";
+import { waitForEmailOTP, getInboxEmail } from "./email-otp.js";
+import { getSsoToken, getSsoClientCredentialsToken } from "./sso-token.js";
+export * from "./engine-router.js";
+export * from "./profiles/domain-profile.js";
+export * from "./policies/crawl-safety.js";
+export * from "./extractors/artifact-envelope.js";
+export * from "./network-observer.js";
+import { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from "./tls-fingerprint.js";
+import { prepareExtensionProfile, adBlockerProfile } from "./browser-extension.js";
+import { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from "./redis-queue.js";
+import { discoverProjectUrls, discoverSitemapUrls } from "./sitemap.js";
+import { httpCrawl } from "./http-engine.js";
+import { extractMarkdown } from "./markdown-extract.js";
+import { TieredProxyManager } from "./proxy.js";
+import { RedisCrawlFrontier, isRedisFrontierEnabled } from "./redis-frontier.js";
+import { startSpan, endSpan, withSpan, increment, flushMetrics } from "./telemetry.js";
+import { analyzeFailure } from "./failure-analysis-agent.js";
+import { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from "./locator-extractor.js";
+export {
+  CHROME_TLS_HEADERS,
+  RedisCrawlFrontier,
+  TieredProxyManager,
+  adBlockerProfile,
+  analyzeFailure,
+  chromeTlsAgent,
+  closeLiveView,
+  completeLiveView,
+  createChromeTlsAgent,
+  createRedisQueue,
+  discoverProjectUrls,
+  discoverSitemapUrls,
+  elementToPlaywrightLocator,
+  elementsToPlaywrightLocators,
+  endSpan,
+  extractMarkdown,
+  flushMetrics,
+  formatLocatorsForPrompt,
+  getFrame,
+  getInboxEmail,
+  getSsoClientCredentialsToken,
+  getSsoToken,
+  httpCrawl,
+  increment,
+  isRedisFrontierEnabled,
+  isRedisQueueEnabled,
+  liveViewNavigate,
+  maybeCreateRedisQueue,
+  prepareExtensionProfile,
+  probeTlsFingerprint,
+  sendInput,
+  startLiveView,
+  startSpan,
+  startSqsWorker,
+  waitForEmailOTP,
+  withSpan
+};
+//# sourceMappingURL=index.js.map

package/dist/input-fuzzer.js

@@ -1,109 +1,105 @@
-/**
- * Input fuzzer — generate test data for form fields.
- *
- * For each field type, produces a set of test values covering:
- * - Happy path (valid, normal input)
- * - Boundary (min/max length, min/max value)
- * - Edge cases (empty, very long, special chars, SQL injection probe, XSS probe)
- * - Type-invalid (wrong type for the field)
- *
- * This feeds ZeTa's test generator to automatically create form validation tests.
- * Values are safe probes (NOT actual exploits) — they test whether the app
- * properly rejects/accepts boundary inputs.
- */
-const LOREM = 'Lorem ipsum dolor sit amet consectetur adipiscing elit';
+const LOREM = "Lorem ipsum dolor sit amet consectetur adipiscing elit";
 function generateEmailCases() {
-    return [
-        { label: 'valid email', value: 'test@example.com', expectedValid: true, category: 'happy' },
-        { label: 'email with plus', value: 'test+tag@example.com', expectedValid: true, category: 'happy' },
-        { label: 'missing @', value: 'notanemail.com', expectedValid: false, category: 'invalid' },
-        { label: 'missing domain', value: 'test@', expectedValid: false, category: 'invalid' },
-        { label: 'empty', value: '', expectedValid: false, category: 'edge' },
-        { label: 'very long email', value: 'a'.repeat(100) + '@example.com', expectedValid: false, category: 'boundary' },
-        { label: 'special chars', value: 'test<script>@example.com', expectedValid: false, category: 'edge' },
-    ];
+  return [
+    { label: "valid email", value: "test@example.com", expectedValid: true, category: "happy" },
+    { label: "email with plus", value: "test+tag@example.com", expectedValid: true, category: "happy" },
+    { label: "missing @", value: "notanemail.com", expectedValid: false, category: "invalid" },
+    { label: "missing domain", value: "test@", expectedValid: false, category: "invalid" },
+    { label: "empty", value: "", expectedValid: false, category: "edge" },
+    { label: "very long email", value: "a".repeat(100) + "@example.com", expectedValid: false, category: "boundary" },
+    { label: "special chars", value: "test<script>@example.com", expectedValid: false, category: "edge" }
+  ];
 }
 function generatePasswordCases(minLength, maxLength) {
-    const min = minLength ?? 8;
-    const max = maxLength ?? 128;
-    return [
-        { label: 'valid password', value: 'ValidP@ss1', expectedValid: true, category: 'happy' },
-        { label: 'minimum length', value: 'A'.repeat(min), expectedValid: true, category: 'boundary' },
-        { label: 'below minimum', value: 'A'.repeat(Math.max(1, min - 1)), expectedValid: false, category: 'boundary' },
-        { label: 'max length', value: 'Aa1!'.repeat(Math.ceil(max / 4)).slice(0, max), expectedValid: true, category: 'boundary' },
-        { label: 'empty', value: '', expectedValid: false, category: 'edge' },
-        { label: 'spaces only', value: '        ', expectedValid: false, category: 'edge' },
-    ];
+  const min = minLength ?? 8;
+  const max = maxLength ?? 128;
+  return [
+    { label: "valid password", value: "ValidP@ss1", expectedValid: true, category: "happy" },
+    { label: "minimum length", value: "A".repeat(min), expectedValid: true, category: "boundary" },
+    { label: "below minimum", value: "A".repeat(Math.max(1, min - 1)), expectedValid: false, category: "boundary" },
+    { label: "max length", value: "Aa1!".repeat(Math.ceil(max / 4)).slice(0, max), expectedValid: true, category: "boundary" },
+    { label: "empty", value: "", expectedValid: false, category: "edge" },
+    { label: "spaces only", value: "        ", expectedValid: false, category: "edge" }
+  ];
 }
 function generateTextCases(minLength, maxLength, required) {
-    const cases = [
-        { label: 'typical text', value: 'Test Input Value', expectedValid: true, category: 'happy' },
-        { label: 'single char', value: 'A', expectedValid: !minLength || minLength <= 1, category: 'boundary' },
-        { label: 'empty', value: '', expectedValid: !required, category: 'edge' },
-        { label: 'spaces only', value: '   ', expectedValid: !required, category: 'edge' },
-        { label: 'very long text', value: LOREM.repeat(20), expectedValid: !maxLength || maxLength > LOREM.length * 20, category: 'boundary' },
-        { label: 'unicode', value: '日本語テスト Ünïcödé Emojitest 🎉', expectedValid: true, category: 'edge' },
-        { label: 'HTML special chars', value: '<script>alert(1)</script>', expectedValid: true, category: 'edge' },
-        { label: 'SQL probe', value: "' OR '1'='1", expectedValid: true, category: 'edge' },
-    ];
-    if (minLength)
-        cases.push({ label: `below min (${minLength})`, value: 'x'.repeat(Math.max(0, minLength - 1)), expectedValid: false, category: 'boundary' });
-    if (maxLength)
-        cases.push({ label: `above max (${maxLength})`, value: 'x'.repeat(maxLength + 1), expectedValid: false, category: 'boundary' });
-    return cases;
+  const cases = [
+    { label: "typical text", value: "Test Input Value", expectedValid: true, category: "happy" },
+    { label: "single char", value: "A", expectedValid: !minLength || minLength <= 1, category: "boundary" },
+    { label: "empty", value: "", expectedValid: !required, category: "edge" },
+    { label: "spaces only", value: "   ", expectedValid: !required, category: "edge" },
+    { label: "very long text", value: LOREM.repeat(20), expectedValid: !maxLength || maxLength > LOREM.length * 20, category: "boundary" },
+    { label: "unicode", value: "\u65E5\u672C\u8A9E\u30C6\u30B9\u30C8 \xDCn\xEFc\xF6d\xE9 Emojitest \u{1F389}", expectedValid: true, category: "edge" },
+    { label: "HTML special chars", value: "<script>alert(1)</script>", expectedValid: true, category: "edge" },
+    { label: "SQL probe", value: "' OR '1'='1", expectedValid: true, category: "edge" }
+  ];
+  if (minLength) cases.push({ label: `below min (${minLength})`, value: "x".repeat(Math.max(0, minLength - 1)), expectedValid: false, category: "boundary" });
+  if (maxLength) cases.push({ label: `above max (${maxLength})`, value: "x".repeat(maxLength + 1), expectedValid: false, category: "boundary" });
+  return cases;
 }
 function generateNumberCases(min, max) {
-    const minN = min !== undefined ? parseFloat(min) : undefined;
-    const maxN = max !== undefined ? parseFloat(max) : undefined;
-    return [
-        { label: 'valid number', value: '42', expectedValid: true, category: 'happy' },
-        { label: 'zero', value: '0', expectedValid: minN === undefined || minN <= 0, category: 'boundary' },
-        { label: 'negative', value: '-1', expectedValid: minN === undefined || minN < 0, category: 'boundary' },
-        { label: 'decimal', value: '3.14', expectedValid: true, category: 'happy' },
-        { label: 'text as number', value: 'notanumber', expectedValid: false, category: 'invalid' },
-        { label: 'empty', value: '', expectedValid: false, category: 'edge' },
-        ...(minN !== undefined ? [{ label: `below min (${min})`, value: String(minN - 1), expectedValid: false, category: 'boundary' }] : []),
-        ...(maxN !== undefined ? [{ label: `above max (${max})`, value: String(maxN + 1), expectedValid: false, category: 'boundary' }] : []),
-    ];
+  const minN = min !== void 0 ? parseFloat(min) : void 0;
+  const maxN = max !== void 0 ? parseFloat(max) : void 0;
+  return [
+    { label: "valid number", value: "42", expectedValid: true, category: "happy" },
+    { label: "zero", value: "0", expectedValid: minN === void 0 || minN <= 0, category: "boundary" },
+    { label: "negative", value: "-1", expectedValid: minN === void 0 || minN < 0, category: "boundary" },
+    { label: "decimal", value: "3.14", expectedValid: true, category: "happy" },
+    { label: "text as number", value: "notanumber", expectedValid: false, category: "invalid" },
+    { label: "empty", value: "", expectedValid: false, category: "edge" },
+    ...minN !== void 0 ? [{ label: `below min (${min})`, value: String(minN - 1), expectedValid: false, category: "boundary" }] : [],
+    ...maxN !== void 0 ? [{ label: `above max (${max})`, value: String(maxN + 1), expectedValid: false, category: "boundary" }] : []
+  ];
 }
 function generateSelectCases(options) {
-    if (!options?.length)
-        return [{ label: 'first option', value: '', expectedValid: true, category: 'happy' }];
-    return [
-        { label: `first option (${options[0]})`, value: options[0], expectedValid: true, category: 'happy' },
-        { label: 'invalid option', value: '__INVALID_OPTION__', expectedValid: false, category: 'invalid' },
-        { label: 'empty selection', value: '', expectedValid: false, category: 'edge' },
-        ...(options.length > 1 ? [{ label: `last option (${options[options.length - 1]})`, value: options[options.length - 1], expectedValid: true, category: 'happy' }] : []),
-    ];
+  if (!options?.length) return [{ label: "first option", value: "", expectedValid: true, category: "happy" }];
+  return [
+    { label: `first option (${options[0]})`, value: options[0], expectedValid: true, category: "happy" },
+    { label: "invalid option", value: "__INVALID_OPTION__", expectedValid: false, category: "invalid" },
+    { label: "empty selection", value: "", expectedValid: false, category: "edge" },
+    ...options.length > 1 ? [{ label: `last option (${options[options.length - 1]})`, value: options[options.length - 1], expectedValid: true, category: "happy" }] : []
+  ];
 }
-export function generateFieldCases(field) {
-    switch (field.type) {
-        case 'email': return generateEmailCases();
-        case 'password': return generatePasswordCases(field.minLength, field.maxLength);
-        case 'number': return generateNumberCases(field.min, field.max);
-        case 'select': return generateSelectCases(field.options);
-        case 'checkbox': return [
-            { label: 'checked', value: 'true', expectedValid: true, category: 'happy' },
-            { label: 'unchecked', value: 'false', expectedValid: !field.required, category: field.required ? 'invalid' : 'happy' },
-        ];
-        case 'date': return [
-            { label: 'valid date', value: '2026-01-15', expectedValid: true, category: 'happy' },
-            { label: 'invalid date', value: '9999-99-99', expectedValid: false, category: 'invalid' },
-            { label: 'empty', value: '', expectedValid: !field.required, category: 'edge' },
-        ];
-        case 'url': return [
-            { label: 'valid URL', value: 'https://example.com', expectedValid: true, category: 'happy' },
-            { label: 'missing protocol', value: 'example.com', expectedValid: false, category: 'invalid' },
-            { label: 'empty', value: '', expectedValid: !field.required, category: 'edge' },
-        ];
-        default:
-            return generateTextCases(field.minLength, field.maxLength, field.required);
-    }
+function generateFieldCases(field) {
+  switch (field.type) {
+    case "email":
+      return generateEmailCases();
+    case "password":
+      return generatePasswordCases(field.minLength, field.maxLength);
+    case "number":
+      return generateNumberCases(field.min, field.max);
+    case "select":
+      return generateSelectCases(field.options);
+    case "checkbox":
+      return [
+        { label: "checked", value: "true", expectedValid: true, category: "happy" },
+        { label: "unchecked", value: "false", expectedValid: !field.required, category: field.required ? "invalid" : "happy" }
+      ];
+    case "date":
+      return [
+        { label: "valid date", value: "2026-01-15", expectedValid: true, category: "happy" },
+        { label: "invalid date", value: "9999-99-99", expectedValid: false, category: "invalid" },
+        { label: "empty", value: "", expectedValid: !field.required, category: "edge" }
+      ];
+    case "url":
+      return [
+        { label: "valid URL", value: "https://example.com", expectedValid: true, category: "happy" },
+        { label: "missing protocol", value: "example.com", expectedValid: false, category: "invalid" },
+        { label: "empty", value: "", expectedValid: !field.required, category: "edge" }
+      ];
+    default:
+      return generateTextCases(field.minLength, field.maxLength, field.required);
+  }
 }
-export function fuzzForm(form) {
-    return {
-        formIndex: form.index,
-        purpose: form.purpose,
-        fieldCases: form.fields.map(field => ({ field, cases: generateFieldCases(field) })),
-    };
+function fuzzForm(form) {
+  return {
+    formIndex: form.index,
+    purpose: form.purpose,
+    fieldCases: form.fields.map((field) => ({ field, cases: generateFieldCases(field) }))
+  };
 }
+export {
+  fuzzForm,
+  generateFieldCases
+};
+//# sourceMappingURL=input-fuzzer.js.map