@detiq/crawler 1.1.8 → 1.1.9

package/dist/incremental-diff.js

@@ -1,61 +1,94 @@
-function hashContent(text) {
-  let h1 = 5381;
-  let h2 = 52711;
-  for (let i = 0; i < text.length; i++) {
-    const c = text.charCodeAt(i);
-    h1 = (Math.imul(h1, 33) ^ c) >>> 0;
-    h2 = (Math.imul(h2, 31) ^ c) >>> 0;
-  }
-  return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export function hashContent(text) {
+    let h1 = 5381;
+    let h2 = 52711;
+    for (let i = 0; i < text.length; i++) {
+        const c = text.charCodeAt(i);
+        h1 = (Math.imul(h1, 33) ^ c) >>> 0;
+        h2 = (Math.imul(h2, 31) ^ c) >>> 0;
+    }
+    return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
 }
-function normalizeText(text) {
-  return text.split("\n").map((l) => l.trim()).filter((l) => l.length > 0).join("\n");
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export function normalizeText(text) {
+    return text
+        .split('\n')
+        .map(l => l.trim())
+        .filter(l => l.length > 0)
+        .join('\n');
 }
-function computeContentHash(url, markdownText) {
-  const normalized = normalizeText(markdownText);
-  return {
-    url,
-    hash: hashContent(normalized),
-    textLength: normalized.length,
-    computedAt: (/* @__PURE__ */ new Date()).toISOString()
-  };
+export function computeContentHash(url, markdownText) {
+    const normalized = normalizeText(markdownText);
+    return {
+        url,
+        hash: hashContent(normalized),
+        textLength: normalized.length,
+        computedAt: new Date().toISOString(),
+    };
 }
-function diffContent(url, currentMarkdown, priorHashes) {
-  const current = computeContentHash(url, currentMarkdown);
-  const prior = priorHashes.get(url);
-  if (!prior) {
-    return { url, changed: true, currentHash: current.hash, reason: "new" };
-  }
-  if (prior === current.hash) {
-    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: "unchanged" };
-  }
-  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: "changed" };
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export function diffContent(url, currentMarkdown, priorHashes) {
+    const current = computeContentHash(url, currentMarkdown);
+    const prior = priorHashes.get(url);
+    if (!prior) {
+        return { url, changed: true, currentHash: current.hash, reason: 'new' };
+    }
+    if (prior === current.hash) {
+        return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };
+    }
+    return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };
 }
-function buildPriorHashMap(screenRecords) {
-  const map = /* @__PURE__ */ new Map();
-  for (const s of screenRecords) {
-    if (s.url && s.elementHash) map.set(s.url, s.elementHash);
-  }
-  return map;
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export function buildPriorHashMap(screenRecords) {
+    const map = new Map();
+    for (const s of screenRecords) {
+        if (s.url && s.elementHash)
+            map.set(s.url, s.elementHash);
+    }
+    return map;
 }
-function partitionByChange(urls, priorHashes) {
-  const toProcess = [];
-  const toSkip = [];
-  for (const url of urls) {
-    if (priorHashes.has(url)) {
-      toProcess.push(url);
-    } else {
-      toProcess.push(url);
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export function partitionByChange(urls, priorHashes) {
+    const toProcess = [];
+    const toSkip = [];
+    for (const url of urls) {
+        if (priorHashes.has(url)) {
+            // Can't know if changed without fetching — still needs fetch, but hash after fetch
+            // will determine skip. Include all — hash check happens post-fetch.
+            toProcess.push(url);
+        }
+        else {
+            toProcess.push(url);
+        }
     }
-  }
-  return { toProcess, toSkip };
+    return { toProcess, toSkip };
 }
-export {
-  buildPriorHashMap,
-  computeContentHash,
-  diffContent,
-  hashContent,
-  normalizeText,
-  partitionByChange
-};
-//# sourceMappingURL=incremental-diff.js.map

package/dist/index.js

@@ -1,67 +1,28 @@
-export * from "./crawler.js";
-export * from "./visual-diff-verdict.js";
-export * from "./auth.js";
-export * from "./screen-reader.js";
-export * from "./job-queue.js";
-import { startSqsWorker } from "./sqs-queue.js";
-export * from "./session-capture.js";
-export * from "./session-export.js";
-export * from "./test-executor.js";
-import { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from "./live-view.js";
-import { waitForEmailOTP, getInboxEmail } from "./email-otp.js";
-import { getSsoToken, getSsoClientCredentialsToken } from "./sso-token.js";
-export * from "./engine-router.js";
-export * from "./profiles/domain-profile.js";
-export * from "./policies/crawl-safety.js";
-export * from "./extractors/artifact-envelope.js";
-export * from "./network-observer.js";
-import { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from "./tls-fingerprint.js";
-import { prepareExtensionProfile, adBlockerProfile } from "./browser-extension.js";
-import { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from "./redis-queue.js";
-import { discoverProjectUrls, discoverSitemapUrls } from "./sitemap.js";
-import { httpCrawl } from "./http-engine.js";
-import { extractMarkdown } from "./markdown-extract.js";
-import { TieredProxyManager } from "./proxy.js";
-import { RedisCrawlFrontier, isRedisFrontierEnabled } from "./redis-frontier.js";
-import { startSpan, endSpan, withSpan, increment, flushMetrics } from "./telemetry.js";
-import { analyzeFailure } from "./failure-analysis-agent.js";
-import { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from "./locator-extractor.js";
-export {
-  CHROME_TLS_HEADERS,
-  RedisCrawlFrontier,
-  TieredProxyManager,
-  adBlockerProfile,
-  analyzeFailure,
-  chromeTlsAgent,
-  closeLiveView,
-  completeLiveView,
-  createChromeTlsAgent,
-  createRedisQueue,
-  discoverProjectUrls,
-  discoverSitemapUrls,
-  elementToPlaywrightLocator,
-  elementsToPlaywrightLocators,
-  endSpan,
-  extractMarkdown,
-  flushMetrics,
-  formatLocatorsForPrompt,
-  getFrame,
-  getInboxEmail,
-  getSsoClientCredentialsToken,
-  getSsoToken,
-  httpCrawl,
-  increment,
-  isRedisFrontierEnabled,
-  isRedisQueueEnabled,
-  liveViewNavigate,
-  maybeCreateRedisQueue,
-  prepareExtensionProfile,
-  probeTlsFingerprint,
-  sendInput,
-  startLiveView,
-  startSpan,
-  startSqsWorker,
-  waitForEmailOTP,
-  withSpan
-};
-//# sourceMappingURL=index.js.map
+export * from './crawler.js';
+export * from './visual-diff-verdict.js';
+export * from './auth.js';
+export * from './screen-reader.js';
+export * from './job-queue.js';
+export { startSqsWorker } from './sqs-queue.js';
+export * from './session-capture.js';
+export * from './session-export.js';
+export * from './test-executor.js';
+export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
+export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
+export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
+export * from './engine-router.js';
+export * from './profiles/domain-profile.js';
+export * from './policies/crawl-safety.js';
+export * from './extractors/artifact-envelope.js';
+export * from './network-observer.js';
+export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
+export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
+export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
+export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
+export { httpCrawl } from './http-engine.js';
+export { extractMarkdown } from './markdown-extract.js';
+export { TieredProxyManager } from './proxy.js';
+export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
+export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
+export { analyzeFailure } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt } from './locator-extractor.js';

package/dist/input-fuzzer.js

@@ -1,105 +1,109 @@
-const LOREM = "Lorem ipsum dolor sit amet consectetur adipiscing elit";
+/**
+ * Input fuzzer — generate test data for form fields.
+ *
+ * For each field type, produces a set of test values covering:
+ * - Happy path (valid, normal input)
+ * - Boundary (min/max length, min/max value)
+ * - Edge cases (empty, very long, special chars, SQL injection probe, XSS probe)
+ * - Type-invalid (wrong type for the field)
+ *
+ * This feeds ZeTa's test generator to automatically create form validation tests.
+ * Values are safe probes (NOT actual exploits) — they test whether the app
+ * properly rejects/accepts boundary inputs.
+ */
+const LOREM = 'Lorem ipsum dolor sit amet consectetur adipiscing elit';
 function generateEmailCases() {
-  return [
-    { label: "valid email", value: "test@example.com", expectedValid: true, category: "happy" },
-    { label: "email with plus", value: "test+tag@example.com", expectedValid: true, category: "happy" },
-    { label: "missing @", value: "notanemail.com", expectedValid: false, category: "invalid" },
-    { label: "missing domain", value: "test@", expectedValid: false, category: "invalid" },
-    { label: "empty", value: "", expectedValid: false, category: "edge" },
-    { label: "very long email", value: "a".repeat(100) + "@example.com", expectedValid: false, category: "boundary" },
-    { label: "special chars", value: "test<script>@example.com", expectedValid: false, category: "edge" }
-  ];
+    return [
+        { label: 'valid email', value: 'test@example.com', expectedValid: true, category: 'happy' },
+        { label: 'email with plus', value: 'test+tag@example.com', expectedValid: true, category: 'happy' },
+        { label: 'missing @', value: 'notanemail.com', expectedValid: false, category: 'invalid' },
+        { label: 'missing domain', value: 'test@', expectedValid: false, category: 'invalid' },
+        { label: 'empty', value: '', expectedValid: false, category: 'edge' },
+        { label: 'very long email', value: 'a'.repeat(100) + '@example.com', expectedValid: false, category: 'boundary' },
+        { label: 'special chars', value: 'test<script>@example.com', expectedValid: false, category: 'edge' },
+    ];
 }
 function generatePasswordCases(minLength, maxLength) {
-  const min = minLength ?? 8;
-  const max = maxLength ?? 128;
-  return [
-    { label: "valid password", value: "ValidP@ss1", expectedValid: true, category: "happy" },
-    { label: "minimum length", value: "A".repeat(min), expectedValid: true, category: "boundary" },
-    { label: "below minimum", value: "A".repeat(Math.max(1, min - 1)), expectedValid: false, category: "boundary" },
-    { label: "max length", value: "Aa1!".repeat(Math.ceil(max / 4)).slice(0, max), expectedValid: true, category: "boundary" },
-    { label: "empty", value: "", expectedValid: false, category: "edge" },
-    { label: "spaces only", value: "        ", expectedValid: false, category: "edge" }
-  ];
+    const min = minLength ?? 8;
+    const max = maxLength ?? 128;
+    return [
+        { label: 'valid password', value: 'ValidP@ss1', expectedValid: true, category: 'happy' },
+        { label: 'minimum length', value: 'A'.repeat(min), expectedValid: true, category: 'boundary' },
+        { label: 'below minimum', value: 'A'.repeat(Math.max(1, min - 1)), expectedValid: false, category: 'boundary' },
+        { label: 'max length', value: 'Aa1!'.repeat(Math.ceil(max / 4)).slice(0, max), expectedValid: true, category: 'boundary' },
+        { label: 'empty', value: '', expectedValid: false, category: 'edge' },
+        { label: 'spaces only', value: '        ', expectedValid: false, category: 'edge' },
+    ];
 }
 function generateTextCases(minLength, maxLength, required) {
-  const cases = [
-    { label: "typical text", value: "Test Input Value", expectedValid: true, category: "happy" },
-    { label: "single char", value: "A", expectedValid: !minLength || minLength <= 1, category: "boundary" },
-    { label: "empty", value: "", expectedValid: !required, category: "edge" },
-    { label: "spaces only", value: "   ", expectedValid: !required, category: "edge" },
-    { label: "very long text", value: LOREM.repeat(20), expectedValid: !maxLength || maxLength > LOREM.length * 20, category: "boundary" },
-    { label: "unicode", value: "\u65E5\u672C\u8A9E\u30C6\u30B9\u30C8 \xDCn\xEFc\xF6d\xE9 Emojitest \u{1F389}", expectedValid: true, category: "edge" },
-    { label: "HTML special chars", value: "<script>alert(1)</script>", expectedValid: true, category: "edge" },
-    { label: "SQL probe", value: "' OR '1'='1", expectedValid: true, category: "edge" }
-  ];
-  if (minLength) cases.push({ label: `below min (${minLength})`, value: "x".repeat(Math.max(0, minLength - 1)), expectedValid: false, category: "boundary" });
-  if (maxLength) cases.push({ label: `above max (${maxLength})`, value: "x".repeat(maxLength + 1), expectedValid: false, category: "boundary" });
-  return cases;
+    const cases = [
+        { label: 'typical text', value: 'Test Input Value', expectedValid: true, category: 'happy' },
+        { label: 'single char', value: 'A', expectedValid: !minLength || minLength <= 1, category: 'boundary' },
+        { label: 'empty', value: '', expectedValid: !required, category: 'edge' },
+        { label: 'spaces only', value: '   ', expectedValid: !required, category: 'edge' },
+        { label: 'very long text', value: LOREM.repeat(20), expectedValid: !maxLength || maxLength > LOREM.length * 20, category: 'boundary' },
+        { label: 'unicode', value: '日本語テスト Ünïcödé Emojitest 🎉', expectedValid: true, category: 'edge' },
+        { label: 'HTML special chars', value: '<script>alert(1)</script>', expectedValid: true, category: 'edge' },
+        { label: 'SQL probe', value: "' OR '1'='1", expectedValid: true, category: 'edge' },
+    ];
+    if (minLength)
+        cases.push({ label: `below min (${minLength})`, value: 'x'.repeat(Math.max(0, minLength - 1)), expectedValid: false, category: 'boundary' });
+    if (maxLength)
+        cases.push({ label: `above max (${maxLength})`, value: 'x'.repeat(maxLength + 1), expectedValid: false, category: 'boundary' });
+    return cases;
 }
 function generateNumberCases(min, max) {
-  const minN = min !== void 0 ? parseFloat(min) : void 0;
-  const maxN = max !== void 0 ? parseFloat(max) : void 0;
-  return [
-    { label: "valid number", value: "42", expectedValid: true, category: "happy" },
-    { label: "zero", value: "0", expectedValid: minN === void 0 || minN <= 0, category: "boundary" },
-    { label: "negative", value: "-1", expectedValid: minN === void 0 || minN < 0, category: "boundary" },
-    { label: "decimal", value: "3.14", expectedValid: true, category: "happy" },
-    { label: "text as number", value: "notanumber", expectedValid: false, category: "invalid" },
-    { label: "empty", value: "", expectedValid: false, category: "edge" },
-    ...minN !== void 0 ? [{ label: `below min (${min})`, value: String(minN - 1), expectedValid: false, category: "boundary" }] : [],
-    ...maxN !== void 0 ? [{ label: `above max (${max})`, value: String(maxN + 1), expectedValid: false, category: "boundary" }] : []
-  ];
+    const minN = min !== undefined ? parseFloat(min) : undefined;
+    const maxN = max !== undefined ? parseFloat(max) : undefined;
+    return [
+        { label: 'valid number', value: '42', expectedValid: true, category: 'happy' },
+        { label: 'zero', value: '0', expectedValid: minN === undefined || minN <= 0, category: 'boundary' },
+        { label: 'negative', value: '-1', expectedValid: minN === undefined || minN < 0, category: 'boundary' },
+        { label: 'decimal', value: '3.14', expectedValid: true, category: 'happy' },
+        { label: 'text as number', value: 'notanumber', expectedValid: false, category: 'invalid' },
+        { label: 'empty', value: '', expectedValid: false, category: 'edge' },
+        ...(minN !== undefined ? [{ label: `below min (${min})`, value: String(minN - 1), expectedValid: false, category: 'boundary' }] : []),
+        ...(maxN !== undefined ? [{ label: `above max (${max})`, value: String(maxN + 1), expectedValid: false, category: 'boundary' }] : []),
+    ];
 }
 function generateSelectCases(options) {
-  if (!options?.length) return [{ label: "first option", value: "", expectedValid: true, category: "happy" }];
-  return [
-    { label: `first option (${options[0]})`, value: options[0], expectedValid: true, category: "happy" },
-    { label: "invalid option", value: "__INVALID_OPTION__", expectedValid: false, category: "invalid" },
-    { label: "empty selection", value: "", expectedValid: false, category: "edge" },
-    ...options.length > 1 ? [{ label: `last option (${options[options.length - 1]})`, value: options[options.length - 1], expectedValid: true, category: "happy" }] : []
-  ];
+    if (!options?.length)
+        return [{ label: 'first option', value: '', expectedValid: true, category: 'happy' }];
+    return [
+        { label: `first option (${options[0]})`, value: options[0], expectedValid: true, category: 'happy' },
+        { label: 'invalid option', value: '__INVALID_OPTION__', expectedValid: false, category: 'invalid' },
+        { label: 'empty selection', value: '', expectedValid: false, category: 'edge' },
+        ...(options.length > 1 ? [{ label: `last option (${options[options.length - 1]})`, value: options[options.length - 1], expectedValid: true, category: 'happy' }] : []),
+    ];
 }
-function generateFieldCases(field) {
-  switch (field.type) {
-    case "email":
-      return generateEmailCases();
-    case "password":
-      return generatePasswordCases(field.minLength, field.maxLength);
-    case "number":
-      return generateNumberCases(field.min, field.max);
-    case "select":
-      return generateSelectCases(field.options);
-    case "checkbox":
-      return [
-        { label: "checked", value: "true", expectedValid: true, category: "happy" },
-        { label: "unchecked", value: "false", expectedValid: !field.required, category: field.required ? "invalid" : "happy" }
-      ];
-    case "date":
-      return [
-        { label: "valid date", value: "2026-01-15", expectedValid: true, category: "happy" },
-        { label: "invalid date", value: "9999-99-99", expectedValid: false, category: "invalid" },
-        { label: "empty", value: "", expectedValid: !field.required, category: "edge" }
-      ];
-    case "url":
-      return [
-        { label: "valid URL", value: "https://example.com", expectedValid: true, category: "happy" },
-        { label: "missing protocol", value: "example.com", expectedValid: false, category: "invalid" },
-        { label: "empty", value: "", expectedValid: !field.required, category: "edge" }
-      ];
-    default:
-      return generateTextCases(field.minLength, field.maxLength, field.required);
-  }
+export function generateFieldCases(field) {
+    switch (field.type) {
+        case 'email': return generateEmailCases();
+        case 'password': return generatePasswordCases(field.minLength, field.maxLength);
+        case 'number': return generateNumberCases(field.min, field.max);
+        case 'select': return generateSelectCases(field.options);
+        case 'checkbox': return [
+            { label: 'checked', value: 'true', expectedValid: true, category: 'happy' },
+            { label: 'unchecked', value: 'false', expectedValid: !field.required, category: field.required ? 'invalid' : 'happy' },
+        ];
+        case 'date': return [
+            { label: 'valid date', value: '2026-01-15', expectedValid: true, category: 'happy' },
+            { label: 'invalid date', value: '9999-99-99', expectedValid: false, category: 'invalid' },
+            { label: 'empty', value: '', expectedValid: !field.required, category: 'edge' },
+        ];
+        case 'url': return [
+            { label: 'valid URL', value: 'https://example.com', expectedValid: true, category: 'happy' },
+            { label: 'missing protocol', value: 'example.com', expectedValid: false, category: 'invalid' },
+            { label: 'empty', value: '', expectedValid: !field.required, category: 'edge' },
+        ];
+        default:
+            return generateTextCases(field.minLength, field.maxLength, field.required);
+    }
 }
-function fuzzForm(form) {
-  return {
-    formIndex: form.index,
-    purpose: form.purpose,
-    fieldCases: form.fields.map((field) => ({ field, cases: generateFieldCases(field) }))
-  };
+export function fuzzForm(form) {
+    return {
+        formIndex: form.index,
+        purpose: form.purpose,
+        fieldCases: form.fields.map(field => ({ field, cases: generateFieldCases(field) })),
+    };
 }
-export {
-  fuzzForm,
-  generateFieldCases
-};
-//# sourceMappingURL=input-fuzzer.js.map