@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/incremental-diff.ts

@@ -0,0 +1,259 @@
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+
+export interface ContentHash {
+  url: string;
+  hash: string;
+  textLength: number;
+  computedAt: string; // ISO timestamp
+}
+
+export interface DiffResult {
+  url: string;
+  changed: boolean;
+  previousHash?: string;
+  currentHash: string;
+  reason: 'new' | 'changed' | 'unchanged';
+}
+
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export function hashContent(text: string): string {
+  let h1 = 5381;
+  let h2 = 52711;
+  for (let i = 0; i < text.length; i++) {
+    const c = text.charCodeAt(i);
+    h1 = (Math.imul(h1, 33) ^ c) >>> 0;
+    h2 = (Math.imul(h2, 31) ^ c) >>> 0;
+  }
+  return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
+}
+
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export function normalizeText(text: string): string {
+  return text
+    .split('\n')
+    .map(l => l.trim())
+    .filter(l => l.length > 0)
+    .join('\n');
+}
+
+export function computeContentHash(url: string, markdownText: string): ContentHash {
+  const normalized = normalizeText(markdownText);
+  return {
+    url,
+    hash: hashContent(normalized),
+    textLength: normalized.length,
+    computedAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export function diffContent(
+  url: string,
+  currentMarkdown: string,
+  priorHashes: Map<string, string>, // url → hash
+): DiffResult {
+  const current = computeContentHash(url, currentMarkdown);
+  const prior = priorHashes.get(url);
+
+  if (!prior) {
+    return { url, changed: true, currentHash: current.hash, reason: 'new' };
+  }
+  if (prior === current.hash) {
+    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };
+  }
+  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };
+}
+
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export function buildPriorHashMap(screenRecords: Array<{ url: string; elementHash?: string | null }>): Map<string, string> {
+  const map = new Map<string, string>();
+  for (const s of screenRecords) {
+    if (s.url && s.elementHash) map.set(s.url, s.elementHash);
+  }
+  return map;
+}
+
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export function partitionByChange(
+  urls: string[],
+  priorHashes: Map<string, string>,
+): { toProcess: string[]; toSkip: string[] } {
+  const toProcess: string[] = [];
+  const toSkip: string[] = [];
+  for (const url of urls) {
+    if (priorHashes.has(url)) {
+      // Can't know if changed without fetching — still needs fetch, but hash after fetch
+      // will determine skip. Include all — hash check happens post-fetch.
+      toProcess.push(url);
+    } else {
+      toProcess.push(url);
+    }
+  }
+  return { toProcess, toSkip };
+}
+
+// ─── Typed monitor diffs (GAP-01) ───────────────────────────────────────────
+
+/** A single line-level change in text content (markdown, html, summary). */
+export interface TextChange {
+  type: 'added' | 'removed';
+  line: number;
+  content: string;
+}
+
+/** A change to a specific JSON path between two scraped JSON objects. */
+export interface JsonChange {
+  path: string;
+  old: unknown;
+  new: unknown;
+}
+
+/** Set-based change for collections like links or images. */
+export interface SetChange {
+  added: string[];
+  removed: string[];
+}
+
+/** Screenshot pixel-level change. changed is percentage (0-100). */
+export interface ImageChange {
+  size: number;
+  changed: number;
+  mask?: string; // base64 diff mask
+}
+
+/** All possible diffs for a single monitor tick. */
+export interface MonitorDiffs {
+  markdown?: TextChange[];
+  html?: TextChange[];
+  json?: JsonChange[];
+  summary?: TextChange[];
+  links?: SetChange;
+  images?: SetChange;
+  branding?: JsonChange[];
+  screenshot?: ImageChange;
+}
+
+/**
+ * Compute line-level text diffs between two strings.
+ * Returns TextChange[] showing exactly which lines were added or removed.
+ * No external dependencies — pure JS Myers-inspired algorithm.
+ */
+export function computeTextDiffs(previous: string, current: string): TextChange[] {
+  const prevLines = previous.split('\n');
+  const currLines = current.split('\n');
+  const changes: TextChange[] = [];
+
+  const prevSet = new Set(prevLines);
+  const currSet = new Set(currLines);
+
+  let lineNum = 0;
+  for (const line of prevLines) {
+    if (!currSet.has(line)) {
+      changes.push({ type: 'removed', line: lineNum, content: line });
+    }
+    lineNum++;
+  }
+  lineNum = 0;
+  for (const line of currLines) {
+    if (!prevSet.has(line)) {
+      changes.push({ type: 'added', line: lineNum, content: line });
+    }
+    lineNum++;
+  }
+
+  return changes;
+}
+
+/**
+ * Compute set-based diffs for URL collections (links, images).
+ */
+export function computeSetDiffs(previous: string[], current: string[]): SetChange {
+  const prevSet = new Set(previous);
+  const currSet = new Set(current);
+  return {
+    added: current.filter(u => !prevSet.has(u)),
+    removed: previous.filter(u => !currSet.has(u)),
+  };
+}
+
+/**
+ * Flatten a JSON object into dot-notation path/value pairs.
+ * Used to compute JsonChange[] between two JSON extraction results.
+ */
+export function flattenJson(obj: unknown, prefix = ''): Record<string, unknown> {
+  if (typeof obj !== 'object' || obj === null) {
+    return { [prefix]: obj };
+  }
+  const result: Record<string, unknown> = {};
+  for (const [k, v] of Object.entries(obj as Record<string, unknown>)) {
+    const path = prefix ? `${prefix}.${k}` : k;
+    if (typeof v === 'object' && v !== null && !Array.isArray(v)) {
+      Object.assign(result, flattenJson(v, path));
+    } else {
+      result[path] = v;
+    }
+  }
+  return result;
+}
+
+/**
+ * Compute JSON path-level diffs between two extraction results.
+ */
+export function computeJsonDiffs(previous: unknown, current: unknown): JsonChange[] {
+  const prevFlat = flattenJson(previous);
+  const currFlat = flattenJson(current);
+  const allKeys = new Set([...Object.keys(prevFlat), ...Object.keys(currFlat)]);
+  const changes: JsonChange[] = [];
+  for (const path of allKeys) {
+    const oldVal = prevFlat[path];
+    const newVal = currFlat[path];
+    if (JSON.stringify(oldVal) !== JSON.stringify(newVal)) {
+      changes.push({ path, old: oldVal, new: newVal });
+    }
+  }
+  return changes;
+}
+
+/**
+ * Count total number of changes across all diff types.
+ */
+export function countMonitorDiffs(diffs: MonitorDiffs): number {
+  let count = 0;
+  if (diffs.markdown) count += diffs.markdown.length;
+  if (diffs.html) count += diffs.html.length;
+  if (diffs.json) count += diffs.json.length;
+  if (diffs.summary) count += diffs.summary.length;
+  if (diffs.links) count += diffs.links.added.length + diffs.links.removed.length;
+  if (diffs.images) count += diffs.images.added.length + diffs.images.removed.length;
+  if (diffs.branding) count += diffs.branding.length;
+  if (diffs.screenshot) count += diffs.screenshot.changed > 0 ? 1 : 0;
+  return count;
+}

package/src/index.ts

@@ -0,0 +1,52 @@
+export * from './crawler.js';
+export * from './visual-diff-verdict.js';
+export * from './auth.js';
+export * from './screen-reader.js';
+export * from './job-queue.js';
+export { startSqsWorker, getDlqStats, redriveFromDlq } from './sqs-queue.js';
+export * from './session-capture.js';
+export * from './session-export.js';
+export * from './test-executor.js';
+export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
+export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
+export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
+export type { SsoConfig, SsoProvider } from './sso-token.js';
+export * from './engine-router.js';
+export * from './profiles/domain-profile.js';
+export * from './policies/crawl-safety.js';
+export * from './extractors/artifact-envelope.js';
+export * from './network-observer.js';
+export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
+export type { TlsFingerprintOptions } from './tls-fingerprint.js';
+export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
+export type { ExtensionSource, ExtensionProfile } from './browser-extension.js';
+export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
+export type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';
+export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
+export type { UrlCandidate } from './sitemap.js';
+export { httpCrawl } from './http-engine.js';
+export { extractMarkdown } from './markdown-extract.js';
+export type { ExtractedMarkdown } from './markdown-extract.js';
+export type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';
+export { TieredProxyManager } from './proxy.js';
+export type { ProxyTier, ProxyTierConfig } from './proxy.js';
+export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
+export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
+export type { Span, FinishedSpan } from './telemetry.js';
+export { analyzeFailure } from './failure-analysis-agent.js';
+export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';
+export * from './dropdown-enumerator.js';
+export * from './crud-interaction.js';
+export * from './filter-pagination-crawler.js';
+export * from './multi-context-handler.js';
+export * from './advanced-ui-probes.js';
+export * from './state-graph.js';
+export * from './nested-window-crawler.js';
+export * from './shadow-dom-crawler.js';
+export * from './network-action-correlator.js';
+export * from './crawl-session-manager.js';
+export * from './accessibility-tree-crawler.js';
+export * from './performance-metrics.js';
+export * from './service-worker-interceptor.js';

package/src/input-fuzzer.ts

@@ -0,0 +1,129 @@
+/**
+ * Input fuzzer — generate test data for form fields.
+ *
+ * For each field type, produces a set of test values covering:
+ * - Happy path (valid, normal input)
+ * - Boundary (min/max length, min/max value)
+ * - Edge cases (empty, very long, special chars, SQL injection probe, XSS probe)
+ * - Type-invalid (wrong type for the field)
+ *
+ * This feeds ZeTa's test generator to automatically create form validation tests.
+ * Values are safe probes (NOT actual exploits) — they test whether the app
+ * properly rejects/accepts boundary inputs.
+ */
+
+import type { FormField } from './form-detector.js';
+
+export type InputCase = {
+  label: string;          // human-readable test case name
+  value: string;
+  expectedValid: boolean; // true if input should be accepted
+  category: 'happy' | 'boundary' | 'edge' | 'invalid';
+};
+
+const LOREM = 'Lorem ipsum dolor sit amet consectetur adipiscing elit';
+
+function generateEmailCases(): InputCase[] {
+  return [
+    { label: 'valid email', value: 'test@example.com', expectedValid: true, category: 'happy' },
+    { label: 'email with plus', value: 'test+tag@example.com', expectedValid: true, category: 'happy' },
+    { label: 'missing @', value: 'notanemail.com', expectedValid: false, category: 'invalid' },
+    { label: 'missing domain', value: 'test@', expectedValid: false, category: 'invalid' },
+    { label: 'empty', value: '', expectedValid: false, category: 'edge' },
+    { label: 'very long email', value: 'a'.repeat(100) + '@example.com', expectedValid: false, category: 'boundary' },
+    { label: 'special chars', value: 'test<script>@example.com', expectedValid: false, category: 'edge' },
+  ];
+}
+
+function generatePasswordCases(minLength?: number, maxLength?: number): InputCase[] {
+  const min = minLength ?? 8;
+  const max = maxLength ?? 128;
+  return [
+    { label: 'valid password', value: 'ValidP@ss1', expectedValid: true, category: 'happy' },
+    { label: 'minimum length', value: 'A'.repeat(min), expectedValid: true, category: 'boundary' },
+    { label: 'below minimum', value: 'A'.repeat(Math.max(1, min - 1)), expectedValid: false, category: 'boundary' },
+    { label: 'max length', value: 'Aa1!'.repeat(Math.ceil(max / 4)).slice(0, max), expectedValid: true, category: 'boundary' },
+    { label: 'empty', value: '', expectedValid: false, category: 'edge' },
+    { label: 'spaces only', value: '        ', expectedValid: false, category: 'edge' },
+  ];
+}
+
+function generateTextCases(minLength?: number, maxLength?: number, required?: boolean): InputCase[] {
+  const cases: InputCase[] = [
+    { label: 'typical text', value: 'Test Input Value', expectedValid: true, category: 'happy' },
+    { label: 'single char', value: 'A', expectedValid: !minLength || minLength <= 1, category: 'boundary' },
+    { label: 'empty', value: '', expectedValid: !required, category: 'edge' },
+    { label: 'spaces only', value: '   ', expectedValid: !required, category: 'edge' },
+    { label: 'very long text', value: LOREM.repeat(20), expectedValid: !maxLength || maxLength > LOREM.length * 20, category: 'boundary' },
+    { label: 'unicode', value: '日本語テスト Ünïcödé Emojitest 🎉', expectedValid: true, category: 'edge' },
+    { label: 'HTML special chars', value: '<script>alert(1)</script>', expectedValid: true, category: 'edge' },
+    { label: 'SQL probe', value: "' OR '1'='1", expectedValid: true, category: 'edge' },
+  ];
+  if (minLength) cases.push({ label: `below min (${minLength})`, value: 'x'.repeat(Math.max(0, minLength - 1)), expectedValid: false, category: 'boundary' });
+  if (maxLength) cases.push({ label: `above max (${maxLength})`, value: 'x'.repeat(maxLength + 1), expectedValid: false, category: 'boundary' });
+  return cases;
+}
+
+function generateNumberCases(min?: string, max?: string): InputCase[] {
+  const minN = min !== undefined ? parseFloat(min) : undefined;
+  const maxN = max !== undefined ? parseFloat(max) : undefined;
+  return [
+    { label: 'valid number', value: '42', expectedValid: true, category: 'happy' },
+    { label: 'zero', value: '0', expectedValid: minN === undefined || minN <= 0, category: 'boundary' },
+    { label: 'negative', value: '-1', expectedValid: minN === undefined || minN < 0, category: 'boundary' },
+    { label: 'decimal', value: '3.14', expectedValid: true, category: 'happy' },
+    { label: 'text as number', value: 'notanumber', expectedValid: false, category: 'invalid' },
+    { label: 'empty', value: '', expectedValid: false, category: 'edge' },
+    ...(minN !== undefined ? [{ label: `below min (${min})`, value: String(minN - 1), expectedValid: false, category: 'boundary' as const }] : []),
+    ...(maxN !== undefined ? [{ label: `above max (${max})`, value: String(maxN + 1), expectedValid: false, category: 'boundary' as const }] : []),
+  ];
+}
+
+function generateSelectCases(options?: string[]): InputCase[] {
+  if (!options?.length) return [{ label: 'first option', value: '', expectedValid: true, category: 'happy' }];
+  return [
+    { label: `first option (${options[0]})`, value: options[0], expectedValid: true, category: 'happy' },
+    { label: 'invalid option', value: '__INVALID_OPTION__', expectedValid: false, category: 'invalid' },
+    { label: 'empty selection', value: '', expectedValid: false, category: 'edge' },
+    ...(options.length > 1 ? [{ label: `last option (${options[options.length - 1]})`, value: options[options.length - 1], expectedValid: true, category: 'happy' as const }] : []),
+  ];
+}
+
+export function generateFieldCases(field: FormField): InputCase[] {
+  switch (field.type) {
+    case 'email':    return generateEmailCases();
+    case 'password': return generatePasswordCases(field.minLength, field.maxLength);
+    case 'number':   return generateNumberCases(field.min, field.max);
+    case 'select':   return generateSelectCases(field.options);
+    case 'checkbox': return [
+      { label: 'checked', value: 'true', expectedValid: true, category: 'happy' },
+      { label: 'unchecked', value: 'false', expectedValid: !field.required, category: field.required ? 'invalid' : 'happy' },
+    ];
+    case 'date':     return [
+      { label: 'valid date', value: '2026-01-15', expectedValid: true, category: 'happy' },
+      { label: 'invalid date', value: '9999-99-99', expectedValid: false, category: 'invalid' },
+      { label: 'empty', value: '', expectedValid: !field.required, category: 'edge' },
+    ];
+    case 'url':      return [
+      { label: 'valid URL', value: 'https://example.com', expectedValid: true, category: 'happy' },
+      { label: 'missing protocol', value: 'example.com', expectedValid: false, category: 'invalid' },
+      { label: 'empty', value: '', expectedValid: !field.required, category: 'edge' },
+    ];
+    default:
+      return generateTextCases(field.minLength, field.maxLength, field.required);
+  }
+}
+
+export interface FuzzedForm {
+  formIndex: number;
+  purpose: string;
+  fieldCases: Array<{ field: FormField; cases: InputCase[] }>;
+}
+
+export function fuzzForm(form: import('./form-detector.js').DetectedForm): FuzzedForm {
+  return {
+    formIndex: form.index,
+    purpose: form.purpose,
+    fieldCases: form.fields.map(field => ({ field, cases: generateFieldCases(field) })),
+  };
+}