@taodigital/crawler 0.0.0-stage → 1.2.1

package/types/frontier.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Configurable crawl traversal strategy — matches crawl4AI's BFS/DFS/
+ * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded
+ * FIFO queue, i.e. always BFS).
+ *
+ * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it
+ * drops into crawler.ts's existing queue call sites with a rename, not a
+ * rewrite — BFS's `shift()` behavior is byte-for-byte the same as the plain
+ * array it replaces, so BFS (the default) has zero behavior change.
+ */
+export type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';
+/**
+ * Simple glob-like pattern matching against a URL's pathname.
+ * Supports `*` (matches any single path segment characters, not `/`)
+ * and `**` (matches anything including `/`).
+ */
+export declare function matchesPattern(url: string, patterns: string[]): boolean;
+export declare class CrawlFrontier {
+    private items;
+    private readonly strategy;
+    private readonly keywords;
+    private readonly urlFilter;
+    private readonly includePatterns;
+    private readonly excludePatterns;
+    constructor(strategy: TraversalStrategy, seedUrls: string[], keywords?: string[], urlFilter?: string, includePatterns?: string[], excludePatterns?: string[]);
+    get length(): number;
+    /**
+     * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+     * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+     * the frontier revisits sparse branches before deep-linked content.
+     */
+    push(url: string, pushOpts?: {
+        parentElements?: number;
+    }): void;
+    /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+    shift(): string | undefined;
+    filter(predicate: (url: string) => boolean): string[];
+    /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+    filterByPattern(pattern: string): number;
+}

package/types/gap-pass.d.ts

@@ -0,0 +1,36 @@
+/**
+ * P6: Gap Pass — LLM-guided second crawl pass
+ *
+ * After the primary BFS crawl completes, this module:
+ * 1. Sends the discovered screen list to an LLM for gap analysis
+ * 2. HEAD-validates each suggested URL
+ * 3. Returns confirmed gap URLs for targeted re-crawl
+ *
+ * The caller (crawler.ts) is responsible for actually visiting the confirmed
+ * URLs with the existing browser — this module only identifies them.
+ */
+export interface GapCandidate {
+    category: string;
+    urlPatterns: string[];
+    confidence: number;
+    rationale: string;
+    confirmedUrl: string | null;
+    headStatus: number | null;
+}
+interface DiscoveredScreen {
+    url: string;
+    name?: string;
+    elements?: number;
+}
+interface LlmCfg {
+    provider: string;
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+/**
+ * Run LLM gap analysis + HEAD validation.
+ * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.
+ */
+export declare function runGapAnalysis(screens: DiscoveredScreen[], appUrl: string, llmCfg: LlmCfg | null | undefined): Promise<GapCandidate[]>;
+export {};

package/types/graphql-detector.d.ts

@@ -0,0 +1,39 @@
+/**
+ * GraphQL detection + introspection.
+ *
+ * During crawl, if a GraphQL endpoint is discovered (via network observer or
+ * known-URL heuristics), fire a minimal introspection query to get the schema.
+ * The schema is stored in the crawl artifact for use during test generation.
+ */
+export interface GqlType {
+    name: string;
+    kind: string;
+    fields?: Array<{
+        name: string;
+        type: string;
+    }>;
+    inputFields?: Array<{
+        name: string;
+        type: string;
+    }>;
+    enumValues?: string[];
+}
+export interface GqlSchema {
+    queryType?: string;
+    mutationType?: string;
+    subscriptionType?: string;
+    types: GqlType[];
+}
+export interface GqlIntrospectionResult {
+    endpoint: string;
+    schema: GqlSchema | null;
+    error?: string;
+    durationMs: number;
+}
+export declare function introspectGraphQL(endpoint: string, opts?: {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    cookies?: string;
+}): Promise<GqlIntrospectionResult>;
+/** Guess GraphQL endpoint URLs from a known app URL. */
+export declare function guessGraphQLEndpoints(appUrl: string): string[];

package/types/http-engine.d.ts

@@ -0,0 +1,80 @@
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
+export interface HttpScreen {
+    url: string;
+    title: string | null;
+    markdown: string;
+    html?: string;
+    metadata: {
+        description: string | null;
+        ogTitle: string | null;
+        ogDescription: string | null;
+        ogImage: string | null;
+        canonical: string | null;
+        schemaTypes: string[];
+        lang: string | null;
+    };
+    tables: Array<Record<string, string>[]>;
+    links: string[];
+    images: Array<{
+        src: string;
+        alt: string;
+    }>;
+    statusCode: number;
+    fromCache: boolean;
+    requiresBrowser?: boolean;
+    durationMs: number;
+}
+export interface HttpCrawlOpts {
+    appUrl: string;
+    maxScreens?: number;
+    urlFilter?: string;
+    traversalStrategy?: 'BFS' | 'DFS';
+    outputFormats?: HttpOutputFormat[];
+    cacheDir?: string;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    respectRobotsTxt?: boolean;
+    includeRawHtml?: boolean;
+    timeoutMs?: number;
+    concurrency?: number;
+    onProgress?: (update: {
+        url: string;
+        screensFound: number;
+        total: number;
+        cached: boolean;
+    }) => void;
+}
+export interface HttpCrawlResult {
+    screens: HttpScreen[];
+    stats: {
+        total: number;
+        cached: number;
+        errors: number;
+        rateLimited: number;
+        durationMs: number;
+        cacheHitRate: number;
+    };
+    jsonLines?: string;
+    csv?: string;
+}
+export declare function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult>;

package/types/incremental-diff.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+export interface ContentHash {
+    url: string;
+    hash: string;
+    textLength: number;
+    computedAt: string;
+}
+export interface DiffResult {
+    url: string;
+    changed: boolean;
+    previousHash?: string;
+    currentHash: string;
+    reason: 'new' | 'changed' | 'unchanged';
+}
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export declare function hashContent(text: string): string;
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export declare function normalizeText(text: string): string;
+export declare function computeContentHash(url: string, markdownText: string): ContentHash;
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export declare function diffContent(url: string, currentMarkdown: string, priorHashes: Map<string, string>): DiffResult;
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export declare function buildPriorHashMap(screenRecords: Array<{
+    url: string;
+    elementHash?: string | null;
+}>): Map<string, string>;
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export declare function partitionByChange(urls: string[], priorHashes: Map<string, string>): {
+    toProcess: string[];
+    toSkip: string[];
+};
+/** A single line-level change in text content (markdown, html, summary). */
+export interface TextChange {
+    type: 'added' | 'removed';
+    line: number;
+    content: string;
+}
+/** A change to a specific JSON path between two scraped JSON objects. */
+export interface JsonChange {
+    path: string;
+    old: unknown;
+    new: unknown;
+}
+/** Set-based change for collections like links or images. */
+export interface SetChange {
+    added: string[];
+    removed: string[];
+}
+/** Screenshot pixel-level change. changed is percentage (0-100). */
+export interface ImageChange {
+    size: number;
+    changed: number;
+    mask?: string;
+}
+/** All possible diffs for a single monitor tick. */
+export interface MonitorDiffs {
+    markdown?: TextChange[];
+    html?: TextChange[];
+    json?: JsonChange[];
+    summary?: TextChange[];
+    links?: SetChange;
+    images?: SetChange;
+    branding?: JsonChange[];
+    screenshot?: ImageChange;
+}
+/**
+ * Compute line-level text diffs between two strings.
+ * Returns TextChange[] showing exactly which lines were added or removed.
+ * No external dependencies — pure JS Myers-inspired algorithm.
+ */
+export declare function computeTextDiffs(previous: string, current: string): TextChange[];
+/**
+ * Compute set-based diffs for URL collections (links, images).
+ */
+export declare function computeSetDiffs(previous: string[], current: string[]): SetChange;
+/**
+ * Flatten a JSON object into dot-notation path/value pairs.
+ * Used to compute JsonChange[] between two JSON extraction results.
+ */
+export declare function flattenJson(obj: unknown, prefix?: string): Record<string, unknown>;
+/**
+ * Compute JSON path-level diffs between two extraction results.
+ */
+export declare function computeJsonDiffs(previous: unknown, current: unknown): JsonChange[];
+/**
+ * Count total number of changes across all diff types.
+ */
+export declare function countMonitorDiffs(diffs: MonitorDiffs): number;

package/types/index.d.ts

@@ -0,0 +1,52 @@
+export * from './crawler.js';
+export * from './visual-diff-verdict.js';
+export * from './auth.js';
+export * from './screen-reader.js';
+export * from './job-queue.js';
+export { startSqsWorker, getDlqStats, redriveFromDlq } from './sqs-queue.js';
+export * from './session-capture.js';
+export * from './session-export.js';
+export * from './test-executor.js';
+export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
+export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
+export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
+export type { SsoConfig, SsoProvider } from './sso-token.js';
+export * from './engine-router.js';
+export * from './profiles/domain-profile.js';
+export * from './policies/crawl-safety.js';
+export * from './extractors/artifact-envelope.js';
+export * from './network-observer.js';
+export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
+export type { TlsFingerprintOptions } from './tls-fingerprint.js';
+export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
+export type { ExtensionSource, ExtensionProfile } from './browser-extension.js';
+export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
+export type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';
+export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
+export type { UrlCandidate } from './sitemap.js';
+export { httpCrawl } from './http-engine.js';
+export { extractMarkdown } from './markdown-extract.js';
+export type { ExtractedMarkdown } from './markdown-extract.js';
+export type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';
+export { TieredProxyManager } from './proxy.js';
+export type { ProxyTier, ProxyTierConfig } from './proxy.js';
+export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
+export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
+export type { Span, FinishedSpan } from './telemetry.js';
+export { analyzeFailure } from './failure-analysis-agent.js';
+export type { FailureContext } from './failure-analysis-agent.js';
+export { elementToPlaywrightLocator, elementsToPlaywrightLocators, formatLocatorsForPrompt, detectAuthWallLocators } from './locator-extractor.js';
+export type { PlaywrightLocator } from './locator-extractor.js';
+export * from './dropdown-enumerator.js';
+export * from './crud-interaction.js';
+export * from './filter-pagination-crawler.js';
+export * from './multi-context-handler.js';
+export * from './advanced-ui-probes.js';
+export * from './state-graph.js';
+export * from './nested-window-crawler.js';
+export * from './shadow-dom-crawler.js';
+export * from './network-action-correlator.js';
+export * from './crawl-session-manager.js';
+export * from './accessibility-tree-crawler.js';
+export * from './performance-metrics.js';
+export * from './service-worker-interceptor.js';

package/types/input-fuzzer.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Input fuzzer — generate test data for form fields.
+ *
+ * For each field type, produces a set of test values covering:
+ * - Happy path (valid, normal input)
+ * - Boundary (min/max length, min/max value)
+ * - Edge cases (empty, very long, special chars, SQL injection probe, XSS probe)
+ * - Type-invalid (wrong type for the field)
+ *
+ * This feeds ZeTa's test generator to automatically create form validation tests.
+ * Values are safe probes (NOT actual exploits) — they test whether the app
+ * properly rejects/accepts boundary inputs.
+ */
+import type { FormField } from './form-detector.js';
+export type InputCase = {
+    label: string;
+    value: string;
+    expectedValid: boolean;
+    category: 'happy' | 'boundary' | 'edge' | 'invalid';
+};
+export declare function generateFieldCases(field: FormField): InputCase[];
+export interface FuzzedForm {
+    formIndex: number;
+    purpose: string;
+    fieldCases: Array<{
+        field: FormField;
+        cases: InputCase[];
+    }>;
+}
+export declare function fuzzForm(form: import('./form-detector.js').DetectedForm): FuzzedForm;

package/types/interactive-probe.d.ts

@@ -0,0 +1,144 @@
+/**
+ * Interactive Probe — Phase 2 Dynamic Observation
+ *
+ * The static crawl captures DOM structure at rest. This module runs AFTER the static crawl
+ * to observe dynamic behaviour by actually interacting with the UI:
+ *
+ *   1. Empty submit  → captures required-field validation error messages
+ *   2. Invalid format → captures format validation errors (bad email, short password)
+ *   3. Button clicks → captures modal/dialog contents
+ *   4. Select changes → captures conditional reveals (show/hide logic)
+ *   5. Success paths → captures toast/banner messages (where input is safe)
+ *
+ * Results are stored as KnowledgeChunks (sourceType='probe') and update the
+ * ERROR_CATALOGUE + INTERACTION_PATTERNS skill files with real observed data
+ * instead of LLM inference.
+ *
+ * Call signature: probeScreen(page, screenId, elements, opts)
+ * Called from crawlProject() after static element extraction when opts.interactive=true.
+ */
+import type { Page } from 'playwright';
+export interface ProbeElementFact {
+    meaning: string;
+    role: string;
+    expectedData?: string;
+}
+export interface ValidationError {
+    field: string;
+    trigger: 'empty_submit' | 'invalid_format' | 'boundary';
+    errorText: string;
+}
+export interface SuccessMessage {
+    type: 'toast' | 'inline_success' | 'redirect';
+    content: string;
+}
+export interface ModalObservation {
+    triggerElement: string;
+    title: string;
+    content: string;
+    fields: string[];
+    stateId?: string;
+    domSnapshot?: string;
+}
+export interface ConditionalReveal {
+    triggerElement: string;
+    triggerValue: string;
+    revealedText: string;
+    stateId?: string;
+    domSnapshot?: string;
+}
+export interface TooltipCapture {
+    triggerLabel: string;
+    tooltipText: string;
+}
+export interface TabPanelCapture {
+    tabLabel: string;
+    panelContent: string;
+    panelHeading?: string;
+}
+export interface CarouselSlide {
+    slideIndex: number;
+    heading?: string;
+    content: string;
+}
+export interface WizardStep {
+    stepIndex: number;
+    stepLabel?: string;
+    content: string;
+    fields: string[];
+}
+export interface FocusTrapResult {
+    modalTrigger: string;
+    focusTrapped: boolean;
+    sampledFocused: string[];
+}
+export interface ContextMenuCapture {
+    trigger: string;
+    items: string[];
+}
+export interface ProbeResult {
+    screenId: string;
+    url: string;
+    validationErrors: ValidationError[];
+    successMessages: SuccessMessage[];
+    modals: ModalObservation[];
+    conditionals: ConditionalReveal[];
+    loadingPatterns: string[];
+    tooltips?: TooltipCapture[];
+    tabPanels?: TabPanelCapture[];
+    carouselSlides?: CarouselSlide[];
+    wizardSteps?: WizardStep[];
+    focusTraps?: FocusTrapResult[];
+    skipNavIssues?: string[];
+    contextMenus?: ContextMenuCapture[];
+}
+/**
+ * Run interactive probing on a single screen page.
+ * The page must already be navigated to the screen URL.
+ * `elements` is the output of the static extractElements pass.
+ */
+export declare function probeScreen(page: Page, screenId: string, url: string, elements: ProbeElementFact[]): Promise<ProbeResult>;
+/** Phase 5: Convert ProbeResult to typed ProbeObservationFact rows for direct DB storage. */
+export declare function probeResultToObservations(result: ProbeResult): Array<{
+    elementMeaning: string;
+    observationType: string;
+    observedText: string;
+    triggeredBy: string;
+}>;
+/**
+ * Convert a ProbeResult into markdown to store as KnowledgeChunks
+ * and update ERROR_CATALOGUE + INTERACTION_PATTERNS skill files.
+ */
+export declare function probeResultToMarkdown(result: ProbeResult): {
+    errorCatalogueChunk: string;
+    interactionPatternsChunk: string;
+};
+/**
+ * Run interactive probing for all screens discovered in a crawl session.
+ * Called from crawlProject() when opts.interactive = true.
+ */
+export declare function probeAllScreens(page: any, tenantId: string, projectId: string, discovered: Array<{
+    url: string;
+    screenId?: string;
+    elements?: any[];
+}>, onProgress?: (pct: number, step: string) => Promise<void>, jobId?: string): Promise<void>;
+export interface FocusableElement {
+    tag: string;
+    role: string | null;
+    ariaLabel: string | null;
+    text: string | null;
+    hasFocusIndicator: boolean;
+    tabIndex: number | null;
+}
+export interface KeyboardNavResult {
+    focusableCount: number;
+    elementsWithNoFocusIndicator: number;
+    sequence: FocusableElement[];
+    issues: string[];
+}
+/**
+ * Probe keyboard navigation accessibility by pressing Tab up to 50 times and
+ * recording each focused element's properties and focus-indicator visibility.
+ * Stops early if focus wraps back to the start or hits the same element twice.
+ */
+export declare function probeKeyboardNav(page: Page): Promise<KeyboardNavResult>;