@detiq/crawler 0.1.3

package/types/extractors/advanced.d.ts

@@ -0,0 +1,54 @@
+export interface ExtractionResult {
+    matches: string[];
+    count: number;
+    method: string;
+}
+export type TableRow = Record<string, string>;
+export interface PageMetadata {
+    title: string | null;
+    description: string | null;
+    keywords: string | null;
+    ogTitle: string | null;
+    ogDescription: string | null;
+    ogImage: string | null;
+    ogUrl: string | null;
+    ogType: string | null;
+    twitterCard: string | null;
+    twitterTitle: string | null;
+    twitterDescription: string | null;
+    canonical: string | null;
+    robots: string | null;
+    jsonLd: object[];
+    schemaTypes: string[];
+    lang: string | null;
+}
+export declare function extractByCssSelector(html: string, selector: string, attr?: string): ExtractionResult;
+export declare function extractByXPath(html: string, xpath: string): ExtractionResult;
+export declare function extractByRegex(text: string, pattern: string, flags?: string): ExtractionResult;
+export declare function extractTables(html: string): TableRow[][];
+export declare function extractMetadata(html: string, url: string): PageMetadata;
+export interface ProductData {
+    name: string | null;
+    description: string | null;
+    brand: string | null;
+    sku: string | null;
+    price: number | null;
+    priceCurrency: string | null;
+    availability: string | null;
+    rating: number | null;
+    reviewCount: number | null;
+    image: string | null;
+    url: string | null;
+}
+export declare function extractStructuredProduct(jsonLd: object[]): ProductData | null;
+export declare function extractImages(html: string, baseUrl: string): Array<{
+    src: string;
+    alt: string;
+    width?: number;
+    height?: number;
+}>;
+export declare function extractMediaFiles(html: string, baseUrl: string): Array<{
+    url: string;
+    type: 'image' | 'video' | 'audio' | 'document';
+    tag: string;
+}>;

package/types/extractors/artifact-envelope.d.ts

@@ -0,0 +1,63 @@
+import type { RawElement } from '../screen-reader.js';
+export interface CrawlEvidenceFact {
+    type: string;
+    payload: unknown;
+}
+export interface CrawlArtifactEnvelope {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    status: 'ok' | 'blocked' | 'auth_expired' | 'error' | 'skipped';
+    outputs: {
+        screenshot?: string;
+        html?: string;
+        markdown?: string;
+        fitMarkdown?: string;
+        accessibilityTree?: string;
+        structuredJson?: unknown;
+        networkTrace?: string;
+        consoleLog?: string;
+    };
+    fingerprints: {
+        textHash?: string;
+        domHash?: string;
+        accessibilityHash?: string;
+        urlCanonicalHash: string;
+    };
+    evidence: CrawlEvidenceFact[];
+}
+export declare function sha256(input: string): string;
+export declare function buildFitMarkdown(markdown: string, keywords?: string[]): string;
+export declare function summarizeStructuredElements(elements: RawElement[]): {
+    elementCount: number;
+    formFieldCount: number;
+    actionCount: number;
+    fields: {
+        label: string;
+        role: string;
+        expectedData: string | undefined;
+    }[];
+    actions: {
+        label: string;
+        role: string;
+    }[];
+};
+export declare function buildCrawlArtifactEnvelope(input: {
+    tenantId: string;
+    projectId: string;
+    jobId?: string;
+    url: string;
+    canonicalUrl: string;
+    screenId?: string;
+    screenshotPath?: string;
+    domPath?: string;
+    domContent?: string;
+    markdown?: string;
+    fitMarkdown?: string;
+    accessibilityElements?: RawElement[];
+    structuredJson?: unknown;
+    evidence?: CrawlEvidenceFact[];
+}): CrawlArtifactEnvelope;

package/types/failure-analysis-agent.d.ts

@@ -0,0 +1,16 @@
+export interface FailureContext {
+    tenantId?: string;
+    error: string;
+    stepDescription?: string;
+    consoleLogs?: Array<{
+        type: string;
+        text: string;
+    }>;
+    networkErrors?: Array<{
+        url: string;
+        status: number;
+        method: string;
+    }>;
+    screenshotPath?: string;
+}
+export declare function analyzeFailure(ctx: FailureContext): Promise<string | null>;

package/types/feature-flag-detector.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Feature flag + A/B test detector.
+ *
+ * Detects popular feature flag / experiment SDKs loaded on the page:
+ * LaunchDarkly, Optimizely, Split.io, Unleash, GrowthBook, Statsig, Flagsmith, Amplitude Experiment.
+ *
+ * Also detects common A/B test patterns: data-testid attributes, gtm.js experiment layers,
+ * Optimizely variation IDs, VWO, etc.
+ *
+ * Why: ZeTa needs to know when a page shows different content to different users
+ * so test generation can account for variants and not generate flaky tests.
+ */
+import type { Page } from 'playwright';
+export interface FeatureFlagInfo {
+    sdk: string;
+    confidence: 'high' | 'medium' | 'low';
+    details?: string;
+}
+export interface AbTestInfo {
+    tool: string;
+    experiments?: string[];
+}
+export interface FeatureFlagDetectionResult {
+    url: string;
+    flagSdks: FeatureFlagInfo[];
+    abTools: AbTestInfo[];
+    hasAnyFlags: boolean;
+    detectedAt: string;
+}
+export declare function detectFeatureFlags(page: Page, url: string): Promise<FeatureFlagDetectionResult>;

package/types/form-detector.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Form detector — extract form metadata from crawled pages.
+ *
+ * For each form found on a page, captures: action URL, method, all input fields
+ * with their types/names/required/validation constraints, and submit button text.
+ *
+ * This data feeds ZeTa's test generator to create form validation test cases
+ * automatically (happy path + boundary/error cases per field type).
+ */
+import type { Page } from 'playwright';
+export interface FormField {
+    name: string;
+    type: string;
+    label?: string;
+    required: boolean;
+    placeholder?: string;
+    minLength?: number;
+    maxLength?: number;
+    min?: string;
+    max?: string;
+    pattern?: string;
+    options?: string[];
+    autocomplete?: string;
+}
+export interface DetectedForm {
+    index: number;
+    action?: string;
+    method: string;
+    fields: FormField[];
+    submitText?: string;
+    hasFileUpload: boolean;
+    purpose: 'login' | 'search' | 'registration' | 'contact' | 'checkout' | 'settings' | 'unknown';
+}
+export declare function detectForms(page: Page): Promise<DetectedForm[]>;

package/types/frontier.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Configurable crawl traversal strategy — matches crawl4AI's BFS/DFS/
+ * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded
+ * FIFO queue, i.e. always BFS).
+ *
+ * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it
+ * drops into crawler.ts's existing queue call sites with a rename, not a
+ * rewrite — BFS's `shift()` behavior is byte-for-byte the same as the plain
+ * array it replaces, so BFS (the default) has zero behavior change.
+ */
+export type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';
+export declare class CrawlFrontier {
+    private items;
+    private readonly strategy;
+    private readonly keywords;
+    private readonly urlFilter;
+    constructor(strategy: TraversalStrategy, seedUrls: string[], keywords?: string[], urlFilter?: string);
+    get length(): number;
+    /**
+     * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+     * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+     * the frontier revisits sparse branches before deep-linked content.
+     */
+    push(url: string, pushOpts?: {
+        parentElements?: number;
+    }): void;
+    /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+    shift(): string | undefined;
+    filter(predicate: (url: string) => boolean): string[];
+    /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+    filterByPattern(pattern: string): number;
+}

package/types/gap-pass.d.ts

@@ -0,0 +1,35 @@
+/**
+ * P6: Gap Pass — LLM-guided second crawl pass
+ *
+ * After the primary BFS crawl completes, this module:
+ * 1. Sends the discovered screen list to an LLM for gap analysis
+ * 2. HEAD-validates each suggested URL
+ * 3. Returns confirmed gap URLs for targeted re-crawl
+ *
+ * The caller (crawler.ts) is responsible for actually visiting the confirmed
+ * URLs with the existing browser — this module only identifies them.
+ */
+export interface GapCandidate {
+    category: string;
+    urlPatterns: string[];
+    confidence: number;
+    rationale: string;
+    confirmedUrl: string | null;
+    headStatus: number | null;
+}
+interface DiscoveredScreen {
+    url: string;
+    name?: string;
+    elements?: number;
+}
+interface LlmCfg {
+    provider: string;
+    model: string;
+    apiKey?: string;
+}
+/**
+ * Run LLM gap analysis + HEAD validation.
+ * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.
+ */
+export declare function runGapAnalysis(screens: DiscoveredScreen[], appUrl: string, llmCfg: LlmCfg | null | undefined): Promise<GapCandidate[]>;
+export {};

package/types/graphql-detector.d.ts

@@ -0,0 +1,39 @@
+/**
+ * GraphQL detection + introspection.
+ *
+ * During crawl, if a GraphQL endpoint is discovered (via network observer or
+ * known-URL heuristics), fire a minimal introspection query to get the schema.
+ * The schema is stored in the crawl artifact for use during test generation.
+ */
+export interface GqlType {
+    name: string;
+    kind: string;
+    fields?: Array<{
+        name: string;
+        type: string;
+    }>;
+    inputFields?: Array<{
+        name: string;
+        type: string;
+    }>;
+    enumValues?: string[];
+}
+export interface GqlSchema {
+    queryType?: string;
+    mutationType?: string;
+    subscriptionType?: string;
+    types: GqlType[];
+}
+export interface GqlIntrospectionResult {
+    endpoint: string;
+    schema: GqlSchema | null;
+    error?: string;
+    durationMs: number;
+}
+export declare function introspectGraphQL(endpoint: string, opts?: {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    cookies?: string;
+}): Promise<GqlIntrospectionResult>;
+/** Guess GraphQL endpoint URLs from a known app URL. */
+export declare function guessGraphQLEndpoints(appUrl: string): string[];

package/types/http-engine.d.ts

@@ -0,0 +1,80 @@
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
+export interface HttpScreen {
+    url: string;
+    title: string | null;
+    markdown: string;
+    html?: string;
+    metadata: {
+        description: string | null;
+        ogTitle: string | null;
+        ogDescription: string | null;
+        ogImage: string | null;
+        canonical: string | null;
+        schemaTypes: string[];
+        lang: string | null;
+    };
+    tables: Array<Record<string, string>[]>;
+    links: string[];
+    images: Array<{
+        src: string;
+        alt: string;
+    }>;
+    statusCode: number;
+    fromCache: boolean;
+    requiresBrowser?: boolean;
+    durationMs: number;
+}
+export interface HttpCrawlOpts {
+    appUrl: string;
+    maxScreens?: number;
+    urlFilter?: string;
+    traversalStrategy?: 'BFS' | 'DFS';
+    outputFormats?: HttpOutputFormat[];
+    cacheDir?: string;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    respectRobotsTxt?: boolean;
+    includeRawHtml?: boolean;
+    timeoutMs?: number;
+    concurrency?: number;
+    onProgress?: (update: {
+        url: string;
+        screensFound: number;
+        total: number;
+        cached: boolean;
+    }) => void;
+}
+export interface HttpCrawlResult {
+    screens: HttpScreen[];
+    stats: {
+        total: number;
+        cached: number;
+        errors: number;
+        rateLimited: number;
+        durationMs: number;
+        cacheHitRate: number;
+    };
+    jsonLines?: string;
+    csv?: string;
+}
+export declare function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult>;

package/types/incremental-diff.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+export interface ContentHash {
+    url: string;
+    hash: string;
+    textLength: number;
+    computedAt: string;
+}
+export interface DiffResult {
+    url: string;
+    changed: boolean;
+    previousHash?: string;
+    currentHash: string;
+    reason: 'new' | 'changed' | 'unchanged';
+}
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export declare function hashContent(text: string): string;
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export declare function normalizeText(text: string): string;
+export declare function computeContentHash(url: string, markdownText: string): ContentHash;
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export declare function diffContent(url: string, currentMarkdown: string, priorHashes: Map<string, string>): DiffResult;
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export declare function buildPriorHashMap(screenRecords: Array<{
+    url: string;
+    elementHash?: string | null;
+}>): Map<string, string>;
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export declare function partitionByChange(urls: string[], priorHashes: Map<string, string>): {
+    toProcess: string[];
+    toSkip: string[];
+};

package/types/index.d.ts

@@ -0,0 +1,35 @@
+export * from './crawler.js';
+export * from './visual-diff-verdict.js';
+export * from './auth.js';
+export * from './screen-reader.js';
+export * from './job-queue.js';
+export { startSqsWorker } from './sqs-queue.js';
+export * from './session-capture.js';
+export * from './session-export.js';
+export * from './test-executor.js';
+export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
+export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
+export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
+export type { SsoConfig, SsoProvider } from './sso-token.js';
+export * from './engine-router.js';
+export * from './profiles/domain-profile.js';
+export * from './policies/crawl-safety.js';
+export * from './extractors/artifact-envelope.js';
+export * from './network-observer.js';
+export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
+export type { TlsFingerprintOptions } from './tls-fingerprint.js';
+export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
+export type { ExtensionSource, ExtensionProfile } from './browser-extension.js';
+export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
+export type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';
+export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
+export type { UrlCandidate } from './sitemap.js';
+export { httpCrawl } from './http-engine.js';
+export { extractMarkdown } from './markdown-extract.js';
+export type { ExtractedMarkdown } from './markdown-extract.js';
+export type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';
+export { TieredProxyManager } from './proxy.js';
+export type { ProxyTier, ProxyTierConfig } from './proxy.js';
+export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
+export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
+export type { Span, FinishedSpan } from './telemetry.js';

package/types/input-fuzzer.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Input fuzzer — generate test data for form fields.
+ *
+ * For each field type, produces a set of test values covering:
+ * - Happy path (valid, normal input)
+ * - Boundary (min/max length, min/max value)
+ * - Edge cases (empty, very long, special chars, SQL injection probe, XSS probe)
+ * - Type-invalid (wrong type for the field)
+ *
+ * This feeds ZeTa's test generator to automatically create form validation tests.
+ * Values are safe probes (NOT actual exploits) — they test whether the app
+ * properly rejects/accepts boundary inputs.
+ */
+import type { FormField } from './form-detector.js';
+export type InputCase = {
+    label: string;
+    value: string;
+    expectedValid: boolean;
+    category: 'happy' | 'boundary' | 'edge' | 'invalid';
+};
+export declare function generateFieldCases(field: FormField): InputCase[];
+export interface FuzzedForm {
+    formIndex: number;
+    purpose: string;
+    fieldCases: Array<{
+        field: FormField;
+        cases: InputCase[];
+    }>;
+}
+export declare function fuzzForm(form: import('./form-detector.js').DetectedForm): FuzzedForm;

package/types/interactive-probe.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Interactive Probe — Phase 2 Dynamic Observation
+ *
+ * The static crawl captures DOM structure at rest. This module runs AFTER the static crawl
+ * to observe dynamic behaviour by actually interacting with the UI:
+ *
+ *   1. Empty submit  → captures required-field validation error messages
+ *   2. Invalid format → captures format validation errors (bad email, short password)
+ *   3. Button clicks → captures modal/dialog contents
+ *   4. Select changes → captures conditional reveals (show/hide logic)
+ *   5. Success paths → captures toast/banner messages (where input is safe)
+ *
+ * Results are stored as KnowledgeChunks (sourceType='probe') and update the
+ * ERROR_CATALOGUE + INTERACTION_PATTERNS skill files with real observed data
+ * instead of LLM inference.
+ *
+ * Call signature: probeScreen(page, screenId, elements, opts)
+ * Called from crawlProject() after static element extraction when opts.interactive=true.
+ */
+import type { Page } from 'playwright';
+export interface ProbeElementFact {
+    meaning: string;
+    role: string;
+    expectedData?: string;
+}
+export interface ValidationError {
+    field: string;
+    trigger: 'empty_submit' | 'invalid_format' | 'boundary';
+    errorText: string;
+}
+export interface SuccessMessage {
+    type: 'toast' | 'inline_success' | 'redirect';
+    content: string;
+}
+export interface ModalObservation {
+    triggerElement: string;
+    title: string;
+    content: string;
+    fields: string[];
+}
+export interface ConditionalReveal {
+    triggerElement: string;
+    triggerValue: string;
+    revealedText: string;
+}
+export interface ProbeResult {
+    screenId: string;
+    url: string;
+    validationErrors: ValidationError[];
+    successMessages: SuccessMessage[];
+    modals: ModalObservation[];
+    conditionals: ConditionalReveal[];
+    loadingPatterns: string[];
+}
+/**
+ * Run interactive probing on a single screen page.
+ * The page must already be navigated to the screen URL.
+ * `elements` is the output of the static extractElements pass.
+ */
+export declare function probeScreen(page: Page, screenId: string, url: string, elements: ProbeElementFact[]): Promise<ProbeResult>;
+/** Phase 5: Convert ProbeResult to typed ProbeObservationFact rows for direct DB storage. */
+export declare function probeResultToObservations(result: ProbeResult): Array<{
+    elementMeaning: string;
+    observationType: string;
+    observedText: string;
+    triggeredBy: string;
+}>;
+/**
+ * Convert a ProbeResult into markdown to store as KnowledgeChunks
+ * and update ERROR_CATALOGUE + INTERACTION_PATTERNS skill files.
+ */
+export declare function probeResultToMarkdown(result: ProbeResult): {
+    errorCatalogueChunk: string;
+    interactionPatternsChunk: string;
+};
+/**
+ * Run interactive probing for all screens discovered in a crawl session.
+ * Called from crawlProject() when opts.interactive = true.
+ */
+export declare function probeAllScreens(page: any, tenantId: string, projectId: string, discovered: Array<{
+    url: string;
+    screenId?: string;
+    elements?: any[];
+}>, onProgress?: (pct: number, step: string) => Promise<void>): Promise<void>;
+export interface FocusableElement {
+    tag: string;
+    role: string | null;
+    ariaLabel: string | null;
+    text: string | null;
+    hasFocusIndicator: boolean;
+    tabIndex: number | null;
+}
+export interface KeyboardNavResult {
+    focusableCount: number;
+    elementsWithNoFocusIndicator: number;
+    sequence: FocusableElement[];
+    issues: string[];
+}
+/**
+ * Probe keyboard navigation accessibility by pressing Tab up to 50 times and
+ * recording each focused element's properties and focus-indicator visibility.
+ * Stops early if focus wraps back to the start or hits the same element twice.
+ */
+export declare function probeKeyboardNav(page: Page): Promise<KeyboardNavResult>;