@taodigital/crawler 0.0.0-stage → 1.2.1

package/types/test-executor.d.ts

@@ -0,0 +1,84 @@
+import { type Page } from 'playwright';
+import type { ExecutionStep } from '@taodigital/agents';
+export interface StepResult {
+    stepNumber: number;
+    description: string;
+    status: 'PASSED' | 'FAILED' | 'SKIPPED';
+    duration: number;
+    error?: string;
+    screenshotPath?: string;
+    healed?: boolean;
+    /** 'mechanical' = same selector text, different Playwright locator strategy.
+     *  'ai' = AI Self-Healing Agent matched a semantically different element
+     *  (the app's selector text itself changed, e.g. "Sign In" -> "Log In"). */
+    healedVia?: 'mechanical' | 'ai';
+    healReasoning?: string;
+    healConfidence?: number;
+    elementBoundingRect?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    } | null;
+    elementSelector?: string;
+}
+export interface ExecutionResult {
+    stepResults: StepResult[];
+    status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';
+    errorStep?: number;
+    totalDuration: number;
+    finalScreenshotPath?: string;
+    retryCount?: number;
+    consoleLogs?: Array<{
+        type: string;
+        text: string;
+        timestamp: number;
+    }>;
+    networkErrors?: Array<{
+        url: string;
+        status: number;
+        method: string;
+        timestamp: number;
+    }>;
+    rootCauseSummary?: string;
+    networkTimings?: Array<{
+        url: string;
+        method: string;
+        status: number;
+        duration: number;
+    }>;
+}
+export declare function executeExecutionStep(page: Page, step: ExecutionStep, options?: {
+    healCtx?: {
+        tenantId: string;
+        projectId: string;
+    };
+    allowUnsafeLocalUrlsForTests?: boolean;
+}): Promise<Pick<StepResult, 'healed' | 'healedVia' | 'healReasoning' | 'healConfidence' | 'elementBoundingRect' | 'elementSelector'>>;
+export declare function executeTestPlan(plan: ExecutionStep[], options: {
+    screenshotDir: string;
+    testRunId: string;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    onStepComplete?: (result: StepResult) => void;
+    /** Run the test up to this many times to detect flakiness (default 1). */
+    maxRuns?: number;
+    browser?: 'chromium' | 'firefox' | 'webkit';
+    /** Data row for parameterized execution: substitutes {{varname}} in step fields. */
+    dataRow?: Record<string, string>;
+    /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved
+     *  even after trying alternate locator strategies. Omit to keep mechanical-only
+     *  healing (no LLM calls during execution). */
+    healCtx?: {
+        tenantId: string;
+        projectId: string;
+    };
+    /** Decrypted Playwright storageState JSON — restores a logged-in session so
+     *  steps don't have to re-authenticate. Without this, any test case behind a
+     *  login wall fails at the first authenticated step. */
+    storageState?: string;
+    /** Test fixtures may run on loopback; production callers must leave this unset. */
+    allowUnsafeLocalUrlsForTests?: boolean;
+}): Promise<ExecutionResult>;

package/types/tls-fingerprint.d.ts

@@ -0,0 +1,50 @@
+/**
+ * TLS fingerprint spoofing — Chrome JA3 cipher suite ordering.
+ *
+ * Node.js TLS doesn't expose JA3 hash control directly, but ordering
+ * cipher suites to match Chrome's TLS ClientHello reduces fingerprint
+ * detectability vs the default OpenSSL order.
+ *
+ * For full JA3 bypass, pair with the browser-stealth engine (Camoufox).
+ */
+import { Agent as HttpsAgent } from 'node:https';
+export declare const CHROME_CIPHERS: string;
+export declare const CHROME_CURVES = "X25519:P-256:P-384";
+export interface TlsFingerprintOptions {
+    rejectUnauthorized?: boolean;
+    keepAlive?: boolean;
+    keepAliveMsecs?: number;
+    maxSockets?: number;
+}
+/**
+ * Returns an HTTPS agent configured with Chrome-like TLS settings.
+ * Drop-in replacement for the default agent in fetch/got/axios.
+ */
+export declare function createChromeTlsAgent(opts?: TlsFingerprintOptions): HttpsAgent;
+/**
+ * Probe a host and return its JA3-relevant TLS parameters.
+ * Useful for verifying that the fingerprint changed vs default.
+ */
+export declare function probeTlsFingerprint(host: string, port?: number): Promise<{
+    protocol: string;
+    cipher: string;
+    authorized: boolean;
+    subjectCN: string | null;
+}>;
+/**
+ * Node.js fetch dispatcher options with Chrome TLS agent.
+ * Pass as the `dispatcher` option to undici or as `agent` to node-fetch.
+ */
+export declare const chromeTlsAgent: HttpsAgent;
+export declare const CHROME_TLS_HEADERS: {
+    'Accept-Encoding': string;
+    'Accept-Language': string;
+    'sec-ch-ua': string;
+    'sec-ch-ua-mobile': string;
+    'sec-ch-ua-platform': string;
+    'Sec-Fetch-Dest': string;
+    'Sec-Fetch-Mode': string;
+    'Sec-Fetch-Site': string;
+    'Sec-Fetch-User': string;
+    'Upgrade-Insecure-Requests': string;
+};

package/types/url-pattern-completer.d.ts

@@ -0,0 +1,28 @@
+/**
+ * P1: URL Pattern LLM Completion
+ *
+ * After BFS crawl completes, runs a single LLM call to predict URLs that exist
+ * but were never linked from within the app. Validates each candidate with
+ * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).
+ *
+ * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on
+ * candidate count. Does NOT launch a browser.
+ */
+export interface UrlCandidate {
+    url: string;
+    confidence: number;
+    reason: string;
+    status: 'hit' | 'miss' | 'error';
+}
+interface LlmCfg {
+    provider: string;
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+/**
+ * Returns validated URL candidates (HEAD=200) predicted by LLM from
+ * the pattern of already-discovered URLs.
+ */
+export declare function runUrlPatternCompletion(discoveredUrls: string[], appUrl: string, llmCfg: LlmCfg | null | undefined): Promise<UrlCandidate[]>;
+export {};

package/types/visual-diff-verdict.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
+ * transitively @taodigital/database, which instantiates a PrismaClient at module load and
+ * so requires a live DATABASE_URL just to import the module).
+ */
+export interface VisualDiffVerdict {
+    changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';
+    recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';
+    changes: Array<{
+        location: string;
+        severity: string;
+        description: string;
+    }>;
+    bugReportDraft?: {
+        title: string;
+        severity: string;
+        steps: string[];
+        expected: string;
+        actual: string;
+    };
+    aiSummary?: string;
+}
+/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
+export declare function pHashVerdict(pHashDistance: number, changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT', changes: Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>): VisualDiffVerdict;
+/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
+ *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
+ *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
+ *  as NONE/MINOR — only what survives AI review is worth a draft). */
+export declare function mergeAiVisualVerdict(pHashResult: VisualDiffVerdict, aiResult: {
+    changeType: string;
+    changes: any[];
+    recommendation: string;
+    diffScore: number;
+    summary?: string;
+}, screenName: string): VisualDiffVerdict;

package/types/webhook-notifier.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
+ *
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
+ * Retries 3x on failure with exponential backoff.
+ */
+export type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';
+export interface WebhookPayload {
+    event: CrawlEventType;
+    jobId: string;
+    projectId: string;
+    tenantId: string;
+    timestamp: string;
+    data: Record<string, unknown>;
+}
+export interface WebhookNotifierOpts {
+    url: string;
+    secret?: string;
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    maxRetries?: number;
+}
+export declare class WebhookNotifier {
+    private readonly opts;
+    constructor(opts: WebhookNotifierOpts);
+    notify(event: CrawlEventType, jobId: string, projectId: string, tenantId: string, data?: Record<string, unknown>): Promise<void>;
+    /** Slack-compatible message format */
+    static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown>;
+}

package/types/websocket-detector.d.ts

@@ -0,0 +1,24 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
+export interface WsConnection {
+    url: string;
+    framesSent: number;
+    framesReceived: number;
+    /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */
+    samples: Array<{
+        direction: 'sent' | 'received';
+        payload: string;
+        ts: string;
+    }>;
+    /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */
+    protocol: string;
+    openedAt: string;
+}
+export declare function attachWebSocketDetector(cdpSession: any): {
+    getConnections(): WsConnection[];
+    detach(): void;
+};

package/types/worker.d.ts

@@ -0,0 +1 @@
+export {};