@detiq/crawler 0.1.3

package/types/skyvern.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Skyvern Vision Fallback
+ *
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
+ * government portals. It's the "break glass" option when Stagehand fails.
+ *
+ * Production setup (Phase 4+):
+ *   1. Self-host: https://github.com/Skyvern-AI/skyvern
+ *   2. Or use cloud: https://app.skyvern.com
+ *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
+ *
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
+ * Use only when the fast path fails.
+ */
+export interface SkyvernTask {
+    taskDescription: string;
+    url: string;
+    credentials?: {
+        username: string;
+        password: string;
+        oneTimeCode?: string;
+    };
+}
+export interface SkyvernConfig {
+    apiUrl: string;
+    apiKey: string;
+    llmKey?: string;
+    llmApiKey?: string;
+}
+export interface SkyvernStep {
+    order: number;
+    action_type: string;
+    element_description?: string;
+    reasoning?: string;
+    success: boolean;
+    screenshot_url?: string;
+    duration_ms?: number;
+    error?: string;
+}
+export interface SkyvernNavigateResult {
+    screenshotUrls: string[];
+    visitedUrls: string[];
+    status: 'success' | 'failed' | 'partial';
+    steps?: SkyvernStep[];
+    taskId?: string;
+    durationMs?: number;
+    triggerReason?: string;
+}
+/**
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
+ */
+export declare function skyvernNavigate(startUrl: string, navigationGoals: string[], tenantConfig?: SkyvernConfig, triggerReason?: string): Promise<SkyvernNavigateResult>;
+export declare function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'>;

package/types/sqs-queue.d.ts

@@ -0,0 +1,2 @@
+export declare function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void>;
+export declare function startSqsWorker(): Promise<void>;

package/types/sso-token.d.ts

@@ -0,0 +1,29 @@
+/**
+ * SSO token retrieval for Okta and Azure AD.
+ * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
+ * without browser-based login.
+ *
+ * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
+ * for service accounts and automated testing scenarios.
+ */
+export type SsoProvider = 'okta' | 'azure_ad' | 'generic_oidc';
+export interface SsoConfig {
+    provider: SsoProvider;
+    /** Okta domain (e.g. 'mycompany.okta.com') or Azure tenant ID */
+    domain: string;
+    /** OAuth2 client ID */
+    clientId: string;
+    /** OAuth2 client secret (for client_credentials or confidential ROPC) */
+    clientSecret?: string;
+    /** Scopes to request (space-separated). Default: 'openid profile' */
+    scope?: string;
+}
+/**
+ * Get an access token using ROPC flow (username + password → token).
+ * Returns the access token string.
+ */
+export declare function getSsoToken(config: SsoConfig, username: string, password: string): Promise<string>;
+/**
+ * Get a client credentials token (no user — for service accounts).
+ */
+export declare function getSsoClientCredentialsToken(config: SsoConfig): Promise<string>;

package/types/stealth/human-behavior.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright';
+export declare function humanDelay(min?: number, max?: number): Promise<void>;
+export declare function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void>;
+export declare function simulateMouseHover(page: Page, selector: string): Promise<void>;
+export declare function simulateClick(page: Page, selector: string): Promise<void>;
+export declare function injectFingerprintOverrides(page: Page): Promise<void>;
+export declare function infiniteScrollToBottom(page: Page, opts?: {
+    maxScrolls?: number;
+    scrollDelay?: number;
+    stabilizeMs?: number;
+}): Promise<{
+    scrollCount: number;
+    heightReached: number;
+}>;
+export declare function extractIframeContent(page: Page): Promise<Array<{
+    src: string;
+    content: string;
+    markdown: string;
+}>>;
+export declare function pierceShadowDom(page: Page): Promise<string>;

package/types/telemetry.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Lightweight OTel-compatible tracing for Zeta crawler.
+ *
+ * Emits spans as structured JSON logs compatible with OpenTelemetry format.
+ * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
+ *
+ * Usage:
+ *   const span = startSpan('crawl.url', { url, engine });
+ *   // ... do work ...
+ *   endSpan(span, { screensFound: 3 });
+ */
+export interface Span {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startTimeMs: number;
+    attributes: Record<string, string | number | boolean>;
+}
+export interface FinishedSpan extends Span {
+    endTimeMs: number;
+    durationMs: number;
+    status: 'ok' | 'error';
+    error?: string;
+}
+/** Start a new span. If a trace is already active, the new span is a child. */
+export declare function startSpan(name: string, attributes?: Record<string, string | number | boolean>): Span;
+/** End a span and emit it as a structured log line. */
+export declare function endSpan(span: Span, extraAttributes?: Record<string, string | number | boolean>, error?: Error): FinishedSpan;
+/** Convenience: wrap an async function in a span. */
+export declare function withSpan<T>(name: string, attributes: Record<string, string | number | boolean>, fn: (span: Span) => Promise<T>): Promise<T>;
+export declare function increment(metric: string, value?: number, tags?: Record<string, string>): void;
+/** Flush counters as a structured log — call periodically (e.g. at job end). */
+export declare function flushMetrics(): Record<string, number>;

package/types/test-executor.d.ts

@@ -0,0 +1,67 @@
+import type { ExecutionStep } from '@detiq/agents';
+export interface StepResult {
+    stepNumber: number;
+    description: string;
+    status: 'PASSED' | 'FAILED' | 'SKIPPED';
+    duration: number;
+    error?: string;
+    screenshotPath?: string;
+    healed?: boolean;
+    /** 'mechanical' = same selector text, different Playwright locator strategy.
+     *  'ai' = AI Self-Healing Agent matched a semantically different element
+     *  (the app's selector text itself changed, e.g. "Sign In" -> "Log In"). */
+    healedVia?: 'mechanical' | 'ai';
+    healReasoning?: string;
+    healConfidence?: number;
+}
+export interface ExecutionResult {
+    stepResults: StepResult[];
+    status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY';
+    errorStep?: number;
+    totalDuration: number;
+    finalScreenshotPath?: string;
+    retryCount?: number;
+    consoleLogs?: Array<{
+        type: string;
+        text: string;
+        timestamp: number;
+    }>;
+    networkErrors?: Array<{
+        url: string;
+        status: number;
+        method: string;
+        timestamp: number;
+    }>;
+    rootCauseSummary?: string;
+    networkTimings?: Array<{
+        url: string;
+        method: string;
+        status: number;
+        duration: number;
+    }>;
+}
+export declare function executeTestPlan(plan: ExecutionStep[], options: {
+    screenshotDir: string;
+    testRunId: string;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    onStepComplete?: (result: StepResult) => void;
+    /** Run the test up to this many times to detect flakiness (default 1). */
+    maxRuns?: number;
+    browser?: 'chromium' | 'firefox' | 'webkit';
+    /** Data row for parameterized execution: substitutes {{varname}} in step fields. */
+    dataRow?: Record<string, string>;
+    /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved
+     *  even after trying alternate locator strategies. Omit to keep mechanical-only
+     *  healing (no LLM calls during execution). */
+    healCtx?: {
+        tenantId: string;
+        projectId: string;
+    };
+    /** Decrypted Playwright storageState JSON — restores a logged-in session so
+     *  steps don't have to re-authenticate. Without this, any test case behind a
+     *  login wall fails at the first authenticated step. */
+    storageState?: string;
+}): Promise<ExecutionResult>;

package/types/tls-fingerprint.d.ts

@@ -0,0 +1,50 @@
+/**
+ * TLS fingerprint spoofing — Chrome JA3 cipher suite ordering.
+ *
+ * Node.js TLS doesn't expose JA3 hash control directly, but ordering
+ * cipher suites to match Chrome's TLS ClientHello reduces fingerprint
+ * detectability vs the default OpenSSL order.
+ *
+ * For full JA3 bypass, pair with the browser-stealth engine (Camoufox).
+ */
+import { Agent as HttpsAgent } from 'node:https';
+export declare const CHROME_CIPHERS: string;
+export declare const CHROME_CURVES = "X25519:P-256:P-384";
+export interface TlsFingerprintOptions {
+    rejectUnauthorized?: boolean;
+    keepAlive?: boolean;
+    keepAliveMsecs?: number;
+    maxSockets?: number;
+}
+/**
+ * Returns an HTTPS agent configured with Chrome-like TLS settings.
+ * Drop-in replacement for the default agent in fetch/got/axios.
+ */
+export declare function createChromeTlsAgent(opts?: TlsFingerprintOptions): HttpsAgent;
+/**
+ * Probe a host and return its JA3-relevant TLS parameters.
+ * Useful for verifying that the fingerprint changed vs default.
+ */
+export declare function probeTlsFingerprint(host: string, port?: number): Promise<{
+    protocol: string;
+    cipher: string;
+    authorized: boolean;
+    subjectCN: string | null;
+}>;
+/**
+ * Node.js fetch dispatcher options with Chrome TLS agent.
+ * Pass as the `dispatcher` option to undici or as `agent` to node-fetch.
+ */
+export declare const chromeTlsAgent: HttpsAgent;
+export declare const CHROME_TLS_HEADERS: {
+    'Accept-Encoding': string;
+    'Accept-Language': string;
+    'sec-ch-ua': string;
+    'sec-ch-ua-mobile': string;
+    'sec-ch-ua-platform': string;
+    'Sec-Fetch-Dest': string;
+    'Sec-Fetch-Mode': string;
+    'Sec-Fetch-Site': string;
+    'Sec-Fetch-User': string;
+    'Upgrade-Insecure-Requests': string;
+};

package/types/url-pattern-completer.d.ts

@@ -0,0 +1,28 @@
+/**
+ * P1: URL Pattern LLM Completion
+ *
+ * After BFS crawl completes, runs a single LLM call to predict URLs that exist
+ * but were never linked from within the app. Validates each candidate with
+ * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).
+ *
+ * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on
+ * candidate count. Does NOT launch a browser.
+ */
+export interface UrlCandidate {
+    url: string;
+    confidence: number;
+    reason: string;
+    status: 'hit' | 'miss' | 'error';
+}
+interface LlmCfg {
+    provider: string;
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+/**
+ * Returns validated URL candidates (HEAD=200) predicted by LLM from
+ * the pattern of already-discovered URLs.
+ */
+export declare function runUrlPatternCompletion(discoveredUrls: string[], appUrl: string, llmCfg: LlmCfg | null | undefined): Promise<UrlCandidate[]>;
+export {};

package/types/visual-diff-verdict.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
+ * transitively @detiq/database, which instantiates a PrismaClient at module load and
+ * so requires a live DATABASE_URL just to import the module).
+ */
+export interface VisualDiffVerdict {
+    changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';
+    recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';
+    changes: Array<{
+        location: string;
+        severity: string;
+        description: string;
+    }>;
+    bugReportDraft?: {
+        title: string;
+        severity: string;
+        steps: string[];
+        expected: string;
+        actual: string;
+    };
+    aiSummary?: string;
+}
+/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
+export declare function pHashVerdict(pHashDistance: number, changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT', changes: Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>): VisualDiffVerdict;
+/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
+ *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
+ *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
+ *  as NONE/MINOR — only what survives AI review is worth a draft). */
+export declare function mergeAiVisualVerdict(pHashResult: VisualDiffVerdict, aiResult: {
+    changeType: string;
+    changes: any[];
+    recommendation: string;
+    diffScore: number;
+    summary?: string;
+}, screenName: string): VisualDiffVerdict;

package/types/webhook-notifier.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
+ *
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
+ * Retries 3x on failure with exponential backoff.
+ */
+export type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';
+export interface WebhookPayload {
+    event: CrawlEventType;
+    jobId: string;
+    projectId: string;
+    tenantId: string;
+    timestamp: string;
+    data: Record<string, unknown>;
+}
+export interface WebhookNotifierOpts {
+    url: string;
+    secret?: string;
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    maxRetries?: number;
+}
+export declare class WebhookNotifier {
+    private readonly opts;
+    constructor(opts: WebhookNotifierOpts);
+    notify(event: CrawlEventType, jobId: string, projectId: string, tenantId: string, data?: Record<string, unknown>): Promise<void>;
+    /** Slack-compatible message format */
+    static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown>;
+}

package/types/websocket-detector.d.ts

@@ -0,0 +1,24 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
+export interface WsConnection {
+    url: string;
+    framesSent: number;
+    framesReceived: number;
+    /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */
+    samples: Array<{
+        direction: 'sent' | 'received';
+        payload: string;
+        ts: string;
+    }>;
+    /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */
+    protocol: string;
+    openedAt: string;
+}
+export declare function attachWebSocketDetector(cdpSession: any): {
+    getConnections(): WsConnection[];
+    detach(): void;
+};

package/types/worker.d.ts

@@ -0,0 +1 @@
+export {};