opendevbrowser 0.0.12 → 0.0.16

package/dist/providers/types.d.ts

@@ -0,0 +1,344 @@
+export type ProviderSource = "web" | "community" | "social" | "shopping";
+export type ProviderSelection = "auto" | ProviderSource | "all";
+export type ProviderOperation = "search" | "fetch" | "crawl" | "post";
+export type ProviderTier = "A" | "B" | "C";
+export type ProviderTierReasonCode = "default_tier" | "operator_override" | "restricted_safe_forced" | "challenge_pressure" | "high_friction_target" | "hybrid_eligible" | "hybrid_unhealthy" | "hybrid_risk_threshold" | "hybrid_latency_budget" | "hybrid_error_budget" | "policy_restricted_safe" | "restricted_safe_recovered" | "hybrid_disabled" | "restricted_safe_disabled" | "fallback_to_tier_a";
+export type ProviderReasonCode = "ip_blocked" | "token_required" | "auth_required" | "challenge_detected" | "rate_limited" | "caption_missing" | "env_limited" | "transcript_unavailable" | "policy_blocked" | "cooldown_active" | "strategy_unapproved";
+export type ProviderCookiePolicy = "off" | "auto" | "required";
+export type ProviderCookieImportRecord = {
+    name: string;
+    value: string;
+    url?: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: "Strict" | "Lax" | "None";
+};
+export type ProviderCookieSourceConfig = {
+    type: "file";
+    value: string;
+} | {
+    type: "env";
+    value: string;
+} | {
+    type: "inline";
+    value: ProviderCookieImportRecord[];
+};
+export type JsonPrimitive = string | number | boolean | null;
+export type JsonValue = JsonPrimitive | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+export type BlockerType = "auth_required" | "anti_bot_challenge" | "rate_limited" | "upstream_block" | "restricted_target" | "env_limited" | "unknown";
+export type BlockerSource = "navigation" | "network" | "console" | "runtime_fetch" | "macro_execution";
+export type BlockerActionHintId = "manual_login" | "manual_challenge" | "retry_after_backoff" | "switch_managed_headed" | "switch_extension_mode" | "collect_debug_trace";
+export interface BlockerEvidence {
+    url?: string;
+    finalUrl?: string;
+    title?: string;
+    status?: number;
+    providerErrorCode?: string;
+    matchedPatterns: string[];
+    networkHosts: string[];
+    traceRequestId?: string;
+}
+export interface BlockerActionHint {
+    id: BlockerActionHintId;
+    reason: string;
+    priority: 1 | 2 | 3;
+}
+export interface BlockerSanitationDiagnostics {
+    entries: number;
+    quarantinedSegments: number;
+}
+export interface BlockerSignalV1 {
+    schemaVersion: "1.0";
+    type: BlockerType;
+    source: BlockerSource;
+    reasonCode?: ProviderReasonCode;
+    confidence: number;
+    retryable: boolean;
+    detectedAt: string;
+    evidence: BlockerEvidence;
+    actionHints: BlockerActionHint[];
+    sanitation?: BlockerSanitationDiagnostics;
+}
+export interface BlockerArtifactCaps {
+    maxNetworkEvents: number;
+    maxConsoleEvents: number;
+    maxExceptionEvents: number;
+    maxHosts: number;
+    maxTextLength: number;
+}
+export interface BlockerArtifactsV1 {
+    schemaVersion: "1.0";
+    network: Array<Record<string, JsonValue>>;
+    console: Array<Record<string, JsonValue>>;
+    exception: Array<Record<string, JsonValue>>;
+    hosts: string[];
+    sanitation: BlockerSanitationDiagnostics;
+}
+export type ProviderErrorCode = "invalid_input" | "timeout" | "network" | "rate_limited" | "auth" | "upstream" | "not_supported" | "policy_blocked" | "circuit_open" | "unavailable" | "internal";
+export interface TraceContext {
+    requestId: string;
+    sessionId?: string;
+    targetId?: string;
+    provider?: string;
+    ts: string;
+}
+export interface NormalizedRecord {
+    id: string;
+    source: ProviderSource;
+    provider: string;
+    url?: string;
+    title?: string;
+    content?: string;
+    timestamp: string;
+    confidence: number;
+    attributes: Record<string, JsonValue>;
+}
+export interface ProviderSearchInput {
+    query: string;
+    limit?: number;
+    filters?: Record<string, JsonValue>;
+}
+export interface ProviderFetchInput {
+    url: string;
+    filters?: Record<string, JsonValue>;
+}
+export type CrawlStrategy = "bfs" | "dfs";
+export interface ProviderCrawlInput {
+    seedUrls: string[];
+    strategy?: CrawlStrategy;
+    maxDepth?: number;
+    maxPages?: number;
+    maxPerDomain?: number;
+    filters?: Record<string, JsonValue>;
+}
+export interface ProviderPostInput {
+    target: string;
+    content: string;
+    mediaUrls?: string[];
+    confirm?: boolean;
+    riskAccepted?: boolean;
+    metadata?: Record<string, JsonValue>;
+}
+export type ProviderInput = ProviderSearchInput | ProviderFetchInput | ProviderCrawlInput | ProviderPostInput;
+export interface ProviderError {
+    code: ProviderErrorCode;
+    message: string;
+    retryable: boolean;
+    reasonCode?: ProviderReasonCode;
+    provider?: string;
+    source?: ProviderSource;
+    details?: Record<string, JsonValue>;
+}
+export interface ProviderContext {
+    trace: TraceContext;
+    timeoutMs: number;
+    attempt: number;
+    signal?: AbortSignal;
+    useCookies?: boolean;
+    cookiePolicyOverride?: ProviderCookiePolicy;
+    browserFallbackPort?: BrowserFallbackPort;
+}
+export interface ProviderHealth {
+    status: "healthy" | "degraded" | "unhealthy";
+    updatedAt: string;
+    reason?: string;
+    latencyMs?: number;
+}
+export interface ProviderCapability {
+    op: ProviderOperation;
+    supported: boolean;
+    description?: string;
+    metadata?: Record<string, JsonValue>;
+}
+export interface ProviderCapabilities {
+    providerId: string;
+    source: ProviderSource;
+    operations: Record<ProviderOperation, ProviderCapability>;
+    policy: {
+        posting: "unsupported" | "gated" | "open";
+        riskNoticeRequired: boolean;
+        confirmationRequired: boolean;
+    };
+    metadata: Record<string, JsonValue>;
+}
+export interface ProviderTierMetadata {
+    selected: ProviderTier;
+    reasonCode: ProviderTierReasonCode;
+}
+export interface ProviderProvenanceMetadata {
+    provider: string;
+    retrievalPath: string;
+    retrievedAt: string;
+}
+export interface ProviderExecutionMetadata {
+    tier: ProviderTierMetadata;
+    provenance: ProviderProvenanceMetadata;
+    blocker?: BlockerSignalV1;
+}
+export interface AdaptiveConcurrencyDiagnostics {
+    enabled: boolean;
+    scope: string;
+    global: {
+        limit: number;
+        min: number;
+        max: number;
+    };
+    scoped: {
+        limit: number;
+        min: number;
+        max: number;
+    };
+}
+export interface PromptGuardAudit {
+    enabled: boolean;
+    quarantinedSegments: number;
+    entries: number;
+}
+export interface RealismDiagnostics {
+    violations: number;
+    patterns: string[];
+}
+export interface ProviderRuntimeDiagnostics {
+    adaptiveConcurrency: AdaptiveConcurrencyDiagnostics;
+    promptGuard: PromptGuardAudit;
+    realism: RealismDiagnostics;
+}
+export interface ProviderOperationSuccess {
+    ok: true;
+    records: NormalizedRecord[];
+    trace: TraceContext;
+    provider: string;
+    source: ProviderSource;
+    latencyMs: number;
+    attempts: number;
+    retries: number;
+    meta?: ProviderExecutionMetadata;
+    provenance?: Record<string, JsonValue>;
+    diagnostics?: ProviderRuntimeDiagnostics;
+}
+export interface ProviderOperationFailure {
+    ok: false;
+    error: ProviderError;
+    trace: TraceContext;
+    provider: string;
+    source: ProviderSource;
+    latencyMs: number;
+    attempts: number;
+    retries: number;
+    meta?: ProviderExecutionMetadata;
+    diagnostics?: ProviderRuntimeDiagnostics;
+}
+export type ProviderOperationResult = ProviderOperationSuccess | ProviderOperationFailure;
+export interface ProviderFailureEntry {
+    provider: string;
+    source: ProviderSource;
+    error: ProviderError;
+}
+export interface ProviderAggregateResult {
+    ok: boolean;
+    records: NormalizedRecord[];
+    trace: TraceContext;
+    partial: boolean;
+    failures: ProviderFailureEntry[];
+    metrics: {
+        attempted: number;
+        succeeded: number;
+        failed: number;
+        retries: number;
+        latencyMs: number;
+    };
+    sourceSelection: ProviderSelection;
+    providerOrder: string[];
+    meta?: ProviderExecutionMetadata;
+    diagnostics?: ProviderRuntimeDiagnostics;
+    error?: ProviderError;
+}
+export interface ProviderAdapter {
+    id: string;
+    source: ProviderSource;
+    search?: (input: ProviderSearchInput, context: ProviderContext) => Promise<NormalizedRecord[]>;
+    fetch?: (input: ProviderFetchInput, context: ProviderContext) => Promise<NormalizedRecord[]>;
+    crawl?: (input: ProviderCrawlInput, context: ProviderContext) => Promise<NormalizedRecord[]>;
+    post?: (input: ProviderPostInput, context: ProviderContext) => Promise<NormalizedRecord[]>;
+    health?: (context: Omit<ProviderContext, "attempt">) => Promise<ProviderHealth>;
+    capabilities: () => ProviderCapabilities;
+}
+export interface ProviderRuntimeBudgets {
+    timeoutMs: Record<ProviderOperation, number>;
+    retries: {
+        read: number;
+        write: number;
+    };
+    concurrency: {
+        global: number;
+        perProvider: number;
+        perDomain?: number;
+    };
+    circuitBreaker: {
+        failureThreshold: number;
+        cooldownMs: number;
+    };
+}
+export interface ProviderRunOptions {
+    source?: ProviderSelection;
+    providerIds?: string[];
+    timeoutMs?: number;
+    trace?: Partial<TraceContext>;
+    useCookies?: boolean;
+    cookiePolicyOverride?: ProviderCookiePolicy;
+    tier?: {
+        preferred?: ProviderTier;
+        forceRestrictedSafe?: boolean;
+        challengePressure?: number;
+        highFrictionTarget?: boolean;
+        riskScore?: number;
+        hybridHealthy?: boolean;
+        policyRestrictedSafe?: boolean;
+        latencyBudgetExceeded?: boolean;
+        errorBudgetExceeded?: boolean;
+        recoveryStableForMs?: number;
+        policyAllowsRecovery?: boolean;
+    };
+}
+export type BrowserFallbackMode = "managed_headed" | "extension";
+export interface BrowserFallbackRequest {
+    provider: string;
+    source: ProviderSource;
+    operation: ProviderOperation;
+    reasonCode: ProviderReasonCode;
+    trace: TraceContext;
+    url?: string;
+    details?: Record<string, JsonValue>;
+    preferredModes?: BrowserFallbackMode[];
+    useCookies?: boolean;
+    cookiePolicyOverride?: ProviderCookiePolicy;
+}
+export interface BrowserFallbackResponse {
+    ok: boolean;
+    reasonCode: ProviderReasonCode;
+    mode?: BrowserFallbackMode;
+    output?: Record<string, JsonValue>;
+    details?: Record<string, JsonValue>;
+}
+export interface BrowserFallbackPort {
+    resolve: (request: BrowserFallbackRequest) => Promise<BrowserFallbackResponse>;
+}
+export type ProviderCallResultByOperation = {
+    search: ProviderSearchInput;
+    fetch: ProviderFetchInput;
+    crawl: ProviderCrawlInput;
+    post: ProviderPostInput;
+};
+declare module "./web/crawler" {
+    interface CrawlOptions {
+        workerThreads?: number;
+        queueMax?: number;
+        forceInlineParse?: boolean;
+    }
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/providers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/providers/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,WAAW,GAAG,QAAQ,GAAG,UAAU,CAAC;AACzE,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,cAAc,GAAG,KAAK,CAAC;AAChE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AACtE,MAAM,MAAM,YAAY,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;AAE3C,MAAM,MAAM,sBAAsB,GAC9B,cAAc,GACd,mBAAmB,GACnB,wBAAwB,GACxB,oBAAoB,GACpB,sBAAsB,GACtB,iBAAiB,GACjB,kBAAkB,GAClB,uBAAuB,GACvB,uBAAuB,GACvB,qBAAqB,GACrB,wBAAwB,GACxB,2BAA2B,GAC3B,iBAAiB,GACjB,0BAA0B,GAC1B,oBAAoB,CAAC;AAEzB,MAAM,MAAM,kBAAkB,GAC1B,YAAY,GACZ,gBAAgB,GAChB,eAAe,GACf,oBAAoB,GACpB,cAAc,GACd,iBAAiB,GACjB,aAAa,GACb,wBAAwB,GACxB,gBAAgB,GAChB,iBAAiB,GACjB,qBAAqB,CAAC;AAE1B,MAAM,MAAM,oBAAoB,GAAG,KAAK,GAAG,MAAM,GAAG,UAAU,CAAC;AAE/D,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAClC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,GACC;IACA,IAAI,EAAE,KAAK,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,GACC;IACA,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,0BAA0B,EAAE,CAAC;CACrC,CAAC;AAEJ,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG,SAAS,EAAE,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEnF,MAAM,MAAM,WAAW,GACnB,eAAe,GACf,oBAAoB,GACpB,cAAc,GACd,gBAAgB,GAChB,mBAAmB,GACnB,aAAa,GACb,SAAS,CAAC;AAEd,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,SAAS,GACT,SAAS,GACT,eAAe,GACf,iBAAiB,CAAC;AAEtB,MAAM,MAAM,mBAAmB,GAC3B,cAAc,GACd,kBAAkB,GAClB,qBAAqB,GACrB,uBAAuB,GACvB,uBAAuB,GACvB,qBAAqB,CAAC;AAE1B,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,mBAAmB,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;CACrB;AAED,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,KAAK,CAAC;IACrB,IAAI,EAAE,WAAW,CAAC;IAClB,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,eAAe,CAAC;IAC1B,WAAW,EAAE,iBAAiB,EAAE,CAAC;IACjC,UAAU,CAAC,EAAE,4BAA4B,CAAC;CAC3C;AAED,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,aAAa,EAAE,KAAK,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC1C,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC1C,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC5C,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,4BAA4B,CAAC;CAC1C;AAED,MAAM,MAAM,iBAAiB,GACzB,eAAe,GACf,SAAS,GACT,SAAS,GACT,cAAc,GACd,MAAM,GACN,UAAU,GACV,eAAe,GACf,gBAAgB,GAChB,cAAc,GACd,aAAa,GACb,UAAU,CAAC;AAEf,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,cAAc,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,KAAK,CAAC;AAE1C,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,kBAAkB,GAClB,kBAAkB,GAClB,iBAAiB,CAAC;AAEtB,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,iBAAiB,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,WAAW,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,iBAAiB,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;IAC1D,MAAM,EAAE;QACN,OAAO,EAAE,aAAa,GAAG,OAAO,GAAG,MAAM,CAAC;QAC1C,kBAAkB,EAAE,OAAO,CAAC;QAC5B,oBAAoB,EAAE,OAAO,CAAC;KAC/B,CAAC;IACF,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,YAAY,CAAC;IACvB,UAAU,EAAE,sBAAsB,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,UAAU,EAAE,0BAA0B,CAAC;IACvC,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;IACF,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,0BAA0B;IACzC,mBAAmB,EAAE,8BAA8B,CAAC;IACpD,WAAW,EAAE,gBAAgB,CAAC;IAC9B,OAAO,EAAE,kBAAkB,CAAC;CAC7B;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,IAAI,CAAC;IACT,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,0BAA0B,CAAC;CAC1C;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,KAAK,CAAC;IACV,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC,WAAW,CAAC,EAAE,0BAA0B,CAAC;CAC1C;AAED,MAAM,MAAM,uBAAuB,GAAG,wBAAwB,GAAG,wBAAwB,CAAC;AAE1F,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,oBAAoB,EAAE,CAAC;IACjC,OAAO,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,eAAe,EAAE,iBAAiB,CAAC;IACnC,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC,WAAW,CAAC,EAAE,0BAA0B,CAAC;IACzC,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,cAAc,CAAC;IACvB,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC/F,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC7F,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC7F,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAChF,YAAY,EAAE,MAAM,oBAAoB,CAAC;CAC1C;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAC7C,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,WAAW,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,cAAc,EAAE;QACd,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,IAAI,CAAC,EAAE;QACL,SAAS,CAAC,EAAE,YAAY,CAAC;QACzB,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,oBAAoB,CAAC,EAAE,OAAO,CAAC;QAC/B,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE,OAAO,CAAC;KAChC,CAAC;CACH;AAED,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,GAAG,WAAW,CAAC;AAEjE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,EAAE,iBAAiB,CAAC;IAC7B,UAAU,EAAE,kBAAkB,CAAC;IAC/B,KAAK,EAAE,YAAY,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpC,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,EAAE,kBAAkB,CAAC;IAC/B,IAAI,CAAC,EAAE,mBAAmB,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,OAAO,CAAC,uBAAuB,CAAC,CAAC;CAChF;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,KAAK,EAAE,kBAAkB,CAAC;IAC1B,IAAI,EAAE,iBAAiB,CAAC;CACzB,CAAC;AAEF,OAAO,QAAQ,eAAe,CAAC;IAC7B,UAAU,YAAY;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B;CACF"}

package/dist/providers/web/crawl-worker.d.ts

@@ -0,0 +1,36 @@
+import { type ExtractedContent } from "./extract";
+export interface CrawlExtractInput {
+    url: string;
+    html: string;
+    selectors: string[];
+}
+export interface CrawlWorkerPoolOptions {
+    workerThreads?: number;
+    queueMax?: number;
+    forceInline?: boolean;
+}
+export declare class CrawlWorkerPool {
+    private readonly queueMax;
+    private readonly queue;
+    private readonly inFlight;
+    private readonly workers;
+    private nextTaskId;
+    private inlineOnly;
+    private closed;
+    constructor(options?: CrawlWorkerPoolOptions);
+    extract(input: CrawlExtractInput): Promise<ExtractedContent>;
+    close(): Promise<void>;
+    private spawnWorker;
+    private dispatch;
+    private handleWorkerMessage;
+    private handleWorkerFailure;
+    private parseWorkerResponse;
+    private resolveTaskInline;
+}
+export declare const extractCrawlContentInline: (input: CrawlExtractInput) => ExtractedContent;
+export declare const createCrawlWorkerPool: (options?: CrawlWorkerPoolOptions) => CrawlWorkerPool;
+export declare const __test__: {
+    supportsWorkerThreads: () => boolean;
+    toError: (error: unknown, fallback: string) => Error;
+};
+//# sourceMappingURL=crawl-worker.d.ts.map

package/dist/providers/web/crawl-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-worker.d.ts","sourceRoot":"","sources":["../../../src/providers/web/crawl-worker.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE5E,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAmJD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmB;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgC;IACzD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,MAAM,CAAS;gBAEX,OAAO,GAAE,sBAA2B;IAe1C,OAAO,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsB5D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAqB5B,OAAO,CAAC,WAAW;IA8BnB,OAAO,CAAC,QAAQ;IA6BhB,OAAO,CAAC,mBAAmB;IA8B3B,OAAO,CAAC,mBAAmB;IA+B3B,OAAO,CAAC,mBAAmB;IAoC3B,OAAO,CAAC,iBAAiB;CAO1B;AAED,eAAO,MAAM,yBAAyB,GAAI,OAAO,iBAAiB,KAAG,gBAEpE,CAAC;AAEF,eAAO,MAAM,qBAAqB,GAChC,UAAS,sBAA2B,KACnC,eAEF,CAAC;AAEF,eAAO,MAAM,QAAQ;iCA3Pa,OAAO;qBAIjB,OAAO,YAAY,MAAM,KAAG,KAAK;CA0PxD,CAAC"}

package/dist/providers/web/crawler.d.ts

@@ -0,0 +1,101 @@
+import { type WebCrawlPolicy } from "./policy";
+import type { CrawlStrategy } from "../types";
+export interface CrawlBudget {
+    maxDepth: number;
+    maxPages: number;
+    maxPerDomain: number;
+}
+export interface CrawlPipelineBudget {
+    workerThreads: number;
+    queueMax: number;
+    fetchConcurrency: number;
+    frontierMax: number;
+}
+export interface CrawlPage {
+    url: string;
+    canonicalUrl: string;
+    depth: number;
+    status: number;
+    text: string;
+    links: string[];
+    selectors: Record<string, string[]>;
+    warnings: string[];
+}
+export interface CrawlMetrics {
+    visited: number;
+    fetched: number;
+    deduped: number;
+    elapsedMs: number;
+    pagesPerMinute: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+}
+export interface CrawlResult {
+    pages: CrawlPage[];
+    graph: Array<{
+        from: string;
+        to: string;
+    }>;
+    warnings: string[];
+    metrics: CrawlMetrics;
+}
+export interface CrawlFetcherResponse {
+    url?: string;
+    html: string;
+    status?: number;
+}
+export type CrawlFetcher = (url: string) => Promise<CrawlFetcherResponse>;
+export interface CrawlOptions {
+    seeds: string[];
+    strategy?: CrawlStrategy;
+    budget?: Partial<CrawlBudget>;
+    policy?: WebCrawlPolicy;
+    selectors?: string[];
+    pipeline?: Partial<CrawlPipelineBudget>;
+    workerThreads?: number;
+    queueMax?: number;
+    forceInlineParse?: boolean;
+    fetcher: CrawlFetcher;
+}
+interface FrontierNode {
+    url: string;
+    depth: number;
+    sequence: number;
+    firstSeenOrder: number;
+    firstSeenAtMs: number;
+    sourcePriority: number;
+    stableRecordId: string;
+}
+interface CrawlTaskResult {
+    sequence: number;
+    firstSeenAtMs: number;
+    sourcePriority: number;
+    stableRecordId: string;
+    page: CrawlPage | null;
+    links: string[];
+    warnings: string[];
+    latencyMs: number;
+}
+interface CrawlPageEntry {
+    page: CrawlPage;
+    firstSeenAtMs: number;
+    sourcePriority: number;
+    stableRecordId: string;
+    sequence: number;
+}
+export declare const canonicalizeUrl: (rawUrl: string) => string;
+export declare const crawlWeb: (options: CrawlOptions) => Promise<CrawlResult>;
+export declare const __test__: {
+    percentile: (values: number[], ratio: number) => number;
+    isHttpUrl: (value: string) => boolean;
+    compareFrontierNodes: (left: FrontierNode, right: FrontierNode, strategy: CrawlStrategy) => number;
+    dequeueNode: (frontier: FrontierNode[], strategy: CrawlStrategy) => FrontierNode | undefined;
+    resolveFrontierDomain: (url: string) => string;
+    enqueueFrontierNode: (frontierByDomain: Map<string, FrontierNode[]>, node: FrontierNode) => void;
+    frontierSize: (frontierByDomain: Map<string, FrontierNode[]>) => number;
+    dequeueFrontierByDomain: (frontierByDomain: Map<string, FrontierNode[]>, strategy: CrawlStrategy) => FrontierNode | undefined;
+    sortPageEntries: (entries: CrawlPageEntry[]) => CrawlPage[];
+    waitForNextTask: (tasks: Map<number, Promise<CrawlTaskResult>>) => Promise<CrawlTaskResult>;
+};
+export {};
+//# sourceMappingURL=crawler.d.ts.map

package/dist/providers/web/crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../../src/providers/web/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,KAAK,cAAc,EAAE,MAAM,UAAU,CAAC;AAEvE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACpC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,YAAY,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAE1E,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACxC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE,YAAY,CAAC;CACvB;AAED,UAAU,YAAY;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,SAAS,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,UAAU,cAAc;IACtB,IAAI,EAAE,SAAS,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAyBD,eAAO,MAAM,eAAe,GAAI,QAAQ,MAAM,KAAG,MAkChD,CAAC;AAkNF,eAAO,MAAM,QAAQ,GAAU,SAAS,YAAY,KAAG,OAAO,CAAC,WAAW,CAwMzE,CAAC;AAEF,eAAO,MAAM,QAAQ;yBA1ZO,MAAM,EAAE,SAAS,MAAM,KAAG,MAAM;uBAOlC,MAAM,KAAG,OAAO;iCAUlC,YAAY,SACX,YAAY,YACT,aAAa,KACtB,MAAM;4BAuBG,YAAY,EAAE,YACd,aAAa,KACtB,YAAY,GAAG,SAAS;iCAMS,MAAM,KAAG,MAAM;4CAS/B,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,QACvC,YAAY,KACjB,IAAI;qCAOiC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,KAAG,MAAM;gDASxD,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,YACnC,aAAa,KACtB,YAAY,GAAG,SAAS;+BAoCO,cAAc,EAAE,KAAG,SAAS,EAAE;6BALvD,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,KAC3C,OAAO,CAAC,eAAe,CAAC;CAqT1B,CAAC"}

package/dist/providers/web/extract.d.ts

@@ -0,0 +1,11 @@
+export interface ExtractedContent {
+    text: string;
+    links: string[];
+    selectors: Record<string, string[]>;
+}
+export declare const extractText: (html: string) => string;
+export declare const extractLinks: (html: string, baseUrl: string) => string[];
+export declare const extractSelectors: (html: string, selectors?: string[]) => Record<string, string[]>;
+export declare const extractStructuredContent: (html: string, baseUrl: string, selectors?: string[]) => ExtractedContent;
+export declare const toSnippet: (text: string, maxChars?: number) => string;
+//# sourceMappingURL=extract.d.ts.map

package/dist/providers/web/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../../src/providers/web/extract.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACrC;AAOD,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,MAM1C,CAAC;AAcF,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,EAAE,SAAS,MAAM,KAAG,MAAM,EAUlE,CAAC;AAeF,eAAO,MAAM,gBAAgB,GAAI,MAAM,MAAM,EAAE,YAAW,MAAM,EAAO,KAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAahG,CAAC;AAEF,eAAO,MAAM,wBAAwB,GACnC,MAAM,MAAM,EACZ,SAAS,MAAM,EACf,YAAW,MAAM,EAAO,KACvB,gBAMF,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,iBAAc,KAAG,MAGxD,CAAC"}

package/dist/providers/web/index.d.ts

@@ -0,0 +1,24 @@
+import { type CrawlBudget, type CrawlFetcher, type CrawlPipelineBudget } from "./crawler";
+import type { JsonValue, ProviderAdapter, ProviderContext, ProviderSearchInput } from "../types";
+import type { WebCrawlPolicy } from "./policy";
+export interface WebSearchRecord {
+    url: string;
+    title?: string;
+    content?: string;
+    confidence?: number;
+    attributes?: Record<string, JsonValue>;
+}
+export interface WebProviderOptions {
+    id?: string;
+    fetcher?: CrawlFetcher;
+    searchIndex?: (input: ProviderSearchInput, context: ProviderContext) => Promise<WebSearchRecord[]>;
+    defaultBudget?: Partial<CrawlBudget>;
+    defaultPipeline?: Partial<CrawlPipelineBudget>;
+    selectors?: string[];
+    workerThreads?: number;
+    queueMax?: number;
+    forceInlineParse?: boolean;
+    policy?: WebCrawlPolicy;
+}
+export declare const createWebProvider: (options?: WebProviderOptions) => ProviderAdapter;
+//# sourceMappingURL=index.d.ts.map

package/dist/providers/web/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/providers/web/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAY,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEpG,OAAO,KAAK,EACV,SAAS,EAET,eAAe,EAEf,eAAe,EAGf,mBAAmB,EACpB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACnG,aAAa,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IACrC,eAAe,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC/C,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AA4DD,eAAO,MAAM,iBAAiB,GAAI,UAAS,kBAAuB,KAAG,eAsLpE,CAAC"}

package/dist/providers/web/policy.d.ts

@@ -0,0 +1,14 @@
+export type RobotsMode = "strict" | "warn" | "off";
+export interface WebCrawlPolicy {
+    robotsMode?: RobotsMode;
+    allowDomains?: string[];
+    denyDomains?: string[];
+    robotsBlockedDomains?: string[];
+}
+export interface CrawlPolicyDecision {
+    allowed: boolean;
+    warnings: string[];
+    reason?: string;
+}
+export declare const evaluateWebCrawlPolicy: (rawUrl: string, policy?: WebCrawlPolicy) => CrawlPolicyDecision;
+//# sourceMappingURL=policy.d.ts.map

package/dist/providers/web/policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy.d.ts","sourceRoot":"","sources":["../../../src/providers/web/policy.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAC;AAEnD,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAUD,eAAO,MAAM,sBAAsB,GACjC,QAAQ,MAAM,EACd,SAAQ,cAAmB,KAC1B,mBAiDF,CAAC"}

package/dist/providers/workflows.d.ts

@@ -0,0 +1,67 @@
+import { type ResearchRecord } from "./enrichment";
+import { type RenderMode } from "./renderer";
+import type { ProviderAggregateResult, ProviderCallResultByOperation, ProviderCookiePolicy, ProviderRunOptions, ProviderSelection, ProviderSource } from "./types";
+export interface ProviderExecutor {
+    search: (input: ProviderCallResultByOperation["search"], options?: ProviderRunOptions) => Promise<ProviderAggregateResult>;
+    fetch: (input: ProviderCallResultByOperation["fetch"], options?: ProviderRunOptions) => Promise<ProviderAggregateResult>;
+}
+export interface ResearchRunInput {
+    topic: string;
+    days?: number;
+    from?: string;
+    to?: string;
+    sourceSelection?: ProviderSelection;
+    sources?: ProviderSource[];
+    mode: RenderMode;
+    includeEngagement?: boolean;
+    limitPerSource?: number;
+    outputDir?: string;
+    ttlHours?: number;
+    useCookies?: boolean;
+    cookiePolicyOverride?: ProviderCookiePolicy;
+}
+export interface ShoppingRunInput {
+    query: string;
+    providers?: string[];
+    budget?: number;
+    region?: string;
+    sort?: "best_deal" | "lowest_price" | "highest_rating" | "fastest_shipping";
+    mode: RenderMode;
+    timeoutMs?: number;
+    outputDir?: string;
+    ttlHours?: number;
+    useCookies?: boolean;
+    cookiePolicyOverride?: ProviderCookiePolicy;
+}
+export interface ProductVideoRunInput {
+    product_url?: string;
+    product_name?: string;
+    provider_hint?: string;
+    include_screenshots?: boolean;
+    include_all_images?: boolean;
+    include_copy?: boolean;
+    output_dir?: string;
+    ttl_hours?: number;
+    useCookies?: boolean;
+    cookiePolicyOverride?: ProviderCookiePolicy;
+}
+export interface ProductVideoWorkflowOptions {
+    captureScreenshot?: (url: string) => Promise<Buffer | null>;
+}
+export declare const workflowTestUtils: {
+    resetProviderSignalState: () => void;
+    getDegradedProviders: () => string[];
+    nextSignalState: (previous: "none" | "warning" | "degraded", warning: boolean, degraded: boolean, healthyWindows: number) => {
+        state: "none" | "warning" | "degraded";
+        healthyWindows: number;
+    };
+    redactRawCapture: (record: unknown) => Record<string, unknown>;
+    toProviderSource: (providerId: string) => ProviderSource | null;
+    resolveShoppingProviderIdForUrl: (url: string) => string | null;
+    fetchBinary: (url: string) => Promise<Buffer | null>;
+    rankResearchRecords: (records: ResearchRecord[]) => ResearchRecord[];
+};
+export declare const runResearchWorkflow: (runtime: ProviderExecutor, input: ResearchRunInput) => Promise<Record<string, unknown>>;
+export declare const runShoppingWorkflow: (runtime: ProviderExecutor, input: ShoppingRunInput) => Promise<Record<string, unknown>>;
+export declare const runProductVideoWorkflow: (runtime: ProviderExecutor, input: ProductVideoRunInput, options?: ProductVideoWorkflowOptions) => Promise<Record<string, unknown>>;
+//# sourceMappingURL=workflows.d.ts.map

package/dist/providers/workflows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflows.d.ts","sourceRoot":"","sources":["../../src/providers/workflows.ts"],"names":[],"mappings":"AAEA,OAAO,EAAyB,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AAC1E,OAAO,EAAkC,KAAK,UAAU,EAAsB,MAAM,YAAY,CAAC;AAYjG,OAAO,KAAK,EAGV,uBAAuB,EACvB,6BAA6B,EAC7B,oBAAoB,EAIpB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACf,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,CACN,KAAK,EAAE,6BAA6B,CAAC,QAAQ,CAAC,EAC9C,OAAO,CAAC,EAAE,kBAAkB,KACzB,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACtC,KAAK,EAAE,CACL,KAAK,EAAE,6BAA6B,CAAC,OAAO,CAAC,EAC7C,OAAO,CAAC,EAAE,kBAAkB,KACzB,OAAO,CAAC,uBAAuB,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,iBAAiB,CAAC;IACpC,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,EAAE,UAAU,CAAC;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,GAAG,cAAc,GAAG,gBAAgB,GAAG,kBAAkB,CAAC;IAC5E,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,oBAAoB;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,2BAA2B;IAC1C,iBAAiB,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7D;AAkZD,eAAO,MAAM,iBAAiB;oCACE,IAAI;gCAGR,MAAM,EAAE;gCAEtB,MAAM,GAAG,SAAS,GAAG,UAAU,WAChC,OAAO,YACN,OAAO,kBACD,MAAM,KACrB;QAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE;+BAE1C,OAAO,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;mCAE7B,MAAM,KAAG,cAAc,GAAG,IAAI;2CACtB,MAAM,KAAG,MAAM,GAAG,IAAI;uBAC1C,MAAM,KAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;mCACnB,cAAc,EAAE,KAAG,cAAc,EAAE;CACnE,CAAC;AAgTF,eAAO,MAAM,mBAAmB,GAC9B,SAAS,gBAAgB,EACzB,OAAO,gBAAgB,KACtB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAkIjC,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAC9B,SAAS,gBAAgB,EACzB,OAAO,gBAAgB,KACtB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAwHjC,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAClC,SAAS,gBAAgB,EACzB,OAAO,oBAAoB,EAC3B,UAAS,2BAAgC,KACxC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA6LjC,CAAC"}