@wenathlan/saddle 2.1.6 → 2.1.8

package/dist/acquisition.d.ts

@@ -1,359 +0,0 @@
-/**
- * captcha contracts separate detection and evidence from any solver provider.
- */
-export declare const captchatypes: readonly string[];
-export declare function captchacontract(options?: {}): {
-    detect(context: any): Promise<{
-        kind: any;
-        detected: boolean;
-        sitekey: any;
-        evidenceurl: any;
-    }>;
-    request(context: any): Promise<{
-        status: string;
-        kind: any;
-        message: string;
-        context: any;
-    }>;
-    solve(context: any): Promise<{
-        status: string;
-        reason: string;
-        passed?: undefined;
-        solver?: undefined;
-        token?: undefined;
-        evidenceurl?: undefined;
-    } | {
-        reason?: undefined;
-        status: string;
-        passed: boolean;
-        solver: any;
-        token: any;
-        evidenceurl: any;
-    }>;
-    assert(result: any): any;
-};
-/**
- * captcha guard blocks silent automation and records a review event for audit.
- */
-export declare function captchaguard(options?: {}): {
-    check(context: any): Promise<{
-        allowed: boolean;
-        result: any;
-        event?: undefined;
-        action?: undefined;
-    } | {
-        allowed: boolean;
-        result: any;
-        event: {
-            type: string;
-            at: number;
-            kind: any;
-            sitekey: any;
-        };
-        action: string;
-    }>;
-    events(): any[];
-};
-/**
- * captcha evidence stores references and hashes, never raw secrets or tokens by default.
- */
-export declare function evidence(options?: {}): {
-    kind: any;
-    passed: boolean;
-    solver: any;
-    evidenceurl: any;
-    sha256: any;
-    createdat: any;
-    metadata: any;
-};
-/**
- * robots context groups local rule parsing, fetch caching and active aliases.
- */
-export interface RobotsDirective {
-    userAgent: string;
-    allow: string[];
-    disallow: string[];
-    crawlDelay?: number;
-}
-export interface RobotsGroup {
-    agents: string[];
-    allow: string[];
-    disallow: string[];
-    delay?: number;
-    sitemaps: string[];
-}
-export interface RobotsTxt {
-    directives: RobotsDirective[];
-    groups: RobotsGroup[];
-    sitemaps: string[];
-    crawlDelay?: number;
-}
-/** Parses robots text into both historical directives and active groups. */
-export declare function robotsrules(text?: string): RobotsTxt;
-/** Fetches and caches a site's robots policy without failing the caller's scrape. */
-export declare function fetchRobotsTxt(siteUrl: string, options?: {
-    timeout?: number;
-    userAgent?: string;
-}): Promise<RobotsTxt>;
-/** Checks whether a URL is allowed for a selected agent. */
-export declare function isAllowed(url: string, robots: RobotsTxt, userAgent?: string): boolean;
-/** Returns the selected agent's crawl delay. */
-export declare function getCrawlDelay(robots: RobotsTxt, userAgent?: string): number | undefined;
-export declare function getSitemaps(robots: RobotsTxt): string[];
-export declare function robotsallowed(rules: RobotsTxt, target: string, agent?: string): boolean;
-export declare function robotsdelay(rules: RobotsTxt, agent?: string): number;
-/**
- * scrape cache context groups the active stale-aware fetch cache and the
- * historical typed cache facade without requiring an external LRU package.
- */
-export interface CacheEntry<T = unknown> {
-    value: T;
-    timestamp: number;
-    expires?: number;
-}
-export interface CacheConfig {
-    maxSize?: number;
-    defaultTtlMs?: number;
-    checkPeriodMs?: number;
-}
-/** Provides a bounded typed cache facade for historical scrape callers. */
-export declare class WebScrapeCache {
-    private readonly store;
-    private readonly config;
-    constructor(config?: Partial<CacheConfig>);
-    get<T = unknown>(key: string): T | undefined;
-    set<T = unknown>(key: string, value: T, ttlMs?: number): void;
-    has(key: string): boolean;
-    delete(key: string): boolean;
-    clear(): void;
-    getOrSet<T = unknown>(key: string, factory: () => Promise<T>, ttlMs?: number): Promise<T>;
-    size(): number;
-    keys(): string[];
-}
-/** Creates the historical typed cache facade. */
-export declare function createCache(config?: Partial<CacheConfig>): WebScrapeCache;
-/** Creates the active stale-aware fetch cache contract. */
-export declare function ttlcache<T = unknown>(options?: {
-    ttl?: number;
-    stale?: boolean;
-}): {
-    get(key: string): T | null;
-    set(key: string, value: T, valueoptions?: {
-        ttl?: number;
-        stale?: number;
-    }): T;
-    delete(key: string): void;
-    clear(): void;
-    size(): number;
-};
-/** Detects a normalized media type from a header, URL suffix or caller fallback. */
-export declare function detectcontenttype(header: any, url: any, fallback?: string): any;
-/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
-export declare function normalizeresult(input: any, options?: {}): {
-    contenttype: any;
-    kind: string;
-    size: number;
-    bytes: Uint8Array<ArrayBufferLike>;
-    data?: undefined;
-    content?: undefined;
-} | {
-    bytes?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-    data: any;
-} | {
-    bytes?: undefined;
-    data?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-};
-/** Reads a caller-provided response body after checking declared and actual size limits. */
-export declare function normalizeresponse(response: any, options?: {}): Promise<{
-    contenttype: any;
-    kind: string;
-    size: number;
-    bytes: Uint8Array<ArrayBufferLike>;
-    data?: undefined;
-    content?: undefined;
-} | {
-    bytes?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-    data: any;
-} | {
-    bytes?: undefined;
-    data?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-}>;
-/**
- * scrape extraction context groups dependency-free HTML extraction and the
- * historical structured extraction facade under one TypeScript contract.
- */
-export interface LinkInfo {
-    href: string;
-    text: string;
-    isInternal: boolean;
-    isExternal: boolean;
-}
-export interface ImageInfo {
-    src: string;
-    alt: string;
-    width?: number;
-    height?: number;
-}
-export interface TableInfo {
-    headers: string[];
-    rows: string[][];
-    caption?: string;
-}
-export interface PageMetadata {
-    title: string;
-    description?: string;
-    favicon?: string;
-    charset?: string;
-    language?: string;
-    author?: string;
-    publishedDate?: string;
-    ogImage?: string;
-    ogType?: string;
-    keywords?: string[];
-}
-export interface ExtractOptions {
-    readable?: boolean;
-    preserveLinks?: boolean;
-    preserveImages?: boolean;
-    preserveTables?: boolean;
-    removeSelectors?: string[];
-    maxLength?: number;
-}
-export interface ExtractedContent {
-    content: string;
-    text: string;
-    links: LinkInfo[];
-    images: ImageInfo[];
-    tables: TableInfo[];
-    metadata: PageMetadata;
-    jsonLd: unknown[];
-}
-/** Extracts the compact active scrape result without a DOM dependency. */
-export declare function extracthtml(html: string, url?: string): {
-    url: string;
-    title: string;
-    description: string;
-    text: string;
-    links: string[];
-};
-/** Extracts structured content for historical TypeScript callers. */
-export declare function extractContent(html: string, options?: ExtractOptions): Promise<ExtractedContent>;
-/** Extracts readable page text from a document string. */
-export declare function extractReadable(html: string): Promise<string>;
-/**
- * schema extraction accepts safe field descriptors and never evaluates source strings.
- */
-export declare function extractwithschema(html: any, schema: {}, url: any): {};
-/** Extracts schema fields with bounded payload and field-level source provenance. */
-export declare function extractstructured(html: any, schema?: {}, options?: {}): {
-    version: number;
-    sourceurl: string;
-    extractedat: string;
-    values: {};
-    provenance: {
-        [k: string]: {
-            sourceurl: string;
-            selector: string;
-            extractedat: string;
-            truncated: boolean;
-        };
-    };
-    bytes: number;
-    maxbytes: number;
-    truncated: any[];
-};
-/**
- * semantic extraction exposes bounded headings, landmarks, controls and links without evaluating page code.
- */
-/** Extracts semantic page facts from HTML using safe built-in parsing heuristics. */
-export declare function extractsemantic(html: any, url: any): {
-    url: any;
-    title: string;
-    headings: {
-        level: number;
-        text: string;
-    }[];
-    landmarks: {
-        role: string;
-        label: any;
-    }[];
-    controls: {
-        ref: string;
-        role: any;
-        name: any;
-        type: any;
-    }[];
-    links: any;
-    semantictext: string;
-};
-/**
- * scrape crawl context owns URL normalization traversal frontier and durable crawl state.
- *
- * The context keeps single page acquisition injectable while grouping the correlated
- * crawl responsibilities that previously lived in four separate top level files.
- */
-/** Removes fragments and tracking parameters before crawl deduplication. */
-export declare function normalizeurl(value: any): string;
-/** Compares two crawl targets by origin. */
-export declare function sameorigin(left: any, right: any): boolean;
-/** Creates a bounded priority frontier for crawler and queue adapters. */
-export declare function crawlfrontier(options?: {}): {
-    add: (input?: {}) => boolean;
-    next: () => any;
-    complete: (url: any) => void;
-    state: () => {
-        maxpages: number;
-        maxperdomain: number;
-        queued: number;
-        discovered: number;
-        completed: number;
-        domains: any;
-    };
-    list(): any[];
-};
-/** Creates a durable crawl queue around a caller-owned store. */
-export declare function persistentqueue(options?: {}): {
-    restore: () => Promise<void>;
-    add: (item: any) => Promise<boolean>;
-    next: () => Promise<any>;
-    complete: (url: any, patch?: {}) => Promise<any>;
-    list(): any[];
-};
-/** Runs bounded breadth first traversal through the injected single page scrape contract. */
-export declare function crawl(start: any, options?: {}): Promise<{
-    results: any[];
-    stats: {
-        maxperdomain: number;
-        queued: number;
-        discovered: number;
-        domains: any;
-        completed: number;
-        maxdepth: any;
-        maxpages: any;
-    };
-}>;
-/**
- * scraper composes robots policy cache transport and extraction without browser assumptions.
- */
-export declare function scraper(options?: {}): {
-    robots(origin: any): Promise<any>;
-    scrape(url: any): Promise<any>;
-    cache: any;
-};
-//# sourceMappingURL=acquisition.d.ts.map

package/dist/acquisition.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"acquisition.d.ts","sourceRoot":"","sources":["../acquisition.ts"],"names":[],"mappings":"AAoBA;;GAEG;AACH,eAAO,MAAM,YAAY,mBAAmE,CAAC;AAE7F,wBAAgB,eAAe,CAAC,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAS3C;AAKD;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,KAAK;;;;;;;;;;;;;;;;;;EAMxC;AAKD;;GAEG;AAEH,wBAAgB,QAAQ,CAAC,OAAO,KAAK;IAE1B,IAAI;IAA6B,MAAM;IAA2B,MAAM;IAA8B,WAAW;IAAuB,MAAM;IAA8C,SAAS;IAAmC,QAAQ;EAC1P;AAKD;;GAEG;AAEH,MAAM,WAAW,eAAe;IAAG,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AACjH,MAAM,WAAW,WAAW;IAAG,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;CAAE;AAC3H,MAAM,WAAW,SAAS;IAAG,UAAU,EAAE,eAAe,EAAE,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AAK7H,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,IAAI,SAAK,GAAG,SAAS,CAA+wC;AAEh0C,qFAAqF;AACrF,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAA4nB;AAE7vB,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,OAAO,CAAwW;AAE3b,gDAAgD;AAChD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,MAAM,GAAG,SAAS,CAAuH;AAC5M,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,EAAE,CAAiC;AACzF,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,OAAO,CAA4C;AACjI,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,SAAM,GAAG,MAAM,CAA6C;AAM/G;;;GAGG;AAEH,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,OAAO;IACrC,KAAK,EAAE,CAAC,CAAC;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAQD,2EAA2E;AAC3E,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAE/C,YAAY,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM,EAAoD;IAEjG,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAA6L;IAEzO,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAyQ;IAEtU,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAwC;IACjE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAmC;IAC/D,KAAK,IAAI,IAAI,CAAwB;IACrC,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAiL;IAC1Q,IAAI,IAAI,MAAM,CAA4B;IAC1C,IAAI,IAAI,MAAM,EAAE,CAAmC;CACpD;AAED,iDAAiD;AACjD,wBAAgB,WAAW,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,cAAc,CAAuC;AAEjH,2DAA2D;AAC3D,wBAAgB,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO;IAIjF,GAAG,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI;IAC1B,GAAG,MAAM,MAAM,SAAS,CAAC,iBAAgB;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAQ,CAAC;IAClF,MAAM,MAAM,MAAM,GAAG,IAAI;IACzB,KAAK,IAAI,IAAI;IACb,IAAI,IAAI,MAAM;EAEjB;AAuBD,oFAAoF;AACpF,wBAAgB,iBAAiB,CAAC,MAAM,KAAA,EAAE,GAAG,KAAA,EAAE,QAAQ,SAA6B,OASnF;AAED,sFAAsF;AACtF,wBAAgB,eAAe,CAAC,KAAK,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;EAalD;AAED,4FAA4F;AAC5F,wBAAsB,iBAAiB,CAAC,QAAQ,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;GAS7D;AAkCD;;;GAGG;AAEH,MAAM,WAAW,QAAQ;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;CAAE;AACnG,MAAM,WAAW,SAAS;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAAE;AACzF,MAAM,WAAW,SAAS;IAAG,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAAE;AACrF,MAAM,WAAW,YAAY;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CAAE;AAC9N,MAAM,WAAW,cAAc;IAAG,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAAE;AACpL,MAAM,WAAW,gBAAgB;IAAG,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAC;CAAE;AAE5K,0EAA0E;AAC1E,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM;;;;;;EAAwkB;AAE9nB,qEAAqE;AACrE,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAO1G;AAED,0DAA0D;AAC1D,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAqE;AAkDzI;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,IAAK,EAAE,GAAG,KAAA,MAIvD;AAED,qFAAqF;AACrF,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,KAAK,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;EAkBhE;AAuDD;;GAEG;AAEH,qFAAqF;AACrF,wBAAgB,eAAe,CAAC,IAAI,KAAA,EAAE,GAAG,KAAA;;;;;;;;;;;;;;;;;;;EAOxC;AAkBD;;;;;GAKG;AAEH,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,KAAK,KAAA,UAMjC;AAED,4CAA4C;AAC5C,wBAAgB,UAAU,CAAC,IAAI,KAAA,EAAE,KAAK,KAAA,WAA4D;AAElG,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,OAAO,KAAK;;;;;;;;;;;;;EAqCzC;AAED,iEAAiE;AACjE,wBAAgB,eAAe,CAAC,OAAO,KAAK;;;;;;EAkB3C;AAED,6FAA6F;AAC7F,wBAAsB,KAAK,CAAC,KAAK,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;GAsB9C;AAKD;;GAEG;AAEH,wBAAgB,OAAO,CAAC,OAAO,KAAK;;;;EA+BnC"}