@wenathlan/saddle 1.8.17 → 2.0.2

package/dist/acquisition.d.ts

@@ -0,0 +1,359 @@
+/**
+ * captcha contracts separate detection and evidence from any solver provider.
+ */
+export declare const captchatypes: readonly string[];
+export declare function captchacontract(options?: {}): {
+    detect(context: any): Promise<{
+        kind: any;
+        detected: boolean;
+        sitekey: any;
+        evidenceurl: any;
+    }>;
+    request(context: any): Promise<{
+        status: string;
+        kind: any;
+        message: string;
+        context: any;
+    }>;
+    solve(context: any): Promise<{
+        status: string;
+        reason: string;
+        passed?: undefined;
+        solver?: undefined;
+        token?: undefined;
+        evidenceurl?: undefined;
+    } | {
+        reason?: undefined;
+        status: string;
+        passed: boolean;
+        solver: any;
+        token: any;
+        evidenceurl: any;
+    }>;
+    assert(result: any): any;
+};
+/**
+ * captcha guard blocks silent automation and records a review event for audit.
+ */
+export declare function captchaguard(options?: {}): {
+    check(context: any): Promise<{
+        allowed: boolean;
+        result: any;
+        event?: undefined;
+        action?: undefined;
+    } | {
+        allowed: boolean;
+        result: any;
+        event: {
+            type: string;
+            at: number;
+            kind: any;
+            sitekey: any;
+        };
+        action: string;
+    }>;
+    events(): any[];
+};
+/**
+ * captcha evidence stores references and hashes, never raw secrets or tokens by default.
+ */
+export declare function evidence(options?: {}): {
+    kind: any;
+    passed: boolean;
+    solver: any;
+    evidenceurl: any;
+    sha256: any;
+    createdat: any;
+    metadata: any;
+};
+/**
+ * robots context groups local rule parsing, fetch caching and active aliases.
+ */
+export interface RobotsDirective {
+    userAgent: string;
+    allow: string[];
+    disallow: string[];
+    crawlDelay?: number;
+}
+export interface RobotsGroup {
+    agents: string[];
+    allow: string[];
+    disallow: string[];
+    delay?: number;
+    sitemaps: string[];
+}
+export interface RobotsTxt {
+    directives: RobotsDirective[];
+    groups: RobotsGroup[];
+    sitemaps: string[];
+    crawlDelay?: number;
+}
+/** Parses robots text into both historical directives and active groups. */
+export declare function robotsrules(text?: string): RobotsTxt;
+/** Fetches and caches a site's robots policy without failing the caller's scrape. */
+export declare function fetchRobotsTxt(siteUrl: string, options?: {
+    timeout?: number;
+    userAgent?: string;
+}): Promise<RobotsTxt>;
+/** Checks whether a URL is allowed for a selected agent. */
+export declare function isAllowed(url: string, robots: RobotsTxt, userAgent?: string): boolean;
+/** Returns the selected agent's crawl delay. */
+export declare function getCrawlDelay(robots: RobotsTxt, userAgent?: string): number | undefined;
+export declare function getSitemaps(robots: RobotsTxt): string[];
+export declare function robotsallowed(rules: RobotsTxt, target: string, agent?: string): boolean;
+export declare function robotsdelay(rules: RobotsTxt, agent?: string): number;
+/**
+ * scrape cache context groups the active stale-aware fetch cache and the
+ * historical typed cache facade without requiring an external LRU package.
+ */
+export interface CacheEntry<T = unknown> {
+    value: T;
+    timestamp: number;
+    expires?: number;
+}
+export interface CacheConfig {
+    maxSize?: number;
+    defaultTtlMs?: number;
+    checkPeriodMs?: number;
+}
+/** Provides a bounded typed cache facade for historical scrape callers. */
+export declare class WebScrapeCache {
+    private readonly store;
+    private readonly config;
+    constructor(config?: Partial<CacheConfig>);
+    get<T = unknown>(key: string): T | undefined;
+    set<T = unknown>(key: string, value: T, ttlMs?: number): void;
+    has(key: string): boolean;
+    delete(key: string): boolean;
+    clear(): void;
+    getOrSet<T = unknown>(key: string, factory: () => Promise<T>, ttlMs?: number): Promise<T>;
+    size(): number;
+    keys(): string[];
+}
+/** Creates the historical typed cache facade. */
+export declare function createCache(config?: Partial<CacheConfig>): WebScrapeCache;
+/** Creates the active stale-aware fetch cache contract. */
+export declare function ttlcache<T = unknown>(options?: {
+    ttl?: number;
+    stale?: boolean;
+}): {
+    get(key: string): T | null;
+    set(key: string, value: T, valueoptions?: {
+        ttl?: number;
+        stale?: number;
+    }): T;
+    delete(key: string): void;
+    clear(): void;
+    size(): number;
+};
+/** Detects a normalized media type from a header, URL suffix or caller fallback. */
+export declare function detectcontenttype(header: any, url: any, fallback?: string): any;
+/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
+export declare function normalizeresult(input: any, options?: {}): {
+    contenttype: any;
+    kind: string;
+    size: number;
+    bytes: Uint8Array<ArrayBufferLike>;
+    data?: undefined;
+    content?: undefined;
+} | {
+    bytes?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+    data: any;
+} | {
+    bytes?: undefined;
+    data?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+};
+/** Reads a caller-provided response body after checking declared and actual size limits. */
+export declare function normalizeresponse(response: any, options?: {}): Promise<{
+    contenttype: any;
+    kind: string;
+    size: number;
+    bytes: Uint8Array<ArrayBufferLike>;
+    data?: undefined;
+    content?: undefined;
+} | {
+    bytes?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+    data: any;
+} | {
+    bytes?: undefined;
+    data?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+}>;
+/**
+ * scrape extraction context groups dependency-free HTML extraction and the
+ * historical structured extraction facade under one TypeScript contract.
+ */
+export interface LinkInfo {
+    href: string;
+    text: string;
+    isInternal: boolean;
+    isExternal: boolean;
+}
+export interface ImageInfo {
+    src: string;
+    alt: string;
+    width?: number;
+    height?: number;
+}
+export interface TableInfo {
+    headers: string[];
+    rows: string[][];
+    caption?: string;
+}
+export interface PageMetadata {
+    title: string;
+    description?: string;
+    favicon?: string;
+    charset?: string;
+    language?: string;
+    author?: string;
+    publishedDate?: string;
+    ogImage?: string;
+    ogType?: string;
+    keywords?: string[];
+}
+export interface ExtractOptions {
+    readable?: boolean;
+    preserveLinks?: boolean;
+    preserveImages?: boolean;
+    preserveTables?: boolean;
+    removeSelectors?: string[];
+    maxLength?: number;
+}
+export interface ExtractedContent {
+    content: string;
+    text: string;
+    links: LinkInfo[];
+    images: ImageInfo[];
+    tables: TableInfo[];
+    metadata: PageMetadata;
+    jsonLd: unknown[];
+}
+/** Extracts the compact active scrape result without a DOM dependency. */
+export declare function extracthtml(html: string, url?: string): {
+    url: string;
+    title: string;
+    description: string;
+    text: string;
+    links: string[];
+};
+/** Extracts structured content for historical TypeScript callers. */
+export declare function extractContent(html: string, options?: ExtractOptions): Promise<ExtractedContent>;
+/** Extracts readable page text from a document string. */
+export declare function extractReadable(html: string): Promise<string>;
+/**
+ * schema extraction accepts safe field descriptors and never evaluates source strings.
+ */
+export declare function extractwithschema(html: any, schema: {}, url: any): {};
+/** Extracts schema fields with bounded payload and field-level source provenance. */
+export declare function extractstructured(html: any, schema?: {}, options?: {}): {
+    version: number;
+    sourceurl: string;
+    extractedat: string;
+    values: {};
+    provenance: {
+        [k: string]: {
+            sourceurl: string;
+            selector: string;
+            extractedat: string;
+            truncated: boolean;
+        };
+    };
+    bytes: number;
+    maxbytes: number;
+    truncated: any[];
+};
+/**
+ * semantic extraction exposes bounded headings, landmarks, controls and links without evaluating page code.
+ */
+/** Extracts semantic page facts from HTML using safe built-in parsing heuristics. */
+export declare function extractsemantic(html: any, url: any): {
+    url: any;
+    title: string;
+    headings: {
+        level: number;
+        text: string;
+    }[];
+    landmarks: {
+        role: string;
+        label: any;
+    }[];
+    controls: {
+        ref: string;
+        role: any;
+        name: any;
+        type: any;
+    }[];
+    links: any;
+    semantictext: string;
+};
+/**
+ * scrape crawl context owns URL normalization traversal frontier and durable crawl state.
+ *
+ * The context keeps single page acquisition injectable while grouping the correlated
+ * crawl responsibilities that previously lived in four separate top level files.
+ */
+/** Removes fragments and tracking parameters before crawl deduplication. */
+export declare function normalizeurl(value: any): string;
+/** Compares two crawl targets by origin. */
+export declare function sameorigin(left: any, right: any): boolean;
+/** Creates a bounded priority frontier for crawler and queue adapters. */
+export declare function crawlfrontier(options?: {}): {
+    add: (input?: {}) => boolean;
+    next: () => any;
+    complete: (url: any) => void;
+    state: () => {
+        maxpages: number;
+        maxperdomain: number;
+        queued: number;
+        discovered: number;
+        completed: number;
+        domains: any;
+    };
+    list(): any[];
+};
+/** Creates a durable crawl queue around a caller-owned store. */
+export declare function persistentqueue(options?: {}): {
+    restore: () => Promise<void>;
+    add: (item: any) => Promise<boolean>;
+    next: () => Promise<any>;
+    complete: (url: any, patch?: {}) => Promise<any>;
+    list(): any[];
+};
+/** Runs bounded breadth first traversal through the injected single page scrape contract. */
+export declare function crawl(start: any, options?: {}): Promise<{
+    results: any[];
+    stats: {
+        maxperdomain: number;
+        queued: number;
+        discovered: number;
+        domains: any;
+        completed: number;
+        maxdepth: any;
+        maxpages: any;
+    };
+}>;
+/**
+ * scraper composes robots policy cache transport and extraction without browser assumptions.
+ */
+export declare function scraper(options?: {}): {
+    robots(origin: any): Promise<any>;
+    scrape(url: any): Promise<any>;
+    cache: any;
+};
+//# sourceMappingURL=acquisition.d.ts.map

package/dist/acquisition.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"acquisition.d.ts","sourceRoot":"","sources":["../acquisition.ts"],"names":[],"mappings":"AAoBA;;GAEG;AACH,eAAO,MAAM,YAAY,mBAAmE,CAAC;AAE7F,wBAAgB,eAAe,CAAC,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAS3C;AAKD;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,KAAK;;;;;;;;;;;;;;;;;;EAMxC;AAKD;;GAEG;AAEH,wBAAgB,QAAQ,CAAC,OAAO,KAAK;IAE1B,IAAI;IAA6B,MAAM;IAA2B,MAAM;IAA8B,WAAW;IAAuB,MAAM;IAA8C,SAAS;IAAmC,QAAQ;EAC1P;AAKD;;GAEG;AAEH,MAAM,WAAW,eAAe;IAAG,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AACjH,MAAM,WAAW,WAAW;IAAG,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;CAAE;AAC3H,MAAM,WAAW,SAAS;IAAG,UAAU,EAAE,eAAe,EAAE,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AAK7H,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,IAAI,SAAK,GAAG,SAAS,CAA+wC;AAEh0C,qFAAqF;AACrF,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAA4nB;AAE7vB,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,OAAO,CAAwW;AAE3b,gDAAgD;AAChD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,MAAM,GAAG,SAAS,CAAuH;AAC5M,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,EAAE,CAAiC;AACzF,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,OAAO,CAA4C;AACjI,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,SAAM,GAAG,MAAM,CAA6C;AAM/G;;;GAGG;AAEH,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,OAAO;IACrC,KAAK,EAAE,CAAC,CAAC;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAQD,2EAA2E;AAC3E,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAE/C,YAAY,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM,EAAoD;IAEjG,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAA6L;IAEzO,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAyQ;IAEtU,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAwC;IACjE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAmC;IAC/D,KAAK,IAAI,IAAI,CAAwB;IACrC,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAiL;IAC1Q,IAAI,IAAI,MAAM,CAA4B;IAC1C,IAAI,IAAI,MAAM,EAAE,CAAmC;CACpD;AAED,iDAAiD;AACjD,wBAAgB,WAAW,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,cAAc,CAAuC;AAEjH,2DAA2D;AAC3D,wBAAgB,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO;IAIjF,GAAG,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI;IAC1B,GAAG,MAAM,MAAM,SAAS,CAAC,iBAAgB;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAQ,CAAC;IAClF,MAAM,MAAM,MAAM,GAAG,IAAI;IACzB,KAAK,IAAI,IAAI;IACb,IAAI,IAAI,MAAM;EAEjB;AAuBD,oFAAoF;AACpF,wBAAgB,iBAAiB,CAAC,MAAM,KAAA,EAAE,GAAG,KAAA,EAAE,QAAQ,SAA6B,OASnF;AAED,sFAAsF;AACtF,wBAAgB,eAAe,CAAC,KAAK,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;EAalD;AAED,4FAA4F;AAC5F,wBAAsB,iBAAiB,CAAC,QAAQ,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;GAS7D;AAkCD;;;GAGG;AAEH,MAAM,WAAW,QAAQ;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;CAAE;AACnG,MAAM,WAAW,SAAS;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAAE;AACzF,MAAM,WAAW,SAAS;IAAG,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAAE;AACrF,MAAM,WAAW,YAAY;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CAAE;AAC9N,MAAM,WAAW,cAAc;IAAG,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAAE;AACpL,MAAM,WAAW,gBAAgB;IAAG,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAC;CAAE;AAE5K,0EAA0E;AAC1E,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM;;;;;;EAAwkB;AAE9nB,qEAAqE;AACrE,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAO1G;AAED,0DAA0D;AAC1D,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAqE;AAkDzI;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,IAAK,EAAE,GAAG,KAAA,MAIvD;AAED,qFAAqF;AACrF,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,KAAK,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;EAkBhE;AAuDD;;GAEG;AAEH,qFAAqF;AACrF,wBAAgB,eAAe,CAAC,IAAI,KAAA,EAAE,GAAG,KAAA;;;;;;;;;;;;;;;;;;;EAOxC;AAkBD;;;;;GAKG;AAEH,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,KAAK,KAAA,UAMjC;AAED,4CAA4C;AAC5C,wBAAgB,UAAU,CAAC,IAAI,KAAA,EAAE,KAAK,KAAA,WAA4D;AAElG,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,OAAO,KAAK;;;;;;;;;;;;;EAqCzC;AAED,iEAAiE;AACjE,wBAAgB,eAAe,CAAC,OAAO,KAAK;;;;;;EAkB3C;AAED,6FAA6F;AAC7F,wBAAsB,KAAK,CAAC,KAAK,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;GAsB9C;AAKD;;GAEG;AAEH,wBAAgB,OAAO,CAAC,OAAO,KAAK;;;;EA+BnC"}