@wenathlan/saddle 1.8.8 → 1.8.9

package/dist/scrape/crawl.js

@@ -0,0 +1,114 @@
+/**
+ * scrape crawl context owns URL normalization traversal frontier and durable crawl state.
+ *
+ * The context keeps single page acquisition injectable while grouping the correlated
+ * crawl responsibilities that previously lived in four separate top level files.
+ */
+/** Removes fragments and tracking parameters before crawl deduplication. */
+export function normalizeurl(value) {
+    const url = new URL(value);
+    url.hash = "";
+    for (const key of [...url.searchParams.keys()])
+        if (/^(utm_|fbclid$|msclkid$|gclid$|gclsrc$|dclid$|gbraid$|wbraid$|twclid$|campaign$|content$|term$|source$|medium$|ref$|share_id$)/i.test(key))
+            url.searchParams.delete(key);
+    if (url.pathname.length > 1)
+        url.pathname = url.pathname.replace(/\/+$/, "");
+    return url.href;
+}
+/** Compares two crawl targets by origin. */
+export function sameorigin(left, right) { return new URL(left).origin === new URL(right).origin; }
+/** Creates a bounded priority frontier for crawler and queue adapters. */
+export function crawlfrontier(options = {}) {
+    const maxpages = Number(options.maxpages ?? 20);
+    const maxperdomain = Number(options.maxperdomain ?? maxpages);
+    const queue = [];
+    const seen = new Set();
+    const completed = new Set();
+    const domains = new Map();
+    /** Adds a URL once while respecting the global page budget. */
+    function add(input = {}) {
+        const url = String(input.url ?? "");
+        if (!url || seen.has(url) || seen.size >= maxpages)
+            return false;
+        seen.add(url);
+        queue.push({ url, depth: Number(input.depth ?? 0), priority: Number(input.priority ?? 0), discoveredat: Number(input.discoveredat ?? Date.now()) });
+        queue.sort((left, right) => right.priority - left.priority || left.discoveredat - right.discoveredat);
+        return true;
+    }
+    /** Removes the next URL that still fits its per-domain budget. */
+    function next() {
+        while (queue.length) {
+            const item = queue.shift();
+            const domain = new URL(item.url).hostname;
+            if ((domains.get(domain) ?? 0) >= maxperdomain)
+                continue;
+            domains.set(domain, (domains.get(domain) ?? 0) + 1);
+            return { ...item };
+        }
+        return null;
+    }
+    /** Records a completed URL for diagnostics and persistence-friendly state. */
+    function complete(url) { completed.add(String(url)); }
+    /** Returns stable frontier diagnostics without exposing mutable collections. */
+    function state() { return { maxpages, maxperdomain, queued: queue.length, discovered: seen.size, completed: completed.size, domains: Object.fromEntries(domains) }; }
+    return { add, next, complete, state, list() { return queue.map((item) => ({ ...item })); } };
+}
+/** Creates a durable crawl queue around a caller-owned store. */
+export function persistentqueue(options = {}) {
+    const store = options.store;
+    const values = [];
+    const seen = new Set();
+    /** Restores unfinished crawl records from the injected store. */
+    async function restore() { if (typeof store?.list !== "function")
+        return; for (const item of await store.list())
+        if (!seen.has(item.url) && item.status !== "done") {
+            seen.add(item.url);
+            values.push(item);
+        } }
+    /** Adds a crawl record and persists it when the store supports writes. */
+    async function add(item) { if (!item?.url || seen.has(item.url))
+        return false; const value = { url: item.url, depth: item.depth ?? 0, status: "queued", createdat: Date.now(), metadata: item.metadata ?? {} }; seen.add(value.url); values.push(value); if (typeof store?.save === "function")
+        await store.save(value); return true; }
+    /** Claims the next queued crawl record. */
+    async function next() { const item = values.find((value) => value.status === "queued"); if (!item)
+        return null; item.status = "running"; if (typeof store?.update === "function")
+        await store.update(item.url, item); return item; }
+    /** Completes a crawl record and persists the resulting status. */
+    async function complete(url, patch = {}) { const item = values.find((value) => value.url === url); if (!item)
+        return null; Object.assign(item, patch, { status: patch.status ?? "done", processedat: Date.now() }); if (typeof store?.update === "function")
+        await store.update(url, item); return item; }
+    return { restore, add, next, complete, list() { return values.map((value) => ({ ...value })); } };
+}
+/** Runs bounded breadth first traversal through the injected single page scrape contract. */
+export async function crawl(start, options = {}) {
+    const maxdepth = options.maxdepth ?? 1;
+    const maxpages = options.maxpages ?? 20;
+    const sameDomain = options.samedomain ?? true;
+    const frontier = crawlfrontier({ maxpages, maxperdomain: options.maxperdomain ?? maxpages });
+    frontier.add({ url: normalizeurl(start), depth: 0, priority: options.startpriority ?? 0 });
+    const results = [];
+    while (frontier.state().queued && results.length < maxpages) {
+        const current = frontier.next();
+        if (!current || current.depth > maxdepth)
+            continue;
+        const result = await options.scrape(current.url);
+        results.push({ ...result, depth: current.depth });
+        frontier.complete(current.url);
+        if (current.depth >= maxdepth)
+            continue;
+        for (const link of result.links ?? []) {
+            let url;
+            try {
+                url = normalizeurl(link);
+            }
+            catch {
+                continue;
+            }
+            if (sameDomain && !sameorigin(start, url))
+                continue;
+            frontier.add({ url, depth: current.depth + 1, priority: Number(options.priority?.(url, result) ?? 0) });
+        }
+    }
+    return { results, stats: { ...frontier.state(), completed: results.length, maxdepth, maxpages } };
+}
+//# sourceMappingURL=crawl.js.map

package/dist/scrape/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../scrape/crawl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,KAAK;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAAE,IAAI,iIAAiI,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9N,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC7E,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,UAAU,CAAC,IAAI,EAAE,KAAK,IAAI,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAElG,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,OAAO,GAAG,EAAE;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;IAE1B,+DAA+D;IAC/D,SAAS,GAAG,CAAC,KAAK,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,QAAQ;YAAE,OAAO,KAAK,CAAC;QACjE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACpJ,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QACtG,OAAO,IAAI,CAAC;IACd,CAAC;IAED,kEAAkE;IAClE,SAAS,IAAI;QACX,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,YAAY;gBAAE,SAAS;YACzD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC;QACrB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8EAA8E;IAC9E,SAAS,QAAQ,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtD,gFAAgF;IAChF,SAAS,KAAK,KAAK,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAErK,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,KAAK,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,eAAe,CAAC,OAAO,GAAG,EAAE;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;IAEvB,iEAAiE;IACjE,KAAK,UAAU,OAAO,KAAK,IAAI,OAAO,KAAK,EAAE,IAAI,KAAK,UAAU;QAAE,OAAO,CAAC,KAAK,MAAM,IAAI,IAAI,MAAM,KAAK,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC,CAAC,CAAC;IAEhN,0EAA0E;IAC1E,KAAK,UAAU,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,OAAO,KAAK,EAAE,IAAI,KAAK,UAAU;QAAE,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IAEvU,2CAA2C;IAC3C,KAAK,UAAU,IAAI,KAAK,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,IAAI,OAAO,KAAK,EAAE,MAAM,KAAK,UAAU;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IAEpO,kEAAkE;IAClE,KAAK,UAAU,QAAQ,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,IAAI,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,KAAK,EAAE,MAAM,KAAK,UAAU;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IAE1S,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,KAAK,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACpG,CAAC;AAED,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,EAAE;IAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IAC9C,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,QAAQ,EAAE,CAAC,CAAC;IAC7F,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,OAAO,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,GAAG,QAAQ;YAAE,SAAS;QACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QAClD,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,IAAI,QAAQ;YAAE,SAAS;QACxC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACtC,IAAI,GAAG,CAAC;YACR,IAAI,CAAC;gBAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,SAAS;YAAC,CAAC;YACrD,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC;gBAAE,SAAS;YACpD,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;AACpG,CAAC"}

package/dist/scrape/extract.d.ts

@@ -0,0 +1,63 @@
+/**
+ * scrape extraction context groups dependency-free HTML extraction and the
+ * historical structured extraction facade under one TypeScript contract.
+ */
+export interface LinkInfo {
+    href: string;
+    text: string;
+    isInternal: boolean;
+    isExternal: boolean;
+}
+export interface ImageInfo {
+    src: string;
+    alt: string;
+    width?: number;
+    height?: number;
+}
+export interface TableInfo {
+    headers: string[];
+    rows: string[][];
+    caption?: string;
+}
+export interface PageMetadata {
+    title: string;
+    description?: string;
+    favicon?: string;
+    charset?: string;
+    language?: string;
+    author?: string;
+    publishedDate?: string;
+    ogImage?: string;
+    ogType?: string;
+    keywords?: string[];
+}
+export interface ExtractOptions {
+    readable?: boolean;
+    preserveLinks?: boolean;
+    preserveImages?: boolean;
+    preserveTables?: boolean;
+    removeSelectors?: string[];
+    maxLength?: number;
+}
+export interface ExtractedContent {
+    content: string;
+    text: string;
+    links: LinkInfo[];
+    images: ImageInfo[];
+    tables: TableInfo[];
+    metadata: PageMetadata;
+    jsonLd: unknown[];
+}
+/** Extracts the compact active scrape result without a DOM dependency. */
+export declare function extracthtml(html: string, url?: string): {
+    url: string;
+    title: string;
+    description: string;
+    text: string;
+    links: string[];
+};
+/** Extracts structured content for historical TypeScript callers. */
+export declare function extractContent(html: string, options?: ExtractOptions): Promise<ExtractedContent>;
+/** Extracts readable page text from a document string. */
+export declare function extractReadable(html: string): Promise<string>;
+//# sourceMappingURL=extract.d.ts.map

package/dist/scrape/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../scrape/extract.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,QAAQ;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;CAAE;AACnG,MAAM,WAAW,SAAS;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAAE;AACzF,MAAM,WAAW,SAAS;IAAG,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAAE;AACrF,MAAM,WAAW,YAAY;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CAAE;AAC9N,MAAM,WAAW,cAAc;IAAG,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAAE;AACpL,MAAM,WAAW,gBAAgB;IAAG,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAC;CAAE;AAE5K,0EAA0E;AAC1E,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM;;;;;;EAAsjB;AAE5mB,qEAAqE;AACrE,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAO1G;AAED,0DAA0D;AAC1D,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAqE"}

package/dist/scrape/extract.js

@@ -0,0 +1,47 @@
+/**
+ * scrape extraction context groups dependency-free HTML extraction and the
+ * historical structured extraction facade under one TypeScript contract.
+ */
+/** Extracts the compact active scrape result without a DOM dependency. */
+export function extracthtml(html, url) { const title = match(html, /<title[^>]*>([\s\S]*?)<\/title>/i); const description = match(html, /<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i) ?? match(html, /<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i); const links = [...html.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>/gi)].map((item) => resolveurl(item[1], url)).filter((value) => Boolean(value)); const text = htmltotext(html); return { url, title: decode(title ?? ""), description: decode(description ?? ""), text, links: [...new Set(links)] }; }
+/** Extracts structured content for historical TypeScript callers. */
+export async function extractContent(html, options = {}) {
+    let source = html;
+    for (const selector of options.removeSelectors ?? [])
+        source = removeSelector(source, selector);
+    const metadata = extractmetadata(source);
+    const text = options.readable ? readabletext(source) : htmltotext(source);
+    const content = options.maxLength ? text.slice(0, options.maxLength) : text;
+    return { content, text, links: options.preserveLinks === false ? [] : extractlinks(source), images: options.preserveImages === false ? [] : extractimages(source), tables: options.preserveTables === false ? [] : extracttables(source), metadata, jsonLd: extractjsonld(source) };
+}
+/** Extracts readable page text from a document string. */
+export async function extractReadable(html) { return (await extractContent(html, { readable: true })).content; }
+function match(value, expression) { return value.match(expression)?.[1]?.trim(); }
+function decode(value) { return value.replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"').replaceAll("&#39;", "'"); }
+function resolveurl(value, base) { try {
+    const parsed = new URL(value, base);
+    return parsed.pathname === "/" && !parsed.search && !parsed.hash ? `${parsed.protocol}//${parsed.host}` : parsed.href;
+}
+catch {
+    return null;
+} }
+function htmltotext(html) { return decode(html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<br\s*\/?>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim()); }
+function readabletext(html) { const selectors = [/ <article[\s\S]*?<\/article>/i, /<main[\s\S]*?<\/main>/i, /<body[\s\S]*?<\/body>/i]; for (const selector of selectors) {
+    const matchvalue = html.match(selector)?.[0];
+    if (matchvalue)
+        return htmltotext(matchvalue);
+} return htmltotext(html); }
+function extractmetadata(html) { return { title: match(html, /<title[^>]*>([\s\S]*?)<\/title>/i) ?? match(html, /<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) ?? "", description: matchmeta(html, "description") ?? matchmeta(html, "og:description"), favicon: match(html, /<link[^>]+rel=["'](?:icon|shortcut icon)["'][^>]+href=["']([^"']*)["']/i), language: match(html, /<html[^>]+lang=["']([^"']*)["']/i), author: matchmeta(html, "author"), ogImage: matchmeta(html, "og:image"), ogType: matchmeta(html, "og:type"), keywords: matchmeta(html, "keywords")?.split(",").map((value) => value.trim()) }; }
+function matchmeta(html, name) { return match(html, new RegExp(`<meta[^>]+(?:name|property)=["']${name}["'][^>]+content=["']([^"']*)["']`, "i")) ?? match(html, new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+(?:name|property)=["']${name}["']`, "i")); }
+function extractlinks(html) { return [...html.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi)].map((item) => ({ href: resolveurl(item[1]) ?? item[1], text: htmltotext(item[2]).slice(0, 200), isInternal: false, isExternal: true })); }
+function extractimages(html) { return [...html.matchAll(/<img[^>]+src=["']([^"']+)["'][^>]*>/gi)].map((item) => ({ src: item[1], alt: match(item[0], /alt=["']([^"']*)["']/i) ?? "", width: numberattribute(item[0], "width"), height: numberattribute(item[0], "height") })); }
+function numberattribute(value, name) { const parsed = Number(match(value, new RegExp(`${name}=["'](\\d+)["']`, "i"))); return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; }
+function extracttables(html) { return [...html.matchAll(/<table[\s\S]*?<\/table>/gi)].map((table) => { const source = table[0]; const body = source.match(/<tbody[^>]*>[\s\S]*?<\/tbody>/i)?.[0] ?? source; return { headers: [...source.matchAll(/<th[^>]*>([\s\S]*?)<\/th>/gi)].map((item) => htmltotext(item[1])), rows: [...body.matchAll(/<tr[^>]*>([\s\S]*?)<\/tr>/gi)].map((row) => [...row[1].matchAll(/<td[^>]*>([\s\S]*?)<\/td>/gi)].map((item) => htmltotext(item[1]))).filter((row) => row.length) }; }); }
+function extractjsonld(html) { const values = []; for (const item of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi))
+    try {
+        values.push(JSON.parse(item[1]));
+    }
+    catch { /* malformed structured data remains non fatal */ } return values; }
+function removeSelector(html, selector) { const tag = selector.match(/^\s*([a-z][a-z0-9]*)\s*$/i)?.[1]; if (tag)
+    return html.replace(new RegExp(`<${tag}[^>]*>[\\s\\S]*?<\/${tag}>`, "gi"), ""); const className = selector.match(/^\.([a-z0-9_-]+)$/i)?.[1]; return className ? html.replace(new RegExp(`<([a-z][a-z0-9]*)[^>]*class=["'][^"']*\\b${className}\\b[^"']*["'][^>]*>[\\s\\S]*?<\\/\\1>`, "gi"), "") : html; }
+//# sourceMappingURL=extract.js.map

package/dist/scrape/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../scrape/extract.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,GAAY,IAAI,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,kCAAkC,CAAC,CAAC,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,EAAE,kEAAkE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,kEAAkE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,sCAAsC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE5mB,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,OAAO,GAAmB,EAAE;IAC7E,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,eAAe,IAAI,EAAE;QAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;AACtR,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,IAAqB,OAAO,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AAEzI,SAAS,KAAK,CAAC,KAAa,EAAE,UAAkB,IAAwB,OAAO,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACtH,SAAS,MAAM,CAAC,KAAa,IAAY,OAAO,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;AACpL,SAAS,UAAU,CAAC,KAAa,EAAE,IAAa,IAAmB,IAAI,CAAC;IAAC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,OAAO,IAAI,CAAC;AAAC,CAAC,CAAC,CAAC;AAC/P,SAAS,UAAU,CAAC,IAAY,IAAY,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC3O,SAAS,YAAY,CAAC,IAAY,IAAY,MAAM,SAAS,GAAG,CAAC,+BAA+B,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC,CAAC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;IAAC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAAC,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;AAAC,CAAC,CAAC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpT,SAAS,eAAe,CAAC,IAAY,IAAkB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,kCAAkC,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,mEAAmE,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,SAAS,CAAC,IAAI,EAAE,gBAAgB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,yEAAyE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,kCAAkC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAChoB,SAAS,SAAS,CAAC,IAAY,EAAE,IAAY,IAAwB,OAAO,KAAK,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,mCAAmC,IAAI,mCAAmC,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,gEAAgE,IAAI,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACnS,SAAS,YAAY,CAAC,IAAY,IAAgB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7Q,SAAS,aAAa,CAAC,IAAY,IAAiB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,uCAAuC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,uBAAuB,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACrS,SAAS,eAAe,CAAC,KAAa,EAAE,IAAY,IAAwB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,MAAM,CAAC,GAAG,IAAI,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AAChO,SAAS,aAAa,CAAC,IAAY,IAAiB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5gB,SAAS,aAAa,CAAC,IAAY,IAAe,MAAM,MAAM,GAAc,EAAE,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,4EAA4E,CAAC;IAAE,IAAI,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,iDAAiD,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC;AACtT,SAAS,cAAc,CAAC,IAAY,EAAE,QAAgB,IAAY,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG;IAAE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,sBAAsB,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,4CAA4C,SAAS,uCAAuC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC"}

package/dist/scrape/normalize.d.ts

@@ -0,0 +1,52 @@
+/**
+ * content normalization classifies bounded response bytes without owning a parser, transport or storage backend.
+ */
+/** Detects a normalized media type from a header, URL suffix or caller fallback. */
+export declare function detectcontenttype(header: any, url: any, fallback?: string): any;
+/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
+export declare function normalizeresult(input: any, options?: {}): {
+    contenttype: any;
+    kind: string;
+    size: number;
+    bytes: Uint8Array<ArrayBufferLike>;
+    data?: undefined;
+    content?: undefined;
+} | {
+    bytes?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+    data: any;
+} | {
+    bytes?: undefined;
+    data?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+};
+/** Reads a caller-provided response body after checking declared and actual size limits. */
+export declare function normalizeresponse(response: any, options?: {}): Promise<{
+    contenttype: any;
+    kind: string;
+    size: number;
+    bytes: Uint8Array<ArrayBufferLike>;
+    data?: undefined;
+    content?: undefined;
+} | {
+    bytes?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+    data: any;
+} | {
+    bytes?: undefined;
+    data?: undefined;
+    contenttype: any;
+    kind: string;
+    size: number;
+    content: string;
+}>;
+//# sourceMappingURL=normalize.d.ts.map

package/dist/scrape/normalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../scrape/normalize.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgBH,oFAAoF;AACpF,wBAAgB,iBAAiB,CAAC,MAAM,KAAA,EAAE,GAAG,KAAA,EAAE,QAAQ,SAA6B,OASnF;AAED,sFAAsF;AACtF,wBAAgB,eAAe,CAAC,KAAK,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;EAalD;AAED,4FAA4F;AAC5F,wBAAsB,iBAAiB,CAAC,QAAQ,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;GAS7D"}

package/dist/scrape/normalize.js

@@ -0,0 +1,102 @@
+/**
+ * content normalization classifies bounded response bytes without owning a parser, transport or storage backend.
+ */
+const extensions = Object.freeze({
+    ".json": "application/json",
+    ".map": "application/json",
+    ".xml": "application/xml",
+    ".rss": "application/rss+xml",
+    ".atom": "application/atom+xml",
+    ".md": "text/markdown",
+    ".markdown": "text/markdown",
+    ".html": "text/html",
+    ".htm": "text/html",
+    ".txt": "text/plain",
+    ".csv": "text/csv"
+});
+/** Detects a normalized media type from a header, URL suffix or caller fallback. */
+export function detectcontenttype(header, url, fallback = "application/octet-stream") {
+    const value = String(header ?? "").split(";", 1)[0].trim().toLowerCase();
+    if (value)
+        return value;
+    try {
+        const pathname = new URL(url).pathname.toLowerCase();
+        const suffix = Object.keys(extensions).find((extension) => pathname.endsWith(extension));
+        if (suffix)
+            return extensions[suffix];
+    }
+    catch { /* invalid or absent URLs remain caller-owned metadata */ }
+    return String(fallback).toLowerCase();
+}
+/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
+export function normalizeresult(input, options = {}) {
+    const contenttype = detectcontenttype(options.contenttype, options.url, options.defaultcontenttype ?? "application/octet-stream");
+    const bytes = tobytes(input);
+    const maxbytes = boundedlimit(options.maxbytes, 2_000_000);
+    if (bytes.byteLength > maxbytes)
+        throw normalizationerror("CONTENT_TOO_LARGE", `content exceeds ${maxbytes} bytes`);
+    const kind = contentkind(contenttype);
+    if (kind === "binary")
+        return { contenttype, kind, size: bytes.byteLength, bytes };
+    const text = new TextDecoder(options.charset ?? "utf-8", { fatal: false }).decode(bytes).replaceAll("\r\n", "\n");
+    if (text.length > boundedlimit(options.maxtext, maxbytes))
+        throw normalizationerror("TEXT_TOO_LARGE", "decoded content exceeds the configured text limit");
+    if (kind === "json") {
+        try {
+            return { contenttype, kind, size: bytes.byteLength, content: text, data: JSON.parse(text) };
+        }
+        catch (error) {
+            throw normalizationerror("INVALID_JSON", `invalid JSON content: ${error.message}`);
+        }
+    }
+    return { contenttype, kind, size: bytes.byteLength, content: text };
+}
+/** Reads a caller-provided response body after checking declared and actual size limits. */
+export async function normalizeresponse(response, options = {}) {
+    if (!response || typeof response !== "object")
+        throw new TypeError("response is required");
+    const headers = response.headers;
+    const header = typeof headers?.get === "function" ? headers.get("content-type") : headers?.["content-type"];
+    const declared = Number(typeof headers?.get === "function" ? headers.get("content-length") : headers?.["content-length"]);
+    const maxbytes = boundedlimit(options.maxbytes, 2_000_000);
+    if (Number.isFinite(declared) && declared > maxbytes)
+        throw normalizationerror("CONTENT_TOO_LARGE", `content exceeds ${maxbytes} bytes`);
+    const body = typeof response.arrayBuffer === "function" ? await response.arrayBuffer() : await response.text();
+    return normalizeresult(body, { ...options, contenttype: header ?? options.contenttype, url: options.url ?? response.url });
+}
+function contentkind(contenttype) {
+    if (contenttype === "application/json" || contenttype.endsWith("+json"))
+        return "json";
+    if (contenttype === "text/html" || contenttype === "application/xhtml+xml")
+        return "html";
+    if (contenttype === "application/xml" || contenttype === "text/xml" || contenttype.endsWith("+xml"))
+        return "xml";
+    if (contenttype === "text/markdown")
+        return "markdown";
+    if (contenttype.startsWith("text/"))
+        return "text";
+    return "binary";
+}
+function tobytes(input) {
+    if (typeof input === "string")
+        return new TextEncoder().encode(input);
+    if (input instanceof Uint8Array)
+        return input;
+    if (input instanceof ArrayBuffer)
+        return new Uint8Array(input);
+    if (ArrayBuffer.isView(input))
+        return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
+    throw new TypeError("content must be a string, ArrayBuffer or typed array");
+}
+function boundedlimit(value, fallback) {
+    const limit = Number(value ?? fallback);
+    if (!Number.isSafeInteger(limit) || limit < 1)
+        throw new TypeError("content limit must be a positive safe integer");
+    return limit;
+}
+function normalizationerror(code, message) {
+    const error = new Error(message);
+    error.code = code;
+    return error;
+}
+//# sourceMappingURL=normalize.js.map

package/dist/scrape/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../scrape/normalize.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/B,OAAO,EAAE,kBAAkB;IAC3B,MAAM,EAAE,kBAAkB;IAC1B,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,qBAAqB;IAC7B,OAAO,EAAE,sBAAsB;IAC/B,KAAK,EAAE,eAAe;IACtB,WAAW,EAAE,eAAe;IAC5B,OAAO,EAAE,WAAW;IACpB,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,UAAU;CACnB,CAAC,CAAC;AAEH,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,0BAA0B;IAClF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzE,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACzF,IAAI,MAAM;YAAE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC,CAAC,yDAAyD,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;AACxC,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,EAAE;IACjD,MAAM,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,kBAAkB,IAAI,0BAA0B,CAAC,CAAC;IAClI,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC3D,IAAI,KAAK,CAAC,UAAU,GAAG,QAAQ;QAAE,MAAM,kBAAkB,CAAC,mBAAmB,EAAE,mBAAmB,QAAQ,QAAQ,CAAC,CAAC;IACpH,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;IACnF,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClH,IAAI,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC;QAAE,MAAM,kBAAkB,CAAC,gBAAgB,EAAE,mDAAmD,CAAC,CAAC;IAC3J,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,IAAI,CAAC;YAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,MAAM,kBAAkB,CAAC,cAAc,EAAE,yBAAyB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;IAC5M,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AACtE,CAAC;AAED,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAQ,EAAE,OAAO,GAAG,EAAE;IAC5D,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;IACjC,MAAM,MAAM,GAAG,OAAO,OAAO,EAAE,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,CAAC;IAC5G,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,OAAO,EAAE,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAC1H,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,QAAQ;QAAE,MAAM,kBAAkB,CAAC,mBAAmB,EAAE,mBAAmB,QAAQ,QAAQ,CAAC,CAAC;IACzI,MAAM,IAAI,GAAG,OAAO,QAAQ,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/G,OAAO,eAAe,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;AAC7H,CAAC;AAED,SAAS,WAAW,CAAC,WAAW;IAC9B,IAAI,WAAW,KAAK,kBAAkB,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IACvF,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,KAAK,uBAAuB;QAAE,OAAO,MAAM,CAAC;IAC1F,IAAI,WAAW,KAAK,iBAAiB,IAAI,WAAW,KAAK,UAAU,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAClH,IAAI,WAAW,KAAK,eAAe;QAAE,OAAO,UAAU,CAAC;IACvD,IAAI,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IACnD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,OAAO,CAAC,KAAK;IACpB,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtE,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IACvG,MAAM,IAAI,SAAS,CAAC,sDAAsD,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,YAAY,CAAC,KAAK,EAAE,QAAQ;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IACpH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAI,EAAE,OAAO;IACvC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;IAClB,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/scrape/robots.d.ts

@@ -0,0 +1,37 @@
+/**
+ * robots context groups local rule parsing, fetch caching and active aliases.
+ */
+export interface RobotsDirective {
+    userAgent: string;
+    allow: string[];
+    disallow: string[];
+    crawlDelay?: number;
+}
+export interface RobotsGroup {
+    agents: string[];
+    allow: string[];
+    disallow: string[];
+    delay?: number;
+    sitemaps: string[];
+}
+export interface RobotsTxt {
+    directives: RobotsDirective[];
+    groups: RobotsGroup[];
+    sitemaps: string[];
+    crawlDelay?: number;
+}
+/** Parses robots text into both historical directives and active groups. */
+export declare function robotsrules(text?: string): RobotsTxt;
+/** Fetches and caches a site's robots policy without failing the caller's scrape. */
+export declare function fetchRobotsTxt(siteUrl: string, options?: {
+    timeout?: number;
+    userAgent?: string;
+}): Promise<RobotsTxt>;
+/** Checks whether a URL is allowed for a selected agent. */
+export declare function isAllowed(url: string, robots: RobotsTxt, userAgent?: string): boolean;
+/** Returns the selected agent's crawl delay. */
+export declare function getCrawlDelay(robots: RobotsTxt, userAgent?: string): number | undefined;
+export declare function getSitemaps(robots: RobotsTxt): string[];
+export declare function robotsallowed(rules: RobotsTxt, target: string, agent?: string): boolean;
+export declare function robotsdelay(rules: RobotsTxt, agent?: string): number;
+//# sourceMappingURL=robots.d.ts.map

package/dist/scrape/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../scrape/robots.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAAG,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AACjH,MAAM,WAAW,WAAW;IAAG,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;CAAE;AAC3H,MAAM,WAAW,SAAS;IAAG,UAAU,EAAE,eAAe,EAAE,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AAK7H,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,IAAI,SAAK,GAAG,SAAS,CAA+wC;AAEh0C,qFAAqF;AACrF,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAA4nB;AAE7vB,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,OAAO,CAAwW;AAE3b,gDAAgD;AAChD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,MAAM,GAAG,SAAS,CAAuH;AAC5M,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,EAAE,CAAiC;AACzF,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,OAAO,CAA4C;AACjI,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,SAAM,GAAG,MAAM,CAA6C"}

package/dist/scrape/robots.js

@@ -0,0 +1,71 @@
+/**
+ * robots context groups local rule parsing, fetch caching and active aliases.
+ */
+const cache = new Map();
+const cachettl = 24 * 60 * 60 * 1000;
+/** Parses robots text into both historical directives and active groups. */
+export function robotsrules(text = "") { const directives = []; const groups = []; const sitemaps = []; let directive = null; let group = null; for (const raw of text.split(/\r?\n/)) {
+    const line = raw.split("#", 1)[0].trim();
+    if (!line)
+        continue;
+    const separator = line.indexOf(":");
+    if (separator < 0)
+        continue;
+    const key = line.slice(0, separator).trim().toLowerCase();
+    const value = line.slice(separator + 1).trim();
+    if (key === "user-agent") {
+        directive = { userAgent: value, allow: [], disallow: [] };
+        directives.push(directive);
+        group = { agents: [value.toLowerCase()], allow: [], disallow: [], sitemaps: [] };
+        groups.push(group);
+    }
+    else if (key === "allow" && value) {
+        directive?.allow.push(value);
+        group?.allow.push(value);
+    }
+    else if (key === "disallow" && value) {
+        directive?.disallow.push(value);
+        group?.disallow.push(value);
+    }
+    else if (key === "crawl-delay" && Number.isFinite(Number(value))) {
+        const delay = Number(value);
+        if (directive)
+            directive.crawlDelay = delay;
+        if (group)
+            group.delay = delay;
+    }
+    else if (key === "sitemap" && value) {
+        sitemaps.push(value);
+        if (group)
+            group.sitemaps.push(value);
+    }
+} return { directives, groups, sitemaps, crawlDelay: directives.find((value) => value.crawlDelay !== undefined)?.crawlDelay }; }
+/** Fetches and caches a site's robots policy without failing the caller's scrape. */
+export async function fetchRobotsTxt(siteUrl, options = {}) { const base = siteUrl.replace(/\/$/, ""); const robotsUrl = `${base}/robots.txt`; const cached = cache.get(robotsUrl); if (cached && Date.now() - cached.fetchedAt < cachettl)
+    return cached.robots; try {
+    const response = await fetch(robotsUrl, { headers: { "User-Agent": options.userAgent ?? "Saddle/1.8.9" }, signal: AbortSignal.timeout(options.timeout ?? 5000) });
+    const robots = response.ok ? robotsrules(await response.text()) : robotsrules();
+    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+    return robots;
+}
+catch {
+    const robots = robotsrules();
+    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+    return robots;
+} }
+/** Checks whether a URL is allowed for a selected agent. */
+export function isAllowed(url, robots, userAgent = "*") { try {
+    const path = new URL(url).pathname;
+    const rules = robots.directives.filter((value) => matchagent(value.userAgent, userAgent));
+    return !rules.some((value) => value.disallow.some((pattern) => pattern && path.startsWith(pattern) && !value.allow.some((allowed) => allowed.length >= pattern.length && path.startsWith(allowed))));
+}
+catch {
+    return true;
+} }
+/** Returns the selected agent's crawl delay. */
+export function getCrawlDelay(robots, userAgent = "*") { return robots.directives.find((value) => matchagent(value.userAgent, userAgent))?.crawlDelay ?? robots.crawlDelay; }
+export function getSitemaps(robots) { return [...robots.sitemaps]; }
+export function robotsallowed(rules, target, agent = "*") { return isAllowed(target, rules, agent); }
+export function robotsdelay(rules, agent = "*") { return getCrawlDelay(rules, agent) ?? 0; }
+function matchagent(pattern, userAgent) { const normalized = pattern.toLowerCase(); const candidate = userAgent.toLowerCase(); return normalized === "*" || candidate.includes(normalized.replaceAll("*", "")); }
+//# sourceMappingURL=robots.js.map

package/dist/scrape/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../scrape/robots.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoD,CAAC;AAC1E,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAErC,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CAAC,IAAI,GAAG,EAAE,IAAe,MAAM,UAAU,GAAsB,EAAE,CAAC,CAAC,MAAM,MAAM,GAAkB,EAAE,CAAC,CAAC,MAAM,QAAQ,GAAa,EAAE,CAAC,CAAC,IAAI,SAAS,GAA2B,IAAI,CAAC,CAAC,IAAI,KAAK,GAAuB,IAAI,CAAC,CAAC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IAAC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,IAAI,CAAC,IAAI;QAAE,SAAS;IAAC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAAC,IAAI,SAAS,GAAG,CAAC;QAAE,SAAS;IAAC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAAC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;QAAC,SAAS,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAAC,KAAK,GAAG,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,OAAO,IAAI,KAAK,EAAE,CAAC;QAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,UAAU,IAAI,KAAK,EAAE,CAAC;QAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAAC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,SAAS;YAAE,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAAC,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,EAAE,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,KAAK;YAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;AAAC,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AAEh0C,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe,EAAE,OAAO,GAA6C,EAAE,IAAwB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,SAAS,GAAG,GAAG,IAAI,aAAa,CAAC,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,QAAQ;IAAE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,cAAc,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC;AAAC,CAAC,CAAC,CAAC;AAE7vB,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,MAAiB,EAAE,SAAS,GAAG,GAAG,IAAa,IAAI,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAAC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,OAAO,IAAI,CAAC;AAAC,CAAC,CAAC,CAAC;AAE3b,gDAAgD;AAChD,MAAM,UAAU,aAAa,CAAC,MAAiB,EAAE,SAAS,GAAG,GAAG,IAAwB,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;AAC5M,MAAM,UAAU,WAAW,CAAC,MAAiB,IAAc,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AACzF,MAAM,UAAU,aAAa,CAAC,KAAgB,EAAE,MAAc,EAAE,KAAK,GAAG,GAAG,IAAa,OAAO,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACjI,MAAM,UAAU,WAAW,CAAC,KAAgB,EAAE,KAAK,GAAG,GAAG,IAAY,OAAO,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC/G,SAAS,UAAU,CAAC,OAAe,EAAE,SAAiB,IAAa,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,UAAU,KAAK,GAAG,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC"}

package/dist/scrape/schema.d.ts

@@ -0,0 +1,5 @@
+/**
+ * schema extraction accepts safe field descriptors and never evaluates source strings.
+ */
+export declare function extractwithschema(html: any, schema: {}, url: any): {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/scrape/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../scrape/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,IAAK,EAAE,GAAG,KAAA,MAIvD"}

package/dist/scrape/schema.js

@@ -0,0 +1,25 @@
+/**
+ * schema extraction accepts safe field descriptors and never evaluates source strings.
+ */
+export function extractwithschema(html, schema = {}, url) {
+    const result = {};
+    for (const [name, descriptor] of Object.entries(schema))
+        result[name] = extractfield(html, descriptor, url);
+    return result;
+}
+function extractfield(html, descriptor, url) {
+    if (typeof descriptor === "function")
+        return descriptor({ html, url });
+    if (typeof descriptor === "string")
+        return textfromselector(html, descriptor);
+    if (!descriptor || typeof descriptor.selector !== "string")
+        throw new TypeError("schema field requires selector or function");
+    if (descriptor.selector === "title")
+        return html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim() ?? null;
+    const escaped = descriptor.selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const attribute = descriptor.attribute;
+    const pattern = attribute ? new RegExp(`<[^>]+${escaped}[^>]*${attribute}=["']([^"']+)["'][^>]*>`, "i") : new RegExp(`<${escaped}[^>]*>([\\s\\S]*?)<\\/${escaped}>`, "i");
+    return pattern.exec(html)?.[1]?.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() ?? null;
+}
+function textfromselector(html, selector) { return extractfield(html, { selector }); }
+//# sourceMappingURL=schema.js.map

package/dist/scrape/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../scrape/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,EAAE,EAAE,GAAG;IACtD,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG;IACzC,IAAI,OAAO,UAAU,KAAK,UAAU;QAAE,OAAO,UAAU,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACvE,IAAI,OAAO,UAAU,KAAK,QAAQ;QAAE,OAAO,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC9E,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IAC9H,IAAI,UAAU,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAChH,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,SAAS,OAAO,QAAQ,SAAS,yBAAyB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,yBAAyB,OAAO,GAAG,EAAE,GAAG,CAAC,CAAC;IAC1K,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;AAC/F,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAI,EAAE,QAAQ,IAAI,OAAO,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC"}

package/dist/scrape/scraper.d.ts

@@ -0,0 +1,6 @@
+export declare function scraper(options?: {}): {
+    robots(origin: any): Promise<any>;
+    scrape(url: any): Promise<any>;
+    cache: any;
+};
+//# sourceMappingURL=scraper.d.ts.map

package/dist/scrape/scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../scrape/scraper.ts"],"names":[],"mappings":"AAQA,wBAAgB,OAAO,CAAC,OAAO,KAAK;;;;EA+BnC"}