@wenathlan/saddle 1.8.17 → 2.0.2

package/dist/scrape/extract.d.ts

@@ -1,63 +0,0 @@
-/**
- * scrape extraction context groups dependency-free HTML extraction and the
- * historical structured extraction facade under one TypeScript contract.
- */
-export interface LinkInfo {
-    href: string;
-    text: string;
-    isInternal: boolean;
-    isExternal: boolean;
-}
-export interface ImageInfo {
-    src: string;
-    alt: string;
-    width?: number;
-    height?: number;
-}
-export interface TableInfo {
-    headers: string[];
-    rows: string[][];
-    caption?: string;
-}
-export interface PageMetadata {
-    title: string;
-    description?: string;
-    favicon?: string;
-    charset?: string;
-    language?: string;
-    author?: string;
-    publishedDate?: string;
-    ogImage?: string;
-    ogType?: string;
-    keywords?: string[];
-}
-export interface ExtractOptions {
-    readable?: boolean;
-    preserveLinks?: boolean;
-    preserveImages?: boolean;
-    preserveTables?: boolean;
-    removeSelectors?: string[];
-    maxLength?: number;
-}
-export interface ExtractedContent {
-    content: string;
-    text: string;
-    links: LinkInfo[];
-    images: ImageInfo[];
-    tables: TableInfo[];
-    metadata: PageMetadata;
-    jsonLd: unknown[];
-}
-/** Extracts the compact active scrape result without a DOM dependency. */
-export declare function extracthtml(html: string, url?: string): {
-    url: string;
-    title: string;
-    description: string;
-    text: string;
-    links: string[];
-};
-/** Extracts structured content for historical TypeScript callers. */
-export declare function extractContent(html: string, options?: ExtractOptions): Promise<ExtractedContent>;
-/** Extracts readable page text from a document string. */
-export declare function extractReadable(html: string): Promise<string>;
-//# sourceMappingURL=extract.d.ts.map

package/dist/scrape/extract.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../scrape/extract.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,QAAQ;IAAG,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;CAAE;AACnG,MAAM,WAAW,SAAS;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAAE;AACzF,MAAM,WAAW,SAAS;IAAG,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAAE;AACrF,MAAM,WAAW,YAAY;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CAAE;AAC9N,MAAM,WAAW,cAAc;IAAG,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAAE;AACpL,MAAM,WAAW,gBAAgB;IAAG,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAC;CAAE;AAE5K,0EAA0E;AAC1E,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM;;;;;;EAAsjB;AAE5mB,qEAAqE;AACrE,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAO1G;AAED,0DAA0D;AAC1D,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAqE"}

package/dist/scrape/extract.js

@@ -1,47 +0,0 @@
-/**
- * scrape extraction context groups dependency-free HTML extraction and the
- * historical structured extraction facade under one TypeScript contract.
- */
-/** Extracts the compact active scrape result without a DOM dependency. */
-export function extracthtml(html, url) { const title = match(html, /<title[^>]*>([\s\S]*?)<\/title>/i); const description = match(html, /<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i) ?? match(html, /<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i); const links = [...html.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>/gi)].map((item) => resolveurl(item[1], url)).filter((value) => Boolean(value)); const text = htmltotext(html); return { url, title: decode(title ?? ""), description: decode(description ?? ""), text, links: [...new Set(links)] }; }
-/** Extracts structured content for historical TypeScript callers. */
-export async function extractContent(html, options = {}) {
-    let source = html;
-    for (const selector of options.removeSelectors ?? [])
-        source = removeSelector(source, selector);
-    const metadata = extractmetadata(source);
-    const text = options.readable ? readabletext(source) : htmltotext(source);
-    const content = options.maxLength ? text.slice(0, options.maxLength) : text;
-    return { content, text, links: options.preserveLinks === false ? [] : extractlinks(source), images: options.preserveImages === false ? [] : extractimages(source), tables: options.preserveTables === false ? [] : extracttables(source), metadata, jsonLd: extractjsonld(source) };
-}
-/** Extracts readable page text from a document string. */
-export async function extractReadable(html) { return (await extractContent(html, { readable: true })).content; }
-function match(value, expression) { return value.match(expression)?.[1]?.trim(); }
-function decode(value) { return value.replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"').replaceAll("&#39;", "'"); }
-function resolveurl(value, base) { try {
-    const parsed = new URL(value, base);
-    return parsed.pathname === "/" && !parsed.search && !parsed.hash ? `${parsed.protocol}//${parsed.host}` : parsed.href;
-}
-catch {
-    return null;
-} }
-function htmltotext(html) { return decode(html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<br\s*\/?>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim()); }
-function readabletext(html) { const selectors = [/ <article[\s\S]*?<\/article>/i, /<main[\s\S]*?<\/main>/i, /<body[\s\S]*?<\/body>/i]; for (const selector of selectors) {
-    const matchvalue = html.match(selector)?.[0];
-    if (matchvalue)
-        return htmltotext(matchvalue);
-} return htmltotext(html); }
-function extractmetadata(html) { return { title: match(html, /<title[^>]*>([\s\S]*?)<\/title>/i) ?? match(html, /<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']*)["']/i) ?? "", description: matchmeta(html, "description") ?? matchmeta(html, "og:description"), favicon: match(html, /<link[^>]+rel=["'](?:icon|shortcut icon)["'][^>]+href=["']([^"']*)["']/i), language: match(html, /<html[^>]+lang=["']([^"']*)["']/i), author: matchmeta(html, "author"), ogImage: matchmeta(html, "og:image"), ogType: matchmeta(html, "og:type"), keywords: matchmeta(html, "keywords")?.split(",").map((value) => value.trim()) }; }
-function matchmeta(html, name) { return match(html, new RegExp(`<meta[^>]+(?:name|property)=["']${name}["'][^>]+content=["']([^"']*)["']`, "i")) ?? match(html, new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+(?:name|property)=["']${name}["']`, "i")); }
-function extractlinks(html) { return [...html.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi)].map((item) => ({ href: resolveurl(item[1]) ?? item[1], text: htmltotext(item[2]).slice(0, 200), isInternal: false, isExternal: true })); }
-function extractimages(html) { return [...html.matchAll(/<img[^>]+src=["']([^"']+)["'][^>]*>/gi)].map((item) => ({ src: item[1], alt: match(item[0], /alt=["']([^"']*)["']/i) ?? "", width: numberattribute(item[0], "width"), height: numberattribute(item[0], "height") })); }
-function numberattribute(value, name) { const parsed = Number(match(value, new RegExp(`${name}=["'](\\d+)["']`, "i"))); return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; }
-function extracttables(html) { return [...html.matchAll(/<table[\s\S]*?<\/table>/gi)].map((table) => { const source = table[0]; const body = source.match(/<tbody[^>]*>[\s\S]*?<\/tbody>/i)?.[0] ?? source; return { headers: [...source.matchAll(/<th[^>]*>([\s\S]*?)<\/th>/gi)].map((item) => htmltotext(item[1])), rows: [...body.matchAll(/<tr[^>]*>([\s\S]*?)<\/tr>/gi)].map((row) => [...row[1].matchAll(/<td[^>]*>([\s\S]*?)<\/td>/gi)].map((item) => htmltotext(item[1]))).filter((row) => row.length) }; }); }
-function extractjsonld(html) { const values = []; for (const item of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi))
-    try {
-        values.push(JSON.parse(item[1]));
-    }
-    catch { /* malformed structured data remains non fatal */ } return values; }
-function removeSelector(html, selector) { const tag = selector.match(/^\s*([a-z][a-z0-9]*)\s*$/i)?.[1]; if (tag)
-    return html.replace(new RegExp(`<${tag}[^>]*>[\\s\\S]*?<\/${tag}>`, "gi"), ""); const className = selector.match(/^\.([a-z0-9_-]+)$/i)?.[1]; return className ? html.replace(new RegExp(`<([a-z][a-z0-9]*)[^>]*class=["'][^"']*\\b${className}\\b[^"']*["'][^>]*>[\\s\\S]*?<\\/\\1>`, "gi"), "") : html; }
-//# sourceMappingURL=extract.js.map

package/dist/scrape/extract.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../scrape/extract.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,GAAY,IAAI,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,kCAAkC,CAAC,CAAC,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,EAAE,kEAAkE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,kEAAkE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,sCAAsC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE5mB,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,OAAO,GAAmB,EAAE;IAC7E,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,eAAe,IAAI,EAAE;QAAE,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;AACtR,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,IAAqB,OAAO,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AAEzI,SAAS,KAAK,CAAC,KAAa,EAAE,UAAkB,IAAwB,OAAO,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACtH,SAAS,MAAM,CAAC,KAAa,IAAY,OAAO,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;AACpL,SAAS,UAAU,CAAC,KAAa,EAAE,IAAa,IAAmB,IAAI,CAAC;IAAC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,OAAO,IAAI,CAAC;AAAC,CAAC,CAAC,CAAC;AAC/P,SAAS,UAAU,CAAC,IAAY,IAAY,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC3O,SAAS,YAAY,CAAC,IAAY,IAAY,MAAM,SAAS,GAAG,CAAC,+BAA+B,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC,CAAC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;IAAC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAAC,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;AAAC,CAAC,CAAC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpT,SAAS,eAAe,CAAC,IAAY,IAAkB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,kCAAkC,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,mEAAmE,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,SAAS,CAAC,IAAI,EAAE,gBAAgB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,yEAAyE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,kCAAkC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAChoB,SAAS,SAAS,CAAC,IAAY,EAAE,IAAY,IAAwB,OAAO,KAAK,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,mCAAmC,IAAI,mCAAmC,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,gEAAgE,IAAI,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACnS,SAAS,YAAY,CAAC,IAAY,IAAgB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7Q,SAAS,aAAa,CAAC,IAAY,IAAiB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,uCAAuC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,uBAAuB,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACrS,SAAS,eAAe,CAAC,KAAa,EAAE,IAAY,IAAwB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,MAAM,CAAC,GAAG,IAAI,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AAChO,SAAS,aAAa,CAAC,IAAY,IAAiB,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5gB,SAAS,aAAa,CAAC,IAAY,IAAe,MAAM,MAAM,GAAc,EAAE,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,4EAA4E,CAAC;IAAE,IAAI,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,iDAAiD,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC;AACtT,SAAS,cAAc,CAAC,IAAY,EAAE,QAAgB,IAAY,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG;IAAE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,sBAAsB,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,4CAA4C,SAAS,uCAAuC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC"}

package/dist/scrape/normalize.d.ts

@@ -1,52 +0,0 @@
-/**
- * content normalization classifies bounded response bytes without owning a parser, transport or storage backend.
- */
-/** Detects a normalized media type from a header, URL suffix or caller fallback. */
-export declare function detectcontenttype(header: any, url: any, fallback?: string): any;
-/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
-export declare function normalizeresult(input: any, options?: {}): {
-    contenttype: any;
-    kind: string;
-    size: number;
-    bytes: Uint8Array<ArrayBufferLike>;
-    data?: undefined;
-    content?: undefined;
-} | {
-    bytes?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-    data: any;
-} | {
-    bytes?: undefined;
-    data?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-};
-/** Reads a caller-provided response body after checking declared and actual size limits. */
-export declare function normalizeresponse(response: any, options?: {}): Promise<{
-    contenttype: any;
-    kind: string;
-    size: number;
-    bytes: Uint8Array<ArrayBufferLike>;
-    data?: undefined;
-    content?: undefined;
-} | {
-    bytes?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-    data: any;
-} | {
-    bytes?: undefined;
-    data?: undefined;
-    contenttype: any;
-    kind: string;
-    size: number;
-    content: string;
-}>;
-//# sourceMappingURL=normalize.d.ts.map

package/dist/scrape/normalize.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../scrape/normalize.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgBH,oFAAoF;AACpF,wBAAgB,iBAAiB,CAAC,MAAM,KAAA,EAAE,GAAG,KAAA,EAAE,QAAQ,SAA6B,OASnF;AAED,sFAAsF;AACtF,wBAAgB,eAAe,CAAC,KAAK,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;EAalD;AAED,4FAA4F;AAC5F,wBAAsB,iBAAiB,CAAC,QAAQ,KAAA,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;GAS7D"}

package/dist/scrape/normalize.js

@@ -1,102 +0,0 @@
-/**
- * content normalization classifies bounded response bytes without owning a parser, transport or storage backend.
- */
-const extensions = Object.freeze({
-    ".json": "application/json",
-    ".map": "application/json",
-    ".xml": "application/xml",
-    ".rss": "application/rss+xml",
-    ".atom": "application/atom+xml",
-    ".md": "text/markdown",
-    ".markdown": "text/markdown",
-    ".html": "text/html",
-    ".htm": "text/html",
-    ".txt": "text/plain",
-    ".csv": "text/csv"
-});
-/** Detects a normalized media type from a header, URL suffix or caller fallback. */
-export function detectcontenttype(header, url, fallback = "application/octet-stream") {
-    const value = String(header ?? "").split(";", 1)[0].trim().toLowerCase();
-    if (value)
-        return value;
-    try {
-        const pathname = new URL(url).pathname.toLowerCase();
-        const suffix = Object.keys(extensions).find((extension) => pathname.endsWith(extension));
-        if (suffix)
-            return extensions[suffix];
-    }
-    catch { /* invalid or absent URLs remain caller-owned metadata */ }
-    return String(fallback).toLowerCase();
-}
-/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
-export function normalizeresult(input, options = {}) {
-    const contenttype = detectcontenttype(options.contenttype, options.url, options.defaultcontenttype ?? "application/octet-stream");
-    const bytes = tobytes(input);
-    const maxbytes = boundedlimit(options.maxbytes, 2_000_000);
-    if (bytes.byteLength > maxbytes)
-        throw normalizationerror("CONTENT_TOO_LARGE", `content exceeds ${maxbytes} bytes`);
-    const kind = contentkind(contenttype);
-    if (kind === "binary")
-        return { contenttype, kind, size: bytes.byteLength, bytes };
-    const text = new TextDecoder(options.charset ?? "utf-8", { fatal: false }).decode(bytes).replaceAll("\r\n", "\n");
-    if (text.length > boundedlimit(options.maxtext, maxbytes))
-        throw normalizationerror("TEXT_TOO_LARGE", "decoded content exceeds the configured text limit");
-    if (kind === "json") {
-        try {
-            return { contenttype, kind, size: bytes.byteLength, content: text, data: JSON.parse(text) };
-        }
-        catch (error) {
-            throw normalizationerror("INVALID_JSON", `invalid JSON content: ${error.message}`);
-        }
-    }
-    return { contenttype, kind, size: bytes.byteLength, content: text };
-}
-/** Reads a caller-provided response body after checking declared and actual size limits. */
-export async function normalizeresponse(response, options = {}) {
-    if (!response || typeof response !== "object")
-        throw new TypeError("response is required");
-    const headers = response.headers;
-    const header = typeof headers?.get === "function" ? headers.get("content-type") : headers?.["content-type"];
-    const declared = Number(typeof headers?.get === "function" ? headers.get("content-length") : headers?.["content-length"]);
-    const maxbytes = boundedlimit(options.maxbytes, 2_000_000);
-    if (Number.isFinite(declared) && declared > maxbytes)
-        throw normalizationerror("CONTENT_TOO_LARGE", `content exceeds ${maxbytes} bytes`);
-    const body = typeof response.arrayBuffer === "function" ? await response.arrayBuffer() : await response.text();
-    return normalizeresult(body, { ...options, contenttype: header ?? options.contenttype, url: options.url ?? response.url });
-}
-function contentkind(contenttype) {
-    if (contenttype === "application/json" || contenttype.endsWith("+json"))
-        return "json";
-    if (contenttype === "text/html" || contenttype === "application/xhtml+xml")
-        return "html";
-    if (contenttype === "application/xml" || contenttype === "text/xml" || contenttype.endsWith("+xml"))
-        return "xml";
-    if (contenttype === "text/markdown")
-        return "markdown";
-    if (contenttype.startsWith("text/"))
-        return "text";
-    return "binary";
-}
-function tobytes(input) {
-    if (typeof input === "string")
-        return new TextEncoder().encode(input);
-    if (input instanceof Uint8Array)
-        return input;
-    if (input instanceof ArrayBuffer)
-        return new Uint8Array(input);
-    if (ArrayBuffer.isView(input))
-        return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
-    throw new TypeError("content must be a string, ArrayBuffer or typed array");
-}
-function boundedlimit(value, fallback) {
-    const limit = Number(value ?? fallback);
-    if (!Number.isSafeInteger(limit) || limit < 1)
-        throw new TypeError("content limit must be a positive safe integer");
-    return limit;
-}
-function normalizationerror(code, message) {
-    const error = new Error(message);
-    error.code = code;
-    return error;
-}
-//# sourceMappingURL=normalize.js.map

package/dist/scrape/normalize.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../scrape/normalize.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/B,OAAO,EAAE,kBAAkB;IAC3B,MAAM,EAAE,kBAAkB;IAC1B,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,qBAAqB;IAC7B,OAAO,EAAE,sBAAsB;IAC/B,KAAK,EAAE,eAAe;IACtB,WAAW,EAAE,eAAe;IAC5B,OAAO,EAAE,WAAW;IACpB,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,UAAU;CACnB,CAAC,CAAC;AAEH,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,0BAA0B;IAClF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzE,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACzF,IAAI,MAAM;YAAE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC,CAAC,yDAAyD,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;AACxC,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,EAAE;IACjD,MAAM,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,kBAAkB,IAAI,0BAA0B,CAAC,CAAC;IAClI,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC3D,IAAI,KAAK,CAAC,UAAU,GAAG,QAAQ;QAAE,MAAM,kBAAkB,CAAC,mBAAmB,EAAE,mBAAmB,QAAQ,QAAQ,CAAC,CAAC;IACpH,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;IACnF,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClH,IAAI,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC;QAAE,MAAM,kBAAkB,CAAC,gBAAgB,EAAE,mDAAmD,CAAC,CAAC;IAC3J,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,IAAI,CAAC;YAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,MAAM,kBAAkB,CAAC,cAAc,EAAE,yBAAyB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;IAC5M,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AACtE,CAAC;AAED,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAQ,EAAE,OAAO,GAAG,EAAE;IAC5D,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;IACjC,MAAM,MAAM,GAAG,OAAO,OAAO,EAAE,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,CAAC;IAC5G,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,OAAO,EAAE,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAC1H,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,QAAQ;QAAE,MAAM,kBAAkB,CAAC,mBAAmB,EAAE,mBAAmB,QAAQ,QAAQ,CAAC,CAAC;IACzI,MAAM,IAAI,GAAG,OAAO,QAAQ,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/G,OAAO,eAAe,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;AAC7H,CAAC;AAED,SAAS,WAAW,CAAC,WAAW;IAC9B,IAAI,WAAW,KAAK,kBAAkB,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IACvF,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,KAAK,uBAAuB;QAAE,OAAO,MAAM,CAAC;IAC1F,IAAI,WAAW,KAAK,iBAAiB,IAAI,WAAW,KAAK,UAAU,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAClH,IAAI,WAAW,KAAK,eAAe;QAAE,OAAO,UAAU,CAAC;IACvD,IAAI,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IACnD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,OAAO,CAAC,KAAK;IACpB,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtE,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IACvG,MAAM,IAAI,SAAS,CAAC,sDAAsD,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,YAAY,CAAC,KAAK,EAAE,QAAQ;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IACpH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAI,EAAE,OAAO;IACvC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;IAClB,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/scrape/robots.d.ts

@@ -1,37 +0,0 @@
-/**
- * robots context groups local rule parsing, fetch caching and active aliases.
- */
-export interface RobotsDirective {
-    userAgent: string;
-    allow: string[];
-    disallow: string[];
-    crawlDelay?: number;
-}
-export interface RobotsGroup {
-    agents: string[];
-    allow: string[];
-    disallow: string[];
-    delay?: number;
-    sitemaps: string[];
-}
-export interface RobotsTxt {
-    directives: RobotsDirective[];
-    groups: RobotsGroup[];
-    sitemaps: string[];
-    crawlDelay?: number;
-}
-/** Parses robots text into both historical directives and active groups. */
-export declare function robotsrules(text?: string): RobotsTxt;
-/** Fetches and caches a site's robots policy without failing the caller's scrape. */
-export declare function fetchRobotsTxt(siteUrl: string, options?: {
-    timeout?: number;
-    userAgent?: string;
-}): Promise<RobotsTxt>;
-/** Checks whether a URL is allowed for a selected agent. */
-export declare function isAllowed(url: string, robots: RobotsTxt, userAgent?: string): boolean;
-/** Returns the selected agent's crawl delay. */
-export declare function getCrawlDelay(robots: RobotsTxt, userAgent?: string): number | undefined;
-export declare function getSitemaps(robots: RobotsTxt): string[];
-export declare function robotsallowed(rules: RobotsTxt, target: string, agent?: string): boolean;
-export declare function robotsdelay(rules: RobotsTxt, agent?: string): number;
-//# sourceMappingURL=robots.d.ts.map

package/dist/scrape/robots.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../scrape/robots.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,eAAe;IAAG,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AACjH,MAAM,WAAW,WAAW;IAAG,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;CAAE;AAC3H,MAAM,WAAW,SAAS;IAAG,UAAU,EAAE,eAAe,EAAE,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AAK7H,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,IAAI,SAAK,GAAG,SAAS,CAA+wC;AAEh0C,qFAAqF;AACrF,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAA6nB;AAE9vB,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,OAAO,CAAwW;AAE3b,gDAAgD;AAChD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,SAAM,GAAG,MAAM,GAAG,SAAS,CAAuH;AAC5M,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,EAAE,CAAiC;AACzF,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,OAAO,CAA4C;AACjI,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,SAAM,GAAG,MAAM,CAA6C"}

package/dist/scrape/robots.js

@@ -1,71 +0,0 @@
-/**
- * robots context groups local rule parsing, fetch caching and active aliases.
- */
-const cache = new Map();
-const cachettl = 24 * 60 * 60 * 1000;
-/** Parses robots text into both historical directives and active groups. */
-export function robotsrules(text = "") { const directives = []; const groups = []; const sitemaps = []; let directive = null; let group = null; for (const raw of text.split(/\r?\n/)) {
-    const line = raw.split("#", 1)[0].trim();
-    if (!line)
-        continue;
-    const separator = line.indexOf(":");
-    if (separator < 0)
-        continue;
-    const key = line.slice(0, separator).trim().toLowerCase();
-    const value = line.slice(separator + 1).trim();
-    if (key === "user-agent") {
-        directive = { userAgent: value, allow: [], disallow: [] };
-        directives.push(directive);
-        group = { agents: [value.toLowerCase()], allow: [], disallow: [], sitemaps: [] };
-        groups.push(group);
-    }
-    else if (key === "allow" && value) {
-        directive?.allow.push(value);
-        group?.allow.push(value);
-    }
-    else if (key === "disallow" && value) {
-        directive?.disallow.push(value);
-        group?.disallow.push(value);
-    }
-    else if (key === "crawl-delay" && Number.isFinite(Number(value))) {
-        const delay = Number(value);
-        if (directive)
-            directive.crawlDelay = delay;
-        if (group)
-            group.delay = delay;
-    }
-    else if (key === "sitemap" && value) {
-        sitemaps.push(value);
-        if (group)
-            group.sitemaps.push(value);
-    }
-} return { directives, groups, sitemaps, crawlDelay: directives.find((value) => value.crawlDelay !== undefined)?.crawlDelay }; }
-/** Fetches and caches a site's robots policy without failing the caller's scrape. */
-export async function fetchRobotsTxt(siteUrl, options = {}) { const base = siteUrl.replace(/\/$/, ""); const robotsUrl = `${base}/robots.txt`; const cached = cache.get(robotsUrl); if (cached && Date.now() - cached.fetchedAt < cachettl)
-    return cached.robots; try {
-    const response = await fetch(robotsUrl, { headers: { "User-Agent": options.userAgent ?? "Saddle/1.8.17" }, signal: AbortSignal.timeout(options.timeout ?? 5000) });
-    const robots = response.ok ? robotsrules(await response.text()) : robotsrules();
-    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
-    return robots;
-}
-catch {
-    const robots = robotsrules();
-    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
-    return robots;
-} }
-/** Checks whether a URL is allowed for a selected agent. */
-export function isAllowed(url, robots, userAgent = "*") { try {
-    const path = new URL(url).pathname;
-    const rules = robots.directives.filter((value) => matchagent(value.userAgent, userAgent));
-    return !rules.some((value) => value.disallow.some((pattern) => pattern && path.startsWith(pattern) && !value.allow.some((allowed) => allowed.length >= pattern.length && path.startsWith(allowed))));
-}
-catch {
-    return true;
-} }
-/** Returns the selected agent's crawl delay. */
-export function getCrawlDelay(robots, userAgent = "*") { return robots.directives.find((value) => matchagent(value.userAgent, userAgent))?.crawlDelay ?? robots.crawlDelay; }
-export function getSitemaps(robots) { return [...robots.sitemaps]; }
-export function robotsallowed(rules, target, agent = "*") { return isAllowed(target, rules, agent); }
-export function robotsdelay(rules, agent = "*") { return getCrawlDelay(rules, agent) ?? 0; }
-function matchagent(pattern, userAgent) { const normalized = pattern.toLowerCase(); const candidate = userAgent.toLowerCase(); return normalized === "*" || candidate.includes(normalized.replaceAll("*", "")); }
-//# sourceMappingURL=robots.js.map

package/dist/scrape/robots.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../scrape/robots.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoD,CAAC;AAC1E,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAErC,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CAAC,IAAI,GAAG,EAAE,IAAe,MAAM,UAAU,GAAsB,EAAE,CAAC,CAAC,MAAM,MAAM,GAAkB,EAAE,CAAC,CAAC,MAAM,QAAQ,GAAa,EAAE,CAAC,CAAC,IAAI,SAAS,GAA2B,IAAI,CAAC,CAAC,IAAI,KAAK,GAAuB,IAAI,CAAC,CAAC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IAAC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,IAAI,CAAC,IAAI;QAAE,SAAS;IAAC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAAC,IAAI,SAAS,GAAG,CAAC;QAAE,SAAS;IAAC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAAC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAAC,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;QAAC,SAAS,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAAC,KAAK,GAAG,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,OAAO,IAAI,KAAK,EAAE,CAAC;QAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,UAAU,IAAI,KAAK,EAAE,CAAC;QAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAAC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,SAAS;YAAE,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAAC,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;IAAC,CAAC;SAAM,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,EAAE,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,KAAK;YAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;AAAC,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AAEh0C,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe,EAAE,OAAO,GAA6C,EAAE,IAAwB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,SAAS,GAAG,GAAG,IAAI,aAAa,CAAC,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,QAAQ;IAAE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,eAAe,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC;AAAC,CAAC,CAAC,CAAC;AAE9vB,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,MAAiB,EAAE,SAAS,GAAG,GAAG,IAAa,IAAI,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAAC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,OAAO,IAAI,CAAC;AAAC,CAAC,CAAC,CAAC;AAE3b,gDAAgD;AAChD,MAAM,UAAU,aAAa,CAAC,MAAiB,EAAE,SAAS,GAAG,GAAG,IAAwB,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;AAC5M,MAAM,UAAU,WAAW,CAAC,MAAiB,IAAc,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AACzF,MAAM,UAAU,aAAa,CAAC,KAAgB,EAAE,MAAc,EAAE,KAAK,GAAG,GAAG,IAAa,OAAO,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACjI,MAAM,UAAU,WAAW,CAAC,KAAgB,EAAE,KAAK,GAAG,GAAG,IAAY,OAAO,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC/G,SAAS,UAAU,CAAC,OAAe,EAAE,SAAiB,IAAa,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,UAAU,KAAK,GAAG,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC"}

package/dist/scrape/schema.d.ts

@@ -1,23 +0,0 @@
-/**
- * schema extraction accepts safe field descriptors and never evaluates source strings.
- */
-export declare function extractwithschema(html: any, schema: {}, url: any): {};
-/** Extracts schema fields with bounded payload and field-level source provenance. */
-export declare function extractstructured(html: any, schema?: {}, options?: {}): {
-    version: number;
-    sourceurl: string;
-    extractedat: string;
-    values: {};
-    provenance: {
-        [k: string]: {
-            sourceurl: string;
-            selector: string;
-            extractedat: string;
-            truncated: boolean;
-        };
-    };
-    bytes: number;
-    maxbytes: number;
-    truncated: any[];
-};
-//# sourceMappingURL=schema.d.ts.map

package/dist/scrape/schema.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../scrape/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,IAAK,EAAE,GAAG,KAAA,MAIvD;AAED,qFAAqF;AACrF,wBAAgB,iBAAiB,CAAC,IAAI,KAAA,EAAE,MAAM,KAAK,EAAE,OAAO,KAAK;;;;;;;;;;;;;;;;EAkBhE"}

package/dist/scrape/schema.js

@@ -1,92 +0,0 @@
-/**
- * schema extraction accepts safe field descriptors and never evaluates source strings.
- */
-export function extractwithschema(html, schema = {}, url) {
-    const result = {};
-    for (const [name, descriptor] of Object.entries(schema))
-        result[name] = extractfield(html, descriptor, url);
-    return result;
-}
-/** Extracts schema fields with bounded payload and field-level source provenance. */
-export function extractstructured(html, schema = {}, options = {}) {
-    if (typeof html !== "string")
-        throw new TypeError("structured extraction requires html text");
-    if (!schema || typeof schema !== "object" || Array.isArray(schema))
-        throw new TypeError("structured extraction requires a schema object");
-    if (options.parser !== undefined && typeof options.parser !== "function")
-        throw new TypeError("structured extraction parser must be a function");
-    const sourceurl = String(options.url ?? "");
-    const extractedat = String(options.extractedat ?? new Date(Number(options.now ?? Date.now())).toISOString());
-    const maxbytes = normalizebound(options.maxbytes ?? 65536, "maxbytes");
-    const parser = options.parser ?? ((input) => extractwithschema(input.html, input.schema, input.url));
-    const values = parser({ html, schema, url: sourceurl });
-    if (!values || typeof values !== "object" || Array.isArray(values))
-        throw new TypeError("structured extraction parser must return an object");
-    const bounded = boundpayload(values, maxbytes);
-    const provenance = Object.fromEntries(Object.entries(schema).map(([name, descriptor]) => [name, {
-            sourceurl,
-            selector: descriptorselector(descriptor),
-            extractedat,
-            truncated: bounded.truncated.includes(name)
-        }]));
-    return { version: 1, sourceurl, extractedat, values: bounded.values, provenance, bytes: bounded.bytes, maxbytes, truncated: bounded.truncated };
-}
-function extractfield(html, descriptor, url) {
-    if (typeof descriptor === "function")
-        return descriptor({ html, url });
-    if (typeof descriptor === "string")
-        return textfromselector(html, descriptor);
-    if (!descriptor || typeof descriptor.selector !== "string")
-        throw new TypeError("schema field requires selector or function");
-    if (descriptor.selector === "title")
-        return html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim() ?? null;
-    const escaped = descriptor.selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-    const attribute = descriptor.attribute;
-    const pattern = attribute ? new RegExp(`<[^>]+${escaped}[^>]*${attribute}=["']([^"']+)["'][^>]*>`, "i") : new RegExp(`<${escaped}[^>]*>([\\s\\S]*?)<\\/${escaped}>`, "i");
-    return pattern.exec(html)?.[1]?.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() ?? null;
-}
-function textfromselector(html, selector) { return extractfield(html, { selector }); }
-function descriptorselector(descriptor) { return typeof descriptor === "string" ? descriptor : typeof descriptor === "function" ? "function" : String(descriptor?.selector ?? "unknown"); }
-function normalizebound(value, name) { const normalized = Number(value); if (!Number.isSafeInteger(normalized) || normalized <= 0)
-    throw new RangeError(`${name} must be a positive safe integer`); return normalized; }
-function boundpayload(values, maxbytes) {
-    const bounded = {};
-    const truncated = [];
-    for (const [name, value] of Object.entries(values)) {
-        const candidate = JSON.stringify({ ...bounded, [name]: value });
-        if (byteLength(candidate) <= maxbytes) {
-            bounded[name] = value;
-            continue;
-        }
-        if (typeof value === "string") {
-            const clipped = clipstring(value, (trial) => byteLength(JSON.stringify({ ...bounded, [name]: trial })) <= maxbytes);
-            bounded[name] = clipped;
-        }
-        else {
-            bounded[name] = null;
-        }
-        truncated.push(name);
-    }
-    const bytes = byteLength(JSON.stringify(bounded));
-    if (bytes > maxbytes)
-        throw new RangeError("structured extraction payload budget is too small");
-    return { values: bounded, bytes, truncated };
-}
-function clipstring(value, fits) {
-    let low = 0;
-    let high = value.length;
-    let result = "";
-    while (low <= high) {
-        const middle = Math.floor((low + high) / 2);
-        const candidate = value.slice(0, middle);
-        if (fits(candidate)) {
-            result = candidate;
-            low = middle + 1;
-        }
-        else
-            high = middle - 1;
-    }
-    return result;
-}
-function byteLength(value) { return new TextEncoder().encode(String(value)).byteLength; }
-//# sourceMappingURL=schema.js.map

package/dist/scrape/schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../scrape/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,EAAE,EAAE,GAAG;IACtD,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,EAAE,EAAE,OAAO,GAAG,EAAE;IAC/D,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,CAAC,CAAC;IAC9F,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,gDAAgD,CAAC,CAAC;IAC1I,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,UAAU;QAAE,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACjJ,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC7G,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE,UAAU,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACrG,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC9I,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;YAC9F,SAAS;YACT,QAAQ,EAAE,kBAAkB,CAAC,UAAU,CAAC;YACxC,WAAW;YACX,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;SAC5C,CAAC,CAAC,CAAC,CAAC;IACL,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;AAClJ,CAAC;AAED,SAAS,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG;IACzC,IAAI,OAAO,UAAU,KAAK,UAAU;QAAE,OAAO,UAAU,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACvE,IAAI,OAAO,UAAU,KAAK,QAAQ;QAAE,OAAO,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC9E,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IAC9H,IAAI,UAAU,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAChH,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,SAAS,OAAO,QAAQ,SAAS,yBAAyB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,yBAAyB,OAAO,GAAG,EAAE,GAAG,CAAC,CAAC;IAC1K,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;AAC/F,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAI,EAAE,QAAQ,IAAI,OAAO,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;AAEtF,SAAS,kBAAkB,CAAC,UAAU,IAAI,OAAO,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,QAAQ,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;AAE3L,SAAS,cAAc,CAAC,KAAK,EAAE,IAAI,IAAI,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC;IAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,kCAAkC,CAAC,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC;AAExN,SAAS,YAAY,CAAC,MAAM,EAAE,QAAQ;IACpC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,QAAQ,EAAE,CAAC;YAAC,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAAC,SAAS;QAAC,CAAC;QAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC;YACpH,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACvB,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAClD,IAAI,KAAK,GAAG,QAAQ;QAAE,MAAM,IAAI,UAAU,CAAC,mDAAmD,CAAC,CAAC;IAChG,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,KAAK,EAAE,IAAI;IAC7B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC;IACxB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAAC,MAAM,GAAG,SAAS,CAAC;YAAC,GAAG,GAAG,MAAM,GAAG,CAAC,CAAC;QAAC,CAAC;;YAAM,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAK,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC"}

package/dist/scrape/scraper.d.ts

@@ -1,6 +0,0 @@
-export declare function scraper(options?: {}): {
-    robots(origin: any): Promise<any>;
-    scrape(url: any): Promise<any>;
-    cache: any;
-};
-//# sourceMappingURL=scraper.d.ts.map

package/dist/scrape/scraper.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../scrape/scraper.ts"],"names":[],"mappings":"AAQA,wBAAgB,OAAO,CAAC,OAAO,KAAK;;;;EA+BnC"}

package/dist/scrape/scraper.js

@@ -1,46 +0,0 @@
-/**
- * scraper composes robots policy cache transport and extraction without browser assumptions.
- */
-import { transport } from "../adapters/transport.js";
-import { extracthtml } from "./extract.js";
-import { robotsallowed, robotsdelay, robotsrules } from "./robots.js";
-import { ttlcache } from "./cache.js";
-export function scraper(options = {}) {
-    const client = options.transport ?? transport({ fetcher: options.fetcher, timeout: options.timeout, attempts: options.attempts });
-    const cache = options.cache ?? ttlcache(options.cacheoptions);
-    const agent = options.agent ?? "*";
-    const policies = new Map();
-    return {
-        async robots(origin) {
-            if (policies.has(origin))
-                return policies.get(origin);
-            const url = new URL("/robots.txt", origin).href;
-            const response = await client.request(url);
-            const rules = robotsrules(response.ok ? await response.text() : "");
-            policies.set(origin, rules);
-            return rules;
-        },
-        async scrape(url) {
-            const target = new URL(url);
-            if (!["http:", "https:"].includes(target.protocol))
-                throw new TypeError("scraper accepts http and https only");
-            const rules = await this.robots(target.origin);
-            if (!robotsallowed(rules, target.href, agent))
-                throw new Error("robots policy disallows target");
-            const cached = cache.get(target.href);
-            if (cached)
-                return cached;
-            const wait = robotsdelay(rules, agent);
-            if (wait)
-                await new Promise((resolve) => setTimeout(resolve, wait * 1000));
-            const response = await client.request(target.href, { headers: options.headers });
-            if (!response.ok)
-                throw new Error(`scrape request failed with ${response.status}`);
-            const result = extracthtml(await response.text(), target.href);
-            cache.set(target.href, result, { ttl: options.ttl });
-            return result;
-        },
-        cache
-    };
-}
-//# sourceMappingURL=scraper.js.map

package/dist/scrape/scraper.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../scrape/scraper.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,UAAU,OAAO,CAAC,OAAO,GAAG,EAAE;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClI,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;IAC3B,OAAO;QACL,KAAK,CAAC,MAAM,CAAC,MAAM;YACjB,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,OAAO,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;YAChD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC5B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG;YACd,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;YAC/G,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACjG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAC1B,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACvC,IAAI,IAAI;gBAAE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;YAC3E,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;YACjF,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACnF,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/D,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACrD,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,KAAK;KACN,CAAC;AACJ,CAAC"}