@wenathlan/saddle 2.0.6 → 2.0.151

package/dist/operations.js

@@ -1,94 +0,0 @@
-/**
- * operations.ts — transport-neutral public library helpers.
- *
- * This file is the v2.0.0 grand-merge consolidation of the former nested
- * folders of the repository (library) into the single
- * root-level domain file of the saddle family standard: one optimized
- * TypeScript file per correlated domain, ordinal sections, JSDoc on every
- * block, intra-domain imports dissolved, cross-domain imports rewritten to
- * ./<domain>.js. Every capability of every source file is preserved here;
- * the pure re-export barrels (none in this domain) folded their surface into this file directly.
- */
-import { crawl, extracthtml, normalizeresponse } from "./acquisition.js";
-import { chunkmarkdown, estimatetokens } from "./intelligence.js";
-import { browseragent } from "./browser.js";
-/* ════════════════════════════════════════════════════════════════════ */
-/* Section 1: library/public.ts — public library helpers compose fetch extraction serialization chunking and crawl contracts. */
-/* ════════════════════════════════════════════════════════════════════ */
-/**
- * public library helpers compose fetch extraction serialization chunking and crawl contracts.
- */
-/** Selects the fetch or browser execution path. */
-export async function saddleurl(url, options = {}) {
-    const mode = options.mode ?? "fetch";
-    if (mode === "browser")
-        return scrapewithbrowser(url, options);
-    return scrapeurl(url, options);
-}
-/** Fetches one URL through the caller supplied transport. */
-export async function scrapeurl(url, options = {}) {
-    const target = safeurl(url);
-    const response = await (options.fetcher ?? fetch)(target, { signal: options.signal, headers: options.headers });
-    if (!response.ok)
-        throw new Error(`scrape request failed with ${response.status}`);
-    const normalized = await normalizeresponse(response, { url: target, defaultcontenttype: "text/html", maxbytes: options.maxbytes });
-    const result = normalized.kind === "html" ? scrapehtml(normalized.content, target, options) : { content: normalized.content, data: normalized.data, metadata: { url: target, contenttype: normalized.contenttype, size: normalized.size }, bytes: normalized.bytes };
-    return formatresult(result, options);
-}
-/** Extracts a serializable result from HTML without network access. */
-export function scrapehtml(html, url, options = {}) {
-    const result = extracthtml(html, url);
-    return { content: result.text, metadata: { url: result.url, title: result.title, description: result.description, links: result.links }, html: options.includehtml ? html : undefined };
-}
-/** Exposes structured extraction for callers that do not need formatting. */
-export function extractcontent(html, options = {}) { return extracthtml(html, options.url); }
-/** Runs a scrape through the injected browser agent contract. */
-export async function scrapewithbrowser(url, options = {}) {
-    const agent = browseragent(options.browser);
-    await agent.navigate({ url, waituntil: options.waituntil ?? "networkidle" });
-    return formatresult({ content: await agent.text(), metadata: { url, title: await agent.title(), html: options.includehtml ? await agent.html() : undefined } }, options);
-}
-/** Serializes a result into a supported output format. */
-export function serializeresult(result, options = {}) {
-    const format = options.format ?? "json";
-    if (format === "json")
-        return JSON.stringify(result, null, options.pretty ? 2 : 0);
-    if (format === "text")
-        return result.content ?? "";
-    if (format === "markdown")
-        return `# ${result.metadata?.title ?? "result"}\n\n${result.content ?? ""}`;
-    if (format === "xml")
-        return `<result><title>${escape(result.metadata?.title ?? "")}</title><content>${escape(result.content ?? "")}</content></result>`;
-    if (format === "redis")
-        return JSON.stringify({ content: result.content, metadata: result.metadata });
-    throw new TypeError(`unsupported format: ${format}`);
-}
-/** Converts local HTML into Markdown. */
-export function serializehtml(html, options = {}) { return serializeresult(scrapehtml(html, options.url), { format: "markdown" }); }
-/** Builds compact context for an agent or a vector pipeline. */
-export function formatforagent(result, options = {}) {
-    const content = result.content ?? "";
-    const chunks = chunkmarkdown(content, { maxtokens: options.maxchunksize ?? 4000 });
-    const lines = content.split(/[.!?]\s+/).filter(Boolean);
-    return { summary: lines.slice(0, 2).join(". "), keypoints: lines.slice(0, options.keypoints ?? 5), relevanturls: result.metadata?.links ?? [], chunks, tokencount: estimatetokens(content, options.model) };
-}
-/** Processes URLs in bounded groups and emits progress events. */
-export async function batchscrape(options = {}) {
-    const urls = options.urls ?? [];
-    const concurrency = options.concurrency ?? 10;
-    const results = [];
-    for (let index = 0; index < urls.length; index += concurrency) {
-        const group = urls.slice(index, index + concurrency);
-        const completed = await Promise.all(group.map((url) => scrapeurl(url, options)));
-        results.push(...completed);
-        options.onprogress?.({ completed: results.length, total: urls.length });
-    }
-    return results;
-}
-/** Runs the crawler through the public scrape contract. */
-export async function crawlurl(url, options = {}) { return crawl(url, { ...options, scrape: (target) => scrapeurl(target, options) }); }
-function safeurl(value) { const url = new URL(value); if (!["http:", "https:"].includes(url.protocol))
-    throw new TypeError("url must use http or https"); return url.href; }
-function formatresult(result, options) { return options.format ? { ...result, serialized: serializeresult(result, options) } : result; }
-function escape(value) { return String(value).replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;"); }
-//# sourceMappingURL=operations.js.map

package/dist/operations.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"operations.js","sourceRoot":"","sources":["../operations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,0EAA0E;AAC1E,gIAAgI;AAChI,0EAA0E;AAC1E;;GAEG;AAEH,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAG,EAAE,OAAO,GAAG,EAAE;IAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC;IACrC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/D,OAAO,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACjC,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAG,EAAE,OAAO,GAAG,EAAE;IAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAChH,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACnF,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,kBAAkB,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnI,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;IACrQ,OAAO,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,GAAG,EAAE;IAChD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACtC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AAC1L,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,EAAE,IAAI,OAAO,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAE7F,iEAAiE;AACjE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAG,EAAE,OAAO,GAAG,EAAE;IACvD,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,aAAa,EAAE,CAAC,CAAC;IAC7E,OAAO,YAAY,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;AAC3K,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,eAAe,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;IAClD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;IACxC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACnD,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,QAAQ,OAAO,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;IACvG,IAAI,MAAM,KAAK,KAAK;QAAE,OAAO,kBAAkB,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,oBAAoB,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,qBAAqB,CAAC;IACzJ,IAAI,MAAM,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtG,MAAM,IAAI,SAAS,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,aAAa,CAAC,IAAI,EAAE,OAAO,GAAG,EAAE,IAAI,OAAO,eAAe,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;AAEpI,gEAAgE;AAChE,MAAM,UAAU,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,EAAE;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxD,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;AAC9M,CAAC;AAED,kEAAkE;AAClE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAO,GAAG,EAAE;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,WAAW,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,WAAW,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACjF,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC3B,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,2DAA2D;AAC3D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,EAAE,IAAI,OAAO,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAExI,SAAS,OAAO,CAAC,KAAK,IAAI,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC;IAAE,MAAM,IAAI,SAAS,CAAC,4BAA4B,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5K,SAAS,YAAY,CAAC,MAAM,EAAE,OAAO,IAAI,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AACxI,SAAS,MAAM,CAAC,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC"}