openclaw-smart-fetch 0.1.6 → 0.1.8

package/README.md

@@ -1,10 +1,43 @@
 # openclaw-smart-fetch
 
-OpenClaw plugin package for browser-fingerprinted fetching via `wreq-js` plus readable extraction via Defuddle.
+`openclaw-smart-fetch` adds a smarter fetching plugin for OpenClaw by exposing `smart_fetch`.
 
-## What it registers
+It combines:
+- `wreq-js` for browser-like transport fingerprints
+- `Defuddle` for readable content extraction
 
-- `defuddle_fetch`
+## Why use this instead of OpenClaw's built-in `web_fetch`
+
+Use this package when the built-in `web_fetch` is not enough.
+
+Typical advantages:
+- **better resistance to bot detection** on sites that inspect TLS/HTTP client fingerprints
+- **more browser-like transport behavior** instead of a generic server-side HTTP client
+- **cleaner extracted content** instead of raw or noisy page output
+- **better article/document readability** for downstream agent analysis
+- **useful metadata** like title, author, published date, site, and language when available
+
+A good rule of thumb:
+- use built-in `web_fetch` for simple pages
+- use `smart_fetch` when pages are blocked, noisy, or extraction quality matters
+
+## Bot-detection focus
+
+This tool is aimed at sites that detect bots through:
+- TLS/client fingerprinting
+- transport/header inconsistencies
+- non-browser HTTP behavior
+
+It does **not** execute JavaScript or solve interactive anti-bot flows.
+
+If a page requires JS execution, login, scrolling, or clicking, use browser automation instead.
+
+## What tool it exposes
+
+This package registers:
+- `smart_fetch`
+
+OpenClaw keeps the separate tool name because overriding/hoisting built-in `web_fetch` is not the desired path here.
 
 ## Install
 
@@ -20,19 +53,47 @@ From a local checkout:
 openclaw plugins install -l /absolute/path/to/agent-smart-fetch/packages/openclaw-smart-fetch
 ```
 
-## Tool parameters
+## Use cases
 
-Supported request parameters:
-- `url`
+Use `smart_fetch` when you want to:
+- fetch pages that reject naive HTTP clients
+- extract the readable body from articles, docs, and blog posts
+- reduce noise before passing content to an agent
+- preserve page metadata for summarization or research
+- use browser-like fetching without paying the cost of full browser automation
+
+## Parameters
+
+| Parameter         | Type                          | Default         | Description                                               |
+|-------------------|-------------------------------|-----------------|-----------------------------------------------------------|
+| `url`             | string                        | required        | URL to fetch                                              |
+| `browser`         | string                        | `chrome_145`    | Browser profile used for transport fingerprinting         |
+| `os`              | string                        | `windows`       | OS profile: `windows`, `macos`, `linux`, `android`, `ios` |
+| `headers`         | object                        | auto            | Extra request headers                                     |
+| `maxChars`        | number                        | `50000`         | Maximum returned characters                               |
+| `format`          | `markdown` \| `html` \| `text` \| `json` | `markdown`      | Output format                                             |
+| `removeImages`    | boolean                       | `false`         | Strip image references from output                        |
+| `includeReplies`  | boolean \| `extractors`       | `extractors`    | Include replies/comments                                  |
+| `proxy`           | string                        | none            | Proxy URL                                                 |
+
+## OpenClaw config
+
+See `openclaw.plugin.json` for plugin config defaults and schema.
+
+Configurable defaults include:
+- `maxChars`
+- `timeoutMs`
 - `browser`
 - `os`
-- `headers`
-- `maxChars`
-- `format`
 - `removeImages`
 - `includeReplies`
-- `proxy`
 
-## OpenClaw config
+## When not to use it
+
+Do not use this tool when:
+- the page requires JS rendering
+- you need login/session flows
+- you need clicks, scrolling, or form submission
+- a full browser session is required
 
-See `openclaw.plugin.json` for the plugin config schema and defaults.
+In those cases, use browser automation instead.

package/dist/index.js

@@ -12,6 +12,7 @@ var DEFAULT_MAX_CHARS = 5e4;
 var DEFAULT_TIMEOUT_MS = 15e3;
 var DEFAULT_INCLUDE_REPLIES = "extractors";
 var DEFAULT_ACCEPT_HEADER = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+var DEFAULT_JSON_ACCEPT_HEADER = "application/json,text/json,application/ld+json;q=0.9,text/plain;q=0.8,*/*;q=0.7";
 var DEFAULT_ACCEPT_LANGUAGE_HEADER = "en-US,en;q=0.9";
 var runtimeDependencies = {
   fetch: fetch,
@@ -75,6 +76,41 @@ function buildFetchResponseText(result, options = {}) {
 
 ${result.content}` : result.content;
 }
+function estimateWordCount(content) {
+  const words = content.trim().match(/\S+/g);
+  return words?.length ?? 0;
+}
+function escapeHtml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+function parseAndFormatJson(raw) {
+  try {
+    return {
+      formatted: JSON.stringify(JSON.parse(raw), null, 2)
+    };
+  } catch {
+    return { error: "Invalid JSON response" };
+  }
+}
+function renderJsonContent(formattedJson, format) {
+  switch (format) {
+    case "json":
+    case "text":
+      return formattedJson;
+    case "html":
+      return `<pre><code class="language-json">${escapeHtml(formattedJson)}</code></pre>`;
+    default:
+      return `\`\`\`json
+${formattedJson}
+\`\`\``;
+  }
+}
+function stripExtractorComments(content, format) {
+  if (format === "html") {
+    return content.replace(/\s*<hr>\s*<div class="[^"]* comments">[\s\S]*$/i, "").trimEnd();
+  }
+  return content.replace(/\n---\n+## Comments\n[\s\S]*$/i, "").trimEnd();
+}
 
 // ../core/src/extract.ts
 var HTML_CONTENT_TYPES = [
@@ -83,6 +119,46 @@ var HTML_CONTENT_TYPES = [
   "text/plain",
   "text/markdown"
 ];
+function resolveAcceptHeader(format) {
+  return format === "json" ? DEFAULT_JSON_ACCEPT_HEADER : DEFAULT_ACCEPT_HEADER;
+}
+function isJsonContentType(contentType) {
+  const normalized = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+  return normalized === "application/json" || normalized === "text/json" || normalized.endsWith("+json");
+}
+function isLikelyJsonBody(body) {
+  const trimmed = body.trim();
+  return trimmed.startsWith("{") || trimmed.startsWith("[");
+}
+function isJsonResponse(contentType, body) {
+  return isJsonContentType(contentType) || isLikelyJsonBody(body);
+}
+function buildJsonResult(opts, finalUrl, rawBody, format, maxChars, browser, os) {
+  const parsedJson = parseAndFormatJson(rawBody);
+  if ("error" in parsedJson) {
+    return parsedJson;
+  }
+  const content = truncateContent(
+    renderJsonContent(parsedJson.formatted, format),
+    maxChars
+  );
+  return {
+    url: opts.url,
+    finalUrl,
+    title: "",
+    author: "",
+    published: "",
+    site: new URL(finalUrl).hostname,
+    language: "",
+    wordCount: estimateWordCount(parsedJson.formatted),
+    content,
+    browser,
+    os
+  };
+}
+function shouldStripReplies(site) {
+  return site === "Hacker News" || site.startsWith("r/") || site.startsWith("GitHub - ");
+}
 function createDefuddleFetch(dependencies = runtimeDependencies) {
   return async function defuddleFetch2(opts) {
     const browser = opts.browser ?? DEFAULT_BROWSER;
@@ -107,7 +183,7 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       browser,
       os,
       headers: {
-        Accept: DEFAULT_ACCEPT_HEADER,
+        Accept: resolveAcceptHeader(format),
         "Accept-Language": DEFAULT_ACCEPT_LANGUAGE_HEADER,
         ...opts.headers
       },
@@ -125,11 +201,37 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
     }
     const finalUrl = response.url ?? opts.url;
     const contentType = response.headers.get("content-type") ?? "";
+    const rawBody = await response.text();
+    const jsonResponse = isJsonResponse(contentType, rawBody);
+    if (format === "json") {
+      if (!jsonResponse) {
+        return { error: `Not a JSON response (content-type: ${contentType})` };
+      }
+      return buildJsonResult(
+        opts,
+        finalUrl,
+        rawBody,
+        format,
+        maxChars,
+        browser,
+        os
+      );
+    }
+    if (jsonResponse) {
+      return buildJsonResult(
+        opts,
+        finalUrl,
+        rawBody,
+        format,
+        maxChars,
+        browser,
+        os
+      );
+    }
     if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {
       return { error: `Not an HTML page (content-type: ${contentType})` };
     }
-    const html = await response.text();
-    const document = parseLinkedomHTML(html, finalUrl);
+    const document = parseLinkedomHTML(rawBody, finalUrl);
     const extracted = await dependencies.defuddle(document, finalUrl, {
       markdown: format !== "html",
       removeImages,
@@ -140,7 +242,18 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
         error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`
       };
     }
-    const normalizedContent = format === "text" ? markdownToText(extracted.content) : extracted.content;
+    let extractedContent = extracted.content;
+    let wordCount = extracted.wordCount;
+    if (includeReplies === false && shouldStripReplies(extracted.site ?? "")) {
+      const strippedContent = stripExtractorComments(extractedContent, format);
+      if (strippedContent !== extractedContent) {
+        extractedContent = strippedContent;
+        wordCount = estimateWordCount(
+          format === "text" ? markdownToText(extractedContent) : extractedContent
+        );
+      }
+    }
+    const normalizedContent = format === "text" ? markdownToText(extractedContent) : extractedContent;
     return {
       url: opts.url,
       finalUrl,
@@ -149,7 +262,7 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       published: extracted.published ?? "",
       site: extracted.site ?? "",
       language: extracted.language ?? "",
-      wordCount: extracted.wordCount,
+      wordCount,
       content: truncateContent(normalizedContent, maxChars),
       browser,
       os
@@ -195,9 +308,14 @@ function createBaseFetchToolParameterProperties(defaults) {
     ),
     format: Type.Optional(
       Type.Union(
-        [Type.Literal("markdown"), Type.Literal("html"), Type.Literal("text")],
+        [
+          Type.Literal("markdown"),
+          Type.Literal("html"),
+          Type.Literal("text"),
+          Type.Literal("json")
+        ],
         {
-          description: 'Output format. "markdown" (default), "html" (cleaned HTML), or "text" (plain text, no formatting)'
+          description: 'Output format. "markdown" (default), "html" (cleaned HTML), "text" (plain text, no formatting), or "json" (pretty-printed JSON)'
         }
       )
     ),
@@ -252,7 +370,7 @@ var plugin = {
   register(api) {
     const defaults = resolvePluginDefaults(api.pluginConfig);
     api.registerTool({
-      name: "defuddle_fetch",
+      name: "smart_fetch",
       description: [
         "Fetch a URL with browser-grade TLS fingerprinting and extract clean, readable content.",
         "Uses Rust native bindings to impersonate real browsers at the TLS/HTTP2 level (JA3/JA4 match).",
@@ -275,7 +393,7 @@ var plugin = {
       }
     });
     api.logger.info(
-      `defuddle_fetch tool registered (default: ${defaults.browser}/${defaults.os})`
+      `smart_fetch tool registered (default: ${defaults.browser}/${defaults.os})`
     );
   }
 };

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../core/src/constants.ts","../../core/src/dependencies.ts","../../core/src/dom.ts","../../core/src/format.ts","../../core/src/extract.ts","../../core/src/tool.ts","../src/index.ts"],"names":["wreqFetch","defuddleFetch","Type"],"mappings":";;;;;;;;AAEO,IAAM,eAAA,GAAkB,YAAA;AACxB,IAAM,UAAA,GAA4B,SAAA;AAClC,IAAM,iBAAA,GAAoB,GAAA;AAC1B,IAAM,kBAAA,GAAqB,IAAA;AAC3B,IAAM,uBAAA,GAA0B,YAAA;AAChC,IAAM,qBAAA,GACX,iEAAA;AACK,IAAM,8BAAA,GAAiC,gBAAA;ACLvC,IAAM,mBAAA,GAAyC;AAAA,EACpD,KAAA,EAAOA,KAAA;AAAA,EACP,QAAA,EAAU,QAAA;AAAA,EACV;AACF,CAAA;ACLO,SAAS,iBAAA,CAAkB,MAAc,GAAA,EAAwB;AACtE,EAAA,MAAM,EAAE,QAAA,EAAS,GAAI,SAAA,CAAU,IAAI,CAAA;AACnC,EAAA,MAAM,GAAA,GAAM,QAAA;AACZ,EAAA,MAAM,cAAc,GAAA,CAAI,WAAA;AASxB,EAAA,IAAI,CAAE,IAAkC,WAAA,EAAa;AACnD,IAAC,GAAA,CAAkC,cACjC,EAAC;AAAA,EACL;AAEA,EAAA,IAAI,WAAA,IAAe,CAAC,WAAA,CAAY,gBAAA,EAAkB;AAChD,IAAA,WAAA,CAAY,oBAAoB,OAAO;AAAA,MACrC,OAAA,EAAS;AAAA,KACX,CAAA,CAAA;AAAA,EACF;AAEA,EAAA,IAAI,GAAA,EAAK;AACP,IAAC,IAAyB,GAAA,GAAM,GAAA;AAAA,EAClC;AAEA,EAAA,OAAO,QAAA;AACT;;;AC7BA,SAAS,YACP,KAAA,EACA;AACA,EAAA,OAAO,KAAA,CACJ,MAAA,CAAO,CAAC,GAAG,KAAK,CAAA,KAAM,KAAA,KAAU,MAAA,IAAa,KAAA,KAAU,EAAE,CAAA,CACzD,IAAI,CAAC,CAAC,KAAA,EAAO,KAAK,CAAA,KAAM,CAAA,EAAA,EAAK,KAAK,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAA,CAC9C,IAAA,CAAK,IAAI,CAAA;AACd;AAEO,SAAS,eAAe,QAAA,EAA0B;AACvD,EAAA,OAAO,QAAA,CACJ,OAAA,CAAQ,cAAA,EAAgB,EAAE,CAAA,CAC1B,OAAA,CAAQ,kBAAA,EAAoB,IAAI,CAAA,CAChC,OAAA,CAAQ,cAAA,EAAgB,IAAI,EAC5B,OAAA,CAAQ,wBAAA,EAA0B,IAAI,CAAA,CACtC,OAAA,CAAQ,uBAAA,EAAyB,EAAE,CAAA,CACnC,QAAQ,SAAA,EAAW,EAAE,CAAA,CACrB,OAAA,CAAQ,aAAA,EAAe,SAAI,CAAA,CAC3B,OAAA,CAAQ,cAAc,IAAI,CAAA;AAC/B;AAEO,SAAS,eAAA,CAAgB,SAAiB,QAAA,EAA0B;AACzE,EAAA,IAAI,OAAA,CAAQ,MAAA,IAAU,QAAA,EAAU,OAAO,OAAA;AACvC,EAAA,OAAO,CAAA,EAAG,OAAA,CAAQ,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAC;;AAAA,eAAA,CAAA;AACtC;AAEO,SAAS,2BAA2B,MAAA,EAA6B;AACtE,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS;AAAA,GAC/B,CAAA;AACH;AAEO,SAAS,oBAAoB,MAAA,EAA6B;AAC/D,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS,CAAA;AAAA,IAC9B,CAAC,MAAA,EAAQ,MAAA,CAAO,IAAI,CAAA;AAAA,IACpB,CAAC,UAAA,EAAY,MAAA,CAAO,QAAQ,CAAA;AAAA,IAC5B,CAAC,OAAA,EAAS,MAAA,CAAO,SAAS,CAAA;AAAA,IAC1B,CAAC,WAAW,CAAA,EAAG,MAAA,CAAO,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,EAAE,CAAA,CAAE;AAAA,GAC7C,CAAA;AACH;AAEO,SAAS,sBAAA,CACd,MAAA,EACA,OAAA,GAAiC,EAAC,EAC1B;AACR,EAAA,MAAM,SAAS,OAAA,CAAQ,OAAA,GACnB,oBAAoB,MAAM,CAAA,GAC1B,2BAA2B,MAAM,CAAA;AAErC,EAAA,OAAO,MAAA,GAAS,GAAG,MAAM;;AAAA,EAAO,MAAA,CAAO,OAAO,CAAA,CAAA,GAAK,MAAA,CAAO,OAAA;AAC5D;;;ACnBA,IAAM,kBAAA,GAAqB;AAAA,EACzB,WAAA;AAAA,EACA,uBAAA;AAAA,EACA,YAAA;AAAA,EACA;AACF,CAAA;AAMO,SAAS,mBAAA,CACd,eAAkC,mBAAA,EAClC;AACA,EAAA,OAAO,eAAeC,eACpB,IAAA,EACmC;AACnC,IAAA,MAAM,OAAA,GAAU,KAAK,OAAA,IAAW,eAAA;AAChC,IAAA,MAAM,EAAA,GAAK,KAAK,EAAA,IAAM,UAAA;AACtB,IAAA,MAAM,MAAA,GAAuB,KAAK,MAAA,IAAU,UAAA;AAC5C,IAAA,MAAM,QAAA,GAAW,KAAK,QAAA,IAAY,iBAAA;AAClC,IAAA,MAAM,YAAA,GAAe,KAAK,YAAA,IAAgB,KAAA;AAC1C,IAAA,MAAM,cAAA,GAAiB,KAAK,cAAA,IAAkB,uBAAA;AAC9C,IAAA,MAAM,SAAA,GAAY,KAAK,SAAA,IAAa,kBAAA;AAEpC,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI;AACF,MAAA,MAAA,GAAS,IAAI,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AAAA,IAC3B,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,aAAA,EAAgB,IAAA,CAAK,GAAG,CAAA,CAAA,EAAG;AAAA,IAC7C;AAEA,IAAA,IAAI,CAAC,CAAC,OAAA,EAAS,QAAQ,EAAE,QAAA,CAAS,MAAA,CAAO,QAAQ,CAAA,EAAG;AAClD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,oCAAA,EAAuC,MAAA,CAAO,QAAQ,CAAA;AAAA,OAC/D;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAwC;AAAA,MAC5C,OAAA;AAAA,MACA,EAAA;AAAA,MACA,OAAA,EAAS;AAAA,QACP,MAAA,EAAQ,qBAAA;AAAA,QACR,iBAAA,EAAmB,8BAAA;AAAA,QACnB,GAAG,IAAA,CAAK;AAAA,OACV;AAAA,MACA,QAAA,EAAU,QAAA;AAAA,MACV,OAAA,EAAS;AAAA,KACX;AAEA,IAAA,IAAI,KAAK,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,QAAQ,IAAA,CAAK,KAAA;AAAA,IAC5B;AAEA,IAAA,MAAM,WAAW,MAAM,YAAA,CAAa,KAAA,CAAM,IAAA,CAAK,KAAK,YAAY,CAAA;AAEhE,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,QAAQ,QAAA,CAAS,MAAM,IAAI,QAAA,CAAS,UAAU,CAAA,KAAA,EAAQ,IAAA,CAAK,GAAG,CAAA;AAAA,OACvE;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,QAAA,CAAS,GAAA,IAAO,IAAA,CAAK,GAAA;AACtC,IAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,GAAA,CAAI,cAAc,CAAA,IAAK,EAAA;AAE5D,IAAA,IAAI,CAAC,mBAAmB,IAAA,CAAK,CAAC,UAAU,WAAA,CAAY,QAAA,CAAS,KAAK,CAAC,CAAA,EAAG;AACpE,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,gCAAA,EAAmC,WAAW,CAAA,CAAA,CAAA,EAAI;AAAA,IACpE;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AACjC,IAAA,MAAM,QAAA,GAAW,iBAAA,CAAkB,IAAA,EAAM,QAAQ,CAAA;AACjD,IAAA,MAAM,SAAA,GAAY,MAAM,YAAA,CAAa,QAAA,CAAS,UAAU,QAAA,EAAU;AAAA,MAChE,UAAU,MAAA,KAAW,MAAA;AAAA,MACrB,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,IAAI,CAAC,SAAA,CAAU,OAAA,IAAW,SAAA,CAAU,cAAc,CAAA,EAAG;AACnD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,0BAAA,EAA6B,IAAA,CAAK,GAAG,CAAA,sCAAA;AAAA,OAC9C;AAAA,IACF;AAEA,IAAA,MAAM,oBACJ,MAAA,KAAW,MAAA,GAAS,eAAe,SAAA,CAAU,OAAO,IAAI,SAAA,CAAU,OAAA;AAEpE,IAAA,OAAO;AAAA,MACL,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,QAAA;AAAA,MACA,KAAA,EAAO,UAAU,KAAA,IAAS,EAAA;AAAA,MAC1B,MAAA,EAAQ,UAAU,MAAA,IAAU,EAAA;AAAA,MAC5B,SAAA,EAAW,UAAU,SAAA,IAAa,EAAA;AAAA,MAClC,IAAA,EAAM,UAAU,IAAA,IAAQ,EAAA;AAAA,MACxB,QAAA,EAAU,UAAU,QAAA,IAAY,EAAA;AAAA,MAChC,WAAW,SAAA,CAAU,SAAA;AAAA,MACrB,OAAA,EAAS,eAAA,CAAgB,iBAAA,EAAmB,QAAQ,CAAA;AAAA,MACpD,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAA;AACF;AAEO,IAAM,gBAAgB,mBAAA,EAAoB;AAG1C,SAAS,QACd,MAAA,EACsB;AACtB,EAAA,OAAO,OAAA,IAAW,MAAA;AACpB;ACrIO,SAAS,wBAAA,CACd,MAAA,GAA0B,EAAC,EACR;AACnB,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,OAAO,QAAA,IAAY,iBAAA;AAAA,IAC7B,SAAA,EAAW,OAAO,SAAA,IAAa,kBAAA;AAAA,IAC/B,OAAA,EAAS,OAAO,OAAA,IAAW,eAAA;AAAA,IAC3B,EAAA,EAAI,OAAO,EAAA,IAAM,UAAA;AAAA,IACjB,YAAA,EAAc,OAAO,YAAA,IAAgB,KAAA;AAAA,IACrC,cAAA,EAAgB,OAAO,cAAA,IAAkB;AAAA,GAC3C;AACF;AAEO,SAAS,uCACd,QAAA,EACyB;AACzB,EAAA,OAAO;AAAA,IACL,KAAK,IAAA,CAAK,MAAA,CAAO,EAAE,WAAA,EAAa,kCAAkC,CAAA;AAAA,IAClE,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,kDAAA,EAAqD,QAAA,CAAS,OAAO,CAAA,oEAAA;AAAA,OACnF;AAAA,KACH;AAAA,IACA,IAAI,IAAA,CAAK,QAAA;AAAA,MACP,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,yCAAA,EAA4C,QAAA,CAAS,EAAE,CAAA,+CAAA;AAAA,OACrE;AAAA,KACH;AAAA,IACA,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO,IAAA,CAAK,QAAO,EAAG,IAAA,CAAK,QAAO,EAAG;AAAA,QACxC,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,UAAU,IAAA,CAAK,QAAA;AAAA,MACb,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,uCAAA,EAA0C,QAAA,CAAS,QAAQ,CAAA;AAAA,OACzE;AAAA,KACH;AAAA,IACA,QAAQ,IAAA,CAAK,QAAA;AAAA,MACX,IAAA,CAAK,KAAA;AAAA,QACH,CAAC,IAAA,CAAK,OAAA,CAAQ,UAAU,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA,EAAG,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAC,CAAA;AAAA,QACrE;AAAA,UACE,WAAA,EACE;AAAA;AACJ;AACF,KACF;AAAA,IACA,cAAc,IAAA,CAAK,QAAA;AAAA,MACjB,KAAK,OAAA,CAAQ;AAAA,QACX,WAAA,EAAa;AAAA,OACd;AAAA,KACH;AAAA,IACA,gBAAgB,IAAA,CAAK,QAAA;AAAA,MACnB,IAAA,CAAK,KAAA,CAAM,CAAC,IAAA,CAAK,OAAA,IAAW,IAAA,CAAK,OAAA,CAAQ,YAAY,CAAC,CAAA,EAAG;AAAA,QACvD,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,OAAO,IAAA,CAAK,QAAA;AAAA,MACV,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EACE;AAAA,OACH;AAAA;AACH,GACF;AACF;AAEA,eAAsB,oBAAA,CACpB,QACA,QAAA,EACmC;AACnC,EAAA,OAAO,aAAA,CAAc;AAAA,IACnB,KAAK,MAAA,CAAO,GAAA;AAAA,IACZ,OAAA,EAAU,MAAA,CAAO,OAAA,IAAsB,QAAA,CAAS,OAAA;AAAA,IAChD,EAAA,EAAK,MAAA,CAAO,EAAA,IAAiB,QAAA,CAAS,EAAA;AAAA,IACtC,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,QAAA,EAAW,MAAA,CAAO,QAAA,IAAuB,QAAA,CAAS,QAAA;AAAA,IAClD,MAAA,EAAS,OAAO,MAAA,IAA2C,UAAA;AAAA,IAC3D,YAAA,EAAe,MAAA,CAAO,YAAA,IAA4B,QAAA,CAAS,YAAA;AAAA,IAC3D,cAAA,EACG,MAAA,CAAO,cAAA,IACR,QAAA,CAAS,cAAA;AAAA,IACX,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,QAAA,CAAS;AAAA,GACrB,CAAA;AACH;;;AC3FO,IAAM,wBAAwB,CAAC,YAAA,GAA6B,EAAC,KAClE,yBAAyB,YAAY;AAEvC,SAAS,mBAAmB,MAAA,EAAqB;AAC/C,EAAA,OAAO;AAAA,IACL,OAAA,EAAS;AAAA,MACP;AAAA,QACE,IAAA,EAAM,MAAA;AAAA,QACN,MAAM,sBAAA,CAAuB,MAAA,EAAQ,EAAE,OAAA,EAAS,MAAM;AAAA;AACxD;AACF,GACF;AACF;AAEA,IAAM,MAAA,GAAS;AAAA,EACb,EAAA,EAAI,aAAA;AAAA,EACJ,IAAA,EAAM,aAAA;AAAA,EACN,WAAA,EACE,8IAAA;AAAA,EAEF,SAAS,GAAA,EAA0B;AACjC,IAAA,MAAM,QAAA,GAAW,qBAAA,CAAsB,GAAA,CAAI,YAAY,CAAA;AAEvD,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM,gBAAA;AAAA,MACN,WAAA,EAAa;AAAA,QACX,wFAAA;AAAA,QACA,gGAAA;AAAA,QACA,8EAAA;AAAA,QACA,0DAAA;AAAA,QACA;AAAA,OACF,CAAE,KAAK,GAAG,CAAA;AAAA,MACV,UAAA,EAAYC,IAAAA,CAAK,MAAA,CAAO,sCAAA,CAAuC,QAAQ,CAAC,CAAA;AAAA,MAExE,MAAM,OAAA,CAAQ,WAAA,EAAqB,MAAA,EAAiC;AAClE,QAAA,MAAM,MAAA,GAAS,MAAM,oBAAA,CAAqB,MAAA,EAAQ,QAAQ,CAAA;AAE1D,QAAA,IAAI,OAAA,CAAQ,MAAM,CAAA,EAAG;AACnB,UAAA,OAAO;AAAA,YACL,OAAA,EAAS;AAAA,cACP,EAAE,IAAA,EAAM,MAAA,EAAiB,MAAM,CAAA,OAAA,EAAU,MAAA,CAAO,KAAK,CAAA,CAAA;AAAG,aAC1D;AAAA,YACA,OAAA,EAAS;AAAA,WACX;AAAA,QACF;AAEA,QAAA,OAAO,mBAAmB,MAAM,CAAA;AAAA,MAClC;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,MAAA,CAAO,IAAA;AAAA,MACT,CAAA,yCAAA,EAA4C,QAAA,CAAS,OAAO,CAAA,CAAA,EAAI,SAAS,EAAE,CAAA,CAAA;AAAA,KAC7E;AAAA,EACF;AACF,CAAA;AAEA,IAAO,aAAA,GAAQ","file":"index.js","sourcesContent":["import type { FingerprintOs } from \"./types\";\n\nexport const DEFAULT_BROWSER = \"chrome_145\";\nexport const DEFAULT_OS: FingerprintOs = \"windows\";\nexport const DEFAULT_MAX_CHARS = 50_000;\nexport const DEFAULT_TIMEOUT_MS = 15_000;\nexport const DEFAULT_INCLUDE_REPLIES = \"extractors\" as const;\nexport const DEFAULT_ACCEPT_HEADER =\n  \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\";\nexport const DEFAULT_ACCEPT_LANGUAGE_HEADER = \"en-US,en;q=0.9\";\n","import { Defuddle } from \"defuddle/node\";\nimport { getProfiles, fetch as wreqFetch } from \"wreq-js\";\nimport type { FetchDependencies } from \"./types\";\n\nexport const runtimeDependencies: FetchDependencies = {\n  fetch: wreqFetch,\n  defuddle: Defuddle,\n  getProfiles,\n};\n","import { parseHTML } from \"linkedom\";\n\n/** Apply linkedom polyfills that Defuddle expects (getComputedStyle, styleSheets). */\nexport function parseLinkedomHTML(html: string, url?: string): Document {\n  const { document } = parseHTML(html);\n  const doc = document as Document & Record<string, unknown>;\n  const defaultView = doc.defaultView as\n    | (Window & {\n        getComputedStyle?: (\n          elt: Element,\n          pseudoElt?: string | null,\n        ) => CSSStyleDeclaration;\n      })\n    | undefined;\n\n  if (!(doc as { styleSheets?: unknown }).styleSheets) {\n    (doc as { styleSheets?: unknown }).styleSheets =\n      [] as unknown as StyleSheetList;\n  }\n\n  if (defaultView && !defaultView.getComputedStyle) {\n    defaultView.getComputedStyle = (() => ({\n      display: \"\",\n    })) as unknown as typeof defaultView.getComputedStyle;\n  }\n\n  if (url) {\n    (doc as { URL?: string }).URL = url;\n  }\n\n  return document;\n}\n","import type { FetchResult } from \"./types\";\n\nfunction buildHeader(\n  parts: Array<[label: string, value: string | number | undefined]>,\n) {\n  return parts\n    .filter(([, value]) => value !== undefined && value !== \"\")\n    .map(([label, value]) => `> ${label}: ${value}`)\n    .join(\"\\n\");\n}\n\nexport function markdownToText(markdown: string): string {\n  return markdown\n    .replace(/^#{1,6}\\s+/gm, \"\")\n    .replace(/\\*\\*([^*]+)\\*\\*/g, \"$1\")\n    .replace(/\\*([^*]+)\\*/g, \"$1\")\n    .replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, \"$1\")\n    .replace(/!\\[[^\\]]*\\]\\([^)]+\\)/g, \"\")\n    .replace(/^>\\s+/gm, \"\")\n    .replace(/^[-*+]\\s+/gm, \"• \")\n    .replace(/`([^`]+)`/g, \"$1\");\n}\n\nexport function truncateContent(content: string, maxChars: number): string {\n  if (content.length <= maxChars) return content;\n  return `${content.slice(0, maxChars)}\\n\\n[... truncated]`;\n}\n\nexport function buildCompactMetadataHeader(result: FetchResult): string {\n  return buildHeader([\n    [\"URL\", result.finalUrl],\n    [\"Title\", result.title],\n    [\"Author\", result.author],\n    [\"Published\", result.published],\n  ]);\n}\n\nexport function buildMetadataHeader(result: FetchResult): string {\n  return buildHeader([\n    [\"URL\", result.finalUrl],\n    [\"Title\", result.title],\n    [\"Author\", result.author],\n    [\"Published\", result.published],\n    [\"Site\", result.site],\n    [\"Language\", result.language],\n    [\"Words\", result.wordCount],\n    [\"Browser\", `${result.browser}/${result.os}`],\n  ]);\n}\n\nexport function buildFetchResponseText(\n  result: FetchResult,\n  options: { verbose?: boolean } = {},\n): string {\n  const header = options.verbose\n    ? buildMetadataHeader(result)\n    : buildCompactMetadataHeader(result);\n\n  return header ? `${header}\\n\\n${result.content}` : result.content;\n}\n","/**\n * Core extraction pipeline: fetch with TLS fingerprinting → parse → Defuddle extract.\n * Separated from the plugin entry so it can be tested independently.\n */\n\nimport {\n  DEFAULT_ACCEPT_HEADER,\n  DEFAULT_ACCEPT_LANGUAGE_HEADER,\n  DEFAULT_BROWSER,\n  DEFAULT_INCLUDE_REPLIES,\n  DEFAULT_MAX_CHARS,\n  DEFAULT_OS,\n  DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { runtimeDependencies } from \"./dependencies\";\nimport { parseLinkedomHTML } from \"./dom\";\nimport { markdownToText, truncateContent } from \"./format\";\nimport { getLatestChromeProfile as getLatestChromeProfileFrom } from \"./profiles\";\nimport type {\n  FetchDependencies,\n  FetchError,\n  FetchOptions,\n  FetchResult,\n  OutputFormat,\n} from \"./types\";\n\nexport {\n  DEFAULT_BROWSER,\n  DEFAULT_INCLUDE_REPLIES,\n  DEFAULT_MAX_CHARS,\n  DEFAULT_OS,\n  DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nexport type {\n  FetchError,\n  FetchOptions,\n  FetchResult,\n  OutputFormat,\n} from \"./types\";\n\nconst HTML_CONTENT_TYPES = [\n  \"text/html\",\n  \"application/xhtml+xml\",\n  \"text/plain\",\n  \"text/markdown\",\n];\n\nexport function getLatestChromeProfile(): string {\n  return getLatestChromeProfileFrom(runtimeDependencies.getProfiles);\n}\n\nexport function createDefuddleFetch(\n  dependencies: FetchDependencies = runtimeDependencies,\n) {\n  return async function defuddleFetch(\n    opts: FetchOptions,\n  ): Promise<FetchResult | FetchError> {\n    const browser = opts.browser ?? DEFAULT_BROWSER;\n    const os = opts.os ?? DEFAULT_OS;\n    const format: OutputFormat = opts.format ?? \"markdown\";\n    const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;\n    const removeImages = opts.removeImages ?? false;\n    const includeReplies = opts.includeReplies ?? DEFAULT_INCLUDE_REPLIES;\n    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\n    let parsed: URL;\n    try {\n      parsed = new URL(opts.url);\n    } catch {\n      return { error: `Invalid URL: ${opts.url}` };\n    }\n\n    if (![\"http:\", \"https:\"].includes(parsed.protocol)) {\n      return {\n        error: `Only http/https URLs supported, got ${parsed.protocol}`,\n      };\n    }\n\n    const fetchOptions: Record<string, unknown> = {\n      browser,\n      os,\n      headers: {\n        Accept: DEFAULT_ACCEPT_HEADER,\n        \"Accept-Language\": DEFAULT_ACCEPT_LANGUAGE_HEADER,\n        ...opts.headers,\n      },\n      redirect: \"follow\",\n      timeout: timeoutMs,\n    };\n\n    if (opts.proxy) {\n      fetchOptions.proxy = opts.proxy;\n    }\n\n    const response = await dependencies.fetch(opts.url, fetchOptions);\n\n    if (!response.ok) {\n      return {\n        error: `HTTP ${response.status} ${response.statusText} for ${opts.url}`,\n      };\n    }\n\n    const finalUrl = response.url ?? opts.url;\n    const contentType = response.headers.get(\"content-type\") ?? \"\";\n\n    if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {\n      return { error: `Not an HTML page (content-type: ${contentType})` };\n    }\n\n    const html = await response.text();\n    const document = parseLinkedomHTML(html, finalUrl);\n    const extracted = await dependencies.defuddle(document, finalUrl, {\n      markdown: format !== \"html\",\n      removeImages,\n      includeReplies,\n    });\n\n    if (!extracted.content || extracted.wordCount === 0) {\n      return {\n        error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`,\n      };\n    }\n\n    const normalizedContent =\n      format === \"text\" ? markdownToText(extracted.content) : extracted.content;\n\n    return {\n      url: opts.url,\n      finalUrl,\n      title: extracted.title ?? \"\",\n      author: extracted.author ?? \"\",\n      published: extracted.published ?? \"\",\n      site: extracted.site ?? \"\",\n      language: extracted.language ?? \"\",\n      wordCount: extracted.wordCount,\n      content: truncateContent(normalizedContent, maxChars),\n      browser,\n      os,\n    };\n  };\n}\n\nexport const defuddleFetch = createDefuddleFetch();\n\n/** Type guard: check if result is an error. */\nexport function isError(\n  result: FetchResult | FetchError,\n): result is FetchError {\n  return \"error\" in result;\n}\n","import { type TSchema, Type } from \"@sinclair/typebox\";\nimport {\n  DEFAULT_BROWSER,\n  DEFAULT_INCLUDE_REPLIES,\n  DEFAULT_MAX_CHARS,\n  DEFAULT_OS,\n  DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { defuddleFetch } from \"./extract\";\nimport type {\n  FetchError,\n  FetchResult,\n  FetchToolConfig,\n  FetchToolDefaults,\n} from \"./types\";\n\nexport function resolveFetchToolDefaults(\n  config: FetchToolConfig = {},\n): FetchToolDefaults {\n  return {\n    maxChars: config.maxChars ?? DEFAULT_MAX_CHARS,\n    timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,\n    browser: config.browser ?? DEFAULT_BROWSER,\n    os: config.os ?? DEFAULT_OS,\n    removeImages: config.removeImages ?? false,\n    includeReplies: config.includeReplies ?? DEFAULT_INCLUDE_REPLIES,\n  };\n}\n\nexport function createBaseFetchToolParameterProperties(\n  defaults: FetchToolDefaults,\n): Record<string, TSchema> {\n  return {\n    url: Type.String({ description: \"URL to fetch (http/https only)\" }),\n    browser: Type.Optional(\n      Type.String({\n        description: `Browser profile for TLS fingerprinting. Default: \"${defaults.browser}\". Examples: chrome_145, firefox_147, safari_26, edge_145, opera_127`,\n      }),\n    ),\n    os: Type.Optional(\n      Type.String({\n        description: `OS profile for fingerprinting. Default: \"${defaults.os}\". Options: windows, macos, linux, android, ios`,\n      }),\n    ),\n    headers: Type.Optional(\n      Type.Record(Type.String(), Type.String(), {\n        description:\n          \"Custom HTTP headers to send. By default, Accept and Accept-Language are set automatically.\",\n      }),\n    ),\n    maxChars: Type.Optional(\n      Type.Number({\n        description: `Maximum characters to return. Default: ${defaults.maxChars}`,\n      }),\n    ),\n    format: Type.Optional(\n      Type.Union(\n        [Type.Literal(\"markdown\"), Type.Literal(\"html\"), Type.Literal(\"text\")],\n        {\n          description:\n            'Output format. \"markdown\" (default), \"html\" (cleaned HTML), or \"text\" (plain text, no formatting)',\n        },\n      ),\n    ),\n    removeImages: Type.Optional(\n      Type.Boolean({\n        description: \"Strip image references from output. Default: false\",\n      }),\n    ),\n    includeReplies: Type.Optional(\n      Type.Union([Type.Boolean(), Type.Literal(\"extractors\")], {\n        description:\n          \"Include replies/comments: 'extractors' for site-specific only (default), true for all, false for none\",\n      }),\n    ),\n    proxy: Type.Optional(\n      Type.String({\n        description:\n          \"Proxy URL (http://user:pass@host:port or socks5://host:port)\",\n      }),\n    ),\n  };\n}\n\nexport async function executeFetchToolCall(\n  params: Record<string, unknown>,\n  defaults: FetchToolDefaults,\n): Promise<FetchResult | FetchError> {\n  return defuddleFetch({\n    url: params.url as string,\n    browser: (params.browser as string) ?? defaults.browser,\n    os: (params.os as string) ?? defaults.os,\n    headers: params.headers as Record<string, string> | undefined,\n    maxChars: (params.maxChars as number) ?? defaults.maxChars,\n    format: (params.format as \"markdown\" | \"html\" | \"text\") ?? \"markdown\",\n    removeImages: (params.removeImages as boolean) ?? defaults.removeImages,\n    includeReplies:\n      (params.includeReplies as boolean | \"extractors\") ??\n      defaults.includeReplies,\n    proxy: params.proxy as string | undefined,\n    timeoutMs: defaults.timeoutMs,\n  });\n}\n","import { Type } from \"@sinclair/typebox\";\nimport {\n  buildFetchResponseText,\n  createBaseFetchToolParameterProperties,\n  executeFetchToolCall,\n  type FetchResult,\n  isError,\n  resolveFetchToolDefaults,\n} from \"smart-fetch-core\";\nimport type { PluginConfig, ToolRegistrationApi } from \"./types\";\n\nexport const resolvePluginDefaults = (pluginConfig: PluginConfig = {}) =>\n  resolveFetchToolDefaults(pluginConfig);\n\nfunction renderToolResponse(result: FetchResult) {\n  return {\n    content: [\n      {\n        type: \"text\" as const,\n        text: buildFetchResponseText(result, { verbose: true }),\n      },\n    ],\n  };\n}\n\nconst plugin = {\n  id: \"smart-fetch\",\n  name: \"Smart Fetch\",\n  description:\n    \"Clean web content extraction with TLS fingerprinting. Uses wreq-js (Rust native bindings) for browser-grade TLS and Defuddle for extraction.\",\n\n  register(api: ToolRegistrationApi) {\n    const defaults = resolvePluginDefaults(api.pluginConfig);\n\n    api.registerTool({\n      name: \"defuddle_fetch\",\n      description: [\n        \"Fetch a URL with browser-grade TLS fingerprinting and extract clean, readable content.\",\n        \"Uses Rust native bindings to impersonate real browsers at the TLS/HTTP2 level (JA3/JA4 match).\",\n        \"Returns markdown with rich metadata (author, publish date, schema.org data).\",\n        \"Better noise removal and anti-bot bypass than web_fetch.\",\n        \"Does NOT execute JavaScript — use the browser tool for JS-heavy SPAs.\",\n      ].join(\" \"),\n      parameters: Type.Object(createBaseFetchToolParameterProperties(defaults)),\n\n      async execute(_toolCallId: string, params: Record<string, unknown>) {\n        const result = await executeFetchToolCall(params, defaults);\n\n        if (isError(result)) {\n          return {\n            content: [\n              { type: \"text\" as const, text: `Error: ${result.error}` },\n            ],\n            isError: true,\n          };\n        }\n\n        return renderToolResponse(result);\n      },\n    });\n\n    api.logger.info(\n      `defuddle_fetch tool registered (default: ${defaults.browser}/${defaults.os})`,\n    );\n  },\n};\n\nexport default plugin;\n"]}
+{"version":3,"sources":["../../core/src/constants.ts","../../core/src/dependencies.ts","../../core/src/dom.ts","../../core/src/format.ts","../../core/src/extract.ts","../../core/src/tool.ts","../src/index.ts"],"names":["wreqFetch","defuddleFetch","Type"],"mappings":";;;;;;;;AAEO,IAAM,eAAA,GAAkB,YAAA;AACxB,IAAM,UAAA,GAA4B,SAAA;AAClC,IAAM,iBAAA,GAAoB,GAAA;AAC1B,IAAM,kBAAA,GAAqB,IAAA;AAC3B,IAAM,uBAAA,GAA0B,YAAA;AAChC,IAAM,qBAAA,GACX,iEAAA;AACK,IAAM,0BAAA,GACX,iFAAA;AACK,IAAM,8BAAA,GAAiC,gBAAA;ACPvC,IAAM,mBAAA,GAAyC;AAAA,EACpD,KAAA,EAAOA,KAAA;AAAA,EACP,QAAA,EAAU,QAAA;AAAA,EACV;AACF,CAAA;ACLO,SAAS,iBAAA,CAAkB,MAAc,GAAA,EAAwB;AACtE,EAAA,MAAM,EAAE,QAAA,EAAS,GAAI,SAAA,CAAU,IAAI,CAAA;AACnC,EAAA,MAAM,GAAA,GAAM,QAAA;AACZ,EAAA,MAAM,cAAc,GAAA,CAAI,WAAA;AASxB,EAAA,IAAI,CAAE,IAAkC,WAAA,EAAa;AACnD,IAAC,GAAA,CAAkC,cACjC,EAAC;AAAA,EACL;AAEA,EAAA,IAAI,WAAA,IAAe,CAAC,WAAA,CAAY,gBAAA,EAAkB;AAChD,IAAA,WAAA,CAAY,oBAAoB,OAAO;AAAA,MACrC,OAAA,EAAS;AAAA,KACX,CAAA,CAAA;AAAA,EACF;AAEA,EAAA,IAAI,GAAA,EAAK;AACP,IAAC,IAAyB,GAAA,GAAM,GAAA;AAAA,EAClC;AAEA,EAAA,OAAO,QAAA;AACT;;;AC7BA,SAAS,YACP,KAAA,EACA;AACA,EAAA,OAAO,KAAA,CACJ,MAAA,CAAO,CAAC,GAAG,KAAK,CAAA,KAAM,KAAA,KAAU,MAAA,IAAa,KAAA,KAAU,EAAE,CAAA,CACzD,IAAI,CAAC,CAAC,KAAA,EAAO,KAAK,CAAA,KAAM,CAAA,EAAA,EAAK,KAAK,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAA,CAC9C,IAAA,CAAK,IAAI,CAAA;AACd;AAEO,SAAS,eAAe,QAAA,EAA0B;AACvD,EAAA,OAAO,QAAA,CACJ,OAAA,CAAQ,cAAA,EAAgB,EAAE,CAAA,CAC1B,OAAA,CAAQ,kBAAA,EAAoB,IAAI,CAAA,CAChC,OAAA,CAAQ,cAAA,EAAgB,IAAI,EAC5B,OAAA,CAAQ,wBAAA,EAA0B,IAAI,CAAA,CACtC,OAAA,CAAQ,uBAAA,EAAyB,EAAE,CAAA,CACnC,QAAQ,SAAA,EAAW,EAAE,CAAA,CACrB,OAAA,CAAQ,aAAA,EAAe,SAAI,CAAA,CAC3B,OAAA,CAAQ,cAAc,IAAI,CAAA;AAC/B;AAEO,SAAS,eAAA,CAAgB,SAAiB,QAAA,EAA0B;AACzE,EAAA,IAAI,OAAA,CAAQ,MAAA,IAAU,QAAA,EAAU,OAAO,OAAA;AACvC,EAAA,OAAO,CAAA,EAAG,OAAA,CAAQ,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAC;;AAAA,eAAA,CAAA;AACtC;AAEO,SAAS,2BAA2B,MAAA,EAA6B;AACtE,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS;AAAA,GAC/B,CAAA;AACH;AAEO,SAAS,oBAAoB,MAAA,EAA6B;AAC/D,EAAA,OAAO,WAAA,CAAY;AAAA,IACjB,CAAC,KAAA,EAAO,MAAA,CAAO,QAAQ,CAAA;AAAA,IACvB,CAAC,OAAA,EAAS,MAAA,CAAO,KAAK,CAAA;AAAA,IACtB,CAAC,QAAA,EAAU,MAAA,CAAO,MAAM,CAAA;AAAA,IACxB,CAAC,WAAA,EAAa,MAAA,CAAO,SAAS,CAAA;AAAA,IAC9B,CAAC,MAAA,EAAQ,MAAA,CAAO,IAAI,CAAA;AAAA,IACpB,CAAC,UAAA,EAAY,MAAA,CAAO,QAAQ,CAAA;AAAA,IAC5B,CAAC,OAAA,EAAS,MAAA,CAAO,SAAS,CAAA;AAAA,IAC1B,CAAC,WAAW,CAAA,EAAG,MAAA,CAAO,OAAO,CAAA,CAAA,EAAI,MAAA,CAAO,EAAE,CAAA,CAAE;AAAA,GAC7C,CAAA;AACH;AAEO,SAAS,sBAAA,CACd,MAAA,EACA,OAAA,GAAiC,EAAC,EAC1B;AACR,EAAA,MAAM,SAAS,OAAA,CAAQ,OAAA,GACnB,oBAAoB,MAAM,CAAA,GAC1B,2BAA2B,MAAM,CAAA;AAErC,EAAA,OAAO,MAAA,GAAS,GAAG,MAAM;;AAAA,EAAO,MAAA,CAAO,OAAO,CAAA,CAAA,GAAK,MAAA,CAAO,OAAA;AAC5D;AAEO,SAAS,kBAAkB,OAAA,EAAyB;AACzD,EAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,IAAA,EAAK,CAAE,MAAM,MAAM,CAAA;AACzC,EAAA,OAAO,OAAO,MAAA,IAAU,CAAA;AAC1B;AAEO,SAAS,WAAW,KAAA,EAAuB;AAChD,EAAA,OAAO,MACJ,OAAA,CAAQ,IAAA,EAAM,OAAO,CAAA,CACrB,OAAA,CAAQ,MAAM,MAAM,CAAA,CACpB,QAAQ,IAAA,EAAM,MAAM,EACpB,OAAA,CAAQ,IAAA,EAAM,QAAQ,CAAA,CACtB,OAAA,CAAQ,MAAM,OAAO,CAAA;AAC1B;AAEO,SAAS,mBACd,GAAA,EACoC;AACpC,EAAA,IAAI;AACF,IAAA,OAAO;AAAA,MACL,SAAA,EAAW,KAAK,SAAA,CAAU,IAAA,CAAK,MAAM,GAAG,CAAA,EAAG,MAAM,CAAC;AAAA,KACpD;AAAA,EACF,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,EAAE,OAAO,uBAAA,EAAwB;AAAA,EAC1C;AACF;AAEO,SAAS,iBAAA,CACd,eACA,MAAA,EACQ;AACR,EAAA,QAAQ,MAAA;AAAQ,IACd,KAAK,MAAA;AAAA,IACL,KAAK,MAAA;AACH,MAAA,OAAO,aAAA;AAAA,IACT,KAAK,MAAA;AACH,MAAA,OAAO,CAAA,iCAAA,EAAoC,UAAA,CAAW,aAAa,CAAC,CAAA,aAAA,CAAA;AAAA,IACtE;AACE,MAAA,OAAO,CAAA;AAAA,EAAe,aAAa;AAAA,MAAA,CAAA;AAAA;AAEzC;AAEO,SAAS,sBAAA,CACd,SACA,MAAA,EACQ;AACR,EAAA,IAAI,WAAW,MAAA,EAAQ;AACrB,IAAA,OAAO,OAAA,CACJ,OAAA,CAAQ,iDAAA,EAAmD,EAAE,EAC7D,OAAA,EAAQ;AAAA,EACb;AAEA,EAAA,OAAO,OAAA,CAAQ,OAAA,CAAQ,gCAAA,EAAkC,EAAE,EAAE,OAAA,EAAQ;AACvE;;;ACjEA,IAAM,kBAAA,GAAqB;AAAA,EACzB,WAAA;AAAA,EACA,uBAAA;AAAA,EACA,YAAA;AAAA,EACA;AACF,CAAA;AAEA,SAAS,oBAAoB,MAAA,EAA8B;AACzD,EAAA,OAAO,MAAA,KAAW,SAAS,0BAAA,GAA6B,qBAAA;AAC1D;AAEA,SAAS,kBAAkB,WAAA,EAA8B;AACvD,EAAA,MAAM,UAAA,GAAa,WAAA,CAAY,KAAA,CAAM,GAAG,CAAA,CAAE,CAAC,CAAA,EAAG,IAAA,EAAK,CAAE,WAAA,EAAY,IAAK,EAAA;AACtE,EAAA,OACE,eAAe,kBAAA,IACf,UAAA,KAAe,WAAA,IACf,UAAA,CAAW,SAAS,OAAO,CAAA;AAE/B;AAEA,SAAS,iBAAiB,IAAA,EAAuB;AAC/C,EAAA,MAAM,OAAA,GAAU,KAAK,IAAA,EAAK;AAC1B,EAAA,OAAO,QAAQ,UAAA,CAAW,GAAG,CAAA,IAAK,OAAA,CAAQ,WAAW,GAAG,CAAA;AAC1D;AAEA,SAAS,cAAA,CAAe,aAAqB,IAAA,EAAuB;AAClE,EAAA,OAAO,iBAAA,CAAkB,WAAW,CAAA,IAAK,gBAAA,CAAiB,IAAI,CAAA;AAChE;AAEA,SAAS,gBACP,IAAA,EACA,QAAA,EACA,SACA,MAAA,EACA,QAAA,EACA,SACA,EAAA,EAC0B;AAC1B,EAAA,MAAM,UAAA,GAAa,mBAAmB,OAAO,CAAA;AAE7C,EAAA,IAAI,WAAW,UAAA,EAAY;AACzB,IAAA,OAAO,UAAA;AAAA,EACT;AAEA,EAAA,MAAM,OAAA,GAAU,eAAA;AAAA,IACd,iBAAA,CAAkB,UAAA,CAAW,SAAA,EAAW,MAAM,CAAA;AAAA,IAC9C;AAAA,GACF;AAEA,EAAA,OAAO;AAAA,IACL,KAAK,IAAA,CAAK,GAAA;AAAA,IACV,QAAA;AAAA,IACA,KAAA,EAAO,EAAA;AAAA,IACP,MAAA,EAAQ,EAAA;AAAA,IACR,SAAA,EAAW,EAAA;AAAA,IACX,IAAA,EAAM,IAAI,GAAA,CAAI,QAAQ,CAAA,CAAE,QAAA;AAAA,IACxB,QAAA,EAAU,EAAA;AAAA,IACV,SAAA,EAAW,iBAAA,CAAkB,UAAA,CAAW,SAAS,CAAA;AAAA,IACjD,OAAA;AAAA,IACA,OAAA;AAAA,IACA;AAAA,GACF;AACF;AAEA,SAAS,mBAAmB,IAAA,EAAuB;AACjD,EAAA,OACE,IAAA,KAAS,iBACT,IAAA,CAAK,UAAA,CAAW,IAAI,CAAA,IACpB,IAAA,CAAK,WAAW,WAAW,CAAA;AAE/B;AAMO,SAAS,mBAAA,CACd,eAAkC,mBAAA,EAClC;AACA,EAAA,OAAO,eAAeC,eACpB,IAAA,EACmC;AACnC,IAAA,MAAM,OAAA,GAAU,KAAK,OAAA,IAAW,eAAA;AAChC,IAAA,MAAM,EAAA,GAAK,KAAK,EAAA,IAAM,UAAA;AACtB,IAAA,MAAM,MAAA,GAAuB,KAAK,MAAA,IAAU,UAAA;AAC5C,IAAA,MAAM,QAAA,GAAW,KAAK,QAAA,IAAY,iBAAA;AAClC,IAAA,MAAM,YAAA,GAAe,KAAK,YAAA,IAAgB,KAAA;AAC1C,IAAA,MAAM,cAAA,GAAiB,KAAK,cAAA,IAAkB,uBAAA;AAC9C,IAAA,MAAM,SAAA,GAAY,KAAK,SAAA,IAAa,kBAAA;AAEpC,IAAA,IAAI,MAAA;AACJ,IAAA,IAAI;AACF,MAAA,MAAA,GAAS,IAAI,GAAA,CAAI,IAAA,CAAK,GAAG,CAAA;AAAA,IAC3B,CAAA,CAAA,MAAQ;AACN,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,aAAA,EAAgB,IAAA,CAAK,GAAG,CAAA,CAAA,EAAG;AAAA,IAC7C;AAEA,IAAA,IAAI,CAAC,CAAC,OAAA,EAAS,QAAQ,EAAE,QAAA,CAAS,MAAA,CAAO,QAAQ,CAAA,EAAG;AAClD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,oCAAA,EAAuC,MAAA,CAAO,QAAQ,CAAA;AAAA,OAC/D;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAwC;AAAA,MAC5C,OAAA;AAAA,MACA,EAAA;AAAA,MACA,OAAA,EAAS;AAAA,QACP,MAAA,EAAQ,oBAAoB,MAAM,CAAA;AAAA,QAClC,iBAAA,EAAmB,8BAAA;AAAA,QACnB,GAAG,IAAA,CAAK;AAAA,OACV;AAAA,MACA,QAAA,EAAU,QAAA;AAAA,MACV,OAAA,EAAS;AAAA,KACX;AAEA,IAAA,IAAI,KAAK,KAAA,EAAO;AACd,MAAA,YAAA,CAAa,QAAQ,IAAA,CAAK,KAAA;AAAA,IAC5B;AAEA,IAAA,MAAM,WAAW,MAAM,YAAA,CAAa,KAAA,CAAM,IAAA,CAAK,KAAK,YAAY,CAAA;AAEhE,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,QAAQ,QAAA,CAAS,MAAM,IAAI,QAAA,CAAS,UAAU,CAAA,KAAA,EAAQ,IAAA,CAAK,GAAG,CAAA;AAAA,OACvE;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,QAAA,CAAS,GAAA,IAAO,IAAA,CAAK,GAAA;AACtC,IAAA,MAAM,WAAA,GAAc,QAAA,CAAS,OAAA,CAAQ,GAAA,CAAI,cAAc,CAAA,IAAK,EAAA;AAC5D,IAAA,MAAM,OAAA,GAAU,MAAM,QAAA,CAAS,IAAA,EAAK;AACpC,IAAA,MAAM,YAAA,GAAe,cAAA,CAAe,WAAA,EAAa,OAAO,CAAA;AAExD,IAAA,IAAI,WAAW,MAAA,EAAQ;AACrB,MAAA,IAAI,CAAC,YAAA,EAAc;AACjB,QAAA,OAAO,EAAE,KAAA,EAAO,CAAA,mCAAA,EAAsC,WAAW,CAAA,CAAA,CAAA,EAAI;AAAA,MACvE;AAEA,MAAA,OAAO,eAAA;AAAA,QACL,IAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA,OACF;AAAA,IACF;AAEA,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,OAAO,eAAA;AAAA,QACL,IAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,OAAA;AAAA,QACA;AAAA,OACF;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,mBAAmB,IAAA,CAAK,CAAC,UAAU,WAAA,CAAY,QAAA,CAAS,KAAK,CAAC,CAAA,EAAG;AACpE,MAAA,OAAO,EAAE,KAAA,EAAO,CAAA,gCAAA,EAAmC,WAAW,CAAA,CAAA,CAAA,EAAI;AAAA,IACpE;AAEA,IAAA,MAAM,QAAA,GAAW,iBAAA,CAAkB,OAAA,EAAS,QAAQ,CAAA;AACpD,IAAA,MAAM,SAAA,GAAY,MAAM,YAAA,CAAa,QAAA,CAAS,UAAU,QAAA,EAAU;AAAA,MAChE,UAAU,MAAA,KAAW,MAAA;AAAA,MACrB,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,IAAI,CAAC,SAAA,CAAU,OAAA,IAAW,SAAA,CAAU,cAAc,CAAA,EAAG;AACnD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,CAAA,0BAAA,EAA6B,IAAA,CAAK,GAAG,CAAA,sCAAA;AAAA,OAC9C;AAAA,IACF;AAEA,IAAA,IAAI,mBAAmB,SAAA,CAAU,OAAA;AACjC,IAAA,IAAI,YAAY,SAAA,CAAU,SAAA;AAE1B,IAAA,IAAI,mBAAmB,KAAA,IAAS,kBAAA,CAAmB,SAAA,CAAU,IAAA,IAAQ,EAAE,CAAA,EAAG;AACxE,MAAA,MAAM,eAAA,GAAkB,sBAAA,CAAuB,gBAAA,EAAkB,MAAM,CAAA;AACvE,MAAA,IAAI,oBAAoB,gBAAA,EAAkB;AACxC,QAAA,gBAAA,GAAmB,eAAA;AACnB,QAAA,SAAA,GAAY,iBAAA;AAAA,UACV,MAAA,KAAW,MAAA,GACP,cAAA,CAAe,gBAAgB,CAAA,GAC/B;AAAA,SACN;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,iBAAA,GACJ,MAAA,KAAW,MAAA,GAAS,cAAA,CAAe,gBAAgB,CAAA,GAAI,gBAAA;AAEzD,IAAA,OAAO;AAAA,MACL,KAAK,IAAA,CAAK,GAAA;AAAA,MACV,QAAA;AAAA,MACA,KAAA,EAAO,UAAU,KAAA,IAAS,EAAA;AAAA,MAC1B,MAAA,EAAQ,UAAU,MAAA,IAAU,EAAA;AAAA,MAC5B,SAAA,EAAW,UAAU,SAAA,IAAa,EAAA;AAAA,MAClC,IAAA,EAAM,UAAU,IAAA,IAAQ,EAAA;AAAA,MACxB,QAAA,EAAU,UAAU,QAAA,IAAY,EAAA;AAAA,MAChC,SAAA;AAAA,MACA,OAAA,EAAS,eAAA,CAAgB,iBAAA,EAAmB,QAAQ,CAAA;AAAA,MACpD,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAA;AACF;AAEO,IAAM,gBAAgB,mBAAA,EAAoB;AAG1C,SAAS,QACd,MAAA,EACsB;AACtB,EAAA,OAAO,OAAA,IAAW,MAAA;AACpB;AC1PO,SAAS,wBAAA,CACd,MAAA,GAA0B,EAAC,EACR;AACnB,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,OAAO,QAAA,IAAY,iBAAA;AAAA,IAC7B,SAAA,EAAW,OAAO,SAAA,IAAa,kBAAA;AAAA,IAC/B,OAAA,EAAS,OAAO,OAAA,IAAW,eAAA;AAAA,IAC3B,EAAA,EAAI,OAAO,EAAA,IAAM,UAAA;AAAA,IACjB,YAAA,EAAc,OAAO,YAAA,IAAgB,KAAA;AAAA,IACrC,cAAA,EAAgB,OAAO,cAAA,IAAkB;AAAA,GAC3C;AACF;AAEO,SAAS,uCACd,QAAA,EACyB;AACzB,EAAA,OAAO;AAAA,IACL,KAAK,IAAA,CAAK,MAAA,CAAO,EAAE,WAAA,EAAa,kCAAkC,CAAA;AAAA,IAClE,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,kDAAA,EAAqD,QAAA,CAAS,OAAO,CAAA,oEAAA;AAAA,OACnF;AAAA,KACH;AAAA,IACA,IAAI,IAAA,CAAK,QAAA;AAAA,MACP,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,yCAAA,EAA4C,QAAA,CAAS,EAAE,CAAA,+CAAA;AAAA,OACrE;AAAA,KACH;AAAA,IACA,SAAS,IAAA,CAAK,QAAA;AAAA,MACZ,KAAK,MAAA,CAAO,IAAA,CAAK,QAAO,EAAG,IAAA,CAAK,QAAO,EAAG;AAAA,QACxC,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,UAAU,IAAA,CAAK,QAAA;AAAA,MACb,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EAAa,CAAA,uCAAA,EAA0C,QAAA,CAAS,QAAQ,CAAA;AAAA,OACzE;AAAA,KACH;AAAA,IACA,QAAQ,IAAA,CAAK,QAAA;AAAA,MACX,IAAA,CAAK,KAAA;AAAA,QACH;AAAA,UACE,IAAA,CAAK,QAAQ,UAAU,CAAA;AAAA,UACvB,IAAA,CAAK,QAAQ,MAAM,CAAA;AAAA,UACnB,IAAA,CAAK,QAAQ,MAAM,CAAA;AAAA,UACnB,IAAA,CAAK,QAAQ,MAAM;AAAA,SACrB;AAAA,QACA;AAAA,UACE,WAAA,EACE;AAAA;AACJ;AACF,KACF;AAAA,IACA,cAAc,IAAA,CAAK,QAAA;AAAA,MACjB,KAAK,OAAA,CAAQ;AAAA,QACX,WAAA,EAAa;AAAA,OACd;AAAA,KACH;AAAA,IACA,gBAAgB,IAAA,CAAK,QAAA;AAAA,MACnB,IAAA,CAAK,KAAA,CAAM,CAAC,IAAA,CAAK,OAAA,IAAW,IAAA,CAAK,OAAA,CAAQ,YAAY,CAAC,CAAA,EAAG;AAAA,QACvD,WAAA,EACE;AAAA,OACH;AAAA,KACH;AAAA,IACA,OAAO,IAAA,CAAK,QAAA;AAAA,MACV,KAAK,MAAA,CAAO;AAAA,QACV,WAAA,EACE;AAAA,OACH;AAAA;AACH,GACF;AACF;AAEA,eAAsB,oBAAA,CACpB,QACA,QAAA,EACmC;AACnC,EAAA,OAAO,aAAA,CAAc;AAAA,IACnB,KAAK,MAAA,CAAO,GAAA;AAAA,IACZ,OAAA,EAAU,MAAA,CAAO,OAAA,IAAsB,QAAA,CAAS,OAAA;AAAA,IAChD,EAAA,EAAK,MAAA,CAAO,EAAA,IAAiB,QAAA,CAAS,EAAA;AAAA,IACtC,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,QAAA,EAAW,MAAA,CAAO,QAAA,IAAuB,QAAA,CAAS,QAAA;AAAA,IAClD,MAAA,EACG,OAAO,MAAA,IAAoD,UAAA;AAAA,IAC9D,YAAA,EAAe,MAAA,CAAO,YAAA,IAA4B,QAAA,CAAS,YAAA;AAAA,IAC3D,cAAA,EACG,MAAA,CAAO,cAAA,IACR,QAAA,CAAS,cAAA;AAAA,IACX,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,QAAA,CAAS;AAAA,GACrB,CAAA;AACH;;;ACjGO,IAAM,wBAAwB,CAAC,YAAA,GAA6B,EAAC,KAClE,yBAAyB,YAAY;AAEvC,SAAS,mBAAmB,MAAA,EAAqB;AAC/C,EAAA,OAAO;AAAA,IACL,OAAA,EAAS;AAAA,MACP;AAAA,QACE,IAAA,EAAM,MAAA;AAAA,QACN,MAAM,sBAAA,CAAuB,MAAA,EAAQ,EAAE,OAAA,EAAS,MAAM;AAAA;AACxD;AACF,GACF;AACF;AAEA,IAAM,MAAA,GAAS;AAAA,EACb,EAAA,EAAI,aAAA;AAAA,EACJ,IAAA,EAAM,aAAA;AAAA,EACN,WAAA,EACE,8IAAA;AAAA,EAEF,SAAS,GAAA,EAA0B;AACjC,IAAA,MAAM,QAAA,GAAW,qBAAA,CAAsB,GAAA,CAAI,YAAY,CAAA;AAEvD,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM,aAAA;AAAA,MACN,WAAA,EAAa;AAAA,QACX,wFAAA;AAAA,QACA,gGAAA;AAAA,QACA,8EAAA;AAAA,QACA,0DAAA;AAAA,QACA;AAAA,OACF,CAAE,KAAK,GAAG,CAAA;AAAA,MACV,UAAA,EAAYC,IAAAA,CAAK,MAAA,CAAO,sCAAA,CAAuC,QAAQ,CAAC,CAAA;AAAA,MAExE,MAAM,OAAA,CAAQ,WAAA,EAAqB,MAAA,EAAiC;AAClE,QAAA,MAAM,MAAA,GAAS,MAAM,oBAAA,CAAqB,MAAA,EAAQ,QAAQ,CAAA;AAE1D,QAAA,IAAI,OAAA,CAAQ,MAAM,CAAA,EAAG;AACnB,UAAA,OAAO;AAAA,YACL,OAAA,EAAS;AAAA,cACP,EAAE,IAAA,EAAM,MAAA,EAAiB,MAAM,CAAA,OAAA,EAAU,MAAA,CAAO,KAAK,CAAA,CAAA;AAAG,aAC1D;AAAA,YACA,OAAA,EAAS;AAAA,WACX;AAAA,QACF;AAEA,QAAA,OAAO,mBAAmB,MAAM,CAAA;AAAA,MAClC;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,MAAA,CAAO,IAAA;AAAA,MACT,CAAA,sCAAA,EAAyC,QAAA,CAAS,OAAO,CAAA,CAAA,EAAI,SAAS,EAAE,CAAA,CAAA;AAAA,KAC1E;AAAA,EACF;AACF,CAAA;AAEA,IAAO,aAAA,GAAQ","file":"index.js","sourcesContent":["import type { FingerprintOs } from \"./types\";\n\nexport const DEFAULT_BROWSER = \"chrome_145\";\nexport const DEFAULT_OS: FingerprintOs = \"windows\";\nexport const DEFAULT_MAX_CHARS = 50_000;\nexport const DEFAULT_TIMEOUT_MS = 15_000;\nexport const DEFAULT_INCLUDE_REPLIES = \"extractors\" as const;\nexport const DEFAULT_ACCEPT_HEADER =\n  \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\";\nexport const DEFAULT_JSON_ACCEPT_HEADER =\n  \"application/json,text/json,application/ld+json;q=0.9,text/plain;q=0.8,*/*;q=0.7\";\nexport const DEFAULT_ACCEPT_LANGUAGE_HEADER = \"en-US,en;q=0.9\";\n","import { Defuddle } from \"defuddle/node\";\nimport { getProfiles, fetch as wreqFetch } from \"wreq-js\";\nimport type { FetchDependencies } from \"./types\";\n\nexport const runtimeDependencies: FetchDependencies = {\n  fetch: wreqFetch,\n  defuddle: Defuddle,\n  getProfiles,\n};\n","import { parseHTML } from \"linkedom\";\n\n/** Apply linkedom polyfills that Defuddle expects (getComputedStyle, styleSheets). */\nexport function parseLinkedomHTML(html: string, url?: string): Document {\n  const { document } = parseHTML(html);\n  const doc = document as Document & Record<string, unknown>;\n  const defaultView = doc.defaultView as\n    | (Window & {\n        getComputedStyle?: (\n          elt: Element,\n          pseudoElt?: string | null,\n        ) => CSSStyleDeclaration;\n      })\n    | undefined;\n\n  if (!(doc as { styleSheets?: unknown }).styleSheets) {\n    (doc as { styleSheets?: unknown }).styleSheets =\n      [] as unknown as StyleSheetList;\n  }\n\n  if (defaultView && !defaultView.getComputedStyle) {\n    defaultView.getComputedStyle = (() => ({\n      display: \"\",\n    })) as unknown as typeof defaultView.getComputedStyle;\n  }\n\n  if (url) {\n    (doc as { URL?: string }).URL = url;\n  }\n\n  return document;\n}\n","import type { FetchError, FetchResult, OutputFormat } from \"./types\";\n\nfunction buildHeader(\n  parts: Array<[label: string, value: string | number | undefined]>,\n) {\n  return parts\n    .filter(([, value]) => value !== undefined && value !== \"\")\n    .map(([label, value]) => `> ${label}: ${value}`)\n    .join(\"\\n\");\n}\n\nexport function markdownToText(markdown: string): string {\n  return markdown\n    .replace(/^#{1,6}\\s+/gm, \"\")\n    .replace(/\\*\\*([^*]+)\\*\\*/g, \"$1\")\n    .replace(/\\*([^*]+)\\*/g, \"$1\")\n    .replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, \"$1\")\n    .replace(/!\\[[^\\]]*\\]\\([^)]+\\)/g, \"\")\n    .replace(/^>\\s+/gm, \"\")\n    .replace(/^[-*+]\\s+/gm, \"• \")\n    .replace(/`([^`]+)`/g, \"$1\");\n}\n\nexport function truncateContent(content: string, maxChars: number): string {\n  if (content.length <= maxChars) return content;\n  return `${content.slice(0, maxChars)}\\n\\n[... truncated]`;\n}\n\nexport function buildCompactMetadataHeader(result: FetchResult): string {\n  return buildHeader([\n    [\"URL\", result.finalUrl],\n    [\"Title\", result.title],\n    [\"Author\", result.author],\n    [\"Published\", result.published],\n  ]);\n}\n\nexport function buildMetadataHeader(result: FetchResult): string {\n  return buildHeader([\n    [\"URL\", result.finalUrl],\n    [\"Title\", result.title],\n    [\"Author\", result.author],\n    [\"Published\", result.published],\n    [\"Site\", result.site],\n    [\"Language\", result.language],\n    [\"Words\", result.wordCount],\n    [\"Browser\", `${result.browser}/${result.os}`],\n  ]);\n}\n\nexport function buildFetchResponseText(\n  result: FetchResult,\n  options: { verbose?: boolean } = {},\n): string {\n  const header = options.verbose\n    ? buildMetadataHeader(result)\n    : buildCompactMetadataHeader(result);\n\n  return header ? `${header}\\n\\n${result.content}` : result.content;\n}\n\nexport function estimateWordCount(content: string): number {\n  const words = content.trim().match(/\\S+/g);\n  return words?.length ?? 0;\n}\n\nexport function escapeHtml(value: string): string {\n  return value\n    .replace(/&/g, \"&amp;\")\n    .replace(/</g, \"&lt;\")\n    .replace(/>/g, \"&gt;\")\n    .replace(/\"/g, \"&quot;\")\n    .replace(/'/g, \"&#39;\");\n}\n\nexport function parseAndFormatJson(\n  raw: string,\n): { formatted: string } | FetchError {\n  try {\n    return {\n      formatted: JSON.stringify(JSON.parse(raw), null, 2),\n    };\n  } catch {\n    return { error: \"Invalid JSON response\" };\n  }\n}\n\nexport function renderJsonContent(\n  formattedJson: string,\n  format: OutputFormat,\n): string {\n  switch (format) {\n    case \"json\":\n    case \"text\":\n      return formattedJson;\n    case \"html\":\n      return `<pre><code class=\"language-json\">${escapeHtml(formattedJson)}</code></pre>`;\n    default:\n      return `\\`\\`\\`json\\n${formattedJson}\\n\\`\\`\\``;\n  }\n}\n\nexport function stripExtractorComments(\n  content: string,\n  format: OutputFormat,\n): string {\n  if (format === \"html\") {\n    return content\n      .replace(/\\s*<hr>\\s*<div class=\"[^\"]* comments\">[\\s\\S]*$/i, \"\")\n      .trimEnd();\n  }\n\n  return content.replace(/\\n---\\n+## Comments\\n[\\s\\S]*$/i, \"\").trimEnd();\n}\n","/**\n * Core extraction pipeline: fetch with TLS fingerprinting → parse → Defuddle extract.\n * Separated from the plugin entry so it can be tested independently.\n */\n\nimport {\n  DEFAULT_ACCEPT_HEADER,\n  DEFAULT_ACCEPT_LANGUAGE_HEADER,\n  DEFAULT_BROWSER,\n  DEFAULT_INCLUDE_REPLIES,\n  DEFAULT_JSON_ACCEPT_HEADER,\n  DEFAULT_MAX_CHARS,\n  DEFAULT_OS,\n  DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { runtimeDependencies } from \"./dependencies\";\nimport { parseLinkedomHTML } from \"./dom\";\nimport {\n  estimateWordCount,\n  markdownToText,\n  parseAndFormatJson,\n  renderJsonContent,\n  stripExtractorComments,\n  truncateContent,\n} from \"./format\";\nimport { getLatestChromeProfile as getLatestChromeProfileFrom } from \"./profiles\";\nimport type {\n  FetchDependencies,\n  FetchError,\n  FetchOptions,\n  FetchResult,\n  OutputFormat,\n} from \"./types\";\n\nexport {\n  DEFAULT_BROWSER,\n  DEFAULT_INCLUDE_REPLIES,\n  DEFAULT_MAX_CHARS,\n  DEFAULT_OS,\n  DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nexport type {\n  FetchError,\n  FetchOptions,\n  FetchResult,\n  OutputFormat,\n} from \"./types\";\n\nconst HTML_CONTENT_TYPES = [\n  \"text/html\",\n  \"application/xhtml+xml\",\n  \"text/plain\",\n  \"text/markdown\",\n];\n\nfunction resolveAcceptHeader(format: OutputFormat): string {\n  return format === \"json\" ? DEFAULT_JSON_ACCEPT_HEADER : DEFAULT_ACCEPT_HEADER;\n}\n\nfunction isJsonContentType(contentType: string): boolean {\n  const normalized = contentType.split(\";\")[0]?.trim().toLowerCase() ?? \"\";\n  return (\n    normalized === \"application/json\" ||\n    normalized === \"text/json\" ||\n    normalized.endsWith(\"+json\")\n  );\n}\n\nfunction isLikelyJsonBody(body: string): boolean {\n  const trimmed = body.trim();\n  return trimmed.startsWith(\"{\") || trimmed.startsWith(\"[\");\n}\n\nfunction isJsonResponse(contentType: string, body: string): boolean {\n  return isJsonContentType(contentType) || isLikelyJsonBody(body);\n}\n\nfunction buildJsonResult(\n  opts: FetchOptions,\n  finalUrl: string,\n  rawBody: string,\n  format: OutputFormat,\n  maxChars: number,\n  browser: string,\n  os: string,\n): FetchResult | FetchError {\n  const parsedJson = parseAndFormatJson(rawBody);\n\n  if (\"error\" in parsedJson) {\n    return parsedJson;\n  }\n\n  const content = truncateContent(\n    renderJsonContent(parsedJson.formatted, format),\n    maxChars,\n  );\n\n  return {\n    url: opts.url,\n    finalUrl,\n    title: \"\",\n    author: \"\",\n    published: \"\",\n    site: new URL(finalUrl).hostname,\n    language: \"\",\n    wordCount: estimateWordCount(parsedJson.formatted),\n    content,\n    browser,\n    os,\n  };\n}\n\nfunction shouldStripReplies(site: string): boolean {\n  return (\n    site === \"Hacker News\" ||\n    site.startsWith(\"r/\") ||\n    site.startsWith(\"GitHub - \")\n  );\n}\n\nexport function getLatestChromeProfile(): string {\n  return getLatestChromeProfileFrom(runtimeDependencies.getProfiles);\n}\n\nexport function createDefuddleFetch(\n  dependencies: FetchDependencies = runtimeDependencies,\n) {\n  return async function defuddleFetch(\n    opts: FetchOptions,\n  ): Promise<FetchResult | FetchError> {\n    const browser = opts.browser ?? DEFAULT_BROWSER;\n    const os = opts.os ?? DEFAULT_OS;\n    const format: OutputFormat = opts.format ?? \"markdown\";\n    const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;\n    const removeImages = opts.removeImages ?? false;\n    const includeReplies = opts.includeReplies ?? DEFAULT_INCLUDE_REPLIES;\n    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n\n    let parsed: URL;\n    try {\n      parsed = new URL(opts.url);\n    } catch {\n      return { error: `Invalid URL: ${opts.url}` };\n    }\n\n    if (![\"http:\", \"https:\"].includes(parsed.protocol)) {\n      return {\n        error: `Only http/https URLs supported, got ${parsed.protocol}`,\n      };\n    }\n\n    const fetchOptions: Record<string, unknown> = {\n      browser,\n      os,\n      headers: {\n        Accept: resolveAcceptHeader(format),\n        \"Accept-Language\": DEFAULT_ACCEPT_LANGUAGE_HEADER,\n        ...opts.headers,\n      },\n      redirect: \"follow\",\n      timeout: timeoutMs,\n    };\n\n    if (opts.proxy) {\n      fetchOptions.proxy = opts.proxy;\n    }\n\n    const response = await dependencies.fetch(opts.url, fetchOptions);\n\n    if (!response.ok) {\n      return {\n        error: `HTTP ${response.status} ${response.statusText} for ${opts.url}`,\n      };\n    }\n\n    const finalUrl = response.url ?? opts.url;\n    const contentType = response.headers.get(\"content-type\") ?? \"\";\n    const rawBody = await response.text();\n    const jsonResponse = isJsonResponse(contentType, rawBody);\n\n    if (format === \"json\") {\n      if (!jsonResponse) {\n        return { error: `Not a JSON response (content-type: ${contentType})` };\n      }\n\n      return buildJsonResult(\n        opts,\n        finalUrl,\n        rawBody,\n        format,\n        maxChars,\n        browser,\n        os,\n      );\n    }\n\n    if (jsonResponse) {\n      return buildJsonResult(\n        opts,\n        finalUrl,\n        rawBody,\n        format,\n        maxChars,\n        browser,\n        os,\n      );\n    }\n\n    if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {\n      return { error: `Not an HTML page (content-type: ${contentType})` };\n    }\n\n    const document = parseLinkedomHTML(rawBody, finalUrl);\n    const extracted = await dependencies.defuddle(document, finalUrl, {\n      markdown: format !== \"html\",\n      removeImages,\n      includeReplies,\n    });\n\n    if (!extracted.content || extracted.wordCount === 0) {\n      return {\n        error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`,\n      };\n    }\n\n    let extractedContent = extracted.content;\n    let wordCount = extracted.wordCount;\n\n    if (includeReplies === false && shouldStripReplies(extracted.site ?? \"\")) {\n      const strippedContent = stripExtractorComments(extractedContent, format);\n      if (strippedContent !== extractedContent) {\n        extractedContent = strippedContent;\n        wordCount = estimateWordCount(\n          format === \"text\"\n            ? markdownToText(extractedContent)\n            : extractedContent,\n        );\n      }\n    }\n\n    const normalizedContent =\n      format === \"text\" ? markdownToText(extractedContent) : extractedContent;\n\n    return {\n      url: opts.url,\n      finalUrl,\n      title: extracted.title ?? \"\",\n      author: extracted.author ?? \"\",\n      published: extracted.published ?? \"\",\n      site: extracted.site ?? \"\",\n      language: extracted.language ?? \"\",\n      wordCount,\n      content: truncateContent(normalizedContent, maxChars),\n      browser,\n      os,\n    };\n  };\n}\n\nexport const defuddleFetch = createDefuddleFetch();\n\n/** Type guard: check if result is an error. */\nexport function isError(\n  result: FetchResult | FetchError,\n): result is FetchError {\n  return \"error\" in result;\n}\n","import { type TSchema, Type } from \"@sinclair/typebox\";\nimport {\n  DEFAULT_BROWSER,\n  DEFAULT_INCLUDE_REPLIES,\n  DEFAULT_MAX_CHARS,\n  DEFAULT_OS,\n  DEFAULT_TIMEOUT_MS,\n} from \"./constants\";\nimport { defuddleFetch } from \"./extract\";\nimport type {\n  FetchError,\n  FetchResult,\n  FetchToolConfig,\n  FetchToolDefaults,\n} from \"./types\";\n\nexport function resolveFetchToolDefaults(\n  config: FetchToolConfig = {},\n): FetchToolDefaults {\n  return {\n    maxChars: config.maxChars ?? DEFAULT_MAX_CHARS,\n    timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,\n    browser: config.browser ?? DEFAULT_BROWSER,\n    os: config.os ?? DEFAULT_OS,\n    removeImages: config.removeImages ?? false,\n    includeReplies: config.includeReplies ?? DEFAULT_INCLUDE_REPLIES,\n  };\n}\n\nexport function createBaseFetchToolParameterProperties(\n  defaults: FetchToolDefaults,\n): Record<string, TSchema> {\n  return {\n    url: Type.String({ description: \"URL to fetch (http/https only)\" }),\n    browser: Type.Optional(\n      Type.String({\n        description: `Browser profile for TLS fingerprinting. Default: \"${defaults.browser}\". Examples: chrome_145, firefox_147, safari_26, edge_145, opera_127`,\n      }),\n    ),\n    os: Type.Optional(\n      Type.String({\n        description: `OS profile for fingerprinting. Default: \"${defaults.os}\". Options: windows, macos, linux, android, ios`,\n      }),\n    ),\n    headers: Type.Optional(\n      Type.Record(Type.String(), Type.String(), {\n        description:\n          \"Custom HTTP headers to send. By default, Accept and Accept-Language are set automatically.\",\n      }),\n    ),\n    maxChars: Type.Optional(\n      Type.Number({\n        description: `Maximum characters to return. Default: ${defaults.maxChars}`,\n      }),\n    ),\n    format: Type.Optional(\n      Type.Union(\n        [\n          Type.Literal(\"markdown\"),\n          Type.Literal(\"html\"),\n          Type.Literal(\"text\"),\n          Type.Literal(\"json\"),\n        ],\n        {\n          description:\n            'Output format. \"markdown\" (default), \"html\" (cleaned HTML), \"text\" (plain text, no formatting), or \"json\" (pretty-printed JSON)',\n        },\n      ),\n    ),\n    removeImages: Type.Optional(\n      Type.Boolean({\n        description: \"Strip image references from output. Default: false\",\n      }),\n    ),\n    includeReplies: Type.Optional(\n      Type.Union([Type.Boolean(), Type.Literal(\"extractors\")], {\n        description:\n          \"Include replies/comments: 'extractors' for site-specific only (default), true for all, false for none\",\n      }),\n    ),\n    proxy: Type.Optional(\n      Type.String({\n        description:\n          \"Proxy URL (http://user:pass@host:port or socks5://host:port)\",\n      }),\n    ),\n  };\n}\n\nexport async function executeFetchToolCall(\n  params: Record<string, unknown>,\n  defaults: FetchToolDefaults,\n): Promise<FetchResult | FetchError> {\n  return defuddleFetch({\n    url: params.url as string,\n    browser: (params.browser as string) ?? defaults.browser,\n    os: (params.os as string) ?? defaults.os,\n    headers: params.headers as Record<string, string> | undefined,\n    maxChars: (params.maxChars as number) ?? defaults.maxChars,\n    format:\n      (params.format as \"markdown\" | \"html\" | \"text\" | \"json\") ?? \"markdown\",\n    removeImages: (params.removeImages as boolean) ?? defaults.removeImages,\n    includeReplies:\n      (params.includeReplies as boolean | \"extractors\") ??\n      defaults.includeReplies,\n    proxy: params.proxy as string | undefined,\n    timeoutMs: defaults.timeoutMs,\n  });\n}\n","import { Type } from \"@sinclair/typebox\";\nimport {\n  buildFetchResponseText,\n  createBaseFetchToolParameterProperties,\n  executeFetchToolCall,\n  type FetchResult,\n  isError,\n  resolveFetchToolDefaults,\n} from \"smart-fetch-core\";\nimport type { PluginConfig, ToolRegistrationApi } from \"./types\";\n\nexport const resolvePluginDefaults = (pluginConfig: PluginConfig = {}) =>\n  resolveFetchToolDefaults(pluginConfig);\n\nfunction renderToolResponse(result: FetchResult) {\n  return {\n    content: [\n      {\n        type: \"text\" as const,\n        text: buildFetchResponseText(result, { verbose: true }),\n      },\n    ],\n  };\n}\n\nconst plugin = {\n  id: \"smart-fetch\",\n  name: \"Smart Fetch\",\n  description:\n    \"Clean web content extraction with TLS fingerprinting. Uses wreq-js (Rust native bindings) for browser-grade TLS and Defuddle for extraction.\",\n\n  register(api: ToolRegistrationApi) {\n    const defaults = resolvePluginDefaults(api.pluginConfig);\n\n    api.registerTool({\n      name: \"smart_fetch\",\n      description: [\n        \"Fetch a URL with browser-grade TLS fingerprinting and extract clean, readable content.\",\n        \"Uses Rust native bindings to impersonate real browsers at the TLS/HTTP2 level (JA3/JA4 match).\",\n        \"Returns markdown with rich metadata (author, publish date, schema.org data).\",\n        \"Better noise removal and anti-bot bypass than web_fetch.\",\n        \"Does NOT execute JavaScript — use the browser tool for JS-heavy SPAs.\",\n      ].join(\" \"),\n      parameters: Type.Object(createBaseFetchToolParameterProperties(defaults)),\n\n      async execute(_toolCallId: string, params: Record<string, unknown>) {\n        const result = await executeFetchToolCall(params, defaults);\n\n        if (isError(result)) {\n          return {\n            content: [\n              { type: \"text\" as const, text: `Error: ${result.error}` },\n            ],\n            isError: true,\n          };\n        }\n\n        return renderToolResponse(result);\n      },\n    });\n\n    api.logger.info(\n      `smart_fetch tool registered (default: ${defaults.browser}/${defaults.os})`,\n    );\n  },\n};\n\nexport default plugin;\n"]}

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "smart-fetch",
   "name": "Smart Fetch",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Clean web content extraction with browser-grade TLS fingerprinting. Uses wreq-js (Rust native bindings) for anti-bot bypass and Defuddle for superior content extraction.",
   "configSchema": {
     "type": "object",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openclaw-smart-fetch",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "type": "module",
   "description": "OpenClaw smart fetch plugin with browser-grade TLS fingerprinting and Defuddle extraction.",
   "license": "MIT",
@@ -39,7 +39,7 @@
   ],
   "engines": {
     "bun": ">=1.3.0",
-    "node": ">=22"
+    "node": ">=24"
   },
   "publishConfig": {
     "access": "public"