beeai-framework 0.1.2

package/dist/tools/search/wikipedia.cjs

@@ -0,0 +1,338 @@
+'use strict';
+
+var wiki = require('wikipedia');
+var decoratorCache_cjs = require('../../cache/decoratorCache.cjs');
+var R = require('remeda');
+var base_cjs = require('./base.cjs');
+var promise_cjs = require('../../internals/helpers/promise.cjs');
+var zod = require('zod');
+var base_cjs$1 = require('../base.cjs');
+var Turndown = require('turndown');
+var turndownPlugin = require('joplin-turndown-plugin-gfm');
+var stringComparison = require('string-comparison');
+var emitter_cjs = require('../../emitter/emitter.cjs');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+function _interopNamespace(e) {
+  if (e && e.__esModule) return e;
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var wiki__default = /*#__PURE__*/_interopDefault(wiki);
+var R__namespace = /*#__PURE__*/_interopNamespace(R);
+var Turndown__default = /*#__PURE__*/_interopDefault(Turndown);
+var turndownPlugin__default = /*#__PURE__*/_interopDefault(turndownPlugin);
+var stringComparison__default = /*#__PURE__*/_interopDefault(stringComparison);
+
+var __defProp = Object.defineProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+function _ts_decorate(decorators, target, key, desc) {
+  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+  return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+__name(_ts_decorate, "_ts_decorate");
+function _ts_metadata(k, v) {
+  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+}
+__name(_ts_metadata, "_ts_metadata");
+wiki__default.default.default.setLang("en");
+class WikipediaToolOutput extends base_cjs.SearchToolOutput {
+  static {
+    __name(this, "WikipediaToolOutput");
+  }
+  results;
+  maxSerializedLength;
+  constructor(results, maxSerializedLength) {
+    super(results), this.results = results, this.maxSerializedLength = maxSerializedLength;
+  }
+  static {
+    this.register();
+  }
+  getTextContent() {
+    if (this.isEmpty()) {
+      return `No results were found. Try to reformat your query.`;
+    }
+    const target = this.results.length === 1 ? this.results[0] : this.results;
+    const response = JSON.stringify(target);
+    return this.maxSerializedLength < Infinity ? response.substring(0, this.maxSerializedLength) : response;
+  }
+  createSnapshot() {
+    return {
+      results: this.results,
+      maxSerializedLength: this.maxSerializedLength
+    };
+  }
+  loadSnapshot(snapshot) {
+    Object.assign(this, snapshot);
+  }
+}
+_ts_decorate([
+  decoratorCache_cjs.Cache(),
+  _ts_metadata("design:type", Function),
+  _ts_metadata("design:paramtypes", []),
+  _ts_metadata("design:returntype", String)
+], WikipediaToolOutput.prototype, "getTextContent", null);
+class WikipediaTool extends base_cjs$1.Tool {
+  static {
+    __name(this, "WikipediaTool");
+  }
+  config;
+  name;
+  description;
+  emitter;
+  inputSchema() {
+    return zod.z.object({
+      query: zod.z.string({
+        description: `Name of the wikipedia page, for example 'New York'`
+      }).min(1).max(128)
+    });
+  }
+  constructor(config = {}) {
+    super(config), this.config = config, this.name = "Wikipedia", this.description = "Search factual and historical information, including biography, history, politics, geography, society, culture, science, technology, people, animal species, mathematics, and other subjects.", this.emitter = emitter_cjs.Emitter.root.child({
+      namespace: [
+        "tool",
+        "search",
+        "wikipedia"
+      ],
+      creator: this
+    });
+  }
+  static {
+    this.register();
+  }
+  get _mappers() {
+    return {
+      categories: /* @__PURE__ */ __name((page) => page.categories(), "categories"),
+      content: /* @__PURE__ */ __name((page) => page.content(), "content"),
+      html: /* @__PURE__ */ __name((page) => page.html(), "html"),
+      markdown: /* @__PURE__ */ __name(async (page, runOptions) => {
+        const html = await page.html().then((result) => {
+          const url = new URL(page.fullurl);
+          const base = `${url.protocol}//${[
+            url.hostname,
+            url.port
+          ].filter(Boolean).join(":")}`;
+          return result.replace(/(<img .*src=)"(\/\/.*)"/gm, `$1"${url.protocol}$2"`).replace(/(<a .*href=)"(\/\/.*)"/gm, `$1"${url.protocol}$2"`).replace(/(<img .*src=)"(\/.*)"/gm, `$1"${base}$2"`).replace(/(<a .*href=)"(\/.*)"/gm, `$1"${base}$2"`);
+        });
+        const service = new Turndown__default.default();
+        service.use(turndownPlugin__default.default.gfm);
+        return service.remove((node) => runOptions.extraction?.fields?.markdown?.filter?.(node) === false).turndown(html);
+      }, "markdown"),
+      images: /* @__PURE__ */ __name((page) => page.images(), "images"),
+      intro: /* @__PURE__ */ __name((page) => page.intro(), "intro"),
+      infobox: /* @__PURE__ */ __name((page) => page.infobox(), "infobox"),
+      links: /* @__PURE__ */ __name((page) => page.links(), "links"),
+      coordinates: /* @__PURE__ */ __name((page) => page.coordinates(), "coordinates"),
+      langLinks: /* @__PURE__ */ __name((page) => page.langLinks(), "langLinks"),
+      references: /* @__PURE__ */ __name((page) => page.references(), "references"),
+      related: /* @__PURE__ */ __name((page) => page.related(), "related"),
+      summary: /* @__PURE__ */ __name((page) => page.summary(), "summary"),
+      tables: /* @__PURE__ */ __name((page) => page.tables(), "tables")
+    };
+  }
+  get _defaultRunOptions() {
+    const unwrapTags = /* @__PURE__ */ new Set([
+      "a",
+      "small",
+      "sup"
+    ]);
+    const unwrapTagsSelector = Array.from(unwrapTags.values()).join(",");
+    const ignoredTags = /* @__PURE__ */ new Set([
+      "img",
+      "link",
+      "style",
+      "abbr",
+      "cite",
+      "input",
+      "sup",
+      "bdi",
+      "q",
+      "figure",
+      "audio",
+      "track",
+      "figcaption"
+    ]);
+    const ignoredClasses = /* @__PURE__ */ new Set([
+      "toc",
+      "reflist",
+      "mw-references-wrap",
+      "box-More_footnotes_needed",
+      "navbox",
+      "navbox-styles",
+      "mw-editsection",
+      "sistersitebox",
+      "navbox-inner",
+      "refbegin",
+      "notpageimage",
+      "mw-file-element",
+      "box-Unreferenced_section",
+      "navigation-not-searchable",
+      "noexcerpt",
+      "infobox-image",
+      "mw-tmh-player",
+      "printfooter",
+      "ambox",
+      "infobox"
+    ]);
+    const ignoredRoles = /* @__PURE__ */ new Set([
+      "navigation"
+    ]);
+    const ignoredSelector = [
+      ...ignoredTags.values(),
+      Array.from(ignoredClasses.values()).map((cls) => `.${cls}`),
+      Array.from(ignoredRoles.values()).map((role) => `[role="${role}"]`),
+      "table>caption"
+    ].join(",");
+    return {
+      extraction: {
+        fields: {
+          markdown: {
+            transform: /* @__PURE__ */ __name((output) => output.replace(/([^\\])(\$)/g, `$1\\$2`), "transform"),
+            filter: /* @__PURE__ */ __name((node) => {
+              const tagName = node.tagName.toLowerCase();
+              if (ignoredTags.has(tagName) || ignoredRoles.has(node.role ?? "")) {
+                return false;
+              }
+              for (const cls of node.className.trim().split(" ").filter(Boolean)) {
+                if (ignoredClasses.has(cls)) {
+                  return false;
+                }
+              }
+              for (const table of Array.from(node.querySelectorAll("table tr>th"))) {
+                let colspan = parseInt(table.getAttribute("colspan") || "1");
+                while (colspan > 1) {
+                  table.insertAdjacentHTML("afterend", `<th>&nbsp;</th>`);
+                  colspan--;
+                }
+                table.removeAttribute("colspan");
+              }
+              const ignoredTargets = Array.from(ignoredSelector ? node.querySelectorAll(ignoredSelector) : []);
+              for (const childNode of ignoredTargets) {
+                childNode.remove();
+              }
+              const unwrapTargets = Array.from(unwrapTagsSelector ? node.querySelectorAll(unwrapTagsSelector) : []).reverse();
+              if (unwrapTags.has(tagName)) {
+                unwrapTargets.push(node);
+              }
+              for (const childNode of unwrapTargets) {
+                childNode.outerHTML = childNode.innerHTML;
+              }
+              return true;
+            }, "filter")
+          }
+        }
+      },
+      filters: {
+        minPageNameSimilarity: 0.5,
+        excludeOthersOnExactMatch: true
+      },
+      search: {
+        limit: 3,
+        suggestion: true
+      },
+      output: {
+        maxSerializedLength: 25e3,
+        maxDescriptionLength: 250
+      }
+    };
+  }
+  _createRunOptions(overrides) {
+    const baseKeys = [
+      "filters",
+      "search",
+      "extraction",
+      "retryOptions",
+      "output"
+    ];
+    return R__namespace.pipe({
+      ...this._defaultRunOptions
+    }, R__namespace.mergeDeep(R__namespace.pick(this.options ?? {}, baseKeys)), R__namespace.mergeDeep({
+      ...overrides
+    }));
+  }
+  async _run({ query: input }, _options) {
+    const runOptions = this._createRunOptions(_options);
+    const { results: searchRawResults, suggestion } = await wiki__default.default.default.search(input, {
+      suggestion: Boolean(_options?.search?.suggestion),
+      ...runOptions.search
+    });
+    if (searchRawResults.length === 0 && suggestion && runOptions.search?.suggestion) {
+      return await this._run({
+        query: suggestion
+      }, _options);
+    }
+    const bestCandidates = this.extractBestCandidates(input, searchRawResults, runOptions?.filters ?? {});
+    const results = await Promise.all(bestCandidates.map(async ({ pageId }) => {
+      const page = await wiki__default.default.default.page(pageId, {
+        redirect: true,
+        preload: false,
+        fields: R.keys(runOptions.extraction?.fields ?? {}).filter((key) => key !== "markdown")
+      });
+      return promise_cjs.asyncProperties({
+        title: page.title,
+        description: (() => {
+          const length = runOptions?.output?.maxDescriptionLength ?? 0;
+          return length <= 0 ? Promise.resolve("") : page.content().then((content) => content.substring(0, length));
+        })(),
+        url: page.fullurl,
+        fields: promise_cjs.asyncProperties(R.mapValues(runOptions?.extraction?.fields ?? {}, (value, key) => this._mappers[key](page, runOptions).then((response) => value.transform ? value.transform(response) : response).catch(() => void 0)))
+      });
+    }));
+    return new WikipediaToolOutput(results, runOptions.output?.maxSerializedLength ?? Infinity);
+  }
+  extractBestCandidates(query, candidates, options) {
+    const normalize = /* @__PURE__ */ __name((text) => text.normalize("NFKD").replace(/[^\w| ]/g, "").replace(/\s\s+/g, " ").trim(), "normalize");
+    const bestCandidates = stringComparison__default.default.jaccardIndex.sortMatch(normalize(query), candidates.map((candidate) => normalize(candidate.title))).map((result) => ({
+      pageId: candidates[result.index].pageid,
+      score: result.rating
+    })).filter((result) => result.score >= (options.minPageNameSimilarity ?? 0)).sort((a, b) => b.score - a.score);
+    if (bestCandidates.at(0)?.score === 1 && options.excludeOthersOnExactMatch) {
+      bestCandidates.length = 1;
+    }
+    return bestCandidates;
+  }
+  createSnapshot() {
+    return {
+      ...super.createSnapshot(),
+      config: this.config
+    };
+  }
+  loadSnapshot(snapshot) {
+    super.loadSnapshot(snapshot);
+  }
+}
+_ts_decorate([
+  decoratorCache_cjs.Cache({
+    enumerable: false
+  }),
+  _ts_metadata("design:type", typeof Record === "undefined" ? Object : Record),
+  _ts_metadata("design:paramtypes", [])
+], WikipediaTool.prototype, "_mappers", null);
+_ts_decorate([
+  decoratorCache_cjs.Cache({
+    enumerable: false
+  }),
+  _ts_metadata("design:type", typeof WikipediaToolRunOptions === "undefined" ? Object : WikipediaToolRunOptions),
+  _ts_metadata("design:paramtypes", [])
+], WikipediaTool.prototype, "_defaultRunOptions", null);
+
+exports.WikipediaTool = WikipediaTool;
+exports.WikipediaToolOutput = WikipediaToolOutput;
+//# sourceMappingURL=wikipedia.cjs.map
+//# sourceMappingURL=wikipedia.cjs.map

package/dist/tools/search/wikipedia.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tools/search/wikipedia.ts"],"names":["wiki","default","setLang","WikipediaToolOutput","SearchToolOutput","constructor","results","maxSerializedLength","register","getTextContent","isEmpty","target","length","response","JSON","stringify","Infinity","substring","createSnapshot","loadSnapshot","snapshot","Object","assign","WikipediaTool","Tool","name","description","emitter","inputSchema","z","object","query","string","min","max","config","Emitter","root","child","namespace","creator","_mappers","categories","page","content","html","markdown","runOptions","then","result","url","URL","fullurl","base","protocol","hostname","port","filter","Boolean","join","replace","service","Turndown","use","turndownPlugin","gfm","remove","node","extraction","fields","turndown","images","intro","infobox","links","coordinates","langLinks","references","related","summary","tables","_defaultRunOptions","unwrapTags","Set","unwrapTagsSelector","Array","from","values","ignoredTags","ignoredClasses","ignoredRoles","ignoredSelector","map","cls","role","transform","output","tagName","toLowerCase","has","className","trim","split","table","querySelectorAll","colspan","parseInt","getAttribute","insertAdjacentHTML","removeAttribute","ignoredTargets","childNode","unwrapTargets","reverse","push","outerHTML","innerHTML","filters","minPageNameSimilarity","excludeOthersOnExactMatch","search","limit","suggestion","maxDescriptionLength","_createRunOptions","overrides","baseKeys","R","pipe","mergeDeep","pick","options","_run","input","_options","searchRawResults","bestCandidates","extractBestCandidates","Promise","all","pageId","redirect","preload","keys","key","asyncProperties","title","resolve","mapValues","value","catch","undefined","candidates","normalize","text","stringComparison","jaccardIndex","sortMatch","candidate","index","pageid","score","rating","sort","a","b","at","enumerable"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCAA,qBAAKC,CAAAA,OAAAA,CAAQC,QAAQ,IAAA,CAAA;AAoDd,MAAMC,4BAA4BC,yBAAAA,CAAAA;AAAAA,EAAAA;;;;;AACvCC,EAAAA,WAAAA,CACkBC,SACGC,mBACnB,EAAA;AACA,IAAA,KAAA,CAAMD,OAAAA,CAAAA,EAAAA,IAAAA,CAHUA,OAAAA,GAAAA,OAAAA,EAAAA,KACGC,mBAAAA,GAAAA,mBAAAA;AAGrB;EAEA;AACE,IAAA,IAAA,CAAKC,QAAQ,EAAA;AACf;EAGAC,cAAyB,GAAA;AACvB,IAAI,IAAA,IAAA,CAAKC,SAAW,EAAA;AAClB,MAAO,OAAA,CAAA,kDAAA,CAAA;AACT;AAEA,IAAMC,MAAAA,MAAAA,GAAS,KAAKL,OAAQM,CAAAA,MAAAA,KAAW,IAAI,IAAKN,CAAAA,OAAAA,CAAQ,CAAA,CAAA,GAAK,IAAKA,CAAAA,OAAAA;AAClE,IAAMO,MAAAA,QAAAA,GAAWC,IAAKC,CAAAA,SAAAA,CAAUJ,MAAAA,CAAAA;AAEhC,IAAO,OAAA,IAAA,CAAKJ,sBAAsBS,QAC9BH,GAAAA,QAAAA,CAASI,UAAU,CAAG,EAAA,IAAA,CAAKV,mBAAmB,CAC9CM,GAAAA,QAAAA;AACN;EAEAK,cAAiB,GAAA;AACf,IAAO,OAAA;AACLZ,MAAAA,OAAAA,EAAS,IAAKA,CAAAA,OAAAA;AACdC,MAAAA,mBAAAA,EAAqB,IAAKA,CAAAA;AAC5B,KAAA;AACF;AAEAY,EAAAA,YAAAA,CAAaC,QAAkD,EAAA;AAC7DC,IAAOC,MAAAA,CAAAA,MAAAA,CAAO,MAAMF,QAAAA,CAAAA;AACtB;AACF;;;;;;;AAOO,MAAMG,sBAAsBC,eAAAA,CAAAA;AAAAA,EAAAA;;;;AAKjCC,EAAAA,IAAAA;AACAC,EAAAA,WAAAA;AAGgBC,EAAAA,OAAAA;EAKhBC,WAAc,GAAA;AACZ,IAAA,OAAOC,MAAEC,MAAO,CAAA;AACdC,MAAAA,KAAAA,EAAOF,MACJG,MAAO,CAAA;QAAEN,WAAa,EAAA,CAAA,kDAAA;AAAqD,OAAA,CAC3EO,CAAAA,GAAAA,CAAI,CAAA,CAAA,CACJC,IAAI,GAAA;KACT,CAAA;AACF;EAEA,WAAmCC,CAAAA,MAAAA,GAA+B,EAAI,EAAA;AACpE,IAAA,KAAA,CAAMA,MAAAA,CAAAA,EAAAA,IAD2BA,CAAAA,MAAAA,GAAAA,QAAAA,IAlBnCV,CAAAA,IAAAA,GAAO,WAAA,EAAA,IAAA,CACPC,cACE,+LAAA,EAAA,IAAA,CAEcC,OAA6DS,GAAAA,mBAAAA,CAAQC,KAAKC,KAAM,CAAA;MAC9FC,SAAW,EAAA;AAAC,QAAA,MAAA;AAAQ,QAAA,QAAA;AAAU,QAAA;;MAC9BC,OAAS,EAAA;KACX,CAAA;AAaA;EAEA;AACE,IAAA,IAAA,CAAKhC,QAAQ,EAAA;AACf;AAEA,EAAA,IAGciC,QAGZ,GAAA;AACA,IAAO,OAAA;AACLC,MAAAA,UAAAA,kBAAaC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKD,CAAAA,UAAAA,EAAf,EAAA,YAAA,CAAA;AACZE,MAAAA,OAAAA,kBAAUD,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKC,CAAAA,OAAAA,EAAf,EAAA,SAAA,CAAA;AACTC,MAAAA,IAAAA,kBAAOF,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKE,CAAAA,IAAAA,EAAf,EAAA,MAAA,CAAA;MACNC,QAAU,kBAAA,MAAA,CAAA,OAAOH,MAAMI,UAAAA,KAAAA;AACrB,QAAA,MAAMF,OAAO,MAAMF,IAAAA,CAAKE,MAAOG,CAAAA,IAAAA,CAAK,CAACC,MAAAA,KAAAA;AACnC,UAAA,MAAMC,GAAM,GAAA,IAAIC,GAAIR,CAAAA,IAAAA,CAAKS,OAAO,CAAA;AAChC,UAAA,MAAMC,IAAO,GAAA,CAAA,EAAGH,GAAII,CAAAA,QAAQ,CAAK,EAAA,EAAA;YAACJ,GAAIK,CAAAA,QAAAA;YAAUL,GAAIM,CAAAA;AAAMC,WAAAA,CAAAA,MAAAA,CAAOC,OAAAA,CAAAA,CAASC,IAAK,CAAA,GAAA,CAAA,CAAA,CAAA;AAC/E,UACEV,OAAAA,MAAAA,CAEGW,OAAQ,CAAA,2BAAA,EAA6B,CAAMV,GAAAA,EAAAA,GAAAA,CAAII,QAAQ,CAAK,GAAA,CAAA,CAAA,CAC5DM,OAAQ,CAAA,0BAAA,EAA4B,CAAMV,GAAAA,EAAAA,GAAAA,CAAII,QAAQ,CAAK,GAAA,CAAA,CAAA,CAG3DM,OAAQ,CAAA,yBAAA,EAA2B,CAAMP,GAAAA,EAAAA,IAAAA,CAAS,GAAA,CAAA,CAAA,CAClDO,OAAQ,CAAA,wBAAA,EAA0B,CAAMP,GAAAA,EAAAA,IAAAA,CAAS,GAAA,CAAA,CAAA;SAExD,CAAA;AAEA,QAAMQ,MAAAA,OAAAA,GAAU,IAAIC,yBAAAA,EAAAA;AACpBD,QAAQE,OAAAA,CAAAA,GAAAA,CAAIC,gCAAeC,GAAG,CAAA;AAC9B,QAAA,OAAOJ,OACJK,CAAAA,MAAAA,CAAO,CAACC,IAAAA,KAASpB,WAAWqB,UAAYC,EAAAA,MAAAA,EAAQvB,QAAUW,EAAAA,MAAAA,GAASU,IAAAA,CAAAA,KAAU,KAAA,CAAA,CAC7EG,SAASzB,IAAAA,CAAAA;OApBJ,EAAA,UAAA,CAAA;AAsBV0B,MAAAA,MAAAA,kBAAS5B,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAK4B,CAAAA,MAAAA,EAAf,EAAA,QAAA,CAAA;AACRC,MAAAA,KAAAA,kBAAQ7B,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAK6B,CAAAA,KAAAA,EAAf,EAAA,OAAA,CAAA;AACPC,MAAAA,OAAAA,kBAAU9B,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAK8B,CAAAA,OAAAA,EAAf,EAAA,SAAA,CAAA;AACTC,MAAAA,KAAAA,kBAAQ/B,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAK+B,CAAAA,KAAAA,EAAf,EAAA,OAAA,CAAA;AACPC,MAAAA,WAAAA,kBAAchC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKgC,CAAAA,WAAAA,EAAf,EAAA,aAAA,CAAA;AACbC,MAAAA,SAAAA,kBAAYjC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKiC,CAAAA,SAAAA,EAAf,EAAA,WAAA,CAAA;AACXC,MAAAA,UAAAA,kBAAalC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKkC,CAAAA,UAAAA,EAAf,EAAA,YAAA,CAAA;AACZC,MAAAA,OAAAA,kBAAUnC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKmC,CAAAA,OAAAA,EAAf,EAAA,SAAA,CAAA;AACTC,MAAAA,OAAAA,kBAAUpC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKoC,CAAAA,OAAAA,EAAf,EAAA,SAAA,CAAA;AACTC,MAAAA,MAAAA,kBAASrC,MAAAA,CAAAA,CAAAA,IAAAA,KAASA,IAAKqC,CAAAA,MAAAA,EAAf,EAAA,QAAA;AACV,KAAA;AACF;AAEA,EAAA,IAGcC,kBAA8C,GAAA;AAC1D,IAAMC,MAAAA,UAAAA,uBAAiBC,GAAI,CAAA;AAAC,MAAA,GAAA;AAAK,MAAA,OAAA;AAAS,MAAA;AAAM,KAAA,CAAA;AAChD,IAAMC,MAAAA,kBAAAA,GAAqBC,MAAMC,IAAKJ,CAAAA,UAAAA,CAAWK,QAAM,CAAA,CAAI5B,KAAK,GAAA,CAAA;AAEhE,IAAM6B,MAAAA,WAAAA,uBAAkBL,GAAI,CAAA;AAC1B,MAAA,KAAA;AACA,MAAA,MAAA;AACA,MAAA,OAAA;AACA,MAAA,MAAA;AACA,MAAA,MAAA;AACA,MAAA,OAAA;AACA,MAAA,KAAA;AACA,MAAA,KAAA;AACA,MAAA,GAAA;AACA,MAAA,QAAA;AACA,MAAA,OAAA;AACA,MAAA,OAAA;AACA,MAAA;AACD,KAAA,CAAA;AACD,IAAMM,MAAAA,cAAAA,uBAAqBN,GAAI,CAAA;AAC7B,MAAA,KAAA;AACA,MAAA,SAAA;AACA,MAAA,oBAAA;AACA,MAAA,2BAAA;AACA,MAAA,QAAA;AACA,MAAA,eAAA;AACA,MAAA,gBAAA;AACA,MAAA,eAAA;AACA,MAAA,cAAA;AACA,MAAA,UAAA;AACA,MAAA,cAAA;AACA,MAAA,iBAAA;AACA,MAAA,0BAAA;AACA,MAAA,2BAAA;AACA,MAAA,WAAA;AACA,MAAA,eAAA;AACA,MAAA,eAAA;AACA,MAAA,aAAA;AACA,MAAA,OAAA;AACA,MAAA;AACD,KAAA,CAAA;AACD,IAAMO,MAAAA,YAAAA,uBAAmBP,GAAI,CAAA;AAAC,MAAA;AAAa,KAAA,CAAA;AAE3C,IAAA,MAAMQ,eAAkB,GAAA;AACnBH,MAAAA,GAAAA,WAAAA,CAAYD,MAAM,EAAA;MACrBF,KAAMC,CAAAA,IAAAA,CAAKG,cAAeF,CAAAA,MAAAA,EAAM,CAAA,CAAIK,IAAI,CAACC,GAAAA,KAAQ,CAAIA,CAAAA,EAAAA,GAAAA,CAAK,CAAA,CAAA;MAC1DR,KAAMC,CAAAA,IAAAA,CAAKI,YAAaH,CAAAA,MAAAA,EAAM,CAAA,CAAIK,IAAI,CAACE,IAAAA,KAAS,CAAUA,OAAAA,EAAAA,IAAAA,CAAQ,EAAA,CAAA,CAAA;AAClE,MAAA;AACAnC,KAAAA,CAAAA,IAAAA,CAAK,GAAA,CAAA;AAEP,IAAO,OAAA;MACLS,UAAY,EAAA;QACVC,MAAQ,EAAA;UACNvB,QAAU,EAAA;AACRiD,YAAAA,SAAAA,0BAAYC,MAAmBA,KAAAA,MAAAA,CAAOpC,OAAQ,CAAA,cAAA,EAAgB,QAAQ,CAA3D,EAAA,WAAA,CAAA;AACXH,YAAAA,MAAAA,0BAASU,IAAAA,KAAAA;AACP,cAAM8B,MAAAA,OAAAA,GAAU9B,IAAK8B,CAAAA,OAAAA,CAAQC,WAAW,EAAA;AACxC,cAAIV,IAAAA,WAAAA,CAAYW,IAAIF,OAAAA,CAAAA,IAAYP,aAAaS,GAAIhC,CAAAA,IAAAA,CAAK2B,IAAQ,IAAA,EAAA,CAAK,EAAA;AACjE,gBAAO,OAAA,KAAA;AACT;AAEA,cAAWD,KAAAA,MAAAA,GAAAA,IAAO1B,IAAKiC,CAAAA,SAAAA,CAAUC,IAAI,EAAA,CAAGC,MAAM,GAAA,CAAA,CAAK7C,MAAOC,CAAAA,OAAAA,CAAU,EAAA;AAClE,gBAAI+B,IAAAA,cAAAA,CAAeU,GAAIN,CAAAA,GAAAA,CAAM,EAAA;AAC3B,kBAAO,OAAA,KAAA;AACT;AACF;AAEA,cAAA,KAAA,MAAWU,SAASlB,KAAMC,CAAAA,IAAAA,CAAKnB,KAAKqC,gBAAiB,CAAA,aAAA,CAAA,CAAiB,EAAA;AACpE,gBAAA,IAAIC,UAAUC,QAASH,CAAAA,KAAAA,CAAMI,YAAa,CAAA,SAAA,KAAc,GAAA,CAAA;AACxD,gBAAA,OAAOF,UAAU,CAAG,EAAA;AAClBF,kBAAMK,KAAAA,CAAAA,kBAAAA,CAAmB,YAAY,CAAiB,eAAA,CAAA,CAAA;AACtDH,kBAAAA,OAAAA,EAAAA;AACF;AACAF,gBAAAA,KAAAA,CAAMM,gBAAgB,SAAA,CAAA;AACxB;AAEA,cAAMC,MAAAA,cAAAA,GAAiBzB,MAAMC,IAC3BK,CAAAA,eAAAA,GAAkBxB,KAAKqC,gBAAiBb,CAAAA,eAAAA,CAAmB,GAAA,EAAE,CAAA;AAE/D,cAAA,KAAA,MAAWoB,aAAaD,cAAgB,EAAA;AACtCC,gBAAAA,SAAAA,CAAU7C,MAAM,EAAA;AAClB;AAEA,cAAM8C,MAAAA,aAAAA,GAAgB3B,KAAMC,CAAAA,IAAAA,CAC1BF,kBAAqBjB,GAAAA,IAAAA,CAAKqC,gBAAiBpB,CAAAA,kBAAAA,CAAsB,GAAA,EAAE,CAAA,CACnE6B,OAAO,EAAA;AACT,cAAI/B,IAAAA,UAAAA,CAAWiB,GAAIF,CAAAA,OAAAA,CAAU,EAAA;AAC3Be,gBAAAA,aAAAA,CAAcE,KAAK/C,IAAAA,CAAAA;AACrB;AACA,cAAA,KAAA,MAAW4C,aAAaC,aAAe,EAAA;AACrCD,gBAAAA,SAAAA,CAAUI,YAAYJ,SAAUK,CAAAA,SAAAA;AAClC;AAEA,cAAO,OAAA,IAAA;aAtCD,EAAA,QAAA;AAwCV;AACF;AACF,OAAA;MACAC,OAAS,EAAA;QACPC,qBAAuB,EAAA,GAAA;QACvBC,yBAA2B,EAAA;AAC7B,OAAA;MACAC,MAAQ,EAAA;QACNC,KAAO,EAAA,CAAA;QACPC,UAAY,EAAA;AACd,OAAA;MACA1B,MAAQ,EAAA;QACNzF,mBAAqB,EAAA,IAAA;QACrBoH,oBAAsB,EAAA;AACxB;AACF,KAAA;AACF;AAEUC,EAAAA,iBAAAA,CAAkBC,SAA8D,EAAA;AACxF,IAAA,MAAMC,QAA6E,GAAA;AACjF,MAAA,SAAA;AACA,MAAA,QAAA;AACA,MAAA,YAAA;AACA,MAAA,cAAA;AACA,MAAA;;AAGF,IAAA,OAAOC,aAAEC,IACP,CAAA;AAAE,MAAA,GAAG,IAAK/C,CAAAA;AAAmB,KAAA,EAC7B8C,YAAEE,CAAAA,SAAAA,CAAUF,YAAEG,CAAAA,IAAAA,CAAK,IAAKC,CAAAA,OAAAA,IAAW,EAAC,EAAGL,QAAAA,CAAAA,CACvCC,EAAAA,YAAAA,CAAEE,SAAU,CAAA;MAAE,GAAGJ;AAAU,KAAA,CAAA,CAAA;AAE/B;AAEA,EAAA,MAAgBO,IACd,CAAA,EAAErG,KAAOsG,EAAAA,KAAAA,IACTC,QAC8B,EAAA;AAC9B,IAAMvF,MAAAA,UAAAA,GAAa,IAAK6E,CAAAA,iBAAAA,CAAkBU,QAAAA,CAAAA;AAE1C,IAAM,MAAA,EAAEhI,SAASiI,gBAAkBb,EAAAA,UAAAA,KAA+B,MAAM1H,qBAAAA,CAAKC,OAAQuH,CAAAA,MAAAA,CACnFa,KACA,EAAA;MACEX,UAAYhE,EAAAA,OAAAA,CAAQ4E,QAAUd,EAAAA,MAAAA,EAAQE,UAAAA,CAAAA;AACtC,MAAA,GAAG3E,UAAWyE,CAAAA;KAChB,CAAA;AAGF,IAAA,IAAIe,iBAAiB3H,MAAW,KAAA,CAAA,IAAK8G,UAAc3E,IAAAA,UAAAA,CAAWyE,QAAQE,UAAY,EAAA;AAChF,MAAO,OAAA,MAAM,KAAKU,IAAK,CAAA;QAAErG,KAAO2F,EAAAA;AAAW,OAAA,EAAGY,QAAAA,CAAAA;AAChD;AAEA,IAAME,MAAAA,cAAAA,GAAiB,KAAKC,qBAC1BJ,CAAAA,KAAAA,EACAE,kBACAxF,UAAYsE,EAAAA,OAAAA,IAAW,EAAC,CAAA;AAG1B,IAAM/G,MAAAA,OAAAA,GAAU,MAAMoI,OAAQC,CAAAA,GAAAA,CAC5BH,eAAe5C,GAAI,CAAA,OAAO,EAAEgD,MAAAA,EAAQ,KAAA;AAElC,MAAA,MAAMjG,IAAO,GAAA,MAAM3C,qBAAKC,CAAAA,OAAAA,CAAQ0C,KAAKiG,MAAQ,EAAA;QAC3CC,QAAU,EAAA,IAAA;QACVC,OAAS,EAAA,KAAA;QACTzE,MAAQ0E,EAAAA,MAAAA,CAAKhG,UAAWqB,CAAAA,UAAAA,EAAYC,MAAU,IAAA,EAAC,CAAA,CAAGZ,MAAO,CAAA,CAACuF,GAAQA,KAAAA,GAAAA,KAAQ,UAAA;OAC5E,CAAA;AAEA,MAAA,OAAOC,2BAAgB,CAAA;AACrBC,QAAAA,KAAAA,EAAOvG,IAAKuG,CAAAA,KAAAA;AACZxH,QAAAA,WAAAA,EAAAA,CAAc,MAAA;AACZ,UAAMd,MAAAA,MAAAA,GAASmC,UAAYiD,EAAAA,MAAAA,EAAQ2B,oBAAwB,IAAA,CAAA;AAC3D,UAAA,OAAO/G,UAAU,CACb8H,GAAAA,OAAAA,CAAQS,OAAQ,CAAA,EAAA,IAChBxG,IAAKC,CAAAA,OAAAA,EAAUI,CAAAA,IAAAA,CAAK,CAACJ,OAAYA,KAAAA,OAAAA,CAAQ3B,SAAU,CAAA,CAAA,EAAGL,MAAAA,CAAAA,CAAAA;SAC5D,GAAA;AACAsC,QAAAA,GAAAA,EAAKP,IAAKS,CAAAA,OAAAA;AACViB,QAAAA,MAAAA,EAAQ4E,2BACNG,CAAAA,WAAAA,CAAUrG,UAAYqB,EAAAA,UAAAA,EAAYC,UAAU,EAAC,EAAG,CAACgF,KAAAA,EAAOL,GACtD,KAAA,IAAA,CAAKvG,QAASuG,CAAAA,GAAAA,EAAKrG,IAAMI,EAAAA,UAAAA,CACtBC,CAAAA,IAAAA,CAAK,CAACnC,QAAAA,KAAcwI,KAAMtD,CAAAA,SAAAA,GAAYsD,MAAMtD,SAAUlF,CAAAA,QAAAA,CAAYA,GAAAA,QAAAA,CAClEyI,CAAAA,KAAAA,CAAM,MAAMC,KAAAA,CAAAA,CAAAA,CAAAA;OAGrB,CAAA;AACF,KAAA,CAAA,CAAA;AAEF,IAAA,OAAO,IAAIpJ,mBAAoBG,CAAAA,OAAAA,EAASyC,UAAWiD,CAAAA,MAAAA,EAAQzF,uBAAuBS,QAAAA,CAAAA;AACpF;EAEUyH,qBACR1G,CAAAA,KAAAA,EACAyH,YACArB,OACA,EAAA;AACA,IAAA,MAAMsB,4BAAaC,MAAAA,CAAAA,CAAAA,IAAAA,KACjBA,IACGD,CAAAA,SAAAA,CAAU,MAAA,CACV7F,CAAAA,OAAAA,CAAQ,UAAY,EAAA,EAAA,EACpBA,OAAQ,CAAA,QAAA,EAAU,GAAA,CAAA,CAClByC,MALa,EAAA,WAAA,CAAA;AAOlB,IAAA,MAAMmC,iBAAiBmB,iCAAiBC,CAAAA,YAAAA,CACrCC,UACCJ,SAAU1H,CAAAA,KAAAA,GACVyH,UAAW5D,CAAAA,GAAAA,CAAI,CAACkE,SAAcL,KAAAA,SAAAA,CAAUK,UAAUZ,KAAK,CAAA,CAAA,CAExDtD,CAAAA,GAAAA,CAAI,CAAC3C,MAAY,MAAA;MAChB2F,MAAQY,EAAAA,UAAAA,CAAWvG,MAAO8G,CAAAA,KAAK,CAAEC,CAAAA,MAAAA;AACjCC,MAAAA,KAAAA,EAAOhH,MAAOiH,CAAAA;AAChB,KAAA,CAAA,EACCzG,MAAO,CAAA,CAACR,MAAWA,KAAAA,MAAAA,CAAOgH,UAAU9B,OAAQb,CAAAA,qBAAAA,IAAyB,CAAA,CAAA,CAAA,CACrE6C,KAAK,CAACC,CAAAA,EAAGC,MAAMA,CAAEJ,CAAAA,KAAAA,GAAQG,EAAEH,KAAK,CAAA;AAEnC,IAAA,IAAIzB,eAAe8B,EAAG,CAAA,CAAA,GAAIL,KAAU,KAAA,CAAA,IAAK9B,QAAQZ,yBAA2B,EAAA;AAC1EiB,MAAAA,cAAAA,CAAe5H,MAAS,GAAA,CAAA;AAC1B;AAEA,IAAO4H,OAAAA,cAAAA;AACT;EAEAtH,cAAiB,GAAA;AACf,IAAO,OAAA;AACL,MAAA,GAAG,MAAMA,cAAAA,EAAAA;AACTiB,MAAAA,MAAAA,EAAQ,IAAKA,CAAAA;AACf,KAAA;AACF;AAEAhB,EAAAA,YAAAA,CAAaC,QAAkD,EAAA;AAC7D,IAAA,KAAA,CAAMD,aAAaC,QAAAA,CAAAA;AACrB;AACF;;;IAjRImJ,UAAY,EAAA;;;;;;;IA8CZA,UAAY,EAAA","file":"wikipedia.cjs","sourcesContent":["/**\n * Copyright 2025 IBM Corp.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { Page, pageFunctions, searchOptions } from \"wikipedia\";\nimport wiki from \"wikipedia\";\nimport { Cache } from \"@/cache/decoratorCache.js\";\nimport * as R from \"remeda\";\nimport { keys, mapValues } from \"remeda\";\nimport { ArrayKeys, Common } from \"@/internals/types.js\";\nimport {\n  SearchToolOptions,\n  SearchToolOutput,\n  SearchToolResult,\n  SearchToolRunOptions,\n} from \"./base.js\";\nimport { asyncProperties } from \"@/internals/helpers/promise.js\";\nimport { z } from \"zod\";\nimport { Tool, ToolEmitter, ToolInput } from \"@/tools/base.js\";\nimport Turndown from \"turndown\";\n// @ts-expect-error missing types\nimport turndownPlugin from \"joplin-turndown-plugin-gfm\";\nimport stringComparison from \"string-comparison\";\nimport { pageResult } from \"wikipedia/dist/resultTypes.js\";\nimport { Emitter } from \"@/emitter/emitter.js\";\n\nwiki.default.setLang(\"en\");\n\nexport interface SearchOptions extends searchOptions {}\n\nexport interface FilterOptions {\n  excludeOthersOnExactMatch?: boolean;\n  minPageNameSimilarity?: number;\n}\n\nexport type PageFunctions = Record<\n  pageFunctions,\n  {\n    transform?: <T>(output: T) => T;\n  }\n> & {\n  markdown: {\n    transform?: (output: string) => string;\n    filter?: (node: HTMLElement) => boolean;\n  };\n};\n\nexport interface ExtractionOptions {\n  fields?: Partial<PageFunctions>;\n}\n\nexport interface OutputOptions {\n  maxSerializedLength?: number;\n  maxDescriptionLength?: number;\n}\n\nexport interface WikipediaToolOptions extends SearchToolOptions {\n  filters?: FilterOptions;\n  search?: SearchOptions;\n  extraction?: ExtractionOptions;\n  output?: OutputOptions;\n}\n\nexport interface WikipediaToolRunOptions extends SearchToolRunOptions {\n  filters?: FilterOptions;\n  search?: SearchOptions;\n  extraction?: ExtractionOptions;\n  output?: OutputOptions;\n}\n\ntype PageWithMarkdown = Page & { markdown: () => Promise<string> };\n\ntype ResultFields = { [K in keyof PageFunctions]: Awaited<ReturnType<PageWithMarkdown[K]>> };\n\nexport interface WikipediaToolResult extends SearchToolResult {\n  fields: Partial<ResultFields>;\n}\n\nexport class WikipediaToolOutput extends SearchToolOutput<WikipediaToolResult> {\n  constructor(\n    public readonly results: WikipediaToolResult[],\n    protected readonly maxSerializedLength: number,\n  ) {\n    super(results);\n  }\n\n  static {\n    this.register();\n  }\n\n  @Cache()\n  getTextContent(): string {\n    if (this.isEmpty()) {\n      return `No results were found. Try to reformat your query.`;\n    }\n\n    const target = this.results.length === 1 ? this.results[0] : this.results;\n    const response = JSON.stringify(target);\n\n    return this.maxSerializedLength < Infinity\n      ? response.substring(0, this.maxSerializedLength)\n      : response;\n  }\n\n  createSnapshot() {\n    return {\n      results: this.results,\n      maxSerializedLength: this.maxSerializedLength,\n    };\n  }\n\n  loadSnapshot(snapshot: ReturnType<typeof this.createSnapshot>) {\n    Object.assign(this, snapshot);\n  }\n}\n\ninterface SearchResponse {\n  results: Pick<pageResult, \"title\" | \"pageid\">[];\n  suggestion: string;\n}\n\nexport class WikipediaTool extends Tool<\n  WikipediaToolOutput,\n  WikipediaToolOptions,\n  WikipediaToolRunOptions\n> {\n  name = \"Wikipedia\";\n  description =\n    \"Search factual and historical information, including biography, history, politics, geography, society, culture, science, technology, people, animal species, mathematics, and other subjects.\";\n\n  public readonly emitter: ToolEmitter<ToolInput<this>, WikipediaToolOutput> = Emitter.root.child({\n    namespace: [\"tool\", \"search\", \"wikipedia\"],\n    creator: this,\n  });\n\n  inputSchema() {\n    return z.object({\n      query: z\n        .string({ description: `Name of the wikipedia page, for example 'New York'` })\n        .min(1)\n        .max(128),\n    });\n  }\n\n  public constructor(public readonly config: WikipediaToolOptions = {}) {\n    super(config);\n  }\n\n  static {\n    this.register();\n  }\n\n  @Cache({\n    enumerable: false,\n  })\n  protected get _mappers(): Record<\n    keyof PageFunctions,\n    (page: Page, runOptions: WikipediaToolRunOptions) => Promise<any>\n  > {\n    return {\n      categories: (page) => page.categories(),\n      content: (page) => page.content(),\n      html: (page) => page.html(),\n      markdown: async (page, runOptions) => {\n        const html = await page.html().then((result) => {\n          const url = new URL(page.fullurl);\n          const base = `${url.protocol}//${[url.hostname, url.port].filter(Boolean).join(\":\")}`;\n          return (\n            result\n              // Missing a protocol\n              .replace(/(<img .*src=)\"(\\/\\/.*)\"/gm, `$1\"${url.protocol}$2\"`)\n              .replace(/(<a .*href=)\"(\\/\\/.*)\"/gm, `$1\"${url.protocol}$2\"`)\n\n              // Missing a hostname\n              .replace(/(<img .*src=)\"(\\/.*)\"/gm, `$1\"${base}$2\"`)\n              .replace(/(<a .*href=)\"(\\/.*)\"/gm, `$1\"${base}$2\"`)\n          );\n        });\n\n        const service = new Turndown();\n        service.use(turndownPlugin.gfm);\n        return service\n          .remove((node) => runOptions.extraction?.fields?.markdown?.filter?.(node) === false)\n          .turndown(html);\n      },\n      images: (page) => page.images(),\n      intro: (page) => page.intro(),\n      infobox: (page) => page.infobox(),\n      links: (page) => page.links(),\n      coordinates: (page) => page.coordinates(),\n      langLinks: (page) => page.langLinks(),\n      references: (page) => page.references(),\n      related: (page) => page.related(),\n      summary: (page) => page.summary(),\n      tables: (page) => page.tables(),\n    };\n  }\n\n  @Cache({\n    enumerable: false,\n  })\n  protected get _defaultRunOptions(): WikipediaToolRunOptions {\n    const unwrapTags = new Set([\"a\", \"small\", \"sup\"]);\n    const unwrapTagsSelector = Array.from(unwrapTags.values()).join(\",\");\n\n    const ignoredTags = new Set([\n      \"img\",\n      \"link\",\n      \"style\",\n      \"abbr\",\n      \"cite\",\n      \"input\",\n      \"sup\",\n      \"bdi\",\n      \"q\",\n      \"figure\",\n      \"audio\",\n      \"track\",\n      \"figcaption\",\n    ]);\n    const ignoredClasses = new Set([\n      \"toc\",\n      \"reflist\",\n      \"mw-references-wrap\",\n      \"box-More_footnotes_needed\",\n      \"navbox\",\n      \"navbox-styles\",\n      \"mw-editsection\",\n      \"sistersitebox\",\n      \"navbox-inner\",\n      \"refbegin\",\n      \"notpageimage\",\n      \"mw-file-element\",\n      \"box-Unreferenced_section\",\n      \"navigation-not-searchable\",\n      \"noexcerpt\",\n      \"infobox-image\",\n      \"mw-tmh-player\",\n      \"printfooter\",\n      \"ambox\",\n      \"infobox\",\n    ]);\n    const ignoredRoles = new Set([\"navigation\"]);\n\n    const ignoredSelector = [\n      ...ignoredTags.values(),\n      Array.from(ignoredClasses.values()).map((cls) => `.${cls}`),\n      Array.from(ignoredRoles.values()).map((role) => `[role=\"${role}\"]`),\n      \"table>caption\",\n    ].join(\",\");\n\n    return {\n      extraction: {\n        fields: {\n          markdown: {\n            transform: (output: string) => output.replace(/([^\\\\])(\\$)/g, `$1\\\\$2`),\n            filter: (node) => {\n              const tagName = node.tagName.toLowerCase();\n              if (ignoredTags.has(tagName) || ignoredRoles.has(node.role ?? \"\")) {\n                return false;\n              }\n\n              for (const cls of node.className.trim().split(\" \").filter(Boolean)) {\n                if (ignoredClasses.has(cls)) {\n                  return false;\n                }\n              }\n\n              for (const table of Array.from(node.querySelectorAll(\"table tr>th\"))) {\n                let colspan = parseInt(table.getAttribute(\"colspan\") || \"1\");\n                while (colspan > 1) {\n                  table.insertAdjacentHTML(\"afterend\", `<th>&nbsp;</th>`);\n                  colspan--;\n                }\n                table.removeAttribute(\"colspan\");\n              }\n\n              const ignoredTargets = Array.from(\n                ignoredSelector ? node.querySelectorAll(ignoredSelector) : [],\n              );\n              for (const childNode of ignoredTargets) {\n                childNode.remove();\n              }\n\n              const unwrapTargets = Array.from(\n                unwrapTagsSelector ? node.querySelectorAll(unwrapTagsSelector) : [],\n              ).reverse();\n              if (unwrapTags.has(tagName)) {\n                unwrapTargets.push(node);\n              }\n              for (const childNode of unwrapTargets) {\n                childNode.outerHTML = childNode.innerHTML;\n              }\n\n              return true;\n            },\n          },\n        },\n      },\n      filters: {\n        minPageNameSimilarity: 0.5,\n        excludeOthersOnExactMatch: true,\n      },\n      search: {\n        limit: 3,\n        suggestion: true,\n      },\n      output: {\n        maxSerializedLength: 25_000,\n        maxDescriptionLength: 250,\n      },\n    };\n  }\n\n  protected _createRunOptions(overrides?: WikipediaToolRunOptions): WikipediaToolRunOptions {\n    const baseKeys: ArrayKeys<Common<WikipediaToolRunOptions, WikipediaToolOptions>> = [\n      \"filters\",\n      \"search\",\n      \"extraction\",\n      \"retryOptions\",\n      \"output\",\n    ];\n\n    return R.pipe(\n      { ...this._defaultRunOptions },\n      R.mergeDeep(R.pick(this.options ?? {}, baseKeys)),\n      R.mergeDeep({ ...overrides }),\n    );\n  }\n\n  protected async _run(\n    { query: input }: ToolInput<WikipediaTool>,\n    _options?: WikipediaToolRunOptions,\n  ): Promise<WikipediaToolOutput> {\n    const runOptions = this._createRunOptions(_options);\n\n    const { results: searchRawResults, suggestion }: SearchResponse = await wiki.default.search(\n      input,\n      {\n        suggestion: Boolean(_options?.search?.suggestion),\n        ...runOptions.search,\n      },\n    );\n\n    if (searchRawResults.length === 0 && suggestion && runOptions.search?.suggestion) {\n      return await this._run({ query: suggestion }, _options);\n    }\n\n    const bestCandidates = this.extractBestCandidates(\n      input,\n      searchRawResults,\n      runOptions?.filters ?? {},\n    );\n\n    const results = await Promise.all(\n      bestCandidates.map(async ({ pageId }) => {\n        // @ts-expect-error wrong library's typing, passing a string would lead to a classic text search instead of a concrete page retrieval\n        const page = await wiki.default.page(pageId, {\n          redirect: true,\n          preload: false,\n          fields: keys(runOptions.extraction?.fields ?? {}).filter((key) => key !== \"markdown\"),\n        });\n\n        return asyncProperties({\n          title: page.title,\n          description: ((): Promise<string> => {\n            const length = runOptions?.output?.maxDescriptionLength ?? 0;\n            return length <= 0\n              ? Promise.resolve(\"\")\n              : page.content().then((content) => content.substring(0, length));\n          })(),\n          url: page.fullurl,\n          fields: asyncProperties(\n            mapValues(runOptions?.extraction?.fields ?? {}, (value, key) =>\n              this._mappers[key](page, runOptions)\n                .then((response) => (value.transform ? value.transform(response) : response))\n                .catch(() => undefined),\n            ),\n          ),\n        });\n      }),\n    );\n    return new WikipediaToolOutput(results, runOptions.output?.maxSerializedLength ?? Infinity);\n  }\n\n  protected extractBestCandidates(\n    query: string,\n    candidates: SearchResponse[\"results\"],\n    options: FilterOptions,\n  ) {\n    const normalize = (text: string) =>\n      text\n        .normalize(\"NFKD\")\n        .replace(/[^\\w| ]/g, \"\") // remove diacritics and special characters (except whitespace)\n        .replace(/\\s\\s+/g, \" \") // collapse multiple whitespaces into one\n        .trim();\n\n    const bestCandidates = stringComparison.jaccardIndex\n      .sortMatch(\n        normalize(query),\n        candidates.map((candidate) => normalize(candidate.title)),\n      )\n      .map((result) => ({\n        pageId: candidates[result.index].pageid,\n        score: result.rating,\n      }))\n      .filter((result) => result.score >= (options.minPageNameSimilarity ?? 0))\n      .sort((a, b) => b.score - a.score);\n\n    if (bestCandidates.at(0)?.score === 1 && options.excludeOthersOnExactMatch) {\n      bestCandidates.length = 1;\n    }\n\n    return bestCandidates;\n  }\n\n  createSnapshot() {\n    return {\n      ...super.createSnapshot(),\n      config: this.config,\n    };\n  }\n\n  loadSnapshot(snapshot: ReturnType<typeof this.createSnapshot>) {\n    super.loadSnapshot(snapshot);\n  }\n}\n"]}

package/dist/tools/search/wikipedia.d.cts

@@ -0,0 +1,108 @@
+import { BaseCache } from '../../cache/base.cjs';
+import * as promise_based_task from 'promise-based-task';
+import { searchOptions, pageFunctions, Page } from 'wikipedia';
+import { SearchToolOptions, SearchToolRunOptions, SearchToolResult, SearchToolOutput } from './base.cjs';
+import { z } from 'zod';
+import { Tool, ToolEmitter, ToolInput } from '../base.cjs';
+import { pageResult } from 'wikipedia/dist/resultTypes.js';
+import { E as Emitter } from '../../emitter-BWtGHYn0.cjs';
+import '../../internals/serializable.cjs';
+import '../../internals/types.cjs';
+import '../../internals/helpers/guards.cjs';
+import 'ajv';
+import '../../context.cjs';
+import '../../internals/helpers/promise.cjs';
+import '../../errors.cjs';
+import '../../internals/helpers/schema.cjs';
+import 'zod-to-json-schema';
+
+interface SearchOptions extends searchOptions {
+}
+interface FilterOptions {
+    excludeOthersOnExactMatch?: boolean;
+    minPageNameSimilarity?: number;
+}
+type PageFunctions = Record<pageFunctions, {
+    transform?: <T>(output: T) => T;
+}> & {
+    markdown: {
+        transform?: (output: string) => string;
+        filter?: (node: HTMLElement) => boolean;
+    };
+};
+interface ExtractionOptions {
+    fields?: Partial<PageFunctions>;
+}
+interface OutputOptions {
+    maxSerializedLength?: number;
+    maxDescriptionLength?: number;
+}
+interface WikipediaToolOptions extends SearchToolOptions {
+    filters?: FilterOptions;
+    search?: SearchOptions;
+    extraction?: ExtractionOptions;
+    output?: OutputOptions;
+}
+interface WikipediaToolRunOptions extends SearchToolRunOptions {
+    filters?: FilterOptions;
+    search?: SearchOptions;
+    extraction?: ExtractionOptions;
+    output?: OutputOptions;
+}
+type PageWithMarkdown = Page & {
+    markdown: () => Promise<string>;
+};
+type ResultFields = {
+    [K in keyof PageFunctions]: Awaited<ReturnType<PageWithMarkdown[K]>>;
+};
+interface WikipediaToolResult extends SearchToolResult {
+    fields: Partial<ResultFields>;
+}
+declare class WikipediaToolOutput extends SearchToolOutput<WikipediaToolResult> {
+    readonly results: WikipediaToolResult[];
+    protected readonly maxSerializedLength: number;
+    constructor(results: WikipediaToolResult[], maxSerializedLength: number);
+    getTextContent(): string;
+    createSnapshot(): {
+        results: WikipediaToolResult[];
+        maxSerializedLength: number;
+    };
+    loadSnapshot(snapshot: ReturnType<typeof this.createSnapshot>): void;
+}
+interface SearchResponse {
+    results: Pick<pageResult, "title" | "pageid">[];
+    suggestion: string;
+}
+declare class WikipediaTool extends Tool<WikipediaToolOutput, WikipediaToolOptions, WikipediaToolRunOptions> {
+    readonly config: WikipediaToolOptions;
+    name: string;
+    description: string;
+    readonly emitter: ToolEmitter<ToolInput<this>, WikipediaToolOutput>;
+    inputSchema(): z.ZodObject<{
+        query: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        query: string;
+    }, {
+        query: string;
+    }>;
+    constructor(config?: WikipediaToolOptions);
+    protected get _mappers(): Record<keyof PageFunctions, (page: Page, runOptions: WikipediaToolRunOptions) => Promise<any>>;
+    protected get _defaultRunOptions(): WikipediaToolRunOptions;
+    protected _createRunOptions(overrides?: WikipediaToolRunOptions): WikipediaToolRunOptions;
+    protected _run({ query: input }: ToolInput<WikipediaTool>, _options?: WikipediaToolRunOptions): Promise<WikipediaToolOutput>;
+    protected extractBestCandidates(query: string, candidates: SearchResponse["results"], options: FilterOptions): {
+        pageId: number;
+        score: number;
+    }[];
+    createSnapshot(): {
+        config: WikipediaToolOptions;
+        name: string;
+        description: string;
+        options: WikipediaToolOptions;
+        cache: BaseCache<promise_based_task.Task<WikipediaToolOutput, any>>;
+        emitter: Emitter<any>;
+    };
+    loadSnapshot(snapshot: ReturnType<typeof this.createSnapshot>): void;
+}
+
+export { type ExtractionOptions, type FilterOptions, type OutputOptions, type PageFunctions, type SearchOptions, WikipediaTool, type WikipediaToolOptions, WikipediaToolOutput, type WikipediaToolResult, type WikipediaToolRunOptions };

package/dist/tools/search/wikipedia.d.ts

@@ -0,0 +1,108 @@
+import { BaseCache } from '../../cache/base.js';
+import * as promise_based_task from 'promise-based-task';
+import { searchOptions, pageFunctions, Page } from 'wikipedia';
+import { SearchToolOptions, SearchToolRunOptions, SearchToolResult, SearchToolOutput } from './base.js';
+import { z } from 'zod';
+import { Tool, ToolEmitter, ToolInput } from '../base.js';
+import { pageResult } from 'wikipedia/dist/resultTypes.js';
+import { E as Emitter } from '../../emitter-l0W9gC1A.js';
+import '../../internals/serializable.js';
+import '../../internals/types.js';
+import '../../internals/helpers/guards.js';
+import 'ajv';
+import '../../context.js';
+import '../../internals/helpers/promise.js';
+import '../../errors.js';
+import '../../internals/helpers/schema.js';
+import 'zod-to-json-schema';
+
+interface SearchOptions extends searchOptions {
+}
+interface FilterOptions {
+    excludeOthersOnExactMatch?: boolean;
+    minPageNameSimilarity?: number;
+}
+type PageFunctions = Record<pageFunctions, {
+    transform?: <T>(output: T) => T;
+}> & {
+    markdown: {
+        transform?: (output: string) => string;
+        filter?: (node: HTMLElement) => boolean;
+    };
+};
+interface ExtractionOptions {
+    fields?: Partial<PageFunctions>;
+}
+interface OutputOptions {
+    maxSerializedLength?: number;
+    maxDescriptionLength?: number;
+}
+interface WikipediaToolOptions extends SearchToolOptions {
+    filters?: FilterOptions;
+    search?: SearchOptions;
+    extraction?: ExtractionOptions;
+    output?: OutputOptions;
+}
+interface WikipediaToolRunOptions extends SearchToolRunOptions {
+    filters?: FilterOptions;
+    search?: SearchOptions;
+    extraction?: ExtractionOptions;
+    output?: OutputOptions;
+}
+type PageWithMarkdown = Page & {
+    markdown: () => Promise<string>;
+};
+type ResultFields = {
+    [K in keyof PageFunctions]: Awaited<ReturnType<PageWithMarkdown[K]>>;
+};
+interface WikipediaToolResult extends SearchToolResult {
+    fields: Partial<ResultFields>;
+}
+declare class WikipediaToolOutput extends SearchToolOutput<WikipediaToolResult> {
+    readonly results: WikipediaToolResult[];
+    protected readonly maxSerializedLength: number;
+    constructor(results: WikipediaToolResult[], maxSerializedLength: number);
+    getTextContent(): string;
+    createSnapshot(): {
+        results: WikipediaToolResult[];
+        maxSerializedLength: number;
+    };
+    loadSnapshot(snapshot: ReturnType<typeof this.createSnapshot>): void;
+}
+interface SearchResponse {
+    results: Pick<pageResult, "title" | "pageid">[];
+    suggestion: string;
+}
+declare class WikipediaTool extends Tool<WikipediaToolOutput, WikipediaToolOptions, WikipediaToolRunOptions> {
+    readonly config: WikipediaToolOptions;
+    name: string;
+    description: string;
+    readonly emitter: ToolEmitter<ToolInput<this>, WikipediaToolOutput>;
+    inputSchema(): z.ZodObject<{
+        query: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        query: string;
+    }, {
+        query: string;
+    }>;
+    constructor(config?: WikipediaToolOptions);
+    protected get _mappers(): Record<keyof PageFunctions, (page: Page, runOptions: WikipediaToolRunOptions) => Promise<any>>;
+    protected get _defaultRunOptions(): WikipediaToolRunOptions;
+    protected _createRunOptions(overrides?: WikipediaToolRunOptions): WikipediaToolRunOptions;
+    protected _run({ query: input }: ToolInput<WikipediaTool>, _options?: WikipediaToolRunOptions): Promise<WikipediaToolOutput>;
+    protected extractBestCandidates(query: string, candidates: SearchResponse["results"], options: FilterOptions): {
+        pageId: number;
+        score: number;
+    }[];
+    createSnapshot(): {
+        config: WikipediaToolOptions;
+        name: string;
+        description: string;
+        options: WikipediaToolOptions;
+        cache: BaseCache<promise_based_task.Task<WikipediaToolOutput, any>>;
+        emitter: Emitter<any>;
+    };
+    loadSnapshot(snapshot: ReturnType<typeof this.createSnapshot>): void;
+}
+
+export { type ExtractionOptions, type FilterOptions, type OutputOptions, type PageFunctions, type SearchOptions, WikipediaTool, type WikipediaToolOptions, WikipediaToolOutput, type WikipediaToolResult, type WikipediaToolRunOptions };