@humain/terminal 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/dist/bundle/chunks/{chunk-U75GS4NL.js → chunk-CBTAWCW5.js} +1 -1
- package/dist/bundle/chunks/{chunk-D6HHRSB5.js → chunk-S43N4JYF.js} +14 -29
- package/dist/bundle/chunks/{chunk-LYOOYM65.js → chunk-WKLAQTQ4.js} +100 -124
- package/dist/bundle/chunks/{compat-22OQ5DBK.js → compat-L5WILLPV.js} +1 -1
- package/dist/bundle/chunks/{extract-CTIFQZDJ.js → extract-2YP6VMY4.js} +3 -3
- package/dist/bundle/cli.js +1 -1
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/humain/web-access/vendor/anysearch.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/anysearch.js +4 -14
- package/dist/humain/web-access/vendor/anysearch.js.map +1 -1
- package/dist/humain/web-access/vendor/bocha.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/bocha.js +3 -24
- package/dist/humain/web-access/vendor/bocha.js.map +1 -1
- package/dist/humain/web-access/vendor/brave.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/brave.js +3 -24
- package/dist/humain/web-access/vendor/brave.js.map +1 -1
- package/dist/humain/web-access/vendor/brightdata.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/brightdata.js +5 -34
- package/dist/humain/web-access/vendor/brightdata.js.map +1 -1
- package/dist/humain/web-access/vendor/content-find.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/content-find.js +186 -51
- package/dist/humain/web-access/vendor/content-find.js.map +1 -1
- package/dist/humain/web-access/vendor/curator-page.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/curator-page.js +4 -1
- package/dist/humain/web-access/vendor/curator-page.js.map +1 -1
- package/dist/humain/web-access/vendor/curator-run.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/curator-run.js +3 -1
- package/dist/humain/web-access/vendor/curator-run.js.map +1 -1
- package/dist/humain/web-access/vendor/curator-server.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/curator-server.js +2 -0
- package/dist/humain/web-access/vendor/curator-server.js.map +1 -1
- package/dist/humain/web-access/vendor/domain-filter-normalization.d.ts +2 -0
- package/dist/humain/web-access/vendor/domain-filter-normalization.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/domain-filter-normalization.js +19 -0
- package/dist/humain/web-access/vendor/domain-filter-normalization.js.map +1 -0
- package/dist/humain/web-access/vendor/duckduckgo.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/duckduckgo.js +3 -24
- package/dist/humain/web-access/vendor/duckduckgo.js.map +1 -1
- package/dist/humain/web-access/vendor/extract.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/extract.js +3 -1
- package/dist/humain/web-access/vendor/extract.js.map +1 -1
- package/dist/humain/web-access/vendor/firecrawl.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/firecrawl.js +5 -32
- package/dist/humain/web-access/vendor/firecrawl.js.map +1 -1
- package/dist/humain/web-access/vendor/gemini-search.d.ts +5 -2
- package/dist/humain/web-access/vendor/gemini-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/gemini-search.js +85 -37
- package/dist/humain/web-access/vendor/gemini-search.js.map +1 -1
- package/dist/humain/web-access/vendor/index.d.ts +3 -3
- package/dist/humain/web-access/vendor/index.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/index.js +135 -157
- package/dist/humain/web-access/vendor/index.js.map +1 -1
- package/dist/humain/web-access/vendor/jina-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/jina-search.js +4 -14
- package/dist/humain/web-access/vendor/jina-search.js.map +1 -1
- package/dist/humain/web-access/vendor/kagi.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/kagi.js +4 -12
- package/dist/humain/web-access/vendor/kagi.js.map +1 -1
- package/dist/humain/web-access/vendor/kimi-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/kimi-search.js +5 -32
- package/dist/humain/web-access/vendor/kimi-search.js.map +1 -1
- package/dist/humain/web-access/vendor/mistral-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/mistral-search.js +1 -18
- package/dist/humain/web-access/vendor/mistral-search.js.map +1 -1
- package/dist/humain/web-access/vendor/ollama.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/ollama.js +2 -6
- package/dist/humain/web-access/vendor/ollama.js.map +1 -1
- package/dist/humain/web-access/vendor/openai-search.d.ts +3 -0
- package/dist/humain/web-access/vendor/openai-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/openai-search.js +181 -38
- package/dist/humain/web-access/vendor/openai-search.js.map +1 -1
- package/dist/humain/web-access/vendor/opencode-session-headers.d.ts +10 -0
- package/dist/humain/web-access/vendor/opencode-session-headers.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/opencode-session-headers.js +23 -0
- package/dist/humain/web-access/vendor/opencode-session-headers.js.map +1 -0
- package/dist/humain/web-access/vendor/page-query.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/page-query.js +10 -2
- package/dist/humain/web-access/vendor/page-query.js.map +1 -1
- package/dist/humain/web-access/vendor/parallel.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/parallel.js +1 -18
- package/dist/humain/web-access/vendor/parallel.js.map +1 -1
- package/dist/humain/web-access/vendor/pdf-extract.d.ts +2 -0
- package/dist/humain/web-access/vendor/pdf-extract.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/pdf-extract.js +1 -0
- package/dist/humain/web-access/vendor/pdf-extract.js.map +1 -1
- package/dist/humain/web-access/vendor/querit.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/querit.js +5 -32
- package/dist/humain/web-access/vendor/querit.js.map +1 -1
- package/dist/humain/web-access/vendor/search-answer-formatting.d.ts +8 -0
- package/dist/humain/web-access/vendor/search-answer-formatting.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/search-answer-formatting.js +6 -0
- package/dist/humain/web-access/vendor/search-answer-formatting.js.map +1 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.d.ts +2 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.js +6 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.js.map +1 -0
- package/dist/humain/web-access/vendor/search1api.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/search1api.js +5 -32
- package/dist/humain/web-access/vendor/search1api.js.map +1 -1
- package/dist/humain/web-access/vendor/searchinfinity.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/searchinfinity.js +5 -32
- package/dist/humain/web-access/vendor/searchinfinity.js.map +1 -1
- package/dist/humain/web-access/vendor/searxng.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/searxng.js +3 -24
- package/dist/humain/web-access/vendor/searxng.js.map +1 -1
- package/dist/humain/web-access/vendor/serpapi.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serpapi.js +5 -30
- package/dist/humain/web-access/vendor/serpapi.js.map +1 -1
- package/dist/humain/web-access/vendor/serpbase.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serpbase.js +3 -24
- package/dist/humain/web-access/vendor/serpbase.js.map +1 -1
- package/dist/humain/web-access/vendor/serpdive.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serpdive.js +3 -24
- package/dist/humain/web-access/vendor/serpdive.js.map +1 -1
- package/dist/humain/web-access/vendor/serper.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serper.js +5 -30
- package/dist/humain/web-access/vendor/serper.js.map +1 -1
- package/dist/humain/web-access/vendor/serply.d.ts +4 -0
- package/dist/humain/web-access/vendor/serply.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/serply.js +191 -0
- package/dist/humain/web-access/vendor/serply.js.map +1 -0
- package/dist/humain/web-access/vendor/source-check.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/source-check.js +11 -47
- package/dist/humain/web-access/vendor/source-check.js.map +1 -1
- package/dist/humain/web-access/vendor/summary-review.d.ts +1 -1
- package/dist/humain/web-access/vendor/summary-review.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/summary-review.js +6 -2
- package/dist/humain/web-access/vendor/summary-review.js.map +1 -1
- package/dist/humain/web-access/vendor/tavily.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/tavily.js +3 -24
- package/dist/humain/web-access/vendor/tavily.js.map +1 -1
- package/dist/humain/web-access/vendor/tinyfish.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/tinyfish.js +5 -32
- package/dist/humain/web-access/vendor/tinyfish.js.map +1 -1
- package/dist/humain/web-access/vendor/utils.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/utils.js +2 -0
- package/dist/humain/web-access/vendor/utils.js.map +1 -1
- package/dist/humain/web-access/vendor/valyu.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/valyu.js +5 -30
- package/dist/humain/web-access/vendor/valyu.js.map +1 -1
- package/dist/humain/web-access/vendor/xai-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/xai-search.js +1 -18
- package/dist/humain/web-access/vendor/xai-search.js.map +1 -1
- package/dist/humain/web-access/vendor/xcrawl.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/xcrawl.js +5 -34
- package/dist/humain/web-access/vendor/xcrawl.js.map +1 -1
- package/npm-shrinkwrap.json +6 -6
- package/package.json +2 -2
- package/src/humain/web-access/UPSTREAM.md +33 -1
- package/src/humain/web-access/upstream.json +6 -5
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import net from "node:net";
|
|
3
3
|
import { activityMonitor } from "./activity.js";
|
|
4
|
+
import { normalizeDomain } from "./domain-filter-normalization.js";
|
|
4
5
|
import { redactCredential, resolveCredential } from "./credential-source.js";
|
|
6
|
+
import { formatSearchResultsAsAnswer } from "./search-answer-formatting.js";
|
|
7
|
+
import { normalizeSearchResultCount } from "./search-result-count-normalization.js";
|
|
5
8
|
import { loadSsrfConfig, validateRemoteUrl } from "./ssrf-protection.js";
|
|
6
9
|
import { getWebSearchConfigPath } from "./utils.js";
|
|
7
10
|
const CONFIG_PATH = getWebSearchConfigPath();
|
|
@@ -178,29 +181,6 @@ function scrapeBody(url) {
|
|
|
178
181
|
...(allowFreshScrape() ? {} : { lockdown: true }),
|
|
179
182
|
};
|
|
180
183
|
}
|
|
181
|
-
function normalizeCount(value) {
|
|
182
|
-
if (typeof value !== "number" || !Number.isFinite(value))
|
|
183
|
-
return 5;
|
|
184
|
-
return Math.max(1, Math.min(Math.floor(value), 20));
|
|
185
|
-
}
|
|
186
|
-
function normalizeDomain(value) {
|
|
187
|
-
let input = value.trim().toLowerCase();
|
|
188
|
-
if (!input)
|
|
189
|
-
return null;
|
|
190
|
-
if (input.startsWith("-"))
|
|
191
|
-
input = input.slice(1).trim();
|
|
192
|
-
if (!input)
|
|
193
|
-
return null;
|
|
194
|
-
try {
|
|
195
|
-
const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
|
|
196
|
-
input = parsed.hostname;
|
|
197
|
-
}
|
|
198
|
-
catch {
|
|
199
|
-
input = input.split("/")[0]?.split(":")[0] ?? "";
|
|
200
|
-
}
|
|
201
|
-
input = input.replace(/^\.+|\.+$/g, "");
|
|
202
|
-
return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
|
|
203
|
-
}
|
|
204
184
|
function parseDomainFilter(domainFilter) {
|
|
205
185
|
const filters = { include: [], exclude: [] };
|
|
206
186
|
for (const raw of domainFilter ?? []) {
|
|
@@ -261,13 +241,6 @@ function firstString(...values) {
|
|
|
261
241
|
}
|
|
262
242
|
return null;
|
|
263
243
|
}
|
|
264
|
-
function buildAnswer(results) {
|
|
265
|
-
return results.map((result) => {
|
|
266
|
-
if (result.snippet)
|
|
267
|
-
return `${result.snippet}\nSource: ${result.title} (${result.url})`;
|
|
268
|
-
return `Source: ${result.title} (${result.url})`;
|
|
269
|
-
}).join("\n\n");
|
|
270
|
-
}
|
|
271
244
|
function mapSearchResults(data, numResults, filters) {
|
|
272
245
|
const web = Array.isArray(data)
|
|
273
246
|
? data
|
|
@@ -350,11 +323,11 @@ export function isFirecrawlAvailable() {
|
|
|
350
323
|
}
|
|
351
324
|
export async function searchWithFirecrawl(query, options = {}) {
|
|
352
325
|
requireBaseUrl();
|
|
353
|
-
const numResults =
|
|
326
|
+
const numResults = normalizeSearchResultCount(options.numResults);
|
|
354
327
|
const filters = parseDomainFilter(options.domainFilter);
|
|
355
328
|
const envelope = await firecrawlFetch("search", searchBody(query, options, numResults, filters), options.signal, { ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS }, "search", { type: "api", query });
|
|
356
329
|
const mapped = mapSearchResults(envelope.data, numResults, filters);
|
|
357
|
-
const response = { answer:
|
|
330
|
+
const response = { answer: formatSearchResultsAsAnswer(mapped.results), results: mapped.results };
|
|
358
331
|
if (options.includeContent && mapped.inlineContent.length > 0)
|
|
359
332
|
response.inlineContent = mapped.inlineContent;
|
|
360
333
|
return response;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAG7E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAe,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC7D,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,IAAI,CAAU,CAAC;AAiDrD,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU;IAClB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC9C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,qBAAqB,WAAW,0BAA0B,CAAC,CAAC;IAC7E,CAAC;IACD,YAAY,GAAG,MAAyB,CAAC;IACzC,OAAO,YAAY,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,yBAAyB;IACxC,YAAY,GAAG,IAAI,CAAC;AACrB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACvC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,mCAAmC,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;IACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,UAAU;IAClB,OAAO,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,gBAAgB,CAAC,CAAC;AAC5G,CAAC;AAED,SAAS,cAAc;IACtB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACd,8CAA8C;YAC9C,gCAAgC,WAAW,IAAI;YAC/C,kDAAkD,CAClD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACrB,MAAM,gBAAgB,GAAG,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,QAAQ;QAC7E,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE;QAC1C,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,GAAG,GAAG,gBAAgB,IAAI,UAAU,EAAE,CAAC,mBAAmB,CAAC;IACjE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAClE,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,WAAW,kCAAkC,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,UAAU;QAAE,OAAO,mBAAmB,CAAC;IAC5C,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,UAAiC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,sCAAsC,GAAG,0BAA0B,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,UAAiC,CAAC;AAC1C,CAAC;AAED,SAAS,gBAAgB;IACxB,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC5D,IAAI,gBAAgB,KAAK,SAAS;QAAE,OAAO,gBAAgB,KAAK,GAAG,IAAI,gBAAgB,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IACjH,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC,oBAAoB,CAAC;IACrD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,OAAO,UAAU,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,WAAW,oBAAoB,CAAC,CAAC;IACjH,OAAO,UAAU,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAoB;IAC5C,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,eAAe,CAAC;IAC/C,OAAO,iBAAiB,CAAC;QACxB,QAAQ,EAAE,WAAW;QACrB,eAAe,EAAE,SAAS;QAC1B,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAC/C,MAAM;KACN,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,MAAoB;IAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC9D,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IACjC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IACjC,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,WAAW,CAAC,OAA0D;IAC9E,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,OAAO;QACN,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QAC7D,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,IAAI,MAAM,CAAC,aAAa;QACnE,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACvF,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAChE,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3C,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC;AACzC,CAAC;AAED,SAAS,uBAAuB,CAC/B,OAAqE,EACrE,aAAsB;IAEtB,OAAO,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,uBAAuB,CAAC,OAA+B;IAC/D,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC5B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC;AACb,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC/B,GAAW,EACX,IAA4F,EAC5F,OAAqE;IAErE,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,uBAAuB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;IAC5F,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC/B,IAAI,SAAS,KAAK,qBAAqB;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE9G,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,uBAAuB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;QAClH,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAC/E,OAAO,GAAG,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC9B,OAAO;QACN,GAAG;QACH,OAAO,EAAE,CAAC,UAAU,CAAC;QACrB,eAAe,EAAE,IAAI;QACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACjD,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACrC,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;QACpF,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AACvE,CAAC;AAED,SAAS,iBAAiB,CAAC,YAAkC;IAC5D,MAAM,OAAO,GAAkB,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC5D,KAAK,MAAM,GAAG,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAE,OAAsB;IAC/D,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9E,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACJ,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,QAAQ,KAAK,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAqC;IAC9D,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,KAAK,EAAE,OAAO,OAAO,CAAC;QAC3B,KAAK,MAAM,EAAE,OAAO,OAAO,CAAC;QAC5B,KAAK,OAAO,EAAE,OAAO,OAAO,CAAC;QAC7B,KAAK,MAAM,EAAE,OAAO,OAAO,CAAC;QAC5B,SAAS,OAAO,SAAS,CAAC;IAC3B,CAAC;AACF,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,OAA+B,EAAE,UAAkB,EAAE,OAAsB;IAC7G,OAAO;QACN,KAAK;QACL,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,CAAC,KAAK,CAAC;QAChB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1G,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;YAC5B,aAAa,EAAE;gBACd,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,eAAe,EAAE,IAAI;gBACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aACjD;SACD,CAAC,CAAC,CAAC,EAAE,CAAC;KACP,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,GAAG,MAAiB;IACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;YAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,WAAW,CAAC,OAAkC;IACtD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC7B,IAAI,MAAM,CAAC,OAAO;YAAE,OAAO,GAAG,MAAM,CAAC,OAAO,aAAa,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,GAAG,GAAG,CAAC;QACxF,OAAO,WAAW,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa,EAAE,UAAkB,EAAE,OAAsB;IAIlF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAC9B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAgC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IACrG,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QAChF,MAAM,IAAI,GAAG,OAA8B,CAAC;QAC5C,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAChF,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC;YAAE,SAAS;QAC1E,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC;QACnE,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC;QAC9F,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,QAAQ;YAAE,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;IACzC,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,cAAc,CAC5B,QAAgB,EAChB,IAA6B,EAC7B,MAA+B,EAC/B,OAAqE,EACrE,KAAK,GAAG,QAAQ,EAChB,QAAQ,GAAgF,SAAS;IAEjG,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,OAAO,GAA2B,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;IAC/E,IAAI,MAAM;QAAE,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,OAAO,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5F,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,UAAU,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,EAAE,SAAS,IAAI,kBAAkB,EAAE,MAAM,CAAC;SACvE,EAAE,OAAO,CAAC,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,UAAU,QAAQ,CAAC,MAAM,KAAK,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;QACD,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,2BAA2B,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,QAAQ,GAAG,IAA+B,CAAC;QACjD,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9G,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,kBAAkB,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC,CAAC,CAAC;QAC7E,CAAC;QACD,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,YAAY,CAAC,GAAG,CAAC;YAAE,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;;YAC7D,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACX,CAAC;AACF,CAAC;AAED,MAAM,UAAU,oBAAoB;IACnC,OAAO,UAAU,EAAE,KAAK,IAAI,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAa,EAAE,OAAO,GAA2B,EAAE;IAC5F,cAAc,EAAE,CAAC;IACjB,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,MAAM,cAAc,CACpC,QAAQ,EACR,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,EAC/C,OAAO,CAAC,MAAM,EACd,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,iBAAiB,EAAE,EACjE,QAAQ,EACR,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CACtB,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAmB,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAClG,IAAI,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAC7G,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,MAAoB,EACpB,OAAiC;IAEjC,cAAc,EAAE,CAAC;IACjB,MAAM,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,MAAM,GAAG,IAA2B,CAAC;IAC3C,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC;IAC7C,MAAM,KAAK,GAAG,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,CAAC,IAAI,EAAE;QACtE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE;QACtB,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC7C,CAAC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport net from \"node:net\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { loadSsrfConfig, validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst DEFAULT_API_VERSION = \"v2\";\nconst EXTRACT_TIMEOUT_MS = 60_000;\nconst SEARCH_TIMEOUT_MS = 60_000;\nconst DEFAULT_MAX_REDIRECTS = 5;\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\nconst SUPPORTED_API_VERSIONS = [\"v1\", \"v2\"] as const;\ntype FirecrawlApiVersion = (typeof SUPPORTED_API_VERSIONS)[number];\n\nexport interface FirecrawlSsrfOptions {\n\tallowRanges: string[];\n\ttrustEnvProxy: boolean;\n}\n\nexport interface FirecrawlExtractOptions extends Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlSearchOptions extends SearchOptions, Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tincludeContent?: boolean;\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlConfig {\n\tfirecrawlBaseUrl?: unknown;\n\tfirecrawlApiKey?: unknown;\n\tfirecrawlApiVersion?: unknown;\n\tfirecrawlFreshScrape?: unknown;\n}\n\ninterface FirecrawlScrapeData {\n\ttitle?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: { title?: unknown };\n}\n\ninterface FirecrawlSearchItem {\n\ttitle?: unknown;\n\tdescription?: unknown;\n\tsnippet?: unknown;\n\turl?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: {\n\t\ttitle?: unknown;\n\t\tdescription?: unknown;\n\t\tsourceURL?: unknown;\n\t\turl?: unknown;\n\t};\n}\n\ninterface DomainFilters {\n\tinclude: string[];\n\texclude: string[];\n}\n\nlet cachedConfig: FirecrawlConfig | null = null;\n\nfunction loadConfig(): FirecrawlConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\tconst raw = readFileSync(CONFIG_PATH, \"utf8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as FirecrawlConfig;\n\treturn cachedConfig;\n}\n\nexport function clearFirecrawlConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeBaseUrl(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst trimmed = value.trim();\n\tif (!trimmed) return null;\n\tlet parsed: URL;\n\ttry {\n\t\tparsed = new URL(trimmed);\n\t} catch {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.username || parsed.password) {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: URL credentials are not allowed`);\n\t}\n\tparsed.pathname = parsed.pathname.replace(/\\/+$/, \"\");\n\tparsed.search = \"\";\n\tparsed.hash = \"\";\n\treturn parsed.toString().replace(/\\/+$/, \"\");\n}\n\nfunction getBaseUrl(): string | null {\n\treturn normalizeBaseUrl(process.env.FIRECRAWL_BASE_URL) ?? normalizeBaseUrl(loadConfig().firecrawlBaseUrl);\n}\n\nfunction requireBaseUrl(): string {\n\tconst baseUrl = getBaseUrl();\n\tif (!baseUrl) {\n\t\tthrow new Error(\n\t\t\t\"Firecrawl base URL not configured. Either:\\n\" +\n\t\t\t` 1. Set firecrawlBaseUrl in ${CONFIG_PATH}\\n` +\n\t\t\t\" 2. Set FIRECRAWL_BASE_URL environment variable\",\n\t\t);\n\t}\n\treturn baseUrl;\n}\n\nfunction getApiVersion(): FirecrawlApiVersion {\n\tconst environmentValue = typeof process.env.FIRECRAWL_API_VERSION === \"string\"\n\t\t? process.env.FIRECRAWL_API_VERSION.trim()\n\t\t: \"\";\n\tconst raw = environmentValue || loadConfig().firecrawlApiVersion;\n\tif (raw === undefined || raw === null) return DEFAULT_API_VERSION;\n\tif (typeof raw !== \"string\") {\n\t\tthrow new Error(`firecrawlApiVersion in ${CONFIG_PATH} must be a string (\"v1\" or \"v2\")`);\n\t}\n\tconst normalized = raw.trim().toLowerCase();\n\tif (!normalized) return DEFAULT_API_VERSION;\n\tif (!SUPPORTED_API_VERSIONS.includes(normalized as FirecrawlApiVersion)) {\n\t\tthrow new Error(`Unsupported Firecrawl API version \"${raw}\". Supported versions: ${SUPPORTED_API_VERSIONS.join(\", \")}`);\n\t}\n\treturn normalized as FirecrawlApiVersion;\n}\n\nfunction allowFreshScrape(): boolean {\n\tconst environmentValue = process.env.FIRECRAWL_FRESH_SCRAPE;\n\tif (environmentValue !== undefined) return environmentValue === \"1\" || environmentValue.toLowerCase() === \"true\";\n\tconst configured = loadConfig().firecrawlFreshScrape;\n\tif (configured === undefined || configured === null) return false;\n\tif (typeof configured !== \"boolean\") throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);\n\treturn configured;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\tconst configKey = loadConfig().firecrawlApiKey;\n\treturn resolveCredential({\n\t\tprovider: \"Firecrawl\",\n\t\tconfiguredValue: configKey,\n\t\tenvironmentValue: process.env.FIRECRAWL_API_KEY,\n\t\tsignal,\n\t});\n}\n\nfunction requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {\n\tconst config = loadSsrfConfig();\n\treturn {\n\t\tallowRanges: options?.ssrf?.allowRanges ?? config.allowRanges,\n\t\ttrustEnvProxy: options?.ssrf?.trustEnvProxy ?? config.trustEnvProxy,\n\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t};\n}\n\nfunction isLoopbackApiUrl(url: URL): boolean {\n\tconst hostname = url.hostname.toLowerCase().replace(/^\\[|\\]$/g, \"\").replace(/\\.$/, \"\");\n\tif (hostname === \"localhost\" || hostname === \"::1\") return true;\n\tif (net.isIP(hostname) !== 4) return false;\n\treturn hostname.split(\".\")[0] === \"127\";\n}\n\nfunction firecrawlApiSsrfOptions(\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n\tallowLoopback: boolean,\n): ReturnType<typeof ssrfOptions> & { allowLoopback: boolean } {\n\treturn { ...ssrfOptions(options), allowLoopback };\n}\n\nfunction withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {\n\tconst next = { ...headers };\n\tdelete next.Authorization;\n\tdelete next.authorization;\n\tdelete next.Cookie;\n\tdelete next.cookie;\n\tdelete next[\"X-API-Key\"];\n\tdelete next[\"x-api-key\"];\n\treturn next;\n}\n\nasync function fetchFirecrawlApi(\n\turl: string,\n\tinit: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n): Promise<Response> {\n\tconst allowLoopback = isLoopbackApiUrl(new URL(url));\n\tlet current = await validateRemoteUrl(url, firecrawlApiSsrfOptions(options, allowLoopback));\n\tlet headers = init.headers;\n\tfor (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {\n\t\tconst response = await fetch(current, { ...init, headers, redirect: \"manual\" });\n\t\tif (!REDIRECT_STATUSES.has(response.status)) return response;\n\n\t\tconst location = response.headers.get(\"location\");\n\t\tif (!location) return response;\n\t\tif (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);\n\n\t\tconst next = await validateRemoteUrl(new URL(location, current), firecrawlApiSsrfOptions(options, allowLoopback));\n\t\tif (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);\n\t\tcurrent = next;\n\t}\n\tthrow new Error(`Too many redirects fetching ${current.toString()}`);\n}\n\nfunction scrapeBody(url: string): Record<string, unknown> {\n\treturn {\n\t\turl,\n\t\tformats: [\"markdown\"],\n\t\tonlyMainContent: true,\n\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t};\n}\n\nfunction normalizeCount(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 5;\n\treturn Math.max(1, Math.min(Math.floor(value), 20));\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction parseDomainFilter(domainFilter: string[] | undefined): DomainFilters {\n\tconst filters: DomainFilters = { include: [], exclude: [] };\n\tfor (const raw of domainFilter ?? []) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? filters.exclude : filters.include;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn filters;\n}\n\nfunction passesDomainFilters(url: string, filters: DomainFilters): boolean {\n\tif (filters.include.length === 0 && filters.exclude.length === 0) return true;\n\tlet hostname: string;\n\ttry {\n\t\thostname = new URL(url).hostname.toLowerCase();\n\t} catch {\n\t\treturn false;\n\t}\n\tconst matches = (domain: string) => hostname === domain || hostname.endsWith(`.${domain}`);\n\tif (filters.exclude.some(matches)) return false;\n\treturn filters.include.length === 0 || filters.include.some(matches);\n}\n\nfunction mapRecencyFilter(value: SearchOptions[\"recencyFilter\"]): string | undefined {\n\tswitch (value) {\n\t\tcase \"day\": return \"qdr:d\";\n\t\tcase \"week\": return \"qdr:w\";\n\t\tcase \"month\": return \"qdr:m\";\n\t\tcase \"year\": return \"qdr:y\";\n\t\tdefault: return undefined;\n\t}\n}\n\nfunction searchBody(query: string, options: FirecrawlSearchOptions, numResults: number, filters: DomainFilters): Record<string, unknown> {\n\treturn {\n\t\tquery,\n\t\tlimit: numResults,\n\t\tsources: [\"web\"],\n\t\t...(filters.include.length > 0 ? { includeDomains: filters.include } : {}),\n\t\t...(filters.include.length === 0 && filters.exclude.length > 0 ? { excludeDomains: filters.exclude } : {}),\n\t\t...(mapRecencyFilter(options.recencyFilter) ? { tbs: mapRecencyFilter(options.recencyFilter) } : {}),\n\t\t...(options.includeContent ? {\n\t\t\tscrapeOptions: {\n\t\t\t\tformats: [\"markdown\"],\n\t\t\t\tonlyMainContent: true,\n\t\t\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t\t\t},\n\t\t} : {}),\n\t};\n}\n\nfunction firstString(...values: unknown[]): string | null {\n\tfor (const value of values) {\n\t\tif (typeof value === \"string\" && value.trim()) return value.trim();\n\t}\n\treturn null;\n}\n\nfunction buildAnswer(results: SearchResponse[\"results\"]): string {\n\treturn results.map((result) => {\n\t\tif (result.snippet) return `${result.snippet}\\nSource: ${result.title} (${result.url})`;\n\t\treturn `Source: ${result.title} (${result.url})`;\n\t}).join(\"\\n\\n\");\n}\n\nfunction mapSearchResults(data: unknown, numResults: number, filters: DomainFilters): {\n\tresults: SearchResponse[\"results\"];\n\tinlineContent: ExtractedContent[];\n} {\n\tconst web = Array.isArray(data)\n\t\t? data\n\t\t: data && typeof data === \"object\" ? (data as Record<string, unknown>).web : undefined;\n\tif (!Array.isArray(web)) throw new Error(\"Firecrawl search returned an unexpected web result shape\");\n\tconst results: SearchResponse[\"results\"] = [];\n\tconst inlineContent: ExtractedContent[] = [];\n\tconst seen = new Set<string>();\n\tfor (const rawItem of web) {\n\t\tif (!rawItem || typeof rawItem !== \"object\" || Array.isArray(rawItem)) continue;\n\t\tconst item = rawItem as FirecrawlSearchItem;\n\t\tconst url = firstString(item.url, item.metadata?.sourceURL, item.metadata?.url);\n\t\tif (!url || seen.has(url) || !passesDomainFilters(url, filters)) continue;\n\t\tseen.add(url);\n\t\tconst title = firstString(item.title, item.metadata?.title) ?? url;\n\t\tconst snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? \"\";\n\t\tresults.push({ title, url, snippet });\n\t\tconst markdown = firstString(item.markdown);\n\t\tif (markdown) inlineContent.push({ url, title, content: markdown, error: null });\n\t\tif (results.length >= numResults) break;\n\t}\n\treturn { results, inlineContent };\n}\n\nasync function firecrawlFetch(\n\tendpoint: string,\n\tbody: Record<string, unknown>,\n\tsignal: AbortSignal | undefined,\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n\tlabel = endpoint,\n\tactivity: { type: \"api\"; query: string } | { type: \"fetch\"; url: string } | undefined = undefined,\n): Promise<Record<string, unknown>> {\n\tconst baseUrl = requireBaseUrl();\n\tconst version = getApiVersion();\n\tconst apiKey = await getApiKey(signal);\n\tconst headers: Record<string, string> = { \"Content-Type\": \"application/json\" };\n\tif (apiKey) headers.Authorization = `Bearer ${apiKey}`;\n\tconst requestUrl = `${baseUrl}/${version}/${endpoint}`;\n\tconst activityId = activityMonitor.logStart(activity ?? { type: \"fetch\", url: requestUrl });\n\ttry {\n\t\tconst response = await fetchFirecrawlApi(requestUrl, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders,\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),\n\t\t}, options);\n\t\tif (!response.ok) {\n\t\t\tconst text = await response.text().catch(() => \"\");\n\t\t\tthrow new Error(`Firecrawl ${label} error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);\n\t\t}\n\t\tlet data: unknown;\n\t\ttry {\n\t\t\tdata = await response.json();\n\t\t} catch (err) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned invalid JSON: ${errorMessage(err)}`);\n\t\t}\n\t\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tconst envelope = data as Record<string, unknown>;\n\t\tif (envelope.success === false) {\n\t\t\tconst reason = typeof envelope.error === \"string\" && envelope.error.trim() ? envelope.error : \"unknown error\";\n\t\t\tthrow new Error(`Firecrawl ${label} unsuccessful: ${redactCredential(reason, apiKey)}`);\n\t\t}\n\t\tif (envelope.success !== true) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn envelope;\n\t} catch (err) {\n\t\tif (isAbortError(err)) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, errorMessage(err));\n\t\tthrow err;\n\t}\n}\n\nexport function isFirecrawlAvailable(): boolean {\n\treturn getBaseUrl() !== null;\n}\n\nexport async function searchWithFirecrawl(query: string, options: FirecrawlSearchOptions = {}): Promise<SearchResponse> {\n\trequireBaseUrl();\n\tconst numResults = normalizeCount(options.numResults);\n\tconst filters = parseDomainFilter(options.domainFilter);\n\tconst envelope = await firecrawlFetch(\n\t\t\"search\",\n\t\tsearchBody(query, options, numResults, filters),\n\t\toptions.signal,\n\t\t{ ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS },\n\t\t\"search\",\n\t\t{ type: \"api\", query },\n\t);\n\tconst mapped = mapSearchResults(envelope.data, numResults, filters);\n\tconst response: SearchResponse = { answer: buildAnswer(mapped.results), results: mapped.results };\n\tif (options.includeContent && mapped.inlineContent.length > 0) response.inlineContent = mapped.inlineContent;\n\treturn response;\n}\n\nexport async function extractWithFirecrawl(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: FirecrawlExtractOptions,\n): Promise<ExtractedContent | null> {\n\trequireBaseUrl();\n\tawait validateRemoteUrl(url, ssrfOptions(options));\n\tconst envelope = await firecrawlFetch(\"scrape\", scrapeBody(url), signal, options);\n\tconst data = envelope.data;\n\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\tthrow new Error(\"Firecrawl scrape returned an unexpected data shape\");\n\t}\n\tconst scrape = data as FirecrawlScrapeData;\n\tif (typeof scrape.markdown !== \"string\") {\n\t\tthrow new Error(\"Firecrawl scrape returned markdown in an unexpected shape\");\n\t}\n\tconst content = scrape.markdown.trim();\n\tif (!content) return null;\n\tconst metadataTitle = scrape.metadata?.title;\n\tconst title = typeof metadataTitle === \"string\" && metadataTitle.trim()\n\t\t? metadataTitle.trim()\n\t\t: typeof scrape.title === \"string\" ? scrape.title.trim() : \"\";\n\treturn { url, title, content, error: null };\n}\n"]}
|
|
1
|
+
{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAG7E,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,wCAAwC,CAAC;AACpF,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAe,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAChC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC7D,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,IAAI,CAAU,CAAC;AAiDrD,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU;IAClB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC9C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,qBAAqB,WAAW,0BAA0B,CAAC,CAAC;IAC7E,CAAC;IACD,YAAY,GAAG,MAAyB,CAAC;IACzC,OAAO,YAAY,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,yBAAyB;IACxC,YAAY,GAAG,IAAI,CAAC;AACrB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACvC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,iCAAiC,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,mCAAmC,CAAC,CAAC;IAClG,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;IACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,UAAU;IAClB,OAAO,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,gBAAgB,CAAC,CAAC;AAC5G,CAAC;AAED,SAAS,cAAc;IACtB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACd,8CAA8C;YAC9C,gCAAgC,WAAW,IAAI;YAC/C,kDAAkD,CAClD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,aAAa;IACrB,MAAM,gBAAgB,GAAG,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,QAAQ;QAC7E,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE;QAC1C,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,GAAG,GAAG,gBAAgB,IAAI,UAAU,EAAE,CAAC,mBAAmB,CAAC;IACjE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAClE,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,WAAW,kCAAkC,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,UAAU;QAAE,OAAO,mBAAmB,CAAC;IAC5C,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,UAAiC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,sCAAsC,GAAG,0BAA0B,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,UAAiC,CAAC;AAC1C,CAAC;AAED,SAAS,gBAAgB;IACxB,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC5D,IAAI,gBAAgB,KAAK,SAAS;QAAE,OAAO,gBAAgB,KAAK,GAAG,IAAI,gBAAgB,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IACjH,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC,oBAAoB,CAAC;IACrD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,OAAO,UAAU,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,WAAW,oBAAoB,CAAC,CAAC;IACjH,OAAO,UAAU,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAoB;IAC5C,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,eAAe,CAAC;IAC/C,OAAO,iBAAiB,CAAC;QACxB,QAAQ,EAAE,WAAW;QACrB,eAAe,EAAE,SAAS;QAC1B,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAC/C,MAAM;KACN,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,MAAoB;IAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC9D,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IACjC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IACjC,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,WAAW,CAAC,OAA0D;IAC9E,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,OAAO;QACN,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,IAAI,MAAM,CAAC,WAAW;QAC7D,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,IAAI,MAAM,CAAC,aAAa;QACnE,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACvF,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAChE,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3C,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC;AACzC,CAAC;AAED,SAAS,uBAAuB,CAC/B,OAAqE,EACrE,aAAsB;IAEtB,OAAO,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,uBAAuB,CAAC,OAA+B;IAC/D,MAAM,IAAI,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC5B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,aAAa,CAAC;IAC1B,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACnB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC;AACb,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC/B,GAAW,EACX,IAA4F,EAC5F,OAAqE;IAErE,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,uBAAuB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;IAC5F,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;QACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC/B,IAAI,SAAS,KAAK,qBAAqB;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE9G,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,uBAAuB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;QAClH,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAC/E,OAAO,GAAG,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC9B,OAAO;QACN,GAAG;QACH,OAAO,EAAE,CAAC,UAAU,CAAC;QACrB,eAAe,EAAE,IAAI;QACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACjD,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,YAAkC;IAC5D,MAAM,OAAO,GAAkB,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC5D,KAAK,MAAM,GAAG,IAAI,YAAY,IAAI,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAE,OAAsB;IAC/D,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9E,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACJ,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,QAAQ,KAAK,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAqC;IAC9D,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,KAAK,EAAE,OAAO,OAAO,CAAC;QAC3B,KAAK,MAAM,EAAE,OAAO,OAAO,CAAC;QAC5B,KAAK,OAAO,EAAE,OAAO,OAAO,CAAC;QAC7B,KAAK,MAAM,EAAE,OAAO,OAAO,CAAC;QAC5B,SAAS,OAAO,SAAS,CAAC;IAC3B,CAAC;AACF,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,OAA+B,EAAE,UAAkB,EAAE,OAAsB;IAC7G,OAAO;QACN,KAAK;QACL,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,CAAC,KAAK,CAAC;QAChB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1G,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;YAC5B,aAAa,EAAE;gBACd,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,eAAe,EAAE,IAAI;gBACrB,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aACjD;SACD,CAAC,CAAC,CAAC,EAAE,CAAC;KACP,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,GAAG,MAAiB;IACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;YAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa,EAAE,UAAkB,EAAE,OAAsB;IAIlF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAC9B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAgC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IACrG,MAAM,OAAO,GAA8B,EAAE,CAAC;IAC9C,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QAChF,MAAM,IAAI,GAAG,OAA8B,CAAC;QAC5C,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAChF,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC;YAAE,SAAS;QAC1E,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC;QACnE,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC;QAC9F,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,QAAQ;YAAE,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;IACzC,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,cAAc,CAC5B,QAAgB,EAChB,IAA6B,EAC7B,MAA+B,EAC/B,OAAqE,EACrE,KAAK,GAAG,QAAQ,EAChB,QAAQ,GAAgF,SAAS;IAEjG,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,OAAO,GAA2B,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;IAC/E,IAAI,MAAM;QAAE,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,OAAO,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5F,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,UAAU,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,EAAE,SAAS,IAAI,kBAAkB,EAAE,MAAM,CAAC;SACvE,EAAE,OAAO,CAAC,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,UAAU,QAAQ,CAAC,MAAM,KAAK,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;QACD,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,2BAA2B,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,QAAQ,GAAG,IAA+B,CAAC;QACjD,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC;YAC9G,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,kBAAkB,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,wCAAwC,CAAC,CAAC;QAC7E,CAAC;QACD,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,IAAI,YAAY,CAAC,GAAG,CAAC;YAAE,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;;YAC7D,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,CAAC;IACX,CAAC;AACF,CAAC;AAED,MAAM,UAAU,oBAAoB;IACnC,OAAO,UAAU,EAAE,KAAK,IAAI,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAa,EAAE,OAAO,GAA2B,EAAE;IAC5F,cAAc,EAAE,CAAC;IACjB,MAAM,UAAU,GAAG,0BAA0B,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,MAAM,cAAc,CACpC,QAAQ,EACR,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,EAC/C,OAAO,CAAC,MAAM,EACd,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,iBAAiB,EAAE,EACjE,QAAQ,EACR,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CACtB,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAmB,EAAE,MAAM,EAAE,2BAA2B,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAClH,IAAI,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAC7G,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAW,EACX,MAAoB,EACpB,OAAiC;IAEjC,cAAc,EAAE,CAAC;IACjB,MAAM,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,MAAM,GAAG,IAA2B,CAAC;IAC3C,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC;IAC7C,MAAM,KAAK,GAAG,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,CAAC,IAAI,EAAE;QACtE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE;QACtB,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC7C,CAAC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport net from \"node:net\";\nimport { activityMonitor } from \"./activity.ts\";\nimport { normalizeDomain } from \"./domain-filter-normalization.ts\";\nimport { redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { formatSearchResultsAsAnswer } from \"./search-answer-formatting.ts\";\nimport { normalizeSearchResultCount } from \"./search-result-count-normalization.ts\";\nimport { loadSsrfConfig, validateRemoteUrl, type Lookup } from \"./ssrf-protection.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst DEFAULT_API_VERSION = \"v2\";\nconst EXTRACT_TIMEOUT_MS = 60_000;\nconst SEARCH_TIMEOUT_MS = 60_000;\nconst DEFAULT_MAX_REDIRECTS = 5;\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\nconst SUPPORTED_API_VERSIONS = [\"v1\", \"v2\"] as const;\ntype FirecrawlApiVersion = (typeof SUPPORTED_API_VERSIONS)[number];\n\nexport interface FirecrawlSsrfOptions {\n\tallowRanges: string[];\n\ttrustEnvProxy: boolean;\n}\n\nexport interface FirecrawlExtractOptions extends Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlSearchOptions extends SearchOptions, Pick<ExtractOptions, \"timeoutMs\" | \"lookup\"> {\n\tincludeContent?: boolean;\n\tssrf?: FirecrawlSsrfOptions;\n}\n\ninterface FirecrawlConfig {\n\tfirecrawlBaseUrl?: unknown;\n\tfirecrawlApiKey?: unknown;\n\tfirecrawlApiVersion?: unknown;\n\tfirecrawlFreshScrape?: unknown;\n}\n\ninterface FirecrawlScrapeData {\n\ttitle?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: { title?: unknown };\n}\n\ninterface FirecrawlSearchItem {\n\ttitle?: unknown;\n\tdescription?: unknown;\n\tsnippet?: unknown;\n\turl?: unknown;\n\tmarkdown?: unknown;\n\tmetadata?: {\n\t\ttitle?: unknown;\n\t\tdescription?: unknown;\n\t\tsourceURL?: unknown;\n\t\turl?: unknown;\n\t};\n}\n\ninterface DomainFilters {\n\tinclude: string[];\n\texclude: string[];\n}\n\nlet cachedConfig: FirecrawlConfig | null = null;\n\nfunction loadConfig(): FirecrawlConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\tconst raw = readFileSync(CONFIG_PATH, \"utf8\");\n\tlet parsed: unknown;\n\ttry {\n\t\tparsed = JSON.parse(raw);\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n\tif (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n\t\tthrow new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);\n\t}\n\tcachedConfig = parsed as FirecrawlConfig;\n\treturn cachedConfig;\n}\n\nexport function clearFirecrawlConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeBaseUrl(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst trimmed = value.trim();\n\tif (!trimmed) return null;\n\tlet parsed: URL;\n\ttry {\n\t\tparsed = new URL(trimmed);\n\t} catch {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: expected an HTTP or HTTPS URL`);\n\t}\n\tif (parsed.username || parsed.password) {\n\t\tthrow new Error(`Invalid Firecrawl base URL in ${CONFIG_PATH}: URL credentials are not allowed`);\n\t}\n\tparsed.pathname = parsed.pathname.replace(/\\/+$/, \"\");\n\tparsed.search = \"\";\n\tparsed.hash = \"\";\n\treturn parsed.toString().replace(/\\/+$/, \"\");\n}\n\nfunction getBaseUrl(): string | null {\n\treturn normalizeBaseUrl(process.env.FIRECRAWL_BASE_URL) ?? normalizeBaseUrl(loadConfig().firecrawlBaseUrl);\n}\n\nfunction requireBaseUrl(): string {\n\tconst baseUrl = getBaseUrl();\n\tif (!baseUrl) {\n\t\tthrow new Error(\n\t\t\t\"Firecrawl base URL not configured. Either:\\n\" +\n\t\t\t` 1. Set firecrawlBaseUrl in ${CONFIG_PATH}\\n` +\n\t\t\t\" 2. Set FIRECRAWL_BASE_URL environment variable\",\n\t\t);\n\t}\n\treturn baseUrl;\n}\n\nfunction getApiVersion(): FirecrawlApiVersion {\n\tconst environmentValue = typeof process.env.FIRECRAWL_API_VERSION === \"string\"\n\t\t? process.env.FIRECRAWL_API_VERSION.trim()\n\t\t: \"\";\n\tconst raw = environmentValue || loadConfig().firecrawlApiVersion;\n\tif (raw === undefined || raw === null) return DEFAULT_API_VERSION;\n\tif (typeof raw !== \"string\") {\n\t\tthrow new Error(`firecrawlApiVersion in ${CONFIG_PATH} must be a string (\"v1\" or \"v2\")`);\n\t}\n\tconst normalized = raw.trim().toLowerCase();\n\tif (!normalized) return DEFAULT_API_VERSION;\n\tif (!SUPPORTED_API_VERSIONS.includes(normalized as FirecrawlApiVersion)) {\n\t\tthrow new Error(`Unsupported Firecrawl API version \"${raw}\". Supported versions: ${SUPPORTED_API_VERSIONS.join(\", \")}`);\n\t}\n\treturn normalized as FirecrawlApiVersion;\n}\n\nfunction allowFreshScrape(): boolean {\n\tconst environmentValue = process.env.FIRECRAWL_FRESH_SCRAPE;\n\tif (environmentValue !== undefined) return environmentValue === \"1\" || environmentValue.toLowerCase() === \"true\";\n\tconst configured = loadConfig().firecrawlFreshScrape;\n\tif (configured === undefined || configured === null) return false;\n\tif (typeof configured !== \"boolean\") throw new Error(`firecrawlFreshScrape in ${CONFIG_PATH} must be a boolean`);\n\treturn configured;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\tconst configKey = loadConfig().firecrawlApiKey;\n\treturn resolveCredential({\n\t\tprovider: \"Firecrawl\",\n\t\tconfiguredValue: configKey,\n\t\tenvironmentValue: process.env.FIRECRAWL_API_KEY,\n\t\tsignal,\n\t});\n}\n\nfunction requestSignal(timeoutMs: number, signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction isAbortError(err: unknown): boolean {\n\treturn errorMessage(err).toLowerCase().includes(\"abort\");\n}\n\nfunction ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions): { lookup?: Lookup; allowRanges: string[]; trustEnvProxy: boolean } {\n\tconst config = loadSsrfConfig();\n\treturn {\n\t\tallowRanges: options?.ssrf?.allowRanges ?? config.allowRanges,\n\t\ttrustEnvProxy: options?.ssrf?.trustEnvProxy ?? config.trustEnvProxy,\n\t\t...(options?.lookup ? { lookup: options.lookup } : {}),\n\t};\n}\n\nfunction isLoopbackApiUrl(url: URL): boolean {\n\tconst hostname = url.hostname.toLowerCase().replace(/^\\[|\\]$/g, \"\").replace(/\\.$/, \"\");\n\tif (hostname === \"localhost\" || hostname === \"::1\") return true;\n\tif (net.isIP(hostname) !== 4) return false;\n\treturn hostname.split(\".\")[0] === \"127\";\n}\n\nfunction firecrawlApiSsrfOptions(\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n\tallowLoopback: boolean,\n): ReturnType<typeof ssrfOptions> & { allowLoopback: boolean } {\n\treturn { ...ssrfOptions(options), allowLoopback };\n}\n\nfunction withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {\n\tconst next = { ...headers };\n\tdelete next.Authorization;\n\tdelete next.authorization;\n\tdelete next.Cookie;\n\tdelete next.cookie;\n\tdelete next[\"X-API-Key\"];\n\tdelete next[\"x-api-key\"];\n\treturn next;\n}\n\nasync function fetchFirecrawlApi(\n\turl: string,\n\tinit: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n): Promise<Response> {\n\tconst allowLoopback = isLoopbackApiUrl(new URL(url));\n\tlet current = await validateRemoteUrl(url, firecrawlApiSsrfOptions(options, allowLoopback));\n\tlet headers = init.headers;\n\tfor (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {\n\t\tconst response = await fetch(current, { ...init, headers, redirect: \"manual\" });\n\t\tif (!REDIRECT_STATUSES.has(response.status)) return response;\n\n\t\tconst location = response.headers.get(\"location\");\n\t\tif (!location) return response;\n\t\tif (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);\n\n\t\tconst next = await validateRemoteUrl(new URL(location, current), firecrawlApiSsrfOptions(options, allowLoopback));\n\t\tif (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);\n\t\tcurrent = next;\n\t}\n\tthrow new Error(`Too many redirects fetching ${current.toString()}`);\n}\n\nfunction scrapeBody(url: string): Record<string, unknown> {\n\treturn {\n\t\turl,\n\t\tformats: [\"markdown\"],\n\t\tonlyMainContent: true,\n\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t};\n}\n\nfunction parseDomainFilter(domainFilter: string[] | undefined): DomainFilters {\n\tconst filters: DomainFilters = { include: [], exclude: [] };\n\tfor (const raw of domainFilter ?? []) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? filters.exclude : filters.include;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn filters;\n}\n\nfunction passesDomainFilters(url: string, filters: DomainFilters): boolean {\n\tif (filters.include.length === 0 && filters.exclude.length === 0) return true;\n\tlet hostname: string;\n\ttry {\n\t\thostname = new URL(url).hostname.toLowerCase();\n\t} catch {\n\t\treturn false;\n\t}\n\tconst matches = (domain: string) => hostname === domain || hostname.endsWith(`.${domain}`);\n\tif (filters.exclude.some(matches)) return false;\n\treturn filters.include.length === 0 || filters.include.some(matches);\n}\n\nfunction mapRecencyFilter(value: SearchOptions[\"recencyFilter\"]): string | undefined {\n\tswitch (value) {\n\t\tcase \"day\": return \"qdr:d\";\n\t\tcase \"week\": return \"qdr:w\";\n\t\tcase \"month\": return \"qdr:m\";\n\t\tcase \"year\": return \"qdr:y\";\n\t\tdefault: return undefined;\n\t}\n}\n\nfunction searchBody(query: string, options: FirecrawlSearchOptions, numResults: number, filters: DomainFilters): Record<string, unknown> {\n\treturn {\n\t\tquery,\n\t\tlimit: numResults,\n\t\tsources: [\"web\"],\n\t\t...(filters.include.length > 0 ? { includeDomains: filters.include } : {}),\n\t\t...(filters.include.length === 0 && filters.exclude.length > 0 ? { excludeDomains: filters.exclude } : {}),\n\t\t...(mapRecencyFilter(options.recencyFilter) ? { tbs: mapRecencyFilter(options.recencyFilter) } : {}),\n\t\t...(options.includeContent ? {\n\t\t\tscrapeOptions: {\n\t\t\t\tformats: [\"markdown\"],\n\t\t\t\tonlyMainContent: true,\n\t\t\t\t...(allowFreshScrape() ? {} : { lockdown: true }),\n\t\t\t},\n\t\t} : {}),\n\t};\n}\n\nfunction firstString(...values: unknown[]): string | null {\n\tfor (const value of values) {\n\t\tif (typeof value === \"string\" && value.trim()) return value.trim();\n\t}\n\treturn null;\n}\n\nfunction mapSearchResults(data: unknown, numResults: number, filters: DomainFilters): {\n\tresults: SearchResponse[\"results\"];\n\tinlineContent: ExtractedContent[];\n} {\n\tconst web = Array.isArray(data)\n\t\t? data\n\t\t: data && typeof data === \"object\" ? (data as Record<string, unknown>).web : undefined;\n\tif (!Array.isArray(web)) throw new Error(\"Firecrawl search returned an unexpected web result shape\");\n\tconst results: SearchResponse[\"results\"] = [];\n\tconst inlineContent: ExtractedContent[] = [];\n\tconst seen = new Set<string>();\n\tfor (const rawItem of web) {\n\t\tif (!rawItem || typeof rawItem !== \"object\" || Array.isArray(rawItem)) continue;\n\t\tconst item = rawItem as FirecrawlSearchItem;\n\t\tconst url = firstString(item.url, item.metadata?.sourceURL, item.metadata?.url);\n\t\tif (!url || seen.has(url) || !passesDomainFilters(url, filters)) continue;\n\t\tseen.add(url);\n\t\tconst title = firstString(item.title, item.metadata?.title) ?? url;\n\t\tconst snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? \"\";\n\t\tresults.push({ title, url, snippet });\n\t\tconst markdown = firstString(item.markdown);\n\t\tif (markdown) inlineContent.push({ url, title, content: markdown, error: null });\n\t\tif (results.length >= numResults) break;\n\t}\n\treturn { results, inlineContent };\n}\n\nasync function firecrawlFetch(\n\tendpoint: string,\n\tbody: Record<string, unknown>,\n\tsignal: AbortSignal | undefined,\n\toptions: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,\n\tlabel = endpoint,\n\tactivity: { type: \"api\"; query: string } | { type: \"fetch\"; url: string } | undefined = undefined,\n): Promise<Record<string, unknown>> {\n\tconst baseUrl = requireBaseUrl();\n\tconst version = getApiVersion();\n\tconst apiKey = await getApiKey(signal);\n\tconst headers: Record<string, string> = { \"Content-Type\": \"application/json\" };\n\tif (apiKey) headers.Authorization = `Bearer ${apiKey}`;\n\tconst requestUrl = `${baseUrl}/${version}/${endpoint}`;\n\tconst activityId = activityMonitor.logStart(activity ?? { type: \"fetch\", url: requestUrl });\n\ttry {\n\t\tconst response = await fetchFirecrawlApi(requestUrl, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders,\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal),\n\t\t}, options);\n\t\tif (!response.ok) {\n\t\t\tconst text = await response.text().catch(() => \"\");\n\t\t\tthrow new Error(`Firecrawl ${label} error ${response.status}: ${redactCredential(text.slice(0, 300), apiKey)}`);\n\t\t}\n\t\tlet data: unknown;\n\t\ttry {\n\t\t\tdata = await response.json();\n\t\t} catch (err) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned invalid JSON: ${errorMessage(err)}`);\n\t\t}\n\t\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tconst envelope = data as Record<string, unknown>;\n\t\tif (envelope.success === false) {\n\t\t\tconst reason = typeof envelope.error === \"string\" && envelope.error.trim() ? envelope.error : \"unknown error\";\n\t\t\tthrow new Error(`Firecrawl ${label} unsuccessful: ${redactCredential(reason, apiKey)}`);\n\t\t}\n\t\tif (envelope.success !== true) {\n\t\t\tthrow new Error(`Firecrawl ${label} returned an unexpected response shape`);\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn envelope;\n\t} catch (err) {\n\t\tif (isAbortError(err)) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, errorMessage(err));\n\t\tthrow err;\n\t}\n}\n\nexport function isFirecrawlAvailable(): boolean {\n\treturn getBaseUrl() !== null;\n}\n\nexport async function searchWithFirecrawl(query: string, options: FirecrawlSearchOptions = {}): Promise<SearchResponse> {\n\trequireBaseUrl();\n\tconst numResults = normalizeSearchResultCount(options.numResults);\n\tconst filters = parseDomainFilter(options.domainFilter);\n\tconst envelope = await firecrawlFetch(\n\t\t\"search\",\n\t\tsearchBody(query, options, numResults, filters),\n\t\toptions.signal,\n\t\t{ ...options, timeoutMs: options.timeoutMs ?? SEARCH_TIMEOUT_MS },\n\t\t\"search\",\n\t\t{ type: \"api\", query },\n\t);\n\tconst mapped = mapSearchResults(envelope.data, numResults, filters);\n\tconst response: SearchResponse = { answer: formatSearchResultsAsAnswer(mapped.results), results: mapped.results };\n\tif (options.includeContent && mapped.inlineContent.length > 0) response.inlineContent = mapped.inlineContent;\n\treturn response;\n}\n\nexport async function extractWithFirecrawl(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions?: FirecrawlExtractOptions,\n): Promise<ExtractedContent | null> {\n\trequireBaseUrl();\n\tawait validateRemoteUrl(url, ssrfOptions(options));\n\tconst envelope = await firecrawlFetch(\"scrape\", scrapeBody(url), signal, options);\n\tconst data = envelope.data;\n\tif (!data || typeof data !== \"object\" || Array.isArray(data)) {\n\t\tthrow new Error(\"Firecrawl scrape returned an unexpected data shape\");\n\t}\n\tconst scrape = data as FirecrawlScrapeData;\n\tif (typeof scrape.markdown !== \"string\") {\n\t\tthrow new Error(\"Firecrawl scrape returned markdown in an unexpected shape\");\n\t}\n\tconst content = scrape.markdown.trim();\n\tif (!content) return null;\n\tconst metadataTitle = scrape.metadata?.title;\n\tconst title = typeof metadataTitle === \"string\" && metadataTitle.trim()\n\t\t? metadataTitle.trim()\n\t\t: typeof scrape.title === \"string\" ? scrape.title.trim() : \"\";\n\treturn { url, title, content, error: null };\n}\n"]}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ExtensionContext } from "../host.ts";
|
|
2
2
|
import { type SearchResponse, type SearchOptions } from "./perplexity.ts";
|
|
3
|
-
export declare const RESOLVED_SEARCH_PROVIDERS: readonly ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "valyu", "bocha", "xcrawl"];
|
|
4
|
-
export declare const SEARCH_PROVIDERS: readonly ["auto", "all", "openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "valyu", "bocha", "xcrawl"];
|
|
3
|
+
export declare const RESOLVED_SEARCH_PROVIDERS: readonly ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "bocha", "xcrawl"];
|
|
4
|
+
export declare const SEARCH_PROVIDERS: readonly ["auto", "all", "openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "bocha", "xcrawl"];
|
|
5
5
|
export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number];
|
|
6
6
|
export type SearchProvider = typeof SEARCH_PROVIDERS[number];
|
|
7
7
|
export type SearchProviderSelection = SearchProvider | ResolvedSearchProvider[];
|
|
@@ -35,11 +35,14 @@ export interface AttributedSearchResponse extends SearchResponse {
|
|
|
35
35
|
}
|
|
36
36
|
export declare const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[];
|
|
37
37
|
export declare function getConfiguredSearchRouting(): SearchRoutingConfig | undefined;
|
|
38
|
+
export declare function assertSearchProviderSelectionAllowed(selection: SearchProviderSelection, label?: string, allowedProviders?: ("anysearch" | "bocha" | "brave" | "brightdata" | "duckduckgo" | "exa" | "firecrawl" | "gemini" | "jina" | "kagi" | "kimi" | "mistral" | "ollama" | "openai" | "parallel" | "parallel-mcp" | "perplexity" | "querit" | "search1api" | "searchinfinity" | "searxng" | "serpapi" | "serpbase" | "serpdive" | "serper" | "serply" | "tavily" | "tinyfish" | "valyu" | "xai" | "xcrawl")[] | undefined): void;
|
|
39
|
+
export declare function getAllowedSearchProviders(): readonly ResolvedSearchProvider[];
|
|
38
40
|
export declare function normalizeSearchProviderSelection(value: unknown, label?: string): SearchProviderSelection;
|
|
39
41
|
export interface FullSearchOptions extends SearchOptions {
|
|
40
42
|
provider?: SearchProviderSelection;
|
|
41
43
|
includeContent?: boolean;
|
|
42
44
|
extensionContext?: ExtensionContext;
|
|
43
45
|
}
|
|
46
|
+
export declare function providerLabel(provider: ResolvedSearchProvider): string;
|
|
44
47
|
export declare function search(query: string, options?: FullSearchOptions): Promise<AttributedSearchResponse>;
|
|
45
48
|
//# sourceMappingURL=gemini-search.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini-search.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/gemini-search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAMnD,OAAO,EAAkE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"gemini-search.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/gemini-search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAMnD,OAAO,EAAkE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAsC1I,eAAO,MAAM,yBAAyB,YAAI,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAC;AACzY,eAAO,MAAM,gBAAgB,gXAAyD,CAAC;AAEvF,MAAM,MAAM,sBAAsB,GAAG,OAAO,yBAAyB,CAAC,MAAM,CAAC,CAAC;AAC9E,MAAM,MAAM,cAAc,GAAG,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;AAC7D,MAAM,MAAM,uBAAuB,GAAG,cAAc,GAAG,sBAAsB,EAAE,CAAC;AAChF,MAAM,MAAM,oBAAoB,GAAG;IAAE,GAAG,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;AAC9F,MAAM,MAAM,uBAAuB,GAChC,WAAW,GACX,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,MAAM,GACN,iBAAiB,GACjB,kBAAkB,GAClB,aAAa,GACb,SAAS,GACT,SAAS,CAAC;AAEb,MAAM,WAAW,mBAAmB;IACnC,SAAS,EAAE,sBAAsB,EAAE,CAAC;IACpC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,uBAAuB,EAAE,WAAW,GAAG,OAAO,GAAG,SAAS,GAAG,kBAAkB,GAAG,aAAa,CAAC,CAAC,CAAC;CAC5H;AAED,qBAAa,mBAAoB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAC;IAC1C,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAE7B,YACC,QAAQ,EAAE,sBAAsB,EAChC,IAAI,EAAE,uBAAuB,EAC7B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,KAAK,EAAE,OAAO,EAQd;CACD;AAED,MAAM,WAAW,sBAAuB,SAAQ,cAAc;IAC7D,QAAQ,EAAE,sBAAsB,CAAC;CACjC;AAED,MAAM,WAAW,qBAAqB;IACrC,QAAQ,EAAE,sBAAsB,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,wBAAyB,SAAQ,cAAc;IAC/D,QAAQ,EAAE,sBAAsB,GAAG,KAAK,CAAC;IACzC,iBAAiB,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAC7C,cAAc,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC;AAMD,eAAO,MAAM,oBAAoB,EAAE,sBAAsB,EAA0M,CAAC;AA4FpQ,wBAAgB,0BAA0B,IAAI,mBAAmB,GAAG,SAAS,CAG5E;AA0BD,wBAAgB,oCAAoC,CACnD,SAAS,EAAE,uBAAuB,EAClC,KAAK,SAAa,EAClB,gBAAgB,qYAAqC,GACnD,IAAI,CASN;AAED,wBAAgB,yBAAyB,IAAI,SAAS,sBAAsB,EAAE,CAE7E;AAED,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,SAAa,GAAG,uBAAuB,CAI5G;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACvD,QAAQ,CAAC,EAAE,uBAAuB,CAAC;IACnC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACpC;AA2ND,wBAAgB,aAAa,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAyBtE;AAgHD,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAmN9G"}
|
|
@@ -7,7 +7,7 @@ import { getGeminiWebAvailabilityDiagnostic, isGeminiWebAvailable, queryWithCook
|
|
|
7
7
|
import { isPerplexityAvailable, searchWithPerplexity } from "./perplexity.js";
|
|
8
8
|
import { isExaAvailable, searchWithExa } from "./exa.js";
|
|
9
9
|
import { isBraveAvailable, searchWithBrave } from "./brave.js";
|
|
10
|
-
import { isCurrentModelHostedSearchEligible, isOpenAISearchAvailable, searchWithCurrentModelOpenAI, searchWithOpenAI, } from "./openai-search.js";
|
|
10
|
+
import { isCurrentModelHostedSearchEligible, isOpenAISearchAvailable, OpenAIAlphaSearchUnsupportedError, searchWithCurrentModelOpenAI, searchWithOpenAI, } from "./openai-search.js";
|
|
11
11
|
import { isParallelAvailable, searchWithParallel } from "./parallel.js";
|
|
12
12
|
import { isParallelMcpAvailable, searchWithParallelMcp } from "./parallel-mcp.js";
|
|
13
13
|
import { isTinyFishAvailable, searchWithTinyFish } from "./tinyfish.js";
|
|
@@ -30,11 +30,12 @@ import { isBrightDataAvailable, searchWithBrightData } from "./brightdata.js";
|
|
|
30
30
|
import { isSerpBaseAvailable, searchWithSerpBase } from "./serpbase.js";
|
|
31
31
|
import { isSerpApiAvailable, searchWithSerpApi } from "./serpapi.js";
|
|
32
32
|
import { isSerperAvailable, searchWithSerper } from "./serper.js";
|
|
33
|
+
import { isSerplyAvailable, searchWithSerply } from "./serply.js";
|
|
33
34
|
import { isValyuAvailable, searchWithValyu } from "./valyu.js";
|
|
34
35
|
import { isKimiSearchAvailable, searchWithKimi } from "./kimi-search.js";
|
|
35
36
|
import { isMistralAvailable, searchWithMistral } from "./mistral-search.js";
|
|
36
37
|
import { getWebSearchConfigPath } from "./utils.js";
|
|
37
|
-
export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "valyu", "bocha", "xcrawl"];
|
|
38
|
+
export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "bocha", "xcrawl"];
|
|
38
39
|
export const SEARCH_PROVIDERS = ["auto", "all", ...RESOLVED_SEARCH_PROVIDERS];
|
|
39
40
|
export class SearchProviderError extends Error {
|
|
40
41
|
constructor(provider, kind, message, status, cause) {
|
|
@@ -48,7 +49,7 @@ export class SearchProviderError extends Error {
|
|
|
48
49
|
}
|
|
49
50
|
const CONFIG_PATH = getWebSearchConfigPath();
|
|
50
51
|
const DEFAULT_SEARCH_MODEL = "gemini-3.6-flash";
|
|
51
|
-
// Explicit-only providers (Parallel MCP, DuckDuckGo, Kimi, AnySearch, XCrawl, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Valyu) are deliberately absent:
|
|
52
|
+
// Explicit-only providers (Parallel MCP, DuckDuckGo, Kimi, AnySearch, XCrawl, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Serply, Valyu) are deliberately absent:
|
|
52
53
|
// `all` must never fan out to an opt-in or paid provider without the user asking for it.
|
|
53
54
|
export const ALL_SEARCH_PROVIDERS = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"];
|
|
54
55
|
const VALID_ROUTING_KINDS = ["transient", "quota", "network", "invalid-response", "unsupported"];
|
|
@@ -74,12 +75,32 @@ function getSearchConfig() {
|
|
|
74
75
|
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
75
76
|
}
|
|
76
77
|
const searchModel = normalizeSearchModel(raw.searchModel);
|
|
78
|
+
const webSearch = raw.webSearch;
|
|
79
|
+
if (webSearch !== undefined && (!webSearch || typeof webSearch !== "object" || Array.isArray(webSearch))) {
|
|
80
|
+
throw new Error(`webSearch in ${CONFIG_PATH} must be an object`);
|
|
81
|
+
}
|
|
82
|
+
const allowedProviders = webSearch && Object.hasOwn(webSearch, "allowedProviders")
|
|
83
|
+
? normalizeResolvedProviderList(webSearch.allowedProviders, `webSearch.allowedProviders in ${CONFIG_PATH}`)
|
|
84
|
+
: undefined;
|
|
77
85
|
const searchProviderConfigured = Object.hasOwn(raw, "searchProvider") || Object.hasOwn(raw, "provider");
|
|
86
|
+
const searchProvider = normalizeSearchProviderSelection(raw.searchProvider ?? raw.provider, `provider in ${CONFIG_PATH}`);
|
|
87
|
+
const searchRouting = Object.hasOwn(raw, "searchRouting") ? normalizeSearchRouting(raw.searchRouting) : undefined;
|
|
88
|
+
if (allowedProviders) {
|
|
89
|
+
if (Object.hasOwn(raw, "searchProvider")) {
|
|
90
|
+
assertSearchProviderSelectionAllowed(normalizeSearchProviderSelection(raw.searchProvider), `searchProvider in ${CONFIG_PATH}`, allowedProviders);
|
|
91
|
+
}
|
|
92
|
+
if (Object.hasOwn(raw, "provider")) {
|
|
93
|
+
assertSearchProviderSelectionAllowed(normalizeSearchProviderSelection(raw.provider), `provider in ${CONFIG_PATH}`, allowedProviders);
|
|
94
|
+
}
|
|
95
|
+
if (searchRouting)
|
|
96
|
+
assertSearchProviderSelectionAllowed(searchRouting.providers, `searchRouting.providers in ${CONFIG_PATH}`, allowedProviders);
|
|
97
|
+
}
|
|
78
98
|
cachedSearchConfig = {
|
|
79
|
-
searchProvider
|
|
99
|
+
searchProvider,
|
|
80
100
|
searchProviderConfigured,
|
|
81
|
-
...(
|
|
101
|
+
...(searchRouting ? { searchRouting } : {}),
|
|
82
102
|
...(searchModel ? { searchModel } : {}),
|
|
103
|
+
...(allowedProviders ? { allowedProviders } : {}),
|
|
83
104
|
};
|
|
84
105
|
return cachedSearchConfig;
|
|
85
106
|
}
|
|
@@ -138,6 +159,20 @@ function normalizeResolvedProviderList(value, label) {
|
|
|
138
159
|
}
|
|
139
160
|
return providers;
|
|
140
161
|
}
|
|
162
|
+
export function assertSearchProviderSelectionAllowed(selection, label = "provider", allowedProviders = getSearchConfig().allowedProviders) {
|
|
163
|
+
if (!allowedProviders)
|
|
164
|
+
return;
|
|
165
|
+
const requested = Array.isArray(selection)
|
|
166
|
+
? selection
|
|
167
|
+
: selection === "auto" || selection === "all" ? [] : [selection];
|
|
168
|
+
const disabled = requested.filter(provider => !allowedProviders.includes(provider));
|
|
169
|
+
if (disabled.length > 0) {
|
|
170
|
+
throw new Error(`${label} ${disabled.length === 1 ? `references disabled provider "${disabled[0]}"` : `references disabled providers: ${disabled.join(", ")}`}; allowed by webSearch.allowedProviders: ${allowedProviders.join(", ")}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
export function getAllowedSearchProviders() {
|
|
174
|
+
return getSearchConfig().allowedProviders ?? RESOLVED_SEARCH_PROVIDERS;
|
|
175
|
+
}
|
|
141
176
|
export function normalizeSearchProviderSelection(value, label = "provider") {
|
|
142
177
|
if (Array.isArray(value))
|
|
143
178
|
return normalizeResolvedProviderList(value, label);
|
|
@@ -227,6 +262,9 @@ function classifyProviderError(provider, err) {
|
|
|
227
262
|
else if (isAbortError(err)) {
|
|
228
263
|
kind = "aborted";
|
|
229
264
|
}
|
|
265
|
+
else if (err instanceof OpenAIAlphaSearchUnsupportedError) {
|
|
266
|
+
kind = "unsupported";
|
|
267
|
+
}
|
|
230
268
|
else if (provider === "xai" && status === 403 && /spending[- ]limit|(?:no|out of) credits?|insufficient quota|quota (?:exceeded|exhausted)|credits? (?:exhausted|depleted|used up)/.test(lower)) {
|
|
231
269
|
kind = "quota";
|
|
232
270
|
}
|
|
@@ -317,6 +355,8 @@ async function searchWithResolvedProvider(provider, query, options, useCurrentMo
|
|
|
317
355
|
return { ...(await searchWithSerpApi(query, options)), provider };
|
|
318
356
|
if (provider === "serper")
|
|
319
357
|
return { ...(await searchWithSerper(query, options)), provider };
|
|
358
|
+
if (provider === "serply")
|
|
359
|
+
return { ...(await searchWithSerply(query, options)), provider };
|
|
320
360
|
if (provider === "valyu")
|
|
321
361
|
return { ...(await searchWithValyu(query, options)), provider };
|
|
322
362
|
if (provider === "xcrawl")
|
|
@@ -392,6 +432,8 @@ async function isResolvedProviderAvailable(provider, options, useCurrentModel =
|
|
|
392
432
|
return isSerpApiAvailable();
|
|
393
433
|
if (provider === "serper")
|
|
394
434
|
return isSerperAvailable();
|
|
435
|
+
if (provider === "serply")
|
|
436
|
+
return isSerplyAvailable();
|
|
395
437
|
if (provider === "valyu")
|
|
396
438
|
return isValyuAvailable();
|
|
397
439
|
if (provider === "xcrawl")
|
|
@@ -416,7 +458,7 @@ async function isGeminiWebOptionallyAvailable() {
|
|
|
416
458
|
return false;
|
|
417
459
|
}
|
|
418
460
|
}
|
|
419
|
-
function providerLabel(provider) {
|
|
461
|
+
export function providerLabel(provider) {
|
|
420
462
|
if (provider === "openai")
|
|
421
463
|
return "OpenAI";
|
|
422
464
|
if (provider === "parallel-mcp")
|
|
@@ -459,6 +501,8 @@ function providerLabel(provider) {
|
|
|
459
501
|
return "SerpApi";
|
|
460
502
|
if (provider === "serper")
|
|
461
503
|
return "Serper";
|
|
504
|
+
if (provider === "serply")
|
|
505
|
+
return "Serply";
|
|
462
506
|
if (provider === "valyu")
|
|
463
507
|
return "Valyu";
|
|
464
508
|
return provider.charAt(0).toUpperCase() + provider.slice(1);
|
|
@@ -472,14 +516,15 @@ async function searchWithAllProvider(provider, query, options) {
|
|
|
472
516
|
throw new Error("Gemini API search returned no results.");
|
|
473
517
|
}
|
|
474
518
|
async function searchWithProviders(query, options, selectedProviders) {
|
|
475
|
-
const
|
|
519
|
+
const allowed = getAllowedSearchProviders();
|
|
520
|
+
const providers = selectedProviders ?? (await Promise.all(ALL_SEARCH_PROVIDERS.filter(provider => allowed.includes(provider)).map(async (provider) => ({
|
|
476
521
|
provider,
|
|
477
522
|
available: provider === "gemini"
|
|
478
523
|
? isGeminiApiAvailable()
|
|
479
524
|
: await isResolvedProviderAvailable(provider, options),
|
|
480
525
|
})))).filter((entry) => entry.available).map((entry) => entry.provider);
|
|
481
526
|
if (providers.length === 0) {
|
|
482
|
-
throw new Error("No configured search provider available for provider \"all\". Parallel MCP, DuckDuckGo, Kimi, AnySearch, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Valyu, and XCrawl are excluded.");
|
|
527
|
+
throw new Error("No configured search provider available for provider \"all\". Parallel MCP, DuckDuckGo, Kimi, AnySearch, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Serply, Valyu, and XCrawl are excluded.");
|
|
483
528
|
}
|
|
484
529
|
const settled = await Promise.allSettled(providers.map((provider) => selectedProviders
|
|
485
530
|
? searchWithResolvedProvider(provider, query, options)
|
|
@@ -558,6 +603,7 @@ export async function search(query, options = {}) {
|
|
|
558
603
|
const provider = options.provider === undefined || options.provider === "auto"
|
|
559
604
|
? config.searchProvider
|
|
560
605
|
: options.provider;
|
|
606
|
+
assertSearchProviderSelectionAllowed(provider, "Requested provider");
|
|
561
607
|
if (Array.isArray(provider)) {
|
|
562
608
|
return searchWithProviders(query, options, normalizeResolvedProviderList(provider, "provider"));
|
|
563
609
|
}
|
|
@@ -569,7 +615,8 @@ export async function search(query, options = {}) {
|
|
|
569
615
|
return searchWithConfiguredRouting(query, options, config.searchRouting);
|
|
570
616
|
}
|
|
571
617
|
const fallbackErrors = [];
|
|
572
|
-
|
|
618
|
+
const allowed = new Set(config.allowedProviders ?? RESOLVED_SEARCH_PROVIDERS);
|
|
619
|
+
if (allowed.has("searxng") && isSearXNGAvailable()) {
|
|
573
620
|
try {
|
|
574
621
|
const result = await searchWithSearXNG(query, options);
|
|
575
622
|
return { ...result, provider: "searxng" };
|
|
@@ -581,13 +628,13 @@ export async function search(query, options = {}) {
|
|
|
581
628
|
}
|
|
582
629
|
}
|
|
583
630
|
let triedOpenAI = false;
|
|
584
|
-
if (!options.extensionContext || isOpenAICodexSelected(options.extensionContext)) {
|
|
631
|
+
if (allowed.has("openai") && (!options.extensionContext || isOpenAICodexSelected(options.extensionContext))) {
|
|
585
632
|
triedOpenAI = true;
|
|
586
633
|
const result = await tryOpenAIInAuto(query, options, fallbackErrors);
|
|
587
634
|
if (result)
|
|
588
635
|
return result;
|
|
589
636
|
}
|
|
590
|
-
if (isExaAvailable()) {
|
|
637
|
+
if (allowed.has("exa") && isExaAvailable()) {
|
|
591
638
|
try {
|
|
592
639
|
const result = await searchWithExa(query, options);
|
|
593
640
|
if (result)
|
|
@@ -599,12 +646,12 @@ export async function search(query, options = {}) {
|
|
|
599
646
|
fallbackErrors.push(`Exa: ${errorMessage(err)}`);
|
|
600
647
|
}
|
|
601
648
|
}
|
|
602
|
-
if (!triedOpenAI) {
|
|
649
|
+
if (allowed.has("openai") && !triedOpenAI) {
|
|
603
650
|
const result = await tryOpenAIInAuto(query, options, fallbackErrors);
|
|
604
651
|
if (result)
|
|
605
652
|
return result;
|
|
606
653
|
}
|
|
607
|
-
if (isBraveAvailable()) {
|
|
654
|
+
if (allowed.has("brave") && isBraveAvailable()) {
|
|
608
655
|
try {
|
|
609
656
|
const result = await searchWithBrave(query, options);
|
|
610
657
|
return { ...result, provider: "brave" };
|
|
@@ -615,7 +662,7 @@ export async function search(query, options = {}) {
|
|
|
615
662
|
fallbackErrors.push(`Brave: ${errorMessage(err)}`);
|
|
616
663
|
}
|
|
617
664
|
}
|
|
618
|
-
if (isParallelAvailable()) {
|
|
665
|
+
if (allowed.has("parallel") && isParallelAvailable()) {
|
|
619
666
|
try {
|
|
620
667
|
const result = await searchWithParallel(query, options);
|
|
621
668
|
return { ...result, provider: "parallel" };
|
|
@@ -626,7 +673,7 @@ export async function search(query, options = {}) {
|
|
|
626
673
|
fallbackErrors.push(`Parallel: ${errorMessage(err)}`);
|
|
627
674
|
}
|
|
628
675
|
}
|
|
629
|
-
if (isTinyFishAvailable()) {
|
|
676
|
+
if (allowed.has("tinyfish") && isTinyFishAvailable()) {
|
|
630
677
|
try {
|
|
631
678
|
const result = await searchWithTinyFish(query, options);
|
|
632
679
|
return { ...result, provider: "tinyfish" };
|
|
@@ -637,7 +684,7 @@ export async function search(query, options = {}) {
|
|
|
637
684
|
fallbackErrors.push(`TinyFish: ${errorMessage(err)}`);
|
|
638
685
|
}
|
|
639
686
|
}
|
|
640
|
-
if (isSearch1APIAvailable()) {
|
|
687
|
+
if (allowed.has("search1api") && isSearch1APIAvailable()) {
|
|
641
688
|
try {
|
|
642
689
|
const result = await searchWithSearch1API(query, options);
|
|
643
690
|
return { ...result, provider: "search1api" };
|
|
@@ -648,7 +695,7 @@ export async function search(query, options = {}) {
|
|
|
648
695
|
fallbackErrors.push(`Search1API: ${errorMessage(err)}`);
|
|
649
696
|
}
|
|
650
697
|
}
|
|
651
|
-
if (isSearchinfinityAvailable()) {
|
|
698
|
+
if (allowed.has("searchinfinity") && isSearchinfinityAvailable()) {
|
|
652
699
|
try {
|
|
653
700
|
const result = await searchWithSearchinfinity(query, options);
|
|
654
701
|
return { ...result, provider: "searchinfinity" };
|
|
@@ -659,7 +706,7 @@ export async function search(query, options = {}) {
|
|
|
659
706
|
fallbackErrors.push(`Searchinfinity: ${errorMessage(err)}`);
|
|
660
707
|
}
|
|
661
708
|
}
|
|
662
|
-
if (isQueritAvailable()) {
|
|
709
|
+
if (allowed.has("querit") && isQueritAvailable()) {
|
|
663
710
|
try {
|
|
664
711
|
const result = await searchWithQuerit(query, options);
|
|
665
712
|
return { ...result, provider: "querit" };
|
|
@@ -670,7 +717,7 @@ export async function search(query, options = {}) {
|
|
|
670
717
|
fallbackErrors.push(`Querit: ${errorMessage(err)}`);
|
|
671
718
|
}
|
|
672
719
|
}
|
|
673
|
-
if (isTavilyAvailable()) {
|
|
720
|
+
if (allowed.has("tavily") && isTavilyAvailable()) {
|
|
674
721
|
try {
|
|
675
722
|
const result = await searchWithTavily(query, options);
|
|
676
723
|
return { ...result, provider: "tavily" };
|
|
@@ -681,7 +728,7 @@ export async function search(query, options = {}) {
|
|
|
681
728
|
fallbackErrors.push(`Tavily: ${errorMessage(err)}`);
|
|
682
729
|
}
|
|
683
730
|
}
|
|
684
|
-
if (isFirecrawlAvailable()) {
|
|
731
|
+
if (allowed.has("firecrawl") && isFirecrawlAvailable()) {
|
|
685
732
|
try {
|
|
686
733
|
const result = await searchWithFirecrawl(query, options);
|
|
687
734
|
return { ...result, provider: "firecrawl" };
|
|
@@ -692,7 +739,7 @@ export async function search(query, options = {}) {
|
|
|
692
739
|
fallbackErrors.push(`Firecrawl: ${errorMessage(err)}`);
|
|
693
740
|
}
|
|
694
741
|
}
|
|
695
|
-
if (isJinaSearchAvailable()) {
|
|
742
|
+
if (allowed.has("jina") && isJinaSearchAvailable()) {
|
|
696
743
|
try {
|
|
697
744
|
const result = await searchWithJina(query, options);
|
|
698
745
|
return { ...result, provider: "jina" };
|
|
@@ -703,7 +750,7 @@ export async function search(query, options = {}) {
|
|
|
703
750
|
fallbackErrors.push(`Jina: ${errorMessage(err)}`);
|
|
704
751
|
}
|
|
705
752
|
}
|
|
706
|
-
if (isSerpdiveAvailable()) {
|
|
753
|
+
if (allowed.has("serpdive") && isSerpdiveAvailable()) {
|
|
707
754
|
try {
|
|
708
755
|
const result = await searchWithSerpdive(query, options);
|
|
709
756
|
return { ...result, provider: "serpdive" };
|
|
@@ -714,7 +761,7 @@ export async function search(query, options = {}) {
|
|
|
714
761
|
fallbackErrors.push(`SERPdive: ${errorMessage(err)}`);
|
|
715
762
|
}
|
|
716
763
|
}
|
|
717
|
-
if (isKagiAvailable()) {
|
|
764
|
+
if (allowed.has("kagi") && isKagiAvailable()) {
|
|
718
765
|
try {
|
|
719
766
|
const result = await searchWithKagi(query, options);
|
|
720
767
|
return { ...result, provider: "kagi" };
|
|
@@ -725,7 +772,7 @@ export async function search(query, options = {}) {
|
|
|
725
772
|
fallbackErrors.push(`Kagi: ${errorMessage(err)}`);
|
|
726
773
|
}
|
|
727
774
|
}
|
|
728
|
-
if (isBochaAvailable()) {
|
|
775
|
+
if (allowed.has("bocha") && isBochaAvailable()) {
|
|
729
776
|
try {
|
|
730
777
|
const result = await searchWithBocha(query, options);
|
|
731
778
|
return { ...result, provider: "bocha" };
|
|
@@ -736,7 +783,7 @@ export async function search(query, options = {}) {
|
|
|
736
783
|
fallbackErrors.push(`Bocha: ${errorMessage(err)}`);
|
|
737
784
|
}
|
|
738
785
|
}
|
|
739
|
-
if (isOllamaAvailable()) {
|
|
786
|
+
if (allowed.has("ollama") && isOllamaAvailable()) {
|
|
740
787
|
try {
|
|
741
788
|
const result = await searchWithOllama(query, options);
|
|
742
789
|
return { ...result, provider: "ollama" };
|
|
@@ -747,7 +794,7 @@ export async function search(query, options = {}) {
|
|
|
747
794
|
fallbackErrors.push(`Ollama: ${errorMessage(err)}`);
|
|
748
795
|
}
|
|
749
796
|
}
|
|
750
|
-
if (isPerplexityAvailable()) {
|
|
797
|
+
if (allowed.has("perplexity") && isPerplexityAvailable()) {
|
|
751
798
|
try {
|
|
752
799
|
const result = await searchWithPerplexity(query, options);
|
|
753
800
|
return { ...result, provider: "perplexity" };
|
|
@@ -758,16 +805,17 @@ export async function search(query, options = {}) {
|
|
|
758
805
|
fallbackErrors.push(`Perplexity: ${errorMessage(err)}`);
|
|
759
806
|
}
|
|
760
807
|
}
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
808
|
+
if (allowed.has("gemini"))
|
|
809
|
+
try {
|
|
810
|
+
const geminiResult = await searchWithGemini(query, options, false);
|
|
811
|
+
if (geminiResult)
|
|
812
|
+
return { ...geminiResult, provider: "gemini" };
|
|
813
|
+
}
|
|
814
|
+
catch (err) {
|
|
815
|
+
if (isAbortError(err))
|
|
816
|
+
throw err;
|
|
817
|
+
fallbackErrors.push(`Gemini: ${errorMessage(err)}`);
|
|
818
|
+
}
|
|
771
819
|
if (fallbackErrors.length > 0) {
|
|
772
820
|
throw new Error(`Auto provider search failed:\n - ${fallbackErrors.join("\n - ")}`);
|
|
773
821
|
}
|
|
@@ -777,7 +825,7 @@ export async function search(query, options = {}) {
|
|
|
777
825
|
" 3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, FIRECRAWL_BASE_URL, JINA_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, BOCHA_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
|
|
778
826
|
" 4. Set GOOGLE_GEMINI_BASE_URL with CLOUDFLARE_API_KEY for Cloudflare AI Gateway routing\n" +
|
|
779
827
|
" 5. Sign into gemini.google.com in a supported Chromium-based browser\n" +
|
|
780
|
-
" 6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xcrawl\" for XCrawl, \"xai\" for Grok, \"mistral\" for Mistral Conversations web search, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, \"serpbase\", \"serpapi\", or \"
|
|
828
|
+
" 6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xcrawl\" for XCrawl, \"xai\" for Grok, \"mistral\" for Mistral Conversations web search, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, \"serpbase\", \"serpapi\", \"serper\", or \"serply\" for Google SERP, or \"valyu\" for research search");
|
|
781
829
|
}
|
|
782
830
|
async function searchWithGeminiApi(query, options = {}) {
|
|
783
831
|
const requestSignal = AbortSignal.any([
|