@humain/terminal 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/dist/bundle/chunks/{chunk-U75GS4NL.js → chunk-CBTAWCW5.js} +1 -1
- package/dist/bundle/chunks/{chunk-D6HHRSB5.js → chunk-S43N4JYF.js} +14 -29
- package/dist/bundle/chunks/{chunk-LYOOYM65.js → chunk-WKLAQTQ4.js} +100 -124
- package/dist/bundle/chunks/{compat-22OQ5DBK.js → compat-L5WILLPV.js} +1 -1
- package/dist/bundle/chunks/{extract-CTIFQZDJ.js → extract-2YP6VMY4.js} +3 -3
- package/dist/bundle/cli.js +1 -1
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/humain/web-access/vendor/anysearch.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/anysearch.js +4 -14
- package/dist/humain/web-access/vendor/anysearch.js.map +1 -1
- package/dist/humain/web-access/vendor/bocha.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/bocha.js +3 -24
- package/dist/humain/web-access/vendor/bocha.js.map +1 -1
- package/dist/humain/web-access/vendor/brave.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/brave.js +3 -24
- package/dist/humain/web-access/vendor/brave.js.map +1 -1
- package/dist/humain/web-access/vendor/brightdata.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/brightdata.js +5 -34
- package/dist/humain/web-access/vendor/brightdata.js.map +1 -1
- package/dist/humain/web-access/vendor/content-find.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/content-find.js +186 -51
- package/dist/humain/web-access/vendor/content-find.js.map +1 -1
- package/dist/humain/web-access/vendor/curator-page.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/curator-page.js +4 -1
- package/dist/humain/web-access/vendor/curator-page.js.map +1 -1
- package/dist/humain/web-access/vendor/curator-run.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/curator-run.js +3 -1
- package/dist/humain/web-access/vendor/curator-run.js.map +1 -1
- package/dist/humain/web-access/vendor/curator-server.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/curator-server.js +2 -0
- package/dist/humain/web-access/vendor/curator-server.js.map +1 -1
- package/dist/humain/web-access/vendor/domain-filter-normalization.d.ts +2 -0
- package/dist/humain/web-access/vendor/domain-filter-normalization.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/domain-filter-normalization.js +19 -0
- package/dist/humain/web-access/vendor/domain-filter-normalization.js.map +1 -0
- package/dist/humain/web-access/vendor/duckduckgo.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/duckduckgo.js +3 -24
- package/dist/humain/web-access/vendor/duckduckgo.js.map +1 -1
- package/dist/humain/web-access/vendor/extract.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/extract.js +3 -1
- package/dist/humain/web-access/vendor/extract.js.map +1 -1
- package/dist/humain/web-access/vendor/firecrawl.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/firecrawl.js +5 -32
- package/dist/humain/web-access/vendor/firecrawl.js.map +1 -1
- package/dist/humain/web-access/vendor/gemini-search.d.ts +5 -2
- package/dist/humain/web-access/vendor/gemini-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/gemini-search.js +85 -37
- package/dist/humain/web-access/vendor/gemini-search.js.map +1 -1
- package/dist/humain/web-access/vendor/index.d.ts +3 -3
- package/dist/humain/web-access/vendor/index.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/index.js +135 -157
- package/dist/humain/web-access/vendor/index.js.map +1 -1
- package/dist/humain/web-access/vendor/jina-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/jina-search.js +4 -14
- package/dist/humain/web-access/vendor/jina-search.js.map +1 -1
- package/dist/humain/web-access/vendor/kagi.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/kagi.js +4 -12
- package/dist/humain/web-access/vendor/kagi.js.map +1 -1
- package/dist/humain/web-access/vendor/kimi-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/kimi-search.js +5 -32
- package/dist/humain/web-access/vendor/kimi-search.js.map +1 -1
- package/dist/humain/web-access/vendor/mistral-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/mistral-search.js +1 -18
- package/dist/humain/web-access/vendor/mistral-search.js.map +1 -1
- package/dist/humain/web-access/vendor/ollama.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/ollama.js +2 -6
- package/dist/humain/web-access/vendor/ollama.js.map +1 -1
- package/dist/humain/web-access/vendor/openai-search.d.ts +3 -0
- package/dist/humain/web-access/vendor/openai-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/openai-search.js +181 -38
- package/dist/humain/web-access/vendor/openai-search.js.map +1 -1
- package/dist/humain/web-access/vendor/opencode-session-headers.d.ts +10 -0
- package/dist/humain/web-access/vendor/opencode-session-headers.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/opencode-session-headers.js +23 -0
- package/dist/humain/web-access/vendor/opencode-session-headers.js.map +1 -0
- package/dist/humain/web-access/vendor/page-query.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/page-query.js +10 -2
- package/dist/humain/web-access/vendor/page-query.js.map +1 -1
- package/dist/humain/web-access/vendor/parallel.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/parallel.js +1 -18
- package/dist/humain/web-access/vendor/parallel.js.map +1 -1
- package/dist/humain/web-access/vendor/pdf-extract.d.ts +2 -0
- package/dist/humain/web-access/vendor/pdf-extract.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/pdf-extract.js +1 -0
- package/dist/humain/web-access/vendor/pdf-extract.js.map +1 -1
- package/dist/humain/web-access/vendor/querit.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/querit.js +5 -32
- package/dist/humain/web-access/vendor/querit.js.map +1 -1
- package/dist/humain/web-access/vendor/search-answer-formatting.d.ts +8 -0
- package/dist/humain/web-access/vendor/search-answer-formatting.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/search-answer-formatting.js +6 -0
- package/dist/humain/web-access/vendor/search-answer-formatting.js.map +1 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.d.ts +2 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.js +6 -0
- package/dist/humain/web-access/vendor/search-result-count-normalization.js.map +1 -0
- package/dist/humain/web-access/vendor/search1api.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/search1api.js +5 -32
- package/dist/humain/web-access/vendor/search1api.js.map +1 -1
- package/dist/humain/web-access/vendor/searchinfinity.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/searchinfinity.js +5 -32
- package/dist/humain/web-access/vendor/searchinfinity.js.map +1 -1
- package/dist/humain/web-access/vendor/searxng.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/searxng.js +3 -24
- package/dist/humain/web-access/vendor/searxng.js.map +1 -1
- package/dist/humain/web-access/vendor/serpapi.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serpapi.js +5 -30
- package/dist/humain/web-access/vendor/serpapi.js.map +1 -1
- package/dist/humain/web-access/vendor/serpbase.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serpbase.js +3 -24
- package/dist/humain/web-access/vendor/serpbase.js.map +1 -1
- package/dist/humain/web-access/vendor/serpdive.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serpdive.js +3 -24
- package/dist/humain/web-access/vendor/serpdive.js.map +1 -1
- package/dist/humain/web-access/vendor/serper.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/serper.js +5 -30
- package/dist/humain/web-access/vendor/serper.js.map +1 -1
- package/dist/humain/web-access/vendor/serply.d.ts +4 -0
- package/dist/humain/web-access/vendor/serply.d.ts.map +1 -0
- package/dist/humain/web-access/vendor/serply.js +191 -0
- package/dist/humain/web-access/vendor/serply.js.map +1 -0
- package/dist/humain/web-access/vendor/source-check.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/source-check.js +11 -47
- package/dist/humain/web-access/vendor/source-check.js.map +1 -1
- package/dist/humain/web-access/vendor/summary-review.d.ts +1 -1
- package/dist/humain/web-access/vendor/summary-review.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/summary-review.js +6 -2
- package/dist/humain/web-access/vendor/summary-review.js.map +1 -1
- package/dist/humain/web-access/vendor/tavily.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/tavily.js +3 -24
- package/dist/humain/web-access/vendor/tavily.js.map +1 -1
- package/dist/humain/web-access/vendor/tinyfish.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/tinyfish.js +5 -32
- package/dist/humain/web-access/vendor/tinyfish.js.map +1 -1
- package/dist/humain/web-access/vendor/utils.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/utils.js +2 -0
- package/dist/humain/web-access/vendor/utils.js.map +1 -1
- package/dist/humain/web-access/vendor/valyu.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/valyu.js +5 -30
- package/dist/humain/web-access/vendor/valyu.js.map +1 -1
- package/dist/humain/web-access/vendor/xai-search.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/xai-search.js +1 -18
- package/dist/humain/web-access/vendor/xai-search.js.map +1 -1
- package/dist/humain/web-access/vendor/xcrawl.d.ts.map +1 -1
- package/dist/humain/web-access/vendor/xcrawl.js +5 -34
- package/dist/humain/web-access/vendor/xcrawl.js.map +1 -1
- package/npm-shrinkwrap.json +6 -6
- package/package.json +2 -2
- package/src/humain/web-access/UPSTREAM.md +33 -1
- package/src/humain/web-access/upstream.json +6 -5
|
@@ -15,8 +15,8 @@ import { findContent } from "./content-find.js";
|
|
|
15
15
|
import { answerFromPage } from "./page-query.js";
|
|
16
16
|
import { rewriteSearchQuery } from "./query-rewrite.js";
|
|
17
17
|
import { clearCloneCache } from "./github-extract.js";
|
|
18
|
-
import { ALL_SEARCH_PROVIDERS, getConfiguredSearchRouting, normalizeSearchProviderSelection,
|
|
19
|
-
import { formatSeconds, getWebSearchConfigDir, getWebSearchConfigPath,
|
|
18
|
+
import { ALL_SEARCH_PROVIDERS, assertSearchProviderSelectionAllowed, getAllowedSearchProviders, getConfiguredSearchRouting, normalizeSearchProviderSelection, providerLabel, RESOLVED_SEARCH_PROVIDERS, search } from "./gemini-search.js";
|
|
19
|
+
import { formatSeconds, getWebSearchConfigDir, getWebSearchConfigPath, resolveCuratorNetworkConfig, runWithProxy } from "./utils.js";
|
|
20
20
|
import { clearResults, deleteResult, generateId, getAllResults, getResult, restoreFromSession, storeFetchedContentResult, storeResult, } from "./storage.js";
|
|
21
21
|
import { activityMonitor } from "./activity.js";
|
|
22
22
|
import { startCuratorServer } from "./curator-server.js";
|
|
@@ -57,6 +57,7 @@ import { isBrightDataAvailable } from "./brightdata.js";
|
|
|
57
57
|
import { isSerpBaseAvailable } from "./serpbase.js";
|
|
58
58
|
import { isSerpApiAvailable } from "./serpapi.js";
|
|
59
59
|
import { isSerperAvailable } from "./serper.js";
|
|
60
|
+
import { isSerplyAvailable } from "./serply.js";
|
|
60
61
|
import { isValyuAvailable } from "./valyu.js";
|
|
61
62
|
import { isXcrawlAvailable } from "./xcrawl.js";
|
|
62
63
|
import { buildSearchErrorPlan } from "./render-search-error.js";
|
|
@@ -148,6 +149,28 @@ const DEFAULT_REMOTE_CURATOR_TIMEOUT_SECONDS = 60;
|
|
|
148
149
|
const MAX_CURATOR_TIMEOUT_SECONDS = 600;
|
|
149
150
|
const MAX_SUMMARY_GENERATION_DEADLINE_MS = 600_000;
|
|
150
151
|
const SEARCH_QUERY_CONCURRENCY = 3;
|
|
152
|
+
const FETCH_MODES = ["readable", "raw", "answer"];
|
|
153
|
+
const FETCH_MODE_DESCRIPTIONS = {
|
|
154
|
+
readable: "extract readable content as markdown",
|
|
155
|
+
raw: "return the exact textual body using direct HTTP only",
|
|
156
|
+
answer: "answer a prompt using only fetched content",
|
|
157
|
+
};
|
|
158
|
+
function resolveFetchModeConfig(config) {
|
|
159
|
+
const configuredModes = config.fetch?.allowedModes ?? FETCH_MODES;
|
|
160
|
+
if (!Array.isArray(configuredModes) || configuredModes.length === 0 || configuredModes.some(mode => !FETCH_MODES.includes(mode))) {
|
|
161
|
+
throw new Error(`fetch.allowedModes in ${WEB_SEARCH_CONFIG_PATH} must be a non-empty array containing only "readable", "raw", or "answer"`);
|
|
162
|
+
}
|
|
163
|
+
const allowedModes = configuredModes;
|
|
164
|
+
const duplicateMode = allowedModes.find((mode, index) => allowedModes.indexOf(mode) !== index);
|
|
165
|
+
if (duplicateMode) {
|
|
166
|
+
throw new Error(`fetch.allowedModes in ${WEB_SEARCH_CONFIG_PATH} must not contain duplicates: "${duplicateMode}"`);
|
|
167
|
+
}
|
|
168
|
+
const defaultMode = config.fetch?.defaultMode ?? "readable";
|
|
169
|
+
if (!allowedModes.includes(defaultMode)) {
|
|
170
|
+
throw new Error(`fetch.defaultMode in ${WEB_SEARCH_CONFIG_PATH} must be one of fetch.allowedModes`);
|
|
171
|
+
}
|
|
172
|
+
return { defaultMode: defaultMode, allowedModes };
|
|
173
|
+
}
|
|
151
174
|
// Limit each batch independently so separate Pi tool calls can still run in parallel.
|
|
152
175
|
function runSearchQueries(queries, run) {
|
|
153
176
|
const limit = pLimit(SEARCH_QUERY_CONCURRENCY);
|
|
@@ -161,10 +184,10 @@ function curatorResultIndex(queryIndex, entryIndex, queryCount) {
|
|
|
161
184
|
function curatorResultIndexCapacity(queryCount) {
|
|
162
185
|
return queryCount * RESOLVED_SEARCH_PROVIDERS.length;
|
|
163
186
|
}
|
|
164
|
-
function searchProviderSchema(description) {
|
|
187
|
+
function searchProviderSchema(description, allowedProviders) {
|
|
165
188
|
return Type.Union([
|
|
166
|
-
StringEnum([...
|
|
167
|
-
Type.Array(StringEnum([...
|
|
189
|
+
StringEnum(["auto", "all", ...allowedProviders]),
|
|
190
|
+
Type.Array(StringEnum([...allowedProviders]), { minItems: 1 }),
|
|
168
191
|
], { description });
|
|
169
192
|
}
|
|
170
193
|
function isToolEnabled(config, key) {
|
|
@@ -231,10 +254,14 @@ function normalizeProviderInput(value, label = "provider") {
|
|
|
231
254
|
}
|
|
232
255
|
function resolveRequestedProvider(requested) {
|
|
233
256
|
const normalizedRequested = normalizeProviderInput(requested);
|
|
234
|
-
if (normalizedRequested && normalizedRequested !== "auto")
|
|
257
|
+
if (normalizedRequested && normalizedRequested !== "auto") {
|
|
258
|
+
assertSearchProviderSelectionAllowed(normalizedRequested, "Requested provider");
|
|
235
259
|
return normalizedRequested;
|
|
260
|
+
}
|
|
236
261
|
const config = loadConfig();
|
|
237
|
-
|
|
262
|
+
const provider = normalizeProviderInput(config.searchProvider ?? config.provider, `provider in ${WEB_SEARCH_CONFIG_PATH}`) ?? "auto";
|
|
263
|
+
assertSearchProviderSelectionAllowed(provider, `configured provider in ${WEB_SEARCH_CONFIG_PATH}`);
|
|
264
|
+
return provider;
|
|
238
265
|
}
|
|
239
266
|
function toCuratorProvider(provider) {
|
|
240
267
|
if (Array.isArray(provider))
|
|
@@ -245,9 +272,7 @@ function resolveCuratorSearchProvider(requested, current) {
|
|
|
245
272
|
const normalized = normalizeProviderInput(requested);
|
|
246
273
|
if (!normalized || normalized === "auto")
|
|
247
274
|
return current;
|
|
248
|
-
|
|
249
|
-
return current;
|
|
250
|
-
return normalized;
|
|
275
|
+
return normalized === "all" && Array.isArray(current) ? current : normalized;
|
|
251
276
|
}
|
|
252
277
|
function normalizeRecencyFilter(value) {
|
|
253
278
|
return value === "day" || value === "week" || value === "month" || value === "year"
|
|
@@ -324,43 +349,44 @@ function shouldAutoOpenCuratorBrowser(config) {
|
|
|
324
349
|
return true;
|
|
325
350
|
}
|
|
326
351
|
async function getProviderAvailability(ctx) {
|
|
327
|
-
const
|
|
328
|
-
const
|
|
352
|
+
const allowedProviders = new Set(getAllowedSearchProviders());
|
|
353
|
+
const geminiWebAvail = allowedProviders.has("gemini") ? await getOptionalGeminiWebAvailability() : null;
|
|
354
|
+
const geminiApiAvail = allowedProviders.has("gemini") && isGeminiApiAvailable();
|
|
329
355
|
const providers = {
|
|
330
|
-
openai: await isOpenAISearchAvailable(ctx),
|
|
331
|
-
brave: isBraveAvailable(),
|
|
332
|
-
parallel: isParallelAvailable(),
|
|
333
|
-
"parallel-mcp": isParallelMcpAvailable(),
|
|
334
|
-
tinyfish: isTinyFishAvailable(),
|
|
335
|
-
search1api: isSearch1APIAvailable(),
|
|
336
|
-
searchinfinity: isSearchinfinityAvailable(),
|
|
337
|
-
querit: isQueritAvailable(),
|
|
338
|
-
tavily: isTavilyAvailable(),
|
|
339
|
-
firecrawl: isFirecrawlAvailable(),
|
|
340
|
-
jina: isJinaSearchAvailable(),
|
|
341
|
-
serpdive: isSerpdiveAvailable(),
|
|
342
|
-
kagi: isKagiAvailable(),
|
|
343
|
-
bocha: isBochaAvailable(),
|
|
344
|
-
ollama: isOllamaAvailable(),
|
|
345
|
-
searxng: isSearXNGAvailable(),
|
|
346
|
-
duckduckgo: isDuckDuckGoAvailable(),
|
|
347
|
-
perplexity: isPerplexityAvailable(),
|
|
348
|
-
exa: isExaAvailable(),
|
|
356
|
+
openai: allowedProviders.has("openai") && await isOpenAISearchAvailable(ctx),
|
|
357
|
+
brave: allowedProviders.has("brave") && isBraveAvailable(),
|
|
358
|
+
parallel: allowedProviders.has("parallel") && isParallelAvailable(),
|
|
359
|
+
"parallel-mcp": allowedProviders.has("parallel-mcp") && isParallelMcpAvailable(),
|
|
360
|
+
tinyfish: allowedProviders.has("tinyfish") && isTinyFishAvailable(),
|
|
361
|
+
search1api: allowedProviders.has("search1api") && isSearch1APIAvailable(),
|
|
362
|
+
searchinfinity: allowedProviders.has("searchinfinity") && isSearchinfinityAvailable(),
|
|
363
|
+
querit: allowedProviders.has("querit") && isQueritAvailable(),
|
|
364
|
+
tavily: allowedProviders.has("tavily") && isTavilyAvailable(),
|
|
365
|
+
firecrawl: allowedProviders.has("firecrawl") && isFirecrawlAvailable(),
|
|
366
|
+
jina: allowedProviders.has("jina") && isJinaSearchAvailable(),
|
|
367
|
+
serpdive: allowedProviders.has("serpdive") && isSerpdiveAvailable(),
|
|
368
|
+
kagi: allowedProviders.has("kagi") && isKagiAvailable(),
|
|
369
|
+
bocha: allowedProviders.has("bocha") && isBochaAvailable(),
|
|
370
|
+
ollama: allowedProviders.has("ollama") && isOllamaAvailable(),
|
|
371
|
+
searxng: allowedProviders.has("searxng") && isSearXNGAvailable(),
|
|
372
|
+
duckduckgo: allowedProviders.has("duckduckgo") && isDuckDuckGoAvailable(),
|
|
373
|
+
perplexity: allowedProviders.has("perplexity") && isPerplexityAvailable(),
|
|
374
|
+
exa: allowedProviders.has("exa") && isExaAvailable(),
|
|
349
375
|
gemini: geminiApiAvail || !!geminiWebAvail,
|
|
350
|
-
kimi: await isKimiSearchAvailable(ctx),
|
|
351
|
-
anysearch: isAnySearchAvailable(),
|
|
352
|
-
xcrawl: isXcrawlAvailable(),
|
|
353
|
-
xai: await isXaiSearchAvailable(ctx),
|
|
354
|
-
mistral: isMistralAvailable(),
|
|
355
|
-
brightdata: isBrightDataAvailable(),
|
|
356
|
-
serpbase: isSerpBaseAvailable(),
|
|
357
|
-
serpapi: isSerpApiAvailable(),
|
|
358
|
-
serper: isSerperAvailable(),
|
|
359
|
-
|
|
376
|
+
kimi: allowedProviders.has("kimi") && await isKimiSearchAvailable(ctx),
|
|
377
|
+
anysearch: allowedProviders.has("anysearch") && isAnySearchAvailable(),
|
|
378
|
+
xcrawl: allowedProviders.has("xcrawl") && isXcrawlAvailable(),
|
|
379
|
+
xai: allowedProviders.has("xai") && await isXaiSearchAvailable(ctx),
|
|
380
|
+
mistral: allowedProviders.has("mistral") && isMistralAvailable(),
|
|
381
|
+
brightdata: allowedProviders.has("brightdata") && isBrightDataAvailable(),
|
|
382
|
+
serpbase: allowedProviders.has("serpbase") && isSerpBaseAvailable(),
|
|
383
|
+
serpapi: allowedProviders.has("serpapi") && isSerpApiAvailable(),
|
|
384
|
+
serper: allowedProviders.has("serper") && isSerperAvailable(),
|
|
385
|
+
serply: allowedProviders.has("serply") && isSerplyAvailable(),
|
|
386
|
+
valyu: allowedProviders.has("valyu") && isValyuAvailable(),
|
|
360
387
|
};
|
|
361
|
-
const allSearchProviders = new Set(ALL_SEARCH_PROVIDERS);
|
|
362
388
|
return {
|
|
363
|
-
all:
|
|
389
|
+
all: ALL_SEARCH_PROVIDERS.some(provider => provider === "gemini" ? geminiApiAvail : providers[provider]),
|
|
364
390
|
...providers,
|
|
365
391
|
};
|
|
366
392
|
}
|
|
@@ -404,39 +430,14 @@ function firstAvailableProvider(available, preferOpenAI, fallback) {
|
|
|
404
430
|
return "openai";
|
|
405
431
|
if (available.exa)
|
|
406
432
|
return "exa";
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
if (available.search1api)
|
|
416
|
-
return "search1api";
|
|
417
|
-
if (available.searchinfinity)
|
|
418
|
-
return "searchinfinity";
|
|
419
|
-
if (available.querit)
|
|
420
|
-
return "querit";
|
|
421
|
-
if (available.tavily)
|
|
422
|
-
return "tavily";
|
|
423
|
-
if (available.firecrawl)
|
|
424
|
-
return "firecrawl";
|
|
425
|
-
if (available.jina)
|
|
426
|
-
return "jina";
|
|
427
|
-
if (available.serpdive)
|
|
428
|
-
return "serpdive";
|
|
429
|
-
if (available.kagi)
|
|
430
|
-
return "kagi";
|
|
431
|
-
if (available.bocha)
|
|
432
|
-
return "bocha";
|
|
433
|
-
if (available.ollama)
|
|
434
|
-
return "ollama";
|
|
435
|
-
if (available.perplexity)
|
|
436
|
-
return "perplexity";
|
|
437
|
-
if (available.gemini)
|
|
438
|
-
return "gemini";
|
|
439
|
-
return fallback;
|
|
433
|
+
for (const provider of ALL_SEARCH_PROVIDERS) {
|
|
434
|
+
if (provider === "ollama" && available.bocha)
|
|
435
|
+
return "bocha";
|
|
436
|
+
if (available[provider])
|
|
437
|
+
return provider;
|
|
438
|
+
}
|
|
439
|
+
const allowed = getAllowedSearchProviders();
|
|
440
|
+
return allowed.includes(fallback) ? fallback : "auto";
|
|
440
441
|
}
|
|
441
442
|
function resolveProvider(provider, available, options, preferOpenAICodexDefault = false, ctx) {
|
|
442
443
|
if (Array.isArray(provider))
|
|
@@ -455,62 +456,11 @@ function resolveProvider(provider, available, options, preferOpenAICodexDefault
|
|
|
455
456
|
}
|
|
456
457
|
return firstAvailableProvider(available, preferOpenAI, "exa");
|
|
457
458
|
}
|
|
458
|
-
if (provider === "all"
|
|
459
|
-
return firstAvailableProvider(available, preferOpenAI, "exa");
|
|
460
|
-
}
|
|
461
|
-
if (provider === "openai" && !available.openai) {
|
|
462
|
-
return firstAvailableProvider(available, false, "openai");
|
|
463
|
-
}
|
|
464
|
-
if (provider === "brave" && !available.brave) {
|
|
465
|
-
return firstAvailableProvider(available, preferOpenAI, "brave");
|
|
466
|
-
}
|
|
467
|
-
if (provider === "parallel" && !available.parallel) {
|
|
468
|
-
return firstAvailableProvider(available, preferOpenAI, "parallel");
|
|
469
|
-
}
|
|
470
|
-
if (provider === "tinyfish" && !available.tinyfish) {
|
|
471
|
-
return firstAvailableProvider(available, preferOpenAI, "tinyfish");
|
|
472
|
-
}
|
|
473
|
-
if (provider === "search1api" && !available.search1api) {
|
|
474
|
-
return firstAvailableProvider(available, preferOpenAI, "search1api");
|
|
475
|
-
}
|
|
476
|
-
if (provider === "searchinfinity" && !available.searchinfinity) {
|
|
477
|
-
return firstAvailableProvider(available, preferOpenAI, "searchinfinity");
|
|
478
|
-
}
|
|
479
|
-
if (provider === "querit" && !available.querit) {
|
|
480
|
-
return firstAvailableProvider(available, preferOpenAI, "querit");
|
|
481
|
-
}
|
|
482
|
-
if (provider === "tavily" && !available.tavily) {
|
|
483
|
-
return firstAvailableProvider(available, preferOpenAI, "tavily");
|
|
484
|
-
}
|
|
485
|
-
if (provider === "firecrawl" && !available.firecrawl) {
|
|
486
|
-
return firstAvailableProvider(available, preferOpenAI, "firecrawl");
|
|
487
|
-
}
|
|
488
|
-
if (provider === "jina" && !available.jina) {
|
|
489
|
-
return firstAvailableProvider(available, preferOpenAI, "jina");
|
|
490
|
-
}
|
|
491
|
-
if (provider === "serpdive" && !available.serpdive) {
|
|
492
|
-
return firstAvailableProvider(available, preferOpenAI, "serpdive");
|
|
493
|
-
}
|
|
494
|
-
if (provider === "kagi" && !available.kagi) {
|
|
495
|
-
return firstAvailableProvider(available, preferOpenAI, "kagi");
|
|
496
|
-
}
|
|
497
|
-
if (provider === "bocha" && !available.bocha) {
|
|
498
|
-
return firstAvailableProvider(available, preferOpenAI, "bocha");
|
|
499
|
-
}
|
|
500
|
-
if (provider === "ollama" && !available.ollama) {
|
|
501
|
-
return firstAvailableProvider(available, preferOpenAI, "ollama");
|
|
502
|
-
}
|
|
503
|
-
if (provider === "searxng" && !available.searxng) {
|
|
504
|
-
return firstAvailableProvider(available, preferOpenAI, "searxng");
|
|
505
|
-
}
|
|
506
|
-
if (provider === "exa" && !available.exa) {
|
|
507
|
-
return firstAvailableProvider(available, preferOpenAI, "exa");
|
|
508
|
-
}
|
|
509
|
-
if (provider === "perplexity" && !available.perplexity) {
|
|
510
|
-
return firstAvailableProvider(available, preferOpenAI, "perplexity");
|
|
459
|
+
if (provider === "all") {
|
|
460
|
+
return available.all ? "all" : firstAvailableProvider(available, preferOpenAI, "exa");
|
|
511
461
|
}
|
|
512
|
-
if (provider
|
|
513
|
-
return firstAvailableProvider(available, preferOpenAI,
|
|
462
|
+
if (ALL_SEARCH_PROVIDERS.includes(provider) && !available[provider]) {
|
|
463
|
+
return firstAvailableProvider(available, provider === "openai" ? false : preferOpenAI, provider);
|
|
514
464
|
}
|
|
515
465
|
return provider;
|
|
516
466
|
}
|
|
@@ -905,8 +855,16 @@ function handleSessionChange(ctx) {
|
|
|
905
855
|
}
|
|
906
856
|
export default function (pi) {
|
|
907
857
|
const initConfig = loadConfigForExtensionInit();
|
|
858
|
+
const fetchModeConfig = resolveFetchModeConfig(initConfig);
|
|
859
|
+
const allowedSearchProviders = initConfig.webSearch?.allowedProviders === undefined ? RESOLVED_SEARCH_PROVIDERS : getAllowedSearchProviders();
|
|
860
|
+
const allEligibleProviders = allowedSearchProviders.filter(provider => ALL_SEARCH_PROVIDERS.includes(provider));
|
|
861
|
+
const allExcludedProviders = allowedSearchProviders.filter(provider => !ALL_SEARCH_PROVIDERS.includes(provider));
|
|
862
|
+
const allPolicyDescription = allEligibleProviders.length === 0
|
|
863
|
+
? `all has no eligible allowed providers; explicit-only allowed providers (${allExcludedProviders.map(providerLabel).join(", ")}) remain excluded`
|
|
864
|
+
: allExcludedProviders.length > 0
|
|
865
|
+
? `all searches eligible allowed providers (${allEligibleProviders.map(providerLabel).join(", ")}); explicit-only allowed providers (${allExcludedProviders.map(providerLabel).join(", ")}) remain excluded`
|
|
866
|
+
: `all searches every eligible allowed provider (${allEligibleProviders.map(providerLabel).join(", ")})`;
|
|
908
867
|
const curatorRunState = registerCuratorRunLifecycle(pi);
|
|
909
|
-
installGlobalProxyFetch();
|
|
910
868
|
const toolNames = resolveToolNames(initConfig);
|
|
911
869
|
const webSearchEnabled = isToolEnabled(initConfig, "webSearch");
|
|
912
870
|
const sourceCheckEnabled = isToolEnabled(initConfig, "sourceCheck");
|
|
@@ -929,6 +887,9 @@ export default function (pi) {
|
|
|
929
887
|
const fetchContentStorageNote = getSearchContentEnabled
|
|
930
888
|
? `Full original content is stored for retrieval with ${toolNames.getSearchContent}.`
|
|
931
889
|
: "Full original content is stored internally, but the retrieval tool is not registered.";
|
|
890
|
+
const fetchModeDescription = fetchModeConfig.allowedModes
|
|
891
|
+
.map(mode => `${mode}${mode === fetchModeConfig.defaultMode ? " (default)" : ""}: ${FETCH_MODE_DESCRIPTIONS[mode]}`)
|
|
892
|
+
.join("; ");
|
|
932
893
|
const curateKey = initConfig.shortcuts?.curate || DEFAULT_SHORTCUTS.curate;
|
|
933
894
|
const activityKey = initConfig.shortcuts?.activity || DEFAULT_SHORTCUTS.activity;
|
|
934
895
|
function startBackgroundFetch(urls, proxy) {
|
|
@@ -1573,18 +1534,18 @@ export default function (pi) {
|
|
|
1573
1534
|
pi.registerTool({
|
|
1574
1535
|
name: toolNames.webSearch,
|
|
1575
1536
|
label: "Web Search",
|
|
1576
|
-
description: `Search the web
|
|
1537
|
+
description: `Search the web with ${allowedSearchProviders.map(providerLabel).join(", ")}. Provider arrays run simultaneously; ${allPolicyDescription}. By default, returns source-linked search results or provider answers. For comprehensive research, prefer queries (plural) with 2-4 varied angles over a single query. When includeContent is true, full page content is fetched in the background. Searches return without the interactive browser curator by default; set workflow to "summary-review" to open the curator with an auto-generated summary draft or "auto-summary" to generate a summary without the browser curator. The configured provider is used when provider is omitted or set to auto; omit provider unless explicitly overriding it.`,
|
|
1577
1538
|
promptSnippet: "Use for web research questions. Prefer {queries:[...]} with 2-4 varied angles over a single query for broader coverage. Omit provider unless explicitly overriding the configured default.",
|
|
1578
1539
|
parameters: Type.Object({
|
|
1579
1540
|
query: Type.Optional(Type.String({ description: "Single search query. For research tasks, prefer 'queries' with multiple varied angles instead." })),
|
|
1580
|
-
queries: Type.Optional(Type.Array(Type.String(), { description: "Multiple queries searched concurrently (up to three at a time), each returning
|
|
1541
|
+
queries: Type.Optional(Type.Array(Type.String(), { description: "Multiple queries searched concurrently (up to three at a time), each returning source-linked search results or a provider answer. Prefer this for research — vary phrasing, scope, and angle across 2-4 queries to maximize coverage. Good: ['React vs Vue performance benchmarks 2026', 'React vs Vue developer experience comparison', 'React ecosystem size vs Vue ecosystem']. Bad: ['React vs Vue', 'React vs Vue comparison', 'React vs Vue review'] (too similar, redundant results)." })),
|
|
1581
1542
|
numResults: Type.Optional(Type.Integer({ minimum: 1, maximum: 20, description: "Results per query (default: 5, max: 20)" })),
|
|
1582
1543
|
includeContent: Type.Optional(Type.Boolean({ description: "Fetch full page content (async)" })),
|
|
1583
1544
|
recencyFilter: Type.Optional(StringEnum(["day", "week", "month", "year"], { description: "Filter by recency" })),
|
|
1584
1545
|
domainFilter: Type.Optional(Type.Array(Type.String(), { description: "Limit to domains (prefix with - to exclude)" })),
|
|
1585
|
-
provider: Type.Optional(searchProviderSchema(
|
|
1546
|
+
provider: Type.Optional(searchProviderSchema(`Search provider or non-empty list of allowed providers to search simultaneously; ${allPolicyDescription}; omit this field to use the configured provider, or use auto when none is configured`, allowedSearchProviders)),
|
|
1586
1547
|
workflow: Type.Optional(StringEnum(["none", "summary-review", "auto-summary"], {
|
|
1587
|
-
description: "Search workflow mode: none = no curator, summary-review = open curator with auto summary draft
|
|
1548
|
+
description: "Search workflow mode: none = no curator (default), summary-review = open curator with auto summary draft, auto-summary = generate summary without opening curator",
|
|
1588
1549
|
})),
|
|
1589
1550
|
proxy: Type.Optional(Type.String({
|
|
1590
1551
|
description: "http(s) or socks proxy URL (e.g. http://host:port or socks5h://host:port) used for every outbound request in this call (search APIs and content fetches). Node fetch ignores HTTP(S)_PROXY env vars, so set this (or `proxy` in web-search.json) when direct access is blocked; empty string forces direct access.",
|
|
@@ -1640,6 +1601,7 @@ export default function (pi) {
|
|
|
1640
1601
|
const summaryContext = {
|
|
1641
1602
|
model: ctx.model,
|
|
1642
1603
|
modelRegistry: ctx.modelRegistry,
|
|
1604
|
+
sessionManager: ctx.sessionManager,
|
|
1643
1605
|
cwd: ctx.cwd,
|
|
1644
1606
|
isProjectTrusted: () => ctx.isProjectTrusted(),
|
|
1645
1607
|
};
|
|
@@ -1870,6 +1832,7 @@ export default function (pi) {
|
|
|
1870
1832
|
const summaryContext = {
|
|
1871
1833
|
model: ctx.model,
|
|
1872
1834
|
modelRegistry: ctx.modelRegistry,
|
|
1835
|
+
sessionManager: ctx.sessionManager,
|
|
1873
1836
|
cwd: ctx.cwd,
|
|
1874
1837
|
isProjectTrusted: () => ctx.isProjectTrusted(),
|
|
1875
1838
|
};
|
|
@@ -2106,16 +2069,16 @@ export default function (pi) {
|
|
|
2106
2069
|
pi.registerTool({
|
|
2107
2070
|
name: toolNames.sourceCheck,
|
|
2108
2071
|
label: "Source Check",
|
|
2109
|
-
description: "
|
|
2110
|
-
promptSnippet: "
|
|
2072
|
+
description: "Gather web sources for a claim and return a bounded machine-readable research artifact with exact passage citations for manual review.",
|
|
2073
|
+
promptSnippet: "Gather structured source evidence and passage-level citations for manual semantic review of a claim.",
|
|
2111
2074
|
parameters: Type.Object({
|
|
2112
|
-
claim: Type.String({ description: "The assertion to
|
|
2075
|
+
claim: Type.String({ description: "The assertion to gather web sources for." }),
|
|
2113
2076
|
queries: Type.Optional(Type.Array(Type.String(), { description: "Search queries (default: the claim)." })),
|
|
2114
2077
|
numResults: Type.Optional(Type.Integer({ minimum: 1, maximum: 20, description: "Results per query (default: 5, max: 20)." })),
|
|
2115
2078
|
fetchContent: Type.Optional(Type.Boolean({ description: "Fetch up to 5 result pages for exact passage extraction." })),
|
|
2116
2079
|
recencyFilter: Type.Optional(StringEnum(["day", "week", "month", "year"], { description: "Filter by recency." })),
|
|
2117
2080
|
domainFilter: Type.Optional(Type.Array(Type.String(), { description: "Limit to domains; prefix with - to exclude." })),
|
|
2118
|
-
provider: Type.Optional(searchProviderSchema(
|
|
2081
|
+
provider: Type.Optional(searchProviderSchema(`Search provider or non-empty list of allowed providers to search simultaneously; ${allPolicyDescription}`, allowedSearchProviders)),
|
|
2119
2082
|
proxy: Type.Optional(Type.String({
|
|
2120
2083
|
description: "http(s) or socks proxy URL (e.g. http://host:port or socks5h://host:port) used for every outbound request in this call (search APIs and result-page fetches). Empty string forces direct access.",
|
|
2121
2084
|
})),
|
|
@@ -2211,8 +2174,8 @@ export default function (pi) {
|
|
|
2211
2174
|
pi.registerTool({
|
|
2212
2175
|
name: toolNames.fetchContent,
|
|
2213
2176
|
label: "Fetch Content",
|
|
2214
|
-
description: `Fetch URL(s)
|
|
2215
|
-
promptSnippet: "Use to fetch
|
|
2177
|
+
description: `Fetch URL(s). Available modes: ${fetchModeDescription}. Direct image URLs return resized image content when supported by the selected mode. Supports YouTube transcripts, GitHub repositories, PDFs, and local videos when supported by the selected mode. ${fetchContentStorageNote}`,
|
|
2178
|
+
promptSnippet: "Use to fetch URL content, direct images, GitHub repos, and videos.",
|
|
2216
2179
|
parameters: Type.Object({
|
|
2217
2180
|
url: Type.Optional(Type.String({ description: "Single URL to fetch" })),
|
|
2218
2181
|
urls: Type.Optional(Type.Array(Type.String(), { description: "Multiple URLs (parallel)" })),
|
|
@@ -2220,14 +2183,18 @@ export default function (pi) {
|
|
|
2220
2183
|
description: "Force cloning large GitHub repositories that exceed the size threshold",
|
|
2221
2184
|
})),
|
|
2222
2185
|
prompt: Type.Optional(Type.String({
|
|
2223
|
-
description: "
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
description: "Fetch mode: readable (default extraction), raw (exact textual HTTP body), or answer (answer prompt using only fetched content).",
|
|
2186
|
+
description: fetchModeConfig.allowedModes.includes("answer")
|
|
2187
|
+
? "Question or instruction for video analysis, or the page-local question required by answer mode."
|
|
2188
|
+
: "Question or instruction for video analysis.",
|
|
2227
2189
|
})),
|
|
2228
|
-
|
|
2229
|
-
description:
|
|
2190
|
+
mode: Type.Optional(StringEnum(fetchModeConfig.allowedModes, {
|
|
2191
|
+
description: `Fetch mode. ${fetchModeDescription}.`,
|
|
2230
2192
|
})),
|
|
2193
|
+
...(fetchModeConfig.allowedModes.includes("answer") ? {
|
|
2194
|
+
answerModel: Type.Optional(Type.String({
|
|
2195
|
+
description: "Optional provider/model-id override for answer mode. Defaults to fetch.answerProvider + fetch.answerModel when configured, otherwise the current Pi model.",
|
|
2196
|
+
})),
|
|
2197
|
+
} : {}),
|
|
2231
2198
|
timestamp: Type.Optional(Type.String({
|
|
2232
2199
|
description: "Extract video frame(s) at a timestamp or time range. Single: '1:23:45', '23:45', or '85' (seconds). Range: '23:41-25:00' extracts evenly-spaced frames across that span (default 6). Use frames with ranges to control density; single+frames uses a fixed 5s interval. YouTube requires yt-dlp + ffmpeg; local videos require ffmpeg. Use a range when you know the approximate area but not the exact moment — you'll get a contact sheet to visually identify the right frame.",
|
|
2233
2200
|
})),
|
|
@@ -2256,8 +2223,12 @@ export default function (pi) {
|
|
|
2256
2223
|
return { content: [{ type: "text", text: `Error: ${error}` }], details: { error } };
|
|
2257
2224
|
}
|
|
2258
2225
|
const { urlList, options } = normalized;
|
|
2226
|
+
const mode = options.mode ?? fetchModeConfig.defaultMode;
|
|
2227
|
+
if (!fetchModeConfig.allowedModes.includes(mode)) {
|
|
2228
|
+
const error = `Fetch mode "${mode}" is disabled by fetch.allowedModes.`;
|
|
2229
|
+
return { content: [{ type: "text", text: `Error: ${error}` }], details: { error } };
|
|
2230
|
+
}
|
|
2259
2231
|
return runWithProxy(options.proxy, async () => {
|
|
2260
|
-
const mode = options.mode ?? "readable";
|
|
2261
2232
|
if (mode === "answer" && !options.prompt) {
|
|
2262
2233
|
return { content: [{ type: "text", text: "Error: mode answer requires prompt." }], details: { error: "mode answer requires prompt" } };
|
|
2263
2234
|
}
|
|
@@ -2293,13 +2264,12 @@ export default function (pi) {
|
|
|
2293
2264
|
content: [{ type: "text", text: `Fetching ${urlList.length} URL(s)...` }],
|
|
2294
2265
|
details: { phase: "fetch", progress: 0 },
|
|
2295
2266
|
});
|
|
2296
|
-
const { answerModel: _answerModel, auth: _auth, ...extractionOptions } = options;
|
|
2297
|
-
const
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
: { ...extractionOptions, ...(authFetchProfile ? { authFetchProfile } : {}) };
|
|
2267
|
+
const { answerModel: _answerModel, auth: _auth, ...extractionOptions } = { ...options, mode };
|
|
2268
|
+
const { prompt: _prompt, ...answerExtractionOptions } = extractionOptions;
|
|
2269
|
+
const fetchOptions = {
|
|
2270
|
+
...(mode === "answer" ? answerExtractionOptions : extractionOptions),
|
|
2271
|
+
...(authFetchProfile ? { authFetchProfile } : {}),
|
|
2272
|
+
};
|
|
2303
2273
|
const fetchResults = await fetchAllContent(urlList, signal, withRegisteredFetchOptions(fetchOptions, registeredToolNames, options.proxy));
|
|
2304
2274
|
const presentedResults = mode === "answer"
|
|
2305
2275
|
? await Promise.all(fetchResults.map(async (result) => {
|
|
@@ -2411,7 +2381,14 @@ export default function (pi) {
|
|
|
2411
2381
|
});
|
|
2412
2382
|
},
|
|
2413
2383
|
renderCall(args, theme) {
|
|
2414
|
-
|
|
2384
|
+
let normalized;
|
|
2385
|
+
try {
|
|
2386
|
+
normalized = normalizeFetchContentParams(args);
|
|
2387
|
+
}
|
|
2388
|
+
catch {
|
|
2389
|
+
return new Text(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("error", "(invalid parameters)"), 0, 0);
|
|
2390
|
+
}
|
|
2391
|
+
const { urlList, options } = normalized;
|
|
2415
2392
|
const { prompt, timestamp, frames, model, mode, answerModel, auth } = options;
|
|
2416
2393
|
if (urlList.length === 0) {
|
|
2417
2394
|
return new Text(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("error", "(no URL)"), 0, 0);
|
|
@@ -2869,6 +2846,7 @@ export default function (pi) {
|
|
|
2869
2846
|
const summaryContext = {
|
|
2870
2847
|
model: ctx.model,
|
|
2871
2848
|
modelRegistry: ctx.modelRegistry,
|
|
2849
|
+
sessionManager: ctx.sessionManager,
|
|
2872
2850
|
cwd: ctx.cwd,
|
|
2873
2851
|
isProjectTrusted: () => ctx.isProjectTrusted(),
|
|
2874
2852
|
};
|