@herbertgao/pi-extensions 2026.10.0 → 2026.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/THIRD_PARTY_NOTICES.md +2 -2
- package/node_modules/@dietrichgebert/ponytail/.opencode/command/ponytail-audit.md +1 -1
- package/node_modules/@dietrichgebert/ponytail/.opencode/command/ponytail-debt.md +1 -1
- package/node_modules/@dietrichgebert/ponytail/.opencode/command/ponytail-gain.md +1 -1
- package/node_modules/@dietrichgebert/ponytail/.opencode/command/ponytail-review.md +1 -1
- package/node_modules/@dietrichgebert/ponytail/.opencode/command/ponytail.md +1 -1
- package/node_modules/@dietrichgebert/ponytail/.opencode/plugins/index.js +4 -0
- package/node_modules/@dietrichgebert/ponytail/.opencode/plugins/ponytail-frontmatter.cjs +43 -3
- package/node_modules/@dietrichgebert/ponytail/.opencode/plugins/ponytail.mjs +142 -59
- package/node_modules/@dietrichgebert/ponytail/.qoder-plugin/plugin.json +1 -1
- package/node_modules/@dietrichgebert/ponytail/README.es.md +87 -204
- package/node_modules/@dietrichgebert/ponytail/README.ja.md +212 -0
- package/node_modules/@dietrichgebert/ponytail/README.ko.md +92 -226
- package/node_modules/@dietrichgebert/ponytail/README.md +58 -244
- package/node_modules/@dietrichgebert/ponytail/README.zh-CN.md +212 -0
- package/node_modules/@dietrichgebert/ponytail/hooks/claude-codex-hooks.json +3 -3
- package/node_modules/@dietrichgebert/ponytail/hooks/copilot-hooks.json +4 -4
- package/node_modules/@dietrichgebert/ponytail/hooks/ponytail-activate.js +63 -16
- package/node_modules/@dietrichgebert/ponytail/hooks/ponytail-config.js +8 -7
- package/node_modules/@dietrichgebert/ponytail/hooks/ponytail-mode-tracker.js +46 -23
- package/node_modules/@dietrichgebert/ponytail/hooks/ponytail-runtime.js +39 -7
- package/node_modules/@dietrichgebert/ponytail/hooks/ponytail-subagent.js +20 -3
- package/node_modules/@dietrichgebert/ponytail/hooks/qoder-hooks.json +3 -3
- package/node_modules/@dietrichgebert/ponytail/package.json +2 -2
- package/node_modules/@dietrichgebert/ponytail/pi-extension/index.js +47 -17
- package/node_modules/@dietrichgebert/ponytail/scripts/uninstall.js +21 -7
- package/node_modules/@dietrichgebert/ponytail/skills/plugin.json +5 -0
- package/node_modules/@dietrichgebert/ponytail/skills/ponytail-audit/SKILL.md +7 -2
- package/node_modules/@dietrichgebert/ponytail/skills/ponytail-debt/SKILL.md +1 -1
- package/node_modules/@dietrichgebert/ponytail/skills/ponytail-gain/SKILL.md +16 -14
- package/node_modules/@dietrichgebert/ponytail/skills/ponytail-help/SKILL.md +3 -3
- package/node_modules/@dietrichgebert/ponytail/skills/ponytail-review/SKILL.md +12 -7
- package/node_modules/@narumitw/pi-caffeinate/package.json +4 -4
- package/node_modules/pi-typesafe/README.md +1 -1
- package/node_modules/pi-typesafe/dist/backends.d.ts +15 -5
- package/node_modules/pi-typesafe/dist/backends.js +18 -0
- package/node_modules/pi-typesafe/dist/client.d.ts +4 -3
- package/node_modules/pi-typesafe/dist/client.js +22 -7
- package/node_modules/pi-typesafe/dist/index.d.ts +2 -2
- package/node_modules/pi-typesafe/dist/index.js +1 -1
- package/node_modules/pi-typesafe/dist/usage.d.ts +14 -4
- package/node_modules/pi-typesafe/dist/usage.js +147 -42
- package/node_modules/pi-typesafe/package.json +1 -1
- package/node_modules/pi-web-access/CHANGELOG.md +48 -0
- package/node_modules/pi-web-access/README.md +57 -11
- package/node_modules/pi-web-access/brave-rate-limit.ts +8 -2
- package/node_modules/pi-web-access/curator-page.ts +5 -1
- package/node_modules/pi-web-access/curator-server.ts +2 -0
- package/node_modules/pi-web-access/declared-web-links.ts +2 -2
- package/node_modules/pi-web-access/degoog.ts +323 -0
- package/node_modules/pi-web-access/dist/index.js +3483 -2745
- package/node_modules/pi-web-access/dist/mcp-cli.js +27443 -0
- package/node_modules/pi-web-access/exa.ts +5 -1
- package/node_modules/pi-web-access/extract.ts +41 -3
- package/node_modules/pi-web-access/firecrawl.ts +35 -4
- package/node_modules/pi-web-access/gemini-search.ts +34 -11
- package/node_modules/pi-web-access/gemini-web.ts +14 -9
- package/node_modules/pi-web-access/index.ts +153 -1202
- package/node_modules/pi-web-access/keenable.ts +221 -0
- package/node_modules/pi-web-access/mcp-cli.ts +40 -0
- package/node_modules/pi-web-access/mcp-server.ts +81 -0
- package/node_modules/pi-web-access/mcp-tool-definitions.ts +117 -0
- package/node_modules/pi-web-access/openai-search.ts +34 -1
- package/node_modules/pi-web-access/package.json +23 -5
- package/node_modules/pi-web-access/perplexity.ts +64 -56
- package/node_modules/pi-web-access/tool-arguments.ts +20 -0
- package/node_modules/pi-web-access/video-extract.ts +1 -1
- package/node_modules/pi-web-access/web-tool-contract.ts +86 -0
- package/node_modules/pi-web-access/web-tool-core.ts +1502 -0
- package/node_modules/pi-web-access/youtube-extract.ts +2 -5
- package/package.json +6 -6
- package/node_modules/@dietrichgebert/ponytail/assets/benchmark-3model.svg +0 -21
- package/node_modules/@dietrichgebert/ponytail/assets/benchmark-agentic.svg +0 -62
- package/node_modules/@dietrichgebert/ponytail/assets/logo-dark.png +0 -0
- package/node_modules/@dietrichgebert/ponytail/assets/logo-dark.svg +0 -115
- package/node_modules/@dietrichgebert/ponytail/assets/logo-greenpt-dark.svg +0 -27
- package/node_modules/@dietrichgebert/ponytail/assets/logo-greenpt.svg +0 -27
- package/node_modules/@dietrichgebert/ponytail/assets/logo.png +0 -0
- package/node_modules/@dietrichgebert/ponytail/assets/retriever-icon.png +0 -0
- package/node_modules/@dietrichgebert/ponytail/assets/retriever-logo-dark.svg +0 -4
- package/node_modules/@dietrichgebert/ponytail/assets/retriever-logo-light.svg +0 -4
- package/node_modules/@dietrichgebert/ponytail/assets/social-preview.png +0 -0
- package/node_modules/@dietrichgebert/ponytail/assets/waitlist-banner-es.png +0 -0
- package/node_modules/@dietrichgebert/ponytail/assets/waitlist-banner-ko.png +0 -0
- package/node_modules/@dietrichgebert/ponytail/assets/waitlist-banner.png +0 -0
|
@@ -43,6 +43,8 @@ export type ExaSearchResult = SearchResponse | null;
|
|
|
43
43
|
|
|
44
44
|
export interface ExaSearchOptions extends SearchOptions {
|
|
45
45
|
includeContent?: boolean;
|
|
46
|
+
/** Restrict results to an Exa content category (e.g. "research paper", "news", "pdf"). */
|
|
47
|
+
category?: string;
|
|
46
48
|
}
|
|
47
49
|
|
|
48
50
|
type McpParsedResult = { title: string; url: string; content: string };
|
|
@@ -132,6 +134,7 @@ function exaSearchArgs(query: string, options: ExaSearchOptions): Record<string,
|
|
|
132
134
|
type: "auto",
|
|
133
135
|
numResults: options.numResults ?? 5,
|
|
134
136
|
...mapDomainFilter(options.domainFilter),
|
|
137
|
+
...(options.category ? { category: options.category } : {}),
|
|
135
138
|
...(startDate ? { startPublishedDate: startDate } : {}),
|
|
136
139
|
};
|
|
137
140
|
}
|
|
@@ -345,6 +348,7 @@ function buildMcpQuery(query: string, options: ExaSearchOptions): string {
|
|
|
345
348
|
case "year": parts.push(String(now.getFullYear())); break;
|
|
346
349
|
}
|
|
347
350
|
}
|
|
351
|
+
if (options.category) parts.push(options.category);
|
|
348
352
|
return parts.join(" ");
|
|
349
353
|
}
|
|
350
354
|
|
|
@@ -414,7 +418,7 @@ async function searchWithFilteredExaMcp(
|
|
|
414
418
|
async function searchWithExaMcp(query: string, options: ExaSearchOptions = {}): Promise<SearchResponse | null> {
|
|
415
419
|
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
416
420
|
const basicArgs = { query: buildMcpQuery(query, options), numResults: options.numResults ?? 5 };
|
|
417
|
-
const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length;
|
|
421
|
+
const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length || !!options.category;
|
|
418
422
|
|
|
419
423
|
try {
|
|
420
424
|
const response = filtered
|
|
@@ -149,6 +149,12 @@ function isRedirectPolicyError(message: string): boolean {
|
|
|
149
149
|
message.startsWith("Failed to resolve ");
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
+
// Errors that end routing: no other provider may be tried after them.
|
|
153
|
+
function isNonRecoverableHttpError(error: string | null): boolean {
|
|
154
|
+
return !!error && (NON_RECOVERABLE_ERRORS.some(prefix => error.startsWith(prefix)) ||
|
|
155
|
+
isRedirectPolicyError(error) || isConfigParseError(error));
|
|
156
|
+
}
|
|
157
|
+
|
|
152
158
|
function imageGateError(): string | null {
|
|
153
159
|
try {
|
|
154
160
|
return isImageEnabled() ? null : "Image fetching is disabled by image.enabled";
|
|
@@ -360,6 +366,9 @@ export interface ExtractOptions {
|
|
|
360
366
|
toolNames?: RegisteredToolNames;
|
|
361
367
|
/** Optional HTTP(S) or SOCKS proxy URL; routed through the curl-backed transport. */
|
|
362
368
|
proxy?: string;
|
|
369
|
+
/** Reject direct image URLs with this error instead of decoding them with Pi's
|
|
370
|
+
* resizeImage. Hosts without the Pi runtime set it; Pi leaves it unset. */
|
|
371
|
+
rejectDirectImages?: string;
|
|
363
372
|
/** Custom DNS resolver used for SSRF validation. Primarily a test seam. */
|
|
364
373
|
lookup?: Lookup;
|
|
365
374
|
}
|
|
@@ -811,7 +820,7 @@ export async function extractContent(
|
|
|
811
820
|
declaredLinks = discoveredLinks;
|
|
812
821
|
if (signal?.aborted) return abortedResult(url);
|
|
813
822
|
if (!httpResult.error) return httpResult;
|
|
814
|
-
if (
|
|
823
|
+
if (isNonRecoverableHttpError(httpResult.error) || httpResult.error === options?.rejectDirectImages) {
|
|
815
824
|
return httpResult;
|
|
816
825
|
}
|
|
817
826
|
return null;
|
|
@@ -1175,11 +1184,19 @@ function responseSizeLimitError(maxBytes: number): Error {
|
|
|
1175
1184
|
return new Error(`Response too large (${Math.round(maxBytes / 1024 / 1024)}MB)`);
|
|
1176
1185
|
}
|
|
1177
1186
|
|
|
1187
|
+
const BROWSER_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8";
|
|
1188
|
+
// Servers that support markdown content negotiation (Cloudflare Markdown for
|
|
1189
|
+
// Agents, Mintlify, Vercel) return markdown directly; others see the same
|
|
1190
|
+
// relative preferences as BROWSER_ACCEPT. Raw mode keeps BROWSER_ACCEPT so it
|
|
1191
|
+
// returns the server's normal representation.
|
|
1192
|
+
const MARKDOWN_FIRST_ACCEPT = "text/markdown,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,image/avif;q=0.9,image/webp;q=0.9,image/apng;q=0.9,*/*;q=0.7";
|
|
1193
|
+
|
|
1178
1194
|
async function extractViaHttp(
|
|
1179
1195
|
url: string,
|
|
1180
1196
|
timeoutMs: number,
|
|
1181
1197
|
signal?: AbortSignal,
|
|
1182
1198
|
options?: ExtractOptions,
|
|
1199
|
+
preferMarkdown = options?.mode !== "raw",
|
|
1183
1200
|
): Promise<HttpExtractedContent> {
|
|
1184
1201
|
const activityId = activityMonitor.logStart({ type: "fetch", url });
|
|
1185
1202
|
|
|
@@ -1199,7 +1216,7 @@ async function extractViaHttp(
|
|
|
1199
1216
|
__proxy: options?.proxy,
|
|
1200
1217
|
headers: {
|
|
1201
1218
|
"User-Agent": "OpenAI File Downloader, XaiImageApiFetch/1.0",
|
|
1202
|
-
"Accept":
|
|
1219
|
+
"Accept": preferMarkdown ? MARKDOWN_FIRST_ACCEPT : BROWSER_ACCEPT,
|
|
1203
1220
|
"Accept-Language": "en-US,en;q=0.9",
|
|
1204
1221
|
"Cache-Control": "no-cache",
|
|
1205
1222
|
"Sec-Fetch-Dest": "document",
|
|
@@ -1275,6 +1292,10 @@ async function extractViaHttp(
|
|
|
1275
1292
|
activityMonitor.logComplete(activityId, response.status);
|
|
1276
1293
|
return { url, title: "", content: "", error: disabled, mimeType, status: response.status };
|
|
1277
1294
|
}
|
|
1295
|
+
if (options?.rejectDirectImages) {
|
|
1296
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
1297
|
+
return { url, title: "", content: "", error: options.rejectDirectImages, mimeType, status: response.status };
|
|
1298
|
+
}
|
|
1278
1299
|
try {
|
|
1279
1300
|
const buffer = await readResponseBufferWithLimit(response, maxResponseSize, () => responseSizeLimitError(maxResponseSize));
|
|
1280
1301
|
const { resizeImage } = await import("@earendil-works/pi-coding-agent");
|
|
@@ -1350,7 +1371,24 @@ async function extractViaHttp(
|
|
|
1350
1371
|
if (!isHTML) {
|
|
1351
1372
|
activityMonitor.logComplete(activityId, response.status);
|
|
1352
1373
|
const title = extractTextTitle(text, url);
|
|
1353
|
-
return { url, title, content: text, error: null };
|
|
1374
|
+
if (mimeType !== "text/markdown" && mimeType !== "text/x-markdown") return { url, title, content: text, error: null };
|
|
1375
|
+
const declaredLinks = discoverDeclaredWebLinks(null, response.headers.get("link"), response.url || url);
|
|
1376
|
+
const markdown = { url, title, content: appendDeclaredWebLinks(text, declaredLinks), error: null, declaredLinks };
|
|
1377
|
+
if (!preferMarkdown || text.trim().length >= MIN_USEFUL_CONTENT) return markdown;
|
|
1378
|
+
// Short negotiated markdown is re-requested with the browser Accept header.
|
|
1379
|
+
// That response stays authoritative for policy/config rejection, not-found
|
|
1380
|
+
// status, cancellation, and any content it has. This markdown only fills a
|
|
1381
|
+
// recoverable empty response, marked incomplete with its status and error.
|
|
1382
|
+
const normal = await extractViaHttp(url, Math.max(1, timeoutMs - (Date.now() - startedAt)), signal, options, false);
|
|
1383
|
+
if (signal?.aborted || isNonRecoverableHttpError(normal.error) ||
|
|
1384
|
+
normal.status === 404 || normal.status === 410 || isAbortError(normal.error)) return normal;
|
|
1385
|
+
const normalHasBody = normal.content.trim() !== appendDeclaredWebLinks("", normal.declaredLinks ?? []).trim();
|
|
1386
|
+
if (normalHasBody || !markdown.content.trim()) return normal;
|
|
1387
|
+
return {
|
|
1388
|
+
...normal,
|
|
1389
|
+
...markdown,
|
|
1390
|
+
error: "Extracted content appears incomplete" + (normal.error ? `\nBrowser retry failed: ${normal.error}` : ""),
|
|
1391
|
+
};
|
|
1354
1392
|
}
|
|
1355
1393
|
|
|
1356
1394
|
const { parseHTML } = await import("linkedom");
|
|
@@ -37,12 +37,18 @@ interface FirecrawlConfig {
|
|
|
37
37
|
firecrawlApiKey?: unknown;
|
|
38
38
|
firecrawlApiVersion?: unknown;
|
|
39
39
|
firecrawlFreshScrape?: unknown;
|
|
40
|
+
firecrawlPdfMaxPages?: unknown;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
interface FirecrawlPageMetadata {
|
|
44
|
+
numPages?: unknown;
|
|
45
|
+
totalPages?: unknown;
|
|
40
46
|
}
|
|
41
47
|
|
|
42
48
|
interface FirecrawlScrapeData {
|
|
43
49
|
title?: unknown;
|
|
44
50
|
markdown?: unknown;
|
|
45
|
-
metadata?: { title?: unknown };
|
|
51
|
+
metadata?: { title?: unknown } & FirecrawlPageMetadata;
|
|
46
52
|
}
|
|
47
53
|
|
|
48
54
|
interface FirecrawlSearchItem {
|
|
@@ -51,7 +57,7 @@ interface FirecrawlSearchItem {
|
|
|
51
57
|
snippet?: unknown;
|
|
52
58
|
url?: unknown;
|
|
53
59
|
markdown?: unknown;
|
|
54
|
-
metadata?: {
|
|
60
|
+
metadata?: FirecrawlPageMetadata & {
|
|
55
61
|
title?: unknown;
|
|
56
62
|
description?: unknown;
|
|
57
63
|
sourceURL?: unknown;
|
|
@@ -155,6 +161,28 @@ function allowFreshScrape(): boolean {
|
|
|
155
161
|
return configured;
|
|
156
162
|
}
|
|
157
163
|
|
|
164
|
+
function pdfParserOptions(): { parsers?: Array<{ type: "pdf"; maxPages: number }> } {
|
|
165
|
+
const configured = loadConfig().firecrawlPdfMaxPages;
|
|
166
|
+
if (configured === undefined || configured === null) return {};
|
|
167
|
+
if (typeof configured !== "number" || !Number.isInteger(configured) || configured < 1) {
|
|
168
|
+
throw new Error(`firecrawlPdfMaxPages in ${CONFIG_PATH} must be a positive integer`);
|
|
169
|
+
}
|
|
170
|
+
if (getApiVersion() === "v1") {
|
|
171
|
+
throw new Error(
|
|
172
|
+
`firecrawlPdfMaxPages in ${CONFIG_PATH} requires Firecrawl API v2; v1 has no PDF page cap. ` +
|
|
173
|
+
"Use firecrawlApiVersion \"v2\" or remove firecrawlPdfMaxPages",
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
return { parsers: [{ type: "pdf", maxPages: configured }] };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function withPdfTruncationNotice(content: string, metadata: FirecrawlPageMetadata | undefined): string {
|
|
180
|
+
const parsed = metadata?.numPages;
|
|
181
|
+
const total = metadata?.totalPages;
|
|
182
|
+
if (typeof parsed !== "number" || typeof total !== "number" || parsed >= total) return content;
|
|
183
|
+
return `${content}\n\n---\n\n*[Truncated: Only first ${parsed} of ${total} PDF pages parsed by Firecrawl]*`;
|
|
184
|
+
}
|
|
185
|
+
|
|
158
186
|
async function getApiKey(signal?: AbortSignal): Promise<string | null> {
|
|
159
187
|
const configKey = loadConfig().firecrawlApiKey;
|
|
160
188
|
return resolveCredential({
|
|
@@ -234,6 +262,7 @@ function scrapeBody(url: string): Record<string, unknown> {
|
|
|
234
262
|
formats: ["markdown"],
|
|
235
263
|
onlyMainContent: true,
|
|
236
264
|
...(allowFreshScrape() ? {} : { lockdown: true }),
|
|
265
|
+
...pdfParserOptions(),
|
|
237
266
|
};
|
|
238
267
|
}
|
|
239
268
|
|
|
@@ -272,6 +301,7 @@ function mapRecencyFilter(value: SearchOptions["recencyFilter"]): string | undef
|
|
|
272
301
|
}
|
|
273
302
|
|
|
274
303
|
function searchBody(query: string, options: FirecrawlSearchOptions, numResults: number, filters: DomainFilters): Record<string, unknown> {
|
|
304
|
+
const parserOptions = pdfParserOptions();
|
|
275
305
|
return {
|
|
276
306
|
query,
|
|
277
307
|
limit: numResults,
|
|
@@ -284,6 +314,7 @@ function searchBody(query: string, options: FirecrawlSearchOptions, numResults:
|
|
|
284
314
|
formats: ["markdown"],
|
|
285
315
|
onlyMainContent: true,
|
|
286
316
|
...(allowFreshScrape() ? {} : { lockdown: true }),
|
|
317
|
+
...parserOptions,
|
|
287
318
|
},
|
|
288
319
|
} : {}),
|
|
289
320
|
};
|
|
@@ -317,7 +348,7 @@ function mapSearchResults(data: unknown, numResults: number, filters: DomainFilt
|
|
|
317
348
|
const snippet = firstString(item.description, item.snippet, item.metadata?.description) ?? "";
|
|
318
349
|
results.push({ title, url, snippet });
|
|
319
350
|
const markdown = firstString(item.markdown);
|
|
320
|
-
if (markdown) inlineContent.push({ url, title, content: markdown, error: null });
|
|
351
|
+
if (markdown) inlineContent.push({ url, title, content: withPdfTruncationNotice(markdown, item.metadata), error: null });
|
|
321
352
|
if (results.length >= numResults) break;
|
|
322
353
|
}
|
|
323
354
|
return { results, inlineContent };
|
|
@@ -419,5 +450,5 @@ export async function extractWithFirecrawl(
|
|
|
419
450
|
const title = typeof metadataTitle === "string" && metadataTitle.trim()
|
|
420
451
|
? metadataTitle.trim()
|
|
421
452
|
: typeof scrape.title === "string" ? scrape.title.trim() : "";
|
|
422
|
-
return { url, title, content, error: null };
|
|
453
|
+
return { url, title, content: withPdfTruncationNotice(content, scrape.metadata), error: null };
|
|
423
454
|
}
|
|
@@ -31,6 +31,7 @@ import { isKagiAvailable, searchWithKagi } from "./kagi.ts";
|
|
|
31
31
|
import { isBochaAvailable, searchWithBocha } from "./bocha.ts";
|
|
32
32
|
import { isOllamaAvailable, searchWithOllama } from "./ollama.ts";
|
|
33
33
|
import { isSearXNGAvailable, searchWithSearXNG } from "./searxng.ts";
|
|
34
|
+
import { isDegoogAvailable, searchWithDegoog } from "./degoog.ts";
|
|
34
35
|
import { isDuckDuckGoAvailable, searchWithDuckDuckGo } from "./duckduckgo.ts";
|
|
35
36
|
import { isAnySearchAvailable, searchWithAnySearch } from "./anysearch.ts";
|
|
36
37
|
import { isXcrawlAvailable, searchWithXCrawl } from "./xcrawl.ts";
|
|
@@ -43,11 +44,12 @@ import { isSerplyAvailable, searchWithSerply } from "./serply.ts";
|
|
|
43
44
|
import { isBaizhiAvailable, searchWithBaizhi } from "./baizhi.ts";
|
|
44
45
|
import { isZaiAvailable, searchWithZai } from "./zai.ts";
|
|
45
46
|
import { isValyuAvailable, searchWithValyu } from "./valyu.ts";
|
|
47
|
+
import { isKeenableAvailable, searchWithKeenable } from "./keenable.ts";
|
|
46
48
|
import { isKimiSearchAvailable, searchWithKimi } from "./kimi-search.ts";
|
|
47
49
|
import { isMistralAvailable, searchWithMistral } from "./mistral-search.ts";
|
|
48
50
|
import { getWebSearchConfigPath } from "./utils.ts";
|
|
49
51
|
|
|
50
|
-
export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "you", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "bocha", "xcrawl", "baizhi", "zai"] as const;
|
|
52
|
+
export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "you", "firecrawl", "jina", "searxng", "degoog", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "bocha", "xcrawl", "baizhi", "zai", "keenable"] as const;
|
|
51
53
|
export const SEARCH_PROVIDERS = ["auto", "all", ...RESOLVED_SEARCH_PROVIDERS] as const;
|
|
52
54
|
|
|
53
55
|
export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number];
|
|
@@ -112,7 +114,7 @@ export interface AttributedSearchResponse extends SearchResponse {
|
|
|
112
114
|
|
|
113
115
|
const CONFIG_PATH = getWebSearchConfigPath();
|
|
114
116
|
const DEFAULT_SEARCH_MODEL = "gemini-3.6-flash";
|
|
115
|
-
// Explicit-only providers (Parallel MCP, DuckDuckGo, Kimi, AnySearch, XCrawl, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Serply, You.com, Valyu, Baizhi, Z.ai) are deliberately absent:
|
|
117
|
+
// Explicit-only providers (Parallel MCP, degoog, DuckDuckGo, Kimi, AnySearch, XCrawl, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Serply, You.com, Valyu, Baizhi, Z.ai, Keenable) are deliberately absent:
|
|
116
118
|
// `all` must never fan out to an opt-in or paid provider without the user asking for it.
|
|
117
119
|
export const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"];
|
|
118
120
|
const VALID_ROUTING_KINDS = ["transient", "quota", "network", "invalid-response", "unsupported"] as const;
|
|
@@ -264,6 +266,8 @@ export interface FullSearchOptions extends SearchOptions {
|
|
|
264
266
|
provider?: SearchProviderSelection;
|
|
265
267
|
includeContent?: boolean;
|
|
266
268
|
extensionContext?: ExtensionContext;
|
|
269
|
+
/** Forwarded to providers that understand it (currently Exa); ignored elsewhere. */
|
|
270
|
+
category?: string;
|
|
267
271
|
}
|
|
268
272
|
|
|
269
273
|
function errorMessage(err: unknown): string {
|
|
@@ -330,6 +334,16 @@ function classifyProviderError(provider: ResolvedSearchProvider, err: unknown):
|
|
|
330
334
|
const message = errorMessage(err);
|
|
331
335
|
const lower = message.toLowerCase();
|
|
332
336
|
const status = providerErrorStatus(message);
|
|
337
|
+
// An HTTP 200 OpenAI stream reports its failure as an error type/code such as
|
|
338
|
+
// rate_limit_exceeded or server_error. Reading only that message with spaces
|
|
339
|
+
// lets the shared keywords below classify it like prose, without widening them
|
|
340
|
+
// for other providers, which echo foreign text (Bright Data zone names,
|
|
341
|
+
// envelope codes) that must keep classifying as before. OpenAI error types
|
|
342
|
+
// the shared keywords don't name (usage limit, authentication_error,
|
|
343
|
+
// permission_error, invalid_api_key, api_error, internal_error, overloaded)
|
|
344
|
+
// are added per branch for these messages only.
|
|
345
|
+
const openAIStreamError = provider === "openai" && lower.includes("openai api stream error");
|
|
346
|
+
const text = openAIStreamError ? lower.replace(/_/g, " ") : lower;
|
|
333
347
|
let kind: SearchProviderErrorKind = "unknown";
|
|
334
348
|
const mentionsUnsupportedWebSearch = /(?:web[_ -]?search|web[_ -]?search_preview|(?:the )?tool)\b.*\b(?:unsupported|not supported|does not support|doesn't support|unknown|unrecognized|unavailable|not found)|\b(?:unsupported|not supported|does not support|doesn't support|unknown|unrecognized|unavailable|not found)\b.*\b(?:web[_ -]?search|web[_ -]?search_preview|(?:the )?tool)/i.test(lower);
|
|
335
349
|
if (err instanceof CredentialResolutionError || /(?:api )?key (?:not found|missing)|credential resolution/.test(lower)) {
|
|
@@ -350,20 +364,23 @@ function classifyProviderError(provider: ResolvedSearchProvider, err: unknown):
|
|
|
350
364
|
kind = "quota";
|
|
351
365
|
} else if (status !== undefined && (status === 408 || status === 425 || status >= 500)) {
|
|
352
366
|
kind = "transient";
|
|
353
|
-
} else if (/rate limit|quota|too many requests/.test(
|
|
367
|
+
} else if (/rate limit|quota|too many requests/.test(text) || (openAIStreamError && /usage limit/.test(text))) {
|
|
354
368
|
kind = "quota";
|
|
355
|
-
} else if (/unauthorized|forbidden|permission denied/.test(
|
|
369
|
+
} else if (/unauthorized|forbidden|permission denied/.test(text) || (openAIStreamError && /authentication error|permission error|invalid api key/.test(text))) {
|
|
356
370
|
kind = "auth";
|
|
357
|
-
} else if (/bad request|invalid request/.test(
|
|
371
|
+
} else if (/bad request|invalid request/.test(text)) {
|
|
358
372
|
kind = "invalid-request";
|
|
359
|
-
} else if (/invalid json|no parseable response|no parseable results|invalid response|returned empty response|no web_search_call/.test(
|
|
373
|
+
} else if (/invalid json|no parseable response|no parseable results|invalid response|returned empty response|no web_search_call/.test(text)) {
|
|
360
374
|
kind = "invalid-response";
|
|
361
|
-
} else if (/temporar|service unavailable|server error/.test(
|
|
375
|
+
} else if (/temporar|service unavailable|server error/.test(text) || (openAIStreamError && /internal error|\bapi error|overloaded/.test(text))) {
|
|
362
376
|
kind = "transient";
|
|
363
|
-
} else if (err instanceof TypeError || /fetch failed|network|econnreset|econnrefused|enotfound|etimedout|timed out|socket/.test(
|
|
377
|
+
} else if (err instanceof TypeError || /fetch failed|network|econnreset|econnrefused|enotfound|etimedout|timed out|socket/.test(text)) {
|
|
364
378
|
kind = "network";
|
|
365
|
-
} else if (/invalid or missing|invalid config|failed to parse|must be an? |configuration/.test(
|
|
379
|
+
} else if (/invalid or missing|invalid config|failed to parse|must be an? |configuration/.test(text)) {
|
|
366
380
|
kind = "config";
|
|
381
|
+
} else if (openAIStreamError) {
|
|
382
|
+
// The same stream threw "no parseable response output" before its error was surfaced.
|
|
383
|
+
kind = "invalid-response";
|
|
367
384
|
}
|
|
368
385
|
return new SearchProviderError(provider, kind, message, status, err);
|
|
369
386
|
}
|
|
@@ -406,9 +423,11 @@ async function searchWithResolvedProvider(
|
|
|
406
423
|
if (provider === "baizhi") return { ...(await searchWithBaizhi(query, options)), provider };
|
|
407
424
|
if (provider === "zai") return { ...(await searchWithZai(query, options)), provider };
|
|
408
425
|
if (provider === "valyu") return { ...(await searchWithValyu(query, options)), provider };
|
|
426
|
+
if (provider === "keenable") return { ...(await searchWithKeenable(query, options)), provider };
|
|
409
427
|
if (provider === "xcrawl") return { ...(await searchWithXCrawl(query, options)), provider };
|
|
410
428
|
if (provider === "perplexity") return { ...(await searchWithPerplexity(query, options)), provider };
|
|
411
429
|
if (provider === "searxng") return { ...(await searchWithSearXNG(query, options)), provider };
|
|
430
|
+
if (provider === "degoog") return { ...(await searchWithDegoog(query, options)), provider };
|
|
412
431
|
if (provider === "duckduckgo") return { ...(await searchWithDuckDuckGo(query, options)), provider };
|
|
413
432
|
if (provider === "kimi") return { ...(await searchWithKimi(query, options, options.extensionContext)), provider };
|
|
414
433
|
if (provider === "gemini") {
|
|
@@ -459,9 +478,11 @@ async function isResolvedProviderAvailable(provider: ResolvedSearchProvider, opt
|
|
|
459
478
|
if (provider === "baizhi") return isBaizhiAvailable();
|
|
460
479
|
if (provider === "zai") return isZaiAvailable();
|
|
461
480
|
if (provider === "valyu") return isValyuAvailable();
|
|
481
|
+
if (provider === "keenable") return isKeenableAvailable();
|
|
462
482
|
if (provider === "xcrawl") return isXcrawlAvailable();
|
|
463
483
|
if (provider === "perplexity") return isPerplexityAvailable();
|
|
464
484
|
if (provider === "searxng") return isSearXNGAvailable();
|
|
485
|
+
if (provider === "degoog") return isDegoogAvailable();
|
|
465
486
|
if (provider === "duckduckgo") return isDuckDuckGoAvailable();
|
|
466
487
|
if (provider === "gemini") return isGeminiApiAvailable() || await isGeminiWebOptionallyAvailable();
|
|
467
488
|
if (provider === "kimi") return isKimiSearchAvailable(options.extensionContext);
|
|
@@ -486,6 +507,7 @@ export function providerLabel(provider: ResolvedSearchProvider): string {
|
|
|
486
507
|
if (provider === "firecrawl") return "Firecrawl";
|
|
487
508
|
if (provider === "serpdive") return "SERPdive";
|
|
488
509
|
if (provider === "searxng") return "SearXNG";
|
|
510
|
+
if (provider === "degoog") return "degoog";
|
|
489
511
|
if (provider === "duckduckgo") return "DuckDuckGo";
|
|
490
512
|
if (provider === "kagi") return "Kagi";
|
|
491
513
|
if (provider === "bocha") return "Bocha";
|
|
@@ -503,6 +525,7 @@ export function providerLabel(provider: ResolvedSearchProvider): string {
|
|
|
503
525
|
if (provider === "baizhi") return "Baizhi";
|
|
504
526
|
if (provider === "zai") return "Z.ai";
|
|
505
527
|
if (provider === "valyu") return "Valyu";
|
|
528
|
+
if (provider === "keenable") return "Keenable";
|
|
506
529
|
return provider.charAt(0).toUpperCase() + provider.slice(1);
|
|
507
530
|
}
|
|
508
531
|
|
|
@@ -530,7 +553,7 @@ async function searchWithProviders(
|
|
|
530
553
|
: await isResolvedProviderAvailable(provider, options),
|
|
531
554
|
})))).filter((entry) => entry.available).map((entry) => entry.provider);
|
|
532
555
|
if (providers.length === 0) {
|
|
533
|
-
throw new Error("No configured search provider available for provider \"all\". Parallel MCP, DuckDuckGo, Kimi, AnySearch, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Serply, You.com, Valyu, Baizhi, Z.ai, and
|
|
556
|
+
throw new Error("No configured search provider available for provider \"all\". Parallel MCP, degoog, DuckDuckGo, Kimi, AnySearch, xAI, Mistral, Bright Data, SerpBase, SerpApi, Serper, Serply, You.com, Valyu, Baizhi, Z.ai, XCrawl, and Keenable are excluded.");
|
|
534
557
|
}
|
|
535
558
|
|
|
536
559
|
const settled = await Promise.allSettled(
|
|
@@ -825,7 +848,7 @@ export async function search(query: string, options: FullSearchOptions = {}): Pr
|
|
|
825
848
|
" 3. Set OPENAI_API_KEY, BRAVE_API_KEY, PARALLEL_API_KEY, TINYFISH_API_KEY, SEARCH1API_KEY, SEARCHINFINITY_API_KEY, QUERIT_API_KEY, TAVILY_API_KEY, FIRECRAWL_BASE_URL, JINA_API_KEY, SERPDIVE_API_KEY, KAGI_API_KEY, BOCHA_API_KEY, OLLAMA_API_KEY, SEARXNG_BASE_URL, EXA_API_KEY, PERPLEXITY_API_KEY, GEMINI_API_KEY, or CLOUDFLARE_API_KEY env vars\n" +
|
|
826
849
|
" 4. Set GOOGLE_GEMINI_BASE_URL with CLOUDFLARE_API_KEY for Cloudflare AI Gateway routing\n" +
|
|
827
850
|
" 5. Sign into gemini.google.com in a supported Chromium-based browser\n" +
|
|
828
|
-
" 6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xcrawl\" for XCrawl, \"xai\" for Grok, \"mistral\" for Mistral Conversations web search, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, \"serpbase\", \"serpapi\", \"serper\", or \"serply\" for Google SERP, \"you\" for You.com, \"zai\" for Z.ai GLM Coding Plan search,
|
|
851
|
+
" 6. Explicitly select provider: \"anysearch\" for anonymous AnySearch, \"xcrawl\" for XCrawl, \"xai\" for Grok, \"mistral\" for Mistral Conversations web search, \"brightdata\" with brightdataSerpZone for paid Bright Data SERP, \"serpbase\", \"serpapi\", \"serper\", or \"serply\" for Google SERP, \"you\" for You.com, \"zai\" for Z.ai GLM Coding Plan search, \"valyu\" for research search, or \"keenable\" for keyless Keenable search"
|
|
829
852
|
);
|
|
830
853
|
}
|
|
831
854
|
|
|
@@ -92,11 +92,16 @@ export async function resolveGeminiFetch(): Promise<typeof fetch> {
|
|
|
92
92
|
export interface GeminiWebOptions {
|
|
93
93
|
youtubeUrl?: string;
|
|
94
94
|
model?: string;
|
|
95
|
-
files?:
|
|
95
|
+
files?: GeminiWebFile[];
|
|
96
96
|
signal?: AbortSignal;
|
|
97
97
|
timeoutMs?: number;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
export interface GeminiWebFile {
|
|
101
|
+
path: string;
|
|
102
|
+
mimeType: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
100
105
|
export async function isGeminiWebAvailable(chromeProfile?: string): Promise<CookieMap | null> {
|
|
101
106
|
if (!isBrowserCookieAccessAllowed()) return null;
|
|
102
107
|
|
|
@@ -180,7 +185,7 @@ async function runGeminiWebOnce(
|
|
|
180
185
|
prompt: string,
|
|
181
186
|
cookieMap: CookieMap,
|
|
182
187
|
model: string,
|
|
183
|
-
files:
|
|
188
|
+
files: GeminiWebFile[] | undefined,
|
|
184
189
|
timeoutMs: number,
|
|
185
190
|
signal?: AbortSignal,
|
|
186
191
|
): Promise<GeminiWebResult> {
|
|
@@ -190,8 +195,8 @@ async function runGeminiWebOnce(
|
|
|
190
195
|
|
|
191
196
|
const uploaded: Array<{ id: string; name: string }> = [];
|
|
192
197
|
if (files) {
|
|
193
|
-
for (const
|
|
194
|
-
uploaded.push(await uploadFile(
|
|
198
|
+
for (const file of files) {
|
|
199
|
+
uploaded.push(await uploadFile(file, cookieHeader, effectiveSignal));
|
|
195
200
|
}
|
|
196
201
|
}
|
|
197
202
|
|
|
@@ -357,14 +362,14 @@ function decodeEmailEscapes(value: string): string {
|
|
|
357
362
|
}
|
|
358
363
|
|
|
359
364
|
async function uploadFile(
|
|
360
|
-
|
|
365
|
+
file: GeminiWebFile,
|
|
361
366
|
cookieHeader: string,
|
|
362
367
|
signal: AbortSignal,
|
|
363
368
|
): Promise<{ id: string; name: string }> {
|
|
364
|
-
const data = readFileSync(
|
|
365
|
-
const fileName = basename(
|
|
369
|
+
const data = readFileSync(file.path);
|
|
370
|
+
const fileName = basename(file.path);
|
|
366
371
|
const boundary = "----FormBoundary" + Math.random().toString(36).slice(2);
|
|
367
|
-
const header = `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\nContent-Type:
|
|
372
|
+
const header = `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${fileName}"\r\nContent-Type: ${file.mimeType}\r\n\r\n`;
|
|
368
373
|
const footer = `\r\n--${boundary}--\r\n`;
|
|
369
374
|
|
|
370
375
|
const body = Buffer.concat([
|
|
@@ -400,7 +405,7 @@ function buildFReqPayload(
|
|
|
400
405
|
): string {
|
|
401
406
|
const promptPayload =
|
|
402
407
|
uploaded.length > 0
|
|
403
|
-
? [prompt, 0, null, uploaded.map((file) => [[file.id, 1]])]
|
|
408
|
+
? [prompt, 0, null, uploaded.map((file) => [[file.id, 1], file.name])]
|
|
404
409
|
: [prompt];
|
|
405
410
|
const innerList = [promptPayload, null, null];
|
|
406
411
|
return JSON.stringify([null, JSON.stringify(innerList)]);
|