@asterxsk/kiln 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +170 -170
- package/agent/AGENTS.md +67 -67
- package/agent/README.md +5 -5
- package/agent/extensions/AGENTS.md +68 -68
- package/agent/extensions/ask-user/index.ts +418 -418
- package/agent/extensions/ask-user/package-lock.json +769 -769
- package/agent/extensions/ask-user/package.json +19 -19
- package/agent/extensions/ask-user/prompt.ts +45 -45
- package/agent/extensions/ask-user/tsconfig.json +7 -7
- package/agent/extensions/background-terminals/docs/implementation-guide.md +942 -942
- package/agent/extensions/background-terminals/index.ts +627 -627
- package/agent/extensions/background-terminals/manager.test.ts +735 -735
- package/agent/extensions/background-terminals/output.test.ts +109 -109
- package/agent/extensions/background-terminals/package-lock.json +769 -769
- package/agent/extensions/background-terminals/package.json +17 -17
- package/agent/extensions/background-terminals/prompt.test.ts +125 -125
- package/agent/extensions/background-terminals/ps.test.ts +82 -82
- package/agent/extensions/background-terminals/result-delivery.test.ts +44 -44
- package/agent/extensions/background-terminals/src/domain.ts +87 -87
- package/agent/extensions/background-terminals/src/manager.ts +907 -907
- package/agent/extensions/background-terminals/src/output.ts +84 -84
- package/agent/extensions/background-terminals/src/prompt.ts +142 -142
- package/agent/extensions/background-terminals/src/result-delivery.ts +27 -27
- package/agent/extensions/background-terminals/src/runtime.ts +36 -36
- package/agent/extensions/background-terminals/src/ui/output-view.ts +79 -79
- package/agent/extensions/background-terminals/src/ui/ps.ts +621 -621
- package/agent/extensions/background-terminals/tsconfig.json +7 -7
- package/agent/extensions/file-search/index.spec.ts +443 -443
- package/agent/extensions/file-search/index.ts +459 -459
- package/agent/extensions/file-search/package-lock.json +2253 -2253
- package/agent/extensions/file-search/package.json +23 -23
- package/agent/extensions/file-search/src/args.ts +122 -122
- package/agent/extensions/file-search/src/binaries.ts +422 -422
- package/agent/extensions/file-search/src/output.ts +126 -126
- package/agent/extensions/file-search/src/process.ts +146 -146
- package/agent/extensions/file-search/src/prompt.ts +52 -52
- package/agent/extensions/file-search/tsconfig.json +7 -7
- package/agent/extensions/modelconf/PLAN.md +915 -915
- package/agent/extensions/modelconf/index.ts +296 -296
- package/agent/extensions/modelconf/src/ui/ModelConfView.ts +1101 -1101
- package/agent/extensions/pi-web-access/CHANGELOG.md +690 -690
- package/agent/extensions/pi-web-access/LICENSE +21 -21
- package/agent/extensions/pi-web-access/README.md +470 -470
- package/agent/extensions/pi-web-access/SECURITY.md +5 -5
- package/agent/extensions/pi-web-access/activity.ts +101 -101
- package/agent/extensions/pi-web-access/auth-fetch.ts +148 -148
- package/agent/extensions/pi-web-access/brightdata-unlocker.ts +272 -272
- package/agent/extensions/pi-web-access/chrome-cookies.ts +669 -669
- package/agent/extensions/pi-web-access/content-find.ts +139 -139
- package/agent/extensions/pi-web-access/credential-source.ts +191 -191
- package/agent/extensions/pi-web-access/data-uri-sanitize.ts +406 -406
- package/agent/extensions/pi-web-access/datalab-pdf-extract.ts +568 -568
- package/agent/extensions/pi-web-access/declared-web-links.ts +173 -173
- package/agent/extensions/pi-web-access/evidence/CONTRACT-EVIDENCE.md +496 -496
- package/agent/extensions/pi-web-access/evidence/contract-probe.mjs +140 -140
- package/agent/extensions/pi-web-access/exa.ts +526 -526
- package/agent/extensions/pi-web-access/extract.ts +1196 -1196
- package/agent/extensions/pi-web-access/feature-config.ts +29 -29
- package/agent/extensions/pi-web-access/fetch-params.ts +111 -111
- package/agent/extensions/pi-web-access/gemini-adc.ts +298 -298
- package/agent/extensions/pi-web-access/gemini-api.ts +353 -353
- package/agent/extensions/pi-web-access/gemini-pdf-extract.ts +108 -108
- package/agent/extensions/pi-web-access/gemini-url-context.ts +128 -128
- package/agent/extensions/pi-web-access/gemini-web-config.ts +101 -101
- package/agent/extensions/pi-web-access/gemini-web.ts +487 -487
- package/agent/extensions/pi-web-access/github-api.ts +197 -197
- package/agent/extensions/pi-web-access/github-extract.ts +746 -746
- package/agent/extensions/pi-web-access/github-issue-pr.ts +700 -700
- package/agent/extensions/pi-web-access/index.ts +1737 -1737
- package/agent/extensions/pi-web-access/package-lock.json +5808 -5808
- package/agent/extensions/pi-web-access/package.json +64 -64
- package/agent/extensions/pi-web-access/page-query.ts +96 -96
- package/agent/extensions/pi-web-access/pdf-extract.ts +409 -409
- package/agent/extensions/pi-web-access/promise-try.d.ts +7 -7
- package/agent/extensions/pi-web-access/query-rewrite.ts +51 -51
- package/agent/extensions/pi-web-access/render-search-error.ts +170 -170
- package/agent/extensions/pi-web-access/rsc-extract.ts +338 -338
- package/agent/extensions/pi-web-access/source-check.ts +282 -282
- package/agent/extensions/pi-web-access/ssrf-protection.ts +526 -526
- package/agent/extensions/pi-web-access/storage.ts +521 -521
- package/agent/extensions/pi-web-access/summary-model-scope.ts +125 -125
- package/agent/extensions/pi-web-access/test/auth-fetch.test.mjs +208 -208
- package/agent/extensions/pi-web-access/test/brightdata-unlocker.test.mjs +840 -840
- package/agent/extensions/pi-web-access/test/chrome-cookie-extraction.test.mjs +441 -441
- package/agent/extensions/pi-web-access/test/config-path.test.mjs +283 -283
- package/agent/extensions/pi-web-access/test/content-find.test.mjs +25 -25
- package/agent/extensions/pi-web-access/test/credential-source.test.mjs +118 -118
- package/agent/extensions/pi-web-access/test/data-uri-sanitize.test.mjs +210 -210
- package/agent/extensions/pi-web-access/test/datalab-pdf-extract.test.mjs +552 -552
- package/agent/extensions/pi-web-access/test/declared-web-links.test.mjs +212 -212
- package/agent/extensions/pi-web-access/test/fetch-answer-storage.test.mjs +40 -40
- package/agent/extensions/pi-web-access/test/fetch-cache-storage.test.mjs +334 -334
- package/agent/extensions/pi-web-access/test/fetch-content-domain-policy.test.mjs +95 -95
- package/agent/extensions/pi-web-access/test/fetch-modes.test.mjs +53 -53
- package/agent/extensions/pi-web-access/test/fetch-not-found-guidance.test.mjs +92 -92
- package/agent/extensions/pi-web-access/test/fetch-params.test.mjs +86 -86
- package/agent/extensions/pi-web-access/test/fetch-render-call.test.mjs +34 -34
- package/agent/extensions/pi-web-access/test/fetch-routing.test.mjs +173 -173
- package/agent/extensions/pi-web-access/test/gemini-adc-auth.test.mjs +257 -257
- package/agent/extensions/pi-web-access/test/gemini-api-transport.test.mjs +170 -170
- package/agent/extensions/pi-web-access/test/gemini-pdf-extract.test.mjs +133 -133
- package/agent/extensions/pi-web-access/test/gemini-web-cookie-opt-in.test.mjs +178 -178
- package/agent/extensions/pi-web-access/test/gemini-web-header-overflow.test.mjs +148 -148
- package/agent/extensions/pi-web-access/test/get-search-content.test.mjs +223 -223
- package/agent/extensions/pi-web-access/test/github-extract.test.mjs +378 -378
- package/agent/extensions/pi-web-access/test/github-issue-pr.test.mjs +565 -565
- package/agent/extensions/pi-web-access/test/inline-content-config.test.mjs +99 -99
- package/agent/extensions/pi-web-access/test/lazy-extract-load.test.mjs +118 -118
- package/agent/extensions/pi-web-access/test/local-video-oversize.test.mjs +52 -52
- package/agent/extensions/pi-web-access/test/package-typebox-dependency.test.mjs +50 -50
- package/agent/extensions/pi-web-access/test/page-query.test.mjs +51 -51
- package/agent/extensions/pi-web-access/test/pdf-config.test.mjs +140 -140
- package/agent/extensions/pi-web-access/test/pdf-extract.test.mjs +500 -500
- package/agent/extensions/pi-web-access/test/proxy-transport.test.mjs +286 -286
- package/agent/extensions/pi-web-access/test/query-rewrite.test.mjs +52 -52
- package/agent/extensions/pi-web-access/test/rsc-fallback.test.mjs +102 -102
- package/agent/extensions/pi-web-access/test/search-error-render.test.mjs +152 -152
- package/agent/extensions/pi-web-access/test/search-providers.test.mjs +274 -274
- package/agent/extensions/pi-web-access/test/source-check.test.mjs +179 -179
- package/agent/extensions/pi-web-access/test/ssrf-allow-ranges-config.test.mjs +205 -205
- package/agent/extensions/pi-web-access/test/ssrf-protection.test.mjs +456 -456
- package/agent/extensions/pi-web-access/test/tool-registration-config.test.mjs +182 -182
- package/agent/extensions/pi-web-access/test/youtube-extract-errors.test.mjs +64 -64
- package/agent/extensions/pi-web-access/tsconfig.json +11 -11
- package/agent/extensions/pi-web-access/utils.ts +451 -451
- package/agent/extensions/pi-web-access/video-extract.ts +392 -392
- package/agent/extensions/pi-web-access/youtube-extract.ts +328 -328
- package/agent/extensions/shared/activity-status.ts +31 -31
- package/agent/extensions/shared/child-session.test.ts +270 -270
- package/agent/extensions/shared/child-session.ts +148 -148
- package/agent/extensions/shared/context-utilization.test.ts +48 -48
- package/agent/extensions/shared/context-utilization.ts +47 -47
- package/agent/extensions/shared/dashboard-state.ts +99 -99
- package/agent/extensions/shared/tool-call-timeout.test.ts +117 -117
- package/agent/extensions/shared/tool-call-timeout.ts +104 -104
- package/agent/extensions/subagents/by-the-way.test.ts +29 -29
- package/agent/extensions/subagents/claude.test.ts +119 -119
- package/agent/extensions/subagents/codex.test.ts +102 -102
- package/agent/extensions/subagents/context-usage.test.ts +107 -107
- package/agent/extensions/subagents/docs/design-plan.md +568 -568
- package/agent/extensions/subagents/docs/effect-v4-extension-guide.md +354 -354
- package/agent/extensions/subagents/docs/effect-v4-notes.md +571 -571
- package/agent/extensions/subagents/index.ts +779 -779
- package/agent/extensions/subagents/manager.test.ts +276 -276
- package/agent/extensions/subagents/package-lock.json +2244 -2244
- package/agent/extensions/subagents/package.json +19 -19
- package/agent/extensions/subagents/result-delivery.test.ts +27 -27
- package/agent/extensions/subagents/src/backend.ts +73 -73
- package/agent/extensions/subagents/src/backends/claude.ts +701 -701
- package/agent/extensions/subagents/src/backends/codex.ts +1060 -1060
- package/agent/extensions/subagents/src/backends/pi.ts +575 -575
- package/agent/extensions/subagents/src/backends/stub.ts +300 -300
- package/agent/extensions/subagents/src/by-the-way.ts +21 -21
- package/agent/extensions/subagents/src/domain.ts +253 -253
- package/agent/extensions/subagents/src/format.ts +74 -74
- package/agent/extensions/subagents/src/manager.ts +736 -736
- package/agent/extensions/subagents/src/prompt.ts +92 -92
- package/agent/extensions/subagents/src/result-delivery.ts +20 -20
- package/agent/extensions/subagents/src/runtime.ts +53 -53
- package/agent/extensions/subagents/src/ui/takeover.ts +583 -583
- package/agent/extensions/subagents/src/ui/transcript.ts +201 -201
- package/agent/extensions/subagents/takeover.test.ts +29 -29
- package/agent/extensions/subagents/tsconfig.json +7 -7
- package/agent/extensions/todo/AGENTS.md +38 -38
- package/agent/extensions/todo/LICENSE +21 -21
- package/agent/extensions/todo/config.ts +55 -55
- package/agent/extensions/todo/index.ts +151 -151
- package/agent/extensions/todo/locales/de.json +17 -17
- package/agent/extensions/todo/locales/en.json +15 -15
- package/agent/extensions/todo/locales/es.json +17 -17
- package/agent/extensions/todo/locales/fr.json +17 -17
- package/agent/extensions/todo/locales/pt-BR.json +17 -17
- package/agent/extensions/todo/locales/pt.json +17 -17
- package/agent/extensions/todo/locales/ru.json +17 -17
- package/agent/extensions/todo/locales/uk.json +17 -17
- package/agent/extensions/todo/locales/zh.json +17 -17
- package/agent/extensions/todo/package-lock.json +3358 -3358
- package/agent/extensions/todo/package.json +67 -67
- package/agent/extensions/todo/state/i18n-bridge.ts +64 -64
- package/agent/extensions/todo/state/invariants.ts +20 -20
- package/agent/extensions/todo/state/replay.ts +38 -38
- package/agent/extensions/todo/state/selectors.ts +107 -107
- package/agent/extensions/todo/state/state-reducer.ts +326 -326
- package/agent/extensions/todo/state/state.ts +18 -18
- package/agent/extensions/todo/state/store.ts +82 -82
- package/agent/extensions/todo/state/task-graph.ts +57 -57
- package/agent/extensions/todo/todo-overlay.ts +200 -200
- package/agent/extensions/todo/todo.ts +155 -155
- package/agent/extensions/todo/tool/response-envelope.ts +109 -109
- package/agent/extensions/todo/tool/types.ts +206 -206
- package/agent/extensions/todo/view/format.ts +177 -177
- package/agent/install.ps1 +637 -527
- package/agent/install.sh +620 -511
- package/agent/keybindings.json +7 -7
- package/bin/kiln.js +124 -11
- package/package.json +8 -2
- package/agent/extensions/taste/index.ts +0 -443
- package/agent/extensions/taste/install.ps1 +0 -23
- package/agent/extensions/taste/install.sh +0 -21
- /package/agent/extensions/{status line → statusline}/index.ts +0 -0
- /package/agent/extensions/{status line → statusline}/install.ps1 +0 -0
- /package/agent/extensions/{status line → statusline}/install.sh +0 -0
|
@@ -1,526 +1,526 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
-
import { activityMonitor } from "./activity.ts";
|
|
3
|
-
import type { ExtractedContent } from "./extract.ts";
|
|
4
|
-
import type { SearchOptions, SearchResponse } from "./search-types.ts";
|
|
5
|
-
import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
|
|
6
|
-
import { fetchWithCredentialRedirects, getWebSearchConfigPath, resolveApiBaseUrl } from "./utils.ts";
|
|
7
|
-
|
|
8
|
-
const EXA_API_BASE_URL = "https://api.exa.ai";
|
|
9
|
-
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
|
|
10
|
-
const CONFIG_PATH = getWebSearchConfigPath();
|
|
11
|
-
const EXA_MCP_ADVANCED_TOOL = "web_search_advanced_exa";
|
|
12
|
-
const EXA_MCP_BASIC_TOOL = "web_search_exa";
|
|
13
|
-
|
|
14
|
-
interface WebSearchConfig {
|
|
15
|
-
exaApiKey?: unknown;
|
|
16
|
-
exaBaseUrl?: unknown;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
interface ExaAnswerResponse {
|
|
20
|
-
answer?: string;
|
|
21
|
-
citations?: Array<{ url?: string; title?: string; text?: string; publishedDate?: string }>;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
interface ExaSearchResponse {
|
|
25
|
-
results?: Array<{
|
|
26
|
-
title?: string;
|
|
27
|
-
url?: string;
|
|
28
|
-
publishedDate?: string;
|
|
29
|
-
author?: string;
|
|
30
|
-
text?: string;
|
|
31
|
-
highlights?: unknown;
|
|
32
|
-
highlightScores?: number[];
|
|
33
|
-
}>;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
interface ExaMcpRpcResponse {
|
|
37
|
-
result?: {
|
|
38
|
-
content?: Array<{ type?: string; text?: string }>;
|
|
39
|
-
isError?: boolean;
|
|
40
|
-
};
|
|
41
|
-
error?: {
|
|
42
|
-
code?: number;
|
|
43
|
-
message?: string;
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export type ExaSearchResult = SearchResponse | null;
|
|
48
|
-
|
|
49
|
-
export interface ExaSearchOptions extends SearchOptions {
|
|
50
|
-
includeContent?: boolean;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
type McpParsedResult = { title: string; url: string; content: string };
|
|
54
|
-
|
|
55
|
-
let cachedConfig: WebSearchConfig | null = null;
|
|
56
|
-
|
|
57
|
-
function loadConfig(): WebSearchConfig {
|
|
58
|
-
if (cachedConfig) return cachedConfig;
|
|
59
|
-
if (!existsSync(CONFIG_PATH)) {
|
|
60
|
-
cachedConfig = {};
|
|
61
|
-
return cachedConfig;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
const raw = readFileSync(CONFIG_PATH, "utf-8");
|
|
65
|
-
try {
|
|
66
|
-
cachedConfig = JSON.parse(raw) as WebSearchConfig;
|
|
67
|
-
return cachedConfig;
|
|
68
|
-
} catch (err) {
|
|
69
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
70
|
-
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async function getApiKey(signal?: AbortSignal): Promise<string | null> {
|
|
75
|
-
return resolveCredential({
|
|
76
|
-
provider: "Exa",
|
|
77
|
-
configuredValue: loadConfig().exaApiKey,
|
|
78
|
-
environmentValue: process.env.EXA_API_KEY,
|
|
79
|
-
signal,
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
function getApiBaseUrl(): string {
|
|
84
|
-
return resolveApiBaseUrl({
|
|
85
|
-
configKey: "exaBaseUrl",
|
|
86
|
-
configuredValue: loadConfig().exaBaseUrl,
|
|
87
|
-
defaultValue: EXA_API_BASE_URL,
|
|
88
|
-
environmentKey: "EXA_BASE_URL",
|
|
89
|
-
environmentValue: process.env.EXA_BASE_URL,
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function exaApiHeaders(apiKey: string): Record<string, string> {
|
|
94
|
-
return {
|
|
95
|
-
"x-api-key": apiKey,
|
|
96
|
-
"Content-Type": "application/json",
|
|
97
|
-
"x-exa-integration": "pi-web-access",
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
function requestSignal(signal?: AbortSignal): AbortSignal {
|
|
102
|
-
const timeout = AbortSignal.timeout(60000);
|
|
103
|
-
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function recencyToStartDate(filter: string): string {
|
|
107
|
-
const now = new Date();
|
|
108
|
-
const offsets: Record<string, number> = {
|
|
109
|
-
day: 1,
|
|
110
|
-
week: 7,
|
|
111
|
-
month: 30,
|
|
112
|
-
year: 365,
|
|
113
|
-
};
|
|
114
|
-
const days = offsets[filter] ?? 0;
|
|
115
|
-
return new Date(now.getTime() - days * 86400000).toISOString();
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
function mapDomainFilter(domainFilter: string[] | undefined): { includeDomains?: string[]; excludeDomains?: string[] } {
|
|
119
|
-
if (!domainFilter?.length) return {};
|
|
120
|
-
const includeDomains = domainFilter
|
|
121
|
-
.filter(d => !d.startsWith("-") && d.trim().length > 0)
|
|
122
|
-
.map(d => d.trim());
|
|
123
|
-
const excludeDomains = domainFilter
|
|
124
|
-
.filter(d => d.startsWith("-"))
|
|
125
|
-
.map(d => d.slice(1).trim())
|
|
126
|
-
.filter(Boolean);
|
|
127
|
-
return {
|
|
128
|
-
...(includeDomains.length ? { includeDomains } : {}),
|
|
129
|
-
...(excludeDomains.length ? { excludeDomains } : {}),
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function exaSearchArgs(query: string, options: ExaSearchOptions): Record<string, unknown> {
|
|
134
|
-
const startDate = options.recencyFilter ? recencyToStartDate(options.recencyFilter) : null;
|
|
135
|
-
return {
|
|
136
|
-
query,
|
|
137
|
-
type: "auto",
|
|
138
|
-
numResults: options.numResults ?? 5,
|
|
139
|
-
...mapDomainFilter(options.domainFilter),
|
|
140
|
-
...(startDate ? { startPublishedDate: startDate } : {}),
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function normalizeHighlights(value: unknown): string[] {
|
|
145
|
-
if (!Array.isArray(value)) return [];
|
|
146
|
-
return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function buildAnswerFromSearchResults(results: ExaSearchResponse["results"]): string {
|
|
150
|
-
if (!results?.length) return "";
|
|
151
|
-
const parts: string[] = [];
|
|
152
|
-
for (let i = 0; i < results.length; i++) {
|
|
153
|
-
const item = results[i];
|
|
154
|
-
if (!item?.url) continue;
|
|
155
|
-
const highlights = normalizeHighlights(item.highlights);
|
|
156
|
-
const content = highlights.length > 0
|
|
157
|
-
? highlights.join(" ")
|
|
158
|
-
: typeof item.text === "string" ? item.text.trim().slice(0, 1000) : "";
|
|
159
|
-
if (!content) continue;
|
|
160
|
-
const sourceTitle = item.title || `Source ${i + 1}`;
|
|
161
|
-
parts.push(`${content}\nSource: ${sourceTitle} (${item.url})`);
|
|
162
|
-
}
|
|
163
|
-
return parts.join("\n\n");
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
function mapResults(results: ExaSearchResponse["results"] | ExaAnswerResponse["citations"]): SearchResponse["results"] {
|
|
167
|
-
if (!Array.isArray(results)) return [];
|
|
168
|
-
const mapped: SearchResponse["results"] = [];
|
|
169
|
-
for (let i = 0; i < results.length; i++) {
|
|
170
|
-
const item = results[i];
|
|
171
|
-
if (!item?.url) continue;
|
|
172
|
-
mapped.push({
|
|
173
|
-
title: item.title || `Source ${i + 1}`,
|
|
174
|
-
url: item.url,
|
|
175
|
-
snippet: "",
|
|
176
|
-
});
|
|
177
|
-
}
|
|
178
|
-
return mapped;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
function mapInlineContent(results: ExaSearchResponse["results"]): ExtractedContent[] {
|
|
182
|
-
if (!results?.length) return [];
|
|
183
|
-
return results
|
|
184
|
-
.filter((r): r is NonNullable<ExaSearchResponse["results"]>[number] & { url: string; text: string } =>
|
|
185
|
-
!!r?.url && typeof r.text === "string" && r.text.length > 0)
|
|
186
|
-
.map(r => ({
|
|
187
|
-
url: r.url,
|
|
188
|
-
title: r.title || "",
|
|
189
|
-
content: r.text,
|
|
190
|
-
error: null,
|
|
191
|
-
}));
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function toSearchResponse(
|
|
195
|
-
answer: string,
|
|
196
|
-
results: SearchResponse["results"],
|
|
197
|
-
inlineContent: ExtractedContent[] | null,
|
|
198
|
-
): SearchResponse {
|
|
199
|
-
const response: SearchResponse = { answer, results };
|
|
200
|
-
if (inlineContent?.length) response.inlineContent = inlineContent;
|
|
201
|
-
return response;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
export async function callExaMcp(
|
|
205
|
-
toolName: string,
|
|
206
|
-
args: Record<string, unknown>,
|
|
207
|
-
signal?: AbortSignal,
|
|
208
|
-
): Promise<string> {
|
|
209
|
-
const response = await fetch(`${EXA_MCP_URL}?tools=${toolName}`, {
|
|
210
|
-
method: "POST",
|
|
211
|
-
headers: {
|
|
212
|
-
"Content-Type": "application/json",
|
|
213
|
-
"Accept": "application/json, text/event-stream",
|
|
214
|
-
"x-exa-source": "pi-web-access",
|
|
215
|
-
},
|
|
216
|
-
body: JSON.stringify({
|
|
217
|
-
jsonrpc: "2.0",
|
|
218
|
-
id: 1,
|
|
219
|
-
method: "tools/call",
|
|
220
|
-
params: {
|
|
221
|
-
name: toolName,
|
|
222
|
-
arguments: args,
|
|
223
|
-
},
|
|
224
|
-
}),
|
|
225
|
-
signal: requestSignal(signal),
|
|
226
|
-
});
|
|
227
|
-
|
|
228
|
-
if (!response.ok) {
|
|
229
|
-
const errorText = await response.text();
|
|
230
|
-
if (response.status === 429) {
|
|
231
|
-
throw new Error(
|
|
232
|
-
`Exa MCP rate limit reached (429). Add "exaApiKey" to ${CONFIG_PATH} for unthrottled Exa search: ${errorText.slice(0, 200)}`,
|
|
233
|
-
);
|
|
234
|
-
}
|
|
235
|
-
throw new Error(`Exa MCP error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
const body = await response.text();
|
|
239
|
-
const dataLines = body.split("\n").filter(line => line.startsWith("data:"));
|
|
240
|
-
|
|
241
|
-
let parsed: ExaMcpRpcResponse | null = null;
|
|
242
|
-
for (const line of dataLines) {
|
|
243
|
-
const payload = line.slice(5).trim();
|
|
244
|
-
if (!payload) continue;
|
|
245
|
-
try {
|
|
246
|
-
const candidate = JSON.parse(payload) as ExaMcpRpcResponse;
|
|
247
|
-
if (candidate?.result || candidate?.error) {
|
|
248
|
-
parsed = candidate;
|
|
249
|
-
break;
|
|
250
|
-
}
|
|
251
|
-
} catch {
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
if (!parsed) {
|
|
256
|
-
try {
|
|
257
|
-
const candidate = JSON.parse(body) as ExaMcpRpcResponse;
|
|
258
|
-
if (candidate?.result || candidate?.error) {
|
|
259
|
-
parsed = candidate;
|
|
260
|
-
}
|
|
261
|
-
} catch {
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
if (!parsed) {
|
|
266
|
-
throw new Error("Exa MCP returned an empty response");
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
if (parsed.error) {
|
|
270
|
-
const code = typeof parsed.error.code === "number" ? ` ${parsed.error.code}` : "";
|
|
271
|
-
const message = parsed.error.message || "Unknown error";
|
|
272
|
-
throw new Error(`Exa MCP error${code}: ${message}`);
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
if (parsed.result?.isError) {
|
|
276
|
-
const message = parsed.result.content
|
|
277
|
-
?.find(item => item.type === "text" && typeof item.text === "string")
|
|
278
|
-
?.text?.trim();
|
|
279
|
-
throw new Error(message || "Exa MCP returned an error");
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
const text = parsed.result?.content
|
|
283
|
-
?.find(item => item.type === "text" && typeof item.text === "string" && item.text.trim().length > 0)
|
|
284
|
-
?.text;
|
|
285
|
-
|
|
286
|
-
if (!text) {
|
|
287
|
-
throw new Error("Exa MCP returned empty content");
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
return text;
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
function parseMcpResults(text: string): McpParsedResult[] | null {
|
|
294
|
-
const blocks = text.split(/(?=^Title: )/m).filter(block => block.trim().length > 0);
|
|
295
|
-
const parsed = blocks.map(block => {
|
|
296
|
-
const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? "";
|
|
297
|
-
const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? "";
|
|
298
|
-
let content = "";
|
|
299
|
-
const textStart = block.indexOf("\nText: ");
|
|
300
|
-
if (textStart >= 0) {
|
|
301
|
-
content = block.slice(textStart + 7).trim();
|
|
302
|
-
} else {
|
|
303
|
-
const hlMatch = block.match(/\nHighlights:\s*\n/);
|
|
304
|
-
if (hlMatch?.index != null) {
|
|
305
|
-
content = block.slice(hlMatch.index + hlMatch[0].length).trim();
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
content = content.replace(/\n---\s*$/, "").trim();
|
|
309
|
-
return { title, url, content };
|
|
310
|
-
}).filter(result => result.url.length > 0);
|
|
311
|
-
return parsed.length > 0 ? parsed : null;
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
function buildAnswerFromMcpResults(results: McpParsedResult[]): string {
|
|
315
|
-
if (results.length === 0) return "";
|
|
316
|
-
const parts: string[] = [];
|
|
317
|
-
for (let i = 0; i < results.length; i++) {
|
|
318
|
-
const result = results[i];
|
|
319
|
-
const snippet = result.content.replace(/\s+/g, " ").trim().slice(0, 500);
|
|
320
|
-
if (!snippet) continue;
|
|
321
|
-
const sourceTitle = result.title || `Source ${i + 1}`;
|
|
322
|
-
parts.push(`${snippet}\nSource: ${sourceTitle} (${result.url})`);
|
|
323
|
-
}
|
|
324
|
-
return parts.join("\n\n");
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
function mapMcpInlineContent(results: McpParsedResult[]): ExtractedContent[] {
|
|
328
|
-
return results
|
|
329
|
-
.filter(result => result.content.length > 0)
|
|
330
|
-
.map(result => ({
|
|
331
|
-
url: result.url,
|
|
332
|
-
title: result.title,
|
|
333
|
-
content: result.content,
|
|
334
|
-
error: null,
|
|
335
|
-
}));
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
function buildMcpQuery(query: string, options: ExaSearchOptions): string {
|
|
339
|
-
const parts = [query];
|
|
340
|
-
if (options.domainFilter?.length) {
|
|
341
|
-
for (const d of options.domainFilter) {
|
|
342
|
-
parts.push(d.startsWith("-") ? `-site:${d.slice(1)}` : `site:${d}`);
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
if (options.recencyFilter) {
|
|
346
|
-
const now = new Date();
|
|
347
|
-
switch (options.recencyFilter) {
|
|
348
|
-
case "day": parts.push("past 24 hours"); break;
|
|
349
|
-
case "week": parts.push("past week"); break;
|
|
350
|
-
case "month": parts.push(`${now.toLocaleString("en", { month: "long" })} ${now.getFullYear()}`); break;
|
|
351
|
-
case "year": parts.push(String(now.getFullYear())); break;
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
return parts.join(" ");
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
function isAbortMessage(message: string): boolean {
|
|
358
|
-
return message.toLowerCase().includes("abort");
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
function parseJsonMcpResults(text: string): ExaSearchResponse["results"] | null {
|
|
362
|
-
try {
|
|
363
|
-
const results = (JSON.parse(text) as ExaSearchResponse).results;
|
|
364
|
-
return Array.isArray(results) && results.length > 0 ? results : null;
|
|
365
|
-
} catch {
|
|
366
|
-
return null;
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
/**
|
|
371
|
-
* Calls one Exa MCP search tool and normalizes its payload. `web_search_advanced_exa`
|
|
372
|
-
* returns the raw Exa search JSON; `web_search_exa` returns a formatted text block.
|
|
373
|
-
*/
|
|
374
|
-
async function searchWithExaMcpTool(
|
|
375
|
-
tool: string,
|
|
376
|
-
args: Record<string, unknown>,
|
|
377
|
-
options: ExaSearchOptions,
|
|
378
|
-
): Promise<SearchResponse | null> {
|
|
379
|
-
const text = await callExaMcp(tool, args, options.signal);
|
|
380
|
-
|
|
381
|
-
const jsonResults = parseJsonMcpResults(text);
|
|
382
|
-
if (jsonResults) {
|
|
383
|
-
return toSearchResponse(
|
|
384
|
-
buildAnswerFromSearchResults(jsonResults),
|
|
385
|
-
mapResults(jsonResults),
|
|
386
|
-
options.includeContent ? mapInlineContent(jsonResults) : null,
|
|
387
|
-
);
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
const textResults = parseMcpResults(text);
|
|
391
|
-
if (!textResults) return null;
|
|
392
|
-
|
|
393
|
-
return toSearchResponse(
|
|
394
|
-
buildAnswerFromMcpResults(textResults),
|
|
395
|
-
mapResults(textResults),
|
|
396
|
-
options.includeContent ? mapMcpInlineContent(textResults) : null,
|
|
397
|
-
);
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
/** Filtered searches need the advanced tool, which not every deployment exposes. */
|
|
401
|
-
async function searchWithFilteredExaMcp(
|
|
402
|
-
query: string,
|
|
403
|
-
options: ExaSearchOptions,
|
|
404
|
-
basicArgs: Record<string, unknown>,
|
|
405
|
-
): Promise<SearchResponse | null> {
|
|
406
|
-
try {
|
|
407
|
-
return await searchWithExaMcpTool(EXA_MCP_ADVANCED_TOOL, {
|
|
408
|
-
...exaSearchArgs(query, options),
|
|
409
|
-
enableHighlights: true,
|
|
410
|
-
textMaxCharacters: options.includeContent ? 50000 : 3000,
|
|
411
|
-
}, options);
|
|
412
|
-
} catch (err) {
|
|
413
|
-
if (isAbortMessage(err instanceof Error ? err.message : String(err))) throw err;
|
|
414
|
-
// The basic tool ignores every argument except query/numResults, so the
|
|
415
|
-
// filters degrade into the query text.
|
|
416
|
-
return searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
async function searchWithExaMcp(query: string, options: ExaSearchOptions = {}): Promise<SearchResponse | null> {
|
|
421
|
-
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
422
|
-
const basicArgs = { query: buildMcpQuery(query, options), numResults: options.numResults ?? 5 };
|
|
423
|
-
const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length;
|
|
424
|
-
|
|
425
|
-
try {
|
|
426
|
-
const response = filtered
|
|
427
|
-
? await searchWithFilteredExaMcp(query, options, basicArgs)
|
|
428
|
-
: await searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
|
|
429
|
-
activityMonitor.logComplete(activityId, 200);
|
|
430
|
-
return response;
|
|
431
|
-
} catch (err) {
|
|
432
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
433
|
-
if (isAbortMessage(message)) {
|
|
434
|
-
activityMonitor.logComplete(activityId, 0);
|
|
435
|
-
} else {
|
|
436
|
-
activityMonitor.logError(activityId, message);
|
|
437
|
-
}
|
|
438
|
-
throw err;
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
export function isExaAvailable(): boolean {
|
|
443
|
-
return true;
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
export function hasExaApiKey(): boolean {
|
|
447
|
-
return hasCredentialSource({
|
|
448
|
-
provider: "Exa",
|
|
449
|
-
configuredValue: loadConfig().exaApiKey,
|
|
450
|
-
environmentValue: process.env.EXA_API_KEY,
|
|
451
|
-
});
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
export async function searchWithExa(query: string, options: ExaSearchOptions = {}): Promise<ExaSearchResult> {
|
|
455
|
-
const apiKey = await getApiKey(options.signal);
|
|
456
|
-
if (!apiKey) {
|
|
457
|
-
return searchWithExaMcp(query, options);
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
const apiBaseUrl = getApiBaseUrl();
|
|
461
|
-
const useSearch = options.includeContent
|
|
462
|
-
|| !!options.recencyFilter
|
|
463
|
-
|| !!options.domainFilter?.length
|
|
464
|
-
|| !!(options.numResults && options.numResults !== 5);
|
|
465
|
-
|
|
466
|
-
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
467
|
-
|
|
468
|
-
try {
|
|
469
|
-
if (!useSearch) {
|
|
470
|
-
const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/answer`, {
|
|
471
|
-
method: "POST",
|
|
472
|
-
headers: exaApiHeaders(apiKey),
|
|
473
|
-
body: JSON.stringify({ query }),
|
|
474
|
-
signal: requestSignal(options.signal),
|
|
475
|
-
}, ["x-api-key"]);
|
|
476
|
-
|
|
477
|
-
if (!response.ok) {
|
|
478
|
-
const errorText = redactCredential(await response.text(), apiKey);
|
|
479
|
-
throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
const data = await response.json() as ExaAnswerResponse;
|
|
483
|
-
activityMonitor.logComplete(activityId, response.status);
|
|
484
|
-
return {
|
|
485
|
-
answer: data.answer || "",
|
|
486
|
-
results: mapResults(data.citations),
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/search`, {
|
|
491
|
-
method: "POST",
|
|
492
|
-
headers: exaApiHeaders(apiKey),
|
|
493
|
-
body: JSON.stringify({
|
|
494
|
-
...exaSearchArgs(query, options),
|
|
495
|
-
contents: options.includeContent
|
|
496
|
-
? { text: true, highlights: true }
|
|
497
|
-
: { highlights: true },
|
|
498
|
-
}),
|
|
499
|
-
signal: requestSignal(options.signal),
|
|
500
|
-
}, ["x-api-key"]);
|
|
501
|
-
|
|
502
|
-
if (!response.ok) {
|
|
503
|
-
const errorText = redactCredential(await response.text(), apiKey);
|
|
504
|
-
throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
const data = await response.json() as ExaSearchResponse;
|
|
508
|
-
activityMonitor.logComplete(activityId, response.status);
|
|
509
|
-
|
|
510
|
-
return toSearchResponse(
|
|
511
|
-
buildAnswerFromSearchResults(data.results),
|
|
512
|
-
mapResults(data.results),
|
|
513
|
-
options.includeContent ? mapInlineContent(data.results) : null,
|
|
514
|
-
);
|
|
515
|
-
} catch (err) {
|
|
516
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
517
|
-
const redactedMessage = redactCredential(message, apiKey);
|
|
518
|
-
if (isAbortMessage(redactedMessage)) {
|
|
519
|
-
activityMonitor.logComplete(activityId, 0);
|
|
520
|
-
} else {
|
|
521
|
-
activityMonitor.logError(activityId, redactedMessage);
|
|
522
|
-
}
|
|
523
|
-
if (redactedMessage === message) throw err;
|
|
524
|
-
throw new Error(redactedMessage);
|
|
525
|
-
}
|
|
526
|
-
}
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { activityMonitor } from "./activity.ts";
|
|
3
|
+
import type { ExtractedContent } from "./extract.ts";
|
|
4
|
+
import type { SearchOptions, SearchResponse } from "./search-types.ts";
|
|
5
|
+
import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
|
|
6
|
+
import { fetchWithCredentialRedirects, getWebSearchConfigPath, resolveApiBaseUrl } from "./utils.ts";
|
|
7
|
+
|
|
8
|
+
const EXA_API_BASE_URL = "https://api.exa.ai";
|
|
9
|
+
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
|
|
10
|
+
const CONFIG_PATH = getWebSearchConfigPath();
|
|
11
|
+
const EXA_MCP_ADVANCED_TOOL = "web_search_advanced_exa";
|
|
12
|
+
const EXA_MCP_BASIC_TOOL = "web_search_exa";
|
|
13
|
+
|
|
14
|
+
interface WebSearchConfig {
|
|
15
|
+
exaApiKey?: unknown;
|
|
16
|
+
exaBaseUrl?: unknown;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface ExaAnswerResponse {
|
|
20
|
+
answer?: string;
|
|
21
|
+
citations?: Array<{ url?: string; title?: string; text?: string; publishedDate?: string }>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface ExaSearchResponse {
|
|
25
|
+
results?: Array<{
|
|
26
|
+
title?: string;
|
|
27
|
+
url?: string;
|
|
28
|
+
publishedDate?: string;
|
|
29
|
+
author?: string;
|
|
30
|
+
text?: string;
|
|
31
|
+
highlights?: unknown;
|
|
32
|
+
highlightScores?: number[];
|
|
33
|
+
}>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
interface ExaMcpRpcResponse {
|
|
37
|
+
result?: {
|
|
38
|
+
content?: Array<{ type?: string; text?: string }>;
|
|
39
|
+
isError?: boolean;
|
|
40
|
+
};
|
|
41
|
+
error?: {
|
|
42
|
+
code?: number;
|
|
43
|
+
message?: string;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type ExaSearchResult = SearchResponse | null;
|
|
48
|
+
|
|
49
|
+
export interface ExaSearchOptions extends SearchOptions {
|
|
50
|
+
includeContent?: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
type McpParsedResult = { title: string; url: string; content: string };
|
|
54
|
+
|
|
55
|
+
let cachedConfig: WebSearchConfig | null = null;
|
|
56
|
+
|
|
57
|
+
function loadConfig(): WebSearchConfig {
|
|
58
|
+
if (cachedConfig) return cachedConfig;
|
|
59
|
+
if (!existsSync(CONFIG_PATH)) {
|
|
60
|
+
cachedConfig = {};
|
|
61
|
+
return cachedConfig;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const raw = readFileSync(CONFIG_PATH, "utf-8");
|
|
65
|
+
try {
|
|
66
|
+
cachedConfig = JSON.parse(raw) as WebSearchConfig;
|
|
67
|
+
return cachedConfig;
|
|
68
|
+
} catch (err) {
|
|
69
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
70
|
+
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function getApiKey(signal?: AbortSignal): Promise<string | null> {
|
|
75
|
+
return resolveCredential({
|
|
76
|
+
provider: "Exa",
|
|
77
|
+
configuredValue: loadConfig().exaApiKey,
|
|
78
|
+
environmentValue: process.env.EXA_API_KEY,
|
|
79
|
+
signal,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function getApiBaseUrl(): string {
|
|
84
|
+
return resolveApiBaseUrl({
|
|
85
|
+
configKey: "exaBaseUrl",
|
|
86
|
+
configuredValue: loadConfig().exaBaseUrl,
|
|
87
|
+
defaultValue: EXA_API_BASE_URL,
|
|
88
|
+
environmentKey: "EXA_BASE_URL",
|
|
89
|
+
environmentValue: process.env.EXA_BASE_URL,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function exaApiHeaders(apiKey: string): Record<string, string> {
|
|
94
|
+
return {
|
|
95
|
+
"x-api-key": apiKey,
|
|
96
|
+
"Content-Type": "application/json",
|
|
97
|
+
"x-exa-integration": "pi-web-access",
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function requestSignal(signal?: AbortSignal): AbortSignal {
|
|
102
|
+
const timeout = AbortSignal.timeout(60000);
|
|
103
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function recencyToStartDate(filter: string): string {
|
|
107
|
+
const now = new Date();
|
|
108
|
+
const offsets: Record<string, number> = {
|
|
109
|
+
day: 1,
|
|
110
|
+
week: 7,
|
|
111
|
+
month: 30,
|
|
112
|
+
year: 365,
|
|
113
|
+
};
|
|
114
|
+
const days = offsets[filter] ?? 0;
|
|
115
|
+
return new Date(now.getTime() - days * 86400000).toISOString();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function mapDomainFilter(domainFilter: string[] | undefined): { includeDomains?: string[]; excludeDomains?: string[] } {
|
|
119
|
+
if (!domainFilter?.length) return {};
|
|
120
|
+
const includeDomains = domainFilter
|
|
121
|
+
.filter(d => !d.startsWith("-") && d.trim().length > 0)
|
|
122
|
+
.map(d => d.trim());
|
|
123
|
+
const excludeDomains = domainFilter
|
|
124
|
+
.filter(d => d.startsWith("-"))
|
|
125
|
+
.map(d => d.slice(1).trim())
|
|
126
|
+
.filter(Boolean);
|
|
127
|
+
return {
|
|
128
|
+
...(includeDomains.length ? { includeDomains } : {}),
|
|
129
|
+
...(excludeDomains.length ? { excludeDomains } : {}),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function exaSearchArgs(query: string, options: ExaSearchOptions): Record<string, unknown> {
|
|
134
|
+
const startDate = options.recencyFilter ? recencyToStartDate(options.recencyFilter) : null;
|
|
135
|
+
return {
|
|
136
|
+
query,
|
|
137
|
+
type: "auto",
|
|
138
|
+
numResults: options.numResults ?? 5,
|
|
139
|
+
...mapDomainFilter(options.domainFilter),
|
|
140
|
+
...(startDate ? { startPublishedDate: startDate } : {}),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function normalizeHighlights(value: unknown): string[] {
|
|
145
|
+
if (!Array.isArray(value)) return [];
|
|
146
|
+
return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function buildAnswerFromSearchResults(results: ExaSearchResponse["results"]): string {
|
|
150
|
+
if (!results?.length) return "";
|
|
151
|
+
const parts: string[] = [];
|
|
152
|
+
for (let i = 0; i < results.length; i++) {
|
|
153
|
+
const item = results[i];
|
|
154
|
+
if (!item?.url) continue;
|
|
155
|
+
const highlights = normalizeHighlights(item.highlights);
|
|
156
|
+
const content = highlights.length > 0
|
|
157
|
+
? highlights.join(" ")
|
|
158
|
+
: typeof item.text === "string" ? item.text.trim().slice(0, 1000) : "";
|
|
159
|
+
if (!content) continue;
|
|
160
|
+
const sourceTitle = item.title || `Source ${i + 1}`;
|
|
161
|
+
parts.push(`${content}\nSource: ${sourceTitle} (${item.url})`);
|
|
162
|
+
}
|
|
163
|
+
return parts.join("\n\n");
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function mapResults(results: ExaSearchResponse["results"] | ExaAnswerResponse["citations"]): SearchResponse["results"] {
|
|
167
|
+
if (!Array.isArray(results)) return [];
|
|
168
|
+
const mapped: SearchResponse["results"] = [];
|
|
169
|
+
for (let i = 0; i < results.length; i++) {
|
|
170
|
+
const item = results[i];
|
|
171
|
+
if (!item?.url) continue;
|
|
172
|
+
mapped.push({
|
|
173
|
+
title: item.title || `Source ${i + 1}`,
|
|
174
|
+
url: item.url,
|
|
175
|
+
snippet: "",
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
return mapped;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function mapInlineContent(results: ExaSearchResponse["results"]): ExtractedContent[] {
|
|
182
|
+
if (!results?.length) return [];
|
|
183
|
+
return results
|
|
184
|
+
.filter((r): r is NonNullable<ExaSearchResponse["results"]>[number] & { url: string; text: string } =>
|
|
185
|
+
!!r?.url && typeof r.text === "string" && r.text.length > 0)
|
|
186
|
+
.map(r => ({
|
|
187
|
+
url: r.url,
|
|
188
|
+
title: r.title || "",
|
|
189
|
+
content: r.text,
|
|
190
|
+
error: null,
|
|
191
|
+
}));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function toSearchResponse(
|
|
195
|
+
answer: string,
|
|
196
|
+
results: SearchResponse["results"],
|
|
197
|
+
inlineContent: ExtractedContent[] | null,
|
|
198
|
+
): SearchResponse {
|
|
199
|
+
const response: SearchResponse = { answer, results };
|
|
200
|
+
if (inlineContent?.length) response.inlineContent = inlineContent;
|
|
201
|
+
return response;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export async function callExaMcp(
|
|
205
|
+
toolName: string,
|
|
206
|
+
args: Record<string, unknown>,
|
|
207
|
+
signal?: AbortSignal,
|
|
208
|
+
): Promise<string> {
|
|
209
|
+
const response = await fetch(`${EXA_MCP_URL}?tools=${toolName}`, {
|
|
210
|
+
method: "POST",
|
|
211
|
+
headers: {
|
|
212
|
+
"Content-Type": "application/json",
|
|
213
|
+
"Accept": "application/json, text/event-stream",
|
|
214
|
+
"x-exa-source": "pi-web-access",
|
|
215
|
+
},
|
|
216
|
+
body: JSON.stringify({
|
|
217
|
+
jsonrpc: "2.0",
|
|
218
|
+
id: 1,
|
|
219
|
+
method: "tools/call",
|
|
220
|
+
params: {
|
|
221
|
+
name: toolName,
|
|
222
|
+
arguments: args,
|
|
223
|
+
},
|
|
224
|
+
}),
|
|
225
|
+
signal: requestSignal(signal),
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
if (!response.ok) {
|
|
229
|
+
const errorText = await response.text();
|
|
230
|
+
if (response.status === 429) {
|
|
231
|
+
throw new Error(
|
|
232
|
+
`Exa MCP rate limit reached (429). Add "exaApiKey" to ${CONFIG_PATH} for unthrottled Exa search: ${errorText.slice(0, 200)}`,
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
throw new Error(`Exa MCP error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const body = await response.text();
|
|
239
|
+
const dataLines = body.split("\n").filter(line => line.startsWith("data:"));
|
|
240
|
+
|
|
241
|
+
let parsed: ExaMcpRpcResponse | null = null;
|
|
242
|
+
for (const line of dataLines) {
|
|
243
|
+
const payload = line.slice(5).trim();
|
|
244
|
+
if (!payload) continue;
|
|
245
|
+
try {
|
|
246
|
+
const candidate = JSON.parse(payload) as ExaMcpRpcResponse;
|
|
247
|
+
if (candidate?.result || candidate?.error) {
|
|
248
|
+
parsed = candidate;
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
} catch {
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
if (!parsed) {
|
|
256
|
+
try {
|
|
257
|
+
const candidate = JSON.parse(body) as ExaMcpRpcResponse;
|
|
258
|
+
if (candidate?.result || candidate?.error) {
|
|
259
|
+
parsed = candidate;
|
|
260
|
+
}
|
|
261
|
+
} catch {
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
if (!parsed) {
|
|
266
|
+
throw new Error("Exa MCP returned an empty response");
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (parsed.error) {
|
|
270
|
+
const code = typeof parsed.error.code === "number" ? ` ${parsed.error.code}` : "";
|
|
271
|
+
const message = parsed.error.message || "Unknown error";
|
|
272
|
+
throw new Error(`Exa MCP error${code}: ${message}`);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
if (parsed.result?.isError) {
|
|
276
|
+
const message = parsed.result.content
|
|
277
|
+
?.find(item => item.type === "text" && typeof item.text === "string")
|
|
278
|
+
?.text?.trim();
|
|
279
|
+
throw new Error(message || "Exa MCP returned an error");
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const text = parsed.result?.content
|
|
283
|
+
?.find(item => item.type === "text" && typeof item.text === "string" && item.text.trim().length > 0)
|
|
284
|
+
?.text;
|
|
285
|
+
|
|
286
|
+
if (!text) {
|
|
287
|
+
throw new Error("Exa MCP returned empty content");
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
return text;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function parseMcpResults(text: string): McpParsedResult[] | null {
|
|
294
|
+
const blocks = text.split(/(?=^Title: )/m).filter(block => block.trim().length > 0);
|
|
295
|
+
const parsed = blocks.map(block => {
|
|
296
|
+
const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? "";
|
|
297
|
+
const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? "";
|
|
298
|
+
let content = "";
|
|
299
|
+
const textStart = block.indexOf("\nText: ");
|
|
300
|
+
if (textStart >= 0) {
|
|
301
|
+
content = block.slice(textStart + 7).trim();
|
|
302
|
+
} else {
|
|
303
|
+
const hlMatch = block.match(/\nHighlights:\s*\n/);
|
|
304
|
+
if (hlMatch?.index != null) {
|
|
305
|
+
content = block.slice(hlMatch.index + hlMatch[0].length).trim();
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
content = content.replace(/\n---\s*$/, "").trim();
|
|
309
|
+
return { title, url, content };
|
|
310
|
+
}).filter(result => result.url.length > 0);
|
|
311
|
+
return parsed.length > 0 ? parsed : null;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function buildAnswerFromMcpResults(results: McpParsedResult[]): string {
|
|
315
|
+
if (results.length === 0) return "";
|
|
316
|
+
const parts: string[] = [];
|
|
317
|
+
for (let i = 0; i < results.length; i++) {
|
|
318
|
+
const result = results[i];
|
|
319
|
+
const snippet = result.content.replace(/\s+/g, " ").trim().slice(0, 500);
|
|
320
|
+
if (!snippet) continue;
|
|
321
|
+
const sourceTitle = result.title || `Source ${i + 1}`;
|
|
322
|
+
parts.push(`${snippet}\nSource: ${sourceTitle} (${result.url})`);
|
|
323
|
+
}
|
|
324
|
+
return parts.join("\n\n");
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function mapMcpInlineContent(results: McpParsedResult[]): ExtractedContent[] {
|
|
328
|
+
return results
|
|
329
|
+
.filter(result => result.content.length > 0)
|
|
330
|
+
.map(result => ({
|
|
331
|
+
url: result.url,
|
|
332
|
+
title: result.title,
|
|
333
|
+
content: result.content,
|
|
334
|
+
error: null,
|
|
335
|
+
}));
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function buildMcpQuery(query: string, options: ExaSearchOptions): string {
|
|
339
|
+
const parts = [query];
|
|
340
|
+
if (options.domainFilter?.length) {
|
|
341
|
+
for (const d of options.domainFilter) {
|
|
342
|
+
parts.push(d.startsWith("-") ? `-site:${d.slice(1)}` : `site:${d}`);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
if (options.recencyFilter) {
|
|
346
|
+
const now = new Date();
|
|
347
|
+
switch (options.recencyFilter) {
|
|
348
|
+
case "day": parts.push("past 24 hours"); break;
|
|
349
|
+
case "week": parts.push("past week"); break;
|
|
350
|
+
case "month": parts.push(`${now.toLocaleString("en", { month: "long" })} ${now.getFullYear()}`); break;
|
|
351
|
+
case "year": parts.push(String(now.getFullYear())); break;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
return parts.join(" ");
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function isAbortMessage(message: string): boolean {
|
|
358
|
+
return message.toLowerCase().includes("abort");
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function parseJsonMcpResults(text: string): ExaSearchResponse["results"] | null {
|
|
362
|
+
try {
|
|
363
|
+
const results = (JSON.parse(text) as ExaSearchResponse).results;
|
|
364
|
+
return Array.isArray(results) && results.length > 0 ? results : null;
|
|
365
|
+
} catch {
|
|
366
|
+
return null;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Calls one Exa MCP search tool and normalizes its payload. `web_search_advanced_exa`
|
|
372
|
+
* returns the raw Exa search JSON; `web_search_exa` returns a formatted text block.
|
|
373
|
+
*/
|
|
374
|
+
async function searchWithExaMcpTool(
|
|
375
|
+
tool: string,
|
|
376
|
+
args: Record<string, unknown>,
|
|
377
|
+
options: ExaSearchOptions,
|
|
378
|
+
): Promise<SearchResponse | null> {
|
|
379
|
+
const text = await callExaMcp(tool, args, options.signal);
|
|
380
|
+
|
|
381
|
+
const jsonResults = parseJsonMcpResults(text);
|
|
382
|
+
if (jsonResults) {
|
|
383
|
+
return toSearchResponse(
|
|
384
|
+
buildAnswerFromSearchResults(jsonResults),
|
|
385
|
+
mapResults(jsonResults),
|
|
386
|
+
options.includeContent ? mapInlineContent(jsonResults) : null,
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
const textResults = parseMcpResults(text);
|
|
391
|
+
if (!textResults) return null;
|
|
392
|
+
|
|
393
|
+
return toSearchResponse(
|
|
394
|
+
buildAnswerFromMcpResults(textResults),
|
|
395
|
+
mapResults(textResults),
|
|
396
|
+
options.includeContent ? mapMcpInlineContent(textResults) : null,
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** Filtered searches need the advanced tool, which not every deployment exposes. */
|
|
401
|
+
async function searchWithFilteredExaMcp(
|
|
402
|
+
query: string,
|
|
403
|
+
options: ExaSearchOptions,
|
|
404
|
+
basicArgs: Record<string, unknown>,
|
|
405
|
+
): Promise<SearchResponse | null> {
|
|
406
|
+
try {
|
|
407
|
+
return await searchWithExaMcpTool(EXA_MCP_ADVANCED_TOOL, {
|
|
408
|
+
...exaSearchArgs(query, options),
|
|
409
|
+
enableHighlights: true,
|
|
410
|
+
textMaxCharacters: options.includeContent ? 50000 : 3000,
|
|
411
|
+
}, options);
|
|
412
|
+
} catch (err) {
|
|
413
|
+
if (isAbortMessage(err instanceof Error ? err.message : String(err))) throw err;
|
|
414
|
+
// The basic tool ignores every argument except query/numResults, so the
|
|
415
|
+
// filters degrade into the query text.
|
|
416
|
+
return searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
async function searchWithExaMcp(query: string, options: ExaSearchOptions = {}): Promise<SearchResponse | null> {
|
|
421
|
+
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
422
|
+
const basicArgs = { query: buildMcpQuery(query, options), numResults: options.numResults ?? 5 };
|
|
423
|
+
const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length;
|
|
424
|
+
|
|
425
|
+
try {
|
|
426
|
+
const response = filtered
|
|
427
|
+
? await searchWithFilteredExaMcp(query, options, basicArgs)
|
|
428
|
+
: await searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
|
|
429
|
+
activityMonitor.logComplete(activityId, 200);
|
|
430
|
+
return response;
|
|
431
|
+
} catch (err) {
|
|
432
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
433
|
+
if (isAbortMessage(message)) {
|
|
434
|
+
activityMonitor.logComplete(activityId, 0);
|
|
435
|
+
} else {
|
|
436
|
+
activityMonitor.logError(activityId, message);
|
|
437
|
+
}
|
|
438
|
+
throw err;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
export function isExaAvailable(): boolean {
|
|
443
|
+
return true;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
export function hasExaApiKey(): boolean {
|
|
447
|
+
return hasCredentialSource({
|
|
448
|
+
provider: "Exa",
|
|
449
|
+
configuredValue: loadConfig().exaApiKey,
|
|
450
|
+
environmentValue: process.env.EXA_API_KEY,
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
export async function searchWithExa(query: string, options: ExaSearchOptions = {}): Promise<ExaSearchResult> {
|
|
455
|
+
const apiKey = await getApiKey(options.signal);
|
|
456
|
+
if (!apiKey) {
|
|
457
|
+
return searchWithExaMcp(query, options);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
const apiBaseUrl = getApiBaseUrl();
|
|
461
|
+
const useSearch = options.includeContent
|
|
462
|
+
|| !!options.recencyFilter
|
|
463
|
+
|| !!options.domainFilter?.length
|
|
464
|
+
|| !!(options.numResults && options.numResults !== 5);
|
|
465
|
+
|
|
466
|
+
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
467
|
+
|
|
468
|
+
try {
|
|
469
|
+
if (!useSearch) {
|
|
470
|
+
const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/answer`, {
|
|
471
|
+
method: "POST",
|
|
472
|
+
headers: exaApiHeaders(apiKey),
|
|
473
|
+
body: JSON.stringify({ query }),
|
|
474
|
+
signal: requestSignal(options.signal),
|
|
475
|
+
}, ["x-api-key"]);
|
|
476
|
+
|
|
477
|
+
if (!response.ok) {
|
|
478
|
+
const errorText = redactCredential(await response.text(), apiKey);
|
|
479
|
+
throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const data = await response.json() as ExaAnswerResponse;
|
|
483
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
484
|
+
return {
|
|
485
|
+
answer: data.answer || "",
|
|
486
|
+
results: mapResults(data.citations),
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/search`, {
|
|
491
|
+
method: "POST",
|
|
492
|
+
headers: exaApiHeaders(apiKey),
|
|
493
|
+
body: JSON.stringify({
|
|
494
|
+
...exaSearchArgs(query, options),
|
|
495
|
+
contents: options.includeContent
|
|
496
|
+
? { text: true, highlights: true }
|
|
497
|
+
: { highlights: true },
|
|
498
|
+
}),
|
|
499
|
+
signal: requestSignal(options.signal),
|
|
500
|
+
}, ["x-api-key"]);
|
|
501
|
+
|
|
502
|
+
if (!response.ok) {
|
|
503
|
+
const errorText = redactCredential(await response.text(), apiKey);
|
|
504
|
+
throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
const data = await response.json() as ExaSearchResponse;
|
|
508
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
509
|
+
|
|
510
|
+
return toSearchResponse(
|
|
511
|
+
buildAnswerFromSearchResults(data.results),
|
|
512
|
+
mapResults(data.results),
|
|
513
|
+
options.includeContent ? mapInlineContent(data.results) : null,
|
|
514
|
+
);
|
|
515
|
+
} catch (err) {
|
|
516
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
517
|
+
const redactedMessage = redactCredential(message, apiKey);
|
|
518
|
+
if (isAbortMessage(redactedMessage)) {
|
|
519
|
+
activityMonitor.logComplete(activityId, 0);
|
|
520
|
+
} else {
|
|
521
|
+
activityMonitor.logError(activityId, redactedMessage);
|
|
522
|
+
}
|
|
523
|
+
if (redactedMessage === message) throw err;
|
|
524
|
+
throw new Error(redactedMessage);
|
|
525
|
+
}
|
|
526
|
+
}
|