@nijaru/pi-search 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +165 -0
- package/THIRD_PARTY_NOTICES.md +31 -0
- package/package.json +69 -0
- package/src/anthropic.ts +350 -0
- package/src/anydoc-worker.mjs +35 -0
- package/src/anydoc.ts +161 -0
- package/src/brave-answers.ts +396 -0
- package/src/brave.ts +441 -0
- package/src/codex.ts +314 -0
- package/src/contracts.ts +644 -0
- package/src/direct-transport.ts +141 -0
- package/src/errors.ts +185 -0
- package/src/exa.ts +204 -0
- package/src/fetch-errors.ts +134 -0
- package/src/fetch-extractor-worker.mjs +85 -0
- package/src/fetch-tool.ts +166 -0
- package/src/fetcher.ts +451 -0
- package/src/fixtures/sample.csv +2 -0
- package/src/fixtures/sample.docx +0 -0
- package/src/fixtures/sample.epub +0 -0
- package/src/fixtures/sample.odt +0 -0
- package/src/fixtures/sample.pdf +32 -0
- package/src/fixtures/sample.pptx +0 -0
- package/src/fixtures/sample.rtf +1 -0
- package/src/fixtures/sample.xlsx +0 -0
- package/src/gemini.ts +298 -0
- package/src/grounding.ts +167 -0
- package/src/http.ts +78 -0
- package/src/index.ts +101 -0
- package/src/local-llama-compat.ts +134 -0
- package/src/meta.ts +137 -0
- package/src/model-selection.ts +134 -0
- package/src/openai.ts +850 -0
- package/src/parallel-responses.ts +260 -0
- package/src/parallel.ts +173 -0
- package/src/pdf.ts +153 -0
- package/src/provider-availability.ts +35 -0
- package/src/provider-http.ts +203 -0
- package/src/render-text.ts +8 -0
- package/src/research-tool.ts +487 -0
- package/src/router.ts +302 -0
- package/src/search-cleanup.ts +219 -0
- package/src/search-tool.ts +551 -0
- package/src/search.ts +420 -0
- package/src/ssrf.ts +302 -0
- package/src/url-rendering.ts +21 -0
- package/src/x.ts +188 -0
- package/src/xai.ts +254 -0
- package/src/youtube.ts +224 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import http, { STATUS_CODES, type IncomingHttpHeaders, type IncomingMessage, type RequestOptions } from "node:http";
|
|
2
|
+
import https from "node:https";
|
|
3
|
+
import net from "node:net";
|
|
4
|
+
import { SafeFetchError } from "./fetch-errors";
|
|
5
|
+
import type { DirectTransportInit, ResponseBody, TransportResponse, ValidatedRemoteUrl } from "./ssrf";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A transport is deliberately narrower than fetch(): the caller has already
|
|
9
|
+
* resolved and validated the URL, and this implementation must connect to the
|
|
10
|
+
* selected address while retaining the URL hostname for Host/SNI validation.
|
|
11
|
+
*/
|
|
12
|
+
export type DirectTransport = (
|
|
13
|
+
target: ValidatedRemoteUrl,
|
|
14
|
+
init: DirectTransportInit,
|
|
15
|
+
) => Promise<TransportResponse>;
|
|
16
|
+
|
|
17
|
+
export const directTransport: DirectTransport = (target, init) => requestDirect(target, init);
|
|
18
|
+
|
|
19
|
+
function requestDirect(target: ValidatedRemoteUrl, init: DirectTransportInit): Promise<TransportResponse> {
|
|
20
|
+
if (init.signal.aborted) {
|
|
21
|
+
return Promise.reject(new SafeFetchError({ kind: "canceled", message: "Fetch canceled" }));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const isHttps = target.url.protocol === "https:";
|
|
25
|
+
const hostname = target.url.hostname.replace(/^\[|\]$/g, "");
|
|
26
|
+
const port = target.url.port === "" ? (isHttps ? 443 : 80) : Number(target.url.port);
|
|
27
|
+
const path = `${target.url.pathname || "/"}${target.url.search}`;
|
|
28
|
+
const requestOptions: RequestOptions = {
|
|
29
|
+
protocol: target.url.protocol,
|
|
30
|
+
hostname,
|
|
31
|
+
port,
|
|
32
|
+
method: "GET",
|
|
33
|
+
path,
|
|
34
|
+
headers: {
|
|
35
|
+
accept: "text/markdown, text/html;q=0.9, text/plain;q=0.8, application/json;q=0.7, application/xml;q=0.6, */*;q=0.1",
|
|
36
|
+
"user-agent": "pi-search (+https://github.com/nijaru/pi-search)",
|
|
37
|
+
...init.headers,
|
|
38
|
+
},
|
|
39
|
+
lookup: (_lookupHostname, lookupOptions, callback) => {
|
|
40
|
+
if (lookupOptions.all) {
|
|
41
|
+
callback(null, [{ address: target.address, family: target.family }]);
|
|
42
|
+
} else {
|
|
43
|
+
callback(null, target.address, target.family);
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
...(isHttps && net.isIP(hostname) === 0 ? { servername: hostname } : {}),
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
return new Promise<TransportResponse>((resolve, reject) => {
|
|
50
|
+
let settled = false;
|
|
51
|
+
let response: IncomingMessage | undefined;
|
|
52
|
+
let removeAbortListener: (() => void) | undefined;
|
|
53
|
+
|
|
54
|
+
const fail = (error: unknown): void => {
|
|
55
|
+
if (settled) return;
|
|
56
|
+
settled = true;
|
|
57
|
+
removeAbortListener?.();
|
|
58
|
+
if (error instanceof SafeFetchError) {
|
|
59
|
+
reject(error);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
reject(
|
|
63
|
+
new SafeFetchError({
|
|
64
|
+
kind: "network",
|
|
65
|
+
message: "Direct HTTP request failed",
|
|
66
|
+
retryable: true,
|
|
67
|
+
cause: error,
|
|
68
|
+
}),
|
|
69
|
+
);
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const req = (isHttps ? https : http).request(requestOptions, (incoming) => {
|
|
73
|
+
response = incoming;
|
|
74
|
+
const body = makeResponseBody(incoming, req, init.signal, () => removeAbortListener?.());
|
|
75
|
+
settled = true;
|
|
76
|
+
removeAbortListener?.();
|
|
77
|
+
resolve({
|
|
78
|
+
status: incoming.statusCode ?? 0,
|
|
79
|
+
statusText: STATUS_CODES[incoming.statusCode ?? 0] ?? "",
|
|
80
|
+
headers: toHeaders(incoming.headers),
|
|
81
|
+
body,
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
const onAbort = (): void => {
|
|
86
|
+
if (response) {
|
|
87
|
+
response.destroy();
|
|
88
|
+
} else {
|
|
89
|
+
req.destroy();
|
|
90
|
+
}
|
|
91
|
+
fail(new SafeFetchError({ kind: "canceled", message: "Fetch canceled" }));
|
|
92
|
+
};
|
|
93
|
+
removeAbortListener = () => init.signal.removeEventListener("abort", onAbort);
|
|
94
|
+
init.signal.addEventListener("abort", onAbort, { once: true });
|
|
95
|
+
req.once("error", fail);
|
|
96
|
+
req.end();
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function makeResponseBody(
|
|
101
|
+
incoming: IncomingMessage,
|
|
102
|
+
req: { destroy: () => void },
|
|
103
|
+
signal: AbortSignal,
|
|
104
|
+
onClose: () => void,
|
|
105
|
+
): ResponseBody {
|
|
106
|
+
let closed = false;
|
|
107
|
+
const close = (): void => {
|
|
108
|
+
if (closed) return;
|
|
109
|
+
closed = true;
|
|
110
|
+
signal.removeEventListener("abort", onAbort);
|
|
111
|
+
onClose();
|
|
112
|
+
incoming.destroy();
|
|
113
|
+
req.destroy();
|
|
114
|
+
};
|
|
115
|
+
const onAbort = (): void => close();
|
|
116
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
117
|
+
return {
|
|
118
|
+
async *[Symbol.asyncIterator](): AsyncIterator<Uint8Array> {
|
|
119
|
+
try {
|
|
120
|
+
for await (const chunk of incoming) {
|
|
121
|
+
yield chunk;
|
|
122
|
+
}
|
|
123
|
+
} finally {
|
|
124
|
+
signal.removeEventListener("abort", onAbort);
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
cancel: close,
|
|
128
|
+
destroy: close,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function toHeaders(raw: IncomingHttpHeaders): Headers {
|
|
133
|
+
const headers = new Headers();
|
|
134
|
+
for (const [name, value] of Object.entries(raw)) {
|
|
135
|
+
if (value === undefined) continue;
|
|
136
|
+
for (const entry of Array.isArray(value) ? value : [value]) {
|
|
137
|
+
headers.append(name, entry);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return headers;
|
|
141
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import type { ProviderError, ProviderId, ProviderRateLimitInfo, RequestId } from "./contracts";
|
|
2
|
+
|
|
3
|
+
export interface ProviderErrorOptions {
|
|
4
|
+
readonly provider: ProviderId;
|
|
5
|
+
readonly kind: ProviderError["kind"];
|
|
6
|
+
readonly message: string;
|
|
7
|
+
readonly retryable: boolean;
|
|
8
|
+
readonly status?: number;
|
|
9
|
+
readonly requestId?: RequestId;
|
|
10
|
+
readonly retryAfterMs?: number;
|
|
11
|
+
readonly rateLimits?: ProviderRateLimitInfo;
|
|
12
|
+
readonly cause?: unknown;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** An adapter error with a stable, provider-neutral classification. */
|
|
16
|
+
export class SearchProviderError extends Error implements ProviderError {
|
|
17
|
+
readonly provider: ProviderId;
|
|
18
|
+
readonly kind: ProviderError["kind"];
|
|
19
|
+
readonly retryable: boolean;
|
|
20
|
+
readonly status?: number;
|
|
21
|
+
readonly requestId?: RequestId;
|
|
22
|
+
readonly retryAfterMs?: number;
|
|
23
|
+
readonly rateLimits?: ProviderRateLimitInfo;
|
|
24
|
+
|
|
25
|
+
constructor(options: ProviderErrorOptions) {
|
|
26
|
+
super(options.message, { cause: options.cause });
|
|
27
|
+
this.name = "SearchProviderError";
|
|
28
|
+
this.provider = options.provider;
|
|
29
|
+
this.kind = options.kind;
|
|
30
|
+
this.retryable = options.retryable;
|
|
31
|
+
this.status = options.status;
|
|
32
|
+
this.requestId = options.requestId;
|
|
33
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
34
|
+
this.rateLimits = options.rateLimits;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function isProviderError(error: unknown): error is ProviderError {
|
|
39
|
+
return (
|
|
40
|
+
error instanceof SearchProviderError ||
|
|
41
|
+
(typeof error === "object" &&
|
|
42
|
+
error !== null &&
|
|
43
|
+
"provider" in error &&
|
|
44
|
+
"kind" in error &&
|
|
45
|
+
"retryable" in error &&
|
|
46
|
+
error instanceof Error)
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function createProviderError(options: ProviderErrorOptions): SearchProviderError {
|
|
51
|
+
return new SearchProviderError(options);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export type SearchToolErrorCode =
|
|
55
|
+
| "WEB_SEARCH_INVALID_REQUEST"
|
|
56
|
+
| "WEB_SEARCH_AUTH"
|
|
57
|
+
| "WEB_SEARCH_RATE_LIMIT"
|
|
58
|
+
| "WEB_SEARCH_BAD_REQUEST"
|
|
59
|
+
| "WEB_SEARCH_MALFORMED_RESPONSE"
|
|
60
|
+
| "WEB_SEARCH_TIMEOUT"
|
|
61
|
+
| "WEB_SEARCH_CANCELED"
|
|
62
|
+
| "WEB_SEARCH_NETWORK"
|
|
63
|
+
| "WEB_SEARCH_HTTP"
|
|
64
|
+
| "WEB_SEARCH_UNSUPPORTED"
|
|
65
|
+
| "WEB_SEARCH_PROVIDER_UNAVAILABLE"
|
|
66
|
+
| "WEB_SEARCH_UNKNOWN"
|
|
67
|
+
| "WEB_RESEARCH_INVALID_REQUEST"
|
|
68
|
+
| "WEB_RESEARCH_BUDGET"
|
|
69
|
+
| "WEB_RESEARCH_TIMEOUT"
|
|
70
|
+
| "WEB_RESEARCH_CANCELED"
|
|
71
|
+
| "WEB_RESEARCH_PROVIDER"
|
|
72
|
+
| "WEB_RESEARCH_UNKNOWN";
|
|
73
|
+
|
|
74
|
+
/** A stable error shape thrown so Pi records an unsuccessful tool call. */
|
|
75
|
+
export class SearchToolError extends Error {
|
|
76
|
+
readonly code: SearchToolErrorCode;
|
|
77
|
+
readonly provider?: ProviderId;
|
|
78
|
+
readonly kind?: ProviderError["kind"];
|
|
79
|
+
readonly retryable?: boolean;
|
|
80
|
+
readonly status?: number;
|
|
81
|
+
readonly requestId?: RequestId;
|
|
82
|
+
readonly retryAfterMs?: number;
|
|
83
|
+
readonly rateLimits?: ProviderRateLimitInfo;
|
|
84
|
+
|
|
85
|
+
constructor(
|
|
86
|
+
code: SearchToolErrorCode,
|
|
87
|
+
message: string,
|
|
88
|
+
options: {
|
|
89
|
+
readonly provider?: ProviderId;
|
|
90
|
+
readonly kind?: ProviderError["kind"];
|
|
91
|
+
readonly retryable?: boolean;
|
|
92
|
+
readonly status?: number;
|
|
93
|
+
readonly requestId?: RequestId;
|
|
94
|
+
readonly retryAfterMs?: number;
|
|
95
|
+
readonly rateLimits?: ProviderRateLimitInfo;
|
|
96
|
+
} = {},
|
|
97
|
+
) {
|
|
98
|
+
super(`${code}: ${message}`);
|
|
99
|
+
this.name = "SearchToolError";
|
|
100
|
+
this.code = code;
|
|
101
|
+
this.provider = options.provider;
|
|
102
|
+
this.kind = options.kind;
|
|
103
|
+
this.retryable = options.retryable;
|
|
104
|
+
this.status = options.status;
|
|
105
|
+
this.requestId = options.requestId;
|
|
106
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
107
|
+
this.rateLimits = options.rateLimits;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function providerDiagnosticMessage(error: ProviderError): string {
|
|
112
|
+
const details = {
|
|
113
|
+
provider: error.provider,
|
|
114
|
+
...(error.status === undefined ? {} : { status: error.status }),
|
|
115
|
+
...(error.requestId === undefined ? {} : { requestId: error.requestId.slice(0, 200) }),
|
|
116
|
+
...(error.retryAfterMs === undefined ? {} : { retryAfterMs: Math.max(0, Math.round(error.retryAfterMs)) }),
|
|
117
|
+
...(error.rateLimits === undefined
|
|
118
|
+
? {}
|
|
119
|
+
: {
|
|
120
|
+
rateLimits: {
|
|
121
|
+
windows: error.rateLimits.windows.slice(0, 8),
|
|
122
|
+
...(error.rateLimits.retryAfterMs === undefined ? {} : { retryAfterMs: error.rateLimits.retryAfterMs }),
|
|
123
|
+
},
|
|
124
|
+
}),
|
|
125
|
+
};
|
|
126
|
+
return `${error.message} [${JSON.stringify(details)}]`;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const TOOL_ERROR_CODE_BY_KIND: Readonly<Record<ProviderError["kind"], SearchToolErrorCode>> = {
|
|
130
|
+
network: "WEB_SEARCH_NETWORK",
|
|
131
|
+
auth: "WEB_SEARCH_AUTH",
|
|
132
|
+
rateLimit: "WEB_SEARCH_RATE_LIMIT",
|
|
133
|
+
badRequest: "WEB_SEARCH_BAD_REQUEST",
|
|
134
|
+
malformed: "WEB_SEARCH_MALFORMED_RESPONSE",
|
|
135
|
+
timeout: "WEB_SEARCH_TIMEOUT",
|
|
136
|
+
canceled: "WEB_SEARCH_CANCELED",
|
|
137
|
+
unsupported: "WEB_SEARCH_UNSUPPORTED",
|
|
138
|
+
http: "WEB_SEARCH_HTTP",
|
|
139
|
+
unavailable: "WEB_SEARCH_PROVIDER_UNAVAILABLE",
|
|
140
|
+
unknown: "WEB_SEARCH_UNKNOWN",
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
/** Convert an adapter failure into the stable shape exposed by web_search. */
|
|
144
|
+
export function toSearchToolError(error: unknown, provider: ProviderId): SearchToolError {
|
|
145
|
+
if (error instanceof SearchToolError) {
|
|
146
|
+
return error;
|
|
147
|
+
}
|
|
148
|
+
if (isProviderError(error)) {
|
|
149
|
+
return new SearchToolError(TOOL_ERROR_CODE_BY_KIND[error.kind], providerDiagnosticMessage(error), {
|
|
150
|
+
provider: error.provider,
|
|
151
|
+
kind: error.kind,
|
|
152
|
+
retryable: error.retryable,
|
|
153
|
+
status: error.status,
|
|
154
|
+
requestId: error.requestId,
|
|
155
|
+
retryAfterMs: error.retryAfterMs,
|
|
156
|
+
rateLimits: error.rateLimits,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
const message = error instanceof Error ? error.message : "Unknown search failure";
|
|
160
|
+
return new SearchToolError("WEB_SEARCH_UNKNOWN", message, { provider, kind: "unknown" });
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export interface SearchToolFailureDetails {
|
|
164
|
+
readonly code: SearchToolErrorCode;
|
|
165
|
+
readonly provider?: ProviderId;
|
|
166
|
+
readonly kind?: ProviderError["kind"];
|
|
167
|
+
readonly retryable?: boolean;
|
|
168
|
+
readonly status?: number;
|
|
169
|
+
readonly requestId?: RequestId;
|
|
170
|
+
readonly retryAfterMs?: number;
|
|
171
|
+
readonly rateLimits?: ProviderRateLimitInfo;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function searchToolFailureDetails(error: SearchToolError): SearchToolFailureDetails {
|
|
175
|
+
return {
|
|
176
|
+
code: error.code,
|
|
177
|
+
...(error.provider === undefined ? {} : { provider: error.provider }),
|
|
178
|
+
...(error.kind === undefined ? {} : { kind: error.kind }),
|
|
179
|
+
...(error.retryable === undefined ? {} : { retryable: error.retryable }),
|
|
180
|
+
...(error.status === undefined ? {} : { status: error.status }),
|
|
181
|
+
...(error.requestId === undefined ? {} : { requestId: error.requestId }),
|
|
182
|
+
...(error.retryAfterMs === undefined ? {} : { retryAfterMs: error.retryAfterMs }),
|
|
183
|
+
...(error.rateLimits === undefined ? {} : { rateLimits: error.rateLimits }),
|
|
184
|
+
};
|
|
185
|
+
}
|
package/src/exa.ts
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Provider,
|
|
3
|
+
ProviderCapabilities,
|
|
4
|
+
ProviderContext,
|
|
5
|
+
ProviderProfile,
|
|
6
|
+
ProviderRateLimitInfo,
|
|
7
|
+
SearchOption,
|
|
8
|
+
SearchRequest,
|
|
9
|
+
SearchResponse,
|
|
10
|
+
SearchResult,
|
|
11
|
+
SearchWarning,
|
|
12
|
+
} from "./contracts";
|
|
13
|
+
import { createProviderError } from "./errors";
|
|
14
|
+
import { postJson, requireApiKey, httpSource, objectValue, optionalString, optionalTimestamp, type SearchHttpFetch } from "./provider-http";
|
|
15
|
+
import { validateSearchRequest } from "./search";
|
|
16
|
+
|
|
17
|
+
export const EXA_SEARCH_ENDPOINT = "https://api.exa.ai/search";
|
|
18
|
+
export const DEFAULT_EXA_RESPONSE_BYTES = 4 * 1024 * 1024;
|
|
19
|
+
/** Current published cost for one standard search returning up to 10 results. */
|
|
20
|
+
export const EXA_ESTIMATED_SEARCH_COST_USD = 0.007;
|
|
21
|
+
|
|
22
|
+
export interface ExaAdapterOptions {
|
|
23
|
+
readonly apiKey?: string;
|
|
24
|
+
readonly endpoint?: string;
|
|
25
|
+
readonly fetchImpl?: SearchHttpFetch;
|
|
26
|
+
readonly maxResponseBytes?: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const capabilities: ProviderCapabilities = {
|
|
30
|
+
keyword: true,
|
|
31
|
+
freshness: true,
|
|
32
|
+
semantic: true,
|
|
33
|
+
excerpts: true,
|
|
34
|
+
domainFilter: true,
|
|
35
|
+
dateFilter: true,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const profile: ProviderProfile = {
|
|
39
|
+
auth: "environment",
|
|
40
|
+
costModel: "usage-based",
|
|
41
|
+
estimatedCostUsd: EXA_ESTIMATED_SEARCH_COST_USD,
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
function domainQuery(request: SearchRequest): Record<string, unknown> {
|
|
45
|
+
return {
|
|
46
|
+
...(request.domains?.include === undefined ? {} : { includeDomains: [...request.domains.include] }),
|
|
47
|
+
...(request.domains?.exclude === undefined ? {} : { excludeDomains: [...request.domains.exclude] }),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function exaDate(value: string, endOfDay: boolean): string {
|
|
52
|
+
if (value.includes("T")) return value;
|
|
53
|
+
return `${value}T${endOfDay ? "23:59:59.999" : "00:00:00.000"}Z`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface ExaRequestPlan {
|
|
57
|
+
readonly body: Record<string, unknown>;
|
|
58
|
+
readonly appliedOptions: readonly SearchOption[];
|
|
59
|
+
readonly warnings: readonly SearchWarning[];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function buildExaRequest(request: SearchRequest): ExaRequestPlan {
|
|
63
|
+
const normalized = validateSearchRequest(request);
|
|
64
|
+
if (normalized.social !== undefined) {
|
|
65
|
+
throw createProviderError({ provider: "exa", kind: "unsupported", message: "Exa search does not provide a dedicated social/X constraint contract", retryable: false });
|
|
66
|
+
}
|
|
67
|
+
const warnings: SearchWarning[] = [];
|
|
68
|
+
const appliedOptions: SearchOption[] = ["maxResults", "mode"];
|
|
69
|
+
if (normalized.mode === "keyword") {
|
|
70
|
+
warnings.push({ code: "unsupported-option", option: "mode", message: "Exa's current search API uses automatic retrieval; keyword-only ranking is not guaranteed" });
|
|
71
|
+
}
|
|
72
|
+
if (normalized.mode === "fresh") {
|
|
73
|
+
warnings.push({ code: "unsupported-option", option: "mode", message: "Exa search can return recent sources but this request does not impose a date cutoff" });
|
|
74
|
+
}
|
|
75
|
+
if (normalized.domains?.include !== undefined || normalized.domains?.exclude !== undefined) appliedOptions.push("domains");
|
|
76
|
+
if (normalized.dateRange !== undefined) appliedOptions.push("dateRange");
|
|
77
|
+
return {
|
|
78
|
+
body: {
|
|
79
|
+
query: normalized.query,
|
|
80
|
+
type: "auto",
|
|
81
|
+
numResults: normalized.maxResults,
|
|
82
|
+
contents: { highlights: true },
|
|
83
|
+
...(normalized.dateRange?.from === undefined ? {} : { startPublishedDate: exaDate(normalized.dateRange.from, false) }),
|
|
84
|
+
...(normalized.dateRange?.to === undefined ? {} : { endPublishedDate: exaDate(normalized.dateRange.to, true) }),
|
|
85
|
+
...domainQuery(normalized),
|
|
86
|
+
},
|
|
87
|
+
appliedOptions,
|
|
88
|
+
warnings,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function malformed(message: string): never {
|
|
93
|
+
throw createProviderError({ provider: "exa", kind: "malformed", message: `Exa returned a malformed response (${message})`, retryable: false });
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
97
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function excerptFromRecord(record: Record<string, unknown>): string | undefined {
|
|
101
|
+
if (Array.isArray(record.highlights)) {
|
|
102
|
+
const highlights = record.highlights
|
|
103
|
+
.filter((item): item is string => typeof item === "string" && item.trim().length > 0)
|
|
104
|
+
.slice(0, 3);
|
|
105
|
+
if (highlights.length > 0) return highlights.join("\n").slice(0, 4_000);
|
|
106
|
+
}
|
|
107
|
+
for (const value of [record.text, record.summary]) {
|
|
108
|
+
const excerpt = optionalString(value, 4_000);
|
|
109
|
+
if (excerpt !== undefined) return excerpt;
|
|
110
|
+
}
|
|
111
|
+
return undefined;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function costFromResponse(value: unknown): number | undefined {
|
|
115
|
+
if (!isRecord(value)) return undefined;
|
|
116
|
+
const total = value.total;
|
|
117
|
+
return typeof total === "number" && Number.isFinite(total) && total >= 0 ? total : undefined;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function normalizeExaResponse(payload: unknown, request: SearchRequest, metadata: { readonly requestId?: string; readonly rateLimits?: ProviderRateLimitInfo }): SearchResponse {
|
|
121
|
+
const normalized = validateSearchRequest(request);
|
|
122
|
+
const root = objectValue(payload, "response", "exa");
|
|
123
|
+
if (!Array.isArray(root.results)) return malformed("results is not an array");
|
|
124
|
+
const results: SearchResult[] = [];
|
|
125
|
+
let discarded = 0;
|
|
126
|
+
for (const value of root.results) {
|
|
127
|
+
// A single malformed result should not hide otherwise usable evidence.
|
|
128
|
+
if (!isRecord(value)) {
|
|
129
|
+
discarded += 1;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const parsed = httpSource(value.url, "exa");
|
|
133
|
+
if (parsed === undefined) {
|
|
134
|
+
discarded += 1;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const score = typeof value.score === "number" && Number.isFinite(value.score) ? Math.max(0, Math.min(1, value.score)) : undefined;
|
|
138
|
+
results.push({
|
|
139
|
+
url: parsed.url,
|
|
140
|
+
title: optionalString(value.title, 500),
|
|
141
|
+
domain: parsed.domain,
|
|
142
|
+
publishedAt: optionalTimestamp(value.publishedDate),
|
|
143
|
+
excerpt: excerptFromRecord(value),
|
|
144
|
+
provider: "exa",
|
|
145
|
+
searchQuery: normalized.query,
|
|
146
|
+
sourceId: optionalString(value.id, 500),
|
|
147
|
+
score,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
if (discarded > 0 && results.length === 0) return malformed("results contained no parseable HTTP URLs");
|
|
151
|
+
// Billing metadata is useful but non-essential. Ignore an invalid optional
|
|
152
|
+
// cost object rather than turning a successful search into a hard failure.
|
|
153
|
+
const costUsd = costFromResponse(root.costDollars);
|
|
154
|
+
const bodyRequestId = optionalString(root.requestId, 500);
|
|
155
|
+
const usage = costUsd === undefined && metadata.rateLimits === undefined
|
|
156
|
+
? undefined
|
|
157
|
+
: { ...(costUsd === undefined ? {} : { costUsd }), ...(metadata.rateLimits === undefined ? {} : { rateLimits: metadata.rateLimits }) };
|
|
158
|
+
return {
|
|
159
|
+
query: normalized.query,
|
|
160
|
+
results: results.slice(0, normalized.maxResults ?? 10),
|
|
161
|
+
provider: "exa",
|
|
162
|
+
appliedOptions: [],
|
|
163
|
+
warnings: discarded > 0 ? [{ code: "partial-results", message: `Exa discarded ${discarded} malformed result entr${discarded === 1 ? "y" : "ies"}` }] : [],
|
|
164
|
+
...(metadata.requestId === undefined && bodyRequestId === undefined ? {} : { requestId: metadata.requestId ?? bodyRequestId }),
|
|
165
|
+
...(usage === undefined ? {} : { usage }),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export class ExaProvider implements Provider {
|
|
170
|
+
readonly id = "exa" as const;
|
|
171
|
+
readonly capabilities = capabilities;
|
|
172
|
+
readonly profile = profile;
|
|
173
|
+
private readonly apiKey?: string;
|
|
174
|
+
private readonly endpoint: string;
|
|
175
|
+
private readonly fetchImpl: SearchHttpFetch;
|
|
176
|
+
private readonly maxResponseBytes: number;
|
|
177
|
+
|
|
178
|
+
constructor(options: ExaAdapterOptions) {
|
|
179
|
+
this.apiKey = options.apiKey;
|
|
180
|
+
this.endpoint = options.endpoint ?? EXA_SEARCH_ENDPOINT;
|
|
181
|
+
this.fetchImpl = options.fetchImpl ?? (fetch as SearchHttpFetch);
|
|
182
|
+
this.maxResponseBytes = options.maxResponseBytes ?? DEFAULT_EXA_RESPONSE_BYTES;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async search(request: SearchRequest, signal: AbortSignal, _context: ProviderContext): Promise<SearchResponse> {
|
|
186
|
+
const normalized = validateSearchRequest(request);
|
|
187
|
+
const plan = buildExaRequest(normalized);
|
|
188
|
+
const result = await postJson({
|
|
189
|
+
provider: this.id,
|
|
190
|
+
url: this.endpoint,
|
|
191
|
+
headers: { "x-api-key": requireApiKey(this.id, this.apiKey) },
|
|
192
|
+
body: plan.body,
|
|
193
|
+
signal,
|
|
194
|
+
fetchImpl: this.fetchImpl,
|
|
195
|
+
maxResponseBytes: this.maxResponseBytes,
|
|
196
|
+
});
|
|
197
|
+
const response = normalizeExaResponse(result.payload, normalized, result);
|
|
198
|
+
return { ...response, appliedOptions: plan.appliedOptions, warnings: [...plan.warnings, ...response.warnings] };
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export function createExaProvider(options: ExaAdapterOptions): ExaProvider {
|
|
203
|
+
return new ExaProvider(options);
|
|
204
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
export type FetchErrorKind =
|
|
2
|
+
| "invalidRequest"
|
|
3
|
+
| "ssrf"
|
|
4
|
+
| "redirect"
|
|
5
|
+
| "network"
|
|
6
|
+
| "http"
|
|
7
|
+
| "timeout"
|
|
8
|
+
| "canceled"
|
|
9
|
+
| "responseTooLarge"
|
|
10
|
+
| "unsupportedContentType"
|
|
11
|
+
| "extraction"
|
|
12
|
+
| "unknown";
|
|
13
|
+
|
|
14
|
+
export interface FetchErrorOptions {
|
|
15
|
+
readonly kind: FetchErrorKind;
|
|
16
|
+
readonly message: string;
|
|
17
|
+
readonly status?: number;
|
|
18
|
+
readonly retryable?: boolean;
|
|
19
|
+
readonly cause?: unknown;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export class SafeFetchError extends Error {
|
|
23
|
+
readonly kind: FetchErrorKind;
|
|
24
|
+
readonly status?: number;
|
|
25
|
+
readonly retryable: boolean;
|
|
26
|
+
|
|
27
|
+
constructor(options: FetchErrorOptions) {
|
|
28
|
+
super(options.message, { cause: options.cause });
|
|
29
|
+
this.name = "SafeFetchError";
|
|
30
|
+
this.kind = options.kind;
|
|
31
|
+
this.status = options.status;
|
|
32
|
+
this.retryable = options.retryable ?? false;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type FetchToolErrorCode =
|
|
37
|
+
| "WEB_FETCH_INVALID_REQUEST"
|
|
38
|
+
| "WEB_FETCH_SSRF_BLOCKED"
|
|
39
|
+
| "WEB_FETCH_REDIRECT_ERROR"
|
|
40
|
+
| "WEB_FETCH_NETWORK"
|
|
41
|
+
| "WEB_FETCH_HTTP"
|
|
42
|
+
| "WEB_FETCH_TIMEOUT"
|
|
43
|
+
| "WEB_FETCH_CANCELED"
|
|
44
|
+
| "WEB_FETCH_RESPONSE_TOO_LARGE"
|
|
45
|
+
| "WEB_FETCH_UNSUPPORTED_CONTENT_TYPE"
|
|
46
|
+
| "WEB_FETCH_EXTRACTION_FAILED"
|
|
47
|
+
| "WEB_FETCH_UNKNOWN";
|
|
48
|
+
|
|
49
|
+
export class FetchToolError extends Error {
|
|
50
|
+
readonly code: FetchToolErrorCode;
|
|
51
|
+
readonly kind: FetchErrorKind;
|
|
52
|
+
readonly status?: number;
|
|
53
|
+
readonly retryable: boolean;
|
|
54
|
+
|
|
55
|
+
constructor(options: {
|
|
56
|
+
readonly code: FetchToolErrorCode;
|
|
57
|
+
readonly kind: FetchErrorKind;
|
|
58
|
+
readonly message: string;
|
|
59
|
+
readonly status?: number;
|
|
60
|
+
readonly retryable?: boolean;
|
|
61
|
+
}) {
|
|
62
|
+
super(`${options.code}: ${options.message}`);
|
|
63
|
+
this.name = "FetchToolError";
|
|
64
|
+
this.code = options.code;
|
|
65
|
+
this.kind = options.kind;
|
|
66
|
+
this.status = options.status;
|
|
67
|
+
this.retryable = options.retryable ?? false;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function isSafeFetchError(error: unknown): error is SafeFetchError {
|
|
72
|
+
return error instanceof SafeFetchError;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function errorCode(kind: FetchErrorKind): FetchToolErrorCode {
|
|
76
|
+
switch (kind) {
|
|
77
|
+
case "invalidRequest":
|
|
78
|
+
return "WEB_FETCH_INVALID_REQUEST";
|
|
79
|
+
case "ssrf":
|
|
80
|
+
return "WEB_FETCH_SSRF_BLOCKED";
|
|
81
|
+
case "redirect":
|
|
82
|
+
return "WEB_FETCH_REDIRECT_ERROR";
|
|
83
|
+
case "network":
|
|
84
|
+
return "WEB_FETCH_NETWORK";
|
|
85
|
+
case "http":
|
|
86
|
+
return "WEB_FETCH_HTTP";
|
|
87
|
+
case "timeout":
|
|
88
|
+
return "WEB_FETCH_TIMEOUT";
|
|
89
|
+
case "canceled":
|
|
90
|
+
return "WEB_FETCH_CANCELED";
|
|
91
|
+
case "responseTooLarge":
|
|
92
|
+
return "WEB_FETCH_RESPONSE_TOO_LARGE";
|
|
93
|
+
case "unsupportedContentType":
|
|
94
|
+
return "WEB_FETCH_UNSUPPORTED_CONTENT_TYPE";
|
|
95
|
+
case "extraction":
|
|
96
|
+
return "WEB_FETCH_EXTRACTION_FAILED";
|
|
97
|
+
case "unknown":
|
|
98
|
+
return "WEB_FETCH_UNKNOWN";
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function toFetchToolError(error: unknown): FetchToolError {
|
|
103
|
+
if (error instanceof FetchToolError) return error;
|
|
104
|
+
if (isSafeFetchError(error)) {
|
|
105
|
+
return new FetchToolError({
|
|
106
|
+
code: errorCode(error.kind),
|
|
107
|
+
kind: error.kind,
|
|
108
|
+
message: error.message,
|
|
109
|
+
status: error.status,
|
|
110
|
+
retryable: error.retryable,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
return new FetchToolError({
|
|
114
|
+
code: "WEB_FETCH_UNKNOWN",
|
|
115
|
+
kind: "unknown",
|
|
116
|
+
message: error instanceof Error ? error.message : "Unknown fetch failure",
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface FetchToolFailureDetails {
|
|
121
|
+
readonly code: FetchToolErrorCode;
|
|
122
|
+
readonly kind: FetchErrorKind;
|
|
123
|
+
readonly status?: number;
|
|
124
|
+
readonly retryable: boolean;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function fetchToolFailureDetails(error: FetchToolError): FetchToolFailureDetails {
|
|
128
|
+
return {
|
|
129
|
+
code: error.code,
|
|
130
|
+
kind: error.kind,
|
|
131
|
+
...(error.status === undefined ? {} : { status: error.status }),
|
|
132
|
+
retryable: error.retryable,
|
|
133
|
+
};
|
|
134
|
+
}
|