@zeldrisho/pi-web-search 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -1
- package/README.md +1 -3
- package/package.json +8 -10
- package/src/brave.ts +57 -124
- package/src/cache.ts +50 -0
- package/src/credentials.ts +10 -0
- package/src/index.ts +12 -104
- package/src/inflight.ts +10 -0
- package/src/provider.ts +20 -0
- package/src/render.ts +3 -0
- package/src/request-validation.ts +125 -0
- package/src/search.ts +70 -1
- package/src/format-results.ts +0 -44
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.8.3] - 2026-09-28
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- Simplify search guidance to favor broad discovery and avoid overly restrictive `site:` paths
|
|
15
|
+
- Expose only core search controls (`mode`, `count`, `freshness`, and `spellcheck`) in the model-facing tool schema
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
|
|
19
|
+
- Document mode-specific result limits and correctly escape square brackets in rendered result links
|
|
20
|
+
|
|
21
|
+
## [0.8.2] - 2026-09-24
|
|
22
|
+
|
|
23
|
+
### Fixed
|
|
24
|
+
|
|
25
|
+
- Handle already-aborted search requests and preserve provider errors when response-body cancellation or reading fails.
|
|
26
|
+
|
|
10
27
|
## [0.8.1] - 2026-09-12
|
|
11
28
|
|
|
12
29
|
### Fixed
|
|
@@ -91,7 +108,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
91
108
|
|
|
92
109
|
- Add bounded Brave web search ([cbc9491](https://github.com/zeldrisho/pi-packages/commit/cbc9491dce36555ab91b6bef203ec8b380596a89))
|
|
93
110
|
|
|
94
|
-
[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.8.
|
|
111
|
+
[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.8.3...HEAD
|
|
112
|
+
[0.8.3]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.8.2...pi-web-search-v0.8.3
|
|
113
|
+
[0.8.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.8.1...pi-web-search-v0.8.2
|
|
95
114
|
[0.8.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.8.0...pi-web-search-v0.8.1
|
|
96
115
|
[0.8.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.7.0...pi-web-search-v0.8.0
|
|
97
116
|
[0.7.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.6.0...pi-web-search-v0.7.0
|
package/README.md
CHANGED
|
@@ -23,9 +23,7 @@ If unset, the tool reads the first matching `BRAVE_SEARCH_API_KEY=` line from th
|
|
|
23
23
|
|
|
24
24
|
## Usage
|
|
25
25
|
|
|
26
|
-
`web_search` returns compact Brave results by default.
|
|
27
|
-
|
|
28
|
-
Web mode supports SafeSearch, extra snippets, search operators, result filters, page `offset` (0–9), UI language, and a version-scoped `dateRange` (`YYYY-MM-DDtoYYYY-MM-DD`). It allows up to 20 results. Context mode supports `threshold` (`strict`, `balanced`, `lenient`, `disabled`) and `depth` (`quick`, `standard`, `deep`), with up to 50 results. Options belonging to the other mode fail before a provider request.
|
|
26
|
+
`web_search` returns compact Brave results by default. Its model-facing options are `mode` (`web` or `context`), `count`, `freshness`, and `spellcheck`. Context mode uses Brave's LLM Context API and never fetches result URLs; use `web_fetch` to inspect a live page. Context queries are limited to 400 characters; web-mode queries are limited to 400 characters.
|
|
29
27
|
|
|
30
28
|
Identical searches are cached in byte-bounded memory and coalesced while in flight; cancelling one caller does not cancel work needed by another. Large output is written to a private temporary file and removed on write failure or session shutdown. Pi's interactive preview is collapsed by default (`Ctrl+O` expands it), and tool output remains bounded.
|
|
31
29
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zeldrisho/pi-web-search",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.3",
|
|
4
4
|
"description": "Pi extension for bounded Brave web search and LLM context retrieval",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"brave-search",
|
|
@@ -23,22 +23,20 @@
|
|
|
23
23
|
},
|
|
24
24
|
"files": [
|
|
25
25
|
"src",
|
|
26
|
-
"
|
|
27
|
-
"CHANGELOG.md",
|
|
28
|
-
"LICENSE"
|
|
26
|
+
"CHANGELOG.md"
|
|
29
27
|
],
|
|
30
28
|
"type": "module",
|
|
31
29
|
"publishConfig": {
|
|
32
30
|
"access": "public"
|
|
33
31
|
},
|
|
34
32
|
"devDependencies": {
|
|
35
|
-
"@earendil-works/pi-ai": "^0.
|
|
36
|
-
"@earendil-works/pi-coding-agent": "^0.
|
|
37
|
-
"@earendil-works/pi-tui": "^0.
|
|
38
|
-
"typebox": "^1.3.
|
|
33
|
+
"@earendil-works/pi-ai": "^0.85.1",
|
|
34
|
+
"@earendil-works/pi-coding-agent": "^0.85.1",
|
|
35
|
+
"@earendil-works/pi-tui": "^0.85.1",
|
|
36
|
+
"typebox": "^1.3.30",
|
|
39
37
|
"typescript": "^7.0.2",
|
|
40
|
-
"vite": "npm:@voidzero-dev/vite-plus-core@0.3.
|
|
41
|
-
"vite-plus": "0.3.
|
|
38
|
+
"vite": "npm:@voidzero-dev/vite-plus-core@0.3.3",
|
|
39
|
+
"vite-plus": "0.3.3"
|
|
42
40
|
},
|
|
43
41
|
"peerDependencies": {
|
|
44
42
|
"@earendil-works/pi-ai": "*",
|
package/src/brave.ts
CHANGED
|
@@ -1,26 +1,23 @@
|
|
|
1
|
-
import {
|
|
2
|
-
SEARCH_CONTEXT_MAX_QUERY_CHARACTERS,
|
|
3
|
-
SEARCH_CONTEXT_MAX_RESULT_COUNT,
|
|
4
|
-
SEARCH_MAX_DATE_RANGE_CHARACTERS,
|
|
5
|
-
SEARCH_MAX_GOGGLES_CHARACTERS,
|
|
6
|
-
SEARCH_MAX_OFFSET,
|
|
7
|
-
SEARCH_MAX_RESULT_FILTER_CHARACTERS,
|
|
8
|
-
SEARCH_MIN_OFFSET,
|
|
9
|
-
SEARCH_MIN_RESULT_COUNT,
|
|
10
|
-
SEARCH_RESULT_FILTER_VALUES,
|
|
11
|
-
SEARCH_WEB_MAX_QUERY_CHARACTERS,
|
|
12
|
-
SEARCH_WEB_MAX_RESULT_COUNT,
|
|
13
|
-
} from "./limits";
|
|
1
|
+
import { SEARCH_RESULT_FILTER_VALUES } from "./limits";
|
|
14
2
|
import { normalizeText, requestJson } from "./provider";
|
|
3
|
+
import { normalizeResultFilter, validateProviderRequest } from "./request-validation";
|
|
4
|
+
|
|
5
|
+
export { normalizeResultFilter, validateProviderRequest } from "./request-validation";
|
|
15
6
|
|
|
16
7
|
export type Provider = "brave";
|
|
8
|
+
|
|
17
9
|
export type Freshness = "day" | "week" | "month" | "year";
|
|
10
|
+
|
|
18
11
|
export type SafesearchMode = "off" | "moderate" | "strict";
|
|
12
|
+
|
|
19
13
|
export type SearchMode = "web" | "context";
|
|
14
|
+
|
|
20
15
|
/** Relevance threshold for the LLM context endpoint. */
|
|
21
16
|
export type ContextThresholdMode = "strict" | "balanced" | "lenient" | "disabled";
|
|
17
|
+
|
|
22
18
|
/** Token-budget preset for the LLM context endpoint (coding-oriented). */
|
|
23
19
|
export type ContextDepth = "quick" | "standard" | "deep";
|
|
20
|
+
|
|
24
21
|
/** Allowed Brave `result_filter` values. */
|
|
25
22
|
export type ResultFilterValue = (typeof SEARCH_RESULT_FILTER_VALUES)[number];
|
|
26
23
|
|
|
@@ -71,18 +68,27 @@ export interface SearchResponse {
|
|
|
71
68
|
export function parseQueryMeta(query: BraveQuery | undefined): BraveQueryMeta {
|
|
72
69
|
if (!query) return {};
|
|
73
70
|
const meta: BraveQueryMeta = {};
|
|
71
|
+
|
|
74
72
|
if (query.altered) meta.altered = query.altered;
|
|
73
|
+
|
|
75
74
|
if (query.cleaned) meta.cleaned = query.cleaned;
|
|
75
|
+
|
|
76
76
|
if (query.spellcheck_off !== undefined) meta.spellcheckOff = query.spellcheck_off;
|
|
77
|
+
|
|
77
78
|
if (query.show_strict_warning !== undefined) meta.showStrictWarning = query.show_strict_warning;
|
|
79
|
+
|
|
78
80
|
if (query.more_results_available !== undefined)
|
|
79
81
|
meta.moreResultsAvailable = query.more_results_available;
|
|
80
82
|
const operators = query.search_operators;
|
|
83
|
+
|
|
81
84
|
if (operators) {
|
|
82
85
|
if (operators.applied !== undefined) meta.operatorsApplied = operators.applied;
|
|
86
|
+
|
|
83
87
|
if (operators.cleaned_query) meta.cleanedQuery = operators.cleaned_query;
|
|
88
|
+
|
|
84
89
|
if (Array.isArray(operators.sites)) meta.operatorSites = [...operators.sites];
|
|
85
90
|
}
|
|
91
|
+
|
|
86
92
|
return meta;
|
|
87
93
|
}
|
|
88
94
|
|
|
@@ -175,8 +181,11 @@ export interface ContextSearchExtras {
|
|
|
175
181
|
*/
|
|
176
182
|
export function classifyResultQuality(result: { title: string; snippet: string }): ResultQuality {
|
|
177
183
|
if (!result.title && !result.snippet) return "low";
|
|
184
|
+
|
|
178
185
|
if (result.snippet.length >= 80) return "high";
|
|
186
|
+
|
|
179
187
|
if (result.title || result.snippet) return "medium";
|
|
188
|
+
|
|
180
189
|
return "low";
|
|
181
190
|
}
|
|
182
191
|
|
|
@@ -188,118 +197,8 @@ const CONTEXT_DEPTH_BUDGETS = {
|
|
|
188
197
|
ContextDepth,
|
|
189
198
|
{ tokens: number; snippets: number; tokensPerUrl: number; snippetsPerUrl: number }
|
|
190
199
|
>;
|
|
191
|
-
const BRAVE_MAX_EXTRA_SNIPPETS = 5;
|
|
192
|
-
const DATE_RANGE_PATTERN = /^\d{4}-\d{2}-\d{2}to\d{4}-\d{2}-\d{2}$/;
|
|
193
|
-
const COUNTRY_PATTERN = /^([A-Za-z]{2}|ALL)$/;
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Validates a search query and requested result count against the provider limits.
|
|
197
|
-
*
|
|
198
|
-
* @param query - The search query to validate
|
|
199
|
-
* @param count - The requested number of results
|
|
200
|
-
* @param mode - The search mode whose query and count limits apply
|
|
201
|
-
* @param extras - Optional per-mode search options to validate
|
|
202
|
-
* @throws If the query, count, or any option is outside the allowed range
|
|
203
|
-
*/
|
|
204
|
-
export function validateProviderRequest(
|
|
205
|
-
query: string,
|
|
206
|
-
count: number,
|
|
207
|
-
mode: SearchMode,
|
|
208
|
-
extras?: ProviderRequestExtras,
|
|
209
|
-
): void {
|
|
210
|
-
if (
|
|
211
|
-
mode === "context" &&
|
|
212
|
-
(extras?.extraSnippets !== undefined ||
|
|
213
|
-
extras?.operators !== undefined ||
|
|
214
|
-
extras?.resultFilter !== undefined ||
|
|
215
|
-
extras?.offset !== undefined ||
|
|
216
|
-
extras?.uiLang !== undefined ||
|
|
217
|
-
extras?.dateRange !== undefined)
|
|
218
|
-
) {
|
|
219
|
-
throw new Error(
|
|
220
|
-
"The extraSnippets, operators, resultFilter, offset, uiLang, and dateRange options are only supported in web mode.",
|
|
221
|
-
);
|
|
222
|
-
}
|
|
223
|
-
if (mode === "web" && (extras?.threshold !== undefined || extras?.depth !== undefined)) {
|
|
224
|
-
throw new Error("The threshold and depth options are only supported in context mode.");
|
|
225
|
-
}
|
|
226
|
-
const maximumQueryCharacters =
|
|
227
|
-
mode === "context" ? SEARCH_CONTEXT_MAX_QUERY_CHARACTERS : SEARCH_WEB_MAX_QUERY_CHARACTERS;
|
|
228
|
-
if (query.length > maximumQueryCharacters) {
|
|
229
|
-
throw new Error(`Search queries cannot exceed ${maximumQueryCharacters} characters.`);
|
|
230
|
-
}
|
|
231
|
-
const maximumCount =
|
|
232
|
-
mode === "context" ? SEARCH_CONTEXT_MAX_RESULT_COUNT : SEARCH_WEB_MAX_RESULT_COUNT;
|
|
233
|
-
if (!Number.isInteger(count) || count < SEARCH_MIN_RESULT_COUNT || count > maximumCount) {
|
|
234
|
-
throw new Error(
|
|
235
|
-
`Search result count must be an integer between ${SEARCH_MIN_RESULT_COUNT} and ${maximumCount} in ${mode} mode.`,
|
|
236
|
-
);
|
|
237
|
-
}
|
|
238
|
-
if (extras?.country !== undefined && !COUNTRY_PATTERN.test(extras.country)) {
|
|
239
|
-
throw new Error('Search country must be a 2-letter code (e.g. "US") or "ALL".');
|
|
240
|
-
}
|
|
241
|
-
if (extras?.goggles !== undefined) {
|
|
242
|
-
if (!extras.goggles || extras.goggles.length > SEARCH_MAX_GOGGLES_CHARACTERS) {
|
|
243
|
-
throw new Error(
|
|
244
|
-
`Search goggles must be between 1 and ${SEARCH_MAX_GOGGLES_CHARACTERS} characters.`,
|
|
245
|
-
);
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
if (extras?.offset !== undefined) {
|
|
249
|
-
if (
|
|
250
|
-
!Number.isInteger(extras.offset) ||
|
|
251
|
-
extras.offset < SEARCH_MIN_OFFSET ||
|
|
252
|
-
extras.offset > SEARCH_MAX_OFFSET
|
|
253
|
-
) {
|
|
254
|
-
throw new Error(
|
|
255
|
-
`Search offset must be an integer between ${SEARCH_MIN_OFFSET} and ${SEARCH_MAX_OFFSET}.`,
|
|
256
|
-
);
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
if (extras?.resultFilter !== undefined) {
|
|
260
|
-
normalizeResultFilter(extras.resultFilter);
|
|
261
|
-
}
|
|
262
|
-
if (extras?.dateRange !== undefined) {
|
|
263
|
-
if (
|
|
264
|
-
extras.dateRange.length > SEARCH_MAX_DATE_RANGE_CHARACTERS ||
|
|
265
|
-
!DATE_RANGE_PATTERN.test(extras.dateRange)
|
|
266
|
-
) {
|
|
267
|
-
throw new Error(
|
|
268
|
-
"Search dateRange must use YYYY-MM-DDtoYYYY-MM-DD (e.g. 2025-01-01to2025-06-30).",
|
|
269
|
-
);
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
if (extras?.uiLang !== undefined && (extras.uiLang.length < 2 || extras.uiLang.length > 20)) {
|
|
273
|
-
throw new Error("Search uiLang must be between 2 and 20 characters (e.g. en-US).");
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
200
|
|
|
277
|
-
|
|
278
|
-
* Normalizes a comma-separated `result_filter` value.
|
|
279
|
-
*
|
|
280
|
-
* @param value - The raw filter list (e.g. `web,discussions,faq`)
|
|
281
|
-
* @returns The normalized, deduplicated filter list
|
|
282
|
-
* @throws If the list is empty, too long, or contains an unknown value
|
|
283
|
-
*/
|
|
284
|
-
export function normalizeResultFilter(value: string): string {
|
|
285
|
-
const values = value
|
|
286
|
-
.split(",")
|
|
287
|
-
.map((item) => item.trim().toLowerCase())
|
|
288
|
-
.filter(Boolean);
|
|
289
|
-
const unique = [...new Set(values)];
|
|
290
|
-
const allowed = new Set<string>(SEARCH_RESULT_FILTER_VALUES);
|
|
291
|
-
if (
|
|
292
|
-
unique.length === 0 ||
|
|
293
|
-
value.length > SEARCH_MAX_RESULT_FILTER_CHARACTERS ||
|
|
294
|
-
unique.some((item) => !allowed.has(item)) ||
|
|
295
|
-
!unique.includes("web")
|
|
296
|
-
) {
|
|
297
|
-
throw new Error(
|
|
298
|
-
`Search resultFilter must be a comma-separated list of: ${SEARCH_RESULT_FILTER_VALUES.join(", ")}, and must include "web" (only web results are mapped).`,
|
|
299
|
-
);
|
|
300
|
-
}
|
|
301
|
-
return unique.join(",");
|
|
302
|
-
}
|
|
201
|
+
const BRAVE_MAX_EXTRA_SNIPPETS = 5;
|
|
303
202
|
|
|
304
203
|
/**
|
|
305
204
|
* Maps `freshness`/`dateRange` tool options to Brave's `freshness` parameter.
|
|
@@ -316,8 +215,11 @@ export function mapFreshness(
|
|
|
316
215
|
if (freshness && dateRange) {
|
|
317
216
|
throw new Error("Search freshness and dateRange are mutually exclusive; set only one.");
|
|
318
217
|
}
|
|
218
|
+
|
|
319
219
|
if (dateRange) return dateRange;
|
|
220
|
+
|
|
320
221
|
if (freshness) return { day: "pd", week: "pw", month: "pm", year: "py" }[freshness];
|
|
222
|
+
|
|
321
223
|
return undefined;
|
|
322
224
|
}
|
|
323
225
|
|
|
@@ -330,6 +232,7 @@ export function mapFreshness(
|
|
|
330
232
|
function normalizeUrl(value: string): string {
|
|
331
233
|
try {
|
|
332
234
|
const url = new URL(value);
|
|
235
|
+
|
|
333
236
|
return url.protocol === "http:" || url.protocol === "https:"
|
|
334
237
|
? url.toString().slice(0, 2048)
|
|
335
238
|
: "";
|
|
@@ -346,30 +249,39 @@ function escapeMarkdownCell(value: string): string {
|
|
|
346
249
|
return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, "<br>");
|
|
347
250
|
}
|
|
348
251
|
|
|
252
|
+
/** Renders a structured Brave snippet as a bounded Markdown table when possible. */
|
|
349
253
|
function structuredSnippetToMarkdown(value: BraveSnippet): string | undefined {
|
|
350
254
|
const table = value.table;
|
|
255
|
+
|
|
351
256
|
if (!table || table.length === 0) return undefined;
|
|
352
257
|
|
|
353
258
|
const rows = table.filter(Boolean);
|
|
354
259
|
const headers = [...new Set(rows.flatMap((row) => Object.keys(row)))];
|
|
260
|
+
|
|
355
261
|
if (headers.length === 0) return undefined;
|
|
356
262
|
|
|
357
263
|
const caption = value.caption ? `**${escapeMarkdownLinkText(value.caption)}**\n\n` : "";
|
|
358
264
|
const header = `| ${headers.map(escapeMarkdownCell).join(" | ")} |`;
|
|
359
265
|
const separator = `| ${headers.map(() => "---").join(" | ")} |`;
|
|
266
|
+
|
|
360
267
|
const body = rows
|
|
361
268
|
.map((row) => `| ${headers.map((key) => escapeMarkdownCell(String(row[key]))).join(" | ")} |`)
|
|
362
269
|
.join("\n");
|
|
270
|
+
|
|
363
271
|
return `${caption}${header}\n${separator}\n${body}`;
|
|
364
272
|
}
|
|
365
273
|
|
|
274
|
+
/** Converts a Brave structured snippet payload to bounded, display-safe Markdown. */
|
|
366
275
|
function braveSnippetToMarkdown(value: string): string {
|
|
367
276
|
try {
|
|
368
277
|
const snippet = value.trim();
|
|
278
|
+
|
|
369
279
|
if (!snippet) return "";
|
|
370
280
|
const parsed: BraveSnippet = JSON.parse(snippet);
|
|
371
281
|
const rendered = structuredSnippetToMarkdown(parsed);
|
|
282
|
+
|
|
372
283
|
if (rendered !== undefined) return rendered.slice(0, 8000);
|
|
284
|
+
|
|
373
285
|
return `\`\`\`json\n${JSON.stringify(parsed, null, 2)}\n\`\`\``.slice(0, 8000);
|
|
374
286
|
} catch {
|
|
375
287
|
return String(value)
|
|
@@ -403,18 +315,28 @@ export async function searchBraveWeb(
|
|
|
403
315
|
url.searchParams.set("count", String(count));
|
|
404
316
|
url.searchParams.set("safesearch", extras.safesearch ?? "moderate");
|
|
405
317
|
url.searchParams.set("text_decorations", "false");
|
|
318
|
+
|
|
406
319
|
if (language) url.searchParams.set("search_lang", language);
|
|
320
|
+
|
|
407
321
|
if (extras.country) url.searchParams.set("country", extras.country.toUpperCase());
|
|
322
|
+
|
|
408
323
|
if (extras.extraSnippets) url.searchParams.set("extra_snippets", "true");
|
|
324
|
+
|
|
409
325
|
if (extras.operators !== undefined) url.searchParams.set("operators", String(extras.operators));
|
|
326
|
+
|
|
410
327
|
if (extras.spellcheck !== undefined)
|
|
411
328
|
url.searchParams.set("spellcheck", String(extras.spellcheck));
|
|
329
|
+
|
|
412
330
|
if (extras.resultFilter)
|
|
413
331
|
url.searchParams.set("result_filter", normalizeResultFilter(extras.resultFilter));
|
|
332
|
+
|
|
414
333
|
if (extras.goggles) url.searchParams.set("goggles", extras.goggles);
|
|
334
|
+
|
|
415
335
|
if (extras.offset !== undefined) url.searchParams.set("offset", String(extras.offset));
|
|
336
|
+
|
|
416
337
|
if (extras.uiLang) url.searchParams.set("ui_lang", extras.uiLang);
|
|
417
338
|
const mappedFreshness = mapFreshness(freshness, extras.dateRange);
|
|
339
|
+
|
|
418
340
|
if (mappedFreshness) url.searchParams.set("freshness", mappedFreshness);
|
|
419
341
|
|
|
420
342
|
const data = await requestJson<BraveWebResponse & { query?: BraveQuery }>(
|
|
@@ -432,15 +354,18 @@ export async function searchBraveWeb(
|
|
|
432
354
|
results: (data.web?.results ?? []).map((item) => {
|
|
433
355
|
const title = normalizeText(item.title ?? "", 300);
|
|
434
356
|
let snippet = normalizeText(item.description ?? "", 600);
|
|
357
|
+
|
|
435
358
|
if (extras.extraSnippets && item.extra_snippets?.length) {
|
|
436
359
|
const additional = item.extra_snippets
|
|
437
360
|
.slice(0, BRAVE_MAX_EXTRA_SNIPPETS)
|
|
438
361
|
.map((value) => normalizeText(value ?? "", 300))
|
|
439
362
|
.filter(Boolean);
|
|
363
|
+
|
|
440
364
|
if (additional.length) {
|
|
441
365
|
snippet = normalizeText([snippet, ...additional].filter(Boolean).join("\n\n"), 2000);
|
|
442
366
|
}
|
|
443
367
|
}
|
|
368
|
+
|
|
444
369
|
return {
|
|
445
370
|
title,
|
|
446
371
|
url: normalizeUrl(item.url ?? ""),
|
|
@@ -480,12 +405,18 @@ export async function searchBraveContext(
|
|
|
480
405
|
url.searchParams.set("maximum_number_of_tokens_per_url", String(budget.tokensPerUrl));
|
|
481
406
|
url.searchParams.set("maximum_number_of_snippets_per_url", String(budget.snippetsPerUrl));
|
|
482
407
|
url.searchParams.set("context_threshold_mode", extras.threshold ?? "strict");
|
|
408
|
+
|
|
483
409
|
if (language) url.searchParams.set("search_lang", language);
|
|
410
|
+
|
|
484
411
|
if (extras.country) url.searchParams.set("country", extras.country.toUpperCase());
|
|
412
|
+
|
|
485
413
|
if (extras.safesearch) url.searchParams.set("safesearch", extras.safesearch);
|
|
414
|
+
|
|
486
415
|
if (extras.spellcheck !== undefined)
|
|
487
416
|
url.searchParams.set("spellcheck", String(extras.spellcheck));
|
|
417
|
+
|
|
488
418
|
if (extras.goggles) url.searchParams.set("goggles", extras.goggles);
|
|
419
|
+
|
|
489
420
|
if (freshness)
|
|
490
421
|
url.searchParams.set(
|
|
491
422
|
"freshness",
|
|
@@ -508,8 +439,10 @@ export async function searchBraveContext(
|
|
|
508
439
|
const snippets = [
|
|
509
440
|
...new Set((item.snippets ?? []).map(braveSnippetToMarkdown).filter(Boolean)),
|
|
510
441
|
];
|
|
442
|
+
|
|
511
443
|
const title = normalizeText(item.title ?? "", 300);
|
|
512
444
|
const snippet = snippets.slice(0, budget.snippetsPerUrl).join("\n\n");
|
|
445
|
+
|
|
513
446
|
return {
|
|
514
447
|
title,
|
|
515
448
|
url: normalizeUrl(item.url ?? ""),
|
package/src/cache.ts
CHANGED
|
@@ -48,9 +48,11 @@ export function stableKeyHash(key: string): string {
|
|
|
48
48
|
export function resolveCachePath(directory: string, keyPath: string): string {
|
|
49
49
|
const base = resolve(directory);
|
|
50
50
|
const full = resolve(base, keyPath);
|
|
51
|
+
|
|
51
52
|
if (full !== base && !full.startsWith(base + sep)) {
|
|
52
53
|
throw new Error(`Refusing to write cache entry outside ${base}: ${keyPath}`);
|
|
53
54
|
}
|
|
55
|
+
|
|
54
56
|
return full;
|
|
55
57
|
}
|
|
56
58
|
|
|
@@ -63,10 +65,12 @@ export function resolveCachePath(directory: string, keyPath: string): string {
|
|
|
63
65
|
function encodeExpiresAt(expiresAt: number): Uint8Array {
|
|
64
66
|
const out = new Uint8Array(8);
|
|
65
67
|
let value = BigInt(Math.round(expiresAt));
|
|
68
|
+
|
|
66
69
|
for (let index = 7; index >= 0; index -= 1) {
|
|
67
70
|
out[index] = Number(value & 0xffn);
|
|
68
71
|
value >>= 8n;
|
|
69
72
|
}
|
|
73
|
+
|
|
70
74
|
return out;
|
|
71
75
|
}
|
|
72
76
|
|
|
@@ -78,7 +82,9 @@ function encodeExpiresAt(expiresAt: number): Uint8Array {
|
|
|
78
82
|
*/
|
|
79
83
|
function decodeExpiresAt(bytes: Uint8Array): number {
|
|
80
84
|
let value = 0n;
|
|
85
|
+
|
|
81
86
|
for (let index = 0; index < 8; index += 1) value = (value << 8n) | BigInt(bytes[index]);
|
|
87
|
+
|
|
82
88
|
return Number(value);
|
|
83
89
|
}
|
|
84
90
|
|
|
@@ -123,19 +129,26 @@ export class ExpiringLruCache<K, V> {
|
|
|
123
129
|
*/
|
|
124
130
|
get(key: K): V | undefined {
|
|
125
131
|
const entry = this.#entries.get(key);
|
|
132
|
+
|
|
126
133
|
if (entry) {
|
|
127
134
|
if (entry.expiresAt <= this.now()) {
|
|
128
135
|
this.#delete(key);
|
|
136
|
+
|
|
129
137
|
return undefined;
|
|
130
138
|
}
|
|
139
|
+
|
|
131
140
|
this.#entries.delete(key);
|
|
132
141
|
this.#entries.set(key, entry);
|
|
142
|
+
|
|
133
143
|
return entry.value;
|
|
134
144
|
}
|
|
145
|
+
|
|
135
146
|
if (this.persistence) {
|
|
136
147
|
const loaded = this.#loadFromDisk(key);
|
|
148
|
+
|
|
137
149
|
if (loaded !== undefined) return loaded.value;
|
|
138
150
|
}
|
|
151
|
+
|
|
139
152
|
return undefined;
|
|
140
153
|
}
|
|
141
154
|
|
|
@@ -150,13 +163,16 @@ export class ExpiringLruCache<K, V> {
|
|
|
150
163
|
set(key: K, value: V, expiresAt: number): boolean {
|
|
151
164
|
this.#delete(key);
|
|
152
165
|
const size = this.sizeOf(value);
|
|
166
|
+
|
|
153
167
|
if (size > this.maxBytes) return false;
|
|
154
168
|
|
|
155
169
|
this.#entries.set(key, { expiresAt, size, value });
|
|
156
170
|
this.#byteSize += size;
|
|
157
171
|
this.#evict();
|
|
158
172
|
const stored = this.#entries.has(key);
|
|
173
|
+
|
|
159
174
|
if (stored && this.persistence) this.#writeToDisk(key, value, expiresAt);
|
|
175
|
+
|
|
160
176
|
return stored;
|
|
161
177
|
}
|
|
162
178
|
|
|
@@ -164,6 +180,7 @@ export class ExpiringLruCache<K, V> {
|
|
|
164
180
|
let bytes: Uint8Array;
|
|
165
181
|
const path = resolveCachePath(this.persistence!.directory, this.persistence!.keyToPath(key));
|
|
166
182
|
let fileSize: number;
|
|
183
|
+
|
|
167
184
|
try {
|
|
168
185
|
// Serialized metadata can exceed the value budget, but never allow an
|
|
169
186
|
// unbounded cache file to be read into memory.
|
|
@@ -171,37 +188,50 @@ export class ExpiringLruCache<K, V> {
|
|
|
171
188
|
} catch {
|
|
172
189
|
return undefined;
|
|
173
190
|
}
|
|
191
|
+
|
|
174
192
|
if (fileSize < 8 || fileSize > this.maxBytes * 2 + 65_536) {
|
|
175
193
|
this.#removeFromDisk(key);
|
|
194
|
+
|
|
176
195
|
return undefined;
|
|
177
196
|
}
|
|
197
|
+
|
|
178
198
|
try {
|
|
179
199
|
bytes = readFileSync(path);
|
|
180
200
|
} catch {
|
|
181
201
|
return undefined;
|
|
182
202
|
}
|
|
203
|
+
|
|
183
204
|
let entry: ExpiringCacheEntry<V>;
|
|
205
|
+
|
|
184
206
|
try {
|
|
185
207
|
const expiresAt = decodeExpiresAt(bytes);
|
|
186
208
|
const value = this.persistence!.deserialize(bytes.subarray(8));
|
|
209
|
+
|
|
187
210
|
if (this.persistence!.validate && !this.persistence!.validate(value)) {
|
|
188
211
|
throw new Error("invalid cache entry");
|
|
189
212
|
}
|
|
213
|
+
|
|
190
214
|
const size = this.sizeOf(value);
|
|
215
|
+
|
|
191
216
|
if (!Number.isFinite(size) || size < 0 || size > this.maxBytes)
|
|
192
217
|
throw new Error("oversized cache entry");
|
|
193
218
|
entry = { expiresAt, size, value };
|
|
194
219
|
} catch {
|
|
195
220
|
this.#removeFromDisk(key);
|
|
221
|
+
|
|
196
222
|
return undefined;
|
|
197
223
|
}
|
|
224
|
+
|
|
198
225
|
if (entry.expiresAt <= this.now()) {
|
|
199
226
|
this.#removeFromDisk(key);
|
|
227
|
+
|
|
200
228
|
return undefined;
|
|
201
229
|
}
|
|
230
|
+
|
|
202
231
|
this.#entries.set(key, entry);
|
|
203
232
|
this.#byteSize += entry.size;
|
|
204
233
|
this.#evict();
|
|
234
|
+
|
|
205
235
|
return this.#entries.get(key);
|
|
206
236
|
}
|
|
207
237
|
|
|
@@ -211,10 +241,12 @@ export class ExpiringLruCache<K, V> {
|
|
|
211
241
|
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
212
242
|
chmodSync(directory, 0o700);
|
|
213
243
|
const payload = this.persistence!.serialize(value);
|
|
244
|
+
|
|
214
245
|
if (payload.byteLength > this.maxBytes * 2 + 65_536)
|
|
215
246
|
throw new Error("oversized cache payload");
|
|
216
247
|
const path = resolveCachePath(directory, this.persistence!.keyToPath(key));
|
|
217
248
|
const temp = `${path}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
249
|
+
|
|
218
250
|
try {
|
|
219
251
|
writeFileSync(temp, Buffer.concat([encodeExpiresAt(expiresAt), payload]), { mode: 0o600 });
|
|
220
252
|
chmodSync(temp, 0o600);
|
|
@@ -241,27 +273,35 @@ export class ExpiringLruCache<K, V> {
|
|
|
241
273
|
|
|
242
274
|
#maintainDisk(): void {
|
|
243
275
|
const persistence = this.persistence!;
|
|
276
|
+
|
|
244
277
|
try {
|
|
245
278
|
const files = readdirSync(persistence.directory);
|
|
246
279
|
const candidates: Array<{ path: string; size: number; mtime: number }> = [];
|
|
280
|
+
|
|
247
281
|
for (const file of files) {
|
|
248
282
|
const path = resolveCachePath(persistence.directory, file);
|
|
283
|
+
|
|
249
284
|
if (file.endsWith(".tmp")) {
|
|
250
285
|
try {
|
|
251
286
|
unlinkSync(path);
|
|
252
287
|
} catch {
|
|
253
288
|
// Best effort cleanup.
|
|
254
289
|
}
|
|
290
|
+
|
|
255
291
|
continue;
|
|
256
292
|
}
|
|
293
|
+
|
|
257
294
|
try {
|
|
258
295
|
const stat = statSync(path);
|
|
296
|
+
|
|
259
297
|
if (!stat.isFile() || stat.size < 8 || stat.size > this.maxBytes * 2 + 65_536) {
|
|
260
298
|
unlinkSync(path);
|
|
261
299
|
continue;
|
|
262
300
|
}
|
|
301
|
+
|
|
263
302
|
const descriptor = openSync(path, "r");
|
|
264
303
|
const header = new Uint8Array(8);
|
|
304
|
+
|
|
265
305
|
try {
|
|
266
306
|
if (
|
|
267
307
|
readSync(descriptor, header, 0, 8, 0) !== 8 ||
|
|
@@ -273,17 +313,22 @@ export class ExpiringLruCache<K, V> {
|
|
|
273
313
|
} finally {
|
|
274
314
|
closeSync(descriptor);
|
|
275
315
|
}
|
|
316
|
+
|
|
276
317
|
candidates.push({ path, size: stat.size, mtime: stat.mtimeMs });
|
|
277
318
|
} catch {
|
|
278
319
|
// Ignore races and inaccessible files in this best-effort sweep.
|
|
279
320
|
}
|
|
280
321
|
}
|
|
322
|
+
|
|
281
323
|
candidates.sort((a, b) => a.mtime - b.mtime);
|
|
282
324
|
let total = candidates.reduce((sum, candidate) => sum + candidate.size, 0);
|
|
325
|
+
|
|
283
326
|
while (candidates.length > this.maxEntries || total > this.maxBytes * 2 + 65_536) {
|
|
284
327
|
const oldest = candidates.shift();
|
|
328
|
+
|
|
285
329
|
if (!oldest) break;
|
|
286
330
|
total -= oldest.size;
|
|
331
|
+
|
|
287
332
|
try {
|
|
288
333
|
unlinkSync(oldest.path);
|
|
289
334
|
} catch {
|
|
@@ -298,6 +343,7 @@ export class ExpiringLruCache<K, V> {
|
|
|
298
343
|
#evict(): void {
|
|
299
344
|
while (this.#entries.size > this.maxEntries || this.#byteSize > this.maxBytes) {
|
|
300
345
|
const oldest = this.#entries.keys().next().value;
|
|
346
|
+
|
|
301
347
|
if (oldest === undefined) break;
|
|
302
348
|
this.#delete(oldest);
|
|
303
349
|
}
|
|
@@ -305,12 +351,16 @@ export class ExpiringLruCache<K, V> {
|
|
|
305
351
|
|
|
306
352
|
#delete(key: K): void {
|
|
307
353
|
const entry = this.#entries.get(key);
|
|
354
|
+
|
|
308
355
|
if (!entry) {
|
|
309
356
|
if (this.persistence) this.#removeFromDisk(key);
|
|
357
|
+
|
|
310
358
|
return;
|
|
311
359
|
}
|
|
360
|
+
|
|
312
361
|
this.#entries.delete(key);
|
|
313
362
|
this.#byteSize -= entry.size;
|
|
363
|
+
|
|
314
364
|
if (this.persistence) this.#removeFromDisk(key);
|
|
315
365
|
}
|
|
316
366
|
}
|