@zeldrisho/pi-web-search 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -20
- package/README.md +1 -1
- package/package.json +7 -6
- package/src/brave.ts +80 -51
- package/src/cache.ts +132 -9
- package/src/index.ts +13 -27
- package/src/provider.ts +5 -5
- package/src/search.ts +39 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,48 +1,70 @@
|
|
|
1
1
|
# Changelog
|
|
2
|
-
## [0.4.1](https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.4.0...pi-web-search-v0.4.1) (2026-08-03)
|
|
3
2
|
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/2.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
## [Unreleased]
|
|
8
9
|
|
|
10
|
+
## [0.5.0] - 2026-08-20
|
|
9
11
|
|
|
10
|
-
###
|
|
12
|
+
### Added
|
|
11
13
|
|
|
12
|
-
-
|
|
14
|
+
- Persist search results to a private cross-session disk cache (24h TTL) behind the existing in-memory cache
|
|
15
|
+
- Report honest-evidence metadata (`evidence` summary and per-result `quality`) in `details`
|
|
13
16
|
|
|
17
|
+
## [0.4.2] - 2026-08-05
|
|
14
18
|
|
|
15
|
-
###
|
|
19
|
+
### Changed
|
|
16
20
|
|
|
17
|
-
-
|
|
21
|
+
- Document runtime enforcement of the context query limit ([347b386](https://github.com/zeldrisho/pi-packages/commit/347b386c62f33d489b6be207aff2a96f491eced0))
|
|
18
22
|
|
|
19
|
-
|
|
23
|
+
### Fixed
|
|
24
|
+
|
|
25
|
+
- Emit provider-compatible object schema for tool parameters ([fe81916](https://github.com/zeldrisho/pi-packages/commit/fe819161e9cf61be09a0d68be713b63cb318155e))
|
|
20
26
|
|
|
27
|
+
## [0.4.1] - 2026-08-03
|
|
21
28
|
|
|
22
|
-
|
|
29
|
+
### Changed
|
|
23
30
|
|
|
31
|
+
- Clarify local package management ([f5b420e](https://github.com/zeldrisho/pi-packages/commit/f5b420eba246eb6a372a4b6a9036a060b9db623b))
|
|
32
|
+
- Strengthen repository maintenance boundaries ([a44da67](https://github.com/zeldrisho/pi-packages/commit/a44da679b93ae6ea7ebdd77389bf84716b195e33))
|
|
33
|
+
- Strengthen maintenance safety ([8a26436](https://github.com/zeldrisho/pi-packages/commit/8a26436780373f21535580224b59bc6dd2a4ad72))
|
|
24
34
|
|
|
25
|
-
###
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- Address pull request review feedback ([3b05033](https://github.com/zeldrisho/pi-packages/commit/3b05033003d173809134d9e2d2a53e82e45050b6))
|
|
26
38
|
|
|
27
|
-
|
|
39
|
+
## [0.4.0] - 2026-07-25
|
|
28
40
|
|
|
29
|
-
|
|
41
|
+
### Added
|
|
30
42
|
|
|
43
|
+
- **web-tools:** strengthen independent tool boundaries ([71aba47](https://github.com/zeldrisho/pi-packages/commit/71aba47db47483b6a75935796a1747603850aa96))
|
|
31
44
|
|
|
32
|
-
|
|
45
|
+
## [0.3.1] - 2026-07-22
|
|
33
46
|
|
|
34
|
-
|
|
47
|
+
### Fixed
|
|
35
48
|
|
|
36
|
-
|
|
49
|
+
- address consolidated review feedback ([cea33a1](https://github.com/zeldrisho/pi-packages/commit/cea33a1f30783bd19231fe9dc69686c58f313785))
|
|
37
50
|
|
|
51
|
+
## [0.3.0] - 2026-07-20
|
|
38
52
|
|
|
39
|
-
###
|
|
53
|
+
### Added
|
|
40
54
|
|
|
41
|
-
|
|
55
|
+
- **web:** Add collapsible web tool results ([e8664d6](https://github.com/zeldrisho/pi-packages/commit/e8664d6c07b719d92f18d9a0048b47cc1970b97c))
|
|
42
56
|
|
|
43
|
-
## 0.2.0
|
|
57
|
+
## [0.2.0] - 2026-07-19
|
|
44
58
|
|
|
59
|
+
### Added
|
|
45
60
|
|
|
46
|
-
|
|
61
|
+
- Add bounded Brave web search ([cbc9491](https://github.com/zeldrisho/pi-packages/commit/cbc9491dce36555ab91b6bef203ec8b380596a89))
|
|
47
62
|
|
|
48
|
-
|
|
63
|
+
[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.5.0...HEAD
|
|
64
|
+
[0.5.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.4.2...pi-web-search-v0.5.0
|
|
65
|
+
[0.4.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.4.1...pi-web-search-v0.4.2
|
|
66
|
+
[0.4.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.4.0...pi-web-search-v0.4.1
|
|
67
|
+
[0.4.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.3.1...pi-web-search-v0.4.0
|
|
68
|
+
[0.3.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.3.0...pi-web-search-v0.3.1
|
|
69
|
+
[0.3.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-search-v0.2.0...pi-web-search-v0.3.0
|
|
70
|
+
[0.2.0]: https://github.com/zeldrisho/pi-packages/releases/tag/pi-web-search-v0.2.0
|
package/README.md
CHANGED
|
@@ -27,7 +27,7 @@ If Pi is already running when you set the environment variable, run `/reload` in
|
|
|
27
27
|
|
|
28
28
|
## Usage
|
|
29
29
|
|
|
30
|
-
The `web_search` tool returns compact web results by default, making it suitable for discovering current sources and URLs. Set `mode` to `context` to call only Brave's LLM Context API and return provider-extracted snippets. Context queries are limited to 400 characters; web-mode queries are limited to 500 characters.
|
|
30
|
+
The `web_search` tool returns compact web results by default, making it suitable for discovering current sources and URLs. Set `mode` to `context` to call only Brave's LLM Context API and return provider-extracted snippets. Context queries are limited to 400 characters; web-mode queries are limited to 500 characters. Because the provider rejects union schemas, the tool schema accepts up to the web-mode query length for either mode; the tighter context-mode limit is enforced at runtime when the request executes, not in the schema.
|
|
31
31
|
|
|
32
32
|
Context snippets may not reflect the current live page. The tool does not fetch result URLs. When live-page inspection is needed, call `web_fetch` explicitly on the relevant URL.
|
|
33
33
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zeldrisho/pi-web-search",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Pi extension for bounded Brave web search and LLM context retrieval",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"brave-search",
|
|
@@ -32,12 +32,13 @@
|
|
|
32
32
|
"access": "public"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
-
"@earendil-works/pi-ai": "^0.
|
|
36
|
-
"@earendil-works/pi-coding-agent": "^0.
|
|
37
|
-
"@earendil-works/pi-tui": "^0.
|
|
35
|
+
"@earendil-works/pi-ai": "^0.84.0",
|
|
36
|
+
"@earendil-works/pi-coding-agent": "^0.84.0",
|
|
37
|
+
"@earendil-works/pi-tui": "^0.84.0",
|
|
38
38
|
"typebox": "^1.1.24",
|
|
39
|
-
"typescript": "^
|
|
40
|
-
"vite
|
|
39
|
+
"typescript": "^7.0.0",
|
|
40
|
+
"vite": "npm:@voidzero-dev/vite-plus-core@0.2.9",
|
|
41
|
+
"vite-plus": "0.2.9"
|
|
41
42
|
},
|
|
42
43
|
"peerDependencies": {
|
|
43
44
|
"@earendil-works/pi-ai": "*",
|
package/src/brave.ts
CHANGED
|
@@ -10,10 +10,52 @@ export type Provider = "brave";
|
|
|
10
10
|
export type Freshness = "day" | "week" | "month" | "year";
|
|
11
11
|
export type SearchMode = "web" | "context";
|
|
12
12
|
|
|
13
|
+
export type ResultQuality = "high" | "medium" | "low";
|
|
14
|
+
|
|
13
15
|
export interface SearchResult {
|
|
14
16
|
title: string;
|
|
15
17
|
url: string;
|
|
16
18
|
snippet: string;
|
|
19
|
+
/** Honest-evidence hint about how useful the result is likely to be as a citation. */
|
|
20
|
+
quality: ResultQuality;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
interface BraveWebResult {
|
|
24
|
+
title?: string;
|
|
25
|
+
url?: string;
|
|
26
|
+
description?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface BraveWebResponse {
|
|
30
|
+
web?: { results?: BraveWebResult[] };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface BraveSnippet {
|
|
34
|
+
caption?: string;
|
|
35
|
+
table?: Array<Record<string, string>>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface BraveContextResult {
|
|
39
|
+
title?: string;
|
|
40
|
+
url?: string;
|
|
41
|
+
snippets?: string[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface BraveContextResponse {
|
|
45
|
+
grounding?: { generic?: BraveContextResult[] };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Classifies a search result by how much usable, sourced information it carries.
|
|
50
|
+
*
|
|
51
|
+
* @param result - The result title and snippet to assess
|
|
52
|
+
* @returns A coarse quality hint for weighting citations
|
|
53
|
+
*/
|
|
54
|
+
export function classifyResultQuality(result: { title: string; snippet: string }): ResultQuality {
|
|
55
|
+
if (!result.title && !result.snippet) return "low";
|
|
56
|
+
if (result.snippet.length >= 80) return "high";
|
|
57
|
+
if (result.title || result.snippet) return "medium";
|
|
58
|
+
return "low";
|
|
17
59
|
}
|
|
18
60
|
|
|
19
61
|
const BRAVE_MAX_CONTEXT_TOKENS = 2_048;
|
|
@@ -49,11 +91,10 @@ export function validateProviderRequest(query: string, count: number, mode: Sear
|
|
|
49
91
|
/**
|
|
50
92
|
* Normalizes an HTTP or HTTPS URL and limits the result to 2,048 characters.
|
|
51
93
|
*
|
|
52
|
-
* @param value - The
|
|
94
|
+
* @param value - The URL text to normalize
|
|
53
95
|
* @returns The normalized URL, or an empty string for invalid values or unsupported protocols
|
|
54
96
|
*/
|
|
55
|
-
function normalizeUrl(value:
|
|
56
|
-
if (typeof value !== "string") return "";
|
|
97
|
+
function normalizeUrl(value: string): string {
|
|
57
98
|
try {
|
|
58
99
|
const url = new URL(value);
|
|
59
100
|
return url.protocol === "http:" || url.protocol === "https:"
|
|
@@ -68,55 +109,37 @@ function escapeMarkdownLinkText(value: string): string {
|
|
|
68
109
|
return value.replace(/([\\[\]])/g, "\\$1");
|
|
69
110
|
}
|
|
70
111
|
|
|
71
|
-
function escapeMarkdownCell(value:
|
|
72
|
-
|
|
73
|
-
if (typeof value === "string") text = value;
|
|
74
|
-
else if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {
|
|
75
|
-
text = String(value);
|
|
76
|
-
} else if (value !== null && value !== undefined) {
|
|
77
|
-
try {
|
|
78
|
-
text = JSON.stringify(value) ?? "";
|
|
79
|
-
} catch {
|
|
80
|
-
text = "";
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
return text.replace(/\|/g, "\\|").replace(/\r?\n/g, "<br>");
|
|
112
|
+
function escapeMarkdownCell(value: string): string {
|
|
113
|
+
return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, "<br>");
|
|
84
114
|
}
|
|
85
115
|
|
|
86
|
-
function structuredSnippetToMarkdown(value:
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
if (!Array.isArray(record.table) || record.table.length === 0) return undefined;
|
|
116
|
+
function structuredSnippetToMarkdown(value: BraveSnippet): string | undefined {
|
|
117
|
+
const table = value.table;
|
|
118
|
+
if (!table || table.length === 0) return undefined;
|
|
90
119
|
|
|
91
|
-
const rows =
|
|
92
|
-
(row): row is Record<string, unknown> => Boolean(row) && typeof row === "object",
|
|
93
|
-
);
|
|
120
|
+
const rows = table.filter(Boolean);
|
|
94
121
|
const headers = [...new Set(rows.flatMap((row) => Object.keys(row)))];
|
|
95
122
|
if (headers.length === 0) return undefined;
|
|
96
123
|
|
|
97
|
-
const caption =
|
|
98
|
-
typeof record.caption === "string" ? `**${escapeMarkdownLinkText(record.caption)}**\n\n` : "";
|
|
124
|
+
const caption = value.caption ? `**${escapeMarkdownLinkText(value.caption)}**\n\n` : "";
|
|
99
125
|
const header = `| ${headers.map(escapeMarkdownCell).join(" | ")} |`;
|
|
100
126
|
const separator = `| ${headers.map(() => "---").join(" | ")} |`;
|
|
101
127
|
const body = rows
|
|
102
|
-
.map((row) => `| ${headers.map((key) => escapeMarkdownCell(row[key])).join(" | ")} |`)
|
|
128
|
+
.map((row) => `| ${headers.map((key) => escapeMarkdownCell(String(row[key]))).join(" | ")} |`)
|
|
103
129
|
.join("\n");
|
|
104
130
|
return `${caption}${header}\n${separator}\n${body}`;
|
|
105
131
|
}
|
|
106
132
|
|
|
107
|
-
function braveSnippetToMarkdown(value:
|
|
108
|
-
if (typeof value !== "string") return "";
|
|
109
|
-
const snippet = value.trim();
|
|
110
|
-
if (!snippet) return "";
|
|
111
|
-
|
|
133
|
+
function braveSnippetToMarkdown(value: string): string {
|
|
112
134
|
try {
|
|
113
|
-
const
|
|
114
|
-
return
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
).slice(0, 8000);
|
|
135
|
+
const snippet = value.trim();
|
|
136
|
+
if (!snippet) return "";
|
|
137
|
+
const parsed: BraveSnippet = JSON.parse(snippet);
|
|
138
|
+
const rendered = structuredSnippetToMarkdown(parsed);
|
|
139
|
+
if (rendered !== undefined) return rendered.slice(0, 8000);
|
|
140
|
+
return `\`\`\`json\n${JSON.stringify(parsed, null, 2)}\n\`\`\``.slice(0, 8000);
|
|
118
141
|
} catch {
|
|
119
|
-
return
|
|
142
|
+
return String(value)
|
|
120
143
|
.replace(/\r\n/g, "\n")
|
|
121
144
|
.replace(/\n{4,}/g, "\n\n\n")
|
|
122
145
|
.slice(0, 8000);
|
|
@@ -150,7 +173,7 @@ export async function searchBraveWeb(
|
|
|
150
173
|
{ day: "pd", week: "pw", month: "pm", year: "py" }[freshness],
|
|
151
174
|
);
|
|
152
175
|
|
|
153
|
-
const data =
|
|
176
|
+
const data = await requestJson<BraveWebResponse>(
|
|
154
177
|
url,
|
|
155
178
|
{
|
|
156
179
|
headers: {
|
|
@@ -159,13 +182,18 @@ export async function searchBraveWeb(
|
|
|
159
182
|
},
|
|
160
183
|
},
|
|
161
184
|
signal,
|
|
162
|
-
)
|
|
185
|
+
);
|
|
163
186
|
|
|
164
|
-
return (data.web?.results ?? []).map((item) =>
|
|
165
|
-
title
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
187
|
+
return (data.web?.results ?? []).map((item) => {
|
|
188
|
+
const title = normalizeText(item.title ?? "", 300);
|
|
189
|
+
const snippet = normalizeText(item.description ?? "", 600);
|
|
190
|
+
return {
|
|
191
|
+
title,
|
|
192
|
+
url: normalizeUrl(item.url ?? ""),
|
|
193
|
+
snippet,
|
|
194
|
+
quality: classifyResultQuality({ title, snippet }),
|
|
195
|
+
};
|
|
196
|
+
});
|
|
169
197
|
}
|
|
170
198
|
|
|
171
199
|
/**
|
|
@@ -198,7 +226,7 @@ export async function searchBraveContext(
|
|
|
198
226
|
{ day: "pd", week: "pw", month: "pm", year: "py" }[freshness],
|
|
199
227
|
);
|
|
200
228
|
|
|
201
|
-
const data =
|
|
229
|
+
const data = await requestJson<BraveContextResponse>(
|
|
202
230
|
url,
|
|
203
231
|
{
|
|
204
232
|
headers: {
|
|
@@ -207,18 +235,19 @@ export async function searchBraveContext(
|
|
|
207
235
|
},
|
|
208
236
|
},
|
|
209
237
|
signal,
|
|
210
|
-
)
|
|
211
|
-
grounding?: { generic?: Array<{ title?: unknown; url?: unknown; snippets?: unknown[] }> };
|
|
212
|
-
};
|
|
238
|
+
);
|
|
213
239
|
|
|
214
240
|
return (data.grounding?.generic ?? []).map((item) => {
|
|
215
241
|
const snippets = [
|
|
216
242
|
...new Set((item.snippets ?? []).map(braveSnippetToMarkdown).filter(Boolean)),
|
|
217
243
|
];
|
|
244
|
+
const title = normalizeText(item.title ?? "", 300);
|
|
245
|
+
const snippet = snippets.slice(0, BRAVE_MAX_SNIPPETS_PER_URL).join("\n\n");
|
|
218
246
|
return {
|
|
219
|
-
title
|
|
220
|
-
url: normalizeUrl(item.url),
|
|
221
|
-
snippet
|
|
247
|
+
title,
|
|
248
|
+
url: normalizeUrl(item.url ?? ""),
|
|
249
|
+
snippet,
|
|
250
|
+
quality: classifyResultQuality({ title, snippet }),
|
|
222
251
|
};
|
|
223
252
|
});
|
|
224
253
|
}
|
package/src/cache.ts
CHANGED
|
@@ -1,9 +1,61 @@
|
|
|
1
|
+
import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { resolve, sep } from "node:path";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
|
|
1
5
|
interface ExpiringCacheEntry<V> {
|
|
2
6
|
expiresAt: number;
|
|
3
7
|
size: number;
|
|
4
8
|
value: V;
|
|
5
9
|
}
|
|
6
10
|
|
|
11
|
+
/** A best-effort on-disk backing store for cache entries. */
|
|
12
|
+
export interface CachePersistence<K, V> {
|
|
13
|
+
/** Directory that holds private cache files, created lazily with 0700 perms. */
|
|
14
|
+
readonly directory: string;
|
|
15
|
+
/** Serialize a value to bytes for on-disk storage. */
|
|
16
|
+
serialize(value: V): Uint8Array;
|
|
17
|
+
/** Deserialize bytes read from disk; throw on corrupt data so it is treated as a miss. */
|
|
18
|
+
deserialize(bytes: Uint8Array): V;
|
|
19
|
+
/** Map a logical cache key to a safe, collision-resistant on-disk filename. */
|
|
20
|
+
keyToPath(key: K): string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Hashes a cache key into a safe, collision-resistant filename segment. */
|
|
24
|
+
export function stableKeyHash(key: string): string {
|
|
25
|
+
return createHash("sha256").update(key).digest("hex");
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Resolves a cache entry's on-disk path and refuses to escape the cache
|
|
30
|
+
* directory, guarding against a `keyToPath` that returns `..` segments. The
|
|
31
|
+
* `keyToPath` implementations in this repo return a hex SHA-256 digest, so this
|
|
32
|
+
* never triggers in normal operation but keeps best-effort persistence safe.
|
|
33
|
+
*/
|
|
34
|
+
export function resolveCachePath(directory: string, keyPath: string): string {
|
|
35
|
+
const base = resolve(directory);
|
|
36
|
+
const full = resolve(base, keyPath);
|
|
37
|
+
if (full !== base && !full.startsWith(base + sep)) {
|
|
38
|
+
throw new Error(`Refusing to write cache entry outside ${base}: ${keyPath}`);
|
|
39
|
+
}
|
|
40
|
+
return full;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function encodeExpiresAt(expiresAt: number): Uint8Array {
|
|
44
|
+
const out = new Uint8Array(8);
|
|
45
|
+
let value = BigInt(Math.round(expiresAt));
|
|
46
|
+
for (let index = 7; index >= 0; index -= 1) {
|
|
47
|
+
out[index] = Number(value & 0xffn);
|
|
48
|
+
value >>= 8n;
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function decodeExpiresAt(bytes: Uint8Array): number {
|
|
54
|
+
let value = 0n;
|
|
55
|
+
for (let index = 0; index < 8; index += 1) value = (value << 8n) | BigInt(bytes[index]);
|
|
56
|
+
return Number(value);
|
|
57
|
+
}
|
|
58
|
+
|
|
7
59
|
/** An expiring least-recently-used cache bounded by entry count and aggregate bytes. */
|
|
8
60
|
export class ExpiringLruCache<K, V> {
|
|
9
61
|
readonly #entries = new Map<K, ExpiringCacheEntry<V>>();
|
|
@@ -14,6 +66,7 @@ export class ExpiringLruCache<K, V> {
|
|
|
14
66
|
readonly maxBytes: number,
|
|
15
67
|
readonly sizeOf: (value: V) => number,
|
|
16
68
|
readonly now: () => number = Date.now,
|
|
69
|
+
readonly persistence?: CachePersistence<K, V>,
|
|
17
70
|
) {}
|
|
18
71
|
|
|
19
72
|
get byteSize(): number {
|
|
@@ -26,14 +79,20 @@ export class ExpiringLruCache<K, V> {
|
|
|
26
79
|
|
|
27
80
|
get(key: K): V | undefined {
|
|
28
81
|
const entry = this.#entries.get(key);
|
|
29
|
-
if (
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
82
|
+
if (entry) {
|
|
83
|
+
if (entry.expiresAt <= this.now()) {
|
|
84
|
+
this.#delete(key);
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
this.#entries.delete(key);
|
|
88
|
+
this.#entries.set(key, entry);
|
|
89
|
+
return entry.value;
|
|
33
90
|
}
|
|
34
|
-
this
|
|
35
|
-
|
|
36
|
-
|
|
91
|
+
if (this.persistence) {
|
|
92
|
+
const loaded = this.#loadFromDisk(key);
|
|
93
|
+
if (loaded !== undefined) return loaded.value;
|
|
94
|
+
}
|
|
95
|
+
return undefined;
|
|
37
96
|
}
|
|
38
97
|
|
|
39
98
|
set(key: K, value: V, expiresAt: number): boolean {
|
|
@@ -43,18 +102,82 @@ export class ExpiringLruCache<K, V> {
|
|
|
43
102
|
|
|
44
103
|
this.#entries.set(key, { expiresAt, size, value });
|
|
45
104
|
this.#byteSize += size;
|
|
105
|
+
this.#evict();
|
|
106
|
+
const stored = this.#entries.has(key);
|
|
107
|
+
if (stored && this.persistence) this.#writeToDisk(key, value, expiresAt);
|
|
108
|
+
return stored;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
#loadFromDisk(key: K): ExpiringCacheEntry<V> | undefined {
|
|
112
|
+
let bytes: Uint8Array;
|
|
113
|
+
try {
|
|
114
|
+
const path = resolveCachePath(this.persistence!.directory, this.persistence!.keyToPath(key));
|
|
115
|
+
bytes = readFileSync(path);
|
|
116
|
+
} catch {
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
let entry: ExpiringCacheEntry<V>;
|
|
120
|
+
try {
|
|
121
|
+
if (bytes.byteLength < 8) throw new Error("cache file too small");
|
|
122
|
+
const expiresAt = decodeExpiresAt(bytes);
|
|
123
|
+
const value = this.persistence!.deserialize(bytes.subarray(8));
|
|
124
|
+
const size = this.sizeOf(value);
|
|
125
|
+
if (size > this.maxBytes) throw new Error("oversized cache entry");
|
|
126
|
+
entry = { expiresAt, size, value };
|
|
127
|
+
} catch {
|
|
128
|
+
this.#removeFromDisk(key);
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
if (entry.expiresAt <= this.now()) {
|
|
132
|
+
this.#removeFromDisk(key);
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
this.#entries.set(key, entry);
|
|
136
|
+
this.#byteSize += entry.size;
|
|
137
|
+
this.#evict();
|
|
138
|
+
return this.#entries.get(key);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
#writeToDisk(key: K, value: V, expiresAt: number): void {
|
|
142
|
+
try {
|
|
143
|
+
const directory = this.persistence!.directory;
|
|
144
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
145
|
+
chmodSync(directory, 0o700);
|
|
146
|
+
const payload = this.persistence!.serialize(value);
|
|
147
|
+
const path = resolveCachePath(directory, this.persistence!.keyToPath(key));
|
|
148
|
+
const temp = `${path}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
149
|
+
writeFileSync(temp, Buffer.concat([encodeExpiresAt(expiresAt), payload]), { mode: 0o600 });
|
|
150
|
+
chmodSync(temp, 0o600);
|
|
151
|
+
renameSync(temp, path);
|
|
152
|
+
} catch {
|
|
153
|
+
// Best-effort persistence: a failed disk write never fails the caller.
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
#removeFromDisk(key: K): void {
|
|
158
|
+
try {
|
|
159
|
+
unlinkSync(resolveCachePath(this.persistence!.directory, this.persistence!.keyToPath(key)));
|
|
160
|
+
} catch {
|
|
161
|
+
// Ignore missing or undeletable files; a cache miss is the correct outcome.
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
#evict(): void {
|
|
46
166
|
while (this.#entries.size > this.maxEntries || this.#byteSize > this.maxBytes) {
|
|
47
167
|
const oldest = this.#entries.keys().next().value;
|
|
48
168
|
if (oldest === undefined) break;
|
|
49
169
|
this.#delete(oldest);
|
|
50
170
|
}
|
|
51
|
-
return this.#entries.has(key);
|
|
52
171
|
}
|
|
53
172
|
|
|
54
173
|
#delete(key: K): void {
|
|
55
174
|
const entry = this.#entries.get(key);
|
|
56
|
-
if (!entry)
|
|
175
|
+
if (!entry) {
|
|
176
|
+
if (this.persistence) this.#removeFromDisk(key);
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
57
179
|
this.#entries.delete(key);
|
|
58
180
|
this.#byteSize -= entry.size;
|
|
181
|
+
if (this.persistence) this.#removeFromDisk(key);
|
|
59
182
|
}
|
|
60
183
|
}
|
package/src/index.ts
CHANGED
|
@@ -22,7 +22,19 @@ export {
|
|
|
22
22
|
type SearchTruncationDetails,
|
|
23
23
|
} from "./search";
|
|
24
24
|
|
|
25
|
-
const
|
|
25
|
+
export const webSearchParameters = Type.Object({
|
|
26
|
+
query: Type.String({
|
|
27
|
+
minLength: 1,
|
|
28
|
+
maxLength: SEARCH_WEB_MAX_QUERY_CHARACTERS,
|
|
29
|
+
description: `The search query (maximum ${SEARCH_WEB_MAX_QUERY_CHARACTERS} characters; context mode limited to ${SEARCH_CONTEXT_MAX_QUERY_CHARACTERS})`,
|
|
30
|
+
}),
|
|
31
|
+
mode: Type.Optional(
|
|
32
|
+
StringEnum(["web", "context"] as const, {
|
|
33
|
+
description:
|
|
34
|
+
"Search mode: compact Brave web results (default) or Brave LLM context extraction",
|
|
35
|
+
default: "web",
|
|
36
|
+
}),
|
|
37
|
+
),
|
|
26
38
|
count: Type.Optional(
|
|
27
39
|
Type.Integer({
|
|
28
40
|
minimum: SEARCH_MIN_RESULT_COUNT,
|
|
@@ -44,32 +56,6 @@ const commonSearchParameters = Type.Object({
|
|
|
44
56
|
),
|
|
45
57
|
});
|
|
46
58
|
|
|
47
|
-
export const webSearchParameters = Type.Intersect([
|
|
48
|
-
commonSearchParameters,
|
|
49
|
-
Type.Union([
|
|
50
|
-
Type.Object({
|
|
51
|
-
query: Type.String({
|
|
52
|
-
minLength: 1,
|
|
53
|
-
maxLength: SEARCH_CONTEXT_MAX_QUERY_CHARACTERS,
|
|
54
|
-
description: `The Brave LLM Context query (maximum ${SEARCH_CONTEXT_MAX_QUERY_CHARACTERS} characters)`,
|
|
55
|
-
}),
|
|
56
|
-
mode: Type.Literal("context", {
|
|
57
|
-
description: "Return provider-extracted Brave LLM context",
|
|
58
|
-
}),
|
|
59
|
-
}),
|
|
60
|
-
Type.Object({
|
|
61
|
-
query: Type.String({
|
|
62
|
-
minLength: 1,
|
|
63
|
-
maxLength: SEARCH_WEB_MAX_QUERY_CHARACTERS,
|
|
64
|
-
description: `The web search query (maximum ${SEARCH_WEB_MAX_QUERY_CHARACTERS} characters)`,
|
|
65
|
-
}),
|
|
66
|
-
mode: Type.Optional(
|
|
67
|
-
Type.Literal("web", { description: "Return compact Brave web results (default)" }),
|
|
68
|
-
),
|
|
69
|
-
}),
|
|
70
|
-
]),
|
|
71
|
-
]);
|
|
72
|
-
|
|
73
59
|
export default function (pi: ExtensionAPI) {
|
|
74
60
|
const runtime = new SearchRuntime();
|
|
75
61
|
pi.on("session_shutdown", () => runtime.shutdown());
|
package/src/provider.ts
CHANGED
|
@@ -9,8 +9,7 @@ export function configuredProvider(): "brave" {
|
|
|
9
9
|
return "brave";
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
-
export function normalizeText(value:
|
|
13
|
-
if (typeof value !== "string") return "";
|
|
12
|
+
export function normalizeText(value: string, maxLength: number): string {
|
|
14
13
|
const text = value
|
|
15
14
|
.replace(/<[^>]*>/g, " ")
|
|
16
15
|
.replace(/\s+/g, " ")
|
|
@@ -72,11 +71,11 @@ async function readResponseText(
|
|
|
72
71
|
return new TextDecoder().decode(await readResponseBytes(response, maxBytes, truncate));
|
|
73
72
|
}
|
|
74
73
|
|
|
75
|
-
export async function requestJson(
|
|
74
|
+
export async function requestJson<T>(
|
|
76
75
|
url: string | URL,
|
|
77
76
|
init: RequestInit,
|
|
78
77
|
signal: AbortSignal | undefined,
|
|
79
|
-
): Promise<
|
|
78
|
+
): Promise<T> {
|
|
80
79
|
const controller = new AbortController();
|
|
81
80
|
let timedOut = false;
|
|
82
81
|
const timeout = setTimeout(() => {
|
|
@@ -100,7 +99,8 @@ export async function requestJson(
|
|
|
100
99
|
}
|
|
101
100
|
throw new Error(`Search provider returned HTTP ${response.status}${body ? `: ${body}` : ""}`);
|
|
102
101
|
}
|
|
103
|
-
|
|
102
|
+
// SAFETY: the caller supplies the expected response shape T; JSON.parse yields the decoded document.
|
|
103
|
+
return JSON.parse(await readResponseText(response, SEARCH_MAX_RESPONSE_BYTES)) as T;
|
|
104
104
|
} catch (error) {
|
|
105
105
|
if (timedOut)
|
|
106
106
|
throw new Error(`Web search timed out after ${REQUEST_TIMEOUT_MS / 1000} seconds.`);
|
package/src/search.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
2
|
-
import { tmpdir } from "node:os";
|
|
2
|
+
import { homedir, tmpdir } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import {
|
|
5
5
|
DEFAULT_MAX_BYTES,
|
|
@@ -17,18 +17,35 @@ import {
|
|
|
17
17
|
type SearchMode,
|
|
18
18
|
type SearchResult,
|
|
19
19
|
} from "./brave";
|
|
20
|
-
import { ExpiringLruCache } from "./cache";
|
|
20
|
+
import { ExpiringLruCache, stableKeyHash, type CachePersistence } from "./cache";
|
|
21
21
|
import { formatResults } from "./format-results";
|
|
22
22
|
import { InflightCoalescer } from "./inflight";
|
|
23
23
|
import { SEARCH_DEFAULT_RESULT_COUNT } from "./limits";
|
|
24
24
|
import { configuredProvider } from "./provider";
|
|
25
25
|
|
|
26
|
-
const CACHE_TTL_MS =
|
|
26
|
+
const CACHE_TTL_MS = 24 * 60 * 60 * 1_000;
|
|
27
27
|
const CACHE_MAX_ENTRIES = 100;
|
|
28
28
|
const CACHE_MAX_RESULT_BYTES = 20 * 1_024 * 1_024;
|
|
29
29
|
const MAX_INFLIGHT_REQUESTS = 100;
|
|
30
30
|
const encoder = new TextEncoder();
|
|
31
31
|
|
|
32
|
+
/** Honest-evidence summary comparing what was requested with what was returned. */
|
|
33
|
+
export interface SearchEvidence {
|
|
34
|
+
requestedCount: number;
|
|
35
|
+
returnedCount: number;
|
|
36
|
+
dropped: number;
|
|
37
|
+
freshness?: Freshness;
|
|
38
|
+
truncated: boolean;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Resolves the private, cross-session cache directory for a web tool. */
|
|
42
|
+
function resolveCacheDirectory(name: string): string {
|
|
43
|
+
const base = process.env.XDG_CACHE_HOME
|
|
44
|
+
? join(process.env.XDG_CACHE_HOME, name)
|
|
45
|
+
: join(homedir(), ".cache", name);
|
|
46
|
+
return base;
|
|
47
|
+
}
|
|
48
|
+
|
|
32
49
|
export interface SearchParameters {
|
|
33
50
|
query: string;
|
|
34
51
|
count?: number;
|
|
@@ -53,6 +70,7 @@ export interface SearchDetails {
|
|
|
53
70
|
mode: SearchMode;
|
|
54
71
|
resultCount: number;
|
|
55
72
|
results: SearchResult[];
|
|
73
|
+
evidence: SearchEvidence;
|
|
56
74
|
cached: boolean;
|
|
57
75
|
truncated: boolean;
|
|
58
76
|
fullOutputPath?: string;
|
|
@@ -64,10 +82,20 @@ interface SearchUpdate {
|
|
|
64
82
|
details: Record<string, never>;
|
|
65
83
|
}
|
|
66
84
|
|
|
85
|
+
const searchCachePersistence: CachePersistence<string, SearchResult[]> = {
|
|
86
|
+
directory: resolveCacheDirectory("pi-web-search"),
|
|
87
|
+
serialize: (results) => encoder.encode(JSON.stringify(results)),
|
|
88
|
+
// SAFETY: cache entries are written by this same serializer, so the decoded
|
|
89
|
+
// JSON always matches the SearchResult[] shape.
|
|
90
|
+
deserialize: (bytes) => JSON.parse(new TextDecoder().decode(bytes)) as SearchResult[],
|
|
91
|
+
keyToPath: (key) => stableKeyHash(key),
|
|
92
|
+
};
|
|
67
93
|
const searchCache = new ExpiringLruCache<string, SearchResult[]>(
|
|
68
94
|
CACHE_MAX_ENTRIES,
|
|
69
95
|
CACHE_MAX_RESULT_BYTES,
|
|
70
96
|
(results) => encoder.encode(JSON.stringify(results)).byteLength,
|
|
97
|
+
undefined,
|
|
98
|
+
searchCachePersistence,
|
|
71
99
|
);
|
|
72
100
|
const inflightSearches = new InflightCoalescer<string, SearchResult[]>(MAX_INFLIGHT_REQUESTS);
|
|
73
101
|
|
|
@@ -145,6 +173,13 @@ export class SearchRuntime {
|
|
|
145
173
|
maxLines: DEFAULT_MAX_LINES,
|
|
146
174
|
maxBytes: DEFAULT_MAX_BYTES,
|
|
147
175
|
});
|
|
176
|
+
const evidence: SearchEvidence = {
|
|
177
|
+
requestedCount: count,
|
|
178
|
+
returnedCount: results.length,
|
|
179
|
+
dropped: Math.max(0, results.length - count),
|
|
180
|
+
freshness: params.freshness,
|
|
181
|
+
truncated: truncation.truncated,
|
|
182
|
+
};
|
|
148
183
|
let text = truncation.content;
|
|
149
184
|
let fullOutputPath: string | undefined;
|
|
150
185
|
|
|
@@ -172,6 +207,7 @@ export class SearchRuntime {
|
|
|
172
207
|
mode,
|
|
173
208
|
resultCount: results.length,
|
|
174
209
|
results,
|
|
210
|
+
evidence,
|
|
175
211
|
cached,
|
|
176
212
|
truncated: truncation.truncated,
|
|
177
213
|
fullOutputPath,
|