@bacnh85/pi-web 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/index.ts +368 -368
- package/extensions/lib/brave.ts +30 -30
- package/extensions/lib/config.ts +97 -97
- package/extensions/lib/content.ts +67 -67
- package/extensions/lib/crawl4ai.ts +97 -97
- package/extensions/lib/extract.ts +137 -137
- package/extensions/lib/firecrawl.ts +28 -28
- package/extensions/lib/format.ts +138 -138
- package/extensions/lib/retry.ts +32 -32
- package/extensions/lib/search.ts +174 -174
- package/extensions/lib/searxng.ts +49 -49
- package/extensions/types.d.ts +1 -1
- package/package.json +1 -1
package/extensions/lib/search.ts
CHANGED
|
@@ -16,218 +16,218 @@ export type SearchBackend = "searxng" | "brave" | "firecrawl";
|
|
|
16
16
|
export type SearchAttemptStatus = "skipped" | "success" | "empty" | "error";
|
|
17
17
|
|
|
18
18
|
export interface SearchParams {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
19
|
+
query: string;
|
|
20
|
+
count?: number;
|
|
21
|
+
freshness?: string;
|
|
22
|
+
country?: string;
|
|
23
|
+
backend?: "auto" | SearchBackend;
|
|
24
|
+
engines?: string;
|
|
25
|
+
include_content?: boolean;
|
|
26
|
+
content_chars?: number;
|
|
27
|
+
timeout_ms?: number;
|
|
28
|
+
signal?: AbortSignal;
|
|
29
|
+
/** Internal: caller-provided ctx for env lookup. */
|
|
30
|
+
_ctx?: Record<string, unknown>;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
export interface SearchResult {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
title: string;
|
|
35
|
+
url: string;
|
|
36
|
+
snippet: string;
|
|
37
|
+
age: string;
|
|
38
|
+
content: string;
|
|
39
|
+
backend: string;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
export interface SearchAttempt {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
43
|
+
backend: SearchBackend;
|
|
44
|
+
status: SearchAttemptStatus;
|
|
45
|
+
message: string;
|
|
46
|
+
resultCount: number;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
export interface SearchDiagnostics {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
results: SearchResult[];
|
|
51
|
+
attempts: SearchAttempt[];
|
|
52
|
+
selectedBackend: SearchBackend | "";
|
|
53
|
+
backendOrder: SearchBackend[];
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
interface BackendConfig {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
57
|
+
searxng: { configured: boolean; baseUrl: string };
|
|
58
|
+
brave: { configured: boolean; apiKey: string };
|
|
59
|
+
firecrawl: { configured: boolean; config: FirecrawlConfig | null };
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
function probeBackends(ctx?: Record<string, unknown>): BackendConfig {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
63
|
+
const cwd = cwdFromContext(ctx ?? {});
|
|
64
|
+
const trusted = includeProjectEnv(ctx ?? {});
|
|
65
|
+
|
|
66
|
+
let searxngBaseUrl: string;
|
|
67
|
+
try {
|
|
68
|
+
const searxngConfig = loadSearxngConfig({}, cwd, trusted);
|
|
69
|
+
searxngBaseUrl = searxngConfig.baseUrl;
|
|
70
|
+
} catch {
|
|
71
|
+
searxngBaseUrl = normalizeSearxngBaseUrl();
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const braveKey = findEnvValue("BRAVE_API_KEY", cwd, trusted);
|
|
75
|
+
|
|
76
|
+
let fcConfig: FirecrawlConfig | null = null;
|
|
77
|
+
try {
|
|
78
|
+
fcConfig = loadFirecrawlConfig({}, cwd, trusted);
|
|
79
|
+
} catch {
|
|
80
|
+
// Not configured or unsafe to use from this context.
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
searxng: { configured: true, baseUrl: searxngBaseUrl },
|
|
85
|
+
brave: { configured: Boolean(braveKey.value), apiKey: braveKey.value ?? "" },
|
|
86
|
+
firecrawl: { configured: fcConfig !== null, config: fcConfig },
|
|
87
|
+
};
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
async function searchSearxng(params: SearchParams): Promise<SearchResult[]> {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
91
|
+
const cfg = loadSearxngConfig(
|
|
92
|
+
{ ...(params.engines ? { engines: params.engines } : {}) } as Record<string, unknown>,
|
|
93
|
+
cwdFromContext(params._ctx ?? {}),
|
|
94
|
+
includeProjectEnv(params._ctx ?? {}),
|
|
95
|
+
);
|
|
96
|
+
const raw = await fetchSearxngResults(
|
|
97
|
+
{ query: params.query, count: params.count ?? 5, engines: params.engines, timeout_ms: params.timeout_ms } as Record<string, unknown>,
|
|
98
|
+
cfg.baseUrl,
|
|
99
|
+
params.signal,
|
|
100
|
+
);
|
|
101
|
+
return (raw.results || []).map((r) => ({
|
|
102
|
+
title: r.title ?? "",
|
|
103
|
+
url: r.url ?? "",
|
|
104
|
+
snippet: r.snippet ?? "",
|
|
105
|
+
age: r.publishedDate ?? "",
|
|
106
|
+
content: "",
|
|
107
|
+
backend: "searxng",
|
|
108
|
+
}));
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
async function searchBrave(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
112
|
+
if (!backends.brave.configured) throw new Error("Brave not configured (no BRAVE_API_KEY)");
|
|
113
|
+
const raw = await fetchBraveResults(
|
|
114
|
+
params.query,
|
|
115
|
+
params.count ?? 5,
|
|
116
|
+
params.country ?? "US",
|
|
117
|
+
params.freshness ?? "",
|
|
118
|
+
backends.brave.apiKey,
|
|
119
|
+
params.signal,
|
|
120
|
+
);
|
|
121
|
+
const results = raw.map((r) => ({
|
|
122
|
+
title: r.title ?? "",
|
|
123
|
+
url: r.url ?? "",
|
|
124
|
+
snippet: r.snippet ?? "",
|
|
125
|
+
age: r.age ?? "",
|
|
126
|
+
content: "",
|
|
127
|
+
backend: "brave",
|
|
128
|
+
}));
|
|
129
|
+
if (params.include_content && results.length > 0) {
|
|
130
|
+
const maxChars = params.content_chars ?? 5000;
|
|
131
|
+
// Cap concurrent content fetches to avoid OOM from parallel JSDOM parsing
|
|
132
|
+
const MAX_CONCURRENT_FETCHES = 5;
|
|
133
|
+
const batchSize = Math.min(results.length, MAX_CONCURRENT_FETCHES);
|
|
134
|
+
for (let i = 0; i < results.length; i += batchSize) {
|
|
135
|
+
const batch = results.slice(i, i + batchSize);
|
|
136
|
+
await Promise.all(
|
|
137
|
+
batch.map(async (r) => {
|
|
138
|
+
try {
|
|
139
|
+
const article = await fetchReadableContent(r.url, params.timeout_ms ?? 10000, params.signal);
|
|
140
|
+
r.content = article.markdown.slice(0, maxChars);
|
|
141
|
+
} catch (e: any) {
|
|
142
|
+
r.content = `(Fetch error for ${r.url}: ${sanitizeError(e)})`;
|
|
143
|
+
}
|
|
144
|
+
}),
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return results;
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
async function searchFirecrawl(params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
152
|
+
if (!backends.firecrawl.configured || !backends.firecrawl.config) {
|
|
153
|
+
throw new Error("Firecrawl not configured (no FIRECRAWL_API_URL)");
|
|
154
|
+
}
|
|
155
|
+
const body: Record<string, unknown> = {
|
|
156
|
+
query: params.query,
|
|
157
|
+
limit: Math.min(100, Math.max(1, params.count ?? 5)),
|
|
158
|
+
sources: [{ type: "web" }],
|
|
159
|
+
...(params.country ? { country: String(params.country).toUpperCase() } : {}),
|
|
160
|
+
};
|
|
161
|
+
const result = await firecrawlRequest(backends.firecrawl.config, "POST", "/search", body, params.signal) as FirecrawlResult;
|
|
162
|
+
const rootData = result.data as Record<string, unknown> | undefined;
|
|
163
|
+
const web = (rootData?.web as unknown[]) || [];
|
|
164
|
+
const news = (rootData?.news as unknown[]) || [];
|
|
165
|
+
const items = [...web, ...news];
|
|
166
|
+
return items.map((r: any) => ({
|
|
167
|
+
title: sanitizeSnippet(r.title ?? r.metadata?.title ?? ""),
|
|
168
|
+
url: r.url ?? r.metadata?.sourceURL ?? "",
|
|
169
|
+
snippet: sanitizeSnippet(r.description ?? r.snippet ?? ""),
|
|
170
|
+
age: "",
|
|
171
|
+
content: r.markdown ? String(r.markdown).slice(0, params.content_chars ?? 5000) : "",
|
|
172
|
+
backend: "firecrawl",
|
|
173
|
+
}));
|
|
174
174
|
}
|
|
175
175
|
|
|
176
176
|
function isPrecisionQuery(params: SearchParams): boolean {
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
177
|
+
const query = params.query.trim();
|
|
178
|
+
const lower = query.toLowerCase();
|
|
179
|
+
if (params.include_content) return true;
|
|
180
|
+
if (/\bsite:[^\s]+/i.test(query)) return true;
|
|
181
|
+
if (/"[^"]+"/.test(query)) return true;
|
|
182
|
+
if (/\b(docs?|documentation|api|github|repo|repository|changelog|release|issue|bug|sdk|package)\b/i.test(query)) return true;
|
|
183
|
+
const terms = query.split(/\s+/).filter(Boolean);
|
|
184
|
+
if (terms.length > 0 && terms.length <= 4 && /\b[A-Z][A-Za-z0-9_-]{2,}\b/.test(query)) return true;
|
|
185
|
+
if (/\b[a-z0-9-]+\.(com|dev|io|org|net|app|ai|tv)\b/i.test(lower)) return true;
|
|
186
|
+
return false;
|
|
187
187
|
}
|
|
188
188
|
|
|
189
189
|
export function selectSearchBackendOrder(params: SearchParams): SearchBackend[] {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
190
|
+
if (params.backend && params.backend !== "auto") return [params.backend];
|
|
191
|
+
if (params.engines && !params.include_content) return ["searxng", "brave", "firecrawl"];
|
|
192
|
+
if (isPrecisionQuery(params)) return ["brave", "searxng", "firecrawl"];
|
|
193
|
+
return ["searxng", "brave", "firecrawl"];
|
|
194
194
|
}
|
|
195
195
|
|
|
196
196
|
function isConfigured(backend: SearchBackend, backends: BackendConfig): boolean {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
197
|
+
if (backend === "searxng") return backends.searxng.configured;
|
|
198
|
+
if (backend === "brave") return backends.brave.configured;
|
|
199
|
+
return backends.firecrawl.configured;
|
|
200
200
|
}
|
|
201
201
|
|
|
202
202
|
async function runBackend(backend: SearchBackend, params: SearchParams, backends: BackendConfig): Promise<SearchResult[]> {
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
203
|
+
if (backend === "searxng") return searchSearxng(params);
|
|
204
|
+
if (backend === "brave") return searchBrave(params, backends);
|
|
205
|
+
return searchFirecrawl(params, backends);
|
|
206
206
|
}
|
|
207
207
|
|
|
208
208
|
export async function searchWithDiagnostics(params: SearchParams): Promise<SearchDiagnostics> {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
209
|
+
const backends = probeBackends(params._ctx);
|
|
210
|
+
const backendOrder = selectSearchBackendOrder(params);
|
|
211
|
+
const attempts: SearchAttempt[] = [];
|
|
212
|
+
|
|
213
|
+
for (const backend of backendOrder) {
|
|
214
|
+
if (!isConfigured(backend, backends)) {
|
|
215
|
+
attempts.push({ backend, status: "skipped", message: `${backend} is not configured`, resultCount: 0 });
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
try {
|
|
219
|
+
const results = await runBackend(backend, params, backends);
|
|
220
|
+
if (results.length > 0) {
|
|
221
|
+
attempts.push({ backend, status: "success", message: `Selected ${backend}`, resultCount: results.length });
|
|
222
|
+
return { results, attempts, selectedBackend: backend, backendOrder };
|
|
223
|
+
}
|
|
224
|
+
attempts.push({ backend, status: "empty", message: `${backend} returned 0 results`, resultCount: 0 });
|
|
225
|
+
} catch (e) {
|
|
226
|
+
attempts.push({ backend, status: "error", message: sanitizeError(e), resultCount: 0 });
|
|
227
|
+
if (params.backend && params.backend !== "auto") break;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return { results: [], attempts, selectedBackend: "" as const, backendOrder };
|
|
232
232
|
}
|
|
233
233
|
|
|
@@ -4,60 +4,60 @@ import { sanitizeSnippet } from "./format";
|
|
|
4
4
|
import { signalWithTimeout, withRetry } from "./retry";
|
|
5
5
|
|
|
6
6
|
export interface SearxngResultItem {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
7
|
+
title: string;
|
|
8
|
+
url: string;
|
|
9
|
+
snippet: string;
|
|
10
|
+
engine: string;
|
|
11
|
+
category: string;
|
|
12
|
+
score?: number;
|
|
13
|
+
publishedDate: string;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
export interface SearxngResponse {
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
results: SearxngResultItem[];
|
|
18
|
+
[key: string]: unknown;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
export async function fetchSearxngResults(
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
params: Record<string, unknown>,
|
|
23
|
+
baseUrl: string,
|
|
24
|
+
signal?: AbortSignal,
|
|
25
25
|
): Promise<SearxngResponse> {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
26
|
+
return withRetry(async () => {
|
|
27
|
+
const limit = Math.min(50, Math.max(1, (params.count as number) ?? 5));
|
|
28
|
+
const url = new URL(`${baseUrl}/search`);
|
|
29
|
+
url.searchParams.set("q", params.query as string);
|
|
30
|
+
url.searchParams.set("format", "json");
|
|
31
|
+
url.searchParams.set("pageno", String(Math.max(1, (params.pageno as number) ?? 1)));
|
|
32
|
+
for (const key of ["categories", "engines", "language", "time_range", "safesearch"] as const) {
|
|
33
|
+
const val = params[key];
|
|
34
|
+
if (val !== undefined && val !== "") url.searchParams.set(key, String(val));
|
|
35
|
+
}
|
|
36
|
+
const timeoutMs = (params.timeout_ms as number) || 15000;
|
|
37
|
+
const response = await fetch(url, {
|
|
38
|
+
headers: { Accept: "application/json", "User-Agent": "pi-web/0.1 SearXNG" },
|
|
39
|
+
signal: signalWithTimeout(timeoutMs, signal),
|
|
40
|
+
});
|
|
41
|
+
const text = await response.text();
|
|
42
|
+
if (!response.ok) {
|
|
43
|
+
const hint =
|
|
44
|
+
response.status === 403
|
|
45
|
+
? "\nHint: enable JSON output in SearXNG settings.yml with search.formats including json."
|
|
46
|
+
: "";
|
|
47
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}${hint}${text ? `\n${text}` : ""}`);
|
|
48
|
+
}
|
|
49
|
+
const data: any = text ? JSON.parse(text) : {};
|
|
50
|
+
const results = (data.results || [])
|
|
51
|
+
.slice(0, limit)
|
|
52
|
+
.map((r: any) => ({
|
|
53
|
+
title: sanitizeSnippet(r.title || ""),
|
|
54
|
+
url: r.url || "",
|
|
55
|
+
snippet: sanitizeSnippet(r.content || r.snippet || ""),
|
|
56
|
+
engine: r.engine || "",
|
|
57
|
+
category: r.category || "",
|
|
58
|
+
score: r.score,
|
|
59
|
+
publishedDate: r.publishedDate || r.published_date || "",
|
|
60
|
+
}));
|
|
61
|
+
return { ...data, results };
|
|
62
|
+
}, undefined, signal);
|
|
63
63
|
}
|
package/extensions/types.d.ts
CHANGED
package/package.json
CHANGED