kovence-cli 0.0.0-stage → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,165 @@
1
+ "use strict";
2
+
3
+ const fs = require("node:fs");
4
+ const path = require("node:path");
5
+
6
+ const ALWAYS_SKIP_DIRS = new Set([
7
+ "node_modules",
8
+ ".git",
9
+ ".ai_harness",
10
+ ".ai_sessions",
11
+ ".kovence",
12
+ "dist",
13
+ "build",
14
+ "__pycache__",
15
+ ]);
16
+
17
+ /**
18
+ * Compile minimal .gitignore lines within a directory into predicates.
19
+ */
20
+ function parseGitignoreFile(gitignorePath, relDir) {
21
+ if (!fs.existsSync(gitignorePath)) return [];
22
+ let raw;
23
+ try {
24
+ raw = fs.readFileSync(gitignorePath, "utf8");
25
+ } catch {
26
+ return [];
27
+ }
28
+ const rules = [];
29
+ for (const rawLine of raw.split(/\r?\n/)) {
30
+ const line = rawLine.trim();
31
+ if (!line || line.startsWith("#")) continue;
32
+ const negated = line.startsWith("!");
33
+ let pat = negated ? line.slice(1).trim() : line;
34
+ if (!pat) continue;
35
+ const dirOnly = pat.endsWith("/");
36
+ if (dirOnly) pat = pat.slice(0, -1);
37
+ const anchored = pat.startsWith("/");
38
+ if (anchored) pat = pat.slice(1);
39
+ const hasSlash = pat.includes("/");
40
+
41
+ const escaped = pat
42
+ .replace(/[.+^${}()|[\]\\]/g, "\\$&")
43
+ .replace(/\*\*/g, "__GLOBSTAR__")
44
+ .replace(/\*/g, "[^/]*")
45
+ .replace(/\?/g, "[^/]")
46
+ .replace(/__GLOBSTAR__/g, ".*");
47
+
48
+ const re = anchored || hasSlash ? new RegExp(`^${escaped}(?:/.*)?$`) : new RegExp(`(^|/)${escaped}(?:/.*)?$`);
49
+
50
+ rules.push({ negated, dirOnly, relBase: relDir, re });
51
+ }
52
+ return rules;
53
+ }
54
+
55
+ function isIgnoredByRules(relPathFromRoot, isDir, rules) {
56
+ const norm = relPathFromRoot.replace(/\\/g, "/");
57
+ let ignored = false;
58
+ for (const rule of rules) {
59
+ let localRel = norm;
60
+ if (rule.relBase) {
61
+ if (!norm.startsWith(rule.relBase + "/")) continue;
62
+ localRel = norm.slice(rule.relBase.length + 1);
63
+ }
64
+ if (rule.dirOnly && !isDir) {
65
+ // Also check parent directory path prefix match via (?:/.*)?$
66
+ if (!localRel.includes("/")) continue;
67
+ }
68
+ if (rule.re.test(localRel)) {
69
+ ignored = !rule.negated;
70
+ }
71
+ }
72
+ return ignored;
73
+ }
74
+
75
+ /**
76
+ * Convert a user glob pattern to a relative-path RegExp.
77
+ */
78
+ function globToRegex(glob) {
79
+ const clean = String(glob || "*")
80
+ .trim()
81
+ .replace(/\\/g, "/")
82
+ .replace(/^\.\//, "");
83
+ const esc = clean
84
+ .replace(/[.+^${}()|[\]\\]/g, "\\$&")
85
+ .replace(/\*\*/g, "__GLOBSTAR__")
86
+ .replace(/\*/g, "[^/]*")
87
+ .replace(/\?/g, "[^/]")
88
+ .replace(/__GLOBSTAR__/g, ".*");
89
+ if (clean.includes("/")) {
90
+ return new RegExp(`^(?:.*/)?${esc}$`);
91
+ }
92
+ return new RegExp(`(^|/)${esc}$`);
93
+ }
94
+
95
+ /**
96
+ * Traverse workspace directories respecting .gitignore, handling 5000+ files without stack overflow.
97
+ */
98
+ function walkWorkspace(baseDir, options = {}) {
99
+ const root = path.resolve(baseDir);
100
+ const maxDepth = options.maxDepth ?? 16;
101
+ const maxEntries = options.maxEntries ?? 10000;
102
+ const includeDirs = options.includeDirs ?? true;
103
+ const cwdRoot = path.resolve(options.cwdRoot || root);
104
+
105
+ const results = [];
106
+ let truncated = false;
107
+ const stack = [{ dir: root, depth: 0, rules: [] }];
108
+
109
+ while (stack.length > 0) {
110
+ if (results.length >= maxEntries) {
111
+ truncated = true;
112
+ break;
113
+ }
114
+ const { dir, depth, rules } = stack.pop();
115
+ let entries;
116
+ try {
117
+ entries = fs.readdirSync(dir, { withFileTypes: true });
118
+ } catch {
119
+ continue;
120
+ }
121
+
122
+ const relDir = path.relative(root, dir).replace(/\\/g, "/");
123
+ const localRules = [...rules, ...parseGitignoreFile(path.join(dir, ".gitignore"), relDir)];
124
+
125
+ entries.sort((a, b) => Number(a.isDirectory()) - Number(b.isDirectory()) || b.name.localeCompare(a.name));
126
+ // Push in reverse so stack pops alphabetically with directories/files ordered naturally
127
+ const subdirsToPush = [];
128
+ const sortedAsc = [...entries].reverse();
129
+
130
+ for (const entry of sortedAsc) {
131
+ if (results.length >= maxEntries) {
132
+ truncated = true;
133
+ break;
134
+ }
135
+ const isDir = entry.isDirectory();
136
+ if (isDir && ALWAYS_SKIP_DIRS.has(entry.name)) continue;
137
+
138
+ const full = path.join(dir, entry.name);
139
+ const relFromWalkRoot = path.relative(root, full).replace(/\\/g, "/");
140
+ if (isIgnoredByRules(relFromWalkRoot, isDir, localRules)) continue;
141
+
142
+ const relFromCwd = path.relative(cwdRoot, full).replace(/\\/g, "/") || ".";
143
+ if (isDir) {
144
+ if (includeDirs) results.push(`${relFromCwd}/`);
145
+ if (depth + 1 < maxDepth) {
146
+ subdirsToPush.push({ dir: full, depth: depth + 1, rules: localRules });
147
+ }
148
+ } else {
149
+ results.push(relFromCwd);
150
+ }
151
+ }
152
+
153
+ for (let i = subdirsToPush.length - 1; i >= 0; i--) {
154
+ stack.push(subdirsToPush[i]);
155
+ }
156
+ }
157
+
158
+ return { entries: results, truncated };
159
+ }
160
+
161
+ module.exports = {
162
+ ALWAYS_SKIP_DIRS,
163
+ globToRegex,
164
+ walkWorkspace,
165
+ };
@@ -0,0 +1,467 @@
1
+ "use strict";
2
+
3
+ const dns = require("node:dns").promises;
4
+ const fs = require("node:fs");
5
+ const http = require("node:http");
6
+ const https = require("node:https");
7
+ const net = require("node:net");
8
+ const path = require("node:path");
9
+ const zlib = require("node:zlib");
10
+
11
+ const MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
12
+
13
+ const BROWSER_HEADERS = {
14
+ "User-Agent":
15
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
16
+ Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
17
+ "Accept-Language": "en-US,en;q=0.9",
18
+ "Accept-Encoding": "gzip, deflate, br",
19
+ "Cache-Control": "no-cache",
20
+ Pragma: "no-cache",
21
+ "Sec-CH-UA": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
22
+ "Sec-CH-UA-Mobile": "?0",
23
+ "Sec-CH-UA-Platform": '"Linux"',
24
+ "Sec-Fetch-Dest": "document",
25
+ "Sec-Fetch-Mode": "navigate",
26
+ "Sec-Fetch-Site": "none",
27
+ "Sec-Fetch-User": "?1",
28
+ "Upgrade-Insecure-Requests": "1",
29
+ DNT: "1",
30
+ };
31
+
32
+ /**
33
+ * Determine whether an IPv4 or IPv6 address belongs to loopback, link-local, or private network space.
34
+ */
35
+ function isPrivateOrLocalIp(ip) {
36
+ const clean = String(ip || "")
37
+ .trim()
38
+ .replace(/^\[|\]$/g, "")
39
+ .split("%")[0]
40
+ .toLowerCase();
41
+ if (!clean) return true;
42
+ if (clean.startsWith("::ffff:")) return isPrivateOrLocalIp(clean.slice(7));
43
+
44
+ if (net.isIPv4(clean)) {
45
+ const parts = clean.split(".").map(Number);
46
+ const [a, b] = parts;
47
+ if (a === 0 || a === 10 || a === 127) return true;
48
+ if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
49
+ if (a === 169 && b === 254) return true; // Link-local / cloud metadata
50
+ if (a === 172 && b >= 16 && b <= 31) return true;
51
+ if (a === 192 && b === 168) return true;
52
+ if (a >= 224) return true;
53
+ return false;
54
+ }
55
+
56
+ if (net.isIPv6(clean)) {
57
+ if (clean === "::1" || clean === "::" || clean === "0:0:0:0:0:0:0:1") return true;
58
+ if (clean.startsWith("fe80:") || clean.startsWith("fc") || clean.startsWith("fd")) return true;
59
+ return false;
60
+ }
61
+ return true;
62
+ }
63
+
64
+ /**
65
+ * Verify URL uses http/https without embedded credentials and does not resolve to private/loopback/metadata IPs.
66
+ */
67
+ async function assertSafePublicUrl(rawUrl, allowPrivate = false) {
68
+ let parsed;
69
+ try {
70
+ parsed = new URL(String(rawUrl || ""));
71
+ } catch {
72
+ throw new Error(`Invalid URL: ${rawUrl}`);
73
+ }
74
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
75
+ throw new Error(`Blocked unsupported URL protocol '${parsed.protocol}'. Only http/https allowed.`);
76
+ }
77
+ if (parsed.username || parsed.password) {
78
+ throw new Error("URLs with embedded credentials are not allowed.");
79
+ }
80
+ if (allowPrivate) return parsed;
81
+
82
+ const host = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase();
83
+ if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || host.endsWith(".internal")) {
84
+ throw new Error(`SSRF blocked: private/local hostname '${host}'`);
85
+ }
86
+ if (net.isIP(host)) {
87
+ if (isPrivateOrLocalIp(host)) throw new Error(`SSRF blocked: private/loopback IP '${host}'`);
88
+ return parsed;
89
+ }
90
+
91
+ const addrs = await dns.lookup(host, { all: true }).catch(() => []);
92
+ if (!addrs.length) throw new Error(`DNS resolution failed for '${host}'`);
93
+ for (const entry of addrs) {
94
+ if (isPrivateOrLocalIp(entry.address)) {
95
+ throw new Error(`SSRF blocked: '${host}' resolved to private/local IP '${entry.address}'`);
96
+ }
97
+ }
98
+ return parsed;
99
+ }
100
+
101
+ function decodeHtmlEntities(str = "") {
102
+ return str
103
+ .replace(/&amp;/g, "&")
104
+ .replace(/&lt;/g, "<")
105
+ .replace(/&gt;/g, ">")
106
+ .replace(/&quot;/g, '"')
107
+ .replace(/&#39;|&#x27;/g, "'")
108
+ .replace(/&nbsp;/g, " ");
109
+ }
110
+
111
+ /**
112
+ * Lightweight zero-dep HTML to Markdown converter for readable fetched web pages.
113
+ */
114
+ function htmlToMarkdown(html = "") {
115
+ let out = String(html || "")
116
+ .replace(/<script[\s\S]*?<\/script>/gi, "")
117
+ .replace(/<style[\s\S]*?<\/style>/gi, "")
118
+ .replace(/<noscript[\s\S]*?<\/noscript>/gi, "")
119
+ .replace(/<nav[\s\S]*?<\/nav>/gi, "")
120
+ .replace(/<footer[\s\S]*?<\/footer>/gi, "");
121
+
122
+ out = out
123
+ .replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n# $1\n")
124
+ .replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n## $1\n")
125
+ .replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n### $1\n")
126
+ .replace(/<h[4-6][^>]*>([\s\S]*?)<\/h[4-6]>/gi, "\n#### $1\n")
127
+ .replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, "\n```\n$1\n```\n")
128
+ .replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, "\n```\n$1\n```\n")
129
+ .replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, "`$1`")
130
+ .replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, "[$2]($1)")
131
+ .replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, "\n- $1")
132
+ .replace(/<br\s*\/?>/gi, "\n")
133
+ .replace(/<\/(p|div|section|article|tr)>/gi, "\n\n")
134
+ .replace(/<[^>]+>/g, " ");
135
+
136
+ out = decodeHtmlEntities(out)
137
+ .replace(/[ \t]+\n/g, "\n")
138
+ .replace(/\n{3,}/g, "\n\n")
139
+ .trim();
140
+ return out;
141
+ }
142
+
143
+ /**
144
+ * Fetch remote URL safely with SSRF checks, redirect verification, size cap, and AbortSignal.
145
+ */
146
+ async function safeHttpGet(targetUrl, options = {}, redirectsLeft = 5) {
147
+ const {
148
+ extraHeaders = {},
149
+ timeout = 15000,
150
+ signal = null,
151
+ maxBytes = MAX_RESPONSE_BYTES,
152
+ allowPrivate = false,
153
+ } = options;
154
+
155
+ if (signal?.aborted) throw new Error("Request aborted");
156
+ const parsed = await assertSafePublicUrl(targetUrl, allowPrivate);
157
+
158
+ return new Promise((resolve, reject) => {
159
+ const client = parsed.protocol === "https:" ? https : http;
160
+ const req = client.request(
161
+ parsed,
162
+ { method: "GET", headers: { ...BROWSER_HEADERS, ...extraHeaders }, timeout },
163
+ (res) => {
164
+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && redirectsLeft > 0) {
165
+ res.resume();
166
+ const nextUrl = new URL(res.headers.location, targetUrl).toString();
167
+ safeHttpGet(
168
+ nextUrl,
169
+ { ...options, extraHeaders: { ...extraHeaders, Referer: targetUrl } },
170
+ redirectsLeft - 1
171
+ ).then(resolve, reject);
172
+ return;
173
+ }
174
+
175
+ const enc = (res.headers["content-encoding"] || "").toLowerCase();
176
+ let stream = res;
177
+ if (enc.includes("gzip")) stream = res.pipe(zlib.createGunzip());
178
+ else if (enc.includes("deflate")) stream = res.pipe(zlib.createInflate());
179
+ else if (enc.includes("br")) stream = res.pipe(zlib.createBrotliDecompress());
180
+
181
+ const chunks = [];
182
+ let total = 0;
183
+ let truncated = false;
184
+
185
+ stream.on("data", (chunk) => {
186
+ if (total + chunk.length > maxBytes) {
187
+ const remain = Math.max(0, maxBytes - total);
188
+ if (remain > 0) chunks.push(chunk.subarray(0, remain));
189
+ truncated = true;
190
+ stream.destroy();
191
+ res.destroy();
192
+ return;
193
+ }
194
+ total += chunk.length;
195
+ chunks.push(chunk);
196
+ });
197
+ stream.on("end", () => {
198
+ resolve({
199
+ status: res.statusCode || 0,
200
+ url: targetUrl,
201
+ headers: res.headers,
202
+ truncated,
203
+ text: Buffer.concat(chunks).toString("utf8"),
204
+ });
205
+ });
206
+ stream.on("close", () => {
207
+ if (truncated) {
208
+ resolve({
209
+ status: res.statusCode || 0,
210
+ url: targetUrl,
211
+ headers: res.headers,
212
+ truncated: true,
213
+ text: Buffer.concat(chunks).toString("utf8"),
214
+ });
215
+ }
216
+ });
217
+ stream.on("error", (err) => {
218
+ if (!truncated) reject(err);
219
+ });
220
+ }
221
+ );
222
+
223
+ const onAbort = () => req.destroy(new Error("Request aborted"));
224
+ signal?.addEventListener("abort", onAbort, { once: true });
225
+ req.on("timeout", () => req.destroy(new Error("Request timed out")));
226
+ req.on("error", (err) => {
227
+ signal?.removeEventListener("abort", onAbort);
228
+ reject(err);
229
+ });
230
+ req.on("close", () => signal?.removeEventListener("abort", onAbort));
231
+ req.end();
232
+ });
233
+ }
234
+
235
+ async function safeHttpPostJson(targetUrl, bodyObj, options = {}) {
236
+ const { headers = {}, timeout = 15000, signal = null, allowPrivate = false } = options;
237
+ if (signal?.aborted) throw new Error("Request aborted");
238
+ await assertSafePublicUrl(targetUrl, allowPrivate);
239
+ const controller = new AbortController();
240
+ const timer = setTimeout(() => controller.abort(new Error("Request timed out")), timeout);
241
+ const onParentAbort = () => controller.abort(signal.reason || new Error("Aborted"));
242
+ signal?.addEventListener("abort", onParentAbort, { once: true });
243
+
244
+ try {
245
+ const res = await fetch(targetUrl, {
246
+ method: "POST",
247
+ headers: { "Content-Type": "application/json", ...headers },
248
+ body: JSON.stringify(bodyObj),
249
+ signal: controller.signal,
250
+ });
251
+ const text = await res.text();
252
+ return { status: res.status, text };
253
+ } finally {
254
+ clearTimeout(timer);
255
+ signal?.removeEventListener("abort", onParentAbort);
256
+ }
257
+ }
258
+
259
+ /**
260
+ * Format nested AggregateError or Node socket errors cleanly (never blank string).
261
+ */
262
+ function formatNetworkError(err) {
263
+ if (!err) return "Unknown network error";
264
+ if (Array.isArray(err.errors) && err.errors.length > 0) {
265
+ const inner = err.errors
266
+ .map((e) => e?.message || e?.code || String(e))
267
+ .filter(Boolean)
268
+ .join("; ");
269
+ if (inner) return inner;
270
+ }
271
+ return err.message || err.code || err.name || String(err);
272
+ }
273
+
274
+ function formatTransferSize(bytes) {
275
+ if (bytes < 1024) return `${bytes} B`;
276
+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
277
+ if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
278
+ return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
279
+ }
280
+
281
+ /**
282
+ * Format a uniform ASCII/Unicode progress bar with percentage and transfer speed.
283
+ */
284
+ function formatEta(sec) {
285
+ if (sec === null || sec === undefined || !Number.isFinite(sec)) return "";
286
+ const s = Math.max(0, Math.ceil(sec));
287
+ if (s >= 3600) return ` · ETA ${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`;
288
+ if (s >= 60) return ` · ETA ${Math.floor(s / 60)}m${s % 60}s`;
289
+ return ` · ETA ${s}s`;
290
+ }
291
+
292
+ function formatDownloadProgress({
293
+ filename,
294
+ bytes,
295
+ totalBytes = 0,
296
+ bytesPerSec = 0,
297
+ etaSec = null,
298
+ status = "downloading",
299
+ note = "",
300
+ attempt = 1,
301
+ maxAttempts = 1,
302
+ resumedFrom = 0,
303
+ }) {
304
+ const barWidth = 20;
305
+ if (status === "retrying") {
306
+ return `⟳ ${filename} · ${note || "reconnecting"} · ${formatTransferSize(bytes)} saved (will resume)`;
307
+ }
308
+ if (status === "connecting") {
309
+ const resumeNote = bytes > 0 ? ` · resuming from ${formatTransferSize(bytes)}` : "";
310
+ const tryNote = attempt > 1 ? ` · try ${attempt}` : "";
311
+ return `⬇ ${filename} · connecting...${resumeNote}${tryNote}`;
312
+ }
313
+ const speedText = `${formatTransferSize(bytesPerSec)}/s`;
314
+ const extra = `${resumedFrom > 0 ? " · resumed" : ""}${attempt > 1 ? ` · try ${attempt}` : ""}`;
315
+ if (totalBytes > 0) {
316
+ const ratio = Math.max(0, Math.min(1, bytes / totalBytes));
317
+ const filled = Math.round(ratio * barWidth);
318
+ const bar = `${"█".repeat(filled)}${"░".repeat(barWidth - filled)}`;
319
+ const pct = (ratio * 100).toFixed(1).padStart(5);
320
+ return `⬇ ${filename} [${bar}] ${pct}% · ${formatTransferSize(bytes)} / ${formatTransferSize(totalBytes)} · ${speedText}${formatEta(etaSec)}${extra}`;
321
+ }
322
+ const pulse = Math.floor(Date.now() / 180) % barWidth;
323
+ let bar = "";
324
+ for (let i = 0; i < barWidth; i++) {
325
+ bar += Math.abs(i - pulse) <= 1 ? "▓" : "░";
326
+ }
327
+ return `⬇ ${filename} [${bar}] ${formatTransferSize(bytes)} · ${speedText}${extra}`;
328
+ }
329
+
330
+ function unwrapDdgUrl(rawUrl) {
331
+ try {
332
+ const u = new URL(rawUrl.startsWith("//") ? `https:${rawUrl}` : rawUrl, "https://duckduckgo.com");
333
+ if (u.pathname === "/l/" && u.searchParams.has("uddg")) return u.searchParams.get("uddg");
334
+ return u.toString();
335
+ } catch {
336
+ return rawUrl;
337
+ }
338
+ }
339
+
340
+ /**
341
+ * Pluggable web search: Tavily -> Brave -> SearXNG -> Google CSE -> DuckDuckGo HTML fallback.
342
+ */
343
+ async function runPluggableSearch(query, limit = 5, signal = null) {
344
+ if (process.env.TAVILY_API_KEY) {
345
+ const res = await safeHttpPostJson(
346
+ "https://api.tavily.com/search",
347
+ { api_key: process.env.TAVILY_API_KEY, query, max_results: limit },
348
+ { signal }
349
+ );
350
+ if (res.status >= 200 && res.status < 300) {
351
+ const data = JSON.parse(res.text);
352
+ return {
353
+ backend: "tavily",
354
+ results: (data.results || []).slice(0, limit).map((r) => ({
355
+ title: r.title || "",
356
+ url: r.url || "",
357
+ snippet: r.content || "",
358
+ })),
359
+ };
360
+ }
361
+ }
362
+
363
+ if (process.env.BRAVE_API_KEY) {
364
+ const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=${limit}`;
365
+ const res = await safeHttpGet(url, {
366
+ extraHeaders: { Accept: "application/json", "X-Subscription-Token": process.env.BRAVE_API_KEY },
367
+ signal,
368
+ });
369
+ if (res.status >= 200 && res.status < 300) {
370
+ const data = JSON.parse(res.text);
371
+ return {
372
+ backend: "brave",
373
+ results: (data.web?.results || []).slice(0, limit).map((r) => ({
374
+ title: r.title || "",
375
+ url: r.url || "",
376
+ snippet: r.description || "",
377
+ })),
378
+ };
379
+ }
380
+ }
381
+
382
+ if (process.env.SEARXNG_URL) {
383
+ const base = process.env.SEARXNG_URL.replace(/\/+$/, "");
384
+ const url = `${base}/search?q=${encodeURIComponent(query)}&format=json`;
385
+ const res = await safeHttpGet(url, {
386
+ extraHeaders: { Accept: "application/json" },
387
+ signal,
388
+ allowPrivate: true, // SearXNG is often self-hosted locally
389
+ });
390
+ if (res.status >= 200 && res.status < 300) {
391
+ const data = JSON.parse(res.text);
392
+ return {
393
+ backend: "searxng",
394
+ results: (data.results || []).slice(0, limit).map((r) => ({
395
+ title: r.title || "",
396
+ url: r.url || "",
397
+ snippet: r.content || "",
398
+ })),
399
+ };
400
+ }
401
+ }
402
+
403
+ if (process.env.GOOGLE_API_KEY && process.env.GOOGLE_CSE_ID) {
404
+ const url = `https://www.googleapis.com/customsearch/v1?key=${encodeURIComponent(
405
+ process.env.GOOGLE_API_KEY
406
+ )}&cx=${encodeURIComponent(process.env.GOOGLE_CSE_ID)}&num=${limit}&q=${encodeURIComponent(query)}`;
407
+ const res = await safeHttpGet(url, { extraHeaders: { Accept: "application/json" }, signal });
408
+ if (res.status >= 200 && res.status < 300) {
409
+ const data = JSON.parse(res.text);
410
+ return {
411
+ backend: "google-cse",
412
+ results: (data.items || []).map((it) => ({
413
+ title: it.title || "",
414
+ url: it.link || "",
415
+ snippet: it.snippet || "",
416
+ })),
417
+ };
418
+ }
419
+ }
420
+
421
+ // Zero-config DuckDuckGo HTML fallback
422
+ const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
423
+ const res = await safeHttpGet(ddgUrl, {
424
+ extraHeaders: { Referer: "https://duckduckgo.com/" },
425
+ signal,
426
+ });
427
+ if (res.status < 200 || res.status >= 300) throw new Error(`Search HTTP ${res.status}`);
428
+ const results = [];
429
+ const blocks = res.text.split(/class="result__body"|class="links_main/i).slice(1);
430
+ for (const block of blocks) {
431
+ if (results.length >= limit) break;
432
+ const linkMatch = block.match(/class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
433
+ if (!linkMatch) continue;
434
+ const snipMatch = block.match(/class="result__snippet"[^>]*>([\s\S]*?)<\/(?:a|div|span)>/i);
435
+ results.push({
436
+ title: decodeHtmlEntities(
437
+ linkMatch[2]
438
+ .replace(/<[^>]+>/g, " ")
439
+ .replace(/\s+/g, " ")
440
+ .trim()
441
+ ),
442
+ url: unwrapDdgUrl(decodeHtmlEntities(linkMatch[1].replace(/<[^>]+>/g, "").trim())),
443
+ snippet: snipMatch
444
+ ? decodeHtmlEntities(
445
+ snipMatch[1]
446
+ .replace(/<[^>]+>/g, " ")
447
+ .replace(/\s+/g, " ")
448
+ .trim()
449
+ )
450
+ : "",
451
+ });
452
+ }
453
+ return { backend: "duckduckgo", results };
454
+ }
455
+
456
+ module.exports = {
457
+ BROWSER_HEADERS,
458
+ isPrivateOrLocalIp,
459
+ assertSafePublicUrl,
460
+ decodeHtmlEntities,
461
+ htmlToMarkdown,
462
+ safeHttpGet,
463
+ formatNetworkError,
464
+ formatTransferSize,
465
+ formatDownloadProgress,
466
+ runPluggableSearch,
467
+ };