@duckmind/dm-windows-x64 0.63.0 → 0.63.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm.exe +0 -0
- package/extensions/.dm-extensions.json +200 -4
- package/extensions/dm-context/package.json +1 -1
- package/extensions/dm-context/skills/context-management/SKILL.md +173 -223
- package/extensions/dm-context/skills/context-management/references/development-and-troubleshooting.md +75 -0
- package/extensions/dm-context/skills/context-management/references/interleaved-async-work.md +143 -0
- package/extensions/dm-context/skills/context-management/references/planning-and-execution.md +81 -0
- package/extensions/dm-context/skills/context-management/references/repeated-items-and-batch-work.md +69 -0
- package/extensions/dm-context/skills/context-management/references/retry-branch-and-pivot.md +80 -0
- package/extensions/dm-context/skills/context-management/references/search-research-and-reading.md +103 -0
- package/extensions/dm-context/skills/context-management/references/task-switching-and-cleanup.md +73 -0
- package/extensions/dm-context/src/context.js +3 -2
- package/extensions/dm-context/src/index.js +168 -84
- package/extensions/dm-skills-manager/THIRD_PARTY_NOTICES.md +27 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/components.js +265 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/constants.js +31 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/creation-fallback.js +20 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/creation.js +145 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/dialog.js +738 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/dm-ai-compat.js +15 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/format.js +125 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/glyphs.js +142 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/layout.js +93 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/paths.js +76 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/registry.js +95 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/settings.js +93 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/startup.js +32 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/toggle.js +54 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/types.js +14 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/ui.js +121 -0
- package/extensions/dm-skills-manager/extensions/skills-manager.js +111 -0
- package/extensions/dm-skills-manager/package.json +121 -0
- package/extensions/dm-web-access/SECURITY.md +5 -0
- package/extensions/dm-web-access/activity.js +65 -0
- package/extensions/dm-web-access/anysearch.js +158 -0
- package/extensions/dm-web-access/auth-fetch.js +131 -0
- package/extensions/dm-web-access/bocha.js +214 -0
- package/extensions/dm-web-access/brave.js +196 -0
- package/extensions/dm-web-access/brightdata-unlocker.js +202 -0
- package/extensions/dm-web-access/brightdata.js +334 -0
- package/extensions/dm-web-access/chrome-cookies.js +627 -0
- package/extensions/dm-web-access/content-find.js +114 -0
- package/extensions/dm-web-access/credential-source.js +150 -0
- package/extensions/dm-web-access/curator-page.js +3559 -0
- package/extensions/dm-web-access/curator-server.js +691 -0
- package/extensions/dm-web-access/data-uri-sanitize.js +312 -0
- package/extensions/dm-web-access/datalab-pdf-extract.js +346 -0
- package/extensions/dm-web-access/declared-web-links.js +167 -0
- package/extensions/dm-web-access/dm-web-fetch-demo.mp4 +0 -0
- package/extensions/dm-web-access/duckduckgo.js +118 -0
- package/extensions/dm-web-access/exa.js +401 -0
- package/extensions/dm-web-access/extract.js +1220 -0
- package/extensions/dm-web-access/feature-config.js +24 -0
- package/extensions/dm-web-access/fetch-params.js +81 -0
- package/extensions/dm-web-access/firecrawl.js +378 -0
- package/extensions/dm-web-access/gemini-adc.js +241 -0
- package/extensions/dm-web-access/gemini-api.js +258 -0
- package/extensions/dm-web-access/gemini-pdf-extract.js +74 -0
- package/extensions/dm-web-access/gemini-search.js +889 -0
- package/extensions/dm-web-access/gemini-url-context.js +97 -0
- package/extensions/dm-web-access/gemini-web-config.js +84 -0
- package/extensions/dm-web-access/gemini-web.js +351 -0
- package/extensions/dm-web-access/github-api.js +166 -0
- package/extensions/dm-web-access/github-extract.js +991 -0
- package/extensions/dm-web-access/github-issue-pr.js +750 -0
- package/extensions/dm-web-access/index.js +3117 -0
- package/extensions/dm-web-access/jina-search.js +242 -0
- package/extensions/dm-web-access/kagi.js +255 -0
- package/extensions/dm-web-access/kimi-search.js +214 -0
- package/extensions/dm-web-access/ollama.js +209 -0
- package/extensions/dm-web-access/openai-search.js +510 -0
- package/extensions/dm-web-access/package.json +34 -0
- package/extensions/dm-web-access/page-query.js +124 -0
- package/extensions/dm-web-access/parallel-mcp.js +223 -0
- package/extensions/dm-web-access/parallel.js +345 -0
- package/extensions/dm-web-access/pdf-extract.js +257 -0
- package/extensions/dm-web-access/perplexity.js +151 -0
- package/extensions/dm-web-access/querit.js +327 -0
- package/extensions/dm-web-access/query-rewrite.js +40 -0
- package/extensions/dm-web-access/render-search-error.js +80 -0
- package/extensions/dm-web-access/rsc-extract.js +347 -0
- package/extensions/dm-web-access/search1api.js +245 -0
- package/extensions/dm-web-access/searchinfinity.js +221 -0
- package/extensions/dm-web-access/searxng.js +223 -0
- package/extensions/dm-web-access/serpbase.js +205 -0
- package/extensions/dm-web-access/serpdive.js +238 -0
- package/extensions/dm-web-access/serper.js +200 -0
- package/extensions/dm-web-access/source-check.js +198 -0
- package/extensions/dm-web-access/ssrf-protection.js +436 -0
- package/extensions/dm-web-access/storage.js +451 -0
- package/extensions/dm-web-access/summary-model-scope.js +83 -0
- package/extensions/dm-web-access/summary-review.js +364 -0
- package/extensions/dm-web-access/tavily.js +199 -0
- package/extensions/dm-web-access/tinyfish.js +325 -0
- package/extensions/dm-web-access/utils.js +476 -0
- package/extensions/dm-web-access/valyu.js +189 -0
- package/extensions/dm-web-access/video-extract.js +336 -0
- package/extensions/dm-web-access/xai-search.js +285 -0
- package/extensions/dm-web-access/xcrawl.js +221 -0
- package/extensions/dm-web-access/youtube-extract.js +279 -0
- package/package.json +9 -1
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
4
|
+
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
5
|
+
import { homedir, hostname, tmpdir } from "node:os";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
let cachedWebSearchConfigDir;
|
|
8
|
+
export function getWebSearchConfigDir() {
|
|
9
|
+
if (cachedWebSearchConfigDir)
|
|
10
|
+
return cachedWebSearchConfigDir;
|
|
11
|
+
const explicitDir = process.env.DM_CODING_AGENT_DIR;
|
|
12
|
+
if (explicitDir)
|
|
13
|
+
return cachedWebSearchConfigDir = explicitDir;
|
|
14
|
+
const xdgConfigHome = process.env.XDG_CONFIG_HOME;
|
|
15
|
+
if (xdgConfigHome) {
|
|
16
|
+
const xdgDir = join(xdgConfigHome, "pi");
|
|
17
|
+
if (existsSync(join(xdgDir, "web-search.json")))
|
|
18
|
+
return cachedWebSearchConfigDir = xdgDir;
|
|
19
|
+
const legacyDir = join(homedir(), ".dm");
|
|
20
|
+
if (existsSync(join(legacyDir, "web-search.json")))
|
|
21
|
+
return cachedWebSearchConfigDir = legacyDir;
|
|
22
|
+
return cachedWebSearchConfigDir = xdgDir;
|
|
23
|
+
}
|
|
24
|
+
return cachedWebSearchConfigDir = join(homedir(), ".dm");
|
|
25
|
+
}
|
|
26
|
+
export function getWebSearchConfigPath() {
|
|
27
|
+
return join(getWebSearchConfigDir(), "web-search.json");
|
|
28
|
+
}
|
|
29
|
+
export function resolveApiBaseUrl(options) {
|
|
30
|
+
const fromEnvironment = options.environmentValue !== undefined;
|
|
31
|
+
const value = fromEnvironment ? options.environmentValue : options.configuredValue;
|
|
32
|
+
if (value === undefined)
|
|
33
|
+
return options.defaultValue;
|
|
34
|
+
const source = fromEnvironment ? options.environmentKey : `${options.configKey} in ${getWebSearchConfigPath()}`;
|
|
35
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
36
|
+
throw new Error(`${source} must be an absolute HTTP(S) URL`);
|
|
37
|
+
}
|
|
38
|
+
let url;
|
|
39
|
+
try {
|
|
40
|
+
url = new URL(value.trim());
|
|
41
|
+
} catch {
|
|
42
|
+
throw new Error(`${source} must be an absolute HTTP(S) URL`);
|
|
43
|
+
}
|
|
44
|
+
if (url.protocol !== "https:") {
|
|
45
|
+
throw new Error(`${source} must be an absolute HTTPS URL`);
|
|
46
|
+
}
|
|
47
|
+
if (url.username || url.password) {
|
|
48
|
+
throw new Error(`${source} must not include credentials`);
|
|
49
|
+
}
|
|
50
|
+
if (url.search || url.hash) {
|
|
51
|
+
throw new Error(`${source} must not include query parameters or fragments`);
|
|
52
|
+
}
|
|
53
|
+
url.search = "";
|
|
54
|
+
url.hash = "";
|
|
55
|
+
url.pathname = url.pathname.replace(/\/+$/, "");
|
|
56
|
+
return url.toString().replace(/\/+$/, "");
|
|
57
|
+
}
|
|
58
|
+
const API_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
59
|
+
const API_REQUEST_BODY_HEADERS = ["Content-Encoding", "Content-Language", "Content-Location", "Content-Type"];
|
|
60
|
+
const MAX_API_REDIRECTS = 5;
|
|
61
|
+
export async function fetchWithCredentialRedirects(url, init, credentialHeaders) {
|
|
62
|
+
let current = new URL(url);
|
|
63
|
+
let requestInit = init;
|
|
64
|
+
for (let redirects = 0;; redirects++) {
|
|
65
|
+
const response = await fetch(current, { ...requestInit, redirect: "manual" });
|
|
66
|
+
if (!API_REDIRECT_STATUSES.has(response.status))
|
|
67
|
+
return response;
|
|
68
|
+
const location = response.headers.get("location");
|
|
69
|
+
if (!location)
|
|
70
|
+
return response;
|
|
71
|
+
if (redirects === MAX_API_REDIRECTS) {
|
|
72
|
+
throw new Error(`Too many API redirects from ${url}`);
|
|
73
|
+
}
|
|
74
|
+
const next = new URL(location, current);
|
|
75
|
+
if (next.protocol !== "http:" && next.protocol !== "https:") {
|
|
76
|
+
throw new Error(`API redirect from ${current.origin} must use HTTP(S)`);
|
|
77
|
+
}
|
|
78
|
+
const method = requestInit.method?.toUpperCase() ?? "GET";
|
|
79
|
+
if ((response.status === 301 || response.status === 302) && method === "POST" || response.status === 303 && method !== "GET" && method !== "HEAD") {
|
|
80
|
+
const headers = new Headers(requestInit.headers);
|
|
81
|
+
for (const name of API_REQUEST_BODY_HEADERS)
|
|
82
|
+
headers.delete(name);
|
|
83
|
+
const { body: _body, ...withoutBody } = requestInit;
|
|
84
|
+
requestInit = { ...withoutBody, method: "GET", headers };
|
|
85
|
+
}
|
|
86
|
+
if (next.origin !== current.origin) {
|
|
87
|
+
const headers = new Headers(requestInit.headers);
|
|
88
|
+
for (const name of credentialHeaders)
|
|
89
|
+
headers.delete(name);
|
|
90
|
+
requestInit = { ...requestInit, headers };
|
|
91
|
+
}
|
|
92
|
+
current = next;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const LOCAL_CURATOR_NETWORK_DEFAULTS = { enabled: false, host: "localhost", bind: "127.0.0.1" };
|
|
96
|
+
function trimmedString(value) {
|
|
97
|
+
if (typeof value !== "string")
|
|
98
|
+
return;
|
|
99
|
+
const trimmed = value.trim();
|
|
100
|
+
return trimmed.length > 0 ? trimmed : undefined;
|
|
101
|
+
}
|
|
102
|
+
export function resolveCuratorNetworkConfig() {
|
|
103
|
+
const configPath = getWebSearchConfigPath();
|
|
104
|
+
if (!existsSync(configPath))
|
|
105
|
+
return LOCAL_CURATOR_NETWORK_DEFAULTS;
|
|
106
|
+
let raw;
|
|
107
|
+
try {
|
|
108
|
+
raw = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
109
|
+
} catch {
|
|
110
|
+
return LOCAL_CURATOR_NETWORK_DEFAULTS;
|
|
111
|
+
}
|
|
112
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
113
|
+
return LOCAL_CURATOR_NETWORK_DEFAULTS;
|
|
114
|
+
const curatorRemote = raw.curatorRemote;
|
|
115
|
+
if (curatorRemote === true)
|
|
116
|
+
return { enabled: true, host: hostname(), bind: "0.0.0.0" };
|
|
117
|
+
if (curatorRemote && typeof curatorRemote === "object" && !Array.isArray(curatorRemote)) {
|
|
118
|
+
const obj = curatorRemote;
|
|
119
|
+
return {
|
|
120
|
+
enabled: true,
|
|
121
|
+
host: trimmedString(obj.host) ?? hostname(),
|
|
122
|
+
bind: trimmedString(obj.bind) ?? "0.0.0.0"
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
return LOCAL_CURATOR_NETWORK_DEFAULTS;
|
|
126
|
+
}
|
|
127
|
+
export function formatSeconds(s) {
|
|
128
|
+
const h = Math.floor(s / 3600);
|
|
129
|
+
const m = Math.floor(s % 3600 / 60);
|
|
130
|
+
const sec = s % 60;
|
|
131
|
+
if (h > 0)
|
|
132
|
+
return `${h}:${String(m).padStart(2, "0")}:${String(sec).padStart(2, "0")}`;
|
|
133
|
+
return `${m}:${String(sec).padStart(2, "0")}`;
|
|
134
|
+
}
|
|
135
|
+
export function readExecError(err) {
|
|
136
|
+
if (!err || typeof err !== "object") {
|
|
137
|
+
return { stderr: "", message: String(err) };
|
|
138
|
+
}
|
|
139
|
+
const code = err.code;
|
|
140
|
+
const message = err.message ?? "";
|
|
141
|
+
const stderrRaw = err.stderr;
|
|
142
|
+
const stderr = Buffer.isBuffer(stderrRaw) ? stderrRaw.toString("utf-8") : typeof stderrRaw === "string" ? stderrRaw : "";
|
|
143
|
+
return { code, stderr, message };
|
|
144
|
+
}
|
|
145
|
+
export function isTimeoutError(err) {
|
|
146
|
+
if (!err || typeof err !== "object")
|
|
147
|
+
return false;
|
|
148
|
+
if (err.killed)
|
|
149
|
+
return true;
|
|
150
|
+
const name = err.name;
|
|
151
|
+
const code = err.code;
|
|
152
|
+
const message = err.message ?? "";
|
|
153
|
+
return name === "AbortError" || code === "ETIMEDOUT" || message.toLowerCase().includes("timed out");
|
|
154
|
+
}
|
|
155
|
+
export function trimErrorText(text) {
|
|
156
|
+
return text.replace(/\s+/g, " ").trim().slice(0, 200);
|
|
157
|
+
}
|
|
158
|
+
export function mapFfmpegError(err) {
|
|
159
|
+
const { code, stderr, message } = readExecError(err);
|
|
160
|
+
if (code === "ENOENT")
|
|
161
|
+
return "ffmpeg is not installed. Install with: brew install ffmpeg";
|
|
162
|
+
if (isTimeoutError(err))
|
|
163
|
+
return "ffmpeg timed out extracting frame";
|
|
164
|
+
if (stderr.includes("403"))
|
|
165
|
+
return "Stream URL returned 403 — may have expired, try again";
|
|
166
|
+
const snippet = trimErrorText(stderr || message);
|
|
167
|
+
return snippet ? `ffmpeg failed: ${snippet}` : "ffmpeg failed";
|
|
168
|
+
}
|
|
169
|
+
const proxyStorage = new AsyncLocalStorage;
|
|
170
|
+
export function normalizeProxyUrl(value, source) {
|
|
171
|
+
if (value === undefined || value === null)
|
|
172
|
+
return null;
|
|
173
|
+
if (typeof value !== "string")
|
|
174
|
+
throw new Error(`${source} must be an http(s) proxy URL string`);
|
|
175
|
+
const trimmed = value.trim();
|
|
176
|
+
if (!trimmed)
|
|
177
|
+
return null;
|
|
178
|
+
let parsed;
|
|
179
|
+
try {
|
|
180
|
+
parsed = new URL(trimmed);
|
|
181
|
+
} catch {
|
|
182
|
+
throw new Error(`${source} must be a valid proxy URL: ${JSON.stringify(trimmed)}`);
|
|
183
|
+
}
|
|
184
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
185
|
+
throw new Error(`${source} must use the http:// or https:// scheme: ${trimmed}`);
|
|
186
|
+
}
|
|
187
|
+
if (!parsed.hostname)
|
|
188
|
+
throw new Error(`${source} must include a proxy host: ${trimmed}`);
|
|
189
|
+
parsed.hash = "";
|
|
190
|
+
parsed.search = "";
|
|
191
|
+
return parsed.toString();
|
|
192
|
+
}
|
|
193
|
+
function redactProxyUrl(value) {
|
|
194
|
+
const parsed = new URL(value);
|
|
195
|
+
if (parsed.username)
|
|
196
|
+
parsed.username = "redacted";
|
|
197
|
+
if (parsed.password)
|
|
198
|
+
parsed.password = "redacted";
|
|
199
|
+
return parsed.toString();
|
|
200
|
+
}
|
|
201
|
+
function loadConfiguredProxy() {
|
|
202
|
+
let configured;
|
|
203
|
+
const path = getWebSearchConfigPath();
|
|
204
|
+
if (existsSync(path)) {
|
|
205
|
+
let raw;
|
|
206
|
+
try {
|
|
207
|
+
raw = JSON.parse(readFileSync(path, "utf-8"));
|
|
208
|
+
} catch (err) {
|
|
209
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
210
|
+
throw new Error(`Failed to load proxy config from ${path}: ${message}`);
|
|
211
|
+
}
|
|
212
|
+
if (raw && typeof raw === "object" && !Array.isArray(raw))
|
|
213
|
+
configured = raw.proxy;
|
|
214
|
+
}
|
|
215
|
+
if (configured === undefined)
|
|
216
|
+
return null;
|
|
217
|
+
return normalizeProxyUrl(configured, `proxy in ${getWebSearchConfigPath()}`);
|
|
218
|
+
}
|
|
219
|
+
export function runWithProxy(proxy, fn) {
|
|
220
|
+
if (proxy === undefined)
|
|
221
|
+
return fn();
|
|
222
|
+
const normalized = normalizeProxyUrl(proxy, "proxy");
|
|
223
|
+
return proxyStorage.run(normalized, fn);
|
|
224
|
+
}
|
|
225
|
+
export function getActiveProxy() {
|
|
226
|
+
const scoped = proxyStorage.getStore();
|
|
227
|
+
return scoped !== undefined ? scoped : loadConfiguredProxy();
|
|
228
|
+
}
|
|
229
|
+
export function hasScopedProxyDecision() {
|
|
230
|
+
return proxyStorage.getStore() !== undefined;
|
|
231
|
+
}
|
|
232
|
+
function noProxyEntryMatches(hostname, entry) {
|
|
233
|
+
if (!entry)
|
|
234
|
+
return false;
|
|
235
|
+
if (entry === "*")
|
|
236
|
+
return true;
|
|
237
|
+
let host = entry;
|
|
238
|
+
if (host.startsWith("[")) {
|
|
239
|
+
const close = host.indexOf("]");
|
|
240
|
+
if (close > 0)
|
|
241
|
+
host = host.slice(0, close + 1);
|
|
242
|
+
} else {
|
|
243
|
+
const colon = host.lastIndexOf(":");
|
|
244
|
+
if (colon > -1 && /^\d+$/.test(host.slice(colon + 1)))
|
|
245
|
+
host = host.slice(0, colon);
|
|
246
|
+
}
|
|
247
|
+
host = host.toLowerCase().replace(/^\[|\]$/g, "");
|
|
248
|
+
if (!host)
|
|
249
|
+
return false;
|
|
250
|
+
return hostname === host || hostname.endsWith(host.startsWith(".") ? host : `.${host}`);
|
|
251
|
+
}
|
|
252
|
+
export function isProxyBypassedUrl(url) {
|
|
253
|
+
const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
254
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "127.0.0.1" || hostname === "::1")
|
|
255
|
+
return true;
|
|
256
|
+
const noProxy = process.env.NO_PROXY || process.env.no_proxy;
|
|
257
|
+
if (noProxy && noProxy.split(",").some((entry) => noProxyEntryMatches(hostname, entry.trim())))
|
|
258
|
+
return true;
|
|
259
|
+
return false;
|
|
260
|
+
}
|
|
261
|
+
export function installGlobalProxyFetch() {
|
|
262
|
+
const current = globalThis.fetch;
|
|
263
|
+
if (typeof current !== "function" || current.__piWebAccessProxyFetch === true)
|
|
264
|
+
return;
|
|
265
|
+
const nativeFetch = current;
|
|
266
|
+
const wrapped = (input, init) => {
|
|
267
|
+
const proxy = init?.__proxy ?? getActiveProxy();
|
|
268
|
+
if (!proxy)
|
|
269
|
+
return nativeFetch(input, init);
|
|
270
|
+
let url = null;
|
|
271
|
+
try {
|
|
272
|
+
url = new URL(typeof input === "string" || input instanceof URL ? input : input.url);
|
|
273
|
+
} catch {
|
|
274
|
+
url = null;
|
|
275
|
+
}
|
|
276
|
+
if (!url || url.protocol !== "http:" && url.protocol !== "https:" || isProxyBypassedUrl(url)) {
|
|
277
|
+
return nativeFetch(input, init);
|
|
278
|
+
}
|
|
279
|
+
return fetchViaCurl(url, init ?? {}, proxy);
|
|
280
|
+
};
|
|
281
|
+
wrapped.__piWebAccessProxyFetch = true;
|
|
282
|
+
globalThis.fetch = wrapped;
|
|
283
|
+
}
|
|
284
|
+
function parseHeaderDump(dump) {
|
|
285
|
+
const blocks = dump.split(/\r?\n\r?\n/).filter((block) => /^HTTP\/[\d.]+\s+\d{3}/.test(block.trim()));
|
|
286
|
+
const block = (blocks.length > 0 ? blocks[blocks.length - 1] : "").trim();
|
|
287
|
+
const lines = block.split(/\r?\n/);
|
|
288
|
+
let status = 0;
|
|
289
|
+
let statusText = "";
|
|
290
|
+
const headers = [];
|
|
291
|
+
for (const line of lines) {
|
|
292
|
+
const match = /^HTTP\/[\d.]+\s+(\d{3})(?:\s+(.*))?$/.exec(line.trim());
|
|
293
|
+
if (match) {
|
|
294
|
+
status = Number(match[1]);
|
|
295
|
+
statusText = match[2] ?? "";
|
|
296
|
+
headers.length = 0;
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
const separator = line.indexOf(":");
|
|
300
|
+
if (separator > 0) {
|
|
301
|
+
const name = line.slice(0, separator).trim();
|
|
302
|
+
if (name.toLowerCase() === "content-encoding" || name.toLowerCase() === "content-length")
|
|
303
|
+
continue;
|
|
304
|
+
headers.push([name, line.slice(separator + 1).trim()]);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return { status, statusText, headers };
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
class CurlTransportError extends Error {
|
|
311
|
+
}
|
|
312
|
+
async function fetchViaCurl(url, init, proxyUrl) {
|
|
313
|
+
let current = url;
|
|
314
|
+
let currentInit = init;
|
|
315
|
+
for (let redirects = 0;; redirects++) {
|
|
316
|
+
const response = await fetchViaCurlOnce(current, currentInit, proxyUrl);
|
|
317
|
+
const location = response.headers.get("location");
|
|
318
|
+
if (!location || ![301, 302, 303, 307, 308].includes(response.status)) {
|
|
319
|
+
if (redirects > 0)
|
|
320
|
+
Object.defineProperty(response, "redirected", { value: true, configurable: true });
|
|
321
|
+
return response;
|
|
322
|
+
}
|
|
323
|
+
if (currentInit.redirect === "manual")
|
|
324
|
+
return response;
|
|
325
|
+
if (currentInit.redirect === "error")
|
|
326
|
+
throw new TypeError(`Proxy fetch redirect blocked from ${current.toString()}`);
|
|
327
|
+
if (redirects === 20)
|
|
328
|
+
throw new Error(`Too many proxy redirects from ${url.toString()}`);
|
|
329
|
+
const next = new URL(location, current);
|
|
330
|
+
if (next.protocol !== "http:" && next.protocol !== "https:")
|
|
331
|
+
throw new Error(`Proxy redirect from ${current.origin} must use HTTP(S)`);
|
|
332
|
+
let headers = new Headers(currentInit.headers);
|
|
333
|
+
let nextInit;
|
|
334
|
+
const method = currentInit.method?.toUpperCase() ?? "GET";
|
|
335
|
+
if ((response.status === 301 || response.status === 302) && method === "POST" || response.status === 303 && method !== "GET" && method !== "HEAD") {
|
|
336
|
+
for (const name of ["Content-Encoding", "Content-Language", "Content-Location", "Content-Type"])
|
|
337
|
+
headers.delete(name);
|
|
338
|
+
const { body: _body, ...withoutBody } = currentInit;
|
|
339
|
+
nextInit = { ...withoutBody, method: "GET", headers };
|
|
340
|
+
} else {
|
|
341
|
+
nextInit = { ...currentInit, headers };
|
|
342
|
+
}
|
|
343
|
+
if (next.origin !== current.origin) {
|
|
344
|
+
headers = new Headers;
|
|
345
|
+
nextInit = { ...nextInit, headers };
|
|
346
|
+
}
|
|
347
|
+
current = next;
|
|
348
|
+
currentInit = nextInit;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
async function fetchViaCurlOnce(url, init, proxyUrl) {
|
|
352
|
+
const method = (init.method ?? "GET").toUpperCase();
|
|
353
|
+
const headers = new Headers(init.headers);
|
|
354
|
+
const dir = await mkdtemp(join(tmpdir(), "dm-web-access-proxy-"));
|
|
355
|
+
const headerFile = join(dir, "headers");
|
|
356
|
+
const bodyFile = join(dir, "body");
|
|
357
|
+
const requestBodyFile = join(dir, "request-body");
|
|
358
|
+
const args = [
|
|
359
|
+
"--silent",
|
|
360
|
+
"--show-error",
|
|
361
|
+
"--compressed",
|
|
362
|
+
"--connect-timeout",
|
|
363
|
+
"20",
|
|
364
|
+
"-x",
|
|
365
|
+
proxyUrl,
|
|
366
|
+
"-D",
|
|
367
|
+
headerFile,
|
|
368
|
+
"--output",
|
|
369
|
+
bodyFile,
|
|
370
|
+
"--write-out",
|
|
371
|
+
"%{json}"
|
|
372
|
+
];
|
|
373
|
+
if (method !== "GET" && method !== "HEAD")
|
|
374
|
+
args.push("-X", method);
|
|
375
|
+
for (const [name, value] of headers.entries()) {
|
|
376
|
+
if (value === "")
|
|
377
|
+
continue;
|
|
378
|
+
args.push("-H", `${name}: ${value}`);
|
|
379
|
+
}
|
|
380
|
+
const body = init.body;
|
|
381
|
+
if (body !== undefined && body !== null) {
|
|
382
|
+
let buffer;
|
|
383
|
+
if (typeof body === "string")
|
|
384
|
+
buffer = Buffer.from(body, "utf-8");
|
|
385
|
+
else if (body instanceof URLSearchParams)
|
|
386
|
+
buffer = Buffer.from(body.toString(), "utf-8");
|
|
387
|
+
else if (body instanceof ArrayBuffer)
|
|
388
|
+
buffer = Buffer.from(body);
|
|
389
|
+
else if (ArrayBuffer.isView(body))
|
|
390
|
+
buffer = Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
|
391
|
+
else
|
|
392
|
+
throw new Error(`Unsupported request body type for proxy fetch: ${typeof body}`);
|
|
393
|
+
await writeFile(requestBodyFile, buffer);
|
|
394
|
+
args.push("--data-binary", `@${requestBodyFile}`);
|
|
395
|
+
if (method === "GET")
|
|
396
|
+
args.unshift("-X", "GET");
|
|
397
|
+
}
|
|
398
|
+
args.push(url.toString());
|
|
399
|
+
const signal = init.signal ?? null;
|
|
400
|
+
let stdout;
|
|
401
|
+
try {
|
|
402
|
+
stdout = await new Promise((resolve, reject) => {
|
|
403
|
+
if (signal?.aborted) {
|
|
404
|
+
reject(new DOMException("The operation was aborted.", "AbortError"));
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
const child = spawn("curl", args, { windowsHide: true });
|
|
408
|
+
let out = "";
|
|
409
|
+
let stderr = "";
|
|
410
|
+
const onAbort = () => {
|
|
411
|
+
try {
|
|
412
|
+
child.kill();
|
|
413
|
+
} catch {}
|
|
414
|
+
};
|
|
415
|
+
if (signal) {
|
|
416
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
417
|
+
}
|
|
418
|
+
child.stdout?.on("data", (chunk) => {
|
|
419
|
+
out += chunk.toString("utf-8");
|
|
420
|
+
});
|
|
421
|
+
child.stderr?.on("data", (chunk) => {
|
|
422
|
+
if (stderr.length < 4096)
|
|
423
|
+
stderr += chunk.toString("utf-8");
|
|
424
|
+
});
|
|
425
|
+
child.once("error", (err) => {
|
|
426
|
+
signal?.removeEventListener("abort", onAbort);
|
|
427
|
+
reject(new CurlTransportError(err.code === "ENOENT" ? "curl executable not found on PATH; proxy transport requires curl" : `curl failed to start: ${err.message}`));
|
|
428
|
+
});
|
|
429
|
+
child.once("close", (code) => {
|
|
430
|
+
signal?.removeEventListener("abort", onAbort);
|
|
431
|
+
if (signal?.aborted)
|
|
432
|
+
return reject(new DOMException("The operation was aborted.", "AbortError"));
|
|
433
|
+
if (code !== 0 && !out.trim()) {
|
|
434
|
+
return reject(new CurlTransportError(`curl exited with code ${code ?? "unknown"} via ${redactProxyUrl(proxyUrl)}${stderr.trim() ? `: ${stderr.trim()}` : ""}`));
|
|
435
|
+
}
|
|
436
|
+
resolve(out);
|
|
437
|
+
});
|
|
438
|
+
});
|
|
439
|
+
} catch (err) {
|
|
440
|
+
await rm(dir, { recursive: true, force: true }).catch(() => {});
|
|
441
|
+
if (err instanceof CurlTransportError)
|
|
442
|
+
throw new Error(err.message);
|
|
443
|
+
throw err;
|
|
444
|
+
}
|
|
445
|
+
let bodyBuffer = Buffer.alloc(0);
|
|
446
|
+
let dump = "";
|
|
447
|
+
try {
|
|
448
|
+
[dump, bodyBuffer] = await Promise.all([readFile(headerFile, "utf-8"), readFile(bodyFile)]);
|
|
449
|
+
} catch {} finally {
|
|
450
|
+
await rm(dir, { recursive: true, force: true }).catch(() => {});
|
|
451
|
+
}
|
|
452
|
+
const { status, statusText, headers: responseHeaders } = parseHeaderDump(dump);
|
|
453
|
+
if (status === 0) {
|
|
454
|
+
throw new Error(`Proxy fetch to ${url.toString()} via ${redactProxyUrl(proxyUrl)} returned no HTTP status`);
|
|
455
|
+
}
|
|
456
|
+
let finalUrl = url.toString();
|
|
457
|
+
let redirected = false;
|
|
458
|
+
try {
|
|
459
|
+
const trimmed = stdout.trim();
|
|
460
|
+
if (trimmed.startsWith("{")) {
|
|
461
|
+
const writeOut = JSON.parse(trimmed);
|
|
462
|
+
if (writeOut.url_effective)
|
|
463
|
+
finalUrl = writeOut.url_effective;
|
|
464
|
+
redirected = (writeOut.num_redirects ?? 0) > 0;
|
|
465
|
+
}
|
|
466
|
+
} catch {}
|
|
467
|
+
const nullBody = status === 204 || status === 205 || status === 304;
|
|
468
|
+
const response = new Response(nullBody ? null : new Uint8Array(bodyBuffer), {
|
|
469
|
+
status,
|
|
470
|
+
statusText: statusText || undefined,
|
|
471
|
+
headers: new Headers(responseHeaders)
|
|
472
|
+
});
|
|
473
|
+
Object.defineProperty(response, "url", { value: finalUrl, configurable: true });
|
|
474
|
+
Object.defineProperty(response, "redirected", { value: redirected, configurable: true });
|
|
475
|
+
return response;
|
|
476
|
+
}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { activityMonitor } from "./activity.js";
|
|
3
|
+
import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
|
|
4
|
+
import { getWebSearchConfigPath } from "./utils.js";
|
|
5
|
+
const VALYU_SEARCH_URL = "https://api.valyu.ai/v1/search";
|
|
6
|
+
const CONFIG_PATH = getWebSearchConfigPath();
|
|
7
|
+
const SEARCH_TIMEOUT_MS = 60000;
|
|
8
|
+
const MAX_SNIPPET_CHARS = 2500;
|
|
9
|
+
const MAX_CONTENT_CHARS = 4000;
|
|
10
|
+
let cachedConfig = null;
|
|
11
|
+
function loadConfig() {
|
|
12
|
+
if (cachedConfig)
|
|
13
|
+
return cachedConfig;
|
|
14
|
+
if (!existsSync(CONFIG_PATH)) {
|
|
15
|
+
cachedConfig = {};
|
|
16
|
+
return cachedConfig;
|
|
17
|
+
}
|
|
18
|
+
const raw = readFileSync(CONFIG_PATH, "utf-8");
|
|
19
|
+
let parsed;
|
|
20
|
+
try {
|
|
21
|
+
parsed = JSON.parse(raw);
|
|
22
|
+
} catch (err) {
|
|
23
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
24
|
+
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
25
|
+
}
|
|
26
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
27
|
+
throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
|
|
28
|
+
}
|
|
29
|
+
cachedConfig = parsed;
|
|
30
|
+
return cachedConfig;
|
|
31
|
+
}
|
|
32
|
+
async function getApiKey(signal) {
|
|
33
|
+
return resolveCredential({
|
|
34
|
+
provider: "Valyu",
|
|
35
|
+
configuredValue: loadConfig().valyuApiKey,
|
|
36
|
+
environmentValue: process.env.VALYU_API_KEY,
|
|
37
|
+
signal
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
async function requireApiKey(signal) {
|
|
41
|
+
const apiKey = await getApiKey(signal);
|
|
42
|
+
if (!apiKey) {
|
|
43
|
+
throw new Error(`Valyu API key not found. Either:
|
|
44
|
+
` + ` 1. Create ${CONFIG_PATH} with { "valyuApiKey": "your-key" }
|
|
45
|
+
` + ` 2. Set VALYU_API_KEY environment variable
|
|
46
|
+
` + "Get a key at https://platform.valyu.ai");
|
|
47
|
+
}
|
|
48
|
+
return apiKey;
|
|
49
|
+
}
|
|
50
|
+
function normalizeCount(value) {
|
|
51
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
52
|
+
return 5;
|
|
53
|
+
return Math.max(1, Math.min(Math.floor(value), 20));
|
|
54
|
+
}
|
|
55
|
+
function normalizeDomain(value) {
|
|
56
|
+
let input = value.trim().toLowerCase();
|
|
57
|
+
if (!input)
|
|
58
|
+
return null;
|
|
59
|
+
if (input.startsWith("-"))
|
|
60
|
+
input = input.slice(1).trim();
|
|
61
|
+
if (!input)
|
|
62
|
+
return null;
|
|
63
|
+
try {
|
|
64
|
+
const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
|
|
65
|
+
input = parsed.hostname;
|
|
66
|
+
} catch {
|
|
67
|
+
input = input.split("/")[0]?.split(":")[0] ?? "";
|
|
68
|
+
}
|
|
69
|
+
input = input.replace(/^\.+|\.+$/g, "");
|
|
70
|
+
return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
|
|
71
|
+
}
|
|
72
|
+
function mapDomainFilter(domainFilter) {
|
|
73
|
+
if (!domainFilter?.length)
|
|
74
|
+
return {};
|
|
75
|
+
const included_sources = [];
|
|
76
|
+
const excluded_sources = [];
|
|
77
|
+
for (const raw of domainFilter) {
|
|
78
|
+
const domain = normalizeDomain(raw);
|
|
79
|
+
if (!domain)
|
|
80
|
+
continue;
|
|
81
|
+
const target = raw.trim().startsWith("-") ? excluded_sources : included_sources;
|
|
82
|
+
if (!target.includes(domain))
|
|
83
|
+
target.push(domain);
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
...included_sources.length > 0 ? { included_sources } : {},
|
|
87
|
+
...excluded_sources.length > 0 ? { excluded_sources } : {}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function recencyToStartDate(filter) {
|
|
91
|
+
if (!filter)
|
|
92
|
+
return;
|
|
93
|
+
const days = { day: 1, week: 7, month: 30, year: 365 }[filter];
|
|
94
|
+
return days ? new Date(Date.now() - days * 86400000).toISOString().slice(0, 10) : undefined;
|
|
95
|
+
}
|
|
96
|
+
function text(value, limit) {
|
|
97
|
+
return typeof value === "string" ? value.replace(/\s+/g, " ").trim().slice(0, limit) : "";
|
|
98
|
+
}
|
|
99
|
+
function errorMessage(err) {
|
|
100
|
+
return err instanceof Error ? err.message : String(err);
|
|
101
|
+
}
|
|
102
|
+
function invalidResponse(message) {
|
|
103
|
+
return new Error(`Valyu API returned invalid response: ${message}`);
|
|
104
|
+
}
|
|
105
|
+
function parseResponse(value) {
|
|
106
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
107
|
+
throw invalidResponse("expected an object envelope");
|
|
108
|
+
const envelope = value;
|
|
109
|
+
if (envelope.success !== true)
|
|
110
|
+
throw invalidResponse("expected success true");
|
|
111
|
+
if (!Array.isArray(envelope.results))
|
|
112
|
+
throw invalidResponse("expected results array");
|
|
113
|
+
return envelope.results;
|
|
114
|
+
}
|
|
115
|
+
function buildAnswer(results) {
|
|
116
|
+
return results.map((result) => result.snippet ? `${result.snippet}
|
|
117
|
+
Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
|
|
118
|
+
|
|
119
|
+
`);
|
|
120
|
+
}
|
|
121
|
+
export function isValyuAvailable() {
|
|
122
|
+
return hasCredentialSource({ provider: "Valyu", configuredValue: loadConfig().valyuApiKey, environmentValue: process.env.VALYU_API_KEY });
|
|
123
|
+
}
|
|
124
|
+
export async function searchWithValyu(query, options = {}) {
|
|
125
|
+
const apiKey = await requireApiKey(options.signal);
|
|
126
|
+
const numResults = normalizeCount(options.numResults);
|
|
127
|
+
const startDate = recencyToStartDate(options.recencyFilter);
|
|
128
|
+
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
129
|
+
let response;
|
|
130
|
+
try {
|
|
131
|
+
response = await fetch(VALYU_SEARCH_URL, {
|
|
132
|
+
method: "POST",
|
|
133
|
+
headers: { "x-api-key": apiKey, "Content-Type": "application/json", Accept: "application/json" },
|
|
134
|
+
body: JSON.stringify({ query, max_num_results: numResults, ...mapDomainFilter(options.domainFilter), ...startDate ? { start_date: startDate } : {} }),
|
|
135
|
+
signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
|
|
136
|
+
});
|
|
137
|
+
} catch (err) {
|
|
138
|
+
const message = errorMessage(err);
|
|
139
|
+
const redactedMessage = redactCredential(message, apiKey);
|
|
140
|
+
if (redactedMessage.toLowerCase().includes("abort"))
|
|
141
|
+
activityMonitor.logComplete(activityId, 0);
|
|
142
|
+
else
|
|
143
|
+
activityMonitor.logError(activityId, redactedMessage);
|
|
144
|
+
if (redactedMessage === message)
|
|
145
|
+
throw err;
|
|
146
|
+
const redactedError = new Error(redactedMessage);
|
|
147
|
+
if (err instanceof Error)
|
|
148
|
+
redactedError.name = err.name;
|
|
149
|
+
throw redactedError;
|
|
150
|
+
}
|
|
151
|
+
if (!response.ok) {
|
|
152
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
153
|
+
const errorText = redactCredential(await response.text(), apiKey);
|
|
154
|
+
throw new Error(`Valyu API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
155
|
+
}
|
|
156
|
+
let rawData;
|
|
157
|
+
try {
|
|
158
|
+
rawData = await response.json();
|
|
159
|
+
} catch (err) {
|
|
160
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
161
|
+
throw new Error(`Valyu API returned invalid JSON: ${errorMessage(err)}`);
|
|
162
|
+
}
|
|
163
|
+
let entries;
|
|
164
|
+
try {
|
|
165
|
+
entries = parseResponse(rawData);
|
|
166
|
+
} catch (err) {
|
|
167
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
168
|
+
throw err;
|
|
169
|
+
}
|
|
170
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
171
|
+
const results = [];
|
|
172
|
+
const inlineContent = [];
|
|
173
|
+
for (const entry of entries) {
|
|
174
|
+
if (!entry || typeof entry !== "object")
|
|
175
|
+
continue;
|
|
176
|
+
const url = text(entry.url, Number.MAX_SAFE_INTEGER);
|
|
177
|
+
if (!url)
|
|
178
|
+
continue;
|
|
179
|
+
const title = text(entry.title, Number.MAX_SAFE_INTEGER) || `Source ${results.length + 1}`;
|
|
180
|
+
const content = text(entry.content, MAX_CONTENT_CHARS);
|
|
181
|
+
const description = text(entry.description, MAX_SNIPPET_CHARS);
|
|
182
|
+
results.push({ title, url, snippet: (content || description).slice(0, MAX_SNIPPET_CHARS) });
|
|
183
|
+
if (options.includeContent && content)
|
|
184
|
+
inlineContent.push({ url, title, content, error: null });
|
|
185
|
+
if (results.length >= numResults)
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
return { answer: buildAnswer(results), results, ...inlineContent.length > 0 ? { inlineContent } : {} };
|
|
189
|
+
}
|