@duckmind/dm-windows-x64 0.63.0 → 0.63.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm.exe +0 -0
- package/extensions/.dm-extensions.json +200 -4
- package/extensions/dm-context/package.json +1 -1
- package/extensions/dm-context/skills/context-management/SKILL.md +173 -223
- package/extensions/dm-context/skills/context-management/references/development-and-troubleshooting.md +75 -0
- package/extensions/dm-context/skills/context-management/references/interleaved-async-work.md +143 -0
- package/extensions/dm-context/skills/context-management/references/planning-and-execution.md +81 -0
- package/extensions/dm-context/skills/context-management/references/repeated-items-and-batch-work.md +69 -0
- package/extensions/dm-context/skills/context-management/references/retry-branch-and-pivot.md +80 -0
- package/extensions/dm-context/skills/context-management/references/search-research-and-reading.md +103 -0
- package/extensions/dm-context/skills/context-management/references/task-switching-and-cleanup.md +73 -0
- package/extensions/dm-context/src/context.js +3 -2
- package/extensions/dm-context/src/index.js +168 -84
- package/extensions/dm-skills-manager/THIRD_PARTY_NOTICES.md +27 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/components.js +265 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/constants.js +31 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/creation-fallback.js +20 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/creation.js +145 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/dialog.js +738 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/dm-ai-compat.js +15 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/format.js +125 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/glyphs.js +142 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/layout.js +93 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/paths.js +76 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/registry.js +95 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/settings.js +93 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/startup.js +32 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/toggle.js +54 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/types.js +14 -0
- package/extensions/dm-skills-manager/extensions/skills-manager/ui.js +121 -0
- package/extensions/dm-skills-manager/extensions/skills-manager.js +111 -0
- package/extensions/dm-skills-manager/package.json +121 -0
- package/extensions/dm-web-access/SECURITY.md +5 -0
- package/extensions/dm-web-access/activity.js +65 -0
- package/extensions/dm-web-access/anysearch.js +158 -0
- package/extensions/dm-web-access/auth-fetch.js +131 -0
- package/extensions/dm-web-access/bocha.js +214 -0
- package/extensions/dm-web-access/brave.js +196 -0
- package/extensions/dm-web-access/brightdata-unlocker.js +202 -0
- package/extensions/dm-web-access/brightdata.js +334 -0
- package/extensions/dm-web-access/chrome-cookies.js +627 -0
- package/extensions/dm-web-access/content-find.js +114 -0
- package/extensions/dm-web-access/credential-source.js +150 -0
- package/extensions/dm-web-access/curator-page.js +3559 -0
- package/extensions/dm-web-access/curator-server.js +691 -0
- package/extensions/dm-web-access/data-uri-sanitize.js +312 -0
- package/extensions/dm-web-access/datalab-pdf-extract.js +346 -0
- package/extensions/dm-web-access/declared-web-links.js +167 -0
- package/extensions/dm-web-access/dm-web-fetch-demo.mp4 +0 -0
- package/extensions/dm-web-access/duckduckgo.js +118 -0
- package/extensions/dm-web-access/exa.js +401 -0
- package/extensions/dm-web-access/extract.js +1220 -0
- package/extensions/dm-web-access/feature-config.js +24 -0
- package/extensions/dm-web-access/fetch-params.js +81 -0
- package/extensions/dm-web-access/firecrawl.js +378 -0
- package/extensions/dm-web-access/gemini-adc.js +241 -0
- package/extensions/dm-web-access/gemini-api.js +258 -0
- package/extensions/dm-web-access/gemini-pdf-extract.js +74 -0
- package/extensions/dm-web-access/gemini-search.js +889 -0
- package/extensions/dm-web-access/gemini-url-context.js +97 -0
- package/extensions/dm-web-access/gemini-web-config.js +84 -0
- package/extensions/dm-web-access/gemini-web.js +351 -0
- package/extensions/dm-web-access/github-api.js +166 -0
- package/extensions/dm-web-access/github-extract.js +991 -0
- package/extensions/dm-web-access/github-issue-pr.js +750 -0
- package/extensions/dm-web-access/index.js +3117 -0
- package/extensions/dm-web-access/jina-search.js +242 -0
- package/extensions/dm-web-access/kagi.js +255 -0
- package/extensions/dm-web-access/kimi-search.js +214 -0
- package/extensions/dm-web-access/ollama.js +209 -0
- package/extensions/dm-web-access/openai-search.js +510 -0
- package/extensions/dm-web-access/package.json +34 -0
- package/extensions/dm-web-access/page-query.js +124 -0
- package/extensions/dm-web-access/parallel-mcp.js +223 -0
- package/extensions/dm-web-access/parallel.js +345 -0
- package/extensions/dm-web-access/pdf-extract.js +257 -0
- package/extensions/dm-web-access/perplexity.js +151 -0
- package/extensions/dm-web-access/querit.js +327 -0
- package/extensions/dm-web-access/query-rewrite.js +40 -0
- package/extensions/dm-web-access/render-search-error.js +80 -0
- package/extensions/dm-web-access/rsc-extract.js +347 -0
- package/extensions/dm-web-access/search1api.js +245 -0
- package/extensions/dm-web-access/searchinfinity.js +221 -0
- package/extensions/dm-web-access/searxng.js +223 -0
- package/extensions/dm-web-access/serpbase.js +205 -0
- package/extensions/dm-web-access/serpdive.js +238 -0
- package/extensions/dm-web-access/serper.js +200 -0
- package/extensions/dm-web-access/source-check.js +198 -0
- package/extensions/dm-web-access/ssrf-protection.js +436 -0
- package/extensions/dm-web-access/storage.js +451 -0
- package/extensions/dm-web-access/summary-model-scope.js +83 -0
- package/extensions/dm-web-access/summary-review.js +364 -0
- package/extensions/dm-web-access/tavily.js +199 -0
- package/extensions/dm-web-access/tinyfish.js +325 -0
- package/extensions/dm-web-access/utils.js +476 -0
- package/extensions/dm-web-access/valyu.js +189 -0
- package/extensions/dm-web-access/video-extract.js +336 -0
- package/extensions/dm-web-access/xai-search.js +285 -0
- package/extensions/dm-web-access/xcrawl.js +221 -0
- package/extensions/dm-web-access/youtube-extract.js +279 -0
- package/package.json +9 -1
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
const MAX_DECLARED_LINKS = 20;
|
|
2
|
+
const MAX_DECLARED_URL_LENGTH = 4096;
|
|
3
|
+
const DECLARATION_RELATIONS = new Set([
|
|
4
|
+
"api-catalog",
|
|
5
|
+
"describedby",
|
|
6
|
+
"service-desc",
|
|
7
|
+
"service-doc",
|
|
8
|
+
"service-meta"
|
|
9
|
+
]);
|
|
10
|
+
const RELATION_LABELS = {
|
|
11
|
+
"api-catalog": "API catalog",
|
|
12
|
+
describedby: "Description",
|
|
13
|
+
"service-desc": "Service description",
|
|
14
|
+
"service-doc": "Service documentation",
|
|
15
|
+
"service-meta": "Service metadata"
|
|
16
|
+
};
|
|
17
|
+
export function discoverDeclaredWebLinks(document, linkHeader, responseUrl) {
|
|
18
|
+
const links = new Map;
|
|
19
|
+
for (const value of splitLinkHeader(linkHeader ?? "")) {
|
|
20
|
+
const target = /^\s*<([^>]*)>/.exec(value);
|
|
21
|
+
if (!target)
|
|
22
|
+
continue;
|
|
23
|
+
const parameters = parseLinkParameters(value.slice(target[0].length));
|
|
24
|
+
if (!parameters || parameters.has("anchor"))
|
|
25
|
+
continue;
|
|
26
|
+
addDeclaredLink(links, {
|
|
27
|
+
url: resolveHttpUrl(target[1], responseUrl),
|
|
28
|
+
relations: declaredRelations(parameters.get("rel")),
|
|
29
|
+
type: parameters.get("type")
|
|
30
|
+
});
|
|
31
|
+
if (links.size >= MAX_DECLARED_LINKS)
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
if (links.size < MAX_DECLARED_LINKS) {
|
|
35
|
+
const declaredBase = document.querySelector("base[href]")?.getAttribute("href");
|
|
36
|
+
const documentBase = resolveHttpUrl(declaredBase, responseUrl) ?? responseUrl;
|
|
37
|
+
for (const element of document.querySelectorAll("link[rel][href], a[rel][href]")) {
|
|
38
|
+
addDeclaredLink(links, {
|
|
39
|
+
url: resolveHttpUrl(element.getAttribute("href"), documentBase),
|
|
40
|
+
relations: declaredRelations(element.getAttribute("rel")),
|
|
41
|
+
type: element.getAttribute("type")
|
|
42
|
+
});
|
|
43
|
+
if (links.size >= MAX_DECLARED_LINKS)
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return [...links.values()];
|
|
48
|
+
}
|
|
49
|
+
export function appendDeclaredWebLinks(content, links) {
|
|
50
|
+
if (links.length === 0)
|
|
51
|
+
return content;
|
|
52
|
+
const section = [
|
|
53
|
+
"## Declared links",
|
|
54
|
+
"",
|
|
55
|
+
...links.map(formatDeclaredLink)
|
|
56
|
+
].join(`
|
|
57
|
+
`);
|
|
58
|
+
return content.trim() ? `${content.trim()}
|
|
59
|
+
|
|
60
|
+
${section}` : section;
|
|
61
|
+
}
|
|
62
|
+
function addDeclaredLink(links, candidate) {
|
|
63
|
+
if (!candidate.url || candidate.relations.length === 0)
|
|
64
|
+
return;
|
|
65
|
+
const existing = links.get(candidate.url);
|
|
66
|
+
if (existing) {
|
|
67
|
+
for (const relation of candidate.relations) {
|
|
68
|
+
if (!existing.relations.includes(relation))
|
|
69
|
+
existing.relations.push(relation);
|
|
70
|
+
}
|
|
71
|
+
if (!existing.type)
|
|
72
|
+
existing.type = normalizeMetadata(candidate.type);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
if (links.size >= MAX_DECLARED_LINKS)
|
|
76
|
+
return;
|
|
77
|
+
const type = normalizeMetadata(candidate.type);
|
|
78
|
+
links.set(candidate.url, {
|
|
79
|
+
url: candidate.url,
|
|
80
|
+
relations: candidate.relations,
|
|
81
|
+
...type ? { type } : {}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
function declaredRelations(value) {
|
|
85
|
+
if (!value)
|
|
86
|
+
return [];
|
|
87
|
+
return [...new Set(value.trim().toLowerCase().split(/\s+/).filter((relation) => DECLARATION_RELATIONS.has(relation)))];
|
|
88
|
+
}
|
|
89
|
+
function resolveHttpUrl(value, baseUrl) {
|
|
90
|
+
if (!value || value.length > MAX_DECLARED_URL_LENGTH)
|
|
91
|
+
return null;
|
|
92
|
+
try {
|
|
93
|
+
const url = new URL(value, baseUrl);
|
|
94
|
+
if (url.href.length > MAX_DECLARED_URL_LENGTH)
|
|
95
|
+
return null;
|
|
96
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
|
|
97
|
+
} catch {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
function splitOutsideSyntax(input, separator, protectTargets) {
|
|
102
|
+
const parts = [];
|
|
103
|
+
let start = 0;
|
|
104
|
+
let inTarget = false;
|
|
105
|
+
let inQuotes = false;
|
|
106
|
+
let escaped = false;
|
|
107
|
+
for (let index = 0;index < input.length; index++) {
|
|
108
|
+
const character = input[index];
|
|
109
|
+
if (inQuotes) {
|
|
110
|
+
if (escaped)
|
|
111
|
+
escaped = false;
|
|
112
|
+
else if (character === "\\")
|
|
113
|
+
escaped = true;
|
|
114
|
+
else if (character === '"')
|
|
115
|
+
inQuotes = false;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (character === '"' && !inTarget)
|
|
119
|
+
inQuotes = true;
|
|
120
|
+
else if (protectTargets && character === "<")
|
|
121
|
+
inTarget = true;
|
|
122
|
+
else if (protectTargets && character === ">")
|
|
123
|
+
inTarget = false;
|
|
124
|
+
else if (character === separator && !inTarget) {
|
|
125
|
+
parts.push(input.slice(start, index));
|
|
126
|
+
start = index + 1;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (inQuotes || inTarget)
|
|
130
|
+
return null;
|
|
131
|
+
parts.push(input.slice(start));
|
|
132
|
+
return parts;
|
|
133
|
+
}
|
|
134
|
+
function splitLinkHeader(header) {
|
|
135
|
+
return (splitOutsideSyntax(header, ",", true) ?? []).map((value) => value.trim()).filter(Boolean);
|
|
136
|
+
}
|
|
137
|
+
function parseLinkParameters(input) {
|
|
138
|
+
const parts = splitOutsideSyntax(input, ";", false);
|
|
139
|
+
if (!parts || parts.shift()?.trim())
|
|
140
|
+
return null;
|
|
141
|
+
const parameters = new Map;
|
|
142
|
+
for (const part of parts) {
|
|
143
|
+
const match = /^\s*([!#$%&'*+\-.^_`|~A-Za-z0-9]+)(?:\s*=\s*(?:"((?:\\.|[^"])*)"|(\S+)))?\s*$/.exec(part);
|
|
144
|
+
if (!match)
|
|
145
|
+
return null;
|
|
146
|
+
const name = match[1].toLowerCase();
|
|
147
|
+
const value = match[2] === undefined ? match[3] ?? "" : match[2].replace(/\\(.)/g, "$1");
|
|
148
|
+
if (!parameters.has(name))
|
|
149
|
+
parameters.set(name, value);
|
|
150
|
+
}
|
|
151
|
+
return parameters;
|
|
152
|
+
}
|
|
153
|
+
function normalizeMetadata(value) {
|
|
154
|
+
if (!value)
|
|
155
|
+
return;
|
|
156
|
+
const normalized = value.replace(/\s+/g, " ").trim().slice(0, 160);
|
|
157
|
+
return normalized || undefined;
|
|
158
|
+
}
|
|
159
|
+
function formatDeclaredLink(link) {
|
|
160
|
+
const relation = link.relations.map(inlineCode).join(", ");
|
|
161
|
+
const type = link.type ? `; ${inlineCode(link.type)}` : "";
|
|
162
|
+
const label = RELATION_LABELS[link.relations[0]] ?? "Declared link";
|
|
163
|
+
return `- ${label} (${relation}${type}): <${link.url}>`;
|
|
164
|
+
}
|
|
165
|
+
function inlineCode(value) {
|
|
166
|
+
return `\`${value.replace(/`/g, "'")}\``;
|
|
167
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { parseHTML } from "linkedom";
|
|
2
|
+
import { activityMonitor } from "./activity.js";
|
|
3
|
+
const SEARCH_URL = "https://html.duckduckgo.com/html/";
|
|
4
|
+
const SEARCH_TIMEOUT_MS = 30000;
|
|
5
|
+
function normalizeCount(value) {
|
|
6
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
7
|
+
return 5;
|
|
8
|
+
return Math.max(1, Math.min(Math.floor(value), 20));
|
|
9
|
+
}
|
|
10
|
+
function normalizeDomain(value) {
|
|
11
|
+
let input = value.trim().toLowerCase();
|
|
12
|
+
if (!input)
|
|
13
|
+
return null;
|
|
14
|
+
if (input.startsWith("-"))
|
|
15
|
+
input = input.slice(1).trim();
|
|
16
|
+
if (!input)
|
|
17
|
+
return null;
|
|
18
|
+
try {
|
|
19
|
+
const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
|
|
20
|
+
input = parsed.hostname;
|
|
21
|
+
} catch {
|
|
22
|
+
input = input.split("/")[0]?.split(":")[0] ?? "";
|
|
23
|
+
}
|
|
24
|
+
input = input.replace(/^\.+|\.+$/g, "");
|
|
25
|
+
return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
|
|
26
|
+
}
|
|
27
|
+
function normalizeDomainFilters(domainFilter) {
|
|
28
|
+
const filters = { allowed: [], blocked: [] };
|
|
29
|
+
for (const raw of domainFilter ?? []) {
|
|
30
|
+
const domain = normalizeDomain(raw);
|
|
31
|
+
if (!domain)
|
|
32
|
+
continue;
|
|
33
|
+
const target = raw.trim().startsWith("-") ? filters.blocked : filters.allowed;
|
|
34
|
+
if (!target.includes(domain))
|
|
35
|
+
target.push(domain);
|
|
36
|
+
}
|
|
37
|
+
return filters;
|
|
38
|
+
}
|
|
39
|
+
function hostMatchesDomain(hostname, domain) {
|
|
40
|
+
return hostname === domain || hostname.endsWith(`.${domain}`);
|
|
41
|
+
}
|
|
42
|
+
function matchesDomainFilters(url, filters) {
|
|
43
|
+
if (filters.allowed.length === 0 && filters.blocked.length === 0)
|
|
44
|
+
return true;
|
|
45
|
+
const hostname = new URL(url).hostname.toLowerCase();
|
|
46
|
+
if (filters.allowed.length > 0 && !filters.allowed.some((domain) => hostMatchesDomain(hostname, domain)))
|
|
47
|
+
return false;
|
|
48
|
+
return !filters.blocked.some((domain) => hostMatchesDomain(hostname, domain));
|
|
49
|
+
}
|
|
50
|
+
function decodeResultUrl(href) {
|
|
51
|
+
try {
|
|
52
|
+
const link = new URL(href, SEARCH_URL);
|
|
53
|
+
const destination = link.searchParams.get("uddg") ?? link.href;
|
|
54
|
+
const url = new URL(destination);
|
|
55
|
+
return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
export function isDuckDuckGoAvailable() {
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
export async function searchWithDuckDuckGo(query, options = {}) {
|
|
64
|
+
const url = new URL(SEARCH_URL);
|
|
65
|
+
url.searchParams.set("q", query);
|
|
66
|
+
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
67
|
+
try {
|
|
68
|
+
const response = await fetch(url, {
|
|
69
|
+
method: "GET",
|
|
70
|
+
headers: {
|
|
71
|
+
Accept: "text/html",
|
|
72
|
+
"User-Agent": "Mozilla/5.0 (compatible; dm-web-access/1.0; +https://github.com/nicobailon/dm-web-access)"
|
|
73
|
+
},
|
|
74
|
+
signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
|
|
75
|
+
});
|
|
76
|
+
if (!response.ok) {
|
|
77
|
+
const body = await response.text();
|
|
78
|
+
throw new Error(`DuckDuckGo search error ${response.status}: ${body.slice(0, 300)}`);
|
|
79
|
+
}
|
|
80
|
+
const { document } = parseHTML(await response.text());
|
|
81
|
+
const filters = normalizeDomainFilters(options.domainFilter);
|
|
82
|
+
const results = [];
|
|
83
|
+
let parseableResults = 0;
|
|
84
|
+
for (const container of document.querySelectorAll(".result")) {
|
|
85
|
+
if (container.classList.contains("result--ad"))
|
|
86
|
+
continue;
|
|
87
|
+
const anchor = container.querySelector(".result__a");
|
|
88
|
+
const title = anchor?.textContent?.trim() ?? "";
|
|
89
|
+
const href = anchor?.getAttribute("href")?.trim() ?? "";
|
|
90
|
+
const resultUrl = href ? decodeResultUrl(href) : null;
|
|
91
|
+
if (!title || !resultUrl)
|
|
92
|
+
continue;
|
|
93
|
+
parseableResults++;
|
|
94
|
+
if (!matchesDomainFilters(resultUrl, filters))
|
|
95
|
+
continue;
|
|
96
|
+
const snippet = container.querySelector(".result__snippet")?.textContent?.trim() ?? "";
|
|
97
|
+
results.push({ title, url: resultUrl, snippet });
|
|
98
|
+
if (results.length >= normalizeCount(options.numResults))
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
if (parseableResults === 0) {
|
|
102
|
+
throw new Error("DuckDuckGo returned no parseable results (invalid response)");
|
|
103
|
+
}
|
|
104
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
105
|
+
const answer = results.map((result) => result.snippet ? `${result.snippet}
|
|
106
|
+
Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
|
|
107
|
+
|
|
108
|
+
`);
|
|
109
|
+
return { answer, results };
|
|
110
|
+
} catch (err) {
|
|
111
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
112
|
+
if (message.toLowerCase().includes("abort"))
|
|
113
|
+
activityMonitor.logComplete(activityId, 0);
|
|
114
|
+
else
|
|
115
|
+
activityMonitor.logError(activityId, message);
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { activityMonitor } from "./activity.js";
|
|
3
|
+
import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
|
|
4
|
+
import { fetchWithCredentialRedirects, getWebSearchConfigPath, resolveApiBaseUrl } from "./utils.js";
|
|
5
|
+
const EXA_API_BASE_URL = "https://api.exa.ai";
|
|
6
|
+
const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
|
|
7
|
+
const CONFIG_PATH = getWebSearchConfigPath();
|
|
8
|
+
const EXA_MCP_ADVANCED_TOOL = "web_search_advanced_exa";
|
|
9
|
+
const EXA_MCP_BASIC_TOOL = "web_search_exa";
|
|
10
|
+
let cachedConfig = null;
|
|
11
|
+
function loadConfig() {
|
|
12
|
+
if (cachedConfig)
|
|
13
|
+
return cachedConfig;
|
|
14
|
+
if (!existsSync(CONFIG_PATH)) {
|
|
15
|
+
cachedConfig = {};
|
|
16
|
+
return cachedConfig;
|
|
17
|
+
}
|
|
18
|
+
const raw = readFileSync(CONFIG_PATH, "utf-8");
|
|
19
|
+
try {
|
|
20
|
+
cachedConfig = JSON.parse(raw);
|
|
21
|
+
return cachedConfig;
|
|
22
|
+
} catch (err) {
|
|
23
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
24
|
+
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async function getApiKey(signal) {
|
|
28
|
+
return resolveCredential({
|
|
29
|
+
provider: "Exa",
|
|
30
|
+
configuredValue: loadConfig().exaApiKey,
|
|
31
|
+
environmentValue: process.env.EXA_API_KEY,
|
|
32
|
+
signal
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
function getApiBaseUrl() {
|
|
36
|
+
return resolveApiBaseUrl({
|
|
37
|
+
configKey: "exaBaseUrl",
|
|
38
|
+
configuredValue: loadConfig().exaBaseUrl,
|
|
39
|
+
defaultValue: EXA_API_BASE_URL,
|
|
40
|
+
environmentKey: "EXA_BASE_URL",
|
|
41
|
+
environmentValue: process.env.EXA_BASE_URL
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
function exaApiHeaders(apiKey) {
|
|
45
|
+
return {
|
|
46
|
+
"x-api-key": apiKey,
|
|
47
|
+
"Content-Type": "application/json",
|
|
48
|
+
"x-exa-integration": "dm-web-access"
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function requestSignal(signal) {
|
|
52
|
+
const timeout = AbortSignal.timeout(60000);
|
|
53
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
54
|
+
}
|
|
55
|
+
function recencyToStartDate(filter) {
|
|
56
|
+
const now = new Date;
|
|
57
|
+
const offsets = {
|
|
58
|
+
day: 1,
|
|
59
|
+
week: 7,
|
|
60
|
+
month: 30,
|
|
61
|
+
year: 365
|
|
62
|
+
};
|
|
63
|
+
const days = offsets[filter] ?? 0;
|
|
64
|
+
return new Date(now.getTime() - days * 86400000).toISOString();
|
|
65
|
+
}
|
|
66
|
+
function mapDomainFilter(domainFilter) {
|
|
67
|
+
if (!domainFilter?.length)
|
|
68
|
+
return {};
|
|
69
|
+
const includeDomains = domainFilter.filter((d) => !d.startsWith("-") && d.trim().length > 0).map((d) => d.trim());
|
|
70
|
+
const excludeDomains = domainFilter.filter((d) => d.startsWith("-")).map((d) => d.slice(1).trim()).filter(Boolean);
|
|
71
|
+
return {
|
|
72
|
+
...includeDomains.length ? { includeDomains } : {},
|
|
73
|
+
...excludeDomains.length ? { excludeDomains } : {}
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function exaSearchArgs(query, options) {
|
|
77
|
+
const startDate = options.recencyFilter ? recencyToStartDate(options.recencyFilter) : null;
|
|
78
|
+
return {
|
|
79
|
+
query,
|
|
80
|
+
type: "auto",
|
|
81
|
+
numResults: options.numResults ?? 5,
|
|
82
|
+
...mapDomainFilter(options.domainFilter),
|
|
83
|
+
...startDate ? { startPublishedDate: startDate } : {}
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
function normalizeHighlights(value) {
|
|
87
|
+
if (!Array.isArray(value))
|
|
88
|
+
return [];
|
|
89
|
+
return value.filter((item) => typeof item === "string" && item.trim().length > 0);
|
|
90
|
+
}
|
|
91
|
+
function buildAnswerFromSearchResults(results) {
|
|
92
|
+
if (!results?.length)
|
|
93
|
+
return "";
|
|
94
|
+
const parts = [];
|
|
95
|
+
for (let i = 0;i < results.length; i++) {
|
|
96
|
+
const item = results[i];
|
|
97
|
+
if (!item?.url)
|
|
98
|
+
continue;
|
|
99
|
+
const highlights = normalizeHighlights(item.highlights);
|
|
100
|
+
const content = highlights.length > 0 ? highlights.join(" ") : typeof item.text === "string" ? item.text.trim().slice(0, 1000) : "";
|
|
101
|
+
if (!content)
|
|
102
|
+
continue;
|
|
103
|
+
const sourceTitle = item.title || `Source ${i + 1}`;
|
|
104
|
+
parts.push(`${content}
|
|
105
|
+
Source: ${sourceTitle} (${item.url})`);
|
|
106
|
+
}
|
|
107
|
+
return parts.join(`
|
|
108
|
+
|
|
109
|
+
`);
|
|
110
|
+
}
|
|
111
|
+
function mapResults(results) {
|
|
112
|
+
if (!Array.isArray(results))
|
|
113
|
+
return [];
|
|
114
|
+
const mapped = [];
|
|
115
|
+
for (let i = 0;i < results.length; i++) {
|
|
116
|
+
const item = results[i];
|
|
117
|
+
if (!item?.url)
|
|
118
|
+
continue;
|
|
119
|
+
mapped.push({
|
|
120
|
+
title: item.title || `Source ${i + 1}`,
|
|
121
|
+
url: item.url,
|
|
122
|
+
snippet: ""
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
return mapped;
|
|
126
|
+
}
|
|
127
|
+
function mapInlineContent(results) {
|
|
128
|
+
if (!results?.length)
|
|
129
|
+
return [];
|
|
130
|
+
return results.filter((r) => !!r?.url && typeof r.text === "string" && r.text.length > 0).map((r) => ({
|
|
131
|
+
url: r.url,
|
|
132
|
+
title: r.title || "",
|
|
133
|
+
content: r.text,
|
|
134
|
+
error: null
|
|
135
|
+
}));
|
|
136
|
+
}
|
|
137
|
+
function toSearchResponse(answer, results, inlineContent) {
|
|
138
|
+
const response = { answer, results };
|
|
139
|
+
if (inlineContent?.length)
|
|
140
|
+
response.inlineContent = inlineContent;
|
|
141
|
+
return response;
|
|
142
|
+
}
|
|
143
|
+
export async function callExaMcp(toolName, args, signal) {
|
|
144
|
+
const response = await fetch(`${EXA_MCP_URL}?tools=${toolName}`, {
|
|
145
|
+
method: "POST",
|
|
146
|
+
headers: {
|
|
147
|
+
"Content-Type": "application/json",
|
|
148
|
+
Accept: "application/json, text/event-stream",
|
|
149
|
+
"x-exa-source": "dm-web-access"
|
|
150
|
+
},
|
|
151
|
+
body: JSON.stringify({
|
|
152
|
+
jsonrpc: "2.0",
|
|
153
|
+
id: 1,
|
|
154
|
+
method: "tools/call",
|
|
155
|
+
params: {
|
|
156
|
+
name: toolName,
|
|
157
|
+
arguments: args
|
|
158
|
+
}
|
|
159
|
+
}),
|
|
160
|
+
signal: requestSignal(signal)
|
|
161
|
+
});
|
|
162
|
+
if (!response.ok) {
|
|
163
|
+
const errorText = await response.text();
|
|
164
|
+
if (response.status === 429) {
|
|
165
|
+
throw new Error(`Exa MCP rate limit reached (429). Add "exaApiKey" to ${CONFIG_PATH} for unthrottled Exa search: ${errorText.slice(0, 200)}`);
|
|
166
|
+
}
|
|
167
|
+
throw new Error(`Exa MCP error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
168
|
+
}
|
|
169
|
+
const body = await response.text();
|
|
170
|
+
const dataLines = body.split(`
|
|
171
|
+
`).filter((line) => line.startsWith("data:"));
|
|
172
|
+
let parsed = null;
|
|
173
|
+
for (const line of dataLines) {
|
|
174
|
+
const payload = line.slice(5).trim();
|
|
175
|
+
if (!payload)
|
|
176
|
+
continue;
|
|
177
|
+
try {
|
|
178
|
+
const candidate = JSON.parse(payload);
|
|
179
|
+
if (candidate?.result || candidate?.error) {
|
|
180
|
+
parsed = candidate;
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
} catch {}
|
|
184
|
+
}
|
|
185
|
+
if (!parsed) {
|
|
186
|
+
try {
|
|
187
|
+
const candidate = JSON.parse(body);
|
|
188
|
+
if (candidate?.result || candidate?.error) {
|
|
189
|
+
parsed = candidate;
|
|
190
|
+
}
|
|
191
|
+
} catch {}
|
|
192
|
+
}
|
|
193
|
+
if (!parsed) {
|
|
194
|
+
throw new Error("Exa MCP returned an empty response");
|
|
195
|
+
}
|
|
196
|
+
if (parsed.error) {
|
|
197
|
+
const code = typeof parsed.error.code === "number" ? ` ${parsed.error.code}` : "";
|
|
198
|
+
const message = parsed.error.message || "Unknown error";
|
|
199
|
+
throw new Error(`Exa MCP error${code}: ${message}`);
|
|
200
|
+
}
|
|
201
|
+
if (parsed.result?.isError) {
|
|
202
|
+
const message = parsed.result.content?.find((item) => item.type === "text" && typeof item.text === "string")?.text?.trim();
|
|
203
|
+
throw new Error(message || "Exa MCP returned an error");
|
|
204
|
+
}
|
|
205
|
+
const text = parsed.result?.content?.find((item) => item.type === "text" && typeof item.text === "string" && item.text.trim().length > 0)?.text;
|
|
206
|
+
if (!text) {
|
|
207
|
+
throw new Error("Exa MCP returned empty content");
|
|
208
|
+
}
|
|
209
|
+
return text;
|
|
210
|
+
}
|
|
211
|
+
function parseMcpResults(text) {
|
|
212
|
+
const blocks = text.split(/(?=^Title: )/m).filter((block) => block.trim().length > 0);
|
|
213
|
+
const parsed = blocks.map((block) => {
|
|
214
|
+
const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? "";
|
|
215
|
+
const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? "";
|
|
216
|
+
let content = "";
|
|
217
|
+
const textStart = block.indexOf(`
|
|
218
|
+
Text: `);
|
|
219
|
+
if (textStart >= 0) {
|
|
220
|
+
content = block.slice(textStart + 7).trim();
|
|
221
|
+
} else {
|
|
222
|
+
const hlMatch = block.match(/\nHighlights:\s*\n/);
|
|
223
|
+
if (hlMatch?.index != null) {
|
|
224
|
+
content = block.slice(hlMatch.index + hlMatch[0].length).trim();
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
content = content.replace(/\n---\s*$/, "").trim();
|
|
228
|
+
return { title, url, content };
|
|
229
|
+
}).filter((result) => result.url.length > 0);
|
|
230
|
+
return parsed.length > 0 ? parsed : null;
|
|
231
|
+
}
|
|
232
|
+
function buildAnswerFromMcpResults(results) {
|
|
233
|
+
if (results.length === 0)
|
|
234
|
+
return "";
|
|
235
|
+
const parts = [];
|
|
236
|
+
for (let i = 0;i < results.length; i++) {
|
|
237
|
+
const result = results[i];
|
|
238
|
+
const snippet = result.content.replace(/\s+/g, " ").trim().slice(0, 500);
|
|
239
|
+
if (!snippet)
|
|
240
|
+
continue;
|
|
241
|
+
const sourceTitle = result.title || `Source ${i + 1}`;
|
|
242
|
+
parts.push(`${snippet}
|
|
243
|
+
Source: ${sourceTitle} (${result.url})`);
|
|
244
|
+
}
|
|
245
|
+
return parts.join(`
|
|
246
|
+
|
|
247
|
+
`);
|
|
248
|
+
}
|
|
249
|
+
function mapMcpInlineContent(results) {
|
|
250
|
+
return results.filter((result) => result.content.length > 0).map((result) => ({
|
|
251
|
+
url: result.url,
|
|
252
|
+
title: result.title,
|
|
253
|
+
content: result.content,
|
|
254
|
+
error: null
|
|
255
|
+
}));
|
|
256
|
+
}
|
|
257
|
+
function buildMcpQuery(query, options) {
|
|
258
|
+
const parts = [query];
|
|
259
|
+
if (options.domainFilter?.length) {
|
|
260
|
+
for (const d of options.domainFilter) {
|
|
261
|
+
parts.push(d.startsWith("-") ? `-site:${d.slice(1)}` : `site:${d}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (options.recencyFilter) {
|
|
265
|
+
const now = new Date;
|
|
266
|
+
switch (options.recencyFilter) {
|
|
267
|
+
case "day":
|
|
268
|
+
parts.push("past 24 hours");
|
|
269
|
+
break;
|
|
270
|
+
case "week":
|
|
271
|
+
parts.push("past week");
|
|
272
|
+
break;
|
|
273
|
+
case "month":
|
|
274
|
+
parts.push(`${now.toLocaleString("en", { month: "long" })} ${now.getFullYear()}`);
|
|
275
|
+
break;
|
|
276
|
+
case "year":
|
|
277
|
+
parts.push(String(now.getFullYear()));
|
|
278
|
+
break;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
return parts.join(" ");
|
|
282
|
+
}
|
|
283
|
+
function isAbortMessage(message) {
|
|
284
|
+
return message.toLowerCase().includes("abort");
|
|
285
|
+
}
|
|
286
|
+
function parseJsonMcpResults(text) {
|
|
287
|
+
try {
|
|
288
|
+
const results = JSON.parse(text).results;
|
|
289
|
+
return Array.isArray(results) && results.length > 0 ? results : null;
|
|
290
|
+
} catch {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
async function searchWithExaMcpTool(tool, args, options) {
|
|
295
|
+
const text = await callExaMcp(tool, args, options.signal);
|
|
296
|
+
const jsonResults = parseJsonMcpResults(text);
|
|
297
|
+
if (jsonResults) {
|
|
298
|
+
return toSearchResponse(buildAnswerFromSearchResults(jsonResults), mapResults(jsonResults), options.includeContent ? mapInlineContent(jsonResults) : null);
|
|
299
|
+
}
|
|
300
|
+
const textResults = parseMcpResults(text);
|
|
301
|
+
if (!textResults)
|
|
302
|
+
return null;
|
|
303
|
+
return toSearchResponse(buildAnswerFromMcpResults(textResults), mapResults(textResults), options.includeContent ? mapMcpInlineContent(textResults) : null);
|
|
304
|
+
}
|
|
305
|
+
async function searchWithFilteredExaMcp(query, options, basicArgs) {
|
|
306
|
+
try {
|
|
307
|
+
return await searchWithExaMcpTool(EXA_MCP_ADVANCED_TOOL, {
|
|
308
|
+
...exaSearchArgs(query, options),
|
|
309
|
+
enableHighlights: true,
|
|
310
|
+
textMaxCharacters: options.includeContent ? 50000 : 3000
|
|
311
|
+
}, options);
|
|
312
|
+
} catch (err) {
|
|
313
|
+
if (isAbortMessage(err instanceof Error ? err.message : String(err)))
|
|
314
|
+
throw err;
|
|
315
|
+
return searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
async function searchWithExaMcp(query, options = {}) {
|
|
319
|
+
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
320
|
+
const basicArgs = { query: buildMcpQuery(query, options), numResults: options.numResults ?? 5 };
|
|
321
|
+
const filtered = !!options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length;
|
|
322
|
+
try {
|
|
323
|
+
const response = filtered ? await searchWithFilteredExaMcp(query, options, basicArgs) : await searchWithExaMcpTool(EXA_MCP_BASIC_TOOL, basicArgs, options);
|
|
324
|
+
activityMonitor.logComplete(activityId, 200);
|
|
325
|
+
return response;
|
|
326
|
+
} catch (err) {
|
|
327
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
328
|
+
if (isAbortMessage(message)) {
|
|
329
|
+
activityMonitor.logComplete(activityId, 0);
|
|
330
|
+
} else {
|
|
331
|
+
activityMonitor.logError(activityId, message);
|
|
332
|
+
}
|
|
333
|
+
throw err;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
export function isExaAvailable() {
|
|
337
|
+
return true;
|
|
338
|
+
}
|
|
339
|
+
export function hasExaApiKey() {
|
|
340
|
+
return hasCredentialSource({
|
|
341
|
+
provider: "Exa",
|
|
342
|
+
configuredValue: loadConfig().exaApiKey,
|
|
343
|
+
environmentValue: process.env.EXA_API_KEY
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
export async function searchWithExa(query, options = {}) {
|
|
347
|
+
const apiKey = await getApiKey(options.signal);
|
|
348
|
+
if (!apiKey) {
|
|
349
|
+
return searchWithExaMcp(query, options);
|
|
350
|
+
}
|
|
351
|
+
const apiBaseUrl = getApiBaseUrl();
|
|
352
|
+
const useSearch = options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length || !!(options.numResults && options.numResults !== 5);
|
|
353
|
+
const activityId = activityMonitor.logStart({ type: "api", query });
|
|
354
|
+
try {
|
|
355
|
+
if (!useSearch) {
|
|
356
|
+
const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/answer`, {
|
|
357
|
+
method: "POST",
|
|
358
|
+
headers: exaApiHeaders(apiKey),
|
|
359
|
+
body: JSON.stringify({ query }),
|
|
360
|
+
signal: requestSignal(options.signal)
|
|
361
|
+
}, ["x-api-key"]);
|
|
362
|
+
if (!response.ok) {
|
|
363
|
+
const errorText = redactCredential(await response.text(), apiKey);
|
|
364
|
+
throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
365
|
+
}
|
|
366
|
+
const data = await response.json();
|
|
367
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
368
|
+
return {
|
|
369
|
+
answer: data.answer || "",
|
|
370
|
+
results: mapResults(data.citations)
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
const response = await fetchWithCredentialRedirects(`${apiBaseUrl}/search`, {
|
|
374
|
+
method: "POST",
|
|
375
|
+
headers: exaApiHeaders(apiKey),
|
|
376
|
+
body: JSON.stringify({
|
|
377
|
+
...exaSearchArgs(query, options),
|
|
378
|
+
contents: options.includeContent ? { text: true, highlights: true } : { highlights: true }
|
|
379
|
+
}),
|
|
380
|
+
signal: requestSignal(options.signal)
|
|
381
|
+
}, ["x-api-key"]);
|
|
382
|
+
if (!response.ok) {
|
|
383
|
+
const errorText = redactCredential(await response.text(), apiKey);
|
|
384
|
+
throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`);
|
|
385
|
+
}
|
|
386
|
+
const data = await response.json();
|
|
387
|
+
activityMonitor.logComplete(activityId, response.status);
|
|
388
|
+
return toSearchResponse(buildAnswerFromSearchResults(data.results), mapResults(data.results), options.includeContent ? mapInlineContent(data.results) : null);
|
|
389
|
+
} catch (err) {
|
|
390
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
391
|
+
const redactedMessage = redactCredential(message, apiKey);
|
|
392
|
+
if (isAbortMessage(redactedMessage)) {
|
|
393
|
+
activityMonitor.logComplete(activityId, 0);
|
|
394
|
+
} else {
|
|
395
|
+
activityMonitor.logError(activityId, redactedMessage);
|
|
396
|
+
}
|
|
397
|
+
if (redactedMessage === message)
|
|
398
|
+
throw err;
|
|
399
|
+
throw new Error(redactedMessage);
|
|
400
|
+
}
|
|
401
|
+
}
|