@easynet/agent-tool-buildin 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Search DuckDuckGo via the Instant Answer API (no API key).
|
|
3
|
-
*
|
|
3
|
+
* When the API returns no web results, falls back to HTML search (html.duckduckgo.com).
|
|
4
|
+
* Requires api.duckduckgo.com in allowedHosts; add html.duckduckgo.com for fallback results.
|
|
4
5
|
* @tool
|
|
5
6
|
* @effect none
|
|
6
7
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgoSearch.d.ts","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"duckduckgoSearch.d.ts","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"AAoCA;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,oBAAoB;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;;;;;;;;iBAgF6B,MAAM;mBAAS,MAAM;sBAAY,MAAM;;;kBAahC,MAAM;kBAAQ,MAAM;;;;;;;;;GAkDxD"}
|
|
@@ -1,11 +1,14 @@
|
|
|
1
|
+
import { parse } from "node-html-parser";
|
|
1
2
|
import { getExtensionContext } from "../../index.js";
|
|
2
3
|
import { DEFAULT_TIMEOUT_MS, DEFAULT_MAX_HTTP_BYTES, DEFAULT_HTTP_USER_AGENT, DEFAULT_BLOCKED_CIDRS, } from "./defaults.js";
|
|
3
4
|
import { validateUrl, createTaggedError } from "@easynet/agent-tool";
|
|
4
5
|
import { overrideWithConfig } from "../util/overrideWithConfig.js";
|
|
5
6
|
const DUCKDUCKGO_API = "https://api.duckduckgo.com/";
|
|
7
|
+
const DUCKDUCKGO_HTML = "https://html.duckduckgo.com/html/";
|
|
6
8
|
/**
|
|
7
9
|
* Search DuckDuckGo via the Instant Answer API (no API key).
|
|
8
|
-
*
|
|
10
|
+
* When the API returns no web results, falls back to HTML search (html.duckduckgo.com).
|
|
11
|
+
* Requires api.duckduckgo.com in allowedHosts; add html.duckduckgo.com for fallback results.
|
|
9
12
|
* @tool
|
|
10
13
|
* @effect none
|
|
11
14
|
*/
|
|
@@ -89,6 +92,14 @@ export async function duckduckgoSearch(args) {
|
|
|
89
92
|
}
|
|
90
93
|
}
|
|
91
94
|
}
|
|
95
|
+
// Step 3b: If Instant Answer API returned no web results, fall back to HTML search.
|
|
96
|
+
// Requires html.duckduckgo.com in allowedHosts.
|
|
97
|
+
if (results.length === 0 && relatedTopics.length === 0) {
|
|
98
|
+
const htmlResults = await fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, userAgent, cfg.allowedHosts ?? defaults.allowedHosts, cfg.blockedHosts ?? defaults.blockedHosts, blockedCidrs);
|
|
99
|
+
if (htmlResults.length > 0) {
|
|
100
|
+
results.push(...htmlResults);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
92
103
|
const result = {
|
|
93
104
|
query,
|
|
94
105
|
abstract: raw.Abstract ?? raw.AbstractText ?? undefined,
|
|
@@ -111,4 +122,58 @@ export async function duckduckgoSearch(args) {
|
|
|
111
122
|
],
|
|
112
123
|
};
|
|
113
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* Fetch DuckDuckGo HTML search (html.duckduckgo.com) and parse result links/snippets.
|
|
127
|
+
* Used when the Instant Answer API returns no results.
|
|
128
|
+
*/
|
|
129
|
+
async function fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, userAgent, allowedHosts, blockedHosts, blockedCidrs) {
|
|
130
|
+
const htmlUrl = `${DUCKDUCKGO_HTML}?q=${encodeURIComponent(query)}`;
|
|
131
|
+
try {
|
|
132
|
+
await validateUrl(htmlUrl, {
|
|
133
|
+
allowedHosts,
|
|
134
|
+
blockedHosts,
|
|
135
|
+
blockedCidrs,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
return [];
|
|
140
|
+
}
|
|
141
|
+
const controller = new AbortController();
|
|
142
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
143
|
+
let response;
|
|
144
|
+
try {
|
|
145
|
+
response = await fetch(htmlUrl, {
|
|
146
|
+
method: "GET",
|
|
147
|
+
headers: { "User-Agent": userAgent },
|
|
148
|
+
signal: controller.signal,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
clearTimeout(timer);
|
|
153
|
+
return [];
|
|
154
|
+
}
|
|
155
|
+
clearTimeout(timer);
|
|
156
|
+
if (!response.ok)
|
|
157
|
+
return [];
|
|
158
|
+
const text = await response.text();
|
|
159
|
+
const bytes = Buffer.byteLength(text, "utf-8");
|
|
160
|
+
if (bytes > maxBytes)
|
|
161
|
+
return [];
|
|
162
|
+
const root = parse(text, { comment: false });
|
|
163
|
+
const out = [];
|
|
164
|
+
// DuckDuckGo HTML: result blocks have .result, link in .result__a, snippet in .result__snippet
|
|
165
|
+
const resultBlocks = root.querySelectorAll(".result");
|
|
166
|
+
for (let i = 0; i < Math.min(resultBlocks.length, maxResults); i++) {
|
|
167
|
+
const block = resultBlocks[i];
|
|
168
|
+
const linkEl = block?.querySelector(".result__a");
|
|
169
|
+
const href = linkEl?.getAttribute("href");
|
|
170
|
+
const title = (linkEl?.textContent ?? "").trim().replace(/\s+/g, " ");
|
|
171
|
+
const snippetEl = block?.querySelector(".result__snippet");
|
|
172
|
+
const snippet = (snippetEl?.textContent ?? "").trim().replace(/\s+/g, " ").slice(0, 500);
|
|
173
|
+
if (href && title) {
|
|
174
|
+
out.push({ url: href, title, snippet: snippet || undefined });
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return out;
|
|
178
|
+
}
|
|
114
179
|
//# sourceMappingURL=duckduckgoSearch.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgoSearch.js","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,MAAM,cAAc,GAAG,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"duckduckgoSearch.js","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,MAAM,cAAc,GAAG,6BAA6B,CAAC;AACrD,MAAM,eAAe,GAAG,mCAAmC,CAAC;AAwB5D;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAQtC;IACC,+EAA+E;IAC/E,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,SAAS,EAAE,uBAAuB;QAClC,YAAY,EAAE,qBAAqB;QACnC,YAAY,EAAE,EAAc;QAC5B,YAAY,EAAE,EAAc;KAC7B,CAAC;IACF,MAAM,GAAG,GAAG,mBAAmB,EAAE,CAAC;IAClC,MAAM,GAAG,GAAG,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IAC7C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,iBAAiB,CAAC,oBAAoB,EAAE,yCAAyC,EAAE,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAC;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;IACnD,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;IAChG,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAC;IAEtD,8DAA8D;IAC9D,MAAM,GAAG,GAAG,GAAG,cAAc,MAAM,kBAAkB,CAAC,KAAK,CAAC,cAAc,CAAC;IAC3E,MAAM,WAAW,CAAC,GAAG,EAAE;QACrB,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY;KACb,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;YACpC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,iBAAiB,CACrB,cAAc,EACd,qCAAqC,SAAS,IAAI,EAClD,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACJ,CAAC;QACD,MAAM,iBAAiB,CACrB,gBAAgB,EAChB,6BAA6B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/E,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;IACD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,MAAM,iBAAiB,CACrB,gBAAgB,EAChB,uBAAuB,KAAK,2BAA2B,QAAQ,QAAQ,EACvE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAClC,CAAC;IACJ,CAAC;IAED,0DAA0D;IAC1D,IAAI,GAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CACrB,gBAAgB,EAChB,kCAAkC,EAClC,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC3C,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAA4D,EAAE,CAAC;IAC5E,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,CAAC,CAAC,QAAQ;oBACf,KAAK,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ;oBAC3B,OAAO,EAAE,CAAC,CAAC,IAAI;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAA0C,EAAE,CAAC;IAChE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;YAChC,IAAI,IAAI,EAAE,CAAC;gBACT,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,oFAAoF;IACpF,gDAAgD;IAChD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,MAAM,yBAAyB,CACjD,KAAK,EACL,UAAU,EACV,SAAS,EACT,QAAQ,EACR,SAAS,EACT,GAAG,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,EACzC,GAAG,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,EACzC,YAAY,CACb,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG;QACb,KAAK;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,IAAI,SAAS;QACvD,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS;QACzC,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,SAAS;QAC/C,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,SAAS;QACjC,OAAO;QACP,aAAa;KACd,CAAC;IAEF,iDAAiD;IACjD,OAAO;QACL,MAAM;QACN,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,GAAG;gBACR,OAAO,EAAE,uBAAuB,KAAK,OAAO,OAAO,CAAC,MAAM,UAAU;gBACpE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;SACF;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,yBAAyB,CACtC,KAAa,EACb,UAAkB,EAClB,SAAiB,EACjB,QAAgB,EAChB,SAAiB,EACjB,YAAsB,EACtB,YAAsB,EACtB,YAAsB;IAEtB,MAAM,OAAO,GAAG,GAAG,eAAe,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,OAAO,EAAE;YACzB,YAAY;YACZ,YAAY;YACZ,YAAY;SACb,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC9B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;YACpC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,GAAG,GAA4D,EAAE,CAAC;IACxE,+FAA+F;IAC/F,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,KAAK,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,CAAC,SAAS,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACzF,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;YAClB,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@easynet/agent-tool-buildin",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.13",
|
|
4
4
|
"description": "Tools extension for @easynet/agent-tool: FS, HTTP, util, security (sandbox, SSRF). Same contract as any other extension.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|