@agentforge/tools 0.16.75 → 0.16.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +260 -48
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +56 -27
- package/dist/index.d.ts +56 -27
- package/dist/index.js +255 -48
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var core = require('@agentforge/core');
|
|
4
|
-
var
|
|
4
|
+
var promises = require('dns/promises');
|
|
5
|
+
var net = require('net');
|
|
6
|
+
var axios13 = require('axios');
|
|
5
7
|
var zod = require('zod');
|
|
6
8
|
var cheerio = require('cheerio');
|
|
7
9
|
var webApi = require('@slack/web-api');
|
|
@@ -37,7 +39,8 @@ function _interopNamespace(e) {
|
|
|
37
39
|
return Object.freeze(n);
|
|
38
40
|
}
|
|
39
41
|
|
|
40
|
-
var
|
|
42
|
+
var net__default = /*#__PURE__*/_interopDefault(net);
|
|
43
|
+
var axios13__default = /*#__PURE__*/_interopDefault(axios13);
|
|
41
44
|
var cheerio__namespace = /*#__PURE__*/_interopNamespace(cheerio);
|
|
42
45
|
var neo4j__default = /*#__PURE__*/_interopDefault(neo4j);
|
|
43
46
|
var path7__namespace = /*#__PURE__*/_interopNamespace(path7);
|
|
@@ -48,6 +51,204 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
48
51
|
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
49
52
|
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
50
53
|
});
|
|
54
|
+
var DEFAULT_MAX_REDIRECTS = 5;
|
|
55
|
+
var REDIRECT_STATUSES = /* @__PURE__ */ new Set([301, 302, 303, 307, 308]);
|
|
56
|
+
var IPV4_MAPPED_NETWORK = "::ffff:0:0";
|
|
57
|
+
var SENSITIVE_REDIRECT_HEADERS = /* @__PURE__ */ new Set(["authorization", "cookie", "proxy-authorization"]);
|
|
58
|
+
var DEFAULT_DESTINATION_POLICY = Object.freeze({
|
|
59
|
+
allowLocalhost: false,
|
|
60
|
+
allowPrivateNetwork: false,
|
|
61
|
+
allowLinkLocal: false,
|
|
62
|
+
allowMetadata: false,
|
|
63
|
+
allowRedirects: true,
|
|
64
|
+
maxRedirects: DEFAULT_MAX_REDIRECTS
|
|
65
|
+
});
|
|
66
|
+
var DestinationPolicyError = class extends Error {
|
|
67
|
+
constructor(url, reason, message) {
|
|
68
|
+
super(message);
|
|
69
|
+
this.url = url;
|
|
70
|
+
this.reason = reason;
|
|
71
|
+
this.name = "DestinationPolicyError";
|
|
72
|
+
}
|
|
73
|
+
code = "WEB_DESTINATION_BLOCKED";
|
|
74
|
+
};
|
|
75
|
+
function ipv4ToNumber(address) {
|
|
76
|
+
const octets = address.split(".").map(Number);
|
|
77
|
+
if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) return void 0;
|
|
78
|
+
return ((octets[0] * 256 + octets[1]) * 256 + octets[2]) * 256 + octets[3] >>> 0;
|
|
79
|
+
}
|
|
80
|
+
function ipv4InCidr(address, network, prefix) {
|
|
81
|
+
const value = ipv4ToNumber(address);
|
|
82
|
+
const networkValue = ipv4ToNumber(network);
|
|
83
|
+
if (value === void 0 || networkValue === void 0) return false;
|
|
84
|
+
const mask = prefix === 0 ? 0 : 4294967295 << 32 - prefix >>> 0;
|
|
85
|
+
return (value & mask) === (networkValue & mask);
|
|
86
|
+
}
|
|
87
|
+
function expandIpv6(address) {
|
|
88
|
+
const normalized = address.toLowerCase().split("%")[0];
|
|
89
|
+
let input = normalized;
|
|
90
|
+
const ipv4Start = normalized.lastIndexOf(":") + 1;
|
|
91
|
+
const ipv4 = normalized.slice(ipv4Start);
|
|
92
|
+
if (net__default.default.isIP(ipv4) === 4) {
|
|
93
|
+
const value = ipv4ToNumber(ipv4);
|
|
94
|
+
if (value === void 0) return void 0;
|
|
95
|
+
input = `${normalized.slice(0, ipv4Start)}${(value >>> 16).toString(16)}:${(value & 65535).toString(16)}`;
|
|
96
|
+
}
|
|
97
|
+
const halves = input.split("::");
|
|
98
|
+
if (halves.length > 2) return void 0;
|
|
99
|
+
const left = halves[0] ? halves[0].split(":").filter(Boolean) : [];
|
|
100
|
+
const right = halves.length === 2 && halves[1] ? halves[1].split(":").filter(Boolean) : [];
|
|
101
|
+
const missing = 8 - left.length - right.length;
|
|
102
|
+
if (missing < 0 || halves.length === 1 && missing !== 0) return void 0;
|
|
103
|
+
const groups = [...left, ...Array.from({ length: missing }, () => "0"), ...right];
|
|
104
|
+
if (groups.length !== 8 || groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))) return void 0;
|
|
105
|
+
return groups.reduce((value, group) => value << 16n | BigInt(parseInt(group, 16)), 0n);
|
|
106
|
+
}
|
|
107
|
+
function ipv6InCidr(address, network, prefix) {
|
|
108
|
+
return prefix === 0 || address >> BigInt(128 - prefix) === network >> BigInt(128 - prefix);
|
|
109
|
+
}
|
|
110
|
+
function ipv6ToIpv4(address) {
|
|
111
|
+
const mappedNetwork = expandIpv6(IPV4_MAPPED_NETWORK);
|
|
112
|
+
if (mappedNetwork === void 0 || !ipv6InCidr(address, mappedNetwork, 96)) return void 0;
|
|
113
|
+
const value = Number(address & 0xffffffffn) >>> 0;
|
|
114
|
+
return [value >>> 24, value >>> 16 & 255, value >>> 8 & 255, value & 255].join(".");
|
|
115
|
+
}
|
|
116
|
+
function classifyIp(address) {
|
|
117
|
+
const normalized = address.replace(/^\[|\]$/g, "").toLowerCase();
|
|
118
|
+
const version = net__default.default.isIP(normalized);
|
|
119
|
+
if (version === 4) {
|
|
120
|
+
if (normalized === "169.254.169.254") return "metadata";
|
|
121
|
+
if (ipv4InCidr(normalized, "127.0.0.0", 8)) return "localhost";
|
|
122
|
+
if (ipv4InCidr(normalized, "169.254.0.0", 16)) return "link-local";
|
|
123
|
+
if (ipv4InCidr(normalized, "10.0.0.0", 8) || ipv4InCidr(normalized, "172.16.0.0", 12) || ipv4InCidr(normalized, "192.168.0.0", 16) || ipv4InCidr(normalized, "100.64.0.0", 10)) return "private-network";
|
|
124
|
+
return void 0;
|
|
125
|
+
}
|
|
126
|
+
if (version !== 6) return void 0;
|
|
127
|
+
const value = expandIpv6(normalized);
|
|
128
|
+
if (value === void 0) return void 0;
|
|
129
|
+
const mappedIpv4 = ipv6ToIpv4(value);
|
|
130
|
+
if (mappedIpv4) return classifyIp(mappedIpv4);
|
|
131
|
+
if (value === expandIpv6("fd00:ec2::254")) return "metadata";
|
|
132
|
+
if (value === expandIpv6("::1")) return "localhost";
|
|
133
|
+
if (ipv6InCidr(value, expandIpv6("fe80::"), 10)) return "link-local";
|
|
134
|
+
if (ipv6InCidr(value, expandIpv6("fc00::"), 7)) return "private-network";
|
|
135
|
+
return void 0;
|
|
136
|
+
}
|
|
137
|
+
function classifyHostname(hostname) {
|
|
138
|
+
const normalized = hostname.toLowerCase().replace(/\.$/, "");
|
|
139
|
+
if (normalized === "localhost" || normalized.endsWith(".localhost") || normalized === "localhost.localdomain") return "localhost";
|
|
140
|
+
if (normalized === "metadata" || normalized === "metadata.google.internal" || normalized === "instance-data") return "metadata";
|
|
141
|
+
return classifyIp(normalized);
|
|
142
|
+
}
|
|
143
|
+
function policyAllows(reason, policy) {
|
|
144
|
+
if (reason === "localhost") return policy.allowLocalhost === true;
|
|
145
|
+
if (reason === "metadata") return policy.allowMetadata === true;
|
|
146
|
+
if (reason === "private-network") return policy.allowPrivateNetwork === true;
|
|
147
|
+
if (reason === "link-local") return policy.allowLinkLocal === true;
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
async function resolveDestinationAddresses(hostname) {
|
|
151
|
+
try {
|
|
152
|
+
const addresses = await promises.lookup(hostname, { all: true, verbatim: true });
|
|
153
|
+
return addresses.map(({ address, family }) => ({ address, family }));
|
|
154
|
+
} catch (error) {
|
|
155
|
+
throw new DestinationPolicyError(hostname, "dns-resolution-failed", `Destination policy could not resolve hostname "${hostname}" before making a request: ${error instanceof Error ? error.message : String(error)}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function blockDestination(url, reason) {
|
|
159
|
+
throw new DestinationPolicyError(url, reason, `Destination blocked by web egress policy (${reason}): ${url}`);
|
|
160
|
+
}
|
|
161
|
+
async function validateDestinationAllowed(url, policy) {
|
|
162
|
+
const effectivePolicy = { ...DEFAULT_DESTINATION_POLICY, ...policy };
|
|
163
|
+
let parsed;
|
|
164
|
+
try {
|
|
165
|
+
parsed = new URL(url);
|
|
166
|
+
} catch {
|
|
167
|
+
blockDestination(url, "unsupported-protocol");
|
|
168
|
+
}
|
|
169
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") blockDestination(url, "unsupported-protocol");
|
|
170
|
+
const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase();
|
|
171
|
+
const directReason = classifyHostname(hostname);
|
|
172
|
+
if (directReason && !policyAllows(directReason, effectivePolicy)) blockDestination(url, directReason);
|
|
173
|
+
const version = net__default.default.isIP(hostname);
|
|
174
|
+
if (version !== 0) return [{ address: hostname, family: version }];
|
|
175
|
+
const addresses = await resolveDestinationAddresses(hostname);
|
|
176
|
+
for (const { address } of addresses) {
|
|
177
|
+
const reason = classifyIp(address);
|
|
178
|
+
if (reason && !policyAllows(reason, effectivePolicy)) blockDestination(url, reason);
|
|
179
|
+
}
|
|
180
|
+
return addresses;
|
|
181
|
+
}
|
|
182
|
+
async function assertDestinationAllowed(url, policy = {}) {
|
|
183
|
+
await validateDestinationAllowed(url, policy);
|
|
184
|
+
}
|
|
185
|
+
function isRedirectStatus(status) {
|
|
186
|
+
return REDIRECT_STATUSES.has(status);
|
|
187
|
+
}
|
|
188
|
+
function responseLocation(response) {
|
|
189
|
+
const headers = response.headers;
|
|
190
|
+
return headers.location ?? headers.Location;
|
|
191
|
+
}
|
|
192
|
+
function stripSensitiveRedirectHeaders(headers) {
|
|
193
|
+
if (!headers) return headers;
|
|
194
|
+
const sanitizedHeaders = { ...headers };
|
|
195
|
+
for (const headerName of Object.keys(sanitizedHeaders)) {
|
|
196
|
+
if (SENSITIVE_REDIRECT_HEADERS.has(headerName.toLowerCase())) delete sanitizedHeaders[headerName];
|
|
197
|
+
}
|
|
198
|
+
return sanitizedHeaders;
|
|
199
|
+
}
|
|
200
|
+
function createPinnedLookup(addresses) {
|
|
201
|
+
return (_hostname, options, callback) => {
|
|
202
|
+
const requestedFamily = "family" in options && (options.family === 4 || options.family === 6) ? options.family : void 0;
|
|
203
|
+
const candidates = requestedFamily ? addresses.filter(({ family: family2 }) => family2 === requestedFamily) : addresses;
|
|
204
|
+
if (candidates.length === 0) {
|
|
205
|
+
callback(new Error(`No validated destination address matches the requested address family: ${requestedFamily}`), "", void 0);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if ("all" in options && options.all === true) {
|
|
209
|
+
callback(null, candidates, void 0);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const [{ address, family }] = candidates;
|
|
213
|
+
callback(null, address, family);
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
async function requestWithDestinationPolicy(config, policy = {}) {
|
|
217
|
+
if (!config.url) throw new Error("A request URL is required for destination policy enforcement");
|
|
218
|
+
const effectivePolicy = { ...DEFAULT_DESTINATION_POLICY, ...policy };
|
|
219
|
+
const maxRedirects = effectivePolicy.maxRedirects;
|
|
220
|
+
if (!Number.isInteger(maxRedirects) || maxRedirects < 0) throw new Error("maxRedirects must be a non-negative integer");
|
|
221
|
+
let currentUrl = config.url;
|
|
222
|
+
let currentConfig = {
|
|
223
|
+
...config,
|
|
224
|
+
url: currentUrl,
|
|
225
|
+
maxRedirects: 0,
|
|
226
|
+
validateStatus: (status) => isRedirectStatus(status) || (config.validateStatus ? config.validateStatus(status) : status >= 200 && status < 300)
|
|
227
|
+
};
|
|
228
|
+
for (let redirectCount = 0; ; redirectCount += 1) {
|
|
229
|
+
const validatedAddresses = await validateDestinationAllowed(currentUrl, effectivePolicy);
|
|
230
|
+
currentConfig.lookup = createPinnedLookup(validatedAddresses);
|
|
231
|
+
const response = await axios13__default.default(currentConfig);
|
|
232
|
+
if (!isRedirectStatus(response.status)) return response;
|
|
233
|
+
if (effectivePolicy.allowRedirects === false) blockDestination(currentUrl, "redirect");
|
|
234
|
+
if (redirectCount >= maxRedirects) throw new DestinationPolicyError(currentUrl, "redirect", `Maximum redirect count (${maxRedirects}) exceeded for ${currentUrl}`);
|
|
235
|
+
const location = responseLocation(response);
|
|
236
|
+
if (!location) throw new DestinationPolicyError(currentUrl, "redirect", `Redirect response did not include a Location header: ${currentUrl}`);
|
|
237
|
+
const previousUrl = currentUrl;
|
|
238
|
+
try {
|
|
239
|
+
currentUrl = new URL(location, currentUrl).toString();
|
|
240
|
+
} catch {
|
|
241
|
+
throw new DestinationPolicyError(currentUrl, "redirect", `Redirect response included an invalid Location header: ${location}`);
|
|
242
|
+
}
|
|
243
|
+
const method = String(currentConfig.method ?? "GET").toUpperCase();
|
|
244
|
+
const nextMethod = response.status === 303 || (response.status === 301 || response.status === 302) && method === "POST" ? "GET" : method;
|
|
245
|
+
currentConfig = { ...currentConfig, url: currentUrl, method: nextMethod };
|
|
246
|
+
if (new URL(previousUrl).origin !== new URL(currentUrl).origin) {
|
|
247
|
+
currentConfig.headers = stripSensitiveRedirectHeaders(currentConfig.headers);
|
|
248
|
+
}
|
|
249
|
+
if (nextMethod === "GET" || nextMethod === "HEAD") delete currentConfig.data;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
51
252
|
var HttpMethod = zod.z.enum(["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"]);
|
|
52
253
|
var httpRequestSchema = zod.z.object({
|
|
53
254
|
url: zod.z.string().url().describe("The URL to make the request to"),
|
|
@@ -69,8 +270,8 @@ var httpPostSchema = zod.z.object({
|
|
|
69
270
|
});
|
|
70
271
|
|
|
71
272
|
// src/web/http/tools/http-client.ts
|
|
72
|
-
function createHttpClientTool(defaultTimeout = 3e4, defaultHeaders = {}) {
|
|
73
|
-
return core.toolBuilder().name("http-client").description("Make HTTP requests to web APIs and services. Supports GET, POST, PUT, DELETE, PATCH methods with custom headers and body.").category(core.ToolCategory.WEB).tags(["http", "api", "request", "web"]).schema(httpRequestSchema).implement(async (input) => {
|
|
273
|
+
function createHttpClientTool(defaultTimeout = 3e4, defaultHeaders = {}, destinationPolicy = {}) {
|
|
274
|
+
return core.toolBuilder().name("http-client").description("Make policy-checked HTTP requests to web APIs and services. Supports GET, POST, PUT, DELETE, PATCH methods with custom headers and body. Local, metadata, link-local, and private-network destinations are blocked by default.").category(core.ToolCategory.WEB).tags(["http", "api", "request", "web"]).schema(httpRequestSchema).implement(async (input) => {
|
|
74
275
|
const config = {
|
|
75
276
|
method: input.method,
|
|
76
277
|
url: input.url,
|
|
@@ -81,37 +282,42 @@ function createHttpClientTool(defaultTimeout = 3e4, defaultHeaders = {}) {
|
|
|
81
282
|
validateStatus: () => true
|
|
82
283
|
// Don't throw on any status code
|
|
83
284
|
};
|
|
84
|
-
const response = await
|
|
285
|
+
const response = await requestWithDestinationPolicy(config, destinationPolicy);
|
|
85
286
|
return {
|
|
86
287
|
status: response.status,
|
|
87
288
|
statusText: response.statusText,
|
|
88
289
|
headers: response.headers,
|
|
89
290
|
data: response.data,
|
|
90
|
-
url: input.url,
|
|
291
|
+
url: response.config.url ?? input.url,
|
|
91
292
|
method: input.method ?? "GET"
|
|
92
293
|
};
|
|
93
294
|
}).build();
|
|
94
295
|
}
|
|
95
|
-
function createHttpGetTool(defaultTimeout = 3e4, defaultHeaders = {}) {
|
|
96
|
-
return core.toolBuilder().name("http-get").description("Make a
|
|
97
|
-
const response = await
|
|
296
|
+
function createHttpGetTool(defaultTimeout = 3e4, defaultHeaders = {}, destinationPolicy = {}) {
|
|
297
|
+
return core.toolBuilder().name("http-get").description("Make a policy-checked HTTP GET request to a URL and return the response data. Local, metadata, link-local, and private-network destinations are blocked by default.").category(core.ToolCategory.WEB).tags(["http", "get", "fetch", "web"]).schema(httpGetSchema).implement(async (input) => {
|
|
298
|
+
const response = await requestWithDestinationPolicy({
|
|
299
|
+
method: "GET",
|
|
300
|
+
url: input.url,
|
|
98
301
|
headers: { ...defaultHeaders, ...input.headers },
|
|
99
302
|
params: input.params,
|
|
100
303
|
timeout: defaultTimeout
|
|
101
|
-
});
|
|
304
|
+
}, destinationPolicy);
|
|
102
305
|
return response.data;
|
|
103
306
|
}).build();
|
|
104
307
|
}
|
|
105
|
-
function createHttpPostTool(defaultTimeout = 3e4, defaultHeaders = {}) {
|
|
106
|
-
return core.toolBuilder().name("http-post").description("Make a
|
|
107
|
-
const response = await
|
|
308
|
+
function createHttpPostTool(defaultTimeout = 3e4, defaultHeaders = {}, destinationPolicy = {}) {
|
|
309
|
+
return core.toolBuilder().name("http-post").description("Make a policy-checked HTTP POST request with JSON body and return the response data. Local, metadata, link-local, and private-network destinations are blocked by default.").category(core.ToolCategory.WEB).tags(["http", "post", "api", "web"]).schema(httpPostSchema).implement(async (input) => {
|
|
310
|
+
const response = await requestWithDestinationPolicy({
|
|
311
|
+
method: "POST",
|
|
312
|
+
url: input.url,
|
|
313
|
+
data: input.body,
|
|
108
314
|
headers: {
|
|
109
315
|
"Content-Type": "application/json",
|
|
110
316
|
...defaultHeaders,
|
|
111
317
|
...input.headers
|
|
112
318
|
},
|
|
113
319
|
timeout: defaultTimeout
|
|
114
|
-
});
|
|
320
|
+
}, destinationPolicy);
|
|
115
321
|
return response.data;
|
|
116
322
|
}).build();
|
|
117
323
|
}
|
|
@@ -122,11 +328,11 @@ var httpGet = createHttpGetTool();
|
|
|
122
328
|
var httpPost = createHttpPostTool();
|
|
123
329
|
var httpTools = [httpClient, httpGet, httpPost];
|
|
124
330
|
function createHttpTools(config = {}) {
|
|
125
|
-
const { defaultTimeout = 3e4, defaultHeaders = {} } = config;
|
|
331
|
+
const { defaultTimeout = 3e4, defaultHeaders = {}, destinationPolicy = {} } = config;
|
|
126
332
|
return [
|
|
127
|
-
createHttpClientTool(defaultTimeout, defaultHeaders),
|
|
128
|
-
createHttpGetTool(defaultTimeout, defaultHeaders),
|
|
129
|
-
createHttpPostTool(defaultTimeout, defaultHeaders)
|
|
333
|
+
createHttpClientTool(defaultTimeout, defaultHeaders, destinationPolicy),
|
|
334
|
+
createHttpGetTool(defaultTimeout, defaultHeaders, destinationPolicy),
|
|
335
|
+
createHttpPostTool(defaultTimeout, defaultHeaders, destinationPolicy)
|
|
130
336
|
];
|
|
131
337
|
}
|
|
132
338
|
var webScraperSchema = zod.z.object({
|
|
@@ -139,18 +345,20 @@ var webScraperSchema = zod.z.object({
|
|
|
139
345
|
extractMetadata: zod.z.boolean().default(false).describe("Extract meta tags (title, description, etc.)"),
|
|
140
346
|
timeout: zod.z.number().default(3e4).describe("Request timeout in milliseconds")
|
|
141
347
|
});
|
|
142
|
-
function createWebScraperTool(defaultTimeout = 3e4, userAgent = "Mozilla/5.0 (compatible; AgentForge/1.0; +https://agentforge.dev)") {
|
|
143
|
-
return core.toolBuilder().name("web-scraper").description("Scrape and extract data from web pages.
|
|
144
|
-
const response = await
|
|
348
|
+
function createWebScraperTool(defaultTimeout = 3e4, userAgent = "Mozilla/5.0 (compatible; AgentForge/1.0; +https://agentforge.dev)", destinationPolicy = {}) {
|
|
349
|
+
return core.toolBuilder().name("web-scraper").description("Scrape and extract data from policy-checked web pages. Local, metadata, link-local, and private-network destinations are blocked by default. Supports text, HTML, links, images, and CSS selectors.").category(core.ToolCategory.WEB).tags(["scraper", "web", "html", "extract", "parse"]).schema(webScraperSchema).implement(async (input) => {
|
|
350
|
+
const response = await requestWithDestinationPolicy({
|
|
351
|
+
method: "GET",
|
|
352
|
+
url: input.url,
|
|
145
353
|
timeout: input.timeout || defaultTimeout,
|
|
146
354
|
headers: {
|
|
147
355
|
"User-Agent": userAgent
|
|
148
356
|
}
|
|
149
|
-
});
|
|
357
|
+
}, destinationPolicy);
|
|
150
358
|
const html = response.data;
|
|
151
359
|
const $ = cheerio__namespace.load(html);
|
|
152
360
|
const result = {
|
|
153
|
-
url: input.url
|
|
361
|
+
url: response.config.url ?? input.url
|
|
154
362
|
};
|
|
155
363
|
const $selected = input.selector ? $(input.selector) : $("body");
|
|
156
364
|
if (input.extractText) {
|
|
@@ -215,8 +423,8 @@ function createWebScraperTool(defaultTimeout = 3e4, userAgent = "Mozilla/5.0 (co
|
|
|
215
423
|
var webScraper = createWebScraperTool();
|
|
216
424
|
var scraperTools = [webScraper];
|
|
217
425
|
function createScraperTools(config = {}) {
|
|
218
|
-
const { defaultTimeout = 3e4, userAgent = "Mozilla/5.0 (compatible; AgentForge/1.0; +https://agentforge.dev)" } = config;
|
|
219
|
-
return [createWebScraperTool(defaultTimeout, userAgent)];
|
|
426
|
+
const { defaultTimeout = 3e4, userAgent = "Mozilla/5.0 (compatible; AgentForge/1.0; +https://agentforge.dev)", destinationPolicy = {} } = config;
|
|
427
|
+
return [createWebScraperTool(defaultTimeout, userAgent, destinationPolicy)];
|
|
220
428
|
}
|
|
221
429
|
var htmlParserSchema = zod.z.object({
|
|
222
430
|
html: zod.z.string().describe("The HTML content to parse"),
|
|
@@ -531,7 +739,7 @@ var DuckDuckGoProvider = class {
|
|
|
531
739
|
async search(query, maxResults, timeout = DEFAULT_TIMEOUT) {
|
|
532
740
|
return retryWithBackoff(async () => {
|
|
533
741
|
try {
|
|
534
|
-
const response = await
|
|
742
|
+
const response = await axios13__default.default.get(
|
|
535
743
|
"https://api.duckduckgo.com/",
|
|
536
744
|
{
|
|
537
745
|
params: {
|
|
@@ -635,7 +843,7 @@ var SerperProvider = class {
|
|
|
635
843
|
}
|
|
636
844
|
return retryWithBackoff(async () => {
|
|
637
845
|
try {
|
|
638
|
-
const response = await
|
|
846
|
+
const response = await axios13__default.default.post(
|
|
639
847
|
"https://google.serper.dev/search",
|
|
640
848
|
{
|
|
641
849
|
q: query,
|
|
@@ -1069,7 +1277,7 @@ function createSearchConfluenceTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1069
1277
|
logger19.info("search-confluence called", { query, limit });
|
|
1070
1278
|
try {
|
|
1071
1279
|
const { ATLASSIAN_SITE_URL } = getAuth();
|
|
1072
|
-
const response = await
|
|
1280
|
+
const response = await axios13__default.default.get(`${ATLASSIAN_SITE_URL}/wiki/rest/api/content/search`, {
|
|
1073
1281
|
headers: {
|
|
1074
1282
|
Authorization: getAuthHeader2(),
|
|
1075
1283
|
Accept: "application/json"
|
|
@@ -1131,7 +1339,7 @@ function createGetConfluencePageTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1131
1339
|
logger19.info("get-confluence-page called", { page_id });
|
|
1132
1340
|
try {
|
|
1133
1341
|
const { ATLASSIAN_SITE_URL } = getAuth();
|
|
1134
|
-
const response = await
|
|
1342
|
+
const response = await axios13__default.default.get(`${ATLASSIAN_SITE_URL}/wiki/rest/api/content/${page_id}`, {
|
|
1135
1343
|
headers: {
|
|
1136
1344
|
Authorization: getAuthHeader2(),
|
|
1137
1345
|
Accept: "application/json"
|
|
@@ -1182,7 +1390,7 @@ function createListConfluenceSpacesTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1182
1390
|
logger19.info("list-confluence-spaces called", { limit });
|
|
1183
1391
|
try {
|
|
1184
1392
|
const { ATLASSIAN_SITE_URL } = getAuth();
|
|
1185
|
-
const response = await
|
|
1393
|
+
const response = await axios13__default.default.get(`${ATLASSIAN_SITE_URL}/wiki/rest/api/space`, {
|
|
1186
1394
|
headers: {
|
|
1187
1395
|
Authorization: getAuthHeader2(),
|
|
1188
1396
|
Accept: "application/json"
|
|
@@ -1228,7 +1436,7 @@ function createGetSpacePagesTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1228
1436
|
logger19.info("get-space-pages called", { space_key, limit });
|
|
1229
1437
|
try {
|
|
1230
1438
|
const { ATLASSIAN_SITE_URL } = getAuth();
|
|
1231
|
-
const response = await
|
|
1439
|
+
const response = await axios13__default.default.get(`${ATLASSIAN_SITE_URL}/wiki/rest/api/content`, {
|
|
1232
1440
|
headers: {
|
|
1233
1441
|
Authorization: getAuthHeader2(),
|
|
1234
1442
|
Accept: "application/json"
|
|
@@ -1302,7 +1510,7 @@ function createCreateConfluencePageTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1302
1510
|
if (parent_page_id) {
|
|
1303
1511
|
pageData.ancestors = [{ id: parent_page_id }];
|
|
1304
1512
|
}
|
|
1305
|
-
const response = await
|
|
1513
|
+
const response = await axios13__default.default.post(
|
|
1306
1514
|
`${ATLASSIAN_SITE_URL}/wiki/rest/api/content`,
|
|
1307
1515
|
pageData,
|
|
1308
1516
|
{
|
|
@@ -1350,7 +1558,7 @@ function createUpdateConfluencePageTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1350
1558
|
logger19.info("update-confluence-page called", { page_id, title });
|
|
1351
1559
|
try {
|
|
1352
1560
|
const { ATLASSIAN_SITE_URL } = getAuth();
|
|
1353
|
-
const getResponse = await
|
|
1561
|
+
const getResponse = await axios13__default.default.get(
|
|
1354
1562
|
`${ATLASSIAN_SITE_URL}/wiki/rest/api/content/${page_id}`,
|
|
1355
1563
|
{
|
|
1356
1564
|
headers: {
|
|
@@ -1360,7 +1568,7 @@ function createUpdateConfluencePageTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1360
1568
|
}
|
|
1361
1569
|
);
|
|
1362
1570
|
const currentVersion = getResponse.data.version.number;
|
|
1363
|
-
const updateResponse = await
|
|
1571
|
+
const updateResponse = await axios13__default.default.put(
|
|
1364
1572
|
`${ATLASSIAN_SITE_URL}/wiki/rest/api/content/${page_id}`,
|
|
1365
1573
|
{
|
|
1366
1574
|
type: "page",
|
|
@@ -1421,7 +1629,7 @@ function createArchiveConfluencePageTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1421
1629
|
});
|
|
1422
1630
|
try {
|
|
1423
1631
|
const { ATLASSIAN_SITE_URL } = getAuth();
|
|
1424
|
-
const getResponse = await
|
|
1632
|
+
const getResponse = await axios13__default.default.get(
|
|
1425
1633
|
`${ATLASSIAN_SITE_URL}/wiki/rest/api/content/${page_id}`,
|
|
1426
1634
|
{
|
|
1427
1635
|
headers: {
|
|
@@ -1432,7 +1640,7 @@ function createArchiveConfluencePageTool(getAuth, getAuthHeader2, logger19) {
|
|
|
1432
1640
|
);
|
|
1433
1641
|
const currentVersion = getResponse.data.version.number;
|
|
1434
1642
|
const pageData = getResponse.data;
|
|
1435
|
-
await
|
|
1643
|
+
await axios13__default.default.put(
|
|
1436
1644
|
`${ATLASSIAN_SITE_URL}/wiki/rest/api/content/${page_id}`,
|
|
1437
1645
|
{
|
|
1438
1646
|
version: { number: currentVersion + 1 },
|
|
@@ -2497,7 +2705,7 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2497
2705
|
const modelToUse = model || this.defaultModel;
|
|
2498
2706
|
return retryWithBackoff2(async () => {
|
|
2499
2707
|
try {
|
|
2500
|
-
const response = await
|
|
2708
|
+
const response = await axios13__default.default.post(
|
|
2501
2709
|
`${this.baseURL}/embeddings`,
|
|
2502
2710
|
{
|
|
2503
2711
|
input: text,
|
|
@@ -2561,7 +2769,7 @@ var OpenAIEmbeddingProvider = class {
|
|
|
2561
2769
|
const modelToUse = model || this.defaultModel;
|
|
2562
2770
|
return retryWithBackoff2(async () => {
|
|
2563
2771
|
try {
|
|
2564
|
-
const response = await
|
|
2772
|
+
const response = await axios13__default.default.post(
|
|
2565
2773
|
`${this.baseURL}/embeddings`,
|
|
2566
2774
|
{
|
|
2567
2775
|
input: texts,
|
|
@@ -2641,7 +2849,7 @@ var CohereEmbeddingProvider = class {
|
|
|
2641
2849
|
const modelToUse = model || this.defaultModel;
|
|
2642
2850
|
return retryWithBackoff2(async () => {
|
|
2643
2851
|
try {
|
|
2644
|
-
const response = await
|
|
2852
|
+
const response = await axios13__default.default.post(
|
|
2645
2853
|
`${this.baseUrl}/embed`,
|
|
2646
2854
|
{
|
|
2647
2855
|
texts,
|
|
@@ -2669,7 +2877,7 @@ var CohereEmbeddingProvider = class {
|
|
|
2669
2877
|
} : void 0
|
|
2670
2878
|
};
|
|
2671
2879
|
} catch (error) {
|
|
2672
|
-
if (
|
|
2880
|
+
if (axios13__default.default.isAxiosError(error)) {
|
|
2673
2881
|
const status = error.response?.status;
|
|
2674
2882
|
const message = error.response?.data?.message || error.message;
|
|
2675
2883
|
let wrappedError;
|
|
@@ -2709,7 +2917,7 @@ var HuggingFaceEmbeddingProvider = class {
|
|
|
2709
2917
|
const modelToUse = model || this.defaultModel;
|
|
2710
2918
|
return retryWithBackoff2(async () => {
|
|
2711
2919
|
try {
|
|
2712
|
-
const response = await
|
|
2920
|
+
const response = await axios13__default.default.post(
|
|
2713
2921
|
`${this.baseUrl}/${modelToUse}`,
|
|
2714
2922
|
{
|
|
2715
2923
|
inputs: text,
|
|
@@ -2731,7 +2939,7 @@ var HuggingFaceEmbeddingProvider = class {
|
|
|
2731
2939
|
dimensions: embedding.length
|
|
2732
2940
|
};
|
|
2733
2941
|
} catch (error) {
|
|
2734
|
-
if (
|
|
2942
|
+
if (axios13__default.default.isAxiosError(error)) {
|
|
2735
2943
|
const status = error.response?.status;
|
|
2736
2944
|
const message = error.response?.data?.error || error.message;
|
|
2737
2945
|
let wrappedError;
|
|
@@ -2758,7 +2966,7 @@ var HuggingFaceEmbeddingProvider = class {
|
|
|
2758
2966
|
const modelToUse = model || this.defaultModel;
|
|
2759
2967
|
return retryWithBackoff2(async () => {
|
|
2760
2968
|
try {
|
|
2761
|
-
const response = await
|
|
2969
|
+
const response = await axios13__default.default.post(
|
|
2762
2970
|
`${this.baseUrl}/${modelToUse}`,
|
|
2763
2971
|
{
|
|
2764
2972
|
inputs: texts,
|
|
@@ -2781,7 +2989,7 @@ var HuggingFaceEmbeddingProvider = class {
|
|
|
2781
2989
|
dimensions
|
|
2782
2990
|
};
|
|
2783
2991
|
} catch (error) {
|
|
2784
|
-
if (
|
|
2992
|
+
if (axios13__default.default.isAxiosError(error)) {
|
|
2785
2993
|
const status = error.response?.status;
|
|
2786
2994
|
const message = error.response?.data?.error || error.message;
|
|
2787
2995
|
let wrappedError;
|
|
@@ -2832,7 +3040,7 @@ var VoyageEmbeddingProvider = class {
|
|
|
2832
3040
|
const modelToUse = model || this.defaultModel;
|
|
2833
3041
|
return retryWithBackoff2(async () => {
|
|
2834
3042
|
try {
|
|
2835
|
-
const response = await
|
|
3043
|
+
const response = await axios13__default.default.post(
|
|
2836
3044
|
`${this.baseUrl}/embeddings`,
|
|
2837
3045
|
{
|
|
2838
3046
|
input: texts,
|
|
@@ -2859,7 +3067,7 @@ var VoyageEmbeddingProvider = class {
|
|
|
2859
3067
|
} : void 0
|
|
2860
3068
|
};
|
|
2861
3069
|
} catch (error) {
|
|
2862
|
-
if (
|
|
3070
|
+
if (axios13__default.default.isAxiosError(error)) {
|
|
2863
3071
|
const status = error.response?.status;
|
|
2864
3072
|
const message = error.response?.data?.message || error.message;
|
|
2865
3073
|
let wrappedError;
|
|
@@ -2898,7 +3106,7 @@ var OllamaEmbeddingProvider = class {
|
|
|
2898
3106
|
const modelToUse = model || this.defaultModel;
|
|
2899
3107
|
return retryWithBackoff2(async () => {
|
|
2900
3108
|
try {
|
|
2901
|
-
const response = await
|
|
3109
|
+
const response = await axios13__default.default.post(
|
|
2902
3110
|
`${this.baseUrl}/api/embeddings`,
|
|
2903
3111
|
{
|
|
2904
3112
|
model: modelToUse,
|
|
@@ -2917,7 +3125,7 @@ var OllamaEmbeddingProvider = class {
|
|
|
2917
3125
|
dimensions: embedding.length
|
|
2918
3126
|
};
|
|
2919
3127
|
} catch (error) {
|
|
2920
|
-
if (
|
|
3128
|
+
if (axios13__default.default.isAxiosError(error)) {
|
|
2921
3129
|
const status = error.response?.status;
|
|
2922
3130
|
const message = error.response?.data?.error || error.message;
|
|
2923
3131
|
let wrappedError;
|
|
@@ -10395,11 +10603,13 @@ exports.CreditCardValidatorSchema = CreditCardValidatorSchema;
|
|
|
10395
10603
|
exports.CurrentDateTimeSchema = CurrentDateTimeSchema;
|
|
10396
10604
|
exports.DEFAULT_BATCH_SIZE = DEFAULT_BATCH_SIZE;
|
|
10397
10605
|
exports.DEFAULT_CHUNK_SIZE = DEFAULT_CHUNK_SIZE;
|
|
10606
|
+
exports.DEFAULT_DESTINATION_POLICY = DEFAULT_DESTINATION_POLICY;
|
|
10398
10607
|
exports.DEFAULT_RETRY_CONFIG = DEFAULT_RETRY_CONFIG2;
|
|
10399
10608
|
exports.DateArithmeticSchema = DateArithmeticSchema;
|
|
10400
10609
|
exports.DateComparisonSchema = DateComparisonSchema;
|
|
10401
10610
|
exports.DateDifferenceSchema = DateDifferenceSchema;
|
|
10402
10611
|
exports.DateFormatterSchema = DateFormatterSchema;
|
|
10612
|
+
exports.DestinationPolicyError = DestinationPolicyError;
|
|
10403
10613
|
exports.DuckDuckGoProvider = DuckDuckGoProvider;
|
|
10404
10614
|
exports.EmailValidatorSchema = EmailValidatorSchema;
|
|
10405
10615
|
exports.EmbeddingManager = EmbeddingManager;
|
|
@@ -10433,6 +10643,7 @@ exports.arrayMapSchema = arrayMapSchema;
|
|
|
10433
10643
|
exports.arraySort = arraySort;
|
|
10434
10644
|
exports.arraySortSchema = arraySortSchema;
|
|
10435
10645
|
exports.askHumanTool = askHumanTool;
|
|
10646
|
+
exports.assertDestinationAllowed = assertDestinationAllowed;
|
|
10436
10647
|
exports.benchmarkBatchExecution = benchmarkBatchExecution;
|
|
10437
10648
|
exports.benchmarkStreamingSelectMemory = benchmarkStreamingSelectMemory;
|
|
10438
10649
|
exports.buildDeleteQuery = buildDeleteQuery;
|
|
@@ -10682,6 +10893,7 @@ exports.relationalInsert = relationalInsert;
|
|
|
10682
10893
|
exports.relationalQuery = relationalQuery;
|
|
10683
10894
|
exports.relationalSelect = relationalSelect;
|
|
10684
10895
|
exports.relationalUpdate = relationalUpdate;
|
|
10896
|
+
exports.requestWithDestinationPolicy = requestWithDestinationPolicy;
|
|
10685
10897
|
exports.retryWithBackoff = retryWithBackoff2;
|
|
10686
10898
|
exports.scraperTools = scraperTools;
|
|
10687
10899
|
exports.searchConfluence = searchConfluence;
|