@openclaw/firecrawl-plugin 2026.11.3 → 2026.11.5-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.setup/{firecrawl-client--j4MjwgU.mjs → firecrawl-client-Bx1Mas0Y.mjs} +108 -143
- package/dist/.setup/{firecrawl-fetch-provider-Br32KRpR.mjs → firecrawl-fetch-provider-DTGxGqmO.mjs} +4 -9
- package/dist/.setup/{firecrawl-search-provider-D_gyroLO.mjs → firecrawl-search-provider-DBouVH9p.mjs} +6 -17
- package/dist/.source-module-urls/ad8205c790d921269d4ea28d22ab658579b9674fd00e025664fcf0b8ea32155c.json +1 -0
- package/dist/api.js +1 -1
- package/dist/index.js +69 -89
- package/dist/web-fetch-contract-api.js +1 -2
- package/dist/web-fetch-provider.js +1 -2
- package/dist/web-search-contract-api.js +1 -2
- package/dist/web-search-provider.js +1 -2
- package/node_modules/typebox/build/format/duration.d.mts +2 -2
- package/node_modules/typebox/build/format/duration.mjs +2 -2
- package/node_modules/typebox/build/format/idn_email.d.mts +1 -0
- package/node_modules/typebox/build/format/idn_email.mjs +1 -0
- package/node_modules/typebox/build/format/idn_hostname.d.mts +1 -1
- package/node_modules/typebox/build/format/idn_hostname.mjs +1 -1
- package/node_modules/typebox/build/format/ipv4.d.mts +2 -2
- package/node_modules/typebox/build/format/ipv4.mjs +2 -2
- package/node_modules/typebox/build/format/ipv6.d.mts +1 -1
- package/node_modules/typebox/build/format/ipv6.mjs +1 -1
- package/node_modules/typebox/build/format/relative_json_pointer.d.mts +2 -2
- package/node_modules/typebox/build/format/relative_json_pointer.mjs +2 -2
- package/node_modules/typebox/build/schema/build.mjs +1 -0
- package/node_modules/typebox/build/schema/engine/_unique.d.mts +4 -2
- package/node_modules/typebox/build/schema/engine/_unique.mjs +7 -3
- package/node_modules/typebox/build/schema/engine/additionalItems.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/additionalProperties.d.mts +0 -3
- package/node_modules/typebox/build/schema/engine/additionalProperties.mjs +15 -17
- package/node_modules/typebox/build/schema/engine/contains.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/index.d.mts +1 -0
- package/node_modules/typebox/build/schema/engine/index.mjs +1 -0
- package/node_modules/typebox/build/schema/engine/items.mjs +7 -4
- package/node_modules/typebox/build/schema/engine/maxContains.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/minContains.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/oneOf.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/patternProperties.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/propertyNames.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/unevaluatedItems.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/unevaluatedProperties.mjs +2 -2
- package/node_modules/typebox/build/system/hashing/hash.mjs +63 -34
- package/node_modules/typebox/build/system/locale/en_US.mjs +2 -1
- package/node_modules/typebox/build/system/memory/assign.d.mts +2 -8
- package/node_modules/typebox/build/system/memory/assign.mjs +13 -6
- package/node_modules/typebox/build/system/memory/clone.d.mts +4 -5
- package/node_modules/typebox/build/system/memory/clone.mjs +8 -96
- package/node_modules/typebox/build/system/memory/create.d.mts +1 -5
- package/node_modules/typebox/build/system/memory/create.mjs +18 -13
- package/node_modules/typebox/build/system/memory/deep.d.mts +2 -0
- package/node_modules/typebox/build/system/memory/deep.mjs +108 -0
- package/node_modules/typebox/build/system/memory/discard.d.mts +1 -1
- package/node_modules/typebox/build/system/memory/discard.mjs +9 -8
- package/node_modules/typebox/build/system/memory/finalize.d.mts +4 -0
- package/node_modules/typebox/build/system/memory/finalize.mjs +30 -0
- package/node_modules/typebox/build/system/memory/keys.d.mts +2 -0
- package/node_modules/typebox/build/system/memory/keys.mjs +8 -0
- package/node_modules/typebox/build/system/memory/update.d.mts +1 -4
- package/node_modules/typebox/build/system/memory/update.mjs +43 -16
- package/node_modules/typebox/build/type/engine/call/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/call/index.mjs +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/composite.d.mts +2 -24
- package/node_modules/typebox/build/type/engine/evaluate/composite.mjs +2 -29
- package/node_modules/typebox/build/type/engine/evaluate/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/index.mjs +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/narrow.d.mts +5 -1
- package/node_modules/typebox/build/type/engine/evaluate/narrow.mjs +6 -1
- package/node_modules/typebox/build/type/engine/evaluate/property.d.mts +30 -0
- package/node_modules/typebox/build/type/engine/evaluate/property.mjs +29 -0
- package/node_modules/typebox/build/type/engine/immutable/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/immutable/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/object/from_intersect.d.mts +2 -2
- package/node_modules/typebox/build/type/engine/object/from_intersect.mjs +2 -2
- package/node_modules/typebox/build/type/engine/object/from_union.d.mts +2 -2
- package/node_modules/typebox/build/type/engine/object/from_union.mjs +2 -2
- package/node_modules/typebox/build/type/engine/optional/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/optional/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/readonly/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/readonly/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/this/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/this/index.mjs +1 -0
- package/node_modules/typebox/build/value/clone/clone.d.mts +4 -3
- package/node_modules/typebox/build/value/clone/clone.mjs +5 -7
- package/node_modules/typebox/build/value/repair/from_cyclic.d.mts +2 -0
- package/node_modules/typebox/build/value/repair/from_cyclic.mjs +6 -0
- package/node_modules/typebox/build/value/shared/union_priority_sort.mjs +25 -14
- package/node_modules/typebox/package.json +1 -1
- package/node_modules/typebox/readme.md +1 -1
- package/package.json +5 -5
- package/node_modules/typebox/build/system/memory/freeze.d.mts +0 -4
- package/node_modules/typebox/build/system/memory/freeze.mjs +0 -6
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import { resolveSiteName } from "openclaw/plugin-sdk/provider-web-search";
|
|
2
|
-
import {
|
|
1
|
+
import { externalContentStamp, readCachedSearchPayload, resolveSearchCacheTtlMs, resolveSiteName, writeCachedSearchPayload } from "openclaw/plugin-sdk/provider-web-search";
|
|
2
|
+
import { sha256Hex } from "openclaw/plugin-sdk/node-crypto-runtime";
|
|
3
3
|
import { asNonNegativeFiniteNumber, asPositiveFiniteNumber, parseDateStringTimestampMs, parseFiniteNumber, resolveIntegerOption } from "openclaw/plugin-sdk/number-runtime";
|
|
4
|
+
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
4
5
|
import { ProviderHttpError, readProviderJsonObjectResponse } from "openclaw/plugin-sdk/provider-http";
|
|
5
|
-
import {
|
|
6
|
+
import { markdownToText, readResponseText, resolvePositiveTimeoutSeconds, withSelfHostedWebToolsEndpoint, withStrictWebToolsEndpoint } from "openclaw/plugin-sdk/provider-web-fetch";
|
|
7
|
+
import { resolveProviderWebSearchPluginConfig } from "openclaw/plugin-sdk/provider-web-search-config-contract";
|
|
6
8
|
import { normalizeSecretInput } from "openclaw/plugin-sdk/secret-input";
|
|
9
|
+
import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
|
|
7
10
|
import { truncateSanitizedExternalContent, wrapExternalContent, wrapWebContent } from "openclaw/plugin-sdk/security-runtime";
|
|
8
11
|
import { SsrFBlockedError, isBlockedHostnameOrIp, isPrivateIpAddress, resolvePinnedHostnameWithPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
|
9
12
|
import { asBoolean, asOptionalObjectRecord, asOptionalRecord, asRecord, normalizeOptionalString, readStringField } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
10
|
-
import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
|
|
11
13
|
//#region \0rolldown/runtime.js
|
|
12
14
|
var __defProp = Object.defineProperty;
|
|
13
15
|
var __exportAll = (all, no_symbols) => {
|
|
@@ -19,64 +21,17 @@ var __exportAll = (all, no_symbols) => {
|
|
|
19
21
|
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
20
22
|
return target;
|
|
21
23
|
};
|
|
22
|
-
const DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS = 30;
|
|
23
|
-
const DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS = 60;
|
|
24
|
-
const DEFAULT_FIRECRAWL_MAX_AGE_MS = 1728e5;
|
|
25
|
-
const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
|
|
26
|
-
function resolveFirecrawlSearchConfig(cfg) {
|
|
27
|
-
const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config;
|
|
28
|
-
return asOptionalRecord(pluginConfig?.webSearch);
|
|
29
|
-
}
|
|
30
|
-
function resolveFirecrawlFetchConfig(cfg) {
|
|
31
|
-
const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config;
|
|
32
|
-
return asOptionalRecord(pluginConfig?.webFetch);
|
|
33
|
-
}
|
|
34
|
-
function resolveConfiguredSecret(value, path, cfg) {
|
|
35
|
-
return resolveReadOnlyEnvSecretRef({
|
|
36
|
-
value,
|
|
37
|
-
path,
|
|
38
|
-
cfg,
|
|
39
|
-
expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
|
|
40
|
-
normalizeValue: normalizeSecretInput
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
function resolveFirecrawlConnection(cfg, scope) {
|
|
44
|
-
const scopes = {
|
|
45
|
-
webSearch: resolveFirecrawlSearchConfig(cfg),
|
|
46
|
-
webFetch: resolveFirecrawlFetchConfig(cfg)
|
|
47
|
-
};
|
|
48
|
-
const primary = scopes[scope];
|
|
49
|
-
const selectedScope = primary?.apiKey === void 0 && !normalizeOptionalString(primary?.baseUrl) ? scope === "webSearch" ? "webFetch" : "webSearch" : scope;
|
|
50
|
-
const connection = scopes[selectedScope];
|
|
51
|
-
const resolved = resolveConfiguredSecret(connection?.apiKey, `plugins.entries.firecrawl.config.${selectedScope}.apiKey`, cfg);
|
|
52
|
-
return {
|
|
53
|
-
apiKey: resolved.status === "available" ? resolved.value : resolved.status === "blocked" ? void 0 : normalizeSecretInput(process.env[FIRECRAWL_API_KEY_ENV_VAR]) || void 0,
|
|
54
|
-
baseUrl: normalizeOptionalString(connection?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || "https://api.firecrawl.dev"
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
function resolveFirecrawlOnlyMainContent(cfg, override) {
|
|
58
|
-
return asBoolean(override) ?? asBoolean(resolveFirecrawlFetchConfig(cfg)?.onlyMainContent) ?? true;
|
|
59
|
-
}
|
|
60
|
-
function resolveFirecrawlMaxAgeMs(cfg, override) {
|
|
61
|
-
return Math.floor(asNonNegativeFiniteNumber(override) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
|
|
62
|
-
}
|
|
63
|
-
function resolveFirecrawlScrapeTimeoutSeconds(cfg, override) {
|
|
64
|
-
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
65
|
-
return resolvePositiveTimeoutSeconds(override, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
|
|
66
|
-
}
|
|
67
|
-
function resolveFirecrawlSearchTimeoutSeconds(override) {
|
|
68
|
-
return resolvePositiveTimeoutSeconds(override, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
|
69
|
-
}
|
|
70
24
|
//#endregion
|
|
71
25
|
//#region extensions/firecrawl/src/firecrawl-client.ts
|
|
72
26
|
var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
|
|
73
|
-
assertFirecrawlScrapeTargetAllowed: () => assertFirecrawlScrapeTargetAllowed,
|
|
74
|
-
parseFirecrawlScrapePayload: () => parseFirecrawlScrapePayload,
|
|
75
27
|
runFirecrawlScrape: () => runFirecrawlScrape,
|
|
76
|
-
runFirecrawlSearch: () => runFirecrawlSearch
|
|
77
|
-
testing: () => testing
|
|
28
|
+
runFirecrawlSearch: () => runFirecrawlSearch
|
|
78
29
|
});
|
|
79
|
-
const
|
|
30
|
+
const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
|
|
31
|
+
const DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS = 30;
|
|
32
|
+
const DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS = 60;
|
|
33
|
+
const DEFAULT_FIRECRAWL_MAX_AGE_MS = 1728e5;
|
|
34
|
+
const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
|
|
80
35
|
const DEFAULT_SEARCH_COUNT = 5;
|
|
81
36
|
const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
|
|
82
37
|
const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
|
|
@@ -88,6 +43,39 @@ const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
|
|
|
88
43
|
const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
|
|
89
44
|
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
|
|
90
45
|
const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
|
|
46
|
+
function resolveFirecrawlFetchConfig(cfg) {
|
|
47
|
+
return asOptionalRecord(resolvePluginConfigObject(cfg, "firecrawl")?.webFetch);
|
|
48
|
+
}
|
|
49
|
+
function resolveFirecrawlApiKey(cfg) {
|
|
50
|
+
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
51
|
+
const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
|
|
52
|
+
const configuredCandidates = [{
|
|
53
|
+
value: fetch?.apiKey,
|
|
54
|
+
path: "plugins.entries.firecrawl.config.webFetch.apiKey"
|
|
55
|
+
}, {
|
|
56
|
+
value: search?.apiKey,
|
|
57
|
+
path: "plugins.entries.firecrawl.config.webSearch.apiKey"
|
|
58
|
+
}];
|
|
59
|
+
let blockedConfiguredSecret = false;
|
|
60
|
+
for (const candidate of configuredCandidates) {
|
|
61
|
+
const resolved = resolveReadOnlyEnvSecretRef({
|
|
62
|
+
value: candidate.value,
|
|
63
|
+
path: candidate.path,
|
|
64
|
+
cfg,
|
|
65
|
+
expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
|
|
66
|
+
normalizeValue: normalizeSecretInput
|
|
67
|
+
});
|
|
68
|
+
if (resolved.status === "available") return resolved.value;
|
|
69
|
+
if (resolved.status === "blocked") blockedConfiguredSecret = true;
|
|
70
|
+
}
|
|
71
|
+
if (blockedConfiguredSecret) return;
|
|
72
|
+
return normalizeSecretInput(process.env[FIRECRAWL_API_KEY_ENV_VAR]) || void 0;
|
|
73
|
+
}
|
|
74
|
+
function resolveFirecrawlBaseUrl(cfg) {
|
|
75
|
+
const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
|
|
76
|
+
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
77
|
+
return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || DEFAULT_FIRECRAWL_BASE_URL;
|
|
78
|
+
}
|
|
91
79
|
function assertFirecrawlScrapeTargetAllowed(url) {
|
|
92
80
|
let parsed;
|
|
93
81
|
try {
|
|
@@ -101,27 +89,24 @@ function assertFirecrawlScrapeTargetAllowed(url) {
|
|
|
101
89
|
function isOfficialFirecrawlEndpoint(url) {
|
|
102
90
|
return url.protocol === "https:" && ALLOWED_FIRECRAWL_HOSTS.has(url.hostname);
|
|
103
91
|
}
|
|
104
|
-
async function firecrawlEndpointTargetsPrivateNetwork(url
|
|
92
|
+
async function firecrawlEndpointTargetsPrivateNetwork(url) {
|
|
105
93
|
if (isBlockedHostnameOrIp(url.hostname)) return true;
|
|
106
94
|
try {
|
|
107
|
-
return (await resolvePinnedHostnameWithPolicy(url.hostname, {
|
|
108
|
-
lookupFn,
|
|
109
|
-
policy: { allowPrivateNetwork: true }
|
|
110
|
-
})).addresses.every((address) => isPrivateIpAddress(address));
|
|
95
|
+
return (await resolvePinnedHostnameWithPolicy(url.hostname, { policy: { allowPrivateNetwork: true } })).addresses.every((address) => isPrivateIpAddress(address));
|
|
111
96
|
} catch {
|
|
112
97
|
return false;
|
|
113
98
|
}
|
|
114
99
|
}
|
|
115
|
-
async function validateFirecrawlBaseUrl(url
|
|
100
|
+
async function validateFirecrawlBaseUrl(url) {
|
|
116
101
|
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("Firecrawl baseUrl must use http:// or https://.");
|
|
117
102
|
if (isOfficialFirecrawlEndpoint(url)) return "strict";
|
|
118
|
-
if (await firecrawlEndpointTargetsPrivateNetwork(url
|
|
103
|
+
if (await firecrawlEndpointTargetsPrivateNetwork(url)) return "selfHosted";
|
|
119
104
|
if (url.protocol === "http:") throw new Error(FIRECRAWL_HTTP_PRIVATE_ERROR);
|
|
120
105
|
throw new Error(`${FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR} Host: ${url.hostname}`);
|
|
121
106
|
}
|
|
122
|
-
async function resolveEndpoint(baseUrl, pathname
|
|
123
|
-
const url = new URL(baseUrl.trim() ||
|
|
124
|
-
const mode = await validateFirecrawlBaseUrl(url
|
|
107
|
+
async function resolveEndpoint(baseUrl, pathname) {
|
|
108
|
+
const url = new URL(baseUrl.trim() || DEFAULT_FIRECRAWL_BASE_URL);
|
|
109
|
+
const mode = await validateFirecrawlBaseUrl(url);
|
|
125
110
|
url.username = "";
|
|
126
111
|
url.password = "";
|
|
127
112
|
url.search = "";
|
|
@@ -231,28 +216,27 @@ async function runFirecrawlSearch(params) {
|
|
|
231
216
|
params.signal?.throwIfAborted();
|
|
232
217
|
const keyless = params.access === "keyless";
|
|
233
218
|
const providerId = keyless ? "firecrawl-free" : "firecrawl";
|
|
234
|
-
const
|
|
235
|
-
const apiKey = keyless ? void 0 : connection.apiKey;
|
|
219
|
+
const apiKey = keyless ? void 0 : resolveFirecrawlApiKey(params.cfg);
|
|
236
220
|
if (!apiKey && !keyless) throw new Error("web_search (firecrawl) needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webSearch.apiKey.");
|
|
237
221
|
const count = resolveIntegerOption(params.count, DEFAULT_SEARCH_COUNT, {
|
|
238
222
|
min: 1,
|
|
239
223
|
max: 100
|
|
240
224
|
});
|
|
241
|
-
const timeoutSeconds =
|
|
225
|
+
const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
|
242
226
|
const scrapeResults = params.scrapeResults === true;
|
|
243
|
-
const sources =
|
|
244
|
-
const categories =
|
|
245
|
-
const includeDomains =
|
|
246
|
-
const excludeDomains =
|
|
227
|
+
const sources = params.sources?.filter(Boolean) ?? [];
|
|
228
|
+
const categories = params.categories?.filter(Boolean) ?? [];
|
|
229
|
+
const includeDomains = params.includeDomains?.filter(Boolean) ?? [];
|
|
230
|
+
const excludeDomains = params.excludeDomains?.filter(Boolean) ?? [];
|
|
247
231
|
if (includeDomains.length > 0 && excludeDomains.length > 0) throw new Error("Firecrawl search accepts includeDomains or excludeDomains, not both.");
|
|
248
232
|
const tbs = normalizeOptionalString(params.tbs);
|
|
249
233
|
const location = normalizeOptionalString(params.location);
|
|
250
234
|
const country = normalizeOptionalString(params.country);
|
|
251
|
-
const baseUrl =
|
|
252
|
-
const cacheKey =
|
|
235
|
+
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
236
|
+
const cacheKey = JSON.stringify({
|
|
253
237
|
type: "firecrawl-search",
|
|
254
238
|
provider: providerId,
|
|
255
|
-
credential: apiKey ?
|
|
239
|
+
credential: apiKey ? sha256Hex(apiKey) : void 0,
|
|
256
240
|
q: params.query,
|
|
257
241
|
count,
|
|
258
242
|
baseUrl,
|
|
@@ -264,13 +248,10 @@ async function runFirecrawlSearch(params) {
|
|
|
264
248
|
location,
|
|
265
249
|
country,
|
|
266
250
|
scrapeResults
|
|
267
|
-
})
|
|
268
|
-
const cacheTtlMs =
|
|
269
|
-
const cached =
|
|
270
|
-
if (cached) return
|
|
271
|
-
...cached.value,
|
|
272
|
-
cached: true
|
|
273
|
-
};
|
|
251
|
+
});
|
|
252
|
+
const cacheTtlMs = resolveSearchCacheTtlMs(params.cfg?.tools?.web?.search);
|
|
253
|
+
const cached = readCachedSearchPayload(cacheKey, cacheTtlMs);
|
|
254
|
+
if (cached) return cached;
|
|
274
255
|
const body = {
|
|
275
256
|
query: params.query,
|
|
276
257
|
limit: count,
|
|
@@ -324,20 +305,52 @@ async function runFirecrawlSearch(params) {
|
|
|
324
305
|
provider: providerId,
|
|
325
306
|
count: items.length,
|
|
326
307
|
tookMs,
|
|
327
|
-
externalContent:
|
|
328
|
-
untrusted: true,
|
|
329
|
-
source: "web_search",
|
|
330
|
-
provider: providerId,
|
|
331
|
-
wrapped: true
|
|
332
|
-
},
|
|
308
|
+
externalContent: externalContentStamp(providerId),
|
|
333
309
|
results,
|
|
334
310
|
...truncated ? { truncated: true } : {}
|
|
335
311
|
};
|
|
336
|
-
|
|
312
|
+
writeCachedSearchPayload(cacheKey, result, cacheTtlMs);
|
|
337
313
|
return result;
|
|
338
314
|
}
|
|
339
|
-
function
|
|
340
|
-
|
|
315
|
+
async function runFirecrawlScrape(params) {
|
|
316
|
+
params.signal?.throwIfAborted();
|
|
317
|
+
assertFirecrawlScrapeTargetAllowed(params.url);
|
|
318
|
+
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
|
319
|
+
if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
|
|
320
|
+
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
321
|
+
const fetch = resolveFirecrawlFetchConfig(params.cfg);
|
|
322
|
+
const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
|
|
323
|
+
const onlyMainContent = asBoolean(params.onlyMainContent) ?? asBoolean(resolveFirecrawlFetchConfig(params.cfg)?.onlyMainContent) ?? true;
|
|
324
|
+
const maxAgeMs = Math.floor(asNonNegativeFiniteNumber(params.maxAgeMs) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(params.cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
|
|
325
|
+
const proxy = params.proxy ?? "auto";
|
|
326
|
+
const storeInCache = params.storeInCache ?? true;
|
|
327
|
+
const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
|
|
328
|
+
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
|
329
|
+
const payload = await postFirecrawlJson({
|
|
330
|
+
url: endpoint.url,
|
|
331
|
+
mode: endpoint.mode,
|
|
332
|
+
timeoutSeconds,
|
|
333
|
+
apiKey,
|
|
334
|
+
errorLabel: "Firecrawl",
|
|
335
|
+
...params.signal ? { signal: params.signal } : {},
|
|
336
|
+
body: {
|
|
337
|
+
url: params.url,
|
|
338
|
+
formats: ["markdown"],
|
|
339
|
+
onlyMainContent,
|
|
340
|
+
timeout: timeoutSeconds * 1e3,
|
|
341
|
+
maxAge: maxAgeMs,
|
|
342
|
+
proxy,
|
|
343
|
+
storeInCache
|
|
344
|
+
}
|
|
345
|
+
}, async (response) => {
|
|
346
|
+
const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
|
|
347
|
+
if (data.success === false) {
|
|
348
|
+
const detail = readFirecrawlErrorDetail(data, response.statusText);
|
|
349
|
+
throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
|
|
350
|
+
}
|
|
351
|
+
return data;
|
|
352
|
+
});
|
|
353
|
+
const data = asRecord(payload.data);
|
|
341
354
|
const metadata = asOptionalObjectRecord(data.metadata);
|
|
342
355
|
const rawStatus = parseFiniteNumber(metadata?.statusCode) ?? parseFiniteNumber(data.statusCode);
|
|
343
356
|
const status = rawStatus === void 0 ? void 0 : Math.floor(rawStatus);
|
|
@@ -345,7 +358,7 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
345
358
|
const markdown = typeof data.markdown === "string" && data.markdown || typeof data.content === "string" && data.content || "";
|
|
346
359
|
if (!markdown) throw new Error("Firecrawl scrape returned no content.");
|
|
347
360
|
const rawText = params.extractMode === "text" ? markdownToText(markdown) : markdown;
|
|
348
|
-
const boundedText = truncateSanitizedExternalContent(rawText,
|
|
361
|
+
const boundedText = truncateSanitizedExternalContent(rawText, maxChars);
|
|
349
362
|
let truncated = boundedText.truncated;
|
|
350
363
|
let remainingMetadataChars = FIRECRAWL_SCRAPE_METADATA_MAX_CHARS;
|
|
351
364
|
const wrapBoundedMetadata = (value) => {
|
|
@@ -362,7 +375,7 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
362
375
|
includeWarning: false
|
|
363
376
|
});
|
|
364
377
|
const title = typeof metadata?.title === "string" && metadata.title ? wrapBoundedMetadata(metadata.title) : void 0;
|
|
365
|
-
const warning = typeof
|
|
378
|
+
const warning = typeof payload.warning === "string" && payload.warning ? wrapBoundedMetadata(payload.warning) : void 0;
|
|
366
379
|
return {
|
|
367
380
|
url: params.url,
|
|
368
381
|
finalUrl: normalizeFirecrawlResultUrl(metadata?.sourceURL) ?? normalizeFirecrawlResultUrl(data.url) ?? params.url,
|
|
@@ -382,53 +395,5 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
382
395
|
...warning ? { warning } : {}
|
|
383
396
|
};
|
|
384
397
|
}
|
|
385
|
-
async function runFirecrawlScrape(params) {
|
|
386
|
-
params.signal?.throwIfAborted();
|
|
387
|
-
assertFirecrawlScrapeTargetAllowed(params.url);
|
|
388
|
-
const { apiKey, baseUrl } = resolveFirecrawlConnection(params.cfg, "webFetch");
|
|
389
|
-
if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
|
|
390
|
-
const timeoutSeconds = resolveFirecrawlScrapeTimeoutSeconds(params.cfg, params.timeoutSeconds);
|
|
391
|
-
const onlyMainContent = resolveFirecrawlOnlyMainContent(params.cfg, params.onlyMainContent);
|
|
392
|
-
const maxAgeMs = resolveFirecrawlMaxAgeMs(params.cfg, params.maxAgeMs);
|
|
393
|
-
const proxy = params.proxy ?? "auto";
|
|
394
|
-
const storeInCache = params.storeInCache ?? true;
|
|
395
|
-
const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
|
|
396
|
-
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
|
397
|
-
return parseFirecrawlScrapePayload({
|
|
398
|
-
payload: await postFirecrawlJson({
|
|
399
|
-
url: endpoint.url,
|
|
400
|
-
mode: endpoint.mode,
|
|
401
|
-
timeoutSeconds,
|
|
402
|
-
apiKey,
|
|
403
|
-
errorLabel: "Firecrawl",
|
|
404
|
-
...params.signal ? { signal: params.signal } : {},
|
|
405
|
-
body: {
|
|
406
|
-
url: params.url,
|
|
407
|
-
formats: ["markdown"],
|
|
408
|
-
onlyMainContent,
|
|
409
|
-
timeout: timeoutSeconds * 1e3,
|
|
410
|
-
maxAge: maxAgeMs,
|
|
411
|
-
proxy,
|
|
412
|
-
storeInCache
|
|
413
|
-
}
|
|
414
|
-
}, async (response) => {
|
|
415
|
-
const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
|
|
416
|
-
if (data.success === false) {
|
|
417
|
-
const detail = readFirecrawlErrorDetail(data, response.statusText);
|
|
418
|
-
throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
|
|
419
|
-
}
|
|
420
|
-
return data;
|
|
421
|
-
}),
|
|
422
|
-
url: params.url,
|
|
423
|
-
extractMode: params.extractMode,
|
|
424
|
-
maxChars
|
|
425
|
-
});
|
|
426
|
-
}
|
|
427
|
-
const testing = {
|
|
428
|
-
assertFirecrawlScrapeTargetAllowed,
|
|
429
|
-
parseFirecrawlScrapePayload,
|
|
430
|
-
resolveEndpoint,
|
|
431
|
-
resolveSearchItems
|
|
432
|
-
};
|
|
433
398
|
//#endregion
|
|
434
399
|
export { runFirecrawlScrape as n, runFirecrawlSearch as r, firecrawl_client_exports as t };
|
package/dist/.setup/{firecrawl-fetch-provider-Br32KRpR.mjs → firecrawl-fetch-provider-DTGxGqmO.mjs}
RENAMED
|
@@ -1,14 +1,8 @@
|
|
|
1
1
|
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
|
2
2
|
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
|
3
3
|
import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch-contract";
|
|
4
|
+
import { ensureRecord } from "openclaw/plugin-sdk/collection-input-runtime";
|
|
4
5
|
//#region extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
|
|
5
|
-
function ensureRecord(target, key) {
|
|
6
|
-
const current = target[key];
|
|
7
|
-
if (current && typeof current === "object" && !Array.isArray(current)) return current;
|
|
8
|
-
const next = {};
|
|
9
|
-
target[key] = next;
|
|
10
|
-
return next;
|
|
11
|
-
}
|
|
12
6
|
const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
|
13
7
|
id: "firecrawl",
|
|
14
8
|
label: "Firecrawl",
|
|
@@ -45,13 +39,14 @@ const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
|
|
45
39
|
const plugins = configTarget.plugins ??= {};
|
|
46
40
|
const entries = plugins.entries ??= {};
|
|
47
41
|
const firecrawlEntry = entries.firecrawl ??= {};
|
|
48
|
-
const
|
|
42
|
+
const pluginConfig = firecrawlEntry.config ??= {};
|
|
43
|
+
const webFetch = ensureRecord(pluginConfig, "webFetch");
|
|
49
44
|
webFetch.apiKey = value;
|
|
50
45
|
}
|
|
51
46
|
};
|
|
52
47
|
//#endregion
|
|
53
48
|
//#region extensions/firecrawl/src/firecrawl-fetch-provider.ts
|
|
54
|
-
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client
|
|
49
|
+
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-Bx1Mas0Y.mjs").then((n) => n.t));
|
|
55
50
|
function createFirecrawlWebFetchProvider() {
|
|
56
51
|
return {
|
|
57
52
|
...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
|
|
@@ -1,29 +1,18 @@
|
|
|
1
|
+
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
2
|
+
import { asOptionalRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
1
3
|
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
|
2
4
|
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
|
3
|
-
import {
|
|
4
|
-
import { createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
|
|
5
|
+
import { WEB_SEARCH_QUERY_COUNT_PROPERTIES, createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
|
|
5
6
|
//#region extensions/firecrawl/src/firecrawl-search-provider.ts
|
|
6
7
|
const FIRECRAWL_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webSearch.apiKey";
|
|
7
|
-
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client
|
|
8
|
+
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-Bx1Mas0Y.mjs").then((n) => n.t));
|
|
8
9
|
const GenericFirecrawlSearchSchema = {
|
|
9
10
|
type: "object",
|
|
10
|
-
properties: {
|
|
11
|
-
query: {
|
|
12
|
-
type: "string",
|
|
13
|
-
description: "Search query string."
|
|
14
|
-
},
|
|
15
|
-
count: {
|
|
16
|
-
type: "integer",
|
|
17
|
-
description: "Number of results to return (1-10).",
|
|
18
|
-
minimum: 1,
|
|
19
|
-
maximum: 10
|
|
20
|
-
}
|
|
21
|
-
},
|
|
11
|
+
properties: { ...WEB_SEARCH_QUERY_COUNT_PROPERTIES },
|
|
22
12
|
additionalProperties: false
|
|
23
13
|
};
|
|
24
14
|
function getConfiguredFirecrawlFetchCredentialFallback(config) {
|
|
25
|
-
const
|
|
26
|
-
const value = asOptionalRecord(pluginConfig?.webFetch)?.apiKey;
|
|
15
|
+
const value = asOptionalRecord(resolvePluginConfigObject(config, "firecrawl")?.webFetch)?.apiKey;
|
|
27
16
|
return value === void 0 ? void 0 : {
|
|
28
17
|
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
|
|
29
18
|
value
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-Bx1Mas0Y.mjs","sha256":"2f749e101e8e3d49510d61a1a4720f9c5866cd17347bc62a45d46e6c7f0247c2","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-client.ts"]},{"output":".setup/firecrawl-fetch-provider-DTGxGqmO.mjs","sha256":"bd3e1a3ef358f1d2f83d56f982f50ceb28360b9a5e4b138e012d256dccf4d2de","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts","extensions/firecrawl/src/firecrawl-fetch-provider.ts"]},{"output":".setup/firecrawl-search-provider-DBouVH9p.mjs","sha256":"dfb053d94025351de94b96962361cde962a5dc68bdc46d6d38e81e4bf353088c","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-search-provider.ts"]},{"output":"api.js","sha256":"d762f8210fd4a9474e0ec4e8e19f3abac0412313be417861578a3b3d1334dd8b","facadeSource":"extensions/firecrawl/api.ts","sources":["extensions/firecrawl/api.ts"]},{"output":"index.js","sha256":"f68acf9490b4d1a11d4fb3bbb1b2cbb126593099bdfc548d0bad697f4632e7da","facadeSource":"extensions/firecrawl/index.ts","sources":["extensions/firecrawl/index.ts"]},{"output":"web-fetch-contract-api.js","sha256":"bb2810073bde8e6640a78714e399d0091117f942f30633e09ec373949f72759a","facadeSource":"extensions/firecrawl/web-fetch-contract-api.ts","sources":[]},{"output":"web-fetch-provider.js","sha256":"d275807de9c003a3a1cb294707cd4e4ace442e3df63878734f6b7237cd550931","facadeSource":"extensions/firecrawl/web-fetch-provider.ts","sources":[]},{"output":"web-search-contract-api.js","sha256":"ad13994cc134d279bbdc467bcc99f8244c8dbe6e30ab1e723868d5be120c7948","facadeSource":"extensions/firecrawl/web-search-contract-api.ts","sources":[]},{"output":"web-search-provider.js","sha256":"f37a3f9840de307207f49c8e064147a2fed045508d43e0d3b1d53d325e07a1cf","facadeSource":"extensions/firecrawl/web-search-provider.ts","sources":[]}],"unrenderedSources":["extensions/firecrawl/web-fetch-contract-api.ts","extensions/firecrawl/web-fetch-provider.ts","extensions/firecrawl/web-search-contract-api.ts","extensions/firecrawl/web-search-provider.ts"]}
|
package/dist/api.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as runFirecrawlScrape } from "./.setup/firecrawl-client
|
|
1
|
+
import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-Bx1Mas0Y.mjs";
|
|
2
2
|
//#region extensions/firecrawl/api.ts
|
|
3
3
|
async function fetchFirecrawlContent(params) {
|
|
4
4
|
const cfg = { plugins: { entries: { firecrawl: {
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { n as runFirecrawlScrape, r as runFirecrawlSearch } from "./.setup/firecrawl-client
|
|
2
|
-
import { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-
|
|
3
|
-
import { n as createFirecrawlWebSearchProvider, t as createFirecrawlFreeWebSearchProvider } from "./.setup/firecrawl-search-provider-
|
|
1
|
+
import { n as runFirecrawlScrape, r as runFirecrawlSearch } from "./.setup/firecrawl-client-Bx1Mas0Y.mjs";
|
|
2
|
+
import { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-DTGxGqmO.mjs";
|
|
3
|
+
import { n as createFirecrawlWebSearchProvider, t as createFirecrawlFreeWebSearchProvider } from "./.setup/firecrawl-search-provider-DBouVH9p.mjs";
|
|
4
|
+
import { optionalNonNegativeIntegerSchema, optionalPositiveIntegerSchema, optionalStringEnum } from "openclaw/plugin-sdk/channel-actions";
|
|
4
5
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
5
|
-
import { optionalStringEnum } from "openclaw/plugin-sdk/channel-actions";
|
|
6
6
|
import { jsonResult, readNonNegativeIntegerParam, readPositiveIntegerParam, readStringArrayParam, readStringParam } from "openclaw/plugin-sdk/provider-web-search";
|
|
7
7
|
import { Type } from "typebox";
|
|
8
|
-
//#region extensions/firecrawl/
|
|
8
|
+
//#region extensions/firecrawl/index.ts
|
|
9
9
|
const FirecrawlScrapeToolSchema = Type.Object({
|
|
10
10
|
url: Type.String({ description: "HTTP or HTTPS URL to scrape via Firecrawl." }),
|
|
11
11
|
extractMode: optionalStringEnum(["markdown", "text"], { description: "Extraction mode (\"markdown\" or \"text\"). Default: markdown." }),
|
|
@@ -14,63 +14,21 @@ const FirecrawlScrapeToolSchema = Type.Object({
|
|
|
14
14
|
minimum: 100
|
|
15
15
|
})),
|
|
16
16
|
onlyMainContent: Type.Optional(Type.Boolean({ description: "Keep only main content when Firecrawl supports it." })),
|
|
17
|
-
maxAgeMs:
|
|
18
|
-
description: "Maximum Firecrawl cache age in milliseconds.",
|
|
19
|
-
minimum: 0
|
|
20
|
-
})),
|
|
17
|
+
maxAgeMs: optionalNonNegativeIntegerSchema({ description: "Maximum Firecrawl cache age in milliseconds." }),
|
|
21
18
|
proxy: optionalStringEnum([
|
|
22
19
|
"auto",
|
|
23
20
|
"basic",
|
|
24
21
|
"stealth"
|
|
25
22
|
], { description: "Firecrawl proxy mode (\"auto\", \"basic\", or \"stealth\")." }),
|
|
26
23
|
storeInCache: Type.Optional(Type.Boolean({ description: "Whether Firecrawl should store the scrape in its cache." })),
|
|
27
|
-
timeoutSeconds:
|
|
28
|
-
description: "Timeout in seconds for the Firecrawl scrape request.",
|
|
29
|
-
minimum: 1
|
|
30
|
-
}))
|
|
24
|
+
timeoutSeconds: optionalPositiveIntegerSchema({ description: "Timeout in seconds for the Firecrawl scrape request." })
|
|
31
25
|
}, { additionalProperties: false });
|
|
32
|
-
function createFirecrawlScrapeTool(api) {
|
|
33
|
-
return {
|
|
34
|
-
name: "firecrawl_scrape",
|
|
35
|
-
label: "Firecrawl Scrape",
|
|
36
|
-
resultContentSource: "network",
|
|
37
|
-
description: "Scrape a page using Firecrawl v2/scrape. Useful for JS-heavy or bot-protected pages where a plain URL fetch is weak.",
|
|
38
|
-
parameters: FirecrawlScrapeToolSchema,
|
|
39
|
-
execute: async (_toolCallId, rawParams, signal) => {
|
|
40
|
-
signal?.throwIfAborted();
|
|
41
|
-
const url = readStringParam(rawParams, "url", { required: true });
|
|
42
|
-
const extractMode = readStringParam(rawParams, "extractMode") === "text" ? "text" : "markdown";
|
|
43
|
-
const maxChars = readPositiveIntegerParam(rawParams, "maxChars");
|
|
44
|
-
const maxAgeMs = readNonNegativeIntegerParam(rawParams, "maxAgeMs");
|
|
45
|
-
const timeoutSeconds = readPositiveIntegerParam(rawParams, "timeoutSeconds");
|
|
46
|
-
const proxyRaw = readStringParam(rawParams, "proxy");
|
|
47
|
-
const proxy = proxyRaw === "basic" || proxyRaw === "stealth" || proxyRaw === "auto" ? proxyRaw : void 0;
|
|
48
|
-
const onlyMainContent = typeof rawParams.onlyMainContent === "boolean" ? rawParams.onlyMainContent : void 0;
|
|
49
|
-
const storeInCache = typeof rawParams.storeInCache === "boolean" ? rawParams.storeInCache : void 0;
|
|
50
|
-
return jsonResult(await runFirecrawlScrape({
|
|
51
|
-
cfg: api.config,
|
|
52
|
-
url,
|
|
53
|
-
extractMode,
|
|
54
|
-
maxChars,
|
|
55
|
-
onlyMainContent,
|
|
56
|
-
maxAgeMs,
|
|
57
|
-
proxy,
|
|
58
|
-
storeInCache,
|
|
59
|
-
timeoutSeconds,
|
|
60
|
-
...signal ? { signal } : {}
|
|
61
|
-
}));
|
|
62
|
-
}
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
//#endregion
|
|
66
|
-
//#region extensions/firecrawl/src/firecrawl-search-tool.ts
|
|
67
26
|
const FirecrawlSearchToolSchema = Type.Object({
|
|
68
27
|
query: Type.String({ description: "Search query string." }),
|
|
69
|
-
count:
|
|
28
|
+
count: optionalPositiveIntegerSchema({
|
|
70
29
|
description: "Number of results to return (1-100).",
|
|
71
|
-
minimum: 1,
|
|
72
30
|
maximum: 100
|
|
73
|
-
})
|
|
31
|
+
}),
|
|
74
32
|
sources: Type.Optional(Type.Array(Type.String(), { description: "Optional sources list, for example [\"web\"], [\"news\"], or [\"images\"]." })),
|
|
75
33
|
categories: Type.Optional(Type.Array(Type.String(), { description: "Optional Firecrawl categories, for example [\"github\"] or [\"research\"]." })),
|
|
76
34
|
includeDomains: Type.Optional(Type.Array(Type.String(), { description: "Restrict results to these hostnames (no protocol or path). Cannot be combined with excludeDomains." })),
|
|
@@ -79,43 +37,8 @@ const FirecrawlSearchToolSchema = Type.Object({
|
|
|
79
37
|
location: Type.Optional(Type.String({ description: "Geo-target location, for example \"Germany\" or \"San Francisco,California,United States\"." })),
|
|
80
38
|
country: Type.Optional(Type.String({ description: "ISO country code for geo-targeting, for example \"US\", \"DE\", or \"JP\"." })),
|
|
81
39
|
scrapeResults: Type.Optional(Type.Boolean({ description: "Include scraped result content when Firecrawl returns it." })),
|
|
82
|
-
timeoutSeconds:
|
|
83
|
-
description: "Timeout in seconds for the Firecrawl Search request.",
|
|
84
|
-
minimum: 1
|
|
85
|
-
}))
|
|
40
|
+
timeoutSeconds: optionalPositiveIntegerSchema({ description: "Timeout in seconds for the Firecrawl Search request." })
|
|
86
41
|
}, { additionalProperties: false });
|
|
87
|
-
function createFirecrawlSearchTool(api) {
|
|
88
|
-
return {
|
|
89
|
-
name: "firecrawl_search",
|
|
90
|
-
label: "Firecrawl Search",
|
|
91
|
-
resultContentSource: "network",
|
|
92
|
-
description: "Search the web using Firecrawl v2/search. Supports includeDomains/excludeDomains filtering and tbs time filters (day/week/month/year). Can optionally include scraped content from result pages.",
|
|
93
|
-
parameters: FirecrawlSearchToolSchema,
|
|
94
|
-
execute: async (_toolCallId, rawParams, signal) => {
|
|
95
|
-
signal?.throwIfAborted();
|
|
96
|
-
return jsonResult(await runFirecrawlSearch({
|
|
97
|
-
query: readStringParam(rawParams, "query", { required: true }),
|
|
98
|
-
count: readPositiveIntegerParam(rawParams, "count", {
|
|
99
|
-
max: 100,
|
|
100
|
-
message: "count must be an integer from 1 to 100"
|
|
101
|
-
}),
|
|
102
|
-
timeoutSeconds: readPositiveIntegerParam(rawParams, "timeoutSeconds"),
|
|
103
|
-
sources: readStringArrayParam(rawParams, "sources"),
|
|
104
|
-
categories: readStringArrayParam(rawParams, "categories"),
|
|
105
|
-
includeDomains: readStringArrayParam(rawParams, "includeDomains"),
|
|
106
|
-
excludeDomains: readStringArrayParam(rawParams, "excludeDomains"),
|
|
107
|
-
tbs: readStringParam(rawParams, "tbs"),
|
|
108
|
-
location: readStringParam(rawParams, "location"),
|
|
109
|
-
country: readStringParam(rawParams, "country"),
|
|
110
|
-
scrapeResults: rawParams.scrapeResults === true,
|
|
111
|
-
cfg: api.config,
|
|
112
|
-
...signal ? { signal } : {}
|
|
113
|
-
}));
|
|
114
|
-
}
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
//#endregion
|
|
118
|
-
//#region extensions/firecrawl/index.ts
|
|
119
42
|
var firecrawl_default = definePluginEntry({
|
|
120
43
|
id: "firecrawl",
|
|
121
44
|
name: "Firecrawl Plugin",
|
|
@@ -124,8 +47,65 @@ var firecrawl_default = definePluginEntry({
|
|
|
124
47
|
api.registerWebFetchProvider(createFirecrawlWebFetchProvider());
|
|
125
48
|
api.registerWebSearchProvider(createFirecrawlWebSearchProvider());
|
|
126
49
|
api.registerWebSearchProvider(createFirecrawlFreeWebSearchProvider());
|
|
127
|
-
api.registerTool(
|
|
128
|
-
|
|
50
|
+
api.registerTool({
|
|
51
|
+
name: "firecrawl_search",
|
|
52
|
+
label: "Firecrawl Search",
|
|
53
|
+
resultContentSource: "network",
|
|
54
|
+
description: "Search the web using Firecrawl v2/search. Supports includeDomains/excludeDomains filtering and tbs time filters (day/week/month/year). Can optionally include scraped content from result pages.",
|
|
55
|
+
parameters: FirecrawlSearchToolSchema,
|
|
56
|
+
execute: async (_toolCallId, rawParams, signal) => {
|
|
57
|
+
signal?.throwIfAborted();
|
|
58
|
+
return jsonResult(await runFirecrawlSearch({
|
|
59
|
+
query: readStringParam(rawParams, "query", { required: true }),
|
|
60
|
+
count: readPositiveIntegerParam(rawParams, "count", {
|
|
61
|
+
max: 100,
|
|
62
|
+
message: "count must be an integer from 1 to 100"
|
|
63
|
+
}),
|
|
64
|
+
timeoutSeconds: readPositiveIntegerParam(rawParams, "timeoutSeconds"),
|
|
65
|
+
sources: readStringArrayParam(rawParams, "sources"),
|
|
66
|
+
categories: readStringArrayParam(rawParams, "categories"),
|
|
67
|
+
includeDomains: readStringArrayParam(rawParams, "includeDomains"),
|
|
68
|
+
excludeDomains: readStringArrayParam(rawParams, "excludeDomains"),
|
|
69
|
+
tbs: readStringParam(rawParams, "tbs"),
|
|
70
|
+
location: readStringParam(rawParams, "location"),
|
|
71
|
+
country: readStringParam(rawParams, "country"),
|
|
72
|
+
scrapeResults: rawParams.scrapeResults === true,
|
|
73
|
+
cfg: api.config,
|
|
74
|
+
...signal ? { signal } : {}
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
api.registerTool({
|
|
79
|
+
name: "firecrawl_scrape",
|
|
80
|
+
label: "Firecrawl Scrape",
|
|
81
|
+
resultContentSource: "network",
|
|
82
|
+
description: "Scrape a page using Firecrawl v2/scrape. Useful for JS-heavy or bot-protected pages where a plain URL fetch is weak.",
|
|
83
|
+
parameters: FirecrawlScrapeToolSchema,
|
|
84
|
+
execute: async (_toolCallId, rawParams, signal) => {
|
|
85
|
+
signal?.throwIfAborted();
|
|
86
|
+
const url = readStringParam(rawParams, "url", { required: true });
|
|
87
|
+
const extractMode = readStringParam(rawParams, "extractMode") === "text" ? "text" : "markdown";
|
|
88
|
+
const maxChars = readPositiveIntegerParam(rawParams, "maxChars");
|
|
89
|
+
const maxAgeMs = readNonNegativeIntegerParam(rawParams, "maxAgeMs");
|
|
90
|
+
const timeoutSeconds = readPositiveIntegerParam(rawParams, "timeoutSeconds");
|
|
91
|
+
const proxyRaw = readStringParam(rawParams, "proxy");
|
|
92
|
+
const proxy = proxyRaw === "basic" || proxyRaw === "stealth" || proxyRaw === "auto" ? proxyRaw : void 0;
|
|
93
|
+
const onlyMainContent = typeof rawParams.onlyMainContent === "boolean" ? rawParams.onlyMainContent : void 0;
|
|
94
|
+
const storeInCache = typeof rawParams.storeInCache === "boolean" ? rawParams.storeInCache : void 0;
|
|
95
|
+
return jsonResult(await runFirecrawlScrape({
|
|
96
|
+
cfg: api.config,
|
|
97
|
+
url,
|
|
98
|
+
extractMode,
|
|
99
|
+
maxChars,
|
|
100
|
+
onlyMainContent,
|
|
101
|
+
maxAgeMs,
|
|
102
|
+
proxy,
|
|
103
|
+
storeInCache,
|
|
104
|
+
timeoutSeconds,
|
|
105
|
+
...signal ? { signal } : {}
|
|
106
|
+
}));
|
|
107
|
+
}
|
|
108
|
+
});
|
|
129
109
|
}
|
|
130
110
|
});
|
|
131
111
|
//#endregion
|
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
export { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-DTGxGqmO.mjs";
|
|
2
2
|
import "./web-fetch-provider.js";
|
|
3
|
-
export { createFirecrawlWebFetchProvider };
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
export { createFirecrawlWebFetchProvider };
|
|
1
|
+
export { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-DTGxGqmO.mjs";
|