@openclaw/firecrawl-plugin 2026.10.10 → 2026.11.5-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.setup/{firecrawl-client-Dk_bi5M_.mjs → firecrawl-client-CDyLQvtN.mjs} +97 -140
- package/dist/.setup/{firecrawl-fetch-provider-EP1XFhDE.mjs → firecrawl-fetch-provider-QmPccHZS.mjs} +4 -9
- package/dist/.setup/{firecrawl-search-provider-Dg-_xx_m.mjs → firecrawl-search-provider-wsRgx6kl.mjs} +6 -17
- package/dist/.source-module-urls/ad8205c790d921269d4ea28d22ab658579b9674fd00e025664fcf0b8ea32155c.json +1 -1
- package/dist/api.js +1 -1
- package/dist/index.js +69 -87
- package/dist/web-fetch-contract-api.js +1 -1
- package/dist/web-fetch-provider.js +1 -1
- package/dist/web-search-contract-api.js +1 -1
- package/dist/web-search-provider.js +1 -1
- package/node_modules/typebox/build/format/duration.d.mts +2 -2
- package/node_modules/typebox/build/format/duration.mjs +2 -2
- package/node_modules/typebox/build/format/idn_email.d.mts +1 -0
- package/node_modules/typebox/build/format/idn_email.mjs +1 -0
- package/node_modules/typebox/build/format/idn_hostname.d.mts +1 -1
- package/node_modules/typebox/build/format/idn_hostname.mjs +1 -1
- package/node_modules/typebox/build/format/ipv4.d.mts +2 -2
- package/node_modules/typebox/build/format/ipv4.mjs +2 -2
- package/node_modules/typebox/build/format/ipv6.d.mts +1 -1
- package/node_modules/typebox/build/format/ipv6.mjs +1 -1
- package/node_modules/typebox/build/format/relative_json_pointer.d.mts +2 -2
- package/node_modules/typebox/build/format/relative_json_pointer.mjs +2 -2
- package/node_modules/typebox/build/schema/build.mjs +1 -0
- package/node_modules/typebox/build/schema/engine/_unique.d.mts +4 -2
- package/node_modules/typebox/build/schema/engine/_unique.mjs +7 -3
- package/node_modules/typebox/build/schema/engine/additionalItems.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/additionalProperties.d.mts +0 -3
- package/node_modules/typebox/build/schema/engine/additionalProperties.mjs +15 -17
- package/node_modules/typebox/build/schema/engine/contains.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/index.d.mts +1 -0
- package/node_modules/typebox/build/schema/engine/index.mjs +1 -0
- package/node_modules/typebox/build/schema/engine/items.mjs +7 -4
- package/node_modules/typebox/build/schema/engine/maxContains.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/minContains.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/oneOf.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/patternProperties.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/propertyNames.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/unevaluatedItems.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/unevaluatedProperties.mjs +2 -2
- package/node_modules/typebox/build/system/hashing/hash.mjs +63 -34
- package/node_modules/typebox/build/system/locale/en_US.mjs +2 -1
- package/node_modules/typebox/build/system/memory/assign.d.mts +2 -8
- package/node_modules/typebox/build/system/memory/assign.mjs +13 -6
- package/node_modules/typebox/build/system/memory/clone.d.mts +4 -5
- package/node_modules/typebox/build/system/memory/clone.mjs +8 -96
- package/node_modules/typebox/build/system/memory/create.d.mts +1 -5
- package/node_modules/typebox/build/system/memory/create.mjs +18 -13
- package/node_modules/typebox/build/system/memory/deep.d.mts +2 -0
- package/node_modules/typebox/build/system/memory/deep.mjs +108 -0
- package/node_modules/typebox/build/system/memory/discard.d.mts +1 -1
- package/node_modules/typebox/build/system/memory/discard.mjs +9 -8
- package/node_modules/typebox/build/system/memory/finalize.d.mts +4 -0
- package/node_modules/typebox/build/system/memory/finalize.mjs +30 -0
- package/node_modules/typebox/build/system/memory/keys.d.mts +2 -0
- package/node_modules/typebox/build/system/memory/keys.mjs +8 -0
- package/node_modules/typebox/build/system/memory/update.d.mts +1 -4
- package/node_modules/typebox/build/system/memory/update.mjs +43 -16
- package/node_modules/typebox/build/type/engine/call/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/call/index.mjs +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/composite.d.mts +2 -24
- package/node_modules/typebox/build/type/engine/evaluate/composite.mjs +2 -29
- package/node_modules/typebox/build/type/engine/evaluate/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/index.mjs +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/narrow.d.mts +5 -1
- package/node_modules/typebox/build/type/engine/evaluate/narrow.mjs +6 -1
- package/node_modules/typebox/build/type/engine/evaluate/property.d.mts +30 -0
- package/node_modules/typebox/build/type/engine/evaluate/property.mjs +29 -0
- package/node_modules/typebox/build/type/engine/immutable/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/immutable/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/object/from_intersect.d.mts +2 -2
- package/node_modules/typebox/build/type/engine/object/from_intersect.mjs +2 -2
- package/node_modules/typebox/build/type/engine/object/from_union.d.mts +2 -2
- package/node_modules/typebox/build/type/engine/object/from_union.mjs +2 -2
- package/node_modules/typebox/build/type/engine/optional/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/optional/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/readonly/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/readonly/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/this/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/this/index.mjs +1 -0
- package/node_modules/typebox/build/value/clone/clone.d.mts +4 -3
- package/node_modules/typebox/build/value/clone/clone.mjs +5 -7
- package/node_modules/typebox/build/value/repair/from_cyclic.d.mts +2 -0
- package/node_modules/typebox/build/value/repair/from_cyclic.mjs +6 -0
- package/node_modules/typebox/build/value/shared/union_priority_sort.mjs +25 -14
- package/node_modules/typebox/package.json +1 -1
- package/node_modules/typebox/readme.md +1 -1
- package/package.json +5 -5
- package/node_modules/typebox/build/system/memory/freeze.d.mts +0 -4
- package/node_modules/typebox/build/system/memory/freeze.mjs +0 -6
|
@@ -1,12 +1,15 @@
|
|
|
1
|
-
import { resolveSiteName } from "openclaw/plugin-sdk/provider-web-search";
|
|
1
|
+
import { externalContentStamp, readCachedSearchPayload, resolveSearchCacheTtlMs, resolveSiteName, writeCachedSearchPayload } from "openclaw/plugin-sdk/provider-web-search";
|
|
2
|
+
import { sha256Hex } from "openclaw/plugin-sdk/node-crypto-runtime";
|
|
2
3
|
import { asNonNegativeFiniteNumber, asPositiveFiniteNumber, parseDateStringTimestampMs, parseFiniteNumber, resolveIntegerOption } from "openclaw/plugin-sdk/number-runtime";
|
|
4
|
+
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
3
5
|
import { ProviderHttpError, readProviderJsonObjectResponse } from "openclaw/plugin-sdk/provider-http";
|
|
4
|
-
import {
|
|
6
|
+
import { markdownToText, readResponseText, resolvePositiveTimeoutSeconds, withSelfHostedWebToolsEndpoint, withStrictWebToolsEndpoint } from "openclaw/plugin-sdk/provider-web-fetch";
|
|
7
|
+
import { resolveProviderWebSearchPluginConfig } from "openclaw/plugin-sdk/provider-web-search-config-contract";
|
|
5
8
|
import { normalizeSecretInput } from "openclaw/plugin-sdk/secret-input";
|
|
9
|
+
import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
|
|
6
10
|
import { truncateSanitizedExternalContent, wrapExternalContent, wrapWebContent } from "openclaw/plugin-sdk/security-runtime";
|
|
7
11
|
import { SsrFBlockedError, isBlockedHostnameOrIp, isPrivateIpAddress, resolvePinnedHostnameWithPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
|
8
12
|
import { asBoolean, asOptionalObjectRecord, asOptionalRecord, asRecord, normalizeOptionalString, readStringField } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
9
|
-
import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
|
|
10
13
|
// \0rolldown/runtime.js
|
|
11
14
|
var __defProp = Object.defineProperty;
|
|
12
15
|
var __exportAll = (all, no_symbols) => {
|
|
@@ -18,32 +21,35 @@ var __exportAll = (all, no_symbols) => {
|
|
|
18
21
|
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
19
22
|
return target;
|
|
20
23
|
};
|
|
24
|
+
// extensions/firecrawl/src/firecrawl-client.ts
|
|
25
|
+
var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
|
|
26
|
+
runFirecrawlScrape: () => runFirecrawlScrape,
|
|
27
|
+
runFirecrawlSearch: () => runFirecrawlSearch
|
|
28
|
+
});
|
|
29
|
+
const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
|
|
21
30
|
const DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS = 30;
|
|
22
31
|
const DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS = 60;
|
|
23
32
|
const DEFAULT_FIRECRAWL_MAX_AGE_MS = 1728e5;
|
|
24
33
|
const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
const DEFAULT_SEARCH_COUNT = 5;
|
|
35
|
+
const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
|
|
36
|
+
const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
|
|
37
|
+
const DEFAULT_SCRAPE_MAX_CHARS = 5e4;
|
|
38
|
+
const FIRECRAWL_SCRAPE_METADATA_MAX_CHARS = 4e3;
|
|
39
|
+
const FIRECRAWL_RESULT_URL_MAX_CHARS = 2048;
|
|
40
|
+
const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 67108864;
|
|
41
|
+
const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
|
|
42
|
+
const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
|
|
43
|
+
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
|
|
44
|
+
const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
|
|
29
45
|
function resolveFirecrawlFetchConfig(cfg) {
|
|
30
|
-
|
|
31
|
-
return asOptionalRecord(pluginConfig?.webFetch);
|
|
32
|
-
}
|
|
33
|
-
function resolveConfiguredSecret(value, path, cfg) {
|
|
34
|
-
return resolveReadOnlyEnvSecretRef({
|
|
35
|
-
value,
|
|
36
|
-
path,
|
|
37
|
-
cfg,
|
|
38
|
-
expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
|
|
39
|
-
normalizeValue: normalizeSecretInput
|
|
40
|
-
});
|
|
46
|
+
return asOptionalRecord(resolvePluginConfigObject(cfg, "firecrawl")?.webFetch);
|
|
41
47
|
}
|
|
42
48
|
function resolveFirecrawlApiKey(cfg) {
|
|
43
|
-
const
|
|
44
|
-
const search =
|
|
49
|
+
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
50
|
+
const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
|
|
45
51
|
const configuredCandidates = [{
|
|
46
|
-
value:
|
|
52
|
+
value: fetch?.apiKey,
|
|
47
53
|
path: "plugins.entries.firecrawl.config.webFetch.apiKey"
|
|
48
54
|
}, {
|
|
49
55
|
value: search?.apiKey,
|
|
@@ -51,7 +57,13 @@ function resolveFirecrawlApiKey(cfg) {
|
|
|
51
57
|
}];
|
|
52
58
|
let blockedConfiguredSecret = false;
|
|
53
59
|
for (const candidate of configuredCandidates) {
|
|
54
|
-
const resolved =
|
|
60
|
+
const resolved = resolveReadOnlyEnvSecretRef({
|
|
61
|
+
value: candidate.value,
|
|
62
|
+
path: candidate.path,
|
|
63
|
+
cfg,
|
|
64
|
+
expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
|
|
65
|
+
normalizeValue: normalizeSecretInput
|
|
66
|
+
});
|
|
55
67
|
if (resolved.status === "available") return resolved.value;
|
|
56
68
|
if (resolved.status === "blocked") blockedConfiguredSecret = true;
|
|
57
69
|
}
|
|
@@ -59,43 +71,10 @@ function resolveFirecrawlApiKey(cfg) {
|
|
|
59
71
|
return normalizeSecretInput(process.env[FIRECRAWL_API_KEY_ENV_VAR]) || void 0;
|
|
60
72
|
}
|
|
61
73
|
function resolveFirecrawlBaseUrl(cfg) {
|
|
62
|
-
const search =
|
|
63
|
-
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
64
|
-
return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || "https://api.firecrawl.dev";
|
|
65
|
-
}
|
|
66
|
-
function resolveFirecrawlOnlyMainContent(cfg, override) {
|
|
67
|
-
return asBoolean(override) ?? asBoolean(resolveFirecrawlFetchConfig(cfg)?.onlyMainContent) ?? true;
|
|
68
|
-
}
|
|
69
|
-
function resolveFirecrawlMaxAgeMs(cfg, override) {
|
|
70
|
-
return Math.floor(asNonNegativeFiniteNumber(override) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
|
|
71
|
-
}
|
|
72
|
-
function resolveFirecrawlScrapeTimeoutSeconds(cfg, override) {
|
|
74
|
+
const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
|
|
73
75
|
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
74
|
-
return
|
|
75
|
-
}
|
|
76
|
-
function resolveFirecrawlSearchTimeoutSeconds(override) {
|
|
77
|
-
return resolvePositiveTimeoutSeconds(override, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
|
76
|
+
return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || DEFAULT_FIRECRAWL_BASE_URL;
|
|
78
77
|
}
|
|
79
|
-
// extensions/firecrawl/src/firecrawl-client.ts
|
|
80
|
-
var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
|
|
81
|
-
assertFirecrawlScrapeTargetAllowed: () => assertFirecrawlScrapeTargetAllowed,
|
|
82
|
-
parseFirecrawlScrapePayload: () => parseFirecrawlScrapePayload,
|
|
83
|
-
runFirecrawlScrape: () => runFirecrawlScrape,
|
|
84
|
-
runFirecrawlSearch: () => runFirecrawlSearch,
|
|
85
|
-
testing: () => testing
|
|
86
|
-
});
|
|
87
|
-
const SEARCH_CACHE = /* @__PURE__ */ new Map();
|
|
88
|
-
const DEFAULT_SEARCH_COUNT = 5;
|
|
89
|
-
const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
|
|
90
|
-
const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
|
|
91
|
-
const DEFAULT_SCRAPE_MAX_CHARS = 5e4;
|
|
92
|
-
const FIRECRAWL_SCRAPE_METADATA_MAX_CHARS = 4e3;
|
|
93
|
-
const FIRECRAWL_RESULT_URL_MAX_CHARS = 2048;
|
|
94
|
-
const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 67108864;
|
|
95
|
-
const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
|
|
96
|
-
const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
|
|
97
|
-
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
|
|
98
|
-
const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
|
|
99
78
|
function assertFirecrawlScrapeTargetAllowed(url) {
|
|
100
79
|
let parsed;
|
|
101
80
|
try {
|
|
@@ -109,27 +88,24 @@ function assertFirecrawlScrapeTargetAllowed(url) {
|
|
|
109
88
|
function isOfficialFirecrawlEndpoint(url) {
|
|
110
89
|
return url.protocol === "https:" && ALLOWED_FIRECRAWL_HOSTS.has(url.hostname);
|
|
111
90
|
}
|
|
112
|
-
async function firecrawlEndpointTargetsPrivateNetwork(url
|
|
91
|
+
async function firecrawlEndpointTargetsPrivateNetwork(url) {
|
|
113
92
|
if (isBlockedHostnameOrIp(url.hostname)) return true;
|
|
114
93
|
try {
|
|
115
|
-
return (await resolvePinnedHostnameWithPolicy(url.hostname, {
|
|
116
|
-
lookupFn,
|
|
117
|
-
policy: { allowPrivateNetwork: true }
|
|
118
|
-
})).addresses.every((address) => isPrivateIpAddress(address));
|
|
94
|
+
return (await resolvePinnedHostnameWithPolicy(url.hostname, { policy: { allowPrivateNetwork: true } })).addresses.every((address) => isPrivateIpAddress(address));
|
|
119
95
|
} catch {
|
|
120
96
|
return false;
|
|
121
97
|
}
|
|
122
98
|
}
|
|
123
|
-
async function validateFirecrawlBaseUrl(url
|
|
99
|
+
async function validateFirecrawlBaseUrl(url) {
|
|
124
100
|
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("Firecrawl baseUrl must use http:// or https://.");
|
|
125
101
|
if (isOfficialFirecrawlEndpoint(url)) return "strict";
|
|
126
|
-
if (await firecrawlEndpointTargetsPrivateNetwork(url
|
|
102
|
+
if (await firecrawlEndpointTargetsPrivateNetwork(url)) return "selfHosted";
|
|
127
103
|
if (url.protocol === "http:") throw new Error(FIRECRAWL_HTTP_PRIVATE_ERROR);
|
|
128
104
|
throw new Error(`${FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR} Host: ${url.hostname}`);
|
|
129
105
|
}
|
|
130
|
-
async function resolveEndpoint(baseUrl, pathname
|
|
131
|
-
const url = new URL(baseUrl.trim() ||
|
|
132
|
-
const mode = await validateFirecrawlBaseUrl(url
|
|
106
|
+
async function resolveEndpoint(baseUrl, pathname) {
|
|
107
|
+
const url = new URL(baseUrl.trim() || DEFAULT_FIRECRAWL_BASE_URL);
|
|
108
|
+
const mode = await validateFirecrawlBaseUrl(url);
|
|
133
109
|
url.username = "";
|
|
134
110
|
url.password = "";
|
|
135
111
|
url.search = "";
|
|
@@ -245,20 +221,21 @@ async function runFirecrawlSearch(params) {
|
|
|
245
221
|
min: 1,
|
|
246
222
|
max: 100
|
|
247
223
|
});
|
|
248
|
-
const timeoutSeconds =
|
|
224
|
+
const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
|
249
225
|
const scrapeResults = params.scrapeResults === true;
|
|
250
|
-
const sources =
|
|
251
|
-
const categories =
|
|
252
|
-
const includeDomains =
|
|
253
|
-
const excludeDomains =
|
|
226
|
+
const sources = params.sources?.filter(Boolean) ?? [];
|
|
227
|
+
const categories = params.categories?.filter(Boolean) ?? [];
|
|
228
|
+
const includeDomains = params.includeDomains?.filter(Boolean) ?? [];
|
|
229
|
+
const excludeDomains = params.excludeDomains?.filter(Boolean) ?? [];
|
|
254
230
|
if (includeDomains.length > 0 && excludeDomains.length > 0) throw new Error("Firecrawl search accepts includeDomains or excludeDomains, not both.");
|
|
255
231
|
const tbs = normalizeOptionalString(params.tbs);
|
|
256
232
|
const location = normalizeOptionalString(params.location);
|
|
257
233
|
const country = normalizeOptionalString(params.country);
|
|
258
234
|
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
259
|
-
const cacheKey =
|
|
235
|
+
const cacheKey = JSON.stringify({
|
|
260
236
|
type: "firecrawl-search",
|
|
261
237
|
provider: providerId,
|
|
238
|
+
credential: apiKey ? sha256Hex(apiKey) : void 0,
|
|
262
239
|
q: params.query,
|
|
263
240
|
count,
|
|
264
241
|
baseUrl,
|
|
@@ -270,13 +247,10 @@ async function runFirecrawlSearch(params) {
|
|
|
270
247
|
location,
|
|
271
248
|
country,
|
|
272
249
|
scrapeResults
|
|
273
|
-
})
|
|
274
|
-
const cacheTtlMs =
|
|
275
|
-
const cached =
|
|
276
|
-
if (cached) return
|
|
277
|
-
...cached.value,
|
|
278
|
-
cached: true
|
|
279
|
-
};
|
|
250
|
+
});
|
|
251
|
+
const cacheTtlMs = resolveSearchCacheTtlMs(params.cfg?.tools?.web?.search);
|
|
252
|
+
const cached = readCachedSearchPayload(cacheKey, cacheTtlMs);
|
|
253
|
+
if (cached) return cached;
|
|
280
254
|
const body = {
|
|
281
255
|
query: params.query,
|
|
282
256
|
limit: count,
|
|
@@ -330,20 +304,52 @@ async function runFirecrawlSearch(params) {
|
|
|
330
304
|
provider: providerId,
|
|
331
305
|
count: items.length,
|
|
332
306
|
tookMs,
|
|
333
|
-
externalContent:
|
|
334
|
-
untrusted: true,
|
|
335
|
-
source: "web_search",
|
|
336
|
-
provider: providerId,
|
|
337
|
-
wrapped: true
|
|
338
|
-
},
|
|
307
|
+
externalContent: externalContentStamp(providerId),
|
|
339
308
|
results,
|
|
340
309
|
...truncated ? { truncated: true } : {}
|
|
341
310
|
};
|
|
342
|
-
|
|
311
|
+
writeCachedSearchPayload(cacheKey, result, cacheTtlMs);
|
|
343
312
|
return result;
|
|
344
313
|
}
|
|
345
|
-
function
|
|
346
|
-
|
|
314
|
+
async function runFirecrawlScrape(params) {
|
|
315
|
+
params.signal?.throwIfAborted();
|
|
316
|
+
assertFirecrawlScrapeTargetAllowed(params.url);
|
|
317
|
+
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
|
318
|
+
if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
|
|
319
|
+
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
320
|
+
const fetch = resolveFirecrawlFetchConfig(params.cfg);
|
|
321
|
+
const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
|
|
322
|
+
const onlyMainContent = asBoolean(params.onlyMainContent) ?? asBoolean(resolveFirecrawlFetchConfig(params.cfg)?.onlyMainContent) ?? true;
|
|
323
|
+
const maxAgeMs = Math.floor(asNonNegativeFiniteNumber(params.maxAgeMs) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(params.cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
|
|
324
|
+
const proxy = params.proxy ?? "auto";
|
|
325
|
+
const storeInCache = params.storeInCache ?? true;
|
|
326
|
+
const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
|
|
327
|
+
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
|
328
|
+
const payload = await postFirecrawlJson({
|
|
329
|
+
url: endpoint.url,
|
|
330
|
+
mode: endpoint.mode,
|
|
331
|
+
timeoutSeconds,
|
|
332
|
+
apiKey,
|
|
333
|
+
errorLabel: "Firecrawl",
|
|
334
|
+
...params.signal ? { signal: params.signal } : {},
|
|
335
|
+
body: {
|
|
336
|
+
url: params.url,
|
|
337
|
+
formats: ["markdown"],
|
|
338
|
+
onlyMainContent,
|
|
339
|
+
timeout: timeoutSeconds * 1e3,
|
|
340
|
+
maxAge: maxAgeMs,
|
|
341
|
+
proxy,
|
|
342
|
+
storeInCache
|
|
343
|
+
}
|
|
344
|
+
}, async (response) => {
|
|
345
|
+
const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
|
|
346
|
+
if (data.success === false) {
|
|
347
|
+
const detail = readFirecrawlErrorDetail(data, response.statusText);
|
|
348
|
+
throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
|
|
349
|
+
}
|
|
350
|
+
return data;
|
|
351
|
+
});
|
|
352
|
+
const data = asRecord(payload.data);
|
|
347
353
|
const metadata = asOptionalObjectRecord(data.metadata);
|
|
348
354
|
const rawStatus = parseFiniteNumber(metadata?.statusCode) ?? parseFiniteNumber(data.statusCode);
|
|
349
355
|
const status = rawStatus === void 0 ? void 0 : Math.floor(rawStatus);
|
|
@@ -351,7 +357,7 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
351
357
|
const markdown = typeof data.markdown === "string" && data.markdown || typeof data.content === "string" && data.content || "";
|
|
352
358
|
if (!markdown) throw new Error("Firecrawl scrape returned no content.");
|
|
353
359
|
const rawText = params.extractMode === "text" ? markdownToText(markdown) : markdown;
|
|
354
|
-
const boundedText = truncateSanitizedExternalContent(rawText,
|
|
360
|
+
const boundedText = truncateSanitizedExternalContent(rawText, maxChars);
|
|
355
361
|
let truncated = boundedText.truncated;
|
|
356
362
|
let remainingMetadataChars = FIRECRAWL_SCRAPE_METADATA_MAX_CHARS;
|
|
357
363
|
const wrapBoundedMetadata = (value) => {
|
|
@@ -368,7 +374,7 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
368
374
|
includeWarning: false
|
|
369
375
|
});
|
|
370
376
|
const title = typeof metadata?.title === "string" && metadata.title ? wrapBoundedMetadata(metadata.title) : void 0;
|
|
371
|
-
const warning = typeof
|
|
377
|
+
const warning = typeof payload.warning === "string" && payload.warning ? wrapBoundedMetadata(payload.warning) : void 0;
|
|
372
378
|
return {
|
|
373
379
|
url: params.url,
|
|
374
380
|
finalUrl: normalizeFirecrawlResultUrl(metadata?.sourceURL) ?? normalizeFirecrawlResultUrl(data.url) ?? params.url,
|
|
@@ -388,53 +394,4 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
388
394
|
...warning ? { warning } : {}
|
|
389
395
|
};
|
|
390
396
|
}
|
|
391
|
-
async function runFirecrawlScrape(params) {
|
|
392
|
-
params.signal?.throwIfAborted();
|
|
393
|
-
assertFirecrawlScrapeTargetAllowed(params.url);
|
|
394
|
-
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
|
395
|
-
if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
|
|
396
|
-
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
397
|
-
const timeoutSeconds = resolveFirecrawlScrapeTimeoutSeconds(params.cfg, params.timeoutSeconds);
|
|
398
|
-
const onlyMainContent = resolveFirecrawlOnlyMainContent(params.cfg, params.onlyMainContent);
|
|
399
|
-
const maxAgeMs = resolveFirecrawlMaxAgeMs(params.cfg, params.maxAgeMs);
|
|
400
|
-
const proxy = params.proxy ?? "auto";
|
|
401
|
-
const storeInCache = params.storeInCache ?? true;
|
|
402
|
-
const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
|
|
403
|
-
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
|
404
|
-
return parseFirecrawlScrapePayload({
|
|
405
|
-
payload: await postFirecrawlJson({
|
|
406
|
-
url: endpoint.url,
|
|
407
|
-
mode: endpoint.mode,
|
|
408
|
-
timeoutSeconds,
|
|
409
|
-
apiKey,
|
|
410
|
-
errorLabel: "Firecrawl",
|
|
411
|
-
...params.signal ? { signal: params.signal } : {},
|
|
412
|
-
body: {
|
|
413
|
-
url: params.url,
|
|
414
|
-
formats: ["markdown"],
|
|
415
|
-
onlyMainContent,
|
|
416
|
-
timeout: timeoutSeconds * 1e3,
|
|
417
|
-
maxAge: maxAgeMs,
|
|
418
|
-
proxy,
|
|
419
|
-
storeInCache
|
|
420
|
-
}
|
|
421
|
-
}, async (response) => {
|
|
422
|
-
const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
|
|
423
|
-
if (data.success === false) {
|
|
424
|
-
const detail = readFirecrawlErrorDetail(data, response.statusText);
|
|
425
|
-
throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
|
|
426
|
-
}
|
|
427
|
-
return data;
|
|
428
|
-
}),
|
|
429
|
-
url: params.url,
|
|
430
|
-
extractMode: params.extractMode,
|
|
431
|
-
maxChars
|
|
432
|
-
});
|
|
433
|
-
}
|
|
434
|
-
const testing = {
|
|
435
|
-
assertFirecrawlScrapeTargetAllowed,
|
|
436
|
-
parseFirecrawlScrapePayload,
|
|
437
|
-
resolveEndpoint,
|
|
438
|
-
resolveSearchItems
|
|
439
|
-
};
|
|
440
397
|
export { runFirecrawlScrape as n, runFirecrawlSearch as r, firecrawl_client_exports as t };
|
package/dist/.setup/{firecrawl-fetch-provider-EP1XFhDE.mjs → firecrawl-fetch-provider-QmPccHZS.mjs}
RENAMED
|
@@ -1,14 +1,8 @@
|
|
|
1
1
|
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
|
2
2
|
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
|
3
3
|
import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch-contract";
|
|
4
|
+
import { ensureRecord } from "openclaw/plugin-sdk/collection-input-runtime";
|
|
4
5
|
// extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
|
|
5
|
-
function ensureRecord(target, key) {
|
|
6
|
-
const current = target[key];
|
|
7
|
-
if (current && typeof current === "object" && !Array.isArray(current)) return current;
|
|
8
|
-
const next = {};
|
|
9
|
-
target[key] = next;
|
|
10
|
-
return next;
|
|
11
|
-
}
|
|
12
6
|
const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
|
13
7
|
id: "firecrawl",
|
|
14
8
|
label: "Firecrawl",
|
|
@@ -45,12 +39,13 @@ const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
|
|
45
39
|
const plugins = configTarget.plugins ??= {};
|
|
46
40
|
const entries = plugins.entries ??= {};
|
|
47
41
|
const firecrawlEntry = entries.firecrawl ??= {};
|
|
48
|
-
const
|
|
42
|
+
const pluginConfig = firecrawlEntry.config ??= {};
|
|
43
|
+
const webFetch = ensureRecord(pluginConfig, "webFetch");
|
|
49
44
|
webFetch.apiKey = value;
|
|
50
45
|
}
|
|
51
46
|
};
|
|
52
47
|
// extensions/firecrawl/src/firecrawl-fetch-provider.ts
|
|
53
|
-
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-
|
|
48
|
+
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CDyLQvtN.mjs").then((n) => n.t));
|
|
54
49
|
function createFirecrawlWebFetchProvider() {
|
|
55
50
|
return {
|
|
56
51
|
...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
|
|
@@ -1,29 +1,18 @@
|
|
|
1
|
+
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
2
|
+
import { asOptionalRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
1
3
|
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
|
2
4
|
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
|
3
|
-
import {
|
|
4
|
-
import { createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
|
|
5
|
+
import { WEB_SEARCH_QUERY_COUNT_PROPERTIES, createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
|
|
5
6
|
// extensions/firecrawl/src/firecrawl-search-provider.ts
|
|
6
7
|
const FIRECRAWL_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webSearch.apiKey";
|
|
7
|
-
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-
|
|
8
|
+
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CDyLQvtN.mjs").then((n) => n.t));
|
|
8
9
|
const GenericFirecrawlSearchSchema = {
|
|
9
10
|
type: "object",
|
|
10
|
-
properties: {
|
|
11
|
-
query: {
|
|
12
|
-
type: "string",
|
|
13
|
-
description: "Search query string."
|
|
14
|
-
},
|
|
15
|
-
count: {
|
|
16
|
-
type: "integer",
|
|
17
|
-
description: "Number of results to return (1-10).",
|
|
18
|
-
minimum: 1,
|
|
19
|
-
maximum: 10
|
|
20
|
-
}
|
|
21
|
-
},
|
|
11
|
+
properties: { ...WEB_SEARCH_QUERY_COUNT_PROPERTIES },
|
|
22
12
|
additionalProperties: false
|
|
23
13
|
};
|
|
24
14
|
function getConfiguredFirecrawlFetchCredentialFallback(config) {
|
|
25
|
-
const
|
|
26
|
-
const value = asOptionalRecord(pluginConfig?.webFetch)?.apiKey;
|
|
15
|
+
const value = asOptionalRecord(resolvePluginConfigObject(config, "firecrawl")?.webFetch)?.apiKey;
|
|
27
16
|
return value === void 0 ? void 0 : {
|
|
28
17
|
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
|
|
29
18
|
value
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-
|
|
1
|
+
{"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-CDyLQvtN.mjs","sha256":"46c98acf8d71a8310851b8a6bfeb3d7e1176edb3d040c8e7d1de958e62cd4028","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-client.ts"]},{"output":".setup/firecrawl-fetch-provider-QmPccHZS.mjs","sha256":"d32ea4f6248fc6c0c388257809db11ac0107a9a7c473ac75a0f665f71dae723c","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts","extensions/firecrawl/src/firecrawl-fetch-provider.ts"]},{"output":".setup/firecrawl-search-provider-wsRgx6kl.mjs","sha256":"45416f284d535281a900a1ea20307652ca911769a3f36e9cbc5916a29a2ceafd","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-search-provider.ts"]},{"output":"api.js","sha256":"ba89572c128911e1fb033a07004fa1c36861338a6704705ea14c6d6a1ea1cea3","facadeSource":"extensions/firecrawl/api.ts","sources":["extensions/firecrawl/api.ts"]},{"output":"index.js","sha256":"1b6689bdf7d803a983f86a4371e153139d127a259720af5d6412d102ecb1a3e5","facadeSource":"extensions/firecrawl/index.ts","sources":["extensions/firecrawl/index.ts"]},{"output":"web-fetch-contract-api.js","sha256":"e9f57ffa95524a8f5d9ac59cd879f41ec5c43e6e3ffd236a3bcfc6f55e2c439d","facadeSource":"extensions/firecrawl/web-fetch-contract-api.ts","sources":[]},{"output":"web-fetch-provider.js","sha256":"55e667f670164dd167c874b72e48fcba2b7ed2624d0d26f66d1fa78067f5b0b5","facadeSource":"extensions/firecrawl/web-fetch-provider.ts","sources":[]},{"output":"web-search-contract-api.js","sha256":"eb2a1d89385a524e030bc88d18d86398e5d9baf5bdeb57f9f4608017692541e4","facadeSource":"extensions/firecrawl/web-search-contract-api.ts","sources":[]},{"output":"web-search-provider.js","sha256":"54d793c602424e24a8e6f17c2f8eb832c5635c77e9094a1ca97c9b1249b17cb0","facadeSource":"extensions/firecrawl/web-search-provider.ts","sources":[]}],"unrenderedSources":["extensions/firecrawl/web-fetch-contract-api.ts","extensions/firecrawl/web-fetch-provider.ts","extensions/firecrawl/web-search-contract-api.ts","extensions/firecrawl/web-search-provider.ts"]}
|
package/dist/api.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-
|
|
1
|
+
import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-CDyLQvtN.mjs";
|
|
2
2
|
// extensions/firecrawl/api.ts
|
|
3
3
|
async function fetchFirecrawlContent(params) {
|
|
4
4
|
const cfg = { plugins: { entries: { firecrawl: {
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { n as runFirecrawlScrape, r as runFirecrawlSearch } from "./.setup/firecrawl-client-
|
|
2
|
-
import { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-
|
|
3
|
-
import { n as createFirecrawlWebSearchProvider, t as createFirecrawlFreeWebSearchProvider } from "./.setup/firecrawl-search-provider-
|
|
1
|
+
import { n as runFirecrawlScrape, r as runFirecrawlSearch } from "./.setup/firecrawl-client-CDyLQvtN.mjs";
|
|
2
|
+
import { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-QmPccHZS.mjs";
|
|
3
|
+
import { n as createFirecrawlWebSearchProvider, t as createFirecrawlFreeWebSearchProvider } from "./.setup/firecrawl-search-provider-wsRgx6kl.mjs";
|
|
4
|
+
import { optionalNonNegativeIntegerSchema, optionalPositiveIntegerSchema, optionalStringEnum } from "openclaw/plugin-sdk/channel-actions";
|
|
4
5
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
5
|
-
import { optionalStringEnum } from "openclaw/plugin-sdk/channel-actions";
|
|
6
6
|
import { jsonResult, readNonNegativeIntegerParam, readPositiveIntegerParam, readStringArrayParam, readStringParam } from "openclaw/plugin-sdk/provider-web-search";
|
|
7
7
|
import { Type } from "typebox";
|
|
8
|
-
// extensions/firecrawl/
|
|
8
|
+
// extensions/firecrawl/index.ts
|
|
9
9
|
const FirecrawlScrapeToolSchema = Type.Object({
|
|
10
10
|
url: Type.String({ description: "HTTP or HTTPS URL to scrape via Firecrawl." }),
|
|
11
11
|
extractMode: optionalStringEnum(["markdown", "text"], { description: "Extraction mode (\"markdown\" or \"text\"). Default: markdown." }),
|
|
@@ -14,62 +14,21 @@ const FirecrawlScrapeToolSchema = Type.Object({
|
|
|
14
14
|
minimum: 100
|
|
15
15
|
})),
|
|
16
16
|
onlyMainContent: Type.Optional(Type.Boolean({ description: "Keep only main content when Firecrawl supports it." })),
|
|
17
|
-
maxAgeMs:
|
|
18
|
-
description: "Maximum Firecrawl cache age in milliseconds.",
|
|
19
|
-
minimum: 0
|
|
20
|
-
})),
|
|
17
|
+
maxAgeMs: optionalNonNegativeIntegerSchema({ description: "Maximum Firecrawl cache age in milliseconds." }),
|
|
21
18
|
proxy: optionalStringEnum([
|
|
22
19
|
"auto",
|
|
23
20
|
"basic",
|
|
24
21
|
"stealth"
|
|
25
22
|
], { description: "Firecrawl proxy mode (\"auto\", \"basic\", or \"stealth\")." }),
|
|
26
23
|
storeInCache: Type.Optional(Type.Boolean({ description: "Whether Firecrawl should store the scrape in its cache." })),
|
|
27
|
-
timeoutSeconds:
|
|
28
|
-
description: "Timeout in seconds for the Firecrawl scrape request.",
|
|
29
|
-
minimum: 1
|
|
30
|
-
}))
|
|
24
|
+
timeoutSeconds: optionalPositiveIntegerSchema({ description: "Timeout in seconds for the Firecrawl scrape request." })
|
|
31
25
|
}, { additionalProperties: false });
|
|
32
|
-
function createFirecrawlScrapeTool(api) {
|
|
33
|
-
return {
|
|
34
|
-
name: "firecrawl_scrape",
|
|
35
|
-
label: "Firecrawl Scrape",
|
|
36
|
-
resultContentSource: "network",
|
|
37
|
-
description: "Scrape a page using Firecrawl v2/scrape. Useful for JS-heavy or bot-protected pages where a plain URL fetch is weak.",
|
|
38
|
-
parameters: FirecrawlScrapeToolSchema,
|
|
39
|
-
execute: async (_toolCallId, rawParams, signal) => {
|
|
40
|
-
signal?.throwIfAborted();
|
|
41
|
-
const url = readStringParam(rawParams, "url", { required: true });
|
|
42
|
-
const extractMode = readStringParam(rawParams, "extractMode") === "text" ? "text" : "markdown";
|
|
43
|
-
const maxChars = readPositiveIntegerParam(rawParams, "maxChars");
|
|
44
|
-
const maxAgeMs = readNonNegativeIntegerParam(rawParams, "maxAgeMs");
|
|
45
|
-
const timeoutSeconds = readPositiveIntegerParam(rawParams, "timeoutSeconds");
|
|
46
|
-
const proxyRaw = readStringParam(rawParams, "proxy");
|
|
47
|
-
const proxy = proxyRaw === "basic" || proxyRaw === "stealth" || proxyRaw === "auto" ? proxyRaw : void 0;
|
|
48
|
-
const onlyMainContent = typeof rawParams.onlyMainContent === "boolean" ? rawParams.onlyMainContent : void 0;
|
|
49
|
-
const storeInCache = typeof rawParams.storeInCache === "boolean" ? rawParams.storeInCache : void 0;
|
|
50
|
-
return jsonResult(await runFirecrawlScrape({
|
|
51
|
-
cfg: api.config,
|
|
52
|
-
url,
|
|
53
|
-
extractMode,
|
|
54
|
-
maxChars,
|
|
55
|
-
onlyMainContent,
|
|
56
|
-
maxAgeMs,
|
|
57
|
-
proxy,
|
|
58
|
-
storeInCache,
|
|
59
|
-
timeoutSeconds,
|
|
60
|
-
...signal ? { signal } : {}
|
|
61
|
-
}));
|
|
62
|
-
}
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
// extensions/firecrawl/src/firecrawl-search-tool.ts
|
|
66
26
|
const FirecrawlSearchToolSchema = Type.Object({
|
|
67
27
|
query: Type.String({ description: "Search query string." }),
|
|
68
|
-
count:
|
|
28
|
+
count: optionalPositiveIntegerSchema({
|
|
69
29
|
description: "Number of results to return (1-100).",
|
|
70
|
-
minimum: 1,
|
|
71
30
|
maximum: 100
|
|
72
|
-
})
|
|
31
|
+
}),
|
|
73
32
|
sources: Type.Optional(Type.Array(Type.String(), { description: "Optional sources list, for example [\"web\"], [\"news\"], or [\"images\"]." })),
|
|
74
33
|
categories: Type.Optional(Type.Array(Type.String(), { description: "Optional Firecrawl categories, for example [\"github\"] or [\"research\"]." })),
|
|
75
34
|
includeDomains: Type.Optional(Type.Array(Type.String(), { description: "Restrict results to these hostnames (no protocol or path). Cannot be combined with excludeDomains." })),
|
|
@@ -78,42 +37,8 @@ const FirecrawlSearchToolSchema = Type.Object({
|
|
|
78
37
|
location: Type.Optional(Type.String({ description: "Geo-target location, for example \"Germany\" or \"San Francisco,California,United States\"." })),
|
|
79
38
|
country: Type.Optional(Type.String({ description: "ISO country code for geo-targeting, for example \"US\", \"DE\", or \"JP\"." })),
|
|
80
39
|
scrapeResults: Type.Optional(Type.Boolean({ description: "Include scraped result content when Firecrawl returns it." })),
|
|
81
|
-
timeoutSeconds:
|
|
82
|
-
description: "Timeout in seconds for the Firecrawl Search request.",
|
|
83
|
-
minimum: 1
|
|
84
|
-
}))
|
|
40
|
+
timeoutSeconds: optionalPositiveIntegerSchema({ description: "Timeout in seconds for the Firecrawl Search request." })
|
|
85
41
|
}, { additionalProperties: false });
|
|
86
|
-
function createFirecrawlSearchTool(api) {
|
|
87
|
-
return {
|
|
88
|
-
name: "firecrawl_search",
|
|
89
|
-
label: "Firecrawl Search",
|
|
90
|
-
resultContentSource: "network",
|
|
91
|
-
description: "Search the web using Firecrawl v2/search. Supports includeDomains/excludeDomains filtering and tbs time filters (day/week/month/year). Can optionally include scraped content from result pages.",
|
|
92
|
-
parameters: FirecrawlSearchToolSchema,
|
|
93
|
-
execute: async (_toolCallId, rawParams, signal) => {
|
|
94
|
-
signal?.throwIfAborted();
|
|
95
|
-
return jsonResult(await runFirecrawlSearch({
|
|
96
|
-
query: readStringParam(rawParams, "query", { required: true }),
|
|
97
|
-
count: readPositiveIntegerParam(rawParams, "count", {
|
|
98
|
-
max: 100,
|
|
99
|
-
message: "count must be an integer from 1 to 100"
|
|
100
|
-
}),
|
|
101
|
-
timeoutSeconds: readPositiveIntegerParam(rawParams, "timeoutSeconds"),
|
|
102
|
-
sources: readStringArrayParam(rawParams, "sources"),
|
|
103
|
-
categories: readStringArrayParam(rawParams, "categories"),
|
|
104
|
-
includeDomains: readStringArrayParam(rawParams, "includeDomains"),
|
|
105
|
-
excludeDomains: readStringArrayParam(rawParams, "excludeDomains"),
|
|
106
|
-
tbs: readStringParam(rawParams, "tbs"),
|
|
107
|
-
location: readStringParam(rawParams, "location"),
|
|
108
|
-
country: readStringParam(rawParams, "country"),
|
|
109
|
-
scrapeResults: rawParams.scrapeResults === true,
|
|
110
|
-
cfg: api.config,
|
|
111
|
-
...signal ? { signal } : {}
|
|
112
|
-
}));
|
|
113
|
-
}
|
|
114
|
-
};
|
|
115
|
-
}
|
|
116
|
-
// extensions/firecrawl/index.ts
|
|
117
42
|
var firecrawl_default = definePluginEntry({
|
|
118
43
|
id: "firecrawl",
|
|
119
44
|
name: "Firecrawl Plugin",
|
|
@@ -122,8 +47,65 @@ var firecrawl_default = definePluginEntry({
|
|
|
122
47
|
api.registerWebFetchProvider(createFirecrawlWebFetchProvider());
|
|
123
48
|
api.registerWebSearchProvider(createFirecrawlWebSearchProvider());
|
|
124
49
|
api.registerWebSearchProvider(createFirecrawlFreeWebSearchProvider());
|
|
125
|
-
api.registerTool(
|
|
126
|
-
|
|
50
|
+
api.registerTool({
|
|
51
|
+
name: "firecrawl_search",
|
|
52
|
+
label: "Firecrawl Search",
|
|
53
|
+
resultContentSource: "network",
|
|
54
|
+
description: "Search the web using Firecrawl v2/search. Supports includeDomains/excludeDomains filtering and tbs time filters (day/week/month/year). Can optionally include scraped content from result pages.",
|
|
55
|
+
parameters: FirecrawlSearchToolSchema,
|
|
56
|
+
execute: async (_toolCallId, rawParams, signal) => {
|
|
57
|
+
signal?.throwIfAborted();
|
|
58
|
+
return jsonResult(await runFirecrawlSearch({
|
|
59
|
+
query: readStringParam(rawParams, "query", { required: true }),
|
|
60
|
+
count: readPositiveIntegerParam(rawParams, "count", {
|
|
61
|
+
max: 100,
|
|
62
|
+
message: "count must be an integer from 1 to 100"
|
|
63
|
+
}),
|
|
64
|
+
timeoutSeconds: readPositiveIntegerParam(rawParams, "timeoutSeconds"),
|
|
65
|
+
sources: readStringArrayParam(rawParams, "sources"),
|
|
66
|
+
categories: readStringArrayParam(rawParams, "categories"),
|
|
67
|
+
includeDomains: readStringArrayParam(rawParams, "includeDomains"),
|
|
68
|
+
excludeDomains: readStringArrayParam(rawParams, "excludeDomains"),
|
|
69
|
+
tbs: readStringParam(rawParams, "tbs"),
|
|
70
|
+
location: readStringParam(rawParams, "location"),
|
|
71
|
+
country: readStringParam(rawParams, "country"),
|
|
72
|
+
scrapeResults: rawParams.scrapeResults === true,
|
|
73
|
+
cfg: api.config,
|
|
74
|
+
...signal ? { signal } : {}
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
api.registerTool({
|
|
79
|
+
name: "firecrawl_scrape",
|
|
80
|
+
label: "Firecrawl Scrape",
|
|
81
|
+
resultContentSource: "network",
|
|
82
|
+
description: "Scrape a page using Firecrawl v2/scrape. Useful for JS-heavy or bot-protected pages where a plain URL fetch is weak.",
|
|
83
|
+
parameters: FirecrawlScrapeToolSchema,
|
|
84
|
+
execute: async (_toolCallId, rawParams, signal) => {
|
|
85
|
+
signal?.throwIfAborted();
|
|
86
|
+
const url = readStringParam(rawParams, "url", { required: true });
|
|
87
|
+
const extractMode = readStringParam(rawParams, "extractMode") === "text" ? "text" : "markdown";
|
|
88
|
+
const maxChars = readPositiveIntegerParam(rawParams, "maxChars");
|
|
89
|
+
const maxAgeMs = readNonNegativeIntegerParam(rawParams, "maxAgeMs");
|
|
90
|
+
const timeoutSeconds = readPositiveIntegerParam(rawParams, "timeoutSeconds");
|
|
91
|
+
const proxyRaw = readStringParam(rawParams, "proxy");
|
|
92
|
+
const proxy = proxyRaw === "basic" || proxyRaw === "stealth" || proxyRaw === "auto" ? proxyRaw : void 0;
|
|
93
|
+
const onlyMainContent = typeof rawParams.onlyMainContent === "boolean" ? rawParams.onlyMainContent : void 0;
|
|
94
|
+
const storeInCache = typeof rawParams.storeInCache === "boolean" ? rawParams.storeInCache : void 0;
|
|
95
|
+
return jsonResult(await runFirecrawlScrape({
|
|
96
|
+
cfg: api.config,
|
|
97
|
+
url,
|
|
98
|
+
extractMode,
|
|
99
|
+
maxChars,
|
|
100
|
+
onlyMainContent,
|
|
101
|
+
maxAgeMs,
|
|
102
|
+
proxy,
|
|
103
|
+
storeInCache,
|
|
104
|
+
timeoutSeconds,
|
|
105
|
+
...signal ? { signal } : {}
|
|
106
|
+
}));
|
|
107
|
+
}
|
|
108
|
+
});
|
|
127
109
|
}
|
|
128
110
|
});
|
|
129
111
|
export { firecrawl_default as default };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-
|
|
1
|
+
export { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-QmPccHZS.mjs";
|
|
2
2
|
import "./web-fetch-provider.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-
|
|
1
|
+
export { t as createFirecrawlWebFetchProvider } from "./.setup/firecrawl-fetch-provider-QmPccHZS.mjs";
|