@openclaw/firecrawl-plugin 2026.10.10-beta.5 → 2026.11.5-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.setup/{firecrawl-client-CMLsufX8.mjs → firecrawl-client-CDyLQvtN.mjs} +98 -143
- package/dist/.setup/{firecrawl-fetch-provider-uRENNKri.mjs → firecrawl-fetch-provider-QmPccHZS.mjs} +6 -13
- package/dist/.setup/{firecrawl-search-provider-CFhWH8gx.mjs → firecrawl-search-provider-wsRgx6kl.mjs} +7 -19
- package/dist/.source-module-urls/ad8205c790d921269d4ea28d22ab658579b9674fd00e025664fcf0b8ea32155c.json +1 -1
- package/dist/api.js +2 -3
- package/dist/index.js +69 -90
- package/dist/web-fetch-contract-api.js +1 -1
- package/dist/web-fetch-provider.js +1 -1
- package/dist/web-search-contract-api.js +1 -1
- package/dist/web-search-provider.js +1 -1
- package/node_modules/typebox/build/format/duration.d.mts +2 -2
- package/node_modules/typebox/build/format/duration.mjs +2 -2
- package/node_modules/typebox/build/format/idn_email.d.mts +1 -0
- package/node_modules/typebox/build/format/idn_email.mjs +1 -0
- package/node_modules/typebox/build/format/idn_hostname.d.mts +1 -1
- package/node_modules/typebox/build/format/idn_hostname.mjs +1 -1
- package/node_modules/typebox/build/format/ipv4.d.mts +2 -2
- package/node_modules/typebox/build/format/ipv4.mjs +2 -2
- package/node_modules/typebox/build/format/ipv6.d.mts +1 -1
- package/node_modules/typebox/build/format/ipv6.mjs +1 -1
- package/node_modules/typebox/build/format/relative_json_pointer.d.mts +2 -2
- package/node_modules/typebox/build/format/relative_json_pointer.mjs +2 -2
- package/node_modules/typebox/build/schema/build.mjs +1 -0
- package/node_modules/typebox/build/schema/engine/_unique.d.mts +4 -2
- package/node_modules/typebox/build/schema/engine/_unique.mjs +7 -3
- package/node_modules/typebox/build/schema/engine/additionalItems.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/additionalProperties.d.mts +0 -3
- package/node_modules/typebox/build/schema/engine/additionalProperties.mjs +15 -17
- package/node_modules/typebox/build/schema/engine/contains.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/index.d.mts +1 -0
- package/node_modules/typebox/build/schema/engine/index.mjs +1 -0
- package/node_modules/typebox/build/schema/engine/items.mjs +7 -4
- package/node_modules/typebox/build/schema/engine/maxContains.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/minContains.mjs +3 -3
- package/node_modules/typebox/build/schema/engine/oneOf.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/patternProperties.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/propertyNames.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/unevaluatedItems.mjs +2 -2
- package/node_modules/typebox/build/schema/engine/unevaluatedProperties.mjs +2 -2
- package/node_modules/typebox/build/system/hashing/hash.mjs +63 -34
- package/node_modules/typebox/build/system/locale/en_US.mjs +2 -1
- package/node_modules/typebox/build/system/memory/assign.d.mts +2 -8
- package/node_modules/typebox/build/system/memory/assign.mjs +13 -6
- package/node_modules/typebox/build/system/memory/clone.d.mts +4 -5
- package/node_modules/typebox/build/system/memory/clone.mjs +8 -96
- package/node_modules/typebox/build/system/memory/create.d.mts +1 -5
- package/node_modules/typebox/build/system/memory/create.mjs +18 -13
- package/node_modules/typebox/build/system/memory/deep.d.mts +2 -0
- package/node_modules/typebox/build/system/memory/deep.mjs +108 -0
- package/node_modules/typebox/build/system/memory/discard.d.mts +1 -1
- package/node_modules/typebox/build/system/memory/discard.mjs +9 -8
- package/node_modules/typebox/build/system/memory/finalize.d.mts +4 -0
- package/node_modules/typebox/build/system/memory/finalize.mjs +30 -0
- package/node_modules/typebox/build/system/memory/keys.d.mts +2 -0
- package/node_modules/typebox/build/system/memory/keys.mjs +8 -0
- package/node_modules/typebox/build/system/memory/update.d.mts +1 -4
- package/node_modules/typebox/build/system/memory/update.mjs +43 -16
- package/node_modules/typebox/build/type/engine/call/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/call/index.mjs +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/composite.d.mts +2 -24
- package/node_modules/typebox/build/type/engine/evaluate/composite.mjs +2 -29
- package/node_modules/typebox/build/type/engine/evaluate/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/index.mjs +1 -0
- package/node_modules/typebox/build/type/engine/evaluate/narrow.d.mts +5 -1
- package/node_modules/typebox/build/type/engine/evaluate/narrow.mjs +6 -1
- package/node_modules/typebox/build/type/engine/evaluate/property.d.mts +30 -0
- package/node_modules/typebox/build/type/engine/evaluate/property.mjs +29 -0
- package/node_modules/typebox/build/type/engine/immutable/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/immutable/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/object/from_intersect.d.mts +2 -2
- package/node_modules/typebox/build/type/engine/object/from_intersect.mjs +2 -2
- package/node_modules/typebox/build/type/engine/object/from_union.d.mts +2 -2
- package/node_modules/typebox/build/type/engine/object/from_union.mjs +2 -2
- package/node_modules/typebox/build/type/engine/optional/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/optional/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/readonly/instantiate_add.d.mts +1 -3
- package/node_modules/typebox/build/type/engine/readonly/instantiate_add.mjs +1 -4
- package/node_modules/typebox/build/type/engine/this/index.d.mts +1 -0
- package/node_modules/typebox/build/type/engine/this/index.mjs +1 -0
- package/node_modules/typebox/build/value/clone/clone.d.mts +4 -3
- package/node_modules/typebox/build/value/clone/clone.mjs +5 -7
- package/node_modules/typebox/build/value/repair/from_cyclic.d.mts +2 -0
- package/node_modules/typebox/build/value/repair/from_cyclic.mjs +6 -0
- package/node_modules/typebox/build/value/shared/union_priority_sort.mjs +25 -14
- package/node_modules/typebox/package.json +1 -1
- package/node_modules/typebox/readme.md +1 -1
- package/package.json +5 -5
- package/node_modules/typebox/build/system/memory/freeze.d.mts +0 -4
- package/node_modules/typebox/build/system/memory/freeze.mjs +0 -6
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
import { resolveSiteName } from "openclaw/plugin-sdk/provider-web-search";
|
|
1
|
+
import { externalContentStamp, readCachedSearchPayload, resolveSearchCacheTtlMs, resolveSiteName, writeCachedSearchPayload } from "openclaw/plugin-sdk/provider-web-search";
|
|
2
|
+
import { sha256Hex } from "openclaw/plugin-sdk/node-crypto-runtime";
|
|
2
3
|
import { asNonNegativeFiniteNumber, asPositiveFiniteNumber, parseDateStringTimestampMs, parseFiniteNumber, resolveIntegerOption } from "openclaw/plugin-sdk/number-runtime";
|
|
4
|
+
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
3
5
|
import { ProviderHttpError, readProviderJsonObjectResponse } from "openclaw/plugin-sdk/provider-http";
|
|
4
|
-
import {
|
|
6
|
+
import { markdownToText, readResponseText, resolvePositiveTimeoutSeconds, withSelfHostedWebToolsEndpoint, withStrictWebToolsEndpoint } from "openclaw/plugin-sdk/provider-web-fetch";
|
|
7
|
+
import { resolveProviderWebSearchPluginConfig } from "openclaw/plugin-sdk/provider-web-search-config-contract";
|
|
5
8
|
import { normalizeSecretInput } from "openclaw/plugin-sdk/secret-input";
|
|
9
|
+
import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
|
|
6
10
|
import { truncateSanitizedExternalContent, wrapExternalContent, wrapWebContent } from "openclaw/plugin-sdk/security-runtime";
|
|
7
11
|
import { SsrFBlockedError, isBlockedHostnameOrIp, isPrivateIpAddress, resolvePinnedHostnameWithPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
|
|
8
12
|
import { asBoolean, asOptionalObjectRecord, asOptionalRecord, asRecord, normalizeOptionalString, readStringField } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
9
|
-
|
|
10
|
-
//#region \0rolldown/runtime.js
|
|
13
|
+
// \0rolldown/runtime.js
|
|
11
14
|
var __defProp = Object.defineProperty;
|
|
12
15
|
var __exportAll = (all, no_symbols) => {
|
|
13
16
|
let target = {};
|
|
@@ -18,32 +21,35 @@ var __exportAll = (all, no_symbols) => {
|
|
|
18
21
|
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
19
22
|
return target;
|
|
20
23
|
};
|
|
24
|
+
// extensions/firecrawl/src/firecrawl-client.ts
|
|
25
|
+
var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
|
|
26
|
+
runFirecrawlScrape: () => runFirecrawlScrape,
|
|
27
|
+
runFirecrawlSearch: () => runFirecrawlSearch
|
|
28
|
+
});
|
|
29
|
+
const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
|
|
21
30
|
const DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS = 30;
|
|
22
31
|
const DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS = 60;
|
|
23
32
|
const DEFAULT_FIRECRAWL_MAX_AGE_MS = 1728e5;
|
|
24
33
|
const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
const DEFAULT_SEARCH_COUNT = 5;
|
|
35
|
+
const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
|
|
36
|
+
const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
|
|
37
|
+
const DEFAULT_SCRAPE_MAX_CHARS = 5e4;
|
|
38
|
+
const FIRECRAWL_SCRAPE_METADATA_MAX_CHARS = 4e3;
|
|
39
|
+
const FIRECRAWL_RESULT_URL_MAX_CHARS = 2048;
|
|
40
|
+
const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 67108864;
|
|
41
|
+
const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
|
|
42
|
+
const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
|
|
43
|
+
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
|
|
44
|
+
const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
|
|
29
45
|
function resolveFirecrawlFetchConfig(cfg) {
|
|
30
|
-
|
|
31
|
-
return asOptionalRecord(pluginConfig?.webFetch);
|
|
32
|
-
}
|
|
33
|
-
function resolveConfiguredSecret(value, path, cfg) {
|
|
34
|
-
return resolveReadOnlyEnvSecretRef({
|
|
35
|
-
value,
|
|
36
|
-
path,
|
|
37
|
-
cfg,
|
|
38
|
-
expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
|
|
39
|
-
normalizeValue: normalizeSecretInput
|
|
40
|
-
});
|
|
46
|
+
return asOptionalRecord(resolvePluginConfigObject(cfg, "firecrawl")?.webFetch);
|
|
41
47
|
}
|
|
42
48
|
function resolveFirecrawlApiKey(cfg) {
|
|
43
|
-
const
|
|
44
|
-
const search =
|
|
49
|
+
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
50
|
+
const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
|
|
45
51
|
const configuredCandidates = [{
|
|
46
|
-
value:
|
|
52
|
+
value: fetch?.apiKey,
|
|
47
53
|
path: "plugins.entries.firecrawl.config.webFetch.apiKey"
|
|
48
54
|
}, {
|
|
49
55
|
value: search?.apiKey,
|
|
@@ -51,7 +57,13 @@ function resolveFirecrawlApiKey(cfg) {
|
|
|
51
57
|
}];
|
|
52
58
|
let blockedConfiguredSecret = false;
|
|
53
59
|
for (const candidate of configuredCandidates) {
|
|
54
|
-
const resolved =
|
|
60
|
+
const resolved = resolveReadOnlyEnvSecretRef({
|
|
61
|
+
value: candidate.value,
|
|
62
|
+
path: candidate.path,
|
|
63
|
+
cfg,
|
|
64
|
+
expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
|
|
65
|
+
normalizeValue: normalizeSecretInput
|
|
66
|
+
});
|
|
55
67
|
if (resolved.status === "available") return resolved.value;
|
|
56
68
|
if (resolved.status === "blocked") blockedConfiguredSecret = true;
|
|
57
69
|
}
|
|
@@ -59,44 +71,10 @@ function resolveFirecrawlApiKey(cfg) {
|
|
|
59
71
|
return normalizeSecretInput(process.env[FIRECRAWL_API_KEY_ENV_VAR]) || void 0;
|
|
60
72
|
}
|
|
61
73
|
function resolveFirecrawlBaseUrl(cfg) {
|
|
62
|
-
const search =
|
|
74
|
+
const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
|
|
63
75
|
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
64
|
-
return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) ||
|
|
76
|
+
return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || DEFAULT_FIRECRAWL_BASE_URL;
|
|
65
77
|
}
|
|
66
|
-
function resolveFirecrawlOnlyMainContent(cfg, override) {
|
|
67
|
-
return asBoolean(override) ?? asBoolean(resolveFirecrawlFetchConfig(cfg)?.onlyMainContent) ?? true;
|
|
68
|
-
}
|
|
69
|
-
function resolveFirecrawlMaxAgeMs(cfg, override) {
|
|
70
|
-
return Math.floor(asNonNegativeFiniteNumber(override) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
|
|
71
|
-
}
|
|
72
|
-
function resolveFirecrawlScrapeTimeoutSeconds(cfg, override) {
|
|
73
|
-
const fetch = resolveFirecrawlFetchConfig(cfg);
|
|
74
|
-
return resolvePositiveTimeoutSeconds(override, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
|
|
75
|
-
}
|
|
76
|
-
function resolveFirecrawlSearchTimeoutSeconds(override) {
|
|
77
|
-
return resolvePositiveTimeoutSeconds(override, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
|
78
|
-
}
|
|
79
|
-
//#endregion
|
|
80
|
-
//#region extensions/firecrawl/src/firecrawl-client.ts
|
|
81
|
-
var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
|
|
82
|
-
assertFirecrawlScrapeTargetAllowed: () => assertFirecrawlScrapeTargetAllowed,
|
|
83
|
-
parseFirecrawlScrapePayload: () => parseFirecrawlScrapePayload,
|
|
84
|
-
runFirecrawlScrape: () => runFirecrawlScrape,
|
|
85
|
-
runFirecrawlSearch: () => runFirecrawlSearch,
|
|
86
|
-
testing: () => testing
|
|
87
|
-
});
|
|
88
|
-
const SEARCH_CACHE = /* @__PURE__ */ new Map();
|
|
89
|
-
const DEFAULT_SEARCH_COUNT = 5;
|
|
90
|
-
const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
|
|
91
|
-
const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
|
|
92
|
-
const DEFAULT_SCRAPE_MAX_CHARS = 5e4;
|
|
93
|
-
const FIRECRAWL_SCRAPE_METADATA_MAX_CHARS = 4e3;
|
|
94
|
-
const FIRECRAWL_RESULT_URL_MAX_CHARS = 2048;
|
|
95
|
-
const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 67108864;
|
|
96
|
-
const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
|
|
97
|
-
const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
|
|
98
|
-
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
|
|
99
|
-
const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
|
|
100
78
|
function assertFirecrawlScrapeTargetAllowed(url) {
|
|
101
79
|
let parsed;
|
|
102
80
|
try {
|
|
@@ -110,27 +88,24 @@ function assertFirecrawlScrapeTargetAllowed(url) {
|
|
|
110
88
|
function isOfficialFirecrawlEndpoint(url) {
|
|
111
89
|
return url.protocol === "https:" && ALLOWED_FIRECRAWL_HOSTS.has(url.hostname);
|
|
112
90
|
}
|
|
113
|
-
async function firecrawlEndpointTargetsPrivateNetwork(url
|
|
91
|
+
async function firecrawlEndpointTargetsPrivateNetwork(url) {
|
|
114
92
|
if (isBlockedHostnameOrIp(url.hostname)) return true;
|
|
115
93
|
try {
|
|
116
|
-
return (await resolvePinnedHostnameWithPolicy(url.hostname, {
|
|
117
|
-
lookupFn,
|
|
118
|
-
policy: { allowPrivateNetwork: true }
|
|
119
|
-
})).addresses.every((address) => isPrivateIpAddress(address));
|
|
94
|
+
return (await resolvePinnedHostnameWithPolicy(url.hostname, { policy: { allowPrivateNetwork: true } })).addresses.every((address) => isPrivateIpAddress(address));
|
|
120
95
|
} catch {
|
|
121
96
|
return false;
|
|
122
97
|
}
|
|
123
98
|
}
|
|
124
|
-
async function validateFirecrawlBaseUrl(url
|
|
99
|
+
async function validateFirecrawlBaseUrl(url) {
|
|
125
100
|
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("Firecrawl baseUrl must use http:// or https://.");
|
|
126
101
|
if (isOfficialFirecrawlEndpoint(url)) return "strict";
|
|
127
|
-
if (await firecrawlEndpointTargetsPrivateNetwork(url
|
|
102
|
+
if (await firecrawlEndpointTargetsPrivateNetwork(url)) return "selfHosted";
|
|
128
103
|
if (url.protocol === "http:") throw new Error(FIRECRAWL_HTTP_PRIVATE_ERROR);
|
|
129
104
|
throw new Error(`${FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR} Host: ${url.hostname}`);
|
|
130
105
|
}
|
|
131
|
-
async function resolveEndpoint(baseUrl, pathname
|
|
132
|
-
const url = new URL(baseUrl.trim() ||
|
|
133
|
-
const mode = await validateFirecrawlBaseUrl(url
|
|
106
|
+
async function resolveEndpoint(baseUrl, pathname) {
|
|
107
|
+
const url = new URL(baseUrl.trim() || DEFAULT_FIRECRAWL_BASE_URL);
|
|
108
|
+
const mode = await validateFirecrawlBaseUrl(url);
|
|
134
109
|
url.username = "";
|
|
135
110
|
url.password = "";
|
|
136
111
|
url.search = "";
|
|
@@ -246,20 +221,21 @@ async function runFirecrawlSearch(params) {
|
|
|
246
221
|
min: 1,
|
|
247
222
|
max: 100
|
|
248
223
|
});
|
|
249
|
-
const timeoutSeconds =
|
|
224
|
+
const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
|
|
250
225
|
const scrapeResults = params.scrapeResults === true;
|
|
251
|
-
const sources =
|
|
252
|
-
const categories =
|
|
253
|
-
const includeDomains =
|
|
254
|
-
const excludeDomains =
|
|
226
|
+
const sources = params.sources?.filter(Boolean) ?? [];
|
|
227
|
+
const categories = params.categories?.filter(Boolean) ?? [];
|
|
228
|
+
const includeDomains = params.includeDomains?.filter(Boolean) ?? [];
|
|
229
|
+
const excludeDomains = params.excludeDomains?.filter(Boolean) ?? [];
|
|
255
230
|
if (includeDomains.length > 0 && excludeDomains.length > 0) throw new Error("Firecrawl search accepts includeDomains or excludeDomains, not both.");
|
|
256
231
|
const tbs = normalizeOptionalString(params.tbs);
|
|
257
232
|
const location = normalizeOptionalString(params.location);
|
|
258
233
|
const country = normalizeOptionalString(params.country);
|
|
259
234
|
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
260
|
-
const cacheKey =
|
|
235
|
+
const cacheKey = JSON.stringify({
|
|
261
236
|
type: "firecrawl-search",
|
|
262
237
|
provider: providerId,
|
|
238
|
+
credential: apiKey ? sha256Hex(apiKey) : void 0,
|
|
263
239
|
q: params.query,
|
|
264
240
|
count,
|
|
265
241
|
baseUrl,
|
|
@@ -271,13 +247,10 @@ async function runFirecrawlSearch(params) {
|
|
|
271
247
|
location,
|
|
272
248
|
country,
|
|
273
249
|
scrapeResults
|
|
274
|
-
})
|
|
275
|
-
const cacheTtlMs =
|
|
276
|
-
const cached =
|
|
277
|
-
if (cached) return
|
|
278
|
-
...cached.value,
|
|
279
|
-
cached: true
|
|
280
|
-
};
|
|
250
|
+
});
|
|
251
|
+
const cacheTtlMs = resolveSearchCacheTtlMs(params.cfg?.tools?.web?.search);
|
|
252
|
+
const cached = readCachedSearchPayload(cacheKey, cacheTtlMs);
|
|
253
|
+
if (cached) return cached;
|
|
281
254
|
const body = {
|
|
282
255
|
query: params.query,
|
|
283
256
|
limit: count,
|
|
@@ -331,20 +304,52 @@ async function runFirecrawlSearch(params) {
|
|
|
331
304
|
provider: providerId,
|
|
332
305
|
count: items.length,
|
|
333
306
|
tookMs,
|
|
334
|
-
externalContent:
|
|
335
|
-
untrusted: true,
|
|
336
|
-
source: "web_search",
|
|
337
|
-
provider: providerId,
|
|
338
|
-
wrapped: true
|
|
339
|
-
},
|
|
307
|
+
externalContent: externalContentStamp(providerId),
|
|
340
308
|
results,
|
|
341
309
|
...truncated ? { truncated: true } : {}
|
|
342
310
|
};
|
|
343
|
-
|
|
311
|
+
writeCachedSearchPayload(cacheKey, result, cacheTtlMs);
|
|
344
312
|
return result;
|
|
345
313
|
}
|
|
346
|
-
function
|
|
347
|
-
|
|
314
|
+
async function runFirecrawlScrape(params) {
|
|
315
|
+
params.signal?.throwIfAborted();
|
|
316
|
+
assertFirecrawlScrapeTargetAllowed(params.url);
|
|
317
|
+
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
|
318
|
+
if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
|
|
319
|
+
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
320
|
+
const fetch = resolveFirecrawlFetchConfig(params.cfg);
|
|
321
|
+
const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
|
|
322
|
+
const onlyMainContent = asBoolean(params.onlyMainContent) ?? asBoolean(resolveFirecrawlFetchConfig(params.cfg)?.onlyMainContent) ?? true;
|
|
323
|
+
const maxAgeMs = Math.floor(asNonNegativeFiniteNumber(params.maxAgeMs) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(params.cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
|
|
324
|
+
const proxy = params.proxy ?? "auto";
|
|
325
|
+
const storeInCache = params.storeInCache ?? true;
|
|
326
|
+
const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
|
|
327
|
+
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
|
328
|
+
const payload = await postFirecrawlJson({
|
|
329
|
+
url: endpoint.url,
|
|
330
|
+
mode: endpoint.mode,
|
|
331
|
+
timeoutSeconds,
|
|
332
|
+
apiKey,
|
|
333
|
+
errorLabel: "Firecrawl",
|
|
334
|
+
...params.signal ? { signal: params.signal } : {},
|
|
335
|
+
body: {
|
|
336
|
+
url: params.url,
|
|
337
|
+
formats: ["markdown"],
|
|
338
|
+
onlyMainContent,
|
|
339
|
+
timeout: timeoutSeconds * 1e3,
|
|
340
|
+
maxAge: maxAgeMs,
|
|
341
|
+
proxy,
|
|
342
|
+
storeInCache
|
|
343
|
+
}
|
|
344
|
+
}, async (response) => {
|
|
345
|
+
const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
|
|
346
|
+
if (data.success === false) {
|
|
347
|
+
const detail = readFirecrawlErrorDetail(data, response.statusText);
|
|
348
|
+
throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
|
|
349
|
+
}
|
|
350
|
+
return data;
|
|
351
|
+
});
|
|
352
|
+
const data = asRecord(payload.data);
|
|
348
353
|
const metadata = asOptionalObjectRecord(data.metadata);
|
|
349
354
|
const rawStatus = parseFiniteNumber(metadata?.statusCode) ?? parseFiniteNumber(data.statusCode);
|
|
350
355
|
const status = rawStatus === void 0 ? void 0 : Math.floor(rawStatus);
|
|
@@ -352,7 +357,7 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
352
357
|
const markdown = typeof data.markdown === "string" && data.markdown || typeof data.content === "string" && data.content || "";
|
|
353
358
|
if (!markdown) throw new Error("Firecrawl scrape returned no content.");
|
|
354
359
|
const rawText = params.extractMode === "text" ? markdownToText(markdown) : markdown;
|
|
355
|
-
const boundedText = truncateSanitizedExternalContent(rawText,
|
|
360
|
+
const boundedText = truncateSanitizedExternalContent(rawText, maxChars);
|
|
356
361
|
let truncated = boundedText.truncated;
|
|
357
362
|
let remainingMetadataChars = FIRECRAWL_SCRAPE_METADATA_MAX_CHARS;
|
|
358
363
|
const wrapBoundedMetadata = (value) => {
|
|
@@ -369,7 +374,7 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
369
374
|
includeWarning: false
|
|
370
375
|
});
|
|
371
376
|
const title = typeof metadata?.title === "string" && metadata.title ? wrapBoundedMetadata(metadata.title) : void 0;
|
|
372
|
-
const warning = typeof
|
|
377
|
+
const warning = typeof payload.warning === "string" && payload.warning ? wrapBoundedMetadata(payload.warning) : void 0;
|
|
373
378
|
return {
|
|
374
379
|
url: params.url,
|
|
375
380
|
finalUrl: normalizeFirecrawlResultUrl(metadata?.sourceURL) ?? normalizeFirecrawlResultUrl(data.url) ?? params.url,
|
|
@@ -389,54 +394,4 @@ function parseFirecrawlScrapePayload(params) {
|
|
|
389
394
|
...warning ? { warning } : {}
|
|
390
395
|
};
|
|
391
396
|
}
|
|
392
|
-
async function runFirecrawlScrape(params) {
|
|
393
|
-
params.signal?.throwIfAborted();
|
|
394
|
-
assertFirecrawlScrapeTargetAllowed(params.url);
|
|
395
|
-
const apiKey = resolveFirecrawlApiKey(params.cfg);
|
|
396
|
-
if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
|
|
397
|
-
const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
|
|
398
|
-
const timeoutSeconds = resolveFirecrawlScrapeTimeoutSeconds(params.cfg, params.timeoutSeconds);
|
|
399
|
-
const onlyMainContent = resolveFirecrawlOnlyMainContent(params.cfg, params.onlyMainContent);
|
|
400
|
-
const maxAgeMs = resolveFirecrawlMaxAgeMs(params.cfg, params.maxAgeMs);
|
|
401
|
-
const proxy = params.proxy ?? "auto";
|
|
402
|
-
const storeInCache = params.storeInCache ?? true;
|
|
403
|
-
const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
|
|
404
|
-
const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
|
|
405
|
-
return parseFirecrawlScrapePayload({
|
|
406
|
-
payload: await postFirecrawlJson({
|
|
407
|
-
url: endpoint.url,
|
|
408
|
-
mode: endpoint.mode,
|
|
409
|
-
timeoutSeconds,
|
|
410
|
-
apiKey,
|
|
411
|
-
errorLabel: "Firecrawl",
|
|
412
|
-
...params.signal ? { signal: params.signal } : {},
|
|
413
|
-
body: {
|
|
414
|
-
url: params.url,
|
|
415
|
-
formats: ["markdown"],
|
|
416
|
-
onlyMainContent,
|
|
417
|
-
timeout: timeoutSeconds * 1e3,
|
|
418
|
-
maxAge: maxAgeMs,
|
|
419
|
-
proxy,
|
|
420
|
-
storeInCache
|
|
421
|
-
}
|
|
422
|
-
}, async (response) => {
|
|
423
|
-
const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
|
|
424
|
-
if (data.success === false) {
|
|
425
|
-
const detail = readFirecrawlErrorDetail(data, response.statusText);
|
|
426
|
-
throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
|
|
427
|
-
}
|
|
428
|
-
return data;
|
|
429
|
-
}),
|
|
430
|
-
url: params.url,
|
|
431
|
-
extractMode: params.extractMode,
|
|
432
|
-
maxChars
|
|
433
|
-
});
|
|
434
|
-
}
|
|
435
|
-
const testing = {
|
|
436
|
-
assertFirecrawlScrapeTargetAllowed,
|
|
437
|
-
parseFirecrawlScrapePayload,
|
|
438
|
-
resolveEndpoint,
|
|
439
|
-
resolveSearchItems
|
|
440
|
-
};
|
|
441
|
-
//#endregion
|
|
442
397
|
export { runFirecrawlScrape as n, runFirecrawlSearch as r, firecrawl_client_exports as t };
|
package/dist/.setup/{firecrawl-fetch-provider-uRENNKri.mjs → firecrawl-fetch-provider-QmPccHZS.mjs}
RENAMED
|
@@ -1,14 +1,8 @@
|
|
|
1
1
|
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
|
2
2
|
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
|
3
3
|
import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch-contract";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const current = target[key];
|
|
7
|
-
if (current && typeof current === "object" && !Array.isArray(current)) return current;
|
|
8
|
-
const next = {};
|
|
9
|
-
target[key] = next;
|
|
10
|
-
return next;
|
|
11
|
-
}
|
|
4
|
+
import { ensureRecord } from "openclaw/plugin-sdk/collection-input-runtime";
|
|
5
|
+
// extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
|
|
12
6
|
const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
|
13
7
|
id: "firecrawl",
|
|
14
8
|
label: "Firecrawl",
|
|
@@ -45,13 +39,13 @@ const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
|
|
|
45
39
|
const plugins = configTarget.plugins ??= {};
|
|
46
40
|
const entries = plugins.entries ??= {};
|
|
47
41
|
const firecrawlEntry = entries.firecrawl ??= {};
|
|
48
|
-
const
|
|
42
|
+
const pluginConfig = firecrawlEntry.config ??= {};
|
|
43
|
+
const webFetch = ensureRecord(pluginConfig, "webFetch");
|
|
49
44
|
webFetch.apiKey = value;
|
|
50
45
|
}
|
|
51
46
|
};
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CMLsufX8.mjs").then((n) => n.t));
|
|
47
|
+
// extensions/firecrawl/src/firecrawl-fetch-provider.ts
|
|
48
|
+
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CDyLQvtN.mjs").then((n) => n.t));
|
|
55
49
|
function createFirecrawlWebFetchProvider() {
|
|
56
50
|
return {
|
|
57
51
|
...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
|
|
@@ -81,5 +75,4 @@ function createFirecrawlWebFetchProvider() {
|
|
|
81
75
|
})
|
|
82
76
|
};
|
|
83
77
|
}
|
|
84
|
-
//#endregion
|
|
85
78
|
export { createFirecrawlWebFetchProvider as t };
|
|
@@ -1,29 +1,18 @@
|
|
|
1
|
+
import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
2
|
+
import { asOptionalRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
1
3
|
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
|
|
2
4
|
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
//#region extensions/firecrawl/src/firecrawl-search-provider.ts
|
|
5
|
+
import { WEB_SEARCH_QUERY_COUNT_PROPERTIES, createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
|
|
6
|
+
// extensions/firecrawl/src/firecrawl-search-provider.ts
|
|
6
7
|
const FIRECRAWL_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webSearch.apiKey";
|
|
7
|
-
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-
|
|
8
|
+
const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CDyLQvtN.mjs").then((n) => n.t));
|
|
8
9
|
const GenericFirecrawlSearchSchema = {
|
|
9
10
|
type: "object",
|
|
10
|
-
properties: {
|
|
11
|
-
query: {
|
|
12
|
-
type: "string",
|
|
13
|
-
description: "Search query string."
|
|
14
|
-
},
|
|
15
|
-
count: {
|
|
16
|
-
type: "integer",
|
|
17
|
-
description: "Number of results to return (1-10).",
|
|
18
|
-
minimum: 1,
|
|
19
|
-
maximum: 10
|
|
20
|
-
}
|
|
21
|
-
},
|
|
11
|
+
properties: { ...WEB_SEARCH_QUERY_COUNT_PROPERTIES },
|
|
22
12
|
additionalProperties: false
|
|
23
13
|
};
|
|
24
14
|
function getConfiguredFirecrawlFetchCredentialFallback(config) {
|
|
25
|
-
const
|
|
26
|
-
const value = asOptionalRecord(pluginConfig?.webFetch)?.apiKey;
|
|
15
|
+
const value = asOptionalRecord(resolvePluginConfigObject(config, "firecrawl")?.webFetch)?.apiKey;
|
|
27
16
|
return value === void 0 ? void 0 : {
|
|
28
17
|
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
|
|
29
18
|
value
|
|
@@ -106,5 +95,4 @@ function createFirecrawlWebSearchProvider() {
|
|
|
106
95
|
function createFirecrawlFreeWebSearchProvider() {
|
|
107
96
|
return createFirecrawlSearchProvider(true);
|
|
108
97
|
}
|
|
109
|
-
//#endregion
|
|
110
98
|
export { createFirecrawlWebSearchProvider as n, createFirecrawlFreeWebSearchProvider as t };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-
|
|
1
|
+
{"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-CDyLQvtN.mjs","sha256":"46c98acf8d71a8310851b8a6bfeb3d7e1176edb3d040c8e7d1de958e62cd4028","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-client.ts"]},{"output":".setup/firecrawl-fetch-provider-QmPccHZS.mjs","sha256":"d32ea4f6248fc6c0c388257809db11ac0107a9a7c473ac75a0f665f71dae723c","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts","extensions/firecrawl/src/firecrawl-fetch-provider.ts"]},{"output":".setup/firecrawl-search-provider-wsRgx6kl.mjs","sha256":"45416f284d535281a900a1ea20307652ca911769a3f36e9cbc5916a29a2ceafd","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-search-provider.ts"]},{"output":"api.js","sha256":"ba89572c128911e1fb033a07004fa1c36861338a6704705ea14c6d6a1ea1cea3","facadeSource":"extensions/firecrawl/api.ts","sources":["extensions/firecrawl/api.ts"]},{"output":"index.js","sha256":"1b6689bdf7d803a983f86a4371e153139d127a259720af5d6412d102ecb1a3e5","facadeSource":"extensions/firecrawl/index.ts","sources":["extensions/firecrawl/index.ts"]},{"output":"web-fetch-contract-api.js","sha256":"e9f57ffa95524a8f5d9ac59cd879f41ec5c43e6e3ffd236a3bcfc6f55e2c439d","facadeSource":"extensions/firecrawl/web-fetch-contract-api.ts","sources":[]},{"output":"web-fetch-provider.js","sha256":"55e667f670164dd167c874b72e48fcba2b7ed2624d0d26f66d1fa78067f5b0b5","facadeSource":"extensions/firecrawl/web-fetch-provider.ts","sources":[]},{"output":"web-search-contract-api.js","sha256":"eb2a1d89385a524e030bc88d18d86398e5d9baf5bdeb57f9f4608017692541e4","facadeSource":"extensions/firecrawl/web-search-contract-api.ts","sources":[]},{"output":"web-search-provider.js","sha256":"54d793c602424e24a8e6f17c2f8eb832c5635c77e9094a1ca97c9b1249b17cb0","facadeSource":"extensions/firecrawl/web-search-provider.ts","sources":[]}],"unrenderedSources":["extensions/firecrawl/web-fetch-contract-api.ts","extensions/firecrawl/web-fetch-provider.ts","extensions/firecrawl/web-search-contract-api.ts","extensions/firecrawl/web-search-provider.ts"]}
|
package/dist/api.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-
|
|
2
|
-
|
|
1
|
+
import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-CDyLQvtN.mjs";
|
|
2
|
+
// extensions/firecrawl/api.ts
|
|
3
3
|
async function fetchFirecrawlContent(params) {
|
|
4
4
|
const cfg = { plugins: { entries: { firecrawl: {
|
|
5
5
|
enabled: true,
|
|
@@ -30,5 +30,4 @@ async function fetchFirecrawlContent(params) {
|
|
|
30
30
|
warning: result.warning
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
//#endregion
|
|
34
33
|
export { fetchFirecrawlContent };
|