@openclaw/firecrawl-plugin 2026.10.10-beta.5 → 2026.11.5-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/.setup/{firecrawl-client-CMLsufX8.mjs → firecrawl-client-CDyLQvtN.mjs} +98 -143
  2. package/dist/.setup/{firecrawl-fetch-provider-uRENNKri.mjs → firecrawl-fetch-provider-QmPccHZS.mjs} +6 -13
  3. package/dist/.setup/{firecrawl-search-provider-CFhWH8gx.mjs → firecrawl-search-provider-wsRgx6kl.mjs} +7 -19
  4. package/dist/.source-module-urls/ad8205c790d921269d4ea28d22ab658579b9674fd00e025664fcf0b8ea32155c.json +1 -1
  5. package/dist/api.js +2 -3
  6. package/dist/index.js +69 -90
  7. package/dist/web-fetch-contract-api.js +1 -1
  8. package/dist/web-fetch-provider.js +1 -1
  9. package/dist/web-search-contract-api.js +1 -1
  10. package/dist/web-search-provider.js +1 -1
  11. package/node_modules/typebox/build/format/duration.d.mts +2 -2
  12. package/node_modules/typebox/build/format/duration.mjs +2 -2
  13. package/node_modules/typebox/build/format/idn_email.d.mts +1 -0
  14. package/node_modules/typebox/build/format/idn_email.mjs +1 -0
  15. package/node_modules/typebox/build/format/idn_hostname.d.mts +1 -1
  16. package/node_modules/typebox/build/format/idn_hostname.mjs +1 -1
  17. package/node_modules/typebox/build/format/ipv4.d.mts +2 -2
  18. package/node_modules/typebox/build/format/ipv4.mjs +2 -2
  19. package/node_modules/typebox/build/format/ipv6.d.mts +1 -1
  20. package/node_modules/typebox/build/format/ipv6.mjs +1 -1
  21. package/node_modules/typebox/build/format/relative_json_pointer.d.mts +2 -2
  22. package/node_modules/typebox/build/format/relative_json_pointer.mjs +2 -2
  23. package/node_modules/typebox/build/schema/build.mjs +1 -0
  24. package/node_modules/typebox/build/schema/engine/_unique.d.mts +4 -2
  25. package/node_modules/typebox/build/schema/engine/_unique.mjs +7 -3
  26. package/node_modules/typebox/build/schema/engine/additionalItems.mjs +3 -3
  27. package/node_modules/typebox/build/schema/engine/additionalProperties.d.mts +0 -3
  28. package/node_modules/typebox/build/schema/engine/additionalProperties.mjs +15 -17
  29. package/node_modules/typebox/build/schema/engine/contains.mjs +3 -3
  30. package/node_modules/typebox/build/schema/engine/index.d.mts +1 -0
  31. package/node_modules/typebox/build/schema/engine/index.mjs +1 -0
  32. package/node_modules/typebox/build/schema/engine/items.mjs +7 -4
  33. package/node_modules/typebox/build/schema/engine/maxContains.mjs +2 -2
  34. package/node_modules/typebox/build/schema/engine/minContains.mjs +3 -3
  35. package/node_modules/typebox/build/schema/engine/oneOf.mjs +2 -2
  36. package/node_modules/typebox/build/schema/engine/patternProperties.mjs +2 -2
  37. package/node_modules/typebox/build/schema/engine/propertyNames.mjs +2 -2
  38. package/node_modules/typebox/build/schema/engine/unevaluatedItems.mjs +2 -2
  39. package/node_modules/typebox/build/schema/engine/unevaluatedProperties.mjs +2 -2
  40. package/node_modules/typebox/build/system/hashing/hash.mjs +63 -34
  41. package/node_modules/typebox/build/system/locale/en_US.mjs +2 -1
  42. package/node_modules/typebox/build/system/memory/assign.d.mts +2 -8
  43. package/node_modules/typebox/build/system/memory/assign.mjs +13 -6
  44. package/node_modules/typebox/build/system/memory/clone.d.mts +4 -5
  45. package/node_modules/typebox/build/system/memory/clone.mjs +8 -96
  46. package/node_modules/typebox/build/system/memory/create.d.mts +1 -5
  47. package/node_modules/typebox/build/system/memory/create.mjs +18 -13
  48. package/node_modules/typebox/build/system/memory/deep.d.mts +2 -0
  49. package/node_modules/typebox/build/system/memory/deep.mjs +108 -0
  50. package/node_modules/typebox/build/system/memory/discard.d.mts +1 -1
  51. package/node_modules/typebox/build/system/memory/discard.mjs +9 -8
  52. package/node_modules/typebox/build/system/memory/finalize.d.mts +4 -0
  53. package/node_modules/typebox/build/system/memory/finalize.mjs +30 -0
  54. package/node_modules/typebox/build/system/memory/keys.d.mts +2 -0
  55. package/node_modules/typebox/build/system/memory/keys.mjs +8 -0
  56. package/node_modules/typebox/build/system/memory/update.d.mts +1 -4
  57. package/node_modules/typebox/build/system/memory/update.mjs +43 -16
  58. package/node_modules/typebox/build/type/engine/call/index.d.mts +1 -0
  59. package/node_modules/typebox/build/type/engine/call/index.mjs +1 -0
  60. package/node_modules/typebox/build/type/engine/evaluate/composite.d.mts +2 -24
  61. package/node_modules/typebox/build/type/engine/evaluate/composite.mjs +2 -29
  62. package/node_modules/typebox/build/type/engine/evaluate/index.d.mts +1 -0
  63. package/node_modules/typebox/build/type/engine/evaluate/index.mjs +1 -0
  64. package/node_modules/typebox/build/type/engine/evaluate/narrow.d.mts +5 -1
  65. package/node_modules/typebox/build/type/engine/evaluate/narrow.mjs +6 -1
  66. package/node_modules/typebox/build/type/engine/evaluate/property.d.mts +30 -0
  67. package/node_modules/typebox/build/type/engine/evaluate/property.mjs +29 -0
  68. package/node_modules/typebox/build/type/engine/immutable/instantiate_add.d.mts +1 -3
  69. package/node_modules/typebox/build/type/engine/immutable/instantiate_add.mjs +1 -4
  70. package/node_modules/typebox/build/type/engine/object/from_intersect.d.mts +2 -2
  71. package/node_modules/typebox/build/type/engine/object/from_intersect.mjs +2 -2
  72. package/node_modules/typebox/build/type/engine/object/from_union.d.mts +2 -2
  73. package/node_modules/typebox/build/type/engine/object/from_union.mjs +2 -2
  74. package/node_modules/typebox/build/type/engine/optional/instantiate_add.d.mts +1 -3
  75. package/node_modules/typebox/build/type/engine/optional/instantiate_add.mjs +1 -4
  76. package/node_modules/typebox/build/type/engine/readonly/instantiate_add.d.mts +1 -3
  77. package/node_modules/typebox/build/type/engine/readonly/instantiate_add.mjs +1 -4
  78. package/node_modules/typebox/build/type/engine/this/index.d.mts +1 -0
  79. package/node_modules/typebox/build/type/engine/this/index.mjs +1 -0
  80. package/node_modules/typebox/build/value/clone/clone.d.mts +4 -3
  81. package/node_modules/typebox/build/value/clone/clone.mjs +5 -7
  82. package/node_modules/typebox/build/value/repair/from_cyclic.d.mts +2 -0
  83. package/node_modules/typebox/build/value/repair/from_cyclic.mjs +6 -0
  84. package/node_modules/typebox/build/value/shared/union_priority_sort.mjs +25 -14
  85. package/node_modules/typebox/package.json +1 -1
  86. package/node_modules/typebox/readme.md +1 -1
  87. package/package.json +5 -5
  88. package/node_modules/typebox/build/system/memory/freeze.d.mts +0 -4
  89. package/node_modules/typebox/build/system/memory/freeze.mjs +0 -6
@@ -1,13 +1,16 @@
1
- import { resolveSiteName } from "openclaw/plugin-sdk/provider-web-search";
1
+ import { externalContentStamp, readCachedSearchPayload, resolveSearchCacheTtlMs, resolveSiteName, writeCachedSearchPayload } from "openclaw/plugin-sdk/provider-web-search";
2
+ import { sha256Hex } from "openclaw/plugin-sdk/node-crypto-runtime";
2
3
  import { asNonNegativeFiniteNumber, asPositiveFiniteNumber, parseDateStringTimestampMs, parseFiniteNumber, resolveIntegerOption } from "openclaw/plugin-sdk/number-runtime";
4
+ import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
3
5
  import { ProviderHttpError, readProviderJsonObjectResponse } from "openclaw/plugin-sdk/provider-http";
4
- import { DEFAULT_CACHE_TTL_MINUTES, markdownToText, normalizeCacheKey, readCache, readResponseText, resolveCacheTtlMs, resolvePositiveTimeoutSeconds, withSelfHostedWebToolsEndpoint, withStrictWebToolsEndpoint, writeCache } from "openclaw/plugin-sdk/provider-web-fetch";
6
+ import { markdownToText, readResponseText, resolvePositiveTimeoutSeconds, withSelfHostedWebToolsEndpoint, withStrictWebToolsEndpoint } from "openclaw/plugin-sdk/provider-web-fetch";
7
+ import { resolveProviderWebSearchPluginConfig } from "openclaw/plugin-sdk/provider-web-search-config-contract";
5
8
  import { normalizeSecretInput } from "openclaw/plugin-sdk/secret-input";
9
+ import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
6
10
  import { truncateSanitizedExternalContent, wrapExternalContent, wrapWebContent } from "openclaw/plugin-sdk/security-runtime";
7
11
  import { SsrFBlockedError, isBlockedHostnameOrIp, isPrivateIpAddress, resolvePinnedHostnameWithPolicy } from "openclaw/plugin-sdk/ssrf-runtime";
8
12
  import { asBoolean, asOptionalObjectRecord, asOptionalRecord, asRecord, normalizeOptionalString, readStringField } from "openclaw/plugin-sdk/string-coerce-runtime";
9
- import { resolveReadOnlyEnvSecretRef } from "openclaw/plugin-sdk/secret-ref-readonly";
10
- //#region \0rolldown/runtime.js
13
+ // \0rolldown/runtime.js
11
14
  var __defProp = Object.defineProperty;
12
15
  var __exportAll = (all, no_symbols) => {
13
16
  let target = {};
@@ -18,32 +21,35 @@ var __exportAll = (all, no_symbols) => {
18
21
  if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
19
22
  return target;
20
23
  };
24
+ // extensions/firecrawl/src/firecrawl-client.ts
25
+ var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
26
+ runFirecrawlScrape: () => runFirecrawlScrape,
27
+ runFirecrawlSearch: () => runFirecrawlSearch
28
+ });
29
+ const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
21
30
  const DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS = 30;
22
31
  const DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS = 60;
23
32
  const DEFAULT_FIRECRAWL_MAX_AGE_MS = 1728e5;
24
33
  const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
25
- function resolveFirecrawlSearchConfig(cfg) {
26
- const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config;
27
- return asOptionalRecord(pluginConfig?.webSearch);
28
- }
34
+ const DEFAULT_SEARCH_COUNT = 5;
35
+ const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
36
+ const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
37
+ const DEFAULT_SCRAPE_MAX_CHARS = 5e4;
38
+ const FIRECRAWL_SCRAPE_METADATA_MAX_CHARS = 4e3;
39
+ const FIRECRAWL_RESULT_URL_MAX_CHARS = 2048;
40
+ const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 67108864;
41
+ const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
42
+ const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
43
+ const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
44
+ const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
29
45
  function resolveFirecrawlFetchConfig(cfg) {
30
- const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config;
31
- return asOptionalRecord(pluginConfig?.webFetch);
32
- }
33
- function resolveConfiguredSecret(value, path, cfg) {
34
- return resolveReadOnlyEnvSecretRef({
35
- value,
36
- path,
37
- cfg,
38
- expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
39
- normalizeValue: normalizeSecretInput
40
- });
46
+ return asOptionalRecord(resolvePluginConfigObject(cfg, "firecrawl")?.webFetch);
41
47
  }
42
48
  function resolveFirecrawlApiKey(cfg) {
43
- const pluginConfig = cfg?.plugins?.entries?.firecrawl?.config;
44
- const search = resolveFirecrawlSearchConfig(cfg);
49
+ const fetch = resolveFirecrawlFetchConfig(cfg);
50
+ const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
45
51
  const configuredCandidates = [{
46
- value: pluginConfig?.webFetch?.apiKey,
52
+ value: fetch?.apiKey,
47
53
  path: "plugins.entries.firecrawl.config.webFetch.apiKey"
48
54
  }, {
49
55
  value: search?.apiKey,
@@ -51,7 +57,13 @@ function resolveFirecrawlApiKey(cfg) {
51
57
  }];
52
58
  let blockedConfiguredSecret = false;
53
59
  for (const candidate of configuredCandidates) {
54
- const resolved = resolveConfiguredSecret(candidate.value, candidate.path, cfg);
60
+ const resolved = resolveReadOnlyEnvSecretRef({
61
+ value: candidate.value,
62
+ path: candidate.path,
63
+ cfg,
64
+ expectedEnvId: FIRECRAWL_API_KEY_ENV_VAR,
65
+ normalizeValue: normalizeSecretInput
66
+ });
55
67
  if (resolved.status === "available") return resolved.value;
56
68
  if (resolved.status === "blocked") blockedConfiguredSecret = true;
57
69
  }
@@ -59,44 +71,10 @@ function resolveFirecrawlApiKey(cfg) {
59
71
  return normalizeSecretInput(process.env[FIRECRAWL_API_KEY_ENV_VAR]) || void 0;
60
72
  }
61
73
  function resolveFirecrawlBaseUrl(cfg) {
62
- const search = resolveFirecrawlSearchConfig(cfg);
74
+ const search = resolveProviderWebSearchPluginConfig(cfg, "firecrawl");
63
75
  const fetch = resolveFirecrawlFetchConfig(cfg);
64
- return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || "https://api.firecrawl.dev";
76
+ return normalizeOptionalString(search?.baseUrl) || normalizeOptionalString(fetch?.baseUrl) || normalizeSecretInput(process.env.FIRECRAWL_BASE_URL) || DEFAULT_FIRECRAWL_BASE_URL;
65
77
  }
66
- function resolveFirecrawlOnlyMainContent(cfg, override) {
67
- return asBoolean(override) ?? asBoolean(resolveFirecrawlFetchConfig(cfg)?.onlyMainContent) ?? true;
68
- }
69
- function resolveFirecrawlMaxAgeMs(cfg, override) {
70
- return Math.floor(asNonNegativeFiniteNumber(override) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
71
- }
72
- function resolveFirecrawlScrapeTimeoutSeconds(cfg, override) {
73
- const fetch = resolveFirecrawlFetchConfig(cfg);
74
- return resolvePositiveTimeoutSeconds(override, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
75
- }
76
- function resolveFirecrawlSearchTimeoutSeconds(override) {
77
- return resolvePositiveTimeoutSeconds(override, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
78
- }
79
- //#endregion
80
- //#region extensions/firecrawl/src/firecrawl-client.ts
81
- var firecrawl_client_exports = /* @__PURE__ */ __exportAll({
82
- assertFirecrawlScrapeTargetAllowed: () => assertFirecrawlScrapeTargetAllowed,
83
- parseFirecrawlScrapePayload: () => parseFirecrawlScrapePayload,
84
- runFirecrawlScrape: () => runFirecrawlScrape,
85
- runFirecrawlSearch: () => runFirecrawlSearch,
86
- testing: () => testing
87
- });
88
- const SEARCH_CACHE = /* @__PURE__ */ new Map();
89
- const DEFAULT_SEARCH_COUNT = 5;
90
- const FIRECRAWL_SEARCH_MAX_RESULTS = 100;
91
- const FIRECRAWL_SEARCH_MAX_CONTENT_CHARS = 2e4;
92
- const DEFAULT_SCRAPE_MAX_CHARS = 5e4;
93
- const FIRECRAWL_SCRAPE_METADATA_MAX_CHARS = 4e3;
94
- const FIRECRAWL_RESULT_URL_MAX_CHARS = 2048;
95
- const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 67108864;
96
- const ALLOWED_FIRECRAWL_HOSTS = /* @__PURE__ */ new Set(["api.firecrawl.dev"]);
97
- const FIRECRAWL_PUBLISHED_DATE_RE = /^\d{4}-\d{2}-\d{2}(?:[T ][\d:.+Z-]{0,20})?$/u;
98
- const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR = "Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
99
- const FIRECRAWL_HTTP_PRIVATE_ERROR = "Firecrawl HTTP baseUrl must target a private or internal self-hosted endpoint. Use https:// for public hosts.";
100
78
  function assertFirecrawlScrapeTargetAllowed(url) {
101
79
  let parsed;
102
80
  try {
@@ -110,27 +88,24 @@ function assertFirecrawlScrapeTargetAllowed(url) {
110
88
  function isOfficialFirecrawlEndpoint(url) {
111
89
  return url.protocol === "https:" && ALLOWED_FIRECRAWL_HOSTS.has(url.hostname);
112
90
  }
113
- async function firecrawlEndpointTargetsPrivateNetwork(url, lookupFn) {
91
+ async function firecrawlEndpointTargetsPrivateNetwork(url) {
114
92
  if (isBlockedHostnameOrIp(url.hostname)) return true;
115
93
  try {
116
- return (await resolvePinnedHostnameWithPolicy(url.hostname, {
117
- lookupFn,
118
- policy: { allowPrivateNetwork: true }
119
- })).addresses.every((address) => isPrivateIpAddress(address));
94
+ return (await resolvePinnedHostnameWithPolicy(url.hostname, { policy: { allowPrivateNetwork: true } })).addresses.every((address) => isPrivateIpAddress(address));
120
95
  } catch {
121
96
  return false;
122
97
  }
123
98
  }
124
- async function validateFirecrawlBaseUrl(url, lookupFn) {
99
+ async function validateFirecrawlBaseUrl(url) {
125
100
  if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("Firecrawl baseUrl must use http:// or https://.");
126
101
  if (isOfficialFirecrawlEndpoint(url)) return "strict";
127
- if (await firecrawlEndpointTargetsPrivateNetwork(url, lookupFn)) return "selfHosted";
102
+ if (await firecrawlEndpointTargetsPrivateNetwork(url)) return "selfHosted";
128
103
  if (url.protocol === "http:") throw new Error(FIRECRAWL_HTTP_PRIVATE_ERROR);
129
104
  throw new Error(`${FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR} Host: ${url.hostname}`);
130
105
  }
131
- async function resolveEndpoint(baseUrl, pathname, lookupFn) {
132
- const url = new URL(baseUrl.trim() || "https://api.firecrawl.dev");
133
- const mode = await validateFirecrawlBaseUrl(url, lookupFn);
106
+ async function resolveEndpoint(baseUrl, pathname) {
107
+ const url = new URL(baseUrl.trim() || DEFAULT_FIRECRAWL_BASE_URL);
108
+ const mode = await validateFirecrawlBaseUrl(url);
134
109
  url.username = "";
135
110
  url.password = "";
136
111
  url.search = "";
@@ -246,20 +221,21 @@ async function runFirecrawlSearch(params) {
246
221
  min: 1,
247
222
  max: 100
248
223
  });
249
- const timeoutSeconds = resolveFirecrawlSearchTimeoutSeconds(params.timeoutSeconds);
224
+ const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, DEFAULT_FIRECRAWL_SEARCH_TIMEOUT_SECONDS);
250
225
  const scrapeResults = params.scrapeResults === true;
251
- const sources = Array.isArray(params.sources) ? params.sources.filter(Boolean) : [];
252
- const categories = Array.isArray(params.categories) ? params.categories.filter(Boolean) : [];
253
- const includeDomains = Array.isArray(params.includeDomains) ? params.includeDomains.filter(Boolean) : [];
254
- const excludeDomains = Array.isArray(params.excludeDomains) ? params.excludeDomains.filter(Boolean) : [];
226
+ const sources = params.sources?.filter(Boolean) ?? [];
227
+ const categories = params.categories?.filter(Boolean) ?? [];
228
+ const includeDomains = params.includeDomains?.filter(Boolean) ?? [];
229
+ const excludeDomains = params.excludeDomains?.filter(Boolean) ?? [];
255
230
  if (includeDomains.length > 0 && excludeDomains.length > 0) throw new Error("Firecrawl search accepts includeDomains or excludeDomains, not both.");
256
231
  const tbs = normalizeOptionalString(params.tbs);
257
232
  const location = normalizeOptionalString(params.location);
258
233
  const country = normalizeOptionalString(params.country);
259
234
  const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
260
- const cacheKey = normalizeCacheKey(JSON.stringify({
235
+ const cacheKey = JSON.stringify({
261
236
  type: "firecrawl-search",
262
237
  provider: providerId,
238
+ credential: apiKey ? sha256Hex(apiKey) : void 0,
263
239
  q: params.query,
264
240
  count,
265
241
  baseUrl,
@@ -271,13 +247,10 @@ async function runFirecrawlSearch(params) {
271
247
  location,
272
248
  country,
273
249
  scrapeResults
274
- }));
275
- const cacheTtlMs = resolveCacheTtlMs(params.cfg?.tools?.web?.search?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES);
276
- const cached = readCache(SEARCH_CACHE, cacheKey, cacheTtlMs);
277
- if (cached) return {
278
- ...cached.value,
279
- cached: true
280
- };
250
+ });
251
+ const cacheTtlMs = resolveSearchCacheTtlMs(params.cfg?.tools?.web?.search);
252
+ const cached = readCachedSearchPayload(cacheKey, cacheTtlMs);
253
+ if (cached) return cached;
281
254
  const body = {
282
255
  query: params.query,
283
256
  limit: count,
@@ -331,20 +304,52 @@ async function runFirecrawlSearch(params) {
331
304
  provider: providerId,
332
305
  count: items.length,
333
306
  tookMs,
334
- externalContent: {
335
- untrusted: true,
336
- source: "web_search",
337
- provider: providerId,
338
- wrapped: true
339
- },
307
+ externalContent: externalContentStamp(providerId),
340
308
  results,
341
309
  ...truncated ? { truncated: true } : {}
342
310
  };
343
- writeCache(SEARCH_CACHE, cacheKey, result, cacheTtlMs);
311
+ writeCachedSearchPayload(cacheKey, result, cacheTtlMs);
344
312
  return result;
345
313
  }
346
- function parseFirecrawlScrapePayload(params) {
347
- const data = asRecord(params.payload.data);
314
+ async function runFirecrawlScrape(params) {
315
+ params.signal?.throwIfAborted();
316
+ assertFirecrawlScrapeTargetAllowed(params.url);
317
+ const apiKey = resolveFirecrawlApiKey(params.cfg);
318
+ if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
319
+ const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
320
+ const fetch = resolveFirecrawlFetchConfig(params.cfg);
321
+ const timeoutSeconds = resolvePositiveTimeoutSeconds(params.timeoutSeconds, resolvePositiveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_FIRECRAWL_SCRAPE_TIMEOUT_SECONDS));
322
+ const onlyMainContent = asBoolean(params.onlyMainContent) ?? asBoolean(resolveFirecrawlFetchConfig(params.cfg)?.onlyMainContent) ?? true;
323
+ const maxAgeMs = Math.floor(asNonNegativeFiniteNumber(params.maxAgeMs) ?? asNonNegativeFiniteNumber(resolveFirecrawlFetchConfig(params.cfg)?.maxAgeMs) ?? DEFAULT_FIRECRAWL_MAX_AGE_MS);
324
+ const proxy = params.proxy ?? "auto";
325
+ const storeInCache = params.storeInCache ?? true;
326
+ const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
327
+ const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
328
+ const payload = await postFirecrawlJson({
329
+ url: endpoint.url,
330
+ mode: endpoint.mode,
331
+ timeoutSeconds,
332
+ apiKey,
333
+ errorLabel: "Firecrawl",
334
+ ...params.signal ? { signal: params.signal } : {},
335
+ body: {
336
+ url: params.url,
337
+ formats: ["markdown"],
338
+ onlyMainContent,
339
+ timeout: timeoutSeconds * 1e3,
340
+ maxAge: maxAgeMs,
341
+ proxy,
342
+ storeInCache
343
+ }
344
+ }, async (response) => {
345
+ const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
346
+ if (data.success === false) {
347
+ const detail = readFirecrawlErrorDetail(data, response.statusText);
348
+ throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
349
+ }
350
+ return data;
351
+ });
352
+ const data = asRecord(payload.data);
348
353
  const metadata = asOptionalObjectRecord(data.metadata);
349
354
  const rawStatus = parseFiniteNumber(metadata?.statusCode) ?? parseFiniteNumber(data.statusCode);
350
355
  const status = rawStatus === void 0 ? void 0 : Math.floor(rawStatus);
@@ -352,7 +357,7 @@ function parseFirecrawlScrapePayload(params) {
352
357
  const markdown = typeof data.markdown === "string" && data.markdown || typeof data.content === "string" && data.content || "";
353
358
  if (!markdown) throw new Error("Firecrawl scrape returned no content.");
354
359
  const rawText = params.extractMode === "text" ? markdownToText(markdown) : markdown;
355
- const boundedText = truncateSanitizedExternalContent(rawText, params.maxChars);
360
+ const boundedText = truncateSanitizedExternalContent(rawText, maxChars);
356
361
  let truncated = boundedText.truncated;
357
362
  let remainingMetadataChars = FIRECRAWL_SCRAPE_METADATA_MAX_CHARS;
358
363
  const wrapBoundedMetadata = (value) => {
@@ -369,7 +374,7 @@ function parseFirecrawlScrapePayload(params) {
369
374
  includeWarning: false
370
375
  });
371
376
  const title = typeof metadata?.title === "string" && metadata.title ? wrapBoundedMetadata(metadata.title) : void 0;
372
- const warning = typeof params.payload.warning === "string" && params.payload.warning ? wrapBoundedMetadata(params.payload.warning) : void 0;
377
+ const warning = typeof payload.warning === "string" && payload.warning ? wrapBoundedMetadata(payload.warning) : void 0;
373
378
  return {
374
379
  url: params.url,
375
380
  finalUrl: normalizeFirecrawlResultUrl(metadata?.sourceURL) ?? normalizeFirecrawlResultUrl(data.url) ?? params.url,
@@ -389,54 +394,4 @@ function parseFirecrawlScrapePayload(params) {
389
394
  ...warning ? { warning } : {}
390
395
  };
391
396
  }
392
- async function runFirecrawlScrape(params) {
393
- params.signal?.throwIfAborted();
394
- assertFirecrawlScrapeTargetAllowed(params.url);
395
- const apiKey = resolveFirecrawlApiKey(params.cfg);
396
- if (!apiKey && params.access !== "keyless") throw new Error("firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.");
397
- const baseUrl = resolveFirecrawlBaseUrl(params.cfg);
398
- const timeoutSeconds = resolveFirecrawlScrapeTimeoutSeconds(params.cfg, params.timeoutSeconds);
399
- const onlyMainContent = resolveFirecrawlOnlyMainContent(params.cfg, params.onlyMainContent);
400
- const maxAgeMs = resolveFirecrawlMaxAgeMs(params.cfg, params.maxAgeMs);
401
- const proxy = params.proxy ?? "auto";
402
- const storeInCache = params.storeInCache ?? true;
403
- const maxChars = Math.floor(Math.min(asPositiveFiniteNumber(params.maxChars) ?? DEFAULT_SCRAPE_MAX_CHARS, asPositiveFiniteNumber(params.cfg?.tools?.web?.fetch?.maxCharsCap) ?? DEFAULT_SCRAPE_MAX_CHARS));
404
- const endpoint = await resolveEndpoint(baseUrl, "/v2/scrape");
405
- return parseFirecrawlScrapePayload({
406
- payload: await postFirecrawlJson({
407
- url: endpoint.url,
408
- mode: endpoint.mode,
409
- timeoutSeconds,
410
- apiKey,
411
- errorLabel: "Firecrawl",
412
- ...params.signal ? { signal: params.signal } : {},
413
- body: {
414
- url: params.url,
415
- formats: ["markdown"],
416
- onlyMainContent,
417
- timeout: timeoutSeconds * 1e3,
418
- maxAge: maxAgeMs,
419
- proxy,
420
- storeInCache
421
- }
422
- }, async (response) => {
423
- const data = await readProviderJsonObjectResponse(response, "Firecrawl fetch failed", { maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES });
424
- if (data.success === false) {
425
- const detail = readFirecrawlErrorDetail(data, response.statusText);
426
- throw new Error(`Firecrawl fetch failed (${response.status}): ${wrapWebContent(truncateSanitizedExternalContent(detail, FIRECRAWL_SCRAPE_METADATA_MAX_CHARS).text, "web_fetch")}`.trim());
427
- }
428
- return data;
429
- }),
430
- url: params.url,
431
- extractMode: params.extractMode,
432
- maxChars
433
- });
434
- }
435
- const testing = {
436
- assertFirecrawlScrapeTargetAllowed,
437
- parseFirecrawlScrapePayload,
438
- resolveEndpoint,
439
- resolveSearchItems
440
- };
441
- //#endregion
442
397
  export { runFirecrawlScrape as n, runFirecrawlSearch as r, firecrawl_client_exports as t };
@@ -1,14 +1,8 @@
1
1
  import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
2
2
  import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
3
3
  import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch-contract";
4
- //#region extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
5
- function ensureRecord(target, key) {
6
- const current = target[key];
7
- if (current && typeof current === "object" && !Array.isArray(current)) return current;
8
- const next = {};
9
- target[key] = next;
10
- return next;
11
- }
4
+ import { ensureRecord } from "openclaw/plugin-sdk/collection-input-runtime";
5
+ // extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
12
6
  const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
13
7
  id: "firecrawl",
14
8
  label: "Firecrawl",
@@ -45,13 +39,13 @@ const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
45
39
  const plugins = configTarget.plugins ??= {};
46
40
  const entries = plugins.entries ??= {};
47
41
  const firecrawlEntry = entries.firecrawl ??= {};
48
- const webFetch = ensureRecord(firecrawlEntry.config ??= {}, "webFetch");
42
+ const pluginConfig = firecrawlEntry.config ??= {};
43
+ const webFetch = ensureRecord(pluginConfig, "webFetch");
49
44
  webFetch.apiKey = value;
50
45
  }
51
46
  };
52
- //#endregion
53
- //#region extensions/firecrawl/src/firecrawl-fetch-provider.ts
54
- const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CMLsufX8.mjs").then((n) => n.t));
47
+ // extensions/firecrawl/src/firecrawl-fetch-provider.ts
48
+ const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CDyLQvtN.mjs").then((n) => n.t));
55
49
  function createFirecrawlWebFetchProvider() {
56
50
  return {
57
51
  ...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
@@ -81,5 +75,4 @@ function createFirecrawlWebFetchProvider() {
81
75
  })
82
76
  };
83
77
  }
84
- //#endregion
85
78
  export { createFirecrawlWebFetchProvider as t };
@@ -1,29 +1,18 @@
1
+ import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime";
2
+ import { asOptionalRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
1
3
  import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
2
4
  import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
3
- import { asOptionalRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
4
- import { createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
5
- //#region extensions/firecrawl/src/firecrawl-search-provider.ts
5
+ import { WEB_SEARCH_QUERY_COUNT_PROPERTIES, createWebSearchProviderContractFields, enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-search-contract";
6
+ // extensions/firecrawl/src/firecrawl-search-provider.ts
6
7
  const FIRECRAWL_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webSearch.apiKey";
7
- const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CMLsufX8.mjs").then((n) => n.t));
8
+ const loadFirecrawlClientModule = createLazyRuntimeModule(() => import("./firecrawl-client-CDyLQvtN.mjs").then((n) => n.t));
8
9
  const GenericFirecrawlSearchSchema = {
9
10
  type: "object",
10
- properties: {
11
- query: {
12
- type: "string",
13
- description: "Search query string."
14
- },
15
- count: {
16
- type: "integer",
17
- description: "Number of results to return (1-10).",
18
- minimum: 1,
19
- maximum: 10
20
- }
21
- },
11
+ properties: { ...WEB_SEARCH_QUERY_COUNT_PROPERTIES },
22
12
  additionalProperties: false
23
13
  };
24
14
  function getConfiguredFirecrawlFetchCredentialFallback(config) {
25
- const pluginConfig = asOptionalRecord(config?.plugins?.entries?.firecrawl?.config);
26
- const value = asOptionalRecord(pluginConfig?.webFetch)?.apiKey;
15
+ const value = asOptionalRecord(resolvePluginConfigObject(config, "firecrawl")?.webFetch)?.apiKey;
27
16
  return value === void 0 ? void 0 : {
28
17
  path: "plugins.entries.firecrawl.config.webFetch.apiKey",
29
18
  value
@@ -106,5 +95,4 @@ function createFirecrawlWebSearchProvider() {
106
95
  function createFirecrawlFreeWebSearchProvider() {
107
96
  return createFirecrawlSearchProvider(true);
108
97
  }
109
- //#endregion
110
98
  export { createFirecrawlWebSearchProvider as n, createFirecrawlFreeWebSearchProvider as t };
@@ -1 +1 @@
1
- {"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-CMLsufX8.mjs","sha256":"cd5508855652ba7cbe5aeb38e4a6971884618194004da6eaccadc5e93f93a662","facadeSource":null,"sources":["extensions/firecrawl/src/config.ts","extensions/firecrawl/src/firecrawl-client.ts"]},{"output":".setup/firecrawl-fetch-provider-uRENNKri.mjs","sha256":"bcf92acf7d875b8f9b2e5dcea0a77477ea8d9a2bca67f77a7158f29c22f13a99","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts","extensions/firecrawl/src/firecrawl-fetch-provider.ts"]},{"output":".setup/firecrawl-search-provider-CFhWH8gx.mjs","sha256":"a5232cb776b35fd0fff26e372244d2d2d70cb359e8d71f23b12828728b171c92","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-search-provider.ts"]},{"output":"api.js","sha256":"f8f738dcc87a923f53352257cf6cb2adea6d343692994ab30a13294e9a990e3e","facadeSource":"extensions/firecrawl/api.ts","sources":["extensions/firecrawl/api.ts"]},{"output":"index.js","sha256":"3a150ed08c7458d7ee14df8246d5ec85d30761bc83e677d9d79acc3013fc2fba","facadeSource":"extensions/firecrawl/index.ts","sources":["extensions/firecrawl/index.ts","extensions/firecrawl/src/firecrawl-scrape-tool.ts","extensions/firecrawl/src/firecrawl-search-tool.ts"]},{"output":"web-fetch-contract-api.js","sha256":"99c45d50f0efd3eb331a4b152e9b806947d164ed773d1294ff913d9b9d74c73a","facadeSource":"extensions/firecrawl/web-fetch-contract-api.ts","sources":[]},{"output":"web-fetch-provider.js","sha256":"914b543ab4ba5d925b8065ba645d6dc943ba9a868f8c101673e7b47704aa50a5","facadeSource":"extensions/firecrawl/web-fetch-provider.ts","sources":[]},{"output":"web-search-contract-api.js","sha256":"73e9ed4ce719f3ad4b3bbb52dc3159c514edb410210c5ae3f059e68b8eb9b0f6","facadeSource":"extensions/firecrawl/web-search-contract-api.ts","sources":[]},{"output":"web-search-provider.js","sha256":"92101bd89ae69eb169fdd068b0663150d46721e28d7e03de84d7cf5dc2752bbd","facadeSource":"extensions/firecrawl/web-search-provider.ts","sources":[]}],"unrenderedSources":["extensions/firecrawl/web-fetch-contract-api.ts","extensions/firecrawl/web-fetch-provider.ts","extensions/firecrawl/web-search-contract-api.ts","extensions/firecrawl/web-search-provider.ts"]}
1
+ {"version":3,"entries":[{"output":"api.js","source":"extensions/firecrawl/api.ts"},{"output":"index.js","source":"extensions/firecrawl/index.ts"},{"output":"web-fetch-contract-api.js","source":"extensions/firecrawl/web-fetch-contract-api.ts"},{"output":"web-fetch-provider.js","source":"extensions/firecrawl/web-fetch-provider.ts"},{"output":"web-search-contract-api.js","source":"extensions/firecrawl/web-search-contract-api.ts"},{"output":"web-search-provider.js","source":"extensions/firecrawl/web-search-provider.ts"}],"chunks":[{"output":".setup/firecrawl-client-CDyLQvtN.mjs","sha256":"46c98acf8d71a8310851b8a6bfeb3d7e1176edb3d040c8e7d1de958e62cd4028","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-client.ts"]},{"output":".setup/firecrawl-fetch-provider-QmPccHZS.mjs","sha256":"d32ea4f6248fc6c0c388257809db11ac0107a9a7c473ac75a0f665f71dae723c","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts","extensions/firecrawl/src/firecrawl-fetch-provider.ts"]},{"output":".setup/firecrawl-search-provider-wsRgx6kl.mjs","sha256":"45416f284d535281a900a1ea20307652ca911769a3f36e9cbc5916a29a2ceafd","facadeSource":null,"sources":["extensions/firecrawl/src/firecrawl-search-provider.ts"]},{"output":"api.js","sha256":"ba89572c128911e1fb033a07004fa1c36861338a6704705ea14c6d6a1ea1cea3","facadeSource":"extensions/firecrawl/api.ts","sources":["extensions/firecrawl/api.ts"]},{"output":"index.js","sha256":"1b6689bdf7d803a983f86a4371e153139d127a259720af5d6412d102ecb1a3e5","facadeSource":"extensions/firecrawl/index.ts","sources":["extensions/firecrawl/index.ts"]},{"output":"web-fetch-contract-api.js","sha256":"e9f57ffa95524a8f5d9ac59cd879f41ec5c43e6e3ffd236a3bcfc6f55e2c439d","facadeSource":"extensions/firecrawl/web-fetch-contract-api.ts","sources":[]},{"output":"web-fetch-provider.js","sha256":"55e667f670164dd167c874b72e48fcba2b7ed2624d0d26f66d1fa78067f5b0b5","facadeSource":"extensions/firecrawl/web-fetch-provider.ts","sources":[]},{"output":"web-search-contract-api.js","sha256":"eb2a1d89385a524e030bc88d18d86398e5d9baf5bdeb57f9f4608017692541e4","facadeSource":"extensions/firecrawl/web-search-contract-api.ts","sources":[]},{"output":"web-search-provider.js","sha256":"54d793c602424e24a8e6f17c2f8eb832c5635c77e9094a1ca97c9b1249b17cb0","facadeSource":"extensions/firecrawl/web-search-provider.ts","sources":[]}],"unrenderedSources":["extensions/firecrawl/web-fetch-contract-api.ts","extensions/firecrawl/web-fetch-provider.ts","extensions/firecrawl/web-search-contract-api.ts","extensions/firecrawl/web-search-provider.ts"]}
package/dist/api.js CHANGED
@@ -1,5 +1,5 @@
1
- import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-CMLsufX8.mjs";
2
- //#region extensions/firecrawl/api.ts
1
+ import { n as runFirecrawlScrape } from "./.setup/firecrawl-client-CDyLQvtN.mjs";
2
+ // extensions/firecrawl/api.ts
3
3
  async function fetchFirecrawlContent(params) {
4
4
  const cfg = { plugins: { entries: { firecrawl: {
5
5
  enabled: true,
@@ -30,5 +30,4 @@ async function fetchFirecrawlContent(params) {
30
30
  warning: result.warning
31
31
  };
32
32
  }
33
- //#endregion
34
33
  export { fetchFirecrawlContent };