firecrawl 4.38.0 → 4.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -2
- package/dist/{chunk-KSTNOGOD.js → chunk-OJ4YJQ4J.js} +1 -1
- package/dist/index.cjs +204 -17
- package/dist/index.d.cts +289 -7
- package/dist/index.d.ts +289 -7
- package/dist/index.js +203 -17
- package/dist/{package-OS7RONXY.js → package-F6O7J2XI.js} +1 -1
- package/package.json +1 -1
- package/pnpm-workspace.yaml +3 -2
- package/src/__tests__/e2e/v2/batch.test.ts +13 -11
- package/src/__tests__/e2e/v2/crawl.test.ts +24 -20
- package/src/__tests__/e2e/v2/extract.test.ts +32 -7
- package/src/__tests__/e2e/v2/map.test.ts +4 -3
- package/src/__tests__/e2e/v2/parse.test.ts +4 -3
- package/src/__tests__/e2e/v2/scrape.test.ts +10 -9
- package/src/__tests__/e2e/v2/search.test.ts +9 -8
- package/src/__tests__/e2e/v2/usage.test.ts +6 -5
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +257 -0
- package/src/__tests__/e2e/v2/watcher.test.ts +4 -3
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +164 -0
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +501 -0
- package/src/__tests__/unit/v2/extract.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +1 -1
- package/src/__tests__/unit/v2/search-country.unit.test.ts +32 -0
- package/src/__tests__/unit/v2/search-data-property.test.ts +1 -0
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +18 -1
- package/src/__tests__/unit/v2/tools.unit.test.ts +240 -0
- package/src/__tests__/unit/v2/v1-aliases.test.ts +1 -0
- package/src/v2/client.ts +52 -4
- package/src/v2/methods/agent.ts +23 -1
- package/src/v2/methods/research.ts +14 -8
- package/src/v2/methods/scrape.ts +3 -1
- package/src/v2/methods/search.ts +8 -1
- package/src/v2/methods/tools.ts +137 -0
- package/src/v2/types.ts +307 -35
- package/src/v2/utils/errorHandler.ts +19 -3
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-OJ4YJQ4J.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -138,18 +138,35 @@ var HttpClient = class {
|
|
|
138
138
|
};
|
|
139
139
|
|
|
140
140
|
// src/v2/types.ts
|
|
141
|
+
function parseRequiresAction(value) {
|
|
142
|
+
if (!value || typeof value !== "object") return void 0;
|
|
143
|
+
const record = value;
|
|
144
|
+
if (typeof record.type !== "string") return void 0;
|
|
145
|
+
const action = { type: record.type };
|
|
146
|
+
if (typeof record.terms === "string") action.terms = record.terms;
|
|
147
|
+
if (typeof record.version === "string") action.version = record.version;
|
|
148
|
+
if (typeof record.url === "string") action.url = record.url;
|
|
149
|
+
return action;
|
|
150
|
+
}
|
|
141
151
|
var SdkError = class extends Error {
|
|
152
|
+
requestId;
|
|
142
153
|
status;
|
|
143
154
|
code;
|
|
144
155
|
details;
|
|
145
156
|
jobId;
|
|
146
|
-
|
|
157
|
+
/** Present on exchange-mediated scrape failures that already captured credits. */
|
|
158
|
+
chargeId;
|
|
159
|
+
/** Present when the API needs an out-of-band step first, such as accepting provider terms. */
|
|
160
|
+
requiresAction;
|
|
161
|
+
constructor(message, status, code, details, jobId, chargeId, requiresAction) {
|
|
147
162
|
super(message);
|
|
148
163
|
this.name = "FirecrawlSdkError";
|
|
149
164
|
this.status = status;
|
|
150
165
|
this.code = code;
|
|
151
166
|
this.details = details;
|
|
152
167
|
this.jobId = jobId;
|
|
168
|
+
this.chargeId = chargeId;
|
|
169
|
+
this.requiresAction = requiresAction;
|
|
153
170
|
}
|
|
154
171
|
};
|
|
155
172
|
var JobTimeoutError = class extends SdkError {
|
|
@@ -401,14 +418,30 @@ function throwForBadResponse(resp, action) {
|
|
|
401
418
|
const status = resp.status;
|
|
402
419
|
const body = resp.data || {};
|
|
403
420
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
404
|
-
throw new SdkError(
|
|
421
|
+
throw new SdkError(
|
|
422
|
+
msg,
|
|
423
|
+
status,
|
|
424
|
+
body?.code,
|
|
425
|
+
body?.details,
|
|
426
|
+
void 0,
|
|
427
|
+
body?.chargeId,
|
|
428
|
+
parseRequiresAction(body?.requiresAction)
|
|
429
|
+
);
|
|
405
430
|
}
|
|
406
431
|
function normalizeAxiosError(err, action) {
|
|
407
432
|
const status = err.response?.status;
|
|
408
433
|
const body = err.response?.data;
|
|
409
434
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
410
435
|
const code = body?.code || err.code;
|
|
411
|
-
throw new SdkError(
|
|
436
|
+
throw new SdkError(
|
|
437
|
+
message,
|
|
438
|
+
status,
|
|
439
|
+
code,
|
|
440
|
+
body?.details ?? body,
|
|
441
|
+
void 0,
|
|
442
|
+
body?.chargeId,
|
|
443
|
+
parseRequiresAction(body?.requiresAction)
|
|
444
|
+
);
|
|
412
445
|
}
|
|
413
446
|
function isRetryableError(err) {
|
|
414
447
|
if (err instanceof JobTimeoutError) {
|
|
@@ -454,9 +487,10 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
454
487
|
throw new Error("URL cannot be empty");
|
|
455
488
|
}
|
|
456
489
|
if (options) ensureValidScrapeOptions(options);
|
|
457
|
-
const { autoResume, ...requestOptions } = options ?? {};
|
|
490
|
+
const { autoResume, domainTools, ...requestOptions } = options ?? {};
|
|
458
491
|
const payload = { url: url.trim() };
|
|
459
492
|
Object.assign(payload, requestOptions);
|
|
493
|
+
if (domainTools) payload.domainTools = domainTools;
|
|
460
494
|
const requestTimeout = autoResume === false ? typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {} : {
|
|
461
495
|
timeoutMs: Math.max(
|
|
462
496
|
typeof options?.timeout === "number" ? options.timeout : 3e5,
|
|
@@ -613,6 +647,7 @@ function prepareSearchPayload(req) {
|
|
|
613
647
|
const payload = {
|
|
614
648
|
query: req.query
|
|
615
649
|
};
|
|
650
|
+
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
616
651
|
if (req.sources) payload.sources = req.sources;
|
|
617
652
|
if (req.categories) payload.categories = req.categories;
|
|
618
653
|
if (req.includeDomains) payload.includeDomains = req.includeDomains;
|
|
@@ -620,6 +655,7 @@ function prepareSearchPayload(req) {
|
|
|
620
655
|
if (req.limit != null) payload.limit = req.limit;
|
|
621
656
|
if (req.tbs != null) payload.tbs = req.tbs;
|
|
622
657
|
if (req.location != null) payload.location = req.location;
|
|
658
|
+
if (req.country != null) payload.country = req.country;
|
|
623
659
|
if (req.ignoreInvalidURLs != null)
|
|
624
660
|
payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
|
|
625
661
|
if (req.timeout != null) payload.timeout = req.timeout;
|
|
@@ -664,10 +700,12 @@ async function search(http, request) {
|
|
|
664
700
|
}
|
|
665
701
|
const data = res.data.data || {};
|
|
666
702
|
const out = {};
|
|
703
|
+
if (res.data.warning) out.warning = res.data.warning;
|
|
667
704
|
if (data.web) out.web = transformArray(data.web);
|
|
668
705
|
if (data.news) out.news = transformArray(data.news);
|
|
669
706
|
if (data.images)
|
|
670
707
|
out.images = transformArray(data.images);
|
|
708
|
+
if (data.tools) out.tools = data.tools;
|
|
671
709
|
Object.defineProperty(out, "data", {
|
|
672
710
|
get() {
|
|
673
711
|
const parts = [];
|
|
@@ -675,7 +713,9 @@ async function search(http, request) {
|
|
|
675
713
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
676
714
|
if (out.images?.length)
|
|
677
715
|
parts.push(`.images (${out.images.length} results)`);
|
|
678
|
-
|
|
716
|
+
if (out.tools?.length)
|
|
717
|
+
parts.push(`.tools (${out.tools.length} results)`);
|
|
718
|
+
const available = parts.length ? parts.join(", ") : ".web, .news, .images, or .tools";
|
|
679
719
|
throw new Error(
|
|
680
720
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
681
721
|
);
|
|
@@ -690,6 +730,101 @@ async function search(http, request) {
|
|
|
690
730
|
}
|
|
691
731
|
}
|
|
692
732
|
|
|
733
|
+
// src/v2/methods/tools.ts
|
|
734
|
+
var ALEXANDRIA_MAX_CALLS = 10;
|
|
735
|
+
function prepareAlexandriaPayload(calls, opts) {
|
|
736
|
+
if (!Array.isArray(calls) || calls.length === 0) {
|
|
737
|
+
throw new Error("alexandria requires at least one call");
|
|
738
|
+
}
|
|
739
|
+
if (calls.length > ALEXANDRIA_MAX_CALLS) {
|
|
740
|
+
throw new Error(`alexandria accepts at most ${ALEXANDRIA_MAX_CALLS} calls`);
|
|
741
|
+
}
|
|
742
|
+
const alexandria = calls.map((call, index) => {
|
|
743
|
+
if (!call || typeof call.provider !== "string" || !call.provider.trim()) {
|
|
744
|
+
throw new Error(`alexandria[${index}].provider cannot be empty`);
|
|
745
|
+
}
|
|
746
|
+
if (typeof call.capability !== "string" || !call.capability.trim()) {
|
|
747
|
+
throw new Error(`alexandria[${index}].capability cannot be empty`);
|
|
748
|
+
}
|
|
749
|
+
if (Object.keys(call).some(
|
|
750
|
+
(key) => !["provider", "capability", "options"].includes(key)
|
|
751
|
+
))
|
|
752
|
+
throw new Error("Unknown alexandria call option");
|
|
753
|
+
const item = {
|
|
754
|
+
provider: call.provider.trim(),
|
|
755
|
+
capability: call.capability.trim()
|
|
756
|
+
};
|
|
757
|
+
if (call.options != null) {
|
|
758
|
+
if (typeof call.options !== "object" || Array.isArray(call.options)) {
|
|
759
|
+
throw new Error(`alexandria[${index}].options must be an object`);
|
|
760
|
+
}
|
|
761
|
+
item.options = call.options;
|
|
762
|
+
}
|
|
763
|
+
return item;
|
|
764
|
+
});
|
|
765
|
+
if (opts.timeout != null && (!Number.isInteger(opts.timeout) || opts.timeout <= 0)) {
|
|
766
|
+
throw new Error("timeout must be a positive integer");
|
|
767
|
+
}
|
|
768
|
+
const payload = { alexandria };
|
|
769
|
+
if (opts.timeout != null) payload.timeout = opts.timeout;
|
|
770
|
+
if (opts.integration && opts.integration.trim())
|
|
771
|
+
payload.integration = opts.integration.trim();
|
|
772
|
+
if (opts.origin) payload.origin = opts.origin;
|
|
773
|
+
return payload;
|
|
774
|
+
}
|
|
775
|
+
async function scrapeAlexandria(http, calls, opts = {}) {
|
|
776
|
+
const payload = prepareAlexandriaPayload(calls, opts);
|
|
777
|
+
const requestId = opts.requestId ?? crypto.randomUUID();
|
|
778
|
+
if (!/^[A-Za-z0-9._:-]{1,128}$/.test(requestId))
|
|
779
|
+
throw new Error("Invalid requestId");
|
|
780
|
+
try {
|
|
781
|
+
const res = await http.post("/v2/scrape", payload, {
|
|
782
|
+
headers: { "x-request-id": requestId },
|
|
783
|
+
// Allow response delivery after the API's capped execution deadline.
|
|
784
|
+
timeoutMs: Math.min(opts.timeout ?? 5e4, 5e4) + 3e4
|
|
785
|
+
});
|
|
786
|
+
if (res.status !== 200 || !res.data?.success) {
|
|
787
|
+
throwForBadResponse(res, "alexandria");
|
|
788
|
+
}
|
|
789
|
+
const data = res.data.data;
|
|
790
|
+
if (!data || !Array.isArray(data.alexandria) || typeof data.creditsCost !== "number" || !Number.isInteger(data.creditsCost) || data.creditsCost < 0) {
|
|
791
|
+
throw new SdkError("Invalid alexandria response");
|
|
792
|
+
}
|
|
793
|
+
return {
|
|
794
|
+
scrapeId: res.data.scrape_id ?? "",
|
|
795
|
+
requestId,
|
|
796
|
+
alexandria: data.alexandria,
|
|
797
|
+
creditsCost: data.creditsCost
|
|
798
|
+
};
|
|
799
|
+
} catch (err) {
|
|
800
|
+
try {
|
|
801
|
+
if (err?.isAxiosError) normalizeAxiosError(err, "alexandria");
|
|
802
|
+
throw err;
|
|
803
|
+
} catch (error) {
|
|
804
|
+
if (error && typeof error === "object")
|
|
805
|
+
Object.assign(error, { requestId });
|
|
806
|
+
throw error;
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
async function findTools(http, options = {}) {
|
|
811
|
+
const result = await scrapeAlexandria(http, [
|
|
812
|
+
{
|
|
813
|
+
provider: "firecrawl",
|
|
814
|
+
capability: "find-tools",
|
|
815
|
+
options: { ...options }
|
|
816
|
+
}
|
|
817
|
+
]);
|
|
818
|
+
const item = result.alexandria[0];
|
|
819
|
+
if (!item) throw new SdkError("Missing Find Tools result");
|
|
820
|
+
if (item.error)
|
|
821
|
+
throw Object.assign(
|
|
822
|
+
new SdkError(item.error.message, item.error.status, item.error.code),
|
|
823
|
+
{ requestId: result.requestId }
|
|
824
|
+
);
|
|
825
|
+
return item.data;
|
|
826
|
+
}
|
|
827
|
+
|
|
693
828
|
// src/v2/methods/developer.ts
|
|
694
829
|
var ENDPOINT = "/v2/search/developer";
|
|
695
830
|
async function developerSearch(http, query, options = {}) {
|
|
@@ -1240,6 +1375,9 @@ function prepareAgentPayload(args) {
|
|
|
1240
1375
|
body.threatProtection = args.threatProtection;
|
|
1241
1376
|
if (args.auditMetadata != null)
|
|
1242
1377
|
body.auditMetadata = args.auditMetadata;
|
|
1378
|
+
if (args.threadId != null) body.threadId = args.threadId;
|
|
1379
|
+
if (args.mode != null) body.mode = args.mode;
|
|
1380
|
+
if (args.exchange != null) body.exchange = args.exchange;
|
|
1243
1381
|
return body;
|
|
1244
1382
|
}
|
|
1245
1383
|
async function startAgent(http, args) {
|
|
@@ -1310,6 +1448,17 @@ async function getAgentSnapshot(http, jobId, snapshotId) {
|
|
|
1310
1448
|
throw err;
|
|
1311
1449
|
}
|
|
1312
1450
|
}
|
|
1451
|
+
async function getAgentThread(http, threadId, options) {
|
|
1452
|
+
try {
|
|
1453
|
+
const query = options?.includeData ? "?includeData=true" : "";
|
|
1454
|
+
const res = await http.get(`/v2/agent/threads/${threadId}${query}`);
|
|
1455
|
+
if (res.status !== 200) throwForBadResponse(res, "agent thread");
|
|
1456
|
+
return res.data;
|
|
1457
|
+
} catch (err) {
|
|
1458
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent thread");
|
|
1459
|
+
throw err;
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1313
1462
|
async function cancelAgent(http, jobId) {
|
|
1314
1463
|
try {
|
|
1315
1464
|
const res = await http.delete(`/v2/agent/${jobId}`);
|
|
@@ -1550,9 +1699,7 @@ var ResearchClient = class {
|
|
|
1550
1699
|
appendParam(params, "query", options.query);
|
|
1551
1700
|
appendParam(params, "k", options.k);
|
|
1552
1701
|
try {
|
|
1553
|
-
const res = await this.http.get(
|
|
1554
|
-
withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params)
|
|
1555
|
-
);
|
|
1702
|
+
const res = await this.http.get(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
|
|
1556
1703
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1557
1704
|
return res.data;
|
|
1558
1705
|
} catch (err) {
|
|
@@ -1578,10 +1725,7 @@ var ResearchClient = class {
|
|
|
1578
1725
|
appendParam(params, "anchor", options.anchor);
|
|
1579
1726
|
try {
|
|
1580
1727
|
const res = await this.http.get(
|
|
1581
|
-
withQuery(
|
|
1582
|
-
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
1583
|
-
params
|
|
1584
|
-
)
|
|
1728
|
+
withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params)
|
|
1585
1729
|
);
|
|
1586
1730
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
1587
1731
|
return res.data;
|
|
@@ -1590,7 +1734,16 @@ var ResearchClient = class {
|
|
|
1590
1734
|
}
|
|
1591
1735
|
}
|
|
1592
1736
|
/**
|
|
1593
|
-
* Search GitHub issue/PR history and repository
|
|
1737
|
+
* Search the research index GitHub slice: issue/PR history and repository
|
|
1738
|
+
* readmes.
|
|
1739
|
+
*
|
|
1740
|
+
* @deprecated Use `developerSearch()` on the client instead. This
|
|
1741
|
+
* endpoint stops responding after 2026-11-03. The developer index searches
|
|
1742
|
+
* GitHub issues, pull requests and readmes plus curated documentation
|
|
1743
|
+
* sources, returns matched passages, and adds filters for repo, language,
|
|
1744
|
+
* license and stars. It does not carry over the `scores` breakdown or the
|
|
1745
|
+
* `resultType: "web"` fallback results. See
|
|
1746
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1594
1747
|
* @param query Search query.
|
|
1595
1748
|
* @param options Optional `k`.
|
|
1596
1749
|
*/
|
|
@@ -1999,7 +2152,28 @@ var FirecrawlClient = class {
|
|
|
1999
2152
|
});
|
|
2000
2153
|
}
|
|
2001
2154
|
async scrape(url, options) {
|
|
2002
|
-
return scrape(this.http, url, options);
|
|
2155
|
+
if (typeof url === "string") return scrape(this.http, url, options);
|
|
2156
|
+
if (!url || options !== void 0 || Object.keys(url).some(
|
|
2157
|
+
(key) => ![
|
|
2158
|
+
"alexandria",
|
|
2159
|
+
"requestId",
|
|
2160
|
+
"timeout",
|
|
2161
|
+
"integration",
|
|
2162
|
+
"origin"
|
|
2163
|
+
].includes(key)
|
|
2164
|
+
)) {
|
|
2165
|
+
throw new Error("Provide an alexandria request without URL scrape options");
|
|
2166
|
+
}
|
|
2167
|
+
const { alexandria, ...opts } = url;
|
|
2168
|
+
return scrapeAlexandria(
|
|
2169
|
+
this.http,
|
|
2170
|
+
Array.isArray(alexandria) ? alexandria : [alexandria],
|
|
2171
|
+
opts
|
|
2172
|
+
);
|
|
2173
|
+
}
|
|
2174
|
+
/** Explore the catalogue without executing the tools it returns. */
|
|
2175
|
+
async findTools(options) {
|
|
2176
|
+
return findTools(this.http, options);
|
|
2003
2177
|
}
|
|
2004
2178
|
/**
|
|
2005
2179
|
* Interact with the browser session associated with a scrape job.
|
|
@@ -2079,6 +2253,9 @@ var FirecrawlClient = class {
|
|
|
2079
2253
|
* Access the v2 research endpoints — Firecrawl's **research paper index**
|
|
2080
2254
|
* (~43M paper abstracts) plus GitHub history/readmes.
|
|
2081
2255
|
*
|
|
2256
|
+
* `research.searchGithub()` is deprecated and stops responding after
|
|
2257
|
+
* 2026-11-03. Use `developerSearch()` instead.
|
|
2258
|
+
*
|
|
2082
2259
|
* The paper corpus is roughly 90% biomedical and life sciences — PubMed,
|
|
2083
2260
|
* bioRxiv and medRxiv — with arXiv covering physics, mathematics and
|
|
2084
2261
|
* computer science.
|
|
@@ -2338,6 +2515,14 @@ var FirecrawlClient = class {
|
|
|
2338
2515
|
async getAgentSnapshot(jobId, snapshotId) {
|
|
2339
2516
|
return getAgentSnapshot(this.http, jobId, snapshotId);
|
|
2340
2517
|
}
|
|
2518
|
+
/**
|
|
2519
|
+
* Get a thread and its runs, oldest turn first.
|
|
2520
|
+
* @param threadId Thread id, as returned by startAgent or getAgentStatus.
|
|
2521
|
+
* @param options.includeData Inline each succeeded run's data.
|
|
2522
|
+
*/
|
|
2523
|
+
async getAgentThread(threadId, options) {
|
|
2524
|
+
return getAgentThread(this.http, threadId, options);
|
|
2525
|
+
}
|
|
2341
2526
|
// Browser
|
|
2342
2527
|
/**
|
|
2343
2528
|
* Create a new browser session.
|
|
@@ -2480,7 +2665,7 @@ var FirecrawlApp = class {
|
|
|
2480
2665
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2481
2666
|
return process.env.npm_package_version;
|
|
2482
2667
|
}
|
|
2483
|
-
const packageJson = await import("./package-
|
|
2668
|
+
const packageJson = await import("./package-F6O7J2XI.js");
|
|
2484
2669
|
return packageJson.default.version;
|
|
2485
2670
|
} catch (error) {
|
|
2486
2671
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -3880,5 +4065,6 @@ export {
|
|
|
3880
4065
|
ResearchClient,
|
|
3881
4066
|
SdkError,
|
|
3882
4067
|
Watcher,
|
|
3883
|
-
index_default as default
|
|
4068
|
+
index_default as default,
|
|
4069
|
+
parseRequiresAction
|
|
3884
4070
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.40.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -8,10 +8,11 @@ overrides:
|
|
|
8
8
|
"picomatch@<4.0.4": ">=4.0.4"
|
|
9
9
|
handlebars: ">=4.7.9"
|
|
10
10
|
"brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
|
|
11
|
-
"js-yaml@>=3.0.0 <3.15.
|
|
12
|
-
"js-yaml@>=4.0.0 <4.3.
|
|
11
|
+
"js-yaml@>=3.0.0 <3.15.2": "3.15.2"
|
|
12
|
+
"js-yaml@>=4.0.0 <4.3.2": "4.3.2"
|
|
13
13
|
"axios@>=1.0.0 <1.16.0": "1.16.1"
|
|
14
14
|
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
|
15
15
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
16
16
|
"@babel/core@>=7.0.0 <7.29.6": "7.29.7"
|
|
17
17
|
"esbuild@>=0.17.0 <0.28.1": "0.28.1"
|
|
18
|
+
"browserslist@<=4.28.6": ">=4.28.7 <5.0.0"
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import Firecrawl from "../../../index";
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
|
+
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
7
8
|
import { describe, test, expect, beforeAll } from "@jest/globals";
|
|
8
9
|
|
|
9
10
|
config();
|
|
@@ -13,7 +14,7 @@ let client: Firecrawl;
|
|
|
13
14
|
|
|
14
15
|
beforeAll(async () => {
|
|
15
16
|
const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
|
|
16
|
-
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
|
17
|
+
client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
|
|
17
18
|
});
|
|
18
19
|
|
|
19
20
|
describe("v2.batch e2e", () => {
|
|
@@ -22,19 +23,21 @@ describe("v2.batch e2e", () => {
|
|
|
22
23
|
"https://docs.firecrawl.dev",
|
|
23
24
|
"https://firecrawl.dev",
|
|
24
25
|
];
|
|
25
|
-
const
|
|
26
|
+
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] } });
|
|
27
|
+
const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
|
|
26
28
|
expect(["completed", "failed"]).toContain(job.status);
|
|
27
29
|
expect(job.completed).toBeGreaterThanOrEqual(0);
|
|
28
30
|
expect(job.total).toBeGreaterThanOrEqual(0);
|
|
29
31
|
expect(Array.isArray(job.data)).toBe(true);
|
|
30
|
-
},
|
|
32
|
+
}, testTimeoutMs(362_000));
|
|
31
33
|
|
|
32
34
|
test("batch scrape with wait returns job id for error retrieval", async () => {
|
|
33
35
|
const urls = [
|
|
34
36
|
"https://docs.firecrawl.dev",
|
|
35
37
|
"https://firecrawl.dev",
|
|
36
38
|
];
|
|
37
|
-
const
|
|
39
|
+
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] } });
|
|
40
|
+
const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
|
|
38
41
|
// Verify job has id field
|
|
39
42
|
expect(job.id).toBeDefined();
|
|
40
43
|
expect(typeof job.id).toBe("string");
|
|
@@ -44,7 +47,7 @@ describe("v2.batch e2e", () => {
|
|
|
44
47
|
expect(errors).toHaveProperty("robotsBlocked");
|
|
45
48
|
expect(Array.isArray(errors.errors)).toBe(true);
|
|
46
49
|
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
|
47
|
-
},
|
|
50
|
+
}, testTimeoutMs(512_000));
|
|
48
51
|
|
|
49
52
|
test("start batch minimal and status", async () => {
|
|
50
53
|
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
|
@@ -57,11 +60,11 @@ describe("v2.batch e2e", () => {
|
|
|
57
60
|
// Verify status includes id field
|
|
58
61
|
expect(status.id).toBeDefined();
|
|
59
62
|
expect(status.id).toBe(start.id);
|
|
60
|
-
},
|
|
63
|
+
}, testTimeoutMs(151_000));
|
|
61
64
|
|
|
62
65
|
test("wait batch with all params", async () => {
|
|
63
66
|
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
|
64
|
-
const
|
|
67
|
+
const start = await client.startBatchScrape(urls, {
|
|
65
68
|
options: {
|
|
66
69
|
formats: [
|
|
67
70
|
"markdown",
|
|
@@ -74,14 +77,13 @@ describe("v2.batch e2e", () => {
|
|
|
74
77
|
ignoreInvalidURLs: true,
|
|
75
78
|
maxConcurrency: 2,
|
|
76
79
|
zeroDataRetention: false,
|
|
77
|
-
pollInterval: 1,
|
|
78
|
-
timeout: 180,
|
|
79
80
|
});
|
|
81
|
+
const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
|
|
80
82
|
expect(["completed", "failed", "cancelled"]).toContain(job.status);
|
|
81
83
|
expect(job.completed).toBeGreaterThanOrEqual(0);
|
|
82
84
|
expect(job.total).toBeGreaterThanOrEqual(0);
|
|
83
85
|
expect(Array.isArray(job.data)).toBe(true);
|
|
84
|
-
},
|
|
86
|
+
}, testTimeoutMs(362_000));
|
|
85
87
|
|
|
86
88
|
test("cancel batch", async () => {
|
|
87
89
|
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
|
@@ -89,6 +91,6 @@ describe("v2.batch e2e", () => {
|
|
|
89
91
|
expect(typeof start.id).toBe("string");
|
|
90
92
|
const cancelled = await client.cancelBatchScrape(start.id);
|
|
91
93
|
expect(cancelled).toBe(true);
|
|
92
|
-
},
|
|
94
|
+
}, testTimeoutMs(151_000));
|
|
93
95
|
});
|
|
94
96
|
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import Firecrawl from "../../../index";
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
|
+
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
7
8
|
import { describe, test, expect, beforeAll } from "@jest/globals";
|
|
8
9
|
|
|
9
10
|
config();
|
|
@@ -13,7 +14,7 @@ let client: Firecrawl;
|
|
|
13
14
|
|
|
14
15
|
beforeAll(async () => {
|
|
15
16
|
const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
|
|
16
|
-
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
|
17
|
+
client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
|
|
17
18
|
});
|
|
18
19
|
|
|
19
20
|
describe("v2.crawl e2e", () => {
|
|
@@ -23,21 +24,21 @@ describe("v2.crawl e2e", () => {
|
|
|
23
24
|
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
|
24
25
|
expect(typeof job.id).toBe("string");
|
|
25
26
|
expect(typeof job.url).toBe("string");
|
|
26
|
-
}, 90_000);
|
|
27
|
+
}, testTimeoutMs(90_000));
|
|
27
28
|
|
|
28
29
|
test("start crawl with options", async () => {
|
|
29
30
|
if (!client) throw new Error();
|
|
30
31
|
const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
|
|
31
32
|
expect(typeof job.id).toBe("string");
|
|
32
33
|
expect(typeof job.url).toBe("string");
|
|
33
|
-
}, 90_000);
|
|
34
|
+
}, testTimeoutMs(90_000));
|
|
34
35
|
|
|
35
36
|
test("start crawl with prompt", async () => {
|
|
36
37
|
if (!client) throw new Error();
|
|
37
38
|
const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
|
|
38
39
|
expect(typeof job.id).toBe("string");
|
|
39
40
|
expect(typeof job.url).toBe("string");
|
|
40
|
-
}, 90_000);
|
|
41
|
+
}, testTimeoutMs(90_000));
|
|
41
42
|
|
|
42
43
|
test("get crawl status", async () => {
|
|
43
44
|
if (!client) throw new Error();
|
|
@@ -50,14 +51,14 @@ describe("v2.crawl e2e", () => {
|
|
|
50
51
|
expect(status.id).toBe(start.id);
|
|
51
52
|
// next/expiresAt may be null/undefined depending on state; check shape
|
|
52
53
|
expect(Array.isArray(status.data)).toBe(true);
|
|
53
|
-
},
|
|
54
|
+
}, testTimeoutMs(151_000));
|
|
54
55
|
|
|
55
56
|
test("cancel crawl", async () => {
|
|
56
57
|
if (!client) throw new Error();
|
|
57
58
|
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
|
|
58
59
|
const ok = await client.cancelCrawl(start.id);
|
|
59
60
|
expect(ok).toBe(true);
|
|
60
|
-
},
|
|
61
|
+
}, testTimeoutMs(151_000));
|
|
61
62
|
|
|
62
63
|
test("get crawl errors", async () => {
|
|
63
64
|
if (!client) throw new Error();
|
|
@@ -73,12 +74,12 @@ describe("v2.crawl e2e", () => {
|
|
|
73
74
|
expect(typeof e.url === "string" || e.url == null).toBe(true);
|
|
74
75
|
expect(typeof e.error === "string" || e.error == null).toBe(true);
|
|
75
76
|
}
|
|
76
|
-
},
|
|
77
|
+
}, testTimeoutMs(151_000));
|
|
77
78
|
|
|
78
79
|
test("get crawl errors with invalid id should throw", async () => {
|
|
79
80
|
if (!client) throw new Error();
|
|
80
81
|
await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
|
|
81
|
-
}, 60_000);
|
|
82
|
+
}, testTimeoutMs(60_000));
|
|
82
83
|
|
|
83
84
|
test("get active crawls", async () => {
|
|
84
85
|
if (!client) throw new Error();
|
|
@@ -93,7 +94,7 @@ describe("v2.crawl e2e", () => {
|
|
|
93
94
|
expect(typeof c.options === "object").toBe(true);
|
|
94
95
|
}
|
|
95
96
|
}
|
|
96
|
-
}, 90_000);
|
|
97
|
+
}, testTimeoutMs(90_000));
|
|
97
98
|
|
|
98
99
|
test("get active crawls with running crawl", async () => {
|
|
99
100
|
if (!client) throw new Error();
|
|
@@ -104,20 +105,22 @@ describe("v2.crawl e2e", () => {
|
|
|
104
105
|
const ids = active.crawls.map(c => c.id);
|
|
105
106
|
expect(ids.includes(start.id)).toBe(true);
|
|
106
107
|
await client.cancelCrawl(start.id);
|
|
107
|
-
},
|
|
108
|
+
}, testTimeoutMs(301_000));
|
|
108
109
|
|
|
109
110
|
test("crawl with wait", async () => {
|
|
110
111
|
if (!client) throw new Error();
|
|
111
|
-
const
|
|
112
|
+
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2 });
|
|
113
|
+
const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
|
|
112
114
|
expect(["completed", "failed"]).toContain(job.status);
|
|
113
115
|
expect(job.completed).toBeGreaterThanOrEqual(0);
|
|
114
116
|
expect(job.total).toBeGreaterThanOrEqual(0);
|
|
115
117
|
expect(Array.isArray(job.data)).toBe(true);
|
|
116
|
-
},
|
|
118
|
+
}, testTimeoutMs(272_000));
|
|
117
119
|
|
|
118
120
|
test("crawl with wait returns job id for error retrieval", async () => {
|
|
119
121
|
if (!client) throw new Error();
|
|
120
|
-
const
|
|
122
|
+
const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2 });
|
|
123
|
+
const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
|
|
121
124
|
// Verify job has id field
|
|
122
125
|
expect(job.id).toBeDefined();
|
|
123
126
|
expect(typeof job.id).toBe("string");
|
|
@@ -127,16 +130,17 @@ describe("v2.crawl e2e", () => {
|
|
|
127
130
|
expect(errors).toHaveProperty("robotsBlocked");
|
|
128
131
|
expect(Array.isArray(errors.errors)).toBe(true);
|
|
129
132
|
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
|
130
|
-
},
|
|
133
|
+
}, testTimeoutMs(422_000));
|
|
131
134
|
|
|
132
135
|
test("crawl with prompt and wait", async () => {
|
|
133
136
|
if (!client) throw new Error();
|
|
134
|
-
const
|
|
137
|
+
const start = await client.startCrawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
|
|
138
|
+
const job = await waitForJob(() => client.getCrawlStatus(start.id), { pollInterval: 1, timeout: 120 });
|
|
135
139
|
expect(["completed", "failed"]).toContain(job.status);
|
|
136
140
|
expect(job.completed).toBeGreaterThanOrEqual(0);
|
|
137
141
|
expect(job.total).toBeGreaterThanOrEqual(0);
|
|
138
142
|
expect(Array.isArray(job.data)).toBe(true);
|
|
139
|
-
},
|
|
143
|
+
}, testTimeoutMs(272_000));
|
|
140
144
|
|
|
141
145
|
test("crawl with scrape options", async () => {
|
|
142
146
|
if (!client) throw new Error();
|
|
@@ -145,7 +149,7 @@ describe("v2.crawl e2e", () => {
|
|
|
145
149
|
scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
|
|
146
150
|
});
|
|
147
151
|
expect(typeof job.id).toBe("string");
|
|
148
|
-
}, 120_000);
|
|
152
|
+
}, testTimeoutMs(120_000));
|
|
149
153
|
|
|
150
154
|
test("crawl with json format object", async () => {
|
|
151
155
|
if (!client) throw new Error();
|
|
@@ -154,7 +158,7 @@ describe("v2.crawl e2e", () => {
|
|
|
154
158
|
scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
|
|
155
159
|
});
|
|
156
160
|
expect(typeof job.id).toBe("string");
|
|
157
|
-
}, 120_000);
|
|
161
|
+
}, testTimeoutMs(120_000));
|
|
158
162
|
|
|
159
163
|
test("crawl all parameters", async () => {
|
|
160
164
|
if (!client) throw new Error();
|
|
@@ -187,13 +191,13 @@ describe("v2.crawl e2e", () => {
|
|
|
187
191
|
zeroDataRetention: false,
|
|
188
192
|
});
|
|
189
193
|
expect(typeof job.id).toBe("string");
|
|
190
|
-
}, 180_000);
|
|
194
|
+
}, testTimeoutMs(180_000));
|
|
191
195
|
|
|
192
196
|
test("crawl params preview", async () => {
|
|
193
197
|
if (!client) throw new Error();
|
|
194
198
|
const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
|
|
195
199
|
expect(params && typeof params === "object").toBe(true);
|
|
196
200
|
// Optional fields may or may not be present; just assert object shape
|
|
197
|
-
}, 60_000);
|
|
201
|
+
}, testTimeoutMs(60_000));
|
|
198
202
|
});
|
|
199
203
|
|