@mendable/firecrawl-js 4.45.0 → 4.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/audit-ci.jsonc +1 -15
- package/dist/chunk-H75FJ6LW.js +114 -0
- package/dist/index.cjs +61 -27
- package/dist/index.d.cts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +58 -21
- package/dist/package-NZI5TDUV.js +40 -0
- package/package.json +5 -9
- package/pnpm-workspace.yaml +0 -6
- package/src/__tests__/e2e/v1/index.test.ts +1 -1
- package/src/__tests__/e2e/v2/batch.test.ts +1 -1
- package/src/__tests__/e2e/v2/crawl.test.ts +1 -1
- package/src/__tests__/e2e/v2/extract.test.ts +1 -1
- package/src/__tests__/e2e/v2/map.test.ts +1 -1
- package/src/__tests__/e2e/v2/parse.test.ts +1 -1
- package/src/__tests__/e2e/v2/scrape.test.ts +1 -1
- package/src/__tests__/e2e/v2/search.test.ts +1 -1
- package/src/__tests__/e2e/v2/usage.test.ts +1 -1
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +2 -2
- package/src/__tests__/e2e/v2/watcher.test.ts +1 -1
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +7 -7
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +6 -6
- package/src/__tests__/unit/v1/next-url-pinning.test.ts +3 -3
- package/src/__tests__/unit/v1/optional-params.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +6 -6
- package/src/__tests__/unit/v2/agent-hints.test.ts +3 -3
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +7 -7
- package/src/__tests__/unit/v2/agent.test.ts +1 -1
- package/src/__tests__/unit/v2/audit-metadata.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/branding.test.ts +2 -2
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/clientOptions.test.ts +1 -0
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/developer.test.ts +4 -4
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +16 -16
- package/src/__tests__/unit/v2/errorHandler.test.ts +24 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/gov.test.ts +4 -4
- package/src/__tests__/unit/v2/menu.test.ts +2 -2
- package/src/__tests__/unit/v2/monitor-search-target.test.ts +1 -0
- package/src/__tests__/unit/v2/next-url-pinning.test.ts +1 -1
- package/src/__tests__/unit/v2/pagination.test.ts +26 -6
- package/src/__tests__/unit/v2/parse-formats.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/parse.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/product.test.ts +2 -2
- package/src/__tests__/unit/v2/prototype-discoverability.test.ts +1 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -1
- package/src/__tests__/unit/v2/scrape-browser.unit.test.ts +8 -8
- package/src/__tests__/unit/v2/scrape.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +13 -13
- package/src/__tests__/unit/v2/search-country.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-data-property.test.ts +3 -3
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/tools.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/v1-aliases.test.ts +2 -2
- package/src/__tests__/unit/v2/validation.test.ts +40 -2
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +1 -1
- package/src/v1/index.ts +1 -3
- package/src/v2/client.ts +1 -1
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +10 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/vitest.config.ts +7 -0
- package/dist/chunk-LKCOCWYW.js +0 -97
- package/dist/package-C6PK3CK7.js +0 -4
- package/jest.config.js +0 -16
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
processingContinuesDelayMs,
|
|
4
4
|
scrape,
|
|
@@ -30,7 +30,7 @@ function processingContinues408(retryAfterSeconds = 10) {
|
|
|
30
30
|
|
|
31
31
|
describe("v2.scrape auto-resume", () => {
|
|
32
32
|
test("resumes after processing_continues and returns the finished document", async () => {
|
|
33
|
-
const post =
|
|
33
|
+
const post = vi
|
|
34
34
|
.fn()
|
|
35
35
|
.mockRejectedValueOnce(processingContinues408(10))
|
|
36
36
|
.mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
|
|
@@ -49,8 +49,8 @@ describe("v2.scrape auto-resume", () => {
|
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
test("autoResume: false surfaces the timeout immediately and never sleeps", async () => {
|
|
52
|
-
const post =
|
|
53
|
-
const sleepImpl =
|
|
52
|
+
const post = vi.fn().mockRejectedValueOnce(processingContinues408());
|
|
53
|
+
const sleepImpl = vi.fn();
|
|
54
54
|
|
|
55
55
|
await expect(
|
|
56
56
|
scrape({ post } as any, "https://example.com/big.pdf", { autoResume: false }, {
|
|
@@ -62,7 +62,7 @@ describe("v2.scrape auto-resume", () => {
|
|
|
62
62
|
});
|
|
63
63
|
|
|
64
64
|
test("autoResume never enters the wire payload", async () => {
|
|
65
|
-
const post =
|
|
65
|
+
const post = vi
|
|
66
66
|
.fn()
|
|
67
67
|
.mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
|
|
68
68
|
await scrape({ post } as any, "https://example.com/x", { autoResume: true });
|
|
@@ -71,8 +71,8 @@ describe("v2.scrape auto-resume", () => {
|
|
|
71
71
|
});
|
|
72
72
|
|
|
73
73
|
test("gives up after the attempt bound and surfaces the error", async () => {
|
|
74
|
-
const post =
|
|
75
|
-
const sleepImpl =
|
|
74
|
+
const post = vi.fn().mockRejectedValue(processingContinues408(10));
|
|
75
|
+
const sleepImpl = vi.fn().mockResolvedValue(undefined);
|
|
76
76
|
|
|
77
77
|
await expect(
|
|
78
78
|
scrape({ post } as any, "https://example.com/big.pdf", undefined, {
|
|
@@ -85,7 +85,7 @@ describe("v2.scrape auto-resume", () => {
|
|
|
85
85
|
});
|
|
86
86
|
|
|
87
87
|
test("plain timeouts without the processing signal do not resume", async () => {
|
|
88
|
-
const post =
|
|
88
|
+
const post = vi.fn().mockRejectedValueOnce({
|
|
89
89
|
isAxiosError: true,
|
|
90
90
|
response: {
|
|
91
91
|
status: 408,
|
|
@@ -93,7 +93,7 @@ describe("v2.scrape auto-resume", () => {
|
|
|
93
93
|
data: { success: false, code: "SCRAPE_TIMEOUT", error: "Request timed out" },
|
|
94
94
|
},
|
|
95
95
|
});
|
|
96
|
-
const sleepImpl =
|
|
96
|
+
const sleepImpl = vi.fn();
|
|
97
97
|
|
|
98
98
|
await expect(
|
|
99
99
|
scrape({ post } as any, "https://example.com/slow", undefined, {
|
|
@@ -107,11 +107,11 @@ describe("v2.scrape auto-resume", () => {
|
|
|
107
107
|
test("request timeout: opt-out keeps legacy behavior, resume adds the cushion", async () => {
|
|
108
108
|
const ok = { status: 200, data: { success: true, data: DOC } };
|
|
109
109
|
// Opt-out, no explicit timeout → no per-request override at all.
|
|
110
|
-
const post1 =
|
|
110
|
+
const post1 = vi.fn().mockResolvedValueOnce(ok);
|
|
111
111
|
await scrape({ post: post1 } as any, "https://example.com/x", { autoResume: false });
|
|
112
112
|
expect(post1.mock.calls[0][2]).toEqual({});
|
|
113
113
|
// Opt-out with explicit timeout → the pre-existing +5s.
|
|
114
|
-
const post2 =
|
|
114
|
+
const post2 = vi.fn().mockResolvedValueOnce(ok);
|
|
115
115
|
await scrape({ post: post2 } as any, "https://example.com/x", {
|
|
116
116
|
autoResume: false,
|
|
117
117
|
timeout: 10_000,
|
|
@@ -119,14 +119,14 @@ describe("v2.scrape auto-resume", () => {
|
|
|
119
119
|
expect(post2.mock.calls[0][2]).toEqual({ timeoutMs: 15_000 });
|
|
120
120
|
// Resume enabled → cushion over the server wall, respecting a larger
|
|
121
121
|
// configured client default as the floor.
|
|
122
|
-
const post3 =
|
|
122
|
+
const post3 = vi.fn().mockResolvedValueOnce(ok);
|
|
123
123
|
await scrape(
|
|
124
124
|
{ post: post3, getTimeoutMs: () => 600_000 } as any,
|
|
125
125
|
"https://example.com/x",
|
|
126
126
|
);
|
|
127
127
|
expect(post3.mock.calls[0][2]).toEqual({ timeoutMs: 630_000 });
|
|
128
128
|
// Resume enabled, no client getter → server wall default + cushion.
|
|
129
|
-
const post4 =
|
|
129
|
+
const post4 = vi.fn().mockResolvedValueOnce(ok);
|
|
130
130
|
await scrape({ post: post4 } as any, "https://example.com/x", { timeout: 900_000 });
|
|
131
131
|
expect(post4.mock.calls[0][2]).toEqual({ timeoutMs: 930_000 });
|
|
132
132
|
});
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
|
|
4
4
|
function httpMock() {
|
|
5
5
|
return {
|
|
6
|
-
post:
|
|
6
|
+
post: vi.fn(async () => ({ status: 200, data: { success: true } })),
|
|
7
7
|
} as any;
|
|
8
8
|
}
|
|
9
9
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
|
|
4
4
|
const fakeHttp = {
|
|
5
|
-
post:
|
|
5
|
+
post: vi.fn(),
|
|
6
6
|
} as any;
|
|
7
7
|
|
|
8
8
|
describe("SearchData .data helpful error", () => {
|
|
9
|
-
beforeEach(() =>
|
|
9
|
+
beforeEach(() => vi.clearAllMocks());
|
|
10
10
|
|
|
11
11
|
it("throws with available sources when .data is accessed", async () => {
|
|
12
12
|
fakeHttp.post.mockResolvedValue({
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
import type { SearchResultWeb } from "../../../v2/types";
|
|
4
4
|
|
|
@@ -13,7 +13,7 @@ const developerResult: SearchResultWeb = {
|
|
|
13
13
|
|
|
14
14
|
function httpWith(data: Record<string, unknown>) {
|
|
15
15
|
return {
|
|
16
|
-
post:
|
|
16
|
+
post: vi.fn(async () => ({ status: 200, data: { success: true, data } })),
|
|
17
17
|
} as any;
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
|
|
4
4
|
describe("v2 search highlights", () => {
|
|
5
5
|
test("forwards the highlights option", async () => {
|
|
6
6
|
const http = {
|
|
7
|
-
post:
|
|
7
|
+
post: vi.fn(async () => ({ status: 200, data: { success: true } })),
|
|
8
8
|
} as any;
|
|
9
9
|
|
|
10
10
|
await search(http, { query: "firecrawl", highlights: false });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
import { startBatchScrape } from "../../../v2/methods/batch";
|
|
4
4
|
import { startCrawl } from "../../../v2/methods/crawl";
|
|
@@ -18,10 +18,10 @@ const threatProtection: ThreatProtectionOptions = {
|
|
|
18
18
|
};
|
|
19
19
|
|
|
20
20
|
function makeHttp(data: Record<string, unknown>) {
|
|
21
|
-
const post =
|
|
21
|
+
const post = vi.fn(async () => ({ status: 200, data }));
|
|
22
22
|
return {
|
|
23
23
|
post,
|
|
24
|
-
prepareHeaders:
|
|
24
|
+
prepareHeaders: vi.fn(() => undefined),
|
|
25
25
|
} as any;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
beforeEach,
|
|
5
5
|
describe,
|
|
6
6
|
expect,
|
|
7
|
-
jest,
|
|
8
7
|
test,
|
|
9
|
-
|
|
8
|
+
vi,
|
|
9
|
+
} from "vitest";
|
|
10
10
|
import { createServer, type Server } from "node:http";
|
|
11
11
|
import { FirecrawlClient } from "../../../v2/client";
|
|
12
12
|
|
|
@@ -281,7 +281,7 @@ test.each([
|
|
|
281
281
|
"Alexandria timeout %s allows response delivery (%s ms)",
|
|
282
282
|
async (timeout, timeoutMs) => {
|
|
283
283
|
const http = {
|
|
284
|
-
post:
|
|
284
|
+
post: vi.fn(async () => ({
|
|
285
285
|
status: 200,
|
|
286
286
|
data: {
|
|
287
287
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
2
|
import { FirecrawlClient } from "../../../v2/client";
|
|
3
3
|
|
|
4
4
|
const ALIAS_MAP: Record<string, string> = {
|
|
@@ -22,7 +22,7 @@ describe("V1 deprecated aliases", () => {
|
|
|
22
22
|
|
|
23
23
|
for (const [alias, target] of Object.entries(ALIAS_MAP)) {
|
|
24
24
|
it(`${alias} delegates to ${target}`, async () => {
|
|
25
|
-
const spy =
|
|
25
|
+
const spy = vi
|
|
26
26
|
.spyOn(app, target as any)
|
|
27
27
|
.mockResolvedValue({ ok: true } as any);
|
|
28
28
|
await (app as any)[alias]("https://example.com");
|
|
@@ -1,9 +1,47 @@
|
|
|
1
|
-
import { describe, test, expect } from "
|
|
2
|
-
import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
|
1
|
+
import { describe, test, expect } from "vitest";
|
|
2
|
+
import { ensureNoV1Options, ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
|
3
3
|
import type { FormatOption } from "../../../v2/types";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
|
|
6
6
|
describe("v2 utils: validation", () => {
|
|
7
|
+
const scrapeLevel = {
|
|
8
|
+
pageOptions: "use scrapeOptions",
|
|
9
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
10
|
+
parsePDF: "use scrapeOptions.parsers",
|
|
11
|
+
};
|
|
12
|
+
const crawlOnly = {
|
|
13
|
+
crawlerOptions: "move its fields to the top level",
|
|
14
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
15
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
16
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
test.each(Object.entries({ ...crawlOnly, ...scrapeLevel }))("ensureNoV1Options: crawl names %s and its replacement", (option, replacement) => {
|
|
20
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", [option]: true }, "crawl")).toThrow(
|
|
21
|
+
`crawl: ${option} is a v1 option, ${replacement}`,
|
|
22
|
+
);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test.each(Object.entries(scrapeLevel))("ensureNoV1Options: search names %s and its replacement", (option, replacement) => {
|
|
26
|
+
expect(() => ensureNoV1Options({ query: "q", [option]: true }, "search")).toThrow(
|
|
27
|
+
`search: ${option} is a v1 option, ${replacement}`,
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("ensureNoV1Options: search does not apply crawl-only hints", () => {
|
|
32
|
+
expect(() => ensureNoV1Options({ query: "q", maxDepth: 2, crawlerOptions: {} }, "search")).not.toThrow();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("ensureNoV1Options: lists every v1 option found", () => {
|
|
36
|
+
expect(() => ensureNoV1Options({ maxDepth: 2, allowBackwardLinks: true }, "crawl")).toThrow(/maxDepth.*maxDiscoveryDepth.*allowBackwardLinks.*crawlEntireDomain/);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("ensureNoV1Options: accepts v2 options and unset v1 keys", () => {
|
|
40
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", limit: 2, maxDiscoveryDepth: 1 }, "crawl")).not.toThrow();
|
|
41
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", maxDepth: undefined }, "crawl")).not.toThrow();
|
|
42
|
+
expect(() => ensureNoV1Options(undefined, "search")).not.toThrow();
|
|
43
|
+
});
|
|
44
|
+
|
|
7
45
|
test("ensureValidFormats: plain 'json' string is invalid", () => {
|
|
8
46
|
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
|
|
9
47
|
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
|
package/src/v1/index.ts
CHANGED
|
@@ -681,9 +681,7 @@ export default class FirecrawlApp {
|
|
|
681
681
|
return packageJson.default.version;
|
|
682
682
|
} catch (error) {
|
|
683
683
|
// Suppress noisy logs under test environments
|
|
684
|
-
const isTest = typeof process !== 'undefined' &&
|
|
685
|
-
process.env.JEST_WORKER_ID != null || process.env.NODE_ENV === 'test'
|
|
686
|
-
);
|
|
684
|
+
const isTest = typeof process !== 'undefined' && process.env.NODE_ENV === 'test';
|
|
687
685
|
if (!isTest) {
|
|
688
686
|
// eslint-disable-next-line no-console
|
|
689
687
|
console.error("Error getting version:", error);
|
package/src/v2/client.ts
CHANGED
|
@@ -644,7 +644,7 @@ export class FirecrawlClient {
|
|
|
644
644
|
// Browser
|
|
645
645
|
/**
|
|
646
646
|
* Create a new browser session.
|
|
647
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
647
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
648
648
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
649
649
|
*/
|
|
650
650
|
async browser(
|
|
@@ -22,6 +22,8 @@ export async function browser(
|
|
|
22
22
|
name: string;
|
|
23
23
|
saveChanges?: boolean;
|
|
24
24
|
};
|
|
25
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
26
|
+
location?: { country: string };
|
|
25
27
|
integration?: string;
|
|
26
28
|
origin?: string;
|
|
27
29
|
} = {},
|
|
@@ -32,6 +34,7 @@ export async function browser(
|
|
|
32
34
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
33
35
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
34
36
|
if (args.profile != null) body.profile = args.profile;
|
|
37
|
+
if (args.location != null) body.location = args.location;
|
|
35
38
|
if (args.integration != null) body.integration = args.integration;
|
|
36
39
|
if (args.origin) body.origin = args.origin;
|
|
37
40
|
|
package/src/v2/methods/crawl.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
SdkError,
|
|
11
11
|
} from "../types";
|
|
12
12
|
import { HttpClient } from "../utils/httpClient";
|
|
13
|
-
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
13
|
+
import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
|
|
14
14
|
import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
|
|
15
15
|
import type { HttpClient as _Http } from "../utils/httpClient";
|
|
16
16
|
import { fetchAllPages } from "../utils/pagination";
|
|
@@ -21,6 +21,7 @@ export type CrawlRequest = CrawlOptions & {
|
|
|
21
21
|
|
|
22
22
|
function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
|
|
23
23
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
24
|
+
ensureNoV1Options(request, "crawl");
|
|
24
25
|
const data: Record<string, unknown> = { url: request.url.trim() };
|
|
25
26
|
if (request.prompt) data.prompt = request.prompt;
|
|
26
27
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
} from "../types";
|
|
10
10
|
import { HttpClient } from "../utils/httpClient";
|
|
11
11
|
import { agentHintMetadata } from "../utils/agentHints";
|
|
12
|
-
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
12
|
+
import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
|
|
13
13
|
import {
|
|
14
14
|
throwForBadResponse,
|
|
15
15
|
normalizeAxiosError,
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
|
|
18
18
|
function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
|
19
19
|
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
|
|
20
|
+
ensureNoV1Options(req, "search");
|
|
20
21
|
if (req.limit != null && req.limit <= 0)
|
|
21
22
|
throw new Error("limit must be positive");
|
|
22
23
|
if (req.timeout != null && req.timeout <= 0)
|
package/src/v2/types.ts
CHANGED
|
@@ -31,6 +31,7 @@ export interface JsonFormat extends Format {
|
|
|
31
31
|
type: "json";
|
|
32
32
|
prompt?: string;
|
|
33
33
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
34
|
+
/** @deprecated Use the top-level `checkPromptInjection` scrape option. */
|
|
34
35
|
checkPromptInjection?: boolean;
|
|
35
36
|
}
|
|
36
37
|
|
|
@@ -276,6 +277,13 @@ export interface ScrapeOptions {
|
|
|
276
277
|
minAge?: number;
|
|
277
278
|
storeInCache?: boolean;
|
|
278
279
|
lockdown?: boolean;
|
|
280
|
+
/**
|
|
281
|
+
* Scans the page content for prompt injection with any format except
|
|
282
|
+
* rawBase64, before LLM-backed formats run. A detection fails the scrape
|
|
283
|
+
* with SCRAPE_PROMPT_INJECTION_DETECTED.
|
|
284
|
+
* Adds 4 credits when the check scans the whole page.
|
|
285
|
+
*/
|
|
286
|
+
checkPromptInjection?: boolean;
|
|
279
287
|
redactPII?: boolean | RedactPIIOptions;
|
|
280
288
|
threatProtection?: ThreatProtectionOptions;
|
|
281
289
|
auditMetadata?: AuditMetadata;
|
|
@@ -907,10 +915,10 @@ export interface FindToolsOptions {
|
|
|
907
915
|
}
|
|
908
916
|
|
|
909
917
|
export interface FindToolsData {
|
|
910
|
-
level: "providers" | "groups" | "tools";
|
|
918
|
+
level: "categories" | "providers" | "groups" | "tools";
|
|
911
919
|
items: Array<{
|
|
912
920
|
id: string;
|
|
913
|
-
name
|
|
921
|
+
name?: string;
|
|
914
922
|
next?: AlexandriaCall;
|
|
915
923
|
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
916
924
|
[key: string]: unknown;
|
|
@@ -2,12 +2,33 @@ import { type AxiosError, type AxiosResponse } from "axios";
|
|
|
2
2
|
import { SdkError, JobTimeoutError, parseRequiresAction } from "../types";
|
|
3
3
|
import { agentHintMetadata } from "./agentHints";
|
|
4
4
|
|
|
5
|
+
/** Names the fields a validation error rejected, so a bare "Bad Request" says what to change. */
|
|
6
|
+
function describeValidationIssues(details: unknown): string | undefined {
|
|
7
|
+
if (!Array.isArray(details)) return undefined;
|
|
8
|
+
const parts = details.slice(0, 3).flatMap((issue) => {
|
|
9
|
+
if (!issue || typeof issue !== "object") return [];
|
|
10
|
+
const { path, message, keys } = issue as { path?: unknown; message?: unknown; keys?: unknown };
|
|
11
|
+
const what = Array.isArray(keys) && keys.length > 0
|
|
12
|
+
? `unrecognized key(s) ${keys.join(", ")}`
|
|
13
|
+
: typeof message === "string" ? message : undefined;
|
|
14
|
+
if (!what) return [];
|
|
15
|
+
return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
|
|
16
|
+
});
|
|
17
|
+
return parts.length > 0 ? parts.join("; ") : undefined;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function withIssues(message: unknown, details: unknown): string {
|
|
21
|
+
const text = typeof message === "string" ? message : JSON.stringify(message);
|
|
22
|
+
const issues = describeValidationIssues(details);
|
|
23
|
+
return issues && !text.includes(issues) ? `${text} (${issues})` : text;
|
|
24
|
+
}
|
|
25
|
+
|
|
5
26
|
export function throwForBadResponse(resp: AxiosResponse, action: string): never {
|
|
6
27
|
const status = resp.status;
|
|
7
28
|
const body = resp.data || {};
|
|
8
29
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
9
30
|
throw Object.assign(new SdkError(
|
|
10
|
-
msg,
|
|
31
|
+
withIssues(msg, body?.details),
|
|
11
32
|
status,
|
|
12
33
|
body?.code,
|
|
13
34
|
body?.details,
|
|
@@ -23,7 +44,7 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {
|
|
|
23
44
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
24
45
|
const code = (body?.code as string) || err.code;
|
|
25
46
|
throw Object.assign(new SdkError(
|
|
26
|
-
message,
|
|
47
|
+
withIssues(message, body?.details),
|
|
27
48
|
status,
|
|
28
49
|
code,
|
|
29
50
|
body?.details ?? body,
|
|
@@ -1,14 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
try {
|
|
3
|
-
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
4
|
-
return process.env.npm_package_version as string;
|
|
5
|
-
}
|
|
1
|
+
import { version } from "../../../package.json";
|
|
6
2
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
} catch {
|
|
11
|
-
return "3.x.x";
|
|
12
|
-
}
|
|
3
|
+
/** The SDK's own version, inlined at build time so a host app's npm_package_version never replaces it. */
|
|
4
|
+
export function getVersion(): string {
|
|
5
|
+
return version;
|
|
13
6
|
}
|
|
14
|
-
|
|
@@ -39,7 +39,10 @@ export async function fetchAllPages<T = Document>(
|
|
|
39
39
|
docs.push(d as T);
|
|
40
40
|
}
|
|
41
41
|
if (maxResults != null && docs.length >= maxResults) break;
|
|
42
|
-
|
|
42
|
+
const next = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
|
|
43
|
+
// A running job's cursor can point back at itself until new documents land; re-requesting it here only spins.
|
|
44
|
+
if (next === current) break;
|
|
45
|
+
current = next;
|
|
43
46
|
pageCount += 1;
|
|
44
47
|
}
|
|
45
48
|
return docs;
|
|
@@ -86,6 +86,34 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
|
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
+
const V1_SCRAPE_OPTION_REPLACEMENTS: Record<string, string> = {
|
|
90
|
+
pageOptions: "use scrapeOptions",
|
|
91
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
92
|
+
parsePDF: "use scrapeOptions.parsers",
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const V1_OPTION_REPLACEMENTS: Record<"crawl" | "search", Record<string, string>> = {
|
|
96
|
+
crawl: {
|
|
97
|
+
crawlerOptions: "move its fields to the top level",
|
|
98
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
99
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
100
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
101
|
+
...V1_SCRAPE_OPTION_REPLACEMENTS,
|
|
102
|
+
},
|
|
103
|
+
search: V1_SCRAPE_OPTION_REPLACEMENTS,
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/** Throws on v1 option names, which the v2 payload builders would otherwise drop without a word. */
|
|
107
|
+
export function ensureNoV1Options(options: object | undefined, method: "crawl" | "search"): void {
|
|
108
|
+
if (!options) return;
|
|
109
|
+
const replacements = V1_OPTION_REPLACEMENTS[method];
|
|
110
|
+
const found = Object.keys(replacements).filter((key) => (options as Record<string, unknown>)[key] != null);
|
|
111
|
+
if (found.length === 0) return;
|
|
112
|
+
throw new Error(
|
|
113
|
+
`${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
89
117
|
export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
|
|
90
118
|
if (!options) return;
|
|
91
119
|
if (options.timeout != null && options.timeout <= 0) {
|
package/vitest.config.ts
ADDED
package/dist/chunk-LKCOCWYW.js
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
2
|
-
var __commonJS = (cb, mod) => function __require() {
|
|
3
|
-
try {
|
|
4
|
-
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
5
|
-
} catch (e) {
|
|
6
|
-
throw mod = 0, e;
|
|
7
|
-
}
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
// package.json
|
|
11
|
-
var require_package = __commonJS({
|
|
12
|
-
"package.json"(exports, module) {
|
|
13
|
-
module.exports = {
|
|
14
|
-
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.45.0",
|
|
16
|
-
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
|
-
main: "dist/index.js",
|
|
18
|
-
types: "dist/index.d.ts",
|
|
19
|
-
exports: {
|
|
20
|
-
"./package.json": "./package.json",
|
|
21
|
-
".": {
|
|
22
|
-
import: "./dist/index.js",
|
|
23
|
-
default: "./dist/index.cjs"
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
type: "module",
|
|
27
|
-
scripts: {
|
|
28
|
-
build: "tsup",
|
|
29
|
-
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
30
|
-
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
31
|
-
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
32
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit"
|
|
33
|
-
},
|
|
34
|
-
repository: {
|
|
35
|
-
type: "git",
|
|
36
|
-
url: "git+https://github.com/firecrawl/firecrawl.git"
|
|
37
|
-
},
|
|
38
|
-
author: "Mendable.ai",
|
|
39
|
-
license: "MIT",
|
|
40
|
-
dependencies: {
|
|
41
|
-
axios: "1.20.0",
|
|
42
|
-
firecrawl: "4.30.1",
|
|
43
|
-
"typescript-event-target": "^1.1.1",
|
|
44
|
-
zod: "^3.23.8",
|
|
45
|
-
"zod-to-json-schema": "^3.23.0"
|
|
46
|
-
},
|
|
47
|
-
bugs: {
|
|
48
|
-
url: "https://github.com/firecrawl/firecrawl/issues"
|
|
49
|
-
},
|
|
50
|
-
homepage: "https://github.com/firecrawl/firecrawl#readme",
|
|
51
|
-
devDependencies: {
|
|
52
|
-
"@jest/globals": "^30.2.0",
|
|
53
|
-
"@types/dotenv": "^8.2.0",
|
|
54
|
-
"@types/jest": "^30.0.0",
|
|
55
|
-
"@types/mocha": "^10.0.6",
|
|
56
|
-
"@types/node": "^20.12.12",
|
|
57
|
-
"@types/uuid": "^9.0.8",
|
|
58
|
-
dotenv: "^16.4.5",
|
|
59
|
-
jest: "^30.2.0",
|
|
60
|
-
"ts-jest": "^29.4.5",
|
|
61
|
-
tsup: "^8.5.0",
|
|
62
|
-
typescript: "^5.4.5",
|
|
63
|
-
uuid: "^14.0.0"
|
|
64
|
-
},
|
|
65
|
-
keywords: [
|
|
66
|
-
"firecrawl",
|
|
67
|
-
"mendable",
|
|
68
|
-
"crawler",
|
|
69
|
-
"web",
|
|
70
|
-
"scraper",
|
|
71
|
-
"api",
|
|
72
|
-
"sdk",
|
|
73
|
-
"web-search",
|
|
74
|
-
"literature-search",
|
|
75
|
-
"research",
|
|
76
|
-
"scientific-papers",
|
|
77
|
-
"academic-search",
|
|
78
|
-
"biomedical",
|
|
79
|
-
"life-sciences",
|
|
80
|
-
"pubmed",
|
|
81
|
-
"biorxiv",
|
|
82
|
-
"medrxiv",
|
|
83
|
-
"arxiv",
|
|
84
|
-
"preprints",
|
|
85
|
-
"citations",
|
|
86
|
-
"bioinformatics"
|
|
87
|
-
],
|
|
88
|
-
engines: {
|
|
89
|
-
node: ">=22.0.0"
|
|
90
|
-
}
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
export {
|
|
96
|
-
require_package
|
|
97
|
-
};
|
package/dist/package-C6PK3CK7.js
DELETED
package/jest.config.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
/** @type {import('ts-jest').JestConfigWithTsJest} **/
|
|
2
|
-
export default {
|
|
3
|
-
testEnvironment: "node",
|
|
4
|
-
"moduleNameMapper": {
|
|
5
|
-
"^(\\.{1,2}/.*)\\.js$": "$1",
|
|
6
|
-
},
|
|
7
|
-
"extensionsToTreatAsEsm": [".ts"],
|
|
8
|
-
"transform": {
|
|
9
|
-
"^.+\\.(mt|t|cj|j)s$": [
|
|
10
|
-
"ts-jest",
|
|
11
|
-
{
|
|
12
|
-
"useESM": true
|
|
13
|
-
}
|
|
14
|
-
]
|
|
15
|
-
},
|
|
16
|
-
};
|