firecrawl 4.44.1 → 4.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/audit-ci.jsonc +1 -15
- package/dist/{chunk-JU6NENYM.js → chunk-R4XT7PME.js} +5 -9
- package/dist/index.cjs +77 -21
- package/dist/index.d.cts +14 -2
- package/dist/index.d.ts +14 -2
- package/dist/index.js +74 -14
- package/dist/{package-FEZU4ANT.js → package-EI4Q33HD.js} +1 -1
- package/package.json +5 -9
- package/pnpm-workspace.yaml +0 -6
- package/src/__tests__/e2e/v1/index.test.ts +1 -1
- package/src/__tests__/e2e/v2/batch.test.ts +1 -1
- package/src/__tests__/e2e/v2/crawl.test.ts +1 -1
- package/src/__tests__/e2e/v2/extract.test.ts +1 -1
- package/src/__tests__/e2e/v2/map.test.ts +1 -1
- package/src/__tests__/e2e/v2/parse.test.ts +1 -1
- package/src/__tests__/e2e/v2/scrape.test.ts +1 -1
- package/src/__tests__/e2e/v2/search.test.ts +1 -1
- package/src/__tests__/e2e/v2/usage.test.ts +1 -1
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +2 -2
- package/src/__tests__/e2e/v2/watcher.test.ts +1 -1
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +7 -7
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +6 -6
- package/src/__tests__/unit/v1/next-url-pinning.test.ts +3 -3
- package/src/__tests__/unit/v1/optional-params.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +6 -6
- package/src/__tests__/unit/v2/agent-hints.test.ts +3 -3
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +7 -7
- package/src/__tests__/unit/v2/agent.test.ts +1 -1
- package/src/__tests__/unit/v2/audit-metadata.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/branding.test.ts +2 -2
- package/src/__tests__/unit/v2/clientOptions.test.ts +1 -0
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +189 -0
- package/src/__tests__/unit/v2/developer.test.ts +4 -4
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +16 -16
- package/src/__tests__/unit/v2/errorHandler.test.ts +1 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/gov.test.ts +4 -4
- package/src/__tests__/unit/v2/menu.test.ts +2 -2
- package/src/__tests__/unit/v2/monitor-search-target.test.ts +1 -0
- package/src/__tests__/unit/v2/next-url-pinning.test.ts +1 -1
- package/src/__tests__/unit/v2/pagination.test.ts +6 -6
- package/src/__tests__/unit/v2/parse-formats.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/parse.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/product.test.ts +2 -2
- package/src/__tests__/unit/v2/prototype-discoverability.test.ts +1 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -1
- package/src/__tests__/unit/v2/scrape-browser.unit.test.ts +8 -8
- package/src/__tests__/unit/v2/scrape.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +13 -13
- package/src/__tests__/unit/v2/search-country.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-data-property.test.ts +3 -3
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/tools.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/v1-aliases.test.ts +2 -2
- package/src/__tests__/unit/v2/validation.test.ts +1 -1
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +1 -1
- package/src/v1/index.ts +1 -3
- package/src/v2/client.ts +8 -5
- package/src/v2/methods/crawl.ts +90 -6
- package/src/v2/types.ts +8 -0
- package/vitest.config.ts +7 -0
- package/jest.config.js +0 -16
|
@@ -6,7 +6,7 @@ import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages }
|
|
|
6
6
|
import { config } from "dotenv";
|
|
7
7
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
8
8
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
9
|
-
import { describe, test, expect, beforeAll } from "
|
|
9
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
10
10
|
|
|
11
11
|
config();
|
|
12
12
|
|
|
@@ -5,7 +5,7 @@ import Firecrawl from "../../../index";
|
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
7
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
8
|
-
import { describe, test, expect, beforeAll } from "
|
|
8
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
9
9
|
|
|
10
10
|
config();
|
|
11
11
|
|
|
@@ -52,7 +52,7 @@ export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
|
|
|
52
52
|
*
|
|
53
53
|
* Every caller now passes its own timeout, sized to the work it waits for.
|
|
54
54
|
* This value is only a backstop for a new caller that forgets one. Without a
|
|
55
|
-
* bound the poll loop runs until the
|
|
55
|
+
* bound the poll loop runs until the test timeout, which hides the reason.
|
|
56
56
|
*
|
|
57
57
|
* The value stays short because it cannot know the work. It fits the smallest
|
|
58
58
|
* base any suite passes to testTimeoutMs, which is 60_000 ms:
|
|
@@ -69,7 +69,7 @@ export const RETRY_BUDGET_MS = (MAX_ATTEMPTS - 1) * MAX_WAIT_MS;
|
|
|
69
69
|
export const DEFAULT_JOB_TIMEOUT_MS = 45_000;
|
|
70
70
|
|
|
71
71
|
/**
|
|
72
|
-
*
|
|
72
|
+
* Vitest timeout for a test that calls a wrapped client.
|
|
73
73
|
*
|
|
74
74
|
* Pass the base: the time the test needs when the API answers at once. The
|
|
75
75
|
* result adds one retry budget on top, so a test whose single call hits the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import Firecrawl from "../../../index";
|
|
2
2
|
import { config } from "dotenv";
|
|
3
|
-
import { describe, test, expect, beforeAll } from "
|
|
3
|
+
import { describe, test, expect, beforeAll } from "vitest";
|
|
4
4
|
import { getIdentity } from "./utils/idmux";
|
|
5
5
|
import { testTimeoutMs, withRateLimitRetry } from "./utils/rateLimit";
|
|
6
6
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, afterEach, beforeEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -20,7 +20,7 @@ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown:
|
|
|
20
20
|
function mockPoll(path: string, nextOrigin = API_URL) {
|
|
21
21
|
const sent: string[] = [];
|
|
22
22
|
let polls = 0;
|
|
23
|
-
|
|
23
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
24
24
|
sent.push(url);
|
|
25
25
|
if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
|
|
26
26
|
polls++;
|
|
@@ -33,18 +33,18 @@ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
|
33
33
|
const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
|
|
34
34
|
|
|
35
35
|
beforeEach(() => {
|
|
36
|
-
|
|
36
|
+
vi.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
|
|
37
37
|
fn();
|
|
38
38
|
return 0;
|
|
39
39
|
}) as any);
|
|
40
40
|
});
|
|
41
41
|
|
|
42
42
|
afterEach(() => {
|
|
43
|
-
|
|
43
|
+
vi.restoreAllMocks();
|
|
44
44
|
});
|
|
45
45
|
|
|
46
46
|
test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
|
|
47
|
-
const post =
|
|
47
|
+
const post = vi.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
48
48
|
const path = "/v1/batch/scrape/abc";
|
|
49
49
|
const sent = mockPoll(path, "https://evil.example");
|
|
50
50
|
|
|
@@ -58,9 +58,9 @@ describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
|
|
|
58
58
|
});
|
|
59
59
|
|
|
60
60
|
test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
|
|
61
|
-
|
|
61
|
+
vi.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
|
|
62
62
|
const sent: string[] = [];
|
|
63
|
-
|
|
63
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string) => {
|
|
64
64
|
sent.push(url);
|
|
65
65
|
return { status: 200, data: { success: true, status: "failed", data: [] } };
|
|
66
66
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import FirecrawlApp from '../../../v1';
|
|
2
|
-
import { describe, test, expect,
|
|
2
|
+
import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
3
3
|
|
|
4
4
|
const successResponse = {
|
|
5
5
|
status: 200,
|
|
@@ -21,17 +21,17 @@ describe('monitorJobStatus retry logic', () => {
|
|
|
21
21
|
beforeEach(() => {
|
|
22
22
|
app = new FirecrawlApp({ apiKey: 'test-key', apiUrl: 'https://test.com' });
|
|
23
23
|
delays = [];
|
|
24
|
-
|
|
24
|
+
vi.useFakeTimers();
|
|
25
25
|
const fakeSetTimeout = globalThis.setTimeout;
|
|
26
|
-
|
|
26
|
+
vi.spyOn(globalThis, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {
|
|
27
27
|
delays.push(ms ?? 0);
|
|
28
28
|
return fakeSetTimeout(fn, ms);
|
|
29
29
|
}) as any);
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
afterEach(() => {
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
vi.restoreAllMocks();
|
|
34
|
+
vi.useRealTimers();
|
|
35
35
|
});
|
|
36
36
|
|
|
37
37
|
function failFirst(n: number, error: Error, statuses: string[] = []) {
|
|
@@ -48,7 +48,7 @@ describe('monitorJobStatus retry logic', () => {
|
|
|
48
48
|
async function monitor() {
|
|
49
49
|
const result = app.monitorJobStatus('test-id', {} as any, 1);
|
|
50
50
|
result.catch(() => {});
|
|
51
|
-
await
|
|
51
|
+
await vi.runAllTimersAsync();
|
|
52
52
|
return result;
|
|
53
53
|
}
|
|
54
54
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, afterEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -16,7 +16,7 @@ const origins: Array<[string, string]> = [
|
|
|
16
16
|
|
|
17
17
|
function mockGet(respond: (url: URL) => unknown) {
|
|
18
18
|
const sent: Array<{ url: string; authorization: unknown }> = [];
|
|
19
|
-
|
|
19
|
+
vi.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {
|
|
20
20
|
sent.push({ url, authorization: config?.headers?.Authorization });
|
|
21
21
|
return { status: 200, data: respond(new URL(url)) };
|
|
22
22
|
});
|
|
@@ -45,7 +45,7 @@ const lastPage = () => ({ success: true, status: "completed", data: [{ markdown:
|
|
|
45
45
|
|
|
46
46
|
describe("v1 pins next URLs to the api_url origin", () => {
|
|
47
47
|
afterEach(() => {
|
|
48
|
-
|
|
48
|
+
vi.restoreAllMocks();
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
describe.each(origins)("%s next", (_label, origin) => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, beforeEach, afterEach } from "vitest";
|
|
2
2
|
import axios from "axios";
|
|
3
3
|
import FirecrawlApp from "../../../v1";
|
|
4
4
|
|
|
@@ -20,11 +20,11 @@ describe("v1 methods work without params", () => {
|
|
|
20
20
|
beforeEach(() => {
|
|
21
21
|
app = new FirecrawlApp({ apiKey: "fc-test", apiUrl: API_URL });
|
|
22
22
|
posts = [];
|
|
23
|
-
|
|
23
|
+
vi.spyOn(axios, "post").mockImplementation(async (url: string, data?: any) => {
|
|
24
24
|
posts.push({ url, data });
|
|
25
25
|
return { status: 200, data: { success: true, id: "job-1", data: [], links: [] } };
|
|
26
26
|
});
|
|
27
|
-
|
|
27
|
+
vi.spyOn(axios, "get").mockImplementation(async () => ({
|
|
28
28
|
status: 200,
|
|
29
29
|
data: { success: true, status: "completed", data: [{ markdown: "ok" }] },
|
|
30
30
|
}));
|
|
@@ -32,7 +32,7 @@ describe("v1 methods work without params", () => {
|
|
|
32
32
|
});
|
|
33
33
|
|
|
34
34
|
afterEach(() => {
|
|
35
|
-
|
|
35
|
+
vi.restoreAllMocks();
|
|
36
36
|
(globalThis as any).WebSocket = originalWebSocket;
|
|
37
37
|
});
|
|
38
38
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
startAgent,
|
|
4
4
|
getAgentTrace,
|
|
@@ -8,7 +8,7 @@ import type { AgentTraceEvent } from "../../../v2/types";
|
|
|
8
8
|
|
|
9
9
|
describe("v2.agent unit", () => {
|
|
10
10
|
test("startAgent forwards effort in request payload", async () => {
|
|
11
|
-
const post =
|
|
11
|
+
const post = vi.fn().mockResolvedValue({
|
|
12
12
|
status: 200,
|
|
13
13
|
data: { success: true, id: "agent-job" },
|
|
14
14
|
});
|
|
@@ -25,7 +25,7 @@ describe("v2.agent unit", () => {
|
|
|
25
25
|
});
|
|
26
26
|
|
|
27
27
|
test("startAgent omits effort when unset", async () => {
|
|
28
|
-
const post =
|
|
28
|
+
const post = vi.fn().mockResolvedValue({
|
|
29
29
|
status: 200,
|
|
30
30
|
data: { success: true, id: "agent-job" },
|
|
31
31
|
});
|
|
@@ -70,7 +70,7 @@ describe("v2.agent unit", () => {
|
|
|
70
70
|
change: "partial",
|
|
71
71
|
},
|
|
72
72
|
};
|
|
73
|
-
const get =
|
|
73
|
+
const get = vi.fn().mockResolvedValue({
|
|
74
74
|
status: 200,
|
|
75
75
|
data: {
|
|
76
76
|
success: true,
|
|
@@ -95,7 +95,7 @@ describe("v2.agent unit", () => {
|
|
|
95
95
|
});
|
|
96
96
|
|
|
97
97
|
test("getAgentTrace appends liveView query param when requested", async () => {
|
|
98
|
-
const get =
|
|
98
|
+
const get = vi.fn().mockResolvedValue({
|
|
99
99
|
status: 200,
|
|
100
100
|
data: {
|
|
101
101
|
success: true,
|
|
@@ -122,7 +122,7 @@ describe("v2.agent unit", () => {
|
|
|
122
122
|
});
|
|
123
123
|
|
|
124
124
|
test("getAgentSnapshot hits the snapshot endpoint", async () => {
|
|
125
|
-
const get =
|
|
125
|
+
const get = vi.fn().mockResolvedValue({
|
|
126
126
|
status: 200,
|
|
127
127
|
data: {
|
|
128
128
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, it,
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
3
|
import { scrape } from "../../../v2/methods/scrape";
|
|
4
4
|
import { parse } from "../../../v2/methods/parse";
|
|
@@ -14,8 +14,8 @@ const hints = [
|
|
|
14
14
|
];
|
|
15
15
|
function httpFor(body: object) {
|
|
16
16
|
return {
|
|
17
|
-
post:
|
|
18
|
-
postMultipart:
|
|
17
|
+
post: vi.fn<any>().mockResolvedValue({ status: 200, data: body }),
|
|
18
|
+
postMultipart: vi
|
|
19
19
|
.fn<any>()
|
|
20
20
|
.mockResolvedValue({ status: 200, data: body }),
|
|
21
21
|
} as any;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { listAgents } from "../../../v2/methods/agent";
|
|
3
3
|
|
|
4
4
|
describe("v2.agent listAgents unit", () => {
|
|
@@ -17,7 +17,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
17
17
|
};
|
|
18
18
|
|
|
19
19
|
test("hits GET /v2/agent without query by default", async () => {
|
|
20
|
-
const get =
|
|
20
|
+
const get = vi.fn().mockResolvedValue({
|
|
21
21
|
status: 200,
|
|
22
22
|
data: { success: true, agents: [sampleAgent] },
|
|
23
23
|
});
|
|
@@ -33,7 +33,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
test("passes before as a query param", async () => {
|
|
36
|
-
const get =
|
|
36
|
+
const get = vi.fn().mockResolvedValue({
|
|
37
37
|
status: 200,
|
|
38
38
|
data: {
|
|
39
39
|
success: true,
|
|
@@ -51,7 +51,7 @@ describe("v2.agent listAgents unit", () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
test("does not auto-paginate when next is present", async () => {
|
|
54
|
-
const get =
|
|
54
|
+
const get = vi.fn().mockResolvedValue({
|
|
55
55
|
status: 200,
|
|
56
56
|
data: {
|
|
57
57
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
getAgentStatus,
|
|
4
4
|
getAgentThread,
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
} from "../../../v2/methods/agent";
|
|
7
7
|
|
|
8
8
|
const okPost = () =>
|
|
9
|
-
|
|
9
|
+
vi.fn().mockResolvedValue({
|
|
10
10
|
status: 200,
|
|
11
11
|
data: { success: true, id: "agent-job", threadId: "thread-1", threadTurn: 2 },
|
|
12
12
|
});
|
|
@@ -51,7 +51,7 @@ describe("v2.agent threads unit", () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
test("getAgentStatus parses a chat-mode status payload", async () => {
|
|
54
|
-
const get =
|
|
54
|
+
const get = vi.fn().mockResolvedValue({
|
|
55
55
|
status: 200,
|
|
56
56
|
data: {
|
|
57
57
|
success: true,
|
|
@@ -95,7 +95,7 @@ describe("v2.agent threads unit", () => {
|
|
|
95
95
|
});
|
|
96
96
|
|
|
97
97
|
test("getAgentStatus parses a status payload without the thread fields", async () => {
|
|
98
|
-
const get =
|
|
98
|
+
const get = vi.fn().mockResolvedValue({
|
|
99
99
|
status: 200,
|
|
100
100
|
data: {
|
|
101
101
|
success: true,
|
|
@@ -116,7 +116,7 @@ describe("v2.agent threads unit", () => {
|
|
|
116
116
|
});
|
|
117
117
|
|
|
118
118
|
test("getAgentThread hits the thread endpoint", async () => {
|
|
119
|
-
const get =
|
|
119
|
+
const get = vi.fn().mockResolvedValue({
|
|
120
120
|
status: 200,
|
|
121
121
|
data: {
|
|
122
122
|
success: true,
|
|
@@ -150,7 +150,7 @@ describe("v2.agent threads unit", () => {
|
|
|
150
150
|
});
|
|
151
151
|
|
|
152
152
|
test("getAgentThread appends includeData when requested", async () => {
|
|
153
|
-
const get =
|
|
153
|
+
const get = vi.fn().mockResolvedValue({
|
|
154
154
|
status: 200,
|
|
155
155
|
data: { success: true, thread: { id: "thread-1", runs: [] } },
|
|
156
156
|
});
|
|
@@ -197,7 +197,7 @@ describe("v2.agent threads unit", () => {
|
|
|
197
197
|
|
|
198
198
|
test("getAgentStatus parses skippedProviders, requiresAction and a terms approval", async () => {
|
|
199
199
|
const approvalId = "0199aaaa-0000-7000-8000-000000000000";
|
|
200
|
-
const get =
|
|
200
|
+
const get = vi.fn().mockResolvedValue({
|
|
201
201
|
status: 200,
|
|
202
202
|
data: {
|
|
203
203
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
import { startBatchScrape } from "../../../v2/methods/batch";
|
|
4
4
|
import { startCrawl } from "../../../v2/methods/crawl";
|
|
@@ -10,10 +10,10 @@ import { startAgent } from "../../../v2/methods/agent";
|
|
|
10
10
|
const auditMetadata = { username: "alice@example.com" };
|
|
11
11
|
|
|
12
12
|
function makeHttp(data: Record<string, unknown>) {
|
|
13
|
-
const post =
|
|
13
|
+
const post = vi.fn(async () => ({ status: 200, data }));
|
|
14
14
|
return {
|
|
15
15
|
post,
|
|
16
|
-
prepareHeaders:
|
|
16
|
+
prepareHeaders: vi.fn(() => undefined),
|
|
17
17
|
} as any;
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
|
|
4
4
|
describe("JS SDK v2 branding format", () => {
|
|
5
5
|
function makeHttp(postImpl: (url: string, data: any) => any) {
|
|
6
|
-
return { post:
|
|
6
|
+
return { post: vi.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
test("scrape with branding format returns branding data", async () => {
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { describe, test, expect } from "vitest";
|
|
2
|
+
import axios, { type AxiosAdapter } from "axios";
|
|
3
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
4
|
+
import { HttpClient } from "../../../v2/utils/httpClient";
|
|
5
|
+
import { waitForCrawlCompletion } from "../../../v2/methods/crawl";
|
|
6
|
+
import { SdkError } from "../../../v2/types";
|
|
7
|
+
|
|
8
|
+
const API_URL = "https://api.firecrawl.dev";
|
|
9
|
+
const JOB_ID = "job-123";
|
|
10
|
+
|
|
11
|
+
type Sent = { method: string; path: string };
|
|
12
|
+
type Reply = { status: number; data: unknown };
|
|
13
|
+
|
|
14
|
+
function withAdapter<T extends { defaults: any }>(instance: T, route: (req: Sent) => Reply): Sent[] {
|
|
15
|
+
const sent: Sent[] = [];
|
|
16
|
+
const adapter: AxiosAdapter = async (config) => {
|
|
17
|
+
const req = {
|
|
18
|
+
method: (config.method || "get").toLowerCase(),
|
|
19
|
+
path: new URL(axios.getUri(config)).pathname,
|
|
20
|
+
};
|
|
21
|
+
sent.push(req);
|
|
22
|
+
const { status, data } = route(req);
|
|
23
|
+
const response = { data, status, statusText: String(status), headers: {}, config };
|
|
24
|
+
const validate = config.validateStatus;
|
|
25
|
+
if (validate && !validate(status)) {
|
|
26
|
+
throw new axios.AxiosError(`Request failed with status code ${status}`, undefined, config, null, response);
|
|
27
|
+
}
|
|
28
|
+
return response;
|
|
29
|
+
};
|
|
30
|
+
instance.defaults.adapter = adapter;
|
|
31
|
+
return sent;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function makeClient(route: (req: Sent) => Reply) {
|
|
35
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
36
|
+
const sent = withAdapter((client as any).http.instance, route);
|
|
37
|
+
return { client, sent };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// The cancel runs in the background, so let its request reach the adapter.
|
|
41
|
+
const flush = () => new Promise((r) => setTimeout(r, 0));
|
|
42
|
+
|
|
43
|
+
const running: Reply = { status: 200, data: { success: true, status: "scraping", completed: 0, total: 1, data: [] } };
|
|
44
|
+
const started: Reply = { status: 200, data: { success: true, id: JOB_ID, url: "https://example.com" } };
|
|
45
|
+
|
|
46
|
+
describe("v2 cancelCrawl", () => {
|
|
47
|
+
test("returns false when the crawl is already completed (409)", async () => {
|
|
48
|
+
const { client, sent } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
|
49
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
|
50
|
+
expect(sent).toEqual([{ method: "delete", path: `/v2/crawl/${JOB_ID}` }]);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("returns false on 409 when the transport does not throw for 4xx", async () => {
|
|
54
|
+
const { client } = makeClient(() => ({ status: 409, data: { error: "Crawl is already completed" } }));
|
|
55
|
+
(client as any).http.instance.defaults.validateStatus = () => true;
|
|
56
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("returns true when the crawl is cancelled", async () => {
|
|
60
|
+
const { client } = makeClient(() => ({ status: 200, data: { status: "cancelled" } }));
|
|
61
|
+
await expect(client.cancelCrawl(JOB_ID)).resolves.toBe(true);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("still throws for other errors", async () => {
|
|
65
|
+
const { client } = makeClient(() => ({ status: 404, data: { error: "Job not found" } }));
|
|
66
|
+
const err = await client.cancelCrawl(JOB_ID).catch((e) => e);
|
|
67
|
+
expect(err).toBeInstanceOf(SdkError);
|
|
68
|
+
expect(err.status).toBe(404);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
describe("v2 crawl with an AbortSignal", () => {
|
|
73
|
+
test("abort during polling stops polling, cancels the job, and rejects with the abort reason", async () => {
|
|
74
|
+
const controller = new AbortController();
|
|
75
|
+
let polls = 0;
|
|
76
|
+
const { client, sent } = makeClient((req) => {
|
|
77
|
+
if (req.method === "post") return started;
|
|
78
|
+
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
|
79
|
+
polls += 1;
|
|
80
|
+
if (polls === 1) setTimeout(() => controller.abort(), 10);
|
|
81
|
+
return running;
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
const t0 = Date.now();
|
|
85
|
+
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
|
86
|
+
|
|
87
|
+
expect(Date.now() - t0).toBeLessThan(5000);
|
|
88
|
+
expect(err).toBe(controller.signal.reason);
|
|
89
|
+
expect((err as Error).name).toBe("AbortError");
|
|
90
|
+
await flush();
|
|
91
|
+
expect(sent.map((r) => r.method)).toEqual(["post", "get", "delete"]);
|
|
92
|
+
expect(sent[2]!.path).toBe(`/v2/crawl/${JOB_ID}`);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("rejects with a custom abort reason and ignores a failing cancel", async () => {
|
|
96
|
+
const controller = new AbortController();
|
|
97
|
+
const reason = new Error("caller gave up");
|
|
98
|
+
const { client, sent } = makeClient((req) => {
|
|
99
|
+
if (req.method === "post") return started;
|
|
100
|
+
if (req.method === "delete") return { status: 500, data: { error: "boom" } };
|
|
101
|
+
setTimeout(() => controller.abort(reason), 10);
|
|
102
|
+
return running;
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
await expect(client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal })).rejects.toBe(reason);
|
|
106
|
+
await flush();
|
|
107
|
+
expect(sent.some((r) => r.method === "delete")).toBe(true);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("a hanging cancel does not delay the rejection", async () => {
|
|
111
|
+
const controller = new AbortController();
|
|
112
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
113
|
+
const methods: string[] = [];
|
|
114
|
+
(client as any).http.instance.defaults.adapter = (async (config) => {
|
|
115
|
+
methods.push(config.method!);
|
|
116
|
+
if (config.method === "delete") return new Promise(() => {});
|
|
117
|
+
if (config.method === "get") setTimeout(() => controller.abort(), 10);
|
|
118
|
+
const data = config.method === "post" ? started.data : running.data;
|
|
119
|
+
return { data, status: 200, statusText: "OK", headers: {}, config };
|
|
120
|
+
}) as AxiosAdapter;
|
|
121
|
+
|
|
122
|
+
// Read the reason after the abort: before it, signal.reason is undefined.
|
|
123
|
+
const err = await client.crawl("https://example.com", { pollInterval: 30, signal: controller.signal }).catch((e) => e);
|
|
124
|
+
expect(err).toBe(controller.signal.reason);
|
|
125
|
+
expect((err as Error).name).toBe("AbortError");
|
|
126
|
+
await flush();
|
|
127
|
+
expect(methods).toEqual(["post", "get", "delete"]);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("abort while the start request is in flight cancels the job once the id arrives", async () => {
|
|
131
|
+
const controller = new AbortController();
|
|
132
|
+
const { client, sent } = makeClient((req) => {
|
|
133
|
+
if (req.method === "post") {
|
|
134
|
+
controller.abort();
|
|
135
|
+
return started;
|
|
136
|
+
}
|
|
137
|
+
if (req.method === "delete") return { status: 200, data: { status: "cancelled" } };
|
|
138
|
+
return running;
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const err = await client.crawl("https://example.com", { signal: controller.signal }).catch((e) => e);
|
|
142
|
+
expect(err).toBe(controller.signal.reason);
|
|
143
|
+
expect((err as Error).name).toBe("AbortError");
|
|
144
|
+
await flush();
|
|
145
|
+
expect(sent.map((r) => r.method)).toEqual(["post", "delete"]);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("an already aborted signal sends no request", async () => {
|
|
149
|
+
const controller = new AbortController();
|
|
150
|
+
controller.abort();
|
|
151
|
+
const { client, sent } = makeClient(() => started);
|
|
152
|
+
|
|
153
|
+
await expect(client.crawl("https://example.com", { signal: controller.signal })).rejects.toBe(controller.signal.reason);
|
|
154
|
+
expect(sent).toHaveLength(0);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("the signal is not sent in the crawl request body", async () => {
|
|
158
|
+
const controller = new AbortController();
|
|
159
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
160
|
+
const bodies: unknown[] = [];
|
|
161
|
+
(client as any).http.instance.defaults.adapter = (async (config) => {
|
|
162
|
+
if (config.method === "post") bodies.push(JSON.parse(config.data as string));
|
|
163
|
+
const data = config.method === "post" ? started.data : { success: true, status: "completed", completed: 1, total: 1, data: [] };
|
|
164
|
+
return { data, status: 200, statusText: "OK", headers: {}, config };
|
|
165
|
+
}) as AxiosAdapter;
|
|
166
|
+
|
|
167
|
+
const job = await client.crawl("https://example.com", { limit: 1, signal: controller.signal });
|
|
168
|
+
expect(job.status).toBe("completed");
|
|
169
|
+
expect(bodies[0]).toMatchObject({ url: "https://example.com", limit: 1 });
|
|
170
|
+
expect(Object.keys(bodies[0] as object)).not.toContain("signal");
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
describe("v2 waitForCrawlCompletion with an AbortSignal", () => {
|
|
175
|
+
test("stops polling and rejects without cancelling the job", async () => {
|
|
176
|
+
const http = new HttpClient({ apiKey: "fc-test", apiUrl: API_URL });
|
|
177
|
+
const controller = new AbortController();
|
|
178
|
+
const sent = withAdapter((http as any).instance, () => {
|
|
179
|
+
setTimeout(() => controller.abort(), 10);
|
|
180
|
+
return running;
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
const err = await waitForCrawlCompletion(http, JOB_ID, 30, undefined, controller.signal).catch((e) => e);
|
|
184
|
+
expect(err).toBe(controller.signal.reason);
|
|
185
|
+
expect((err as Error).name).toBe("AbortError");
|
|
186
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
187
|
+
expect(sent.map((r) => r.method)).toEqual(["get"]);
|
|
188
|
+
});
|
|
189
|
+
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { FirecrawlClient } from "../../../v2/client";
|
|
3
3
|
import { SdkError } from "../../../v2/types";
|
|
4
4
|
|
|
@@ -31,7 +31,7 @@ const response = {
|
|
|
31
31
|
describe("developerSearch", () => {
|
|
32
32
|
test("forwards every dedicated developer filter", async () => {
|
|
33
33
|
const http = {
|
|
34
|
-
post:
|
|
34
|
+
post: vi.fn(async () => ({ status: 200, data: response })),
|
|
35
35
|
} as any;
|
|
36
36
|
|
|
37
37
|
const client = new FirecrawlClient({
|
|
@@ -77,7 +77,7 @@ describe("developerSearch", () => {
|
|
|
77
77
|
|
|
78
78
|
test("normalizes transport errors to SdkError", async () => {
|
|
79
79
|
const http = {
|
|
80
|
-
post:
|
|
80
|
+
post: vi.fn(async () => {
|
|
81
81
|
throw {
|
|
82
82
|
isAxiosError: true,
|
|
83
83
|
code: "ECONNABORTED",
|
|
@@ -102,7 +102,7 @@ describe("developerSearch", () => {
|
|
|
102
102
|
results: [{ ...response.results[0], license: "MIT" }],
|
|
103
103
|
};
|
|
104
104
|
const http = {
|
|
105
|
-
post:
|
|
105
|
+
post: vi.fn(async () => ({ status: 200, data: flattened })),
|
|
106
106
|
} as any;
|
|
107
107
|
|
|
108
108
|
const client = new FirecrawlClient({
|