@mendable/firecrawl-js 4.45.0 → 4.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/audit-ci.jsonc +1 -15
- package/dist/chunk-H75FJ6LW.js +114 -0
- package/dist/index.cjs +61 -27
- package/dist/index.d.cts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +58 -21
- package/dist/package-NZI5TDUV.js +40 -0
- package/package.json +5 -9
- package/pnpm-workspace.yaml +0 -6
- package/src/__tests__/e2e/v1/index.test.ts +1 -1
- package/src/__tests__/e2e/v2/batch.test.ts +1 -1
- package/src/__tests__/e2e/v2/crawl.test.ts +1 -1
- package/src/__tests__/e2e/v2/extract.test.ts +1 -1
- package/src/__tests__/e2e/v2/map.test.ts +1 -1
- package/src/__tests__/e2e/v2/parse.test.ts +1 -1
- package/src/__tests__/e2e/v2/scrape.test.ts +1 -1
- package/src/__tests__/e2e/v2/search.test.ts +1 -1
- package/src/__tests__/e2e/v2/usage.test.ts +1 -1
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +2 -2
- package/src/__tests__/e2e/v2/watcher.test.ts +1 -1
- package/src/__tests__/unit/v1/batch-scrape-poll.test.ts +7 -7
- package/src/__tests__/unit/v1/monitor-job-status-retry.test.ts +6 -6
- package/src/__tests__/unit/v1/next-url-pinning.test.ts +3 -3
- package/src/__tests__/unit/v1/optional-params.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +6 -6
- package/src/__tests__/unit/v2/agent-hints.test.ts +3 -3
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +7 -7
- package/src/__tests__/unit/v2/agent.test.ts +1 -1
- package/src/__tests__/unit/v2/audit-metadata.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/branding.test.ts +2 -2
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/clientOptions.test.ts +1 -0
- package/src/__tests__/unit/v2/crawl-cancel-abort.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/developer.test.ts +4 -4
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +16 -16
- package/src/__tests__/unit/v2/errorHandler.test.ts +24 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +4 -4
- package/src/__tests__/unit/v2/gov.test.ts +4 -4
- package/src/__tests__/unit/v2/menu.test.ts +2 -2
- package/src/__tests__/unit/v2/monitor-search-target.test.ts +1 -0
- package/src/__tests__/unit/v2/next-url-pinning.test.ts +1 -1
- package/src/__tests__/unit/v2/pagination.test.ts +26 -6
- package/src/__tests__/unit/v2/parse-formats.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/parse.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/product.test.ts +2 -2
- package/src/__tests__/unit/v2/prototype-discoverability.test.ts +1 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -1
- package/src/__tests__/unit/v2/scrape-browser.unit.test.ts +8 -8
- package/src/__tests__/unit/v2/scrape.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +13 -13
- package/src/__tests__/unit/v2/search-country.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-data-property.test.ts +3 -3
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/tools.unit.test.ts +3 -3
- package/src/__tests__/unit/v2/v1-aliases.test.ts +2 -2
- package/src/__tests__/unit/v2/validation.test.ts +40 -2
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +1 -1
- package/src/v1/index.ts +1 -3
- package/src/v2/client.ts +1 -1
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +10 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/vitest.config.ts +7 -0
- package/dist/chunk-LKCOCWYW.js +0 -97
- package/dist/package-C6PK3CK7.js +0 -4
- package/jest.config.js +0 -16
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
getAgentStatus,
|
|
4
4
|
getAgentThread,
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
} from "../../../v2/methods/agent";
|
|
7
7
|
|
|
8
8
|
const okPost = () =>
|
|
9
|
-
|
|
9
|
+
vi.fn().mockResolvedValue({
|
|
10
10
|
status: 200,
|
|
11
11
|
data: { success: true, id: "agent-job", threadId: "thread-1", threadTurn: 2 },
|
|
12
12
|
});
|
|
@@ -51,7 +51,7 @@ describe("v2.agent threads unit", () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
test("getAgentStatus parses a chat-mode status payload", async () => {
|
|
54
|
-
const get =
|
|
54
|
+
const get = vi.fn().mockResolvedValue({
|
|
55
55
|
status: 200,
|
|
56
56
|
data: {
|
|
57
57
|
success: true,
|
|
@@ -95,7 +95,7 @@ describe("v2.agent threads unit", () => {
|
|
|
95
95
|
});
|
|
96
96
|
|
|
97
97
|
test("getAgentStatus parses a status payload without the thread fields", async () => {
|
|
98
|
-
const get =
|
|
98
|
+
const get = vi.fn().mockResolvedValue({
|
|
99
99
|
status: 200,
|
|
100
100
|
data: {
|
|
101
101
|
success: true,
|
|
@@ -116,7 +116,7 @@ describe("v2.agent threads unit", () => {
|
|
|
116
116
|
});
|
|
117
117
|
|
|
118
118
|
test("getAgentThread hits the thread endpoint", async () => {
|
|
119
|
-
const get =
|
|
119
|
+
const get = vi.fn().mockResolvedValue({
|
|
120
120
|
status: 200,
|
|
121
121
|
data: {
|
|
122
122
|
success: true,
|
|
@@ -150,7 +150,7 @@ describe("v2.agent threads unit", () => {
|
|
|
150
150
|
});
|
|
151
151
|
|
|
152
152
|
test("getAgentThread appends includeData when requested", async () => {
|
|
153
|
-
const get =
|
|
153
|
+
const get = vi.fn().mockResolvedValue({
|
|
154
154
|
status: 200,
|
|
155
155
|
data: { success: true, thread: { id: "thread-1", runs: [] } },
|
|
156
156
|
});
|
|
@@ -197,7 +197,7 @@ describe("v2.agent threads unit", () => {
|
|
|
197
197
|
|
|
198
198
|
test("getAgentStatus parses skippedProviders, requiresAction and a terms approval", async () => {
|
|
199
199
|
const approvalId = "0199aaaa-0000-7000-8000-000000000000";
|
|
200
|
-
const get =
|
|
200
|
+
const get = vi.fn().mockResolvedValue({
|
|
201
201
|
status: 200,
|
|
202
202
|
data: {
|
|
203
203
|
success: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
import { startBatchScrape } from "../../../v2/methods/batch";
|
|
4
4
|
import { startCrawl } from "../../../v2/methods/crawl";
|
|
@@ -10,10 +10,10 @@ import { startAgent } from "../../../v2/methods/agent";
|
|
|
10
10
|
const auditMetadata = { username: "alice@example.com" };
|
|
11
11
|
|
|
12
12
|
function makeHttp(data: Record<string, unknown>) {
|
|
13
|
-
const post =
|
|
13
|
+
const post = vi.fn(async () => ({ status: 200, data }));
|
|
14
14
|
return {
|
|
15
15
|
post,
|
|
16
|
-
prepareHeaders:
|
|
16
|
+
prepareHeaders: vi.fn(() => undefined),
|
|
17
17
|
} as any;
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
|
|
4
4
|
describe("JS SDK v2 branding format", () => {
|
|
5
5
|
function makeHttp(postImpl: (url: string, data: any) => any) {
|
|
6
|
-
return { post:
|
|
6
|
+
return { post: vi.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
test("scrape with branding format returns branding data", async () => {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
|
+
import { browser } from "../../../v2/methods/browser";
|
|
3
|
+
|
|
4
|
+
describe("JS SDK v2 browser", () => {
|
|
5
|
+
test("sends location when creating a session", async () => {
|
|
6
|
+
const post = vi.fn(async () => ({
|
|
7
|
+
status: 200,
|
|
8
|
+
data: { success: true, id: "session-1" },
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
await browser({ post } as any, { location: { country: "GB" } });
|
|
12
|
+
|
|
13
|
+
expect(post).toHaveBeenCalledWith("/v2/browser", {
|
|
14
|
+
location: { country: "GB" },
|
|
15
|
+
});
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("omits location by default", async () => {
|
|
19
|
+
const post = vi.fn(async () => ({
|
|
20
|
+
status: 200,
|
|
21
|
+
data: { success: true, id: "session-1" },
|
|
22
|
+
}));
|
|
23
|
+
|
|
24
|
+
await browser({ post } as any);
|
|
25
|
+
|
|
26
|
+
expect(post).toHaveBeenCalledWith("/v2/browser", {});
|
|
27
|
+
});
|
|
28
|
+
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect } from "
|
|
1
|
+
import { describe, test, expect } from "vitest";
|
|
2
2
|
import axios, { type AxiosAdapter } from "axios";
|
|
3
3
|
import { FirecrawlClient } from "../../../v2/client";
|
|
4
4
|
import { HttpClient } from "../../../v2/utils/httpClient";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { FirecrawlClient } from "../../../v2/client";
|
|
3
3
|
import { SdkError } from "../../../v2/types";
|
|
4
4
|
|
|
@@ -31,7 +31,7 @@ const response = {
|
|
|
31
31
|
describe("developerSearch", () => {
|
|
32
32
|
test("forwards every dedicated developer filter", async () => {
|
|
33
33
|
const http = {
|
|
34
|
-
post:
|
|
34
|
+
post: vi.fn(async () => ({ status: 200, data: response })),
|
|
35
35
|
} as any;
|
|
36
36
|
|
|
37
37
|
const client = new FirecrawlClient({
|
|
@@ -77,7 +77,7 @@ describe("developerSearch", () => {
|
|
|
77
77
|
|
|
78
78
|
test("normalizes transport errors to SdkError", async () => {
|
|
79
79
|
const http = {
|
|
80
|
-
post:
|
|
80
|
+
post: vi.fn(async () => {
|
|
81
81
|
throw {
|
|
82
82
|
isAxiosError: true,
|
|
83
83
|
code: "ECONNABORTED",
|
|
@@ -102,7 +102,7 @@ describe("developerSearch", () => {
|
|
|
102
102
|
results: [{ ...response.results[0], license: "MIT" }],
|
|
103
103
|
};
|
|
104
104
|
const http = {
|
|
105
|
-
post:
|
|
105
|
+
post: vi.fn(async () => ({ status: 200, data: flattened })),
|
|
106
106
|
} as any;
|
|
107
107
|
|
|
108
108
|
const client = new FirecrawlClient({
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { readFileSync, readdirSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import Firecrawl from "../../../index";
|
|
@@ -225,7 +225,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
225
225
|
});
|
|
226
226
|
|
|
227
227
|
test("runs the call again after a rate-limit error", async () => {
|
|
228
|
-
const scrape =
|
|
228
|
+
const scrape = vi
|
|
229
229
|
.fn<() => Promise<string>>()
|
|
230
230
|
.mockRejectedValueOnce(rateLimitError(1))
|
|
231
231
|
.mockResolvedValueOnce("ok");
|
|
@@ -236,7 +236,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
236
236
|
}, 30_000);
|
|
237
237
|
|
|
238
238
|
test("stops after the attempt bound and surfaces the error", async () => {
|
|
239
|
-
const scrape =
|
|
239
|
+
const scrape = vi
|
|
240
240
|
.fn<() => Promise<string>>()
|
|
241
241
|
.mockRejectedValue(rateLimitError(1));
|
|
242
242
|
const client = withRateLimitRetry({ scrape });
|
|
@@ -246,7 +246,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
246
246
|
}, 30_000);
|
|
247
247
|
|
|
248
248
|
test("passes other errors through without waiting", async () => {
|
|
249
|
-
const scrape =
|
|
249
|
+
const scrape = vi
|
|
250
250
|
.fn<() => Promise<string>>()
|
|
251
251
|
.mockRejectedValue(new Error("invalid url"));
|
|
252
252
|
const client = withRateLimitRetry({ scrape });
|
|
@@ -257,7 +257,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
257
257
|
|
|
258
258
|
test("never runs a composite method again after a rate-limit error", async () => {
|
|
259
259
|
for (const name of COMPOSITE_METHODS) {
|
|
260
|
-
const method =
|
|
260
|
+
const method = vi.fn<AsyncCall>().mockRejectedValue(rateLimitError(1));
|
|
261
261
|
const client = withRateLimitRetry<Record<string, AsyncCall>>({
|
|
262
262
|
[name]: method,
|
|
263
263
|
});
|
|
@@ -273,7 +273,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
273
273
|
});
|
|
274
274
|
|
|
275
275
|
test("returns the result of a composite method that succeeds", async () => {
|
|
276
|
-
const crawl =
|
|
276
|
+
const crawl = vi.fn<AsyncCall>().mockResolvedValue("job");
|
|
277
277
|
const client = withRateLimitRetry({ crawl });
|
|
278
278
|
|
|
279
279
|
await expect(client.crawl()).resolves.toBe("job");
|
|
@@ -335,7 +335,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
335
335
|
});
|
|
336
336
|
|
|
337
337
|
test("waitForJob polls until the job reaches a terminal state", async () => {
|
|
338
|
-
const getStatus =
|
|
338
|
+
const getStatus = vi
|
|
339
339
|
.fn<() => Promise<{ status: string }>>()
|
|
340
340
|
.mockResolvedValueOnce({ status: "scraping" })
|
|
341
341
|
.mockResolvedValueOnce({ status: "scraping" })
|
|
@@ -349,7 +349,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
349
349
|
}, 30_000);
|
|
350
350
|
|
|
351
351
|
test("waitForJob gives up when the job outlives its timeout", async () => {
|
|
352
|
-
const getStatus =
|
|
352
|
+
const getStatus = vi
|
|
353
353
|
.fn<() => Promise<{ status: string }>>()
|
|
354
354
|
.mockResolvedValue({ status: "scraping" });
|
|
355
355
|
|
|
@@ -359,7 +359,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
359
359
|
}, 30_000);
|
|
360
360
|
|
|
361
361
|
test("waitForJob bounds a caller that passes no timeout", async () => {
|
|
362
|
-
const getStatus =
|
|
362
|
+
const getStatus = vi
|
|
363
363
|
.fn<() => Promise<{ status: string }>>()
|
|
364
364
|
.mockResolvedValue({ status: "scraping" });
|
|
365
365
|
|
|
@@ -373,15 +373,15 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
373
373
|
);
|
|
374
374
|
|
|
375
375
|
// Fake timers run the whole bound without waiting for it.
|
|
376
|
-
|
|
376
|
+
vi.useFakeTimers();
|
|
377
377
|
try {
|
|
378
378
|
const settled = expect(waitForJob(getStatus)).rejects.toThrow(
|
|
379
379
|
new RegExp(`did not finish in ${DEFAULT_JOB_TIMEOUT_MS / 1000}s`),
|
|
380
380
|
);
|
|
381
|
-
await
|
|
381
|
+
await vi.advanceTimersByTimeAsync(DEFAULT_JOB_TIMEOUT_MS + 5_000);
|
|
382
382
|
await settled;
|
|
383
383
|
} finally {
|
|
384
|
-
|
|
384
|
+
vi.useRealTimers();
|
|
385
385
|
}
|
|
386
386
|
|
|
387
387
|
expect(getStatus).toHaveBeenCalled();
|
|
@@ -430,8 +430,8 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
430
430
|
test("every wrapped call in a test fits its budget", () => {
|
|
431
431
|
// Guard for the whole class of budget faults. A test spends the retry
|
|
432
432
|
// budget once per wrapped call, not once in total, so the budget has to
|
|
433
|
-
// cover the sum. Otherwise
|
|
434
|
-
// message this helper exists to surface.
|
|
433
|
+
// cover the sum. Otherwise the test timeout fires first and hides the
|
|
434
|
+
// specific message this helper exists to surface.
|
|
435
435
|
const dir = path.resolve(process.cwd(), "src/__tests__/e2e/v2");
|
|
436
436
|
const suites = readdirSync(dir).filter(name => name.endsWith(".test.ts"));
|
|
437
437
|
expect(suites.length).toBeGreaterThan(0);
|
|
@@ -464,7 +464,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
464
464
|
if (calls === 0) continue;
|
|
465
465
|
|
|
466
466
|
// A test that calls the wrapped client needs a budget of its own.
|
|
467
|
-
// Without one
|
|
467
|
+
// Without one Vitest allows 5s, which any single wait outlasts.
|
|
468
468
|
const budgets = body.match(/testTimeoutMs\((\d[\d_]*)\)/g);
|
|
469
469
|
expect(budgets).not.toBeNull();
|
|
470
470
|
|
|
@@ -492,7 +492,7 @@ describe("e2e rate-limit retry helper", () => {
|
|
|
492
492
|
});
|
|
493
493
|
|
|
494
494
|
test("leaves values that are not promises alone", () => {
|
|
495
|
-
const watcher =
|
|
495
|
+
const watcher = vi.fn(() => ({ kind: "crawl" }));
|
|
496
496
|
const client = withRateLimitRetry({ watcher, apiUrl: "https://example.com" });
|
|
497
497
|
|
|
498
498
|
expect(client.watcher()).toEqual({ kind: "crawl" });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect } from "
|
|
1
|
+
import { describe, test, expect } from "vitest";
|
|
2
2
|
import { throwForBadResponse, normalizeAxiosError } from "../../../v2/utils/errorHandler";
|
|
3
3
|
|
|
4
4
|
describe("v2 utils: errorHandler", () => {
|
|
@@ -15,5 +15,28 @@ describe("v2 utils: errorHandler", () => {
|
|
|
15
15
|
};
|
|
16
16
|
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
|
|
17
17
|
});
|
|
18
|
+
|
|
19
|
+
test("normalizeAxiosError: names the fields a 400 validation error rejected", () => {
|
|
20
|
+
const err: any = {
|
|
21
|
+
isAxiosError: true,
|
|
22
|
+
response: {
|
|
23
|
+
status: 400,
|
|
24
|
+
data: {
|
|
25
|
+
error: "Unrecognized key in body -- please review the v2 API documentation for request body changes",
|
|
26
|
+
details: [{ code: "unrecognized_keys", keys: ["parsePDF"], path: [], message: "Unrecognized key(s) in object: 'parsePDF'" }],
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
message: "Request failed with status code 400",
|
|
30
|
+
};
|
|
31
|
+
expect(() => normalizeAxiosError(err, "scrape")).toThrow(/unrecognized key\(s\) parsePDF/);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("throwForBadResponse: appends the path and message of each issue", () => {
|
|
35
|
+
const resp: any = {
|
|
36
|
+
status: 400,
|
|
37
|
+
data: { error: "Bad Request", details: [{ code: "invalid_union", path: ["formats", 0], message: "Invalid input" }] },
|
|
38
|
+
};
|
|
39
|
+
expect(() => throwForBadResponse(resp, "scrape")).toThrow("Bad Request (formats.0: Invalid input)");
|
|
40
|
+
});
|
|
18
41
|
});
|
|
19
42
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { startExtract } from "../../../v2/methods/extract";
|
|
3
3
|
import type { WebhookConfig } from "../../../v2/types";
|
|
4
4
|
|
|
5
5
|
describe("v2.extract unit", () => {
|
|
6
6
|
test("omits discovery options from extract without mutating shared options", async () => {
|
|
7
|
-
const post =
|
|
7
|
+
const post = vi
|
|
8
8
|
.fn()
|
|
9
9
|
.mockResolvedValue({ status: 200, data: { id: "extract-job" } });
|
|
10
10
|
const scrapeOptions = {
|
|
@@ -27,7 +27,7 @@ describe("v2.extract unit", () => {
|
|
|
27
27
|
});
|
|
28
28
|
});
|
|
29
29
|
test("startExtract forwards string webhook in request payload", async () => {
|
|
30
|
-
const post =
|
|
30
|
+
const post = vi.fn().mockResolvedValue({
|
|
31
31
|
status: 200,
|
|
32
32
|
data: { id: "extract-job" },
|
|
33
33
|
});
|
|
@@ -46,7 +46,7 @@ describe("v2.extract unit", () => {
|
|
|
46
46
|
});
|
|
47
47
|
|
|
48
48
|
test("startExtract forwards object webhook in request payload", async () => {
|
|
49
|
-
const post =
|
|
49
|
+
const post = vi.fn().mockResolvedValue({
|
|
50
50
|
status: 200,
|
|
51
51
|
data: { id: "extract-job" },
|
|
52
52
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect,
|
|
1
|
+
import { describe, expect, vi, test } from "vitest";
|
|
2
2
|
import { FirecrawlClient } from "../../../v2/client";
|
|
3
3
|
|
|
4
4
|
const response = {
|
|
@@ -30,7 +30,7 @@ describe("govSearch", () => {
|
|
|
30
30
|
[undefined, { query: "food labeling requirements" }],
|
|
31
31
|
])("posts %p and returns web results", async (options, body) => {
|
|
32
32
|
const http = {
|
|
33
|
-
post:
|
|
33
|
+
post: vi.fn(async () => ({ status: 200, data: response })),
|
|
34
34
|
} as any;
|
|
35
35
|
|
|
36
36
|
const result = await clientWith(http).govSearch(
|
|
@@ -43,7 +43,7 @@ describe("govSearch", () => {
|
|
|
43
43
|
});
|
|
44
44
|
|
|
45
45
|
test("rejects an empty query", async () => {
|
|
46
|
-
const http = { post:
|
|
46
|
+
const http = { post: vi.fn() } as any;
|
|
47
47
|
|
|
48
48
|
await expect(clientWith(http).govSearch(" ")).rejects.toThrow(
|
|
49
49
|
"query cannot be empty",
|
|
@@ -53,7 +53,7 @@ describe("govSearch", () => {
|
|
|
53
53
|
|
|
54
54
|
test("throws on an unsuccessful response body", async () => {
|
|
55
55
|
const http = {
|
|
56
|
-
post:
|
|
56
|
+
post: vi.fn(async () => ({
|
|
57
57
|
status: 200,
|
|
58
58
|
data: { success: false, error: "Search failed" },
|
|
59
59
|
})),
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
|
|
4
4
|
describe("JS SDK v2 menu format", () => {
|
|
5
5
|
function makeHttp(postImpl: (url: string, data: any) => any) {
|
|
6
|
-
return { post:
|
|
6
|
+
return { post: vi.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
test("scrape with menu format returns menu data", async () => {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Unit test: a search-target monitor request is forwarded intact to the API.
|
|
3
3
|
*/
|
|
4
|
+
import { describe, test, expect } from "vitest";
|
|
4
5
|
import { createMonitor } from "../../../v2/methods/monitor";
|
|
5
6
|
import type { CreateMonitorRequest, MonitorSearchTarget } from "../../../v2/types";
|
|
6
7
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect } from "
|
|
1
|
+
import { describe, test, expect } from "vitest";
|
|
2
2
|
import axios, { type AxiosAdapter } from "axios";
|
|
3
3
|
import { FirecrawlClient } from "../../../v2/client";
|
|
4
4
|
import { pinToApiOrigin } from "../../../utils/apiOrigin";
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi, type Mock } from "vitest";
|
|
2
2
|
import { getCrawlStatus } from "../../../v2/methods/crawl";
|
|
3
3
|
import { getBatchScrapeStatus } from "../../../v2/methods/batch";
|
|
4
4
|
import { getMonitorCheck } from "../../../v2/methods/monitor";
|
|
5
5
|
|
|
6
6
|
describe("JS SDK v2 pagination", () => {
|
|
7
7
|
function makeHttp(getImpl: (url: string) => any) {
|
|
8
|
-
return { get:
|
|
8
|
+
return { get: vi.fn(async (u: string) => getImpl(u)) } as any;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
test("crawl: autoPaginate=false returns next", async () => {
|
|
@@ -93,7 +93,7 @@ describe("JS SDK v2 pagination", () => {
|
|
|
93
93
|
if (url.endsWith("n1")) return p1;
|
|
94
94
|
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
|
95
95
|
});
|
|
96
|
-
const nowSpy =
|
|
96
|
+
const nowSpy = vi.spyOn(Date, "now");
|
|
97
97
|
try {
|
|
98
98
|
nowSpy
|
|
99
99
|
.mockImplementationOnce(() => 0) // started
|
|
@@ -101,12 +101,32 @@ describe("JS SDK v2 pagination", () => {
|
|
|
101
101
|
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
|
|
102
102
|
const res = await getCrawlStatus(http, "jobC", { autoPaginate: true, maxWaitTime: 1 });
|
|
103
103
|
expect(res.data.length).toBe(2); // initial + first page
|
|
104
|
-
expect((http.get as
|
|
104
|
+
expect((http.get as Mock).mock.calls.length).toBe(2); // initial + n1 only
|
|
105
105
|
} finally {
|
|
106
106
|
nowSpy.mockRestore();
|
|
107
107
|
}
|
|
108
108
|
});
|
|
109
109
|
|
|
110
|
+
test("crawl: stops when a running job's cursor does not advance", async () => {
|
|
111
|
+
const next = "https://api/v2/crawl/jobR?skip=2";
|
|
112
|
+
const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
|
|
113
|
+
const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
|
|
114
|
+
const http = makeHttp((url) => (url === next ? stalled : first));
|
|
115
|
+
const res = await getCrawlStatus(http, "jobR");
|
|
116
|
+
expect(res.data.length).toBe(2);
|
|
117
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test("batch: stops when a running job's cursor does not advance", async () => {
|
|
121
|
+
const next = "https://api/v2/batch/scrape/jobR?skip=2";
|
|
122
|
+
const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
|
|
123
|
+
const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
|
|
124
|
+
const http = makeHttp((url) => (url === next ? stalled : first));
|
|
125
|
+
const res = await getBatchScrapeStatus(http, "jobR");
|
|
126
|
+
expect(res.data.length).toBe(2);
|
|
127
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
128
|
+
});
|
|
129
|
+
|
|
110
130
|
test("batch: maxWaitTime stops pagination after first page", async () => {
|
|
111
131
|
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
|
112
132
|
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
|
@@ -115,7 +135,7 @@ describe("JS SDK v2 pagination", () => {
|
|
|
115
135
|
if (url.endsWith("b1")) return p1;
|
|
116
136
|
return { status: 200, data: { success: true, next: null, data: [{ markdown: "c" }] } };
|
|
117
137
|
});
|
|
118
|
-
const nowSpy =
|
|
138
|
+
const nowSpy = vi.spyOn(Date, "now");
|
|
119
139
|
try {
|
|
120
140
|
nowSpy
|
|
121
141
|
.mockImplementationOnce(() => 0) // started
|
|
@@ -123,7 +143,7 @@ describe("JS SDK v2 pagination", () => {
|
|
|
123
143
|
.mockImplementationOnce(() => 3000); // second loop check > maxWaitTime
|
|
124
144
|
const res = await getBatchScrapeStatus(http, "jobB", { autoPaginate: true, maxWaitTime: 1 });
|
|
125
145
|
expect(res.data.length).toBe(2);
|
|
126
|
-
expect((http.get as
|
|
146
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
127
147
|
} finally {
|
|
128
148
|
nowSpy.mockRestore();
|
|
129
149
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { scrape } from "../../../v2/methods/scrape";
|
|
3
3
|
|
|
4
4
|
describe("JS SDK v2 product format", () => {
|
|
5
5
|
function makeHttp(postImpl: (url: string, data: any) => any) {
|
|
6
|
-
return { post:
|
|
6
|
+
return { post: vi.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
test("scrape with product format returns product data", async () => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, test, expect } from "
|
|
1
|
+
import { describe, test, expect } from "vitest";
|
|
2
2
|
import { ResearchClient } from "../../../v2/methods/research";
|
|
3
3
|
import { SdkError } from "../../../v2/types";
|
|
4
4
|
import type { HttpClient } from "../../../v2/utils/httpClient";
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { describe, test, expect,
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
2
|
import { interact, stopInteraction } from "../../../v2/methods/scrape";
|
|
3
3
|
import { SdkError } from "../../../v2/types";
|
|
4
4
|
|
|
5
5
|
describe("JS SDK v2 scrape-browser methods", () => {
|
|
6
6
|
test("interact posts to scrape interact endpoint", async () => {
|
|
7
|
-
const post =
|
|
7
|
+
const post = vi.fn(async () => ({
|
|
8
8
|
status: 200,
|
|
9
9
|
data: {
|
|
10
10
|
success: true,
|
|
@@ -28,7 +28,7 @@ describe("JS SDK v2 scrape-browser methods", () => {
|
|
|
28
28
|
});
|
|
29
29
|
|
|
30
30
|
test("interact with prompt posts prompt to endpoint", async () => {
|
|
31
|
-
const post =
|
|
31
|
+
const post = vi.fn(async () => ({
|
|
32
32
|
status: 200,
|
|
33
33
|
data: {
|
|
34
34
|
success: true,
|
|
@@ -61,14 +61,14 @@ describe("JS SDK v2 scrape-browser methods", () => {
|
|
|
61
61
|
});
|
|
62
62
|
|
|
63
63
|
test("interact throws when neither code nor prompt provided", async () => {
|
|
64
|
-
const http = { post:
|
|
64
|
+
const http = { post: vi.fn() } as any;
|
|
65
65
|
await expect(interact(http, "job-123", {})).rejects.toThrow(
|
|
66
66
|
"Either 'code' or 'prompt' must be provided",
|
|
67
67
|
);
|
|
68
68
|
});
|
|
69
69
|
|
|
70
70
|
test("interact throws on non-200 response", async () => {
|
|
71
|
-
const post =
|
|
71
|
+
const post = vi.fn(async () => ({
|
|
72
72
|
status: 400,
|
|
73
73
|
data: {
|
|
74
74
|
success: false,
|
|
@@ -83,7 +83,7 @@ describe("JS SDK v2 scrape-browser methods", () => {
|
|
|
83
83
|
});
|
|
84
84
|
|
|
85
85
|
test("stopInteraction calls delete endpoint", async () => {
|
|
86
|
-
const del =
|
|
86
|
+
const del = vi.fn(async () => ({
|
|
87
87
|
status: 200,
|
|
88
88
|
data: {
|
|
89
89
|
success: true,
|
|
@@ -98,7 +98,7 @@ describe("JS SDK v2 scrape-browser methods", () => {
|
|
|
98
98
|
});
|
|
99
99
|
|
|
100
100
|
test("stopInteraction throws on non-200 response", async () => {
|
|
101
|
-
const del =
|
|
101
|
+
const del = vi.fn(async () => ({
|
|
102
102
|
status: 404,
|
|
103
103
|
data: {
|
|
104
104
|
success: false,
|
|
@@ -113,7 +113,7 @@ describe("JS SDK v2 scrape-browser methods", () => {
|
|
|
113
113
|
});
|
|
114
114
|
|
|
115
115
|
test("interact converts seconds-based body timeout to ms axios timeout", async () => {
|
|
116
|
-
const post =
|
|
116
|
+
const post = vi.fn(async () => ({
|
|
117
117
|
status: 200,
|
|
118
118
|
data: { success: true, stdout: "ok", exitCode: 0 },
|
|
119
119
|
}));
|