firecrawl 4.38.0 → 4.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +53 -2
  2. package/dist/{chunk-KSTNOGOD.js → chunk-OJ4YJQ4J.js} +1 -1
  3. package/dist/index.cjs +204 -17
  4. package/dist/index.d.cts +289 -7
  5. package/dist/index.d.ts +289 -7
  6. package/dist/index.js +203 -17
  7. package/dist/{package-OS7RONXY.js → package-F6O7J2XI.js} +1 -1
  8. package/package.json +1 -1
  9. package/pnpm-workspace.yaml +3 -2
  10. package/src/__tests__/e2e/v2/batch.test.ts +13 -11
  11. package/src/__tests__/e2e/v2/crawl.test.ts +24 -20
  12. package/src/__tests__/e2e/v2/extract.test.ts +32 -7
  13. package/src/__tests__/e2e/v2/map.test.ts +4 -3
  14. package/src/__tests__/e2e/v2/parse.test.ts +4 -3
  15. package/src/__tests__/e2e/v2/scrape.test.ts +10 -9
  16. package/src/__tests__/e2e/v2/search.test.ts +9 -8
  17. package/src/__tests__/e2e/v2/usage.test.ts +6 -5
  18. package/src/__tests__/e2e/v2/utils/rateLimit.ts +257 -0
  19. package/src/__tests__/e2e/v2/watcher.test.ts +4 -3
  20. package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +1 -1
  21. package/src/__tests__/unit/v2/agent-thread.unit.test.ts +164 -0
  22. package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +501 -0
  23. package/src/__tests__/unit/v2/extract.unit.test.ts +1 -1
  24. package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +1 -1
  25. package/src/__tests__/unit/v2/search-country.unit.test.ts +32 -0
  26. package/src/__tests__/unit/v2/search-data-property.test.ts +1 -0
  27. package/src/__tests__/unit/v2/threat-protection.unit.test.ts +18 -1
  28. package/src/__tests__/unit/v2/tools.unit.test.ts +240 -0
  29. package/src/__tests__/unit/v2/v1-aliases.test.ts +1 -0
  30. package/src/v2/client.ts +52 -4
  31. package/src/v2/methods/agent.ts +23 -1
  32. package/src/v2/methods/research.ts +14 -8
  33. package/src/v2/methods/scrape.ts +3 -1
  34. package/src/v2/methods/search.ts +8 -1
  35. package/src/v2/methods/tools.ts +137 -0
  36. package/src/v2/types.ts +307 -35
  37. package/src/v2/utils/errorHandler.ts +19 -3
@@ -0,0 +1,164 @@
1
+ import { describe, expect, jest, test } from "@jest/globals";
2
+ import {
3
+ getAgentStatus,
4
+ getAgentThread,
5
+ startAgent,
6
+ } from "../../../v2/methods/agent";
7
+
8
+ const okPost = () =>
9
+ jest.fn().mockResolvedValue({
10
+ status: 200,
11
+ data: { success: true, id: "agent-job", threadId: "thread-1", threadTurn: 2 },
12
+ });
13
+
14
+ describe("v2.agent threads unit", () => {
15
+ test("startAgent forwards the thread fields when set", async () => {
16
+ const post = okPost();
17
+
18
+ await startAgent({ post } as any, {
19
+ prompt: "Which tier has SSO?",
20
+ threadId: "thread-1",
21
+ mode: "chat",
22
+ exchange: { enabled: true, toolkits: ["a", "b"] },
23
+ });
24
+
25
+ expect(post).toHaveBeenCalledWith("/v2/agent", {
26
+ prompt: "Which tier has SSO?",
27
+ threadId: "thread-1",
28
+ mode: "chat",
29
+ exchange: { enabled: true, toolkits: ["a", "b"] },
30
+ });
31
+ });
32
+
33
+ test("startAgent omits the thread fields when unset", async () => {
34
+ const post = okPost();
35
+
36
+ await startAgent({ post } as any, { prompt: "List the pricing tiers" });
37
+
38
+ expect(post).toHaveBeenCalledWith("/v2/agent", {
39
+ prompt: "List the pricing tiers",
40
+ });
41
+ });
42
+
43
+ test("startAgent returns the thread the run belongs to", async () => {
44
+ const started = await startAgent({ post: okPost() } as any, {
45
+ prompt: "Which tier has SSO?",
46
+ threadId: "thread-1",
47
+ });
48
+
49
+ expect(started.threadId).toBe("thread-1");
50
+ expect(started.threadTurn).toBe(2);
51
+ });
52
+
53
+ test("getAgentStatus parses a chat-mode status payload", async () => {
54
+ const get = jest.fn().mockResolvedValue({
55
+ status: 200,
56
+ data: {
57
+ success: true,
58
+ status: "completed",
59
+ data: null,
60
+ model: "spark-2",
61
+ expiresAt: "2026-09-02T00:00:00.000Z",
62
+ threadId: "thread-1",
63
+ threadTurn: 2,
64
+ mode: "chat",
65
+ message: "Only the Enterprise tier lists SSO.",
66
+ suggestions: [{ label: "Seat caps?", prompt: "Does Team cap seats?" }],
67
+ pendingApproval: {
68
+ id: "approval-1",
69
+ reason: "One paid call answers this.",
70
+ calls: [
71
+ {
72
+ id: "call-1",
73
+ provider: "provider-1",
74
+ capability: "capability-1",
75
+ input: { query: "sso" },
76
+ creditsEstimate: 5,
77
+ },
78
+ ],
79
+ resolution: null,
80
+ },
81
+ exchange: { enabled: true, paidCalls: 0, creditsUsed: null },
82
+ },
83
+ });
84
+
85
+ const status = await getAgentStatus({ get } as any, "agent-job");
86
+
87
+ expect(get).toHaveBeenCalledWith("/v2/agent/agent-job");
88
+ expect(status.data).toBeNull();
89
+ expect(status.message).toBe("Only the Enterprise tier lists SSO.");
90
+ expect(status.mode).toBe("chat");
91
+ expect(status.threadTurn).toBe(2);
92
+ expect(status.suggestions).toHaveLength(1);
93
+ expect(status.pendingApproval!.calls[0].creditsEstimate).toBe(5);
94
+ expect(status.exchange!.paidCalls).toBe(0);
95
+ });
96
+
97
+ test("getAgentStatus parses a status payload without the thread fields", async () => {
98
+ const get = jest.fn().mockResolvedValue({
99
+ status: 200,
100
+ data: {
101
+ success: true,
102
+ status: "completed",
103
+ data: { price: 42 },
104
+ model: "spark-2",
105
+ expiresAt: "2026-09-02T00:00:00.000Z",
106
+ creditsUsed: 12,
107
+ },
108
+ });
109
+
110
+ const status = await getAgentStatus({ get } as any, "agent-job");
111
+
112
+ expect(status.data).toEqual({ price: 42 });
113
+ expect(status.threadId).toBeUndefined();
114
+ expect(status.message).toBeUndefined();
115
+ expect(status.pendingApproval).toBeUndefined();
116
+ });
117
+
118
+ test("getAgentThread hits the thread endpoint", async () => {
119
+ const get = jest.fn().mockResolvedValue({
120
+ status: 200,
121
+ data: {
122
+ success: true,
123
+ thread: {
124
+ id: "thread-1",
125
+ createdAt: "2026-09-01T00:00:00.000Z",
126
+ updatedAt: "2026-09-01T00:01:00.000Z",
127
+ status: "idle",
128
+ runs: [
129
+ {
130
+ id: "agent-job",
131
+ turn: 1,
132
+ mode: "chat",
133
+ prompt: "List the pricing tiers",
134
+ status: "succeeded",
135
+ createdAt: "2026-09-01T00:00:00.000Z",
136
+ finishedAt: "2026-09-01T00:00:30.000Z",
137
+ creditsUsed: 212,
138
+ message: null,
139
+ },
140
+ ],
141
+ },
142
+ },
143
+ });
144
+
145
+ const thread = await getAgentThread({ get } as any, "thread-1");
146
+
147
+ expect(get).toHaveBeenCalledWith("/v2/agent/threads/thread-1");
148
+ expect(thread.thread!.runs).toHaveLength(1);
149
+ expect(thread.thread!.runs[0].turn).toBe(1);
150
+ });
151
+
152
+ test("getAgentThread appends includeData when requested", async () => {
153
+ const get = jest.fn().mockResolvedValue({
154
+ status: 200,
155
+ data: { success: true, thread: { id: "thread-1", runs: [] } },
156
+ });
157
+
158
+ await getAgentThread({ get } as any, "thread-1", { includeData: true });
159
+
160
+ expect(get).toHaveBeenCalledWith(
161
+ "/v2/agent/threads/thread-1?includeData=true",
162
+ );
163
+ });
164
+ });
@@ -0,0 +1,501 @@
1
+ import { describe, expect, jest, test } from "@jest/globals";
2
+ import { readFileSync, readdirSync } from "node:fs";
3
+ import path from "node:path";
4
+ import Firecrawl from "../../../index";
5
+ import {
6
+ COMPOSITE_METHODS,
7
+ DEFAULT_JOB_TIMEOUT_MS,
8
+ RETRY_BUDGET_MS,
9
+ isRateLimitError,
10
+ rateLimitWaitMs,
11
+ testTimeoutMs,
12
+ waitForJob,
13
+ withRateLimitRetry,
14
+ } from "../../e2e/v2/utils/rateLimit";
15
+
16
+ type AsyncCall = () => Promise<string>;
17
+
18
+ /** Reproduces the error the API returns when the per-minute limit is spent. */
19
+ function rateLimitError(seconds: number): Error & { status: number } {
20
+ const err = new Error(
21
+ `Rate limit exceeded. Consumed (req/min): 3, Remaining (req/min): 0. ` +
22
+ `Upgrade your plan at https://firecrawl.dev/pricing for increased rate ` +
23
+ `limits or please retry after ${seconds}s, resets at Tue Sep 08 2026`,
24
+ ) as Error & { status: number };
25
+ err.status = 429;
26
+ return err;
27
+ }
28
+
29
+ /**
30
+ * Source with comments blanked out, newlines kept.
31
+ *
32
+ * The scans below count brackets and calls. A comment holding `waitForJob(`,
33
+ * a client call or a lone brace would corrupt both, so drop comments first.
34
+ * Strings are not comments: a suite passes paths such as "/blog/*", and a
35
+ * scan that took that for a comment would swallow the rest of the file.
36
+ */
37
+ function stripComments(source: string): string {
38
+ let out = "";
39
+ let quote: string | undefined;
40
+
41
+ for (let i = 0; i < source.length; i++) {
42
+ const c = source[i];
43
+
44
+ if (quote !== undefined) {
45
+ out += c;
46
+ if (c === "\\") {
47
+ out += source[i + 1] ?? "";
48
+ i += 1;
49
+ } else if (c === quote) {
50
+ quote = undefined;
51
+ }
52
+ continue;
53
+ }
54
+
55
+ if (c === "'" || c === '"' || c === "`") {
56
+ quote = c;
57
+ out += c;
58
+ continue;
59
+ }
60
+
61
+ if (source.startsWith("/*", i)) {
62
+ const close = source.indexOf("*/", i + 2);
63
+ const end = close < 0 ? source.length : close + 2;
64
+ // Keep the newlines so line structure survives.
65
+ out += source.slice(i, end).replace(/[^\n]/g, "");
66
+ i = end - 1;
67
+ continue;
68
+ }
69
+
70
+ if (source.startsWith("//", i)) {
71
+ const line = source.indexOf("\n", i);
72
+ if (line < 0) break;
73
+ i = line - 1;
74
+ continue;
75
+ }
76
+
77
+ out += c;
78
+ }
79
+
80
+ return out;
81
+ }
82
+
83
+ /**
84
+ * Text of each `name(...)` call in source, by matching parentheses.
85
+ *
86
+ * A regex cannot do this. The calls hold arrow functions, objects and
87
+ * template strings, so the closing bracket is not the first one.
88
+ *
89
+ * Pass source that stripComments has already cleaned.
90
+ */
91
+ function callsOf(source: string, name: string): string[] {
92
+ const found: string[] = [];
93
+
94
+ for (const match of source.matchAll(new RegExp(`\\b${name}\\(`, "g"))) {
95
+ const from = match.index!;
96
+ let i = from + match[0].length - 1;
97
+ let depth = 0;
98
+ let quote: string | undefined;
99
+
100
+ for (; i < source.length; i++) {
101
+ const c = source[i];
102
+ if (quote !== undefined) {
103
+ if (c === "\\") i += 1;
104
+ else if (c === quote) quote = undefined;
105
+ continue;
106
+ }
107
+ if (c === "'" || c === '"' || c === "`") {
108
+ quote = c;
109
+ continue;
110
+ }
111
+ if (c === "(" || c === "[" || c === "{") depth += 1;
112
+ else if (c === ")" || c === "]" || c === "}") {
113
+ depth -= 1;
114
+ if (depth === 0) break;
115
+ }
116
+ }
117
+ found.push(source.slice(from, i + 1));
118
+ }
119
+
120
+ return found;
121
+ }
122
+
123
+ /**
124
+ * Body of each test in a suite, including the test.each form.
125
+ *
126
+ * test.each takes the cases first and the test second, so the test body is
127
+ * the group after the cases.
128
+ */
129
+ function testBodies(source: string): string[] {
130
+ const found: string[] = [];
131
+
132
+ for (const match of source.matchAll(/\btest(\.each)?\s*\(/g)) {
133
+ if (!match[1]) {
134
+ found.push(callsOf(source.slice(match.index!), "test")[0]);
135
+ continue;
136
+ }
137
+ const cases = callsOf(source.slice(match.index!), "test.each")[0];
138
+ const rest = source.slice(match.index! + cases.length);
139
+ const open = rest.indexOf("(");
140
+ // Reuse the matcher by giving it a name to anchor on.
141
+ const group = callsOf(`each${rest.slice(open)}`, "each")[0];
142
+ found.push(group);
143
+ }
144
+
145
+ return found;
146
+ }
147
+
148
+ /** Local async helpers in a suite, by name, with the body each one runs. */
149
+ function localHelpers(source: string): Map<string, string> {
150
+ const found = new Map<string, string>();
151
+
152
+ for (const match of source.matchAll(/\n(\s*)async function (\w+)\(/g)) {
153
+ const start = match.index! + 1;
154
+ const close = source.indexOf(`\n${match[1]}}`, start);
155
+ found.set(match[2], source.slice(start, close < 0 ? undefined : close));
156
+ }
157
+
158
+ return found;
159
+ }
160
+
161
+ /**
162
+ * Worst case of one test body, and the wrapped calls it counts.
163
+ *
164
+ * Each waitForJob costs its bound, one poll interval and the retry budget of
165
+ * the status read. Each other wrapped client call costs the retry budget on
166
+ * its own. A local helper runs inside the test, so its calls count too.
167
+ */
168
+ function worstCaseMs(
169
+ body: string,
170
+ helpers: Map<string, string>,
171
+ ): { calls: number; ms: number } {
172
+ let calls = 0;
173
+ let ms = 0;
174
+
175
+ const waits = callsOf(body, "waitForJob");
176
+ for (const wait of waits) {
177
+ const timeout = wait.match(/timeout:\s*(\d+)/);
178
+ const poll = wait.match(/pollInterval:\s*(\d+)/);
179
+ const bound = timeout
180
+ ? Number(timeout[1]) * 1000
181
+ : DEFAULT_JOB_TIMEOUT_MS;
182
+ ms +=
183
+ bound +
184
+ Math.max(1000, (poll ? Number(poll[1]) : 2) * 1000) +
185
+ RETRY_BUDGET_MS;
186
+ calls += 1;
187
+ }
188
+
189
+ // The status read sits inside waitForJob, so drop those calls before
190
+ // counting the rest. Longest first, so a shorter call is not cut twice.
191
+ let rest = body;
192
+ for (const wait of [...waits].sort((a, b) => b.length - a.length)) {
193
+ rest = rest.split(wait).join("");
194
+ }
195
+
196
+ const direct = rest.match(/client\.\w+\s*\(/g) ?? [];
197
+ ms += direct.length * RETRY_BUDGET_MS;
198
+ calls += direct.length;
199
+
200
+ for (const [name, helperBody] of helpers) {
201
+ const uses = rest.match(new RegExp(`\\b${name}\\s*\\(`, "g")) ?? [];
202
+ if (uses.length === 0) continue;
203
+ const inner = worstCaseMs(helperBody, new Map());
204
+ ms += uses.length * inner.ms;
205
+ calls += uses.length * inner.calls;
206
+ }
207
+
208
+ return { calls, ms };
209
+ }
210
+
211
+ describe("e2e rate-limit retry helper", () => {
212
+ test("recognises the rate-limit error and reads its reset time", () => {
213
+ expect(isRateLimitError(rateLimitError(1))).toBe(true);
214
+ expect(isRateLimitError({ status: 429 })).toBe(true);
215
+ expect(isRateLimitError(new Error("Bad request"))).toBe(false);
216
+ expect(isRateLimitError(undefined)).toBe(false);
217
+
218
+ // 1s reset plus the 1s buffer.
219
+ expect(rateLimitWaitMs(rateLimitError(1))).toBe(2000);
220
+ // A reset longer than the window is capped.
221
+ expect(rateLimitWaitMs(rateLimitError(600))).toBe(75_000);
222
+ // No reset time in the error, so the fallback applies.
223
+ expect(rateLimitWaitMs(new Error("Rate limit exceeded."))).toBe(30_000);
224
+ expect(rateLimitWaitMs({ details: { retryAfter: 2 } })).toBe(3000);
225
+ });
226
+
227
+ test("runs the call again after a rate-limit error", async () => {
228
+ const scrape = jest
229
+ .fn<() => Promise<string>>()
230
+ .mockRejectedValueOnce(rateLimitError(1))
231
+ .mockResolvedValueOnce("ok");
232
+ const client = withRateLimitRetry({ scrape });
233
+
234
+ await expect(client.scrape()).resolves.toBe("ok");
235
+ expect(scrape).toHaveBeenCalledTimes(2);
236
+ }, 30_000);
237
+
238
+ test("stops after the attempt bound and surfaces the error", async () => {
239
+ const scrape = jest
240
+ .fn<() => Promise<string>>()
241
+ .mockRejectedValue(rateLimitError(1));
242
+ const client = withRateLimitRetry({ scrape });
243
+
244
+ await expect(client.scrape()).rejects.toThrow(/Rate limit exceeded/);
245
+ expect(scrape).toHaveBeenCalledTimes(3);
246
+ }, 30_000);
247
+
248
+ test("passes other errors through without waiting", async () => {
249
+ const scrape = jest
250
+ .fn<() => Promise<string>>()
251
+ .mockRejectedValue(new Error("invalid url"));
252
+ const client = withRateLimitRetry({ scrape });
253
+
254
+ await expect(client.scrape()).rejects.toThrow("invalid url");
255
+ expect(scrape).toHaveBeenCalledTimes(1);
256
+ });
257
+
258
+ test("never runs a composite method again after a rate-limit error", async () => {
259
+ for (const name of COMPOSITE_METHODS) {
260
+ const method = jest.fn<AsyncCall>().mockRejectedValue(rateLimitError(1));
261
+ const client = withRateLimitRetry<Record<string, AsyncCall>>({
262
+ [name]: method,
263
+ });
264
+
265
+ const startedAt = Date.now();
266
+ await expect(client[name]()).rejects.toThrow(/Rate limit exceeded/);
267
+
268
+ // One call only. A second call would start a second job.
269
+ expect(method).toHaveBeenCalledTimes(1);
270
+ // The error surfaces at once, so no wait ran either.
271
+ expect(Date.now() - startedAt).toBeLessThan(1_000);
272
+ }
273
+ });
274
+
275
+ test("returns the result of a composite method that succeeds", async () => {
276
+ const crawl = jest.fn<AsyncCall>().mockResolvedValue("job");
277
+ const client = withRateLimitRetry({ crawl });
278
+
279
+ await expect(client.crawl()).resolves.toBe("job");
280
+ expect(crawl).toHaveBeenCalledTimes(1);
281
+ });
282
+
283
+ test("every composite name is a method on the client", () => {
284
+ const client = new Firecrawl({ apiKey: "test-key" }) as unknown as Record<
285
+ string,
286
+ unknown
287
+ >;
288
+
289
+ for (const name of COMPOSITE_METHODS) {
290
+ expect(typeof client[name]).toBe("function");
291
+ }
292
+ });
293
+
294
+ test("the client has no composite method the list misses", () => {
295
+ // Guard for a new waiter method. A composite starts a job through a
296
+ // *Waiter call, or forwards to another composite on this.
297
+ const source = readFileSync(
298
+ path.resolve(process.cwd(), "src/v2/client.ts"),
299
+ "utf-8",
300
+ );
301
+
302
+ const found = new Set<string>();
303
+ let current: string | undefined;
304
+ for (const line of source.split("\n")) {
305
+ const signature = line.match(/^\s{2}(?:async\s+)?([A-Za-z_$][\w$]*)\s*\(/);
306
+ if (signature) current = signature[1];
307
+ if (!current) continue;
308
+ const startsJob =
309
+ /\b\w+Waiter\s*\(/.test(line) ||
310
+ new RegExp(`this\\.(${[...COMPOSITE_METHODS].join("|")})\\s*\\(`).test(
311
+ line,
312
+ );
313
+ if (startsJob) found.add(current);
314
+ }
315
+
316
+ // The scan must see the known composites, or it stopped working.
317
+ expect([...found].sort()).toEqual([...COMPOSITE_METHODS].sort());
318
+ });
319
+
320
+ test("no e2e suite calls a composite method", () => {
321
+ // A composite call in a suite has no retry, so a rate limit fails the test
322
+ // at once. The suites start the job and poll it instead.
323
+ const dir = path.resolve(process.cwd(), "src/__tests__/e2e/v2");
324
+ const suites = readdirSync(dir).filter(name => name.endsWith(".test.ts"));
325
+ expect(suites.length).toBeGreaterThan(0);
326
+
327
+ const calls = new RegExp(
328
+ `client\\.(${[...COMPOSITE_METHODS].join("|")})\\s*\\(`,
329
+ );
330
+ const offenders = suites.filter(name =>
331
+ calls.test(readFileSync(path.join(dir, name), "utf-8")),
332
+ );
333
+
334
+ expect(offenders).toEqual([]);
335
+ });
336
+
337
+ test("waitForJob polls until the job reaches a terminal state", async () => {
338
+ const getStatus = jest
339
+ .fn<() => Promise<{ status: string }>>()
340
+ .mockResolvedValueOnce({ status: "scraping" })
341
+ .mockResolvedValueOnce({ status: "scraping" })
342
+ .mockResolvedValueOnce({ status: "completed" });
343
+
344
+ // pollInterval floors at 1s, so three reads take about two seconds.
345
+ await expect(
346
+ waitForJob(getStatus, { pollInterval: 1, timeout: 30 }),
347
+ ).resolves.toEqual({ status: "completed" });
348
+ expect(getStatus).toHaveBeenCalledTimes(3);
349
+ }, 30_000);
350
+
351
+ test("waitForJob gives up when the job outlives its timeout", async () => {
352
+ const getStatus = jest
353
+ .fn<() => Promise<{ status: string }>>()
354
+ .mockResolvedValue({ status: "scraping" });
355
+
356
+ await expect(
357
+ waitForJob(getStatus, { pollInterval: 1, timeout: 1 }),
358
+ ).rejects.toThrow(/did not finish in 1s/);
359
+ }, 30_000);
360
+
361
+ test("waitForJob bounds a caller that passes no timeout", async () => {
362
+ const getStatus = jest
363
+ .fn<() => Promise<{ status: string }>>()
364
+ .mockResolvedValue({ status: "scraping" });
365
+
366
+ // The default is a backstop for a caller that forgets a timeout, not a
367
+ // work allowance. A caller that waits for real work passes its own bound.
368
+ // So the default only has to fit the smallest base any suite uses, which
369
+ // is 60_000 ms. The last read can start just inside the bound and then
370
+ // spend the whole retry budget, so count both.
371
+ expect(DEFAULT_JOB_TIMEOUT_MS + 2_000 + RETRY_BUDGET_MS).toBeLessThan(
372
+ testTimeoutMs(60_000),
373
+ );
374
+
375
+ // Fake timers run the whole bound without waiting for it.
376
+ jest.useFakeTimers();
377
+ try {
378
+ const settled = expect(waitForJob(getStatus)).rejects.toThrow(
379
+ new RegExp(`did not finish in ${DEFAULT_JOB_TIMEOUT_MS / 1000}s`),
380
+ );
381
+ await jest.advanceTimersByTimeAsync(DEFAULT_JOB_TIMEOUT_MS + 5_000);
382
+ await settled;
383
+ } finally {
384
+ jest.useRealTimers();
385
+ }
386
+
387
+ expect(getStatus).toHaveBeenCalled();
388
+ }, 30_000);
389
+
390
+ test("the scan reads past comments and strings", () => {
391
+ // The guard is only as good as its parser, so hold the parser too. Each
392
+ // trap below broke an earlier version: a block comment with a call and a
393
+ // lone brace, a commented-out call, and a path that starts with "/*".
394
+ const suite = [
395
+ 'describe("fixture", () => {',
396
+ ' test("one start and one wait", async () => {',
397
+ ' /* waitForJob( here, with an unbalanced { brace */',
398
+ ' // client.notCounted(',
399
+ ' const started = await client.startThing({',
400
+ ' includePaths: ["/blog/*", "/docs/*"],',
401
+ ' });',
402
+ ' const job = await waitForJob(() => client.getThing(started.id), {',
403
+ ' pollInterval: 1,',
404
+ ' timeout: 60,',
405
+ ' });',
406
+ ' expect(job.status).toBe("completed");',
407
+ ' }, testTimeoutMs(220_000));',
408
+ "});",
409
+ ].join("\n");
410
+
411
+ const source = stripComments(suite);
412
+
413
+ // The traps are gone, and the code around them survived.
414
+ expect(source).not.toMatch(/waitForJob\( here/);
415
+ expect(source).not.toMatch(/notCounted/);
416
+ expect(source).toContain('includePaths: ["/blog/*", "/docs/*"]');
417
+ expect(source).toContain("timeout: 60,");
418
+
419
+ const bodies = testBodies(source);
420
+ expect(bodies).toHaveLength(1);
421
+
422
+ // One start call at the retry budget, and one wait at bound plus poll
423
+ // plus the retry budget of its status read.
424
+ const { calls, ms } = worstCaseMs(bodies[0], new Map());
425
+ expect(calls).toBe(2);
426
+ expect(ms).toBe(RETRY_BUDGET_MS + 60_000 + 1_000 + RETRY_BUDGET_MS);
427
+ expect(ms).toBeLessThan(testTimeoutMs(220_000));
428
+ });
429
+
430
+ test("every wrapped call in a test fits its budget", () => {
431
+ // Guard for the whole class of budget faults. A test spends the retry
432
+ // budget once per wrapped call, not once in total, so the budget has to
433
+ // cover the sum. Otherwise jest fires first and hides the specific
434
+ // message this helper exists to surface.
435
+ const dir = path.resolve(process.cwd(), "src/__tests__/e2e/v2");
436
+ const suites = readdirSync(dir).filter(name => name.endsWith(".test.ts"));
437
+ expect(suites.length).toBeGreaterThan(0);
438
+
439
+ let checked = 0;
440
+ let widest = 0;
441
+
442
+ for (const name of suites) {
443
+ const source = stripComments(
444
+ readFileSync(path.join(dir, name), "utf-8"),
445
+ );
446
+ const helpers = localHelpers(source);
447
+
448
+ // A helper that calls another helper would hide calls from the sum.
449
+ for (const [self, body] of helpers) {
450
+ for (const other of helpers.keys()) {
451
+ if (other === self) continue;
452
+ expect(body).not.toMatch(new RegExp(`\\b${other}\\s*\\(`));
453
+ }
454
+ }
455
+
456
+ const bodies = testBodies(source);
457
+ // The scan must see every test, or a budget goes unchecked.
458
+ expect(bodies.length).toBe(
459
+ (source.match(/\btest(?:\.each)?\s*\(/g) ?? []).length,
460
+ );
461
+
462
+ for (const body of bodies) {
463
+ const { calls, ms } = worstCaseMs(body, helpers);
464
+ if (calls === 0) continue;
465
+
466
+ // A test that calls the wrapped client needs a budget of its own.
467
+ // Without one jest allows 5s, which any single wait outlasts.
468
+ const budgets = body.match(/testTimeoutMs\((\d[\d_]*)\)/g);
469
+ expect(budgets).not.toBeNull();
470
+
471
+ const base = Number(budgets![budgets!.length - 1].replace(/\D/g, ""));
472
+ expect(ms).toBeLessThan(testTimeoutMs(base));
473
+
474
+ checked += 1;
475
+ widest = Math.max(widest, calls);
476
+ }
477
+ }
478
+
479
+ // The scan must see the known tests, or it stopped working.
480
+ expect(checked).toBeGreaterThanOrEqual(45);
481
+ expect(widest).toBeGreaterThanOrEqual(3);
482
+ });
483
+
484
+ test("the test timeout fits the worst-case serial retry", () => {
485
+ // Two waits at the 75s cap follow the first attempt.
486
+ expect(RETRY_BUDGET_MS).toBe(150_000);
487
+ expect(testTimeoutMs(60_000)).toBe(210_000);
488
+
489
+ // No pair of waits the helper can ask for exceeds the budget.
490
+ const longestWait = rateLimitWaitMs(rateLimitError(600));
491
+ expect(longestWait * 2).toBeLessThanOrEqual(RETRY_BUDGET_MS);
492
+ });
493
+
494
+ test("leaves values that are not promises alone", () => {
495
+ const watcher = jest.fn(() => ({ kind: "crawl" }));
496
+ const client = withRateLimitRetry({ watcher, apiUrl: "https://example.com" });
497
+
498
+ expect(client.watcher()).toEqual({ kind: "crawl" });
499
+ expect(client.apiUrl).toBe("https://example.com");
500
+ });
501
+ });
@@ -1,4 +1,4 @@
1
- import { describe, expect, test } from "@jest/globals";
1
+ import { describe, expect, jest, test } from "@jest/globals";
2
2
  import { startExtract } from "../../../v2/methods/extract";
3
3
  import type { WebhookConfig } from "../../../v2/types";
4
4
 
@@ -1,4 +1,4 @@
1
- import { describe, expect, test } from "@jest/globals";
1
+ import { describe, expect, jest, test } from "@jest/globals";
2
2
  import {
3
3
  processingContinuesDelayMs,
4
4
  scrape,