firecrawl 4.35.0 → 4.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-666CR5HH.js → chunk-LFX7WDGT.js} +1 -1
- package/dist/index.cjs +92 -15
- package/dist/index.d.cts +174 -2
- package/dist/index.d.ts +174 -2
- package/dist/index.js +93 -16
- package/dist/{package-ZOVFJE7Z.js → package-SHMK2LJE.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +140 -0
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +145 -0
- package/src/v1/index.ts +1 -0
- package/src/v2/client.ts +22 -4
- package/src/v2/methods/agent.ts +34 -1
- package/src/v2/methods/scrape.ts +123 -19
- package/src/v2/types.ts +165 -0
- package/src/v2/utils/httpClient.ts +6 -0
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-LFX7WDGT.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -48,6 +48,11 @@ var HttpClient = class {
|
|
|
48
48
|
getApiKey() {
|
|
49
49
|
return this.apiKey;
|
|
50
50
|
}
|
|
51
|
+
/** The client-wide default request timeout, for callers that need to
|
|
52
|
+
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
53
|
+
getTimeoutMs() {
|
|
54
|
+
return this.instance.defaults.timeout ?? 3e5;
|
|
55
|
+
}
|
|
51
56
|
async request(config) {
|
|
52
57
|
const version = getVersion();
|
|
53
58
|
config.headers = {
|
|
@@ -428,26 +433,60 @@ function isRetryableError(err) {
|
|
|
428
433
|
}
|
|
429
434
|
|
|
430
435
|
// src/v2/methods/scrape.ts
|
|
431
|
-
|
|
436
|
+
var RESUME_MAX_ATTEMPTS = 5;
|
|
437
|
+
var RESUME_MAX_TOTAL_WAIT_MS = 20 * 60 * 1e3;
|
|
438
|
+
var RESUME_MIN_DELAY_MS = 5 * 1e3;
|
|
439
|
+
var RESUME_MAX_DELAY_MS = 10 * 60 * 1e3;
|
|
440
|
+
function processingContinuesDelayMs(err) {
|
|
441
|
+
const response = err?.isAxiosError ? err.response : void 0;
|
|
442
|
+
if (!response || response.status !== 408) return void 0;
|
|
443
|
+
const body = response.data;
|
|
444
|
+
if (body?.code !== "SCRAPE_TIMEOUT" || body?.details?.state !== "processing_continues") {
|
|
445
|
+
return void 0;
|
|
446
|
+
}
|
|
447
|
+
const headerSeconds = Number(response.headers?.["retry-after"]);
|
|
448
|
+
const seconds = typeof body.details.retryAfterSeconds === "number" && Number.isFinite(body.details.retryAfterSeconds) ? body.details.retryAfterSeconds : Number.isFinite(headerSeconds) ? headerSeconds : 60;
|
|
449
|
+
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1e3));
|
|
450
|
+
}
|
|
451
|
+
async function scrape(http, url, options, deps = {}) {
|
|
452
|
+
const sleep = deps.sleepImpl ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
432
453
|
if (!url || !url.trim()) {
|
|
433
454
|
throw new Error("URL cannot be empty");
|
|
434
455
|
}
|
|
435
456
|
if (options) ensureValidScrapeOptions(options);
|
|
457
|
+
const { autoResume, ...requestOptions } = options ?? {};
|
|
436
458
|
const payload = { url: url.trim() };
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
"
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
459
|
+
Object.assign(payload, requestOptions);
|
|
460
|
+
const requestTimeout = autoResume === false ? typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {} : {
|
|
461
|
+
timeoutMs: Math.max(
|
|
462
|
+
typeof options?.timeout === "number" ? options.timeout : 3e5,
|
|
463
|
+
typeof http.getTimeoutMs === "function" ? http.getTimeoutMs() : 0
|
|
464
|
+
) + 3e4
|
|
465
|
+
};
|
|
466
|
+
let resumes = 0;
|
|
467
|
+
let resumeWaitedMs = 0;
|
|
468
|
+
while (true) {
|
|
469
|
+
try {
|
|
470
|
+
const res = await http.post(
|
|
471
|
+
"/v2/scrape",
|
|
472
|
+
payload,
|
|
473
|
+
requestTimeout
|
|
474
|
+
);
|
|
475
|
+
if (res.status !== 200 || !res.data?.success) {
|
|
476
|
+
throwForBadResponse(res, "scrape");
|
|
477
|
+
}
|
|
478
|
+
return res.data.data || {};
|
|
479
|
+
} catch (err) {
|
|
480
|
+
const delayMs = autoResume !== false ? processingContinuesDelayMs(err) : void 0;
|
|
481
|
+
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
482
|
+
resumes += 1;
|
|
483
|
+
resumeWaitedMs += delayMs;
|
|
484
|
+
await sleep(delayMs);
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
488
|
+
throw err;
|
|
446
489
|
}
|
|
447
|
-
return res.data.data || {};
|
|
448
|
-
} catch (err) {
|
|
449
|
-
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
450
|
-
throw err;
|
|
451
490
|
}
|
|
452
491
|
}
|
|
453
492
|
async function interact(http, jobId, args) {
|
|
@@ -1195,6 +1234,7 @@ function prepareAgentPayload(args) {
|
|
|
1195
1234
|
if (args.maxCredits !== null && args.maxCredits !== void 0) body.maxCredits = args.maxCredits;
|
|
1196
1235
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== void 0) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
1197
1236
|
if (args.model !== null && args.model !== void 0) body.model = args.model;
|
|
1237
|
+
if (args.effort !== null && args.effort !== void 0) body.effort = args.effort;
|
|
1198
1238
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
1199
1239
|
if (args.threatProtection != null)
|
|
1200
1240
|
body.threatProtection = args.threatProtection;
|
|
@@ -1238,6 +1278,27 @@ async function agent(http, args) {
|
|
|
1238
1278
|
if (!jobId) return started;
|
|
1239
1279
|
return waitAgent(http, jobId, args.pollInterval ?? 2, args.timeout);
|
|
1240
1280
|
}
|
|
1281
|
+
async function getAgentTrace(http, jobId, options) {
|
|
1282
|
+
try {
|
|
1283
|
+
const query = options?.liveView ? "?liveView=true" : "";
|
|
1284
|
+
const res = await http.get(`/v2/agent/${jobId}/trace${query}`);
|
|
1285
|
+
if (res.status !== 200) throwForBadResponse(res, "agent trace");
|
|
1286
|
+
return res.data;
|
|
1287
|
+
} catch (err) {
|
|
1288
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent trace");
|
|
1289
|
+
throw err;
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
async function getAgentSnapshot(http, jobId, snapshotId) {
|
|
1293
|
+
try {
|
|
1294
|
+
const res = await http.get(`/v2/agent/${jobId}/snapshots/${snapshotId}`);
|
|
1295
|
+
if (res.status !== 200) throwForBadResponse(res, "agent snapshot");
|
|
1296
|
+
return res.data;
|
|
1297
|
+
} catch (err) {
|
|
1298
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent snapshot");
|
|
1299
|
+
throw err;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1241
1302
|
async function cancelAgent(http, jobId) {
|
|
1242
1303
|
try {
|
|
1243
1304
|
const res = await http.delete(`/v2/agent/${jobId}`);
|
|
@@ -2239,6 +2300,22 @@ var FirecrawlClient = class {
|
|
|
2239
2300
|
async cancelAgent(jobId) {
|
|
2240
2301
|
return cancelAgent(this.http, jobId);
|
|
2241
2302
|
}
|
|
2303
|
+
/**
|
|
2304
|
+
* Get the execution trace of an agent job (spark-2 runs only).
|
|
2305
|
+
* @param jobId Agent job id.
|
|
2306
|
+
* @param options.liveView Also include currently active browser sessions with live view URLs.
|
|
2307
|
+
*/
|
|
2308
|
+
async getAgentTrace(jobId, options) {
|
|
2309
|
+
return getAgentTrace(this.http, jobId, options);
|
|
2310
|
+
}
|
|
2311
|
+
/**
|
|
2312
|
+
* Get the full content of an artifact snapshot referenced by a trace event.
|
|
2313
|
+
* @param jobId Agent job id.
|
|
2314
|
+
* @param snapshotId Snapshot id from an artifact.updated trace event.
|
|
2315
|
+
*/
|
|
2316
|
+
async getAgentSnapshot(jobId, snapshotId) {
|
|
2317
|
+
return getAgentSnapshot(this.http, jobId, snapshotId);
|
|
2318
|
+
}
|
|
2242
2319
|
// Browser
|
|
2243
2320
|
/**
|
|
2244
2321
|
* Create a new browser session.
|
|
@@ -2381,7 +2458,7 @@ var FirecrawlApp = class {
|
|
|
2381
2458
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2382
2459
|
return process.env.npm_package_version;
|
|
2383
2460
|
}
|
|
2384
|
-
const packageJson = await import("./package-
|
|
2461
|
+
const packageJson = await import("./package-SHMK2LJE.js");
|
|
2385
2462
|
return packageJson.default.version;
|
|
2386
2463
|
} catch (error) {
|
|
2387
2464
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.37.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { describe, expect, test } from "@jest/globals";
|
|
2
|
+
import {
|
|
3
|
+
startAgent,
|
|
4
|
+
getAgentTrace,
|
|
5
|
+
getAgentSnapshot,
|
|
6
|
+
} from "../../../v2/methods/agent";
|
|
7
|
+
import type { AgentTraceEvent } from "../../../v2/types";
|
|
8
|
+
|
|
9
|
+
describe("v2.agent unit", () => {
|
|
10
|
+
test("startAgent forwards effort in request payload", async () => {
|
|
11
|
+
const post = jest.fn().mockResolvedValue({
|
|
12
|
+
status: 200,
|
|
13
|
+
data: { success: true, id: "agent-job" },
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
await startAgent({ post } as any, {
|
|
17
|
+
prompt: "Find the pricing",
|
|
18
|
+
effort: "high",
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
expect(post).toHaveBeenCalledWith("/v2/agent", {
|
|
22
|
+
prompt: "Find the pricing",
|
|
23
|
+
effort: "high",
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("startAgent omits effort when unset", async () => {
|
|
28
|
+
const post = jest.fn().mockResolvedValue({
|
|
29
|
+
status: 200,
|
|
30
|
+
data: { success: true, id: "agent-job" },
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
await startAgent({ post } as any, { prompt: "Find the pricing" });
|
|
34
|
+
|
|
35
|
+
expect(post).toHaveBeenCalledWith("/v2/agent", {
|
|
36
|
+
prompt: "Find the pricing",
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("getAgentTrace hits the trace endpoint and returns typed events", async () => {
|
|
41
|
+
const runStarted: AgentTraceEvent = {
|
|
42
|
+
schemaVersion: 1,
|
|
43
|
+
eventId: "018f3c5e-0000-7000-8000-000000000000",
|
|
44
|
+
runId: "018f3c5e-0000-7000-8000-000000000001",
|
|
45
|
+
occurredAt: "2026-08-26T12:00:00+00:00",
|
|
46
|
+
producerSequence: 1,
|
|
47
|
+
agent: {
|
|
48
|
+
id: "018f3c5e-0000-7000-8000-000000000002",
|
|
49
|
+
role: "orchestrator",
|
|
50
|
+
name: "main",
|
|
51
|
+
},
|
|
52
|
+
type: "run.started",
|
|
53
|
+
};
|
|
54
|
+
const artifactUpdated: AgentTraceEvent = {
|
|
55
|
+
schemaVersion: 1,
|
|
56
|
+
eventId: "018f3c5e-0000-7000-8000-000000000010",
|
|
57
|
+
runId: "018f3c5e-0000-7000-8000-000000000001",
|
|
58
|
+
occurredAt: "2026-08-26T12:00:01+00:00",
|
|
59
|
+
producerSequence: 2,
|
|
60
|
+
agent: {
|
|
61
|
+
id: "018f3c5e-0000-7000-8000-000000000002",
|
|
62
|
+
role: "orchestrator",
|
|
63
|
+
name: "main",
|
|
64
|
+
},
|
|
65
|
+
type: "artifact.updated",
|
|
66
|
+
artifact: {
|
|
67
|
+
kind: "json",
|
|
68
|
+
artifactId: "result",
|
|
69
|
+
snapshotId: "018f3c5e-0000-7000-8000-000000000003",
|
|
70
|
+
change: "partial",
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
const get = jest.fn().mockResolvedValue({
|
|
74
|
+
status: 200,
|
|
75
|
+
data: {
|
|
76
|
+
success: true,
|
|
77
|
+
id: "agent-job",
|
|
78
|
+
events: [runStarted, artifactUpdated],
|
|
79
|
+
creditsUsed: 5,
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const trace = await getAgentTrace({ get } as any, "agent-job");
|
|
84
|
+
|
|
85
|
+
expect(get).toHaveBeenCalledWith("/v2/agent/agent-job/trace");
|
|
86
|
+
expect(trace.events).toHaveLength(2);
|
|
87
|
+
expect(trace.events![0].type).toBe("run.started");
|
|
88
|
+
expect(trace.events![1].type).toBe("artifact.updated");
|
|
89
|
+
if (trace.events![1].type === "artifact.updated") {
|
|
90
|
+
expect(trace.events![1].artifact.snapshotId).toBe(
|
|
91
|
+
"018f3c5e-0000-7000-8000-000000000003",
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
expect(trace.creditsUsed).toBe(5);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("getAgentTrace appends liveView query param when requested", async () => {
|
|
98
|
+
const get = jest.fn().mockResolvedValue({
|
|
99
|
+
status: 200,
|
|
100
|
+
data: {
|
|
101
|
+
success: true,
|
|
102
|
+
id: "agent-job",
|
|
103
|
+
events: [],
|
|
104
|
+
creditsUsed: 0,
|
|
105
|
+
activeBrowserSessions: [
|
|
106
|
+
{
|
|
107
|
+
id: "sess-1",
|
|
108
|
+
liveViewUrl: "https://browser.example.com/sess-1",
|
|
109
|
+
viewport: { width: 1280, height: 720 },
|
|
110
|
+
},
|
|
111
|
+
],
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
const trace = await getAgentTrace({ get } as any, "agent-job", {
|
|
116
|
+
liveView: true,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
expect(get).toHaveBeenCalledWith("/v2/agent/agent-job/trace?liveView=true");
|
|
120
|
+
expect(trace.activeBrowserSessions).toHaveLength(1);
|
|
121
|
+
expect(trace.activeBrowserSessions![0].viewport.width).toBe(1280);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("getAgentSnapshot hits the snapshot endpoint", async () => {
|
|
125
|
+
const get = jest.fn().mockResolvedValue({
|
|
126
|
+
status: 200,
|
|
127
|
+
data: {
|
|
128
|
+
success: true,
|
|
129
|
+
id: "agent-job",
|
|
130
|
+
snapshotId: "snap-1",
|
|
131
|
+
snapshot: '{"price": 42}',
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const snapshot = await getAgentSnapshot({ get } as any, "agent-job", "snap-1");
|
|
136
|
+
|
|
137
|
+
expect(get).toHaveBeenCalledWith("/v2/agent/agent-job/snapshots/snap-1");
|
|
138
|
+
expect(snapshot.snapshot).toBe('{"price": 42}');
|
|
139
|
+
});
|
|
140
|
+
});
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { describe, expect, test } from "@jest/globals";
|
|
2
|
+
import {
|
|
3
|
+
processingContinuesDelayMs,
|
|
4
|
+
scrape,
|
|
5
|
+
} from "../../../v2/methods/scrape";
|
|
6
|
+
|
|
7
|
+
const DOC = { markdown: "# big doc" };
|
|
8
|
+
|
|
9
|
+
function processingContinues408(retryAfterSeconds = 10) {
|
|
10
|
+
return {
|
|
11
|
+
isAxiosError: true,
|
|
12
|
+
response: {
|
|
13
|
+
status: 408,
|
|
14
|
+
headers: { "retry-after": String(retryAfterSeconds) },
|
|
15
|
+
data: {
|
|
16
|
+
success: false,
|
|
17
|
+
code: "SCRAPE_TIMEOUT",
|
|
18
|
+
error: "Request timed out, but this 700-page PDF is still being processed on our side.",
|
|
19
|
+
details: {
|
|
20
|
+
state: "processing_continues",
|
|
21
|
+
documentPages: 700,
|
|
22
|
+
jobStatus: "running",
|
|
23
|
+
estimatedRemainingSeconds: retryAfterSeconds,
|
|
24
|
+
retryAfterSeconds,
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
describe("v2.scrape auto-resume", () => {
|
|
32
|
+
test("resumes after processing_continues and returns the finished document", async () => {
|
|
33
|
+
const post = jest
|
|
34
|
+
.fn()
|
|
35
|
+
.mockRejectedValueOnce(processingContinues408(10))
|
|
36
|
+
.mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
|
|
37
|
+
const sleeps: number[] = [];
|
|
38
|
+
const sleepImpl = async (ms: number) => {
|
|
39
|
+
sleeps.push(ms);
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const doc = await scrape({ post } as any, "https://example.com/big.pdf", undefined, {
|
|
43
|
+
sleepImpl,
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
expect(doc).toEqual(DOC);
|
|
47
|
+
expect(post).toHaveBeenCalledTimes(2);
|
|
48
|
+
expect(sleeps).toEqual([10_000]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("autoResume: false surfaces the timeout immediately and never sleeps", async () => {
|
|
52
|
+
const post = jest.fn().mockRejectedValueOnce(processingContinues408());
|
|
53
|
+
const sleepImpl = jest.fn();
|
|
54
|
+
|
|
55
|
+
await expect(
|
|
56
|
+
scrape({ post } as any, "https://example.com/big.pdf", { autoResume: false }, {
|
|
57
|
+
sleepImpl: sleepImpl as any,
|
|
58
|
+
}),
|
|
59
|
+
).rejects.toBeDefined();
|
|
60
|
+
expect(post).toHaveBeenCalledTimes(1);
|
|
61
|
+
expect(sleepImpl).not.toHaveBeenCalled();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("autoResume never enters the wire payload", async () => {
|
|
65
|
+
const post = jest
|
|
66
|
+
.fn()
|
|
67
|
+
.mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
|
|
68
|
+
await scrape({ post } as any, "https://example.com/x", { autoResume: true });
|
|
69
|
+
const payload = post.mock.calls[0][1];
|
|
70
|
+
expect(payload).toEqual({ url: "https://example.com/x" });
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("gives up after the attempt bound and surfaces the error", async () => {
|
|
74
|
+
const post = jest.fn().mockRejectedValue(processingContinues408(10));
|
|
75
|
+
const sleepImpl = jest.fn().mockResolvedValue(undefined);
|
|
76
|
+
|
|
77
|
+
await expect(
|
|
78
|
+
scrape({ post } as any, "https://example.com/big.pdf", undefined, {
|
|
79
|
+
sleepImpl: sleepImpl as any,
|
|
80
|
+
}),
|
|
81
|
+
).rejects.toBeDefined();
|
|
82
|
+
// 1 initial + 5 resumes.
|
|
83
|
+
expect(post).toHaveBeenCalledTimes(6);
|
|
84
|
+
expect(sleepImpl).toHaveBeenCalledTimes(5);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("plain timeouts without the processing signal do not resume", async () => {
|
|
88
|
+
const post = jest.fn().mockRejectedValueOnce({
|
|
89
|
+
isAxiosError: true,
|
|
90
|
+
response: {
|
|
91
|
+
status: 408,
|
|
92
|
+
headers: {},
|
|
93
|
+
data: { success: false, code: "SCRAPE_TIMEOUT", error: "Request timed out" },
|
|
94
|
+
},
|
|
95
|
+
});
|
|
96
|
+
const sleepImpl = jest.fn();
|
|
97
|
+
|
|
98
|
+
await expect(
|
|
99
|
+
scrape({ post } as any, "https://example.com/slow", undefined, {
|
|
100
|
+
sleepImpl: sleepImpl as any,
|
|
101
|
+
}),
|
|
102
|
+
).rejects.toBeDefined();
|
|
103
|
+
expect(post).toHaveBeenCalledTimes(1);
|
|
104
|
+
expect(sleepImpl).not.toHaveBeenCalled();
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("request timeout: opt-out keeps legacy behavior, resume adds the cushion", async () => {
|
|
108
|
+
const ok = { status: 200, data: { success: true, data: DOC } };
|
|
109
|
+
// Opt-out, no explicit timeout → no per-request override at all.
|
|
110
|
+
const post1 = jest.fn().mockResolvedValueOnce(ok);
|
|
111
|
+
await scrape({ post: post1 } as any, "https://example.com/x", { autoResume: false });
|
|
112
|
+
expect(post1.mock.calls[0][2]).toEqual({});
|
|
113
|
+
// Opt-out with explicit timeout → the pre-existing +5s.
|
|
114
|
+
const post2 = jest.fn().mockResolvedValueOnce(ok);
|
|
115
|
+
await scrape({ post: post2 } as any, "https://example.com/x", {
|
|
116
|
+
autoResume: false,
|
|
117
|
+
timeout: 10_000,
|
|
118
|
+
});
|
|
119
|
+
expect(post2.mock.calls[0][2]).toEqual({ timeoutMs: 15_000 });
|
|
120
|
+
// Resume enabled → cushion over the server wall, respecting a larger
|
|
121
|
+
// configured client default as the floor.
|
|
122
|
+
const post3 = jest.fn().mockResolvedValueOnce(ok);
|
|
123
|
+
await scrape(
|
|
124
|
+
{ post: post3, getTimeoutMs: () => 600_000 } as any,
|
|
125
|
+
"https://example.com/x",
|
|
126
|
+
);
|
|
127
|
+
expect(post3.mock.calls[0][2]).toEqual({ timeoutMs: 630_000 });
|
|
128
|
+
// Resume enabled, no client getter → server wall default + cushion.
|
|
129
|
+
const post4 = jest.fn().mockResolvedValueOnce(ok);
|
|
130
|
+
await scrape({ post: post4 } as any, "https://example.com/x", { timeout: 900_000 });
|
|
131
|
+
expect(post4.mock.calls[0][2]).toEqual({ timeoutMs: 930_000 });
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("processingContinuesDelayMs clamps and falls back sensibly", () => {
|
|
135
|
+
expect(processingContinuesDelayMs(processingContinues408(1))).toBe(5_000);
|
|
136
|
+
expect(processingContinuesDelayMs(processingContinues408(3_600))).toBe(600_000);
|
|
137
|
+
// No payload field → Retry-After header.
|
|
138
|
+
const headerOnly = processingContinues408(0);
|
|
139
|
+
delete (headerOnly.response.data.details as any).retryAfterSeconds;
|
|
140
|
+
headerOnly.response.headers["retry-after"] = "90";
|
|
141
|
+
expect(processingContinuesDelayMs(headerOnly)).toBe(90_000);
|
|
142
|
+
expect(processingContinuesDelayMs(new Error("net"))).toBeUndefined();
|
|
143
|
+
expect(processingContinuesDelayMs(undefined)).toBeUndefined();
|
|
144
|
+
});
|
|
145
|
+
});
|
package/src/v1/index.ts
CHANGED
package/src/v2/client.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { HttpClient } from "./utils/httpClient";
|
|
2
2
|
import {
|
|
3
|
-
scrape,
|
|
3
|
+
scrape, type ScrapeCallOptions,
|
|
4
4
|
interact as interactMethod,
|
|
5
5
|
stopInteraction as stopInteractionMethod,
|
|
6
6
|
} from "./methods/scrape";
|
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
batchScrape as batchWaiter,
|
|
27
27
|
} from "./methods/batch";
|
|
28
28
|
import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
|
|
29
|
-
import { startAgent, getAgentStatus, cancelAgent, agent as agentWaiter } from "./methods/agent";
|
|
29
|
+
import { startAgent, getAgentStatus, getAgentTrace, getAgentSnapshot, cancelAgent, agent as agentWaiter } from "./methods/agent";
|
|
30
30
|
import {
|
|
31
31
|
browser as browserMethod,
|
|
32
32
|
browserExecute,
|
|
@@ -68,6 +68,8 @@ import type {
|
|
|
68
68
|
ExtractResponse,
|
|
69
69
|
AgentResponse,
|
|
70
70
|
AgentStatusResponse,
|
|
71
|
+
AgentTraceResponse,
|
|
72
|
+
AgentSnapshotResponse,
|
|
71
73
|
CrawlOptions,
|
|
72
74
|
BatchScrapeOptions,
|
|
73
75
|
PaginationConfig,
|
|
@@ -168,8 +170,8 @@ export class FirecrawlClient {
|
|
|
168
170
|
url: string,
|
|
169
171
|
options: Opts
|
|
170
172
|
): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
|
|
171
|
-
async scrape(url: string, options?:
|
|
172
|
-
async scrape(url: string, options?:
|
|
173
|
+
async scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
|
|
174
|
+
async scrape(url: string, options?: ScrapeCallOptions): Promise<Document> {
|
|
173
175
|
return scrape(this.http, url, options);
|
|
174
176
|
}
|
|
175
177
|
/**
|
|
@@ -535,6 +537,22 @@ export class FirecrawlClient {
|
|
|
535
537
|
async cancelAgent(jobId: string): Promise<boolean> {
|
|
536
538
|
return cancelAgent(this.http, jobId);
|
|
537
539
|
}
|
|
540
|
+
/**
|
|
541
|
+
* Get the execution trace of an agent job (spark-2 runs only).
|
|
542
|
+
* @param jobId Agent job id.
|
|
543
|
+
* @param options.liveView Also include currently active browser sessions with live view URLs.
|
|
544
|
+
*/
|
|
545
|
+
async getAgentTrace(jobId: string, options?: { liveView?: boolean }): Promise<AgentTraceResponse> {
|
|
546
|
+
return getAgentTrace(this.http, jobId, options);
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Get the full content of an artifact snapshot referenced by a trace event.
|
|
550
|
+
* @param jobId Agent job id.
|
|
551
|
+
* @param snapshotId Snapshot id from an artifact.updated trace event.
|
|
552
|
+
*/
|
|
553
|
+
async getAgentSnapshot(jobId: string, snapshotId: string): Promise<AgentSnapshotResponse> {
|
|
554
|
+
return getAgentSnapshot(this.http, jobId, snapshotId);
|
|
555
|
+
}
|
|
538
556
|
|
|
539
557
|
// Browser
|
|
540
558
|
/**
|
package/src/v2/methods/agent.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
|
|
1
|
+
import { type AgentEffort, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentTraceResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
|
4
4
|
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
|
@@ -13,6 +13,7 @@ function prepareAgentPayload(args: {
|
|
|
13
13
|
maxCredits?: number;
|
|
14
14
|
strictConstrainToURLs?: boolean;
|
|
15
15
|
model?: "spark-1-pro" | "spark-1-mini" | "spark-2";
|
|
16
|
+
effort?: AgentEffort;
|
|
16
17
|
webhook?: string | AgentWebhookConfig;
|
|
17
18
|
threatProtection?: ThreatProtectionOptions;
|
|
18
19
|
auditMetadata?: AuditMetadata;
|
|
@@ -28,6 +29,7 @@ function prepareAgentPayload(args: {
|
|
|
28
29
|
if (args.maxCredits !== null && args.maxCredits !== undefined) body.maxCredits = args.maxCredits;
|
|
29
30
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
30
31
|
if (args.model !== null && args.model !== undefined) body.model = args.model;
|
|
32
|
+
if (args.effort !== null && args.effort !== undefined) body.effort = args.effort;
|
|
31
33
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
32
34
|
if (args.threatProtection != null)
|
|
33
35
|
body.threatProtection = args.threatProtection;
|
|
@@ -84,6 +86,37 @@ export async function agent(
|
|
|
84
86
|
return waitAgent(http, jobId, args.pollInterval ?? 2, args.timeout);
|
|
85
87
|
}
|
|
86
88
|
|
|
89
|
+
export async function getAgentTrace(
|
|
90
|
+
http: HttpClient,
|
|
91
|
+
jobId: string,
|
|
92
|
+
options?: { liveView?: boolean }
|
|
93
|
+
): Promise<AgentTraceResponse> {
|
|
94
|
+
try {
|
|
95
|
+
const query = options?.liveView ? "?liveView=true" : "";
|
|
96
|
+
const res = await http.get<AgentTraceResponse>(`/v2/agent/${jobId}/trace${query}`);
|
|
97
|
+
if (res.status !== 200) throwForBadResponse(res, "agent trace");
|
|
98
|
+
return res.data;
|
|
99
|
+
} catch (err: any) {
|
|
100
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent trace");
|
|
101
|
+
throw err;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export async function getAgentSnapshot(
|
|
106
|
+
http: HttpClient,
|
|
107
|
+
jobId: string,
|
|
108
|
+
snapshotId: string
|
|
109
|
+
): Promise<AgentSnapshotResponse> {
|
|
110
|
+
try {
|
|
111
|
+
const res = await http.get<AgentSnapshotResponse>(`/v2/agent/${jobId}/snapshots/${snapshotId}`);
|
|
112
|
+
if (res.status !== 200) throwForBadResponse(res, "agent snapshot");
|
|
113
|
+
return res.data;
|
|
114
|
+
} catch (err: any) {
|
|
115
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent snapshot");
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
87
120
|
export async function cancelAgent(http: HttpClient, jobId: string): Promise<boolean> {
|
|
88
121
|
try {
|
|
89
122
|
const res = await http.delete<{ success: boolean }>(`/v2/agent/${jobId}`);
|