firecrawl 4.34.2 → 4.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-PFIWJELS.js → chunk-LFX7WDGT.js} +1 -1
- package/dist/index.cjs +92 -15
- package/dist/index.d.cts +182 -2
- package/dist/index.d.ts +182 -2
- package/dist/index.js +93 -16
- package/dist/{package-H4IIQQQK.js → package-SHMK2LJE.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +140 -0
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +145 -0
- package/src/__tests__/unit/v2/validation.test.ts +3 -2
- package/src/v1/index.ts +1 -0
- package/src/v2/client.ts +22 -4
- package/src/v2/methods/agent.ts +34 -1
- package/src/v2/methods/scrape.ts +123 -19
- package/src/v2/types.ts +173 -0
- package/src/v2/utils/httpClient.ts +6 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { describe, expect, test } from "@jest/globals";
|
|
2
|
+
import {
|
|
3
|
+
processingContinuesDelayMs,
|
|
4
|
+
scrape,
|
|
5
|
+
} from "../../../v2/methods/scrape";
|
|
6
|
+
|
|
7
|
+
const DOC = { markdown: "# big doc" };
|
|
8
|
+
|
|
9
|
+
function processingContinues408(retryAfterSeconds = 10) {
|
|
10
|
+
return {
|
|
11
|
+
isAxiosError: true,
|
|
12
|
+
response: {
|
|
13
|
+
status: 408,
|
|
14
|
+
headers: { "retry-after": String(retryAfterSeconds) },
|
|
15
|
+
data: {
|
|
16
|
+
success: false,
|
|
17
|
+
code: "SCRAPE_TIMEOUT",
|
|
18
|
+
error: "Request timed out, but this 700-page PDF is still being processed on our side.",
|
|
19
|
+
details: {
|
|
20
|
+
state: "processing_continues",
|
|
21
|
+
documentPages: 700,
|
|
22
|
+
jobStatus: "running",
|
|
23
|
+
estimatedRemainingSeconds: retryAfterSeconds,
|
|
24
|
+
retryAfterSeconds,
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
describe("v2.scrape auto-resume", () => {
|
|
32
|
+
test("resumes after processing_continues and returns the finished document", async () => {
|
|
33
|
+
const post = jest
|
|
34
|
+
.fn()
|
|
35
|
+
.mockRejectedValueOnce(processingContinues408(10))
|
|
36
|
+
.mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
|
|
37
|
+
const sleeps: number[] = [];
|
|
38
|
+
const sleepImpl = async (ms: number) => {
|
|
39
|
+
sleeps.push(ms);
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const doc = await scrape({ post } as any, "https://example.com/big.pdf", undefined, {
|
|
43
|
+
sleepImpl,
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
expect(doc).toEqual(DOC);
|
|
47
|
+
expect(post).toHaveBeenCalledTimes(2);
|
|
48
|
+
expect(sleeps).toEqual([10_000]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("autoResume: false surfaces the timeout immediately and never sleeps", async () => {
|
|
52
|
+
const post = jest.fn().mockRejectedValueOnce(processingContinues408());
|
|
53
|
+
const sleepImpl = jest.fn();
|
|
54
|
+
|
|
55
|
+
await expect(
|
|
56
|
+
scrape({ post } as any, "https://example.com/big.pdf", { autoResume: false }, {
|
|
57
|
+
sleepImpl: sleepImpl as any,
|
|
58
|
+
}),
|
|
59
|
+
).rejects.toBeDefined();
|
|
60
|
+
expect(post).toHaveBeenCalledTimes(1);
|
|
61
|
+
expect(sleepImpl).not.toHaveBeenCalled();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("autoResume never enters the wire payload", async () => {
|
|
65
|
+
const post = jest
|
|
66
|
+
.fn()
|
|
67
|
+
.mockResolvedValueOnce({ status: 200, data: { success: true, data: DOC } });
|
|
68
|
+
await scrape({ post } as any, "https://example.com/x", { autoResume: true });
|
|
69
|
+
const payload = post.mock.calls[0][1];
|
|
70
|
+
expect(payload).toEqual({ url: "https://example.com/x" });
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("gives up after the attempt bound and surfaces the error", async () => {
|
|
74
|
+
const post = jest.fn().mockRejectedValue(processingContinues408(10));
|
|
75
|
+
const sleepImpl = jest.fn().mockResolvedValue(undefined);
|
|
76
|
+
|
|
77
|
+
await expect(
|
|
78
|
+
scrape({ post } as any, "https://example.com/big.pdf", undefined, {
|
|
79
|
+
sleepImpl: sleepImpl as any,
|
|
80
|
+
}),
|
|
81
|
+
).rejects.toBeDefined();
|
|
82
|
+
// 1 initial + 5 resumes.
|
|
83
|
+
expect(post).toHaveBeenCalledTimes(6);
|
|
84
|
+
expect(sleepImpl).toHaveBeenCalledTimes(5);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("plain timeouts without the processing signal do not resume", async () => {
|
|
88
|
+
const post = jest.fn().mockRejectedValueOnce({
|
|
89
|
+
isAxiosError: true,
|
|
90
|
+
response: {
|
|
91
|
+
status: 408,
|
|
92
|
+
headers: {},
|
|
93
|
+
data: { success: false, code: "SCRAPE_TIMEOUT", error: "Request timed out" },
|
|
94
|
+
},
|
|
95
|
+
});
|
|
96
|
+
const sleepImpl = jest.fn();
|
|
97
|
+
|
|
98
|
+
await expect(
|
|
99
|
+
scrape({ post } as any, "https://example.com/slow", undefined, {
|
|
100
|
+
sleepImpl: sleepImpl as any,
|
|
101
|
+
}),
|
|
102
|
+
).rejects.toBeDefined();
|
|
103
|
+
expect(post).toHaveBeenCalledTimes(1);
|
|
104
|
+
expect(sleepImpl).not.toHaveBeenCalled();
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("request timeout: opt-out keeps legacy behavior, resume adds the cushion", async () => {
|
|
108
|
+
const ok = { status: 200, data: { success: true, data: DOC } };
|
|
109
|
+
// Opt-out, no explicit timeout → no per-request override at all.
|
|
110
|
+
const post1 = jest.fn().mockResolvedValueOnce(ok);
|
|
111
|
+
await scrape({ post: post1 } as any, "https://example.com/x", { autoResume: false });
|
|
112
|
+
expect(post1.mock.calls[0][2]).toEqual({});
|
|
113
|
+
// Opt-out with explicit timeout → the pre-existing +5s.
|
|
114
|
+
const post2 = jest.fn().mockResolvedValueOnce(ok);
|
|
115
|
+
await scrape({ post: post2 } as any, "https://example.com/x", {
|
|
116
|
+
autoResume: false,
|
|
117
|
+
timeout: 10_000,
|
|
118
|
+
});
|
|
119
|
+
expect(post2.mock.calls[0][2]).toEqual({ timeoutMs: 15_000 });
|
|
120
|
+
// Resume enabled → cushion over the server wall, respecting a larger
|
|
121
|
+
// configured client default as the floor.
|
|
122
|
+
const post3 = jest.fn().mockResolvedValueOnce(ok);
|
|
123
|
+
await scrape(
|
|
124
|
+
{ post: post3, getTimeoutMs: () => 600_000 } as any,
|
|
125
|
+
"https://example.com/x",
|
|
126
|
+
);
|
|
127
|
+
expect(post3.mock.calls[0][2]).toEqual({ timeoutMs: 630_000 });
|
|
128
|
+
// Resume enabled, no client getter → server wall default + cushion.
|
|
129
|
+
const post4 = jest.fn().mockResolvedValueOnce(ok);
|
|
130
|
+
await scrape({ post: post4 } as any, "https://example.com/x", { timeout: 900_000 });
|
|
131
|
+
expect(post4.mock.calls[0][2]).toEqual({ timeoutMs: 930_000 });
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("processingContinuesDelayMs clamps and falls back sensibly", () => {
|
|
135
|
+
expect(processingContinuesDelayMs(processingContinues408(1))).toBe(5_000);
|
|
136
|
+
expect(processingContinuesDelayMs(processingContinues408(3_600))).toBe(600_000);
|
|
137
|
+
// No payload field → Retry-After header.
|
|
138
|
+
const headerOnly = processingContinues408(0);
|
|
139
|
+
delete (headerOnly.response.data.details as any).retryAfterSeconds;
|
|
140
|
+
headerOnly.response.headers["retry-after"] = "90";
|
|
141
|
+
expect(processingContinuesDelayMs(headerOnly)).toBe(90_000);
|
|
142
|
+
expect(processingContinuesDelayMs(new Error("net"))).toBeUndefined();
|
|
143
|
+
expect(processingContinuesDelayMs(undefined)).toBeUndefined();
|
|
144
|
+
});
|
|
145
|
+
});
|
|
@@ -89,9 +89,9 @@ describe("v2 utils: validation", () => {
|
|
|
89
89
|
expect(options.parsers).toEqual(before);
|
|
90
90
|
});
|
|
91
91
|
|
|
92
|
-
test("ensureValidScrapeOptions: leaves PDF parser pages and
|
|
92
|
+
test("ensureValidScrapeOptions: leaves PDF parser pages, blocks, and pageMarkers options untouched", () => {
|
|
93
93
|
const options = {
|
|
94
|
-
parsers: [{ type: "pdf", mode: "auto", blocks: true, pages: true }],
|
|
94
|
+
parsers: [{ type: "pdf", mode: "auto", blocks: true, pages: true, pageMarkers: true }],
|
|
95
95
|
} as any;
|
|
96
96
|
expect(() => ensureValidScrapeOptions(options)).not.toThrow();
|
|
97
97
|
expect(options.parsers[0]).toEqual({
|
|
@@ -99,6 +99,7 @@ describe("v2 utils: validation", () => {
|
|
|
99
99
|
mode: "auto",
|
|
100
100
|
blocks: true,
|
|
101
101
|
pages: true,
|
|
102
|
+
pageMarkers: true,
|
|
102
103
|
});
|
|
103
104
|
});
|
|
104
105
|
|
package/src/v1/index.ts
CHANGED
package/src/v2/client.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { HttpClient } from "./utils/httpClient";
|
|
2
2
|
import {
|
|
3
|
-
scrape,
|
|
3
|
+
scrape, type ScrapeCallOptions,
|
|
4
4
|
interact as interactMethod,
|
|
5
5
|
stopInteraction as stopInteractionMethod,
|
|
6
6
|
} from "./methods/scrape";
|
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
batchScrape as batchWaiter,
|
|
27
27
|
} from "./methods/batch";
|
|
28
28
|
import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
|
|
29
|
-
import { startAgent, getAgentStatus, cancelAgent, agent as agentWaiter } from "./methods/agent";
|
|
29
|
+
import { startAgent, getAgentStatus, getAgentTrace, getAgentSnapshot, cancelAgent, agent as agentWaiter } from "./methods/agent";
|
|
30
30
|
import {
|
|
31
31
|
browser as browserMethod,
|
|
32
32
|
browserExecute,
|
|
@@ -68,6 +68,8 @@ import type {
|
|
|
68
68
|
ExtractResponse,
|
|
69
69
|
AgentResponse,
|
|
70
70
|
AgentStatusResponse,
|
|
71
|
+
AgentTraceResponse,
|
|
72
|
+
AgentSnapshotResponse,
|
|
71
73
|
CrawlOptions,
|
|
72
74
|
BatchScrapeOptions,
|
|
73
75
|
PaginationConfig,
|
|
@@ -168,8 +170,8 @@ export class FirecrawlClient {
|
|
|
168
170
|
url: string,
|
|
169
171
|
options: Opts
|
|
170
172
|
): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
|
|
171
|
-
async scrape(url: string, options?:
|
|
172
|
-
async scrape(url: string, options?:
|
|
173
|
+
async scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
|
|
174
|
+
async scrape(url: string, options?: ScrapeCallOptions): Promise<Document> {
|
|
173
175
|
return scrape(this.http, url, options);
|
|
174
176
|
}
|
|
175
177
|
/**
|
|
@@ -535,6 +537,22 @@ export class FirecrawlClient {
|
|
|
535
537
|
async cancelAgent(jobId: string): Promise<boolean> {
|
|
536
538
|
return cancelAgent(this.http, jobId);
|
|
537
539
|
}
|
|
540
|
+
/**
|
|
541
|
+
* Get the execution trace of an agent job (spark-2 runs only).
|
|
542
|
+
* @param jobId Agent job id.
|
|
543
|
+
* @param options.liveView Also include currently active browser sessions with live view URLs.
|
|
544
|
+
*/
|
|
545
|
+
async getAgentTrace(jobId: string, options?: { liveView?: boolean }): Promise<AgentTraceResponse> {
|
|
546
|
+
return getAgentTrace(this.http, jobId, options);
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Get the full content of an artifact snapshot referenced by a trace event.
|
|
550
|
+
* @param jobId Agent job id.
|
|
551
|
+
* @param snapshotId Snapshot id from an artifact.updated trace event.
|
|
552
|
+
*/
|
|
553
|
+
async getAgentSnapshot(jobId: string, snapshotId: string): Promise<AgentSnapshotResponse> {
|
|
554
|
+
return getAgentSnapshot(this.http, jobId, snapshotId);
|
|
555
|
+
}
|
|
538
556
|
|
|
539
557
|
// Browser
|
|
540
558
|
/**
|
package/src/v2/methods/agent.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
|
|
1
|
+
import { type AgentEffort, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentTraceResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
|
4
4
|
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
|
@@ -13,6 +13,7 @@ function prepareAgentPayload(args: {
|
|
|
13
13
|
maxCredits?: number;
|
|
14
14
|
strictConstrainToURLs?: boolean;
|
|
15
15
|
model?: "spark-1-pro" | "spark-1-mini" | "spark-2";
|
|
16
|
+
effort?: AgentEffort;
|
|
16
17
|
webhook?: string | AgentWebhookConfig;
|
|
17
18
|
threatProtection?: ThreatProtectionOptions;
|
|
18
19
|
auditMetadata?: AuditMetadata;
|
|
@@ -28,6 +29,7 @@ function prepareAgentPayload(args: {
|
|
|
28
29
|
if (args.maxCredits !== null && args.maxCredits !== undefined) body.maxCredits = args.maxCredits;
|
|
29
30
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
30
31
|
if (args.model !== null && args.model !== undefined) body.model = args.model;
|
|
32
|
+
if (args.effort !== null && args.effort !== undefined) body.effort = args.effort;
|
|
31
33
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
32
34
|
if (args.threatProtection != null)
|
|
33
35
|
body.threatProtection = args.threatProtection;
|
|
@@ -84,6 +86,37 @@ export async function agent(
|
|
|
84
86
|
return waitAgent(http, jobId, args.pollInterval ?? 2, args.timeout);
|
|
85
87
|
}
|
|
86
88
|
|
|
89
|
+
export async function getAgentTrace(
|
|
90
|
+
http: HttpClient,
|
|
91
|
+
jobId: string,
|
|
92
|
+
options?: { liveView?: boolean }
|
|
93
|
+
): Promise<AgentTraceResponse> {
|
|
94
|
+
try {
|
|
95
|
+
const query = options?.liveView ? "?liveView=true" : "";
|
|
96
|
+
const res = await http.get<AgentTraceResponse>(`/v2/agent/${jobId}/trace${query}`);
|
|
97
|
+
if (res.status !== 200) throwForBadResponse(res, "agent trace");
|
|
98
|
+
return res.data;
|
|
99
|
+
} catch (err: any) {
|
|
100
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent trace");
|
|
101
|
+
throw err;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export async function getAgentSnapshot(
|
|
106
|
+
http: HttpClient,
|
|
107
|
+
jobId: string,
|
|
108
|
+
snapshotId: string
|
|
109
|
+
): Promise<AgentSnapshotResponse> {
|
|
110
|
+
try {
|
|
111
|
+
const res = await http.get<AgentSnapshotResponse>(`/v2/agent/${jobId}/snapshots/${snapshotId}`);
|
|
112
|
+
if (res.status !== 200) throwForBadResponse(res, "agent snapshot");
|
|
113
|
+
return res.data;
|
|
114
|
+
} catch (err: any) {
|
|
115
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent snapshot");
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
87
120
|
export async function cancelAgent(http: HttpClient, jobId: string): Promise<boolean> {
|
|
88
121
|
try {
|
|
89
122
|
const res = await http.delete<{ success: boolean }>(`/v2/agent/${jobId}`);
|
package/src/v2/methods/scrape.ts
CHANGED
|
@@ -12,38 +12,142 @@ import {
|
|
|
12
12
|
normalizeAxiosError,
|
|
13
13
|
} from "../utils/errorHandler";
|
|
14
14
|
|
|
15
|
+
/** Auto-resume bounds: a resume only ever follows the server's explicit
|
|
16
|
+
* "processing continues" signal, and stops after this many resumes or
|
|
17
|
+
* this much total sleeping — whichever comes first. */
|
|
18
|
+
const RESUME_MAX_ATTEMPTS = 5;
|
|
19
|
+
const RESUME_MAX_TOTAL_WAIT_MS = 20 * 60 * 1000;
|
|
20
|
+
const RESUME_MIN_DELAY_MS = 5 * 1000;
|
|
21
|
+
const RESUME_MAX_DELAY_MS = 10 * 60 * 1000;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* When a request failed because the document is still processing
|
|
25
|
+
* server-side (large PDFs outlive their request window by design), the
|
|
26
|
+
* error carries `details.state === "processing_continues"` plus a retry
|
|
27
|
+
* delay. Returns that delay in ms, or undefined for every other error.
|
|
28
|
+
*/
|
|
29
|
+
export function processingContinuesDelayMs(err: unknown): number | undefined {
|
|
30
|
+
const response = (err as { isAxiosError?: boolean; response?: any })
|
|
31
|
+
?.isAxiosError
|
|
32
|
+
? (err as { response?: any }).response
|
|
33
|
+
: undefined;
|
|
34
|
+
if (!response || response.status !== 408) return undefined;
|
|
35
|
+
const body = response.data as
|
|
36
|
+
| { code?: string; details?: { state?: string; retryAfterSeconds?: number } }
|
|
37
|
+
| undefined;
|
|
38
|
+
if (
|
|
39
|
+
body?.code !== "SCRAPE_TIMEOUT" ||
|
|
40
|
+
body?.details?.state !== "processing_continues"
|
|
41
|
+
) {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
const headerSeconds = Number(response.headers?.["retry-after"]);
|
|
45
|
+
const seconds =
|
|
46
|
+
typeof body.details.retryAfterSeconds === "number" &&
|
|
47
|
+
Number.isFinite(body.details.retryAfterSeconds)
|
|
48
|
+
? body.details.retryAfterSeconds
|
|
49
|
+
: Number.isFinite(headerSeconds)
|
|
50
|
+
? headerSeconds
|
|
51
|
+
: 60;
|
|
52
|
+
return Math.min(RESUME_MAX_DELAY_MS, Math.max(RESUME_MIN_DELAY_MS, seconds * 1000));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Options for a direct scrape call. `autoResume` lives here — NOT on the
|
|
57
|
+
* shared ScrapeOptions — because that type is reused verbatim inside
|
|
58
|
+
* batch/crawl/search/extract payloads, where an SDK-only key would leak
|
|
59
|
+
* onto the wire and fail strict endpoints.
|
|
60
|
+
*/
|
|
61
|
+
export type ScrapeCallOptions = ScrapeOptions & {
|
|
62
|
+
/**
|
|
63
|
+
* SDK-only (never sent to the API). Large documents (big PDFs) that
|
|
64
|
+
* outlive the request window keep processing server-side; the API's
|
|
65
|
+
* timeout error then carries `details.state === "processing_continues"`
|
|
66
|
+
* and a Retry-After. When this is not `false`, the SDK sleeps that
|
|
67
|
+
* long and re-issues the same request — the retry attaches to the
|
|
68
|
+
* in-flight job and returns the finished result, so a large document
|
|
69
|
+
* behaves like one slow successful call. Bounded (at most 5 resumes /
|
|
70
|
+
* 20 minutes total wait); set `false` to surface the timeout error
|
|
71
|
+
* immediately instead.
|
|
72
|
+
*/
|
|
73
|
+
autoResume?: boolean;
|
|
74
|
+
};
|
|
75
|
+
|
|
15
76
|
export async function scrape(
|
|
16
77
|
http: HttpClient,
|
|
17
78
|
url: string,
|
|
18
|
-
options?:
|
|
79
|
+
options?: ScrapeCallOptions,
|
|
80
|
+
deps: { sleepImpl?: (ms: number) => Promise<void> } = {},
|
|
19
81
|
): Promise<Document> {
|
|
82
|
+
const sleep =
|
|
83
|
+
deps.sleepImpl ??
|
|
84
|
+
((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)));
|
|
20
85
|
if (!url || !url.trim()) {
|
|
21
86
|
throw new Error("URL cannot be empty");
|
|
22
87
|
}
|
|
23
88
|
if (options) ensureValidScrapeOptions(options);
|
|
24
89
|
|
|
90
|
+
// autoResume is SDK behavior, never part of the wire payload.
|
|
91
|
+
const { autoResume, ...requestOptions } = options ?? {};
|
|
25
92
|
const payload: Record<string, unknown> = { url: url.trim() };
|
|
26
|
-
|
|
93
|
+
Object.assign(payload, requestOptions);
|
|
27
94
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
95
|
+
// Per-request timeout. With auto-resume disabled this is exactly the
|
|
96
|
+
// pre-existing behavior (explicit timeout +5s, else the client's own
|
|
97
|
+
// default). With auto-resume on, the client-side timeout must outlast
|
|
98
|
+
// the SERVER's wall — options.timeout, or the API's 5-minute default —
|
|
99
|
+
// otherwise axios can abort in a photo-finish with the server's
|
|
100
|
+
// processing_continues response and the resume signal is lost exactly
|
|
101
|
+
// when it matters. The caller's configured client timeout is
|
|
102
|
+
// respected as a floor, never overridden downward.
|
|
103
|
+
const requestTimeout: { timeoutMs?: number } =
|
|
104
|
+
autoResume === false
|
|
105
|
+
? typeof options?.timeout === "number"
|
|
37
106
|
? { timeoutMs: options.timeout + 5000 }
|
|
38
|
-
: {}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
107
|
+
: {}
|
|
108
|
+
: {
|
|
109
|
+
timeoutMs:
|
|
110
|
+
Math.max(
|
|
111
|
+
typeof options?.timeout === "number" ? options.timeout : 300_000,
|
|
112
|
+
typeof http.getTimeoutMs === "function" ? http.getTimeoutMs() : 0,
|
|
113
|
+
) + 30_000,
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
let resumes = 0;
|
|
117
|
+
let resumeWaitedMs = 0;
|
|
118
|
+
while (true) {
|
|
119
|
+
try {
|
|
120
|
+
const res = await http.post<{
|
|
121
|
+
success: boolean;
|
|
122
|
+
data?: Document;
|
|
123
|
+
error?: string;
|
|
124
|
+
}>(
|
|
125
|
+
"/v2/scrape",
|
|
126
|
+
payload,
|
|
127
|
+
requestTimeout,
|
|
128
|
+
);
|
|
129
|
+
if (res.status !== 200 || !res.data?.success) {
|
|
130
|
+
throwForBadResponse(res, "scrape");
|
|
131
|
+
}
|
|
132
|
+
return (res.data.data || {}) as Document;
|
|
133
|
+
} catch (err: any) {
|
|
134
|
+
const delayMs = autoResume !== false ? processingContinuesDelayMs(err) : undefined;
|
|
135
|
+
if (
|
|
136
|
+
delayMs !== undefined &&
|
|
137
|
+
resumes < RESUME_MAX_ATTEMPTS &&
|
|
138
|
+
resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS
|
|
139
|
+
) {
|
|
140
|
+
// The document keeps processing server-side; the retry attaches
|
|
141
|
+
// to the same in-flight job (content adoption) and returns the
|
|
142
|
+
// finished result instead of restarting the work.
|
|
143
|
+
resumes += 1;
|
|
144
|
+
resumeWaitedMs += delayMs;
|
|
145
|
+
await sleep(delayMs);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
149
|
+
throw err;
|
|
42
150
|
}
|
|
43
|
-
return (res.data.data || {}) as Document;
|
|
44
|
-
} catch (err: any) {
|
|
45
|
-
if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
|
|
46
|
-
throw err;
|
|
47
151
|
}
|
|
48
152
|
}
|
|
49
153
|
|
package/src/v2/types.ts
CHANGED
|
@@ -31,6 +31,7 @@ export interface JsonFormat extends Format {
|
|
|
31
31
|
type: "json";
|
|
32
32
|
prompt?: string;
|
|
33
33
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
34
|
+
checkPromptInjection?: boolean;
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
export interface ScreenshotFormat {
|
|
@@ -199,6 +200,14 @@ export type PDFParser = {
|
|
|
199
200
|
pageMarkdown?: boolean;
|
|
200
201
|
/** Include per-page typed layout blocks (bounding boxes, block types, reading order). */
|
|
201
202
|
blocks?: boolean;
|
|
203
|
+
/**
|
|
204
|
+
* Join PDF pages in `document.markdown` with `\n\n---\n\n<!-- page N -->\n\n`,
|
|
205
|
+
* where N is the 1-based physical page of the content that follows. Markers
|
|
206
|
+
* appear between pages only (no leading marker for page 1), and numbering may
|
|
207
|
+
* skip pages merged by cross-page stitching — use `pages: true` when every
|
|
208
|
+
* physical page is needed. No new response field.
|
|
209
|
+
*/
|
|
210
|
+
pageMarkers?: boolean;
|
|
202
211
|
};
|
|
203
212
|
|
|
204
213
|
export interface PdfPage {
|
|
@@ -1319,10 +1328,174 @@ export interface AgentStatusResponse {
|
|
|
1319
1328
|
* names makes every future model a type error at the call site.
|
|
1320
1329
|
*/
|
|
1321
1330
|
model?: "spark-1-pro" | "spark-1-mini" | "spark-2" | (string & {});
|
|
1331
|
+
/**
|
|
1332
|
+
* Reasoning effort the job ran with. Only set for runs that specified it;
|
|
1333
|
+
* older rows and default runs have no effort.
|
|
1334
|
+
*/
|
|
1335
|
+
effort?: "low" | "medium" | "high";
|
|
1322
1336
|
expiresAt: string;
|
|
1323
1337
|
creditsUsed?: number;
|
|
1324
1338
|
}
|
|
1325
1339
|
|
|
1340
|
+
/** Reasoning effort for agent jobs. Every level runs spark-2. */
|
|
1341
|
+
export type AgentEffort = "low" | "medium" | "high";
|
|
1342
|
+
|
|
1343
|
+
export type AgentTraceAgentRole = "orchestrator" | "subagent" | "browser" | "system";
|
|
1344
|
+
|
|
1345
|
+
export interface AgentTraceAgentIdentity {
|
|
1346
|
+
id: string;
|
|
1347
|
+
role: AgentTraceAgentRole;
|
|
1348
|
+
name: string;
|
|
1349
|
+
parentId?: string;
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
export interface AgentTraceError {
|
|
1353
|
+
code: "cancelled" | "credit_limit_reached" | "parent_finished" | "refused" | "internal";
|
|
1354
|
+
source: "agent" | "tool" | "billing" | "system";
|
|
1355
|
+
retryable: boolean;
|
|
1356
|
+
message: string;
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
export interface AgentTraceArtifactChange {
|
|
1360
|
+
kind: "json" | "markdown" | "html" | "screenshot" | "text";
|
|
1361
|
+
artifactId: string;
|
|
1362
|
+
path?: string;
|
|
1363
|
+
/** Fetch the full content via getAgentSnapshot(jobId, snapshotId). */
|
|
1364
|
+
snapshotId: string;
|
|
1365
|
+
change: "init" | "partial" | "append" | "modify" | "update";
|
|
1366
|
+
changedFields?: string[];
|
|
1367
|
+
itemCount?: number;
|
|
1368
|
+
sourceToolCallId?: string;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
interface AgentTraceEventBase {
|
|
1372
|
+
schemaVersion: 1;
|
|
1373
|
+
eventId: string;
|
|
1374
|
+
runId: string;
|
|
1375
|
+
occurredAt: string;
|
|
1376
|
+
producerSequence: number;
|
|
1377
|
+
agent: AgentTraceAgentIdentity;
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
export interface AgentTraceRunStartedEvent extends AgentTraceEventBase {
|
|
1381
|
+
type: "run.started";
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
export interface AgentTraceRunCancelRequestedEvent extends AgentTraceEventBase {
|
|
1385
|
+
type: "run.cancel_requested";
|
|
1386
|
+
reason: "user";
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
export interface AgentTraceRunFinishedEvent extends AgentTraceEventBase {
|
|
1390
|
+
type: "run.finished";
|
|
1391
|
+
outcome: "succeeded" | "failed" | "cancelled" | "refused" | "credit_limit_reached";
|
|
1392
|
+
/** The canonical schema always writes this key (nullable), but older rows may omit it. */
|
|
1393
|
+
error?: AgentTraceError | null;
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
export interface AgentTraceAgentStartedEvent extends AgentTraceEventBase {
|
|
1397
|
+
type: "agent.started";
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
export interface AgentTraceAgentFinishedEvent extends AgentTraceEventBase {
|
|
1401
|
+
type: "agent.finished";
|
|
1402
|
+
outcome: "succeeded" | "failed" | "cancelled" | "refused";
|
|
1403
|
+
durationMs: number;
|
|
1404
|
+
/** The canonical schema always writes this key (nullable), but older rows may omit it. */
|
|
1405
|
+
error?: AgentTraceError | null;
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
export interface AgentTraceBrowserSessionStartedEvent extends AgentTraceEventBase {
|
|
1409
|
+
type: "browser.session.started";
|
|
1410
|
+
sessionId: string;
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
export interface AgentTraceBrowserSessionFinishedEvent extends AgentTraceEventBase {
|
|
1414
|
+
type: "browser.session.finished";
|
|
1415
|
+
sessionId: string;
|
|
1416
|
+
durationMs: number;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
export interface AgentTraceProgressReportedEvent extends AgentTraceEventBase {
|
|
1420
|
+
type: "progress.reported";
|
|
1421
|
+
phase: "planning" | "working" | "finalizing";
|
|
1422
|
+
message: string;
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
export interface AgentTraceReasoningSummaryEvent extends AgentTraceEventBase {
|
|
1426
|
+
type: "reasoning.summary";
|
|
1427
|
+
text: string;
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
export interface AgentTraceToolCallStartedEvent extends AgentTraceEventBase {
|
|
1431
|
+
type: "tool_call.started";
|
|
1432
|
+
toolCallId: string;
|
|
1433
|
+
toolName: string;
|
|
1434
|
+
parameters: unknown;
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
export interface AgentTraceToolCallFinishedEvent extends AgentTraceEventBase {
|
|
1438
|
+
type: "tool_call.finished";
|
|
1439
|
+
toolCallId: string;
|
|
1440
|
+
toolName: string;
|
|
1441
|
+
result: unknown;
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
export interface AgentTraceArtifactUpdatedEvent extends AgentTraceEventBase {
|
|
1445
|
+
type: "artifact.updated";
|
|
1446
|
+
artifact: AgentTraceArtifactChange;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
export interface AgentTraceErrorOccurredEvent extends AgentTraceEventBase {
|
|
1450
|
+
type: "error.occurred";
|
|
1451
|
+
error: AgentTraceError;
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
/**
|
|
1455
|
+
* One event in an agent job's trace. Mirrors the canonical event schema of
|
|
1456
|
+
* the agent service (schemaVersion 1); `usage.recorded` events are withheld
|
|
1457
|
+
* server-side and `agent.started` carries no model name.
|
|
1458
|
+
*/
|
|
1459
|
+
export type AgentTraceEvent =
|
|
1460
|
+
| AgentTraceRunStartedEvent
|
|
1461
|
+
| AgentTraceRunCancelRequestedEvent
|
|
1462
|
+
| AgentTraceRunFinishedEvent
|
|
1463
|
+
| AgentTraceAgentStartedEvent
|
|
1464
|
+
| AgentTraceAgentFinishedEvent
|
|
1465
|
+
| AgentTraceBrowserSessionStartedEvent
|
|
1466
|
+
| AgentTraceBrowserSessionFinishedEvent
|
|
1467
|
+
| AgentTraceProgressReportedEvent
|
|
1468
|
+
| AgentTraceReasoningSummaryEvent
|
|
1469
|
+
| AgentTraceToolCallStartedEvent
|
|
1470
|
+
| AgentTraceToolCallFinishedEvent
|
|
1471
|
+
| AgentTraceArtifactUpdatedEvent
|
|
1472
|
+
| AgentTraceErrorOccurredEvent;
|
|
1473
|
+
|
|
1474
|
+
export interface AgentTraceActiveBrowserSession {
|
|
1475
|
+
id: string;
|
|
1476
|
+
liveViewUrl: string;
|
|
1477
|
+
viewport: { width: number; height: number };
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
export interface AgentTraceResponse {
|
|
1481
|
+
success: boolean;
|
|
1482
|
+
id?: string;
|
|
1483
|
+
events?: AgentTraceEvent[];
|
|
1484
|
+
creditsUsed?: number;
|
|
1485
|
+
/** Present only when the trace was requested with liveView: true. */
|
|
1486
|
+
activeBrowserSessions?: AgentTraceActiveBrowserSession[];
|
|
1487
|
+
error?: string;
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
export interface AgentSnapshotResponse {
|
|
1491
|
+
success: boolean;
|
|
1492
|
+
id?: string;
|
|
1493
|
+
snapshotId?: string;
|
|
1494
|
+
/** Full artifact content as a JSON/string blob. */
|
|
1495
|
+
snapshot?: string;
|
|
1496
|
+
error?: string;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1326
1499
|
export interface AgentOptions {
|
|
1327
1500
|
model: "FIRE-1" | "v3-beta";
|
|
1328
1501
|
}
|
|
@@ -51,6 +51,12 @@ export class HttpClient {
|
|
|
51
51
|
return this.apiKey;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
/** The client-wide default request timeout, for callers that need to
|
|
55
|
+
* extend a specific request beyond it (e.g. scrape auto-resume). */
|
|
56
|
+
getTimeoutMs(): number {
|
|
57
|
+
return (this.instance.defaults.timeout as number | undefined) ?? 300000;
|
|
58
|
+
}
|
|
59
|
+
|
|
54
60
|
private async request<T = any>(
|
|
55
61
|
config: AxiosRequestConfig,
|
|
56
62
|
): Promise<AxiosResponse<T>> {
|