firecrawl 4.47.0 → 4.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/{chunk-H75FJ6LW.js → chunk-JNLB66PW.js} +1 -1
- package/dist/index.cjs +158 -46
- package/dist/index.d.cts +49 -23
- package/dist/index.d.ts +49 -23
- package/dist/index.js +159 -47
- package/dist/{package-NZI5TDUV.js → package-7E5DZETG.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/__tests__/unit/v2/v1-aliases.test.ts +89 -4
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +41 -31
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/utils/httpClient.ts +10 -3
- package/src/v2/utils/v1Compat.ts +38 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, test, expect } from "vitest";
|
|
2
|
+
import type { AxiosAdapter } from "axios";
|
|
3
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
4
|
+
import { Firecrawl } from "../../../index";
|
|
5
|
+
import { getVersion } from "../../../v2/utils/getVersion";
|
|
6
|
+
import { detectCallerPackage } from "../../../utils/callerOrigin";
|
|
7
|
+
|
|
8
|
+
function withCapture<T extends { http: any }>(client: T) {
|
|
9
|
+
const bodies: Array<Record<string, unknown>> = [];
|
|
10
|
+
const adapter: AxiosAdapter = async config => {
|
|
11
|
+
bodies.push(JSON.parse(config.data));
|
|
12
|
+
return {
|
|
13
|
+
data: { success: true, data: { markdown: "# hello" } },
|
|
14
|
+
status: 200,
|
|
15
|
+
statusText: "OK",
|
|
16
|
+
headers: {},
|
|
17
|
+
config,
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
client.http.instance.defaults.adapter = adapter;
|
|
21
|
+
return { client, bodies };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe("v2 request origin", () => {
|
|
25
|
+
const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
if (previousOptOut === undefined) {
|
|
28
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
29
|
+
} else {
|
|
30
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("defaults to the plain SDK origin when application code creates the client", async () => {
|
|
35
|
+
const { client, bodies } = withCapture(new FirecrawlClient({ apiKey: "fc-test" }));
|
|
36
|
+
await client.scrape("https://example.com");
|
|
37
|
+
expect(bodies[0]!.origin).toBe(`js-sdk@${getVersion()}`);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("the origin client option is stamped into request bodies", async () => {
|
|
41
|
+
const { client, bodies } = withCapture(
|
|
42
|
+
new FirecrawlClient({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" }),
|
|
43
|
+
);
|
|
44
|
+
await client.scrape("https://example.com");
|
|
45
|
+
expect(bodies[0]!.origin).toBe("js-sdk.example@1.0.0");
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("a per-call origin wins over the client origin", async () => {
|
|
49
|
+
const { client, bodies } = withCapture(
|
|
50
|
+
new FirecrawlClient({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" }),
|
|
51
|
+
);
|
|
52
|
+
await client.scrape("https://example.com", { origin: "per-call" } as any);
|
|
53
|
+
await client.scrape("https://example.com", { origin: "mcp-server" } as any);
|
|
54
|
+
expect(bodies.map(b => b.origin)).toEqual(["per-call", "mcp-server"]);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("the unified client forwards the origin option to v1", () => {
|
|
58
|
+
const app = new Firecrawl({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" });
|
|
59
|
+
expect((app.v1 as any).clientOrigin).toBe("js-sdk.example@1.0.0");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe("detectCallerPackage", () => {
|
|
65
|
+
const sdkFrames = [
|
|
66
|
+
"Error",
|
|
67
|
+
" at detectCallerPackage (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:10:5)",
|
|
68
|
+
" at new FirecrawlClient (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:20:9)",
|
|
69
|
+
];
|
|
70
|
+
const stackFrom = (...callers: string[]) =>
|
|
71
|
+
[...sdkFrames, ...callers.map(c => ` at ${c}`)].join("\n");
|
|
72
|
+
const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
73
|
+
|
|
74
|
+
beforeEach(() => {
|
|
75
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
afterEach(() => {
|
|
79
|
+
if (previousOptOut === undefined) {
|
|
80
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
81
|
+
} else {
|
|
82
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("names the scoped package that created the client", () => {
|
|
87
|
+
const stack = stackFrom(
|
|
88
|
+
"make (file:///srv/app/node_modules/@Scope/Some_Agent/dist/index.js:3:12)",
|
|
89
|
+
"main (file:///srv/app/src/main.js:1:1)",
|
|
90
|
+
);
|
|
91
|
+
expect(detectCallerPackage(stack)).toBe("scope-some-agent");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("uses the last node_modules segment of a nested install", () => {
|
|
95
|
+
const stack = stackFrom(
|
|
96
|
+
"make (/srv/app/node_modules/.pnpm/outer@1.0.0/node_modules/inner-pkg/lib/index.cjs:3:12)",
|
|
97
|
+
);
|
|
98
|
+
expect(detectCallerPackage(stack)).toBe("inner-pkg");
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("handles windows paths with parentheses and anonymous frames", () => {
|
|
102
|
+
const stack = stackFrom(
|
|
103
|
+
"Array.map (<anonymous>)",
|
|
104
|
+
"C:\\Program Files (x86)\\app\\node_modules\\wrapper\\index.js:4:2",
|
|
105
|
+
);
|
|
106
|
+
expect(detectCallerPackage(stack)).toBe("wrapper");
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("returns null when application code created the client", () => {
|
|
110
|
+
const stack = stackFrom(
|
|
111
|
+
"main (file:///srv/app/src/main.js:1:1)",
|
|
112
|
+
"make (file:///srv/app/node_modules/wrapper/index.js:3:12)",
|
|
113
|
+
);
|
|
114
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("returns null for a package whose slug would read as another traffic class", () => {
|
|
118
|
+
const stack = stackFrom("make (/srv/app/node_modules/my-playground-kit/index.js:3:12)");
|
|
119
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("returns null for a bundle without node_modules in its path", () => {
|
|
123
|
+
expect(detectCallerPackage(stackFrom("make (/srv/app/dist/bundle.js:1:1)"))).toBeNull();
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("FIRECRAWL_DISABLE_CALLER_ORIGIN turns detection off", () => {
|
|
127
|
+
const stack = stackFrom("make (/srv/app/node_modules/wrapper/index.js:3:12)");
|
|
128
|
+
expect(detectCallerPackage(stack)).toBe("wrapper");
|
|
129
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = "1";
|
|
130
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
131
|
+
});
|
|
132
|
+
});
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { describe, expect, it, vi } from "vitest";
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
import type { AxiosAdapter } from "axios";
|
|
2
3
|
import { FirecrawlClient } from "../../../v2/client";
|
|
3
4
|
|
|
4
5
|
const ALIAS_MAP: Record<string, string> = {
|
|
@@ -20,14 +21,98 @@ describe("V1 deprecated aliases", () => {
|
|
|
20
21
|
apiUrl: "http://localhost:9",
|
|
21
22
|
});
|
|
22
23
|
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
vi.restoreAllMocks();
|
|
26
|
+
});
|
|
27
|
+
|
|
23
28
|
for (const [alias, target] of Object.entries(ALIAS_MAP)) {
|
|
24
|
-
it(`${alias} delegates to ${target}`, async () => {
|
|
29
|
+
it(`${alias} delegates to ${target} and adds success: true`, async () => {
|
|
25
30
|
const spy = vi
|
|
26
31
|
.spyOn(app, target as any)
|
|
27
32
|
.mockResolvedValue({ ok: true } as any);
|
|
28
|
-
await (app as any)[alias]("https://example.com");
|
|
33
|
+
const result = await (app as any)[alias]("https://example.com");
|
|
29
34
|
expect(spy).toHaveBeenCalled();
|
|
30
|
-
|
|
35
|
+
expect(result).toEqual({ ok: true, success: true });
|
|
31
36
|
});
|
|
32
37
|
}
|
|
38
|
+
|
|
39
|
+
it("scrapeUrl keeps the document fields beside success, as v1 did", async () => {
|
|
40
|
+
vi.spyOn(app, "scrape").mockResolvedValue({ markdown: "# hi", metadata: { sourceURL: "https://example.com" } } as any);
|
|
41
|
+
const result = await app.scrapeUrl("https://example.com", { formats: ["markdown"] });
|
|
42
|
+
expect(app.scrape).toHaveBeenCalledWith("https://example.com", { formats: ["markdown"] });
|
|
43
|
+
expect(result.success).toBe(true);
|
|
44
|
+
expect(result.markdown).toBe("# hi");
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it("crawlUrl and asyncCrawlUrl rename v1 crawl options to their v2 names", async () => {
|
|
48
|
+
vi.spyOn(app, "crawl").mockResolvedValue({ status: "completed", data: [] } as any);
|
|
49
|
+
vi.spyOn(app, "startCrawl").mockResolvedValue({ id: "job-1", url: "u" } as any);
|
|
50
|
+
const v1Options = { limit: 2, maxDepth: 3, allowBackwardLinks: true, ignoreSitemap: true, scrapeOptions: { formats: ["markdown" as const] } };
|
|
51
|
+
const expected = { limit: 2, maxDiscoveryDepth: 3, crawlEntireDomain: true, sitemap: "skip", scrapeOptions: { formats: ["markdown"] } };
|
|
52
|
+
|
|
53
|
+
await app.crawlUrl("https://example.com", v1Options);
|
|
54
|
+
await app.asyncCrawlUrl("https://example.com", v1Options);
|
|
55
|
+
|
|
56
|
+
expect(app.crawl).toHaveBeenCalledWith("https://example.com", expected);
|
|
57
|
+
expect(app.startCrawl).toHaveBeenCalledWith("https://example.com", expected);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("v2 methods return results without success", async () => {
|
|
61
|
+
const client = new FirecrawlClient({ apiKey: "fc-test", apiUrl: "http://localhost:9" });
|
|
62
|
+
const adapter: AxiosAdapter = async config => ({
|
|
63
|
+
data: { success: true, data: { markdown: "# hi" } },
|
|
64
|
+
status: 200,
|
|
65
|
+
statusText: "OK",
|
|
66
|
+
headers: {},
|
|
67
|
+
config,
|
|
68
|
+
});
|
|
69
|
+
(client as any).http.instance.defaults.adapter = adapter;
|
|
70
|
+
const result = await client.scrape("https://example.com", { formats: ["markdown"] });
|
|
71
|
+
expect(result).toEqual({ markdown: "# hi" });
|
|
72
|
+
expect("success" in result).toBe(false);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
describe("V1 deprecation warning", () => {
|
|
77
|
+
afterEach(() => {
|
|
78
|
+
vi.restoreAllMocks();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
async function freshClient() {
|
|
82
|
+
vi.resetModules();
|
|
83
|
+
const emitWarning = vi.spyOn(process, "emitWarning").mockImplementation(() => {});
|
|
84
|
+
const { FirecrawlClient: FreshClient } = await import("../../../v2/client");
|
|
85
|
+
const app = new FreshClient({ apiKey: "fc-test", apiUrl: "http://localhost:9" });
|
|
86
|
+
const v1Warnings = () => emitWarning.mock.calls.filter(([, opts]) => (opts as any)?.code === "FIRECRAWL_V1_METHOD");
|
|
87
|
+
return { app, v1Warnings };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
it("is emitted once per process", async () => {
|
|
91
|
+
const { app, v1Warnings } = await freshClient();
|
|
92
|
+
vi.spyOn(app, "scrape").mockResolvedValue({ markdown: "# hi" } as any);
|
|
93
|
+
vi.spyOn(app, "map").mockResolvedValue({ links: [] } as any);
|
|
94
|
+
|
|
95
|
+
await app.scrapeUrl("https://example.com");
|
|
96
|
+
await app.scrapeUrl("https://example.com");
|
|
97
|
+
await app.mapUrl("https://example.com");
|
|
98
|
+
|
|
99
|
+
expect(v1Warnings()).toHaveLength(1);
|
|
100
|
+
expect((v1Warnings()[0]![1] as any).type).toBe("DeprecationWarning");
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("is not emitted by v2 methods", async () => {
|
|
104
|
+
const { app, v1Warnings } = await freshClient();
|
|
105
|
+
const adapter: AxiosAdapter = async config => ({
|
|
106
|
+
data: { success: true, data: { markdown: "# hi" } },
|
|
107
|
+
status: 200,
|
|
108
|
+
statusText: "OK",
|
|
109
|
+
headers: {},
|
|
110
|
+
config,
|
|
111
|
+
});
|
|
112
|
+
(app as any).http.instance.defaults.adapter = adapter;
|
|
113
|
+
|
|
114
|
+
await app.scrape("https://example.com", { formats: ["markdown"] });
|
|
115
|
+
|
|
116
|
+
expect(v1Warnings()).toHaveLength(0);
|
|
117
|
+
});
|
|
33
118
|
});
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
const NODE_MODULES = "/node_modules/";
|
|
2
|
+
|
|
3
|
+
function packageOf(file: string): string | null {
|
|
4
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
5
|
+
try {
|
|
6
|
+
path = decodeURIComponent(path);
|
|
7
|
+
} catch {}
|
|
8
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
9
|
+
if (at < 0) return null;
|
|
10
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
11
|
+
const name = first?.startsWith("@") ? (second ? `${first}/${second}` : "") : first;
|
|
12
|
+
return name && !name.startsWith(".") ? name : null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** File of a V8 stack line, or null for frames without one (native, anonymous, runtime internals). */
|
|
16
|
+
function frameFile(line: string): string | null {
|
|
17
|
+
const text = line.trim().replace(/^at /, "");
|
|
18
|
+
const open = text.indexOf(" (");
|
|
19
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
20
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
21
|
+
return match && !match[1]!.startsWith("node:") ? match[1]! : null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Downstream surface classification matches "playground" anywhere in an origin.
|
|
25
|
+
function slugify(name: string): string {
|
|
26
|
+
return name
|
|
27
|
+
.toLowerCase()
|
|
28
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
29
|
+
.replace(/^-+|-+$/g, "")
|
|
30
|
+
.slice(0, 48)
|
|
31
|
+
.replace(/-+$/, "");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Slug of the installed package that created the client, or null when the
|
|
36
|
+
* caller is application code or detection is disabled. Never throws.
|
|
37
|
+
*/
|
|
38
|
+
export function detectCallerPackage(stack?: string): string | null {
|
|
39
|
+
try {
|
|
40
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
41
|
+
if (stack === undefined) {
|
|
42
|
+
const limit = Error.stackTraceLimit;
|
|
43
|
+
Error.stackTraceLimit = 50;
|
|
44
|
+
try {
|
|
45
|
+
stack = new Error().stack ?? "";
|
|
46
|
+
} finally {
|
|
47
|
+
Error.stackTraceLimit = limit;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
let ownFile: string | null = null;
|
|
51
|
+
let ownPackage: string | null = null;
|
|
52
|
+
for (const line of stack.split("\n")) {
|
|
53
|
+
if (!/^\s*at /.test(line)) continue;
|
|
54
|
+
const file = frameFile(line);
|
|
55
|
+
if (!file) continue;
|
|
56
|
+
if (ownFile === null) {
|
|
57
|
+
ownFile = file;
|
|
58
|
+
ownPackage = packageOf(file);
|
|
59
|
+
}
|
|
60
|
+
if (file === ownFile) continue;
|
|
61
|
+
const name = packageOf(file);
|
|
62
|
+
if (name && name === ownPackage) continue;
|
|
63
|
+
const slug = name ? slugify(name) : "";
|
|
64
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
65
|
+
}
|
|
66
|
+
} catch {}
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Origin sent with requests: `js-sdk@<version>`, or `js-sdk.<package>@<version>` for wrapper libraries. */
|
|
71
|
+
export function defaultOrigin(version: string): string {
|
|
72
|
+
const caller = detectCallerPackage();
|
|
73
|
+
return caller ? `js-sdk.${caller}@${version}` : `js-sdk@${version}`;
|
|
74
|
+
}
|
package/src/v1/index.ts
CHANGED
|
@@ -4,15 +4,18 @@ import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
|
|
|
4
4
|
import { TypedEventTarget } from "typescript-event-target";
|
|
5
5
|
import type { RequiresAction } from "../v2/types";
|
|
6
6
|
import { pinToApiOrigin } from "../utils/apiOrigin";
|
|
7
|
+
import { detectCallerPackage } from "../utils/callerOrigin";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Configuration interface for FirecrawlApp.
|
|
10
11
|
* @param apiKey - Optional API key for authentication.
|
|
11
12
|
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
|
|
13
|
+
* @param origin - Optional attribution tag stamped into request bodies.
|
|
12
14
|
*/
|
|
13
15
|
export interface FirecrawlAppConfig {
|
|
14
16
|
apiKey?: string | null;
|
|
15
17
|
apiUrl?: string | null;
|
|
18
|
+
origin?: string;
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
/**
|
|
@@ -666,6 +669,8 @@ export default class FirecrawlApp {
|
|
|
666
669
|
public apiKey: string;
|
|
667
670
|
public apiUrl: string;
|
|
668
671
|
public version: string = "1.25.1";
|
|
672
|
+
private readonly clientOrigin?: string;
|
|
673
|
+
private readonly callerPackage: string | null;
|
|
669
674
|
|
|
670
675
|
private isCloudService(url: string): boolean {
|
|
671
676
|
return url.includes('api.firecrawl.dev');
|
|
@@ -698,7 +703,7 @@ export default class FirecrawlApp {
|
|
|
698
703
|
* Initializes a new instance of the FirecrawlApp class.
|
|
699
704
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
700
705
|
*/
|
|
701
|
-
constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
|
|
706
|
+
constructor({ apiKey = null, apiUrl = null, origin }: FirecrawlAppConfig) {
|
|
702
707
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
703
708
|
|
|
704
709
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
@@ -707,9 +712,19 @@ export default class FirecrawlApp {
|
|
|
707
712
|
|
|
708
713
|
this.apiKey = apiKey || '';
|
|
709
714
|
this.apiUrl = baseUrl;
|
|
715
|
+
this.clientOrigin = origin || undefined;
|
|
716
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
710
717
|
this.init();
|
|
711
718
|
}
|
|
712
719
|
|
|
720
|
+
private resolveOrigin(params?: any): string {
|
|
721
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
722
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
723
|
+
return this.callerPackage
|
|
724
|
+
? `js-sdk.${this.callerPackage}@${this.version}`
|
|
725
|
+
: `js-sdk@${this.version}`;
|
|
726
|
+
}
|
|
727
|
+
|
|
713
728
|
/**
|
|
714
729
|
* Scrapes a URL using the Firecrawl API.
|
|
715
730
|
* @param url - The URL to scrape.
|
|
@@ -724,7 +739,7 @@ export default class FirecrawlApp {
|
|
|
724
739
|
"Content-Type": "application/json",
|
|
725
740
|
Authorization: `Bearer ${this.apiKey}`,
|
|
726
741
|
} as AxiosRequestHeaders;
|
|
727
|
-
let jsonData: any = { url, ...params, origin:
|
|
742
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
728
743
|
if (jsonData?.extract?.schema) {
|
|
729
744
|
jsonData = {
|
|
730
745
|
...jsonData,
|
|
@@ -791,7 +806,7 @@ export default class FirecrawlApp {
|
|
|
791
806
|
lang: params?.lang ?? "en",
|
|
792
807
|
country: params?.country ?? "us",
|
|
793
808
|
location: params?.location,
|
|
794
|
-
origin:
|
|
809
|
+
origin: this.resolveOrigin(params),
|
|
795
810
|
timeout: params?.timeout ?? 60000,
|
|
796
811
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] },
|
|
797
812
|
};
|
|
@@ -855,7 +870,7 @@ export default class FirecrawlApp {
|
|
|
855
870
|
idempotencyKey?: string
|
|
856
871
|
): Promise<CrawlStatusResponse | ErrorResponse> {
|
|
857
872
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
858
|
-
let jsonData: any = { url, ...params, origin:
|
|
873
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
859
874
|
try {
|
|
860
875
|
const response: AxiosResponse = await this.postRequest(
|
|
861
876
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -884,7 +899,7 @@ export default class FirecrawlApp {
|
|
|
884
899
|
idempotencyKey?: string
|
|
885
900
|
): Promise<CrawlResponse | ErrorResponse> {
|
|
886
901
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
887
|
-
let jsonData: any = { url, ...params, origin:
|
|
902
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
888
903
|
try {
|
|
889
904
|
const response: AxiosResponse = await this.postRequest(
|
|
890
905
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -1060,7 +1075,7 @@ export default class FirecrawlApp {
|
|
|
1060
1075
|
*/
|
|
1061
1076
|
async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
|
|
1062
1077
|
const headers = this.prepareHeaders();
|
|
1063
|
-
let jsonData: any = { url, ...params, origin:
|
|
1078
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
1064
1079
|
|
|
1065
1080
|
try {
|
|
1066
1081
|
const response: AxiosResponse = await this.postRequest(
|
|
@@ -1099,7 +1114,7 @@ export default class FirecrawlApp {
|
|
|
1099
1114
|
maxConcurrency?: number,
|
|
1100
1115
|
): Promise<BatchScrapeStatusResponse | ErrorResponse> {
|
|
1101
1116
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1102
|
-
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
1117
|
+
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
1103
1118
|
if (jsonData?.extract?.schema) {
|
|
1104
1119
|
jsonData = {
|
|
1105
1120
|
...jsonData,
|
|
@@ -1148,7 +1163,7 @@ export default class FirecrawlApp {
|
|
|
1148
1163
|
ignoreInvalidURLs?: boolean,
|
|
1149
1164
|
): Promise<BatchScrapeResponse | ErrorResponse> {
|
|
1150
1165
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1151
|
-
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
1166
|
+
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
1152
1167
|
try {
|
|
1153
1168
|
const response: AxiosResponse = await this.postRequest(
|
|
1154
1169
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -1312,7 +1327,7 @@ export default class FirecrawlApp {
|
|
|
1312
1327
|
try {
|
|
1313
1328
|
const response: AxiosResponse = await this.postRequest(
|
|
1314
1329
|
this.apiUrl + `/v1/extract`,
|
|
1315
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
1330
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
1316
1331
|
headers
|
|
1317
1332
|
);
|
|
1318
1333
|
|
|
@@ -1372,7 +1387,7 @@ export default class FirecrawlApp {
|
|
|
1372
1387
|
try {
|
|
1373
1388
|
const response: AxiosResponse = await this.postRequest(
|
|
1374
1389
|
this.apiUrl + `/v1/extract`,
|
|
1375
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
1390
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
1376
1391
|
headers
|
|
1377
1392
|
);
|
|
1378
1393
|
|
|
@@ -1740,7 +1755,7 @@ export default class FirecrawlApp {
|
|
|
1740
1755
|
*/
|
|
1741
1756
|
async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1742
1757
|
const headers = this.prepareHeaders();
|
|
1743
|
-
let jsonData: any = { query, ...params, origin:
|
|
1758
|
+
let jsonData: any = { query, ...params, origin: this.resolveOrigin(params) };
|
|
1744
1759
|
|
|
1745
1760
|
if (jsonData?.jsonOptions?.schema) {
|
|
1746
1761
|
jsonData = {
|
|
@@ -1888,7 +1903,7 @@ export default class FirecrawlApp {
|
|
|
1888
1903
|
async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1889
1904
|
const headers = this.prepareHeaders();
|
|
1890
1905
|
try {
|
|
1891
|
-
let jsonData: any = { topic, ...params, origin:
|
|
1906
|
+
let jsonData: any = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
1892
1907
|
const response: AxiosResponse = await this.postRequest(
|
|
1893
1908
|
`${this.apiUrl}/v1/deep-research`,
|
|
1894
1909
|
jsonData,
|
|
@@ -2003,7 +2018,7 @@ export default class FirecrawlApp {
|
|
|
2003
2018
|
*/
|
|
2004
2019
|
async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
|
|
2005
2020
|
const headers = this.prepareHeaders();
|
|
2006
|
-
let jsonData: any = { url, ...params, origin:
|
|
2021
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2007
2022
|
try {
|
|
2008
2023
|
const response: AxiosResponse = await this.postRequest(
|
|
2009
2024
|
`${this.apiUrl}/v1/llmstxt`,
|
package/src/v2/client.ts
CHANGED
|
@@ -101,7 +101,10 @@ import type {
|
|
|
101
101
|
GetMonitorCheckOptions,
|
|
102
102
|
UpdateMonitorRequest,
|
|
103
103
|
} from "./types";
|
|
104
|
+
import { defaultOrigin } from "../utils/callerOrigin";
|
|
105
|
+
import { getVersion } from "./utils/getVersion";
|
|
104
106
|
import { Watcher } from "./watcher";
|
|
107
|
+
import { callV1, toV2CrawlOptions, type V1CrawlOptions, type V1Result } from "./utils/v1Compat";
|
|
105
108
|
import type { WatcherOptions } from "./watcher";
|
|
106
109
|
import * as zt from "zod";
|
|
107
110
|
|
|
@@ -130,6 +133,12 @@ export interface FirecrawlClientOptions {
|
|
|
130
133
|
maxRetries?: number;
|
|
131
134
|
/** Exponential backoff factor for retries (optional). */
|
|
132
135
|
backoffFactor?: number;
|
|
136
|
+
/**
|
|
137
|
+
* Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
|
|
138
|
+
* or `js-sdk.<package>@<version>` when an installed package creates the client
|
|
139
|
+
* (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
|
|
140
|
+
*/
|
|
141
|
+
origin?: string;
|
|
133
142
|
}
|
|
134
143
|
|
|
135
144
|
/** Accepts a plain API key string or a full options object. */
|
|
@@ -140,7 +149,7 @@ export type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
|
140
149
|
*/
|
|
141
150
|
|
|
142
151
|
export class FirecrawlClient {
|
|
143
|
-
|
|
152
|
+
protected readonly http: HttpClient;
|
|
144
153
|
private _research?: ResearchClient;
|
|
145
154
|
|
|
146
155
|
private isCloudService(url: string): boolean {
|
|
@@ -168,6 +177,7 @@ export class FirecrawlClient {
|
|
|
168
177
|
timeoutMs: opts.timeoutMs,
|
|
169
178
|
maxRetries: opts.maxRetries,
|
|
170
179
|
backoffFactor: opts.backoffFactor,
|
|
180
|
+
origin: opts.origin || defaultOrigin(getVersion()),
|
|
171
181
|
});
|
|
172
182
|
}
|
|
173
183
|
|
|
@@ -721,54 +731,54 @@ export class FirecrawlClient {
|
|
|
721
731
|
return new Watcher(this.http, jobId, opts);
|
|
722
732
|
}
|
|
723
733
|
|
|
724
|
-
/** @deprecated
|
|
725
|
-
async scrapeUrl(url: string, options?: ScrapeOptions): Promise<Document
|
|
726
|
-
return this.scrape(url, options);
|
|
734
|
+
/** @deprecated v1 alias for scrape(); its result also carries v1's `success: true`. */
|
|
735
|
+
async scrapeUrl(url: string, options?: ScrapeOptions): Promise<V1Result<Document>> {
|
|
736
|
+
return callV1(() => this.scrape(url, options));
|
|
727
737
|
}
|
|
728
738
|
|
|
729
|
-
/** @deprecated
|
|
730
|
-
async crawlUrl(url: string, req: CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<CrawlJob
|
|
731
|
-
return this.crawl(url, req);
|
|
739
|
+
/** @deprecated v1 alias for crawl(); its result also carries v1's `success: true`. */
|
|
740
|
+
async crawlUrl(url: string, req: CrawlOptions & V1CrawlOptions & { pollInterval?: number; timeout?: number; signal?: AbortSignal } = {}): Promise<V1Result<CrawlJob>> {
|
|
741
|
+
return callV1(() => this.crawl(url, toV2CrawlOptions(req)));
|
|
732
742
|
}
|
|
733
743
|
|
|
734
|
-
/** @deprecated
|
|
735
|
-
async asyncCrawlUrl(url: string, req: CrawlOptions = {}): Promise<CrawlResponse
|
|
736
|
-
return this.startCrawl(url, req);
|
|
744
|
+
/** @deprecated v1 alias for startCrawl(); its result also carries v1's `success: true`. */
|
|
745
|
+
async asyncCrawlUrl(url: string, req: CrawlOptions & V1CrawlOptions = {}): Promise<V1Result<CrawlResponse>> {
|
|
746
|
+
return callV1(() => this.startCrawl(url, toV2CrawlOptions(req)));
|
|
737
747
|
}
|
|
738
748
|
|
|
739
|
-
/** @deprecated
|
|
740
|
-
async checkCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<CrawlJob
|
|
741
|
-
return this.getCrawlStatus(jobId, pagination);
|
|
749
|
+
/** @deprecated v1 alias for getCrawlStatus(); its result also carries v1's `success: true`. */
|
|
750
|
+
async checkCrawlStatus(jobId: string, pagination?: PaginationConfig): Promise<V1Result<CrawlJob>> {
|
|
751
|
+
return callV1(() => this.getCrawlStatus(jobId, pagination));
|
|
742
752
|
}
|
|
743
753
|
|
|
744
|
-
/** @deprecated
|
|
745
|
-
async checkCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse
|
|
746
|
-
return this.getCrawlErrors(crawlId);
|
|
754
|
+
/** @deprecated v1 alias for getCrawlErrors(); its result also carries v1's `success: true`. */
|
|
755
|
+
async checkCrawlErrors(crawlId: string): Promise<V1Result<CrawlErrorsResponse>> {
|
|
756
|
+
return callV1(() => this.getCrawlErrors(crawlId));
|
|
747
757
|
}
|
|
748
758
|
|
|
749
|
-
/** @deprecated
|
|
750
|
-
async mapUrl(url: string, options?: MapOptions): Promise<MapData
|
|
751
|
-
return this.map(url, options);
|
|
759
|
+
/** @deprecated v1 alias for map(); its result also carries v1's `success: true`. */
|
|
760
|
+
async mapUrl(url: string, options?: MapOptions): Promise<V1Result<MapData>> {
|
|
761
|
+
return callV1(() => this.map(url, options));
|
|
752
762
|
}
|
|
753
763
|
|
|
754
|
-
/** @deprecated
|
|
755
|
-
async batchScrapeUrls(urls: string[], opts?: BatchScrapeOptions & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob
|
|
756
|
-
return this.batchScrape(urls, opts);
|
|
764
|
+
/** @deprecated v1 alias for batchScrape(); its result also carries v1's `success: true`. */
|
|
765
|
+
async batchScrapeUrls(urls: string[], opts?: BatchScrapeOptions & { pollInterval?: number; timeout?: number }): Promise<V1Result<BatchScrapeJob>> {
|
|
766
|
+
return callV1(() => this.batchScrape(urls, opts));
|
|
757
767
|
}
|
|
758
768
|
|
|
759
|
-
/** @deprecated
|
|
760
|
-
async asyncBatchScrapeUrls(urls: string[], opts?: BatchScrapeOptions): Promise<BatchScrapeResponse
|
|
761
|
-
return this.startBatchScrape(urls, opts);
|
|
769
|
+
/** @deprecated v1 alias for startBatchScrape(); its result also carries v1's `success: true`. */
|
|
770
|
+
async asyncBatchScrapeUrls(urls: string[], opts?: BatchScrapeOptions): Promise<V1Result<BatchScrapeResponse>> {
|
|
771
|
+
return callV1(() => this.startBatchScrape(urls, opts));
|
|
762
772
|
}
|
|
763
773
|
|
|
764
|
-
/** @deprecated
|
|
765
|
-
async checkBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<BatchScrapeJob
|
|
766
|
-
return this.getBatchScrapeStatus(jobId, pagination);
|
|
774
|
+
/** @deprecated v1 alias for getBatchScrapeStatus(); its result also carries v1's `success: true`. */
|
|
775
|
+
async checkBatchScrapeStatus(jobId: string, pagination?: PaginationConfig): Promise<V1Result<BatchScrapeJob>> {
|
|
776
|
+
return callV1(() => this.getBatchScrapeStatus(jobId, pagination));
|
|
767
777
|
}
|
|
768
778
|
|
|
769
|
-
/** @deprecated
|
|
770
|
-
async checkBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse
|
|
771
|
-
return this.getBatchScrapeErrors(jobId);
|
|
779
|
+
/** @deprecated v1 alias for getBatchScrapeErrors(); its result also carries v1's `success: true`. */
|
|
780
|
+
async checkBatchScrapeErrors(jobId: string): Promise<V1Result<CrawlErrorsResponse>> {
|
|
781
|
+
return callV1(() => this.getBatchScrapeErrors(jobId));
|
|
772
782
|
}
|
|
773
783
|
}
|
|
774
784
|
|
package/src/v2/methods/parse.ts
CHANGED
|
@@ -6,7 +6,6 @@ import {
|
|
|
6
6
|
throwForBadResponse,
|
|
7
7
|
normalizeAxiosError,
|
|
8
8
|
} from "../utils/errorHandler";
|
|
9
|
-
import { getVersion } from "../utils/getVersion";
|
|
10
9
|
|
|
11
10
|
function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {
|
|
12
11
|
if (typeof Blob !== "undefined" && input instanceof Blob) {
|
|
@@ -57,13 +56,9 @@ export async function parse(
|
|
|
57
56
|
|
|
58
57
|
if (options) ensureValidParseOptions(options);
|
|
59
58
|
|
|
60
|
-
const version = getVersion();
|
|
61
59
|
const normalizedOptions: ParseOptions = {
|
|
62
60
|
...(options ?? {}),
|
|
63
|
-
origin:
|
|
64
|
-
typeof options?.origin === "string" && options.origin.includes("mcp")
|
|
65
|
-
? options.origin
|
|
66
|
-
: (options?.origin ?? `js-sdk@${version}`),
|
|
61
|
+
origin: options?.origin || http.getOrigin(),
|
|
67
62
|
};
|
|
68
63
|
|
|
69
64
|
const formData = new FormData();
|