@mendable/firecrawl-js 4.46.0 → 4.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -8
- package/dist/chunk-GD463K4D.js +114 -0
- package/dist/index.cjs +163 -39
- package/dist/index.d.cts +24 -5
- package/dist/index.d.ts +24 -5
- package/dist/index.js +164 -41
- package/dist/package-WONXBIJS.js +40 -0
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +1 -0
- package/src/__tests__/unit/v2/browser.unit.test.ts +28 -0
- package/src/__tests__/unit/v2/errorHandler.test.ts +23 -0
- package/src/__tests__/unit/v2/origin.test.ts +132 -0
- package/src/__tests__/unit/v2/pagination.test.ts +20 -0
- package/src/__tests__/unit/v2/research.test.ts +1 -0
- package/src/__tests__/unit/v2/validation.test.ts +39 -1
- package/src/index.ts +1 -0
- package/src/utils/callerOrigin.ts +74 -0
- package/src/v1/index.ts +28 -13
- package/src/v2/client.ts +11 -2
- package/src/v2/methods/browser.ts +3 -0
- package/src/v2/methods/crawl.ts +2 -1
- package/src/v2/methods/parse.ts +1 -6
- package/src/v2/methods/research.ts +20 -7
- package/src/v2/methods/search.ts +2 -1
- package/src/v2/types.ts +2 -2
- package/src/v2/utils/errorHandler.ts +23 -2
- package/src/v2/utils/getVersion.ts +4 -12
- package/src/v2/utils/httpClient.ts +10 -3
- package/src/v2/utils/pagination.ts +4 -1
- package/src/v2/utils/validation.ts +28 -0
- package/dist/chunk-R4XT7PME.js +0 -93
- package/dist/package-EI4Q33HD.js +0 -4
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import {
|
|
2
|
+
author,
|
|
3
|
+
bugs,
|
|
4
|
+
dependencies,
|
|
5
|
+
description,
|
|
6
|
+
devDependencies,
|
|
7
|
+
engines,
|
|
8
|
+
exports,
|
|
9
|
+
homepage,
|
|
10
|
+
keywords,
|
|
11
|
+
license,
|
|
12
|
+
main,
|
|
13
|
+
name,
|
|
14
|
+
package_default,
|
|
15
|
+
repository,
|
|
16
|
+
scripts,
|
|
17
|
+
type,
|
|
18
|
+
types,
|
|
19
|
+
version
|
|
20
|
+
} from "./chunk-GD463K4D.js";
|
|
21
|
+
export {
|
|
22
|
+
author,
|
|
23
|
+
bugs,
|
|
24
|
+
package_default as default,
|
|
25
|
+
dependencies,
|
|
26
|
+
description,
|
|
27
|
+
devDependencies,
|
|
28
|
+
engines,
|
|
29
|
+
exports,
|
|
30
|
+
homepage,
|
|
31
|
+
keywords,
|
|
32
|
+
license,
|
|
33
|
+
main,
|
|
34
|
+
name,
|
|
35
|
+
repository,
|
|
36
|
+
scripts,
|
|
37
|
+
type,
|
|
38
|
+
types,
|
|
39
|
+
version
|
|
40
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.48.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
|
+
import { browser } from "../../../v2/methods/browser";
|
|
3
|
+
|
|
4
|
+
describe("JS SDK v2 browser", () => {
|
|
5
|
+
test("sends location when creating a session", async () => {
|
|
6
|
+
const post = vi.fn(async () => ({
|
|
7
|
+
status: 200,
|
|
8
|
+
data: { success: true, id: "session-1" },
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
await browser({ post } as any, { location: { country: "GB" } });
|
|
12
|
+
|
|
13
|
+
expect(post).toHaveBeenCalledWith("/v2/browser", {
|
|
14
|
+
location: { country: "GB" },
|
|
15
|
+
});
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("omits location by default", async () => {
|
|
19
|
+
const post = vi.fn(async () => ({
|
|
20
|
+
status: 200,
|
|
21
|
+
data: { success: true, id: "session-1" },
|
|
22
|
+
}));
|
|
23
|
+
|
|
24
|
+
await browser({ post } as any);
|
|
25
|
+
|
|
26
|
+
expect(post).toHaveBeenCalledWith("/v2/browser", {});
|
|
27
|
+
});
|
|
28
|
+
});
|
|
@@ -15,5 +15,28 @@ describe("v2 utils: errorHandler", () => {
|
|
|
15
15
|
};
|
|
16
16
|
expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
|
|
17
17
|
});
|
|
18
|
+
|
|
19
|
+
test("normalizeAxiosError: names the fields a 400 validation error rejected", () => {
|
|
20
|
+
const err: any = {
|
|
21
|
+
isAxiosError: true,
|
|
22
|
+
response: {
|
|
23
|
+
status: 400,
|
|
24
|
+
data: {
|
|
25
|
+
error: "Unrecognized key in body -- please review the v2 API documentation for request body changes",
|
|
26
|
+
details: [{ code: "unrecognized_keys", keys: ["parsePDF"], path: [], message: "Unrecognized key(s) in object: 'parsePDF'" }],
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
message: "Request failed with status code 400",
|
|
30
|
+
};
|
|
31
|
+
expect(() => normalizeAxiosError(err, "scrape")).toThrow(/unrecognized key\(s\) parsePDF/);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("throwForBadResponse: appends the path and message of each issue", () => {
|
|
35
|
+
const resp: any = {
|
|
36
|
+
status: 400,
|
|
37
|
+
data: { error: "Bad Request", details: [{ code: "invalid_union", path: ["formats", 0], message: "Invalid input" }] },
|
|
38
|
+
};
|
|
39
|
+
expect(() => throwForBadResponse(resp, "scrape")).toThrow("Bad Request (formats.0: Invalid input)");
|
|
40
|
+
});
|
|
18
41
|
});
|
|
19
42
|
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, test, expect } from "vitest";
|
|
2
|
+
import type { AxiosAdapter } from "axios";
|
|
3
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
4
|
+
import { Firecrawl } from "../../../index";
|
|
5
|
+
import { getVersion } from "../../../v2/utils/getVersion";
|
|
6
|
+
import { detectCallerPackage } from "../../../utils/callerOrigin";
|
|
7
|
+
|
|
8
|
+
function withCapture<T extends { http: any }>(client: T) {
|
|
9
|
+
const bodies: Array<Record<string, unknown>> = [];
|
|
10
|
+
const adapter: AxiosAdapter = async config => {
|
|
11
|
+
bodies.push(JSON.parse(config.data));
|
|
12
|
+
return {
|
|
13
|
+
data: { success: true, data: { markdown: "# hello" } },
|
|
14
|
+
status: 200,
|
|
15
|
+
statusText: "OK",
|
|
16
|
+
headers: {},
|
|
17
|
+
config,
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
client.http.instance.defaults.adapter = adapter;
|
|
21
|
+
return { client, bodies };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe("v2 request origin", () => {
|
|
25
|
+
const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
if (previousOptOut === undefined) {
|
|
28
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
29
|
+
} else {
|
|
30
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("defaults to the plain SDK origin when application code creates the client", async () => {
|
|
35
|
+
const { client, bodies } = withCapture(new FirecrawlClient({ apiKey: "fc-test" }));
|
|
36
|
+
await client.scrape("https://example.com");
|
|
37
|
+
expect(bodies[0]!.origin).toBe(`js-sdk@${getVersion()}`);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("the origin client option is stamped into request bodies", async () => {
|
|
41
|
+
const { client, bodies } = withCapture(
|
|
42
|
+
new FirecrawlClient({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" }),
|
|
43
|
+
);
|
|
44
|
+
await client.scrape("https://example.com");
|
|
45
|
+
expect(bodies[0]!.origin).toBe("js-sdk.example@1.0.0");
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("a per-call origin wins over the client origin", async () => {
|
|
49
|
+
const { client, bodies } = withCapture(
|
|
50
|
+
new FirecrawlClient({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" }),
|
|
51
|
+
);
|
|
52
|
+
await client.scrape("https://example.com", { origin: "per-call" } as any);
|
|
53
|
+
await client.scrape("https://example.com", { origin: "mcp-server" } as any);
|
|
54
|
+
expect(bodies.map(b => b.origin)).toEqual(["per-call", "mcp-server"]);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("the unified client forwards the origin option to v1", () => {
|
|
58
|
+
const app = new Firecrawl({ apiKey: "fc-test", origin: "js-sdk.example@1.0.0" });
|
|
59
|
+
expect((app.v1 as any).clientOrigin).toBe("js-sdk.example@1.0.0");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe("detectCallerPackage", () => {
|
|
65
|
+
const sdkFrames = [
|
|
66
|
+
"Error",
|
|
67
|
+
" at detectCallerPackage (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:10:5)",
|
|
68
|
+
" at new FirecrawlClient (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:20:9)",
|
|
69
|
+
];
|
|
70
|
+
const stackFrom = (...callers: string[]) =>
|
|
71
|
+
[...sdkFrames, ...callers.map(c => ` at ${c}`)].join("\n");
|
|
72
|
+
const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
73
|
+
|
|
74
|
+
beforeEach(() => {
|
|
75
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
afterEach(() => {
|
|
79
|
+
if (previousOptOut === undefined) {
|
|
80
|
+
delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
|
|
81
|
+
} else {
|
|
82
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("names the scoped package that created the client", () => {
|
|
87
|
+
const stack = stackFrom(
|
|
88
|
+
"make (file:///srv/app/node_modules/@Scope/Some_Agent/dist/index.js:3:12)",
|
|
89
|
+
"main (file:///srv/app/src/main.js:1:1)",
|
|
90
|
+
);
|
|
91
|
+
expect(detectCallerPackage(stack)).toBe("scope-some-agent");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("uses the last node_modules segment of a nested install", () => {
|
|
95
|
+
const stack = stackFrom(
|
|
96
|
+
"make (/srv/app/node_modules/.pnpm/outer@1.0.0/node_modules/inner-pkg/lib/index.cjs:3:12)",
|
|
97
|
+
);
|
|
98
|
+
expect(detectCallerPackage(stack)).toBe("inner-pkg");
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("handles windows paths with parentheses and anonymous frames", () => {
|
|
102
|
+
const stack = stackFrom(
|
|
103
|
+
"Array.map (<anonymous>)",
|
|
104
|
+
"C:\\Program Files (x86)\\app\\node_modules\\wrapper\\index.js:4:2",
|
|
105
|
+
);
|
|
106
|
+
expect(detectCallerPackage(stack)).toBe("wrapper");
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("returns null when application code created the client", () => {
|
|
110
|
+
const stack = stackFrom(
|
|
111
|
+
"main (file:///srv/app/src/main.js:1:1)",
|
|
112
|
+
"make (file:///srv/app/node_modules/wrapper/index.js:3:12)",
|
|
113
|
+
);
|
|
114
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("returns null for a package whose slug would read as another traffic class", () => {
|
|
118
|
+
const stack = stackFrom("make (/srv/app/node_modules/my-playground-kit/index.js:3:12)");
|
|
119
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("returns null for a bundle without node_modules in its path", () => {
|
|
123
|
+
expect(detectCallerPackage(stackFrom("make (/srv/app/dist/bundle.js:1:1)"))).toBeNull();
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("FIRECRAWL_DISABLE_CALLER_ORIGIN turns detection off", () => {
|
|
127
|
+
const stack = stackFrom("make (/srv/app/node_modules/wrapper/index.js:3:12)");
|
|
128
|
+
expect(detectCallerPackage(stack)).toBe("wrapper");
|
|
129
|
+
process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = "1";
|
|
130
|
+
expect(detectCallerPackage(stack)).toBeNull();
|
|
131
|
+
});
|
|
132
|
+
});
|
|
@@ -107,6 +107,26 @@ describe("JS SDK v2 pagination", () => {
|
|
|
107
107
|
}
|
|
108
108
|
});
|
|
109
109
|
|
|
110
|
+
test("crawl: stops when a running job's cursor does not advance", async () => {
|
|
111
|
+
const next = "https://api/v2/crawl/jobR?skip=2";
|
|
112
|
+
const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
|
|
113
|
+
const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
|
|
114
|
+
const http = makeHttp((url) => (url === next ? stalled : first));
|
|
115
|
+
const res = await getCrawlStatus(http, "jobR");
|
|
116
|
+
expect(res.data.length).toBe(2);
|
|
117
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test("batch: stops when a running job's cursor does not advance", async () => {
|
|
121
|
+
const next = "https://api/v2/batch/scrape/jobR?skip=2";
|
|
122
|
+
const first = { status: 200, data: { success: true, status: "scraping", completed: 2, total: 2, next, data: [{ markdown: "a" }, { markdown: "b" }] } };
|
|
123
|
+
const stalled = { status: 200, data: { success: true, status: "scraping", next, data: [] } };
|
|
124
|
+
const http = makeHttp((url) => (url === next ? stalled : first));
|
|
125
|
+
const res = await getBatchScrapeStatus(http, "jobR");
|
|
126
|
+
expect(res.data.length).toBe(2);
|
|
127
|
+
expect((http.get as Mock).mock.calls.length).toBe(2);
|
|
128
|
+
});
|
|
129
|
+
|
|
110
130
|
test("batch: maxWaitTime stops pagination after first page", async () => {
|
|
111
131
|
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/b1", data: [{ markdown: "a" }] } };
|
|
112
132
|
const p1 = { status: 200, data: { success: true, next: "https://api/b2", data: [{ markdown: "b" }] } };
|
|
@@ -1,9 +1,47 @@
|
|
|
1
1
|
import { describe, test, expect } from "vitest";
|
|
2
|
-
import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
|
2
|
+
import { ensureNoV1Options, ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
|
|
3
3
|
import type { FormatOption } from "../../../v2/types";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
|
|
6
6
|
describe("v2 utils: validation", () => {
|
|
7
|
+
const scrapeLevel = {
|
|
8
|
+
pageOptions: "use scrapeOptions",
|
|
9
|
+
includeLinks: "add 'links' to scrapeOptions.formats",
|
|
10
|
+
parsePDF: "use scrapeOptions.parsers",
|
|
11
|
+
};
|
|
12
|
+
const crawlOnly = {
|
|
13
|
+
crawlerOptions: "move its fields to the top level",
|
|
14
|
+
maxDepth: "use maxDiscoveryDepth",
|
|
15
|
+
allowBackwardLinks: "use crawlEntireDomain",
|
|
16
|
+
ignoreSitemap: "use sitemap: 'skip'",
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
test.each(Object.entries({ ...crawlOnly, ...scrapeLevel }))("ensureNoV1Options: crawl names %s and its replacement", (option, replacement) => {
|
|
20
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", [option]: true }, "crawl")).toThrow(
|
|
21
|
+
`crawl: ${option} is a v1 option, ${replacement}`,
|
|
22
|
+
);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test.each(Object.entries(scrapeLevel))("ensureNoV1Options: search names %s and its replacement", (option, replacement) => {
|
|
26
|
+
expect(() => ensureNoV1Options({ query: "q", [option]: true }, "search")).toThrow(
|
|
27
|
+
`search: ${option} is a v1 option, ${replacement}`,
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("ensureNoV1Options: search does not apply crawl-only hints", () => {
|
|
32
|
+
expect(() => ensureNoV1Options({ query: "q", maxDepth: 2, crawlerOptions: {} }, "search")).not.toThrow();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("ensureNoV1Options: lists every v1 option found", () => {
|
|
36
|
+
expect(() => ensureNoV1Options({ maxDepth: 2, allowBackwardLinks: true }, "crawl")).toThrow(/maxDepth.*maxDiscoveryDepth.*allowBackwardLinks.*crawlEntireDomain/);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("ensureNoV1Options: accepts v2 options and unset v1 keys", () => {
|
|
40
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", limit: 2, maxDiscoveryDepth: 1 }, "crawl")).not.toThrow();
|
|
41
|
+
expect(() => ensureNoV1Options({ url: "https://a.dev", maxDepth: undefined }, "crawl")).not.toThrow();
|
|
42
|
+
expect(() => ensureNoV1Options(undefined, "search")).not.toThrow();
|
|
43
|
+
});
|
|
44
|
+
|
|
7
45
|
test("ensureValidFormats: plain 'json' string is invalid", () => {
|
|
8
46
|
const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
|
|
9
47
|
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
const NODE_MODULES = "/node_modules/";
|
|
2
|
+
|
|
3
|
+
function packageOf(file: string): string | null {
|
|
4
|
+
let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
|
|
5
|
+
try {
|
|
6
|
+
path = decodeURIComponent(path);
|
|
7
|
+
} catch {}
|
|
8
|
+
const at = path.lastIndexOf(NODE_MODULES);
|
|
9
|
+
if (at < 0) return null;
|
|
10
|
+
const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
|
|
11
|
+
const name = first?.startsWith("@") ? (second ? `${first}/${second}` : "") : first;
|
|
12
|
+
return name && !name.startsWith(".") ? name : null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** File of a V8 stack line, or null for frames without one (native, anonymous, runtime internals). */
|
|
16
|
+
function frameFile(line: string): string | null {
|
|
17
|
+
const text = line.trim().replace(/^at /, "");
|
|
18
|
+
const open = text.indexOf(" (");
|
|
19
|
+
const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
|
|
20
|
+
const match = /^(.*):\d+:\d+$/.exec(location);
|
|
21
|
+
return match && !match[1]!.startsWith("node:") ? match[1]! : null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Downstream surface classification matches "playground" anywhere in an origin.
|
|
25
|
+
function slugify(name: string): string {
|
|
26
|
+
return name
|
|
27
|
+
.toLowerCase()
|
|
28
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
29
|
+
.replace(/^-+|-+$/g, "")
|
|
30
|
+
.slice(0, 48)
|
|
31
|
+
.replace(/-+$/, "");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Slug of the installed package that created the client, or null when the
|
|
36
|
+
* caller is application code or detection is disabled. Never throws.
|
|
37
|
+
*/
|
|
38
|
+
export function detectCallerPackage(stack?: string): string | null {
|
|
39
|
+
try {
|
|
40
|
+
if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
|
|
41
|
+
if (stack === undefined) {
|
|
42
|
+
const limit = Error.stackTraceLimit;
|
|
43
|
+
Error.stackTraceLimit = 50;
|
|
44
|
+
try {
|
|
45
|
+
stack = new Error().stack ?? "";
|
|
46
|
+
} finally {
|
|
47
|
+
Error.stackTraceLimit = limit;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
let ownFile: string | null = null;
|
|
51
|
+
let ownPackage: string | null = null;
|
|
52
|
+
for (const line of stack.split("\n")) {
|
|
53
|
+
if (!/^\s*at /.test(line)) continue;
|
|
54
|
+
const file = frameFile(line);
|
|
55
|
+
if (!file) continue;
|
|
56
|
+
if (ownFile === null) {
|
|
57
|
+
ownFile = file;
|
|
58
|
+
ownPackage = packageOf(file);
|
|
59
|
+
}
|
|
60
|
+
if (file === ownFile) continue;
|
|
61
|
+
const name = packageOf(file);
|
|
62
|
+
if (name && name === ownPackage) continue;
|
|
63
|
+
const slug = name ? slugify(name) : "";
|
|
64
|
+
return slug && !slug.includes("playground") ? slug : null;
|
|
65
|
+
}
|
|
66
|
+
} catch {}
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Origin sent with requests: `js-sdk@<version>`, or `js-sdk.<package>@<version>` for wrapper libraries. */
|
|
71
|
+
export function defaultOrigin(version: string): string {
|
|
72
|
+
const caller = detectCallerPackage();
|
|
73
|
+
return caller ? `js-sdk.${caller}@${version}` : `js-sdk@${version}`;
|
|
74
|
+
}
|
package/src/v1/index.ts
CHANGED
|
@@ -4,15 +4,18 @@ import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
|
|
|
4
4
|
import { TypedEventTarget } from "typescript-event-target";
|
|
5
5
|
import type { RequiresAction } from "../v2/types";
|
|
6
6
|
import { pinToApiOrigin } from "../utils/apiOrigin";
|
|
7
|
+
import { detectCallerPackage } from "../utils/callerOrigin";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Configuration interface for FirecrawlApp.
|
|
10
11
|
* @param apiKey - Optional API key for authentication.
|
|
11
12
|
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
|
|
13
|
+
* @param origin - Optional attribution tag stamped into request bodies.
|
|
12
14
|
*/
|
|
13
15
|
export interface FirecrawlAppConfig {
|
|
14
16
|
apiKey?: string | null;
|
|
15
17
|
apiUrl?: string | null;
|
|
18
|
+
origin?: string;
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
/**
|
|
@@ -666,6 +669,8 @@ export default class FirecrawlApp {
|
|
|
666
669
|
public apiKey: string;
|
|
667
670
|
public apiUrl: string;
|
|
668
671
|
public version: string = "1.25.1";
|
|
672
|
+
private readonly clientOrigin?: string;
|
|
673
|
+
private readonly callerPackage: string | null;
|
|
669
674
|
|
|
670
675
|
private isCloudService(url: string): boolean {
|
|
671
676
|
return url.includes('api.firecrawl.dev');
|
|
@@ -698,7 +703,7 @@ export default class FirecrawlApp {
|
|
|
698
703
|
* Initializes a new instance of the FirecrawlApp class.
|
|
699
704
|
* @param config - Configuration options for the FirecrawlApp instance.
|
|
700
705
|
*/
|
|
701
|
-
constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
|
|
706
|
+
constructor({ apiKey = null, apiUrl = null, origin }: FirecrawlAppConfig) {
|
|
702
707
|
const baseUrl = apiUrl || "https://api.firecrawl.dev";
|
|
703
708
|
|
|
704
709
|
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
|
|
@@ -707,9 +712,19 @@ export default class FirecrawlApp {
|
|
|
707
712
|
|
|
708
713
|
this.apiKey = apiKey || '';
|
|
709
714
|
this.apiUrl = baseUrl;
|
|
715
|
+
this.clientOrigin = origin || undefined;
|
|
716
|
+
this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
|
|
710
717
|
this.init();
|
|
711
718
|
}
|
|
712
719
|
|
|
720
|
+
private resolveOrigin(params?: any): string {
|
|
721
|
+
if (typeof params?.origin === "string" && params.origin) return params.origin;
|
|
722
|
+
if (this.clientOrigin) return this.clientOrigin;
|
|
723
|
+
return this.callerPackage
|
|
724
|
+
? `js-sdk.${this.callerPackage}@${this.version}`
|
|
725
|
+
: `js-sdk@${this.version}`;
|
|
726
|
+
}
|
|
727
|
+
|
|
713
728
|
/**
|
|
714
729
|
* Scrapes a URL using the Firecrawl API.
|
|
715
730
|
* @param url - The URL to scrape.
|
|
@@ -724,7 +739,7 @@ export default class FirecrawlApp {
|
|
|
724
739
|
"Content-Type": "application/json",
|
|
725
740
|
Authorization: `Bearer ${this.apiKey}`,
|
|
726
741
|
} as AxiosRequestHeaders;
|
|
727
|
-
let jsonData: any = { url, ...params, origin:
|
|
742
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
728
743
|
if (jsonData?.extract?.schema) {
|
|
729
744
|
jsonData = {
|
|
730
745
|
...jsonData,
|
|
@@ -791,7 +806,7 @@ export default class FirecrawlApp {
|
|
|
791
806
|
lang: params?.lang ?? "en",
|
|
792
807
|
country: params?.country ?? "us",
|
|
793
808
|
location: params?.location,
|
|
794
|
-
origin:
|
|
809
|
+
origin: this.resolveOrigin(params),
|
|
795
810
|
timeout: params?.timeout ?? 60000,
|
|
796
811
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] },
|
|
797
812
|
};
|
|
@@ -855,7 +870,7 @@ export default class FirecrawlApp {
|
|
|
855
870
|
idempotencyKey?: string
|
|
856
871
|
): Promise<CrawlStatusResponse | ErrorResponse> {
|
|
857
872
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
858
|
-
let jsonData: any = { url, ...params, origin:
|
|
873
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
859
874
|
try {
|
|
860
875
|
const response: AxiosResponse = await this.postRequest(
|
|
861
876
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -884,7 +899,7 @@ export default class FirecrawlApp {
|
|
|
884
899
|
idempotencyKey?: string
|
|
885
900
|
): Promise<CrawlResponse | ErrorResponse> {
|
|
886
901
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
887
|
-
let jsonData: any = { url, ...params, origin:
|
|
902
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
888
903
|
try {
|
|
889
904
|
const response: AxiosResponse = await this.postRequest(
|
|
890
905
|
this.apiUrl + `/v1/crawl`,
|
|
@@ -1060,7 +1075,7 @@ export default class FirecrawlApp {
|
|
|
1060
1075
|
*/
|
|
1061
1076
|
async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
|
|
1062
1077
|
const headers = this.prepareHeaders();
|
|
1063
|
-
let jsonData: any = { url, ...params, origin:
|
|
1078
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
1064
1079
|
|
|
1065
1080
|
try {
|
|
1066
1081
|
const response: AxiosResponse = await this.postRequest(
|
|
@@ -1099,7 +1114,7 @@ export default class FirecrawlApp {
|
|
|
1099
1114
|
maxConcurrency?: number,
|
|
1100
1115
|
): Promise<BatchScrapeStatusResponse | ErrorResponse> {
|
|
1101
1116
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1102
|
-
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin:
|
|
1117
|
+
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
|
|
1103
1118
|
if (jsonData?.extract?.schema) {
|
|
1104
1119
|
jsonData = {
|
|
1105
1120
|
...jsonData,
|
|
@@ -1148,7 +1163,7 @@ export default class FirecrawlApp {
|
|
|
1148
1163
|
ignoreInvalidURLs?: boolean,
|
|
1149
1164
|
): Promise<BatchScrapeResponse | ErrorResponse> {
|
|
1150
1165
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1151
|
-
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin:
|
|
1166
|
+
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
|
|
1152
1167
|
try {
|
|
1153
1168
|
const response: AxiosResponse = await this.postRequest(
|
|
1154
1169
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -1312,7 +1327,7 @@ export default class FirecrawlApp {
|
|
|
1312
1327
|
try {
|
|
1313
1328
|
const response: AxiosResponse = await this.postRequest(
|
|
1314
1329
|
this.apiUrl + `/v1/extract`,
|
|
1315
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
1330
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
1316
1331
|
headers
|
|
1317
1332
|
);
|
|
1318
1333
|
|
|
@@ -1372,7 +1387,7 @@ export default class FirecrawlApp {
|
|
|
1372
1387
|
try {
|
|
1373
1388
|
const response: AxiosResponse = await this.postRequest(
|
|
1374
1389
|
this.apiUrl + `/v1/extract`,
|
|
1375
|
-
{ ...jsonData, schema: jsonSchema, origin:
|
|
1390
|
+
{ ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
|
|
1376
1391
|
headers
|
|
1377
1392
|
);
|
|
1378
1393
|
|
|
@@ -1740,7 +1755,7 @@ export default class FirecrawlApp {
|
|
|
1740
1755
|
*/
|
|
1741
1756
|
async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1742
1757
|
const headers = this.prepareHeaders();
|
|
1743
|
-
let jsonData: any = { query, ...params, origin:
|
|
1758
|
+
let jsonData: any = { query, ...params, origin: this.resolveOrigin(params) };
|
|
1744
1759
|
|
|
1745
1760
|
if (jsonData?.jsonOptions?.schema) {
|
|
1746
1761
|
jsonData = {
|
|
@@ -1888,7 +1903,7 @@ export default class FirecrawlApp {
|
|
|
1888
1903
|
async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1889
1904
|
const headers = this.prepareHeaders();
|
|
1890
1905
|
try {
|
|
1891
|
-
let jsonData: any = { topic, ...params, origin:
|
|
1906
|
+
let jsonData: any = { topic, ...params, origin: this.resolveOrigin(params) };
|
|
1892
1907
|
const response: AxiosResponse = await this.postRequest(
|
|
1893
1908
|
`${this.apiUrl}/v1/deep-research`,
|
|
1894
1909
|
jsonData,
|
|
@@ -2003,7 +2018,7 @@ export default class FirecrawlApp {
|
|
|
2003
2018
|
*/
|
|
2004
2019
|
async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
|
|
2005
2020
|
const headers = this.prepareHeaders();
|
|
2006
|
-
let jsonData: any = { url, ...params, origin:
|
|
2021
|
+
let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
|
|
2007
2022
|
try {
|
|
2008
2023
|
const response: AxiosResponse = await this.postRequest(
|
|
2009
2024
|
`${this.apiUrl}/v1/llmstxt`,
|
package/src/v2/client.ts
CHANGED
|
@@ -101,6 +101,8 @@ import type {
|
|
|
101
101
|
GetMonitorCheckOptions,
|
|
102
102
|
UpdateMonitorRequest,
|
|
103
103
|
} from "./types";
|
|
104
|
+
import { defaultOrigin } from "../utils/callerOrigin";
|
|
105
|
+
import { getVersion } from "./utils/getVersion";
|
|
104
106
|
import { Watcher } from "./watcher";
|
|
105
107
|
import type { WatcherOptions } from "./watcher";
|
|
106
108
|
import * as zt from "zod";
|
|
@@ -130,6 +132,12 @@ export interface FirecrawlClientOptions {
|
|
|
130
132
|
maxRetries?: number;
|
|
131
133
|
/** Exponential backoff factor for retries (optional). */
|
|
132
134
|
backoffFactor?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
|
|
137
|
+
* or `js-sdk.<package>@<version>` when an installed package creates the client
|
|
138
|
+
* (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
|
|
139
|
+
*/
|
|
140
|
+
origin?: string;
|
|
133
141
|
}
|
|
134
142
|
|
|
135
143
|
/** Accepts a plain API key string or a full options object. */
|
|
@@ -140,7 +148,7 @@ export type FirecrawlClientInput = FirecrawlClientOptions | string;
|
|
|
140
148
|
*/
|
|
141
149
|
|
|
142
150
|
export class FirecrawlClient {
|
|
143
|
-
|
|
151
|
+
protected readonly http: HttpClient;
|
|
144
152
|
private _research?: ResearchClient;
|
|
145
153
|
|
|
146
154
|
private isCloudService(url: string): boolean {
|
|
@@ -168,6 +176,7 @@ export class FirecrawlClient {
|
|
|
168
176
|
timeoutMs: opts.timeoutMs,
|
|
169
177
|
maxRetries: opts.maxRetries,
|
|
170
178
|
backoffFactor: opts.backoffFactor,
|
|
179
|
+
origin: opts.origin || defaultOrigin(getVersion()),
|
|
171
180
|
});
|
|
172
181
|
}
|
|
173
182
|
|
|
@@ -644,7 +653,7 @@ export class FirecrawlClient {
|
|
|
644
653
|
// Browser
|
|
645
654
|
/**
|
|
646
655
|
* Create a new browser session.
|
|
647
|
-
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
|
|
656
|
+
* @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile, location).
|
|
648
657
|
* @returns Session id, CDP URL, live view URL, and expiration time.
|
|
649
658
|
*/
|
|
650
659
|
async browser(
|
|
@@ -22,6 +22,8 @@ export async function browser(
|
|
|
22
22
|
name: string;
|
|
23
23
|
saveChanges?: boolean;
|
|
24
24
|
};
|
|
25
|
+
/** Country the session browses from, as an ISO 3166-1 alpha-2 code (default US). */
|
|
26
|
+
location?: { country: string };
|
|
25
27
|
integration?: string;
|
|
26
28
|
origin?: string;
|
|
27
29
|
} = {},
|
|
@@ -32,6 +34,7 @@ export async function browser(
|
|
|
32
34
|
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
|
|
33
35
|
if (args.blockAds != null) body.blockAds = args.blockAds;
|
|
34
36
|
if (args.profile != null) body.profile = args.profile;
|
|
37
|
+
if (args.location != null) body.location = args.location;
|
|
35
38
|
if (args.integration != null) body.integration = args.integration;
|
|
36
39
|
if (args.origin) body.origin = args.origin;
|
|
37
40
|
|
package/src/v2/methods/crawl.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
SdkError,
|
|
11
11
|
} from "../types";
|
|
12
12
|
import { HttpClient } from "../utils/httpClient";
|
|
13
|
-
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
13
|
+
import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
|
|
14
14
|
import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
|
|
15
15
|
import type { HttpClient as _Http } from "../utils/httpClient";
|
|
16
16
|
import { fetchAllPages } from "../utils/pagination";
|
|
@@ -21,6 +21,7 @@ export type CrawlRequest = CrawlOptions & {
|
|
|
21
21
|
|
|
22
22
|
function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
|
|
23
23
|
if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
|
|
24
|
+
ensureNoV1Options(request, "crawl");
|
|
24
25
|
const data: Record<string, unknown> = { url: request.url.trim() };
|
|
25
26
|
if (request.prompt) data.prompt = request.prompt;
|
|
26
27
|
if (request.excludePaths) data.excludePaths = request.excludePaths;
|