firecrawl 4.49.2 → 4.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{chunk-JRVJFY7G.js → chunk-NXSNF7F3.js} +2 -2
- package/dist/index.cjs +96 -35
- package/dist/index.d.cts +25 -2
- package/dist/index.d.ts +25 -2
- package/dist/index.js +96 -35
- package/dist/{package-DW3K6CBX.js → package-KFOQKI5Z.js} +1 -1
- package/package.json +2 -2
- package/src/v1/index.ts +4 -1
- package/src/v2/client.ts +3 -2
- package/src/v2/utils/httpClient.ts +13 -0
- package/src/v2/watcher.ts +92 -30
- package/src/types/node-undici.d.ts +0 -10
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ npm install firecrawl
|
|
|
15
15
|
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
|
|
16
16
|
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.
|
|
17
17
|
|
|
18
|
-
Here's an example of how to use the SDK with error handling:
|
|
18
|
+
Here's an example of how to use the SDK with error handling. It uses ES modules and top-level `await`, so save it as an `.mjs` file or set `"type": "module"` in your `package.json`:
|
|
19
19
|
|
|
20
20
|
```js
|
|
21
21
|
import { Firecrawl } from 'firecrawl';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// package.json
|
|
2
2
|
var name = "@mendable/firecrawl-js";
|
|
3
|
-
var version = "4.
|
|
3
|
+
var version = "4.50.0";
|
|
4
4
|
var description = "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)";
|
|
5
5
|
var main = "dist/index.js";
|
|
6
6
|
var types = "dist/index.d.ts";
|
|
@@ -70,7 +70,7 @@ var keywords = [
|
|
|
70
70
|
"bioinformatics"
|
|
71
71
|
];
|
|
72
72
|
var engines = {
|
|
73
|
-
node: ">=
|
|
73
|
+
node: ">=18.0.0"
|
|
74
74
|
};
|
|
75
75
|
var package_default = {
|
|
76
76
|
name,
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.50.0",
|
|
43
43
|
description: "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -109,7 +109,7 @@ var require_package = __commonJS({
|
|
|
109
109
|
"bioinformatics"
|
|
110
110
|
],
|
|
111
111
|
engines: {
|
|
112
|
-
node: ">=
|
|
112
|
+
node: ">=18.0.0"
|
|
113
113
|
}
|
|
114
114
|
};
|
|
115
115
|
}
|
|
@@ -155,6 +155,15 @@ function pinToApiOrigin(apiUrl, url) {
|
|
|
155
155
|
}
|
|
156
156
|
|
|
157
157
|
// src/v2/utils/httpClient.ts
|
|
158
|
+
var rejectsDefaultCacheMode = () => {
|
|
159
|
+
if (typeof Request !== "function") return false;
|
|
160
|
+
try {
|
|
161
|
+
new Request("https://api.firecrawl.dev", { cache: "default" });
|
|
162
|
+
return false;
|
|
163
|
+
} catch {
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
};
|
|
158
167
|
var HttpClient = class {
|
|
159
168
|
instance;
|
|
160
169
|
apiKey;
|
|
@@ -177,7 +186,8 @@ var HttpClient = class {
|
|
|
177
186
|
// grants it when no Authorization header is present).
|
|
178
187
|
...this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
|
|
179
188
|
},
|
|
180
|
-
transitional: { clarifyTimeoutError: true }
|
|
189
|
+
transitional: { clarifyTimeoutError: true },
|
|
190
|
+
...rejectsDefaultCacheMode() ? { adapter: "fetch", env: { Request: null } } : {}
|
|
181
191
|
});
|
|
182
192
|
}
|
|
183
193
|
getApiUrl() {
|
|
@@ -2246,37 +2256,84 @@ function defaultOrigin(version2) {
|
|
|
2246
2256
|
}
|
|
2247
2257
|
|
|
2248
2258
|
// src/v2/watcher.ts
|
|
2249
|
-
var
|
|
2250
|
-
var hasGlobalWebSocket = () => {
|
|
2251
|
-
if (typeof globalThis === "undefined") return void 0;
|
|
2259
|
+
var getWebSocketCtor = () => {
|
|
2252
2260
|
const candidate = globalThis.WebSocket;
|
|
2253
2261
|
return typeof candidate === "function" ? candidate : void 0;
|
|
2254
2262
|
};
|
|
2255
|
-
var
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2263
|
+
var Emitter = class {
|
|
2264
|
+
listenersByEvent = /* @__PURE__ */ new Map();
|
|
2265
|
+
maxListeners = 10;
|
|
2266
|
+
on(event, listener) {
|
|
2267
|
+
this.listenersByEvent.set(event, [...this.listenersByEvent.get(event) ?? [], listener]);
|
|
2268
|
+
return this;
|
|
2269
|
+
}
|
|
2270
|
+
prependListener(event, listener) {
|
|
2271
|
+
this.listenersByEvent.set(event, [listener, ...this.listenersByEvent.get(event) ?? []]);
|
|
2272
|
+
return this;
|
|
2273
|
+
}
|
|
2274
|
+
prependOnceListener(event, listener) {
|
|
2275
|
+
return this.prependListener(event, this.onceWrapper(event, listener));
|
|
2276
|
+
}
|
|
2277
|
+
listeners(event) {
|
|
2278
|
+
return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);
|
|
2279
|
+
}
|
|
2280
|
+
rawListeners(event) {
|
|
2281
|
+
return [...this.listenersByEvent.get(event) ?? []];
|
|
2282
|
+
}
|
|
2283
|
+
eventNames() {
|
|
2284
|
+
return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);
|
|
2285
|
+
}
|
|
2286
|
+
setMaxListeners(n) {
|
|
2287
|
+
this.maxListeners = n;
|
|
2288
|
+
return this;
|
|
2289
|
+
}
|
|
2290
|
+
getMaxListeners() {
|
|
2291
|
+
return this.maxListeners;
|
|
2292
|
+
}
|
|
2293
|
+
addListener(event, listener) {
|
|
2294
|
+
return this.on(event, listener);
|
|
2295
|
+
}
|
|
2296
|
+
once(event, listener) {
|
|
2297
|
+
return this.on(event, this.onceWrapper(event, listener));
|
|
2298
|
+
}
|
|
2299
|
+
onceWrapper(event, listener) {
|
|
2300
|
+
const wrapped = (...args) => {
|
|
2301
|
+
this.off(event, wrapped);
|
|
2302
|
+
listener.apply(this, args);
|
|
2303
|
+
};
|
|
2304
|
+
wrapped.listener = listener;
|
|
2305
|
+
return wrapped;
|
|
2306
|
+
}
|
|
2307
|
+
off(event, listener) {
|
|
2308
|
+
const current = this.listenersByEvent.get(event);
|
|
2309
|
+
if (!current) return this;
|
|
2310
|
+
for (let index = current.length - 1; index >= 0; index--) {
|
|
2311
|
+
if (current[index] !== listener && current[index].listener !== listener) continue;
|
|
2312
|
+
this.listenersByEvent.set(event, current.filter((_, i) => i !== index));
|
|
2313
|
+
break;
|
|
2314
|
+
}
|
|
2315
|
+
return this;
|
|
2316
|
+
}
|
|
2317
|
+
removeListener(event, listener) {
|
|
2318
|
+
return this.off(event, listener);
|
|
2319
|
+
}
|
|
2320
|
+
removeAllListeners(event) {
|
|
2321
|
+
if (event === void 0) this.listenersByEvent.clear();
|
|
2322
|
+
else this.listenersByEvent.delete(event);
|
|
2323
|
+
return this;
|
|
2324
|
+
}
|
|
2325
|
+
listenerCount(event) {
|
|
2326
|
+
return this.listenersByEvent.get(event)?.length ?? 0;
|
|
2327
|
+
}
|
|
2328
|
+
emit(event, ...args) {
|
|
2329
|
+
const current = this.listenersByEvent.get(event);
|
|
2330
|
+
if (!current?.length) {
|
|
2331
|
+
if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));
|
|
2332
|
+
return false;
|
|
2333
|
+
}
|
|
2334
|
+
for (const listener of [...current]) listener.apply(this, args);
|
|
2335
|
+
return true;
|
|
2266
2336
|
}
|
|
2267
|
-
};
|
|
2268
|
-
var getWebSocketCtor = async () => {
|
|
2269
|
-
if (cachedWebSocket) return cachedWebSocket;
|
|
2270
|
-
const globalWs = hasGlobalWebSocket();
|
|
2271
|
-
if (globalWs) {
|
|
2272
|
-
cachedWebSocket = globalWs;
|
|
2273
|
-
return cachedWebSocket;
|
|
2274
|
-
}
|
|
2275
|
-
if (!loadPromise) {
|
|
2276
|
-
loadPromise = loadNodeWebSocket();
|
|
2277
|
-
}
|
|
2278
|
-
cachedWebSocket = await loadPromise;
|
|
2279
|
-
return cachedWebSocket;
|
|
2280
2337
|
};
|
|
2281
2338
|
var decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : void 0;
|
|
2282
2339
|
var ensureUtf8String = (data) => {
|
|
@@ -2298,7 +2355,7 @@ var ensureUtf8String = (data) => {
|
|
|
2298
2355
|
}
|
|
2299
2356
|
return void 0;
|
|
2300
2357
|
};
|
|
2301
|
-
var Watcher = class extends
|
|
2358
|
+
var Watcher = class extends Emitter {
|
|
2302
2359
|
http;
|
|
2303
2360
|
jobId;
|
|
2304
2361
|
kind;
|
|
@@ -2340,7 +2397,7 @@ var Watcher = class extends import_events.EventEmitter {
|
|
|
2340
2397
|
(async () => {
|
|
2341
2398
|
try {
|
|
2342
2399
|
const url = this.buildWsUrl();
|
|
2343
|
-
const wsCtor =
|
|
2400
|
+
const wsCtor = getWebSocketCtor();
|
|
2344
2401
|
if (!wsCtor) {
|
|
2345
2402
|
this.pollLoop();
|
|
2346
2403
|
return;
|
|
@@ -2526,8 +2583,9 @@ var FirecrawlClient = class {
|
|
|
2526
2583
|
*/
|
|
2527
2584
|
constructor(options = {}) {
|
|
2528
2585
|
const opts = typeof options === "string" ? { apiKey: options } : options;
|
|
2529
|
-
const
|
|
2530
|
-
const
|
|
2586
|
+
const hasEnv = typeof process !== "undefined" && !!process.env;
|
|
2587
|
+
const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : void 0) ?? "").trim();
|
|
2588
|
+
const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : void 0) ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
|
2531
2589
|
this.http = new HttpClient({
|
|
2532
2590
|
apiKey,
|
|
2533
2591
|
apiUrl,
|
|
@@ -3073,7 +3131,7 @@ var FirecrawlApp = class {
|
|
|
3073
3131
|
const packageJson = await Promise.resolve().then(() => __toESM(require_package(), 1));
|
|
3074
3132
|
return packageJson.default.version;
|
|
3075
3133
|
} catch (error) {
|
|
3076
|
-
const isTest = typeof process !== "undefined" && false;
|
|
3134
|
+
const isTest = typeof process !== "undefined" && !!process.env && false;
|
|
3077
3135
|
if (!isTest) {
|
|
3078
3136
|
console.error("Error getting version:", error);
|
|
3079
3137
|
}
|
|
@@ -4362,6 +4420,9 @@ var CrawlWatcher = class extends e {
|
|
|
4362
4420
|
super();
|
|
4363
4421
|
this.id = id;
|
|
4364
4422
|
const wsUrl = app.apiUrl.replace(/^http/, "ws");
|
|
4423
|
+
if (typeof WebSocket === "undefined") {
|
|
4424
|
+
throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");
|
|
4425
|
+
}
|
|
4365
4426
|
this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
4366
4427
|
this.status = "scraping";
|
|
4367
4428
|
this.data = [];
|
package/dist/index.d.cts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as zt from 'zod';
|
|
2
2
|
import { ZodTypeAny } from 'zod';
|
|
3
3
|
import { AxiosResponse, AxiosRequestHeaders } from 'axios';
|
|
4
|
-
import { EventEmitter } from 'events';
|
|
5
4
|
import { TypedEventTarget } from 'typescript-event-target';
|
|
6
5
|
|
|
7
6
|
type FormatString = "markdown" | "html" | "rawHtml" | "links" | "images" | "screenshot" | "summary" | "changeTracking" | "json" | "attributes" | "branding" | "product" | "menu" | "audio" | "video";
|
|
@@ -2222,13 +2221,37 @@ declare class ResearchClient {
|
|
|
2222
2221
|
searchGithub(query: string, options?: SearchGithubOptions): Promise<GitHubSearchResponse>;
|
|
2223
2222
|
}
|
|
2224
2223
|
|
|
2224
|
+
type Listener = ((...args: any[]) => void) & {
|
|
2225
|
+
listener?: Listener;
|
|
2226
|
+
};
|
|
2227
|
+
/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */
|
|
2228
|
+
declare class Emitter {
|
|
2229
|
+
private readonly listenersByEvent;
|
|
2230
|
+
private maxListeners;
|
|
2231
|
+
on(event: string | symbol, listener: Listener): this;
|
|
2232
|
+
prependListener(event: string | symbol, listener: Listener): this;
|
|
2233
|
+
prependOnceListener(event: string | symbol, listener: Listener): this;
|
|
2234
|
+
listeners(event: string | symbol): Listener[];
|
|
2235
|
+
rawListeners(event: string | symbol): Listener[];
|
|
2236
|
+
eventNames(): (string | symbol)[];
|
|
2237
|
+
setMaxListeners(n: number): this;
|
|
2238
|
+
getMaxListeners(): number;
|
|
2239
|
+
addListener(event: string | symbol, listener: Listener): this;
|
|
2240
|
+
once(event: string | symbol, listener: Listener): this;
|
|
2241
|
+
private onceWrapper;
|
|
2242
|
+
off(event: string | symbol, listener: Listener): this;
|
|
2243
|
+
removeListener(event: string | symbol, listener: Listener): this;
|
|
2244
|
+
removeAllListeners(event?: string | symbol): this;
|
|
2245
|
+
listenerCount(event: string | symbol): number;
|
|
2246
|
+
emit(event: string | symbol, ...args: any[]): boolean;
|
|
2247
|
+
}
|
|
2225
2248
|
type JobKind = "crawl" | "batch";
|
|
2226
2249
|
interface WatcherOptions {
|
|
2227
2250
|
kind?: JobKind;
|
|
2228
2251
|
pollInterval?: number;
|
|
2229
2252
|
timeout?: number;
|
|
2230
2253
|
}
|
|
2231
|
-
declare class Watcher extends
|
|
2254
|
+
declare class Watcher extends Emitter {
|
|
2232
2255
|
private readonly http;
|
|
2233
2256
|
private readonly jobId;
|
|
2234
2257
|
private readonly kind;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as zt from 'zod';
|
|
2
2
|
import { ZodTypeAny } from 'zod';
|
|
3
3
|
import { AxiosResponse, AxiosRequestHeaders } from 'axios';
|
|
4
|
-
import { EventEmitter } from 'events';
|
|
5
4
|
import { TypedEventTarget } from 'typescript-event-target';
|
|
6
5
|
|
|
7
6
|
type FormatString = "markdown" | "html" | "rawHtml" | "links" | "images" | "screenshot" | "summary" | "changeTracking" | "json" | "attributes" | "branding" | "product" | "menu" | "audio" | "video";
|
|
@@ -2222,13 +2221,37 @@ declare class ResearchClient {
|
|
|
2222
2221
|
searchGithub(query: string, options?: SearchGithubOptions): Promise<GitHubSearchResponse>;
|
|
2223
2222
|
}
|
|
2224
2223
|
|
|
2224
|
+
type Listener = ((...args: any[]) => void) & {
|
|
2225
|
+
listener?: Listener;
|
|
2226
|
+
};
|
|
2227
|
+
/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */
|
|
2228
|
+
declare class Emitter {
|
|
2229
|
+
private readonly listenersByEvent;
|
|
2230
|
+
private maxListeners;
|
|
2231
|
+
on(event: string | symbol, listener: Listener): this;
|
|
2232
|
+
prependListener(event: string | symbol, listener: Listener): this;
|
|
2233
|
+
prependOnceListener(event: string | symbol, listener: Listener): this;
|
|
2234
|
+
listeners(event: string | symbol): Listener[];
|
|
2235
|
+
rawListeners(event: string | symbol): Listener[];
|
|
2236
|
+
eventNames(): (string | symbol)[];
|
|
2237
|
+
setMaxListeners(n: number): this;
|
|
2238
|
+
getMaxListeners(): number;
|
|
2239
|
+
addListener(event: string | symbol, listener: Listener): this;
|
|
2240
|
+
once(event: string | symbol, listener: Listener): this;
|
|
2241
|
+
private onceWrapper;
|
|
2242
|
+
off(event: string | symbol, listener: Listener): this;
|
|
2243
|
+
removeListener(event: string | symbol, listener: Listener): this;
|
|
2244
|
+
removeAllListeners(event?: string | symbol): this;
|
|
2245
|
+
listenerCount(event: string | symbol): number;
|
|
2246
|
+
emit(event: string | symbol, ...args: any[]): boolean;
|
|
2247
|
+
}
|
|
2225
2248
|
type JobKind = "crawl" | "batch";
|
|
2226
2249
|
interface WatcherOptions {
|
|
2227
2250
|
kind?: JobKind;
|
|
2228
2251
|
pollInterval?: number;
|
|
2229
2252
|
timeout?: number;
|
|
2230
2253
|
}
|
|
2231
|
-
declare class Watcher extends
|
|
2254
|
+
declare class Watcher extends Emitter {
|
|
2232
2255
|
private readonly http;
|
|
2233
2256
|
private readonly jobId;
|
|
2234
2257
|
private readonly kind;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
version
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-NXSNF7F3.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -26,6 +26,15 @@ function pinToApiOrigin(apiUrl, url) {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
// src/v2/utils/httpClient.ts
|
|
29
|
+
var rejectsDefaultCacheMode = () => {
|
|
30
|
+
if (typeof Request !== "function") return false;
|
|
31
|
+
try {
|
|
32
|
+
new Request("https://api.firecrawl.dev", { cache: "default" });
|
|
33
|
+
return false;
|
|
34
|
+
} catch {
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
29
38
|
var HttpClient = class {
|
|
30
39
|
instance;
|
|
31
40
|
apiKey;
|
|
@@ -48,7 +57,8 @@ var HttpClient = class {
|
|
|
48
57
|
// grants it when no Authorization header is present).
|
|
49
58
|
...this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
|
|
50
59
|
},
|
|
51
|
-
transitional: { clarifyTimeoutError: true }
|
|
60
|
+
transitional: { clarifyTimeoutError: true },
|
|
61
|
+
...rejectsDefaultCacheMode() ? { adapter: "fetch", env: { Request: null } } : {}
|
|
52
62
|
});
|
|
53
63
|
}
|
|
54
64
|
getApiUrl() {
|
|
@@ -2117,37 +2127,84 @@ function defaultOrigin(version2) {
|
|
|
2117
2127
|
}
|
|
2118
2128
|
|
|
2119
2129
|
// src/v2/watcher.ts
|
|
2120
|
-
|
|
2121
|
-
var hasGlobalWebSocket = () => {
|
|
2122
|
-
if (typeof globalThis === "undefined") return void 0;
|
|
2130
|
+
var getWebSocketCtor = () => {
|
|
2123
2131
|
const candidate = globalThis.WebSocket;
|
|
2124
2132
|
return typeof candidate === "function" ? candidate : void 0;
|
|
2125
2133
|
};
|
|
2126
|
-
var
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2134
|
+
var Emitter = class {
|
|
2135
|
+
listenersByEvent = /* @__PURE__ */ new Map();
|
|
2136
|
+
maxListeners = 10;
|
|
2137
|
+
on(event, listener) {
|
|
2138
|
+
this.listenersByEvent.set(event, [...this.listenersByEvent.get(event) ?? [], listener]);
|
|
2139
|
+
return this;
|
|
2140
|
+
}
|
|
2141
|
+
prependListener(event, listener) {
|
|
2142
|
+
this.listenersByEvent.set(event, [listener, ...this.listenersByEvent.get(event) ?? []]);
|
|
2143
|
+
return this;
|
|
2144
|
+
}
|
|
2145
|
+
prependOnceListener(event, listener) {
|
|
2146
|
+
return this.prependListener(event, this.onceWrapper(event, listener));
|
|
2147
|
+
}
|
|
2148
|
+
listeners(event) {
|
|
2149
|
+
return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);
|
|
2150
|
+
}
|
|
2151
|
+
rawListeners(event) {
|
|
2152
|
+
return [...this.listenersByEvent.get(event) ?? []];
|
|
2153
|
+
}
|
|
2154
|
+
eventNames() {
|
|
2155
|
+
return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);
|
|
2156
|
+
}
|
|
2157
|
+
setMaxListeners(n) {
|
|
2158
|
+
this.maxListeners = n;
|
|
2159
|
+
return this;
|
|
2160
|
+
}
|
|
2161
|
+
getMaxListeners() {
|
|
2162
|
+
return this.maxListeners;
|
|
2163
|
+
}
|
|
2164
|
+
addListener(event, listener) {
|
|
2165
|
+
return this.on(event, listener);
|
|
2166
|
+
}
|
|
2167
|
+
once(event, listener) {
|
|
2168
|
+
return this.on(event, this.onceWrapper(event, listener));
|
|
2169
|
+
}
|
|
2170
|
+
onceWrapper(event, listener) {
|
|
2171
|
+
const wrapped = (...args) => {
|
|
2172
|
+
this.off(event, wrapped);
|
|
2173
|
+
listener.apply(this, args);
|
|
2174
|
+
};
|
|
2175
|
+
wrapped.listener = listener;
|
|
2176
|
+
return wrapped;
|
|
2177
|
+
}
|
|
2178
|
+
off(event, listener) {
|
|
2179
|
+
const current = this.listenersByEvent.get(event);
|
|
2180
|
+
if (!current) return this;
|
|
2181
|
+
for (let index = current.length - 1; index >= 0; index--) {
|
|
2182
|
+
if (current[index] !== listener && current[index].listener !== listener) continue;
|
|
2183
|
+
this.listenersByEvent.set(event, current.filter((_, i) => i !== index));
|
|
2184
|
+
break;
|
|
2185
|
+
}
|
|
2186
|
+
return this;
|
|
2187
|
+
}
|
|
2188
|
+
removeListener(event, listener) {
|
|
2189
|
+
return this.off(event, listener);
|
|
2190
|
+
}
|
|
2191
|
+
removeAllListeners(event) {
|
|
2192
|
+
if (event === void 0) this.listenersByEvent.clear();
|
|
2193
|
+
else this.listenersByEvent.delete(event);
|
|
2194
|
+
return this;
|
|
2195
|
+
}
|
|
2196
|
+
listenerCount(event) {
|
|
2197
|
+
return this.listenersByEvent.get(event)?.length ?? 0;
|
|
2198
|
+
}
|
|
2199
|
+
emit(event, ...args) {
|
|
2200
|
+
const current = this.listenersByEvent.get(event);
|
|
2201
|
+
if (!current?.length) {
|
|
2202
|
+
if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));
|
|
2203
|
+
return false;
|
|
2204
|
+
}
|
|
2205
|
+
for (const listener of [...current]) listener.apply(this, args);
|
|
2206
|
+
return true;
|
|
2137
2207
|
}
|
|
2138
|
-
};
|
|
2139
|
-
var getWebSocketCtor = async () => {
|
|
2140
|
-
if (cachedWebSocket) return cachedWebSocket;
|
|
2141
|
-
const globalWs = hasGlobalWebSocket();
|
|
2142
|
-
if (globalWs) {
|
|
2143
|
-
cachedWebSocket = globalWs;
|
|
2144
|
-
return cachedWebSocket;
|
|
2145
|
-
}
|
|
2146
|
-
if (!loadPromise) {
|
|
2147
|
-
loadPromise = loadNodeWebSocket();
|
|
2148
|
-
}
|
|
2149
|
-
cachedWebSocket = await loadPromise;
|
|
2150
|
-
return cachedWebSocket;
|
|
2151
2208
|
};
|
|
2152
2209
|
var decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : void 0;
|
|
2153
2210
|
var ensureUtf8String = (data) => {
|
|
@@ -2169,7 +2226,7 @@ var ensureUtf8String = (data) => {
|
|
|
2169
2226
|
}
|
|
2170
2227
|
return void 0;
|
|
2171
2228
|
};
|
|
2172
|
-
var Watcher = class extends
|
|
2229
|
+
var Watcher = class extends Emitter {
|
|
2173
2230
|
http;
|
|
2174
2231
|
jobId;
|
|
2175
2232
|
kind;
|
|
@@ -2211,7 +2268,7 @@ var Watcher = class extends EventEmitter {
|
|
|
2211
2268
|
(async () => {
|
|
2212
2269
|
try {
|
|
2213
2270
|
const url = this.buildWsUrl();
|
|
2214
|
-
const wsCtor =
|
|
2271
|
+
const wsCtor = getWebSocketCtor();
|
|
2215
2272
|
if (!wsCtor) {
|
|
2216
2273
|
this.pollLoop();
|
|
2217
2274
|
return;
|
|
@@ -2397,8 +2454,9 @@ var FirecrawlClient = class {
|
|
|
2397
2454
|
*/
|
|
2398
2455
|
constructor(options = {}) {
|
|
2399
2456
|
const opts = typeof options === "string" ? { apiKey: options } : options;
|
|
2400
|
-
const
|
|
2401
|
-
const
|
|
2457
|
+
const hasEnv = typeof process !== "undefined" && !!process.env;
|
|
2458
|
+
const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : void 0) ?? "").trim();
|
|
2459
|
+
const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : void 0) ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
|
2402
2460
|
this.http = new HttpClient({
|
|
2403
2461
|
apiKey,
|
|
2404
2462
|
apiUrl,
|
|
@@ -2941,10 +2999,10 @@ var FirecrawlApp = class {
|
|
|
2941
2999
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2942
3000
|
return process.env.npm_package_version;
|
|
2943
3001
|
}
|
|
2944
|
-
const packageJson = await import("./package-
|
|
3002
|
+
const packageJson = await import("./package-KFOQKI5Z.js");
|
|
2945
3003
|
return packageJson.default.version;
|
|
2946
3004
|
} catch (error) {
|
|
2947
|
-
const isTest = typeof process !== "undefined" && false;
|
|
3005
|
+
const isTest = typeof process !== "undefined" && !!process.env && false;
|
|
2948
3006
|
if (!isTest) {
|
|
2949
3007
|
console.error("Error getting version:", error);
|
|
2950
3008
|
}
|
|
@@ -4233,6 +4291,9 @@ var CrawlWatcher = class extends e {
|
|
|
4233
4291
|
super();
|
|
4234
4292
|
this.id = id;
|
|
4235
4293
|
const wsUrl = app.apiUrl.replace(/^http/, "ws");
|
|
4294
|
+
if (typeof WebSocket === "undefined") {
|
|
4295
|
+
throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");
|
|
4296
|
+
}
|
|
4236
4297
|
this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
4237
4298
|
this.status = "scraping";
|
|
4238
4299
|
this.data = [];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.50.0",
|
|
4
4
|
"description": "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
"bioinformatics"
|
|
63
63
|
],
|
|
64
64
|
"engines": {
|
|
65
|
-
"node": ">=
|
|
65
|
+
"node": ">=18.0.0"
|
|
66
66
|
},
|
|
67
67
|
"scripts": {
|
|
68
68
|
"build": "tsup",
|
package/src/v1/index.ts
CHANGED
|
@@ -686,7 +686,7 @@ export default class FirecrawlApp {
|
|
|
686
686
|
return packageJson.default.version;
|
|
687
687
|
} catch (error) {
|
|
688
688
|
// Suppress noisy logs under test environments
|
|
689
|
-
const isTest = typeof process !== 'undefined' && process.env.NODE_ENV === 'test';
|
|
689
|
+
const isTest = typeof process !== 'undefined' && !!process.env && process.env.NODE_ENV === 'test';
|
|
690
690
|
if (!isTest) {
|
|
691
691
|
// eslint-disable-next-line no-console
|
|
692
692
|
console.error("Error getting version:", error);
|
|
@@ -2220,6 +2220,9 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
|
|
|
2220
2220
|
this.id = id;
|
|
2221
2221
|
// replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)
|
|
2222
2222
|
const wsUrl = app.apiUrl.replace(/^http/, "ws");
|
|
2223
|
+
if (typeof WebSocket === "undefined") {
|
|
2224
|
+
throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");
|
|
2225
|
+
}
|
|
2223
2226
|
this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
2224
2227
|
this.status = "scraping";
|
|
2225
2228
|
this.data = [];
|
package/src/v2/client.ts
CHANGED
|
@@ -164,8 +164,9 @@ export class FirecrawlClient {
|
|
|
164
164
|
const opts: FirecrawlClientOptions =
|
|
165
165
|
typeof options === "string" ? { apiKey: options } : options;
|
|
166
166
|
|
|
167
|
-
const
|
|
168
|
-
const
|
|
167
|
+
const hasEnv = typeof process !== "undefined" && !!process.env;
|
|
168
|
+
const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : undefined) ?? "").trim();
|
|
169
|
+
const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : undefined) ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
|
169
170
|
|
|
170
171
|
// No API key is allowed: scrape, search, and interact fall back to the
|
|
171
172
|
// keyless free tier (rate-limited per IP). Other methods will return 401
|
|
@@ -6,6 +6,18 @@ import axios, {
|
|
|
6
6
|
import { getVersion } from "./getVersion";
|
|
7
7
|
import { pinToApiOrigin } from "../../utils/apiOrigin";
|
|
8
8
|
|
|
9
|
+
// Runtimes such as Cloudflare Workers reject the Request `cache: "default"` that
|
|
10
|
+
// axios's fetch adapter always sets; there axios calls fetch(url, init) without it.
|
|
11
|
+
const rejectsDefaultCacheMode = (): boolean => {
|
|
12
|
+
if (typeof Request !== "function") return false;
|
|
13
|
+
try {
|
|
14
|
+
new Request("https://api.firecrawl.dev", { cache: "default" });
|
|
15
|
+
return false;
|
|
16
|
+
} catch {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
|
|
9
21
|
export interface HttpClientOptions {
|
|
10
22
|
apiKey: string;
|
|
11
23
|
apiUrl: string;
|
|
@@ -44,6 +56,7 @@ export class HttpClient {
|
|
|
44
56
|
...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
|
|
45
57
|
},
|
|
46
58
|
transitional: { clarifyTimeoutError: true },
|
|
59
|
+
...(rejectsDefaultCacheMode() ? { adapter: "fetch", env: { Request: null as any } } : {}),
|
|
47
60
|
});
|
|
48
61
|
}
|
|
49
62
|
|
package/src/v2/watcher.ts
CHANGED
|
@@ -1,48 +1,110 @@
|
|
|
1
|
-
import { EventEmitter } from "events";
|
|
2
1
|
import type { BatchScrapeJob, CrawlJob, Document } from "./types";
|
|
3
2
|
import type { HttpClient } from "./utils/httpClient";
|
|
4
3
|
import { getBatchScrapeStatus } from "./methods/batch";
|
|
5
4
|
import { getCrawlStatus } from "./methods/crawl";
|
|
6
|
-
// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without
|
|
7
|
-
// experimental flags) do not. We lazily fall back to node:undici.
|
|
8
5
|
|
|
9
6
|
type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;
|
|
10
7
|
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
// Runtimes without a global WebSocket (Node before 22) poll instead.
|
|
9
|
+
const getWebSocketCtor = (): WebSocketConstructor | undefined => {
|
|
13
10
|
const candidate = (globalThis as any).WebSocket;
|
|
14
11
|
return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;
|
|
15
12
|
};
|
|
16
13
|
|
|
17
|
-
|
|
14
|
+
type Listener = ((...args: any[]) => void) & { listener?: Listener };
|
|
18
15
|
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */
|
|
17
|
+
class Emitter {
|
|
18
|
+
private readonly listenersByEvent = new Map<string | symbol, Listener[]>();
|
|
19
|
+
private maxListeners = 10;
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const undici = await import("node:undici");
|
|
26
|
-
const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;
|
|
27
|
-
return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;
|
|
28
|
-
} catch {
|
|
29
|
-
return undefined;
|
|
21
|
+
on(event: string | symbol, listener: Listener): this {
|
|
22
|
+
this.listenersByEvent.set(event, [...(this.listenersByEvent.get(event) ?? []), listener]);
|
|
23
|
+
return this;
|
|
30
24
|
}
|
|
31
|
-
};
|
|
32
25
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if (globalWs) {
|
|
37
|
-
cachedWebSocket = globalWs;
|
|
38
|
-
return cachedWebSocket;
|
|
26
|
+
prependListener(event: string | symbol, listener: Listener): this {
|
|
27
|
+
this.listenersByEvent.set(event, [listener, ...(this.listenersByEvent.get(event) ?? [])]);
|
|
28
|
+
return this;
|
|
39
29
|
}
|
|
40
|
-
|
|
41
|
-
|
|
30
|
+
|
|
31
|
+
prependOnceListener(event: string | symbol, listener: Listener): this {
|
|
32
|
+
return this.prependListener(event, this.onceWrapper(event, listener));
|
|
42
33
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
34
|
+
|
|
35
|
+
listeners(event: string | symbol): Listener[] {
|
|
36
|
+
return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
rawListeners(event: string | symbol): Listener[] {
|
|
40
|
+
return [...(this.listenersByEvent.get(event) ?? [])];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
eventNames(): (string | symbol)[] {
|
|
44
|
+
return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
setMaxListeners(n: number): this {
|
|
48
|
+
this.maxListeners = n;
|
|
49
|
+
return this;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
getMaxListeners(): number {
|
|
53
|
+
return this.maxListeners;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
addListener(event: string | symbol, listener: Listener): this {
|
|
57
|
+
return this.on(event, listener);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
once(event: string | symbol, listener: Listener): this {
|
|
61
|
+
return this.on(event, this.onceWrapper(event, listener));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
private onceWrapper(event: string | symbol, listener: Listener): Listener {
|
|
65
|
+
const wrapped: Listener = (...args) => {
|
|
66
|
+
this.off(event, wrapped);
|
|
67
|
+
listener.apply(this, args);
|
|
68
|
+
};
|
|
69
|
+
wrapped.listener = listener;
|
|
70
|
+
return wrapped;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
off(event: string | symbol, listener: Listener): this {
|
|
74
|
+
const current = this.listenersByEvent.get(event);
|
|
75
|
+
if (!current) return this;
|
|
76
|
+
for (let index = current.length - 1; index >= 0; index--) {
|
|
77
|
+
if (current[index] !== listener && current[index]!.listener !== listener) continue;
|
|
78
|
+
this.listenersByEvent.set(event, current.filter((_, i) => i !== index));
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
return this;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
removeListener(event: string | symbol, listener: Listener): this {
|
|
85
|
+
return this.off(event, listener);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
removeAllListeners(event?: string | symbol): this {
|
|
89
|
+
if (event === undefined) this.listenersByEvent.clear();
|
|
90
|
+
else this.listenersByEvent.delete(event);
|
|
91
|
+
return this;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
listenerCount(event: string | symbol): number {
|
|
95
|
+
return this.listenersByEvent.get(event)?.length ?? 0;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
emit(event: string | symbol, ...args: any[]): boolean {
|
|
99
|
+
const current = this.listenersByEvent.get(event);
|
|
100
|
+
if (!current?.length) {
|
|
101
|
+
if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
for (const listener of [...current]) listener.apply(this, args);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
46
108
|
|
|
47
109
|
const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;
|
|
48
110
|
|
|
@@ -81,7 +143,7 @@ export interface WatcherOptions {
|
|
|
81
143
|
|
|
82
144
|
type Snapshot = CrawlJob | BatchScrapeJob;
|
|
83
145
|
|
|
84
|
-
export class Watcher extends
|
|
146
|
+
export class Watcher extends Emitter {
|
|
85
147
|
private readonly http: HttpClient;
|
|
86
148
|
private readonly jobId: string;
|
|
87
149
|
private readonly kind: JobKind;
|
|
@@ -122,7 +184,7 @@ export class Watcher extends EventEmitter {
|
|
|
122
184
|
(async () => {
|
|
123
185
|
try {
|
|
124
186
|
const url = this.buildWsUrl();
|
|
125
|
-
const wsCtor =
|
|
187
|
+
const wsCtor = getWebSocketCtor();
|
|
126
188
|
if (!wsCtor) {
|
|
127
189
|
this.pollLoop();
|
|
128
190
|
return;
|