@mendable/firecrawl-js 4.49.2 → 4.50.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{chunk-JRVJFY7G.js → chunk-UGAMW2BL.js} +3 -3
- package/dist/index.cjs +99 -36
- package/dist/index.d.cts +36 -3
- package/dist/index.d.ts +36 -3
- package/dist/index.js +98 -35
- package/dist/{package-DW3K6CBX.js → package-TFTOUFDX.js} +1 -1
- package/package.json +4 -4
- package/pnpm-workspace.yaml +3 -1
- package/src/__tests__/unit/v2/agent-location.unit.test.ts +34 -0
- package/src/v1/index.ts +4 -1
- package/src/v2/client.ts +3 -2
- package/src/v2/methods/agent.ts +8 -0
- package/src/v2/types.ts +52 -12
- package/src/v2/utils/httpClient.ts +13 -0
- package/src/v2/watcher.ts +92 -30
- package/src/types/node-undici.d.ts +0 -10
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ npm install firecrawl
|
|
|
15
15
|
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
|
|
16
16
|
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.
|
|
17
17
|
|
|
18
|
-
Here's an example of how to use the SDK with error handling:
|
|
18
|
+
Here's an example of how to use the SDK with error handling. It uses ES modules and top-level `await`, so save it as an `.mjs` file or set `"type": "module"` in your `package.json`:
|
|
19
19
|
|
|
20
20
|
```js
|
|
21
21
|
import { Firecrawl } from 'firecrawl';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// package.json
|
|
2
2
|
var name = "@mendable/firecrawl-js";
|
|
3
|
-
var version = "4.
|
|
3
|
+
var version = "4.50.1";
|
|
4
4
|
var description = "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)";
|
|
5
5
|
var main = "dist/index.js";
|
|
6
6
|
var types = "dist/index.d.ts";
|
|
@@ -44,7 +44,7 @@ var devDependencies = {
|
|
|
44
44
|
tsup: "^8.5.0",
|
|
45
45
|
typescript: "^5.4.5",
|
|
46
46
|
uuid: "^14.0.0",
|
|
47
|
-
vitest: "^4.1.
|
|
47
|
+
vitest: "^4.1.12"
|
|
48
48
|
};
|
|
49
49
|
var keywords = [
|
|
50
50
|
"firecrawl",
|
|
@@ -70,7 +70,7 @@ var keywords = [
|
|
|
70
70
|
"bioinformatics"
|
|
71
71
|
];
|
|
72
72
|
var engines = {
|
|
73
|
-
node: ">=
|
|
73
|
+
node: ">=18.0.0"
|
|
74
74
|
};
|
|
75
75
|
var package_default = {
|
|
76
76
|
name,
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.50.1",
|
|
43
43
|
description: "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -83,7 +83,7 @@ var require_package = __commonJS({
|
|
|
83
83
|
tsup: "^8.5.0",
|
|
84
84
|
typescript: "^5.4.5",
|
|
85
85
|
uuid: "^14.0.0",
|
|
86
|
-
vitest: "^4.1.
|
|
86
|
+
vitest: "^4.1.12"
|
|
87
87
|
},
|
|
88
88
|
keywords: [
|
|
89
89
|
"firecrawl",
|
|
@@ -109,7 +109,7 @@ var require_package = __commonJS({
|
|
|
109
109
|
"bioinformatics"
|
|
110
110
|
],
|
|
111
111
|
engines: {
|
|
112
|
-
node: ">=
|
|
112
|
+
node: ">=18.0.0"
|
|
113
113
|
}
|
|
114
114
|
};
|
|
115
115
|
}
|
|
@@ -155,6 +155,15 @@ function pinToApiOrigin(apiUrl, url) {
|
|
|
155
155
|
}
|
|
156
156
|
|
|
157
157
|
// src/v2/utils/httpClient.ts
|
|
158
|
+
var rejectsDefaultCacheMode = () => {
|
|
159
|
+
if (typeof Request !== "function") return false;
|
|
160
|
+
try {
|
|
161
|
+
new Request("https://api.firecrawl.dev", { cache: "default" });
|
|
162
|
+
return false;
|
|
163
|
+
} catch {
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
};
|
|
158
167
|
var HttpClient = class {
|
|
159
168
|
instance;
|
|
160
169
|
apiKey;
|
|
@@ -177,7 +186,8 @@ var HttpClient = class {
|
|
|
177
186
|
// grants it when no Authorization header is present).
|
|
178
187
|
...this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
|
|
179
188
|
},
|
|
180
|
-
transitional: { clarifyTimeoutError: true }
|
|
189
|
+
transitional: { clarifyTimeoutError: true },
|
|
190
|
+
...rejectsDefaultCacheMode() ? { adapter: "fetch", env: { Request: null } } : {}
|
|
181
191
|
});
|
|
182
192
|
}
|
|
183
193
|
getApiUrl() {
|
|
@@ -1664,6 +1674,8 @@ function prepareAgentPayload(args) {
|
|
|
1664
1674
|
if (args.threadId != null) body.threadId = args.threadId;
|
|
1665
1675
|
if (args.mode != null) body.mode = args.mode;
|
|
1666
1676
|
if (args.exchange != null) body.exchange = args.exchange;
|
|
1677
|
+
if (args.location != null) body.location = args.location;
|
|
1678
|
+
if (args.zeroDataRetention != null) body.zeroDataRetention = args.zeroDataRetention;
|
|
1667
1679
|
return body;
|
|
1668
1680
|
}
|
|
1669
1681
|
async function startAgent(http, args) {
|
|
@@ -2246,37 +2258,84 @@ function defaultOrigin(version2) {
|
|
|
2246
2258
|
}
|
|
2247
2259
|
|
|
2248
2260
|
// src/v2/watcher.ts
|
|
2249
|
-
var
|
|
2250
|
-
var hasGlobalWebSocket = () => {
|
|
2251
|
-
if (typeof globalThis === "undefined") return void 0;
|
|
2261
|
+
var getWebSocketCtor = () => {
|
|
2252
2262
|
const candidate = globalThis.WebSocket;
|
|
2253
2263
|
return typeof candidate === "function" ? candidate : void 0;
|
|
2254
2264
|
};
|
|
2255
|
-
var
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2265
|
+
var Emitter = class {
|
|
2266
|
+
listenersByEvent = /* @__PURE__ */ new Map();
|
|
2267
|
+
maxListeners = 10;
|
|
2268
|
+
on(event, listener) {
|
|
2269
|
+
this.listenersByEvent.set(event, [...this.listenersByEvent.get(event) ?? [], listener]);
|
|
2270
|
+
return this;
|
|
2271
|
+
}
|
|
2272
|
+
prependListener(event, listener) {
|
|
2273
|
+
this.listenersByEvent.set(event, [listener, ...this.listenersByEvent.get(event) ?? []]);
|
|
2274
|
+
return this;
|
|
2275
|
+
}
|
|
2276
|
+
prependOnceListener(event, listener) {
|
|
2277
|
+
return this.prependListener(event, this.onceWrapper(event, listener));
|
|
2278
|
+
}
|
|
2279
|
+
listeners(event) {
|
|
2280
|
+
return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);
|
|
2281
|
+
}
|
|
2282
|
+
rawListeners(event) {
|
|
2283
|
+
return [...this.listenersByEvent.get(event) ?? []];
|
|
2284
|
+
}
|
|
2285
|
+
eventNames() {
|
|
2286
|
+
return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);
|
|
2287
|
+
}
|
|
2288
|
+
setMaxListeners(n) {
|
|
2289
|
+
this.maxListeners = n;
|
|
2290
|
+
return this;
|
|
2291
|
+
}
|
|
2292
|
+
getMaxListeners() {
|
|
2293
|
+
return this.maxListeners;
|
|
2294
|
+
}
|
|
2295
|
+
addListener(event, listener) {
|
|
2296
|
+
return this.on(event, listener);
|
|
2297
|
+
}
|
|
2298
|
+
once(event, listener) {
|
|
2299
|
+
return this.on(event, this.onceWrapper(event, listener));
|
|
2300
|
+
}
|
|
2301
|
+
onceWrapper(event, listener) {
|
|
2302
|
+
const wrapped = (...args) => {
|
|
2303
|
+
this.off(event, wrapped);
|
|
2304
|
+
listener.apply(this, args);
|
|
2305
|
+
};
|
|
2306
|
+
wrapped.listener = listener;
|
|
2307
|
+
return wrapped;
|
|
2308
|
+
}
|
|
2309
|
+
off(event, listener) {
|
|
2310
|
+
const current = this.listenersByEvent.get(event);
|
|
2311
|
+
if (!current) return this;
|
|
2312
|
+
for (let index = current.length - 1; index >= 0; index--) {
|
|
2313
|
+
if (current[index] !== listener && current[index].listener !== listener) continue;
|
|
2314
|
+
this.listenersByEvent.set(event, current.filter((_, i) => i !== index));
|
|
2315
|
+
break;
|
|
2316
|
+
}
|
|
2317
|
+
return this;
|
|
2318
|
+
}
|
|
2319
|
+
removeListener(event, listener) {
|
|
2320
|
+
return this.off(event, listener);
|
|
2321
|
+
}
|
|
2322
|
+
removeAllListeners(event) {
|
|
2323
|
+
if (event === void 0) this.listenersByEvent.clear();
|
|
2324
|
+
else this.listenersByEvent.delete(event);
|
|
2325
|
+
return this;
|
|
2326
|
+
}
|
|
2327
|
+
listenerCount(event) {
|
|
2328
|
+
return this.listenersByEvent.get(event)?.length ?? 0;
|
|
2329
|
+
}
|
|
2330
|
+
emit(event, ...args) {
|
|
2331
|
+
const current = this.listenersByEvent.get(event);
|
|
2332
|
+
if (!current?.length) {
|
|
2333
|
+
if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));
|
|
2334
|
+
return false;
|
|
2335
|
+
}
|
|
2336
|
+
for (const listener of [...current]) listener.apply(this, args);
|
|
2337
|
+
return true;
|
|
2266
2338
|
}
|
|
2267
|
-
};
|
|
2268
|
-
var getWebSocketCtor = async () => {
|
|
2269
|
-
if (cachedWebSocket) return cachedWebSocket;
|
|
2270
|
-
const globalWs = hasGlobalWebSocket();
|
|
2271
|
-
if (globalWs) {
|
|
2272
|
-
cachedWebSocket = globalWs;
|
|
2273
|
-
return cachedWebSocket;
|
|
2274
|
-
}
|
|
2275
|
-
if (!loadPromise) {
|
|
2276
|
-
loadPromise = loadNodeWebSocket();
|
|
2277
|
-
}
|
|
2278
|
-
cachedWebSocket = await loadPromise;
|
|
2279
|
-
return cachedWebSocket;
|
|
2280
2339
|
};
|
|
2281
2340
|
var decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : void 0;
|
|
2282
2341
|
var ensureUtf8String = (data) => {
|
|
@@ -2298,7 +2357,7 @@ var ensureUtf8String = (data) => {
|
|
|
2298
2357
|
}
|
|
2299
2358
|
return void 0;
|
|
2300
2359
|
};
|
|
2301
|
-
var Watcher = class extends
|
|
2360
|
+
var Watcher = class extends Emitter {
|
|
2302
2361
|
http;
|
|
2303
2362
|
jobId;
|
|
2304
2363
|
kind;
|
|
@@ -2340,7 +2399,7 @@ var Watcher = class extends import_events.EventEmitter {
|
|
|
2340
2399
|
(async () => {
|
|
2341
2400
|
try {
|
|
2342
2401
|
const url = this.buildWsUrl();
|
|
2343
|
-
const wsCtor =
|
|
2402
|
+
const wsCtor = getWebSocketCtor();
|
|
2344
2403
|
if (!wsCtor) {
|
|
2345
2404
|
this.pollLoop();
|
|
2346
2405
|
return;
|
|
@@ -2526,8 +2585,9 @@ var FirecrawlClient = class {
|
|
|
2526
2585
|
*/
|
|
2527
2586
|
constructor(options = {}) {
|
|
2528
2587
|
const opts = typeof options === "string" ? { apiKey: options } : options;
|
|
2529
|
-
const
|
|
2530
|
-
const
|
|
2588
|
+
const hasEnv = typeof process !== "undefined" && !!process.env;
|
|
2589
|
+
const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : void 0) ?? "").trim();
|
|
2590
|
+
const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : void 0) ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
|
2531
2591
|
this.http = new HttpClient({
|
|
2532
2592
|
apiKey,
|
|
2533
2593
|
apiUrl,
|
|
@@ -3073,7 +3133,7 @@ var FirecrawlApp = class {
|
|
|
3073
3133
|
const packageJson = await Promise.resolve().then(() => __toESM(require_package(), 1));
|
|
3074
3134
|
return packageJson.default.version;
|
|
3075
3135
|
} catch (error) {
|
|
3076
|
-
const isTest = typeof process !== "undefined" && false;
|
|
3136
|
+
const isTest = typeof process !== "undefined" && !!process.env && false;
|
|
3077
3137
|
if (!isTest) {
|
|
3078
3138
|
console.error("Error getting version:", error);
|
|
3079
3139
|
}
|
|
@@ -4362,6 +4422,9 @@ var CrawlWatcher = class extends e {
|
|
|
4362
4422
|
super();
|
|
4363
4423
|
this.id = id;
|
|
4364
4424
|
const wsUrl = app.apiUrl.replace(/^http/, "ws");
|
|
4425
|
+
if (typeof WebSocket === "undefined") {
|
|
4426
|
+
throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");
|
|
4427
|
+
}
|
|
4365
4428
|
this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
4366
4429
|
this.status = "scraping";
|
|
4367
4430
|
this.data = [];
|
package/dist/index.d.cts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as zt from 'zod';
|
|
2
2
|
import { ZodTypeAny } from 'zod';
|
|
3
3
|
import { AxiosResponse, AxiosRequestHeaders } from 'axios';
|
|
4
|
-
import { EventEmitter } from 'events';
|
|
5
4
|
import { TypedEventTarget } from 'typescript-event-target';
|
|
6
5
|
|
|
7
6
|
type FormatString = "markdown" | "html" | "rawHtml" | "links" | "images" | "screenshot" | "summary" | "changeTracking" | "json" | "attributes" | "branding" | "product" | "menu" | "audio" | "video";
|
|
@@ -1553,7 +1552,7 @@ interface AgentTraceAgentIdentity {
|
|
|
1553
1552
|
parentId?: string;
|
|
1554
1553
|
}
|
|
1555
1554
|
interface AgentTraceError {
|
|
1556
|
-
code: "cancelled" | "credit_limit_reached" | "parent_finished" | "refused" | "internal";
|
|
1555
|
+
code: "cancelled" | "credit_limit_reached" | "insufficient_credits" | "parent_finished" | "refused" | "internal";
|
|
1557
1556
|
source: "agent" | "tool" | "billing" | "system";
|
|
1558
1557
|
retryable: boolean;
|
|
1559
1558
|
message: string;
|
|
@@ -1659,6 +1658,8 @@ interface AgentTraceResponse {
|
|
|
1659
1658
|
creditsUsed?: number;
|
|
1660
1659
|
/** Present only when the trace was requested with liveView: true. */
|
|
1661
1660
|
activeBrowserSessions?: AgentTraceActiveBrowserSession[];
|
|
1661
|
+
/** True for a zero data retention run: events are limited to content-free lifecycle events. */
|
|
1662
|
+
zeroDataRetention?: boolean;
|
|
1662
1663
|
error?: string;
|
|
1663
1664
|
}
|
|
1664
1665
|
interface AgentSnapshotResponse {
|
|
@@ -2110,6 +2111,14 @@ declare function prepareAgentPayload(args: {
|
|
|
2110
2111
|
threadId?: string;
|
|
2111
2112
|
mode?: AgentMode;
|
|
2112
2113
|
exchange?: AgentExchangeOptions;
|
|
2114
|
+
/**
|
|
2115
|
+
* Country for the whole run (ISO 3166-1 alpha-2). Scrapes and browser sessions are proxied from it,
|
|
2116
|
+
* and search results are localized to it without changing where requests come from. The agent cannot change it.
|
|
2117
|
+
*/
|
|
2118
|
+
location?: {
|
|
2119
|
+
country: string;
|
|
2120
|
+
};
|
|
2121
|
+
zeroDataRetention?: boolean;
|
|
2113
2122
|
}): Record<string, unknown>;
|
|
2114
2123
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
2115
2124
|
|
|
@@ -2222,13 +2231,37 @@ declare class ResearchClient {
|
|
|
2222
2231
|
searchGithub(query: string, options?: SearchGithubOptions): Promise<GitHubSearchResponse>;
|
|
2223
2232
|
}
|
|
2224
2233
|
|
|
2234
|
+
type Listener = ((...args: any[]) => void) & {
|
|
2235
|
+
listener?: Listener;
|
|
2236
|
+
};
|
|
2237
|
+
/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */
|
|
2238
|
+
declare class Emitter {
|
|
2239
|
+
private readonly listenersByEvent;
|
|
2240
|
+
private maxListeners;
|
|
2241
|
+
on(event: string | symbol, listener: Listener): this;
|
|
2242
|
+
prependListener(event: string | symbol, listener: Listener): this;
|
|
2243
|
+
prependOnceListener(event: string | symbol, listener: Listener): this;
|
|
2244
|
+
listeners(event: string | symbol): Listener[];
|
|
2245
|
+
rawListeners(event: string | symbol): Listener[];
|
|
2246
|
+
eventNames(): (string | symbol)[];
|
|
2247
|
+
setMaxListeners(n: number): this;
|
|
2248
|
+
getMaxListeners(): number;
|
|
2249
|
+
addListener(event: string | symbol, listener: Listener): this;
|
|
2250
|
+
once(event: string | symbol, listener: Listener): this;
|
|
2251
|
+
private onceWrapper;
|
|
2252
|
+
off(event: string | symbol, listener: Listener): this;
|
|
2253
|
+
removeListener(event: string | symbol, listener: Listener): this;
|
|
2254
|
+
removeAllListeners(event?: string | symbol): this;
|
|
2255
|
+
listenerCount(event: string | symbol): number;
|
|
2256
|
+
emit(event: string | symbol, ...args: any[]): boolean;
|
|
2257
|
+
}
|
|
2225
2258
|
type JobKind = "crawl" | "batch";
|
|
2226
2259
|
interface WatcherOptions {
|
|
2227
2260
|
kind?: JobKind;
|
|
2228
2261
|
pollInterval?: number;
|
|
2229
2262
|
timeout?: number;
|
|
2230
2263
|
}
|
|
2231
|
-
declare class Watcher extends
|
|
2264
|
+
declare class Watcher extends Emitter {
|
|
2232
2265
|
private readonly http;
|
|
2233
2266
|
private readonly jobId;
|
|
2234
2267
|
private readonly kind;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as zt from 'zod';
|
|
2
2
|
import { ZodTypeAny } from 'zod';
|
|
3
3
|
import { AxiosResponse, AxiosRequestHeaders } from 'axios';
|
|
4
|
-
import { EventEmitter } from 'events';
|
|
5
4
|
import { TypedEventTarget } from 'typescript-event-target';
|
|
6
5
|
|
|
7
6
|
type FormatString = "markdown" | "html" | "rawHtml" | "links" | "images" | "screenshot" | "summary" | "changeTracking" | "json" | "attributes" | "branding" | "product" | "menu" | "audio" | "video";
|
|
@@ -1553,7 +1552,7 @@ interface AgentTraceAgentIdentity {
|
|
|
1553
1552
|
parentId?: string;
|
|
1554
1553
|
}
|
|
1555
1554
|
interface AgentTraceError {
|
|
1556
|
-
code: "cancelled" | "credit_limit_reached" | "parent_finished" | "refused" | "internal";
|
|
1555
|
+
code: "cancelled" | "credit_limit_reached" | "insufficient_credits" | "parent_finished" | "refused" | "internal";
|
|
1557
1556
|
source: "agent" | "tool" | "billing" | "system";
|
|
1558
1557
|
retryable: boolean;
|
|
1559
1558
|
message: string;
|
|
@@ -1659,6 +1658,8 @@ interface AgentTraceResponse {
|
|
|
1659
1658
|
creditsUsed?: number;
|
|
1660
1659
|
/** Present only when the trace was requested with liveView: true. */
|
|
1661
1660
|
activeBrowserSessions?: AgentTraceActiveBrowserSession[];
|
|
1661
|
+
/** True for a zero data retention run: events are limited to content-free lifecycle events. */
|
|
1662
|
+
zeroDataRetention?: boolean;
|
|
1662
1663
|
error?: string;
|
|
1663
1664
|
}
|
|
1664
1665
|
interface AgentSnapshotResponse {
|
|
@@ -2110,6 +2111,14 @@ declare function prepareAgentPayload(args: {
|
|
|
2110
2111
|
threadId?: string;
|
|
2111
2112
|
mode?: AgentMode;
|
|
2112
2113
|
exchange?: AgentExchangeOptions;
|
|
2114
|
+
/**
|
|
2115
|
+
* Country for the whole run (ISO 3166-1 alpha-2). Scrapes and browser sessions are proxied from it,
|
|
2116
|
+
* and search results are localized to it without changing where requests come from. The agent cannot change it.
|
|
2117
|
+
*/
|
|
2118
|
+
location?: {
|
|
2119
|
+
country: string;
|
|
2120
|
+
};
|
|
2121
|
+
zeroDataRetention?: boolean;
|
|
2113
2122
|
}): Record<string, unknown>;
|
|
2114
2123
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
2115
2124
|
|
|
@@ -2222,13 +2231,37 @@ declare class ResearchClient {
|
|
|
2222
2231
|
searchGithub(query: string, options?: SearchGithubOptions): Promise<GitHubSearchResponse>;
|
|
2223
2232
|
}
|
|
2224
2233
|
|
|
2234
|
+
type Listener = ((...args: any[]) => void) & {
|
|
2235
|
+
listener?: Listener;
|
|
2236
|
+
};
|
|
2237
|
+
/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */
|
|
2238
|
+
declare class Emitter {
|
|
2239
|
+
private readonly listenersByEvent;
|
|
2240
|
+
private maxListeners;
|
|
2241
|
+
on(event: string | symbol, listener: Listener): this;
|
|
2242
|
+
prependListener(event: string | symbol, listener: Listener): this;
|
|
2243
|
+
prependOnceListener(event: string | symbol, listener: Listener): this;
|
|
2244
|
+
listeners(event: string | symbol): Listener[];
|
|
2245
|
+
rawListeners(event: string | symbol): Listener[];
|
|
2246
|
+
eventNames(): (string | symbol)[];
|
|
2247
|
+
setMaxListeners(n: number): this;
|
|
2248
|
+
getMaxListeners(): number;
|
|
2249
|
+
addListener(event: string | symbol, listener: Listener): this;
|
|
2250
|
+
once(event: string | symbol, listener: Listener): this;
|
|
2251
|
+
private onceWrapper;
|
|
2252
|
+
off(event: string | symbol, listener: Listener): this;
|
|
2253
|
+
removeListener(event: string | symbol, listener: Listener): this;
|
|
2254
|
+
removeAllListeners(event?: string | symbol): this;
|
|
2255
|
+
listenerCount(event: string | symbol): number;
|
|
2256
|
+
emit(event: string | symbol, ...args: any[]): boolean;
|
|
2257
|
+
}
|
|
2225
2258
|
type JobKind = "crawl" | "batch";
|
|
2226
2259
|
interface WatcherOptions {
|
|
2227
2260
|
kind?: JobKind;
|
|
2228
2261
|
pollInterval?: number;
|
|
2229
2262
|
timeout?: number;
|
|
2230
2263
|
}
|
|
2231
|
-
declare class Watcher extends
|
|
2264
|
+
declare class Watcher extends Emitter {
|
|
2232
2265
|
private readonly http;
|
|
2233
2266
|
private readonly jobId;
|
|
2234
2267
|
private readonly kind;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
version
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-UGAMW2BL.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -26,6 +26,15 @@ function pinToApiOrigin(apiUrl, url) {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
// src/v2/utils/httpClient.ts
|
|
29
|
+
var rejectsDefaultCacheMode = () => {
|
|
30
|
+
if (typeof Request !== "function") return false;
|
|
31
|
+
try {
|
|
32
|
+
new Request("https://api.firecrawl.dev", { cache: "default" });
|
|
33
|
+
return false;
|
|
34
|
+
} catch {
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
29
38
|
var HttpClient = class {
|
|
30
39
|
instance;
|
|
31
40
|
apiKey;
|
|
@@ -48,7 +57,8 @@ var HttpClient = class {
|
|
|
48
57
|
// grants it when no Authorization header is present).
|
|
49
58
|
...this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
|
|
50
59
|
},
|
|
51
|
-
transitional: { clarifyTimeoutError: true }
|
|
60
|
+
transitional: { clarifyTimeoutError: true },
|
|
61
|
+
...rejectsDefaultCacheMode() ? { adapter: "fetch", env: { Request: null } } : {}
|
|
52
62
|
});
|
|
53
63
|
}
|
|
54
64
|
getApiUrl() {
|
|
@@ -1535,6 +1545,8 @@ function prepareAgentPayload(args) {
|
|
|
1535
1545
|
if (args.threadId != null) body.threadId = args.threadId;
|
|
1536
1546
|
if (args.mode != null) body.mode = args.mode;
|
|
1537
1547
|
if (args.exchange != null) body.exchange = args.exchange;
|
|
1548
|
+
if (args.location != null) body.location = args.location;
|
|
1549
|
+
if (args.zeroDataRetention != null) body.zeroDataRetention = args.zeroDataRetention;
|
|
1538
1550
|
return body;
|
|
1539
1551
|
}
|
|
1540
1552
|
async function startAgent(http, args) {
|
|
@@ -2117,37 +2129,84 @@ function defaultOrigin(version2) {
|
|
|
2117
2129
|
}
|
|
2118
2130
|
|
|
2119
2131
|
// src/v2/watcher.ts
|
|
2120
|
-
|
|
2121
|
-
var hasGlobalWebSocket = () => {
|
|
2122
|
-
if (typeof globalThis === "undefined") return void 0;
|
|
2132
|
+
var getWebSocketCtor = () => {
|
|
2123
2133
|
const candidate = globalThis.WebSocket;
|
|
2124
2134
|
return typeof candidate === "function" ? candidate : void 0;
|
|
2125
2135
|
};
|
|
2126
|
-
var
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2136
|
+
var Emitter = class {
|
|
2137
|
+
listenersByEvent = /* @__PURE__ */ new Map();
|
|
2138
|
+
maxListeners = 10;
|
|
2139
|
+
on(event, listener) {
|
|
2140
|
+
this.listenersByEvent.set(event, [...this.listenersByEvent.get(event) ?? [], listener]);
|
|
2141
|
+
return this;
|
|
2142
|
+
}
|
|
2143
|
+
prependListener(event, listener) {
|
|
2144
|
+
this.listenersByEvent.set(event, [listener, ...this.listenersByEvent.get(event) ?? []]);
|
|
2145
|
+
return this;
|
|
2146
|
+
}
|
|
2147
|
+
prependOnceListener(event, listener) {
|
|
2148
|
+
return this.prependListener(event, this.onceWrapper(event, listener));
|
|
2149
|
+
}
|
|
2150
|
+
listeners(event) {
|
|
2151
|
+
return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);
|
|
2152
|
+
}
|
|
2153
|
+
rawListeners(event) {
|
|
2154
|
+
return [...this.listenersByEvent.get(event) ?? []];
|
|
2155
|
+
}
|
|
2156
|
+
eventNames() {
|
|
2157
|
+
return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);
|
|
2158
|
+
}
|
|
2159
|
+
setMaxListeners(n) {
|
|
2160
|
+
this.maxListeners = n;
|
|
2161
|
+
return this;
|
|
2162
|
+
}
|
|
2163
|
+
getMaxListeners() {
|
|
2164
|
+
return this.maxListeners;
|
|
2165
|
+
}
|
|
2166
|
+
addListener(event, listener) {
|
|
2167
|
+
return this.on(event, listener);
|
|
2168
|
+
}
|
|
2169
|
+
once(event, listener) {
|
|
2170
|
+
return this.on(event, this.onceWrapper(event, listener));
|
|
2171
|
+
}
|
|
2172
|
+
onceWrapper(event, listener) {
|
|
2173
|
+
const wrapped = (...args) => {
|
|
2174
|
+
this.off(event, wrapped);
|
|
2175
|
+
listener.apply(this, args);
|
|
2176
|
+
};
|
|
2177
|
+
wrapped.listener = listener;
|
|
2178
|
+
return wrapped;
|
|
2179
|
+
}
|
|
2180
|
+
off(event, listener) {
|
|
2181
|
+
const current = this.listenersByEvent.get(event);
|
|
2182
|
+
if (!current) return this;
|
|
2183
|
+
for (let index = current.length - 1; index >= 0; index--) {
|
|
2184
|
+
if (current[index] !== listener && current[index].listener !== listener) continue;
|
|
2185
|
+
this.listenersByEvent.set(event, current.filter((_, i) => i !== index));
|
|
2186
|
+
break;
|
|
2187
|
+
}
|
|
2188
|
+
return this;
|
|
2189
|
+
}
|
|
2190
|
+
removeListener(event, listener) {
|
|
2191
|
+
return this.off(event, listener);
|
|
2192
|
+
}
|
|
2193
|
+
removeAllListeners(event) {
|
|
2194
|
+
if (event === void 0) this.listenersByEvent.clear();
|
|
2195
|
+
else this.listenersByEvent.delete(event);
|
|
2196
|
+
return this;
|
|
2197
|
+
}
|
|
2198
|
+
listenerCount(event) {
|
|
2199
|
+
return this.listenersByEvent.get(event)?.length ?? 0;
|
|
2200
|
+
}
|
|
2201
|
+
emit(event, ...args) {
|
|
2202
|
+
const current = this.listenersByEvent.get(event);
|
|
2203
|
+
if (!current?.length) {
|
|
2204
|
+
if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));
|
|
2205
|
+
return false;
|
|
2206
|
+
}
|
|
2207
|
+
for (const listener of [...current]) listener.apply(this, args);
|
|
2208
|
+
return true;
|
|
2137
2209
|
}
|
|
2138
|
-
};
|
|
2139
|
-
var getWebSocketCtor = async () => {
|
|
2140
|
-
if (cachedWebSocket) return cachedWebSocket;
|
|
2141
|
-
const globalWs = hasGlobalWebSocket();
|
|
2142
|
-
if (globalWs) {
|
|
2143
|
-
cachedWebSocket = globalWs;
|
|
2144
|
-
return cachedWebSocket;
|
|
2145
|
-
}
|
|
2146
|
-
if (!loadPromise) {
|
|
2147
|
-
loadPromise = loadNodeWebSocket();
|
|
2148
|
-
}
|
|
2149
|
-
cachedWebSocket = await loadPromise;
|
|
2150
|
-
return cachedWebSocket;
|
|
2151
2210
|
};
|
|
2152
2211
|
var decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : void 0;
|
|
2153
2212
|
var ensureUtf8String = (data) => {
|
|
@@ -2169,7 +2228,7 @@ var ensureUtf8String = (data) => {
|
|
|
2169
2228
|
}
|
|
2170
2229
|
return void 0;
|
|
2171
2230
|
};
|
|
2172
|
-
var Watcher = class extends
|
|
2231
|
+
var Watcher = class extends Emitter {
|
|
2173
2232
|
http;
|
|
2174
2233
|
jobId;
|
|
2175
2234
|
kind;
|
|
@@ -2211,7 +2270,7 @@ var Watcher = class extends EventEmitter {
|
|
|
2211
2270
|
(async () => {
|
|
2212
2271
|
try {
|
|
2213
2272
|
const url = this.buildWsUrl();
|
|
2214
|
-
const wsCtor =
|
|
2273
|
+
const wsCtor = getWebSocketCtor();
|
|
2215
2274
|
if (!wsCtor) {
|
|
2216
2275
|
this.pollLoop();
|
|
2217
2276
|
return;
|
|
@@ -2397,8 +2456,9 @@ var FirecrawlClient = class {
|
|
|
2397
2456
|
*/
|
|
2398
2457
|
constructor(options = {}) {
|
|
2399
2458
|
const opts = typeof options === "string" ? { apiKey: options } : options;
|
|
2400
|
-
const
|
|
2401
|
-
const
|
|
2459
|
+
const hasEnv = typeof process !== "undefined" && !!process.env;
|
|
2460
|
+
const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : void 0) ?? "").trim();
|
|
2461
|
+
const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : void 0) ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
|
2402
2462
|
this.http = new HttpClient({
|
|
2403
2463
|
apiKey,
|
|
2404
2464
|
apiUrl,
|
|
@@ -2941,10 +3001,10 @@ var FirecrawlApp = class {
|
|
|
2941
3001
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2942
3002
|
return process.env.npm_package_version;
|
|
2943
3003
|
}
|
|
2944
|
-
const packageJson = await import("./package-
|
|
3004
|
+
const packageJson = await import("./package-TFTOUFDX.js");
|
|
2945
3005
|
return packageJson.default.version;
|
|
2946
3006
|
} catch (error) {
|
|
2947
|
-
const isTest = typeof process !== "undefined" && false;
|
|
3007
|
+
const isTest = typeof process !== "undefined" && !!process.env && false;
|
|
2948
3008
|
if (!isTest) {
|
|
2949
3009
|
console.error("Error getting version:", error);
|
|
2950
3010
|
}
|
|
@@ -4233,6 +4293,9 @@ var CrawlWatcher = class extends e {
|
|
|
4233
4293
|
super();
|
|
4234
4294
|
this.id = id;
|
|
4235
4295
|
const wsUrl = app.apiUrl.replace(/^http/, "ws");
|
|
4296
|
+
if (typeof WebSocket === "undefined") {
|
|
4297
|
+
throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");
|
|
4298
|
+
}
|
|
4236
4299
|
this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
4237
4300
|
this.status = "scraping";
|
|
4238
4301
|
this.data = [];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.50.1",
|
|
4
4
|
"description": "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"license": "MIT",
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"axios": "1.20.0",
|
|
23
|
-
"firecrawl": "4.
|
|
23
|
+
"firecrawl": "4.50.1",
|
|
24
24
|
"typescript-event-target": "^1.1.1",
|
|
25
25
|
"zod": "^3.23.8",
|
|
26
26
|
"zod-to-json-schema": "^3.23.0"
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"tsup": "^8.5.0",
|
|
38
38
|
"typescript": "^5.4.5",
|
|
39
39
|
"uuid": "^14.0.0",
|
|
40
|
-
"vitest": "^4.1.
|
|
40
|
+
"vitest": "^4.1.12"
|
|
41
41
|
},
|
|
42
42
|
"keywords": [
|
|
43
43
|
"firecrawl",
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
"bioinformatics"
|
|
64
64
|
],
|
|
65
65
|
"engines": {
|
|
66
|
-
"node": ">=
|
|
66
|
+
"node": ">=18.0.0"
|
|
67
67
|
},
|
|
68
68
|
"scripts": {
|
|
69
69
|
"build": "tsup",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -14,6 +14,8 @@ overrides:
|
|
|
14
14
|
# GHSA-m8m8-qj5v-23w3 / GHSA-44g4-m2mj-wpvx / GHSA-r4gj-5m52-g5wh:
|
|
15
15
|
# pin the 1.x line only. Exact 1.20.0 cannot resolve to 2.x.
|
|
16
16
|
"axios@>=1.0.0 <1.20.0": "1.20.0"
|
|
17
|
-
|
|
17
|
+
# GHSA-Q3JH-WRC6-JGQ8 / GHSA-8R9P-F939-6H3C: fixed in 1.16.1.
|
|
18
|
+
# Replacement is bounded <2.0.0; cannot resolve to follow-redirects 2.x.
|
|
19
|
+
"follow-redirects@<1.16.1": ">=1.16.1 <2.0.0"
|
|
18
20
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
19
21
|
"esbuild@>=0.17.0 <0.28.1": "0.28.1"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { describe, test, expect, vi } from "vitest";
|
|
2
|
+
import { startAgent } from "../../../v2/methods/agent";
|
|
3
|
+
|
|
4
|
+
describe("JS SDK v2 agent location", () => {
|
|
5
|
+
const post = () =>
|
|
6
|
+
vi.fn(async () => ({
|
|
7
|
+
status: 200,
|
|
8
|
+
data: { success: true, id: "agent-1" },
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
test("sends location when starting an agent", async () => {
|
|
12
|
+
const http = { post: post() };
|
|
13
|
+
|
|
14
|
+
await startAgent(http as any, {
|
|
15
|
+
prompt: "Find the pricing page",
|
|
16
|
+
location: { country: "DE" },
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
expect(http.post).toHaveBeenCalledWith("/v2/agent", {
|
|
20
|
+
prompt: "Find the pricing page",
|
|
21
|
+
location: { country: "DE" },
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("omits location by default", async () => {
|
|
26
|
+
const http = { post: post() };
|
|
27
|
+
|
|
28
|
+
await startAgent(http as any, { prompt: "Find the pricing page" });
|
|
29
|
+
|
|
30
|
+
expect(http.post).toHaveBeenCalledWith("/v2/agent", {
|
|
31
|
+
prompt: "Find the pricing page",
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
});
|
package/src/v1/index.ts
CHANGED
|
@@ -686,7 +686,7 @@ export default class FirecrawlApp {
|
|
|
686
686
|
return packageJson.default.version;
|
|
687
687
|
} catch (error) {
|
|
688
688
|
// Suppress noisy logs under test environments
|
|
689
|
-
const isTest = typeof process !== 'undefined' && process.env.NODE_ENV === 'test';
|
|
689
|
+
const isTest = typeof process !== 'undefined' && !!process.env && process.env.NODE_ENV === 'test';
|
|
690
690
|
if (!isTest) {
|
|
691
691
|
// eslint-disable-next-line no-console
|
|
692
692
|
console.error("Error getting version:", error);
|
|
@@ -2220,6 +2220,9 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
|
|
|
2220
2220
|
this.id = id;
|
|
2221
2221
|
// replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)
|
|
2222
2222
|
const wsUrl = app.apiUrl.replace(/^http/, "ws");
|
|
2223
|
+
if (typeof WebSocket === "undefined") {
|
|
2224
|
+
throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");
|
|
2225
|
+
}
|
|
2223
2226
|
this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
2224
2227
|
this.status = "scraping";
|
|
2225
2228
|
this.data = [];
|
package/src/v2/client.ts
CHANGED
|
@@ -164,8 +164,9 @@ export class FirecrawlClient {
|
|
|
164
164
|
const opts: FirecrawlClientOptions =
|
|
165
165
|
typeof options === "string" ? { apiKey: options } : options;
|
|
166
166
|
|
|
167
|
-
const
|
|
168
|
-
const
|
|
167
|
+
const hasEnv = typeof process !== "undefined" && !!process.env;
|
|
168
|
+
const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : undefined) ?? "").trim();
|
|
169
|
+
const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : undefined) ?? "https://api.firecrawl.dev").replace(/\/$/, "");
|
|
169
170
|
|
|
170
171
|
// No API key is allowed: scrape, search, and interact fall back to the
|
|
171
172
|
// keyless free tier (rate-limited per IP). Other methods will return 401
|
package/src/v2/methods/agent.ts
CHANGED
|
@@ -21,6 +21,12 @@ function prepareAgentPayload(args: {
|
|
|
21
21
|
threadId?: string;
|
|
22
22
|
mode?: AgentMode;
|
|
23
23
|
exchange?: AgentExchangeOptions;
|
|
24
|
+
/**
|
|
25
|
+
* Country for the whole run (ISO 3166-1 alpha-2). Scrapes and browser sessions are proxied from it,
|
|
26
|
+
* and search results are localized to it without changing where requests come from. The agent cannot change it.
|
|
27
|
+
*/
|
|
28
|
+
location?: { country: string };
|
|
29
|
+
zeroDataRetention?: boolean;
|
|
24
30
|
}): Record<string, unknown> {
|
|
25
31
|
const body: Record<string, unknown> = {};
|
|
26
32
|
if (args.urls) body.urls = args.urls;
|
|
@@ -42,6 +48,8 @@ function prepareAgentPayload(args: {
|
|
|
42
48
|
if (args.threadId != null) body.threadId = args.threadId;
|
|
43
49
|
if (args.mode != null) body.mode = args.mode;
|
|
44
50
|
if (args.exchange != null) body.exchange = args.exchange;
|
|
51
|
+
if (args.location != null) body.location = args.location;
|
|
52
|
+
if (args.zeroDataRetention != null) body.zeroDataRetention = args.zeroDataRetention;
|
|
45
53
|
return body;
|
|
46
54
|
}
|
|
47
55
|
|
package/src/v2/types.ts
CHANGED
|
@@ -300,7 +300,12 @@ export interface ScrapeOptions {
|
|
|
300
300
|
}
|
|
301
301
|
|
|
302
302
|
export type RedactPIIEntity =
|
|
303
|
-
|
|
303
|
+
| "PERSON"
|
|
304
|
+
| "EMAIL"
|
|
305
|
+
| "PHONE"
|
|
306
|
+
| "LOCATION"
|
|
307
|
+
| "FINANCIAL"
|
|
308
|
+
| "SECRET";
|
|
304
309
|
|
|
305
310
|
export interface RedactPIIOptions {
|
|
306
311
|
/**
|
|
@@ -347,7 +352,12 @@ export interface ThreatProtectionOptions {
|
|
|
347
352
|
}
|
|
348
353
|
|
|
349
354
|
export type ParseFileData =
|
|
350
|
-
|
|
355
|
+
| Blob
|
|
356
|
+
| File
|
|
357
|
+
| Buffer
|
|
358
|
+
| Uint8Array
|
|
359
|
+
| ArrayBuffer
|
|
360
|
+
| string;
|
|
351
361
|
|
|
352
362
|
export interface ParseFile {
|
|
353
363
|
data: ParseFileData;
|
|
@@ -395,7 +405,11 @@ export interface WebhookConfig {
|
|
|
395
405
|
|
|
396
406
|
// Agent webhook events differ from crawl: has 'action' and 'cancelled', no 'page'
|
|
397
407
|
export type AgentWebhookEvent =
|
|
398
|
-
|
|
408
|
+
| "started"
|
|
409
|
+
| "action"
|
|
410
|
+
| "completed"
|
|
411
|
+
| "failed"
|
|
412
|
+
| "cancelled";
|
|
399
413
|
|
|
400
414
|
export interface AgentWebhookConfig {
|
|
401
415
|
url: string;
|
|
@@ -516,7 +530,10 @@ export interface BrandingProfile {
|
|
|
516
530
|
headerHeight?: string;
|
|
517
531
|
footerHeight?: string;
|
|
518
532
|
[key: string]:
|
|
519
|
-
|
|
533
|
+
| number
|
|
534
|
+
| string
|
|
535
|
+
| Record<string, number | string | undefined>
|
|
536
|
+
| undefined;
|
|
520
537
|
};
|
|
521
538
|
tone?: {
|
|
522
539
|
voice?: string;
|
|
@@ -1014,7 +1031,10 @@ export interface SearchRequest {
|
|
|
1014
1031
|
/** Compact by default; summary adds metadata, full includes contracts. */
|
|
1015
1032
|
toolDetail?: "compact" | "summary" | "full";
|
|
1016
1033
|
sources?: Array<
|
|
1017
|
-
|
|
1034
|
+
| "web"
|
|
1035
|
+
| "news"
|
|
1036
|
+
| "images"
|
|
1037
|
+
| "alexandria"
|
|
1018
1038
|
| { type: "web" | "news" | "images" | "alexandria" }
|
|
1019
1039
|
>;
|
|
1020
1040
|
/**
|
|
@@ -1265,7 +1285,9 @@ export interface MonitorSearchTarget {
|
|
|
1265
1285
|
}
|
|
1266
1286
|
|
|
1267
1287
|
export type MonitorTarget =
|
|
1268
|
-
|
|
1288
|
+
| MonitorScrapeTarget
|
|
1289
|
+
| MonitorCrawlTarget
|
|
1290
|
+
| MonitorSearchTarget;
|
|
1269
1291
|
|
|
1270
1292
|
export interface CreateMonitorRequest {
|
|
1271
1293
|
name: string;
|
|
@@ -1388,7 +1410,11 @@ export interface MonitorCheck {
|
|
|
1388
1410
|
reservedCredits?: number | null;
|
|
1389
1411
|
actualCredits?: number | null;
|
|
1390
1412
|
billingStatus:
|
|
1391
|
-
|
|
1413
|
+
| "not_applicable"
|
|
1414
|
+
| "reserved"
|
|
1415
|
+
| "confirmed"
|
|
1416
|
+
| "released"
|
|
1417
|
+
| "failed";
|
|
1392
1418
|
summary: MonitorSummary;
|
|
1393
1419
|
targetResults?: MonitorTargetResult[];
|
|
1394
1420
|
notificationStatus?: unknown;
|
|
@@ -1779,7 +1805,10 @@ export interface AgentListOptions {
|
|
|
1779
1805
|
}
|
|
1780
1806
|
|
|
1781
1807
|
export type AgentTraceAgentRole =
|
|
1782
|
-
|
|
1808
|
+
| "orchestrator"
|
|
1809
|
+
| "subagent"
|
|
1810
|
+
| "browser"
|
|
1811
|
+
| "system";
|
|
1783
1812
|
|
|
1784
1813
|
export interface AgentTraceAgentIdentity {
|
|
1785
1814
|
id: string;
|
|
@@ -1792,6 +1821,7 @@ export interface AgentTraceError {
|
|
|
1792
1821
|
code:
|
|
1793
1822
|
| "cancelled"
|
|
1794
1823
|
| "credit_limit_reached"
|
|
1824
|
+
| "insufficient_credits"
|
|
1795
1825
|
| "parent_finished"
|
|
1796
1826
|
| "refused"
|
|
1797
1827
|
| "internal";
|
|
@@ -1833,7 +1863,11 @@ export interface AgentTraceRunCancelRequestedEvent extends AgentTraceEventBase {
|
|
|
1833
1863
|
export interface AgentTraceRunFinishedEvent extends AgentTraceEventBase {
|
|
1834
1864
|
type: "run.finished";
|
|
1835
1865
|
outcome:
|
|
1836
|
-
|
|
1866
|
+
| "succeeded"
|
|
1867
|
+
| "failed"
|
|
1868
|
+
| "cancelled"
|
|
1869
|
+
| "refused"
|
|
1870
|
+
| "credit_limit_reached";
|
|
1837
1871
|
/** The canonical schema always writes this key (nullable), but older rows may omit it. */
|
|
1838
1872
|
error?: AgentTraceError | null;
|
|
1839
1873
|
}
|
|
@@ -1850,12 +1884,14 @@ export interface AgentTraceAgentFinishedEvent extends AgentTraceEventBase {
|
|
|
1850
1884
|
error?: AgentTraceError | null;
|
|
1851
1885
|
}
|
|
1852
1886
|
|
|
1853
|
-
export interface AgentTraceBrowserSessionStartedEvent
|
|
1887
|
+
export interface AgentTraceBrowserSessionStartedEvent
|
|
1888
|
+
extends AgentTraceEventBase {
|
|
1854
1889
|
type: "browser.session.started";
|
|
1855
1890
|
sessionId: string;
|
|
1856
1891
|
}
|
|
1857
1892
|
|
|
1858
|
-
export interface AgentTraceBrowserSessionFinishedEvent
|
|
1893
|
+
export interface AgentTraceBrowserSessionFinishedEvent
|
|
1894
|
+
extends AgentTraceEventBase {
|
|
1859
1895
|
type: "browser.session.finished";
|
|
1860
1896
|
sessionId: string;
|
|
1861
1897
|
durationMs: number;
|
|
@@ -1929,6 +1965,8 @@ export interface AgentTraceResponse {
|
|
|
1929
1965
|
creditsUsed?: number;
|
|
1930
1966
|
/** Present only when the trace was requested with liveView: true. */
|
|
1931
1967
|
activeBrowserSessions?: AgentTraceActiveBrowserSession[];
|
|
1968
|
+
/** True for a zero data retention run: events are limited to content-free lifecycle events. */
|
|
1969
|
+
zeroDataRetention?: boolean;
|
|
1932
1970
|
error?: string;
|
|
1933
1971
|
}
|
|
1934
1972
|
|
|
@@ -2028,7 +2066,9 @@ export interface RequiresAction {
|
|
|
2028
2066
|
url?: string;
|
|
2029
2067
|
}
|
|
2030
2068
|
|
|
2031
|
-
export function parseRequiresAction(
|
|
2069
|
+
export function parseRequiresAction(
|
|
2070
|
+
value: unknown,
|
|
2071
|
+
): RequiresAction | undefined {
|
|
2032
2072
|
if (!value || typeof value !== "object") return undefined;
|
|
2033
2073
|
const record = value as Record<string, unknown>;
|
|
2034
2074
|
if (typeof record.type !== "string") return undefined;
|
|
@@ -6,6 +6,18 @@ import axios, {
|
|
|
6
6
|
import { getVersion } from "./getVersion";
|
|
7
7
|
import { pinToApiOrigin } from "../../utils/apiOrigin";
|
|
8
8
|
|
|
9
|
+
// Runtimes such as Cloudflare Workers reject the Request `cache: "default"` that
|
|
10
|
+
// axios's fetch adapter always sets; there axios calls fetch(url, init) without it.
|
|
11
|
+
const rejectsDefaultCacheMode = (): boolean => {
|
|
12
|
+
if (typeof Request !== "function") return false;
|
|
13
|
+
try {
|
|
14
|
+
new Request("https://api.firecrawl.dev", { cache: "default" });
|
|
15
|
+
return false;
|
|
16
|
+
} catch {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
|
|
9
21
|
export interface HttpClientOptions {
|
|
10
22
|
apiKey: string;
|
|
11
23
|
apiUrl: string;
|
|
@@ -44,6 +56,7 @@ export class HttpClient {
|
|
|
44
56
|
...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
|
|
45
57
|
},
|
|
46
58
|
transitional: { clarifyTimeoutError: true },
|
|
59
|
+
...(rejectsDefaultCacheMode() ? { adapter: "fetch", env: { Request: null as any } } : {}),
|
|
47
60
|
});
|
|
48
61
|
}
|
|
49
62
|
|
package/src/v2/watcher.ts
CHANGED
|
@@ -1,48 +1,110 @@
|
|
|
1
|
-
import { EventEmitter } from "events";
|
|
2
1
|
import type { BatchScrapeJob, CrawlJob, Document } from "./types";
|
|
3
2
|
import type { HttpClient } from "./utils/httpClient";
|
|
4
3
|
import { getBatchScrapeStatus } from "./methods/batch";
|
|
5
4
|
import { getCrawlStatus } from "./methods/crawl";
|
|
6
|
-
// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without
|
|
7
|
-
// experimental flags) do not. We lazily fall back to node:undici.
|
|
8
5
|
|
|
9
6
|
type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;
|
|
10
7
|
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
// Runtimes without a global WebSocket (Node before 22) poll instead.
|
|
9
|
+
const getWebSocketCtor = (): WebSocketConstructor | undefined => {
|
|
13
10
|
const candidate = (globalThis as any).WebSocket;
|
|
14
11
|
return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;
|
|
15
12
|
};
|
|
16
13
|
|
|
17
|
-
|
|
14
|
+
type Listener = ((...args: any[]) => void) & { listener?: Listener };
|
|
18
15
|
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */
|
|
17
|
+
class Emitter {
|
|
18
|
+
private readonly listenersByEvent = new Map<string | symbol, Listener[]>();
|
|
19
|
+
private maxListeners = 10;
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const undici = await import("node:undici");
|
|
26
|
-
const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;
|
|
27
|
-
return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;
|
|
28
|
-
} catch {
|
|
29
|
-
return undefined;
|
|
21
|
+
on(event: string | symbol, listener: Listener): this {
|
|
22
|
+
this.listenersByEvent.set(event, [...(this.listenersByEvent.get(event) ?? []), listener]);
|
|
23
|
+
return this;
|
|
30
24
|
}
|
|
31
|
-
};
|
|
32
25
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if (globalWs) {
|
|
37
|
-
cachedWebSocket = globalWs;
|
|
38
|
-
return cachedWebSocket;
|
|
26
|
+
prependListener(event: string | symbol, listener: Listener): this {
|
|
27
|
+
this.listenersByEvent.set(event, [listener, ...(this.listenersByEvent.get(event) ?? [])]);
|
|
28
|
+
return this;
|
|
39
29
|
}
|
|
40
|
-
|
|
41
|
-
|
|
30
|
+
|
|
31
|
+
prependOnceListener(event: string | symbol, listener: Listener): this {
|
|
32
|
+
return this.prependListener(event, this.onceWrapper(event, listener));
|
|
42
33
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
34
|
+
|
|
35
|
+
listeners(event: string | symbol): Listener[] {
|
|
36
|
+
return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
rawListeners(event: string | symbol): Listener[] {
|
|
40
|
+
return [...(this.listenersByEvent.get(event) ?? [])];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
eventNames(): (string | symbol)[] {
|
|
44
|
+
return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
setMaxListeners(n: number): this {
|
|
48
|
+
this.maxListeners = n;
|
|
49
|
+
return this;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
getMaxListeners(): number {
|
|
53
|
+
return this.maxListeners;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
addListener(event: string | symbol, listener: Listener): this {
|
|
57
|
+
return this.on(event, listener);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
once(event: string | symbol, listener: Listener): this {
|
|
61
|
+
return this.on(event, this.onceWrapper(event, listener));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
private onceWrapper(event: string | symbol, listener: Listener): Listener {
|
|
65
|
+
const wrapped: Listener = (...args) => {
|
|
66
|
+
this.off(event, wrapped);
|
|
67
|
+
listener.apply(this, args);
|
|
68
|
+
};
|
|
69
|
+
wrapped.listener = listener;
|
|
70
|
+
return wrapped;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
off(event: string | symbol, listener: Listener): this {
|
|
74
|
+
const current = this.listenersByEvent.get(event);
|
|
75
|
+
if (!current) return this;
|
|
76
|
+
for (let index = current.length - 1; index >= 0; index--) {
|
|
77
|
+
if (current[index] !== listener && current[index]!.listener !== listener) continue;
|
|
78
|
+
this.listenersByEvent.set(event, current.filter((_, i) => i !== index));
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
return this;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
removeListener(event: string | symbol, listener: Listener): this {
|
|
85
|
+
return this.off(event, listener);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
removeAllListeners(event?: string | symbol): this {
|
|
89
|
+
if (event === undefined) this.listenersByEvent.clear();
|
|
90
|
+
else this.listenersByEvent.delete(event);
|
|
91
|
+
return this;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
listenerCount(event: string | symbol): number {
|
|
95
|
+
return this.listenersByEvent.get(event)?.length ?? 0;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
emit(event: string | symbol, ...args: any[]): boolean {
|
|
99
|
+
const current = this.listenersByEvent.get(event);
|
|
100
|
+
if (!current?.length) {
|
|
101
|
+
if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
for (const listener of [...current]) listener.apply(this, args);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
46
108
|
|
|
47
109
|
const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;
|
|
48
110
|
|
|
@@ -81,7 +143,7 @@ export interface WatcherOptions {
|
|
|
81
143
|
|
|
82
144
|
type Snapshot = CrawlJob | BatchScrapeJob;
|
|
83
145
|
|
|
84
|
-
export class Watcher extends
|
|
146
|
+
export class Watcher extends Emitter {
|
|
85
147
|
private readonly http: HttpClient;
|
|
86
148
|
private readonly jobId: string;
|
|
87
149
|
private readonly kind: JobKind;
|
|
@@ -122,7 +184,7 @@ export class Watcher extends EventEmitter {
|
|
|
122
184
|
(async () => {
|
|
123
185
|
try {
|
|
124
186
|
const url = this.buildWsUrl();
|
|
125
|
-
const wsCtor =
|
|
187
|
+
const wsCtor = getWebSocketCtor();
|
|
126
188
|
if (!wsCtor) {
|
|
127
189
|
this.pollLoop();
|
|
128
190
|
return;
|