@mendable/firecrawl 3.0.3 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-OIZ6OKY4.js → chunk-QPAPMZLC.js} +1 -1
- package/dist/index.cjs +24 -20
- package/dist/index.d.cts +29 -7
- package/dist/index.d.ts +29 -7
- package/dist/index.js +25 -21
- package/dist/{package-V5IPFKBE.js → package-VNFDXLYR.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/clientOptions.test.ts +55 -0
- package/src/__tests__/unit/v2/validation.test.ts +12 -5
- package/src/index.ts +10 -4
- package/src/v2/methods/search.ts +30 -29
- package/src/v2/types.ts +30 -5
|
@@ -8,7 +8,7 @@ var require_package = __commonJS({
|
|
|
8
8
|
"package.json"(exports, module) {
|
|
9
9
|
module.exports = {
|
|
10
10
|
name: "@mendable/firecrawl-js",
|
|
11
|
-
version: "3.
|
|
11
|
+
version: "3.2.1",
|
|
12
12
|
description: "JavaScript SDK for Firecrawl API",
|
|
13
13
|
main: "dist/index.js",
|
|
14
14
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -35,7 +35,7 @@ var require_package = __commonJS({
|
|
|
35
35
|
"package.json"(exports2, module2) {
|
|
36
36
|
module2.exports = {
|
|
37
37
|
name: "@mendable/firecrawl-js",
|
|
38
|
-
version: "3.
|
|
38
|
+
version: "3.2.1",
|
|
39
39
|
description: "JavaScript SDK for Firecrawl API",
|
|
40
40
|
main: "dist/index.js",
|
|
41
41
|
types: "dist/index.d.ts",
|
|
@@ -316,6 +316,7 @@ function prepareSearchPayload(req) {
|
|
|
316
316
|
query: req.query
|
|
317
317
|
};
|
|
318
318
|
if (req.sources) payload.sources = req.sources;
|
|
319
|
+
if (req.categories) payload.categories = req.categories;
|
|
319
320
|
if (req.limit != null) payload.limit = req.limit;
|
|
320
321
|
if (req.tbs != null) payload.tbs = req.tbs;
|
|
321
322
|
if (req.location != null) payload.location = req.location;
|
|
@@ -327,6 +328,21 @@ function prepareSearchPayload(req) {
|
|
|
327
328
|
}
|
|
328
329
|
return payload;
|
|
329
330
|
}
|
|
331
|
+
function transformArray(arr) {
|
|
332
|
+
const results = [];
|
|
333
|
+
for (const item of arr) {
|
|
334
|
+
if (item && typeof item === "object") {
|
|
335
|
+
if ("markdown" in item || "html" in item || "rawHtml" in item || "links" in item || "screenshot" in item || "changeTracking" in item || "summary" in item || "json" in item) {
|
|
336
|
+
results.push(item);
|
|
337
|
+
} else {
|
|
338
|
+
results.push(item);
|
|
339
|
+
}
|
|
340
|
+
} else {
|
|
341
|
+
results.push({ url: item });
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return results;
|
|
345
|
+
}
|
|
330
346
|
async function search(http, request) {
|
|
331
347
|
const payload = prepareSearchPayload(request);
|
|
332
348
|
try {
|
|
@@ -336,24 +352,9 @@ async function search(http, request) {
|
|
|
336
352
|
}
|
|
337
353
|
const data = res.data.data || {};
|
|
338
354
|
const out = {};
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const results = [];
|
|
343
|
-
for (const item of arr) {
|
|
344
|
-
if (item && typeof item === "object") {
|
|
345
|
-
if ("markdown" in item || "html" in item || "rawHtml" in item || "links" in item || "screenshot" in item || "changeTracking" in item || "summary" in item || "json" in item) {
|
|
346
|
-
results.push(item);
|
|
347
|
-
} else {
|
|
348
|
-
results.push({ url: item.url, title: item.title, description: item.description });
|
|
349
|
-
}
|
|
350
|
-
} else if (typeof item === "string") {
|
|
351
|
-
results.push({ url: item });
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
out[key] = results;
|
|
355
|
-
}
|
|
356
|
-
}
|
|
355
|
+
if (data.web) out.web = transformArray(data.web);
|
|
356
|
+
if (data.news) out.news = transformArray(data.news);
|
|
357
|
+
if (data.images) out.images = transformArray(data.images);
|
|
357
358
|
return out;
|
|
358
359
|
} catch (err) {
|
|
359
360
|
if (err?.isAxiosError) return normalizeAxiosError(err, "search");
|
|
@@ -2347,7 +2348,10 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
2347
2348
|
/** @param opts API credentials and base URL. */
|
|
2348
2349
|
constructor(opts = {}) {
|
|
2349
2350
|
super(opts);
|
|
2350
|
-
this._v1Opts =
|
|
2351
|
+
this._v1Opts = {
|
|
2352
|
+
apiKey: opts.apiKey,
|
|
2353
|
+
apiUrl: opts.apiUrl
|
|
2354
|
+
};
|
|
2351
2355
|
}
|
|
2352
2356
|
/** Access the legacy v1 client (instantiated on first access). */
|
|
2353
2357
|
get v1() {
|
package/dist/index.d.cts
CHANGED
|
@@ -138,21 +138,43 @@ interface Document {
|
|
|
138
138
|
warning?: string;
|
|
139
139
|
changeTracking?: Record<string, unknown>;
|
|
140
140
|
}
|
|
141
|
-
interface
|
|
141
|
+
interface SearchResultWeb {
|
|
142
142
|
url: string;
|
|
143
143
|
title?: string;
|
|
144
144
|
description?: string;
|
|
145
|
+
category?: string;
|
|
146
|
+
}
|
|
147
|
+
interface SearchResultNews {
|
|
148
|
+
title?: string;
|
|
149
|
+
url?: string;
|
|
150
|
+
snippet?: string;
|
|
151
|
+
date?: string;
|
|
152
|
+
imageUrl?: string;
|
|
153
|
+
position?: number;
|
|
154
|
+
category?: string;
|
|
155
|
+
}
|
|
156
|
+
interface SearchResultImages {
|
|
157
|
+
title?: string;
|
|
158
|
+
imageUrl?: string;
|
|
159
|
+
imageWidth?: number;
|
|
160
|
+
imageHeight?: number;
|
|
161
|
+
url?: string;
|
|
162
|
+
position?: number;
|
|
145
163
|
}
|
|
146
164
|
interface SearchData {
|
|
147
|
-
web?: Array<
|
|
148
|
-
news?: Array<
|
|
149
|
-
images?: Array<
|
|
165
|
+
web?: Array<SearchResultWeb | Document>;
|
|
166
|
+
news?: Array<SearchResultNews | Document>;
|
|
167
|
+
images?: Array<SearchResultImages | Document>;
|
|
168
|
+
}
|
|
169
|
+
interface CategoryOption {
|
|
170
|
+
type: "github" | "research";
|
|
150
171
|
}
|
|
151
172
|
interface SearchRequest {
|
|
152
173
|
query: string;
|
|
153
174
|
sources?: Array<"web" | "news" | "images" | {
|
|
154
175
|
type: "web" | "news" | "images";
|
|
155
176
|
}>;
|
|
177
|
+
categories?: Array<"github" | "research" | CategoryOption>;
|
|
156
178
|
limit?: number;
|
|
157
179
|
tbs?: string;
|
|
158
180
|
location?: string;
|
|
@@ -215,7 +237,7 @@ interface BatchScrapeJob {
|
|
|
215
237
|
data: Document[];
|
|
216
238
|
}
|
|
217
239
|
interface MapData {
|
|
218
|
-
links:
|
|
240
|
+
links: SearchResultWeb[];
|
|
219
241
|
}
|
|
220
242
|
interface MapOptions {
|
|
221
243
|
search?: string;
|
|
@@ -1321,9 +1343,9 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
1321
1343
|
private _v1?;
|
|
1322
1344
|
private _v1Opts;
|
|
1323
1345
|
/** @param opts API credentials and base URL. */
|
|
1324
|
-
constructor(opts?:
|
|
1346
|
+
constructor(opts?: FirecrawlClientOptions);
|
|
1325
1347
|
/** Access the legacy v1 client (instantiated on first access). */
|
|
1326
1348
|
get v1(): FirecrawlApp;
|
|
1327
1349
|
}
|
|
1328
1350
|
|
|
1329
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig, type MapData, type MapOptions, type PDFAction, type PressAction, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type
|
|
1351
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig, type MapData, type MapOptions, type PDFAction, type PressAction, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -138,21 +138,43 @@ interface Document {
|
|
|
138
138
|
warning?: string;
|
|
139
139
|
changeTracking?: Record<string, unknown>;
|
|
140
140
|
}
|
|
141
|
-
interface
|
|
141
|
+
interface SearchResultWeb {
|
|
142
142
|
url: string;
|
|
143
143
|
title?: string;
|
|
144
144
|
description?: string;
|
|
145
|
+
category?: string;
|
|
146
|
+
}
|
|
147
|
+
interface SearchResultNews {
|
|
148
|
+
title?: string;
|
|
149
|
+
url?: string;
|
|
150
|
+
snippet?: string;
|
|
151
|
+
date?: string;
|
|
152
|
+
imageUrl?: string;
|
|
153
|
+
position?: number;
|
|
154
|
+
category?: string;
|
|
155
|
+
}
|
|
156
|
+
interface SearchResultImages {
|
|
157
|
+
title?: string;
|
|
158
|
+
imageUrl?: string;
|
|
159
|
+
imageWidth?: number;
|
|
160
|
+
imageHeight?: number;
|
|
161
|
+
url?: string;
|
|
162
|
+
position?: number;
|
|
145
163
|
}
|
|
146
164
|
interface SearchData {
|
|
147
|
-
web?: Array<
|
|
148
|
-
news?: Array<
|
|
149
|
-
images?: Array<
|
|
165
|
+
web?: Array<SearchResultWeb | Document>;
|
|
166
|
+
news?: Array<SearchResultNews | Document>;
|
|
167
|
+
images?: Array<SearchResultImages | Document>;
|
|
168
|
+
}
|
|
169
|
+
interface CategoryOption {
|
|
170
|
+
type: "github" | "research";
|
|
150
171
|
}
|
|
151
172
|
interface SearchRequest {
|
|
152
173
|
query: string;
|
|
153
174
|
sources?: Array<"web" | "news" | "images" | {
|
|
154
175
|
type: "web" | "news" | "images";
|
|
155
176
|
}>;
|
|
177
|
+
categories?: Array<"github" | "research" | CategoryOption>;
|
|
156
178
|
limit?: number;
|
|
157
179
|
tbs?: string;
|
|
158
180
|
location?: string;
|
|
@@ -215,7 +237,7 @@ interface BatchScrapeJob {
|
|
|
215
237
|
data: Document[];
|
|
216
238
|
}
|
|
217
239
|
interface MapData {
|
|
218
|
-
links:
|
|
240
|
+
links: SearchResultWeb[];
|
|
219
241
|
}
|
|
220
242
|
interface MapOptions {
|
|
221
243
|
search?: string;
|
|
@@ -1321,9 +1343,9 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
1321
1343
|
private _v1?;
|
|
1322
1344
|
private _v1Opts;
|
|
1323
1345
|
/** @param opts API credentials and base URL. */
|
|
1324
|
-
constructor(opts?:
|
|
1346
|
+
constructor(opts?: FirecrawlClientOptions);
|
|
1325
1347
|
/** Access the legacy v1 client (instantiated on first access). */
|
|
1326
1348
|
get v1(): FirecrawlApp;
|
|
1327
1349
|
}
|
|
1328
1350
|
|
|
1329
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig, type MapData, type MapOptions, type PDFAction, type PressAction, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type
|
|
1351
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig, type MapData, type MapOptions, type PDFAction, type PressAction, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-QPAPMZLC.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -200,6 +200,7 @@ function prepareSearchPayload(req) {
|
|
|
200
200
|
query: req.query
|
|
201
201
|
};
|
|
202
202
|
if (req.sources) payload.sources = req.sources;
|
|
203
|
+
if (req.categories) payload.categories = req.categories;
|
|
203
204
|
if (req.limit != null) payload.limit = req.limit;
|
|
204
205
|
if (req.tbs != null) payload.tbs = req.tbs;
|
|
205
206
|
if (req.location != null) payload.location = req.location;
|
|
@@ -211,6 +212,21 @@ function prepareSearchPayload(req) {
|
|
|
211
212
|
}
|
|
212
213
|
return payload;
|
|
213
214
|
}
|
|
215
|
+
function transformArray(arr) {
|
|
216
|
+
const results = [];
|
|
217
|
+
for (const item of arr) {
|
|
218
|
+
if (item && typeof item === "object") {
|
|
219
|
+
if ("markdown" in item || "html" in item || "rawHtml" in item || "links" in item || "screenshot" in item || "changeTracking" in item || "summary" in item || "json" in item) {
|
|
220
|
+
results.push(item);
|
|
221
|
+
} else {
|
|
222
|
+
results.push(item);
|
|
223
|
+
}
|
|
224
|
+
} else {
|
|
225
|
+
results.push({ url: item });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return results;
|
|
229
|
+
}
|
|
214
230
|
async function search(http, request) {
|
|
215
231
|
const payload = prepareSearchPayload(request);
|
|
216
232
|
try {
|
|
@@ -220,24 +236,9 @@ async function search(http, request) {
|
|
|
220
236
|
}
|
|
221
237
|
const data = res.data.data || {};
|
|
222
238
|
const out = {};
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
const results = [];
|
|
227
|
-
for (const item of arr) {
|
|
228
|
-
if (item && typeof item === "object") {
|
|
229
|
-
if ("markdown" in item || "html" in item || "rawHtml" in item || "links" in item || "screenshot" in item || "changeTracking" in item || "summary" in item || "json" in item) {
|
|
230
|
-
results.push(item);
|
|
231
|
-
} else {
|
|
232
|
-
results.push({ url: item.url, title: item.title, description: item.description });
|
|
233
|
-
}
|
|
234
|
-
} else if (typeof item === "string") {
|
|
235
|
-
results.push({ url: item });
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
out[key] = results;
|
|
239
|
-
}
|
|
240
|
-
}
|
|
239
|
+
if (data.web) out.web = transformArray(data.web);
|
|
240
|
+
if (data.news) out.news = transformArray(data.news);
|
|
241
|
+
if (data.images) out.images = transformArray(data.images);
|
|
241
242
|
return out;
|
|
242
243
|
} catch (err) {
|
|
243
244
|
if (err?.isAxiosError) return normalizeAxiosError(err, "search");
|
|
@@ -932,7 +933,7 @@ var FirecrawlApp = class {
|
|
|
932
933
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
933
934
|
return process.env.npm_package_version;
|
|
934
935
|
}
|
|
935
|
-
const packageJson = await import("./package-
|
|
936
|
+
const packageJson = await import("./package-VNFDXLYR.js");
|
|
936
937
|
return packageJson.default.version;
|
|
937
938
|
} catch (error) {
|
|
938
939
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -2231,7 +2232,10 @@ var Firecrawl = class extends FirecrawlClient {
|
|
|
2231
2232
|
/** @param opts API credentials and base URL. */
|
|
2232
2233
|
constructor(opts = {}) {
|
|
2233
2234
|
super(opts);
|
|
2234
|
-
this._v1Opts =
|
|
2235
|
+
this._v1Opts = {
|
|
2236
|
+
apiKey: opts.apiKey,
|
|
2237
|
+
apiUrl: opts.apiUrl
|
|
2238
|
+
};
|
|
2235
2239
|
}
|
|
2236
2240
|
/** Access the legacy v1 client (instantiated on first access). */
|
|
2237
2241
|
get v1() {
|
package/package.json
CHANGED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { Firecrawl, type FirecrawlClientOptions } from '../../../index';
|
|
2
|
+
|
|
3
|
+
describe('Firecrawl v2 Client Options', () => {
|
|
4
|
+
it('should accept v2 options including timeoutMs, maxRetries, and backoffFactor', () => {
|
|
5
|
+
const options: FirecrawlClientOptions = {
|
|
6
|
+
apiKey: 'test-key',
|
|
7
|
+
timeoutMs: 300,
|
|
8
|
+
maxRetries: 5,
|
|
9
|
+
backoffFactor: 0.5,
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
// Should not throw any type errors
|
|
13
|
+
const client = new Firecrawl(options);
|
|
14
|
+
|
|
15
|
+
expect(client).toBeDefined();
|
|
16
|
+
expect(client).toBeInstanceOf(Firecrawl);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('should work with minimal options', () => {
|
|
20
|
+
const options: FirecrawlClientOptions = {
|
|
21
|
+
apiKey: 'test-key',
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const client = new Firecrawl(options);
|
|
25
|
+
|
|
26
|
+
expect(client).toBeDefined();
|
|
27
|
+
expect(client).toBeInstanceOf(Firecrawl);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('should work with all v2 options', () => {
|
|
31
|
+
const options: FirecrawlClientOptions = {
|
|
32
|
+
apiKey: 'test-key',
|
|
33
|
+
apiUrl: 'https://custom-api.firecrawl.dev',
|
|
34
|
+
timeoutMs: 60000,
|
|
35
|
+
maxRetries: 3,
|
|
36
|
+
backoffFactor: 1.0,
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const client = new Firecrawl(options);
|
|
40
|
+
|
|
41
|
+
expect(client).toBeDefined();
|
|
42
|
+
expect(client).toBeInstanceOf(Firecrawl);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('should export FirecrawlClientOptions type', () => {
|
|
46
|
+
// This test ensures the type is properly exported
|
|
47
|
+
const options: FirecrawlClientOptions = {
|
|
48
|
+
apiKey: 'test-key',
|
|
49
|
+
timeoutMs: 300,
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
expect(options.timeoutMs).toBe(300);
|
|
53
|
+
expect(options.apiKey).toBe('test-key');
|
|
54
|
+
});
|
|
55
|
+
});
|
|
@@ -9,11 +9,18 @@ describe("v2 utils: validation", () => {
|
|
|
9
9
|
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
|
|
10
10
|
});
|
|
11
11
|
|
|
12
|
-
test("ensureValidFormats: json format requires prompt
|
|
13
|
-
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
test("ensureValidFormats: json format requires prompt or schema", () => {
|
|
13
|
+
// Valid cases - should not throw
|
|
14
|
+
const valid1: FormatOption[] = [{ type: "json", prompt: "p" } as any];
|
|
15
|
+
const valid2: FormatOption[] = [{ type: "json", schema: {} } as any];
|
|
16
|
+
const valid3: FormatOption[] = [{ type: "json", prompt: "p", schema: {} } as any];
|
|
17
|
+
expect(() => ensureValidFormats(valid1)).not.toThrow();
|
|
18
|
+
expect(() => ensureValidFormats(valid2)).not.toThrow();
|
|
19
|
+
expect(() => ensureValidFormats(valid3)).not.toThrow();
|
|
20
|
+
|
|
21
|
+
// Invalid case - should throw when both are missing
|
|
22
|
+
const bad: FormatOption[] = [{ type: "json" } as any];
|
|
23
|
+
expect(() => ensureValidFormats(bad)).toThrow(/requires either 'prompt' or 'schema'/i);
|
|
17
24
|
});
|
|
18
25
|
|
|
19
26
|
test("ensureValidFormats: converts zod schema to JSON schema", () => {
|
package/src/index.ts
CHANGED
|
@@ -13,9 +13,12 @@ export * from "./v2/types";
|
|
|
13
13
|
export { default as FirecrawlAppV1 } from "./v1";
|
|
14
14
|
|
|
15
15
|
import V1 from "./v1";
|
|
16
|
-
import { FirecrawlClient as V2 } from "./v2/client";
|
|
16
|
+
import { FirecrawlClient as V2, type FirecrawlClientOptions } from "./v2/client";
|
|
17
17
|
import type { FirecrawlAppConfig } from "./v1";
|
|
18
18
|
|
|
19
|
+
// Re-export v2 client options for convenience
|
|
20
|
+
export type { FirecrawlClientOptions } from "./v2/client";
|
|
21
|
+
|
|
19
22
|
/** Unified client: extends v2 and adds `.v1` for backward compatibility. */
|
|
20
23
|
export class Firecrawl extends V2 {
|
|
21
24
|
/** Feature‑frozen v1 client (lazy). */
|
|
@@ -23,9 +26,12 @@ export class Firecrawl extends V2 {
|
|
|
23
26
|
private _v1Opts: FirecrawlAppConfig;
|
|
24
27
|
|
|
25
28
|
/** @param opts API credentials and base URL. */
|
|
26
|
-
constructor(opts:
|
|
27
|
-
super(opts
|
|
28
|
-
this._v1Opts =
|
|
29
|
+
constructor(opts: FirecrawlClientOptions = {}) {
|
|
30
|
+
super(opts);
|
|
31
|
+
this._v1Opts = {
|
|
32
|
+
apiKey: opts.apiKey,
|
|
33
|
+
apiUrl: opts.apiUrl,
|
|
34
|
+
};
|
|
29
35
|
}
|
|
30
36
|
|
|
31
37
|
/** Access the legacy v1 client (instantiated on first access). */
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Document, type SearchData, type SearchRequest, type
|
|
1
|
+
import { type Document, type SearchData, type SearchRequest, type SearchResultWeb, type ScrapeOptions, type SearchResultNews, type SearchResultImages } from "../types";
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
4
4
|
import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
|
|
@@ -11,6 +11,7 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
|
|
11
11
|
query: req.query,
|
|
12
12
|
};
|
|
13
13
|
if (req.sources) payload.sources = req.sources;
|
|
14
|
+
if (req.categories) payload.categories = req.categories;
|
|
14
15
|
if (req.limit != null) payload.limit = req.limit;
|
|
15
16
|
if (req.tbs != null) payload.tbs = req.tbs;
|
|
16
17
|
if (req.location != null) payload.location = req.location;
|
|
@@ -23,6 +24,31 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
|
|
23
24
|
return payload;
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
function transformArray<ResultType>(arr: any[]): Array<ResultType | Document> {
|
|
28
|
+
const results: Array<ResultType | Document> = [] as any;
|
|
29
|
+
for (const item of arr) {
|
|
30
|
+
if (item && typeof item === "object") {
|
|
31
|
+
if (
|
|
32
|
+
"markdown" in item ||
|
|
33
|
+
"html" in item ||
|
|
34
|
+
"rawHtml" in item ||
|
|
35
|
+
"links" in item ||
|
|
36
|
+
"screenshot" in item ||
|
|
37
|
+
"changeTracking" in item ||
|
|
38
|
+
"summary" in item ||
|
|
39
|
+
"json" in item
|
|
40
|
+
) {
|
|
41
|
+
results.push(item as Document);
|
|
42
|
+
} else {
|
|
43
|
+
results.push(item as ResultType);
|
|
44
|
+
}
|
|
45
|
+
} else {
|
|
46
|
+
results.push({ url: item } as ResultType);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return results;
|
|
50
|
+
}
|
|
51
|
+
|
|
26
52
|
export async function search(http: HttpClient, request: SearchRequest): Promise<SearchData> {
|
|
27
53
|
const payload = prepareSearchPayload(request);
|
|
28
54
|
try {
|
|
@@ -32,34 +58,9 @@ export async function search(http: HttpClient, request: SearchRequest): Promise<
|
|
|
32
58
|
}
|
|
33
59
|
const data = (res.data.data || {}) as Record<string, any>;
|
|
34
60
|
const out: SearchData = {};
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const results: Array<SearchResult | Document> = [] as any;
|
|
39
|
-
for (const item of arr) {
|
|
40
|
-
if (item && typeof item === "object") {
|
|
41
|
-
// If scraped page fields present, treat as Document; otherwise SearchResult
|
|
42
|
-
if (
|
|
43
|
-
"markdown" in item ||
|
|
44
|
-
"html" in item ||
|
|
45
|
-
"rawHtml" in item ||
|
|
46
|
-
"links" in item ||
|
|
47
|
-
"screenshot" in item ||
|
|
48
|
-
"changeTracking" in item ||
|
|
49
|
-
"summary" in item ||
|
|
50
|
-
"json" in item
|
|
51
|
-
) {
|
|
52
|
-
results.push(item as Document);
|
|
53
|
-
} else {
|
|
54
|
-
results.push({ url: item.url, title: item.title, description: item.description } as SearchResult);
|
|
55
|
-
}
|
|
56
|
-
} else if (typeof item === "string") {
|
|
57
|
-
results.push({ url: item } as SearchResult);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
(out as any)[key] = results;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
61
|
+
if (data.web) out.web = transformArray<SearchResultWeb>(data.web);
|
|
62
|
+
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
|
|
63
|
+
if (data.images) out.images = transformArray<SearchResultImages>(data.images);
|
|
63
64
|
return out;
|
|
64
65
|
} catch (err: any) {
|
|
65
66
|
if (err?.isAxiosError) return normalizeAxiosError(err, "search");
|
package/src/v2/types.ts
CHANGED
|
@@ -173,21 +173,46 @@ export interface Document {
|
|
|
173
173
|
changeTracking?: Record<string, unknown>;
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
export interface
|
|
176
|
+
export interface SearchResultWeb {
|
|
177
177
|
url: string;
|
|
178
178
|
title?: string;
|
|
179
179
|
description?: string;
|
|
180
|
+
category?: string;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export interface SearchResultNews {
|
|
184
|
+
title?: string;
|
|
185
|
+
url?: string;
|
|
186
|
+
snippet?: string;
|
|
187
|
+
date?: string;
|
|
188
|
+
imageUrl?: string;
|
|
189
|
+
position?: number;
|
|
190
|
+
category?: string;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export interface SearchResultImages {
|
|
194
|
+
title?: string;
|
|
195
|
+
imageUrl?: string;
|
|
196
|
+
imageWidth?: number;
|
|
197
|
+
imageHeight?: number;
|
|
198
|
+
url?: string;
|
|
199
|
+
position?: number;
|
|
180
200
|
}
|
|
181
201
|
|
|
182
202
|
export interface SearchData {
|
|
183
|
-
web?: Array<
|
|
184
|
-
news?: Array<
|
|
185
|
-
images?: Array<
|
|
203
|
+
web?: Array<SearchResultWeb | Document>;
|
|
204
|
+
news?: Array<SearchResultNews | Document>;
|
|
205
|
+
images?: Array<SearchResultImages | Document>;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export interface CategoryOption {
|
|
209
|
+
type: "github" | "research";
|
|
186
210
|
}
|
|
187
211
|
|
|
188
212
|
export interface SearchRequest {
|
|
189
213
|
query: string;
|
|
190
214
|
sources?: Array<"web" | "news" | "images" | { type: "web" | "news" | "images" }>;
|
|
215
|
+
categories?: Array<"github" | "research" | CategoryOption>;
|
|
191
216
|
limit?: number;
|
|
192
217
|
tbs?: string;
|
|
193
218
|
location?: string;
|
|
@@ -257,7 +282,7 @@ export interface BatchScrapeJob {
|
|
|
257
282
|
}
|
|
258
283
|
|
|
259
284
|
export interface MapData {
|
|
260
|
-
links:
|
|
285
|
+
links: SearchResultWeb[];
|
|
261
286
|
}
|
|
262
287
|
|
|
263
288
|
export interface MapOptions {
|