@mendable/firecrawl 4.11.1 → 4.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-SHZTLDEM.js → chunk-TLAC3APA.js} +3 -3
- package/dist/index.cjs +62 -92
- package/dist/index.d.cts +9 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.js +62 -92
- package/dist/{package-LM7L2L66.js → package-3BZQ6TAW.js} +1 -1
- package/package.json +8 -17
- package/src/__tests__/unit/v2/agent.test.ts +80 -0
- package/src/__tests__/unit/v2/zodSchemaToJson.test.ts +107 -0
- package/src/utils/zodSchemaToJson.ts +67 -0
- package/src/v1/index.ts +10 -83
- package/src/v2/methods/agent.ts +5 -5
- package/src/v2/methods/extract.ts +2 -4
- package/src/v2/types.ts +10 -0
- package/src/v2/utils/validation.ts +7 -32
|
@@ -8,7 +8,7 @@ var require_package = __commonJS({
|
|
|
8
8
|
"package.json"(exports, module) {
|
|
9
9
|
module.exports = {
|
|
10
10
|
name: "@mendable/firecrawl-js",
|
|
11
|
-
version: "4.11.
|
|
11
|
+
version: "4.11.3",
|
|
12
12
|
description: "JavaScript SDK for Firecrawl API",
|
|
13
13
|
main: "dist/index.js",
|
|
14
14
|
types: "dist/index.d.ts",
|
|
@@ -22,8 +22,8 @@ var require_package = __commonJS({
|
|
|
22
22
|
type: "module",
|
|
23
23
|
scripts: {
|
|
24
24
|
build: "tsup",
|
|
25
|
-
"build-and-publish": "
|
|
26
|
-
"publish-beta": "
|
|
25
|
+
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
26
|
+
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
27
27
|
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
28
28
|
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
|
|
29
29
|
},
|
package/dist/index.cjs
CHANGED
|
@@ -35,7 +35,7 @@ var require_package = __commonJS({
|
|
|
35
35
|
"package.json"(exports2, module2) {
|
|
36
36
|
module2.exports = {
|
|
37
37
|
name: "@mendable/firecrawl-js",
|
|
38
|
-
version: "4.11.
|
|
38
|
+
version: "4.11.3",
|
|
39
39
|
description: "JavaScript SDK for Firecrawl API",
|
|
40
40
|
main: "dist/index.js",
|
|
41
41
|
types: "dist/index.d.ts",
|
|
@@ -49,8 +49,8 @@ var require_package = __commonJS({
|
|
|
49
49
|
type: "module",
|
|
50
50
|
scripts: {
|
|
51
51
|
build: "tsup",
|
|
52
|
-
"build-and-publish": "
|
|
53
|
-
"publish-beta": "
|
|
52
|
+
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
53
|
+
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
54
54
|
test: "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
55
55
|
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
|
|
56
56
|
},
|
|
@@ -249,14 +249,54 @@ var JobTimeoutError = class extends SdkError {
|
|
|
249
249
|
}
|
|
250
250
|
};
|
|
251
251
|
|
|
252
|
-
// src/
|
|
252
|
+
// src/utils/zodSchemaToJson.ts
|
|
253
253
|
var import_zod_to_json_schema = require("zod-to-json-schema");
|
|
254
|
+
function isZodSchema(value) {
|
|
255
|
+
if (!value || typeof value !== "object") return false;
|
|
256
|
+
const schema = value;
|
|
257
|
+
const hasV3Markers = "_def" in schema && (typeof schema.safeParse === "function" || typeof schema.parse === "function");
|
|
258
|
+
const hasV4Markers = "_zod" in schema && typeof schema._zod === "object";
|
|
259
|
+
return hasV3Markers || hasV4Markers;
|
|
260
|
+
}
|
|
261
|
+
function isZodV4Schema(schema) {
|
|
262
|
+
if (!schema || typeof schema !== "object") return false;
|
|
263
|
+
return "_zod" in schema && typeof schema._zod === "object";
|
|
264
|
+
}
|
|
265
|
+
function tryZodV4Conversion(schema) {
|
|
266
|
+
if (!isZodV4Schema(schema)) return null;
|
|
267
|
+
try {
|
|
268
|
+
const zodModule = schema.constructor?.prototype?.constructor;
|
|
269
|
+
if (zodModule && typeof zodModule.toJSONSchema === "function") {
|
|
270
|
+
return zodModule.toJSONSchema(schema);
|
|
271
|
+
}
|
|
272
|
+
} catch {
|
|
273
|
+
}
|
|
274
|
+
return null;
|
|
275
|
+
}
|
|
276
|
+
function zodSchemaToJsonSchema(schema) {
|
|
277
|
+
if (!isZodSchema(schema)) {
|
|
278
|
+
return schema;
|
|
279
|
+
}
|
|
280
|
+
const v4Result = tryZodV4Conversion(schema);
|
|
281
|
+
if (v4Result) {
|
|
282
|
+
return v4Result;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
return (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
|
|
286
|
+
} catch {
|
|
287
|
+
return schema;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
254
290
|
function looksLikeZodShape(obj) {
|
|
255
291
|
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return false;
|
|
256
292
|
const values = Object.values(obj);
|
|
257
293
|
if (values.length === 0) return false;
|
|
258
|
-
return values.some(
|
|
294
|
+
return values.some(
|
|
295
|
+
(v) => v && typeof v === "object" && v._def && typeof v.safeParse === "function"
|
|
296
|
+
);
|
|
259
297
|
}
|
|
298
|
+
|
|
299
|
+
// src/v2/utils/validation.ts
|
|
260
300
|
function ensureValidFormats(formats) {
|
|
261
301
|
if (!formats) return;
|
|
262
302
|
for (const fmt of formats) {
|
|
@@ -272,12 +312,8 @@ function ensureValidFormats(formats) {
|
|
|
272
312
|
throw new Error("json format requires either 'prompt' or 'schema' (or both)");
|
|
273
313
|
}
|
|
274
314
|
const maybeSchema = j.schema;
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
try {
|
|
278
|
-
j.schema = (0, import_zod_to_json_schema.zodToJsonSchema)(maybeSchema);
|
|
279
|
-
} catch {
|
|
280
|
-
}
|
|
315
|
+
if (isZodSchema(maybeSchema)) {
|
|
316
|
+
j.schema = zodSchemaToJsonSchema(maybeSchema);
|
|
281
317
|
} else if (looksLikeZodShape(maybeSchema)) {
|
|
282
318
|
throw new Error(
|
|
283
319
|
"json format schema appears to be a Zod schema's .shape property. Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. The SDK will automatically convert Zod schemas to JSON Schema format."
|
|
@@ -288,12 +324,8 @@ function ensureValidFormats(formats) {
|
|
|
288
324
|
if (fmt.type === "changeTracking") {
|
|
289
325
|
const ct = fmt;
|
|
290
326
|
const maybeSchema = ct.schema;
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
try {
|
|
294
|
-
ct.schema = (0, import_zod_to_json_schema.zodToJsonSchema)(maybeSchema);
|
|
295
|
-
} catch {
|
|
296
|
-
}
|
|
327
|
+
if (isZodSchema(maybeSchema)) {
|
|
328
|
+
ct.schema = zodSchemaToJsonSchema(maybeSchema);
|
|
297
329
|
} else if (looksLikeZodShape(maybeSchema)) {
|
|
298
330
|
throw new Error(
|
|
299
331
|
"changeTracking format schema appears to be a Zod schema's .shape property. Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. The SDK will automatically convert Zod schemas to JSON Schema format."
|
|
@@ -780,15 +812,12 @@ async function batchScrape(http, urls, opts = {}) {
|
|
|
780
812
|
}
|
|
781
813
|
|
|
782
814
|
// src/v2/methods/extract.ts
|
|
783
|
-
var import_zod_to_json_schema2 = require("zod-to-json-schema");
|
|
784
815
|
function prepareExtractPayload(args) {
|
|
785
816
|
const body = {};
|
|
786
817
|
if (args.urls) body.urls = args.urls;
|
|
787
818
|
if (args.prompt != null) body.prompt = args.prompt;
|
|
788
819
|
if (args.schema != null) {
|
|
789
|
-
|
|
790
|
-
const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
|
|
791
|
-
body.schema = isZod ? (0, import_zod_to_json_schema2.zodToJsonSchema)(s) : args.schema;
|
|
820
|
+
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
|
792
821
|
}
|
|
793
822
|
if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
|
|
794
823
|
if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
|
|
@@ -841,20 +870,18 @@ async function extract(http, args) {
|
|
|
841
870
|
}
|
|
842
871
|
|
|
843
872
|
// src/v2/methods/agent.ts
|
|
844
|
-
var import_zod_to_json_schema3 = require("zod-to-json-schema");
|
|
845
873
|
function prepareAgentPayload(args) {
|
|
846
874
|
const body = {};
|
|
847
875
|
if (args.urls) body.urls = args.urls;
|
|
848
876
|
body.prompt = args.prompt;
|
|
849
877
|
if (args.schema != null) {
|
|
850
|
-
|
|
851
|
-
const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
|
|
852
|
-
body.schema = isZod ? (0, import_zod_to_json_schema3.zodToJsonSchema)(s) : args.schema;
|
|
878
|
+
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
|
853
879
|
}
|
|
854
880
|
if (args.integration && args.integration.trim()) body.integration = args.integration.trim();
|
|
855
881
|
if (args.maxCredits !== null && args.maxCredits !== void 0) body.maxCredits = args.maxCredits;
|
|
856
882
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== void 0) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
857
883
|
if (args.model !== null && args.model !== void 0) body.model = args.model;
|
|
884
|
+
if (args.webhook != null) body.webhook = args.webhook;
|
|
858
885
|
return body;
|
|
859
886
|
}
|
|
860
887
|
async function startAgent(http, args) {
|
|
@@ -1436,9 +1463,8 @@ var FirecrawlClient = class {
|
|
|
1436
1463
|
// src/v1/index.ts
|
|
1437
1464
|
var import_axios3 = __toESM(require("axios"), 1);
|
|
1438
1465
|
var zt2 = require("zod");
|
|
1439
|
-
var import_zod_to_json_schema4 = require("zod-to-json-schema");
|
|
1440
1466
|
|
|
1441
|
-
// node_modules/typescript-event-target/dist/index.mjs
|
|
1467
|
+
// node_modules/.pnpm/typescript-event-target@1.1.1/node_modules/typescript-event-target/dist/index.mjs
|
|
1442
1468
|
var e = class extends EventTarget {
|
|
1443
1469
|
dispatchTypedEvent(s, t) {
|
|
1444
1470
|
return super.dispatchEvent(t);
|
|
@@ -1506,30 +1532,20 @@ var FirecrawlApp = class {
|
|
|
1506
1532
|
};
|
|
1507
1533
|
let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
1508
1534
|
if (jsonData?.extract?.schema) {
|
|
1509
|
-
let schema = jsonData.extract.schema;
|
|
1510
|
-
try {
|
|
1511
|
-
schema = (0, import_zod_to_json_schema4.zodToJsonSchema)(schema);
|
|
1512
|
-
} catch (error) {
|
|
1513
|
-
}
|
|
1514
1535
|
jsonData = {
|
|
1515
1536
|
...jsonData,
|
|
1516
1537
|
extract: {
|
|
1517
1538
|
...jsonData.extract,
|
|
1518
|
-
schema
|
|
1539
|
+
schema: zodSchemaToJsonSchema(jsonData.extract.schema)
|
|
1519
1540
|
}
|
|
1520
1541
|
};
|
|
1521
1542
|
}
|
|
1522
1543
|
if (jsonData?.jsonOptions?.schema) {
|
|
1523
|
-
let schema = jsonData.jsonOptions.schema;
|
|
1524
|
-
try {
|
|
1525
|
-
schema = (0, import_zod_to_json_schema4.zodToJsonSchema)(schema);
|
|
1526
|
-
} catch (error) {
|
|
1527
|
-
}
|
|
1528
1544
|
jsonData = {
|
|
1529
1545
|
...jsonData,
|
|
1530
1546
|
jsonOptions: {
|
|
1531
1547
|
...jsonData.jsonOptions,
|
|
1532
|
-
schema
|
|
1548
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema)
|
|
1533
1549
|
}
|
|
1534
1550
|
};
|
|
1535
1551
|
}
|
|
@@ -1583,18 +1599,13 @@ var FirecrawlApp = class {
|
|
|
1583
1599
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
1584
1600
|
};
|
|
1585
1601
|
if (jsonData?.scrapeOptions?.extract?.schema) {
|
|
1586
|
-
let schema = jsonData.scrapeOptions.extract.schema;
|
|
1587
|
-
try {
|
|
1588
|
-
schema = (0, import_zod_to_json_schema4.zodToJsonSchema)(schema);
|
|
1589
|
-
} catch (error) {
|
|
1590
|
-
}
|
|
1591
1602
|
jsonData = {
|
|
1592
1603
|
...jsonData,
|
|
1593
1604
|
scrapeOptions: {
|
|
1594
1605
|
...jsonData.scrapeOptions,
|
|
1595
1606
|
extract: {
|
|
1596
1607
|
...jsonData.scrapeOptions.extract,
|
|
1597
|
-
schema
|
|
1608
|
+
schema: zodSchemaToJsonSchema(jsonData.scrapeOptions.extract.schema)
|
|
1598
1609
|
}
|
|
1599
1610
|
}
|
|
1600
1611
|
};
|
|
@@ -1852,30 +1863,20 @@ var FirecrawlApp = class {
|
|
|
1852
1863
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1853
1864
|
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
1854
1865
|
if (jsonData?.extract?.schema) {
|
|
1855
|
-
let schema = jsonData.extract.schema;
|
|
1856
|
-
try {
|
|
1857
|
-
schema = (0, import_zod_to_json_schema4.zodToJsonSchema)(schema);
|
|
1858
|
-
} catch (error) {
|
|
1859
|
-
}
|
|
1860
1866
|
jsonData = {
|
|
1861
1867
|
...jsonData,
|
|
1862
1868
|
extract: {
|
|
1863
1869
|
...jsonData.extract,
|
|
1864
|
-
schema
|
|
1870
|
+
schema: zodSchemaToJsonSchema(jsonData.extract.schema)
|
|
1865
1871
|
}
|
|
1866
1872
|
};
|
|
1867
1873
|
}
|
|
1868
1874
|
if (jsonData?.jsonOptions?.schema) {
|
|
1869
|
-
let schema = jsonData.jsonOptions.schema;
|
|
1870
|
-
try {
|
|
1871
|
-
schema = (0, import_zod_to_json_schema4.zodToJsonSchema)(schema);
|
|
1872
|
-
} catch (error) {
|
|
1873
|
-
}
|
|
1874
1875
|
jsonData = {
|
|
1875
1876
|
...jsonData,
|
|
1876
1877
|
jsonOptions: {
|
|
1877
1878
|
...jsonData.jsonOptions,
|
|
1878
|
-
schema
|
|
1879
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema)
|
|
1879
1880
|
}
|
|
1880
1881
|
};
|
|
1881
1882
|
}
|
|
@@ -2041,20 +2042,7 @@ var FirecrawlApp = class {
|
|
|
2041
2042
|
async extract(urls, params) {
|
|
2042
2043
|
const headers = this.prepareHeaders();
|
|
2043
2044
|
let jsonData = { urls, ...params };
|
|
2044
|
-
|
|
2045
|
-
try {
|
|
2046
|
-
if (!params?.schema) {
|
|
2047
|
-
jsonSchema = void 0;
|
|
2048
|
-
} else {
|
|
2049
|
-
try {
|
|
2050
|
-
jsonSchema = (0, import_zod_to_json_schema4.zodToJsonSchema)(params.schema);
|
|
2051
|
-
} catch (_) {
|
|
2052
|
-
jsonSchema = params.schema;
|
|
2053
|
-
}
|
|
2054
|
-
}
|
|
2055
|
-
} catch (error) {
|
|
2056
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
2057
|
-
}
|
|
2045
|
+
const jsonSchema = params?.schema ? zodSchemaToJsonSchema(params.schema) : void 0;
|
|
2058
2046
|
try {
|
|
2059
2047
|
const response = await this.postRequest(
|
|
2060
2048
|
this.apiUrl + `/v1/extract`,
|
|
@@ -2104,21 +2092,8 @@ var FirecrawlApp = class {
|
|
|
2104
2092
|
*/
|
|
2105
2093
|
async asyncExtract(urls, params, idempotencyKey) {
|
|
2106
2094
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
try {
|
|
2110
|
-
if (!params?.schema) {
|
|
2111
|
-
jsonSchema = void 0;
|
|
2112
|
-
} else {
|
|
2113
|
-
try {
|
|
2114
|
-
jsonSchema = (0, import_zod_to_json_schema4.zodToJsonSchema)(params.schema);
|
|
2115
|
-
} catch (_) {
|
|
2116
|
-
jsonSchema = params.schema;
|
|
2117
|
-
}
|
|
2118
|
-
}
|
|
2119
|
-
} catch (error) {
|
|
2120
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
2121
|
-
}
|
|
2095
|
+
const jsonData = { urls, ...params };
|
|
2096
|
+
const jsonSchema = params?.schema ? zodSchemaToJsonSchema(params.schema) : void 0;
|
|
2122
2097
|
try {
|
|
2123
2098
|
const response = await this.postRequest(
|
|
2124
2099
|
this.apiUrl + `/v1/extract`,
|
|
@@ -2400,16 +2375,11 @@ var FirecrawlApp = class {
|
|
|
2400
2375
|
const headers = this.prepareHeaders();
|
|
2401
2376
|
let jsonData = { query, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2402
2377
|
if (jsonData?.jsonOptions?.schema) {
|
|
2403
|
-
let schema = jsonData.jsonOptions.schema;
|
|
2404
|
-
try {
|
|
2405
|
-
schema = (0, import_zod_to_json_schema4.zodToJsonSchema)(schema);
|
|
2406
|
-
} catch (error) {
|
|
2407
|
-
}
|
|
2408
2378
|
jsonData = {
|
|
2409
2379
|
...jsonData,
|
|
2410
2380
|
jsonOptions: {
|
|
2411
2381
|
...jsonData.jsonOptions,
|
|
2412
|
-
schema
|
|
2382
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema)
|
|
2413
2383
|
}
|
|
2414
2384
|
};
|
|
2415
2385
|
}
|
package/dist/index.d.cts
CHANGED
|
@@ -122,6 +122,13 @@ interface WebhookConfig {
|
|
|
122
122
|
metadata?: Record<string, string>;
|
|
123
123
|
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
|
124
124
|
}
|
|
125
|
+
type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';
|
|
126
|
+
interface AgentWebhookConfig {
|
|
127
|
+
url: string;
|
|
128
|
+
headers?: Record<string, string>;
|
|
129
|
+
metadata?: Record<string, string>;
|
|
130
|
+
events?: AgentWebhookEvent[];
|
|
131
|
+
}
|
|
125
132
|
interface BrandingProfile {
|
|
126
133
|
colorScheme?: 'light' | 'dark';
|
|
127
134
|
logo?: string | null;
|
|
@@ -594,6 +601,7 @@ declare function prepareAgentPayload(args: {
|
|
|
594
601
|
maxCredits?: number;
|
|
595
602
|
strictConstrainToURLs?: boolean;
|
|
596
603
|
model?: "spark-1-pro" | "spark-1-mini";
|
|
604
|
+
webhook?: string | AgentWebhookConfig;
|
|
597
605
|
}): Record<string, unknown>;
|
|
598
606
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
599
607
|
|
|
@@ -1727,4 +1735,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
1727
1735
|
get v1(): FirecrawlApp;
|
|
1728
1736
|
}
|
|
1729
1737
|
|
|
1730
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, JobTimeoutError, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
1738
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, JobTimeoutError, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -122,6 +122,13 @@ interface WebhookConfig {
|
|
|
122
122
|
metadata?: Record<string, string>;
|
|
123
123
|
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
|
124
124
|
}
|
|
125
|
+
type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';
|
|
126
|
+
interface AgentWebhookConfig {
|
|
127
|
+
url: string;
|
|
128
|
+
headers?: Record<string, string>;
|
|
129
|
+
metadata?: Record<string, string>;
|
|
130
|
+
events?: AgentWebhookEvent[];
|
|
131
|
+
}
|
|
125
132
|
interface BrandingProfile {
|
|
126
133
|
colorScheme?: 'light' | 'dark';
|
|
127
134
|
logo?: string | null;
|
|
@@ -594,6 +601,7 @@ declare function prepareAgentPayload(args: {
|
|
|
594
601
|
maxCredits?: number;
|
|
595
602
|
strictConstrainToURLs?: boolean;
|
|
596
603
|
model?: "spark-1-pro" | "spark-1-mini";
|
|
604
|
+
webhook?: string | AgentWebhookConfig;
|
|
597
605
|
}): Record<string, unknown>;
|
|
598
606
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
599
607
|
|
|
@@ -1727,4 +1735,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
1727
1735
|
get v1(): FirecrawlApp;
|
|
1728
1736
|
}
|
|
1729
1737
|
|
|
1730
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, JobTimeoutError, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
1738
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, JobTimeoutError, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-TLAC3APA.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -130,14 +130,54 @@ var JobTimeoutError = class extends SdkError {
|
|
|
130
130
|
}
|
|
131
131
|
};
|
|
132
132
|
|
|
133
|
-
// src/
|
|
134
|
-
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
133
|
+
// src/utils/zodSchemaToJson.ts
|
|
134
|
+
import { zodToJsonSchema as zodToJsonSchemaLib } from "zod-to-json-schema";
|
|
135
|
+
function isZodSchema(value) {
|
|
136
|
+
if (!value || typeof value !== "object") return false;
|
|
137
|
+
const schema = value;
|
|
138
|
+
const hasV3Markers = "_def" in schema && (typeof schema.safeParse === "function" || typeof schema.parse === "function");
|
|
139
|
+
const hasV4Markers = "_zod" in schema && typeof schema._zod === "object";
|
|
140
|
+
return hasV3Markers || hasV4Markers;
|
|
141
|
+
}
|
|
142
|
+
function isZodV4Schema(schema) {
|
|
143
|
+
if (!schema || typeof schema !== "object") return false;
|
|
144
|
+
return "_zod" in schema && typeof schema._zod === "object";
|
|
145
|
+
}
|
|
146
|
+
function tryZodV4Conversion(schema) {
|
|
147
|
+
if (!isZodV4Schema(schema)) return null;
|
|
148
|
+
try {
|
|
149
|
+
const zodModule = schema.constructor?.prototype?.constructor;
|
|
150
|
+
if (zodModule && typeof zodModule.toJSONSchema === "function") {
|
|
151
|
+
return zodModule.toJSONSchema(schema);
|
|
152
|
+
}
|
|
153
|
+
} catch {
|
|
154
|
+
}
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
function zodSchemaToJsonSchema(schema) {
|
|
158
|
+
if (!isZodSchema(schema)) {
|
|
159
|
+
return schema;
|
|
160
|
+
}
|
|
161
|
+
const v4Result = tryZodV4Conversion(schema);
|
|
162
|
+
if (v4Result) {
|
|
163
|
+
return v4Result;
|
|
164
|
+
}
|
|
165
|
+
try {
|
|
166
|
+
return zodToJsonSchemaLib(schema);
|
|
167
|
+
} catch {
|
|
168
|
+
return schema;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
135
171
|
function looksLikeZodShape(obj) {
|
|
136
172
|
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return false;
|
|
137
173
|
const values = Object.values(obj);
|
|
138
174
|
if (values.length === 0) return false;
|
|
139
|
-
return values.some(
|
|
175
|
+
return values.some(
|
|
176
|
+
(v) => v && typeof v === "object" && v._def && typeof v.safeParse === "function"
|
|
177
|
+
);
|
|
140
178
|
}
|
|
179
|
+
|
|
180
|
+
// src/v2/utils/validation.ts
|
|
141
181
|
function ensureValidFormats(formats) {
|
|
142
182
|
if (!formats) return;
|
|
143
183
|
for (const fmt of formats) {
|
|
@@ -153,12 +193,8 @@ function ensureValidFormats(formats) {
|
|
|
153
193
|
throw new Error("json format requires either 'prompt' or 'schema' (or both)");
|
|
154
194
|
}
|
|
155
195
|
const maybeSchema = j.schema;
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
try {
|
|
159
|
-
j.schema = zodToJsonSchema(maybeSchema);
|
|
160
|
-
} catch {
|
|
161
|
-
}
|
|
196
|
+
if (isZodSchema(maybeSchema)) {
|
|
197
|
+
j.schema = zodSchemaToJsonSchema(maybeSchema);
|
|
162
198
|
} else if (looksLikeZodShape(maybeSchema)) {
|
|
163
199
|
throw new Error(
|
|
164
200
|
"json format schema appears to be a Zod schema's .shape property. Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. The SDK will automatically convert Zod schemas to JSON Schema format."
|
|
@@ -169,12 +205,8 @@ function ensureValidFormats(formats) {
|
|
|
169
205
|
if (fmt.type === "changeTracking") {
|
|
170
206
|
const ct = fmt;
|
|
171
207
|
const maybeSchema = ct.schema;
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
try {
|
|
175
|
-
ct.schema = zodToJsonSchema(maybeSchema);
|
|
176
|
-
} catch {
|
|
177
|
-
}
|
|
208
|
+
if (isZodSchema(maybeSchema)) {
|
|
209
|
+
ct.schema = zodSchemaToJsonSchema(maybeSchema);
|
|
178
210
|
} else if (looksLikeZodShape(maybeSchema)) {
|
|
179
211
|
throw new Error(
|
|
180
212
|
"changeTracking format schema appears to be a Zod schema's .shape property. Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. The SDK will automatically convert Zod schemas to JSON Schema format."
|
|
@@ -661,15 +693,12 @@ async function batchScrape(http, urls, opts = {}) {
|
|
|
661
693
|
}
|
|
662
694
|
|
|
663
695
|
// src/v2/methods/extract.ts
|
|
664
|
-
import { zodToJsonSchema as zodToJsonSchema2 } from "zod-to-json-schema";
|
|
665
696
|
function prepareExtractPayload(args) {
|
|
666
697
|
const body = {};
|
|
667
698
|
if (args.urls) body.urls = args.urls;
|
|
668
699
|
if (args.prompt != null) body.prompt = args.prompt;
|
|
669
700
|
if (args.schema != null) {
|
|
670
|
-
|
|
671
|
-
const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
|
|
672
|
-
body.schema = isZod ? zodToJsonSchema2(s) : args.schema;
|
|
701
|
+
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
|
673
702
|
}
|
|
674
703
|
if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
|
|
675
704
|
if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
|
|
@@ -722,20 +751,18 @@ async function extract(http, args) {
|
|
|
722
751
|
}
|
|
723
752
|
|
|
724
753
|
// src/v2/methods/agent.ts
|
|
725
|
-
import { zodToJsonSchema as zodToJsonSchema3 } from "zod-to-json-schema";
|
|
726
754
|
function prepareAgentPayload(args) {
|
|
727
755
|
const body = {};
|
|
728
756
|
if (args.urls) body.urls = args.urls;
|
|
729
757
|
body.prompt = args.prompt;
|
|
730
758
|
if (args.schema != null) {
|
|
731
|
-
|
|
732
|
-
const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
|
|
733
|
-
body.schema = isZod ? zodToJsonSchema3(s) : args.schema;
|
|
759
|
+
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
|
734
760
|
}
|
|
735
761
|
if (args.integration && args.integration.trim()) body.integration = args.integration.trim();
|
|
736
762
|
if (args.maxCredits !== null && args.maxCredits !== void 0) body.maxCredits = args.maxCredits;
|
|
737
763
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== void 0) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
738
764
|
if (args.model !== null && args.model !== void 0) body.model = args.model;
|
|
765
|
+
if (args.webhook != null) body.webhook = args.webhook;
|
|
739
766
|
return body;
|
|
740
767
|
}
|
|
741
768
|
async function startAgent(http, args) {
|
|
@@ -1317,9 +1344,8 @@ var FirecrawlClient = class {
|
|
|
1317
1344
|
// src/v1/index.ts
|
|
1318
1345
|
import axios2, { AxiosError } from "axios";
|
|
1319
1346
|
import "zod";
|
|
1320
|
-
import { zodToJsonSchema as zodToJsonSchema4 } from "zod-to-json-schema";
|
|
1321
1347
|
|
|
1322
|
-
// node_modules/typescript-event-target/dist/index.mjs
|
|
1348
|
+
// node_modules/.pnpm/typescript-event-target@1.1.1/node_modules/typescript-event-target/dist/index.mjs
|
|
1323
1349
|
var e = class extends EventTarget {
|
|
1324
1350
|
dispatchTypedEvent(s, t) {
|
|
1325
1351
|
return super.dispatchEvent(t);
|
|
@@ -1348,7 +1374,7 @@ var FirecrawlApp = class {
|
|
|
1348
1374
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
1349
1375
|
return process.env.npm_package_version;
|
|
1350
1376
|
}
|
|
1351
|
-
const packageJson = await import("./package-
|
|
1377
|
+
const packageJson = await import("./package-3BZQ6TAW.js");
|
|
1352
1378
|
return packageJson.default.version;
|
|
1353
1379
|
} catch (error) {
|
|
1354
1380
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -1387,30 +1413,20 @@ var FirecrawlApp = class {
|
|
|
1387
1413
|
};
|
|
1388
1414
|
let jsonData = { url, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
1389
1415
|
if (jsonData?.extract?.schema) {
|
|
1390
|
-
let schema = jsonData.extract.schema;
|
|
1391
|
-
try {
|
|
1392
|
-
schema = zodToJsonSchema4(schema);
|
|
1393
|
-
} catch (error) {
|
|
1394
|
-
}
|
|
1395
1416
|
jsonData = {
|
|
1396
1417
|
...jsonData,
|
|
1397
1418
|
extract: {
|
|
1398
1419
|
...jsonData.extract,
|
|
1399
|
-
schema
|
|
1420
|
+
schema: zodSchemaToJsonSchema(jsonData.extract.schema)
|
|
1400
1421
|
}
|
|
1401
1422
|
};
|
|
1402
1423
|
}
|
|
1403
1424
|
if (jsonData?.jsonOptions?.schema) {
|
|
1404
|
-
let schema = jsonData.jsonOptions.schema;
|
|
1405
|
-
try {
|
|
1406
|
-
schema = zodToJsonSchema4(schema);
|
|
1407
|
-
} catch (error) {
|
|
1408
|
-
}
|
|
1409
1425
|
jsonData = {
|
|
1410
1426
|
...jsonData,
|
|
1411
1427
|
jsonOptions: {
|
|
1412
1428
|
...jsonData.jsonOptions,
|
|
1413
|
-
schema
|
|
1429
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema)
|
|
1414
1430
|
}
|
|
1415
1431
|
};
|
|
1416
1432
|
}
|
|
@@ -1464,18 +1480,13 @@ var FirecrawlApp = class {
|
|
|
1464
1480
|
scrapeOptions: params?.scrapeOptions ?? { formats: [] }
|
|
1465
1481
|
};
|
|
1466
1482
|
if (jsonData?.scrapeOptions?.extract?.schema) {
|
|
1467
|
-
let schema = jsonData.scrapeOptions.extract.schema;
|
|
1468
|
-
try {
|
|
1469
|
-
schema = zodToJsonSchema4(schema);
|
|
1470
|
-
} catch (error) {
|
|
1471
|
-
}
|
|
1472
1483
|
jsonData = {
|
|
1473
1484
|
...jsonData,
|
|
1474
1485
|
scrapeOptions: {
|
|
1475
1486
|
...jsonData.scrapeOptions,
|
|
1476
1487
|
extract: {
|
|
1477
1488
|
...jsonData.scrapeOptions.extract,
|
|
1478
|
-
schema
|
|
1489
|
+
schema: zodSchemaToJsonSchema(jsonData.scrapeOptions.extract.schema)
|
|
1479
1490
|
}
|
|
1480
1491
|
}
|
|
1481
1492
|
};
|
|
@@ -1733,30 +1744,20 @@ var FirecrawlApp = class {
|
|
|
1733
1744
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1734
1745
|
let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
1735
1746
|
if (jsonData?.extract?.schema) {
|
|
1736
|
-
let schema = jsonData.extract.schema;
|
|
1737
|
-
try {
|
|
1738
|
-
schema = zodToJsonSchema4(schema);
|
|
1739
|
-
} catch (error) {
|
|
1740
|
-
}
|
|
1741
1747
|
jsonData = {
|
|
1742
1748
|
...jsonData,
|
|
1743
1749
|
extract: {
|
|
1744
1750
|
...jsonData.extract,
|
|
1745
|
-
schema
|
|
1751
|
+
schema: zodSchemaToJsonSchema(jsonData.extract.schema)
|
|
1746
1752
|
}
|
|
1747
1753
|
};
|
|
1748
1754
|
}
|
|
1749
1755
|
if (jsonData?.jsonOptions?.schema) {
|
|
1750
|
-
let schema = jsonData.jsonOptions.schema;
|
|
1751
|
-
try {
|
|
1752
|
-
schema = zodToJsonSchema4(schema);
|
|
1753
|
-
} catch (error) {
|
|
1754
|
-
}
|
|
1755
1756
|
jsonData = {
|
|
1756
1757
|
...jsonData,
|
|
1757
1758
|
jsonOptions: {
|
|
1758
1759
|
...jsonData.jsonOptions,
|
|
1759
|
-
schema
|
|
1760
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema)
|
|
1760
1761
|
}
|
|
1761
1762
|
};
|
|
1762
1763
|
}
|
|
@@ -1922,20 +1923,7 @@ var FirecrawlApp = class {
|
|
|
1922
1923
|
async extract(urls, params) {
|
|
1923
1924
|
const headers = this.prepareHeaders();
|
|
1924
1925
|
let jsonData = { urls, ...params };
|
|
1925
|
-
|
|
1926
|
-
try {
|
|
1927
|
-
if (!params?.schema) {
|
|
1928
|
-
jsonSchema = void 0;
|
|
1929
|
-
} else {
|
|
1930
|
-
try {
|
|
1931
|
-
jsonSchema = zodToJsonSchema4(params.schema);
|
|
1932
|
-
} catch (_) {
|
|
1933
|
-
jsonSchema = params.schema;
|
|
1934
|
-
}
|
|
1935
|
-
}
|
|
1936
|
-
} catch (error) {
|
|
1937
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
1938
|
-
}
|
|
1926
|
+
const jsonSchema = params?.schema ? zodSchemaToJsonSchema(params.schema) : void 0;
|
|
1939
1927
|
try {
|
|
1940
1928
|
const response = await this.postRequest(
|
|
1941
1929
|
this.apiUrl + `/v1/extract`,
|
|
@@ -1985,21 +1973,8 @@ var FirecrawlApp = class {
|
|
|
1985
1973
|
*/
|
|
1986
1974
|
async asyncExtract(urls, params, idempotencyKey) {
|
|
1987
1975
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
try {
|
|
1991
|
-
if (!params?.schema) {
|
|
1992
|
-
jsonSchema = void 0;
|
|
1993
|
-
} else {
|
|
1994
|
-
try {
|
|
1995
|
-
jsonSchema = zodToJsonSchema4(params.schema);
|
|
1996
|
-
} catch (_) {
|
|
1997
|
-
jsonSchema = params.schema;
|
|
1998
|
-
}
|
|
1999
|
-
}
|
|
2000
|
-
} catch (error) {
|
|
2001
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
2002
|
-
}
|
|
1976
|
+
const jsonData = { urls, ...params };
|
|
1977
|
+
const jsonSchema = params?.schema ? zodSchemaToJsonSchema(params.schema) : void 0;
|
|
2003
1978
|
try {
|
|
2004
1979
|
const response = await this.postRequest(
|
|
2005
1980
|
this.apiUrl + `/v1/extract`,
|
|
@@ -2281,16 +2256,11 @@ var FirecrawlApp = class {
|
|
|
2281
2256
|
const headers = this.prepareHeaders();
|
|
2282
2257
|
let jsonData = { query, ...params, origin: typeof params.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
|
|
2283
2258
|
if (jsonData?.jsonOptions?.schema) {
|
|
2284
|
-
let schema = jsonData.jsonOptions.schema;
|
|
2285
|
-
try {
|
|
2286
|
-
schema = zodToJsonSchema4(schema);
|
|
2287
|
-
} catch (error) {
|
|
2288
|
-
}
|
|
2289
2259
|
jsonData = {
|
|
2290
2260
|
...jsonData,
|
|
2291
2261
|
jsonOptions: {
|
|
2292
2262
|
...jsonData.jsonOptions,
|
|
2293
|
-
schema
|
|
2263
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema)
|
|
2294
2264
|
}
|
|
2295
2265
|
};
|
|
2296
2266
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl",
|
|
3
|
-
"version": "4.11.
|
|
3
|
+
"version": "4.11.3",
|
|
4
4
|
"description": "JavaScript SDK for Firecrawl API",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -12,13 +12,6 @@
|
|
|
12
12
|
}
|
|
13
13
|
},
|
|
14
14
|
"type": "module",
|
|
15
|
-
"scripts": {
|
|
16
|
-
"build": "tsup",
|
|
17
|
-
"build-and-publish": "npm run build && npm publish --access public",
|
|
18
|
-
"publish-beta": "npm run build && npm publish --access public --tag beta",
|
|
19
|
-
"test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
20
|
-
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
|
|
21
|
-
},
|
|
22
15
|
"repository": {
|
|
23
16
|
"type": "git",
|
|
24
17
|
"url": "git+https://github.com/firecrawl/firecrawl.git"
|
|
@@ -61,13 +54,11 @@
|
|
|
61
54
|
"engines": {
|
|
62
55
|
"node": ">=22.0.0"
|
|
63
56
|
},
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
"glob@>=10.2.0 <10.5.0": ">=10.5.0"
|
|
71
|
-
}
|
|
57
|
+
"scripts": {
|
|
58
|
+
"build": "tsup",
|
|
59
|
+
"build-and-publish": "pnpm run build && pnpm publish --access public",
|
|
60
|
+
"publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
|
|
61
|
+
"test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
|
|
62
|
+
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
|
|
72
63
|
}
|
|
73
|
-
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { describe, test, expect } from "@jest/globals";
|
|
2
|
+
|
|
3
|
+
// We need to test the prepareAgentPayload function, but it's not exported.
|
|
4
|
+
// Since the function is internal, we'll test the behavior through type checking
|
|
5
|
+
// and verify the types are properly exported.
|
|
6
|
+
|
|
7
|
+
import type { AgentWebhookConfig, AgentWebhookEvent } from "../../../v2/types";
|
|
8
|
+
|
|
9
|
+
describe("v2 types: Agent webhook types", () => {
|
|
10
|
+
test("AgentWebhookConfig accepts string webhook", () => {
|
|
11
|
+
// Type check - this should compile without errors
|
|
12
|
+
const webhook: string | AgentWebhookConfig = "https://example.com/webhook";
|
|
13
|
+
expect(typeof webhook).toBe("string");
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test("AgentWebhookConfig accepts config object", () => {
|
|
17
|
+
const config: AgentWebhookConfig = {
|
|
18
|
+
url: "https://example.com/webhook",
|
|
19
|
+
headers: { Authorization: "Bearer token" },
|
|
20
|
+
events: ["completed", "failed"],
|
|
21
|
+
};
|
|
22
|
+
expect(config.url).toBe("https://example.com/webhook");
|
|
23
|
+
expect(config.headers).toEqual({ Authorization: "Bearer token" });
|
|
24
|
+
expect(config.events).toEqual(["completed", "failed"]);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("AgentWebhookConfig accepts minimal config", () => {
|
|
28
|
+
const config: AgentWebhookConfig = {
|
|
29
|
+
url: "https://example.com/webhook",
|
|
30
|
+
};
|
|
31
|
+
expect(config.url).toBe("https://example.com/webhook");
|
|
32
|
+
expect(config.headers).toBeUndefined();
|
|
33
|
+
expect(config.metadata).toBeUndefined();
|
|
34
|
+
expect(config.events).toBeUndefined();
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("AgentWebhookEvent includes agent-specific events", () => {
|
|
38
|
+
const events: AgentWebhookEvent[] = [
|
|
39
|
+
"started",
|
|
40
|
+
"action",
|
|
41
|
+
"completed",
|
|
42
|
+
"failed",
|
|
43
|
+
"cancelled",
|
|
44
|
+
];
|
|
45
|
+
expect(events).toContain("action");
|
|
46
|
+
expect(events).toContain("cancelled");
|
|
47
|
+
expect(events.length).toBe(5);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("AgentWebhookConfig accepts all fields", () => {
|
|
51
|
+
const config: AgentWebhookConfig = {
|
|
52
|
+
url: "https://example.com/webhook",
|
|
53
|
+
headers: {
|
|
54
|
+
Authorization: "Bearer token",
|
|
55
|
+
"X-Custom-Header": "value",
|
|
56
|
+
},
|
|
57
|
+
metadata: {
|
|
58
|
+
project: "test",
|
|
59
|
+
environment: "staging",
|
|
60
|
+
},
|
|
61
|
+
events: ["started", "action", "completed", "failed", "cancelled"],
|
|
62
|
+
};
|
|
63
|
+
expect(config.url).toBe("https://example.com/webhook");
|
|
64
|
+
expect(Object.keys(config.headers!).length).toBe(2);
|
|
65
|
+
expect(config.metadata!.project).toBe("test");
|
|
66
|
+
expect(config.events!.length).toBe(5);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("AgentWebhookConfig events are agent-specific (not crawl)", () => {
|
|
70
|
+
// Agent has 'action' and 'cancelled', but not 'page'
|
|
71
|
+
const config: AgentWebhookConfig = {
|
|
72
|
+
url: "https://example.com/webhook",
|
|
73
|
+
events: ["action", "cancelled"],
|
|
74
|
+
};
|
|
75
|
+
expect(config.events).toContain("action");
|
|
76
|
+
expect(config.events).toContain("cancelled");
|
|
77
|
+
// 'page' is a crawl-specific event, not valid for agent
|
|
78
|
+
// This is enforced at the type level
|
|
79
|
+
});
|
|
80
|
+
});
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { describe, test, expect } from "@jest/globals";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import {
|
|
4
|
+
isZodSchema,
|
|
5
|
+
zodSchemaToJsonSchema,
|
|
6
|
+
looksLikeZodShape,
|
|
7
|
+
} from "../../../utils/zodSchemaToJson";
|
|
8
|
+
|
|
9
|
+
describe("zodSchemaToJson utility", () => {
|
|
10
|
+
test("isZodSchema detects Zod schemas and rejects non-Zod values", () => {
|
|
11
|
+
expect(isZodSchema(z.object({ name: z.string() }))).toBe(true);
|
|
12
|
+
expect(isZodSchema(z.string())).toBe(true);
|
|
13
|
+
expect(isZodSchema(z.number())).toBe(true);
|
|
14
|
+
expect(isZodSchema(z.array(z.string()))).toBe(true);
|
|
15
|
+
expect(isZodSchema(z.enum(["A", "B"]))).toBe(true);
|
|
16
|
+
expect(isZodSchema(z.union([z.string(), z.number()]))).toBe(true);
|
|
17
|
+
expect(isZodSchema(z.string().optional())).toBe(true);
|
|
18
|
+
expect(isZodSchema(z.string().nullable())).toBe(true);
|
|
19
|
+
|
|
20
|
+
expect(isZodSchema(null)).toBe(false);
|
|
21
|
+
expect(isZodSchema(undefined)).toBe(false);
|
|
22
|
+
expect(isZodSchema({ name: "test" })).toBe(false);
|
|
23
|
+
expect(isZodSchema({ type: "object", properties: {} })).toBe(false);
|
|
24
|
+
expect(isZodSchema("string")).toBe(false);
|
|
25
|
+
expect(isZodSchema(42)).toBe(false);
|
|
26
|
+
expect(isZodSchema([1, 2, 3])).toBe(false);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("zodSchemaToJsonSchema converts Zod schemas to JSON Schema", () => {
|
|
30
|
+
const simpleSchema = z.object({ name: z.string() });
|
|
31
|
+
const simpleResult = zodSchemaToJsonSchema(simpleSchema) as Record<string, unknown>;
|
|
32
|
+
expect(simpleResult.type).toBe("object");
|
|
33
|
+
expect(simpleResult.properties).toBeDefined();
|
|
34
|
+
expect((simpleResult.properties as Record<string, unknown>).name).toBeDefined();
|
|
35
|
+
|
|
36
|
+
const complexSchema = z.object({
|
|
37
|
+
id: z.string().uuid(),
|
|
38
|
+
name: z.string().min(1).max(100),
|
|
39
|
+
age: z.number().min(0).max(150).optional(),
|
|
40
|
+
tags: z.array(z.string()),
|
|
41
|
+
status: z.enum(["active", "inactive"]),
|
|
42
|
+
metadata: z.object({
|
|
43
|
+
createdAt: z.string(),
|
|
44
|
+
nested: z.object({ value: z.number() }),
|
|
45
|
+
}),
|
|
46
|
+
});
|
|
47
|
+
const complexResult = zodSchemaToJsonSchema(complexSchema) as Record<string, unknown>;
|
|
48
|
+
expect(complexResult.type).toBe("object");
|
|
49
|
+
expect(complexResult.properties).toBeDefined();
|
|
50
|
+
expect(complexResult.required).toContain("id");
|
|
51
|
+
expect(complexResult.required).not.toContain("age");
|
|
52
|
+
|
|
53
|
+
const enumResult = zodSchemaToJsonSchema(z.enum(["a", "b", "c"])) as Record<string, unknown>;
|
|
54
|
+
expect(enumResult.enum).toEqual(["a", "b", "c"]);
|
|
55
|
+
|
|
56
|
+
const arrayResult = zodSchemaToJsonSchema(z.array(z.number())) as Record<string, unknown>;
|
|
57
|
+
expect(arrayResult.type).toBe("array");
|
|
58
|
+
expect(arrayResult.items).toBeDefined();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("zodSchemaToJsonSchema passes through non-Zod values unchanged", () => {
|
|
62
|
+
const jsonSchema = { type: "object", properties: { name: { type: "string" } } };
|
|
63
|
+
expect(zodSchemaToJsonSchema(jsonSchema)).toEqual(jsonSchema);
|
|
64
|
+
expect(zodSchemaToJsonSchema(null)).toBe(null);
|
|
65
|
+
expect(zodSchemaToJsonSchema(undefined)).toBe(undefined);
|
|
66
|
+
expect(zodSchemaToJsonSchema("string")).toBe("string");
|
|
67
|
+
expect(zodSchemaToJsonSchema(42)).toBe(42);
|
|
68
|
+
expect(zodSchemaToJsonSchema({ foo: "bar" })).toEqual({ foo: "bar" });
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("looksLikeZodShape detects .shape property misuse", () => {
|
|
72
|
+
const schema = z.object({ title: z.string(), count: z.number() });
|
|
73
|
+
expect(looksLikeZodShape(schema.shape)).toBe(true);
|
|
74
|
+
expect(looksLikeZodShape(schema)).toBe(false);
|
|
75
|
+
expect(looksLikeZodShape(null)).toBe(false);
|
|
76
|
+
expect(looksLikeZodShape(undefined)).toBe(false);
|
|
77
|
+
expect(looksLikeZodShape({ name: "test" })).toBe(false);
|
|
78
|
+
expect(looksLikeZodShape({})).toBe(false);
|
|
79
|
+
expect(looksLikeZodShape([1, 2, 3])).toBe(false);
|
|
80
|
+
expect(looksLikeZodShape({ type: "object", properties: {} })).toBe(false);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("SDK-like usage: convert Zod schema or pass through JSON schema", () => {
|
|
84
|
+
const zodSchema = z.object({
|
|
85
|
+
name: z.string(),
|
|
86
|
+
email: z.string().email(),
|
|
87
|
+
age: z.number().min(0),
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
if (isZodSchema(zodSchema)) {
|
|
91
|
+
const result = zodSchemaToJsonSchema(zodSchema) as Record<string, unknown>;
|
|
92
|
+
expect(result.type).toBe("object");
|
|
93
|
+
expect(result.properties).toBeDefined();
|
|
94
|
+
} else {
|
|
95
|
+
throw new Error("Should detect Zod schema");
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const existingJsonSchema = {
|
|
99
|
+
type: "object" as const,
|
|
100
|
+
properties: { title: { type: "string" as const } },
|
|
101
|
+
required: ["title"] as string[],
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
expect(isZodSchema(existingJsonSchema)).toBe(false);
|
|
105
|
+
expect(zodSchemaToJsonSchema(existingJsonSchema)).toEqual(existingJsonSchema);
|
|
106
|
+
});
|
|
107
|
+
});
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { zodToJsonSchema as zodToJsonSchemaLib } from "zod-to-json-schema";
|
|
2
|
+
|
|
3
|
+
type SchemaConverter = (schema: unknown) => unknown;
|
|
4
|
+
|
|
5
|
+
export function isZodSchema(value: unknown): boolean {
|
|
6
|
+
if (!value || typeof value !== "object") return false;
|
|
7
|
+
const schema = value as Record<string, unknown>;
|
|
8
|
+
|
|
9
|
+
const hasV3Markers =
|
|
10
|
+
"_def" in schema &&
|
|
11
|
+
(typeof schema.safeParse === "function" ||
|
|
12
|
+
typeof schema.parse === "function");
|
|
13
|
+
|
|
14
|
+
const hasV4Markers = "_zod" in schema && typeof schema._zod === "object";
|
|
15
|
+
|
|
16
|
+
return hasV3Markers || hasV4Markers;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function isZodV4Schema(schema: unknown): boolean {
|
|
20
|
+
if (!schema || typeof schema !== "object") return false;
|
|
21
|
+
return "_zod" in schema && typeof (schema as Record<string, unknown>)._zod === "object";
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function tryZodV4Conversion(schema: unknown): Record<string, unknown> | null {
|
|
25
|
+
if (!isZodV4Schema(schema)) return null;
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const zodModule = (schema as Record<string, unknown>).constructor?.prototype?.constructor;
|
|
29
|
+
if (zodModule && typeof (zodModule as Record<string, unknown>).toJSONSchema === "function") {
|
|
30
|
+
return (zodModule as { toJSONSchema: SchemaConverter }).toJSONSchema(schema) as Record<string, unknown>;
|
|
31
|
+
}
|
|
32
|
+
} catch {
|
|
33
|
+
// V4 conversion not available
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function zodSchemaToJsonSchema(schema: unknown): Record<string, unknown> | unknown {
|
|
40
|
+
if (!isZodSchema(schema)) {
|
|
41
|
+
return schema;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const v4Result = tryZodV4Conversion(schema);
|
|
45
|
+
if (v4Result) {
|
|
46
|
+
return v4Result;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
try {
|
|
50
|
+
return zodToJsonSchemaLib(schema as Parameters<typeof zodToJsonSchemaLib>[0]) as Record<string, unknown>;
|
|
51
|
+
} catch {
|
|
52
|
+
return schema;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function looksLikeZodShape(obj: unknown): boolean {
|
|
57
|
+
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return false;
|
|
58
|
+
const values = Object.values(obj);
|
|
59
|
+
if (values.length === 0) return false;
|
|
60
|
+
return values.some(
|
|
61
|
+
(v) =>
|
|
62
|
+
v &&
|
|
63
|
+
typeof v === "object" &&
|
|
64
|
+
(v as Record<string, unknown>)._def &&
|
|
65
|
+
typeof (v as Record<string, unknown>).safeParse === "function"
|
|
66
|
+
);
|
|
67
|
+
}
|
package/src/v1/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";
|
|
2
2
|
import * as zt from "zod";
|
|
3
|
-
import {
|
|
3
|
+
import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
|
|
4
4
|
import { TypedEventTarget } from "typescript-event-target";
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -707,36 +707,21 @@ export default class FirecrawlApp {
|
|
|
707
707
|
} as AxiosRequestHeaders;
|
|
708
708
|
let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
709
709
|
if (jsonData?.extract?.schema) {
|
|
710
|
-
let schema = jsonData.extract.schema;
|
|
711
|
-
|
|
712
|
-
// Try parsing the schema as a Zod schema
|
|
713
|
-
try {
|
|
714
|
-
schema = zodToJsonSchema(schema);
|
|
715
|
-
} catch (error) {
|
|
716
|
-
|
|
717
|
-
}
|
|
718
710
|
jsonData = {
|
|
719
711
|
...jsonData,
|
|
720
712
|
extract: {
|
|
721
713
|
...jsonData.extract,
|
|
722
|
-
schema: schema,
|
|
714
|
+
schema: zodSchemaToJsonSchema(jsonData.extract.schema),
|
|
723
715
|
},
|
|
724
716
|
};
|
|
725
717
|
}
|
|
726
718
|
|
|
727
719
|
if (jsonData?.jsonOptions?.schema) {
|
|
728
|
-
let schema = jsonData.jsonOptions.schema;
|
|
729
|
-
// Try parsing the schema as a Zod schema
|
|
730
|
-
try {
|
|
731
|
-
schema = zodToJsonSchema(schema);
|
|
732
|
-
} catch (error) {
|
|
733
|
-
|
|
734
|
-
}
|
|
735
720
|
jsonData = {
|
|
736
721
|
...jsonData,
|
|
737
722
|
jsonOptions: {
|
|
738
723
|
...jsonData.jsonOptions,
|
|
739
|
-
schema: schema,
|
|
724
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema),
|
|
740
725
|
},
|
|
741
726
|
};
|
|
742
727
|
}
|
|
@@ -793,21 +778,13 @@ export default class FirecrawlApp {
|
|
|
793
778
|
};
|
|
794
779
|
|
|
795
780
|
if (jsonData?.scrapeOptions?.extract?.schema) {
|
|
796
|
-
let schema = jsonData.scrapeOptions.extract.schema;
|
|
797
|
-
|
|
798
|
-
// Try parsing the schema as a Zod schema
|
|
799
|
-
try {
|
|
800
|
-
schema = zodToJsonSchema(schema);
|
|
801
|
-
} catch (error) {
|
|
802
|
-
|
|
803
|
-
}
|
|
804
781
|
jsonData = {
|
|
805
782
|
...jsonData,
|
|
806
783
|
scrapeOptions: {
|
|
807
784
|
...jsonData.scrapeOptions,
|
|
808
785
|
extract: {
|
|
809
786
|
...jsonData.scrapeOptions.extract,
|
|
810
|
-
schema: schema,
|
|
787
|
+
schema: zodSchemaToJsonSchema(jsonData.scrapeOptions.extract.schema),
|
|
811
788
|
},
|
|
812
789
|
},
|
|
813
790
|
};
|
|
@@ -1105,36 +1082,20 @@ export default class FirecrawlApp {
|
|
|
1105
1082
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1106
1083
|
let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1107
1084
|
if (jsonData?.extract?.schema) {
|
|
1108
|
-
let schema = jsonData.extract.schema;
|
|
1109
|
-
|
|
1110
|
-
// Try parsing the schema as a Zod schema
|
|
1111
|
-
try {
|
|
1112
|
-
schema = zodToJsonSchema(schema);
|
|
1113
|
-
} catch (error) {
|
|
1114
|
-
|
|
1115
|
-
}
|
|
1116
1085
|
jsonData = {
|
|
1117
1086
|
...jsonData,
|
|
1118
1087
|
extract: {
|
|
1119
1088
|
...jsonData.extract,
|
|
1120
|
-
schema: schema,
|
|
1089
|
+
schema: zodSchemaToJsonSchema(jsonData.extract.schema),
|
|
1121
1090
|
},
|
|
1122
1091
|
};
|
|
1123
1092
|
}
|
|
1124
1093
|
if (jsonData?.jsonOptions?.schema) {
|
|
1125
|
-
let schema = jsonData.jsonOptions.schema;
|
|
1126
|
-
|
|
1127
|
-
// Try parsing the schema as a Zod schema
|
|
1128
|
-
try {
|
|
1129
|
-
schema = zodToJsonSchema(schema);
|
|
1130
|
-
} catch (error) {
|
|
1131
|
-
|
|
1132
|
-
}
|
|
1133
1094
|
jsonData = {
|
|
1134
1095
|
...jsonData,
|
|
1135
1096
|
jsonOptions: {
|
|
1136
1097
|
...jsonData.jsonOptions,
|
|
1137
|
-
schema: schema,
|
|
1098
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema),
|
|
1138
1099
|
},
|
|
1139
1100
|
};
|
|
1140
1101
|
}
|
|
@@ -1326,20 +1287,7 @@ export default class FirecrawlApp {
|
|
|
1326
1287
|
const headers = this.prepareHeaders();
|
|
1327
1288
|
|
|
1328
1289
|
let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };
|
|
1329
|
-
|
|
1330
|
-
try {
|
|
1331
|
-
if (!params?.schema) {
|
|
1332
|
-
jsonSchema = undefined;
|
|
1333
|
-
} else {
|
|
1334
|
-
try {
|
|
1335
|
-
jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
|
|
1336
|
-
} catch (_) {
|
|
1337
|
-
jsonSchema = params.schema;
|
|
1338
|
-
}
|
|
1339
|
-
}
|
|
1340
|
-
} catch (error: any) {
|
|
1341
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
1342
|
-
}
|
|
1290
|
+
const jsonSchema = params?.schema ? zodSchemaToJsonSchema(params.schema) : undefined;
|
|
1343
1291
|
|
|
1344
1292
|
try {
|
|
1345
1293
|
const response: AxiosResponse = await this.postRequest(
|
|
@@ -1396,22 +1344,8 @@ export default class FirecrawlApp {
|
|
|
1396
1344
|
idempotencyKey?: string
|
|
1397
1345
|
): Promise<ExtractResponse | ErrorResponse> {
|
|
1398
1346
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
try {
|
|
1403
|
-
if (!params?.schema) {
|
|
1404
|
-
jsonSchema = undefined;
|
|
1405
|
-
} else {
|
|
1406
|
-
try {
|
|
1407
|
-
jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
|
|
1408
|
-
} catch (_) {
|
|
1409
|
-
jsonSchema = params.schema;
|
|
1410
|
-
}
|
|
1411
|
-
}
|
|
1412
|
-
} catch (error: any) {
|
|
1413
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
1414
|
-
}
|
|
1347
|
+
const jsonData: any = { urls, ...params };
|
|
1348
|
+
const jsonSchema = params?.schema ? zodSchemaToJsonSchema(params.schema) : undefined;
|
|
1415
1349
|
|
|
1416
1350
|
try {
|
|
1417
1351
|
const response: AxiosResponse = await this.postRequest(
|
|
@@ -1780,18 +1714,11 @@ export default class FirecrawlApp {
|
|
|
1780
1714
|
let jsonData: any = { query, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
|
|
1781
1715
|
|
|
1782
1716
|
if (jsonData?.jsonOptions?.schema) {
|
|
1783
|
-
let schema = jsonData.jsonOptions.schema;
|
|
1784
|
-
// Try parsing the schema as a Zod schema
|
|
1785
|
-
try {
|
|
1786
|
-
schema = zodToJsonSchema(schema);
|
|
1787
|
-
} catch (error) {
|
|
1788
|
-
// Ignore error if schema can't be parsed as Zod
|
|
1789
|
-
}
|
|
1790
1717
|
jsonData = {
|
|
1791
1718
|
...jsonData,
|
|
1792
1719
|
jsonOptions: {
|
|
1793
1720
|
...jsonData.jsonOptions,
|
|
1794
|
-
schema: schema,
|
|
1721
|
+
schema: zodSchemaToJsonSchema(jsonData.jsonOptions.schema),
|
|
1795
1722
|
},
|
|
1796
1723
|
};
|
|
1797
1724
|
}
|
package/src/v2/methods/agent.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { type AgentResponse, type AgentStatusResponse } from "../types";
|
|
1
|
+
import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig } from "../types";
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
|
4
|
-
import {
|
|
4
|
+
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
|
5
5
|
import type { ZodTypeAny } from "zod";
|
|
6
6
|
|
|
7
7
|
function prepareAgentPayload(args: {
|
|
@@ -12,19 +12,19 @@ function prepareAgentPayload(args: {
|
|
|
12
12
|
maxCredits?: number;
|
|
13
13
|
strictConstrainToURLs?: boolean;
|
|
14
14
|
model?: "spark-1-pro" | "spark-1-mini";
|
|
15
|
+
webhook?: string | AgentWebhookConfig;
|
|
15
16
|
}): Record<string, unknown> {
|
|
16
17
|
const body: Record<string, unknown> = {};
|
|
17
18
|
if (args.urls) body.urls = args.urls;
|
|
18
19
|
body.prompt = args.prompt;
|
|
19
20
|
if (args.schema != null) {
|
|
20
|
-
|
|
21
|
-
const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
|
|
22
|
-
body.schema = isZod ? zodToJsonSchema(s) : args.schema;
|
|
21
|
+
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
|
23
22
|
}
|
|
24
23
|
if (args.integration && args.integration.trim()) body.integration = args.integration.trim();
|
|
25
24
|
if (args.maxCredits !== null && args.maxCredits !== undefined) body.maxCredits = args.maxCredits;
|
|
26
25
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
27
26
|
if (args.model !== null && args.model !== undefined) body.model = args.model;
|
|
27
|
+
if (args.webhook != null) body.webhook = args.webhook;
|
|
28
28
|
return body;
|
|
29
29
|
}
|
|
30
30
|
|
|
@@ -2,7 +2,7 @@ import { type ExtractResponse, type ScrapeOptions, type AgentOptions } from "../
|
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
4
4
|
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
|
5
|
-
import {
|
|
5
|
+
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
|
6
6
|
import type { ZodTypeAny } from "zod";
|
|
7
7
|
|
|
8
8
|
function prepareExtractPayload(args: {
|
|
@@ -22,9 +22,7 @@ function prepareExtractPayload(args: {
|
|
|
22
22
|
if (args.urls) body.urls = args.urls;
|
|
23
23
|
if (args.prompt != null) body.prompt = args.prompt;
|
|
24
24
|
if (args.schema != null) {
|
|
25
|
-
|
|
26
|
-
const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
|
|
27
|
-
body.schema = isZod ? zodToJsonSchema(s) : args.schema;
|
|
25
|
+
body.schema = isZodSchema(args.schema) ? zodSchemaToJsonSchema(args.schema) : args.schema;
|
|
28
26
|
}
|
|
29
27
|
if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
|
|
30
28
|
if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
|
package/src/v2/types.ts
CHANGED
|
@@ -167,6 +167,16 @@ export interface WebhookConfig {
|
|
|
167
167
|
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
// Agent webhook events differ from crawl: has 'action' and 'cancelled', no 'page'
|
|
171
|
+
export type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';
|
|
172
|
+
|
|
173
|
+
export interface AgentWebhookConfig {
|
|
174
|
+
url: string;
|
|
175
|
+
headers?: Record<string, string>;
|
|
176
|
+
metadata?: Record<string, string>;
|
|
177
|
+
events?: AgentWebhookEvent[];
|
|
178
|
+
}
|
|
179
|
+
|
|
170
180
|
export interface BrandingProfile {
|
|
171
181
|
colorScheme?: 'light' | 'dark';
|
|
172
182
|
logo?: string | null;
|
|
@@ -1,18 +1,5 @@
|
|
|
1
1
|
import { type FormatOption, type JsonFormat, type ScrapeOptions, type ScreenshotFormat, type ChangeTrackingFormat } from "../types";
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Detects if an object looks like a Zod schema's `.shape` property.
|
|
6
|
-
* When users mistakenly pass `schema.shape` instead of `schema`, the object
|
|
7
|
-
* will have Zod types as values but won't be a Zod schema itself.
|
|
8
|
-
*/
|
|
9
|
-
function looksLikeZodShape(obj: unknown): boolean {
|
|
10
|
-
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return false;
|
|
11
|
-
const values = Object.values(obj);
|
|
12
|
-
if (values.length === 0) return false;
|
|
13
|
-
// Check if at least one value looks like a Zod type
|
|
14
|
-
return values.some((v: any) => v && typeof v === "object" && v._def && typeof v.safeParse === "function");
|
|
15
|
-
}
|
|
2
|
+
import { isZodSchema, zodSchemaToJsonSchema, looksLikeZodShape } from "../../utils/zodSchemaToJson";
|
|
16
3
|
|
|
17
4
|
export function ensureValidFormats(formats?: FormatOption[]): void {
|
|
18
5
|
if (!formats) return;
|
|
@@ -28,17 +15,10 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
|
|
|
28
15
|
if (!j.prompt && !j.schema) {
|
|
29
16
|
throw new Error("json format requires either 'prompt' or 'schema' (or both)");
|
|
30
17
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if (isZod) {
|
|
35
|
-
try {
|
|
36
|
-
(j as any).schema = zodToJsonSchema(maybeSchema);
|
|
37
|
-
} catch {
|
|
38
|
-
// If conversion fails, leave as-is; server-side may still handle, or request will fail explicitly
|
|
39
|
-
}
|
|
18
|
+
const maybeSchema = j.schema;
|
|
19
|
+
if (isZodSchema(maybeSchema)) {
|
|
20
|
+
(j as any).schema = zodSchemaToJsonSchema(maybeSchema);
|
|
40
21
|
} else if (looksLikeZodShape(maybeSchema)) {
|
|
41
|
-
// User likely passed schema.shape instead of the schema itself
|
|
42
22
|
throw new Error(
|
|
43
23
|
"json format schema appears to be a Zod schema's .shape property. " +
|
|
44
24
|
"Pass the Zod schema directly (e.g., `schema: MySchema`) instead of `schema: MySchema.shape`. " +
|
|
@@ -49,14 +29,9 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
|
|
|
49
29
|
}
|
|
50
30
|
if ((fmt as ChangeTrackingFormat).type === "changeTracking") {
|
|
51
31
|
const ct = fmt as ChangeTrackingFormat;
|
|
52
|
-
const maybeSchema
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
try {
|
|
56
|
-
(ct as any).schema = zodToJsonSchema(maybeSchema);
|
|
57
|
-
} catch {
|
|
58
|
-
// Best-effort conversion; if it fails, leave original value
|
|
59
|
-
}
|
|
32
|
+
const maybeSchema = ct.schema;
|
|
33
|
+
if (isZodSchema(maybeSchema)) {
|
|
34
|
+
(ct as any).schema = zodSchemaToJsonSchema(maybeSchema);
|
|
60
35
|
} else if (looksLikeZodShape(maybeSchema)) {
|
|
61
36
|
throw new Error(
|
|
62
37
|
"changeTracking format schema appears to be a Zod schema's .shape property. " +
|