firecrawl-cli 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/commands/crawl.test.d.ts +5 -0
- package/dist/__tests__/commands/crawl.test.d.ts.map +1 -0
- package/dist/__tests__/commands/crawl.test.js +415 -0
- package/dist/__tests__/commands/crawl.test.js.map +1 -0
- package/dist/__tests__/commands/credit-usage.test.d.ts +5 -0
- package/dist/__tests__/commands/credit-usage.test.d.ts.map +1 -0
- package/dist/__tests__/commands/credit-usage.test.js +200 -0
- package/dist/__tests__/commands/credit-usage.test.js.map +1 -0
- package/dist/__tests__/commands/map.test.d.ts +5 -0
- package/dist/__tests__/commands/map.test.d.ts.map +1 -0
- package/dist/__tests__/commands/map.test.js +287 -0
- package/dist/__tests__/commands/map.test.js.map +1 -0
- package/dist/__tests__/commands/scrape.test.d.ts +5 -0
- package/dist/__tests__/commands/scrape.test.d.ts.map +1 -0
- package/dist/__tests__/commands/scrape.test.js +231 -0
- package/dist/__tests__/commands/scrape.test.js.map +1 -0
- package/dist/__tests__/utils/job.test.d.ts +5 -0
- package/dist/__tests__/utils/job.test.d.ts.map +1 -0
- package/dist/__tests__/utils/job.test.js +53 -0
- package/dist/__tests__/utils/job.test.js.map +1 -0
- package/dist/__tests__/utils/mock-client.d.ts +23 -0
- package/dist/__tests__/utils/mock-client.d.ts.map +1 -0
- package/dist/__tests__/utils/mock-client.js +24 -0
- package/dist/__tests__/utils/mock-client.js.map +1 -0
- package/dist/commands/config.d.ts.map +1 -1
- package/dist/commands/config.js +5 -3
- package/dist/commands/config.js.map +1 -1
- package/dist/commands/crawl.d.ts +13 -0
- package/dist/commands/crawl.d.ts.map +1 -0
- package/dist/commands/crawl.js +242 -0
- package/dist/commands/crawl.js.map +1 -0
- package/dist/commands/credit-usage.d.ts +32 -0
- package/dist/commands/credit-usage.d.ts.map +1 -0
- package/dist/commands/credit-usage.js +175 -0
- package/dist/commands/credit-usage.js.map +1 -0
- package/dist/commands/map.d.ts +13 -0
- package/dist/commands/map.d.ts.map +1 -0
- package/dist/commands/map.js +96 -0
- package/dist/commands/map.js.map +1 -0
- package/dist/commands/scrape.d.ts.map +1 -1
- package/dist/commands/scrape.js.map +1 -1
- package/dist/index.js +134 -1
- package/dist/index.js.map +1 -1
- package/dist/types/crawl.d.ts +63 -0
- package/dist/types/crawl.d.ts.map +1 -0
- package/dist/types/crawl.js +6 -0
- package/dist/types/crawl.js.map +1 -0
- package/dist/types/map.d.ts +43 -0
- package/dist/types/map.d.ts.map +1 -0
- package/dist/types/map.js +6 -0
- package/dist/types/map.js.map +1 -0
- package/dist/utils/client.d.ts.map +1 -1
- package/dist/utils/client.js.map +1 -1
- package/dist/utils/config.d.ts.map +1 -1
- package/dist/utils/config.js +7 -3
- package/dist/utils/config.js.map +1 -1
- package/dist/utils/credentials.d.ts.map +1 -1
- package/dist/utils/credentials.js.map +1 -1
- package/dist/utils/job.d.ts +13 -0
- package/dist/utils/job.d.ts.map +1 -0
- package/dist/utils/job.js +29 -0
- package/dist/utils/job.js.map +1 -0
- package/dist/utils/options.d.ts.map +1 -1
- package/dist/utils/options.js +6 -2
- package/dist/utils/options.js.map +1 -1
- package/dist/utils/output.d.ts.map +1 -1
- package/dist/utils/output.js +9 -2
- package/dist/utils/output.js.map +1 -1
- package/dist/utils/url.d.ts.map +1 -1
- package/dist/utils/url.js.map +1 -1
- package/package.json +19 -3
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Map command implementation
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.executeMap = executeMap;
|
|
7
|
+
exports.handleMapCommand = handleMapCommand;
|
|
8
|
+
const client_1 = require("../utils/client");
|
|
9
|
+
const config_1 = require("../utils/config");
|
|
10
|
+
const output_1 = require("../utils/output");
|
|
11
|
+
/**
|
|
12
|
+
* Execute map command
|
|
13
|
+
*/
|
|
14
|
+
async function executeMap(options) {
|
|
15
|
+
try {
|
|
16
|
+
// Update global config if API key is provided
|
|
17
|
+
if (options.apiKey) {
|
|
18
|
+
(0, config_1.updateConfig)({ apiKey: options.apiKey });
|
|
19
|
+
}
|
|
20
|
+
const app = (0, client_1.getClient)();
|
|
21
|
+
const { urlOrJobId } = options;
|
|
22
|
+
// Build map options
|
|
23
|
+
const mapOptions = {};
|
|
24
|
+
if (options.limit !== undefined) {
|
|
25
|
+
mapOptions.limit = options.limit;
|
|
26
|
+
}
|
|
27
|
+
if (options.search) {
|
|
28
|
+
mapOptions.search = options.search;
|
|
29
|
+
}
|
|
30
|
+
if (options.sitemap) {
|
|
31
|
+
mapOptions.sitemap = options.sitemap;
|
|
32
|
+
}
|
|
33
|
+
if (options.includeSubdomains !== undefined) {
|
|
34
|
+
mapOptions.includeSubdomains = options.includeSubdomains;
|
|
35
|
+
}
|
|
36
|
+
if (options.ignoreQueryParameters !== undefined) {
|
|
37
|
+
mapOptions.ignoreQueryParameters = options.ignoreQueryParameters;
|
|
38
|
+
}
|
|
39
|
+
if (options.timeout !== undefined) {
|
|
40
|
+
mapOptions.timeout = options.timeout * 1000; // Convert to milliseconds
|
|
41
|
+
}
|
|
42
|
+
// Execute map (seems synchronous in SDK)
|
|
43
|
+
const mapData = await app.map(urlOrJobId, mapOptions);
|
|
44
|
+
return {
|
|
45
|
+
success: true,
|
|
46
|
+
data: {
|
|
47
|
+
links: mapData.links.map((link) => ({
|
|
48
|
+
url: link.url,
|
|
49
|
+
title: link.title,
|
|
50
|
+
description: link.description,
|
|
51
|
+
})),
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
return {
|
|
57
|
+
success: false,
|
|
58
|
+
error: error instanceof Error ? error.message : 'Unknown error occurred',
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Format map data in human-readable way
|
|
64
|
+
*/
|
|
65
|
+
function formatMapReadable(data) {
|
|
66
|
+
if (!data || !data.links)
|
|
67
|
+
return '';
|
|
68
|
+
// Output one URL per line (like curl)
|
|
69
|
+
return data.links.map((link) => link.url).join('\n') + '\n';
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Handle map command output
|
|
73
|
+
*/
|
|
74
|
+
async function handleMapCommand(options) {
|
|
75
|
+
const result = await executeMap(options);
|
|
76
|
+
if (!result.success) {
|
|
77
|
+
console.error('Error:', result.error);
|
|
78
|
+
process.exit(1);
|
|
79
|
+
}
|
|
80
|
+
if (!result.data) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
let outputContent;
|
|
84
|
+
// Use JSON format if --json flag is set
|
|
85
|
+
if (options.json) {
|
|
86
|
+
outputContent = options.pretty
|
|
87
|
+
? JSON.stringify({ success: true, data: result.data }, null, 2)
|
|
88
|
+
: JSON.stringify({ success: true, data: result.data });
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
// Default to human-readable format (one URL per line)
|
|
92
|
+
outputContent = formatMapReadable(result.data);
|
|
93
|
+
}
|
|
94
|
+
(0, output_1.writeOutput)(outputContent, options.output, !!options.output);
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=map.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/commands/map.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAUH,gCAmDC;AAeD,4CAyBC;AAlGD,4CAA4C;AAC5C,4CAA+C;AAC/C,4CAA8C;AAE9C;;GAEG;AACI,KAAK,UAAU,UAAU,CAAC,OAAmB;IAClD,IAAI,CAAC;QACH,8CAA8C;QAC9C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,GAAG,GAAG,IAAA,kBAAS,GAAE,CAAC;QACxB,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;QAE/B,oBAAoB;QACpB,MAAM,UAAU,GAAQ,EAAE,CAAC;QAE3B,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,UAAU,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QACnC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,UAAU,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QACrC,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAC5C,UAAU,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QAC3D,CAAC;QACD,IAAI,OAAO,CAAC,qBAAqB,KAAK,SAAS,EAAE,CAAC;YAChD,UAAU,CAAC,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;QACnE,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,0BAA0B;QACzE,CAAC;QAED,yCAAyC;QACzC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAEtD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE;gBACJ,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;oBACvC,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;iBAC9B,CAAC,CAAC;aACJ;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAuB;IAChD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IAEpC,sCAAsC;IACtC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAC9D,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,gBAAgB,CAAC,OAAmB;IACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;IAEzC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,IAAI,aAAqB,CAAC;IAE1B,wCAAwC;IACxC,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,aAAa,GAAG,OAAO,CAAC,MAAM;YAC5B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;SAAM,CAAC;QACN,sDAAsD;QACtD,aAAa,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,IAAA,oBAAW,EAAC,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAC/D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAKnE;;GAEG;AACH,wBAAsB,aAAa,
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAKnE;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAoEvB;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,IAAI,CAAC,CAGf"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAWH,
|
|
1
|
+
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAWH,sCAsEC;AAKD,kDAKC;AAvFD,4CAA4C;AAC5C,4CAA+C;AAC/C,4CAAqD;AAErD;;GAEG;AACI,KAAK,UAAU,aAAa,CACjC,OAAsB;IAEtB,IAAI,CAAC;QACH,0DAA0D;QAC1D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,2CAA2C;QAC3C,MAAM,GAAG,GAAG,IAAA,kBAAS,GAAE,CAAC;QAExB,uBAAuB;QACvB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,gDAAgD;YAChD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,YAAY,GAMd;YACF,OAAO;SACR,CAAC;QAEF,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC1C,YAAY,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QACzD,CAAC;QAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACjD,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACjD,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAE3D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,MAAM;SACb,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CACvC,OAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAA,2BAAkB,EAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC7E,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -9,8 +9,12 @@ const commander_1 = require("commander");
|
|
|
9
9
|
const scrape_1 = require("./commands/scrape");
|
|
10
10
|
const config_1 = require("./utils/config");
|
|
11
11
|
const config_2 = require("./commands/config");
|
|
12
|
+
const credit_usage_1 = require("./commands/credit-usage");
|
|
13
|
+
const crawl_1 = require("./commands/crawl");
|
|
14
|
+
const map_1 = require("./commands/map");
|
|
12
15
|
const url_1 = require("./utils/url");
|
|
13
16
|
const options_1 = require("./utils/options");
|
|
17
|
+
const job_1 = require("./utils/job");
|
|
14
18
|
// Initialize global configuration from environment variables
|
|
15
19
|
(0, config_1.initializeConfig)();
|
|
16
20
|
const program = new commander_1.Command();
|
|
@@ -61,12 +65,135 @@ function createScrapeCommand() {
|
|
|
61
65
|
}
|
|
62
66
|
// Add scrape command to main program
|
|
63
67
|
program.addCommand(createScrapeCommand());
|
|
68
|
+
/**
|
|
69
|
+
* Create and configure the crawl command
|
|
70
|
+
*/
|
|
71
|
+
function createCrawlCommand() {
|
|
72
|
+
const crawlCmd = new commander_1.Command('crawl')
|
|
73
|
+
.description('Crawl a website using Firecrawl')
|
|
74
|
+
.argument('[url-or-job-id]', 'URL to crawl or job ID to check status')
|
|
75
|
+
.option('-u, --url <url>', 'URL to crawl (alternative to positional argument)')
|
|
76
|
+
.option('--status', 'Check status of existing crawl job', false)
|
|
77
|
+
.option('--wait', 'Wait for crawl to complete before returning results', false)
|
|
78
|
+
.option('--poll-interval <seconds>', 'Polling interval in seconds when waiting (default: 5)', parseFloat)
|
|
79
|
+
.option('--timeout <seconds>', 'Timeout in seconds when waiting (default: no timeout)', parseFloat)
|
|
80
|
+
.option('--progress', 'Show progress dots while waiting', false)
|
|
81
|
+
.option('--limit <number>', 'Maximum number of pages to crawl', parseInt)
|
|
82
|
+
.option('--max-depth <number>', 'Maximum crawl depth', parseInt)
|
|
83
|
+
.option('--exclude-paths <paths>', 'Comma-separated list of paths to exclude')
|
|
84
|
+
.option('--include-paths <paths>', 'Comma-separated list of paths to include')
|
|
85
|
+
.option('--sitemap <mode>', 'Sitemap handling: skip, include', 'include')
|
|
86
|
+
.option('--ignore-query-parameters', 'Ignore query parameters when crawling', false)
|
|
87
|
+
.option('--crawl-entire-domain', 'Crawl entire domain', false)
|
|
88
|
+
.option('--allow-external-links', 'Allow external links', false)
|
|
89
|
+
.option('--allow-subdomains', 'Allow subdomains', false)
|
|
90
|
+
.option('--delay <ms>', 'Delay between requests in milliseconds', parseInt)
|
|
91
|
+
.option('--max-concurrency <number>', 'Maximum concurrent requests', parseInt)
|
|
92
|
+
.option('-k, --api-key <key>', 'Firecrawl API key (overrides global --api-key)')
|
|
93
|
+
.option('-o, --output <path>', 'Output file path (default: stdout)')
|
|
94
|
+
.option('--pretty', 'Pretty print JSON output', false)
|
|
95
|
+
.action(async (positionalUrlOrJobId, options) => {
|
|
96
|
+
// Use positional argument if provided, otherwise use --url option
|
|
97
|
+
const urlOrJobId = positionalUrlOrJobId || options.url;
|
|
98
|
+
if (!urlOrJobId) {
|
|
99
|
+
console.error('Error: URL or job ID is required. Provide it as argument or use --url option.');
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
// Auto-detect if it's a job ID (UUID format)
|
|
103
|
+
const isStatusCheck = options.status || (0, job_1.isJobId)(urlOrJobId);
|
|
104
|
+
const crawlOptions = {
|
|
105
|
+
urlOrJobId,
|
|
106
|
+
status: isStatusCheck,
|
|
107
|
+
wait: options.wait,
|
|
108
|
+
pollInterval: options.pollInterval,
|
|
109
|
+
timeout: options.timeout,
|
|
110
|
+
progress: options.progress,
|
|
111
|
+
output: options.output,
|
|
112
|
+
pretty: options.pretty,
|
|
113
|
+
apiKey: options.apiKey,
|
|
114
|
+
limit: options.limit,
|
|
115
|
+
maxDepth: options.maxDepth,
|
|
116
|
+
excludePaths: options.excludePaths
|
|
117
|
+
? options.excludePaths.split(',').map((p) => p.trim())
|
|
118
|
+
: undefined,
|
|
119
|
+
includePaths: options.includePaths
|
|
120
|
+
? options.includePaths.split(',').map((p) => p.trim())
|
|
121
|
+
: undefined,
|
|
122
|
+
sitemap: options.sitemap,
|
|
123
|
+
ignoreQueryParameters: options.ignoreQueryParameters,
|
|
124
|
+
crawlEntireDomain: options.crawlEntireDomain,
|
|
125
|
+
allowExternalLinks: options.allowExternalLinks,
|
|
126
|
+
allowSubdomains: options.allowSubdomains,
|
|
127
|
+
delay: options.delay,
|
|
128
|
+
maxConcurrency: options.maxConcurrency,
|
|
129
|
+
};
|
|
130
|
+
await (0, crawl_1.handleCrawlCommand)(crawlOptions);
|
|
131
|
+
});
|
|
132
|
+
return crawlCmd;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Create and configure the map command
|
|
136
|
+
*/
|
|
137
|
+
function createMapCommand() {
|
|
138
|
+
const mapCmd = new commander_1.Command('map')
|
|
139
|
+
.description('Map URLs on a website using Firecrawl')
|
|
140
|
+
.argument('[url]', 'URL to map')
|
|
141
|
+
.option('-u, --url <url>', 'URL to map (alternative to positional argument)')
|
|
142
|
+
.option('--wait', 'Wait for map to complete', false)
|
|
143
|
+
.option('--limit <number>', 'Maximum URLs to discover', parseInt)
|
|
144
|
+
.option('--search <query>', 'Search query to filter URLs')
|
|
145
|
+
.option('--sitemap <mode>', 'Sitemap handling: only, include, skip', 'include')
|
|
146
|
+
.option('--include-subdomains', 'Include subdomains', false)
|
|
147
|
+
.option('--ignore-query-parameters', 'Ignore query parameters', false)
|
|
148
|
+
.option('--timeout <seconds>', 'Timeout in seconds', parseFloat)
|
|
149
|
+
.option('-k, --api-key <key>', 'Firecrawl API key (overrides global --api-key)')
|
|
150
|
+
.option('-o, --output <path>', 'Output file path (default: stdout)')
|
|
151
|
+
.option('--json', 'Output as JSON format', false)
|
|
152
|
+
.option('--pretty', 'Pretty print JSON output', false)
|
|
153
|
+
.action(async (positionalUrl, options) => {
|
|
154
|
+
// Use positional URL if provided, otherwise use --url option
|
|
155
|
+
const url = positionalUrl || options.url;
|
|
156
|
+
if (!url) {
|
|
157
|
+
console.error('Error: URL is required. Provide it as argument or use --url option.');
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
160
|
+
const mapOptions = {
|
|
161
|
+
urlOrJobId: url,
|
|
162
|
+
wait: options.wait,
|
|
163
|
+
output: options.output,
|
|
164
|
+
json: options.json,
|
|
165
|
+
pretty: options.pretty,
|
|
166
|
+
apiKey: options.apiKey,
|
|
167
|
+
limit: options.limit,
|
|
168
|
+
search: options.search,
|
|
169
|
+
sitemap: options.sitemap,
|
|
170
|
+
includeSubdomains: options.includeSubdomains,
|
|
171
|
+
ignoreQueryParameters: options.ignoreQueryParameters,
|
|
172
|
+
timeout: options.timeout,
|
|
173
|
+
};
|
|
174
|
+
await (0, map_1.handleMapCommand)(mapOptions);
|
|
175
|
+
});
|
|
176
|
+
return mapCmd;
|
|
177
|
+
}
|
|
178
|
+
// Add crawl and map commands to main program
|
|
179
|
+
program.addCommand(createCrawlCommand());
|
|
180
|
+
program.addCommand(createMapCommand());
|
|
64
181
|
program
|
|
65
182
|
.command('config')
|
|
66
183
|
.description('Configure API URL and API key (interactive)')
|
|
67
184
|
.action(async () => {
|
|
68
185
|
await (0, config_2.configure)();
|
|
69
186
|
});
|
|
187
|
+
program
|
|
188
|
+
.command('credit-usage')
|
|
189
|
+
.description('Get team credit usage information')
|
|
190
|
+
.option('-k, --api-key <key>', 'Firecrawl API key (overrides global --api-key)')
|
|
191
|
+
.option('-o, --output <path>', 'Output file path (default: stdout)')
|
|
192
|
+
.option('--json', 'Output as JSON format', false)
|
|
193
|
+
.option('--pretty', 'Pretty print JSON output (only applies with --json)', false)
|
|
194
|
+
.action(async (options) => {
|
|
195
|
+
await (0, credit_usage_1.handleCreditUsageCommand)(options);
|
|
196
|
+
});
|
|
70
197
|
// Parse arguments
|
|
71
198
|
const args = process.argv.slice(2);
|
|
72
199
|
// Check if first argument is a URL (and not a command)
|
|
@@ -75,7 +202,13 @@ if (args.length > 0 && !args[0].startsWith('-') && (0, url_1.isUrl)(args[0])) {
|
|
|
75
202
|
const url = (0, url_1.normalizeUrl)(args[0]);
|
|
76
203
|
// Modify argv to include scrape command with URL as positional argument
|
|
77
204
|
// This allows commander to parse it normally with all hooks and options
|
|
78
|
-
const modifiedArgv = [
|
|
205
|
+
const modifiedArgv = [
|
|
206
|
+
process.argv[0],
|
|
207
|
+
process.argv[1],
|
|
208
|
+
'scrape',
|
|
209
|
+
url,
|
|
210
|
+
...args.slice(1),
|
|
211
|
+
];
|
|
79
212
|
// Parse using the main program (which includes hooks and global options)
|
|
80
213
|
program.parseAsync(modifiedArgv).catch((error) => {
|
|
81
214
|
console.error('Error:', error instanceof Error ? error.message : 'Unknown error');
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA;;;GAGG;;AAEH,yCAAoC;AACpC,8CAAwD;AACxD,2CAAgE;AAChE,8CAA8C;AAC9C,qCAAkD;AAClD,6CAAqD;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA;;;GAGG;;AAEH,yCAAoC;AACpC,8CAAwD;AACxD,2CAAgE;AAChE,8CAA8C;AAC9C,0DAAmE;AACnE,4CAAsD;AACtD,wCAAkD;AAClD,qCAAkD;AAClD,6CAAqD;AACrD,qCAAsC;AAEtC,6DAA6D;AAC7D,IAAA,yBAAgB,GAAE,CAAC;AAEnB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,qCAAqC,CAAC;KAClD,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,CACL,qBAAqB,EACrB,iFAAiF,CAClF;KACA,kBAAkB,EAAE,CAAC,oDAAoD;KACzE,IAAI,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE;IAChD,gEAAgE;IAChE,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,SAAS,mBAAmB;IAC1B,MAAM,SAAS,GAAG,IAAI,mBAAO,CAAC,QAAQ,CAAC;SACpC,WAAW,CAAC,8BAA8B,CAAC;SAC3C,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;SAClC,MAAM,CACL,iBAAiB,EACjB,oDAAoD,CACrD;SACA,MAAM,CAAC,YAAY,EAAE,8CAA8C,CAAC;SACpE,MAAM,CACL,uBAAuB,EACvB,wHAAwH,EACxH,UAAU,CACX;SACA,MAAM,CAAC,qBAAqB,EAAE,2BAA2B,EAAE,KAAK,CAAC;SACjE,MAAM,CACL,iBAAiB,EACjB,2CAA2C,EAC3C,QAAQ,CACT;SACA,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,KAAK,CAAC;SAClD,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;SACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE;QACvC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;QACzC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,8BAA8B;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtD,MAAM,aAAa,GAAG,IAAA,4BAAkB,EAAC,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QACtE,MAAM,IAAA,4BAAmB,EAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEL,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qCAAqC;AACrC,OAAO,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAE1C;;GAEG;AACH,SAAS,kBAAkB;IACzB,MAAM,QAAQ,GAAG,IAAI,mBAAO,CAAC,OAAO,CAAC;SAClC,WAAW,CAAC,iCAAiC,CAAC;SAC9C,QAAQ,CAAC,iBAAiB,EAAE,wCAAwC,CAAC;SACrE,MAAM,CACL,iBAAiB,EACjB,mDAAmD,CACpD;SACA,MAAM,CAAC,UAAU,EAAE,oCAAoC,EAAE,KAAK,CAAC;SAC/D,MAAM,CACL,QAAQ,EACR,qDAAqD,EACrD,KAAK,CACN;SACA,MAAM,CACL,2BAA2B,EAC3B,uDAAuD,EACvD,UAAU,CACX;SACA,MAAM,CACL,qBAAqB,EACrB,uDAAuD,EACvD,UAAU,CACX;SACA,MAAM,CAAC,YAAY,EAAE,kCAAkC,EAAE,KAAK,CAAC;SAC/D,MAAM,CAAC,kBAAkB,EAAE,kCAAkC,EAAE,QAAQ,CAAC;SACxE,MAAM,CAAC,sBAAsB,EAAE,qBAAqB,EAAE,QAAQ,CAAC;SAC/D,MAAM,CACL,yBAAyB,EACzB,0CAA0C,CAC3C;SACA,MAAM,CACL,yBAAyB,EACzB,0CAA0C,CAC3C;SACA,MAAM,CAAC,kBAAkB,EAAE,iCAAiC,EAAE,SAAS,CAAC;SACxE,MAAM,CACL,2BAA2B,EAC3B,uCAAuC,EACvC,KAAK,CACN;SACA,MAAM,CAAC,uBAAuB,EAAE,qBAAqB,EAAE,KAAK,CAAC;SAC7D,MAAM,CAAC,wBAAwB,EAAE,sBAAsB,EAAE,KAAK,CAAC;SAC/D,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,KAAK,CAAC;SACvD,MAAM,CAAC,cAAc,EAAE,wCAAwC,EAAE,QAAQ,CAAC;SAC1E,MAAM,CACL,4BAA4B,EAC5B,6BAA6B,EAC7B,QAAQ,CACT;SACA,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;SACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,EAAE;QAC9C,kEAAkE;QAClE,MAAM,UAAU,GAAG,oBAAoB,IAAI,OAAO,CAAC,GAAG,CAAC;QACvD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CACX,+EAA+E,CAChF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,6CAA6C;QAC7C,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,IAAI,IAAA,aAAO,EAAC,UAAU,CAAC,CAAC;QAE5D,MAAM,YAAY,GAAG;YACnB,UAAU;YACV,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;gBAChC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9D,CAAC,CAAC,SAAS;YACb,YAAY,EAAE,OAAO,CAAC,YAAY;gBAChC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9D,CAAC,CAAC,SAAS;YACb,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,qBAAqB,EAAE,OAAO,CAAC,qBAAqB;YACpD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;YAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC;QAEF,MAAM,IAAA,0BAAkB,EAAC,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEL,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB;IACvB,MAAM,MAAM,GAAG,IAAI,mBAAO,CAAC,KAAK,CAAC;SAC9B,WAAW,CAAC,uCAAuC,CAAC;SACpD,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;SAC/B,MAAM,CACL,iBAAiB,EACjB,iDAAiD,CAClD;SACA,MAAM,CAAC,QAAQ,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACnD,MAAM,CAAC,kBAAkB,EAAE,0BAA0B,EAAE,QAAQ,CAAC;SAChE,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;SACzD,MAAM,CACL,kBAAkB,EAClB,uCAAuC,EACvC,SAAS,CACV;SACA,MAAM,CAAC,sBAAsB,EAAE,oBAAoB,EAAE,KAAK,CAAC;SAC3D,MAAM,CAAC,2BAA2B,EAAE,yBAAyB,EAAE,KAAK,CAAC;SACrE,MAAM,CAAC,qBAAqB,EAAE,oBAAoB,EAAE,UAAU,CAAC;SAC/D,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;SACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,QAAQ,EAAE,uBAAuB,EAAE,KAAK,CAAC;SAChD,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE;QACvC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;QACzC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,UAAU,GAAG;YACjB,UAAU,EAAE,GAAG;YACf,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,qBAAqB,EAAE,OAAO,CAAC,qBAAqB;YACpD,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC;QAEF,MAAM,IAAA,sBAAgB,EAAC,UAAU,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEL,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,6CAA6C;AAC7C,OAAO,CAAC,UAAU,CAAC,kBAAkB,EAAE,CAAC,CAAC;AACzC,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC;AAEvC,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,IAAA,kBAAS,GAAE,CAAC;AACpB,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,cAAc,CAAC;KACvB,WAAW,CAAC,mCAAmC,CAAC;KAChD,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;KACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;KACnE,MAAM,CAAC,QAAQ,EAAE,uBAAuB,EAAE,KAAK,CAAC;KAChD,MAAM,CACL,UAAU,EACV,qDAAqD,EACrD,KAAK,CACN;KACA,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,MAAM,IAAA,uCAAwB,EAAC,OAAO,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,uDAAuD;AACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAA,WAAK,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,+DAA+D;IAC/D,MAAM,GAAG,GAAG,IAAA,kBAAY,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,YAAY,GAAG;QACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACf,QAAQ;QACR,GAAG;QACH,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;KACjB,CAAC;IAEF,yEAAyE;IACzE,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/C,OAAO,CAAC,KAAK,CACX,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CACzD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;KAAM,CAAC;IACN,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for crawl command
|
|
3
|
+
*/
|
|
4
|
+
export interface CrawlOptions {
|
|
5
|
+
/** API key for Firecrawl */
|
|
6
|
+
apiKey?: string;
|
|
7
|
+
/** URL to crawl or job ID to check status */
|
|
8
|
+
urlOrJobId: string;
|
|
9
|
+
/** Check status of existing crawl job */
|
|
10
|
+
status?: boolean;
|
|
11
|
+
/** Wait for crawl to complete */
|
|
12
|
+
wait?: boolean;
|
|
13
|
+
/** Polling interval in seconds when waiting */
|
|
14
|
+
pollInterval?: number;
|
|
15
|
+
/** Timeout in seconds when waiting */
|
|
16
|
+
timeout?: number;
|
|
17
|
+
/** Show progress dots while waiting */
|
|
18
|
+
progress?: boolean;
|
|
19
|
+
/** Output file path */
|
|
20
|
+
output?: string;
|
|
21
|
+
/** Pretty print JSON output */
|
|
22
|
+
pretty?: boolean;
|
|
23
|
+
/** Maximum number of pages to crawl */
|
|
24
|
+
limit?: number;
|
|
25
|
+
/** Maximum crawl depth */
|
|
26
|
+
maxDepth?: number;
|
|
27
|
+
/** Exclude paths */
|
|
28
|
+
excludePaths?: string[];
|
|
29
|
+
/** Include paths */
|
|
30
|
+
includePaths?: string[];
|
|
31
|
+
/** Sitemap handling */
|
|
32
|
+
sitemap?: 'skip' | 'include';
|
|
33
|
+
/** Ignore query parameters */
|
|
34
|
+
ignoreQueryParameters?: boolean;
|
|
35
|
+
/** Crawl entire domain */
|
|
36
|
+
crawlEntireDomain?: boolean;
|
|
37
|
+
/** Allow external links */
|
|
38
|
+
allowExternalLinks?: boolean;
|
|
39
|
+
/** Allow subdomains */
|
|
40
|
+
allowSubdomains?: boolean;
|
|
41
|
+
/** Delay between requests */
|
|
42
|
+
delay?: number;
|
|
43
|
+
/** Maximum concurrency */
|
|
44
|
+
maxConcurrency?: number;
|
|
45
|
+
}
|
|
46
|
+
export interface CrawlResult {
|
|
47
|
+
success: boolean;
|
|
48
|
+
data?: any;
|
|
49
|
+
error?: string;
|
|
50
|
+
}
|
|
51
|
+
export interface CrawlStatusResult {
|
|
52
|
+
success: boolean;
|
|
53
|
+
data?: {
|
|
54
|
+
id: string;
|
|
55
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
56
|
+
total: number;
|
|
57
|
+
completed: number;
|
|
58
|
+
creditsUsed?: number;
|
|
59
|
+
expiresAt?: string;
|
|
60
|
+
};
|
|
61
|
+
error?: string;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=crawl.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/types/crawl.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,YAAY;IAC3B,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iCAAiC;IACjC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,+CAA+C;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sCAAsC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,oBAAoB;IACpB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,uBAAuB;IACvB,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,8BAA8B;IAC9B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,0BAA0B;IAC1B,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,2BAA2B;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,uBAAuB;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6BAA6B;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE;QACL,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,UAAU,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;QAC1D,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/types/crawl.ts"],"names":[],"mappings":";AAAA;;GAEG"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for map command
|
|
3
|
+
*/
|
|
4
|
+
export interface MapOptions {
|
|
5
|
+
/** API key for Firecrawl */
|
|
6
|
+
apiKey?: string;
|
|
7
|
+
/** URL to map or job ID to check status */
|
|
8
|
+
urlOrJobId: string;
|
|
9
|
+
/** Check status of existing map job */
|
|
10
|
+
status?: boolean;
|
|
11
|
+
/** Wait for map to complete */
|
|
12
|
+
wait?: boolean;
|
|
13
|
+
/** Output file path */
|
|
14
|
+
output?: string;
|
|
15
|
+
/** Output as JSON format */
|
|
16
|
+
json?: boolean;
|
|
17
|
+
/** Pretty print JSON output */
|
|
18
|
+
pretty?: boolean;
|
|
19
|
+
/** Maximum URLs to discover */
|
|
20
|
+
limit?: number;
|
|
21
|
+
/** Search query */
|
|
22
|
+
search?: string;
|
|
23
|
+
/** Sitemap handling */
|
|
24
|
+
sitemap?: 'only' | 'include' | 'skip';
|
|
25
|
+
/** Include subdomains */
|
|
26
|
+
includeSubdomains?: boolean;
|
|
27
|
+
/** Ignore query parameters */
|
|
28
|
+
ignoreQueryParameters?: boolean;
|
|
29
|
+
/** Timeout in seconds */
|
|
30
|
+
timeout?: number;
|
|
31
|
+
}
|
|
32
|
+
export interface MapResult {
|
|
33
|
+
success: boolean;
|
|
34
|
+
data?: {
|
|
35
|
+
links: Array<{
|
|
36
|
+
url: string;
|
|
37
|
+
title?: string;
|
|
38
|
+
description?: string;
|
|
39
|
+
}>;
|
|
40
|
+
};
|
|
41
|
+
error?: string;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=map.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/types/map.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,UAAU;IACzB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mBAAmB;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC;IACtC,yBAAyB;IACzB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,8BAA8B;IAC9B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE;QACL,KAAK,EAAE,KAAK,CAAC;YACX,GAAG,EAAE,MAAM,CAAC;YACZ,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC,CAAC;KACJ,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/types/map.ts"],"names":[],"mappings":";AAAA;;GAEG"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,SAAS,MAAM,wBAAwB,CAAC;AAC/C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAA6B,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAIxE;;;GAGG;AACH,wBAAgB,SAAS,
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,SAAS,MAAM,wBAAwB,CAAC;AAC/C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAA6B,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAIxE;;;GAGG;AACH,wBAAgB,SAAS,CACvB,OAAO,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GACxC,SAAS,CA6CX;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,SAAS,CAS1E;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
|
package/dist/utils/client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;AAYH,
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;AAYH,8BA+CC;AAMD,4CASC;AAKD,kCAEC;AA/ED,0EAA+C;AAE/C,qCAAwE;AAExE,IAAI,cAAc,GAAqB,IAAI,CAAC;AAE5C;;;GAGG;AACH,SAAgB,SAAS,CACvB,OAAyC;IAEzC,+EAA+E;IAC/E,MAAM,eAAe,GAAG,CACtB,KAAgC,EACZ,EAAE,CACtB,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IAE5D,qFAAqF;IACrF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAChE,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAEhE,8DAA8D;QAC9D,MAAM,gBAAgB,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9D,IAAA,uBAAc,EAAC,gBAAgB,CAAC,CAAC;QAEjC,MAAM,aAAa,GAA2B;YAC5C,MAAM,EAAE,gBAAgB,IAAI,SAAS;YACrC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAC5C,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;YAChD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU;YACnD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC,aAAa;SAC7D,CAAC;QAEF,OAAO,IAAI,sBAAS,CAAC,aAAa,CAAC,CAAC;IACtC,CAAC;IAED,0CAA0C;IAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,IAAA,uBAAc,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE9B,MAAM,aAAa,GAA2B;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC;QAEF,cAAc,GAAG,IAAI,sBAAS,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,MAA8B;IAC7D,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjD,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,qDAAqD;IACrD,cAAc,GAAG,IAAI,CAAC;IACtB,OAAO,SAAS,EAAE,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,cAAc,GAAG,IAAI,CAAC;AACxB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,IAAI,CAiBzE;AAED;;GAEG;AACH,wBAAgB,SAAS,IAAI,YAAY,CAExC;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAKhE;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAQlE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAOpD;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
|
package/dist/utils/config.js
CHANGED
|
@@ -23,8 +23,12 @@ function initializeConfig(config = {}) {
|
|
|
23
23
|
// Priority: provided config > env vars > stored credentials
|
|
24
24
|
const storedCredentials = (0, credentials_1.loadCredentials)();
|
|
25
25
|
globalConfig = {
|
|
26
|
-
apiKey: config.apiKey ||
|
|
27
|
-
|
|
26
|
+
apiKey: config.apiKey ||
|
|
27
|
+
process.env.FIRECRAWL_API_KEY ||
|
|
28
|
+
storedCredentials?.apiKey,
|
|
29
|
+
apiUrl: config.apiUrl ||
|
|
30
|
+
process.env.FIRECRAWL_API_URL ||
|
|
31
|
+
storedCredentials?.apiUrl,
|
|
28
32
|
timeoutMs: config.timeoutMs,
|
|
29
33
|
maxRetries: config.maxRetries,
|
|
30
34
|
backoffFactor: config.backoffFactor,
|
|
@@ -66,7 +70,7 @@ function getApiKey(providedKey) {
|
|
|
66
70
|
function validateConfig(apiKey) {
|
|
67
71
|
const key = getApiKey(apiKey);
|
|
68
72
|
if (!key) {
|
|
69
|
-
throw new Error('API key is required. Set FIRECRAWL_API_KEY environment variable, use --api-key flag, run "firecrawl config set
|
|
73
|
+
throw new Error('API key is required. Set FIRECRAWL_API_KEY environment variable, use --api-key flag, or run "firecrawl config" to set the API key.');
|
|
70
74
|
}
|
|
71
75
|
}
|
|
72
76
|
/**
|
package/dist/utils/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAsBH,
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAsBH,4CAiBC;AAKD,8BAEC;AAKD,oCAKC;AAMD,8BAQC;AAKD,wCAOC;AAKD,kCAEC;AAvFD,+CAAgD;AAUhD;;GAEG;AACH,IAAI,YAAY,GAAiB,EAAE,CAAC;AAEpC;;;;GAIG;AACH,SAAgB,gBAAgB,CAAC,SAAgC,EAAE;IACjE,4DAA4D;IAC5D,MAAM,iBAAiB,GAAG,IAAA,6BAAe,GAAE,CAAC;IAE5C,YAAY,GAAG;QACb,MAAM,EACJ,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAC7B,iBAAiB,EAAE,MAAM;QAC3B,MAAM,EACJ,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAC7B,iBAAiB,EAAE,MAAM;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,SAAS;IACvB,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,MAA6B;IACxD,YAAY,GAAG;QACb,GAAG,YAAY;QACf,GAAG,MAAM;KACV,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAgB,SAAS,CAAC,WAAoB;IAC5C,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,IAAI,YAAY,CAAC,MAAM;QAAE,OAAO,YAAY,CAAC,MAAM,CAAC;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAExE,uDAAuD;IACvD,MAAM,iBAAiB,GAAG,IAAA,6BAAe,GAAE,CAAC;IAC5C,OAAO,iBAAiB,EAAE,MAAM,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,MAAe;IAC5C,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,oIAAoI,CACrI,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,YAAY,GAAG,EAAE,CAAC;AACpB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;
|
|
1
|
+
{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAoDD;;GAEG;AACH,wBAAgB,eAAe,IAAI,iBAAiB,GAAG,IAAI,CAc1D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,iBAAiB,GAAG,IAAI,CAsBpE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAWxC;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgEH,0CAcC;AAKD,0CAsBC;AAKD,8CAWC;AAKD,wDAEC;AA9HD,uCAAyB;AACzB,2CAA6B;AAC7B,uCAAyB;AAOzB;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE/B,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,EAAE,QAAQ;YACrB,OAAO,IAAI,CAAC,IAAI,CACd,OAAO,EACP,SAAS,EACT,qBAAqB,EACrB,eAAe,CAChB,CAAC;QACJ,KAAK,OAAO,EAAE,UAAU;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,mBAAmB;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,kBAAkB,CAAC,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe;IACtB,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY;IACzE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oDAAoD;IACtD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe;IAC7B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAsB,CAAC;QAC1D,OAAO,WAAW,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kDAAkD;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe,CAAC,WAA8B;IAC5D,IAAI,CAAC;QACH,eAAe,EAAE,CAAC;QAClB,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAE7C,sCAAsC;QACtC,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;QACnC,MAAM,MAAM,GAAsB;YAChC,GAAG,QAAQ;YACX,GAAG,WAAW;SACf,CAAC;QAEF,gBAAgB;QAChB,EAAE,CAAC,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAE5E,yBAAyB;QACzB,oBAAoB,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB;IAC/B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACnC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC5F,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,sBAAsB;IACpC,OAAO,YAAY,EAAE,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Utility functions for job ID detection and validation
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Check if a string looks like a UUID/job ID
|
|
6
|
+
* Firecrawl job IDs are UUIDs (e.g., "550e8400-e29b-41d4-a716-446655440000")
|
|
7
|
+
*/
|
|
8
|
+
export declare function isJobId(str: string): boolean;
|
|
9
|
+
/**
|
|
10
|
+
* Check if a string is a valid URL
|
|
11
|
+
*/
|
|
12
|
+
export declare function isValidUrl(str: string): boolean;
|
|
13
|
+
//# sourceMappingURL=job.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"job.d.ts","sourceRoot":"","sources":["../../src/utils/job.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAK5C;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAO/C"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Utility functions for job ID detection and validation
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.isJobId = isJobId;
|
|
7
|
+
exports.isValidUrl = isValidUrl;
|
|
8
|
+
/**
|
|
9
|
+
* Check if a string looks like a UUID/job ID
|
|
10
|
+
* Firecrawl job IDs are UUIDs (e.g., "550e8400-e29b-41d4-a716-446655440000")
|
|
11
|
+
*/
|
|
12
|
+
function isJobId(str) {
|
|
13
|
+
// UUID v4 pattern
|
|
14
|
+
const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
15
|
+
return uuidPattern.test(str);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Check if a string is a valid URL
|
|
19
|
+
*/
|
|
20
|
+
function isValidUrl(str) {
|
|
21
|
+
try {
|
|
22
|
+
new URL(str);
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=job.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"job.js","sourceRoot":"","sources":["../../src/utils/job.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAMH,0BAKC;AAKD,gCAOC;AArBD;;;GAGG;AACH,SAAgB,OAAO,CAAC,GAAW;IACjC,kBAAkB;IAClB,MAAM,WAAW,GACf,wEAAwE,CAAC;IAC3E,OAAO,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,SAAgB,UAAU,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,CAiB9D"}
|
package/dist/utils/options.js
CHANGED
|
@@ -14,8 +14,12 @@ function parseScrapeOptions(options) {
|
|
|
14
14
|
onlyMainContent: options.onlyMainContent,
|
|
15
15
|
waitFor: options.waitFor,
|
|
16
16
|
screenshot: options.screenshot,
|
|
17
|
-
includeTags: options.includeTags
|
|
18
|
-
|
|
17
|
+
includeTags: options.includeTags
|
|
18
|
+
? options.includeTags.split(',').map((t) => t.trim())
|
|
19
|
+
: undefined,
|
|
20
|
+
excludeTags: options.excludeTags
|
|
21
|
+
? options.excludeTags.split(',').map((t) => t.trim())
|
|
22
|
+
: undefined,
|
|
19
23
|
apiKey: options.apiKey,
|
|
20
24
|
output: options.output,
|
|
21
25
|
pretty: options.pretty,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAOH,
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAOH,gDAiBC;AApBD;;GAEG;AACH,SAAgB,kBAAkB,CAAC,OAAY;IAC7C,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAC9B,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,SAAS;QACb,WAAW,EAAE,OAAO,CAAC,WAAW;YAC9B,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,SAAS;QACb,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAiDpD;;GAEG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAmBN;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,
|
|
1
|
+
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAiDpD;;GAEG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAmBN;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAiDN"}
|