firecrawl-cli 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/commands/config.d.ts +10 -0
- package/dist/commands/config.d.ts.map +1 -0
- package/dist/commands/config.js +102 -0
- package/dist/commands/config.js.map +1 -0
- package/dist/commands/scrape.d.ts +13 -0
- package/dist/commands/scrape.d.ts.map +1 -0
- package/dist/commands/scrape.js +73 -0
- package/dist/commands/scrape.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +89 -0
- package/dist/index.js.map +1 -0
- package/dist/types/scrape.d.ts +32 -0
- package/dist/types/scrape.d.ts.map +1 -0
- package/dist/types/scrape.js +6 -0
- package/dist/types/scrape.js.map +1 -0
- package/dist/utils/client.d.ts +22 -0
- package/dist/utils/client.d.ts.map +1 -0
- package/dist/utils/client.js +74 -0
- package/dist/utils/client.js.map +1 -0
- package/dist/utils/config.d.ts +38 -0
- package/dist/utils/config.d.ts.map +1 -0
- package/dist/utils/config.js +78 -0
- package/dist/utils/config.js.map +1 -0
- package/dist/utils/credentials.d.ts +25 -0
- package/dist/utils/credentials.d.ts.map +1 -0
- package/dist/utils/credentials.js +148 -0
- package/dist/utils/credentials.js.map +1 -0
- package/dist/utils/options.d.ts +9 -0
- package/dist/utils/options.d.ts.map +1 -0
- package/dist/utils/options.js +24 -0
- package/dist/utils/options.js.map +1 -0
- package/dist/utils/output.d.ts +16 -0
- package/dist/utils/output.d.ts.map +1 -0
- package/dist/utils/output.js +150 -0
- package/dist/utils/output.js.map +1 -0
- package/dist/utils/url.d.ts +12 -0
- package/dist/utils/url.d.ts.map +1 -0
- package/dist/utils/url.js +40 -0
- package/dist/utils/url.js.map +1 -0
- package/package.json +53 -0
package/README.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# 🔥 Firecrawl CLI
|
|
2
|
+
|
|
3
|
+
Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Install Firecrawl CLI globally using npm:
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install -g firecrawl-cli
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Quick Start
|
|
14
|
+
|
|
15
|
+
Set your API key:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
firecrawl config
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Scrape a URL:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
firecrawl https://firecrawl.dev
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
For detailed usage instructions, examples, and all available commands, visit the [CLI documentation](https://docs.firecrawl.dev/cli).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config command implementation
|
|
3
|
+
* Manages stored credentials and configuration
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Interactive configuration setup
|
|
7
|
+
* Asks for API URL and API key
|
|
8
|
+
*/
|
|
9
|
+
export declare function configure(): Promise<void>;
|
|
10
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/commands/config.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2BH;;;GAGG;AACH,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CA2C/C"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Config command implementation
|
|
4
|
+
* Manages stored credentials and configuration
|
|
5
|
+
*/
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
18
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
19
|
+
}) : function(o, v) {
|
|
20
|
+
o["default"] = v;
|
|
21
|
+
});
|
|
22
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
23
|
+
var ownKeys = function(o) {
|
|
24
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
25
|
+
var ar = [];
|
|
26
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
27
|
+
return ar;
|
|
28
|
+
};
|
|
29
|
+
return ownKeys(o);
|
|
30
|
+
};
|
|
31
|
+
return function (mod) {
|
|
32
|
+
if (mod && mod.__esModule) return mod;
|
|
33
|
+
var result = {};
|
|
34
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
35
|
+
__setModuleDefault(result, mod);
|
|
36
|
+
return result;
|
|
37
|
+
};
|
|
38
|
+
})();
|
|
39
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
40
|
+
exports.configure = configure;
|
|
41
|
+
const readline = __importStar(require("readline"));
|
|
42
|
+
const credentials_1 = require("../utils/credentials");
|
|
43
|
+
const config_1 = require("../utils/config");
|
|
44
|
+
const DEFAULT_API_URL = 'https://api.firecrawl.dev';
|
|
45
|
+
/**
|
|
46
|
+
* Prompt for input (for secure API key entry)
|
|
47
|
+
*/
|
|
48
|
+
function promptInput(question, defaultValue) {
|
|
49
|
+
const rl = readline.createInterface({
|
|
50
|
+
input: process.stdin,
|
|
51
|
+
output: process.stdout,
|
|
52
|
+
});
|
|
53
|
+
const promptText = defaultValue ? `${question} [${defaultValue}]: ` : `${question} `;
|
|
54
|
+
return new Promise((resolve) => {
|
|
55
|
+
rl.question(promptText, (answer) => {
|
|
56
|
+
rl.close();
|
|
57
|
+
resolve(answer.trim() || defaultValue || '');
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Interactive configuration setup
|
|
63
|
+
* Asks for API URL and API key
|
|
64
|
+
*/
|
|
65
|
+
async function configure() {
|
|
66
|
+
console.log('Firecrawl Configuration Setup\n');
|
|
67
|
+
// Prompt for API URL with default
|
|
68
|
+
let url = await promptInput('Enter API URL', DEFAULT_API_URL);
|
|
69
|
+
// Ensure URL doesn't end with trailing slash
|
|
70
|
+
url = url.replace(/\/$/, '');
|
|
71
|
+
// Prompt for API key
|
|
72
|
+
const key = await promptInput('Enter your Firecrawl API key: ');
|
|
73
|
+
if (!key || key.trim().length === 0) {
|
|
74
|
+
console.error('Error: API key cannot be empty');
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
77
|
+
if (!url || url.trim().length === 0) {
|
|
78
|
+
console.error('Error: API URL cannot be empty');
|
|
79
|
+
process.exit(1);
|
|
80
|
+
}
|
|
81
|
+
// Normalize URL (remove trailing slash)
|
|
82
|
+
const normalizedUrl = url.trim().replace(/\/$/, '');
|
|
83
|
+
try {
|
|
84
|
+
(0, credentials_1.saveCredentials)({
|
|
85
|
+
apiKey: key.trim(),
|
|
86
|
+
apiUrl: normalizedUrl
|
|
87
|
+
});
|
|
88
|
+
console.log('\n✓ Configuration saved successfully');
|
|
89
|
+
console.log(` API URL: ${normalizedUrl}`);
|
|
90
|
+
console.log(` Stored in: ${(0, credentials_1.getConfigDirectoryPath)()}`);
|
|
91
|
+
// Update global config
|
|
92
|
+
(0, config_1.updateConfig)({
|
|
93
|
+
apiKey: key.trim(),
|
|
94
|
+
apiUrl: normalizedUrl
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
console.error('Error saving configuration:', error instanceof Error ? error.message : 'Unknown error');
|
|
99
|
+
process.exit(1);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/commands/config.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BH,8BA2CC;AAxED,mDAAqC;AACrC,sDAA+E;AAC/E,4CAA+C;AAE/C,MAAM,eAAe,GAAG,2BAA2B,CAAC;AAEpD;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB,EAAE,YAAqB;IAC1D,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,QAAQ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,CAAC;IAErF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE;YACjC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,YAAY,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,SAAS;IAC7B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAE/C,kCAAkC;IAClC,IAAI,GAAG,GAAG,MAAM,WAAW,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAE9D,6CAA6C;IAC7C,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAE7B,qBAAqB;IACrB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,gCAAgC,CAAC,CAAC;IAEhE,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,wCAAwC;IACxC,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,IAAA,6BAAe,EAAC;YACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE;YAClB,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,cAAc,aAAa,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAA,oCAAsB,GAAE,EAAE,CAAC,CAAC;QAExD,uBAAuB;QACvB,IAAA,qBAAY,EAAC;YACX,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE;YAClB,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scrape command implementation
|
|
3
|
+
*/
|
|
4
|
+
import type { ScrapeOptions, ScrapeResult } from '../types/scrape';
|
|
5
|
+
/**
|
|
6
|
+
* Execute the scrape command
|
|
7
|
+
*/
|
|
8
|
+
export declare function executeScrape(options: ScrapeOptions): Promise<ScrapeResult>;
|
|
9
|
+
/**
|
|
10
|
+
* Handle scrape command output
|
|
11
|
+
*/
|
|
12
|
+
export declare function handleScrapeCommand(options: ScrapeOptions): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=scrape.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAKnE;;GAEG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAoEjF;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/E"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Scrape command implementation
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.executeScrape = executeScrape;
|
|
7
|
+
exports.handleScrapeCommand = handleScrapeCommand;
|
|
8
|
+
const client_1 = require("../utils/client");
|
|
9
|
+
const config_1 = require("../utils/config");
|
|
10
|
+
const output_1 = require("../utils/output");
|
|
11
|
+
/**
|
|
12
|
+
* Execute the scrape command
|
|
13
|
+
*/
|
|
14
|
+
async function executeScrape(options) {
|
|
15
|
+
try {
|
|
16
|
+
// Update global config if API key is provided via options
|
|
17
|
+
if (options.apiKey) {
|
|
18
|
+
(0, config_1.updateConfig)({ apiKey: options.apiKey });
|
|
19
|
+
}
|
|
20
|
+
// Get client instance (uses global config)
|
|
21
|
+
const app = (0, client_1.getClient)();
|
|
22
|
+
// Build scrape options
|
|
23
|
+
const formats = [];
|
|
24
|
+
if (options.format) {
|
|
25
|
+
formats.push(options.format);
|
|
26
|
+
}
|
|
27
|
+
if (options.screenshot) {
|
|
28
|
+
// Add screenshot format if not already included
|
|
29
|
+
if (!formats.includes('screenshot')) {
|
|
30
|
+
formats.push('screenshot');
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
// If no formats specified, default to markdown
|
|
34
|
+
if (formats.length === 0) {
|
|
35
|
+
formats.push('markdown');
|
|
36
|
+
}
|
|
37
|
+
const scrapeParams = {
|
|
38
|
+
formats,
|
|
39
|
+
};
|
|
40
|
+
if (options.onlyMainContent !== undefined) {
|
|
41
|
+
scrapeParams.onlyMainContent = options.onlyMainContent;
|
|
42
|
+
}
|
|
43
|
+
if (options.waitFor !== undefined) {
|
|
44
|
+
scrapeParams.waitFor = options.waitFor;
|
|
45
|
+
}
|
|
46
|
+
if (options.includeTags && options.includeTags.length > 0) {
|
|
47
|
+
scrapeParams.includeTags = options.includeTags;
|
|
48
|
+
}
|
|
49
|
+
if (options.excludeTags && options.excludeTags.length > 0) {
|
|
50
|
+
scrapeParams.excludeTags = options.excludeTags;
|
|
51
|
+
}
|
|
52
|
+
// Execute scrape
|
|
53
|
+
const result = await app.scrape(options.url, scrapeParams);
|
|
54
|
+
return {
|
|
55
|
+
success: true,
|
|
56
|
+
data: result,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
return {
|
|
61
|
+
success: false,
|
|
62
|
+
error: error instanceof Error ? error.message : 'Unknown error occurred',
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Handle scrape command output
|
|
68
|
+
*/
|
|
69
|
+
async function handleScrapeCommand(options) {
|
|
70
|
+
const result = await executeScrape(options);
|
|
71
|
+
(0, output_1.handleScrapeOutput)(result, options.format, options.output, options.pretty);
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=scrape.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAWH,sCAoEC;AAKD,kDAGC;AAnFD,4CAA4C;AAC5C,4CAA+C;AAC/C,4CAAqD;AAErD;;GAEG;AACI,KAAK,UAAU,aAAa,CAAC,OAAsB;IACxD,IAAI,CAAC;QACH,0DAA0D;QAC1D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,2CAA2C;QAC3C,MAAM,GAAG,GAAG,IAAA,kBAAS,GAAE,CAAC;QAExB,uBAAuB;QACvB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,gDAAgD;YAChD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,YAAY,GAMd;YACF,OAAO;SACR,CAAC;QAEF,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC1C,YAAY,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QACzD,CAAC;QAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACjD,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACjD,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAE3D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,MAAM;SACb,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CAAC,OAAsB;IAC9D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAA,2BAAkB,EAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC7E,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;GAGG"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* Firecrawl CLI
|
|
5
|
+
* Entry point for the CLI application
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
const commander_1 = require("commander");
|
|
9
|
+
const scrape_1 = require("./commands/scrape");
|
|
10
|
+
const config_1 = require("./utils/config");
|
|
11
|
+
const config_2 = require("./commands/config");
|
|
12
|
+
const url_1 = require("./utils/url");
|
|
13
|
+
const options_1 = require("./utils/options");
|
|
14
|
+
// Initialize global configuration from environment variables
|
|
15
|
+
(0, config_1.initializeConfig)();
|
|
16
|
+
const program = new commander_1.Command();
|
|
17
|
+
program
|
|
18
|
+
.name('firecrawl')
|
|
19
|
+
.description('CLI tool for Firecrawl web scraping')
|
|
20
|
+
.version('1.0.0')
|
|
21
|
+
.option('-k, --api-key <key>', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var, or use "firecrawl config")')
|
|
22
|
+
.allowUnknownOption() // Allow unknown options when URL is passed directly
|
|
23
|
+
.hook('preAction', (thisCommand, actionCommand) => {
|
|
24
|
+
// Update global config if API key is provided via global option
|
|
25
|
+
const globalOptions = thisCommand.opts();
|
|
26
|
+
if (globalOptions.apiKey) {
|
|
27
|
+
(0, config_1.updateConfig)({ apiKey: globalOptions.apiKey });
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
/**
|
|
31
|
+
* Create and configure the scrape command
|
|
32
|
+
*/
|
|
33
|
+
function createScrapeCommand() {
|
|
34
|
+
const scrapeCmd = new commander_1.Command('scrape')
|
|
35
|
+
.description('Scrape a URL using Firecrawl')
|
|
36
|
+
.argument('[url]', 'URL to scrape')
|
|
37
|
+
.option('-u, --url <url>', 'URL to scrape (alternative to positional argument)')
|
|
38
|
+
.option('-H, --html', 'Output raw HTML (shortcut for --format html)')
|
|
39
|
+
.option('-f, --format <format>', 'Output format: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding', 'markdown')
|
|
40
|
+
.option('--only-main-content', 'Include only main content', false)
|
|
41
|
+
.option('--wait-for <ms>', 'Wait time before scraping in milliseconds', parseInt)
|
|
42
|
+
.option('--screenshot', 'Take a screenshot', false)
|
|
43
|
+
.option('--include-tags <tags>', 'Comma-separated list of tags to include')
|
|
44
|
+
.option('--exclude-tags <tags>', 'Comma-separated list of tags to exclude')
|
|
45
|
+
.option('-k, --api-key <key>', 'Firecrawl API key (overrides global --api-key)')
|
|
46
|
+
.option('-o, --output <path>', 'Output file path (default: stdout)')
|
|
47
|
+
.option('--pretty', 'Pretty print JSON output', false)
|
|
48
|
+
.action(async (positionalUrl, options) => {
|
|
49
|
+
// Use positional URL if provided, otherwise use --url option
|
|
50
|
+
const url = positionalUrl || options.url;
|
|
51
|
+
if (!url) {
|
|
52
|
+
console.error('Error: URL is required. Provide it as argument or use --url option.');
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
// Handle --html shortcut flag
|
|
56
|
+
const format = options.html ? 'html' : options.format;
|
|
57
|
+
const scrapeOptions = (0, options_1.parseScrapeOptions)({ ...options, url, format });
|
|
58
|
+
await (0, scrape_1.handleScrapeCommand)(scrapeOptions);
|
|
59
|
+
});
|
|
60
|
+
return scrapeCmd;
|
|
61
|
+
}
|
|
62
|
+
// Add scrape command to main program
|
|
63
|
+
program.addCommand(createScrapeCommand());
|
|
64
|
+
program
|
|
65
|
+
.command('config')
|
|
66
|
+
.description('Configure API URL and API key (interactive)')
|
|
67
|
+
.action(async () => {
|
|
68
|
+
await (0, config_2.configure)();
|
|
69
|
+
});
|
|
70
|
+
// Parse arguments
|
|
71
|
+
const args = process.argv.slice(2);
|
|
72
|
+
// Check if first argument is a URL (and not a command)
|
|
73
|
+
if (args.length > 0 && !args[0].startsWith('-') && (0, url_1.isUrl)(args[0])) {
|
|
74
|
+
// Treat as scrape command with URL - reuse commander's parsing
|
|
75
|
+
const url = (0, url_1.normalizeUrl)(args[0]);
|
|
76
|
+
// Modify argv to include scrape command with URL as positional argument
|
|
77
|
+
// This allows commander to parse it normally with all hooks and options
|
|
78
|
+
const modifiedArgv = [process.argv[0], process.argv[1], 'scrape', url, ...args.slice(1)];
|
|
79
|
+
// Parse using the main program (which includes hooks and global options)
|
|
80
|
+
program.parseAsync(modifiedArgv).catch((error) => {
|
|
81
|
+
console.error('Error:', error instanceof Error ? error.message : 'Unknown error');
|
|
82
|
+
process.exit(1);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
// Normal command parsing
|
|
87
|
+
program.parse();
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA;;;GAGG;;AAEH,yCAAoC;AACpC,8CAAwD;AACxD,2CAAgE;AAChE,8CAA8C;AAC9C,qCAAkD;AAClD,6CAAqD;AAErD,6DAA6D;AAC7D,IAAA,yBAAgB,GAAE,CAAC;AAEnB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,qCAAqC,CAAC;KAClD,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,CAAC,qBAAqB,EAAE,iFAAiF,CAAC;KAChH,kBAAkB,EAAE,CAAC,oDAAoD;KACzE,IAAI,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE;IAChD,gEAAgE;IAChE,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,SAAS,mBAAmB;IAC1B,MAAM,SAAS,GAAG,IAAI,mBAAO,CAAC,QAAQ,CAAC;SACpC,WAAW,CAAC,8BAA8B,CAAC;SAC3C,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;SAClC,MAAM,CAAC,iBAAiB,EAAE,oDAAoD,CAAC;SAC/E,MAAM,CAAC,YAAY,EAAE,8CAA8C,CAAC;SACpE,MAAM,CAAC,uBAAuB,EAAE,wHAAwH,EAAE,UAAU,CAAC;SACrK,MAAM,CAAC,qBAAqB,EAAE,2BAA2B,EAAE,KAAK,CAAC;SACjE,MAAM,CAAC,iBAAiB,EAAE,2CAA2C,EAAE,QAAQ,CAAC;SAChF,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,KAAK,CAAC;SAClD,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CAAC,qBAAqB,EAAE,gDAAgD,CAAC;SAC/E,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE;QACvC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;QACzC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,8BAA8B;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtD,MAAM,aAAa,GAAG,IAAA,4BAAkB,EAAC,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QACtE,MAAM,IAAA,4BAAmB,EAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEL,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qCAAqC;AACrC,OAAO,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAE1C,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,IAAA,kBAAS,GAAE,CAAC;AACpB,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,uDAAuD;AACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAA,WAAK,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,+DAA+D;IAC/D,MAAM,GAAG,GAAG,IAAA,kBAAY,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzF,yEAAyE;IACzE,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/C,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;KAAM,CAAC;IACN,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types and interfaces for the scrape command
|
|
3
|
+
*/
|
|
4
|
+
export type ScrapeFormat = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes' | 'branding';
|
|
5
|
+
export interface ScrapeOptions {
|
|
6
|
+
/** URL to scrape */
|
|
7
|
+
url: string;
|
|
8
|
+
/** Output format (markdown, html, etc.) */
|
|
9
|
+
format?: ScrapeFormat;
|
|
10
|
+
/** Include only main content */
|
|
11
|
+
onlyMainContent?: boolean;
|
|
12
|
+
/** Wait time before scraping (ms) */
|
|
13
|
+
waitFor?: number;
|
|
14
|
+
/** Take screenshot */
|
|
15
|
+
screenshot?: boolean;
|
|
16
|
+
/** Include tags */
|
|
17
|
+
includeTags?: string[];
|
|
18
|
+
/** Exclude tags */
|
|
19
|
+
excludeTags?: string[];
|
|
20
|
+
/** API key for Firecrawl */
|
|
21
|
+
apiKey?: string;
|
|
22
|
+
/** Output file path */
|
|
23
|
+
output?: string;
|
|
24
|
+
/** Pretty print JSON output */
|
|
25
|
+
pretty?: boolean;
|
|
26
|
+
}
|
|
27
|
+
export interface ScrapeResult {
|
|
28
|
+
success: boolean;
|
|
29
|
+
data?: any;
|
|
30
|
+
error?: string;
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=scrape.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,MAAM,GACN,SAAS,GACT,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,SAAS,GACT,gBAAgB,GAChB,MAAM,GACN,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gCAAgC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Firecrawl client utility
|
|
3
|
+
* Provides a singleton client instance initialized with global configuration
|
|
4
|
+
*/
|
|
5
|
+
import Firecrawl from '@mendable/firecrawl-js';
|
|
6
|
+
import type { FirecrawlClientOptions } from '@mendable/firecrawl-js';
|
|
7
|
+
import { type GlobalConfig } from './config';
|
|
8
|
+
/**
|
|
9
|
+
* Get or create the Firecrawl client instance
|
|
10
|
+
* Uses global configuration if available, otherwise creates with provided options
|
|
11
|
+
*/
|
|
12
|
+
export declare function getClient(options?: Partial<FirecrawlClientOptions>): Firecrawl;
|
|
13
|
+
/**
|
|
14
|
+
* Initialize the client with configuration
|
|
15
|
+
* This should be called early in the application lifecycle
|
|
16
|
+
*/
|
|
17
|
+
export declare function initializeClient(config?: Partial<GlobalConfig>): Firecrawl;
|
|
18
|
+
/**
|
|
19
|
+
* Reset the client instance (useful for testing)
|
|
20
|
+
*/
|
|
21
|
+
export declare function resetClient(): void;
|
|
22
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,SAAS,MAAM,wBAAwB,CAAC;AAC/C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAA6B,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAIxE;;;GAGG;AACH,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,SAAS,CA2C9E;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,SAAS,CAS1E;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Firecrawl client utility
|
|
4
|
+
* Provides a singleton client instance initialized with global configuration
|
|
5
|
+
*/
|
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
|
+
};
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.getClient = getClient;
|
|
11
|
+
exports.initializeClient = initializeClient;
|
|
12
|
+
exports.resetClient = resetClient;
|
|
13
|
+
const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
|
|
14
|
+
const config_1 = require("./config");
|
|
15
|
+
let clientInstance = null;
|
|
16
|
+
/**
|
|
17
|
+
* Get or create the Firecrawl client instance
|
|
18
|
+
* Uses global configuration if available, otherwise creates with provided options
|
|
19
|
+
*/
|
|
20
|
+
function getClient(options) {
|
|
21
|
+
// Helper to convert null to undefined and ensure we have a string or undefined
|
|
22
|
+
const normalizeApiKey = (value) => value === null || value === undefined ? undefined : value;
|
|
23
|
+
// If options provided, create a new instance (useful for command-specific overrides)
|
|
24
|
+
if (options) {
|
|
25
|
+
const config = (0, config_1.getConfig)();
|
|
26
|
+
const apiKey = normalizeApiKey(options.apiKey) ?? config.apiKey;
|
|
27
|
+
const apiUrl = normalizeApiKey(options.apiUrl) ?? config.apiUrl;
|
|
28
|
+
// Normalize apiKey for validation (convert null to undefined)
|
|
29
|
+
const normalizedApiKey = apiKey === null ? undefined : apiKey;
|
|
30
|
+
(0, config_1.validateConfig)(normalizedApiKey);
|
|
31
|
+
const clientOptions = {
|
|
32
|
+
apiKey: normalizedApiKey || undefined,
|
|
33
|
+
apiUrl: apiUrl === null ? undefined : apiUrl,
|
|
34
|
+
timeoutMs: options.timeoutMs ?? config.timeoutMs,
|
|
35
|
+
maxRetries: options.maxRetries ?? config.maxRetries,
|
|
36
|
+
backoffFactor: options.backoffFactor ?? config.backoffFactor,
|
|
37
|
+
};
|
|
38
|
+
return new firecrawl_js_1.default(clientOptions);
|
|
39
|
+
}
|
|
40
|
+
// Return singleton instance or create one
|
|
41
|
+
if (!clientInstance) {
|
|
42
|
+
const config = (0, config_1.getConfig)();
|
|
43
|
+
(0, config_1.validateConfig)(config.apiKey);
|
|
44
|
+
const clientOptions = {
|
|
45
|
+
apiKey: config.apiKey || undefined,
|
|
46
|
+
apiUrl: config.apiUrl || undefined,
|
|
47
|
+
timeoutMs: config.timeoutMs,
|
|
48
|
+
maxRetries: config.maxRetries,
|
|
49
|
+
backoffFactor: config.backoffFactor,
|
|
50
|
+
};
|
|
51
|
+
clientInstance = new firecrawl_js_1.default(clientOptions);
|
|
52
|
+
}
|
|
53
|
+
return clientInstance;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Initialize the client with configuration
|
|
57
|
+
* This should be called early in the application lifecycle
|
|
58
|
+
*/
|
|
59
|
+
function initializeClient(config) {
|
|
60
|
+
if (config) {
|
|
61
|
+
const { initializeConfig } = require('./config');
|
|
62
|
+
initializeConfig(config);
|
|
63
|
+
}
|
|
64
|
+
// Reset instance to force recreation with new config
|
|
65
|
+
clientInstance = null;
|
|
66
|
+
return getClient();
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Reset the client instance (useful for testing)
|
|
70
|
+
*/
|
|
71
|
+
function resetClient() {
|
|
72
|
+
clientInstance = null;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;AAYH,8BA2CC;AAMD,4CASC;AAKD,kCAEC;AA3ED,0EAA+C;AAE/C,qCAAwE;AAExE,IAAI,cAAc,GAAqB,IAAI,CAAC;AAE5C;;;GAGG;AACH,SAAgB,SAAS,CAAC,OAAyC;IACjE,+EAA+E;IAC/E,MAAM,eAAe,GAAG,CAAC,KAAgC,EAAsB,EAAE,CAC/E,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IAE5D,qFAAqF;IACrF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAChE,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAEhE,8DAA8D;QAC9D,MAAM,gBAAgB,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9D,IAAA,uBAAc,EAAC,gBAAgB,CAAC,CAAC;QAEjC,MAAM,aAAa,GAA2B;YAC5C,MAAM,EAAE,gBAAgB,IAAI,SAAS;YACrC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAC5C,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;YAChD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU;YACnD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC,aAAa;SAC7D,CAAC;QAEF,OAAO,IAAI,sBAAS,CAAC,aAAa,CAAC,CAAC;IACtC,CAAC;IAED,0CAA0C;IAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,IAAA,uBAAc,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE9B,MAAM,aAAa,GAA2B;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC;QAEF,cAAc,GAAG,IAAI,sBAAS,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,MAA8B;IAC7D,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjD,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,qDAAqD;IACrD,cAAc,GAAG,IAAI,CAAC;IACtB,OAAO,SAAS,EAAE,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,cAAc,GAAG,IAAI,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Global configuration system
|
|
3
|
+
*/
|
|
4
|
+
export interface GlobalConfig {
|
|
5
|
+
apiKey?: string;
|
|
6
|
+
apiUrl?: string;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
maxRetries?: number;
|
|
9
|
+
backoffFactor?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Initialize global configuration
|
|
13
|
+
* Loads from: provided config > environment variables > OS credential storage
|
|
14
|
+
* @param config Configuration options
|
|
15
|
+
*/
|
|
16
|
+
export declare function initializeConfig(config?: Partial<GlobalConfig>): void;
|
|
17
|
+
/**
|
|
18
|
+
* Get the current global configuration
|
|
19
|
+
*/
|
|
20
|
+
export declare function getConfig(): GlobalConfig;
|
|
21
|
+
/**
|
|
22
|
+
* Update global configuration (merges with existing)
|
|
23
|
+
*/
|
|
24
|
+
export declare function updateConfig(config: Partial<GlobalConfig>): void;
|
|
25
|
+
/**
|
|
26
|
+
* Get API key from global config or provided value
|
|
27
|
+
* Priority: provided key > global config > env var > stored credentials
|
|
28
|
+
*/
|
|
29
|
+
export declare function getApiKey(providedKey?: string): string | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* Validate that required configuration is present
|
|
32
|
+
*/
|
|
33
|
+
export declare function validateConfig(apiKey?: string): void;
|
|
34
|
+
/**
|
|
35
|
+
* Reset global configuration (useful for testing)
|
|
36
|
+
*/
|
|
37
|
+
export declare function resetConfig(): void;
|
|
38
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,IAAI,CAWzE;AAED;;GAEG;AACH,wBAAgB,SAAS,IAAI,YAAY,CAExC;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAKhE;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAQlE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAOpD;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Global configuration system
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.initializeConfig = initializeConfig;
|
|
7
|
+
exports.getConfig = getConfig;
|
|
8
|
+
exports.updateConfig = updateConfig;
|
|
9
|
+
exports.getApiKey = getApiKey;
|
|
10
|
+
exports.validateConfig = validateConfig;
|
|
11
|
+
exports.resetConfig = resetConfig;
|
|
12
|
+
const credentials_1 = require("./credentials");
|
|
13
|
+
/**
|
|
14
|
+
* Global configuration instance
|
|
15
|
+
*/
|
|
16
|
+
let globalConfig = {};
|
|
17
|
+
/**
|
|
18
|
+
* Initialize global configuration
|
|
19
|
+
* Loads from: provided config > environment variables > OS credential storage
|
|
20
|
+
* @param config Configuration options
|
|
21
|
+
*/
|
|
22
|
+
function initializeConfig(config = {}) {
|
|
23
|
+
// Priority: provided config > env vars > stored credentials
|
|
24
|
+
const storedCredentials = (0, credentials_1.loadCredentials)();
|
|
25
|
+
globalConfig = {
|
|
26
|
+
apiKey: config.apiKey || process.env.FIRECRAWL_API_KEY || storedCredentials?.apiKey,
|
|
27
|
+
apiUrl: config.apiUrl || process.env.FIRECRAWL_API_URL || storedCredentials?.apiUrl,
|
|
28
|
+
timeoutMs: config.timeoutMs,
|
|
29
|
+
maxRetries: config.maxRetries,
|
|
30
|
+
backoffFactor: config.backoffFactor,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Get the current global configuration
|
|
35
|
+
*/
|
|
36
|
+
function getConfig() {
|
|
37
|
+
return { ...globalConfig };
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Update global configuration (merges with existing)
|
|
41
|
+
*/
|
|
42
|
+
function updateConfig(config) {
|
|
43
|
+
globalConfig = {
|
|
44
|
+
...globalConfig,
|
|
45
|
+
...config,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Get API key from global config or provided value
|
|
50
|
+
* Priority: provided key > global config > env var > stored credentials
|
|
51
|
+
*/
|
|
52
|
+
function getApiKey(providedKey) {
|
|
53
|
+
if (providedKey)
|
|
54
|
+
return providedKey;
|
|
55
|
+
if (globalConfig.apiKey)
|
|
56
|
+
return globalConfig.apiKey;
|
|
57
|
+
if (process.env.FIRECRAWL_API_KEY)
|
|
58
|
+
return process.env.FIRECRAWL_API_KEY;
|
|
59
|
+
// Fallback to stored credentials if not already loaded
|
|
60
|
+
const storedCredentials = (0, credentials_1.loadCredentials)();
|
|
61
|
+
return storedCredentials?.apiKey;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Validate that required configuration is present
|
|
65
|
+
*/
|
|
66
|
+
function validateConfig(apiKey) {
|
|
67
|
+
const key = getApiKey(apiKey);
|
|
68
|
+
if (!key) {
|
|
69
|
+
throw new Error('API key is required. Set FIRECRAWL_API_KEY environment variable, use --api-key flag, run "firecrawl config set-api-key", or initialize config.');
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Reset global configuration (useful for testing)
|
|
74
|
+
*/
|
|
75
|
+
function resetConfig() {
|
|
76
|
+
globalConfig = {};
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAsBH,4CAWC;AAKD,8BAEC;AAKD,oCAKC;AAMD,8BAQC;AAKD,wCAOC;AAKD,kCAEC;AAjFD,+CAAgD;AAUhD;;GAEG;AACH,IAAI,YAAY,GAAiB,EAAE,CAAC;AAEpC;;;;GAIG;AACH,SAAgB,gBAAgB,CAAC,SAAgC,EAAE;IACjE,4DAA4D;IAC5D,MAAM,iBAAiB,GAAG,IAAA,6BAAe,GAAE,CAAC;IAE5C,YAAY,GAAG;QACb,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,iBAAiB,EAAE,MAAM;QACnF,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,iBAAiB,EAAE,MAAM;QACnF,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,SAAS;IACvB,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,MAA6B;IACxD,YAAY,GAAG;QACb,GAAG,YAAY;QACf,GAAG,MAAM;KACV,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAgB,SAAS,CAAC,WAAoB;IAC5C,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,IAAI,YAAY,CAAC,MAAM;QAAE,OAAO,YAAY,CAAC,MAAM,CAAC;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAExE,uDAAuD;IACvD,MAAM,iBAAiB,GAAG,IAAA,6BAAe,GAAE,CAAC;IAC5C,OAAO,iBAAiB,EAAE,MAAM,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,MAAe;IAC5C,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gJAAgJ,CACjJ,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,YAAY,GAAG,EAAE,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OS-level credential storage utility
|
|
3
|
+
* Stores credentials in platform-specific application data directories
|
|
4
|
+
*/
|
|
5
|
+
export interface StoredCredentials {
|
|
6
|
+
apiKey?: string;
|
|
7
|
+
apiUrl?: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Load credentials from OS storage
|
|
11
|
+
*/
|
|
12
|
+
export declare function loadCredentials(): StoredCredentials | null;
|
|
13
|
+
/**
|
|
14
|
+
* Save credentials to OS storage
|
|
15
|
+
*/
|
|
16
|
+
export declare function saveCredentials(credentials: StoredCredentials): void;
|
|
17
|
+
/**
|
|
18
|
+
* Delete stored credentials
|
|
19
|
+
*/
|
|
20
|
+
export declare function deleteCredentials(): void;
|
|
21
|
+
/**
|
|
22
|
+
* Get the config directory path (for informational purposes)
|
|
23
|
+
*/
|
|
24
|
+
export declare function getConfigDirectoryPath(): string;
|
|
25
|
+
//# sourceMappingURL=credentials.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA+CD;;GAEG;AACH,wBAAgB,eAAe,IAAI,iBAAiB,GAAG,IAAI,CAc1D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,iBAAiB,GAAG,IAAI,CAwBpE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CASxC;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* OS-level credential storage utility
|
|
4
|
+
* Stores credentials in platform-specific application data directories
|
|
5
|
+
*/
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
18
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
19
|
+
}) : function(o, v) {
|
|
20
|
+
o["default"] = v;
|
|
21
|
+
});
|
|
22
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
23
|
+
var ownKeys = function(o) {
|
|
24
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
25
|
+
var ar = [];
|
|
26
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
27
|
+
return ar;
|
|
28
|
+
};
|
|
29
|
+
return ownKeys(o);
|
|
30
|
+
};
|
|
31
|
+
return function (mod) {
|
|
32
|
+
if (mod && mod.__esModule) return mod;
|
|
33
|
+
var result = {};
|
|
34
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
35
|
+
__setModuleDefault(result, mod);
|
|
36
|
+
return result;
|
|
37
|
+
};
|
|
38
|
+
})();
|
|
39
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
40
|
+
exports.loadCredentials = loadCredentials;
|
|
41
|
+
exports.saveCredentials = saveCredentials;
|
|
42
|
+
exports.deleteCredentials = deleteCredentials;
|
|
43
|
+
exports.getConfigDirectoryPath = getConfigDirectoryPath;
|
|
44
|
+
const fs = __importStar(require("fs"));
|
|
45
|
+
const path = __importStar(require("path"));
|
|
46
|
+
const os = __importStar(require("os"));
|
|
47
|
+
/**
|
|
48
|
+
* Get the platform-specific config directory
|
|
49
|
+
*/
|
|
50
|
+
function getConfigDir() {
|
|
51
|
+
const homeDir = os.homedir();
|
|
52
|
+
const platform = os.platform();
|
|
53
|
+
switch (platform) {
|
|
54
|
+
case 'darwin': // macOS
|
|
55
|
+
return path.join(homeDir, 'Library', 'Application Support', 'firecrawl-cli');
|
|
56
|
+
case 'win32': // Windows
|
|
57
|
+
return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli');
|
|
58
|
+
default: // Linux and others
|
|
59
|
+
return path.join(homeDir, '.config', 'firecrawl-cli');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Get the credentials file path
|
|
64
|
+
*/
|
|
65
|
+
function getCredentialsPath() {
|
|
66
|
+
return path.join(getConfigDir(), 'credentials.json');
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Ensure the config directory exists
|
|
70
|
+
*/
|
|
71
|
+
function ensureConfigDir() {
|
|
72
|
+
const configDir = getConfigDir();
|
|
73
|
+
if (!fs.existsSync(configDir)) {
|
|
74
|
+
fs.mkdirSync(configDir, { recursive: true, mode: 0o700 }); // rwx------
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Set file permissions to be readable/writable only by the owner
|
|
79
|
+
*/
|
|
80
|
+
function setSecurePermissions(filePath) {
|
|
81
|
+
try {
|
|
82
|
+
fs.chmodSync(filePath, 0o600); // rw-------
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
// Ignore errors on Windows or if file doesn't exist
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Load credentials from OS storage
|
|
90
|
+
*/
|
|
91
|
+
function loadCredentials() {
|
|
92
|
+
try {
|
|
93
|
+
const credentialsPath = getCredentialsPath();
|
|
94
|
+
if (!fs.existsSync(credentialsPath)) {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
const data = fs.readFileSync(credentialsPath, 'utf-8');
|
|
98
|
+
const credentials = JSON.parse(data);
|
|
99
|
+
return credentials;
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
// If file is corrupted or unreadable, return null
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Save credentials to OS storage
|
|
108
|
+
*/
|
|
109
|
+
function saveCredentials(credentials) {
|
|
110
|
+
try {
|
|
111
|
+
ensureConfigDir();
|
|
112
|
+
const credentialsPath = getCredentialsPath();
|
|
113
|
+
// Read existing credentials and merge
|
|
114
|
+
const existing = loadCredentials();
|
|
115
|
+
const merged = {
|
|
116
|
+
...existing,
|
|
117
|
+
...credentials,
|
|
118
|
+
};
|
|
119
|
+
// Write to file
|
|
120
|
+
fs.writeFileSync(credentialsPath, JSON.stringify(merged, null, 2), 'utf-8');
|
|
121
|
+
// Set secure permissions
|
|
122
|
+
setSecurePermissions(credentialsPath);
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
throw new Error(`Failed to save credentials: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Delete stored credentials
|
|
130
|
+
*/
|
|
131
|
+
function deleteCredentials() {
|
|
132
|
+
try {
|
|
133
|
+
const credentialsPath = getCredentialsPath();
|
|
134
|
+
if (fs.existsSync(credentialsPath)) {
|
|
135
|
+
fs.unlinkSync(credentialsPath);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
throw new Error(`Failed to delete credentials: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Get the config directory path (for informational purposes)
|
|
144
|
+
*/
|
|
145
|
+
function getConfigDirectoryPath() {
|
|
146
|
+
return getConfigDir();
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=credentials.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DH,0CAcC;AAKD,0CAwBC;AAKD,8CASC;AAKD,wDAEC;AAzHD,uCAAyB;AACzB,2CAA6B;AAC7B,uCAAyB;AAOzB;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE/B,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,EAAE,QAAQ;YACrB,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,eAAe,CAAC,CAAC;QAC/E,KAAK,OAAO,EAAE,UAAU;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,mBAAmB;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,kBAAkB,CAAC,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe;IACtB,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY;IACzE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oDAAoD;IACtD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe;IAC7B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAsB,CAAC;QAC1D,OAAO,WAAW,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kDAAkD;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe,CAAC,WAA8B;IAC5D,IAAI,CAAC;QACH,eAAe,EAAE,CAAC;QAClB,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAE7C,sCAAsC;QACtC,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;QACnC,MAAM,MAAM,GAAsB;YAChC,GAAG,QAAQ;YACX,GAAG,WAAW;SACf,CAAC;QAEF,gBAAgB;QAChB,EAAE,CAAC,aAAa,CACd,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC/B,OAAO,CACR,CAAC;QAEF,yBAAyB;QACzB,oBAAoB,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC7G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB;IAC/B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACnC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,sBAAsB;IACpC,OAAO,YAAY,EAAE,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Option parsing utilities
|
|
3
|
+
*/
|
|
4
|
+
import type { ScrapeOptions } from '../types/scrape';
|
|
5
|
+
/**
|
|
6
|
+
* Convert commander options to ScrapeOptions
|
|
7
|
+
*/
|
|
8
|
+
export declare function parseScrapeOptions(options: any): ScrapeOptions;
|
|
9
|
+
//# sourceMappingURL=options.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,CAa9D"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Option parsing utilities
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.parseScrapeOptions = parseScrapeOptions;
|
|
7
|
+
/**
|
|
8
|
+
* Convert commander options to ScrapeOptions
|
|
9
|
+
*/
|
|
10
|
+
function parseScrapeOptions(options) {
|
|
11
|
+
return {
|
|
12
|
+
url: options.url,
|
|
13
|
+
format: options.format,
|
|
14
|
+
onlyMainContent: options.onlyMainContent,
|
|
15
|
+
waitFor: options.waitFor,
|
|
16
|
+
screenshot: options.screenshot,
|
|
17
|
+
includeTags: options.includeTags ? options.includeTags.split(',').map((t) => t.trim()) : undefined,
|
|
18
|
+
excludeTags: options.excludeTags ? options.excludeTags.split(',').map((t) => t.trim()) : undefined,
|
|
19
|
+
apiKey: options.apiKey,
|
|
20
|
+
output: options.output,
|
|
21
|
+
pretty: options.pretty,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=options.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAOH,gDAaC;AAhBD;;GAEG;AACH,SAAgB,kBAAkB,CAAC,OAAY;IAC7C,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1G,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1G,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Output utilities for CLI
|
|
3
|
+
*/
|
|
4
|
+
import type { ScrapeResult } from '../types/scrape';
|
|
5
|
+
import type { ScrapeFormat } from '../types/scrape';
|
|
6
|
+
/**
|
|
7
|
+
* Write output to file or stdout
|
|
8
|
+
*/
|
|
9
|
+
export declare function writeOutput(content: string, outputPath?: string, silent?: boolean): void;
|
|
10
|
+
/**
|
|
11
|
+
* Handle scrape result output
|
|
12
|
+
* For text formats (html, markdown, etc.), outputs raw content directly
|
|
13
|
+
* For complex formats, outputs JSON
|
|
14
|
+
*/
|
|
15
|
+
export declare function handleScrapeOutput(result: ScrapeResult, format?: ScrapeFormat, outputPath?: string, pretty?: boolean): void;
|
|
16
|
+
//# sourceMappingURL=output.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAiDpD;;GAEG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAmBN;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CA0CN"}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Output utilities for CLI
|
|
4
|
+
*/
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.writeOutput = writeOutput;
|
|
40
|
+
exports.handleScrapeOutput = handleScrapeOutput;
|
|
41
|
+
const fs = __importStar(require("fs"));
|
|
42
|
+
const path = __importStar(require("path"));
|
|
43
|
+
/**
|
|
44
|
+
* Extract content from Firecrawl Document based on format
|
|
45
|
+
*/
|
|
46
|
+
function extractContent(data, format) {
|
|
47
|
+
if (!data)
|
|
48
|
+
return null;
|
|
49
|
+
// If format is specified, try to extract that specific content
|
|
50
|
+
if (format) {
|
|
51
|
+
// Handle html/rawHtml formats - extract HTML content directly
|
|
52
|
+
if (format === 'html' || format === 'rawHtml') {
|
|
53
|
+
return data.html || data.rawHtml || data[format] || null;
|
|
54
|
+
}
|
|
55
|
+
// Handle markdown format
|
|
56
|
+
if (format === 'markdown') {
|
|
57
|
+
return data.markdown || data[format] || null;
|
|
58
|
+
}
|
|
59
|
+
// Handle links format
|
|
60
|
+
if (format === 'links') {
|
|
61
|
+
return data.links || data[format] || null;
|
|
62
|
+
}
|
|
63
|
+
// Handle images format
|
|
64
|
+
if (format === 'images') {
|
|
65
|
+
return data.images || data[format] || null;
|
|
66
|
+
}
|
|
67
|
+
// Handle summary format
|
|
68
|
+
if (format === 'summary') {
|
|
69
|
+
return data.summary || data[format] || null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// Fallback: try common content fields
|
|
73
|
+
if (typeof data === 'string') {
|
|
74
|
+
return data;
|
|
75
|
+
}
|
|
76
|
+
// If it's an object, try to find string content
|
|
77
|
+
if (typeof data === 'object') {
|
|
78
|
+
return data.html || data.markdown || data.rawHtml || data.content || null;
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Write output to file or stdout
|
|
84
|
+
*/
|
|
85
|
+
function writeOutput(content, outputPath, silent = false) {
|
|
86
|
+
if (outputPath) {
|
|
87
|
+
const dir = path.dirname(outputPath);
|
|
88
|
+
if (dir && !fs.existsSync(dir)) {
|
|
89
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
90
|
+
}
|
|
91
|
+
fs.writeFileSync(outputPath, content, 'utf-8');
|
|
92
|
+
if (!silent) {
|
|
93
|
+
// Always use stderr for file confirmation messages
|
|
94
|
+
console.error(`Output written to: ${outputPath}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
// Use process.stdout.write for raw output (like curl)
|
|
99
|
+
// Ensure content ends with newline for proper piping
|
|
100
|
+
if (!content.endsWith('\n')) {
|
|
101
|
+
content += '\n';
|
|
102
|
+
}
|
|
103
|
+
process.stdout.write(content);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Handle scrape result output
|
|
108
|
+
* For text formats (html, markdown, etc.), outputs raw content directly
|
|
109
|
+
* For complex formats, outputs JSON
|
|
110
|
+
*/
|
|
111
|
+
function handleScrapeOutput(result, format, outputPath, pretty = false) {
|
|
112
|
+
if (!result.success) {
|
|
113
|
+
// Always use stderr for errors to allow piping
|
|
114
|
+
console.error('Error:', result.error);
|
|
115
|
+
process.exit(1);
|
|
116
|
+
}
|
|
117
|
+
if (!result.data) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
// Text formats that should output raw content (curl-like)
|
|
121
|
+
const rawTextFormats = ['html', 'rawHtml', 'markdown', 'links', 'images', 'summary'];
|
|
122
|
+
const shouldOutputRaw = format && rawTextFormats.includes(format);
|
|
123
|
+
if (shouldOutputRaw) {
|
|
124
|
+
// Extract and output raw content
|
|
125
|
+
const content = extractContent(result.data, format);
|
|
126
|
+
if (content !== null) {
|
|
127
|
+
writeOutput(content, outputPath, !!outputPath);
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// For JSON format or complex formats (branding, json, etc.), output clean JSON
|
|
132
|
+
// Always stringify the entire data object to ensure valid JSON
|
|
133
|
+
let jsonContent;
|
|
134
|
+
try {
|
|
135
|
+
jsonContent = pretty
|
|
136
|
+
? JSON.stringify(result.data, null, 2)
|
|
137
|
+
: JSON.stringify(result.data);
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
// If stringification fails, try to create a minimal error response
|
|
141
|
+
jsonContent = JSON.stringify({
|
|
142
|
+
error: 'Failed to serialize response',
|
|
143
|
+
message: error instanceof Error ? error.message : 'Unknown error'
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
// Ensure clean JSON output (no extra newlines or text before JSON)
|
|
147
|
+
// Output directly to stdout without any prefix
|
|
148
|
+
writeOutput(jsonContent, outputPath, !!outputPath);
|
|
149
|
+
}
|
|
150
|
+
//# sourceMappingURL=output.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDH,kCAuBC;AAOD,gDA+CC;AApID,uCAAyB;AACzB,2CAA6B;AAI7B;;GAEG;AACH,SAAS,cAAc,CAAC,IAAS,EAAE,MAAqB;IACtD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,+DAA+D;IAC/D,IAAI,MAAM,EAAE,CAAC;QACX,8DAA8D;QAC9D,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC3D,CAAC;QAED,yBAAyB;QACzB,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC/C,CAAC;QAED,sBAAsB;QACtB,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC5C,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC7C,CAAC;QAED,wBAAwB;QACxB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;IAC5E,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW,CACzB,OAAe,EACf,UAAmB,EACnB,SAAkB,KAAK;IAEvB,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,mDAAmD;YACnD,OAAO,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sDAAsD;QACtD,qDAAqD;QACrD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAgB,kBAAkB,CAChC,MAAoB,EACpB,MAAqB,EACrB,UAAmB,EACnB,SAAkB,KAAK;IAEvB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,+CAA+C;QAC/C,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,0DAA0D;IAC1D,MAAM,cAAc,GAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IACrG,MAAM,eAAe,GAAG,MAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAElE,IAAI,eAAe,EAAE,CAAC;QACpB,iCAAiC;QACjC,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,+DAA+D;IAC/D,IAAI,WAAmB,CAAC;IACxB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM;YAClB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mEAAmE;QACnE,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;YAC3B,KAAK,EAAE,8BAA8B;YACrC,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAClE,CAAC,CAAC;IACL,CAAC;IAED,mEAAmE;IACnE,+CAA+C;IAC/C,WAAW,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;AACrD,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL utility functions
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Check if a string looks like a URL (with or without protocol)
|
|
6
|
+
*/
|
|
7
|
+
export declare function isUrl(str: string): boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Normalize URL by adding https:// if missing
|
|
10
|
+
*/
|
|
11
|
+
export declare function normalizeUrl(url: string): string;
|
|
12
|
+
//# sourceMappingURL=url.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,wBAAgB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAoB1C;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKhD"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* URL utility functions
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.isUrl = isUrl;
|
|
7
|
+
exports.normalizeUrl = normalizeUrl;
|
|
8
|
+
/**
|
|
9
|
+
* Check if a string looks like a URL (with or without protocol)
|
|
10
|
+
*/
|
|
11
|
+
function isUrl(str) {
|
|
12
|
+
// If it has a protocol, validate it
|
|
13
|
+
if (/^https?:\/\//i.test(str)) {
|
|
14
|
+
try {
|
|
15
|
+
const url = new URL(str);
|
|
16
|
+
return url.protocol === 'http:' || url.protocol === 'https:';
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return true; // Assume valid if it starts with http:// or https://
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
// Check if it looks like a domain (has dots and valid characters)
|
|
23
|
+
// Exclude common commands and flags
|
|
24
|
+
if (str.includes('.') && !str.startsWith('-') && !str.includes(' ')) {
|
|
25
|
+
// Basic domain validation: contains at least one dot and valid characters
|
|
26
|
+
const domainPattern = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/;
|
|
27
|
+
return domainPattern.test(str);
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Normalize URL by adding https:// if missing
|
|
33
|
+
*/
|
|
34
|
+
function normalizeUrl(url) {
|
|
35
|
+
if (/^https?:\/\//i.test(url)) {
|
|
36
|
+
return url;
|
|
37
|
+
}
|
|
38
|
+
return `https://${url}`;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=url.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAKH,sBAoBC;AAKD,oCAKC;AAjCD;;GAEG;AACH,SAAgB,KAAK,CAAC,GAAW;IAC/B,oCAAoC;IACpC,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,CAAC,qDAAqD;QACpE,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,oCAAoC;IACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpE,0EAA0E;QAC1E,MAAM,aAAa,GAAG,oHAAoH,CAAC;QAC3I,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,GAAW;IACtC,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,WAAW,GAAG,EAAE,CAAC;AAC1B,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "firecrawl-cli",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"bin": {
|
|
7
|
+
"firecrawl": "dist/index.js"
|
|
8
|
+
},
|
|
9
|
+
"scripts": {
|
|
10
|
+
"build": "tsc",
|
|
11
|
+
"dev": "tsc --watch",
|
|
12
|
+
"start": "node dist/index.js",
|
|
13
|
+
"clean": "rm -rf dist",
|
|
14
|
+
"prepublishOnly": "pnpm run build",
|
|
15
|
+
"test": "echo \"Error: no test specified\" && exit 1"
|
|
16
|
+
},
|
|
17
|
+
"keywords": [
|
|
18
|
+
"firecrawl",
|
|
19
|
+
"cli",
|
|
20
|
+
"web-scraping",
|
|
21
|
+
"crawler",
|
|
22
|
+
"scraper",
|
|
23
|
+
"data-extraction",
|
|
24
|
+
"llm",
|
|
25
|
+
"markdown"
|
|
26
|
+
],
|
|
27
|
+
"author": "Firecrawl",
|
|
28
|
+
"license": "ISC",
|
|
29
|
+
"repository": {
|
|
30
|
+
"type": "git",
|
|
31
|
+
"url": "git+https://github.com/firecrawl/firecrawl-cli.git"
|
|
32
|
+
},
|
|
33
|
+
"bugs": {
|
|
34
|
+
"url": "https://github.com/firecrawl/firecrawl-cli/issues"
|
|
35
|
+
},
|
|
36
|
+
"homepage": "https://docs.firecrawl.dev/cli",
|
|
37
|
+
"engines": {
|
|
38
|
+
"node": ">=18.0.0"
|
|
39
|
+
},
|
|
40
|
+
"files": [
|
|
41
|
+
"dist",
|
|
42
|
+
"README.md"
|
|
43
|
+
],
|
|
44
|
+
"packageManager": "pnpm@10.12.1",
|
|
45
|
+
"devDependencies": {
|
|
46
|
+
"@types/node": "^20.0.0",
|
|
47
|
+
"typescript": "^5.0.0"
|
|
48
|
+
},
|
|
49
|
+
"dependencies": {
|
|
50
|
+
"@mendable/firecrawl-js": "^4.10.0",
|
|
51
|
+
"commander": "^14.0.2"
|
|
52
|
+
}
|
|
53
|
+
}
|