firecrawl-cli 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +27 -0
  2. package/dist/commands/config.d.ts +10 -0
  3. package/dist/commands/config.d.ts.map +1 -0
  4. package/dist/commands/config.js +102 -0
  5. package/dist/commands/config.js.map +1 -0
  6. package/dist/commands/scrape.d.ts +13 -0
  7. package/dist/commands/scrape.d.ts.map +1 -0
  8. package/dist/commands/scrape.js +73 -0
  9. package/dist/commands/scrape.js.map +1 -0
  10. package/dist/index.d.ts +7 -0
  11. package/dist/index.d.ts.map +1 -0
  12. package/dist/index.js +89 -0
  13. package/dist/index.js.map +1 -0
  14. package/dist/types/scrape.d.ts +32 -0
  15. package/dist/types/scrape.d.ts.map +1 -0
  16. package/dist/types/scrape.js +6 -0
  17. package/dist/types/scrape.js.map +1 -0
  18. package/dist/utils/client.d.ts +22 -0
  19. package/dist/utils/client.d.ts.map +1 -0
  20. package/dist/utils/client.js +74 -0
  21. package/dist/utils/client.js.map +1 -0
  22. package/dist/utils/config.d.ts +38 -0
  23. package/dist/utils/config.d.ts.map +1 -0
  24. package/dist/utils/config.js +78 -0
  25. package/dist/utils/config.js.map +1 -0
  26. package/dist/utils/credentials.d.ts +25 -0
  27. package/dist/utils/credentials.d.ts.map +1 -0
  28. package/dist/utils/credentials.js +148 -0
  29. package/dist/utils/credentials.js.map +1 -0
  30. package/dist/utils/options.d.ts +9 -0
  31. package/dist/utils/options.d.ts.map +1 -0
  32. package/dist/utils/options.js +24 -0
  33. package/dist/utils/options.js.map +1 -0
  34. package/dist/utils/output.d.ts +16 -0
  35. package/dist/utils/output.d.ts.map +1 -0
  36. package/dist/utils/output.js +150 -0
  37. package/dist/utils/output.js.map +1 -0
  38. package/dist/utils/url.d.ts +12 -0
  39. package/dist/utils/url.d.ts.map +1 -0
  40. package/dist/utils/url.js +40 -0
  41. package/dist/utils/url.js.map +1 -0
  42. package/package.json +53 -0
package/README.md ADDED
@@ -0,0 +1,27 @@
1
+ # 🔥 Firecrawl CLI
2
+
3
+ Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.
4
+
5
+ ## Installation
6
+
7
+ Install Firecrawl CLI globally using npm:
8
+
9
+ ```bash
10
+ npm install -g firecrawl-cli
11
+ ```
12
+
13
+ ## Quick Start
14
+
15
+ Set your API key:
16
+
17
+ ```bash
18
+ firecrawl config
19
+ ```
20
+
21
+ Scrape a URL:
22
+
23
+ ```bash
24
+ firecrawl https://firecrawl.dev
25
+ ```
26
+
27
+ For detailed usage instructions, examples, and all available commands, visit the [CLI documentation](https://docs.firecrawl.dev/cli).
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Config command implementation
3
+ * Manages stored credentials and configuration
4
+ */
5
+ /**
6
+ * Interactive configuration setup
7
+ * Asks for API URL and API key
8
+ */
9
+ export declare function configure(): Promise<void>;
10
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/commands/config.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2BH;;;GAGG;AACH,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CA2C/C"}
@@ -0,0 +1,102 @@
1
+ "use strict";
2
+ /**
3
+ * Config command implementation
4
+ * Manages stored credentials and configuration
5
+ */
6
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
7
+ if (k2 === undefined) k2 = k;
8
+ var desc = Object.getOwnPropertyDescriptor(m, k);
9
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
10
+ desc = { enumerable: true, get: function() { return m[k]; } };
11
+ }
12
+ Object.defineProperty(o, k2, desc);
13
+ }) : (function(o, m, k, k2) {
14
+ if (k2 === undefined) k2 = k;
15
+ o[k2] = m[k];
16
+ }));
17
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
18
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
19
+ }) : function(o, v) {
20
+ o["default"] = v;
21
+ });
22
+ var __importStar = (this && this.__importStar) || (function () {
23
+ var ownKeys = function(o) {
24
+ ownKeys = Object.getOwnPropertyNames || function (o) {
25
+ var ar = [];
26
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
27
+ return ar;
28
+ };
29
+ return ownKeys(o);
30
+ };
31
+ return function (mod) {
32
+ if (mod && mod.__esModule) return mod;
33
+ var result = {};
34
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
35
+ __setModuleDefault(result, mod);
36
+ return result;
37
+ };
38
+ })();
39
+ Object.defineProperty(exports, "__esModule", { value: true });
40
+ exports.configure = configure;
41
+ const readline = __importStar(require("readline"));
42
+ const credentials_1 = require("../utils/credentials");
43
+ const config_1 = require("../utils/config");
44
+ const DEFAULT_API_URL = 'https://api.firecrawl.dev';
45
+ /**
46
+ * Prompt for input (for secure API key entry)
47
+ */
48
+ function promptInput(question, defaultValue) {
49
+ const rl = readline.createInterface({
50
+ input: process.stdin,
51
+ output: process.stdout,
52
+ });
53
+ const promptText = defaultValue ? `${question} [${defaultValue}]: ` : `${question} `;
54
+ return new Promise((resolve) => {
55
+ rl.question(promptText, (answer) => {
56
+ rl.close();
57
+ resolve(answer.trim() || defaultValue || '');
58
+ });
59
+ });
60
+ }
61
+ /**
62
+ * Interactive configuration setup
63
+ * Asks for API URL and API key
64
+ */
65
+ async function configure() {
66
+ console.log('Firecrawl Configuration Setup\n');
67
+ // Prompt for API URL with default
68
+ let url = await promptInput('Enter API URL', DEFAULT_API_URL);
69
+ // Ensure URL doesn't end with trailing slash
70
+ url = url.replace(/\/$/, '');
71
+ // Prompt for API key
72
+ const key = await promptInput('Enter your Firecrawl API key: ');
73
+ if (!key || key.trim().length === 0) {
74
+ console.error('Error: API key cannot be empty');
75
+ process.exit(1);
76
+ }
77
+ if (!url || url.trim().length === 0) {
78
+ console.error('Error: API URL cannot be empty');
79
+ process.exit(1);
80
+ }
81
+ // Normalize URL (remove trailing slash)
82
+ const normalizedUrl = url.trim().replace(/\/$/, '');
83
+ try {
84
+ (0, credentials_1.saveCredentials)({
85
+ apiKey: key.trim(),
86
+ apiUrl: normalizedUrl
87
+ });
88
+ console.log('\n✓ Configuration saved successfully');
89
+ console.log(` API URL: ${normalizedUrl}`);
90
+ console.log(` Stored in: ${(0, credentials_1.getConfigDirectoryPath)()}`);
91
+ // Update global config
92
+ (0, config_1.updateConfig)({
93
+ apiKey: key.trim(),
94
+ apiUrl: normalizedUrl
95
+ });
96
+ }
97
+ catch (error) {
98
+ console.error('Error saving configuration:', error instanceof Error ? error.message : 'Unknown error');
99
+ process.exit(1);
100
+ }
101
+ }
102
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/commands/config.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BH,8BA2CC;AAxED,mDAAqC;AACrC,sDAA+E;AAC/E,4CAA+C;AAE/C,MAAM,eAAe,GAAG,2BAA2B,CAAC;AAEpD;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB,EAAE,YAAqB;IAC1D,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,QAAQ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,CAAC;IAErF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE;YACjC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,YAAY,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,SAAS;IAC7B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAE/C,kCAAkC;IAClC,IAAI,GAAG,GAAG,MAAM,WAAW,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAE9D,6CAA6C;IAC7C,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAE7B,qBAAqB;IACrB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,gCAAgC,CAAC,CAAC;IAEhE,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,wCAAwC;IACxC,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,IAAA,6BAAe,EAAC;YACd,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE;YAClB,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,cAAc,aAAa,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAA,oCAAsB,GAAE,EAAE,CAAC,CAAC;QAExD,uBAAuB;QACvB,IAAA,qBAAY,EAAC;YACX,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE;YAClB,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Scrape command implementation
3
+ */
4
+ import type { ScrapeOptions, ScrapeResult } from '../types/scrape';
5
+ /**
6
+ * Execute the scrape command
7
+ */
8
+ export declare function executeScrape(options: ScrapeOptions): Promise<ScrapeResult>;
9
+ /**
10
+ * Handle scrape command output
11
+ */
12
+ export declare function handleScrapeCommand(options: ScrapeOptions): Promise<void>;
13
+ //# sourceMappingURL=scrape.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAKnE;;GAEG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAoEjF;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/E"}
@@ -0,0 +1,73 @@
1
+ "use strict";
2
+ /**
3
+ * Scrape command implementation
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.executeScrape = executeScrape;
7
+ exports.handleScrapeCommand = handleScrapeCommand;
8
+ const client_1 = require("../utils/client");
9
+ const config_1 = require("../utils/config");
10
+ const output_1 = require("../utils/output");
11
+ /**
12
+ * Execute the scrape command
13
+ */
14
+ async function executeScrape(options) {
15
+ try {
16
+ // Update global config if API key is provided via options
17
+ if (options.apiKey) {
18
+ (0, config_1.updateConfig)({ apiKey: options.apiKey });
19
+ }
20
+ // Get client instance (uses global config)
21
+ const app = (0, client_1.getClient)();
22
+ // Build scrape options
23
+ const formats = [];
24
+ if (options.format) {
25
+ formats.push(options.format);
26
+ }
27
+ if (options.screenshot) {
28
+ // Add screenshot format if not already included
29
+ if (!formats.includes('screenshot')) {
30
+ formats.push('screenshot');
31
+ }
32
+ }
33
+ // If no formats specified, default to markdown
34
+ if (formats.length === 0) {
35
+ formats.push('markdown');
36
+ }
37
+ const scrapeParams = {
38
+ formats,
39
+ };
40
+ if (options.onlyMainContent !== undefined) {
41
+ scrapeParams.onlyMainContent = options.onlyMainContent;
42
+ }
43
+ if (options.waitFor !== undefined) {
44
+ scrapeParams.waitFor = options.waitFor;
45
+ }
46
+ if (options.includeTags && options.includeTags.length > 0) {
47
+ scrapeParams.includeTags = options.includeTags;
48
+ }
49
+ if (options.excludeTags && options.excludeTags.length > 0) {
50
+ scrapeParams.excludeTags = options.excludeTags;
51
+ }
52
+ // Execute scrape
53
+ const result = await app.scrape(options.url, scrapeParams);
54
+ return {
55
+ success: true,
56
+ data: result,
57
+ };
58
+ }
59
+ catch (error) {
60
+ return {
61
+ success: false,
62
+ error: error instanceof Error ? error.message : 'Unknown error occurred',
63
+ };
64
+ }
65
+ }
66
+ /**
67
+ * Handle scrape command output
68
+ */
69
+ async function handleScrapeCommand(options) {
70
+ const result = await executeScrape(options);
71
+ (0, output_1.handleScrapeOutput)(result, options.format, options.output, options.pretty);
72
+ }
73
+ //# sourceMappingURL=scrape.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAWH,sCAoEC;AAKD,kDAGC;AAnFD,4CAA4C;AAC5C,4CAA+C;AAC/C,4CAAqD;AAErD;;GAEG;AACI,KAAK,UAAU,aAAa,CAAC,OAAsB;IACxD,IAAI,CAAC;QACH,0DAA0D;QAC1D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,2CAA2C;QAC3C,MAAM,GAAG,GAAG,IAAA,kBAAS,GAAE,CAAC;QAExB,uBAAuB;QACvB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,gDAAgD;YAChD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,YAAY,GAMd;YACF,OAAO;SACR,CAAC;QAEF,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YAC1C,YAAY,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QACzD,CAAC;QAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAClC,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACjD,CAAC;QAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACjD,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAE3D,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,MAAM;SACb,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CAAC,OAAsB;IAC9D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAA,2BAAkB,EAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC7E,CAAC"}
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Firecrawl CLI
4
+ * Entry point for the CLI application
5
+ */
6
+ export {};
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;GAGG"}
package/dist/index.js ADDED
@@ -0,0 +1,89 @@
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+ /**
4
+ * Firecrawl CLI
5
+ * Entry point for the CLI application
6
+ */
7
+ Object.defineProperty(exports, "__esModule", { value: true });
8
+ const commander_1 = require("commander");
9
+ const scrape_1 = require("./commands/scrape");
10
+ const config_1 = require("./utils/config");
11
+ const config_2 = require("./commands/config");
12
+ const url_1 = require("./utils/url");
13
+ const options_1 = require("./utils/options");
14
+ // Initialize global configuration from environment variables
15
+ (0, config_1.initializeConfig)();
16
+ const program = new commander_1.Command();
17
+ program
18
+ .name('firecrawl')
19
+ .description('CLI tool for Firecrawl web scraping')
20
+ .version('1.0.0')
21
+ .option('-k, --api-key <key>', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var, or use "firecrawl config")')
22
+ .allowUnknownOption() // Allow unknown options when URL is passed directly
23
+ .hook('preAction', (thisCommand, actionCommand) => {
24
+ // Update global config if API key is provided via global option
25
+ const globalOptions = thisCommand.opts();
26
+ if (globalOptions.apiKey) {
27
+ (0, config_1.updateConfig)({ apiKey: globalOptions.apiKey });
28
+ }
29
+ });
30
+ /**
31
+ * Create and configure the scrape command
32
+ */
33
+ function createScrapeCommand() {
34
+ const scrapeCmd = new commander_1.Command('scrape')
35
+ .description('Scrape a URL using Firecrawl')
36
+ .argument('[url]', 'URL to scrape')
37
+ .option('-u, --url <url>', 'URL to scrape (alternative to positional argument)')
38
+ .option('-H, --html', 'Output raw HTML (shortcut for --format html)')
39
+ .option('-f, --format <format>', 'Output format: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding', 'markdown')
40
+ .option('--only-main-content', 'Include only main content', false)
41
+ .option('--wait-for <ms>', 'Wait time before scraping in milliseconds', parseInt)
42
+ .option('--screenshot', 'Take a screenshot', false)
43
+ .option('--include-tags <tags>', 'Comma-separated list of tags to include')
44
+ .option('--exclude-tags <tags>', 'Comma-separated list of tags to exclude')
45
+ .option('-k, --api-key <key>', 'Firecrawl API key (overrides global --api-key)')
46
+ .option('-o, --output <path>', 'Output file path (default: stdout)')
47
+ .option('--pretty', 'Pretty print JSON output', false)
48
+ .action(async (positionalUrl, options) => {
49
+ // Use positional URL if provided, otherwise use --url option
50
+ const url = positionalUrl || options.url;
51
+ if (!url) {
52
+ console.error('Error: URL is required. Provide it as argument or use --url option.');
53
+ process.exit(1);
54
+ }
55
+ // Handle --html shortcut flag
56
+ const format = options.html ? 'html' : options.format;
57
+ const scrapeOptions = (0, options_1.parseScrapeOptions)({ ...options, url, format });
58
+ await (0, scrape_1.handleScrapeCommand)(scrapeOptions);
59
+ });
60
+ return scrapeCmd;
61
+ }
62
+ // Add scrape command to main program
63
+ program.addCommand(createScrapeCommand());
64
+ program
65
+ .command('config')
66
+ .description('Configure API URL and API key (interactive)')
67
+ .action(async () => {
68
+ await (0, config_2.configure)();
69
+ });
70
+ // Parse arguments
71
+ const args = process.argv.slice(2);
72
+ // Check if first argument is a URL (and not a command)
73
+ if (args.length > 0 && !args[0].startsWith('-') && (0, url_1.isUrl)(args[0])) {
74
+ // Treat as scrape command with URL - reuse commander's parsing
75
+ const url = (0, url_1.normalizeUrl)(args[0]);
76
+ // Modify argv to include scrape command with URL as positional argument
77
+ // This allows commander to parse it normally with all hooks and options
78
+ const modifiedArgv = [process.argv[0], process.argv[1], 'scrape', url, ...args.slice(1)];
79
+ // Parse using the main program (which includes hooks and global options)
80
+ program.parseAsync(modifiedArgv).catch((error) => {
81
+ console.error('Error:', error instanceof Error ? error.message : 'Unknown error');
82
+ process.exit(1);
83
+ });
84
+ }
85
+ else {
86
+ // Normal command parsing
87
+ program.parse();
88
+ }
89
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA;;;GAGG;;AAEH,yCAAoC;AACpC,8CAAwD;AACxD,2CAAgE;AAChE,8CAA8C;AAC9C,qCAAkD;AAClD,6CAAqD;AAErD,6DAA6D;AAC7D,IAAA,yBAAgB,GAAE,CAAC;AAEnB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,qCAAqC,CAAC;KAClD,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,CAAC,qBAAqB,EAAE,iFAAiF,CAAC;KAChH,kBAAkB,EAAE,CAAC,oDAAoD;KACzE,IAAI,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE;IAChD,gEAAgE;IAChE,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,IAAA,qBAAY,EAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,SAAS,mBAAmB;IAC1B,MAAM,SAAS,GAAG,IAAI,mBAAO,CAAC,QAAQ,CAAC;SACpC,WAAW,CAAC,8BAA8B,CAAC;SAC3C,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;SAClC,MAAM,CAAC,iBAAiB,EAAE,oDAAoD,CAAC;SAC/E,MAAM,CAAC,YAAY,EAAE,8CAA8C,CAAC;SACpE,MAAM,CAAC,uBAAuB,EAAE,wHAAwH,EAAE,UAAU,CAAC;SACrK,MAAM,CAAC,qBAAqB,EAAE,2BAA2B,EAAE,KAAK,CAAC;SACjE,MAAM,CAAC,iBAAiB,EAAE,2CAA2C,EAAE,QAAQ,CAAC;SAChF,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,KAAK,CAAC;SAClD,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CAAC,qBAAqB,EAAE,gDAAgD,CAAC;SAC/E,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE;QACvC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;QACzC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;YACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,8BAA8B;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtD,MAAM,aAAa,GAAG,IAAA,4BAAkB,EAAC,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QACtE,MAAM,IAAA,4BAAmB,EAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEL,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qCAAqC;AACrC,OAAO,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAE1C,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,IAAA,kBAAS,GAAE,CAAC;AACpB,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,uDAAuD;AACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAA,WAAK,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,+DAA+D;IAC/D,MAAM,GAAG,GAAG,IAAA,kBAAY,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzF,yEAAyE;IACzE,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/C,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;KAAM,CAAC;IACN,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC"}
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Types and interfaces for the scrape command
3
+ */
4
+ export type ScrapeFormat = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes' | 'branding';
5
+ export interface ScrapeOptions {
6
+ /** URL to scrape */
7
+ url: string;
8
+ /** Output format (markdown, html, etc.) */
9
+ format?: ScrapeFormat;
10
+ /** Include only main content */
11
+ onlyMainContent?: boolean;
12
+ /** Wait time before scraping (ms) */
13
+ waitFor?: number;
14
+ /** Take screenshot */
15
+ screenshot?: boolean;
16
+ /** Include tags */
17
+ includeTags?: string[];
18
+ /** Exclude tags */
19
+ excludeTags?: string[];
20
+ /** API key for Firecrawl */
21
+ apiKey?: string;
22
+ /** Output file path */
23
+ output?: string;
24
+ /** Pretty print JSON output */
25
+ pretty?: boolean;
26
+ }
27
+ export interface ScrapeResult {
28
+ success: boolean;
29
+ data?: any;
30
+ error?: string;
31
+ }
32
+ //# sourceMappingURL=scrape.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,MAAM,GACN,SAAS,GACT,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,SAAS,GACT,gBAAgB,GAChB,MAAM,GACN,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,gCAAgC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ /**
3
+ * Types and interfaces for the scrape command
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ //# sourceMappingURL=scrape.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG"}
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Firecrawl client utility
3
+ * Provides a singleton client instance initialized with global configuration
4
+ */
5
+ import Firecrawl from '@mendable/firecrawl-js';
6
+ import type { FirecrawlClientOptions } from '@mendable/firecrawl-js';
7
+ import { type GlobalConfig } from './config';
8
+ /**
9
+ * Get or create the Firecrawl client instance
10
+ * Uses global configuration if available, otherwise creates with provided options
11
+ */
12
+ export declare function getClient(options?: Partial<FirecrawlClientOptions>): Firecrawl;
13
+ /**
14
+ * Initialize the client with configuration
15
+ * This should be called early in the application lifecycle
16
+ */
17
+ export declare function initializeClient(config?: Partial<GlobalConfig>): Firecrawl;
18
+ /**
19
+ * Reset the client instance (useful for testing)
20
+ */
21
+ export declare function resetClient(): void;
22
+ //# sourceMappingURL=client.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,SAAS,MAAM,wBAAwB,CAAC;AAC/C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAA6B,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAIxE;;;GAGG;AACH,wBAAgB,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,SAAS,CA2C9E;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,SAAS,CAS1E;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
@@ -0,0 +1,74 @@
1
+ "use strict";
2
+ /**
3
+ * Firecrawl client utility
4
+ * Provides a singleton client instance initialized with global configuration
5
+ */
6
+ var __importDefault = (this && this.__importDefault) || function (mod) {
7
+ return (mod && mod.__esModule) ? mod : { "default": mod };
8
+ };
9
+ Object.defineProperty(exports, "__esModule", { value: true });
10
+ exports.getClient = getClient;
11
+ exports.initializeClient = initializeClient;
12
+ exports.resetClient = resetClient;
13
+ const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
14
+ const config_1 = require("./config");
15
+ let clientInstance = null;
16
+ /**
17
+ * Get or create the Firecrawl client instance
18
+ * Uses global configuration if available, otherwise creates with provided options
19
+ */
20
+ function getClient(options) {
21
+ // Helper to convert null to undefined and ensure we have a string or undefined
22
+ const normalizeApiKey = (value) => value === null || value === undefined ? undefined : value;
23
+ // If options provided, create a new instance (useful for command-specific overrides)
24
+ if (options) {
25
+ const config = (0, config_1.getConfig)();
26
+ const apiKey = normalizeApiKey(options.apiKey) ?? config.apiKey;
27
+ const apiUrl = normalizeApiKey(options.apiUrl) ?? config.apiUrl;
28
+ // Normalize apiKey for validation (convert null to undefined)
29
+ const normalizedApiKey = apiKey === null ? undefined : apiKey;
30
+ (0, config_1.validateConfig)(normalizedApiKey);
31
+ const clientOptions = {
32
+ apiKey: normalizedApiKey || undefined,
33
+ apiUrl: apiUrl === null ? undefined : apiUrl,
34
+ timeoutMs: options.timeoutMs ?? config.timeoutMs,
35
+ maxRetries: options.maxRetries ?? config.maxRetries,
36
+ backoffFactor: options.backoffFactor ?? config.backoffFactor,
37
+ };
38
+ return new firecrawl_js_1.default(clientOptions);
39
+ }
40
+ // Return singleton instance or create one
41
+ if (!clientInstance) {
42
+ const config = (0, config_1.getConfig)();
43
+ (0, config_1.validateConfig)(config.apiKey);
44
+ const clientOptions = {
45
+ apiKey: config.apiKey || undefined,
46
+ apiUrl: config.apiUrl || undefined,
47
+ timeoutMs: config.timeoutMs,
48
+ maxRetries: config.maxRetries,
49
+ backoffFactor: config.backoffFactor,
50
+ };
51
+ clientInstance = new firecrawl_js_1.default(clientOptions);
52
+ }
53
+ return clientInstance;
54
+ }
55
+ /**
56
+ * Initialize the client with configuration
57
+ * This should be called early in the application lifecycle
58
+ */
59
+ function initializeClient(config) {
60
+ if (config) {
61
+ const { initializeConfig } = require('./config');
62
+ initializeConfig(config);
63
+ }
64
+ // Reset instance to force recreation with new config
65
+ clientInstance = null;
66
+ return getClient();
67
+ }
68
+ /**
69
+ * Reset the client instance (useful for testing)
70
+ */
71
+ function resetClient() {
72
+ clientInstance = null;
73
+ }
74
+ //# sourceMappingURL=client.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/utils/client.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;AAYH,8BA2CC;AAMD,4CASC;AAKD,kCAEC;AA3ED,0EAA+C;AAE/C,qCAAwE;AAExE,IAAI,cAAc,GAAqB,IAAI,CAAC;AAE5C;;;GAGG;AACH,SAAgB,SAAS,CAAC,OAAyC;IACjE,+EAA+E;IAC/E,MAAM,eAAe,GAAG,CAAC,KAAgC,EAAsB,EAAE,CAC/E,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IAE5D,qFAAqF;IACrF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAChE,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;QAEhE,8DAA8D;QAC9D,MAAM,gBAAgB,GAAG,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9D,IAAA,uBAAc,EAAC,gBAAgB,CAAC,CAAC;QAEjC,MAAM,aAAa,GAA2B;YAC5C,MAAM,EAAE,gBAAgB,IAAI,SAAS;YACrC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAC5C,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;YAChD,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU;YACnD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC,aAAa;SAC7D,CAAC;QAEF,OAAO,IAAI,sBAAS,CAAC,aAAa,CAAC,CAAC;IACtC,CAAC;IAED,0CAA0C;IAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,IAAA,uBAAc,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE9B,MAAM,aAAa,GAA2B;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC;QAEF,cAAc,GAAG,IAAI,sBAAS,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,MAA8B;IAC7D,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjD,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,qDAAqD;IACrD,cAAc,GAAG,IAAI,CAAC;IACtB,OAAO,SAAS,EAAE,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,cAAc,GAAG,IAAI,CAAC;AACxB,CAAC"}
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Global configuration system
3
+ */
4
+ export interface GlobalConfig {
5
+ apiKey?: string;
6
+ apiUrl?: string;
7
+ timeoutMs?: number;
8
+ maxRetries?: number;
9
+ backoffFactor?: number;
10
+ }
11
+ /**
12
+ * Initialize global configuration
13
+ * Loads from: provided config > environment variables > OS credential storage
14
+ * @param config Configuration options
15
+ */
16
+ export declare function initializeConfig(config?: Partial<GlobalConfig>): void;
17
+ /**
18
+ * Get the current global configuration
19
+ */
20
+ export declare function getConfig(): GlobalConfig;
21
+ /**
22
+ * Update global configuration (merges with existing)
23
+ */
24
+ export declare function updateConfig(config: Partial<GlobalConfig>): void;
25
+ /**
26
+ * Get API key from global config or provided value
27
+ * Priority: provided key > global config > env var > stored credentials
28
+ */
29
+ export declare function getApiKey(providedKey?: string): string | undefined;
30
+ /**
31
+ * Validate that required configuration is present
32
+ */
33
+ export declare function validateConfig(apiKey?: string): void;
34
+ /**
35
+ * Reset global configuration (useful for testing)
36
+ */
37
+ export declare function resetConfig(): void;
38
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,GAAE,OAAO,CAAC,YAAY,CAAM,GAAG,IAAI,CAWzE;AAED;;GAEG;AACH,wBAAgB,SAAS,IAAI,YAAY,CAExC;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAKhE;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAQlE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAOpD;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
@@ -0,0 +1,78 @@
1
+ "use strict";
2
+ /**
3
+ * Global configuration system
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.initializeConfig = initializeConfig;
7
+ exports.getConfig = getConfig;
8
+ exports.updateConfig = updateConfig;
9
+ exports.getApiKey = getApiKey;
10
+ exports.validateConfig = validateConfig;
11
+ exports.resetConfig = resetConfig;
12
+ const credentials_1 = require("./credentials");
13
+ /**
14
+ * Global configuration instance
15
+ */
16
+ let globalConfig = {};
17
+ /**
18
+ * Initialize global configuration
19
+ * Loads from: provided config > environment variables > OS credential storage
20
+ * @param config Configuration options
21
+ */
22
+ function initializeConfig(config = {}) {
23
+ // Priority: provided config > env vars > stored credentials
24
+ const storedCredentials = (0, credentials_1.loadCredentials)();
25
+ globalConfig = {
26
+ apiKey: config.apiKey || process.env.FIRECRAWL_API_KEY || storedCredentials?.apiKey,
27
+ apiUrl: config.apiUrl || process.env.FIRECRAWL_API_URL || storedCredentials?.apiUrl,
28
+ timeoutMs: config.timeoutMs,
29
+ maxRetries: config.maxRetries,
30
+ backoffFactor: config.backoffFactor,
31
+ };
32
+ }
33
+ /**
34
+ * Get the current global configuration
35
+ */
36
+ function getConfig() {
37
+ return { ...globalConfig };
38
+ }
39
+ /**
40
+ * Update global configuration (merges with existing)
41
+ */
42
+ function updateConfig(config) {
43
+ globalConfig = {
44
+ ...globalConfig,
45
+ ...config,
46
+ };
47
+ }
48
+ /**
49
+ * Get API key from global config or provided value
50
+ * Priority: provided key > global config > env var > stored credentials
51
+ */
52
+ function getApiKey(providedKey) {
53
+ if (providedKey)
54
+ return providedKey;
55
+ if (globalConfig.apiKey)
56
+ return globalConfig.apiKey;
57
+ if (process.env.FIRECRAWL_API_KEY)
58
+ return process.env.FIRECRAWL_API_KEY;
59
+ // Fallback to stored credentials if not already loaded
60
+ const storedCredentials = (0, credentials_1.loadCredentials)();
61
+ return storedCredentials?.apiKey;
62
+ }
63
+ /**
64
+ * Validate that required configuration is present
65
+ */
66
+ function validateConfig(apiKey) {
67
+ const key = getApiKey(apiKey);
68
+ if (!key) {
69
+ throw new Error('API key is required. Set FIRECRAWL_API_KEY environment variable, use --api-key flag, run "firecrawl config set-api-key", or initialize config.');
70
+ }
71
+ }
72
+ /**
73
+ * Reset global configuration (useful for testing)
74
+ */
75
+ function resetConfig() {
76
+ globalConfig = {};
77
+ }
78
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/utils/config.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAsBH,4CAWC;AAKD,8BAEC;AAKD,oCAKC;AAMD,8BAQC;AAKD,wCAOC;AAKD,kCAEC;AAjFD,+CAAgD;AAUhD;;GAEG;AACH,IAAI,YAAY,GAAiB,EAAE,CAAC;AAEpC;;;;GAIG;AACH,SAAgB,gBAAgB,CAAC,SAAgC,EAAE;IACjE,4DAA4D;IAC5D,MAAM,iBAAiB,GAAG,IAAA,6BAAe,GAAE,CAAC;IAE5C,YAAY,GAAG;QACb,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,iBAAiB,EAAE,MAAM;QACnF,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,iBAAiB,EAAE,MAAM;QACnF,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,SAAS;IACvB,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,MAA6B;IACxD,YAAY,GAAG;QACb,GAAG,YAAY;QACf,GAAG,MAAM;KACV,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAgB,SAAS,CAAC,WAAoB;IAC5C,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,IAAI,YAAY,CAAC,MAAM;QAAE,OAAO,YAAY,CAAC,MAAM,CAAC;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAExE,uDAAuD;IACvD,MAAM,iBAAiB,GAAG,IAAA,6BAAe,GAAE,CAAC;IAC5C,OAAO,iBAAiB,EAAE,MAAM,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,MAAe;IAC5C,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CACb,gJAAgJ,CACjJ,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,YAAY,GAAG,EAAE,CAAC;AACpB,CAAC"}
@@ -0,0 +1,25 @@
1
+ /**
2
+ * OS-level credential storage utility
3
+ * Stores credentials in platform-specific application data directories
4
+ */
5
+ export interface StoredCredentials {
6
+ apiKey?: string;
7
+ apiUrl?: string;
8
+ }
9
+ /**
10
+ * Load credentials from OS storage
11
+ */
12
+ export declare function loadCredentials(): StoredCredentials | null;
13
+ /**
14
+ * Save credentials to OS storage
15
+ */
16
+ export declare function saveCredentials(credentials: StoredCredentials): void;
17
+ /**
18
+ * Delete stored credentials
19
+ */
20
+ export declare function deleteCredentials(): void;
21
+ /**
22
+ * Get the config directory path (for informational purposes)
23
+ */
24
+ export declare function getConfigDirectoryPath(): string;
25
+ //# sourceMappingURL=credentials.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"credentials.d.ts","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA+CD;;GAEG;AACH,wBAAgB,eAAe,IAAI,iBAAiB,GAAG,IAAI,CAc1D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,iBAAiB,GAAG,IAAI,CAwBpE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CASxC;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C"}
@@ -0,0 +1,148 @@
1
+ "use strict";
2
+ /**
3
+ * OS-level credential storage utility
4
+ * Stores credentials in platform-specific application data directories
5
+ */
6
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
7
+ if (k2 === undefined) k2 = k;
8
+ var desc = Object.getOwnPropertyDescriptor(m, k);
9
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
10
+ desc = { enumerable: true, get: function() { return m[k]; } };
11
+ }
12
+ Object.defineProperty(o, k2, desc);
13
+ }) : (function(o, m, k, k2) {
14
+ if (k2 === undefined) k2 = k;
15
+ o[k2] = m[k];
16
+ }));
17
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
18
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
19
+ }) : function(o, v) {
20
+ o["default"] = v;
21
+ });
22
+ var __importStar = (this && this.__importStar) || (function () {
23
+ var ownKeys = function(o) {
24
+ ownKeys = Object.getOwnPropertyNames || function (o) {
25
+ var ar = [];
26
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
27
+ return ar;
28
+ };
29
+ return ownKeys(o);
30
+ };
31
+ return function (mod) {
32
+ if (mod && mod.__esModule) return mod;
33
+ var result = {};
34
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
35
+ __setModuleDefault(result, mod);
36
+ return result;
37
+ };
38
+ })();
39
+ Object.defineProperty(exports, "__esModule", { value: true });
40
+ exports.loadCredentials = loadCredentials;
41
+ exports.saveCredentials = saveCredentials;
42
+ exports.deleteCredentials = deleteCredentials;
43
+ exports.getConfigDirectoryPath = getConfigDirectoryPath;
44
+ const fs = __importStar(require("fs"));
45
+ const path = __importStar(require("path"));
46
+ const os = __importStar(require("os"));
47
+ /**
48
+ * Get the platform-specific config directory
49
+ */
50
+ function getConfigDir() {
51
+ const homeDir = os.homedir();
52
+ const platform = os.platform();
53
+ switch (platform) {
54
+ case 'darwin': // macOS
55
+ return path.join(homeDir, 'Library', 'Application Support', 'firecrawl-cli');
56
+ case 'win32': // Windows
57
+ return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli');
58
+ default: // Linux and others
59
+ return path.join(homeDir, '.config', 'firecrawl-cli');
60
+ }
61
+ }
62
+ /**
63
+ * Get the credentials file path
64
+ */
65
+ function getCredentialsPath() {
66
+ return path.join(getConfigDir(), 'credentials.json');
67
+ }
68
+ /**
69
+ * Ensure the config directory exists
70
+ */
71
+ function ensureConfigDir() {
72
+ const configDir = getConfigDir();
73
+ if (!fs.existsSync(configDir)) {
74
+ fs.mkdirSync(configDir, { recursive: true, mode: 0o700 }); // rwx------
75
+ }
76
+ }
77
+ /**
78
+ * Set file permissions to be readable/writable only by the owner
79
+ */
80
+ function setSecurePermissions(filePath) {
81
+ try {
82
+ fs.chmodSync(filePath, 0o600); // rw-------
83
+ }
84
+ catch (error) {
85
+ // Ignore errors on Windows or if file doesn't exist
86
+ }
87
+ }
88
+ /**
89
+ * Load credentials from OS storage
90
+ */
91
+ function loadCredentials() {
92
+ try {
93
+ const credentialsPath = getCredentialsPath();
94
+ if (!fs.existsSync(credentialsPath)) {
95
+ return null;
96
+ }
97
+ const data = fs.readFileSync(credentialsPath, 'utf-8');
98
+ const credentials = JSON.parse(data);
99
+ return credentials;
100
+ }
101
+ catch (error) {
102
+ // If file is corrupted or unreadable, return null
103
+ return null;
104
+ }
105
+ }
106
+ /**
107
+ * Save credentials to OS storage
108
+ */
109
+ function saveCredentials(credentials) {
110
+ try {
111
+ ensureConfigDir();
112
+ const credentialsPath = getCredentialsPath();
113
+ // Read existing credentials and merge
114
+ const existing = loadCredentials();
115
+ const merged = {
116
+ ...existing,
117
+ ...credentials,
118
+ };
119
+ // Write to file
120
+ fs.writeFileSync(credentialsPath, JSON.stringify(merged, null, 2), 'utf-8');
121
+ // Set secure permissions
122
+ setSecurePermissions(credentialsPath);
123
+ }
124
+ catch (error) {
125
+ throw new Error(`Failed to save credentials: ${error instanceof Error ? error.message : 'Unknown error'}`);
126
+ }
127
+ }
128
+ /**
129
+ * Delete stored credentials
130
+ */
131
+ function deleteCredentials() {
132
+ try {
133
+ const credentialsPath = getCredentialsPath();
134
+ if (fs.existsSync(credentialsPath)) {
135
+ fs.unlinkSync(credentialsPath);
136
+ }
137
+ }
138
+ catch (error) {
139
+ throw new Error(`Failed to delete credentials: ${error instanceof Error ? error.message : 'Unknown error'}`);
140
+ }
141
+ }
142
+ /**
143
+ * Get the config directory path (for informational purposes)
144
+ */
145
+ function getConfigDirectoryPath() {
146
+ return getConfigDir();
147
+ }
148
+ //# sourceMappingURL=credentials.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"credentials.js","sourceRoot":"","sources":["../../src/utils/credentials.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DH,0CAcC;AAKD,0CAwBC;AAKD,8CASC;AAKD,wDAEC;AAzHD,uCAAyB;AACzB,2CAA6B;AAC7B,uCAAyB;AAOzB;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE/B,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,EAAE,QAAQ;YACrB,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,eAAe,CAAC,CAAC;QAC/E,KAAK,OAAO,EAAE,UAAU;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;QACnE,SAAS,mBAAmB;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,kBAAkB,CAAC,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe;IACtB,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY;IACzE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oDAAoD;IACtD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe;IAC7B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAsB,CAAC;QAC1D,OAAO,WAAW,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kDAAkD;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe,CAAC,WAA8B;IAC5D,IAAI,CAAC;QACH,eAAe,EAAE,CAAC;QAClB,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAE7C,sCAAsC;QACtC,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;QACnC,MAAM,MAAM,GAAsB;YAChC,GAAG,QAAQ;YACX,GAAG,WAAW;SACf,CAAC;QAEF,gBAAgB;QAChB,EAAE,CAAC,aAAa,CACd,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC/B,OAAO,CACR,CAAC;QAEF,yBAAyB;QACzB,oBAAoB,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC7G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB;IAC/B,IAAI,CAAC;QACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACnC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,sBAAsB;IACpC,OAAO,YAAY,EAAE,CAAC;AACxB,CAAC"}
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Option parsing utilities
3
+ */
4
+ import type { ScrapeOptions } from '../types/scrape';
5
+ /**
6
+ * Convert commander options to ScrapeOptions
7
+ */
8
+ export declare function parseScrapeOptions(options: any): ScrapeOptions;
9
+ //# sourceMappingURL=options.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,CAa9D"}
@@ -0,0 +1,24 @@
1
+ "use strict";
2
+ /**
3
+ * Option parsing utilities
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.parseScrapeOptions = parseScrapeOptions;
7
+ /**
8
+ * Convert commander options to ScrapeOptions
9
+ */
10
+ function parseScrapeOptions(options) {
11
+ return {
12
+ url: options.url,
13
+ format: options.format,
14
+ onlyMainContent: options.onlyMainContent,
15
+ waitFor: options.waitFor,
16
+ screenshot: options.screenshot,
17
+ includeTags: options.includeTags ? options.includeTags.split(',').map((t) => t.trim()) : undefined,
18
+ excludeTags: options.excludeTags ? options.excludeTags.split(',').map((t) => t.trim()) : undefined,
19
+ apiKey: options.apiKey,
20
+ output: options.output,
21
+ pretty: options.pretty,
22
+ };
23
+ }
24
+ //# sourceMappingURL=options.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAOH,gDAaC;AAhBD;;GAEG;AACH,SAAgB,kBAAkB,CAAC,OAAY;IAC7C,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1G,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1G,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Output utilities for CLI
3
+ */
4
+ import type { ScrapeResult } from '../types/scrape';
5
+ import type { ScrapeFormat } from '../types/scrape';
6
+ /**
7
+ * Write output to file or stdout
8
+ */
9
+ export declare function writeOutput(content: string, outputPath?: string, silent?: boolean): void;
10
+ /**
11
+ * Handle scrape result output
12
+ * For text formats (html, markdown, etc.), outputs raw content directly
13
+ * For complex formats, outputs JSON
14
+ */
15
+ export declare function handleScrapeOutput(result: ScrapeResult, format?: ScrapeFormat, outputPath?: string, pretty?: boolean): void;
16
+ //# sourceMappingURL=output.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAiDpD;;GAEG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAmBN;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CA0CN"}
@@ -0,0 +1,150 @@
1
+ "use strict";
2
+ /**
3
+ * Output utilities for CLI
4
+ */
5
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
6
+ if (k2 === undefined) k2 = k;
7
+ var desc = Object.getOwnPropertyDescriptor(m, k);
8
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
9
+ desc = { enumerable: true, get: function() { return m[k]; } };
10
+ }
11
+ Object.defineProperty(o, k2, desc);
12
+ }) : (function(o, m, k, k2) {
13
+ if (k2 === undefined) k2 = k;
14
+ o[k2] = m[k];
15
+ }));
16
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
17
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
18
+ }) : function(o, v) {
19
+ o["default"] = v;
20
+ });
21
+ var __importStar = (this && this.__importStar) || (function () {
22
+ var ownKeys = function(o) {
23
+ ownKeys = Object.getOwnPropertyNames || function (o) {
24
+ var ar = [];
25
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
26
+ return ar;
27
+ };
28
+ return ownKeys(o);
29
+ };
30
+ return function (mod) {
31
+ if (mod && mod.__esModule) return mod;
32
+ var result = {};
33
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
34
+ __setModuleDefault(result, mod);
35
+ return result;
36
+ };
37
+ })();
38
+ Object.defineProperty(exports, "__esModule", { value: true });
39
+ exports.writeOutput = writeOutput;
40
+ exports.handleScrapeOutput = handleScrapeOutput;
41
+ const fs = __importStar(require("fs"));
42
+ const path = __importStar(require("path"));
43
+ /**
44
+ * Extract content from Firecrawl Document based on format
45
+ */
46
+ function extractContent(data, format) {
47
+ if (!data)
48
+ return null;
49
+ // If format is specified, try to extract that specific content
50
+ if (format) {
51
+ // Handle html/rawHtml formats - extract HTML content directly
52
+ if (format === 'html' || format === 'rawHtml') {
53
+ return data.html || data.rawHtml || data[format] || null;
54
+ }
55
+ // Handle markdown format
56
+ if (format === 'markdown') {
57
+ return data.markdown || data[format] || null;
58
+ }
59
+ // Handle links format
60
+ if (format === 'links') {
61
+ return data.links || data[format] || null;
62
+ }
63
+ // Handle images format
64
+ if (format === 'images') {
65
+ return data.images || data[format] || null;
66
+ }
67
+ // Handle summary format
68
+ if (format === 'summary') {
69
+ return data.summary || data[format] || null;
70
+ }
71
+ }
72
+ // Fallback: try common content fields
73
+ if (typeof data === 'string') {
74
+ return data;
75
+ }
76
+ // If it's an object, try to find string content
77
+ if (typeof data === 'object') {
78
+ return data.html || data.markdown || data.rawHtml || data.content || null;
79
+ }
80
+ return null;
81
+ }
82
+ /**
83
+ * Write output to file or stdout
84
+ */
85
+ function writeOutput(content, outputPath, silent = false) {
86
+ if (outputPath) {
87
+ const dir = path.dirname(outputPath);
88
+ if (dir && !fs.existsSync(dir)) {
89
+ fs.mkdirSync(dir, { recursive: true });
90
+ }
91
+ fs.writeFileSync(outputPath, content, 'utf-8');
92
+ if (!silent) {
93
+ // Always use stderr for file confirmation messages
94
+ console.error(`Output written to: ${outputPath}`);
95
+ }
96
+ }
97
+ else {
98
+ // Use process.stdout.write for raw output (like curl)
99
+ // Ensure content ends with newline for proper piping
100
+ if (!content.endsWith('\n')) {
101
+ content += '\n';
102
+ }
103
+ process.stdout.write(content);
104
+ }
105
+ }
106
+ /**
107
+ * Handle scrape result output
108
+ * For text formats (html, markdown, etc.), outputs raw content directly
109
+ * For complex formats, outputs JSON
110
+ */
111
+ function handleScrapeOutput(result, format, outputPath, pretty = false) {
112
+ if (!result.success) {
113
+ // Always use stderr for errors to allow piping
114
+ console.error('Error:', result.error);
115
+ process.exit(1);
116
+ }
117
+ if (!result.data) {
118
+ return;
119
+ }
120
+ // Text formats that should output raw content (curl-like)
121
+ const rawTextFormats = ['html', 'rawHtml', 'markdown', 'links', 'images', 'summary'];
122
+ const shouldOutputRaw = format && rawTextFormats.includes(format);
123
+ if (shouldOutputRaw) {
124
+ // Extract and output raw content
125
+ const content = extractContent(result.data, format);
126
+ if (content !== null) {
127
+ writeOutput(content, outputPath, !!outputPath);
128
+ return;
129
+ }
130
+ }
131
+ // For JSON format or complex formats (branding, json, etc.), output clean JSON
132
+ // Always stringify the entire data object to ensure valid JSON
133
+ let jsonContent;
134
+ try {
135
+ jsonContent = pretty
136
+ ? JSON.stringify(result.data, null, 2)
137
+ : JSON.stringify(result.data);
138
+ }
139
+ catch (error) {
140
+ // If stringification fails, try to create a minimal error response
141
+ jsonContent = JSON.stringify({
142
+ error: 'Failed to serialize response',
143
+ message: error instanceof Error ? error.message : 'Unknown error'
144
+ });
145
+ }
146
+ // Ensure clean JSON output (no extra newlines or text before JSON)
147
+ // Output directly to stdout without any prefix
148
+ writeOutput(jsonContent, outputPath, !!outputPath);
149
+ }
150
+ //# sourceMappingURL=output.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDH,kCAuBC;AAOD,gDA+CC;AApID,uCAAyB;AACzB,2CAA6B;AAI7B;;GAEG;AACH,SAAS,cAAc,CAAC,IAAS,EAAE,MAAqB;IACtD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,+DAA+D;IAC/D,IAAI,MAAM,EAAE,CAAC;QACX,8DAA8D;QAC9D,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC3D,CAAC;QAED,yBAAyB;QACzB,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC/C,CAAC;QAED,sBAAsB;QACtB,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC5C,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC7C,CAAC;QAED,wBAAwB;QACxB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;IAC5E,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW,CACzB,OAAe,EACf,UAAmB,EACnB,SAAkB,KAAK;IAEvB,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,mDAAmD;YACnD,OAAO,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sDAAsD;QACtD,qDAAqD;QACrD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAgB,kBAAkB,CAChC,MAAoB,EACpB,MAAqB,EACrB,UAAmB,EACnB,SAAkB,KAAK;IAEvB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,+CAA+C;QAC/C,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,0DAA0D;IAC1D,MAAM,cAAc,GAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IACrG,MAAM,eAAe,GAAG,MAAM,IAAI,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAElE,IAAI,eAAe,EAAE,CAAC;QACpB,iCAAiC;QACjC,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,+DAA+D;IAC/D,IAAI,WAAmB,CAAC;IACxB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM;YAClB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mEAAmE;QACnE,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;YAC3B,KAAK,EAAE,8BAA8B;YACrC,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAClE,CAAC,CAAC;IACL,CAAC;IAED,mEAAmE;IACnE,+CAA+C;IAC/C,WAAW,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;AACrD,CAAC"}
@@ -0,0 +1,12 @@
1
+ /**
2
+ * URL utility functions
3
+ */
4
+ /**
5
+ * Check if a string looks like a URL (with or without protocol)
6
+ */
7
+ export declare function isUrl(str: string): boolean;
8
+ /**
9
+ * Normalize URL by adding https:// if missing
10
+ */
11
+ export declare function normalizeUrl(url: string): string;
12
+ //# sourceMappingURL=url.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,wBAAgB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAoB1C;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAKhD"}
@@ -0,0 +1,40 @@
1
+ "use strict";
2
+ /**
3
+ * URL utility functions
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.isUrl = isUrl;
7
+ exports.normalizeUrl = normalizeUrl;
8
+ /**
9
+ * Check if a string looks like a URL (with or without protocol)
10
+ */
11
+ function isUrl(str) {
12
+ // If it has a protocol, validate it
13
+ if (/^https?:\/\//i.test(str)) {
14
+ try {
15
+ const url = new URL(str);
16
+ return url.protocol === 'http:' || url.protocol === 'https:';
17
+ }
18
+ catch {
19
+ return true; // Assume valid if it starts with http:// or https://
20
+ }
21
+ }
22
+ // Check if it looks like a domain (has dots and valid characters)
23
+ // Exclude common commands and flags
24
+ if (str.includes('.') && !str.startsWith('-') && !str.includes(' ')) {
25
+ // Basic domain validation: contains at least one dot and valid characters
26
+ const domainPattern = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/;
27
+ return domainPattern.test(str);
28
+ }
29
+ return false;
30
+ }
31
+ /**
32
+ * Normalize URL by adding https:// if missing
33
+ */
34
+ function normalizeUrl(url) {
35
+ if (/^https?:\/\//i.test(url)) {
36
+ return url;
37
+ }
38
+ return `https://${url}`;
39
+ }
40
+ //# sourceMappingURL=url.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/utils/url.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAKH,sBAoBC;AAKD,oCAKC;AAjCD;;GAEG;AACH,SAAgB,KAAK,CAAC,GAAW;IAC/B,oCAAoC;IACpC,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,CAAC,qDAAqD;QACpE,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,oCAAoC;IACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpE,0EAA0E;QAC1E,MAAM,aAAa,GAAG,oHAAoH,CAAC;QAC3I,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,GAAW;IACtC,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,WAAW,GAAG,EAAE,CAAC;AAC1B,CAAC"}
package/package.json ADDED
@@ -0,0 +1,53 @@
1
+ {
2
+ "name": "firecrawl-cli",
3
+ "version": "0.0.1",
4
+ "description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.",
5
+ "main": "dist/index.js",
6
+ "bin": {
7
+ "firecrawl": "dist/index.js"
8
+ },
9
+ "scripts": {
10
+ "build": "tsc",
11
+ "dev": "tsc --watch",
12
+ "start": "node dist/index.js",
13
+ "clean": "rm -rf dist",
14
+ "prepublishOnly": "pnpm run build",
15
+ "test": "echo \"Error: no test specified\" && exit 1"
16
+ },
17
+ "keywords": [
18
+ "firecrawl",
19
+ "cli",
20
+ "web-scraping",
21
+ "crawler",
22
+ "scraper",
23
+ "data-extraction",
24
+ "llm",
25
+ "markdown"
26
+ ],
27
+ "author": "Firecrawl",
28
+ "license": "ISC",
29
+ "repository": {
30
+ "type": "git",
31
+ "url": "git+https://github.com/firecrawl/firecrawl-cli.git"
32
+ },
33
+ "bugs": {
34
+ "url": "https://github.com/firecrawl/firecrawl-cli/issues"
35
+ },
36
+ "homepage": "https://docs.firecrawl.dev/cli",
37
+ "engines": {
38
+ "node": ">=18.0.0"
39
+ },
40
+ "files": [
41
+ "dist",
42
+ "README.md"
43
+ ],
44
+ "packageManager": "pnpm@10.12.1",
45
+ "devDependencies": {
46
+ "@types/node": "^20.0.0",
47
+ "typescript": "^5.0.0"
48
+ },
49
+ "dependencies": {
50
+ "@mendable/firecrawl-js": "^4.10.0",
51
+ "commander": "^14.0.2"
52
+ }
53
+ }