browserlessapi 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +98 -2
- package/bin/browserlessapi.js +365 -0
- package/package.json +10 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nero Softwares & IT Services Pvt. Ltd. (BrowserlessAPI)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,99 @@
|
|
|
1
|
-
#
|
|
1
|
+
# browserlessapi
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The command-line tool for [BrowserlessAPI](https://browserlessapi.com/): turn any web
|
|
4
|
+
page into Markdown, take screenshots, run browser steps, solve captchas, and connect
|
|
5
|
+
BrowserlessAPI to AI apps such as Claude, ChatGPT, Cursor and VS Code.
|
|
6
|
+
|
|
7
|
+
Pages are fetched by a real browser when needed, including JavaScript-heavy and
|
|
8
|
+
Cloudflare-protected sites.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
npm install -g browserlessapi
|
|
14
|
+
# or run without installing:
|
|
15
|
+
npx browserlessapi --help
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Node.js 18 or later. No other dependencies.
|
|
19
|
+
|
|
20
|
+
## Sign in
|
|
21
|
+
|
|
22
|
+
Get an API key at [browserlessapi.com](https://browserlessapi.com/register/), then:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
browserlessapi auth # paste your key when asked
|
|
26
|
+
browserlessapi auth --api-key YOUR_KEY # or non-interactive
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The key is checked, then saved to `~/.config/browserlessapi/config.json` (readable only
|
|
30
|
+
by you). You can also set `BROWSERLESSAPI_API_KEY`, or pass `--api-key` to any command.
|
|
31
|
+
`browserlessapi logout` removes the saved key.
|
|
32
|
+
|
|
33
|
+
## Commands
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
# A page as Markdown
|
|
37
|
+
browserlessapi scrape https://example.com
|
|
38
|
+
|
|
39
|
+
# Several formats at once (prints JSON)
|
|
40
|
+
browserlessapi scrape https://example.com -f markdown,links,metadata
|
|
41
|
+
|
|
42
|
+
# Save to a file; always use a browser; keep headers and footers
|
|
43
|
+
browserlessapi scrape https://example.com --render true --full -o page.md
|
|
44
|
+
|
|
45
|
+
# Many pages: one URL per line, 5 at a time, a file per page plus manifest.json
|
|
46
|
+
browserlessapi scrape --input-file urls.txt --output-dir pages -c 5
|
|
47
|
+
|
|
48
|
+
# Screenshot
|
|
49
|
+
browserlessapi screenshot https://example.com -o example.png --wait-for h1
|
|
50
|
+
|
|
51
|
+
# Browser steps in one session (click, type, extract, screenshot)
|
|
52
|
+
browserlessapi steps '[{"action":"goto","url":"https://example.com"},{"action":"extract","selector":"h1","as":"title"}]'
|
|
53
|
+
|
|
54
|
+
# Solve a captcha
|
|
55
|
+
browserlessapi captcha --task '{"type":"RecaptchaV2Task","websiteURL":"https://site/page","websiteKey":"SITE_KEY"}'
|
|
56
|
+
|
|
57
|
+
# Plan, credits and features
|
|
58
|
+
browserlessapi account
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Scrape options: `--format`, `--render auto|true|false`, `--captcha auto|always|off`,
|
|
62
|
+
`--full`, `--engine puppeteer|playwright`, `--json`, `--output-file`.
|
|
63
|
+
Status lines go to stderr, content to stdout, so output can be piped.
|
|
64
|
+
|
|
65
|
+
## Use BrowserlessAPI in AI apps (MCP)
|
|
66
|
+
|
|
67
|
+
BrowserlessAPI runs a remote MCP server at `https://browserlessapi.com/mcp/v2`.
|
|
68
|
+
Print the setup for your app, with your key filled in:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
browserlessapi mcp claude # claude.ai and Claude desktop
|
|
72
|
+
browserlessapi mcp claude-code # Claude Code
|
|
73
|
+
browserlessapi mcp chatgpt # ChatGPT
|
|
74
|
+
browserlessapi mcp cursor # Cursor
|
|
75
|
+
browserlessapi mcp vscode # VS Code
|
|
76
|
+
browserlessapi mcp codex # OpenAI Codex
|
|
77
|
+
browserlessapi mcp neoncodex # NeonCodex
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Tools available to the AI: `scrape_page`, `take_screenshot`, `run_browser_steps`,
|
|
81
|
+
`solve_captcha`, `check_account` and `fetch`. Needs a plan that includes the MCP server.
|
|
82
|
+
|
|
83
|
+
## Credits
|
|
84
|
+
|
|
85
|
+
Every request uses credits from your BrowserlessAPI account, the same as calling the API
|
|
86
|
+
directly. `browserlessapi account` shows your balance.
|
|
87
|
+
|
|
88
|
+
## Safety
|
|
89
|
+
|
|
90
|
+
This tool never runs shell commands based on page content, so text on a scraped page
|
|
91
|
+
cannot make it execute anything. Keep your API key out of shared files and repositories.
|
|
92
|
+
|
|
93
|
+
## Exit codes
|
|
94
|
+
|
|
95
|
+
`0` success, `1` request failed, `2` usage error.
|
|
96
|
+
|
|
97
|
+
## License
|
|
98
|
+
|
|
99
|
+
MIT
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* BrowserlessAPI command-line tool.
|
|
4
|
+
* Docs: https://browserlessapi.com/api-docs/
|
|
5
|
+
*
|
|
6
|
+
* Scrape pages, take screenshots, run browser steps, solve captchas and
|
|
7
|
+
* connect BrowserlessAPI to AI apps (Claude, ChatGPT, Cursor, VS Code,
|
|
8
|
+
* Codex, NeonCodex) from your terminal. No dependencies; Node 18 or later.
|
|
9
|
+
*/
|
|
10
|
+
'use strict';
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const os = require('os');
|
|
14
|
+
const path = require('path');
|
|
15
|
+
const readline = require('readline');
|
|
16
|
+
|
|
17
|
+
const VERSION = '1.0.0';
|
|
18
|
+
const BASE = (process.env.BROWSERLESSAPI_BASE_URL || 'https://browserlessapi.com').replace(/\/+$/, '');
|
|
19
|
+
const API = BASE + '/wp-json/browserless';
|
|
20
|
+
const MCP_URL = BASE + '/mcp/v2';
|
|
21
|
+
const CONFIG_DIR = process.env.BROWSERLESSAPI_CONFIG_DIR || path.join(os.homedir(), '.config', 'browserlessapi');
|
|
22
|
+
const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
|
|
23
|
+
const FORMATS = ['markdown', 'text', 'links', 'metadata', 'html'];
|
|
24
|
+
|
|
25
|
+
// ------------------------------------------------------------------- output
|
|
26
|
+
const out = (s) => process.stdout.write(s.endsWith('\n') ? s : s + '\n');
|
|
27
|
+
const err = (s) => process.stderr.write(s.endsWith('\n') ? s : s + '\n');
|
|
28
|
+
class UsageError extends Error {}
|
|
29
|
+
class ApiError extends Error {}
|
|
30
|
+
|
|
31
|
+
// ------------------------------------------------------------------ args
|
|
32
|
+
// Flags that never take a value (so "--json URL" doesn't eat the URL).
|
|
33
|
+
const BOOL_FLAGS = new Set(['json', 'full', 'show', 'help', 'version', 'no-key']);
|
|
34
|
+
function parseArgs(argv) {
|
|
35
|
+
const pos = [];
|
|
36
|
+
const opt = {};
|
|
37
|
+
const short = { o: 'output-file', f: 'format', h: 'help', v: 'version', k: 'api-key', c: 'concurrency' };
|
|
38
|
+
for (let i = 0; i < argv.length; i++) {
|
|
39
|
+
let a = argv[i];
|
|
40
|
+
if (a === '--') { pos.push(...argv.slice(i + 1)); break; }
|
|
41
|
+
if (/^-[a-z]$/.test(a)) a = '--' + short[a[1]];
|
|
42
|
+
if (a && a.startsWith('--')) {
|
|
43
|
+
let [k, v] = a.slice(2).split(/=(.*)/s);
|
|
44
|
+
if (v === undefined) {
|
|
45
|
+
const next = argv[i + 1];
|
|
46
|
+
if (!BOOL_FLAGS.has(k) && next !== undefined && (next === '-' || !next.startsWith('-'))) { v = next; i++; } else v = true;
|
|
47
|
+
}
|
|
48
|
+
opt[k] = v;
|
|
49
|
+
} else pos.push(a);
|
|
50
|
+
}
|
|
51
|
+
return { pos, opt };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ------------------------------------------------------------------ config
|
|
55
|
+
function readConfig() {
|
|
56
|
+
try { return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')); } catch { return {}; }
|
|
57
|
+
}
|
|
58
|
+
function writeConfig(cfg) {
|
|
59
|
+
fs.mkdirSync(CONFIG_DIR, { recursive: true, mode: 0o700 });
|
|
60
|
+
fs.writeFileSync(CONFIG_FILE, JSON.stringify(cfg, null, 2) + '\n', { mode: 0o600 });
|
|
61
|
+
try { fs.chmodSync(CONFIG_FILE, 0o600); } catch {}
|
|
62
|
+
}
|
|
63
|
+
function apiKey(opt, required = true) {
|
|
64
|
+
const k = (typeof opt['api-key'] === 'string' && opt['api-key']) || process.env.BROWSERLESSAPI_API_KEY || readConfig().api_key || '';
|
|
65
|
+
if (!k && required) throw new UsageError('No API key. Run "browserlessapi auth", set BROWSERLESSAPI_API_KEY, or pass --api-key. Get a key at https://browserlessapi.com/register/');
|
|
66
|
+
return k;
|
|
67
|
+
}
|
|
68
|
+
const mask = (k) => (k.length > 8 ? k.slice(0, 4) + '...' + k.slice(-4) : '****');
|
|
69
|
+
|
|
70
|
+
// --------------------------------------------------------------------- http
|
|
71
|
+
async function call(method, p, key, body, timeoutMs = 200000) {
|
|
72
|
+
const ctrl = new AbortController();
|
|
73
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
74
|
+
let res;
|
|
75
|
+
try {
|
|
76
|
+
res = await fetch(API + p, {
|
|
77
|
+
method,
|
|
78
|
+
headers: { 'Content-Type': 'application/json', 'X-API-Key': key, 'User-Agent': `browserlessapi-cli/${VERSION}` },
|
|
79
|
+
body: body === undefined ? undefined : JSON.stringify(body),
|
|
80
|
+
signal: ctrl.signal,
|
|
81
|
+
});
|
|
82
|
+
} catch (e) {
|
|
83
|
+
throw new ApiError(e.name === 'AbortError' ? 'The request timed out.' : `Could not reach ${BASE}: ${e.message}`);
|
|
84
|
+
} finally {
|
|
85
|
+
clearTimeout(t);
|
|
86
|
+
}
|
|
87
|
+
const text = await res.text();
|
|
88
|
+
let data;
|
|
89
|
+
try { data = JSON.parse(text); } catch { data = { raw: text.slice(0, 300) }; }
|
|
90
|
+
if (!res.ok || data.status === 'error') {
|
|
91
|
+
const e = data.error;
|
|
92
|
+
const msg = e && typeof e === 'object' ? `${e.message || e.code}${e.code ? ` (${e.code})` : ''}` : (e || data.message || `HTTP ${res.status}`);
|
|
93
|
+
throw new ApiError(msg);
|
|
94
|
+
}
|
|
95
|
+
return data;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function save(file, content) {
|
|
99
|
+
fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
|
|
100
|
+
fs.writeFileSync(file, content);
|
|
101
|
+
}
|
|
102
|
+
function readJsonArg(v, label) {
|
|
103
|
+
if (v === undefined || v === true) throw new UsageError(`${label} is required.`);
|
|
104
|
+
const src = v === '-' ? fs.readFileSync(0, 'utf8') : (fs.existsSync(v) ? fs.readFileSync(v, 'utf8') : v);
|
|
105
|
+
try { return JSON.parse(src); } catch { throw new UsageError(`${label} is not valid JSON (give JSON, a file path, or - for stdin).`); }
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// ---------------------------------------------------------------- commands
|
|
109
|
+
async function cmdAuth(opt) {
|
|
110
|
+
if (opt.show) {
|
|
111
|
+
const k = apiKey(opt, false);
|
|
112
|
+
const src = opt['api-key'] ? 'flag' : process.env.BROWSERLESSAPI_API_KEY ? 'BROWSERLESSAPI_API_KEY' : readConfig().api_key ? CONFIG_FILE : null;
|
|
113
|
+
out(k ? `API key ${mask(k)} (from ${src})` : 'No API key saved.');
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
let key = typeof opt['api-key'] === 'string' ? opt['api-key'] : '';
|
|
117
|
+
if (!key) {
|
|
118
|
+
if (!process.stdin.isTTY) key = fs.readFileSync(0, 'utf8').trim();
|
|
119
|
+
else {
|
|
120
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stderr });
|
|
121
|
+
key = (await new Promise((r) => rl.question('BrowserlessAPI API key (from https://browserlessapi.com/portal/ > API Keys): ', r))).trim();
|
|
122
|
+
rl.close();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (!key) throw new UsageError('No key entered.');
|
|
126
|
+
const acct = await call('GET', '/v2/account', key, undefined, 20000);
|
|
127
|
+
writeConfig({ ...readConfig(), api_key: key });
|
|
128
|
+
out(`Saved. Plan: ${acct.plan}, credits: ${acct.credits}. Key stored in ${CONFIG_FILE}`);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function cmdLogout() {
|
|
132
|
+
const cfg = readConfig();
|
|
133
|
+
delete cfg.api_key;
|
|
134
|
+
if (fs.existsSync(CONFIG_FILE)) writeConfig(cfg);
|
|
135
|
+
out('API key removed from ' + CONFIG_FILE);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function cmdAccount(opt) {
|
|
139
|
+
const d = await call('GET', '/v2/account', apiKey(opt), undefined, 20000);
|
|
140
|
+
if (opt.json) return out(JSON.stringify(d, null, 2));
|
|
141
|
+
const on = Object.entries(d.features || {}).filter(([, v]) => v).map(([k]) => k);
|
|
142
|
+
out(`Plan: ${d.plan}\nCredits: ${d.credits}\nConcurrency: ${d.max_concurrency}\nFeatures: ${on.join(', ') || 'none'}`);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function scrapeBody(url, opt) {
|
|
146
|
+
let formats = typeof opt.format === 'string' ? opt.format.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean) : ['markdown'];
|
|
147
|
+
const bad = formats.filter((f) => !FORMATS.includes(f));
|
|
148
|
+
if (bad.length) throw new UsageError(`Unknown format: ${bad.join(', ')}. Use: ${FORMATS.join(', ')}`);
|
|
149
|
+
const body = { url, formats };
|
|
150
|
+
if (opt.render !== undefined) {
|
|
151
|
+
const r = String(opt.render);
|
|
152
|
+
if (!['auto', 'true', 'false'].includes(r)) throw new UsageError('--render must be auto, true or false.');
|
|
153
|
+
body.render = r === 'auto' ? 'auto' : r === 'true';
|
|
154
|
+
}
|
|
155
|
+
if (opt.captcha !== undefined) {
|
|
156
|
+
if (!['auto', 'always', 'off'].includes(String(opt.captcha))) throw new UsageError('--captcha must be auto, always or off.');
|
|
157
|
+
body.captcha = String(opt.captcha);
|
|
158
|
+
}
|
|
159
|
+
if (opt.full) body.only_main_content = false;
|
|
160
|
+
if (typeof opt.engine === 'string') body.engine = opt.engine;
|
|
161
|
+
return body;
|
|
162
|
+
}
|
|
163
|
+
function renderScrape(d, body) {
|
|
164
|
+
const f = d.formats || {};
|
|
165
|
+
if (body.formats.length === 1) {
|
|
166
|
+
const v = f[body.formats[0]];
|
|
167
|
+
return typeof v === 'string' ? v : JSON.stringify(v, null, 2);
|
|
168
|
+
}
|
|
169
|
+
return JSON.stringify(f, null, 2);
|
|
170
|
+
}
|
|
171
|
+
function reportScrape(d) {
|
|
172
|
+
let s = `[${d.status_code}] ${d.final_url} rendered_with=${d.rendered_with} credits=${d.credits_charged} ${d.timing_ms}ms`;
|
|
173
|
+
if (d.captcha) s += ` captcha=${d.captcha.type}:${d.captcha.solved ? 'solved' : d.captcha.reason}`;
|
|
174
|
+
err(s);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async function cmdScrape(pos, opt) {
|
|
178
|
+
const key = apiKey(opt);
|
|
179
|
+
if (opt['input-file']) return scrapeBatch(key, opt);
|
|
180
|
+
if (!pos.length) throw new UsageError('Give a URL: browserlessapi scrape https://example.com');
|
|
181
|
+
const body = scrapeBody(pos[0], opt);
|
|
182
|
+
const d = await call('POST', '/v2/scrape', key, body);
|
|
183
|
+
reportScrape(d);
|
|
184
|
+
const content = opt.json ? JSON.stringify(d, null, 2) : renderScrape(d, body);
|
|
185
|
+
if (typeof opt['output-file'] === 'string') { save(opt['output-file'], content); err('Saved ' + opt['output-file']); }
|
|
186
|
+
else out(content);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async function scrapeBatch(key, opt) {
|
|
190
|
+
const src = opt['input-file'] === '-' ? fs.readFileSync(0, 'utf8') : fs.readFileSync(String(opt['input-file']), 'utf8');
|
|
191
|
+
const urls = [...new Set(src.split(/\r?\n/).map((l) => l.split(',')[0].trim()).filter((l) => /^https?:\/\//i.test(l)))];
|
|
192
|
+
if (!urls.length) throw new UsageError('No http(s) URLs found in the input file (one per line).');
|
|
193
|
+
const dir = typeof opt['output-dir'] === 'string' ? opt['output-dir'] : 'browserlessapi-' + new Date().toISOString().slice(0, 19).replace(/[:T]/g, '');
|
|
194
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
195
|
+
const conc = Math.max(1, Math.min(20, parseInt(opt.concurrency, 10) || 3));
|
|
196
|
+
const probe = scrapeBody(urls[0], opt);
|
|
197
|
+
const ext = probe.formats.length === 1 && ['markdown', 'text', 'html'].includes(probe.formats[0])
|
|
198
|
+
? { markdown: '.md', text: '.txt', html: '.html' }[probe.formats[0]] : '.json';
|
|
199
|
+
const manifest = [];
|
|
200
|
+
let next = 0, done = 0, failed = 0, credits = 0;
|
|
201
|
+
err(`Scraping ${urls.length} URLs, ${conc} at a time, into ${dir}/`);
|
|
202
|
+
async function worker() {
|
|
203
|
+
while (next < urls.length) {
|
|
204
|
+
const i = next++;
|
|
205
|
+
const url = urls[i];
|
|
206
|
+
const file = path.join(dir, String(i + 1).padStart(4, '0') + ext);
|
|
207
|
+
try {
|
|
208
|
+
const body = scrapeBody(url, opt);
|
|
209
|
+
const d = await call('POST', '/v2/scrape', key, body);
|
|
210
|
+
save(file, opt.json ? JSON.stringify(d, null, 2) : renderScrape(d, body));
|
|
211
|
+
credits += d.credits_charged || 0;
|
|
212
|
+
manifest.push({ n: i + 1, url, file: path.basename(file), status: 'ok', final_url: d.final_url, credits: d.credits_charged, captcha: d.captcha || null });
|
|
213
|
+
} catch (e) {
|
|
214
|
+
failed++;
|
|
215
|
+
manifest.push({ n: i + 1, url, file: null, status: 'error', error: e.message });
|
|
216
|
+
}
|
|
217
|
+
done++;
|
|
218
|
+
err(` ${done}/${urls.length} ${manifest[manifest.length - 1].status === 'ok' ? 'ok ' : 'error'} ${url}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
await Promise.all(Array.from({ length: conc }, worker));
|
|
222
|
+
manifest.sort((a, b) => a.n - b.n);
|
|
223
|
+
save(path.join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
224
|
+
err(`Done: ${done - failed} ok, ${failed} failed, ${credits} credits. Manifest: ${path.join(dir, 'manifest.json')}`);
|
|
225
|
+
if (failed) process.exitCode = 1;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function bqlResult(d) {
|
|
229
|
+
for (const c of [d, d && d.result, d && d.data]) if (c && (Array.isArray(c.extracted) || Array.isArray(c.screenshots) || 'success' in c)) return c;
|
|
230
|
+
return d || {};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async function cmdScreenshot(pos, opt) {
|
|
234
|
+
if (!pos.length) throw new UsageError('Give a URL: browserlessapi screenshot https://example.com -o page.png');
|
|
235
|
+
const steps = [{ action: 'goto', url: pos[0] }];
|
|
236
|
+
if (typeof opt['wait-for'] === 'string') steps.push({ action: 'waitForSelector', selector: opt['wait-for'] });
|
|
237
|
+
steps.push({ action: 'screenshot', as: 'page' });
|
|
238
|
+
const key = apiKey(opt);
|
|
239
|
+
const res = bqlResult(await call('POST', '/v1/bql', key, { key, steps }));
|
|
240
|
+
const shot = (res.screenshots || [])[0];
|
|
241
|
+
if (!shot || !shot.base64) throw new ApiError(res.error || 'No screenshot returned.');
|
|
242
|
+
const file = typeof opt['output-file'] === 'string' ? opt['output-file'] : 'screenshot.png';
|
|
243
|
+
save(file, Buffer.from(String(shot.base64).replace(/^data:image\/\w+;base64,/, ''), 'base64'));
|
|
244
|
+
out('Saved ' + file);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
async function cmdSteps(pos, opt) {
|
|
248
|
+
let steps = readJsonArg(pos[0], 'A steps file (or JSON, or - for stdin)');
|
|
249
|
+
if (steps && Array.isArray(steps.steps)) steps = steps.steps;
|
|
250
|
+
if (!Array.isArray(steps) || !steps.length || steps[0].action !== 'goto') throw new UsageError('Steps must be a JSON array whose first step is {"action":"goto","url":"..."}.');
|
|
251
|
+
const key = apiKey(opt);
|
|
252
|
+
const res = bqlResult(await call('POST', '/v1/bql', key, { key, steps }));
|
|
253
|
+
if (res.success === false) throw new ApiError(res.error || 'The steps failed.');
|
|
254
|
+
const dir = typeof opt['output-dir'] === 'string' ? opt['output-dir'] : '.';
|
|
255
|
+
const shots = [];
|
|
256
|
+
(res.screenshots || []).forEach((s, i) => {
|
|
257
|
+
if (!s || !s.base64) return;
|
|
258
|
+
const f = path.join(dir, (s.as || 'screenshot-' + (i + 1)).replace(/[^\w.-]/g, '_') + '.png');
|
|
259
|
+
save(f, Buffer.from(String(s.base64).replace(/^data:image\/\w+;base64,/, ''), 'base64'));
|
|
260
|
+
shots.push(f);
|
|
261
|
+
});
|
|
262
|
+
const summary = { finalUrl: res.finalUrl, extracted: res.extracted || [], screenshots: shots };
|
|
263
|
+
if (res.evaluated !== undefined) summary.evaluated = res.evaluated;
|
|
264
|
+
const content = JSON.stringify(summary, null, 2);
|
|
265
|
+
if (typeof opt['output-file'] === 'string') { save(opt['output-file'], content); err('Saved ' + opt['output-file']); } else out(content);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function cmdCaptcha(opt) {
|
|
269
|
+
const task = opt['task-file'] ? readJsonArg(String(opt['task-file']), '--task-file') : readJsonArg(opt.task, '--task');
|
|
270
|
+
if (!task || typeof task.type !== 'string') throw new UsageError('The task needs a "type", e.g. {"type":"RecaptchaV2Task","websiteURL":"...","websiteKey":"..."}');
|
|
271
|
+
const body = { task };
|
|
272
|
+
if (opt.timeout) body.timeout = parseInt(opt.timeout, 10);
|
|
273
|
+
const d = await call('POST', '/v2/captcha', apiKey(opt), body);
|
|
274
|
+
if (d.status === 'pending') { err(`Still solving. Task ${d.task_id}; try again with a longer --timeout.`); process.exitCode = 1; return; }
|
|
275
|
+
out(opt.json ? JSON.stringify(d, null, 2) : JSON.stringify(d.solution, null, 2));
|
|
276
|
+
err(`task ${d.task_id}, credits ${d.credits_charged}`);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function cmdMcp(pos, opt) {
|
|
280
|
+
const real = opt['no-key'] ? '' : apiKey(opt, false);
|
|
281
|
+
const key = real || 'YOUR_API_KEY';
|
|
282
|
+
const urlKey = `${MCP_URL}?api_key=${key}`;
|
|
283
|
+
const client = (pos[0] || '').toLowerCase();
|
|
284
|
+
const guides = {
|
|
285
|
+
'claude-code': () => `Run in a terminal:\n\n claude mcp add --transport http browserlessapi ${MCP_URL} --header "Authorization: Bearer ${key}"\n\nThen run /mcp inside Claude Code to check it is connected.`,
|
|
286
|
+
'claude': () => `claude.ai and the Claude desktop app (Settings > Connectors > Add custom connector):\n\n Name: BrowserlessAPI\n URL: ${urlKey}\n\nOr, in claude_desktop_config.json (Settings > Developer > Edit Config), then restart Claude:\n\n${JSON.stringify({ mcpServers: { browserlessapi: { command: 'npx', args: ['-y', 'mcp-remote', MCP_URL, '--header', `Authorization: Bearer ${key}`] } } }, null, 2)}`,
|
|
287
|
+
'chatgpt': () => `ChatGPT (chatgpt.com, plan with developer mode):\n\n 1. Settings > Apps > Advanced settings > turn on Developer mode\n 2. Settings > Apps > Create\n 3. Name: BrowserlessAPI\n 4. MCP server URL: ${urlKey}\n 5. Authentication: No authentication\n 6. Tick the confirmation box, click Create, then Connect\n\nUse it with @BrowserlessAPI in a chat.`,
|
|
288
|
+
'cursor': () => `Add to ~/.cursor/mcp.json (all projects) or .cursor/mcp.json (one project):\n\n${JSON.stringify({ mcpServers: { browserlessapi: { url: MCP_URL, headers: { Authorization: `Bearer ${key}` } } } }, null, 2)}\n\nThen Cursor Settings > MCP: turn browserlessapi on.`,
|
|
289
|
+
'vscode': () => `Add to .vscode/mcp.json (or your user mcp.json):\n\n${JSON.stringify({ servers: { browserlessapi: { type: 'http', url: MCP_URL, headers: { Authorization: `Bearer ${key}` } } } }, null, 2)}`,
|
|
290
|
+
'codex': () => `Add to ~/.codex/config.toml (needs Node.js 18+ for npx):\n\n[mcp_servers.browserlessapi]\ncommand = "npx"\nargs = ["-y", "mcp-remote", "${urlKey}"]\n\nRestart Codex, then run "codex mcp list" to check it.`,
|
|
291
|
+
'neoncodex': () => `In NeonCodex, add a custom MCP server:\n\n Name: BrowserlessAPI\n URL: ${MCP_URL}\n Header: Authorization: Bearer ${key}\n\n(or use the URL ${urlKey} if only a URL can be entered)`,
|
|
292
|
+
'url': () => urlKey,
|
|
293
|
+
};
|
|
294
|
+
if (!guides[client]) {
|
|
295
|
+
out(`Connect BrowserlessAPI to an AI app. Pick one:\n\n browserlessapi mcp claude claude.ai and Claude desktop\n browserlessapi mcp claude-code Claude Code\n browserlessapi mcp chatgpt ChatGPT\n browserlessapi mcp cursor Cursor\n browserlessapi mcp vscode VS Code (Copilot)\n browserlessapi mcp codex OpenAI Codex\n browserlessapi mcp neoncodex NeonCodex\n browserlessapi mcp url just the server URL\n\nAny other MCP client: ${MCP_URL} with "Authorization: Bearer YOUR_API_KEY",\nor ${MCP_URL}?api_key=YOUR_API_KEY. Add --no-key to print a placeholder instead of your key.`);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
out(guides[client]());
|
|
299
|
+
if (real) err('\nThis contains your API key. Do not commit it to a repository or share it.');
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const HELP = `browserlessapi ${VERSION}: web scraping, screenshots, browser automation and captcha solving.
|
|
303
|
+
|
|
304
|
+
Usage: browserlessapi <command> [options]
|
|
305
|
+
|
|
306
|
+
Commands
|
|
307
|
+
auth [--api-key KEY] Save your API key (checks it first). --show shows the saved key.
|
|
308
|
+
logout Remove the saved key.
|
|
309
|
+
account Plan, credits and features.
|
|
310
|
+
scrape <url> Page as Markdown (default) or other formats.
|
|
311
|
+
-f, --format LIST markdown,text,links,metadata,html (comma-separated)
|
|
312
|
+
--render auto|true|false Use a browser: when needed (default), always, never
|
|
313
|
+
--captcha auto|always|off What to do when a captcha blocks the page (default auto)
|
|
314
|
+
--full Keep navigation, headers and footers in markdown/text
|
|
315
|
+
--engine NAME puppeteer or playwright
|
|
316
|
+
--json Print the full API response
|
|
317
|
+
-o, --output-file FILE Save instead of printing
|
|
318
|
+
--input-file FILE|- Batch: one URL per line (- reads stdin)
|
|
319
|
+
--output-dir DIR Batch output folder (default browserlessapi-<time>)
|
|
320
|
+
-c, --concurrency N Batch: pages at once (default 3)
|
|
321
|
+
screenshot <url> Save a PNG screenshot. -o FILE, --wait-for SELECTOR
|
|
322
|
+
steps <file|json|-> Run browser steps (BrowserQL). --output-dir for screenshots
|
|
323
|
+
captcha --task JSON Solve a captcha (CapMonster task). --task-file FILE, --timeout SEC
|
|
324
|
+
mcp [client] Setup for Claude, ChatGPT, Cursor, VS Code, Codex, NeonCodex
|
|
325
|
+
|
|
326
|
+
Global
|
|
327
|
+
-k, --api-key KEY Use this key (else BROWSERLESSAPI_API_KEY, else the saved key)
|
|
328
|
+
-h, --help This help
|
|
329
|
+
-v, --version Version
|
|
330
|
+
|
|
331
|
+
Examples
|
|
332
|
+
browserlessapi scrape https://example.com
|
|
333
|
+
browserlessapi scrape https://example.com -f markdown,links,metadata
|
|
334
|
+
browserlessapi scrape --input-file urls.txt --output-dir pages -c 5
|
|
335
|
+
browserlessapi screenshot https://example.com -o example.png
|
|
336
|
+
browserlessapi steps '[{"action":"goto","url":"https://example.com"},{"action":"extract","selector":"h1","as":"title"}]'
|
|
337
|
+
browserlessapi captcha --task '{"type":"RecaptchaV2Task","websiteURL":"https://site/page","websiteKey":"SITE_KEY"}'
|
|
338
|
+
browserlessapi mcp claude-code
|
|
339
|
+
|
|
340
|
+
Each request uses credits from your account. Docs: ${BASE}/api-docs/`;
|
|
341
|
+
|
|
342
|
+
async function main() {
|
|
343
|
+
const { pos, opt } = parseArgs(process.argv.slice(2));
|
|
344
|
+
if (opt.version) return out(VERSION);
|
|
345
|
+
const cmd = pos.shift();
|
|
346
|
+
if (!cmd || opt.help || cmd === 'help') return out(HELP);
|
|
347
|
+
switch (cmd) {
|
|
348
|
+
case 'auth': return cmdAuth(opt);
|
|
349
|
+
case 'logout': return cmdLogout();
|
|
350
|
+
case 'account': case 'usage': case 'credits': return cmdAccount(opt);
|
|
351
|
+
case 'scrape': return cmdScrape(pos, opt);
|
|
352
|
+
case 'screenshot': return cmdScreenshot(pos, opt);
|
|
353
|
+
case 'steps': return cmdSteps(pos, opt);
|
|
354
|
+
case 'captcha': return cmdCaptcha(opt);
|
|
355
|
+
case 'mcp': return cmdMcp(pos, opt);
|
|
356
|
+
default: throw new UsageError(`Unknown command "${cmd}". Run "browserlessapi --help".`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
main().catch((e) => {
|
|
361
|
+
if (e instanceof UsageError) { err('Error: ' + e.message); process.exit(2); }
|
|
362
|
+
if (e instanceof ApiError) { err('Error: ' + e.message); process.exit(1); }
|
|
363
|
+
err('Error: ' + (e && e.message ? e.message : String(e)));
|
|
364
|
+
process.exit(1);
|
|
365
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "browserlessapi",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "BrowserlessAPI command-line tool: scrape any page to Markdown, take screenshots, run browser steps, solve captchas, and connect BrowserlessAPI to Claude, ChatGPT, Cursor and other AI apps over MCP.",
|
|
5
|
+
"bin": { "browserlessapi": "bin/browserlessapi.js" },
|
|
6
|
+
"files": ["bin/", "README.md", "LICENSE"],
|
|
7
|
+
"engines": { "node": ">=18" },
|
|
8
|
+
"keywords": ["web scraping", "scraper", "headless browser", "cloudflare", "captcha", "markdown", "mcp", "ai agents", "cli"],
|
|
9
|
+
"homepage": "https://browserlessapi.com/",
|
|
10
|
+
"author": "BrowserlessAPI (Nero Softwares & IT Services Pvt. Ltd.)",
|
|
11
|
+
"license": "MIT"
|
|
12
|
+
}
|