lobster-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +147 -269
- package/dist/browser/chrome-attach.js +102 -0
- package/dist/browser/chrome-attach.js.map +1 -0
- package/dist/browser/dom/compact-snapshot.js +162 -0
- package/dist/browser/dom/compact-snapshot.js.map +1 -0
- package/dist/browser/dom/index.js +160 -0
- package/dist/browser/dom/index.js.map +1 -1
- package/dist/browser/index.js +907 -70
- package/dist/browser/index.js.map +1 -1
- package/dist/browser/manager.js +443 -11
- package/dist/browser/manager.js.map +1 -1
- package/dist/browser/page-adapter.js +370 -1
- package/dist/browser/page-adapter.js.map +1 -1
- package/dist/browser/profiles.js +238 -0
- package/dist/browser/profiles.js.map +1 -0
- package/dist/browser/semantic-find.js +152 -0
- package/dist/browser/semantic-find.js.map +1 -0
- package/dist/browser/stealth.js +187 -0
- package/dist/browser/stealth.js.map +1 -0
- package/dist/config/index.js +8 -1
- package/dist/config/index.js.map +1 -1
- package/dist/config/schema.js +8 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/domain-guard.js +103 -0
- package/dist/domain-guard.js.map +1 -0
- package/dist/index.js +851 -48
- package/dist/index.js.map +1 -1
- package/dist/lib.js +1141 -244
- package/dist/lib.js.map +1 -1
- package/dist/router/index.js +862 -61
- package/dist/router/index.js.map +1 -1
- package/package.json +2 -1
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 iexcalibur
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
</p>
|
|
11
11
|
|
|
12
12
|
<p align="center">
|
|
13
|
+
<a href="https://www.npmjs.com/package/lobster-cli"><img src="https://img.shields.io/npm/v/lobster-cli?style=flat-square&color=c9a84c" alt="npm" /></a>
|
|
13
14
|
<img src="https://img.shields.io/badge/node-%3E%3D20-black?style=flat-square" />
|
|
14
15
|
<img src="https://img.shields.io/badge/license-MIT-black?style=flat-square" />
|
|
15
16
|
<img src="https://img.shields.io/badge/AI-optional-c9a84c?style=flat-square" />
|
|
@@ -17,373 +18,250 @@
|
|
|
17
18
|
|
|
18
19
|
---
|
|
19
20
|
|
|
20
|
-
##
|
|
21
|
+
## Why LobsterCLI
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
Every web automation tool today does one thing. Puppeteer gives you browser control — but no intelligence. LangChain gives you AI chains — but no browser. Scrapy gives you crawling — but no JavaScript. You end up wiring 3-4 tools together, writing glue code, and deciding which tool to use for each task.
|
|
23
24
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
| **Chrome Extension** | Side panel chat UI — analyze any page you're browsing | Load `extension/` folder in Chrome |
|
|
28
|
-
| **Library** | Import into your own Node.js project | `import { ... } from 'lobster-cli'` |
|
|
25
|
+
LobsterCLI is a single engine that does all of it. It has a **Smart Router** that automatically picks the best approach for your task — from a simple HTTP fetch to a full AI agent — so you never have to choose.
|
|
26
|
+
|
|
27
|
+
**What makes it unique:**
|
|
29
28
|
|
|
30
|
-
|
|
29
|
+
- **One tool, not four.** Fetch, scrape, explore, and AI-navigate — all from `lobster`.
|
|
30
|
+
- **AI is optional.** 80% of features work without any API key. The free tier covers most use cases.
|
|
31
|
+
- **Smart Brain.** When you do use AI, the Brain classifies your intent and gathers only what's needed — no wasted tokens on screenshots when text is enough.
|
|
32
|
+
- **Three interfaces, one engine.** CLI for developers, Chrome extension for everyone, npm library for builders. Same core, same results.
|
|
33
|
+
- **Persistent sessions.** Login once with `--profile`, your cookies and auth survive forever. Attach to your running Chrome with `--attach` — use your real logged-in sessions.
|
|
34
|
+
- **Stealth built-in.** Anti-bot detection out of the box. No separate puppeteer-stealth plugin, no extra config.
|
|
35
|
+
- **Domain restrictions.** Build vertical products on top — a finance tool that only works on Bloomberg, a legal tool that only works on case databases. One config line.
|
|
36
|
+
- **Actually works on modern sites.** Full React/Vue event simulation, shadow DOM, iframe extraction, dynamic content detection. Not just static HTML scraping.
|
|
31
37
|
|
|
32
38
|
---
|
|
33
39
|
|
|
34
|
-
##
|
|
40
|
+
## What is LobsterCLI
|
|
35
41
|
|
|
36
|
-
|
|
42
|
+
LobsterCLI is a web automation engine that works in three ways:
|
|
37
43
|
|
|
38
|
-
|
|
44
|
+
| Product | What it is | Install |
|
|
45
|
+
|---------|-----------|---------|
|
|
46
|
+
| **CLI** | Terminal tool — fetch, scrape, explore, automate | `npm install -g lobster-cli` |
|
|
47
|
+
| **Chrome Extension** | Side panel chat UI — analyze any page you're browsing | Load `extension/` in Chrome |
|
|
48
|
+
| **Library** | Import into your own Node.js project | `npm install lobster-cli` |
|
|
39
49
|
|
|
40
|
-
|
|
41
|
-
|---------|-----------|-------------|
|
|
42
|
-
| `lobster fetch <url>` | **No** | Fetch page, extract as markdown/text/snapshot/HTML. Uses in-house parser, no Chrome needed. |
|
|
43
|
-
| `lobster fetch <url> -e chrome` | **No** | Same but with full Chrome for JS-heavy pages. |
|
|
44
|
-
| `lobster explore <url>` | **No** | Discover APIs, intercept network calls, detect frameworks, generate adapters. |
|
|
45
|
-
| `lobster run <url>` | **No** | Run pre-built site adapters and YAML pipelines. |
|
|
46
|
-
| `lobster list` | **No** | List all registered adapters. |
|
|
47
|
-
| `lobster config` | **No** | View/edit settings. |
|
|
48
|
-
| `lobster doctor` | **No** | Diagnose setup, check Chrome, verify API key. |
|
|
49
|
-
| `lobster setup` | **No** | Interactive setup wizard (AI provider selection is optional). |
|
|
50
|
-
| `lobster plugin install` | **No** | Install community adapters from GitHub. |
|
|
51
|
-
| `lobster agent "task"` | **Yes** | AI agent that reads pages, reasons, clicks, types, and navigates autonomously. |
|
|
50
|
+
All three share the same core engine. **80% of features work without any AI key.**
|
|
52
51
|
|
|
53
|
-
|
|
52
|
+
---
|
|
54
53
|
|
|
55
|
-
|
|
54
|
+
## Install
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
| Click **"Extract as Markdown"** | **No** | Converts full page DOM to clean Markdown. Copy to clipboard. |
|
|
61
|
-
| Click **"Detect all forms"** | **No** | Scans all form fields — labels, types, values, required/disabled state. |
|
|
62
|
-
| Click **"Show key links"** | **No** | Extracts all meaningful links from the page. |
|
|
63
|
-
| Click **"Monitor API calls"** | **No** | Intercepts fetch/XHR, shows live API calls with method/URL/status. |
|
|
64
|
-
| Click **"DOM snapshot"** | **No** | 12-stage pruned DOM snapshot, LLM-optimized. |
|
|
65
|
-
| Type any question | **Yes** | Brain classifies intent, gathers right data, sends to LLM for answer. |
|
|
66
|
-
| Click **"What's on screen?"** | **Yes** | Captures screenshot + sends to vision model for visual analysis. |
|
|
56
|
+
```bash
|
|
57
|
+
npm install -g lobster-cli
|
|
58
|
+
```
|
|
67
59
|
|
|
68
|
-
|
|
60
|
+
Or clone and build:
|
|
69
61
|
|
|
70
|
-
|
|
62
|
+
```bash
|
|
63
|
+
git clone https://github.com/iexcalibur/lobster-cli.git
|
|
64
|
+
cd lobster-cli
|
|
65
|
+
npm install && npm run build && npm link
|
|
66
|
+
```
|
|
71
67
|
|
|
72
|
-
|
|
68
|
+
Requires Node.js 20+. Chrome/Chromium needed only for browser features.
|
|
73
69
|
|
|
74
|
-
|
|
75
|
-
"summarize this page" → Brain: { screenshot: false, markdown: true }
|
|
76
|
-
Cost: ~$0.001 (text only)
|
|
70
|
+
---
|
|
77
71
|
|
|
78
|
-
|
|
79
|
-
Cost: ~$0.001 (text only)
|
|
72
|
+
## Quick Start
|
|
80
73
|
|
|
81
|
-
|
|
82
|
-
|
|
74
|
+
```bash
|
|
75
|
+
# Extract content from any page (no AI, no Chrome)
|
|
76
|
+
lobster fetch https://example.com
|
|
83
77
|
|
|
84
|
-
|
|
85
|
-
|
|
78
|
+
# Discover hidden APIs on a site
|
|
79
|
+
lobster explore https://reddit.com
|
|
86
80
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
```
|
|
81
|
+
# AI agent navigates autonomously (needs API key)
|
|
82
|
+
lobster agent "find pricing on example.com" --url https://example.com
|
|
90
83
|
|
|
91
|
-
|
|
84
|
+
# Use stealth mode to avoid bot detection
|
|
85
|
+
lobster fetch https://linkedin.com --stealth
|
|
92
86
|
|
|
93
|
-
|
|
87
|
+
# Use a persistent profile (cookies survive restarts)
|
|
88
|
+
lobster agent "check my notifications" --url https://github.com --profile work
|
|
94
89
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
import { PuppeteerPage } from 'lobster-cli/page' // No AI needed
|
|
98
|
-
import { MARKDOWN_SCRIPT } from 'lobster-cli/dom' // No AI needed
|
|
99
|
-
import { exploreSite } from 'lobster-cli/discover' // No AI needed
|
|
100
|
-
import { AgentCore } from 'lobster-cli/agent' // Needs AI
|
|
101
|
-
import { LLM } from 'lobster-cli/llm' // Needs AI
|
|
90
|
+
# Attach to your running Chrome (use your logged-in sessions)
|
|
91
|
+
lobster agent "check inbox" --url https://gmail.com --attach
|
|
102
92
|
```
|
|
103
93
|
|
|
104
94
|
---
|
|
105
95
|
|
|
106
|
-
##
|
|
96
|
+
## What's Free vs What Needs AI
|
|
107
97
|
|
|
108
98
|
### CLI
|
|
109
99
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
Requires Node.js 20+. Chrome/Chromium needed only for browser features.
|
|
100
|
+
| Command | AI? | What it does |
|
|
101
|
+
|---------|-----|-------------|
|
|
102
|
+
| `lobster fetch <url>` | No | Extract as markdown, text, snapshot, HTML, semantic tree |
|
|
103
|
+
| `lobster explore <url>` | No | Discover APIs, detect frameworks, generate adapters |
|
|
104
|
+
| `lobster run <url>` | No | Run site adapters and YAML pipelines |
|
|
105
|
+
| `lobster list` | No | List registered adapters |
|
|
106
|
+
| `lobster config` | No | View/edit settings |
|
|
107
|
+
| `lobster doctor` | No | Diagnose setup |
|
|
108
|
+
| `lobster setup` | No | Interactive setup wizard |
|
|
109
|
+
| `lobster plugin install` | No | Install community adapters |
|
|
110
|
+
| `lobster agent "task"` | **Yes** | AI agent that reasons, clicks, types, navigates |
|
|
123
111
|
|
|
124
112
|
### Chrome Extension
|
|
125
113
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
```typescript
|
|
139
|
-
import { classifyIntent, heuristicClassify } from 'lobster-cli/brain'
|
|
140
|
-
import { PuppeteerPage, BrowserManager } from 'lobster-cli/browser'
|
|
141
|
-
import { SNAPSHOT_SCRIPT, MARKDOWN_SCRIPT } from 'lobster-cli/dom'
|
|
142
|
-
import { AgentCore } from 'lobster-cli/agent'
|
|
143
|
-
import { exploreSite } from 'lobster-cli/discover'
|
|
144
|
-
import { executePipeline } from 'lobster-cli/pipeline'
|
|
145
|
-
import { LLM } from 'lobster-cli/llm'
|
|
146
|
-
```
|
|
114
|
+
| Action | AI? | What it does |
|
|
115
|
+
|--------|-----|-------------|
|
|
116
|
+
| Summarize this page | No | Headings, word count, content preview, framework detection |
|
|
117
|
+
| Extract as Markdown | No | Full DOM-to-Markdown with copy button |
|
|
118
|
+
| Detect all forms | No | Labels, types, values, required/disabled state |
|
|
119
|
+
| Show key links | No | All meaningful links on the page |
|
|
120
|
+
| Monitor API calls | No | Live fetch/XHR interception |
|
|
121
|
+
| DOM snapshot | No | 12-stage pruned snapshot |
|
|
122
|
+
| Type any question | **Yes** | Brain classifies intent, gathers data, sends to LLM |
|
|
123
|
+
| What's on screen? | **Yes** | Captures screenshot + vision model analysis |
|
|
147
124
|
|
|
148
125
|
---
|
|
149
126
|
|
|
150
127
|
## AI Setup (optional)
|
|
151
128
|
|
|
152
|
-
AI is only needed for `lobster agent` and typed chat questions in the extension.
|
|
153
|
-
|
|
154
129
|
```bash
|
|
155
130
|
lobster setup
|
|
156
131
|
```
|
|
157
132
|
|
|
158
|
-
| Provider |
|
|
159
|
-
|
|
160
|
-
| **Google Gemini** | gemini-2.5-flash |
|
|
161
|
-
| **OpenAI** | gpt-4o | Pay per token |
|
|
162
|
-
| **Anthropic** | claude-sonnet-4 | Pay per token |
|
|
163
|
-
| **Ollama** | llama3.1 |
|
|
164
|
-
|
|
165
|
-
Or set manually:
|
|
166
|
-
|
|
167
|
-
```bash
|
|
168
|
-
lobster config set llm.provider gemini
|
|
169
|
-
lobster config set llm.apiKey AIza...
|
|
170
|
-
lobster config set llm.model gemini-2.5-flash
|
|
171
|
-
```
|
|
133
|
+
| Provider | Model | Cost |
|
|
134
|
+
|----------|-------|------|
|
|
135
|
+
| **Google Gemini** | gemini-2.5-flash | Free tier available |
|
|
136
|
+
| **OpenAI** | gpt-4o | Pay per token |
|
|
137
|
+
| **Anthropic** | claude-sonnet-4 | Pay per token |
|
|
138
|
+
| **Ollama** | llama3.1 | Free (runs locally) |
|
|
172
139
|
|
|
173
140
|
---
|
|
174
141
|
|
|
175
|
-
##
|
|
142
|
+
## Key Features
|
|
176
143
|
|
|
177
|
-
###
|
|
144
|
+
### Smart Brain
|
|
145
|
+
When you ask a question, the Brain classifies your intent and gathers only what's needed:
|
|
178
146
|
|
|
179
|
-
```bash
|
|
180
|
-
lobster fetch https://example.com # markdown output
|
|
181
|
-
lobster fetch https://example.com -d snapshot # LLM-optimized DOM
|
|
182
|
-
lobster fetch https://example.com -d text # plain text
|
|
183
|
-
lobster fetch https://example.com -d semantic # W3C accessible tree
|
|
184
|
-
lobster fetch https://example.com -d html # raw HTML
|
|
185
|
-
lobster fetch https://example.com -e chrome -w 5 # force Chrome, wait 5s
|
|
186
147
|
```
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
### `lobster run <url>` — Smart router (no AI)
|
|
191
|
-
|
|
192
|
-
```bash
|
|
193
|
-
lobster run https://api.github.com/users/octocat # direct HTTP fetch
|
|
194
|
-
lobster run https://news.ycombinator.com # adapter if registered
|
|
195
|
-
lobster run https://example.com -f yaml # output as YAML
|
|
148
|
+
"summarize this page" → reads text only (~$0.001)
|
|
149
|
+
"what images are showing" → captures screenshot + text (~$0.01)
|
|
150
|
+
"what forms are on this page" → extracts form data (~$0.001)
|
|
196
151
|
```
|
|
197
152
|
|
|
198
|
-
###
|
|
153
|
+
### Persistent Profiles
|
|
154
|
+
Store Chrome sessions that survive restarts — cookies, auth, extensions.
|
|
199
155
|
|
|
200
156
|
```bash
|
|
201
|
-
lobster
|
|
202
|
-
lobster agent "
|
|
203
|
-
lobster agent "log in and check my notifications" --url https://github.com
|
|
157
|
+
lobster profile create work
|
|
158
|
+
lobster agent "check gmail" --profile work # login persists
|
|
204
159
|
```
|
|
205
160
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
### `lobster explore <url>` — Discover APIs (no AI)
|
|
161
|
+
### Chrome Attach
|
|
162
|
+
Connect to your running Chrome with all your logins.
|
|
209
163
|
|
|
210
164
|
```bash
|
|
211
|
-
lobster
|
|
212
|
-
lobster explore https://twitter.com -w 5
|
|
165
|
+
lobster agent "check notifications" --attach
|
|
213
166
|
```
|
|
214
167
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
### `lobster config` — Settings
|
|
168
|
+
### Stealth Mode
|
|
169
|
+
Anti-bot detection — hides headless Chrome fingerprint.
|
|
218
170
|
|
|
219
171
|
```bash
|
|
220
|
-
lobster
|
|
221
|
-
lobster config set llm.provider anthropic
|
|
222
|
-
lobster config set browser.headless false
|
|
172
|
+
lobster fetch https://protected-site.com --stealth
|
|
223
173
|
```
|
|
224
174
|
|
|
225
|
-
###
|
|
175
|
+
### Compact Snapshot
|
|
176
|
+
Token-efficient DOM snapshots (~800 tokens vs ~3000+).
|
|
226
177
|
|
|
227
178
|
```bash
|
|
228
|
-
lobster
|
|
229
|
-
lobster plugin list
|
|
179
|
+
lobster fetch https://example.com -d snapshot --compact
|
|
230
180
|
```
|
|
231
181
|
|
|
232
|
-
|
|
182
|
+
### Semantic Element Finding
|
|
183
|
+
AI agent finds elements by name instead of guessing index numbers.
|
|
233
184
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
### Free features (no API key)
|
|
185
|
+
```
|
|
186
|
+
"click the login button" → finds [1] button "Sign In" (score: 0.81)
|
|
187
|
+
```
|
|
239
188
|
|
|
240
|
-
|
|
189
|
+
### Domain Guard
|
|
190
|
+
Restrict which websites the engine can operate on.
|
|
241
191
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
- **Detect all forms** — every form field with label, type, value, required state
|
|
245
|
-
- **Show key links** — all meaningful links on the page
|
|
246
|
-
- **Monitor API calls** — live fetch/XHR interception
|
|
247
|
-
- **DOM snapshot** — 12-stage pruned snapshot
|
|
192
|
+
```typescript
|
|
193
|
+
import { DomainGuard } from 'lobster-cli/domain-guard'
|
|
248
194
|
|
|
249
|
-
|
|
195
|
+
const guard = new DomainGuard({
|
|
196
|
+
allowDomains: ['bloomberg.com', 'yahoo.com'],
|
|
197
|
+
blockMessage: 'This tool only works on finance sites.',
|
|
198
|
+
})
|
|
199
|
+
```
|
|
250
200
|
|
|
251
|
-
|
|
201
|
+
### Site Explorer
|
|
202
|
+
Discover hidden APIs on any website — no AI needed.
|
|
252
203
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
- "What color is the header?" — captures screenshot, analyzes visually
|
|
204
|
+
```bash
|
|
205
|
+
lobster explore https://reddit.com
|
|
206
|
+
```
|
|
257
207
|
|
|
258
|
-
|
|
208
|
+
Intercepts network calls, clicks buttons to trigger hidden APIs, detects frameworks, scores endpoints, and generates YAML adapters.
|
|
259
209
|
|
|
260
|
-
###
|
|
210
|
+
### Pipeline Engine
|
|
211
|
+
Declarative YAML pipelines with 17 steps and template expressions.
|
|
261
212
|
|
|
262
|
-
|
|
213
|
+
```yaml
|
|
214
|
+
steps:
|
|
215
|
+
- navigate: https://news.ycombinator.com
|
|
216
|
+
- evaluate: |
|
|
217
|
+
[...document.querySelectorAll('.titleline > a')]
|
|
218
|
+
.map(a => ({ title: a.textContent, url: a.href }))
|
|
219
|
+
- limit: 10
|
|
220
|
+
```
|
|
263
221
|
|
|
264
222
|
---
|
|
265
223
|
|
|
266
|
-
## Use as a
|
|
224
|
+
## Use as a Library
|
|
267
225
|
|
|
268
|
-
|
|
226
|
+
```bash
|
|
227
|
+
npm install lobster-cli
|
|
228
|
+
```
|
|
269
229
|
|
|
270
230
|
```typescript
|
|
271
|
-
|
|
272
|
-
import { classifyIntent, heuristicClassify } from 'lobster-cli/brain'
|
|
273
|
-
|
|
274
|
-
// Browser — page control
|
|
231
|
+
import { classifyIntent } from 'lobster-cli/brain'
|
|
275
232
|
import { BrowserManager, PuppeteerPage } from 'lobster-cli/browser'
|
|
276
|
-
|
|
277
|
-
// DOM scripts — run in any browser context
|
|
278
|
-
import { SNAPSHOT_SCRIPT, MARKDOWN_SCRIPT, FORM_STATE_SCRIPT } from 'lobster-cli/dom'
|
|
279
|
-
|
|
280
|
-
// Agent — autonomous web navigation
|
|
233
|
+
import { SNAPSHOT_SCRIPT, MARKDOWN_SCRIPT } from 'lobster-cli/dom'
|
|
281
234
|
import { AgentCore } from 'lobster-cli/agent'
|
|
282
|
-
|
|
283
|
-
// LLM — multi-provider client
|
|
284
|
-
import { LLM } from 'lobster-cli/llm'
|
|
285
|
-
|
|
286
|
-
// Pipeline — declarative YAML execution
|
|
287
|
-
import { executePipeline } from 'lobster-cli/pipeline'
|
|
288
|
-
|
|
289
|
-
// Discovery — find site APIs
|
|
235
|
+
import { DomainGuard } from 'lobster-cli/domain-guard'
|
|
290
236
|
import { exploreSite } from 'lobster-cli/discover'
|
|
291
|
-
|
|
292
|
-
// Config — load/save settings
|
|
293
|
-
import { loadConfig, saveConfig } from 'lobster-cli/config'
|
|
294
|
-
```
|
|
295
|
-
|
|
296
|
-
### Example: build a search agent
|
|
297
|
-
|
|
298
|
-
```typescript
|
|
299
|
-
import { BrowserManager } from 'lobster-cli/browser'
|
|
300
|
-
import { AgentCore } from 'lobster-cli/agent'
|
|
237
|
+
import { executePipeline } from 'lobster-cli/pipeline'
|
|
301
238
|
import { LLM } from 'lobster-cli/llm'
|
|
302
|
-
import { classifyIntent } from 'lobster-cli/brain'
|
|
303
|
-
|
|
304
|
-
async function search(query) {
|
|
305
|
-
// Brain decides what data is needed
|
|
306
|
-
const intent = await classifyIntent(query, 'Google Search')
|
|
307
|
-
|
|
308
|
-
// Launch browser
|
|
309
|
-
const browser = new BrowserManager({ headless: true })
|
|
310
|
-
const page = await browser.launch('https://google.com')
|
|
311
|
-
|
|
312
|
-
// Run agent
|
|
313
|
-
const agent = new AgentCore({ page, llm: new LLM(config), maxSteps: 20 })
|
|
314
|
-
const result = await agent.execute(query)
|
|
315
|
-
|
|
316
|
-
await page.close()
|
|
317
|
-
return result
|
|
318
|
-
}
|
|
319
239
|
```
|
|
320
240
|
|
|
321
241
|
---
|
|
322
242
|
|
|
323
|
-
##
|
|
324
|
-
|
|
325
|
-
```
|
|
326
|
-
lobster-cli/
|
|
327
|
-
├── src/
|
|
328
|
-
│ ├── brain/ → Intent classifier (LLM + heuristic fallback)
|
|
329
|
-
│ ├── browser/ → IPage interface, Puppeteer adapter, DOM scripts
|
|
330
|
-
│ │ └── dom/ → 6 extraction strategies (snapshot, markdown, semantic, etc.)
|
|
331
|
-
│ ├── agent/ → Observe-think-act loop, 8 tools, auto-fixer
|
|
332
|
-
│ ├── llm/ → Multi-provider client (OpenAI, Anthropic, Gemini, Ollama)
|
|
333
|
-
│ ├── pipeline/ → YAML pipeline engine, 17 steps, template expressions
|
|
334
|
-
│ ├── adapter/ → Site adapter registry, YAML/TS loaders
|
|
335
|
-
│ ├── router/ → Smart routing (HTTP → Engine → Adapter → Agent)
|
|
336
|
-
│ ├── discover/ → API discovery, endpoint scoring, adapter generation
|
|
337
|
-
│ ├── cascade/ → Auth strategy detection (public → cookie → header → intercept)
|
|
338
|
-
│ ├── config/ → Settings (~/.lobster/config.yaml)
|
|
339
|
-
│ ├── output/ → Formatters (table, JSON, YAML, CSV, Markdown)
|
|
340
|
-
│ ├── plugin/ → GitHub plugin install/uninstall
|
|
341
|
-
│ ├── lib.ts → Library exports (for npm import)
|
|
342
|
-
│ ├── cli.ts → CLI commands (commander)
|
|
343
|
-
│ └── index.ts → CLI entry point
|
|
344
|
-
│
|
|
345
|
-
├── extension/ → Chrome extension (side panel)
|
|
346
|
-
│ ├── sidepanel/ → Chat UI (HTML/CSS/JS)
|
|
347
|
-
│ ├── background/ → Service worker (LLM calls, screenshot capture)
|
|
348
|
-
│ ├── shared/ → DOM scripts (same code as CLI, ported to browser JS)
|
|
349
|
-
│ ├── options/ → Settings page
|
|
350
|
-
│ └── manifest.json → Chrome extension manifest (v3, side panel API)
|
|
351
|
-
│
|
|
352
|
-
├── logo.svg → Logo (SVG)
|
|
353
|
-
├── logo.png → Logo (512px PNG)
|
|
354
|
-
└── package.json → Dual: CLI binary + library exports
|
|
355
|
-
```
|
|
243
|
+
## Chrome Extension
|
|
356
244
|
|
|
357
|
-
|
|
245
|
+
Chat-style side panel docked to the right of your browser (like Gemini or Claude).
|
|
358
246
|
|
|
359
|
-
|
|
247
|
+
**Install:**
|
|
248
|
+
1. Clone this repo
|
|
249
|
+
2. Open `chrome://extensions/` → enable Developer mode
|
|
250
|
+
3. Click Load unpacked → select `extension/` folder
|
|
251
|
+
4. Click LobsterCLI icon → side panel opens
|
|
360
252
|
|
|
361
|
-
|
|
362
|
-
|----------|---------|
|
|
363
|
-
| `LOBSTER_API_KEY` | LLM API key (overrides config) |
|
|
364
|
-
| `LOBSTER_MODEL` | LLM model name |
|
|
365
|
-
| `LOBSTER_BASE_URL` | LLM API base URL |
|
|
366
|
-
| `LOBSTER_CDP_ENDPOINT` | Chrome DevTools Protocol endpoint |
|
|
367
|
-
| `LOBSTER_BROWSER_PATH` | Path to Chrome/Chromium binary |
|
|
253
|
+
**AI setup:** Click gear icon → Settings → pick provider → enter key → Save.
|
|
368
254
|
|
|
369
255
|
---
|
|
370
256
|
|
|
371
|
-
##
|
|
372
|
-
|
|
373
|
-
| Package | Purpose |
|
|
374
|
-
|---------|---------|
|
|
375
|
-
| `commander` | CLI framework |
|
|
376
|
-
| `puppeteer-core` | Chrome control (no bundled Chrome) |
|
|
377
|
-
| `zod` | Schema validation |
|
|
378
|
-
| `chalk` | Terminal colors |
|
|
379
|
-
| `cli-table3` | Table formatting |
|
|
380
|
-
| `js-yaml` | YAML parsing |
|
|
381
|
-
| `ws` | WebSocket |
|
|
257
|
+
## Documentation
|
|
382
258
|
|
|
383
|
-
|
|
259
|
+
For detailed technical documentation, see **[DOCS.md](DOCS.md)**.
|
|
384
260
|
|
|
385
261
|
---
|
|
386
262
|
|
|
387
263
|
## License
|
|
388
264
|
|
|
389
|
-
MIT
|
|
265
|
+
MIT License — Copyright (c) 2025 iexcalibur
|
|
266
|
+
|
|
267
|
+
See [LICENSE](LICENSE) for full text.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// src/browser/chrome-attach.ts
|
|
2
|
+
import http from "http";
|
|
3
|
+
|
|
4
|
+
// src/utils/logger.ts
|
|
5
|
+
import chalk from "chalk";
|
|
6
|
+
var log = {
|
|
7
|
+
info: (msg) => console.log(chalk.blue("\u2139"), msg),
|
|
8
|
+
success: (msg) => console.log(chalk.green("\u2713"), msg),
|
|
9
|
+
warn: (msg) => console.log(chalk.yellow("\u26A0"), msg),
|
|
10
|
+
error: (msg) => console.error(chalk.red("\u2717"), msg),
|
|
11
|
+
debug: (msg) => {
|
|
12
|
+
if (process.env.LOBSTER_DEBUG) console.log(chalk.gray("\u22EF"), msg);
|
|
13
|
+
},
|
|
14
|
+
step: (n, msg) => console.log(chalk.cyan(`[${n}]`), msg),
|
|
15
|
+
dim: (msg) => console.log(chalk.dim(msg))
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
// src/browser/chrome-attach.ts
|
|
19
|
+
var DEFAULT_PORTS = [9222, 9229, 9333, 9515];
|
|
20
|
+
var PROBE_TIMEOUT = 1500;
|
|
21
|
+
function probePort(port) {
|
|
22
|
+
return new Promise((resolve) => {
|
|
23
|
+
const req = http.get(`http://127.0.0.1:${port}/json/version`, {
|
|
24
|
+
timeout: PROBE_TIMEOUT
|
|
25
|
+
}, (res) => {
|
|
26
|
+
let data = "";
|
|
27
|
+
res.on("data", (chunk) => {
|
|
28
|
+
data += chunk;
|
|
29
|
+
});
|
|
30
|
+
res.on("end", () => {
|
|
31
|
+
try {
|
|
32
|
+
const info = JSON.parse(data);
|
|
33
|
+
if (info.webSocketDebuggerUrl) {
|
|
34
|
+
resolve({
|
|
35
|
+
wsEndpoint: info.webSocketDebuggerUrl,
|
|
36
|
+
port,
|
|
37
|
+
version: info["Protocol-Version"] || "",
|
|
38
|
+
browser: info.Browser || ""
|
|
39
|
+
});
|
|
40
|
+
} else {
|
|
41
|
+
resolve(null);
|
|
42
|
+
}
|
|
43
|
+
} catch {
|
|
44
|
+
resolve(null);
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
req.on("error", () => resolve(null));
|
|
49
|
+
req.on("timeout", () => {
|
|
50
|
+
req.destroy();
|
|
51
|
+
resolve(null);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
async function discoverChrome(ports) {
|
|
56
|
+
const portsToCheck = ports || DEFAULT_PORTS;
|
|
57
|
+
log.debug(`Scanning ports for Chrome: ${portsToCheck.join(", ")}`);
|
|
58
|
+
const results = await Promise.all(portsToCheck.map(probePort));
|
|
59
|
+
const found = results.find(Boolean) || null;
|
|
60
|
+
if (found) {
|
|
61
|
+
log.info(`Found Chrome on port ${found.port}: ${found.browser}`);
|
|
62
|
+
} else {
|
|
63
|
+
log.debug("No running Chrome instance found on debug ports.");
|
|
64
|
+
}
|
|
65
|
+
return found;
|
|
66
|
+
}
|
|
67
|
+
async function getWebSocketDebuggerUrl(port) {
|
|
68
|
+
const result = await probePort(port);
|
|
69
|
+
return result?.wsEndpoint || null;
|
|
70
|
+
}
|
|
71
|
+
async function resolveAttachTarget(target) {
|
|
72
|
+
if (target === true || target === "true") {
|
|
73
|
+
const result = await discoverChrome();
|
|
74
|
+
if (!result) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
"No running Chrome found. Start Chrome with:\n google-chrome --remote-debugging-port=9222\n # or on Mac:\n /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222"
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
return result.wsEndpoint;
|
|
80
|
+
}
|
|
81
|
+
if (typeof target === "string") {
|
|
82
|
+
if (target.startsWith("ws://") || target.startsWith("wss://")) {
|
|
83
|
+
return target;
|
|
84
|
+
}
|
|
85
|
+
const port = parseInt(target, 10);
|
|
86
|
+
if (!isNaN(port) && port > 0 && port < 65536) {
|
|
87
|
+
const url = await getWebSocketDebuggerUrl(port);
|
|
88
|
+
if (!url) {
|
|
89
|
+
throw new Error(`No Chrome found on port ${port}. Make sure Chrome is running with --remote-debugging-port=${port}`);
|
|
90
|
+
}
|
|
91
|
+
return url;
|
|
92
|
+
}
|
|
93
|
+
throw new Error(`Invalid attach target: "${target}". Use "true" for auto-discover, a port number, or a ws:// URL.`);
|
|
94
|
+
}
|
|
95
|
+
throw new Error("Invalid attach target.");
|
|
96
|
+
}
|
|
97
|
+
export {
|
|
98
|
+
discoverChrome,
|
|
99
|
+
getWebSocketDebuggerUrl,
|
|
100
|
+
resolveAttachTarget
|
|
101
|
+
};
|
|
102
|
+
//# sourceMappingURL=chrome-attach.js.map
|