imperium-crawl 1.5.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -32
- package/dist/constants.d.ts +2 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +3 -1
- package/dist/constants.js.map +1 -1
- package/dist/network/interceptor.d.ts +19 -0
- package/dist/network/interceptor.d.ts.map +1 -0
- package/dist/network/interceptor.js +82 -0
- package/dist/network/interceptor.js.map +1 -0
- package/dist/network/types.d.ts +27 -0
- package/dist/network/types.d.ts.map +1 -0
- package/dist/network/types.js +2 -0
- package/dist/network/types.js.map +1 -0
- package/dist/security/action-policy.d.ts +26 -0
- package/dist/security/action-policy.d.ts.map +1 -0
- package/dist/security/action-policy.js +136 -0
- package/dist/security/action-policy.js.map +1 -0
- package/dist/security/auth-vault.d.ts +49 -0
- package/dist/security/auth-vault.d.ts.map +1 -0
- package/dist/security/auth-vault.js +133 -0
- package/dist/security/auth-vault.js.map +1 -0
- package/dist/security/domain-filter.d.ts +19 -0
- package/dist/security/domain-filter.d.ts.map +1 -0
- package/dist/security/domain-filter.js +114 -0
- package/dist/security/domain-filter.js.map +1 -0
- package/dist/security/types.d.ts +19 -0
- package/dist/security/types.d.ts.map +1 -0
- package/dist/security/types.js +2 -0
- package/dist/security/types.js.map +1 -0
- package/dist/sessions/encryption.d.ts +37 -0
- package/dist/sessions/encryption.d.ts.map +1 -0
- package/dist/sessions/encryption.js +108 -0
- package/dist/sessions/encryption.js.map +1 -0
- package/dist/sessions/index.d.ts +1 -0
- package/dist/sessions/index.d.ts.map +1 -1
- package/dist/sessions/index.js +1 -0
- package/dist/sessions/index.js.map +1 -1
- package/dist/sessions/manager.d.ts +3 -0
- package/dist/sessions/manager.d.ts.map +1 -1
- package/dist/sessions/manager.js +28 -2
- package/dist/sessions/manager.js.map +1 -1
- package/dist/snapshot/annotator.d.ts +21 -0
- package/dist/snapshot/annotator.d.ts.map +1 -0
- package/dist/snapshot/annotator.js +152 -0
- package/dist/snapshot/annotator.js.map +1 -0
- package/dist/snapshot/boundary.d.ts +7 -0
- package/dist/snapshot/boundary.d.ts.map +1 -0
- package/dist/snapshot/boundary.js +12 -0
- package/dist/snapshot/boundary.js.map +1 -0
- package/dist/snapshot/differ.d.ts +40 -0
- package/dist/snapshot/differ.d.ts.map +1 -0
- package/dist/snapshot/differ.js +194 -0
- package/dist/snapshot/differ.js.map +1 -0
- package/dist/snapshot/extractor.d.ts +27 -0
- package/dist/snapshot/extractor.d.ts.map +1 -0
- package/dist/snapshot/extractor.js +265 -0
- package/dist/snapshot/extractor.js.map +1 -0
- package/dist/snapshot/index.d.ts +8 -0
- package/dist/snapshot/index.d.ts.map +1 -0
- package/dist/snapshot/index.js +6 -0
- package/dist/snapshot/index.js.map +1 -0
- package/dist/snapshot/store.d.ts +28 -0
- package/dist/snapshot/store.d.ts.map +1 -0
- package/dist/snapshot/store.js +65 -0
- package/dist/snapshot/store.js.map +1 -0
- package/dist/snapshot/types.d.ts +42 -0
- package/dist/snapshot/types.d.ts.map +1 -0
- package/dist/snapshot/types.js +2 -0
- package/dist/snapshot/types.js.map +1 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +2 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/interact.d.ts +194 -5
- package/dist/tools/interact.d.ts.map +1 -1
- package/dist/tools/interact.js +355 -20
- package/dist/tools/interact.js.map +1 -1
- package/dist/tools/snapshot.d.ts +53 -0
- package/dist/tools/snapshot.d.ts.map +1 -0
- package/dist/tools/snapshot.js +160 -0
- package/dist/tools/snapshot.js.map +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
**The most powerful open-source MCP server for web scraping, crawling, and data extraction.**
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
23 tools. Zero API keys required. One `npx` command.
|
|
8
8
|
|
|
9
9
|
[](https://www.npmjs.com/package/imperium-crawl)
|
|
10
10
|
[](./LICENSE)
|
|
@@ -44,7 +44,7 @@ npx -y imperium-crawl scrape --url https://example.com
|
|
|
44
44
|
npm install -g imperium-crawl
|
|
45
45
|
```
|
|
46
46
|
|
|
47
|
-
> That's it.
|
|
47
|
+
> That's it. 17 of 23 tools work with zero API keys. Add optional keys later to unlock search, AI extraction, and CAPTCHA solving.
|
|
48
48
|
|
|
49
49
|
---
|
|
50
50
|
|
|
@@ -120,7 +120,7 @@ Scraping 4 URLs (concurrency: 3)...
|
|
|
120
120
|
## Why imperium-crawl?
|
|
121
121
|
|
|
122
122
|
🔓 **Zero API Keys Required**
|
|
123
|
-
|
|
123
|
+
17 of 23 tools work out of the box. No accounts, no tokens, no credit cards. Just `npx` and go.
|
|
124
124
|
|
|
125
125
|
🛡️ **3-Level Auto-Escalating Stealth**
|
|
126
126
|
Headers → TLS fingerprinting → headless browser + CAPTCHA solving. Automatically escalates until it gets through.
|
|
@@ -128,7 +128,7 @@ Headers → TLS fingerprinting → headless browser + CAPTCHA solving. Automatic
|
|
|
128
128
|
🧠 **Self-Improving**
|
|
129
129
|
Adaptive learning engine remembers what works per domain. Second visit is 3x faster. The more you use it, the smarter it gets.
|
|
130
130
|
|
|
131
|
-
🧰 **
|
|
131
|
+
🧰 **23 Tools, 3 Modes**
|
|
132
132
|
MCP server, CLI tool, or interactive TUI. Scraping, crawling, search, extraction, API discovery, WebSocket monitoring, browser automation, batch processing.
|
|
133
133
|
|
|
134
134
|
📜 **10 Built-in Recipes**
|
|
@@ -144,7 +144,7 @@ Teach it once, run forever. Auto-detect patterns on any page, save as reusable s
|
|
|
144
144
|
| Feature | **imperium-crawl** | Firecrawl MCP | fetch MCP | Crawl4AI MCP | Browserbase MCP |
|
|
145
145
|
|---------|:------------------:|:-------------:|:---------:|:------------:|:---------------:|
|
|
146
146
|
| Price | **Free forever** | $19+/month | Free | Free | $0.01/min |
|
|
147
|
-
| Total tools | **
|
|
147
|
+
| Total tools | **23** | 5 | 2 | 2 | 4 |
|
|
148
148
|
| Stealth levels | **3 (auto-escalate)** | Cloud-based | None | 1 | Cloud-based |
|
|
149
149
|
| Anti-bot detection | **7 systems** | Partial | None | Partial | Partial |
|
|
150
150
|
| TLS fingerprinting | **JA3/JA4** | No | No | No | No |
|
|
@@ -154,6 +154,10 @@ Teach it once, run forever. Auto-detect patterns on any page, save as reusable s
|
|
|
154
154
|
| AI-powered extraction | **Yes** | No | No | No | No |
|
|
155
155
|
| Adaptive learning | **Yes** | No | No | No | No |
|
|
156
156
|
| Batch processing | **Yes** | No | No | No | No |
|
|
157
|
+
| ARIA Snapshots | **Yes** | No | No | No | No |
|
|
158
|
+
| Session Encryption | **Yes** | No | No | No | No |
|
|
159
|
+
| Action Policy | **Yes** | No | No | No | No |
|
|
160
|
+
| Domain Sandboxing | **Yes** | No | No | No | No |
|
|
157
161
|
| Self-hosted | **Yes** | No | N/A | Yes | No |
|
|
158
162
|
| Requires external service | **No** | Yes | No | No | Yes |
|
|
159
163
|
|
|
@@ -235,7 +239,7 @@ Second visit to cloudflare.com:
|
|
|
235
239
|
|
|
236
240
|
---
|
|
237
241
|
|
|
238
|
-
## All
|
|
242
|
+
## All 23 Tools
|
|
239
243
|
|
|
240
244
|
### 📄 Scraping (no API key needed)
|
|
241
245
|
|
|
@@ -283,7 +287,8 @@ Second visit to cloudflare.com:
|
|
|
283
287
|
|
|
284
288
|
| Tool | What It Does |
|
|
285
289
|
|------|-------------|
|
|
286
|
-
| **interact** | Browser automation with
|
|
290
|
+
| **interact** | Browser automation with 18 action types (click, type, scroll, wait, screenshot, evaluate, select, hover, press, navigate, drag, upload, storage, cookies, pdf, auth_login). Ref targeting via ARIA snapshot, session encryption, action policy, domain filter, network interception, device emulation. |
|
|
291
|
+
| **snapshot** | ARIA-based page snapshot with interactive element refs. Use refs in interact for precise targeting. Annotated screenshots. |
|
|
287
292
|
|
|
288
293
|
### 📦 Batch Processing (no API key needed)
|
|
289
294
|
|
|
@@ -311,6 +316,7 @@ Full configuration with all optional environment variables:
|
|
|
311
316
|
"TWOCAPTCHA_API_KEY": "your-2captcha-api-key",
|
|
312
317
|
"LLM_API_KEY": "your-api-key",
|
|
313
318
|
"LLM_PROVIDER": "anthropic",
|
|
319
|
+
"SESSION_ENCRYPTION_KEY": "your-64-char-hex-key",
|
|
314
320
|
"PROXY_URL": "http://user:pass@proxy:8080",
|
|
315
321
|
"PROXY_URLS": "http://proxy1:8080,socks5://proxy2:1080"
|
|
316
322
|
}
|
|
@@ -455,7 +461,7 @@ Turn any website into an API. No documentation needed.
|
|
|
455
461
|
|
|
456
462
|
## AI Agent Guide
|
|
457
463
|
|
|
458
|
-
imperium-crawl ships with [`SKILL/`](./SKILL/) — a structured guide that teaches AI agents how to use all
|
|
464
|
+
imperium-crawl ships with [`SKILL/`](./SKILL/) — a structured guide that teaches AI agents how to use all 23 tools effectively. Includes proven workflows, decision trees, error recovery, and advanced patterns.
|
|
459
465
|
|
|
460
466
|
### Three Ways to Connect
|
|
461
467
|
|
|
@@ -494,30 +500,31 @@ Every tool tested against production websites with real anti-bot defenses:
|
|
|
494
500
|
|
|
495
501
|
| Tool | Target | Result |
|
|
496
502
|
|------|--------|--------|
|
|
497
|
-
| **scrape** | BBC News | Full markdown, stealth level 3 auto-escalation |
|
|
498
|
-
| **crawl** | Cloudflare Blog | 213K characters crawled with depth control |
|
|
499
|
-
| **map** | BBC | Full URL discovery via sitemap + link extraction |
|
|
500
|
-
| **extract** | Amazon (AirPods Pro 2) | Product title, 45,297 reviews, brand extracted |
|
|
501
|
-
| **readability** | Medium article | Clean — title, author, content, publish date |
|
|
502
|
-
| **screenshot** | ProductHunt | Captured Cloudflare Turnstile challenge page |
|
|
503
|
-
| **search** | Brave Web | Web results with snippets and URLs |
|
|
504
|
-
| **news_search** | Brave News | News results with freshness ranking |
|
|
505
|
-
| **image_search** | Brave Image | Images with thumbnails and source URLs |
|
|
506
|
-
| **video_search** | Brave Video | Video results across platforms |
|
|
507
|
-
| **create_skill** | Hacker News | Auto-detected 30 stories with CSS selectors |
|
|
508
|
-
| **run_skill** | Saved skill | Fresh structured data from saved config |
|
|
509
|
-
| **list_skills** | — | Lists all skills with configurations |
|
|
510
|
-
| **discover_apis** | Airbnb Paris | **34 hidden APIs** — DataDome, Google Maps key, internal APIs |
|
|
511
|
-
| **query_api** | jsonplaceholder | Direct JSON API call with stealth headers |
|
|
512
|
-
| **monitor_websocket** | Binance BTC/USDT | 3 WebSocket connections, 23 live messages — BTC price live |
|
|
513
|
-
| **ai_extract** | Amazon product | AI extracted name, price, rating, review count |
|
|
514
|
-
| **
|
|
515
|
-
| **
|
|
516
|
-
| **
|
|
517
|
-
| **
|
|
518
|
-
| **
|
|
519
|
-
|
|
520
|
-
|
|
503
|
+
| 📄 **scrape** | BBC News | Full markdown, stealth level 3 auto-escalation |
|
|
504
|
+
| 🕸️ **crawl** | Cloudflare Blog | 213K characters crawled with depth control |
|
|
505
|
+
| 🗺️ **map** | BBC | Full URL discovery via sitemap + link extraction |
|
|
506
|
+
| 🕷️ **extract** | Amazon (AirPods Pro 2) | Product title, 45,297 reviews, brand extracted |
|
|
507
|
+
| 📖 **readability** | Medium article | Clean — title, author, content, publish date |
|
|
508
|
+
| 📸 **screenshot** | ProductHunt | Captured Cloudflare Turnstile challenge page |
|
|
509
|
+
| 🔍 **search** | Brave Web | Web results with snippets and URLs |
|
|
510
|
+
| 📰 **news_search** | Brave News | News results with freshness ranking |
|
|
511
|
+
| 🖼️ **image_search** | Brave Image | Images with thumbnails and source URLs |
|
|
512
|
+
| 🎬 **video_search** | Brave Video | Video results across platforms |
|
|
513
|
+
| 🛠️ **create_skill** | Hacker News | Auto-detected 30 stories with CSS selectors |
|
|
514
|
+
| ▶️ **run_skill** | Saved skill | Fresh structured data from saved config |
|
|
515
|
+
| 📋 **list_skills** | — | Lists all skills with configurations |
|
|
516
|
+
| 🔓 **discover_apis** | Airbnb Paris | **34 hidden APIs** — DataDome, Google Maps key, internal APIs |
|
|
517
|
+
| ⚡ **query_api** | jsonplaceholder | Direct JSON API call with stealth headers |
|
|
518
|
+
| 📡 **monitor_websocket** | Binance BTC/USDT | 3 WebSocket connections, 23 live messages — BTC price live |
|
|
519
|
+
| 🧠 **ai_extract** | Amazon product | AI extracted name, price, rating, review count |
|
|
520
|
+
| 🎯 **snapshot** | GitHub, Wikipedia | ARIA tree with 107/113 refs, annotated screenshots |
|
|
521
|
+
| 🖱️ **interact** | Login flow | Click → type → submit — ref targeting, session encryption, 18 action types |
|
|
522
|
+
| 📦 **batch_scrape** | 10 news sites | Parallel, concurrency 3, soft failure, 9/10 succeeded |
|
|
523
|
+
| 📋 **list_jobs** | — | Batch jobs with status and progress |
|
|
524
|
+
| 📊 **job_status** | Batch job | Full per-URL results with timing |
|
|
525
|
+
| 🗑️ **delete_job** | Completed job | Cleaned up job data from disk |
|
|
526
|
+
|
|
527
|
+
> **23/23 tools. 34 hidden APIs on Airbnb. Live BTC feed. Zero API keys for scraping.**
|
|
521
528
|
|
|
522
529
|
---
|
|
523
530
|
|
|
@@ -530,6 +537,7 @@ Every tool tested against production websites with real anti-bot defenses:
|
|
|
530
537
|
| `LLM_API_KEY` | No | Anthropic or OpenAI API key (enables `ai_extract`) |
|
|
531
538
|
| `LLM_PROVIDER` | No | `anthropic`, `openai`, or `minimax` (default: `anthropic`) |
|
|
532
539
|
| `LLM_MODEL` | No | Override default LLM model |
|
|
540
|
+
| `SESSION_ENCRYPTION_KEY` | No | 32-byte hex key for encrypting session files at rest |
|
|
533
541
|
| `TRANSPORT` | No | `stdio` (default) or `http` |
|
|
534
542
|
| `PORT` | No | HTTP port (default: 3000) |
|
|
535
543
|
| `PROXY_URL` | No | Single proxy URL (http/https/socks4/socks5) |
|
package/dist/constants.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare const PACKAGE_NAME = "imperium-crawl";
|
|
2
|
-
export declare const PACKAGE_VERSION = "
|
|
2
|
+
export declare const PACKAGE_VERSION = "2.0.0";
|
|
3
3
|
export declare const DEFAULT_TIMEOUT_MS = 30000;
|
|
4
4
|
export declare const DEFAULT_MAX_PAGES = 10;
|
|
5
5
|
export declare const DEFAULT_MAX_DEPTH = 2;
|
|
@@ -36,4 +36,5 @@ export declare const MAX_DURATION_SECONDS = 300;
|
|
|
36
36
|
export declare const MAX_TIMEOUT_MS = 300000;
|
|
37
37
|
export declare const MAX_SELECTOR_KEYS = 50;
|
|
38
38
|
export declare const MAX_CRAWL_CONTENT_PER_PAGE = 102400;
|
|
39
|
+
export declare const MAX_STORED_SNAPSHOTS = 100;
|
|
39
40
|
//# sourceMappingURL=constants.d.ts.map
|
package/dist/constants.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,mBAAmB,CAAC;AAC7C,eAAO,MAAM,eAAe,UAAU,CAAC;AAEvC,eAAO,MAAM,kBAAkB,QAAS,CAAC;AACzC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,iBAAiB,IAAI,CAAC;AACnC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,2BAA2B,QAAiB,CAAC;AAE1D,eAAO,MAAM,eAAe,oBAAoB,CAAC;AACjD,eAAO,MAAM,aAAa,WAAW,CAAC;AACtC,eAAO,MAAM,eAAe,aAAa,CAAC;AAC1C,eAAO,MAAM,WAAW,SAAS,CAAC;AAElC,eAAO,MAAM,cAAc,wCAAwC,CAAC;AAEpE,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAC3C,eAAO,MAAM,+BAA+B,SAAU,CAAC;AAEvD,eAAO,MAAM,cAAc,mBAAmB,CAAC;AAI/C,eAAO,MAAM,YAAY,EAAE,MAAM,EAgBhC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;CAAgC,CAAC;AAE9D,eAAO,MAAM,kBAAkB,MAAM,CAAC;AACtC,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAEvC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAIxC,eAAO,MAAM,iBAAiB,QAAS,CAAC;AACxC,eAAO,MAAM,eAAe,UAAY,CAAC;AACzC,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AACtC,eAAO,MAAM,mBAAmB,OAAQ,CAAC;AACzC,eAAO,MAAM,cAAc,OAAQ,CAAC;AACpC,eAAO,MAAM,SAAS,MAAM,CAAC;AAC7B,eAAO,MAAM,QAAQ,QAAS,CAAC;AAC/B,eAAO,MAAM,SAAS,OAAQ,CAAC;AAC/B,eAAO,MAAM,YAAY,OAAQ,CAAC;AAClC,eAAO,MAAM,eAAe,KAAK,CAAC;AAClC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,oBAAoB,MAAM,CAAC;AACxC,eAAO,MAAM,cAAc,SAAU,CAAC;AACtC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,0BAA0B,SAAU,CAAC"}
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,mBAAmB,CAAC;AAC7C,eAAO,MAAM,eAAe,UAAU,CAAC;AAEvC,eAAO,MAAM,kBAAkB,QAAS,CAAC;AACzC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,iBAAiB,IAAI,CAAC;AACnC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,2BAA2B,QAAiB,CAAC;AAE1D,eAAO,MAAM,eAAe,oBAAoB,CAAC;AACjD,eAAO,MAAM,aAAa,WAAW,CAAC;AACtC,eAAO,MAAM,eAAe,aAAa,CAAC;AAC1C,eAAO,MAAM,WAAW,SAAS,CAAC;AAElC,eAAO,MAAM,cAAc,wCAAwC,CAAC;AAEpE,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAC3C,eAAO,MAAM,+BAA+B,SAAU,CAAC;AAEvD,eAAO,MAAM,cAAc,mBAAmB,CAAC;AAI/C,eAAO,MAAM,YAAY,EAAE,MAAM,EAgBhC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;CAAgC,CAAC;AAE9D,eAAO,MAAM,kBAAkB,MAAM,CAAC;AACtC,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAEvC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAIxC,eAAO,MAAM,iBAAiB,QAAS,CAAC;AACxC,eAAO,MAAM,eAAe,UAAY,CAAC;AACzC,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AACtC,eAAO,MAAM,mBAAmB,OAAQ,CAAC;AACzC,eAAO,MAAM,cAAc,OAAQ,CAAC;AACpC,eAAO,MAAM,SAAS,MAAM,CAAC;AAC7B,eAAO,MAAM,QAAQ,QAAS,CAAC;AAC/B,eAAO,MAAM,SAAS,OAAQ,CAAC;AAC/B,eAAO,MAAM,YAAY,OAAQ,CAAC;AAClC,eAAO,MAAM,eAAe,KAAK,CAAC;AAClC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,oBAAoB,MAAM,CAAC;AACxC,eAAO,MAAM,cAAc,SAAU,CAAC;AACtC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,eAAO,MAAM,0BAA0B,SAAU,CAAC;AAIlD,eAAO,MAAM,oBAAoB,MAAM,CAAC"}
|
package/dist/constants.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const PACKAGE_NAME = "imperium-crawl";
|
|
2
|
-
export const PACKAGE_VERSION = "
|
|
2
|
+
export const PACKAGE_VERSION = "2.0.0";
|
|
3
3
|
export const DEFAULT_TIMEOUT_MS = 30_000;
|
|
4
4
|
export const DEFAULT_MAX_PAGES = 10;
|
|
5
5
|
export const DEFAULT_MAX_DEPTH = 2;
|
|
@@ -51,4 +51,6 @@ export const MAX_DURATION_SECONDS = 300;
|
|
|
51
51
|
export const MAX_TIMEOUT_MS = 300_000;
|
|
52
52
|
export const MAX_SELECTOR_KEYS = 50;
|
|
53
53
|
export const MAX_CRAWL_CONTENT_PER_PAGE = 102_400; // 100KB
|
|
54
|
+
// ── Snapshot system ──
|
|
55
|
+
export const MAX_STORED_SNAPSHOTS = 100;
|
|
54
56
|
//# sourceMappingURL=constants.js.map
|
package/dist/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,gBAAgB,CAAC;AAC7C,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC;AAEvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AACzC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AACpC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AACrC,MAAM,CAAC,MAAM,2BAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAEpE,MAAM,CAAC,MAAM,eAAe,GAAG,iBAAiB,CAAC;AACjD,MAAM,CAAC,MAAM,aAAa,GAAG,QAAQ,CAAC;AACtC,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;AAC1C,MAAM,CAAC,MAAM,WAAW,GAAG,MAAM,CAAC;AAElC,MAAM,CAAC,MAAM,cAAc,GAAG,qCAAqC,CAAC;AAEpE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAC3C,MAAM,CAAC,MAAM,+BAA+B,GAAG,OAAO,CAAC,CAAC,QAAQ;AAEhE,MAAM,CAAC,MAAM,cAAc,GAAG,gBAAgB,CAAC;AAE/C,yBAAyB;AAEzB,MAAM,CAAC,MAAM,YAAY,GAAa;IACpC,+CAA+C;IAC/C,yCAAyC;IACzC,oBAAoB;IACpB,gBAAgB;IAChB,4BAA4B;IAC5B,iCAAiC;IACjC,4BAA4B;IAC5B,wBAAwB;IACxB,wBAAwB;IACxB,0BAA0B;IAC1B,4BAA4B;IAC5B,gBAAgB;IAChB,0BAA0B;IAC1B,sBAAsB;IACtB,wBAAwB;CACzB,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAE9D,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACtC,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEvC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAExC,gCAAgC;AAEhC,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACxC,MAAM,CAAC,MAAM,eAAe,GAAG,SAAS,CAAC,CAAI,MAAM;AACnD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AACtC,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AACzC,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAC;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAC7B,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC/B,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC;AAC/B,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC;AAClC,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAClC,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AACpC,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG,OAAO,CAAC;AACtC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AACpC,MAAM,CAAC,MAAM,0BAA0B,GAAG,OAAO,CAAC,CAAC,QAAQ"}
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,gBAAgB,CAAC;AAC7C,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC;AAEvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AACzC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AACpC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AACrC,MAAM,CAAC,MAAM,2BAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAEpE,MAAM,CAAC,MAAM,eAAe,GAAG,iBAAiB,CAAC;AACjD,MAAM,CAAC,MAAM,aAAa,GAAG,QAAQ,CAAC;AACtC,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;AAC1C,MAAM,CAAC,MAAM,WAAW,GAAG,MAAM,CAAC;AAElC,MAAM,CAAC,MAAM,cAAc,GAAG,qCAAqC,CAAC;AAEpE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAC3C,MAAM,CAAC,MAAM,+BAA+B,GAAG,OAAO,CAAC,CAAC,QAAQ;AAEhE,MAAM,CAAC,MAAM,cAAc,GAAG,gBAAgB,CAAC;AAE/C,yBAAyB;AAEzB,MAAM,CAAC,MAAM,YAAY,GAAa;IACpC,+CAA+C;IAC/C,yCAAyC;IACzC,oBAAoB;IACpB,gBAAgB;IAChB,4BAA4B;IAC5B,iCAAiC;IACjC,4BAA4B;IAC5B,wBAAwB;IACxB,wBAAwB;IACxB,0BAA0B;IAC1B,4BAA4B;IAC5B,gBAAgB;IAChB,0BAA0B;IAC1B,sBAAsB;IACtB,wBAAwB;CACzB,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAE9D,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACtC,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEvC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAExC,gCAAgC;AAEhC,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACxC,MAAM,CAAC,MAAM,eAAe,GAAG,SAAS,CAAC,CAAI,MAAM;AACnD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AACtC,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AACzC,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAC;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAC7B,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC/B,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC;AAC/B,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC;AAClC,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAClC,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AACpC,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG,OAAO,CAAC;AACtC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AACpC,MAAM,CAAC,MAAM,0BAA0B,GAAG,OAAO,CAAC,CAAC,QAAQ;AAE3D,wBAAwB;AAExB,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Network Interceptor — Request interception and logging via page.route().
|
|
3
|
+
*
|
|
4
|
+
* Supports block, mock, modify, and log actions.
|
|
5
|
+
* Captures request log with timing for analysis.
|
|
6
|
+
*/
|
|
7
|
+
import type { InterceptRule, NetworkRequest } from "./types.js";
|
|
8
|
+
type Page = import("rebrowser-playwright").Page;
|
|
9
|
+
/**
|
|
10
|
+
* Set up network interception rules on a page.
|
|
11
|
+
* Also starts request logging.
|
|
12
|
+
*/
|
|
13
|
+
export declare function setupInterception(page: Page, rules: InterceptRule[]): Promise<void>;
|
|
14
|
+
/**
|
|
15
|
+
* Get the captured request log for a page.
|
|
16
|
+
*/
|
|
17
|
+
export declare function getRequestLog(page: Page): NetworkRequest[];
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=interceptor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interceptor.d.ts","sourceRoot":"","sources":["../../src/network/interceptor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEhE,KAAK,IAAI,GAAG,OAAO,sBAAsB,EAAE,IAAI,CAAC;AAMhD;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,aAAa,EAAE,GACrB,OAAO,CAAC,IAAI,CAAC,CAoEf;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,cAAc,EAAE,CAE1D"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Network Interceptor — Request interception and logging via page.route().
|
|
3
|
+
*
|
|
4
|
+
* Supports block, mock, modify, and log actions.
|
|
5
|
+
* Captures request log with timing for analysis.
|
|
6
|
+
*/
|
|
7
|
+
// Per-page request logs
|
|
8
|
+
const requestLogs = new WeakMap();
|
|
9
|
+
/**
|
|
10
|
+
* Set up network interception rules on a page.
|
|
11
|
+
* Also starts request logging.
|
|
12
|
+
*/
|
|
13
|
+
export async function setupInterception(page, rules) {
|
|
14
|
+
const log = [];
|
|
15
|
+
requestLogs.set(page, log);
|
|
16
|
+
// Set up request logging
|
|
17
|
+
page.on("request", (request) => {
|
|
18
|
+
log.push({
|
|
19
|
+
url: request.url(),
|
|
20
|
+
method: request.method(),
|
|
21
|
+
resourceType: request.resourceType(),
|
|
22
|
+
timing: { startTime: Date.now() },
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
page.on("response", (response) => {
|
|
26
|
+
const url = response.url();
|
|
27
|
+
const entry = log.find((e) => e.url === url && !e.status);
|
|
28
|
+
if (entry) {
|
|
29
|
+
entry.status = response.status();
|
|
30
|
+
entry.timing.duration = Date.now() - entry.timing.startTime;
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
// Apply interception rules
|
|
34
|
+
for (const rule of rules) {
|
|
35
|
+
await page.route(rule.url_pattern, async (route) => {
|
|
36
|
+
switch (rule.action) {
|
|
37
|
+
case "block":
|
|
38
|
+
await route.abort("blockedbyclient");
|
|
39
|
+
break;
|
|
40
|
+
case "mock":
|
|
41
|
+
await route.fulfill({
|
|
42
|
+
status: rule.response?.status ?? 200,
|
|
43
|
+
body: rule.response?.body ?? "",
|
|
44
|
+
headers: rule.response?.headers,
|
|
45
|
+
contentType: rule.response?.contentType ?? "text/plain",
|
|
46
|
+
});
|
|
47
|
+
break;
|
|
48
|
+
case "modify":
|
|
49
|
+
// Modify response: fetch original, then override parts
|
|
50
|
+
try {
|
|
51
|
+
const response = await route.fetch();
|
|
52
|
+
const body = rule.response?.body ?? await response.text();
|
|
53
|
+
await route.fulfill({
|
|
54
|
+
status: rule.response?.status ?? response.status(),
|
|
55
|
+
body,
|
|
56
|
+
headers: {
|
|
57
|
+
...response.headers(),
|
|
58
|
+
...(rule.response?.headers ?? {}),
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
await route.continue();
|
|
64
|
+
}
|
|
65
|
+
break;
|
|
66
|
+
case "log":
|
|
67
|
+
// Just continue — logging is handled by event listeners above
|
|
68
|
+
await route.continue();
|
|
69
|
+
break;
|
|
70
|
+
default:
|
|
71
|
+
await route.continue();
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Get the captured request log for a page.
|
|
78
|
+
*/
|
|
79
|
+
export function getRequestLog(page) {
|
|
80
|
+
return requestLogs.get(page) ?? [];
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=interceptor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interceptor.js","sourceRoot":"","sources":["../../src/network/interceptor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,wBAAwB;AACxB,MAAM,WAAW,GAAG,IAAI,OAAO,EAA4B,CAAC;AAE5D;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAU,EACV,KAAsB;IAEtB,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAE3B,yBAAyB;IACzB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;QAC7B,GAAG,CAAC,IAAI,CAAC;YACP,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE;YACpC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE;SAClC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;QAC/B,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,KAAY,EAAE,EAAE;YACxD,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,KAAK,OAAO;oBACV,MAAM,KAAK,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBACrC,MAAM;gBAER,KAAK,MAAM;oBACT,MAAM,KAAK,CAAC,OAAO,CAAC;wBAClB,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG;wBACpC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,EAAE;wBAC/B,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO;wBAC/B,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,WAAW,IAAI,YAAY;qBACxD,CAAC,CAAC;oBACH,MAAM;gBAER,KAAK,QAAQ;oBACX,uDAAuD;oBACvD,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;wBACrC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;wBAC1D,MAAM,KAAK,CAAC,OAAO,CAAC;4BAClB,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;4BAClD,IAAI;4BACJ,OAAO,EAAE;gCACP,GAAG,QAAQ,CAAC,OAAO,EAAE;gCACrB,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC;6BAClC;yBACF,CAAC,CAAC;oBACL,CAAC;oBAAC,MAAM,CAAC;wBACP,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACzB,CAAC;oBACD,MAAM;gBAER,KAAK,KAAK;oBACR,8DAA8D;oBAC9D,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACvB,MAAM;gBAER;oBACE,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC3B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,IAAU;IACtC,OAAO,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** Rule for network request interception */
|
|
2
|
+
export interface InterceptRule {
|
|
3
|
+
url_pattern: string;
|
|
4
|
+
/** Action to take on matching requests */
|
|
5
|
+
action: "block" | "mock" | "modify" | "log";
|
|
6
|
+
/** Mock response (required for "mock" action) */
|
|
7
|
+
response?: {
|
|
8
|
+
status?: number;
|
|
9
|
+
body?: string;
|
|
10
|
+
headers?: Record<string, string>;
|
|
11
|
+
contentType?: string;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
/** Captured network request */
|
|
15
|
+
export interface NetworkRequest {
|
|
16
|
+
url: string;
|
|
17
|
+
method: string;
|
|
18
|
+
resourceType: string;
|
|
19
|
+
status?: number;
|
|
20
|
+
timing: {
|
|
21
|
+
startTime: number;
|
|
22
|
+
duration?: number;
|
|
23
|
+
};
|
|
24
|
+
headers?: Record<string, string>;
|
|
25
|
+
responseSize?: number;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/network/types.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,MAAM,WAAW,aAAa;IAE5B,WAAW,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IAC5C,iDAAiD;IACjD,QAAQ,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,+BAA+B;AAC/B,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/network/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Action Policy — Granular access control for interact actions.
|
|
3
|
+
*
|
|
4
|
+
* Maps action types to categories, then evaluates against a policy config.
|
|
5
|
+
* Supports hot-reload: checks file mtime every 5s.
|
|
6
|
+
*/
|
|
7
|
+
import type { PolicyDecision } from "./types.js";
|
|
8
|
+
/**
|
|
9
|
+
* Get the category for an action type.
|
|
10
|
+
*/
|
|
11
|
+
export declare function getActionCategory(actionType: string): string;
|
|
12
|
+
/**
|
|
13
|
+
* Check policy for a given action type.
|
|
14
|
+
*
|
|
15
|
+
* @param actionType - The action type (e.g. "click", "evaluate")
|
|
16
|
+
* @param policyPath - Path to JSON policy file
|
|
17
|
+
* @returns Policy decision: allow, deny, or confirm
|
|
18
|
+
*/
|
|
19
|
+
export declare function checkPolicy(actionType: string, policyPath: string): Promise<PolicyDecision>;
|
|
20
|
+
/**
|
|
21
|
+
* Human-readable description of an action for confirm prompts.
|
|
22
|
+
*/
|
|
23
|
+
export declare function describeAction(actionType: string, details?: Record<string, unknown>): string;
|
|
24
|
+
/** Clear policy cache (for testing) */
|
|
25
|
+
export declare function resetPolicyCache(): void;
|
|
26
|
+
//# sourceMappingURL=action-policy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action-policy.d.ts","sourceRoot":"","sources":["../../src/security/action-policy.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAsB,cAAc,EAAE,MAAM,YAAY,CAAC;AAwCrE;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE5D;AAgDD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,CAAC,CAmBzB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CA2B5F;AAED,uCAAuC;AACvC,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Action Policy — Granular access control for interact actions.
|
|
3
|
+
*
|
|
4
|
+
* Maps action types to categories, then evaluates against a policy config.
|
|
5
|
+
* Supports hot-reload: checks file mtime every 5s.
|
|
6
|
+
*/
|
|
7
|
+
import fs from "node:fs/promises";
|
|
8
|
+
// ── Action → Category mapping ──
|
|
9
|
+
const ACTION_CATEGORIES = {
|
|
10
|
+
// Navigation
|
|
11
|
+
navigate: "navigate",
|
|
12
|
+
// Click/interact
|
|
13
|
+
click: "click",
|
|
14
|
+
hover: "click",
|
|
15
|
+
drag: "click",
|
|
16
|
+
// Form input
|
|
17
|
+
type: "fill",
|
|
18
|
+
select: "fill",
|
|
19
|
+
upload: "fill",
|
|
20
|
+
// Script execution
|
|
21
|
+
evaluate: "eval",
|
|
22
|
+
// Read state
|
|
23
|
+
screenshot: "snapshot",
|
|
24
|
+
pdf: "snapshot",
|
|
25
|
+
cookie_get: "state",
|
|
26
|
+
storage_get: "state",
|
|
27
|
+
// Write state
|
|
28
|
+
cookie_set: "state_write",
|
|
29
|
+
storage_set: "state_write",
|
|
30
|
+
// Passive
|
|
31
|
+
scroll: "scroll",
|
|
32
|
+
wait: "wait",
|
|
33
|
+
press: "interact",
|
|
34
|
+
};
|
|
35
|
+
// Internal actions always allowed
|
|
36
|
+
const INTERNAL_CATEGORY = "_internal";
|
|
37
|
+
/**
|
|
38
|
+
* Get the category for an action type.
|
|
39
|
+
*/
|
|
40
|
+
export function getActionCategory(actionType) {
|
|
41
|
+
return ACTION_CATEGORIES[actionType] ?? "unknown";
|
|
42
|
+
}
|
|
43
|
+
const policyCache = new Map();
|
|
44
|
+
const POLICY_CHECK_INTERVAL_MS = 5000;
|
|
45
|
+
/**
|
|
46
|
+
* Load and cache a policy file with hot-reload support.
|
|
47
|
+
*/
|
|
48
|
+
async function loadPolicy(policyPath) {
|
|
49
|
+
const now = Date.now();
|
|
50
|
+
const cached = policyCache.get(policyPath);
|
|
51
|
+
// Check if cached and recent enough
|
|
52
|
+
if (cached && now - cached.loadedAt < POLICY_CHECK_INTERVAL_MS) {
|
|
53
|
+
return cached.config;
|
|
54
|
+
}
|
|
55
|
+
// Check mtime for hot-reload
|
|
56
|
+
try {
|
|
57
|
+
const stat = await fs.stat(policyPath);
|
|
58
|
+
const mtime = stat.mtimeMs;
|
|
59
|
+
if (cached && cached.mtime === mtime) {
|
|
60
|
+
cached.loadedAt = now;
|
|
61
|
+
return cached.config;
|
|
62
|
+
}
|
|
63
|
+
const raw = await fs.readFile(policyPath, "utf-8");
|
|
64
|
+
const config = JSON.parse(raw);
|
|
65
|
+
policyCache.set(policyPath, { config, mtime, loadedAt: now });
|
|
66
|
+
return config;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// If file doesn't exist or is invalid, use permissive default
|
|
70
|
+
const defaultConfig = { default: "allow" };
|
|
71
|
+
policyCache.set(policyPath, { config: defaultConfig, mtime: 0, loadedAt: now });
|
|
72
|
+
return defaultConfig;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Check policy for a given action type.
|
|
77
|
+
*
|
|
78
|
+
* @param actionType - The action type (e.g. "click", "evaluate")
|
|
79
|
+
* @param policyPath - Path to JSON policy file
|
|
80
|
+
* @returns Policy decision: allow, deny, or confirm
|
|
81
|
+
*/
|
|
82
|
+
export async function checkPolicy(actionType, policyPath) {
|
|
83
|
+
const category = getActionCategory(actionType);
|
|
84
|
+
// Internal actions always allowed
|
|
85
|
+
if (category === INTERNAL_CATEGORY)
|
|
86
|
+
return "allow";
|
|
87
|
+
const config = await loadPolicy(policyPath);
|
|
88
|
+
// Deny takes highest priority
|
|
89
|
+
if (config.deny?.includes(category))
|
|
90
|
+
return "deny";
|
|
91
|
+
// Confirm takes second priority
|
|
92
|
+
if (config.confirm?.includes(category))
|
|
93
|
+
return "confirm";
|
|
94
|
+
// Explicit allow
|
|
95
|
+
if (config.allow?.includes(category))
|
|
96
|
+
return "allow";
|
|
97
|
+
// Fall through to default
|
|
98
|
+
return config.default;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Human-readable description of an action for confirm prompts.
|
|
102
|
+
*/
|
|
103
|
+
export function describeAction(actionType, details) {
|
|
104
|
+
const descriptions = {
|
|
105
|
+
navigate: "Navigate to a URL",
|
|
106
|
+
click: "Click an element",
|
|
107
|
+
hover: "Hover over an element",
|
|
108
|
+
drag: "Drag and drop an element",
|
|
109
|
+
type: "Type text into a field",
|
|
110
|
+
select: "Select an option",
|
|
111
|
+
upload: "Upload files",
|
|
112
|
+
evaluate: "Execute JavaScript code",
|
|
113
|
+
screenshot: "Take a screenshot",
|
|
114
|
+
pdf: "Generate a PDF",
|
|
115
|
+
cookie_get: "Read cookies",
|
|
116
|
+
cookie_set: "Set cookies",
|
|
117
|
+
storage_get: "Read browser storage",
|
|
118
|
+
storage_set: "Write to browser storage",
|
|
119
|
+
scroll: "Scroll the page",
|
|
120
|
+
wait: "Wait for element/timeout",
|
|
121
|
+
press: "Press a key",
|
|
122
|
+
};
|
|
123
|
+
let desc = descriptions[actionType] ?? `Execute ${actionType}`;
|
|
124
|
+
if (details?.url)
|
|
125
|
+
desc += ` (${details.url})`;
|
|
126
|
+
if (details?.selector)
|
|
127
|
+
desc += ` on ${details.selector}`;
|
|
128
|
+
if (details?.ref)
|
|
129
|
+
desc += ` on ref ${details.ref}`;
|
|
130
|
+
return desc;
|
|
131
|
+
}
|
|
132
|
+
/** Clear policy cache (for testing) */
|
|
133
|
+
export function resetPolicyCache() {
|
|
134
|
+
policyCache.clear();
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=action-policy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action-policy.js","sourceRoot":"","sources":["../../src/security/action-policy.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAGlC,kCAAkC;AAElC,MAAM,iBAAiB,GAA2B;IAChD,aAAa;IACb,QAAQ,EAAE,UAAU;IAEpB,iBAAiB;IACjB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,OAAO;IAEb,aAAa;IACb,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,MAAM;IAEd,mBAAmB;IACnB,QAAQ,EAAE,MAAM;IAEhB,aAAa;IACb,UAAU,EAAE,UAAU;IACtB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,OAAO;IACnB,WAAW,EAAE,OAAO;IAEpB,cAAc;IACd,UAAU,EAAE,aAAa;IACzB,WAAW,EAAE,aAAa;IAE1B,UAAU;IACV,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,UAAU;CAClB,CAAC;AAEF,kCAAkC;AAClC,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAEtC;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,UAAkB;IAClD,OAAO,iBAAiB,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC;AACpD,CAAC;AAUD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAwB,CAAC;AACpD,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAEtC;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,UAAkB;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAE3C,oCAAoC;IACpC,IAAI,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,wBAAwB,EAAE,CAAC;QAC/D,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;QAE3B,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;YACrC,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;YACtB,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAuB,CAAC;QAErD,WAAW,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,MAAM,aAAa,GAAuB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QAC/D,WAAW,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QAChF,OAAO,aAAa,CAAC;IACvB,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAAkB,EAClB,UAAkB;IAElB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAE/C,kCAAkC;IAClC,IAAI,QAAQ,KAAK,iBAAiB;QAAE,OAAO,OAAO,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;IAE5C,8BAA8B;IAC9B,IAAI,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAEnD,gCAAgC;IAChC,IAAI,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,SAAS,CAAC;IAEzD,iBAAiB;IACjB,IAAI,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,OAAO,CAAC;IAErD,0BAA0B;IAC1B,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,OAAiC;IAClF,MAAM,YAAY,GAA2B;QAC3C,QAAQ,EAAE,mBAAmB;QAC7B,KAAK,EAAE,kBAAkB;QACzB,KAAK,EAAE,uBAAuB;QAC9B,IAAI,EAAE,0BAA0B;QAChC,IAAI,EAAE,wBAAwB;QAC9B,MAAM,EAAE,kBAAkB;QAC1B,MAAM,EAAE,cAAc;QACtB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE,mBAAmB;QAC/B,GAAG,EAAE,gBAAgB;QACrB,UAAU,EAAE,cAAc;QAC1B,UAAU,EAAE,aAAa;QACzB,WAAW,EAAE,sBAAsB;QACnC,WAAW,EAAE,0BAA0B;QACvC,MAAM,EAAE,iBAAiB;QACzB,IAAI,EAAE,0BAA0B;QAChC,KAAK,EAAE,aAAa;KACrB,CAAC;IAEF,IAAI,IAAI,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,WAAW,UAAU,EAAE,CAAC;IAC/D,IAAI,OAAO,EAAE,GAAG;QAAE,IAAI,IAAI,KAAK,OAAO,CAAC,GAAG,GAAG,CAAC;IAC9C,IAAI,OAAO,EAAE,QAAQ;QAAE,IAAI,IAAI,OAAO,OAAO,CAAC,QAAQ,EAAE,CAAC;IACzD,IAAI,OAAO,EAAE,GAAG;QAAE,IAAI,IAAI,WAAW,OAAO,CAAC,GAAG,EAAE,CAAC;IAEnD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uCAAuC;AACvC,MAAM,UAAU,gBAAgB;IAC9B,WAAW,CAAC,KAAK,EAAE,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auth Vault — Encrypted credential storage for login automation.
|
|
3
|
+
*
|
|
4
|
+
* Stores login profiles (URL, username, password, form selectors)
|
|
5
|
+
* encrypted on disk. Profiles are decrypted on-demand for login flows.
|
|
6
|
+
*/
|
|
7
|
+
export interface AuthProfile {
|
|
8
|
+
name: string;
|
|
9
|
+
url: string;
|
|
10
|
+
username: string;
|
|
11
|
+
password: string;
|
|
12
|
+
selectors: {
|
|
13
|
+
username: string;
|
|
14
|
+
password: string;
|
|
15
|
+
submit: string;
|
|
16
|
+
};
|
|
17
|
+
lastLogin?: string;
|
|
18
|
+
createdAt: string;
|
|
19
|
+
updatedAt: string;
|
|
20
|
+
}
|
|
21
|
+
export interface AuthProfileMeta {
|
|
22
|
+
name: string;
|
|
23
|
+
url: string;
|
|
24
|
+
username: string;
|
|
25
|
+
lastLogin?: string;
|
|
26
|
+
createdAt: string;
|
|
27
|
+
updatedAt: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Save an auth profile (encrypted on disk).
|
|
31
|
+
*/
|
|
32
|
+
export declare function saveAuthProfile(profile: Omit<AuthProfile, "createdAt" | "updatedAt">): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Get a decrypted auth profile by name.
|
|
35
|
+
*/
|
|
36
|
+
export declare function getAuthProfile(name: string): Promise<AuthProfile | null>;
|
|
37
|
+
/**
|
|
38
|
+
* List all auth profiles (meta only — no passwords).
|
|
39
|
+
*/
|
|
40
|
+
export declare function listAuthProfiles(): Promise<AuthProfileMeta[]>;
|
|
41
|
+
/**
|
|
42
|
+
* Delete an auth profile.
|
|
43
|
+
*/
|
|
44
|
+
export declare function deleteAuthProfile(name: string): Promise<boolean>;
|
|
45
|
+
/**
|
|
46
|
+
* Update the lastLogin timestamp for a profile.
|
|
47
|
+
*/
|
|
48
|
+
export declare function updateLastLogin(name: string): Promise<void>;
|
|
49
|
+
//# sourceMappingURL=auth-vault.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth-vault.d.ts","sourceRoot":"","sources":["../../src/security/auth-vault.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAaD;;GAEG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA8B1G;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAgB9E;AAED;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAiCnE;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAOtE;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQjE"}
|