mcp-scraper 0.93.1 → 0.93.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -25
- package/README.md +3 -3
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-2SP57VCG.js +15 -15
- package/dist/{chunk-OD2WLHEN.js → chunk-4FGLXXZ2.js} +4 -4
- package/dist/{chunk-X6RD3Y4W.js → chunk-BS4V4CVX.js} +1 -1
- package/dist/{chunk-5QYW4CSJ.js → chunk-DFJT2YX6.js} +5 -5
- package/dist/chunk-DT2FYN6N.js +1 -1
- package/dist/chunk-M5QHXNFZ.js +3 -3
- package/dist/chunk-M5VZVMPC.js +3 -3
- package/dist/chunk-ORB4RHCK.js +4 -4
- package/dist/chunk-SH5KB4P7.js +1 -1
- package/dist/chunk-SWCAKYBP.js +1 -0
- package/dist/chunk-WJ4XFLS4.js +15 -15
- package/dist/chunk-Y6MKMSOC.js +1 -1
- package/dist/chunk-ZG52SKGD.js +6 -0
- package/dist/index.cjs +21 -21
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/{server-KWGXYWYO.js → server-G2ZCSWXW.js} +42 -42
- package/dist/{worker-ASXALAY7.js → worker-P2BICG36.js} +1 -1
- package/package.json +136 -17
- package/THIRD_PARTY_NOTICES.html +0 -203
- package/dist/chunk-AYQSKD7D.js +0 -1
- package/dist/chunk-KPQM4UDJ.js +0 -6
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,19 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.93.3] - 2026-09-24
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
|
|
11
|
+
- Render the CEO report headline and detail from one cost-ledger snapshot, and label cost-linked surcharge multiples as pricing targets rather than guaranteed realized margin.
|
|
12
|
+
- Apply the owner-confirmed $1.50 per 1,000 successful Bright Data SERP requests to delivered provider receipts, with a guarded backfill for earlier successful requests. Failed attempts remain visibly unresolved until provider billing evidence is available.
|
|
13
|
+
|
|
14
|
+
## [0.93.2] - 2026-09-24
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- Show Google Search and other provider costs in the CEO report with separate calculated, actual-or-measured, and unresolved receipt coverage. Failed queries now stop the report, missing Bright Data SERP rates leave an explicit delivered-but-unpriced receipt, and historical cost rows no longer imply a margin from subscription MRR.
|
|
19
|
+
|
|
7
20
|
## [0.93.1] - 2026-09-24
|
|
8
21
|
|
|
9
22
|
### Fixed
|
|
@@ -101,7 +114,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
101
114
|
|
|
102
115
|
- Retire the inactive Personal Assistant from server startup, HTTP and MCP routing, web navigation, Scheduler transitions, generated contracts, and Memory tool registration while preserving its source history, persisted data, secrets, and provider resources behind the verified archive branch.
|
|
103
116
|
- Make two-page SERP capture perform a distinct page-two request, preserve page provenance and query/location intent, and report local-pack evidence as present, absent, incomplete, or unknown instead of silently claiming completeness.
|
|
104
|
-
- Make Maps retries follow one immutable egress plan, retain location intent, close every browser/proxy attempt, and treat an already-gone
|
|
117
|
+
- Make Maps retries follow one immutable egress plan, retain location intent, close every browser/proxy attempt, and treat an already-gone Kernel session as successful cleanup.
|
|
105
118
|
|
|
106
119
|
### Fixed
|
|
107
120
|
|
|
@@ -115,13 +128,13 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
115
128
|
|
|
116
129
|
- Add the fixture-tested operation, attempt, provider-receipt, reconciliation, and billing-link foundation needed to trace failed and retried SERP, PAA, Maps, extraction, and transcription work without treating missing provider cost as zero.
|
|
117
130
|
- Add a generated MCP cost-coverage manifest and release gate that tracks every hosted input flag plus each named execution method's retry, timeout, billing, and cost-accounting owner.
|
|
118
|
-
- Trace direct
|
|
119
|
-
- Add a protected four-cell PAA benchmark runner for
|
|
131
|
+
- Trace direct Bright Data SERP requests and Bright Data/Kernel browser attempts into the shared operation timeline, including provider identities, retry or fallback causality, immediate estimates, hourly due-gated Browser API reconciliation, and customer billing links.
|
|
132
|
+
- Add a protected four-cell PAA benchmark runner for Kernel and Bright Data at 20 and 40 complete questions, with a dry-run contract gate, one paid attempt per cell, sanitized provider evidence, and no automatic replacement run.
|
|
120
133
|
- Add a once-daily due-gated purge of raw provider identifiers and verbose diagnostic errors after 30 days while preserving normalized cost facts and hashed correlation keys.
|
|
121
134
|
|
|
122
135
|
### Fixed
|
|
123
136
|
|
|
124
|
-
- Skip
|
|
137
|
+
- Skip Kernel proxy resolution when the active attempt uses Bright Data Browser, removing avoidable Kernel API traffic from Bright Data-first SERP and PAA work.
|
|
125
138
|
|
|
126
139
|
## [0.90.4] - 2026-09-22
|
|
127
140
|
|
|
@@ -133,21 +146,21 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
133
146
|
|
|
134
147
|
### Fixed
|
|
135
148
|
|
|
136
|
-
- Route ordinary
|
|
149
|
+
- Route ordinary Bright Data SERP searches through the zone's native parsed proxy, returning Google results within the existing bounded deadline while preserving the REST endpoint as a configuration fallback.
|
|
137
150
|
- Keep Browser API credentials and interactive PAA behavior independent from the native SERP transport, with one provider request and no hidden retry or browser fallback.
|
|
138
151
|
|
|
139
152
|
## [0.90.2] - 2026-09-21
|
|
140
153
|
|
|
141
154
|
### Fixed
|
|
142
155
|
|
|
143
|
-
- Unwrap
|
|
156
|
+
- Unwrap Bright Data's object-valued REST response body before mapping parsed SERP fields, so successful direct searches return their organic results instead of an empty collection.
|
|
144
157
|
|
|
145
158
|
## [0.90.1] - 2026-09-21
|
|
146
159
|
|
|
147
160
|
### Changed
|
|
148
161
|
|
|
149
|
-
- Route ordinary `search_serp` calls through one parsed
|
|
150
|
-
- Keep saved SERP identities on their existing
|
|
162
|
+
- Route ordinary `search_serp` calls through one parsed Bright Data SERP API request when the dedicated production zone is configured, avoiding browser startup and cleanup while leaving interactive PAA on Browser API.
|
|
163
|
+
- Keep saved SERP identities on their existing Kernel-backed path and retain the browser path as the configuration fallback for local and unconfigured environments.
|
|
151
164
|
|
|
152
165
|
### Fixed
|
|
153
166
|
|
|
@@ -190,7 +203,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
190
203
|
|
|
191
204
|
### Changed
|
|
192
205
|
|
|
193
|
-
- Cap every
|
|
206
|
+
- Cap every Kernel browser session at a ten-minute absolute lifetime and move expired-session reconciliation from the minute root cron to a dedicated hourly audit.
|
|
194
207
|
- Disable the inactive Personal Assistant reminder, reconciliation, and inbound cron schedules while preserving their routes and implementation.
|
|
195
208
|
- Disable the inactive Personal Memory heartbeat and weekly rollup registrations in the production scheduler while preserving their implementation.
|
|
196
209
|
- Pause twice-daily automatic memory optimization while preserving the workflow for deliberate use, preventing all-vault fan-out from consuming background execution capacity.
|
|
@@ -199,10 +212,10 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
199
212
|
|
|
200
213
|
### Fixed
|
|
201
214
|
|
|
202
|
-
- Treat
|
|
215
|
+
- Treat Kernel's not-found response during legacy session deletion as successful cleanup, preventing already-closed sessions from retrying forever.
|
|
203
216
|
- Show browser sessions awaiting provider cleanup separately in the CTO report instead of hiding them behind a non-null close timestamp.
|
|
204
217
|
- Cast the analytics pruning clock before PostgreSQL interval arithmetic so the root cron no longer fails every minute while pruning scheduled occurrences.
|
|
205
|
-
- Explicitly delete
|
|
218
|
+
- Explicitly delete Kernel screenshot sessions after capture, including when closing the browser connection fails.
|
|
206
219
|
|
|
207
220
|
## [0.89.7] - 2026-09-18
|
|
208
221
|
|
|
@@ -321,7 +334,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
321
334
|
|
|
322
335
|
### Fixed
|
|
323
336
|
|
|
324
|
-
- Give each `reddit_thread` retrieval a 300-second end-to-end deadline, with two 60-second
|
|
337
|
+
- Give each `reddit_thread` retrieval a 300-second end-to-end deadline, with two 60-second Kernel attempts and two 60-second managed-browser backup attempts, instead of exhausting the full retry ladder in about 50 seconds. The MCP client now waits long enough to receive the endpoint's structured terminal result.
|
|
325
338
|
|
|
326
339
|
## [0.88.2] - 2026-09-02
|
|
327
340
|
|
|
@@ -358,19 +371,19 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
358
371
|
|
|
359
372
|
### Fixed
|
|
360
373
|
|
|
361
|
-
- Kept
|
|
374
|
+
- Kept Bright Data telemetry lookup off the Reddit response critical path and reallocated the saved time to 17-second backup attempts, so all four provider attempts can finish before production ends the request.
|
|
362
375
|
|
|
363
376
|
## [0.86.4] - 2026-09-02
|
|
364
377
|
|
|
365
378
|
### Fixed
|
|
366
379
|
|
|
367
|
-
- Kept the complete two-primary, two-backup Reddit retry ladder inside the production request window by limiting
|
|
380
|
+
- Kept the complete two-primary, two-backup Reddit retry ladder inside the production request window by limiting Kernel attempts to 8 seconds, Bright Data attempts to 14 seconds, and browser cleanup to 1 second.
|
|
368
381
|
|
|
369
382
|
## [0.86.3] - 2026-09-02
|
|
370
383
|
|
|
371
384
|
### Fixed
|
|
372
385
|
|
|
373
|
-
- Applied 45-second
|
|
386
|
+
- Applied 45-second Kernel and 35-second Bright Data deadlines to the complete Reddit browser-attempt lifecycle, and made known-thread primary attempts find and click the target through DuckDuckGo before the residential landing.
|
|
374
387
|
|
|
375
388
|
## [0.86.2] - 2026-09-02
|
|
376
389
|
|
|
@@ -519,13 +532,13 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
519
532
|
|
|
520
533
|
### Added
|
|
521
534
|
|
|
522
|
-
- Added a
|
|
523
|
-
- Added a bounded managed-browser backup for Reddit thread hydration after the primary
|
|
535
|
+
- Added a Kernel-only Reddit workflow that searches DuckDuckGo with a `site:reddit.com` query, switches the same browser to a residential proxy before clicking the selected result, and reads modern Reddit posts plus bounded rendered-comment expansion through dedicated search, thread, and combined REST endpoints.
|
|
536
|
+
- Added a bounded managed-browser backup for Reddit thread hydration after the primary Kernel attempt fails or returns fewer than the semantic target, capped at 25 comments with measured bandwidth, duration, CAPTCHA, closure, and provider-cost telemetry.
|
|
524
537
|
|
|
525
538
|
### Changed
|
|
526
539
|
|
|
527
|
-
- Routed the production `reddit_thread` and `reddit_trending` MCP tools through modern Reddit on
|
|
528
|
-
- Cost probes now include Reddit
|
|
540
|
+
- Routed the production `reddit_thread` and `reddit_trending` MCP tools through modern Reddit on Kernel residential sessions, with DuckDuckGo site search for trend discovery; removed Google and old Reddit from their active execution path while preserving tool names, billing rates, bounded partial results, and refunds.
|
|
541
|
+
- Cost probes now include Reddit Kernel sessions and any managed-browser fallback bytes and cost in the same request receipt, and identify when the backup contributed to total cost.
|
|
529
542
|
|
|
530
543
|
### Fixed
|
|
531
544
|
|
|
@@ -570,7 +583,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
570
583
|
|
|
571
584
|
### Fixed
|
|
572
585
|
|
|
573
|
-
- Persisted per-control PAA dispatch and 0.7/1.0/1.4-second confirmation telemetry in durable checkpoints, exposed recent interaction and attempt correlation through MCP status, attached
|
|
586
|
+
- Persisted per-control PAA dispatch and 0.7/1.0/1.4-second confirmation telemetry in durable checkpoints, exposed recent interaction and attempt correlation through MCP status, attached Bright Data session IDs immediately after browser launch, and finalized dangling attempt rows during lease recovery without blocking customer settlement.
|
|
574
587
|
- Prevented inline style, script, and hidden DOM text inside Google answer containers from falsely confirming that PAA answer material loaded.
|
|
575
588
|
- Routed canonical `/assistant` page loads to the web app and the redacted private Assistant readiness endpoint to the main API function, preventing production 404s after the 0.79.1 launch.
|
|
576
589
|
|
|
@@ -594,7 +607,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
594
607
|
- Added Scheduling as the canonical Personal Assistant setup surface, with connection readiness for Gmail, Calendar, Zoom, browser profiles, Memory, SMS, and email; exact schedule confirmation; approval and spend review; run history; and explicit watch/takeover states.
|
|
595
608
|
- Added owner-scoped browser profiles that can hold multiple independently verified login bindings, while every browser schedule grant selects one exact profile, login, domain, and action set.
|
|
596
609
|
- Added immutable schedule revisions, readiness receipts, append-only activation records, additive legacy schedule projection, and single-owner occurrence transition receipts so migration cannot silently infer browser authority or double-dispatch work.
|
|
597
|
-
- Added
|
|
610
|
+
- Added Kernel and private-Mac browser runtime boundaries with collision-resistant tenant namespaces, per-owner concurrency ceilings, bounded sessions, explicit and timeout cleanup, owner-qualified account deletion, and provider deletion readback.
|
|
598
611
|
- Added an owner-controlled Personal Assistant that brings SMS/MMS, Gmail, Google Calendar, Zoom, browser work, reminders, and Memory context packets into one governed workflow with immutable plans, approval checkpoints, spend limits, and durable receipts.
|
|
599
612
|
- Added Twilio number discovery, owned-number attachment, purchase and registration previews, Messaging Service readiness, signed inbound and delivery webhooks, safe MMS ingestion, deterministic opt-out handling, single and reviewed bulk messaging, and reconciliation for unknown provider outcomes.
|
|
600
613
|
- Added immutable, revisioned Memory context packets with source and attachment provenance, Gmail full-message imports, MMS media metadata, lifecycle controls, and readback verification against the selected vault.
|
|
@@ -631,7 +644,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
631
644
|
- Made `maxQuestions` an explicit target count rather than a traversal-depth control, with separate discovery and material-completeness diagnostics.
|
|
632
645
|
- Preserved complete People Also Ask, AI Overview, and organic-result link provenance in JSON, structured MCP output, and CSV while classifying plain links and Google redirect links explicitly.
|
|
633
646
|
- Resolved opaque Google `/goto` targets through bounded concurrent manual-redirect requests with active-browser interception as a fallback, without following publisher destinations and without dropping unresolved material.
|
|
634
|
-
- Aligned the bounded PAA production-provider canary with the public `maxQuestions` contract and made
|
|
647
|
+
- Aligned the bounded PAA production-provider canary with the public `maxQuestions` contract and made Bright Data the default test provider.
|
|
635
648
|
|
|
636
649
|
### Fixed
|
|
637
650
|
|
|
@@ -839,7 +852,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
839
852
|
|
|
840
853
|
- Added portable `harvest_paa_start` and `harvest_paa_status` tools for durable long-running PAA research, with stable idempotency recovery, progress, attempt provenance, completeness, billing state, and bounded provider telemetry.
|
|
841
854
|
- Added progressive PAA checkpoints that preserve and merge the best unique rows across browser retries and stale-job recovery instead of losing already captured questions when a provider session or caller is interrupted.
|
|
842
|
-
- Added exact
|
|
855
|
+
- Added exact Bright Data browser-session identity, sanitized Session Logs enrichment, disconnect attribution, bandwidth usage telemetry, and retryable reconciliation without making provider telemetry a prerequisite for result delivery.
|
|
843
856
|
|
|
844
857
|
### Changed
|
|
845
858
|
|
|
@@ -1572,7 +1585,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1572
1585
|
|
|
1573
1586
|
### Changed
|
|
1574
1587
|
|
|
1575
|
-
- PAA browser work now uses
|
|
1588
|
+
- PAA browser work now uses Kernel's co-located Playwright execution with stealth mode's default managed proxy and native browser metadata. Location is expressed only through Google UULE, CAPTCHA solver waiting is capped at 60 seconds, and a fresh session is allowed once only when no useful data was captured.
|
|
1576
1589
|
- PAA invocations stop browser work at 250 seconds inside the 280-second application budget, reserving 30 seconds for persistence, cleanup, and settlement. The legacy cron worker no longer claims Inngest-owned PAA jobs.
|
|
1577
1590
|
|
|
1578
1591
|
### Fixed
|
|
@@ -1808,7 +1821,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1808
1821
|
|
|
1809
1822
|
### Changed
|
|
1810
1823
|
|
|
1811
|
-
- `maps_search` now applies a transport ladder across its retry attempts so it can recover from Google soft-blocks instead of only retrying the same way. The first attempt is unchanged (
|
|
1824
|
+
- `maps_search` now applies a transport ladder across its retry attempts so it can recover from Google soft-blocks instead of only retrying the same way. The first attempt is unchanged (Kernel's default stealth ISP proxy, direct navigation). Subsequent retries switch to direct egress and arrive at Google through a cross-site redirect (the combination that measurably clears blocks a cold navigation triggers); the final escalation attempt uses direct egress without the redirect and accepts any egress country. This only affects the `proxyMode: 'none'` default path and only its retries — a first-attempt success behaves exactly as before.
|
|
1812
1825
|
|
|
1813
1826
|
## [0.32.1] - 2026-07-22
|
|
1814
1827
|
|
package/README.md
CHANGED
|
@@ -175,7 +175,7 @@ Build the branded one-click bundle:
|
|
|
175
175
|
npm run build:mcpb
|
|
176
176
|
```
|
|
177
177
|
|
|
178
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.93.
|
|
178
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.93.3`, SHA-256 `47f9bb3f04cb5788065b39a9f59e91d4680b0add2a80a7a04fa95ce3db7c8061`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
179
179
|
|
|
180
180
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
181
181
|
|
|
@@ -366,8 +366,8 @@ The `mcp-scraper` NPX stdio server also exposes saved reports as MCP resources:
|
|
|
366
366
|
- `MCP_SCRAPER_OUTPUT_DIR` is optional and defaults to `~/Downloads/mcp-scraper`.
|
|
367
367
|
- `MCP_SCRAPER_SAVE_REPORTS=false` disables automatic Markdown report files.
|
|
368
368
|
- `MCP_SCRAPER_KEY_PATH` is optional. When no API key env var is set, the server also reads `~/.mcp-scraper-key` for compatibility with older installs.
|
|
369
|
-
- `BROWSER_AGENT_PROFILE_NAME` is optional and sets the default saved hosted browser profile for `mcp-scraper` stdio sessions. Aliases: `BROWSER_SERVICE_PROFILE_NAME`, `
|
|
370
|
-
- `BROWSER_AGENT_PROFILE_SAVE_CHANGES=true` is optional hosted setup behavior. It persists cookies and storage back to the named profile when `browser_close` deletes the hosted browser session. Aliases: `BROWSER_SERVICE_PROFILE_SAVE_CHANGES`, `
|
|
369
|
+
- `BROWSER_AGENT_PROFILE_NAME` is optional and sets the default saved hosted browser profile for `mcp-scraper` stdio sessions. Aliases: `BROWSER_SERVICE_PROFILE_NAME`, `KERNEL_BROWSER_PROFILE_NAME`, `KERNEL_PROFILE_NAME`.
|
|
370
|
+
- `BROWSER_AGENT_PROFILE_SAVE_CHANGES=true` is optional hosted setup behavior. It persists cookies and storage back to the named profile when `browser_close` deletes the hosted browser session. Aliases: `BROWSER_SERVICE_PROFILE_SAVE_CHANGES`, `KERNEL_BROWSER_PROFILE_SAVE_CHANGES`, `KERNEL_PROFILE_SAVE_CHANGES`.
|
|
371
371
|
|
|
372
372
|
Hosted operators can isolate authorization state in a dedicated Turso/libSQL database without changing the public MCP tool catalog or API-key authentication. The secured store uses atomic authorization-code exchange and refresh rotation, keyed secret lookup, bounded encrypted replay receipts, authority epochs, and fail-closed maintenance behavior. Production migration and rollback are controlled data moves, not ordinary mode flips; see [MCP OAuth operations](docs/operations/mcp-oauth-runbook.md). Existing client setup and reconnect behavior are unchanged in Phase 1.
|
|
373
373
|
|
package/dist/bin/api-server.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import{readFileSync as s}from"fs";function c(){try{for(let r of s(".env","utf8").split(`
|
|
3
|
-
`)){let o=r.indexOf("=");if(o<1||r.trimStart().startsWith("#"))continue;let e=r.slice(0,o).trim();process.env[e]||(process.env[e]=r.slice(o+1).trim())}}catch{}}c();async function a(){let[{serve:r},{app:o},{startWorker:e},{migrate:i}]=await Promise.all([import("@hono/node-server"),import("../server-
|
|
3
|
+
`)){let o=r.indexOf("=");if(o<1||r.trimStart().startsWith("#"))continue;let e=r.slice(0,o).trim();process.env[e]||(process.env[e]=r.slice(o+1).trim())}}catch{}}c();async function a(){let[{serve:r},{app:o},{startWorker:e},{migrate:i}]=await Promise.all([import("@hono/node-server"),import("../server-G2ZCSWXW.js"),import("../worker-P2BICG36.js"),import("../db-B5XJTOGN.js")]),n=parseInt(process.env.PORT??"3001");try{if(await i(),process.env.ANALYTICS_DATABASE_URL){let{migrateAnalytics:t}=await import("../analytics-repository-IHOFBSUV.js");await t()}e(),r({fetch:o.fetch,port:n},t=>{console.log(`[server] http://localhost:${t.port}`),console.log(`[server] admin auth: ${process.env.ADMIN_KEY?"configured":"not configured"}`)})}catch(t){console.error("[startup] server preflight failed",t instanceof Error?t.name:"unknown_error"),process.exit(1)}}a();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-W2T4NTCE.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-Y6MKMSOC.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import{a as P}from"../chunk-
|
|
2
|
+
import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-W2T4NTCE.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-Y6MKMSOC.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import{a as P}from"../chunk-SWCAKYBP.js";import{Command as he}from"commander";import{spawn as ne}from"child_process";import{mkdir as ke,writeFile as Pe}from"fs/promises";import{basename as Ce,join as Z}from"path";function se(e){return e.apiKey?.trim()||"sk_live_your_key"}function ce(e){return e.packageSpec?.trim()||"mcp-scraper@latest"}function A(e={}){return["-y","--package",ce(e),"mcp-scraper"]}function pe(e){let n={MCP_SCRAPER_API_KEY:se(e)},c=e.browserProfileName?.trim();return c&&(n.BROWSER_AGENT_PROFILE_NAME=c),e.browserProfileSaveChanges===!0&&(n.BROWSER_AGENT_PROFILE_SAVE_CHANGES="true"),n}function q(){return["mcp","remove","mcp-scraper","-s","user"]}function J(){return["mcp","get","mcp-scraper"]}function B(e){let n=e.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];if(!n)return null;let c=e.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1]??"",i=c.length?c.split(/\s+/):[],p={},u=e.split(/^\s*Environment:\s*$/m)[1];if(u)for(let a of u.split(`
|
|
3
3
|
`)){let l=a.match(/^\s{2,}([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);if(!l){if(a.trim().length&&!/^\s{2,}/.test(a))break;continue}p[l[1]]=l[2]}return{command:n,args:i,env:p}}function j(e){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(e.env))n.push("--env",`${c}=${i}`);return n.push("--",e.command,...e.args),n}function G(e={}){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(pe(e)))n.push("--env",`${c}=${i}`);return n.push("--","npx",...A(e)),n}function O(e){if(e==="claude-code")return"claude";if(e==="claude"||D.hosts.some(n=>n.id===e))return e;throw new Error('Unknown host "'+e+'". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only')}function ue(e){return K(e==="claude"?"claude-code":e)}function W(e,n={}){let c=O(e),i=ue(c),p="Restart the MCP client so it starts a fresh npx process.",u='MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply',a=`X-Ray install protocol: ${v} (${b})`;return c==="codex"?["# Codex MCP config",a,i.exactConfig,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
4
4
|
`):c==="claude"?["# Claude Code command",a,i.exactConfig,"","# One-command Claude Code setup",u,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
5
5
|
`):c==="claude-desktop"?["# Claude Desktop config",a,i.exactConfig,"","Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,p].join(`
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as e}from"../chunk-
|
|
2
|
+
import{a as e}from"../chunk-BS4V4CVX.js";import"../chunk-4FGLXXZ2.js";import"../chunk-DT2FYN6N.js";import"../chunk-W2BVJ7S2.js";import"../chunk-RK2VCTZI.js";import"../chunk-ORB4RHCK.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-YGBTTW5D.js";import"../chunk-UN6FVDJQ.js";import"../chunk-SH5KB4P7.js";import"../chunk-2SP57VCG.js";import"../chunk-6DTXIZY2.js";import"../chunk-DFJT2YX6.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-SWCAKYBP.js";import"../chunk-WJ4XFLS4.js";var _=["harvest_paa","search_serp","extract_url","diff_page","map_site_urls","map_wayback_snapshots","extract_site","analyze_site_similarity","audit_site","check_site_export","site_export_read","site_export_image","archive_read","youtube_harvest","youtube_transcribe","facebook_page_intel","facebook_ad_search","reddit_thread","reddit_trending","video_frame_analysis","video_frame_analysis_status","facebook_ad_transcribe","google_ads_search","google_ads_page_intel","google_ads_transcribe","facebook_video_transcribe","instagram_profile_content","instagram_media_download","maps_place_intel","maps_search","trustpilot_reviews","g2_reviews","capture_serp_snapshot","capture_serp_page_snapshots"];e({toolsets:new Set(["paa","serp"]),allowedToolNames:_});
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as s}from"../chunk-
|
|
2
|
+
import{a as s}from"../chunk-DFJT2YX6.js";import{a as e}from"../chunk-SWCAKYBP.js";var r=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY,n=process.argv.includes("--help")||process.argv.includes("-h");n&&(process.stdout.write(["Usage: mcp-scraper-install [--no-color]","","Prints the branded MCP Scraper terminal install card and copyable install commands.","mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.",""].join(`
|
|
3
3
|
`)),process.exit(0));process.stdout.write(s({version:e,color:!r,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()}));
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as r}from"../chunk-
|
|
2
|
+
import{a as r}from"../chunk-BS4V4CVX.js";import"../chunk-4FGLXXZ2.js";import"../chunk-DT2FYN6N.js";import"../chunk-W2BVJ7S2.js";import"../chunk-RK2VCTZI.js";import"../chunk-ORB4RHCK.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-YGBTTW5D.js";import"../chunk-UN6FVDJQ.js";import"../chunk-SH5KB4P7.js";import"../chunk-2SP57VCG.js";import"../chunk-6DTXIZY2.js";import"../chunk-DFJT2YX6.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-SWCAKYBP.js";import"../chunk-WJ4XFLS4.js";r();
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{w as t}from"../chunk-M5VZVMPC.js";import"../chunk-M5QHXNFZ.js";import{b as r}from"../chunk-SH5KB4P7.js";import"../chunk-2SP57VCG.js";import"../chunk-6DTXIZY2.js";import"../chunk-Y6MKMSOC.js";import"../chunk-WJ4XFLS4.js";import{Command as s,Option as a}from"commander";var i=new s;i.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>","Seed query").option("-l, --location <location>",'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>","Google country code","us").option("--hl <hl>","Google language code","en").option("-d, --depth <depth>","BFS depth (1-30)","3").option("-m, --max-questions <n>","Max questions to harvest","100").option("-o, --output <dir>","Output directory","./paa-output").option("-f, --format <format>","Output format: json, csv, or both","both").option("--headless","Run browser in headless mode",!1).option("--profile <dir>","Persistent browser profile directory").option("--proxy <url>","Proxy server URL").option("--browser-api-key <key>","Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").addOption(new a("
|
|
2
|
+
import{w as t}from"../chunk-M5VZVMPC.js";import"../chunk-M5QHXNFZ.js";import{b as r}from"../chunk-SH5KB4P7.js";import"../chunk-2SP57VCG.js";import"../chunk-6DTXIZY2.js";import"../chunk-Y6MKMSOC.js";import"../chunk-WJ4XFLS4.js";import{Command as s,Option as a}from"commander";var i=new s;i.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>","Seed query").option("-l, --location <location>",'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>","Google country code","us").option("--hl <hl>","Google language code","en").option("-d, --depth <depth>","BFS depth (1-30)","3").option("-m, --max-questions <n>","Max questions to harvest","100").option("-o, --output <dir>","Output directory","./paa-output").option("-f, --format <format>","Output format: json, csv, or both","both").option("--headless","Run browser in headless mode",!1).option("--profile <dir>","Persistent browser profile directory").option("--proxy <url>","Proxy server URL").option("--browser-api-key <key>","Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").addOption(new a("--kernel-api-key <key>").hideHelp()).action(async e=>{try{let o=await t({query:e.query,location:e.location,gl:e.gl,hl:e.hl,depth:parseInt(e.depth,10),maxQuestions:parseInt(e.maxQuestions,10),outputDir:e.output,format:e.format,headless:e.headless,profileDir:e.profile,proxy:e.proxy,kernelApiKey:e.browserApiKey??e.kernelApiKey??r()});console.log(JSON.stringify({totalQuestions:o.totalQuestions,outputDir:o.stats.seed}))}catch(o){console.error(o instanceof Error?o.message:String(o)),process.exit(1)}});async function n(){await i.parseAsync()}n();
|
package/dist/chunk-2SP57VCG.js
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-WJ4XFLS4.js";var c=166667e-10,N=.0001333336,I=new Set(["serp","fb_search","fb_ad","instagram"]),p=.00111,L=4,R=4e-4;function o(e,r){let t=process.env[e]?.trim();if(!t)return r;let s=Number(t);return Number.isFinite(s)&&s>=0?s:r}var A=o("NANGO_USD_PER_CONNECTION_MONTH",1),U=o("NANGO_USD_PER_FUNCTION_RUN",1e-4),m=o("NANGO_USD_PER_PROXY_REQUEST",1e-4),g=o("NANGO_USD_PER_COMPUTE_SEC",2e-4),O=o("
|
|
1
|
+
import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-WJ4XFLS4.js";var c=166667e-10,N=.0001333336,I=new Set(["serp","fb_search","fb_ad","instagram"]),p=.00111,L=4,R=4e-4;function o(e,r){let t=process.env[e]?.trim();if(!t)return r;let s=Number(t);return Number.isFinite(s)&&s>=0?s:r}var A=o("NANGO_USD_PER_CONNECTION_MONTH",1),U=o("NANGO_USD_PER_FUNCTION_RUN",1e-4),m=o("NANGO_USD_PER_PROXY_REQUEST",1e-4),g=o("NANGO_USD_PER_COMPUTE_SEC",2e-4),O=o("BRIGHTDATA_BROWSER_USD_PER_GB",8),D=o("BRIGHTDATA_SERP_USD_PER_REQUEST",.002836168);function u(e,r){return Math.max(0,e)/1e3*(r?N:c)}function d(e,r){return e==="fal_wizper"?Math.max(0,r)/L*p:e==="deepinfra_qwen"?Math.max(0,r)/1e3*R:e==="openrouter"||e==="mcp_memory_video"||e==="mcp_memory_ai"?Math.max(0,r):e==="nango_connection"?Math.max(0,r)*A:e==="nango_function_run"?Math.max(0,r)*U:e==="nango_proxy_request"?Math.max(0,r)*m:e==="nango_compute"?Math.max(0,r)*g:e==="brightdata_browser_api"?Math.max(0,r)/1e9*O:0}import{randomUUID as i}from"crypto";var E=!1,n=null;async function T(){if(!E)return n||(n=S().finally(()=>{n=null}),n)}async function S(){let e=_(),r=await e.execute(`
|
|
2
2
|
SELECT
|
|
3
|
-
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('
|
|
4
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('
|
|
3
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
4
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
5
5
|
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
|
|
6
6
|
AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
|
|
7
7
|
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
8
8
|
AS ready
|
|
9
9
|
`);if(Number(r.rows[0]?.ready??0)===1){E=!0;return}await e.execute(`
|
|
10
|
-
CREATE TABLE IF NOT EXISTS
|
|
10
|
+
CREATE TABLE IF NOT EXISTS kernel_session_log (
|
|
11
11
|
id TEXT PRIMARY KEY,
|
|
12
|
-
|
|
12
|
+
kernel_session_id TEXT,
|
|
13
13
|
op TEXT,
|
|
14
14
|
source TEXT NOT NULL,
|
|
15
15
|
probe_run_id TEXT,
|
|
@@ -26,7 +26,7 @@ import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-WJ4XFLS4.js";
|
|
|
26
26
|
error TEXT,
|
|
27
27
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
28
28
|
)
|
|
29
|
-
`),await e.execute("CREATE INDEX IF NOT EXISTS
|
|
29
|
+
`),await e.execute("CREATE INDEX IF NOT EXISTS kernel_session_log_op ON kernel_session_log(op)"),await e.execute("CREATE INDEX IF NOT EXISTS kernel_session_log_probe ON kernel_session_log(probe_run_id)"),await e.execute("CREATE INDEX IF NOT EXISTS kernel_session_log_created ON kernel_session_log(created_at)");try{await e.execute("ALTER TABLE kernel_session_log ADD COLUMN proxy_source TEXT")}catch{}try{await e.execute("ALTER TABLE kernel_session_log ADD COLUMN proxy_type TEXT")}catch{}try{await e.execute("ALTER TABLE kernel_session_log ADD COLUMN method TEXT")}catch{}await e.execute(`
|
|
30
30
|
CREATE TABLE IF NOT EXISTS vendor_usage_log (
|
|
31
31
|
id TEXT PRIMARY KEY,
|
|
32
32
|
op TEXT,
|
|
@@ -54,20 +54,20 @@ import{c as l}from"./chunk-6DTXIZY2.js";import{t as _}from"./chunk-WJ4XFLS4.js";
|
|
|
54
54
|
success INTEGER NOT NULL DEFAULT 0,
|
|
55
55
|
error TEXT,
|
|
56
56
|
http_only INTEGER NOT NULL DEFAULT 0,
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
kernel_used INTEGER NOT NULL DEFAULT 0,
|
|
58
|
+
kernel_sessions INTEGER NOT NULL DEFAULT 0,
|
|
59
|
+
kernel_seconds_total REAL NOT NULL DEFAULT 0,
|
|
60
|
+
kernel_tier_observed TEXT,
|
|
61
|
+
est_kernel_cost_usd_headless REAL NOT NULL DEFAULT 0,
|
|
62
|
+
est_kernel_cost_usd_headful REAL NOT NULL DEFAULT 0,
|
|
63
63
|
vendor_cost_usd REAL NOT NULL DEFAULT 0,
|
|
64
64
|
charged_credits REAL NOT NULL DEFAULT 0,
|
|
65
65
|
margin_usd_headless_starter REAL NOT NULL DEFAULT 0,
|
|
66
66
|
margin_usd_headful_starter REAL NOT NULL DEFAULT 0,
|
|
67
67
|
notes TEXT
|
|
68
68
|
)
|
|
69
|
-
`),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function f(e){try{await T();let r=l(),t=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO
|
|
70
|
-
(id,
|
|
71
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e
|
|
69
|
+
`),await e.execute("CREATE INDEX IF NOT EXISTS cost_probe_runs_tool ON cost_probe_runs(tool)");try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN units REAL")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN unit_type TEXT")}catch{}try{await e.execute("ALTER TABLE cost_probe_runs ADD COLUMN mode TEXT")}catch{}E=!0}async function f(e){try{await T();let r=l(),t=Math.max(0,e.closedAtMs-e.openedAtMs);await _().execute({sql:`INSERT INTO kernel_session_log
|
|
70
|
+
(id, kernel_session_id, op, source, probe_run_id, user_id, stealth, headless_sent, proxy_used, proxy_source, proxy_type, fallback, opened_at, closed_at, duration_ms, est_cost_usd_headless, est_cost_usd_headful, error, method)
|
|
71
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.kernelSessionId??null,r?.op??null,e.source,r?.probeRunId??null,r?.userId??null,a(e.stealth),a(e.headlessSent),a(e.proxyUsed),e.proxySource??null,e.proxyType??null,e.fallback?1:0,new Date(e.openedAtMs).toISOString(),new Date(e.closedAtMs).toISOString(),t,u(t,!1),u(t,!0),e.error??null,r?.subOp??null]})}catch(r){console.warn("[cost-telemetry] recordKernelSession failed:",r instanceof Error?r.message:String(r))}}async function v(e){try{await T();let r=l();return(await _().execute({sql:`INSERT OR IGNORE INTO vendor_usage_log
|
|
72
72
|
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
|
|
73
73
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,args:[i(),e.op??r?.op??null,e.probeRunId??r?.probeRunId??null,e.userId??r?.userId??null,e.vendor,e.model??null,e.units,e.unitType,d(e.vendor,e.units),e.error??null,e.method??r?.subOp??null,e.sourceKey??null,e.providerDurationMs??null,a(e.providerCaptcha),e.providerStatus??null]})).rowsAffected===1}catch(r){return console.warn("[cost-telemetry] recordVendorUsage failed:",r instanceof Error?r.message:String(r)),!1}}function a(e){return e==null?null:e?1:0}export{c as a,N as b,I as c,u as d,d as e,T as f,f as g,v as h};
|