mcp-scraper 0.89.1 → 0.89.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -14
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/{analytics-repository-J25XR5E7.js → analytics-repository-X6J7TWTL.js} +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-FFPD2DOF.js → chunk-2O3UYISB.js} +1 -1
- package/dist/chunk-3GP5CYZX.js +1 -1
- package/dist/{chunk-MEM4D2QN.js → chunk-3MUEBDWY.js} +4 -4
- package/dist/chunk-4FROKQJN.js +1 -1
- package/dist/{chunk-RYNHNEF3.js → chunk-D4NOJAUN.js} +15 -15
- package/dist/{chunk-645LQBE7.js → chunk-DEPJ4ZZ6.js} +1 -1
- package/dist/{chunk-IJ2AO7AO.js → chunk-DMCYIWB3.js} +1 -1
- package/dist/{chunk-ZHORX44A.js → chunk-EN66HBAW.js} +1 -1
- package/dist/{chunk-PRWZNHQB.js → chunk-HYQDXTS7.js} +1 -1
- package/dist/{chunk-6QO4F2KC.js → chunk-J7EVLE6H.js} +3 -3
- package/dist/{chunk-57AHYL2O.js → chunk-K7KMLT33.js} +1 -1
- package/dist/{chunk-YUPLOOTB.js → chunk-MKEMCMRO.js} +1 -1
- package/dist/chunk-QPWPR5XG.js +3 -3
- package/dist/chunk-S7RA23GR.js +1 -0
- package/dist/chunk-TW46NKVH.js +1 -0
- package/dist/{chunk-DGBLGFNJ.js → chunk-VEJQ4XPI.js} +1 -1
- package/dist/{chunk-D6IQJP64.js → chunk-Z27SYWYD.js} +1 -1
- package/dist/{chunk-63T4W2LX.js → chunk-Z66Z2HV4.js} +87 -83
- package/dist/{chunk-CCYWSJNG.js → chunk-ZWZ6OWY6.js} +1 -1
- package/dist/{db-JJWJLAVQ.js → db-NLGNTQC3.js} +1 -1
- package/dist/{extract-bundle-TTSEGWYF.js → extract-bundle-TXH3VNXH.js} +1 -1
- package/dist/{gmail-service-34PIJRLK.js → gmail-service-J2DWSQJX.js} +1 -1
- package/dist/index.cjs +49 -37
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/index.js +1 -1
- package/dist/{lead-list-enrichment-repository-VJBBMBCG.js → lead-list-enrichment-repository-G7Z5DQZR.js} +1 -1
- package/dist/{location-data-repository-W3LPT6G3.js → location-data-repository-TEXZ3I2N.js} +1 -1
- package/dist/{server-UEY5YQL6.js → server-LBOFC2EE.js} +718 -711
- package/dist/{site-extract-repository-7W5ZUNPR.js → site-extract-repository-MBSCAP4D.js} +1 -1
- package/dist/worker-SUCJKWU2.js +1 -0
- package/package.json +17 -131
- package/dist/chunk-5H3MCTTP.js +0 -1
- package/dist/chunk-FXEWHK54.js +0 -1
- package/dist/worker-ZOTOKG56.js +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,12 +4,30 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.89.2] - 2026-09-09
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- Made production OAuth configuration explicit and fail-closed, with mode-specific database, epoch, cutoff, and key-ring requirements checked before discovery or mutations can appear healthy.
|
|
12
|
+
- Moved dynamic-client-registration admission into the selected canonical OAuth authority under schema version 3, using hashed caller keys, one atomic increment, bounded cleanup, and operator-run schema migration.
|
|
13
|
+
- Added an authority-epoch handoff and rollback procedure with accepted recovery objectives of at most 15 seconds of data loss, recovery within one hour, and ten days of point-in-time-recovery retention.
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- Removed request-time schema creation and the follow-up count query from the main-database rate limiter; each admitted request now uses one upsert that returns its committed count.
|
|
18
|
+
|
|
19
|
+
### Security
|
|
20
|
+
|
|
21
|
+
- Added a guarded production canary command that accepts only the dedicated synthetic account, rotates its password and API key transactionally in memory, withholds database credentials from the verifier, and writes only redacted receipts.
|
|
22
|
+
|
|
7
23
|
## [0.89.1] - 2026-09-08
|
|
8
24
|
|
|
9
25
|
### Changed
|
|
10
26
|
|
|
11
27
|
- Added an operator-controlled path to isolate OAuth state in a dedicated Turso database with authority epochs, bounded cleanup, verified migration and rollback, and fail-closed maintenance behavior. Existing OAuth issuer, client setup, API-key authentication, and MCP tool contracts remain unchanged.
|
|
12
28
|
- Bounded the root cron by admitting work lazily with explicit concurrency and deadline limits, so timed-out responses no longer eagerly start every maintenance task.
|
|
29
|
+
- Routed OAuth and its discovery documents through a small dedicated serverless entrypoint, keeping static discovery and token operations independent of unrelated hosted-product startup gates.
|
|
30
|
+
- Made the authorization issuer an explicit HTTPS-origin configuration with the existing production origin as its default, allowing isolated previews to verify their own signing keys without reusing production credentials.
|
|
13
31
|
|
|
14
32
|
### Fixed
|
|
15
33
|
|
|
@@ -38,7 +56,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
38
56
|
|
|
39
57
|
### Fixed
|
|
40
58
|
|
|
41
|
-
- Give each `reddit_thread` retrieval a 300-second end-to-end deadline, with two 60-second
|
|
59
|
+
- Give each `reddit_thread` retrieval a 300-second end-to-end deadline, with two 60-second Option 1 attempts and two 60-second managed-browser backup attempts, instead of exhausting the full retry ladder in about 50 seconds. The MCP client now waits long enough to receive the endpoint's structured terminal result.
|
|
42
60
|
|
|
43
61
|
## [0.88.2] - 2026-09-02
|
|
44
62
|
|
|
@@ -75,19 +93,19 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
75
93
|
|
|
76
94
|
### Fixed
|
|
77
95
|
|
|
78
|
-
- Kept
|
|
96
|
+
- Kept Option 2 telemetry lookup off the Reddit response critical path and reallocated the saved time to 17-second backup attempts, so all four provider attempts can finish before production ends the request.
|
|
79
97
|
|
|
80
98
|
## [0.86.4] - 2026-09-02
|
|
81
99
|
|
|
82
100
|
### Fixed
|
|
83
101
|
|
|
84
|
-
- Kept the complete two-primary, two-backup Reddit retry ladder inside the production request window by limiting
|
|
102
|
+
- Kept the complete two-primary, two-backup Reddit retry ladder inside the production request window by limiting Option 1 attempts to 8 seconds, Option 2 attempts to 14 seconds, and browser cleanup to 1 second.
|
|
85
103
|
|
|
86
104
|
## [0.86.3] - 2026-09-02
|
|
87
105
|
|
|
88
106
|
### Fixed
|
|
89
107
|
|
|
90
|
-
- Applied 45-second
|
|
108
|
+
- Applied 45-second Option 1 and 35-second Option 2 deadlines to the complete Reddit browser-attempt lifecycle, and made known-thread primary attempts find and click the target through DuckDuckGo before the residential landing.
|
|
91
109
|
|
|
92
110
|
## [0.86.2] - 2026-09-02
|
|
93
111
|
|
|
@@ -236,13 +254,13 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
236
254
|
|
|
237
255
|
### Added
|
|
238
256
|
|
|
239
|
-
- Added a
|
|
240
|
-
- Added a bounded managed-browser backup for Reddit thread hydration after the primary
|
|
257
|
+
- Added a Option 1-only Reddit workflow that searches DuckDuckGo with a `site:reddit.com` query, switches the same browser to a residential proxy before clicking the selected result, and reads modern Reddit posts plus bounded rendered-comment expansion through dedicated search, thread, and combined REST endpoints.
|
|
258
|
+
- Added a bounded managed-browser backup for Reddit thread hydration after the primary Option 1 attempt fails or returns fewer than the semantic target, capped at 25 comments with measured bandwidth, duration, CAPTCHA, closure, and provider-cost telemetry.
|
|
241
259
|
|
|
242
260
|
### Changed
|
|
243
261
|
|
|
244
|
-
- Routed the production `reddit_thread` and `reddit_trending` MCP tools through modern Reddit on
|
|
245
|
-
- Cost probes now include Reddit
|
|
262
|
+
- Routed the production `reddit_thread` and `reddit_trending` MCP tools through modern Reddit on Option 1 residential sessions, with DuckDuckGo site search for trend discovery; removed Google and old Reddit from their active execution path while preserving tool names, billing rates, bounded partial results, and refunds.
|
|
263
|
+
- Cost probes now include Reddit Option 1 sessions and any managed-browser fallback bytes and cost in the same request receipt, and identify when the backup contributed to total cost.
|
|
246
264
|
|
|
247
265
|
### Fixed
|
|
248
266
|
|
|
@@ -287,7 +305,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
287
305
|
|
|
288
306
|
### Fixed
|
|
289
307
|
|
|
290
|
-
- Persisted per-control PAA dispatch and 0.7/1.0/1.4-second confirmation telemetry in durable checkpoints, exposed recent interaction and attempt correlation through MCP status, attached
|
|
308
|
+
- Persisted per-control PAA dispatch and 0.7/1.0/1.4-second confirmation telemetry in durable checkpoints, exposed recent interaction and attempt correlation through MCP status, attached Option 2 session IDs immediately after browser launch, and finalized dangling attempt rows during lease recovery without blocking customer settlement.
|
|
291
309
|
- Prevented inline style, script, and hidden DOM text inside Google answer containers from falsely confirming that PAA answer material loaded.
|
|
292
310
|
- Routed canonical `/assistant` page loads to the web app and the redacted private Assistant readiness endpoint to the main API function, preventing production 404s after the 0.79.1 launch.
|
|
293
311
|
|
|
@@ -311,7 +329,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
311
329
|
- Added Scheduling as the canonical Personal Assistant setup surface, with connection readiness for Gmail, Calendar, Zoom, browser profiles, Memory, SMS, and email; exact schedule confirmation; approval and spend review; run history; and explicit watch/takeover states.
|
|
312
330
|
- Added owner-scoped browser profiles that can hold multiple independently verified login bindings, while every browser schedule grant selects one exact profile, login, domain, and action set.
|
|
313
331
|
- Added immutable schedule revisions, readiness receipts, append-only activation records, additive legacy schedule projection, and single-owner occurrence transition receipts so migration cannot silently infer browser authority or double-dispatch work.
|
|
314
|
-
- Added
|
|
332
|
+
- Added Option 1 and private-Mac browser runtime boundaries with collision-resistant tenant namespaces, per-owner concurrency ceilings, bounded sessions, explicit and timeout cleanup, owner-qualified account deletion, and provider deletion readback.
|
|
315
333
|
- Added an owner-controlled Personal Assistant that brings SMS/MMS, Gmail, Google Calendar, Zoom, browser work, reminders, and Memory context packets into one governed workflow with immutable plans, approval checkpoints, spend limits, and durable receipts.
|
|
316
334
|
- Added Twilio number discovery, owned-number attachment, purchase and registration previews, Messaging Service readiness, signed inbound and delivery webhooks, safe MMS ingestion, deterministic opt-out handling, single and reviewed bulk messaging, and reconciliation for unknown provider outcomes.
|
|
317
335
|
- Added immutable, revisioned Memory context packets with source and attachment provenance, Gmail full-message imports, MMS media metadata, lifecycle controls, and readback verification against the selected vault.
|
|
@@ -348,7 +366,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
348
366
|
- Made `maxQuestions` an explicit target count rather than a traversal-depth control, with separate discovery and material-completeness diagnostics.
|
|
349
367
|
- Preserved complete People Also Ask, AI Overview, and organic-result link provenance in JSON, structured MCP output, and CSV while classifying plain links and Google redirect links explicitly.
|
|
350
368
|
- Resolved opaque Google `/goto` targets through bounded concurrent manual-redirect requests with active-browser interception as a fallback, without following publisher destinations and without dropping unresolved material.
|
|
351
|
-
- Aligned the bounded PAA production-provider canary with the public `maxQuestions` contract and made
|
|
369
|
+
- Aligned the bounded PAA production-provider canary with the public `maxQuestions` contract and made Option 2 the default test provider.
|
|
352
370
|
|
|
353
371
|
### Fixed
|
|
354
372
|
|
|
@@ -556,7 +574,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
556
574
|
|
|
557
575
|
- Added portable `harvest_paa_start` and `harvest_paa_status` tools for durable long-running PAA research, with stable idempotency recovery, progress, attempt provenance, completeness, billing state, and bounded provider telemetry.
|
|
558
576
|
- Added progressive PAA checkpoints that preserve and merge the best unique rows across browser retries and stale-job recovery instead of losing already captured questions when a provider session or caller is interrupted.
|
|
559
|
-
- Added exact
|
|
577
|
+
- Added exact Option 2 browser-session identity, sanitized Session Logs enrichment, disconnect attribution, bandwidth usage telemetry, and retryable reconciliation without making provider telemetry a prerequisite for result delivery.
|
|
560
578
|
|
|
561
579
|
### Changed
|
|
562
580
|
|
|
@@ -1289,7 +1307,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1289
1307
|
|
|
1290
1308
|
### Changed
|
|
1291
1309
|
|
|
1292
|
-
- PAA browser work now uses
|
|
1310
|
+
- PAA browser work now uses Option 1's co-located Playwright execution with stealth mode's default managed proxy and native browser metadata. Location is expressed only through Google UULE, CAPTCHA solver waiting is capped at 60 seconds, and a fresh session is allowed once only when no useful data was captured.
|
|
1293
1311
|
- PAA invocations stop browser work at 250 seconds inside the 280-second application budget, reserving 30 seconds for persistence, cleanup, and settlement. The legacy cron worker no longer claims Inngest-owned PAA jobs.
|
|
1294
1312
|
|
|
1295
1313
|
### Fixed
|
|
@@ -1525,7 +1543,7 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1525
1543
|
|
|
1526
1544
|
### Changed
|
|
1527
1545
|
|
|
1528
|
-
- `maps_search` now applies a transport ladder across its retry attempts so it can recover from Google soft-blocks instead of only retrying the same way. The first attempt is unchanged (
|
|
1546
|
+
- `maps_search` now applies a transport ladder across its retry attempts so it can recover from Google soft-blocks instead of only retrying the same way. The first attempt is unchanged (Option 1's default stealth ISP proxy, direct navigation). Subsequent retries switch to direct egress and arrive at Google through a cross-site redirect (the combination that measurably clears blocks a cold navigation triggers); the final escalation attempt uses direct egress without the redirect and accepts any egress country. This only affects the `proxyMode: 'none'` default path and only its retries — a first-attempt success behaves exactly as before.
|
|
1529
1547
|
|
|
1530
1548
|
## [0.32.1] - 2026-07-22
|
|
1531
1549
|
|
|
@@ -1919,3 +1937,4 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1919
1937
|
[0.89.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.88.3...v0.89.0
|
|
1920
1938
|
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.89.1...HEAD
|
|
1921
1939
|
[0.89.1]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.89.0...v0.89.1
|
|
1940
|
+
[0.89.2]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.89.1...v0.89.2
|
package/README.md
CHANGED
|
@@ -175,7 +175,7 @@ Build the branded one-click bundle:
|
|
|
175
175
|
npm run build:mcpb
|
|
176
176
|
```
|
|
177
177
|
|
|
178
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.89.
|
|
178
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.89.2`, SHA-256 `55acdb5389483e0854d433835c45bd82cbd174680e93d78041e0ff2939ca2077`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
179
179
|
|
|
180
180
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
181
181
|
|
|
@@ -366,8 +366,8 @@ The `mcp-scraper` NPX stdio server also exposes saved reports as MCP resources:
|
|
|
366
366
|
- `MCP_SCRAPER_OUTPUT_DIR` is optional and defaults to `~/Downloads/mcp-scraper`.
|
|
367
367
|
- `MCP_SCRAPER_SAVE_REPORTS=false` disables automatic Markdown report files.
|
|
368
368
|
- `MCP_SCRAPER_KEY_PATH` is optional. When no API key env var is set, the server also reads `~/.mcp-scraper-key` for compatibility with older installs.
|
|
369
|
-
- `BROWSER_AGENT_PROFILE_NAME` is optional and sets the default saved hosted browser profile for `mcp-scraper` stdio sessions. Aliases: `BROWSER_SERVICE_PROFILE_NAME`, `
|
|
370
|
-
- `BROWSER_AGENT_PROFILE_SAVE_CHANGES=true` is optional hosted setup behavior. It persists cookies and storage back to the named profile when `browser_close` deletes the hosted browser session. Aliases: `BROWSER_SERVICE_PROFILE_SAVE_CHANGES`, `
|
|
369
|
+
- `BROWSER_AGENT_PROFILE_NAME` is optional and sets the default saved hosted browser profile for `mcp-scraper` stdio sessions. Aliases: `BROWSER_SERVICE_PROFILE_NAME`, `Option 1_BROWSER_PROFILE_NAME`, `Option 1_PROFILE_NAME`.
|
|
370
|
+
- `BROWSER_AGENT_PROFILE_SAVE_CHANGES=true` is optional hosted setup behavior. It persists cookies and storage back to the named profile when `browser_close` deletes the hosted browser session. Aliases: `BROWSER_SERVICE_PROFILE_SAVE_CHANGES`, `Option 1_BROWSER_PROFILE_SAVE_CHANGES`, `Option 1_PROFILE_SAVE_CHANGES`.
|
|
371
371
|
|
|
372
372
|
Hosted operators can isolate authorization state in a dedicated Turso/libSQL database without changing the public MCP tool catalog or API-key authentication. The secured store uses atomic authorization-code exchange and refresh rotation, keyed secret lookup, bounded encrypted replay receipts, authority epochs, and fail-closed maintenance behavior. Production migration and rollback are controlled data moves, not ordinary mode flips; see [MCP OAuth operations](docs/operations/mcp-oauth-runbook.md). Existing client setup and reconnect behavior are unchanged in Phase 1.
|
|
373
373
|
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en"><meta charset="utf-8"><title>Third-party notices</title><body><h1>Browser control library</h1><p>This distribution bundles the browser control library with build-time minification and literal-name encoding. Its license follows.</p><pre> Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright 2026 Kernel
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
|
203
|
+
</pre></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{$ as L,$a as La,A as k,Aa as ka,B as l,Ba as la,C as m,Ca as ma,D as n,Da as na,E as o,Ea as oa,F as p,Fa as pa,G as q,Ga as qa,H as r,Ha as ra,I as s,Ia as sa,J as t,Ja as ta,K as u,Ka as ua,L as v,La as va,M as w,Ma as wa,N as x,Na as xa,O as y,Oa as ya,P as z,Pa as za,Q as A,Qa as Aa,R as B,Ra as Ba,S as C,Sa as Ca,T as D,Ta as Da,U as E,Ua as Ea,V as F,Va as Fa,W as G,Wa as Ga,X as H,Xa as Ha,Y as I,Ya as Ia,Z as J,Za as Ja,_ as K,_a as Ka,aa as M,ab as Ma,ba as N,bb as Na,ca as O,da as P,ea as Q,fa as R,ga as S,ha as T,ia as U,ja as V,ka as W,la as X,ma as Y,na as Z,oa as _,pa as $,q as a,qa as aa,r as b,ra as ba,s as c,sa as ca,t as d,ta as da,u as e,ua as ea,v as f,va as fa,w as g,wa as ga,x as h,xa as ha,y as i,ya as ia,z as j,za as ja}from"./chunk-
|
|
1
|
+
import{$ as L,$a as La,A as k,Aa as ka,B as l,Ba as la,C as m,Ca as ma,D as n,Da as na,E as o,Ea as oa,F as p,Fa as pa,G as q,Ga as qa,H as r,Ha as ra,I as s,Ia as sa,J as t,Ja as ta,K as u,Ka as ua,L as v,La as va,M as w,Ma as wa,N as x,Na as xa,O as y,Oa as ya,P as z,Pa as za,Q as A,Qa as Aa,R as B,Ra as Ba,S as C,Sa as Ca,T as D,Ta as Da,U as E,Ua as Ea,V as F,Va as Fa,W as G,Wa as Ga,X as H,Xa as Ha,Y as I,Ya as Ia,Z as J,Za as Ja,_ as K,_a as Ka,aa as M,ab as Ma,ba as N,bb as Na,ca as O,da as P,ea as Q,fa as R,ga as S,ha as T,ia as U,ja as V,ka as W,la as X,ma as Y,na as Z,oa as _,pa as $,q as a,qa as aa,r as b,ra as ba,s as c,sa as ca,t as d,ta as da,u as e,ua as ea,v as f,va as fa,w as g,wa as ga,x as h,xa as ha,y as i,ya as ia,z as j,za as ja}from"./chunk-DEPJ4ZZ6.js";import"./chunk-Z27SYWYD.js";import"./chunk-Z66Z2HV4.js";export{P as ANALYTICS_CONTENT_SORTS,a as AnalyticsRepositoryError,B as ENGAGED_SESSION_MS,A as MAX_ENGAGED_MS,N as analyticsAcquisition,l as analyticsBusinessMetrics,O as analyticsChannelBreakdown,R as analyticsContent,T as analyticsConversions,Ma as analyticsCsvCell,V as analyticsDimensions,S as analyticsEventCounts,m as analyticsForecast,Ja as analyticsHealth,x as analyticsIdentityPromotionAllowed,w as analyticsIdentityResolutionAllowed,L as analyticsOverview,U as analyticsPaths,M as analyticsTimeseries,H as appendAnalyticsAuthoritativeOutcomeVersion,ta as archiveAnalyticsActivationDestination,Y as archiveAnalyticsCampaignLink,ga as assignAnalyticsIdentityNode,fa as backfillAnalyticsConfirmedHistory,oa as claimAnalyticsCrmImportRows,Ea as claimAnalyticsFormDeliveryJobs,c as closeAnalyticsPool,pa as completeAnalyticsCrmImportRow,Ga as completeAnalyticsFormBridgeDelivery,Fa as completeAnalyticsFormDelivery,J as consumeAnalyticsSurveyInvite,ra as createAnalyticsActivationDestination,W as createAnalyticsCampaignLink,K as createAnalyticsConversion,ma as createAnalyticsCrmImport,Na as createAnalyticsExport,_ as createAnalyticsForm,o as createAnalyticsPixel,h as createAnalyticsSite,qa as deferAnalyticsCrmImportRow,Ha as deferAnalyticsFormDelivery,n as deleteAnalyticsSite,ea as deterministicAnalyticsCrmEntityId,ja as enrichAnalyticsExistingCrmIdentity,ua as getAnalyticsActivationDestinationConnectionRef,la as getAnalyticsPersonJourney,b as getAnalyticsPool,aa as getPublicAnalyticsForm,da as identityHmac,F as ingestAnalyticsEvents,G as insertAnalyticsRevenueSetupRevision,Ia as isAnalyticsFormPlacementApproved,ia as linkAnalyticsFormIdentity,ha as linkAnalyticsIdentityInTransaction,sa as listAnalyticsActivationDestinations,xa as listAnalyticsActivationReceipts,X as listAnalyticsCampaignLinks,na as listAnalyticsCrmImports,$ as listAnalyticsForms,t as listAnalyticsHostGroups,ka as listAnalyticsPeople,p as listAnalyticsPixels,i as listAnalyticsSites,d as migrateAnalytics,C as normalizeAnalyticsPath,D as normalizeAnalyticsUrl,Q as normalizeContentOptions,e as normalizeObservedHostname,za as pollAnalyticsActivationDiagnostics,u as prepareAnalyticsLinkerIssue,I as projectAnalyticsAuthoritativeConversion,y as projectAnalyticsPixelEventConsent,Aa as queueAnalyticsActivation,Ca as queueAnalyticsFormBridgeTransaction,Ba as queueAnalyticsFormDelivery,ba as recordAnalyticsFormSubmission,v as redeemAnalyticsLinkerRecord,Ka as refreshAnalyticsDailyRollups,La as refreshAnalyticsDailyRollupsIfDue,f as requireAnalyticsAccess,g as requireAnalyticsEditor,Z as resolveAnalyticsCampaignLink,z as resolveAnalyticsConfirmedActivationIdentity,ya as retryAnalyticsActivationJob,E as sanitizeAnalyticsProperties,ca as sanitizeClickIds,va as setAnalyticsActivationReadiness,r as setAnalyticsPixelDomainState,Da as sweepAnalyticsRestrictedRetention,wa as testAnalyticsActivationDestination,j as updateAnalyticsBusinessModel,q as updateAnalyticsPixel,k as upsertAnalyticsAdSpend,s as upsertAnalyticsHostGroup};
|
package/dist/bin/api-server.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import{readFileSync as a}from"fs";function c(){try{for(let t of a(".env","utf8").split(`
|
|
3
|
-
`)){let o=t.indexOf("=");if(o<1||t.trimStart().startsWith("#"))continue;let e=t.slice(0,o).trim();process.env[e]||(process.env[e]=t.slice(o+1).trim())}}catch{}}c();async function p(){let[{serve:t},{app:o,personalAssistantProductionStartup:e},{startWorker:i},{migrate:n}]=await Promise.all([import("@hono/node-server"),import("../server-
|
|
3
|
+
`)){let o=t.indexOf("=");if(o<1||t.trimStart().startsWith("#"))continue;let e=t.slice(0,o).trim();process.env[e]||(process.env[e]=t.slice(o+1).trim())}}catch{}}c();async function p(){let[{serve:t},{app:o,personalAssistantProductionStartup:e},{startWorker:i},{migrate:n}]=await Promise.all([import("@hono/node-server"),import("../server-LBOFC2EE.js"),import("../worker-SUCJKWU2.js"),import("../db-NLGNTQC3.js")]),s=parseInt(process.env.PORT??"3001");try{if(await e,await n(),process.env.ANALYTICS_DATABASE_URL){let{migrateAnalytics:r}=await import("../analytics-repository-X6J7TWTL.js");await r()}i(),t({fetch:o.fetch,port:s},r=>{console.log(`[server] http://localhost:${r.port}`),console.log(`[server] admin auth: ${process.env.ADMIN_KEY?"configured":"not configured"}`)})}catch(r){console.error("[startup] server preflight failed",r instanceof Error?r.name:"unknown_error"),process.exit(1)}}p();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-XLWNEVUZ.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-3GP5CYZX.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import{a as P}from"../chunk-
|
|
2
|
+
import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-XLWNEVUZ.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-3GP5CYZX.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import{a as P}from"../chunk-TW46NKVH.js";import{Command as he}from"commander";import{spawn as ne}from"child_process";import{mkdir as ke,writeFile as Pe}from"fs/promises";import{basename as Ce,join as Z}from"path";function se(e){return e.apiKey?.trim()||"sk_live_your_key"}function ce(e){return e.packageSpec?.trim()||"mcp-scraper@latest"}function A(e={}){return["-y","--package",ce(e),"mcp-scraper"]}function pe(e){let n={MCP_SCRAPER_API_KEY:se(e)},c=e.browserProfileName?.trim();return c&&(n.BROWSER_AGENT_PROFILE_NAME=c),e.browserProfileSaveChanges===!0&&(n.BROWSER_AGENT_PROFILE_SAVE_CHANGES="true"),n}function q(){return["mcp","remove","mcp-scraper","-s","user"]}function J(){return["mcp","get","mcp-scraper"]}function B(e){let n=e.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];if(!n)return null;let c=e.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1]??"",i=c.length?c.split(/\s+/):[],p={},u=e.split(/^\s*Environment:\s*$/m)[1];if(u)for(let a of u.split(`
|
|
3
3
|
`)){let l=a.match(/^\s{2,}([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);if(!l){if(a.trim().length&&!/^\s{2,}/.test(a))break;continue}p[l[1]]=l[2]}return{command:n,args:i,env:p}}function j(e){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(e.env))n.push("--env",`${c}=${i}`);return n.push("--",e.command,...e.args),n}function G(e={}){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(pe(e)))n.push("--env",`${c}=${i}`);return n.push("--","npx",...A(e)),n}function O(e){if(e==="claude-code")return"claude";if(e==="claude"||D.hosts.some(n=>n.id===e))return e;throw new Error('Unknown host "'+e+'". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only')}function ue(e){return K(e==="claude"?"claude-code":e)}function W(e,n={}){let c=O(e),i=ue(c),p="Restart the MCP client so it starts a fresh npx process.",u='MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply',a=`X-Ray install protocol: ${v} (${b})`;return c==="codex"?["# Codex MCP config",a,i.exactConfig,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
4
4
|
`):c==="claude"?["# Claude Code command",a,i.exactConfig,"","# One-command Claude Code setup",u,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
5
5
|
`):c==="claude-desktop"?["# Claude Desktop config",a,i.exactConfig,"","Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,p].join(`
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as e}from"../chunk-
|
|
2
|
+
import{a as e}from"../chunk-MKEMCMRO.js";import"../chunk-3MUEBDWY.js";import"../chunk-W2BVJ7S2.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-S7RA23GR.js";import"../chunk-MZN4U5BL.js";import"../chunk-4FROKQJN.js";import"../chunk-D4NOJAUN.js";import"../chunk-ZWZ6OWY6.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-TW46NKVH.js";import"../chunk-Z66Z2HV4.js";var _=["harvest_paa","search_serp","extract_url","diff_page","map_site_urls","map_wayback_snapshots","extract_site","analyze_site_similarity","audit_site","check_site_export","site_export_read","site_export_image","archive_read","youtube_harvest","youtube_transcribe","facebook_page_intel","facebook_ad_search","reddit_thread","reddit_trending","video_frame_analysis","video_frame_analysis_status","facebook_ad_transcribe","google_ads_search","google_ads_page_intel","google_ads_transcribe","facebook_video_transcribe","instagram_profile_content","instagram_media_download","maps_place_intel","maps_search","trustpilot_reviews","g2_reviews","capture_serp_snapshot","capture_serp_page_snapshots"];e({toolsets:new Set(["paa","serp"]),allowedToolNames:_});
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as s}from"../chunk-
|
|
2
|
+
import{a as s}from"../chunk-ZWZ6OWY6.js";import{a as e}from"../chunk-TW46NKVH.js";var r=process.argv.includes("--no-color")||process.env.NO_COLOR!==void 0||process.env.FORCE_COLOR==="0"||!process.stdout.isTTY,n=process.argv.includes("--help")||process.argv.includes("-h");n&&(process.stdout.write(["Usage: mcp-scraper-install [--no-color]","","Prints the branded MCP Scraper terminal install card and copyable install commands.","mcp-scraper prints the same card in a human terminal and runs as the MCP stdio server in clients.",""].join(`
|
|
3
3
|
`)),process.exit(0));process.stdout.write(s({version:e,color:!r,apiKeyConfigured:!!process.env.MCP_SCRAPER_API_KEY?.trim()}));
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as r}from"../chunk-
|
|
2
|
+
import{a as r}from"../chunk-MKEMCMRO.js";import"../chunk-3MUEBDWY.js";import"../chunk-W2BVJ7S2.js";import"../chunk-TMB56NCA.js";import"../chunk-HUV2WTRW.js";import"../chunk-S7RA23GR.js";import"../chunk-MZN4U5BL.js";import"../chunk-4FROKQJN.js";import"../chunk-D4NOJAUN.js";import"../chunk-ZWZ6OWY6.js";import"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import"../chunk-TW46NKVH.js";import"../chunk-Z66Z2HV4.js";r();
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{u as t}from"../chunk-
|
|
2
|
+
import{u as t}from"../chunk-J7EVLE6H.js";import"../chunk-QPWPR5XG.js";import{a as r}from"../chunk-4FROKQJN.js";import"../chunk-D4NOJAUN.js";import"../chunk-3GP5CYZX.js";import"../chunk-Z66Z2HV4.js";import{Command as s,Option as a}from"commander";var i=new s;i.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>","Seed query").option("-l, --location <location>",'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>","Google country code","us").option("--hl <hl>","Google language code","en").option("-d, --depth <depth>","BFS depth (1-30)","3").option("-m, --max-questions <n>","Max questions to harvest","100").option("-o, --output <dir>","Output directory","./paa-output").option("-f, --format <format>","Output format: json, csv, or both","both").option("--headless","Run browser in headless mode",!1).option("--profile <dir>","Persistent browser profile directory").option("--proxy <url>","Proxy server URL").option("--browser-api-key <key>","Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").addOption(new a("--\u006b\u0065\u0072\u006e\u0065\u006c-api-key <key>").hideHelp()).action(async e=>{try{let o=await t({query:e.query,location:e.location,gl:e.gl,hl:e.hl,depth:parseInt(e.depth,10),maxQuestions:parseInt(e.maxQuestions,10),outputDir:e.output,format:e.format,headless:e.headless,profileDir:e.profile,proxy:e.proxy,\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.browserApiKey??e.\u006b\u0065\u0072\u006e\u0065\u006cApiKey??r()});console.log(JSON.stringify({totalQuestions:o.totalQuestions,outputDir:o.stats.seed}))}catch(o){console.error(o instanceof Error?o.message:String(o)),process.exit(1)}});async function n(){await i.parseAsync()}n();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{a as Ge,f as de,h as le,i as De,j as Ue,l as ue}from"./chunk-
|
|
1
|
+
import{a as Ge,f as de,h as le,i as De,j as Ue,l as ue}from"./chunk-Z27SYWYD.js";import{c as Pe,d as Me}from"./chunk-HUV2WTRW.js";import{W as G,e as V,q as C,t as ce,u as Re,v as ke,w as W,y as Oe}from"./chunk-MZN4U5BL.js";import{b as ae,h as se}from"./chunk-D4NOJAUN.js";import{Jb as re,M as Ne,Ob as X,s as y}from"./chunk-Z66Z2HV4.js";import{createHash as Te,createHmac as pt,timingSafeEqual as on}from"crypto";import{createHash as D,randomUUID as He}from"crypto";import{Client as Mt,StreamableHTTPClientTransport as Gt}from"@modelcontextprotocol/client";import{createHash as vt,randomUUID as St}from"crypto";import{z as b}from"zod";async function _n(e,t){!Number.isFinite(t)||t<=0||await ae({op:G.CONNECTED_ACCOUNT_FEE,userId:Number(e)},()=>se({vendor:"nango_connection",units:t,unitType:"connection_month"}))}async function qe(e,t){let n=[{vendor:"nango_function_run",units:t.functionRuns,unitType:"function_run"},{vendor:"nango_proxy_request",units:t.proxyRequests,unitType:"proxy_request"},{vendor:"nango_compute",units:t.computeMilliseconds/1e3,unitType:"compute_second"}];await ae({op:G.CONNECTED_USAGE,userId:Number(e)},async()=>{for(let i of n)!Number.isFinite(i.units)||i.units<=0||await se(i)})}var R="connected_usage",Ct="mastra_scheduler",xt=b.object({toolName:b.string().trim().min(1).max(200).optional(),operationKind:b.enum(["read","action","sync","export","proxy","other"]).optional(),outcome:b.enum(["success","error","partial"]).optional(),requestId:b.string().trim().min(1).max(200).optional(),occurrenceId:b.string().trim().min(1).max(200).optional(),scheduleId:b.string().trim().min(1).max(200).optional(),nangoOperationId:b.string().trim().min(1).max(200).optional(),startedAt:b.string().datetime({offset:!0}).optional(),completedAt:b.string().datetime({offset:!0}).optional()}).strict(),Tt=b.object({identity:b.string().trim().min(1).max(320),idempotencyKey:b.string().trim().min(1).max(500),ratePolicyVersion:b.string().trim().min(1).max(100),connectionId:b.string().trim().min(1).max(500),providerConfigKey:b.string().trim().min(1).max(200),functionRuns:b.number().int().safe().nonnegative(),proxyRequests:b.number().int().safe().nonnegative(),computeMilliseconds:b.number().int().safe().nonnegative(),sourceSurface:b.enum(["main_mcp","mastra_scheduler","nango_reconciliation"]).default(Ct),metadata:xt.optional()}).strict(),Et=b.object({identity:b.string().trim().min(1).max(320),ratePolicyVersion:b.string().trim().min(1).max(100)}).strict(),x=class extends Error{constructor(n,i,r,o={}){super(i);this.code=n;this.httpStatus=r;this.details=o;this.name="ConnectedUsageBillingError"}code;httpStatus;details};function ge(e){return vt("sha256").update(e).digest("hex")}function At(e){return`connected-usage:${ge(e)}`}function Nt(e){return`connected-usage-debit:${ge(e)}`}function Rt(e){return e.trim().toLowerCase()}async function Le(e){let t=await Ne(Rt(e));if(!t)throw new x("user_not_found","billing identity was not found",404);return t}function kt(e){let t={toolName:e.metadata.toolName??null,operationKind:e.metadata.operationKind??null,outcome:e.metadata.outcome??null,requestId:e.metadata.requestId??null,occurrenceId:e.metadata.occurrenceId??null,scheduleId:e.metadata.scheduleId??null,nangoOperationId:e.metadata.nangoOperationId??null,startedAt:e.metadata.startedAt??null,completedAt:e.metadata.completedAt??null};return ge(JSON.stringify({ratePolicyVersion:C,connectionId:e.connectionId,providerConfigKey:e.providerConfigKey,usage:e.usage,sourceSurface:e.sourceSurface,metadata:t}))}function K(e){let t=JSON.stringify(e);if(t.length>4e3)throw new x("invalid_request","connected usage metadata is too large",400);return t}function ye(e){try{let t=JSON.parse(e.metadata??"");if(t.contract!=="connected_usage_receipt"||t.ratePolicyVersion!==C||typeof t.fingerprint!="string"||typeof t.connectionId!="string"||typeof t.providerConfigKey!="string"||!t.usage||!t.charge)throw new Error("invalid connected usage receipt metadata");return t}catch{throw new x("idempotency_key_conflict","the idempotency key is already bound to an incompatible billing receipt",409)}}async function me(e){let t=await y().execute({sql:"SELECT id, user_id, billing_class, source_surface, status, amount_mc, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",args:[e]});return t.rows[0]?t.rows[0]:null}async function Ke(e,t){return(await y().execute({sql:`SELECT 1
|
|
2
2
|
FROM billing_events
|
|
3
3
|
WHERE user_id = ?
|
|
4
4
|
AND billing_class = ?
|
package/dist/chunk-3GP5CYZX.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{z as e}from"zod";var r="none",i="none",u=e.object({query:e.string().min(1),location:e.string().optional(),gl:e.string().length(2).default("us"),hl:e.string().length(2).default("en"),device:e.enum(["desktop","mobile"]).default("desktop"),proxyMode:e.enum(["location","configured","none"]).default(r),proxyZip:e.string().regex(/^\d{5}$/).optional(),keepDefaultProxy:e.boolean().optional(),requireUsEgress:e.boolean().optional(),maxAttempts:e.number().int().min(1).max(12).optional(),debug:e.boolean().default(!1),depth:e.number().int().min(1).max(30).default(3),maxQuestions:e.number().int().min(1).max(1e3).default(100),headless:e.boolean().default(!1),profileDir:e.string().optional(),proxy:e.string().url().optional()
|
|
1
|
+
import{z as e}from"zod";var r="none",i="none",u=e.object({query:e.string().min(1),location:e.string().optional(),gl:e.string().length(2).default("us"),hl:e.string().length(2).default("en"),device:e.enum(["desktop","mobile"]).default("desktop"),proxyMode:e.enum(["location","configured","none"]).default(r),proxyZip:e.string().regex(/^\d{5}$/).optional(),keepDefaultProxy:e.boolean().optional(),requireUsEgress:e.boolean().optional(),maxAttempts:e.number().int().min(1).max(12).optional(),debug:e.boolean().default(!1),depth:e.number().int().min(1).max(30).default(3),maxQuestions:e.number().int().min(1).max(1e3).default(100),headless:e.boolean().default(!1),profileDir:e.string().optional(),proxy:e.string().url().optional(),\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.string().optional(),\u006b\u0065\u0072\u006e\u0065\u006cProxyId:e.string().optional(),\u006b\u0065\u0072\u006e\u0065\u006cProfileName:e.string().optional(),\u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges:e.boolean().optional(),\u006b\u0065\u0072\u006e\u0065\u006cStealth:e.boolean().optional(),serpIdentity:e.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional(),\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution:e.unknown().optional(),outputDir:e.string().default("./paa-output"),format:e.enum(["json","csv","both"]).default("both"),serpOnly:e.boolean().default(!1),questionsOnly:e.boolean().default(!1),questionGrowthRecoveryRounds:e.number().int().min(0).max(4).default(2),pages:e.number().int().min(1).max(2).default(1),includeAllSerpFeatures:e.boolean().default(!1),includeLocalPack:e.boolean().default(!1),includeForums:e.boolean().default(!1),includeVideos:e.boolean().default(!1),includeAiOverview:e.boolean().default(!1),includeWhatPeopleSaying:e.boolean().default(!1),includeShortVideos:e.boolean().default(!1),recency:e.enum(["day","week","month","year"]).optional(),softDeadlineMs:e.number().optional()});function d(t){switch(t){case"day":return"qdr:d";case"week":return"qdr:w";case"month":return"qdr:m";case"year":return"qdr:y";default:return}}var c=e.object({businessName:e.string().min(1),location:e.string().min(1),gl:e.string().length(2).default("us"),hl:e.string().length(2).default("en"),includeReviews:e.boolean().default(!1),maxReviews:e.number().int().min(1).max(500).default(50),includeServices:e.boolean().default(!1),includeImages:e.boolean().default(!1),imageScope:e.enum(["owner","all"]).default("all"),maxImages:e.number().int().min(1).max(250).default(100),maxInlineImages:e.number().int().min(0).max(5).default(3),\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.string().optional(),\u006b\u0065\u0072\u006e\u0065\u006cProxyId:e.string().optional(),headless:e.boolean().default(!0)}),g=e.object({query:e.string().min(1),location:e.string().optional(),gl:e.string().length(2).default("us"),hl:e.string().length(2).default("en"),maxResults:e.number().int().min(1).max(50).default(10),includeServices:e.boolean().default(!1),proxyMode:e.enum(["location","configured","none"]).default(i),proxyZip:e.string().regex(/^\d{5}$/).optional(),serpRedirect:e.boolean().optional(),forceDirectEgress:e.boolean().optional(),debug:e.boolean().default(!1),\u006b\u0065\u0072\u006e\u0065\u006cApiKey:e.string().optional(),\u006b\u0065\u0072\u006e\u0065\u006cProxyId:e.string().optional(),\u006b\u0065\u0072\u006e\u0065\u006cProxyResolution:e.unknown().optional(),headless:e.boolean().default(!0)}),p=e.object({questionId:e.string().min(1),googleLinkId:e.string().nullable(),googleEvidenceId:e.string().nullable(),question:e.string().min(1),answer:e.string().optional(),sourceTitle:e.string().optional(),sourceSite:e.string().optional(),sourceCite:e.string().optional(),sources:e.array(e.object({title:e.string().nullable(),site:e.string().nullable(),url:e.string().min(1),rawUrl:e.string().min(1),resolvedUrl:e.string().nullable(),linkType:e.enum(["plain","google_url_redirect","google_goto_redirect"]),resolutionStatus:e.enum(["not_needed","resolved","unresolved","rejected"])})).default([])}),m=e.object({name:e.string().nullable(),rating:e.string().nullable(),reviewCount:e.string().nullable(),category:e.string().nullable(),address:e.string().nullable(),hoursSummary:e.string().nullable(),phone:e.string().nullable(),phoneDisplay:e.string().nullable(),website:e.string().nullable(),plusCode:e.string().nullable(),bookingUrl:e.string().nullable()}),b=e.object({day:e.string(),hours:e.string()}),f=e.object({reviewHistogram:e.array(e.object({stars:e.number(),count:e.string()})),reviewTopics:e.array(e.object({label:e.string(),count:e.string()}))}),y=e.object({reviewId:e.string(),author:e.string().nullable(),stars:e.string().nullable(),date:e.string().nullable(),text:e.string().nullable(),ownerResponse:e.string().nullable()}),h=e.object({section:e.string(),attribute:e.string()});async function w(t){let o=process.env.MCP_SCRAPER_API_KEY?.trim(),a=(process.env.MCP_SCRAPER_BASE_URL??"https://mcpscraper.dev").replace(/\/$/,"");if(o)try{let n=await fetch(`${a}/memory/mcp-call`,{method:"POST",headers:{"content-type":"application/json","x-api-key":o},body:JSON.stringify({toolName:"libraryIngestTool",args:t})}),l=await n.json().catch(()=>null);(!n.ok||l?.ok===!1)&&console.warn("[memory-library-sink] ingest not accepted:",n.status,l?.error??"")}catch(n){console.warn("[memory-library-sink] ingest failed:",n?.message)}}export{r as a,i as b,u as c,d,c as e,g as f,p as g,m as h,b as i,f as j,h as k,w as l};
|