mcp-scraper 0.80.0 → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -1
- package/README.md +3 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +5 -5
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-P4447RNF.js → chunk-M2A7YSJV.js} +1 -1
- package/dist/{chunk-KNGTQD6U.js → chunk-NH4QAH3X.js} +3 -1
- package/dist/{chunk-5UAFBKJR.js → chunk-OPQIGAFB.js} +24 -5
- package/dist/{chunk-YPOOARYN.js → chunk-RSXDHQBA.js} +1 -1
- package/dist/{chunk-62W6UER7.js → chunk-VWC7XATD.js} +1 -1
- package/dist/{chunk-OY25W46Q.js → chunk-XWXCLKXF.js} +15 -13
- package/dist/{chunk-ID7J23WP.js → chunk-Z63DQAO7.js} +3 -3
- package/dist/{gmail-service-IMUHMRH4.js → gmail-service-2A3ZZT3M.js} +2 -2
- package/dist/index.cjs +23 -5
- package/dist/index.js +2 -2
- package/dist/{server-XBRLYGTR.js → server-QRYEMF4H.js} +2935 -2308
- package/dist/{worker-ICCZR443.js → worker-2MY77WYY.js} +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,26 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.81.0] - 2026-08-31
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Added a Kernel-only Reddit workflow that searches DuckDuckGo with a `site:reddit.com` query, switches the same browser to a residential proxy before clicking the selected result, and reads modern Reddit posts plus bounded rendered-comment expansion through dedicated search, thread, and combined REST endpoints.
|
|
12
|
+
- Added a bounded managed-browser backup for Reddit thread hydration after the primary Kernel attempt fails or returns fewer than the semantic target, capped at 25 comments with measured bandwidth, duration, CAPTCHA, closure, and provider-cost telemetry.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- Routed the production `reddit_thread` and `reddit_trending` MCP tools through modern Reddit on Kernel residential sessions, with DuckDuckGo site search for trend discovery; removed Google and old Reddit from their active execution path while preserving tool names, billing rates, bounded partial results, and refunds.
|
|
17
|
+
- Cost probes now include Reddit Kernel sessions and any managed-browser fallback bytes and cost in the same request receipt, and identify when the backup contributed to total cost.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- Reconciled delayed managed-browser fallback telemetry back into its linked cost probe, correcting stored vendor totals and both margin estimates after provider finalization; cost probes now label unresolved provider cost as pending instead of reporting a false zero.
|
|
22
|
+
- Measured the package-entrypoint size guard against files selected by the package publish allowlist while continuing to reject source maps anywhere in the built distribution.
|
|
23
|
+
- Account publishing lists accept hosted routing metadata while retaining strict pagination, filtering, and account isolation.
|
|
24
|
+
- Added a private, paginated publishing inventory for each account’s wiki contributions, publication subdomains and editions, and Local Sourcebook listings, with canonical links and current publication status.
|
|
25
|
+
- Validated Twilio and HubSpot X-Ray webhook signatures against the exact public callback URL instead of an internal deployment origin or Vercel rewrite query, allowing genuine provider events to become live evidence after reverse-proxy delivery.
|
|
26
|
+
|
|
7
27
|
## [0.80.0] - 2026-08-29
|
|
8
28
|
|
|
9
29
|
### Added
|
|
@@ -1524,7 +1544,8 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
1524
1544
|
- Write actions remain unavailable until the account owner explicitly enables them.
|
|
1525
1545
|
- Provider-specific connection data is normalized into one agent-facing contract.
|
|
1526
1546
|
|
|
1527
|
-
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.
|
|
1547
|
+
[Unreleased]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.81.0...HEAD
|
|
1548
|
+
[0.81.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.80.0...v0.81.0
|
|
1528
1549
|
[0.80.0]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.4...v0.80.0
|
|
1529
1550
|
[0.79.4]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.3...v0.79.4
|
|
1530
1551
|
[0.79.3]: https://github.com/VilovietaSEO/mcp-scraper/compare/v0.79.2...v0.79.3
|
package/README.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
MCP Scraper is an MCP server for live web intelligence tools backed by `https://mcpscraper.dev`.
|
|
4
4
|
|
|
5
|
+
The authenticated [Kernel Reddit API](docs/kernel-reddit.md) provides an ordered DuckDuckGo search, runtime residential-proxy handoff, real result click, and modern Reddit reader at `/kernel-reddit`.
|
|
6
|
+
|
|
5
7
|
## Product repository
|
|
6
8
|
|
|
7
9
|
This repository owns the complete private product runtime while preserving independent service and
|
|
@@ -173,7 +175,7 @@ Build the branded one-click bundle:
|
|
|
173
175
|
npm run build:mcpb
|
|
174
176
|
```
|
|
175
177
|
|
|
176
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.
|
|
178
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.81.0`, SHA-256 `866faf8d18568514b29f407c6789bc779e895c9780cfcee27eb958cb079cf244`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
177
179
|
|
|
178
180
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
179
181
|
|
package/dist/bin/api-server.js
CHANGED
|
@@ -17,8 +17,8 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
21
|
-
import("../worker-
|
|
20
|
+
import("../server-QRYEMF4H.js"),
|
|
21
|
+
import("../worker-2MY77WYY.js"),
|
|
22
22
|
import("../db-Z34LPZNR.js")
|
|
23
23
|
]);
|
|
24
24
|
const PORT = parseInt(process.env.PORT ?? "3001");
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-Z63DQAO7.js";
|
|
5
|
+
import "../chunk-XWXCLKXF.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
9
9
|
import "../chunk-6HAV7LCE.js";
|
|
10
10
|
import "../chunk-PMONKQGJ.js";
|
|
11
11
|
import "../chunk-OM7HVEJ3.js";
|
|
12
|
-
import "../chunk-
|
|
13
|
-
import "../chunk-
|
|
12
|
+
import "../chunk-OPQIGAFB.js";
|
|
13
|
+
import "../chunk-M2A7YSJV.js";
|
|
14
14
|
import "../chunk-VXLU74YZ.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-RSXDHQBA.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// src/mcp/thorbit-restricted-tool-surface.ts
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
renderInstallTerminal
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-M2A7YSJV.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-RSXDHQBA.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-Z63DQAO7.js";
|
|
5
|
+
import "../chunk-XWXCLKXF.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
9
9
|
import "../chunk-6HAV7LCE.js";
|
|
10
10
|
import "../chunk-PMONKQGJ.js";
|
|
11
11
|
import "../chunk-OM7HVEJ3.js";
|
|
12
|
-
import "../chunk-
|
|
13
|
-
import "../chunk-
|
|
12
|
+
import "../chunk-OPQIGAFB.js";
|
|
13
|
+
import "../chunk-M2A7YSJV.js";
|
|
14
14
|
import "../chunk-VXLU74YZ.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-RSXDHQBA.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// bin/mcp-stdio-server.ts
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
harvest
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-NH4QAH3X.js";
|
|
5
5
|
import {
|
|
6
6
|
browserServiceApiKey
|
|
7
7
|
} from "../chunk-OM7HVEJ3.js";
|
|
8
|
-
import "../chunk-
|
|
8
|
+
import "../chunk-OPQIGAFB.js";
|
|
9
9
|
import "../chunk-OWF2JJKN.js";
|
|
10
10
|
import "../chunk-YXNDOQXN.js";
|
|
11
11
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// release-message.json
|
|
2
2
|
var release_message_default = {
|
|
3
|
-
message: "
|
|
3
|
+
message: "Reddit thread and trending tools now use residential browser retrieval with a bounded backup, honest partial-result handling, and measured fallback cost telemetry."
|
|
4
4
|
};
|
|
5
5
|
|
|
6
6
|
// src/install-terminal.ts
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
currentCostContext,
|
|
9
9
|
recordKernelSession,
|
|
10
10
|
runWithCostContext
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-OPQIGAFB.js";
|
|
12
12
|
import {
|
|
13
13
|
DEFAULT_PROXY_MODE,
|
|
14
14
|
HarvestOptionsSchema,
|
|
@@ -5120,7 +5120,9 @@ export {
|
|
|
5120
5120
|
serpArrivalUrl,
|
|
5121
5121
|
buildYouTubeChannelVideosUrl,
|
|
5122
5122
|
BrowserDriver,
|
|
5123
|
+
sanitizeProviderLocalMessage,
|
|
5123
5124
|
validateProviderSessionId,
|
|
5125
|
+
brightDataBrowserEndpoint,
|
|
5124
5126
|
brightDataSerpEnabled,
|
|
5125
5127
|
questionIdFor,
|
|
5126
5128
|
classifyGoogleOutboundLink,
|
|
@@ -33,7 +33,7 @@ var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1
|
|
|
33
33
|
var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
|
|
34
34
|
var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
|
|
35
35
|
var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
|
|
36
|
-
var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB",
|
|
36
|
+
var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 8);
|
|
37
37
|
function kernelCostUsd(ms, headful) {
|
|
38
38
|
const sec = Math.max(0, ms) / 1e3;
|
|
39
39
|
return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
|
|
@@ -74,7 +74,7 @@ async function runCostTelemetryMigration() {
|
|
|
74
74
|
SELECT
|
|
75
75
|
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
76
76
|
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
77
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key')) =
|
|
77
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
|
|
78
78
|
AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
|
|
79
79
|
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
80
80
|
AS ready
|
|
@@ -132,6 +132,9 @@ async function runCostTelemetryMigration() {
|
|
|
132
132
|
est_cost_usd REAL NOT NULL DEFAULT 0,
|
|
133
133
|
error TEXT,
|
|
134
134
|
source_key TEXT,
|
|
135
|
+
provider_duration_ms INTEGER,
|
|
136
|
+
provider_captcha INTEGER,
|
|
137
|
+
provider_status TEXT,
|
|
135
138
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
136
139
|
)
|
|
137
140
|
`);
|
|
@@ -146,6 +149,18 @@ async function runCostTelemetryMigration() {
|
|
|
146
149
|
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
|
|
147
150
|
} catch {
|
|
148
151
|
}
|
|
152
|
+
try {
|
|
153
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_duration_ms INTEGER`);
|
|
154
|
+
} catch {
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_captcha INTEGER`);
|
|
158
|
+
} catch {
|
|
159
|
+
}
|
|
160
|
+
try {
|
|
161
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_status TEXT`);
|
|
162
|
+
} catch {
|
|
163
|
+
}
|
|
149
164
|
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
|
|
150
165
|
await db.execute(`
|
|
151
166
|
CREATE TABLE IF NOT EXISTS cost_probe_runs (
|
|
@@ -228,8 +243,8 @@ async function recordVendorUsage(r) {
|
|
|
228
243
|
const db = getDb();
|
|
229
244
|
const result = await db.execute({
|
|
230
245
|
sql: `INSERT OR IGNORE INTO vendor_usage_log
|
|
231
|
-
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key)
|
|
232
|
-
VALUES (
|
|
246
|
+
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
|
|
247
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
233
248
|
args: [
|
|
234
249
|
randomUUID(),
|
|
235
250
|
r.op ?? ctx?.op ?? null,
|
|
@@ -242,7 +257,10 @@ async function recordVendorUsage(r) {
|
|
|
242
257
|
vendorCostUsd(r.vendor, r.units),
|
|
243
258
|
r.error ?? null,
|
|
244
259
|
r.method ?? ctx?.subOp ?? null,
|
|
245
|
-
r.sourceKey ?? null
|
|
260
|
+
r.sourceKey ?? null,
|
|
261
|
+
r.providerDurationMs ?? null,
|
|
262
|
+
boolToInt(r.providerCaptcha),
|
|
263
|
+
r.providerStatus ?? null
|
|
246
264
|
]
|
|
247
265
|
});
|
|
248
266
|
return result.rowsAffected === 1;
|
|
@@ -262,6 +280,7 @@ export {
|
|
|
262
280
|
currentCostContext,
|
|
263
281
|
HEADLESS_OPS,
|
|
264
282
|
vendorCostUsd,
|
|
283
|
+
migrateCostTelemetry,
|
|
265
284
|
recordKernelSession,
|
|
266
285
|
recordVendorUsage
|
|
267
286
|
};
|
|
@@ -34,7 +34,7 @@ import {
|
|
|
34
34
|
import {
|
|
35
35
|
recordVendorUsage,
|
|
36
36
|
vendorCostUsd
|
|
37
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-OPQIGAFB.js";
|
|
38
38
|
import {
|
|
39
39
|
buildSerpEmailQuery,
|
|
40
40
|
classifyEmailScope,
|
|
@@ -45,7 +45,7 @@ import {
|
|
|
45
45
|
} from "./chunk-VXLU74YZ.js";
|
|
46
46
|
import {
|
|
47
47
|
PACKAGE_VERSION
|
|
48
|
-
} from "./chunk-
|
|
48
|
+
} from "./chunk-RSXDHQBA.js";
|
|
49
49
|
import {
|
|
50
50
|
PUBLIC_ERROR_CODES,
|
|
51
51
|
buildPublicErrorEnvelope,
|
|
@@ -2839,7 +2839,7 @@ ${questionList || "_No questions extracted._"}`,
|
|
|
2839
2839
|
candidatesFound: Number(d.candidatesFound ?? threads.length),
|
|
2840
2840
|
partial: Boolean(d.partial),
|
|
2841
2841
|
searchQuery: d.searchQuery ?? "",
|
|
2842
|
-
discoverySource: d.discoverySource ?? "
|
|
2842
|
+
discoverySource: d.discoverySource ?? "web_search",
|
|
2843
2843
|
resultQuality: d.resultQuality ?? "complete",
|
|
2844
2844
|
degradedResult: Boolean(d.degradedResult),
|
|
2845
2845
|
degradationReasons: d.degradationReasons ?? [],
|
|
@@ -5723,12 +5723,14 @@ seam is noted so you can chain them.
|
|
|
5723
5723
|
url the user gives).
|
|
5724
5724
|
|
|
5725
5725
|
## Reddit
|
|
5726
|
-
- Read ONE known reddit.com thread/post URL -> **reddit_thread**. It opens
|
|
5727
|
-
|
|
5726
|
+
- Read ONE known reddit.com thread/post URL -> **reddit_thread**. It opens current Reddit through a managed
|
|
5727
|
+
residential browser and returns a bounded rendered comment tree. After primary failure, its managed-browser
|
|
5728
|
+
backup can recover the post and up to 25 comments.
|
|
5728
5729
|
- DISCOVER what a niche is talking about (topic, no known thread) -> **reddit_trending** (takes a topic, optional
|
|
5729
5730
|
subreddit and a week, 30-day, or all-time window; returns top threads ranked by engagement plus the questions people asked \u2014
|
|
5730
|
-
feed winning \`rankedThreads[].url\` values into \`reddit_thread\` for the full comment tree). It uses
|
|
5731
|
-
|
|
5731
|
+
feed winning \`rankedThreads[].url\` values into \`reddit_thread\` for the full comment tree). It uses a site-restricted
|
|
5732
|
+
web search, then opens selected threads on current Reddit through managed residential browsers; each failed hydration can use the same
|
|
5733
|
+
bounded backup for up to 25 comments. Before interpreting an empty result, inspect
|
|
5732
5734
|
\`resultQuality\`, \`discoverySource\`, \`degradationReasons\`, \`retryRecommended\`, and \`billingRefunded\`;
|
|
5733
5735
|
a degraded empty result is not evidence that the topic has no Reddit discussion.
|
|
5734
5736
|
|
|
@@ -8811,13 +8813,13 @@ var FacebookAdSearchInputSchema = {
|
|
|
8811
8813
|
maxResults: z7.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
|
|
8812
8814
|
};
|
|
8813
8815
|
var RedditThreadInputSchema = {
|
|
8814
|
-
url: z7.string().min(1).describe("A reddit.com thread/post URL.
|
|
8815
|
-
maxComments: z7.number().int().min(1).max(2e3).optional().describe("Optional cap on comments returned. Omit to return
|
|
8816
|
+
url: z7.string().min(1).describe("A reddit.com thread/post URL. Legacy reddit.com hostnames are normalized, but retrieval uses the current Reddit page only."),
|
|
8817
|
+
maxComments: z7.number().int().min(1).max(2e3).optional().describe("Optional cap on comments returned. Omit to return the largest set captured within the bounded expansion window, up to 2,000.")
|
|
8816
8818
|
};
|
|
8817
8819
|
var RedditTrendingInputSchema = {
|
|
8818
8820
|
topic: z7.string().min(1).describe('Topic to scan (e.g. "crm for small business"). Include quotation marks inside the value for an exact-phrase site search. Not a URL \u2014 pass a known thread URL to reddit_thread instead.'),
|
|
8819
8821
|
subreddit: z7.string().min(1).optional().describe('Bare subreddit name to scope the scan to one community, e.g. "SEO" (no r/ prefix, no URL). Omit to scan all of Reddit.'),
|
|
8820
|
-
window: z7.enum(["week", "month", "all"]).default("month").describe('
|
|
8822
|
+
window: z7.enum(["week", "month", "all"]).default("month").describe('Web-discovery time range: "week", "month" (default), or "all" (no date filter).'),
|
|
8821
8823
|
maxThreads: z7.number().int().min(1).max(40).default(20).describe("How many discovered threads to scrape and rank. Default 20 (scrape-all). Each scraped thread is billed like reddit_thread + its comments, so lower this to cap cost; raise toward 40 for a wider sweep. Scraping runs in parallel and stops early if it nears the request time limit (partial:true in the response)."),
|
|
8822
8824
|
includeComments: z7.boolean().default(true).describe("Scrape each discovered thread for real upvotes, comments, and the questions people asked, then rank by engagement. Set false for a fast, cheap discovery-only sweep \u2014 returns the discovered threads (title + url) in relevance order with NO engagement stats and NO per-thread billing, so you can then call reddit_thread on the ones you want."),
|
|
8823
8825
|
maxCommentsPerThread: z7.number().int().min(1).max(200).default(50).describe("Comments captured per scraped thread when includeComments is true. Default 50. Billed per captured comment.")
|
|
@@ -10804,7 +10806,7 @@ var RedditTrendingOutputSchema = {
|
|
|
10804
10806
|
candidatesFound: z7.number().int().min(0),
|
|
10805
10807
|
partial: z7.boolean(),
|
|
10806
10808
|
searchQuery: z7.string(),
|
|
10807
|
-
discoverySource: z7.enum(["google_serp", "reddit_search_fallback", "none"]),
|
|
10809
|
+
discoverySource: z7.enum(["web_search", "google_serp", "reddit_search_fallback", "none"]),
|
|
10808
10810
|
resultQuality: z7.enum(["complete", "partial", "degraded"]),
|
|
10809
10811
|
degradedResult: z7.boolean(),
|
|
10810
10812
|
degradationReasons: z7.array(z7.string()),
|
|
@@ -16363,14 +16365,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
16363
16365
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
16364
16366
|
server.registerTool("reddit_thread", {
|
|
16365
16367
|
title: "Reddit Thread + Comments",
|
|
16366
|
-
description: "
|
|
16368
|
+
description: "Read one known reddit.com post and its rendered comment tree on current Reddit. The managed residential browser is primary; a bounded managed-browser backup can recover the post and up to 25 comments when the primary fails. Use reddit_trending when no thread URL is known. Costs one Reddit base lookup plus the per-comment rate for comments returned; pass maxComments to bound both output and cost.",
|
|
16367
16369
|
inputSchema: RedditThreadInputSchema,
|
|
16368
16370
|
outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
|
|
16369
16371
|
annotations: liveWebToolAnnotations("Reddit Thread + Comments")
|
|
16370
16372
|
}, async (input) => formatRedditThread(await executor.redditThread(input), input));
|
|
16371
16373
|
server.registerTool("reddit_trending", {
|
|
16372
16374
|
title: "Reddit Trending",
|
|
16373
|
-
description: "Discover Reddit conversations through a
|
|
16375
|
+
description: "Discover Reddit conversations through a site-restricted web search for the last week, month, or all time. With includeComments:true, each candidate is opened on current Reddit through a managed residential browser, with a bounded backup of up to 25 comments after primary failure, and ranked from rendered engagement; discovery, each hydrated thread, and returned comments are billed separately. Set includeComments:false for the faster discovery-only result, and inspect resultQuality and discoverySource before treating an empty result as meaningful.",
|
|
16374
16376
|
inputSchema: RedditTrendingInputSchema,
|
|
16375
16377
|
outputSchema: recordOutputSchema("reddit_trending", RedditTrendingOutputSchema),
|
|
16376
16378
|
annotations: liveWebToolAnnotations("Reddit Trending")
|
|
@@ -12,13 +12,13 @@ import {
|
|
|
12
12
|
registerScheduledResultsMcpTools,
|
|
13
13
|
registerSerpIntelligenceCaptureTools,
|
|
14
14
|
resolveDeploymentProfile
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-XWXCLKXF.js";
|
|
16
16
|
import {
|
|
17
17
|
renderInstallTerminal
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-M2A7YSJV.js";
|
|
19
19
|
import {
|
|
20
20
|
PACKAGE_VERSION
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-RSXDHQBA.js";
|
|
22
22
|
|
|
23
23
|
// src/mcp/stdio-runtime.ts
|
|
24
24
|
import { readFileSync } from "fs";
|
|
@@ -7,11 +7,11 @@ import {
|
|
|
7
7
|
GmailServiceError,
|
|
8
8
|
normalizeGmailMessage,
|
|
9
9
|
parseGmailAddresses
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-VWC7XATD.js";
|
|
11
11
|
import "./chunk-C5Z4OFKW.js";
|
|
12
12
|
import "./chunk-T3MZISOF.js";
|
|
13
13
|
import "./chunk-PMONKQGJ.js";
|
|
14
|
-
import "./chunk-
|
|
14
|
+
import "./chunk-OPQIGAFB.js";
|
|
15
15
|
import "./chunk-YXNDOQXN.js";
|
|
16
16
|
export {
|
|
17
17
|
GMAIL_ATTACHMENT_REF_TTL_MS,
|
package/dist/index.cjs
CHANGED
|
@@ -458,7 +458,7 @@ var NANGO_USD_PER_CONNECTION_MONTH = envRate("NANGO_USD_PER_CONNECTION_MONTH", 1
|
|
|
458
458
|
var NANGO_USD_PER_FUNCTION_RUN = envRate("NANGO_USD_PER_FUNCTION_RUN", 1e-4);
|
|
459
459
|
var NANGO_USD_PER_PROXY_REQUEST = envRate("NANGO_USD_PER_PROXY_REQUEST", 1e-4);
|
|
460
460
|
var NANGO_USD_PER_COMPUTE_SEC = envRate("NANGO_USD_PER_COMPUTE_SEC", 2e-4);
|
|
461
|
-
var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB",
|
|
461
|
+
var BRIGHTDATA_BROWSER_USD_PER_GB = envRate("BRIGHTDATA_BROWSER_USD_PER_GB", 8);
|
|
462
462
|
function kernelCostUsd(ms, headful) {
|
|
463
463
|
const sec = Math.max(0, ms) / 1e3;
|
|
464
464
|
return sec * (headful ? KERNEL_HEADFUL_USD_PER_SEC : KERNEL_HEADLESS_USD_PER_SEC);
|
|
@@ -499,7 +499,7 @@ async function runCostTelemetryMigration() {
|
|
|
499
499
|
SELECT
|
|
500
500
|
(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('kernel_session_log', 'vendor_usage_log', 'cost_probe_runs')) = 3
|
|
501
501
|
AND (SELECT COUNT(*) FROM pragma_table_info('kernel_session_log') WHERE name IN ('proxy_source', 'proxy_type', 'method')) = 3
|
|
502
|
-
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key')) =
|
|
502
|
+
AND (SELECT COUNT(*) FROM pragma_table_info('vendor_usage_log') WHERE name IN ('method', 'source_key', 'provider_duration_ms', 'provider_captcha', 'provider_status')) = 5
|
|
503
503
|
AND (SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'vendor_usage_log_vendor_source_key') = 1
|
|
504
504
|
AND (SELECT COUNT(*) FROM pragma_table_info('cost_probe_runs') WHERE name IN ('units', 'unit_type', 'mode')) = 3
|
|
505
505
|
AS ready
|
|
@@ -557,6 +557,9 @@ async function runCostTelemetryMigration() {
|
|
|
557
557
|
est_cost_usd REAL NOT NULL DEFAULT 0,
|
|
558
558
|
error TEXT,
|
|
559
559
|
source_key TEXT,
|
|
560
|
+
provider_duration_ms INTEGER,
|
|
561
|
+
provider_captcha INTEGER,
|
|
562
|
+
provider_status TEXT,
|
|
560
563
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
561
564
|
)
|
|
562
565
|
`);
|
|
@@ -571,6 +574,18 @@ async function runCostTelemetryMigration() {
|
|
|
571
574
|
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN source_key TEXT`);
|
|
572
575
|
} catch {
|
|
573
576
|
}
|
|
577
|
+
try {
|
|
578
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_duration_ms INTEGER`);
|
|
579
|
+
} catch {
|
|
580
|
+
}
|
|
581
|
+
try {
|
|
582
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_captcha INTEGER`);
|
|
583
|
+
} catch {
|
|
584
|
+
}
|
|
585
|
+
try {
|
|
586
|
+
await db.execute(`ALTER TABLE vendor_usage_log ADD COLUMN provider_status TEXT`);
|
|
587
|
+
} catch {
|
|
588
|
+
}
|
|
574
589
|
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS vendor_usage_log_vendor_source_key ON vendor_usage_log(vendor, source_key) WHERE source_key IS NOT NULL`);
|
|
575
590
|
await db.execute(`
|
|
576
591
|
CREATE TABLE IF NOT EXISTS cost_probe_runs (
|
|
@@ -653,8 +668,8 @@ async function recordVendorUsage(r) {
|
|
|
653
668
|
const db = getDb();
|
|
654
669
|
const result = await db.execute({
|
|
655
670
|
sql: `INSERT OR IGNORE INTO vendor_usage_log
|
|
656
|
-
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key)
|
|
657
|
-
VALUES (
|
|
671
|
+
(id, op, probe_run_id, user_id, vendor, model, units, unit_type, est_cost_usd, error, method, source_key, provider_duration_ms, provider_captcha, provider_status)
|
|
672
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
658
673
|
args: [
|
|
659
674
|
(0, import_node_crypto2.randomUUID)(),
|
|
660
675
|
r.op ?? ctx?.op ?? null,
|
|
@@ -667,7 +682,10 @@ async function recordVendorUsage(r) {
|
|
|
667
682
|
vendorCostUsd(r.vendor, r.units),
|
|
668
683
|
r.error ?? null,
|
|
669
684
|
r.method ?? ctx?.subOp ?? null,
|
|
670
|
-
r.sourceKey ?? null
|
|
685
|
+
r.sourceKey ?? null,
|
|
686
|
+
r.providerDurationMs ?? null,
|
|
687
|
+
boolToInt(r.providerCaptcha),
|
|
688
|
+
r.providerStatus ?? null
|
|
671
689
|
]
|
|
672
690
|
});
|
|
673
691
|
return result.rowsAffected === 1;
|
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
harvest
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-NH4QAH3X.js";
|
|
4
4
|
import "./chunk-OM7HVEJ3.js";
|
|
5
5
|
import {
|
|
6
6
|
recordVendorUsage
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-OPQIGAFB.js";
|
|
8
8
|
import "./chunk-OWF2JJKN.js";
|
|
9
9
|
import "./chunk-YXNDOQXN.js";
|
|
10
10
|
|