mcp-scraper 0.33.7 → 0.34.0

package/dist/index.d.cts

@@ -24,6 +24,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     format: z.ZodDefault<z.ZodEnum<["json", "csv", "both"]>>;
     serpOnly: z.ZodDefault<z.ZodBoolean>;
     pages: z.ZodDefault<z.ZodNumber>;
+    recency: z.ZodOptional<z.ZodEnum<["day", "week", "month", "year"]>>;
     softDeadlineMs: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     query: string;
@@ -49,6 +50,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     kernelApiKey?: string | undefined;
     kernelProxyId?: string | undefined;
     kernelProxyResolution?: unknown;
+    recency?: "day" | "week" | "month" | "year" | undefined;
     softDeadlineMs?: number | undefined;
 }, {
     query: string;
@@ -74,6 +76,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     format?: "json" | "csv" | "both" | undefined;
     serpOnly?: boolean | undefined;
     pages?: number | undefined;
+    recency?: "day" | "week" | "month" | "year" | undefined;
     softDeadlineMs?: number | undefined;
 }>;
 

package/dist/index.d.ts

@@ -24,6 +24,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     format: z.ZodDefault<z.ZodEnum<["json", "csv", "both"]>>;
     serpOnly: z.ZodDefault<z.ZodBoolean>;
     pages: z.ZodDefault<z.ZodNumber>;
+    recency: z.ZodOptional<z.ZodEnum<["day", "week", "month", "year"]>>;
     softDeadlineMs: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     query: string;
@@ -49,6 +50,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     kernelApiKey?: string | undefined;
     kernelProxyId?: string | undefined;
     kernelProxyResolution?: unknown;
+    recency?: "day" | "week" | "month" | "year" | undefined;
     softDeadlineMs?: number | undefined;
 }, {
     query: string;
@@ -74,6 +76,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
     format?: "json" | "csv" | "both" | undefined;
     serpOnly?: boolean | undefined;
     pages?: number | undefined;
+    recency?: "day" | "week" | "month" | "year" | undefined;
     softDeadlineMs?: number | undefined;
 }>;
 

package/dist/index.js

@@ -1,11 +1,11 @@
 import {
   harvest
-} from "./chunk-XVCETVYY.js";
+} from "./chunk-4KQVIYIH.js";
 import {
   recordVendorUsage
 } from "./chunk-V36LS5YV.js";
 import "./chunk-K443GQY5.js";
-import "./chunk-V73MPRU6.js";
+import "./chunk-CB5C3BPB.js";
 import "./chunk-M2S27J6Z.js";
 import "./chunk-62DQAWPF.js";
 

package/dist/{server-B35CQJB3.js → server-LKVPEQFE.js}

@@ -20,7 +20,7 @@ import {
   resolveBackupKernelProxyId,
   resolveKernelProxyId,
   serpArrivalUrl
-} from "./chunk-XVCETVYY.js";
+} from "./chunk-4KQVIYIH.js";
 import {
   HttpMcpToolExecutor,
   assessTranscriptSignal,
@@ -40,7 +40,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-H6QECE2U.js";
+} from "./chunk-J32XSMJI.js";
 import {
   auditImages,
   buildLinkReport,
@@ -69,7 +69,7 @@ import {
   workflowOutputBaseDir,
   workflowStepCount,
   workflowSupportsSteps
-} from "./chunk-AT2PFWHW.js";
+} from "./chunk-ZID3WQID.js";
 import "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
@@ -80,8 +80,8 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
-} from "./chunk-V73MPRU6.js";
-import "./chunk-HAMN42SP.js";
+} from "./chunk-CB5C3BPB.js";
+import "./chunk-NFMIHEYE.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -10367,7 +10367,8 @@ var HarvestBodySchema = z14.object({
   proxyZip: z14.string().regex(/^\d{5}$/).optional(),
   debug: z14.boolean().optional(),
   serpOnly: z14.boolean().optional(),
-  pages: z14.number().int().min(1).max(2).optional()
+  pages: z14.number().int().min(1).max(2).optional(),
+  recency: z14.enum(["day", "week", "month", "year"]).optional()
 });
 var ExtractUrlBodySchema = z14.object({
   url: z14.string().min(1, "url is required"),
@@ -13163,33 +13164,44 @@ import { Hono as Hono10 } from "hono";
 import { z as z19 } from "zod";
 
 // src/api/reddit-trending.ts
-var PARSE_REDDIT_SEARCH = `(() => {
-  const txt = el => ((el && el.innerText) || '').trim();
-  const bodyText = (document.body && document.body.innerText) || '';
-  const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
-  const listing = document.querySelector('.search-result-listing');
-  const rows = [];
-  document.querySelectorAll('.search-result.search-result-link').forEach(r => {
-    const titleEl = r.querySelector('header.search-result-header a.search-title');
-    rows.push({
-      title: txt(titleEl),
-      url: (titleEl && titleEl.getAttribute('href')) || '',
-      subreddit: txt(r.querySelector('a.search-subreddit-link')),
-      scoreText: txt(r.querySelector('.search-score')),
-      commentsText: txt(r.querySelector('a.search-comments')),
-      ageText: txt(r.querySelector('.search-time time')),
+function buildSerpQuery(topic, subreddit) {
+  const scope = subreddit ? `site:reddit.com/r/${subreddit}` : "site:reddit.com";
+  return `${scope} ${topic}`.trim();
+}
+function subredditFromUrl(url) {
+  const m = url.match(/\/r\/([^/]+)\//i);
+  return m ? m[1] : "";
+}
+function extractRedditThreadUrls(organicResults, limit) {
+  if (!Array.isArray(organicResults)) return [];
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const raw of organicResults) {
+    if (!raw || typeof raw !== "object") continue;
+    const r = raw;
+    const url = canonicalThreadUrl(r.url);
+    if (!url || seen.has(url)) continue;
+    seen.add(url);
+    out.push({
+      title: typeof r.title === "string" ? r.title.trim() : "",
+      url,
+      subreddit: subredditFromUrl(url)
     });
+    if (out.length >= limit) break;
+  }
+  return out;
+}
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await fn(items[i], i);
+    }
   });
-  return { blocked: blocked, listingFound: Boolean(listing), rowCount: rows.length, rows: rows };
-})()`;
-function buildRedditSearchUrl(topic, subreddit) {
-  const base = subreddit ? `https://old.reddit.com/r/${encodeURIComponent(subreddit)}/search` : "https://old.reddit.com/search";
-  const params = new URLSearchParams();
-  params.set("q", topic);
-  params.set("sort", "top");
-  params.set("t", "month");
-  if (subreddit) params.set("restrict_sr", "on");
-  return `${base}?${params.toString()}`;
+  await Promise.all(workers);
+  return results;
 }
 function parseCountText(text2) {
   if (typeof text2 !== "string") return 0;
@@ -13214,35 +13226,6 @@ function canonicalThreadUrl(href) {
 function engagementScore(score, commentCount) {
   return score + 2 * commentCount;
 }
-function normalizeSearchRows(rows) {
-  if (!Array.isArray(rows)) return [];
-  const seen = /* @__PURE__ */ new Set();
-  const out = [];
-  for (const raw of rows) {
-    if (!raw || typeof raw !== "object") continue;
-    const r = raw;
-    const url = canonicalThreadUrl(r.url);
-    const title = typeof r.title === "string" ? r.title.trim() : "";
-    if (!url || !title || seen.has(url)) continue;
-    seen.add(url);
-    const score = parseCountText(r.scoreText);
-    const commentCount = parseCountText(r.commentsText);
-    out.push({
-      title,
-      url,
-      subreddit: typeof r.subreddit === "string" ? r.subreddit.trim() : "",
-      score,
-      commentCount,
-      engagementScore: engagementScore(score, commentCount),
-      ageText: typeof r.ageText === "string" ? r.ageText.trim() : "",
-      topQuestions: []
-    });
-  }
-  return out;
-}
-function rankThreads(rows, maxThreads) {
-  return [...rows].sort((a, b) => b.engagementScore - a.engagementScore).slice(0, maxThreads);
-}
 var INTERROGATIVE_STARTERS = /* @__PURE__ */ new Set([
   "what",
   "whats",
@@ -13335,7 +13318,8 @@ var RedditThreadBodySchema = z19.object({
 var RedditTrendingBodySchema = z19.object({
   topic: z19.string().trim().min(1, "topic is required"),
   subreddit: z19.string().trim().regex(/^(r\/)?[A-Za-z0-9_]{2,24}$/i, "subreddit must be a bare name like SEO").optional(),
-  maxThreads: z19.number().int().min(1).max(25).default(10),
+  window: z19.enum(["week", "month"]).default("month"),
+  maxThreads: z19.number().int().min(1).max(40).default(20),
   includeComments: z19.boolean().default(true),
   maxCommentsPerThread: z19.number().int().min(1).max(200).default(50)
 });
@@ -13486,50 +13470,76 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+var TRENDING_SOFT_DEADLINE_MS = 24e4;
+var TRENDING_SCRAPE_CONCURRENCY = 4;
 redditApp.post("/trending", createApiKeyAuth(), async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const parsed = RedditTrendingBodySchema.safeParse(raw);
   if (!parsed.success) return c.json(invalidRequest4(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
   const body = parsed.data;
   const subreddit = body.subreddit ? body.subreddit.replace(/^r\//i, "") : void 0;
-  const searchUrl = buildRedditSearchUrl(body.topic, subreddit);
+  const serpQuery = buildSerpQuery(body.topic, subreddit);
   const user = c.get("user");
   const gate = await acquireConcurrencyGate(user, "reddit_trending", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
     metadata: { topic: body.topic }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  const softDeadlineMs = Date.now() + TRENDING_SOFT_DEADLINE_MS;
   let discoveryDebited = false;
   let discoveryRefunded = false;
-  let threadDebitMcOutstanding = 0;
-  let commentDebitMcOutstanding = 0;
+  const outstanding = { thread: 0, comment: 0 };
   try {
-    const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, searchUrl);
+    const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, serpQuery);
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
     discoveryDebited = true;
-    const discovery = await scrapeOldRedditWithRetries(
-      searchUrl,
-      PARSE_REDDIT_SEARCH,
-      (data) => !data.blocked && data.listingFound
-    );
-    if (!discovery) {
-      await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
-      discoveryRefunded = true;
-      await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: "reddit blocked/empty after retries" });
-      return c.json({ error: "Reddit blocked the request after retries (refunded)" }, 503);
+    let candidates = [];
+    try {
+      const serp = await harvest({
+        query: serpQuery,
+        serpOnly: true,
+        proxyMode: "none",
+        pages: 2,
+        recency: body.window,
+        kernelApiKey: browserServiceApiKey(),
+        format: "json",
+        outputDir: "/tmp/reddit-trending-serp",
+        softDeadlineMs
+      });
+      candidates = extractRedditThreadUrls(serp.organicResults, body.maxThreads);
+    } catch {
+      candidates = [];
     }
-    const rows = normalizeSearchRows(discovery.data.rows);
-    const ranked = rankThreads(rows, body.maxThreads);
-    let threadsScraped = 0;
-    if (body.includeComments) {
-      let insufficient = false;
-      for (const thread of ranked) {
-        if (insufficient) break;
-        const oldUrl = toOldReddit(thread.url);
-        if (!oldUrl) continue;
-        const threadDebit = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, thread.url);
-        if (!threadDebit.ok) break;
-        threadDebitMcOutstanding += MC_COSTS.reddit_thread;
+    if (candidates.length === 0) {
+      await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "no reddit threads discovered");
+      discoveryRefunded = true;
+      await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: "no reddit threads discovered" });
+      return c.json({ error: "No Reddit threads found for that topic and window (refunded)" }, 404);
+    }
+    let ranked;
+    if (!body.includeComments) {
+      ranked = candidates.map((cand) => ({
+        title: cand.title,
+        url: cand.url,
+        subreddit: cand.subreddit,
+        score: 0,
+        commentCount: 0,
+        engagementScore: 0,
+        ageText: "",
+        topQuestions: []
+      }));
+    } else {
+      let balanceExhausted = false;
+      const scrapedThreads = await mapWithConcurrency(candidates, TRENDING_SCRAPE_CONCURRENCY, async (cand) => {
+        if (balanceExhausted || Date.now() >= softDeadlineMs) return null;
+        const oldUrl = toOldReddit(cand.url);
+        if (!oldUrl) return null;
+        const threadDebit = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, cand.url);
+        if (!threadDebit.ok) {
+          balanceExhausted = true;
+          return null;
+        }
+        outstanding.thread += MC_COSTS.reddit_thread;
         const scraped = await scrapeOldRedditWithRetries(
           oldUrl,
           PARSE_REDDIT,
@@ -13537,46 +13547,57 @@ redditApp.post("/trending", createApiKeyAuth(), async (c) => {
         );
         if (!scraped) {
           await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
-          threadDebitMcOutstanding -= MC_COSTS.reddit_thread;
-          continue;
+          outstanding.thread -= MC_COSTS.reddit_thread;
+          return null;
         }
+        const score = parseCountText(scraped.data.score);
         let comments = Array.isArray(scraped.data.comments) ? scraped.data.comments : [];
+        const commentCount = comments.length;
         comments = comments.slice(0, body.maxCommentsPerThread);
         let commentMc = comments.length * MC_COSTS.reddit_comment;
         if (commentMc > 0) {
-          const commentDebit = await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, thread.url);
+          const commentDebit = await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, cand.url);
           if (!commentDebit.ok) {
+            balanceExhausted = true;
             const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
             comments = comments.slice(0, affordable);
             commentMc = affordable * MC_COSTS.reddit_comment;
-            if (commentMc > 0) await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, thread.url);
-            insufficient = true;
+            if (commentMc > 0) await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, cand.url);
           }
-          commentDebitMcOutstanding += commentMc;
+          outstanding.comment += commentMc;
         }
         const topLevelBodies = comments.filter((cm) => cm && cm.depth === 0).map((cm) => cm.body);
-        thread.topQuestions = extractQuestions([scraped.data.postBody, ...topLevelBodies]);
-        threadsScraped++;
-      }
+        return {
+          title: cand.title || scraped.data.title || "",
+          url: cand.url,
+          subreddit: cand.subreddit,
+          score,
+          commentCount,
+          engagementScore: engagementScore(score, commentCount),
+          ageText: "",
+          topQuestions: extractQuestions([scraped.data.postBody, ...topLevelBodies])
+        };
+      });
+      ranked = scrapedThreads.filter((t) => t !== null).sort((a, b) => b.engagementScore - a.engagementScore);
     }
     const result = {
       topic: body.topic,
       subreddit: subreddit ?? null,
-      window: "30d",
+      window: body.window === "week" ? "7d" : "30d",
       totals: trendingTotals(ranked),
       rankedThreads: ranked,
       questions: flattenQuestions(ranked),
-      threadsScraped,
-      searchUrl,
-      attempts: discovery.attempts,
-      proxied: discovery.proxied
+      threadsScraped: body.includeComments ? ranked.length : 0,
+      candidatesFound: candidates.length,
+      partial: body.includeComments && ranked.length < candidates.length,
+      searchQuery: serpQuery
     };
     await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "done", query: body.topic, resultCount: ranked.length, result });
     return c.json(result);
   } catch (err) {
     if (discoveryDebited && !discoveryRefunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
-    if (threadDebitMcOutstanding > 0) await creditMc(user.id, threadDebitMcOutstanding, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
-    if (commentDebitMcOutstanding > 0) await creditMc(user.id, commentDebitMcOutstanding, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
+    if (outstanding.thread > 0) await creditMc(user.id, outstanding.thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
+    if (outstanding.comment > 0) await creditMc(user.id, outstanding.comment, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
     const msg = err instanceof Error ? err.message : String(err);
     await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: msg });
     return c.json({ error: msg }, 500);
@@ -21923,7 +21944,7 @@ async function readMemory(tool, args, token) {
   }
   return result;
 }
-async function mapWithConcurrency(items, limit, map) {
+async function mapWithConcurrency2(items, limit, map) {
   const values = new Array(items.length);
   let cursor = 0;
   const worker = async () => {
@@ -21940,7 +21961,7 @@ async function readVault(vault, token) {
   const list = await readMemory("listTool", { vault }, token);
   if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
   const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
-  const notes = await mapWithConcurrency(
+  const notes = await mapWithConcurrency2(
     noteRefs,
     MAX_MEMORY_READ_CONCURRENCY,
     ({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token)
@@ -28160,7 +28181,8 @@ app.post("/harvest/sync", auth2, async (c) => {
     proxyZip: body.proxyZip,
     debug: body.debug ?? false,
     serpOnly: body.serpOnly ?? false,
-    pages: Math.min(2, Math.max(1, body.pages ?? 1))
+    pages: Math.min(2, Math.max(1, body.pages ?? 1)),
+    recency: body.recency
   };
   const syncCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount(options.maxQuestions);
   const { ok: syncOk, balance_mc: syncBal } = await debitMc(user.id, syncCost, options.serpOnly ? LedgerOperation.SERP : LedgerOperation.PAA, options.query);
@@ -28915,7 +28937,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-3LYNVGLF.js");
+  const { drainQueue } = await import("./worker-ZZHSYI3Y.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -29120,4 +29142,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-B35CQJB3.js.map
+//# sourceMappingURL=server-LKVPEQFE.js.map