mcp-scraper 0.33.7 → 0.34.1

package/dist/{server-B35CQJB3.js → server-IWDHTES2.js}

@@ -20,7 +20,7 @@ import {
   resolveBackupKernelProxyId,
   resolveKernelProxyId,
   serpArrivalUrl
-} from "./chunk-XVCETVYY.js";
+} from "./chunk-XVVNKASZ.js";
 import {
   HttpMcpToolExecutor,
   assessTranscriptSignal,
@@ -40,7 +40,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-H6QECE2U.js";
+} from "./chunk-6OPHG76G.js";
 import {
   auditImages,
   buildLinkReport,
@@ -53,11 +53,10 @@ import {
   browserServiceProfileSaveChanges,
   browserServiceProxyId,
   currentCostContext,
-  kernelCostUsd,
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-V36LS5YV.js";
+} from "./chunk-4ZB3X6BQ.js";
 import {
   csvRecords,
   listWorkflowDefinitions,
@@ -69,7 +68,7 @@ import {
   workflowOutputBaseDir,
   workflowStepCount,
   workflowSupportsSteps
-} from "./chunk-AT2PFWHW.js";
+} from "./chunk-ZID3WQID.js";
 import "./chunk-K443GQY5.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
@@ -80,8 +79,8 @@ import {
   RawMapsHoursRowSchema,
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
-} from "./chunk-V73MPRU6.js";
-import "./chunk-HAMN42SP.js";
+} from "./chunk-CB5C3BPB.js";
+import "./chunk-7N2KYL4U.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -92,7 +91,7 @@ import {
   saveExtractPages,
   setExtractJobTotal,
   settleExtractJob
-} from "./chunk-SUOHUXQS.js";
+} from "./chunk-5PZ6N2QM.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -110,7 +109,6 @@ import {
   MC_PER_CREDIT,
   MC_PER_USD,
   MEDIA_TRANSCRIBE_HOLD_MC,
-  MEDIA_TRANSCRIBE_MARGIN_MULTIPLE,
   MEMORY_AI_MARGIN_MULTIPLE,
   MEMORY_PLANS,
   MEMORY_PLAN_QUOTA,
@@ -131,13 +129,14 @@ import {
   getPublicRatesPayload,
   insufficientBalanceResponse,
   mcToCredits,
+  mediaTranscribeChargeMc,
   offerClaimPathForTier,
   serpActualCostMc,
   tierAmountChargedUsd,
   tierBillingInterval,
   tierCreditsPerInterval,
   vendorCostUsdToMc
-} from "./chunk-JBSGBSGT.js";
+} from "./chunk-BWXLTWF7.js";
 import {
   CaptchaError,
   ExtractionError,
@@ -10367,7 +10366,8 @@ var HarvestBodySchema = z14.object({
   proxyZip: z14.string().regex(/^\d{5}$/).optional(),
   debug: z14.boolean().optional(),
   serpOnly: z14.boolean().optional(),
-  pages: z14.number().int().min(1).max(2).optional()
+  pages: z14.number().int().min(1).max(2).optional(),
+  recency: z14.enum(["day", "week", "month", "year"]).optional()
 });
 var ExtractUrlBodySchema = z14.object({
   url: z14.string().min(1, "url is required"),
@@ -10653,8 +10653,7 @@ youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
   const { videoId } = parsed.data;
   const id = videoId.trim();
   const user = c.get("user");
-  const holdMins = 5;
-  const holdMc = MC_COSTS.yt_transcription * holdMins;
+  const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
   const gate = await acquireConcurrencyGate(user, "youtube_transcribe", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
     metadata: { videoId: id }
@@ -10670,8 +10669,7 @@ youtubeApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     const chunkSecs = lastChunk && Number.isFinite(lastChunk.timestamp[1]) ? lastChunk.timestamp[1] : 0;
     const durationSecs = Number.isFinite(result.durationMs) ? result.durationMs / 1e3 : 0;
     const videoSecs = Math.max(chunkSecs, durationSecs) || 60;
-    const actualMins = Math.max(1, Math.ceil(videoSecs / 60));
-    const actualMc = MC_COSTS.yt_transcription * actualMins;
+    const actualMc = mediaTranscribeChargeMc(videoSecs);
     const diff = holdMc - actualMc;
     if (diff > 0) await creditMc(user.id, diff, LedgerOperation.TRANSCRIPTION_REFUND, "overestimate refund");
     else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.TRANSCRIPTION, id);
@@ -11940,7 +11938,7 @@ facebookAdApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
     debited = true;
     const transcript = await transcribeFacebookVideoUrl(videoUrl, "# Facebook Ad Transcript");
-    const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+    const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
     const diff = holdMc - settleMc;
     if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
     else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, videoUrl);
@@ -11991,7 +11989,7 @@ facebookAdApp.post("/video-transcribe", createApiKeyAuth(), async (c) => {
     const video = await extractFacebookOrganicVideoFromPage(page, sourceUrl.href, body.quality);
     const kernelMs = Date.now() - kernelStartMs;
     const transcript = await transcribeFacebookVideoUrl(video.videoUrl, "# Facebook Organic Video Transcript");
-    const settleMc = Math.round((transcript.costUsd + kernelCostUsd(kernelMs, true)) * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+    const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
     const diff = holdMc - settleMc;
     if (diff > 0) await creditMc(fbUser.id, diff, LedgerOperation.FB_TRANSCRIBE_REFUND, "real-cost reconcile");
     else if (diff < 0) await debitMc(fbUser.id, -diff, LedgerOperation.FB_TRANSCRIBE, sourceUrl.href);
@@ -12474,7 +12472,7 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, holdMc), 402);
     debited = true;
     const transcript = await runWithCostContext({ ...currentCostContext(), op: "google_ads_transcribe", userId: user.id }, () => transcribeMediaUrl(videoUrl, "# Google Ad Transcript"));
-    const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+    const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
     const diff = holdMc - settleMc;
     if (diff > 0) await creditMc(user.id, diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE_REFUND, "real-cost reconcile");
     else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.GOOGLE_ADS_TRANSCRIBE, videoUrl);
@@ -13129,7 +13127,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
         transcriptDebited = true;
         transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
-        const settleMc = Math.round(transcript.costUsd * MEDIA_TRANSCRIBE_MARGIN_MULTIPLE * MC_PER_USD);
+        const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
         const diff = holdMc - settleMc;
         if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
         else if (diff < 0) await debitMc(user.id, -diff, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
@@ -13163,33 +13161,44 @@ import { Hono as Hono10 } from "hono";
 import { z as z19 } from "zod";
 
 // src/api/reddit-trending.ts
-var PARSE_REDDIT_SEARCH = `(() => {
-  const txt = el => ((el && el.innerText) || '').trim();
-  const bodyText = (document.body && document.body.innerText) || '';
-  const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
-  const listing = document.querySelector('.search-result-listing');
-  const rows = [];
-  document.querySelectorAll('.search-result.search-result-link').forEach(r => {
-    const titleEl = r.querySelector('header.search-result-header a.search-title');
-    rows.push({
-      title: txt(titleEl),
-      url: (titleEl && titleEl.getAttribute('href')) || '',
-      subreddit: txt(r.querySelector('a.search-subreddit-link')),
-      scoreText: txt(r.querySelector('.search-score')),
-      commentsText: txt(r.querySelector('a.search-comments')),
-      ageText: txt(r.querySelector('.search-time time')),
+function buildSerpQuery(topic, subreddit) {
+  const scope = subreddit ? `site:reddit.com/r/${subreddit}` : "site:reddit.com";
+  return `${scope} ${topic}`.trim();
+}
+function subredditFromUrl(url) {
+  const m = url.match(/\/r\/([^/]+)\//i);
+  return m ? m[1] : "";
+}
+function extractRedditThreadUrls(organicResults, limit) {
+  if (!Array.isArray(organicResults)) return [];
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const raw of organicResults) {
+    if (!raw || typeof raw !== "object") continue;
+    const r = raw;
+    const url = canonicalThreadUrl(r.url);
+    if (!url || seen.has(url)) continue;
+    seen.add(url);
+    out.push({
+      title: typeof r.title === "string" ? r.title.trim() : "",
+      url,
+      subreddit: subredditFromUrl(url)
     });
+    if (out.length >= limit) break;
+  }
+  return out;
+}
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await fn(items[i], i);
+    }
   });
-  return { blocked: blocked, listingFound: Boolean(listing), rowCount: rows.length, rows: rows };
-})()`;
-function buildRedditSearchUrl(topic, subreddit) {
-  const base = subreddit ? `https://old.reddit.com/r/${encodeURIComponent(subreddit)}/search` : "https://old.reddit.com/search";
-  const params = new URLSearchParams();
-  params.set("q", topic);
-  params.set("sort", "top");
-  params.set("t", "month");
-  if (subreddit) params.set("restrict_sr", "on");
-  return `${base}?${params.toString()}`;
+  await Promise.all(workers);
+  return results;
 }
 function parseCountText(text2) {
   if (typeof text2 !== "string") return 0;
@@ -13214,35 +13223,6 @@ function canonicalThreadUrl(href) {
 function engagementScore(score, commentCount) {
   return score + 2 * commentCount;
 }
-function normalizeSearchRows(rows) {
-  if (!Array.isArray(rows)) return [];
-  const seen = /* @__PURE__ */ new Set();
-  const out = [];
-  for (const raw of rows) {
-    if (!raw || typeof raw !== "object") continue;
-    const r = raw;
-    const url = canonicalThreadUrl(r.url);
-    const title = typeof r.title === "string" ? r.title.trim() : "";
-    if (!url || !title || seen.has(url)) continue;
-    seen.add(url);
-    const score = parseCountText(r.scoreText);
-    const commentCount = parseCountText(r.commentsText);
-    out.push({
-      title,
-      url,
-      subreddit: typeof r.subreddit === "string" ? r.subreddit.trim() : "",
-      score,
-      commentCount,
-      engagementScore: engagementScore(score, commentCount),
-      ageText: typeof r.ageText === "string" ? r.ageText.trim() : "",
-      topQuestions: []
-    });
-  }
-  return out;
-}
-function rankThreads(rows, maxThreads) {
-  return [...rows].sort((a, b) => b.engagementScore - a.engagementScore).slice(0, maxThreads);
-}
 var INTERROGATIVE_STARTERS = /* @__PURE__ */ new Set([
   "what",
   "whats",
@@ -13335,7 +13315,8 @@ var RedditThreadBodySchema = z19.object({
 var RedditTrendingBodySchema = z19.object({
   topic: z19.string().trim().min(1, "topic is required"),
   subreddit: z19.string().trim().regex(/^(r\/)?[A-Za-z0-9_]{2,24}$/i, "subreddit must be a bare name like SEO").optional(),
-  maxThreads: z19.number().int().min(1).max(25).default(10),
+  window: z19.enum(["week", "month"]).default("month"),
+  maxThreads: z19.number().int().min(1).max(40).default(20),
   includeComments: z19.boolean().default(true),
   maxCommentsPerThread: z19.number().int().min(1).max(200).default(50)
 });
@@ -13486,50 +13467,76 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+var TRENDING_SOFT_DEADLINE_MS = 24e4;
+var TRENDING_SCRAPE_CONCURRENCY = 4;
 redditApp.post("/trending", createApiKeyAuth(), async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const parsed = RedditTrendingBodySchema.safeParse(raw);
   if (!parsed.success) return c.json(invalidRequest4(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
   const body = parsed.data;
   const subreddit = body.subreddit ? body.subreddit.replace(/^r\//i, "") : void 0;
-  const searchUrl = buildRedditSearchUrl(body.topic, subreddit);
+  const serpQuery = buildSerpQuery(body.topic, subreddit);
   const user = c.get("user");
   const gate = await acquireConcurrencyGate(user, "reddit_trending", {
     reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
     metadata: { topic: body.topic }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  const softDeadlineMs = Date.now() + TRENDING_SOFT_DEADLINE_MS;
   let discoveryDebited = false;
   let discoveryRefunded = false;
-  let threadDebitMcOutstanding = 0;
-  let commentDebitMcOutstanding = 0;
+  const outstanding = { thread: 0, comment: 0 };
   try {
-    const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, searchUrl);
+    const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, serpQuery);
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
     discoveryDebited = true;
-    const discovery = await scrapeOldRedditWithRetries(
-      searchUrl,
-      PARSE_REDDIT_SEARCH,
-      (data) => !data.blocked && data.listingFound
-    );
-    if (!discovery) {
-      await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
-      discoveryRefunded = true;
-      await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: "reddit blocked/empty after retries" });
-      return c.json({ error: "Reddit blocked the request after retries (refunded)" }, 503);
+    let candidates = [];
+    try {
+      const serp = await harvest({
+        query: serpQuery,
+        serpOnly: true,
+        proxyMode: "none",
+        pages: 2,
+        recency: body.window,
+        kernelApiKey: browserServiceApiKey(),
+        format: "json",
+        outputDir: "/tmp/reddit-trending-serp",
+        softDeadlineMs
+      });
+      candidates = extractRedditThreadUrls(serp.organicResults, body.maxThreads);
+    } catch {
+      candidates = [];
     }
-    const rows = normalizeSearchRows(discovery.data.rows);
-    const ranked = rankThreads(rows, body.maxThreads);
-    let threadsScraped = 0;
-    if (body.includeComments) {
-      let insufficient = false;
-      for (const thread of ranked) {
-        if (insufficient) break;
-        const oldUrl = toOldReddit(thread.url);
-        if (!oldUrl) continue;
-        const threadDebit = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, thread.url);
-        if (!threadDebit.ok) break;
-        threadDebitMcOutstanding += MC_COSTS.reddit_thread;
+    if (candidates.length === 0) {
+      await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "no reddit threads discovered");
+      discoveryRefunded = true;
+      await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: "no reddit threads discovered" });
+      return c.json({ error: "No Reddit threads found for that topic and window (refunded)" }, 404);
+    }
+    let ranked;
+    if (!body.includeComments) {
+      ranked = candidates.map((cand) => ({
+        title: cand.title,
+        url: cand.url,
+        subreddit: cand.subreddit,
+        score: 0,
+        commentCount: 0,
+        engagementScore: 0,
+        ageText: "",
+        topQuestions: []
+      }));
+    } else {
+      let balanceExhausted = false;
+      const scrapedThreads = await mapWithConcurrency(candidates, TRENDING_SCRAPE_CONCURRENCY, async (cand) => {
+        if (balanceExhausted || Date.now() >= softDeadlineMs) return null;
+        const oldUrl = toOldReddit(cand.url);
+        if (!oldUrl) return null;
+        const threadDebit = await debitMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD, cand.url);
+        if (!threadDebit.ok) {
+          balanceExhausted = true;
+          return null;
+        }
+        outstanding.thread += MC_COSTS.reddit_thread;
         const scraped = await scrapeOldRedditWithRetries(
           oldUrl,
           PARSE_REDDIT,
@@ -13537,46 +13544,57 @@ redditApp.post("/trending", createApiKeyAuth(), async (c) => {
         );
         if (!scraped) {
           await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "reddit blocked/empty after retries");
-          threadDebitMcOutstanding -= MC_COSTS.reddit_thread;
-          continue;
+          outstanding.thread -= MC_COSTS.reddit_thread;
+          return null;
         }
+        const score = parseCountText(scraped.data.score);
         let comments = Array.isArray(scraped.data.comments) ? scraped.data.comments : [];
+        const commentCount = comments.length;
         comments = comments.slice(0, body.maxCommentsPerThread);
         let commentMc = comments.length * MC_COSTS.reddit_comment;
         if (commentMc > 0) {
-          const commentDebit = await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, thread.url);
+          const commentDebit = await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, cand.url);
           if (!commentDebit.ok) {
+            balanceExhausted = true;
             const affordable = Math.floor((commentDebit.balance_mc ?? 0) / MC_COSTS.reddit_comment);
             comments = comments.slice(0, affordable);
             commentMc = affordable * MC_COSTS.reddit_comment;
-            if (commentMc > 0) await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, thread.url);
-            insufficient = true;
+            if (commentMc > 0) await debitMc(user.id, commentMc, LedgerOperation.REDDIT_COMMENT, cand.url);
           }
-          commentDebitMcOutstanding += commentMc;
+          outstanding.comment += commentMc;
         }
         const topLevelBodies = comments.filter((cm) => cm && cm.depth === 0).map((cm) => cm.body);
-        thread.topQuestions = extractQuestions([scraped.data.postBody, ...topLevelBodies]);
-        threadsScraped++;
-      }
+        return {
+          title: cand.title || scraped.data.title || "",
+          url: cand.url,
+          subreddit: cand.subreddit,
+          score,
+          commentCount,
+          engagementScore: engagementScore(score, commentCount),
+          ageText: "",
+          topQuestions: extractQuestions([scraped.data.postBody, ...topLevelBodies])
+        };
+      });
+      ranked = scrapedThreads.filter((t) => t !== null).sort((a, b) => b.engagementScore - a.engagementScore);
     }
     const result = {
       topic: body.topic,
       subreddit: subreddit ?? null,
-      window: "30d",
+      window: body.window === "week" ? "7d" : "30d",
       totals: trendingTotals(ranked),
       rankedThreads: ranked,
       questions: flattenQuestions(ranked),
-      threadsScraped,
-      searchUrl,
-      attempts: discovery.attempts,
-      proxied: discovery.proxied
+      threadsScraped: body.includeComments ? ranked.length : 0,
+      candidatesFound: candidates.length,
+      partial: body.includeComments && ranked.length < candidates.length,
+      searchQuery: serpQuery
     };
     await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "done", query: body.topic, resultCount: ranked.length, result });
     return c.json(result);
   } catch (err) {
     if (discoveryDebited && !discoveryRefunded) await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
-    if (threadDebitMcOutstanding > 0) await creditMc(user.id, threadDebitMcOutstanding, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
-    if (commentDebitMcOutstanding > 0) await creditMc(user.id, commentDebitMcOutstanding, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
+    if (outstanding.thread > 0) await creditMc(user.id, outstanding.thread, LedgerOperation.REDDIT_THREAD_REFUND, "failed call");
+    if (outstanding.comment > 0) await creditMc(user.id, outstanding.comment, LedgerOperation.REDDIT_COMMENT_REFUND, "failed call");
     const msg = err instanceof Error ? err.message : String(err);
     await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: msg });
     return c.json({ error: msg }, 500);
@@ -20523,6 +20541,19 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
 import Stripe from "stripe";
 import { Hono as Hono20 } from "hono";
 var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
+async function cancelStandaloneMemorySub(memSubId, baseSubId) {
+  if (!memSubId || memSubId === baseSubId) return;
+  try {
+    const memSub = await stripe.subscriptions.retrieve(memSubId);
+    const priceId = memSub.items.data[0]?.price?.id;
+    const stillOpen = memSub.status !== "canceled" && !memSub.cancel_at_period_end;
+    if (priceId && priceId in MEMORY_PLANS && stillOpen) {
+      await stripe.subscriptions.update(memSubId, { cancel_at_period_end: true });
+    }
+  } catch (err) {
+    console.error("[stripe] could not wind down standalone memory subscription", memSubId, err instanceof Error ? err.message : err);
+  }
+}
 var stripeApp = new Hono20();
 stripeApp.post("/webhooks", async (c) => {
   const sig = c.req.header("stripe-signature");
@@ -20554,14 +20585,7 @@ stripeApp.post("/webhooks", async (c) => {
           await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id, { neverExpires: tier.credits_never_expire });
           await setSubscriptionTier(user.id, tier.tier, tier.concurrency, subId ?? user.subscription_id);
           if (tier.includes_memory) {
-            const standaloneMemorySub = user.memory_subscription_id;
-            if (standaloneMemorySub) {
-              try {
-                await stripe.subscriptions.update(standaloneMemorySub, { cancel_at_period_end: true });
-              } catch (err) {
-                console.error("[stripe] could not wind down standalone memory subscription", standaloneMemorySub, err instanceof Error ? err.message : err);
-              }
-            }
+            await cancelStandaloneMemorySub(user.memory_subscription_id, subId ?? user.subscription_id);
             await setMemoryPlan(user.id, "pro", null);
             await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "pro", subscription_tier: tier.tier }));
           }
@@ -20613,6 +20637,7 @@ stripeApp.post("/webhooks", async (c) => {
       await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
       if (tier.includes_memory) {
         const newMemoryPlan = live ? "pro" : "free";
+        if (live) await cancelStandaloneMemorySub(user.memory_subscription_id, sub.id);
         await setMemoryPlan(user.id, newMemoryPlan, live ? sub.id : null);
         await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: newMemoryPlan }));
       } else if (user.memory_subscription_id === sub.id) {
@@ -21923,7 +21948,7 @@ async function readMemory(tool, args, token) {
   }
   return result;
 }
-async function mapWithConcurrency(items, limit, map) {
+async function mapWithConcurrency2(items, limit, map) {
   const values = new Array(items.length);
   let cursor = 0;
   const worker = async () => {
@@ -21940,7 +21965,7 @@ async function readVault(vault, token) {
   const list = await readMemory("listTool", { vault }, token);
   if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
   const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
-  const notes = await mapWithConcurrency(
+  const notes = await mapWithConcurrency2(
     noteRefs,
     MAX_MEMORY_READ_CONCURRENCY,
     ({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token)
@@ -28160,7 +28185,8 @@ app.post("/harvest/sync", auth2, async (c) => {
     proxyZip: body.proxyZip,
     debug: body.debug ?? false,
     serpOnly: body.serpOnly ?? false,
-    pages: Math.min(2, Math.max(1, body.pages ?? 1))
+    pages: Math.min(2, Math.max(1, body.pages ?? 1)),
+    recency: body.recency
   };
   const syncCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount(options.maxQuestions);
   const { ok: syncOk, balance_mc: syncBal } = await debitMc(user.id, syncCost, options.serpOnly ? LedgerOperation.SERP : LedgerOperation.PAA, options.query);
@@ -28667,15 +28693,25 @@ app.post("/billing/subscribe", requireAllowedOrigin, sessionAuth, async (c) => {
       customerId = customer.id;
       await setStripeCustomerId(user.id, customerId);
     }
-    if (user.subscription_id) {
-      const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
+    let existingSubId = user.subscription_id;
+    if (!existingSubId) {
+      try {
+        const active = await stripeClient.subscriptions.list({ customer: customerId, status: "active", limit: 20 });
+        const baseSub = active.data.find((s) => findBasePlanItem(s));
+        if (baseSub) existingSubId = baseSub.id;
+      } catch (err) {
+        console.warn("[billing/subscribe] could not check for existing subscription", err instanceof Error ? err.message : err);
+      }
+    }
+    if (existingSubId) {
+      const sub = await stripeClient.subscriptions.retrieve(existingSubId);
       const itemId = findBasePlanItem(sub)?.id;
       if (itemId) {
-        await stripeClient.subscriptions.update(user.subscription_id, {
+        await stripeClient.subscriptions.update(existingSubId, {
           items: [{ id: itemId, price: tier.price_id }],
           proration_behavior: "create_prorations"
         });
-        const billed = await chargeTierChangeNow(stripeClient, user.subscription_id, customerId);
+        const billed = await chargeTierChangeNow(stripeClient, existingSubId, customerId);
         if (!billed.ok) {
           return c.json({
             updated: true,
@@ -28793,15 +28829,25 @@ app.post("/billing/subscribe/terminal-checkout", auth2, async (c) => {
       customerId = customer.id;
       await setStripeCustomerId(user.id, customerId);
     }
-    if (user.subscription_id) {
-      const sub = await stripeClient.subscriptions.retrieve(user.subscription_id);
+    let existingSubId = user.subscription_id;
+    if (!existingSubId) {
+      try {
+        const active = await stripeClient.subscriptions.list({ customer: customerId, status: "active", limit: 20 });
+        const baseSub = active.data.find((s) => findBasePlanItem(s));
+        if (baseSub) existingSubId = baseSub.id;
+      } catch (err) {
+        console.warn("[billing/subscribe] could not check for existing subscription", err instanceof Error ? err.message : err);
+      }
+    }
+    if (existingSubId) {
+      const sub = await stripeClient.subscriptions.retrieve(existingSubId);
       const itemId = findBasePlanItem(sub)?.id;
       if (itemId) {
-        await stripeClient.subscriptions.update(user.subscription_id, {
+        await stripeClient.subscriptions.update(existingSubId, {
           items: [{ id: itemId, price: tier.price_id }],
           proration_behavior: "create_prorations"
         });
-        const billed = await chargeTierChangeNow(stripeClient, user.subscription_id, customerId);
+        const billed = await chargeTierChangeNow(stripeClient, existingSubId, customerId);
         if (!billed.ok) {
           return c.json({
             updated: true,
@@ -28915,7 +28961,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-3LYNVGLF.js");
+  const { drainQueue } = await import("./worker-645BZPEK.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
@@ -28943,7 +28989,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const jobId = c.req.param("id");
-  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-JHMVHENZ.js");
+  const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-OLVWMOU2.js");
   const { assembleExtractArtifacts } = await import("./extract-bundle-GPZRLBVY.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
@@ -28951,7 +28997,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   await completeExtractJob2(jobId, stored);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-JHMVHENZ.js");
+    const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-OLVWMOU2.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -29120,4 +29166,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-B35CQJB3.js.map
+//# sourceMappingURL=server-IWDHTES2.js.map