mcp-scraper 0.50.0 → 0.51.0

package/dist/editorial-reading-room/assets/app.js

@@ -32,6 +32,39 @@ function escapeHtml(value = "") {
     .replace(/"/g, """);
 }
 
+function socialImage(article = null) {
+  return article?.ogImage || article?.image || state.data.site.ogImage || null;
+}
+
+function setMeta(selector, attribute, value) {
+  const element = document.querySelector(selector);
+  if (!element) return;
+  if (value) element.setAttribute(attribute, value);
+  else element.removeAttribute(attribute);
+}
+
+function updateSocialMeta(article = null) {
+  const title = article ? `${article.title} — ${state.data.site.title}` : state.data.site.title;
+  const description = article?.summary || state.data.deck || state.data.site.heroTitle;
+  const image = socialImage(article);
+  setMeta('meta[property="og:type"]', "content", article ? "article" : "website");
+  setMeta('meta[property="og:title"]', "content", title);
+  setMeta('meta[property="og:description"]', "content", description);
+  setMeta('meta[name="twitter:title"]', "content", title);
+  setMeta('meta[name="twitter:description"]', "content", description);
+  setMeta('meta[property="og:image"]', "content", image?.url || "");
+  setMeta('meta[property="og:image:alt"]', "content", image?.alt || "");
+  setMeta('meta[property="og:image:width"]', "content", image?.width ? String(image.width) : "");
+  setMeta('meta[property="og:image:height"]', "content", image?.height ? String(image.height) : "");
+  setMeta('meta[name="twitter:image"]', "content", image?.url || "");
+  setMeta('meta[name="twitter:image:alt"]', "content", image?.alt || "");
+}
+
+function imageMarkup(image, className) {
+  if (!image) return "";
+  return `<img class="${className}" src="${escapeHtml(image.url)}" alt="${escapeHtml(image.alt)}" loading="lazy">`;
+}
+
 function setTypeMode() {
   const mode = typeModes[state.typeIndex % typeModes.length];
   if (mode === "default") document.documentElement.removeAttribute("data-type");
@@ -132,6 +165,7 @@ function renderHome() {
   elements.article.hidden = true;
   elements.progress.style.width = "0";
   document.title = state.data.site.title;
+  updateSocialMeta();
   document.querySelector("#masthead-product").textContent = state.data.site.product;
   document.querySelector("#masthead-title").textContent = state.data.site.edition;
   document.querySelector("#edition-label").textContent = state.data.site.editionLabel;
@@ -151,6 +185,7 @@ function renderHome() {
     .map(
       (article) => `<article class="feature-card">
         <button data-article="${article.slug}">
+          ${imageMarkup(article.image, "feature-card-image")}
           <p class="card-kicker">${escapeHtml(article.kicker)}</p>
           <h2>${escapeHtml(article.title)}</h2>
           <p>${escapeHtml(article.summary)}</p>
@@ -190,6 +225,7 @@ function renderArticle(article) {
   elements.home.hidden = true;
   elements.article.hidden = false;
   document.title = `${article.title} — ${state.data.site.title}`;
+  updateSocialMeta(article);
   document.querySelector("#article-category").textContent = article.category;
   document.querySelector("#article-kicker").textContent = article.kicker;
   document.querySelector("#article-title").textContent = article.title;
@@ -199,6 +235,32 @@ function renderArticle(article) {
   document.querySelector("#article-revision").textContent = article.revision
     ? `Revision ${article.revision}`
     : article.sourceType || "Source material";
+  const imageFigure = document.querySelector("#article-image");
+  const imageElement = document.querySelector("#article-image-src");
+  const imageCaption = document.querySelector("#article-image-caption");
+  const imageCaptionText = document.querySelector("#article-image-caption-text");
+  const imageSource = document.querySelector("#article-image-source");
+  if (article.image) {
+    imageFigure.hidden = false;
+    imageElement.src = article.image.url;
+    imageElement.alt = article.image.alt;
+    imageCaptionText.textContent = [article.image.caption, article.image.credit].filter(Boolean).join(" — ");
+    if (article.image.sourceUrl) {
+      imageSource.href = article.image.sourceUrl;
+      imageSource.hidden = false;
+    } else {
+      imageSource.removeAttribute("href");
+      imageSource.hidden = true;
+    }
+    imageCaption.hidden = !imageCaptionText.textContent && !article.image.sourceUrl;
+  } else {
+    imageFigure.hidden = true;
+    imageElement.removeAttribute("src");
+    imageElement.alt = "";
+    imageCaptionText.textContent = "";
+    imageSource.removeAttribute("href");
+    imageSource.hidden = true;
+  }
   document.querySelector("#article-body").innerHTML = article.html;
   document.querySelector("#article-source").textContent = article.sourceLabel;
   renderDesktopJumps(article);

package/dist/editorial-reading-room/assets/index.html

@@ -8,6 +8,7 @@
       content="A mobile-first editorial reading room."
     />
     <meta name="theme-color" content="#f5f1e8" />
+    <!--__READING_ROOM_SOCIAL_META__-->
     <title>Editorial Reading Room</title>
     <style>/*__READING_ROOM_STYLES__*/</style>
   </head>
@@ -99,6 +100,13 @@
                 <span id="article-word-count"></span>
                 <span id="article-revision"></span>
               </div>
+              <figure class="article-image" id="article-image" hidden>
+                <img id="article-image-src" alt="" />
+                <figcaption id="article-image-caption">
+                  <span id="article-image-caption-text"></span>
+                  <a id="article-image-source" target="_blank" rel="noopener noreferrer">Image source</a>
+                </figcaption>
+              </figure>
             </header>
             <div class="article-body" id="article-body"></div>
             <footer class="article-source">

package/dist/editorial-reading-room/assets/styles.css

@@ -277,6 +277,14 @@ a {
   text-align: left;
 }
 
+.feature-card-image {
+  width: 100%;
+  aspect-ratio: 16 / 9;
+  margin-bottom: 20px;
+  border: 1px solid var(--rule);
+  object-fit: cover;
+}
+
 .card-kicker {
   margin: 0 0 18px;
   color: var(--accent);
@@ -594,6 +602,37 @@ body.drawer-open .drawer-backdrop {
   text-transform: uppercase;
 }
 
+.article-image {
+  margin: 34px 0 0;
+}
+
+.article-image img {
+  display: block;
+  width: 100%;
+  max-height: 680px;
+  border: 1px solid var(--rule);
+  object-fit: cover;
+}
+
+.article-image figcaption {
+  margin-top: 9px;
+  color: var(--muted);
+  font-size: 12px;
+  line-height: 1.45;
+}
+
+.article-image figcaption a {
+  margin-left: 0.45em;
+  color: inherit;
+}
+
+.article-body img {
+  display: block;
+  max-width: 100%;
+  height: auto;
+  margin: 1.8em auto;
+}
+
 .article-body {
   padding-top: 28px;
   font-family: var(--serif);

package/dist/{server-3ND2EXYO.js → server-IGIIEHXY.js}

@@ -158,7 +158,7 @@ import {
   resolveDeploymentProfile,
   resolveLocalSourcebookSchemaType,
   transcribeMediaUrl
-} from "./chunk-44LNKZTE.js";
+} from "./chunk-O63VVCCE.js";
 import {
   auditImageUrls,
   auditImages,
@@ -214,7 +214,7 @@ import {
 } from "./chunk-3ZUBQQPQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-C7NN57WH.js";
+} from "./chunk-73MUQSWC.js";
 import {
   abandonExtractSettlement,
   countSuccessfulPages,
@@ -20000,6 +20000,34 @@ import { Hono as Hono11 } from "hono";
 import { z as z21 } from "zod";
 
 // src/api/reddit-trending.ts
+var PARSE_REDDIT_SEARCH = `(() => {
+  const txt = el => ((el && el.innerText) || '').trim();
+  const bodyText = (document.body && document.body.innerText) || '';
+  const blocked = /whoa there|blocked by network|you've been blocked|network (policy|security)|log in to your reddit account/i.test(bodyText);
+  const listing = document.querySelector('.search-result-listing');
+  const rows = [];
+  document.querySelectorAll('.search-result.search-result-link').forEach(r => {
+    const titleEl = r.querySelector('header.search-result-header a.search-title');
+    rows.push({
+      title: txt(titleEl),
+      url: (titleEl && titleEl.getAttribute('href')) || '',
+      subreddit: txt(r.querySelector('a.search-subreddit-link')),
+      scoreText: txt(r.querySelector('.search-score')),
+      commentsText: txt(r.querySelector('a.search-comments')),
+      ageText: txt(r.querySelector('.search-time time')),
+    });
+  });
+  return { blocked: blocked, listingFound: Boolean(listing), rowCount: rows.length, rows: rows };
+})()`;
+function buildRedditSearchUrl(topic, subreddit, window2 = "month") {
+  const base = subreddit ? `https://old.reddit.com/r/${encodeURIComponent(subreddit)}/search` : "https://old.reddit.com/search";
+  const params = new URLSearchParams();
+  params.set("q", topic);
+  params.set("sort", "top");
+  params.set("t", window2);
+  if (subreddit) params.set("restrict_sr", "on");
+  return `${base}?${params.toString()}`;
+}
 function buildSerpQuery(topic, subreddit) {
   const scope = subreddit ? `site:reddit.com/r/${subreddit}` : "site:reddit.com";
   return `${scope} ${topic}`.trim();
@@ -20062,6 +20090,32 @@ function canonicalThreadUrl(href) {
 function engagementScore(score, commentCount) {
   return score + 2 * commentCount;
 }
+function normalizeSearchRows(rows) {
+  if (!Array.isArray(rows)) return [];
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const raw of rows) {
+    if (!raw || typeof raw !== "object") continue;
+    const r = raw;
+    const url = canonicalThreadUrl(r.url);
+    const title = typeof r.title === "string" ? r.title.trim() : "";
+    if (!url || !title || seen.has(url)) continue;
+    seen.add(url);
+    const score = parseCountText(r.scoreText);
+    const commentCount = parseCountText(r.commentsText);
+    out.push({
+      title,
+      url,
+      subreddit: typeof r.subreddit === "string" ? r.subreddit.trim() : "",
+      score,
+      commentCount,
+      engagementScore: engagementScore(score, commentCount),
+      ageText: typeof r.ageText === "string" ? r.ageText.trim() : "",
+      topQuestions: []
+    });
+  }
+  return out;
+}
 var INTERROGATIVE_STARTERS = /* @__PURE__ */ new Set([
   "what",
   "whats",
@@ -20213,8 +20267,8 @@ var PARSE_REDDIT = `(() => {
   });
   return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
 })()`;
-async function scrapeOldRedditWithRetries(url, script, accept) {
-  for (let attempt = 0; attempt < 4; attempt++) {
+async function scrapeOldRedditWithRetries(url, script, accept, maxAttempts = 4) {
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
     const backupAttempt = attempt === 3;
     if (backupAttempt && !backupProxyAvailable()) break;
     const proxyId = backupAttempt ? await createBackupProxyIdSafe(browserServiceApiKey(), attempt) : await residentialProxyId(attempt);
@@ -20330,6 +20384,9 @@ redditApp.post("/trending", createApiKeyAuth(), async (c) => {
     if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.reddit_thread), 402);
     discoveryDebited = true;
     let candidates = [];
+    let discoverySource = "google_serp";
+    let primaryDiscoveryDegraded = false;
+    let primaryDegradationReasons = [];
     try {
       const serp = await harvest({
         query: serpQuery,
@@ -20343,14 +20400,59 @@ redditApp.post("/trending", createApiKeyAuth(), async (c) => {
         softDeadlineMs
       });
       candidates = extractRedditThreadUrls(serp.organicResults, body.maxThreads);
+      primaryDiscoveryDegraded = serp.diagnostics?.degradedResult === true;
+      primaryDegradationReasons = Array.isArray(serp.diagnostics?.degradationReasons) ? serp.diagnostics.degradationReasons.filter((reason) => typeof reason === "string") : [];
     } catch {
       candidates = [];
+      primaryDiscoveryDegraded = true;
+      primaryDegradationReasons = ["primary_serp_failed"];
+    }
+    if (candidates.length === 0) {
+      const fallback = await scrapeOldRedditWithRetries(
+        buildRedditSearchUrl(body.topic, subreddit, body.window),
+        PARSE_REDDIT_SEARCH,
+        (data) => !data.blocked && data.listingFound,
+        2
+      );
+      if (fallback) {
+        candidates = normalizeSearchRows(fallback.data.rows).slice(0, body.maxThreads).map((thread) => ({ title: thread.title, url: thread.url, subreddit: thread.subreddit.replace(/^r\//i, "") }));
+        discoverySource = "reddit_search_fallback";
+      } else {
+        discoverySource = "none";
+      }
     }
     if (candidates.length === 0) {
       await creditMc(user.id, MC_COSTS.reddit_thread, LedgerOperation.REDDIT_THREAD_REFUND, "no reddit threads discovered");
       discoveryRefunded = true;
-      await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "failed", query: body.topic, error: "no reddit threads discovered" });
-      return c.json({ error: "No Reddit threads found for that topic and window (refunded)" }, 404);
+      const degradedResult = discoverySource === "none";
+      const degradationReasons = degradedResult ? [.../* @__PURE__ */ new Set([...primaryDegradationReasons, "reddit_fallback_empty_or_blocked"])] : [];
+      const result2 = {
+        topic: body.topic,
+        subreddit: subreddit ?? null,
+        window: body.window === "week" ? "7d" : "30d",
+        totals: trendingTotals([]),
+        rankedThreads: [],
+        questions: [],
+        threadsScraped: 0,
+        candidatesFound: 0,
+        partial: false,
+        searchQuery: serpQuery,
+        discoverySource,
+        resultQuality: degradedResult ? "degraded" : "complete",
+        degradedResult,
+        degradationReasons,
+        retryRecommended: degradedResult,
+        billingRefunded: true
+      };
+      await logRequestEvent({
+        userId: user.id,
+        source: "reddit_trending",
+        status: "done",
+        query: body.topic,
+        resultCount: 0,
+        result: result2
+      });
+      return c.json(result2);
     }
     let ranked;
     if (!body.includeComments) {
@@ -20426,7 +20528,13 @@ redditApp.post("/trending", createApiKeyAuth(), async (c) => {
       threadsScraped: body.includeComments ? ranked.length : 0,
       candidatesFound: candidates.length,
       partial: body.includeComments && ranked.length < candidates.length,
-      searchQuery: serpQuery
+      searchQuery: serpQuery,
+      discoverySource,
+      resultQuality: primaryDiscoveryDegraded ? "partial" : "complete",
+      degradedResult: false,
+      degradationReasons: primaryDiscoveryDegraded ? primaryDegradationReasons : [],
+      retryRecommended: false,
+      billingRefunded: false
     };
     await logRequestEvent({ userId: user.id, source: "reddit_trending", status: "done", query: body.topic, resultCount: ranked.length, result });
     return c.json(result);
@@ -23886,6 +23994,24 @@ import { Hono as Hono21 } from "hono";
 import { createMcpHandler } from "@modelcontextprotocol/server";
 
 // src/mcp/memory-mcp-tool-executor.ts
+function folderInventory(result) {
+  if (!Array.isArray(result.notes)) return [];
+  const folders = /* @__PURE__ */ new Set();
+  for (const note of result.notes) {
+    if (!note || typeof note !== "object" || Array.isArray(note)) continue;
+    const path5 = note.path;
+    if (typeof path5 !== "string") continue;
+    const segments = path5.replace(/\\/g, "/").split("/").filter(Boolean);
+    for (let depth = 1; depth < segments.length; depth += 1) {
+      folders.add(segments.slice(0, depth).join("/"));
+    }
+  }
+  return [...folders].sort((a, b) => a.localeCompare(b));
+}
+function enrichMemoryResult(toolName, result) {
+  if (toolName !== "listTool" || result.ok !== true) return result;
+  return { ...result, folders: folderInventory(result) };
+}
 var MemoryMcpToolExecutor = class {
   baseUrl;
   apiKey;
@@ -23908,7 +24034,10 @@ var MemoryMcpToolExecutor = class {
         const message = data?.error ?? `memory ${toolName} failed (HTTP ${res.status})`;
         return { content: [{ type: "text", text: message }], isError: true };
       }
-      const result = data ?? { ok: false, error: `memory ${toolName} returned no result` };
+      const result = enrichMemoryResult(
+        toolName,
+        data ?? { ok: false, error: `memory ${toolName} returned no result` }
+      );
       return {
         content: [{ type: "text", text: JSON.stringify(result) }],
         structuredContent: result,
@@ -30749,6 +30878,24 @@ function safeFilename(requested, slug2) {
 function escapedJsonForHtml(value) {
   return JSON.stringify(value).replace(/</g, "\\u003c").replace(/\u2028/g, "\\u2028").replace(/\u2029/g, "\\u2029");
 }
+function socialMeta(input) {
+  const image = input.site.ogImage;
+  const tags = [
+    '<meta property="og:type" content="website">',
+    `<meta property="og:title" content="${escapeHtml5(input.site.title)}">`,
+    `<meta property="og:description" content="${escapeHtml5(input.deck || input.site.heroTitle)}">`,
+    '<meta name="twitter:card" content="summary_large_image">',
+    `<meta name="twitter:title" content="${escapeHtml5(input.site.title)}">`,
+    `<meta name="twitter:description" content="${escapeHtml5(input.deck || input.site.heroTitle)}">`,
+    `<meta property="og:image" content="${image ? escapeHtml5(image.url) : ""}">`,
+    `<meta property="og:image:alt" content="${image ? escapeHtml5(image.alt) : ""}">`,
+    `<meta property="og:image:width" content="${image?.width ?? ""}">`,
+    `<meta property="og:image:height" content="${image?.height ?? ""}">`,
+    `<meta name="twitter:image" content="${image ? escapeHtml5(image.url) : ""}">`,
+    `<meta name="twitter:image:alt" content="${image ? escapeHtml5(image.alt) : ""}">`
+  ];
+  return tags.join("\n    ");
+}
 function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
   const sourceBytes = input.articles.reduce((total, article) => total + Buffer.byteLength(article.markdown), 0);
   if (sourceBytes > 2e6) throw new Error("Editorial reading-room Markdown must be 2,000,000 bytes or fewer in total.");
@@ -30781,7 +30928,7 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
   const generatedAt = now.toISOString();
   const data = { ...input, generatedAt, articles };
   const template = readAsset("index.html");
-  const html = template.replace("<title>Editorial Reading Room</title>", `<title>${escapeHtml5(input.site.title)}</title>`).replace('content="A mobile-first editorial reading room."', `content="${escapeHtml5(input.deck || input.site.heroTitle)}"`).replace("/*__READING_ROOM_STYLES__*/", readAsset("styles.css")).replace("/*__READING_ROOM_DATA__*/", escapedJsonForHtml(data)).replace("/*__READING_ROOM_SCRIPT__*/", readAsset("app.js"));
+  const html = template.replace("<title>Editorial Reading Room</title>", `<title>${escapeHtml5(input.site.title)}</title>`).replace('content="A mobile-first editorial reading room."', `content="${escapeHtml5(input.deck || input.site.heroTitle)}"`).replace("<!--__READING_ROOM_SOCIAL_META__-->", socialMeta(input)).replace("/*__READING_ROOM_STYLES__*/", readAsset("styles.css")).replace("/*__READING_ROOM_DATA__*/", escapedJsonForHtml(data)).replace("/*__READING_ROOM_SCRIPT__*/", readAsset("app.js"));
   const bytes = Buffer.byteLength(html);
   return {
     html,
@@ -44403,4 +44550,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-3ND2EXYO.js.map
+//# sourceMappingURL=server-IGIIEHXY.js.map