metanova 0.2.0 → 0.2.1

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # CHANGELOG
 
+## v0.2.1
+
+Release date: 2026-06-04
+
+### Fixed
+
+- Fixed Reddit verification/block pages that returned HTTP 200 being treated as successful metadata.
+- Fixed blocked Reddit fallback handling so provider diagnostics clearly report blocked status, reason, status code, and suggested action.
+- Fixed Reddit verification titles such as `Reddit - Please wait for verification` leaking into `metadata.title`.
+
 ## v0.2.0
 
 Release date: 2026-06-04

package/dist/index.cjs

@@ -2897,8 +2897,8 @@ var redditAdapter = {
       type: reddit.isPost ? "social_post" : "website",
       siteName: "Reddit",
       canonicalUrl: context.raw.openGraph.url ?? context.raw.html.canonicalUrl,
-      title: cleanSocialTitle(titleSelection.value),
-      description: descriptionSelection.value,
+      title: cleanRedditTitle(titleSelection.value),
+      description: cleanRedditDescription(descriptionSelection.value),
       images: markAdapterMedia(mediaFromContext(context).images, "redditAdapter"),
       videos: markAdapterMedia(mediaFromContext(context).videos, "redditAdapter"),
       author: username ? { name: username } : entityFromContext(context, ["author", "submitter", "user"]),
@@ -3572,6 +3572,20 @@ function parseRedditUrl(url) {
 function cleanSocialTitle(title) {
   return title?.replace(/\s*:\s*r\/[A-Za-z0-9_]+$/i, "").trim();
 }
+function cleanRedditTitle(title) {
+  const cleaned = cleanSocialTitle(title);
+  if (!cleaned || /reddit\s*-\s*please wait for verification|please wait for verification|whoa there, pardner/i.test(cleaned)) {
+    return void 0;
+  }
+  return cleaned;
+}
+function cleanRedditDescription(description) {
+  const cleaned = description?.replace(/\s+/g, " ").trim();
+  if (!cleaned || /please wait for verification|whoa there, pardner|request has been blocked/i.test(cleaned)) {
+    return void 0;
+  }
+  return cleaned;
+}
 function hostMatches(url, domains) {
   const host = url.hostname.toLowerCase().replace(/^www\./, "");
   return domains.some((domain) => host === domain || host.endsWith(`.${domain}`));
@@ -3961,12 +3975,17 @@ function ascii(bytes, offset, length) {
 }
 
 // src/fetchMetadata.ts
+var REDDIT_BLOCKED_METADATA_WARNING = "Reddit returned a verification/block page; metadata is incomplete.";
+var PROVIDER_BLOCKED_SUGGESTED_ACTION = "retry_on_different_host_or_use_supported_proxy";
 async function fetchMetadata(url, options = {}) {
   const startedAt = Date.now();
   try {
     const requestedUrl = normalizeUrl(url);
     const fetchResult = await fetchPageWithStrategies(requestedUrl, options);
     const page = fetchResult.page;
+    if (fetchResult.providerDiagnostics?.blocked) {
+      return createBlockedProviderMetadata(requestedUrl, fetchResult, Date.now() - startedAt);
+    }
     const directMedia = createDirectMediaMetadata(page, requestedUrl, Date.now() - startedAt);
     if (directMedia) {
       return directMedia;
@@ -4035,6 +4054,62 @@ async function fetchMetadata(url, options = {}) {
     };
   }
 }
+function createBlockedProviderMetadata(requestedUrl, fetchResult, fetchDurationMs) {
+  const page = fetchResult.page;
+  const providerDiagnostics = fetchResult.providerDiagnostics;
+  const trace = uniqueStrings3([
+    ...page.isShortUrl ? [`detected short URL provider: ${page.shortUrlProvider ?? "unknown"}`] : [],
+    ...page.redirects.length > 0 ? [`resolved ${page.redirects.length} redirect${page.redirects.length === 1 ? "" : "s"}`] : [],
+    ...fetchResult.trace,
+    "detected blocked provider response"
+  ]);
+  const warnings = uniqueStrings3([
+    ...fetchResult.warnings,
+    REDDIT_BLOCKED_METADATA_WARNING,
+    ...page.statusCode < 200 || page.statusCode >= 300 ? [`Fetch completed with non-success status code ${page.statusCode}.`] : []
+  ]);
+  return {
+    ok: false,
+    url: requestedUrl,
+    finalUrl: page.finalUrl,
+    type: "unknown",
+    siteName: providerDiagnostics?.platform === "reddit" ? "Reddit" : void 0,
+    confidence: 0,
+    completeness: 0,
+    reliability: 0,
+    images: [],
+    videos: [],
+    audio: [],
+    favicons: [],
+    trace,
+    diagnostics: {
+      originalUrl: requestedUrl,
+      finalUrl: page.finalUrl,
+      isShortUrl: page.isShortUrl,
+      shortUrlProvider: page.shortUrlProvider,
+      statusCode: page.statusCode,
+      contentType: page.contentType,
+      redirects: page.redirects,
+      sourcesUsed: [],
+      warnings,
+      fallbacksAttempted: mergeFallbackAttempts2(void 0, fetchResult.fallbacksAttempted),
+      trace,
+      sourcePriority: fetchResult.sourcePriority,
+      extractionMethod: fetchResult.extractionMethod,
+      retryInfo: fetchResult.retryInfo,
+      providerDiagnostics,
+      confidenceBreakdown: {
+        title: 0,
+        description: 0,
+        image: 0,
+        structuredData: 0,
+        adapter: 0
+      },
+      fetchDurationMs,
+      extractedAt: (/* @__PURE__ */ new Date()).toISOString()
+    }
+  };
+}
 async function fetchPageWithStrategies(requestedUrl, options) {
   if (isRedditUrl(requestedUrl)) {
     return fetchRedditPageWithStrategy(requestedUrl, options);
@@ -4059,7 +4134,7 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
     });
     attempts.push(attempt);
     lastError = attempt.error;
-    if (attempt.page && attempt.ok && !attempt.blocked) {
+    if (attempt.page && attempt.ok) {
       const redditPost = parseRedditJsonPayload(attempt.page.html);
       if (redditPost?.title) {
         return {
@@ -4082,7 +4157,7 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
     const attempt = await attemptFetch("oldReddit", oldRedditUrl, options);
     attempts.push(attempt);
     lastError = attempt.error;
-    if (attempt.page && attempt.ok && !attempt.blocked) {
+    if (attempt.page && attempt.ok) {
       return {
         page: attempt.page,
         fallbacksAttempted: attempts,
@@ -4100,10 +4175,7 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
   const htmlAttempt = await attemptFetch("redditHtmlFallback", requestedUrl, options);
   attempts.push(htmlAttempt);
   lastError = htmlAttempt.error;
-  if (htmlAttempt.page) {
-    if (htmlAttempt.blocked) {
-      warnings.push("Reddit HTML fallback appears to have been blocked; metadata may be incomplete.");
-    }
+  if (htmlAttempt.page && htmlAttempt.ok) {
     return {
       page: htmlAttempt.page,
       fallbacksAttempted: attempts,
@@ -4114,19 +4186,37 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
       retryInfo: redditRetryInfo(attempts)
     };
   }
+  if (htmlAttempt.blocked) {
+    warnings.push("Reddit HTML fallback appears to have been blocked; metadata may be incomplete.");
+  }
+  const providerDiagnostics = redditProviderDiagnosticsFromAttempts(attempts);
+  if (providerDiagnostics) {
+    return {
+      page: synthesizeRedditBlockedPage(requestedUrl, attempts, providerDiagnostics),
+      fallbacksAttempted: attempts,
+      warnings: uniqueStrings3([...warnings, REDDIT_BLOCKED_METADATA_WARNING]),
+      trace: ["Reddit provider blocked metadata extraction"],
+      sourcePriority,
+      extractionMethod: "reddit:blockedProvider",
+      retryInfo: redditRetryInfo(attempts),
+      providerDiagnostics
+    };
+  }
   throw lastError ?? new Error("All Reddit extraction fetch attempts failed.");
 }
 async function attemptFetch(method, url, options) {
   try {
     const page = await fetchPage(url, options);
     const retryAfter = page.headers["retry-after"];
-    const blocked = isRedditBlocked(page);
+    const blockReason = redditBlockReason(page);
+    const blocked = Boolean(blockReason);
     return {
       method,
       url,
       ok: page.statusCode >= 200 && page.statusCode < 300 && !blocked,
       statusCode: page.statusCode,
       blocked,
+      blockReason,
       retryAfter,
       page
     };
@@ -4377,8 +4467,52 @@ function synthesizeRedditJsonPage(jsonPage, requestedUrl, post) {
     statusCode: jsonPage.statusCode
   };
 }
-function isRedditBlocked(page) {
-  return page.statusCode === 403 || page.statusCode === 429 || /please wait for verification|whoa there, pardner|blocked|forbidden|too many requests|request has been blocked/i.test(page.html);
+function redditProviderDiagnosticsFromAttempts(attempts) {
+  const blockedAttempts = attempts.filter((attempt) => attempt.blocked);
+  if (blockedAttempts.length === 0) {
+    return void 0;
+  }
+  const selectedAttempt = blockedAttempts.find((attempt) => attempt.blockReason === "provider_verification_required") ?? blockedAttempts.at(-1);
+  return {
+    platform: "reddit",
+    blocked: true,
+    statusCode: selectedAttempt?.statusCode,
+    reason: selectedAttempt?.blockReason ?? "provider_blocked_request",
+    suggestedAction: PROVIDER_BLOCKED_SUGGESTED_ACTION
+  };
+}
+function synthesizeRedditBlockedPage(requestedUrl, attempts, providerDiagnostics) {
+  const selectedPage = attempts.find((attempt) => attempt.blockReason === providerDiagnostics.reason)?.page ?? attempts.slice().reverse().find((attempt) => attempt.page)?.page;
+  return {
+    url: requestedUrl,
+    originalUrl: requestedUrl,
+    finalUrl: requestedUrl,
+    isShortUrl: selectedPage?.isShortUrl ?? false,
+    shortUrlProvider: selectedPage?.shortUrlProvider,
+    html: "",
+    bytes: new Uint8Array(),
+    statusCode: providerDiagnostics.statusCode ?? selectedPage?.statusCode ?? 403,
+    contentType: selectedPage?.contentType,
+    redirects: selectedPage?.redirects ?? [],
+    headers: selectedPage?.headers ?? {}
+  };
+}
+function redditBlockReason(page) {
+  const title = htmlTitle(page.html);
+  const text = normalizeText(`${title ?? ""} ${page.html}`);
+  if (/reddit\s*-\s*please wait for verification/i.test(title ?? "") || /please wait for verification|verification required|verify you are human/i.test(text)) {
+    return "provider_verification_required";
+  }
+  if (page.statusCode === 403 || page.statusCode === 429 || /whoa there, pardner|request has been blocked|too many requests|forbidden|you're blocked|you are blocked|youre blocked|blocked by network security/i.test(text) || /^blocked$/i.test(title ?? "")) {
+    return "provider_blocked_request";
+  }
+  return void 0;
+}
+function htmlTitle(html) {
+  return normalizeText(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]);
+}
+function normalizeText(value) {
+  return value?.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim() ?? "";
 }
 function redditRetryInfo(attempts) {
   const blockedAttempts = attempts.filter((attempt) => attempt.blocked || attempt.statusCode === 429 || attempt.statusCode === 403);
@@ -4412,7 +4546,7 @@ function mergeFallbackAttempts2(existing, incoming) {
   }
   const seen = /* @__PURE__ */ new Set();
   return attempts.map((value) => {
-    const { page: _page, ...attempt } = value;
+    const { page: _page, blockReason: _blockReason, ...attempt } = value;
     return attempt;
   }).filter((attempt) => {
     const key = `${attempt.method}:${attempt.url ?? ""}:${attempt.statusCode ?? ""}:${attempt.error ?? ""}`;