metanova 0.2.0 → 0.2.1

package/dist/index.d.cts

@@ -118,6 +118,7 @@ interface ExtractionDiagnostics {
     };
     errors?: string[];
     retryInfo?: ExtractionRetryInfo;
+    providerDiagnostics?: ProviderDiagnostics;
     selectedImageReason?: string;
     confidenceBreakdown?: ConfidenceBreakdown;
     fetchDurationMs?: number;
@@ -139,6 +140,13 @@ interface ExtractionRetryInfo {
     retryAfterMs?: number;
     attempts?: number;
 }
+interface ProviderDiagnostics {
+    platform: string;
+    blocked: boolean;
+    statusCode?: number;
+    reason?: "provider_verification_required" | "provider_blocked_request" | string;
+    suggestedAction?: "retry_on_different_host_or_use_supported_proxy" | string;
+}
 interface ConfidenceBreakdown {
     title: number;
     description: number;
@@ -521,4 +529,4 @@ interface MetaNovaRegistry {
 declare function registerGlobalPlugin(plugin: MetaNovaPlugin): void;
 declare function createRegistry(options?: ParseMetadataOptions): MetaNovaRegistry;
 
-export { type AdapterContext, type AdapterExtractionResult, type AdapterRawData, type ApplicationMetadata, type ArticleMetadata, type CompletenessInput, type ConfidenceBreakdown, type ConfidenceEngineInput, type CustomExtractor, DEFAULT_ACCEPT, DEFAULT_ACCEPT_ENCODING, DEFAULT_ACCEPT_LANGUAGE, DEFAULT_BROWSER_USER_AGENT, type EmbeddedDataItem, type EmbeddedDataMetadata, type Entity, type ExtractionDiagnostics, type ExtractionFallbackAttempt, type ExtractionRetryInfo, type ExtractorContext, type FetchMetadataOptions, type FetchedPage, type HtmlMetadata, type ImageScorer, type ImageScoringContext, type JsonLdMetadata, type JsonLdNode, type MediaAsset, type MediaDiscoveryResult, type MediaKind, MetaNova, type MetaNovaCache, type MetaNovaCacheEntry, type MetaNovaPlugin, type MetaNovaPluginApi, type MetaNovaRegistry, type MetadataSource, type MetadataSourceAttribution, type MetadataType, type OEmbedData, type OEmbedLink, type OEmbedMetadata, type OpenGraphMetadata, type ParseMetadataOptions, type PlaylistMetadata, type PlaylistVideo, type PluginExtractionResult, type PreviewCard, type ProductMetadata, type RawMetadataSources, type RedirectEntry, type RedirectResolution, type ReliabilityInput, SecurityError, type SiteAdapter, type TwitterMetadata, type UnifiedMetadata, type VideoMetadata, addWarning, assertSafeRequestUrl, behanceAdapter, calculateCompleteness, calculateConfidence, calculateConfidenceBreakdown, calculateReliability, createDiagnostics, createPreviewCard, createRegistry, MetaNova as default, defaultAdapters, detectShortUrl, discoverMedia, extractAudio, extractEmbeddedData, extractHtmlMetadata, extractImages, extractJsonLd, extractOEmbed, extractOpenGraph, extractTwitterCards, extractVideos, facebookAdapter, fetchMetadata, fetchPage, instagramAdapter, normalizeMetadata, normalizeUrl, parseMetadata, parseMetadataAsync, pinterestAdapter, redditAdapter, registerGlobalPlugin, resolveCanonicalUrl, resolveRedirects, resolveUrl, scoreImages, tiktokAdapter, twitterAdapter, validateUrl, youtubeAdapter };
+export { type AdapterContext, type AdapterExtractionResult, type AdapterRawData, type ApplicationMetadata, type ArticleMetadata, type CompletenessInput, type ConfidenceBreakdown, type ConfidenceEngineInput, type CustomExtractor, DEFAULT_ACCEPT, DEFAULT_ACCEPT_ENCODING, DEFAULT_ACCEPT_LANGUAGE, DEFAULT_BROWSER_USER_AGENT, type EmbeddedDataItem, type EmbeddedDataMetadata, type Entity, type ExtractionDiagnostics, type ExtractionFallbackAttempt, type ExtractionRetryInfo, type ExtractorContext, type FetchMetadataOptions, type FetchedPage, type HtmlMetadata, type ImageScorer, type ImageScoringContext, type JsonLdMetadata, type JsonLdNode, type MediaAsset, type MediaDiscoveryResult, type MediaKind, MetaNova, type MetaNovaCache, type MetaNovaCacheEntry, type MetaNovaPlugin, type MetaNovaPluginApi, type MetaNovaRegistry, type MetadataSource, type MetadataSourceAttribution, type MetadataType, type OEmbedData, type OEmbedLink, type OEmbedMetadata, type OpenGraphMetadata, type ParseMetadataOptions, type PlaylistMetadata, type PlaylistVideo, type PluginExtractionResult, type PreviewCard, type ProductMetadata, type ProviderDiagnostics, type RawMetadataSources, type RedirectEntry, type RedirectResolution, type ReliabilityInput, SecurityError, type SiteAdapter, type TwitterMetadata, type UnifiedMetadata, type VideoMetadata, addWarning, assertSafeRequestUrl, behanceAdapter, calculateCompleteness, calculateConfidence, calculateConfidenceBreakdown, calculateReliability, createDiagnostics, createPreviewCard, createRegistry, MetaNova as default, defaultAdapters, detectShortUrl, discoverMedia, extractAudio, extractEmbeddedData, extractHtmlMetadata, extractImages, extractJsonLd, extractOEmbed, extractOpenGraph, extractTwitterCards, extractVideos, facebookAdapter, fetchMetadata, fetchPage, instagramAdapter, normalizeMetadata, normalizeUrl, parseMetadata, parseMetadataAsync, pinterestAdapter, redditAdapter, registerGlobalPlugin, resolveCanonicalUrl, resolveRedirects, resolveUrl, scoreImages, tiktokAdapter, twitterAdapter, validateUrl, youtubeAdapter };

package/dist/index.d.ts

@@ -118,6 +118,7 @@ interface ExtractionDiagnostics {
     };
     errors?: string[];
     retryInfo?: ExtractionRetryInfo;
+    providerDiagnostics?: ProviderDiagnostics;
     selectedImageReason?: string;
     confidenceBreakdown?: ConfidenceBreakdown;
     fetchDurationMs?: number;
@@ -139,6 +140,13 @@ interface ExtractionRetryInfo {
     retryAfterMs?: number;
     attempts?: number;
 }
+interface ProviderDiagnostics {
+    platform: string;
+    blocked: boolean;
+    statusCode?: number;
+    reason?: "provider_verification_required" | "provider_blocked_request" | string;
+    suggestedAction?: "retry_on_different_host_or_use_supported_proxy" | string;
+}
 interface ConfidenceBreakdown {
     title: number;
     description: number;
@@ -521,4 +529,4 @@ interface MetaNovaRegistry {
 declare function registerGlobalPlugin(plugin: MetaNovaPlugin): void;
 declare function createRegistry(options?: ParseMetadataOptions): MetaNovaRegistry;
 
-export { type AdapterContext, type AdapterExtractionResult, type AdapterRawData, type ApplicationMetadata, type ArticleMetadata, type CompletenessInput, type ConfidenceBreakdown, type ConfidenceEngineInput, type CustomExtractor, DEFAULT_ACCEPT, DEFAULT_ACCEPT_ENCODING, DEFAULT_ACCEPT_LANGUAGE, DEFAULT_BROWSER_USER_AGENT, type EmbeddedDataItem, type EmbeddedDataMetadata, type Entity, type ExtractionDiagnostics, type ExtractionFallbackAttempt, type ExtractionRetryInfo, type ExtractorContext, type FetchMetadataOptions, type FetchedPage, type HtmlMetadata, type ImageScorer, type ImageScoringContext, type JsonLdMetadata, type JsonLdNode, type MediaAsset, type MediaDiscoveryResult, type MediaKind, MetaNova, type MetaNovaCache, type MetaNovaCacheEntry, type MetaNovaPlugin, type MetaNovaPluginApi, type MetaNovaRegistry, type MetadataSource, type MetadataSourceAttribution, type MetadataType, type OEmbedData, type OEmbedLink, type OEmbedMetadata, type OpenGraphMetadata, type ParseMetadataOptions, type PlaylistMetadata, type PlaylistVideo, type PluginExtractionResult, type PreviewCard, type ProductMetadata, type RawMetadataSources, type RedirectEntry, type RedirectResolution, type ReliabilityInput, SecurityError, type SiteAdapter, type TwitterMetadata, type UnifiedMetadata, type VideoMetadata, addWarning, assertSafeRequestUrl, behanceAdapter, calculateCompleteness, calculateConfidence, calculateConfidenceBreakdown, calculateReliability, createDiagnostics, createPreviewCard, createRegistry, MetaNova as default, defaultAdapters, detectShortUrl, discoverMedia, extractAudio, extractEmbeddedData, extractHtmlMetadata, extractImages, extractJsonLd, extractOEmbed, extractOpenGraph, extractTwitterCards, extractVideos, facebookAdapter, fetchMetadata, fetchPage, instagramAdapter, normalizeMetadata, normalizeUrl, parseMetadata, parseMetadataAsync, pinterestAdapter, redditAdapter, registerGlobalPlugin, resolveCanonicalUrl, resolveRedirects, resolveUrl, scoreImages, tiktokAdapter, twitterAdapter, validateUrl, youtubeAdapter };
+export { type AdapterContext, type AdapterExtractionResult, type AdapterRawData, type ApplicationMetadata, type ArticleMetadata, type CompletenessInput, type ConfidenceBreakdown, type ConfidenceEngineInput, type CustomExtractor, DEFAULT_ACCEPT, DEFAULT_ACCEPT_ENCODING, DEFAULT_ACCEPT_LANGUAGE, DEFAULT_BROWSER_USER_AGENT, type EmbeddedDataItem, type EmbeddedDataMetadata, type Entity, type ExtractionDiagnostics, type ExtractionFallbackAttempt, type ExtractionRetryInfo, type ExtractorContext, type FetchMetadataOptions, type FetchedPage, type HtmlMetadata, type ImageScorer, type ImageScoringContext, type JsonLdMetadata, type JsonLdNode, type MediaAsset, type MediaDiscoveryResult, type MediaKind, MetaNova, type MetaNovaCache, type MetaNovaCacheEntry, type MetaNovaPlugin, type MetaNovaPluginApi, type MetaNovaRegistry, type MetadataSource, type MetadataSourceAttribution, type MetadataType, type OEmbedData, type OEmbedLink, type OEmbedMetadata, type OpenGraphMetadata, type ParseMetadataOptions, type PlaylistMetadata, type PlaylistVideo, type PluginExtractionResult, type PreviewCard, type ProductMetadata, type ProviderDiagnostics, type RawMetadataSources, type RedirectEntry, type RedirectResolution, type ReliabilityInput, SecurityError, type SiteAdapter, type TwitterMetadata, type UnifiedMetadata, type VideoMetadata, addWarning, assertSafeRequestUrl, behanceAdapter, calculateCompleteness, calculateConfidence, calculateConfidenceBreakdown, calculateReliability, createDiagnostics, createPreviewCard, createRegistry, MetaNova as default, defaultAdapters, detectShortUrl, discoverMedia, extractAudio, extractEmbeddedData, extractHtmlMetadata, extractImages, extractJsonLd, extractOEmbed, extractOpenGraph, extractTwitterCards, extractVideos, facebookAdapter, fetchMetadata, fetchPage, instagramAdapter, normalizeMetadata, normalizeUrl, parseMetadata, parseMetadataAsync, pinterestAdapter, redditAdapter, registerGlobalPlugin, resolveCanonicalUrl, resolveRedirects, resolveUrl, scoreImages, tiktokAdapter, twitterAdapter, validateUrl, youtubeAdapter };

package/dist/index.js

@@ -2814,8 +2814,8 @@ var redditAdapter = {
       type: reddit.isPost ? "social_post" : "website",
       siteName: "Reddit",
       canonicalUrl: context.raw.openGraph.url ?? context.raw.html.canonicalUrl,
-      title: cleanSocialTitle(titleSelection.value),
-      description: descriptionSelection.value,
+      title: cleanRedditTitle(titleSelection.value),
+      description: cleanRedditDescription(descriptionSelection.value),
       images: markAdapterMedia(mediaFromContext(context).images, "redditAdapter"),
       videos: markAdapterMedia(mediaFromContext(context).videos, "redditAdapter"),
       author: username ? { name: username } : entityFromContext(context, ["author", "submitter", "user"]),
@@ -3489,6 +3489,20 @@ function parseRedditUrl(url) {
 function cleanSocialTitle(title) {
   return title?.replace(/\s*:\s*r\/[A-Za-z0-9_]+$/i, "").trim();
 }
+function cleanRedditTitle(title) {
+  const cleaned = cleanSocialTitle(title);
+  if (!cleaned || /reddit\s*-\s*please wait for verification|please wait for verification|whoa there, pardner/i.test(cleaned)) {
+    return void 0;
+  }
+  return cleaned;
+}
+function cleanRedditDescription(description) {
+  const cleaned = description?.replace(/\s+/g, " ").trim();
+  if (!cleaned || /please wait for verification|whoa there, pardner|request has been blocked/i.test(cleaned)) {
+    return void 0;
+  }
+  return cleaned;
+}
 function hostMatches(url, domains) {
   const host = url.hostname.toLowerCase().replace(/^www\./, "");
   return domains.some((domain) => host === domain || host.endsWith(`.${domain}`));
@@ -3878,12 +3892,17 @@ function ascii(bytes, offset, length) {
 }
 
 // src/fetchMetadata.ts
+var REDDIT_BLOCKED_METADATA_WARNING = "Reddit returned a verification/block page; metadata is incomplete.";
+var PROVIDER_BLOCKED_SUGGESTED_ACTION = "retry_on_different_host_or_use_supported_proxy";
 async function fetchMetadata(url, options = {}) {
   const startedAt = Date.now();
   try {
     const requestedUrl = normalizeUrl(url);
     const fetchResult = await fetchPageWithStrategies(requestedUrl, options);
     const page = fetchResult.page;
+    if (fetchResult.providerDiagnostics?.blocked) {
+      return createBlockedProviderMetadata(requestedUrl, fetchResult, Date.now() - startedAt);
+    }
     const directMedia = createDirectMediaMetadata(page, requestedUrl, Date.now() - startedAt);
     if (directMedia) {
       return directMedia;
@@ -3952,6 +3971,62 @@ async function fetchMetadata(url, options = {}) {
     };
   }
 }
+function createBlockedProviderMetadata(requestedUrl, fetchResult, fetchDurationMs) {
+  const page = fetchResult.page;
+  const providerDiagnostics = fetchResult.providerDiagnostics;
+  const trace = uniqueStrings3([
+    ...page.isShortUrl ? [`detected short URL provider: ${page.shortUrlProvider ?? "unknown"}`] : [],
+    ...page.redirects.length > 0 ? [`resolved ${page.redirects.length} redirect${page.redirects.length === 1 ? "" : "s"}`] : [],
+    ...fetchResult.trace,
+    "detected blocked provider response"
+  ]);
+  const warnings = uniqueStrings3([
+    ...fetchResult.warnings,
+    REDDIT_BLOCKED_METADATA_WARNING,
+    ...page.statusCode < 200 || page.statusCode >= 300 ? [`Fetch completed with non-success status code ${page.statusCode}.`] : []
+  ]);
+  return {
+    ok: false,
+    url: requestedUrl,
+    finalUrl: page.finalUrl,
+    type: "unknown",
+    siteName: providerDiagnostics?.platform === "reddit" ? "Reddit" : void 0,
+    confidence: 0,
+    completeness: 0,
+    reliability: 0,
+    images: [],
+    videos: [],
+    audio: [],
+    favicons: [],
+    trace,
+    diagnostics: {
+      originalUrl: requestedUrl,
+      finalUrl: page.finalUrl,
+      isShortUrl: page.isShortUrl,
+      shortUrlProvider: page.shortUrlProvider,
+      statusCode: page.statusCode,
+      contentType: page.contentType,
+      redirects: page.redirects,
+      sourcesUsed: [],
+      warnings,
+      fallbacksAttempted: mergeFallbackAttempts2(void 0, fetchResult.fallbacksAttempted),
+      trace,
+      sourcePriority: fetchResult.sourcePriority,
+      extractionMethod: fetchResult.extractionMethod,
+      retryInfo: fetchResult.retryInfo,
+      providerDiagnostics,
+      confidenceBreakdown: {
+        title: 0,
+        description: 0,
+        image: 0,
+        structuredData: 0,
+        adapter: 0
+      },
+      fetchDurationMs,
+      extractedAt: (/* @__PURE__ */ new Date()).toISOString()
+    }
+  };
+}
 async function fetchPageWithStrategies(requestedUrl, options) {
   if (isRedditUrl(requestedUrl)) {
     return fetchRedditPageWithStrategy(requestedUrl, options);
@@ -3976,7 +4051,7 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
     });
     attempts.push(attempt);
     lastError = attempt.error;
-    if (attempt.page && attempt.ok && !attempt.blocked) {
+    if (attempt.page && attempt.ok) {
       const redditPost = parseRedditJsonPayload(attempt.page.html);
       if (redditPost?.title) {
         return {
@@ -3999,7 +4074,7 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
     const attempt = await attemptFetch("oldReddit", oldRedditUrl, options);
     attempts.push(attempt);
     lastError = attempt.error;
-    if (attempt.page && attempt.ok && !attempt.blocked) {
+    if (attempt.page && attempt.ok) {
       return {
         page: attempt.page,
         fallbacksAttempted: attempts,
@@ -4017,10 +4092,7 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
   const htmlAttempt = await attemptFetch("redditHtmlFallback", requestedUrl, options);
   attempts.push(htmlAttempt);
   lastError = htmlAttempt.error;
-  if (htmlAttempt.page) {
-    if (htmlAttempt.blocked) {
-      warnings.push("Reddit HTML fallback appears to have been blocked; metadata may be incomplete.");
-    }
+  if (htmlAttempt.page && htmlAttempt.ok) {
     return {
       page: htmlAttempt.page,
       fallbacksAttempted: attempts,
@@ -4031,19 +4103,37 @@ async function fetchRedditPageWithStrategy(requestedUrl, options) {
       retryInfo: redditRetryInfo(attempts)
     };
   }
+  if (htmlAttempt.blocked) {
+    warnings.push("Reddit HTML fallback appears to have been blocked; metadata may be incomplete.");
+  }
+  const providerDiagnostics = redditProviderDiagnosticsFromAttempts(attempts);
+  if (providerDiagnostics) {
+    return {
+      page: synthesizeRedditBlockedPage(requestedUrl, attempts, providerDiagnostics),
+      fallbacksAttempted: attempts,
+      warnings: uniqueStrings3([...warnings, REDDIT_BLOCKED_METADATA_WARNING]),
+      trace: ["Reddit provider blocked metadata extraction"],
+      sourcePriority,
+      extractionMethod: "reddit:blockedProvider",
+      retryInfo: redditRetryInfo(attempts),
+      providerDiagnostics
+    };
+  }
   throw lastError ?? new Error("All Reddit extraction fetch attempts failed.");
 }
 async function attemptFetch(method, url, options) {
   try {
     const page = await fetchPage(url, options);
     const retryAfter = page.headers["retry-after"];
-    const blocked = isRedditBlocked(page);
+    const blockReason = redditBlockReason(page);
+    const blocked = Boolean(blockReason);
     return {
       method,
       url,
       ok: page.statusCode >= 200 && page.statusCode < 300 && !blocked,
       statusCode: page.statusCode,
       blocked,
+      blockReason,
       retryAfter,
       page
     };
@@ -4294,8 +4384,52 @@ function synthesizeRedditJsonPage(jsonPage, requestedUrl, post) {
     statusCode: jsonPage.statusCode
   };
 }
-function isRedditBlocked(page) {
-  return page.statusCode === 403 || page.statusCode === 429 || /please wait for verification|whoa there, pardner|blocked|forbidden|too many requests|request has been blocked/i.test(page.html);
+function redditProviderDiagnosticsFromAttempts(attempts) {
+  const blockedAttempts = attempts.filter((attempt) => attempt.blocked);
+  if (blockedAttempts.length === 0) {
+    return void 0;
+  }
+  const selectedAttempt = blockedAttempts.find((attempt) => attempt.blockReason === "provider_verification_required") ?? blockedAttempts.at(-1);
+  return {
+    platform: "reddit",
+    blocked: true,
+    statusCode: selectedAttempt?.statusCode,
+    reason: selectedAttempt?.blockReason ?? "provider_blocked_request",
+    suggestedAction: PROVIDER_BLOCKED_SUGGESTED_ACTION
+  };
+}
+function synthesizeRedditBlockedPage(requestedUrl, attempts, providerDiagnostics) {
+  const selectedPage = attempts.find((attempt) => attempt.blockReason === providerDiagnostics.reason)?.page ?? attempts.slice().reverse().find((attempt) => attempt.page)?.page;
+  return {
+    url: requestedUrl,
+    originalUrl: requestedUrl,
+    finalUrl: requestedUrl,
+    isShortUrl: selectedPage?.isShortUrl ?? false,
+    shortUrlProvider: selectedPage?.shortUrlProvider,
+    html: "",
+    bytes: new Uint8Array(),
+    statusCode: providerDiagnostics.statusCode ?? selectedPage?.statusCode ?? 403,
+    contentType: selectedPage?.contentType,
+    redirects: selectedPage?.redirects ?? [],
+    headers: selectedPage?.headers ?? {}
+  };
+}
+function redditBlockReason(page) {
+  const title = htmlTitle(page.html);
+  const text = normalizeText(`${title ?? ""} ${page.html}`);
+  if (/reddit\s*-\s*please wait for verification/i.test(title ?? "") || /please wait for verification|verification required|verify you are human/i.test(text)) {
+    return "provider_verification_required";
+  }
+  if (page.statusCode === 403 || page.statusCode === 429 || /whoa there, pardner|request has been blocked|too many requests|forbidden|you're blocked|you are blocked|youre blocked|blocked by network security/i.test(text) || /^blocked$/i.test(title ?? "")) {
+    return "provider_blocked_request";
+  }
+  return void 0;
+}
+function htmlTitle(html) {
+  return normalizeText(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]);
+}
+function normalizeText(value) {
+  return value?.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim() ?? "";
 }
 function redditRetryInfo(attempts) {
   const blockedAttempts = attempts.filter((attempt) => attempt.blocked || attempt.statusCode === 429 || attempt.statusCode === 403);
@@ -4329,7 +4463,7 @@ function mergeFallbackAttempts2(existing, incoming) {
   }
   const seen = /* @__PURE__ */ new Set();
   return attempts.map((value) => {
-    const { page: _page, ...attempt } = value;
+    const { page: _page, blockReason: _blockReason, ...attempt } = value;
     return attempt;
   }).filter((attempt) => {
     const key = `${attempt.method}:${attempt.url ?? ""}:${attempt.statusCode ?? ""}:${attempt.error ?? ""}`;