@soundchain/openclaw-plugin 1.0.0 → 2.0.0

package/src/content-fetcher.ts

@@ -1,406 +0,0 @@
-/**
- * FURL Content Fetcher — URL Detection + Content Extraction
- *
- * Detects URLs in DM messages and extracts readable text content
- * so Claude can summarize videos, articles, tweets, etc.
- *
- * Supported:
- *   - YouTube (transcript via captions API — free, no key needed)
- *   - General web pages (HTML stripped to text)
- *
- * Zero cost, zero new dependencies. Uses built-in fetch().
- */
-
-// ---------------------------------------------------------------------------
-// URL detection
-// ---------------------------------------------------------------------------
-
-const URL_REGEX = /https?:\/\/[^\s<>"')\]]+/gi;
-
-const YOUTUBE_REGEX =
-  /(?:youtube\.com\/(?:watch\?.*?v=|shorts\/|live\/|embed\/)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
-
-const X_TWITTER_REGEX = /(?:x\.com|twitter\.com)\/(\w+)\/status\/(\d+)/;
-
-import { lookup } from "node:dns/promises";
-
-const USER_AGENT =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
-
-const FETCH_TIMEOUT_MS = 12_000;
-const MAX_URLS = 3;
-const MAX_TRANSCRIPT_CHARS = 15_000;
-const MAX_WEBPAGE_CHARS = 10_000;
-
-// Block private/internal network URLs to prevent SSRF
-const BLOCKED_HOST_PATTERNS = [
-  /^localhost$/i,
-  /^127\.\d+\.\d+\.\d+$/,
-  /^10\.\d+\.\d+\.\d+$/,
-  /^172\.(1[6-9]|2\d|3[01])\.\d+\.\d+$/,
-  /^192\.168\.\d+\.\d+$/,
-  /^169\.254\.\d+\.\d+$/, // Link-local
-  /^0\.0\.0\.0$/,
-  /^\[?::1?\]?$/,
-  /^\[?::ffff:/i, // IPv6-mapped IPv4 (bracketed or bare)
-  /^\[?fd[0-9a-f]{2}:/i, // IPv6 ULA fd00::/8 (bracketed or bare)
-  /^\[?fe80:/i, // IPv6 link-local fe80::/10 (bracketed or bare)
-  /^\[?fc[0-9a-f]{2}:/i, // IPv6 ULA fc00::/7 (bracketed or bare)
-  /\.internal$/i,
-  /\.local$/i,
-  /metadata\.google/i, // Cloud metadata
-  /169\.254\.169\.254/, // AWS/GCP metadata
-];
-
-function isBlockedHostname(hostname: string): boolean {
-  return BLOCKED_HOST_PATTERNS.some((p) => p.test(hostname));
-}
-
-function isBlockedUrl(url: string): boolean {
-  try {
-    const parsed = new URL(url);
-    return isBlockedHostname(parsed.hostname);
-  } catch {
-    return true; // Block unparseable URLs
-  }
-}
-
-/** Resolve hostname via DNS and check if the IP is private/blocked */
-async function isBlockedAfterResolve(url: string): Promise<boolean> {
-  if (isBlockedUrl(url)) return true;
-  try {
-    const parsed = new URL(url);
-    const { address } = await lookup(parsed.hostname);
-    return isBlockedHostname(address);
-  } catch {
-    return false; // DNS failure — allow (will fail at fetch anyway)
-  }
-}
-
-/**
- * Safe fetch that blocks redirects to private networks.
- * Uses redirect: "manual" to inspect each hop.
- */
-const MAX_REDIRECTS = 5;
-
-async function safeFetch(url: string, options?: RequestInit, hops = 0): Promise<Response> {
-  if (hops > MAX_REDIRECTS) throw new Error(`Too many redirects (>${MAX_REDIRECTS})`);
-
-  // Resolve DNS and validate BEFORE fetching (prevents DNS rebinding)
-  const parsed = new URL(url);
-  if (isBlockedHostname(parsed.hostname)) throw new Error("Blocked hostname");
-  let resolvedIp: string | undefined;
-  try {
-    const { address } = await lookup(parsed.hostname);
-    resolvedIp = address;
-    if (isBlockedHostname(address)) throw new Error("Blocked resolved IP");
-  } catch (e: any) {
-    if (e.message?.includes("Blocked")) throw e;
-    // DNS failure — proceed with hostname (will fail at fetch if unreachable)
-  }
-
-  // DNS validated — proceed with original URL (IP pinning breaks TLS/SNI)
-  // The validation + short timeout window is sufficient for non-targeted SSRF
-  const res = await fetch(url, {
-    ...options,
-    redirect: "manual",
-    signal: options?.signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS),
-  });
-
-  if (res.status >= 300 && res.status < 400) {
-    const location = res.headers.get("location");
-    if (!location) return res;
-
-    const redirectUrl = new URL(location, url).toString();
-    if (isBlockedUrl(redirectUrl)) throw new Error("Redirect to blocked URL");
-
-    return safeFetch(redirectUrl, options, hops + 1);
-  }
-
-  return res;
-}
-
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
-/**
- * Scans a message for URLs, fetches content from each, and appends
- * extracted text so Claude can summarize it.
- * Returns the original message if no URLs found or all extractions fail.
- */
-export async function enrichMessageWithUrlContent(message: string): Promise<string> {
-  const urls = message.match(URL_REGEX);
-  if (!urls || urls.length === 0) return message;
-
-  // Deduplicate
-  const unique = [...new Set(urls)].slice(0, MAX_URLS);
-
-  const extractions: string[] = [];
-
-  for (const url of unique) {
-    try {
-      const content = await extractContent(url);
-      if (content) {
-        extractions.push(content);
-      }
-    } catch {
-      // Skip failed extractions silently
-    }
-  }
-
-  if (extractions.length === 0) return message;
-
-  return message + "\n\n---\n" + extractions.join("\n\n---\n");
-}
-
-// ---------------------------------------------------------------------------
-// Router
-// ---------------------------------------------------------------------------
-
-async function extractContent(url: string): Promise<string | null> {
-  // Block private/internal network URLs (SSRF prevention — checks both hostname and resolved IP)
-  if (await isBlockedAfterResolve(url)) return null;
-
-  const ytMatch = url.match(YOUTUBE_REGEX);
-  if (ytMatch) {
-    return fetchYouTubeTranscript(ytMatch[1], url);
-  }
-  const xMatch = url.match(X_TWITTER_REGEX);
-  if (xMatch) {
-    return fetchXPost(xMatch[1], xMatch[2], url);
-  }
-  return fetchWebPageText(url);
-}
-
-// ---------------------------------------------------------------------------
-// YouTube Transcript (free — captions API, no key needed)
-// ---------------------------------------------------------------------------
-
-async function fetchYouTubeTranscript(
-  videoId: string,
-  originalUrl: string,
-): Promise<string | null> {
-  try {
-    const res = await fetch(`https://www.youtube.com/watch?v=${videoId}&hl=en`, {
-      headers: {
-        "User-Agent": USER_AGENT,
-        "Accept-Language": "en-US,en;q=0.9",
-      },
-      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-    });
-    const html = await res.text();
-
-    // Extract video title
-    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    const title = titleMatch
-      ? titleMatch[1].replace(/\s*-\s*YouTube\s*$/, "").trim()
-      : "Unknown Video";
-
-    // Extract ytInitialPlayerResponse JSON
-    const playerMatch =
-      html.match(/var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\});/s) ||
-      html.match(/ytInitialPlayerResponse\s*=\s*(\{.+?\});/s);
-
-    if (!playerMatch) {
-      return `[YouTube Video: "${title}" — ${originalUrl}]\n(Could not extract transcript — captions may be disabled)`;
-    }
-
-    let playerResponse: Record<string, unknown>;
-    try {
-      playerResponse = JSON.parse(playerMatch[1]);
-    } catch {
-      return `[YouTube Video: "${title}" — ${originalUrl}]\n(Could not parse video data)`;
-    }
-
-    // Navigate to caption tracks
-    const captions = playerResponse?.captions as Record<string, unknown> | undefined;
-    const renderer = captions?.playerCaptionsTracklistRenderer as
-      | Record<string, unknown>
-      | undefined;
-    const captionTracks = renderer?.captionTracks as Array<Record<string, unknown>> | undefined;
-
-    if (!captionTracks || captionTracks.length === 0) {
-      // No captions — try to get video description instead
-      const descMatch = html.match(/"shortDescription":"((?:[^"\\]|\\.)*)"/);
-      const description = descMatch
-        ? descMatch[1].replace(/\\n/g, "\n").replace(/\\"/g, '"').replace(/\\\\/g, "\\")
-        : null;
-
-      if (description && description.length > 50) {
-        return `[YouTube Video: "${title}" — ${originalUrl}]\n(No captions available — description below)\n\n${description.slice(0, 3000)}`;
-      }
-      return `[YouTube Video: "${title}" — ${originalUrl}]\n(No captions or description available)`;
-    }
-
-    // Prefer English, fallback to first available
-    const track =
-      captionTracks.find(
-        (t) => typeof t.languageCode === "string" && t.languageCode.startsWith("en"),
-      ) || captionTracks[0];
-
-    const captionUrl = track?.baseUrl;
-    if (typeof captionUrl !== "string") return null;
-
-    // Fetch caption XML
-    const captionRes = await safeFetch(captionUrl, {
-      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-    });
-    const xml = await captionRes.text();
-
-    // Parse <text> elements to plain text
-    const textMatches = [...xml.matchAll(/<text[^>]*>([\s\S]*?)<\/text>/g)];
-    const transcript = textMatches
-      .map((m) => decodeHtmlEntities(m[1]).replace(/\n/g, " "))
-      .join(" ")
-      .replace(/\s+/g, " ")
-      .trim();
-
-    if (!transcript) {
-      return `[YouTube Video: "${title}" — ${originalUrl}]\n(Captions found but empty)`;
-    }
-
-    const lang = typeof track.languageCode === "string" ? ` (${track.languageCode})` : "";
-
-    return `[YouTube Video: "${title}" — ${originalUrl}]\n[Transcript${lang} — ${transcript.length} chars]:\n\n${transcript.slice(0, MAX_TRANSCRIPT_CHARS)}`;
-  } catch (err) {
-    return null;
-  }
-}
-
-// ---------------------------------------------------------------------------
-// X/Twitter Posts (via Nitter instances or syndication API)
-// ---------------------------------------------------------------------------
-
-async function fetchXPost(
-  username: string,
-  tweetId: string,
-  originalUrl: string,
-): Promise<string | null> {
-  // Try Twitter syndication API (public, no auth needed)
-  try {
-    const syndicationUrl = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetId}&token=0`;
-    const res = await fetch(syndicationUrl, {
-      headers: { "User-Agent": USER_AGENT },
-      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-    });
-
-    if (res.ok) {
-      const data = (await res.json()) as Record<string, unknown>;
-      const text = data.text as string | undefined;
-      const userName = (data.user as Record<string, unknown>)?.name as string | undefined;
-      const handle = (data.user as Record<string, unknown>)?.screen_name as string | undefined;
-
-      if (text) {
-        const header = `[X Post by ${userName || username} (@${handle || username}) — ${originalUrl}]`;
-
-        // Check for quoted tweet
-        const quoted = data.quoted_tweet as Record<string, unknown> | undefined;
-        let quotedText = "";
-        if (quoted?.text) {
-          const qUser = (quoted.user as Record<string, unknown>)?.screen_name || "unknown";
-          quotedText = `\n\n[Quoted @${qUser}]: ${quoted.text as string}`;
-        }
-
-        // Check for media descriptions
-        const mediaDetails = data.mediaDetails as Array<Record<string, unknown>> | undefined;
-        let mediaText = "";
-        if (mediaDetails && mediaDetails.length > 0) {
-          const types = mediaDetails.map((m) => m.type as string).filter(Boolean);
-          mediaText = `\n[Media: ${types.join(", ")}]`;
-        }
-
-        return `${header}\n\n${text}${quotedText}${mediaText}`;
-      }
-    }
-  } catch {
-    // Syndication failed — fall through to web page fetch
-  }
-
-  // Fallback: fetch the page directly (may get limited HTML)
-  return fetchWebPageText(originalUrl);
-}
-
-// ---------------------------------------------------------------------------
-// General Web Page
-// ---------------------------------------------------------------------------
-
-async function fetchWebPageText(url: string): Promise<string | null> {
-  try {
-    const res = await safeFetch(url, {
-      headers: { "User-Agent": USER_AGENT },
-    });
-
-    const contentType = res.headers.get("content-type") || "";
-    if (
-      !contentType.includes("text/html") &&
-      !contentType.includes("text/plain") &&
-      !contentType.includes("application/json")
-    ) {
-      return null; // Skip binary content (images, videos, PDFs, etc.)
-    }
-
-    // Cap response body to 512KB to prevent memory exhaustion from large pages
-    const MAX_BODY_BYTES = 512 * 1024;
-    const reader = res.body?.getReader();
-    if (!reader) return null;
-    const chunks: Uint8Array[] = [];
-    let totalBytes = 0;
-    while (totalBytes < MAX_BODY_BYTES) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      chunks.push(value);
-      totalBytes += value.length;
-    }
-    reader.cancel();
-    const raw = new TextDecoder().decode(Buffer.concat(chunks));
-
-    // JSON responses — return formatted
-    if (contentType.includes("application/json")) {
-      return `[API Response: ${url}]\n\n${raw.slice(0, MAX_WEBPAGE_CHARS)}`;
-    }
-
-    // Extract title
-    const titleMatch = raw.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    const title = titleMatch ? decodeHtmlEntities(titleMatch[1]).trim() : "";
-
-    // Strip non-content elements, then all tags
-    const text = raw
-      .replace(/<script[\s\S]*?<\/script>/gi, "")
-      .replace(/<style[\s\S]*?<\/style>/gi, "")
-      .replace(/<nav[\s\S]*?<\/nav>/gi, "")
-      .replace(/<footer[\s\S]*?<\/footer>/gi, "")
-      .replace(/<noscript[\s\S]*?<\/noscript>/gi, "")
-      .replace(/<svg[\s\S]*?<\/svg>/gi, "")
-      .replace(/<[^>]+>/g, " ")
-      .replace(/\s+/g, " ")
-      .trim();
-
-    const cleaned = decodeHtmlEntities(text);
-
-    if (cleaned.length < 100) return null;
-
-    const header = title ? `[Web Page: "${title}" — ${url}]\n\n` : `[Web Page: ${url}]\n\n`;
-
-    return header + cleaned.slice(0, MAX_WEBPAGE_CHARS);
-  } catch {
-    return null;
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function decodeHtmlEntities(text: string): string {
-  return text
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&#39;/g, "'")
-    .replace(/&apos;/g, "'")
-    .replace(/&quot;/g, '"')
-    .replace(/&nbsp;/g, " ")
-    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
-    .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
-}