@syengup/friday-channel-next 0.1.26 → 0.1.28

package/src/link-preview/preview-service.ts

@@ -0,0 +1,247 @@
+/**
+ * Link-preview orchestration: fetch page → parse Open Graph → re-host the cover image through
+ * the gateway's stored files → cache.
+ *
+ * The cover image is downloaded server-side and served from /friday-next/files/ so the app only
+ * ever talks to the trusted gateway host (same rationale as `downloadRemoteMedia` for outbound
+ * media). Failures degrade to "no card" on the app side, so every error path returns a typed
+ * error instead of throwing.
+ */
+
+import { createFridayNextLogger } from "../logging.js";
+import { storeFile } from "../http/handlers/files.js";
+import { parseOpenGraph } from "./og-parse.js";
+import { BlockedUrlError, fetchPublicUrl, parseHttpUrl } from "./ssrf-guard.js";
+
+const HTML_MAX_BYTES = 2 * 1024 * 1024;
+const HTML_TIMEOUT_MS = 10_000;
+const IMAGE_MAX_BYTES = 8 * 1024 * 1024;
+const IMAGE_TIMEOUT_MS = 10_000;
+
+const SUCCESS_TTL_MS = 24 * 60 * 60 * 1000;
+const FAILURE_TTL_MS = 10 * 60 * 1000;
+const MAX_CACHE_ENTRIES = 1000;
+
+const logger = createFridayNextLogger("link-preview");
+
+export interface LinkPreviewPayload {
+  url: string;
+  finalUrl: string;
+  siteName: string | null;
+  title: string;
+  description: string | null;
+  /** Gateway-relative cover URL ("/friday-next/files/{token}") or null. */
+  imageUrl: string | null;
+  /** Gateway-relative favicon URL ("/friday-next/files/{token}") or null. */
+  iconUrl: string | null;
+  fetchedAt: number;
+}
+
+export type LinkPreviewError = "invalid_url" | "blocked_url" | "fetch_failed" | "no_metadata";
+
+export type LinkPreviewResult =
+  | { ok: true; preview: LinkPreviewPayload }
+  | { ok: false; error: LinkPreviewError };
+
+interface CacheEntry {
+  result: LinkPreviewResult;
+  cachedAt: number;
+}
+
+const cache = new Map<string, CacheEntry>();
+const inFlight = new Map<string, Promise<LinkPreviewResult>>();
+
+export function resetLinkPreviewCacheForTest(): void {
+  cache.clear();
+  inFlight.clear();
+}
+
+export async function getLinkPreview(rawUrl: string): Promise<LinkPreviewResult> {
+  const parsed = parseHttpUrl(rawUrl);
+  if (!parsed) return { ok: false, error: "invalid_url" };
+  const key = parsed.toString();
+
+  const cached = cache.get(key);
+  if (cached) {
+    const ttl = cached.result.ok ? SUCCESS_TTL_MS : FAILURE_TTL_MS;
+    if (Date.now() - cached.cachedAt < ttl) return cached.result;
+    cache.delete(key);
+  }
+
+  const pending = inFlight.get(key);
+  if (pending) return pending;
+
+  const task = buildPreview(key)
+    .then((result) => {
+      writeCache(key, result);
+      return result;
+    })
+    .finally(() => {
+      inFlight.delete(key);
+    });
+  inFlight.set(key, task);
+  return task;
+}
+
+function writeCache(key: string, result: LinkPreviewResult): void {
+  if (cache.size >= MAX_CACHE_ENTRIES) {
+    let oldestKey: string | null = null;
+    let oldestAt = Infinity;
+    for (const [k, entry] of cache) {
+      if (entry.cachedAt < oldestAt) {
+        oldestAt = entry.cachedAt;
+        oldestKey = k;
+      }
+    }
+    if (oldestKey) cache.delete(oldestKey);
+  }
+  cache.set(key, { result, cachedAt: Date.now() });
+}
+
+async function buildPreview(pageUrl: string): Promise<LinkPreviewResult> {
+  let page;
+  try {
+    page = await fetchPublicUrl(pageUrl, {
+      maxBytes: HTML_MAX_BYTES,
+      timeoutMs: HTML_TIMEOUT_MS,
+      accept: "text/html,application/xhtml+xml",
+      requireContentTypePrefixes: ["text/html", "application/xhtml+xml"],
+    });
+  } catch (err) {
+    if (err instanceof BlockedUrlError) {
+      logger.warn(`link-preview blocked: ${pageUrl} (${err.reason})`);
+      return { ok: false, error: "blocked_url" };
+    }
+    page = null; // network/timeout — fall through to a favicon-only minimal card
+  }
+
+  const finalUrl = page?.finalUrl ?? pageUrl;
+  const og = page ? parseOpenGraph(page.body.toString("utf8"), finalUrl) : null;
+  const hostname = (() => {
+    try {
+      return new URL(finalUrl).hostname;
+    } catch {
+      return null;
+    }
+  })();
+
+  // Favicon: parsed <link rel icon> first, then the conventional /favicon.ico (which is reachable
+  // even for pages that block bots, e.g. zhihu → redirects to its CDN icon).
+  const iconUrl = await resolveFavicon(og?.iconUrl ?? null, finalUrl);
+
+  // A failed page fetch only yields a (minimal) card when the favicon resolved — that proves the
+  // domain is real/reachable (e.g. bot-blocked zhihu). A dead domain (favicon also fails) collapses.
+  const reachable = page !== null || iconUrl !== null;
+  const title = og?.title ?? hostname;
+  if (!reachable || !title) {
+    return { ok: false, error: page ? "no_metadata" : "fetch_failed" };
+  }
+
+  const imageUrl = og?.imageUrl ? await rehostCoverImage(og.imageUrl) : null;
+
+  return {
+    ok: true,
+    preview: {
+      url: pageUrl,
+      finalUrl,
+      siteName: og?.siteName ?? hostname,
+      title: title ?? hostname ?? pageUrl,
+      description: og?.description ?? null,
+      imageUrl,
+      iconUrl,
+      fetchedAt: Date.now(),
+    },
+  };
+}
+
+/** Re-host a favicon: try the parsed `<link rel icon>`, then `<origin>/favicon.ico`. */
+async function resolveFavicon(parsedIconUrl: string | null, finalUrl: string): Promise<string | null> {
+  const candidates: string[] = [];
+  if (parsedIconUrl) candidates.push(parsedIconUrl);
+  try {
+    candidates.push(new URL("/favicon.ico", finalUrl).toString());
+  } catch {
+    // finalUrl unparseable — skip the conventional fallback
+  }
+  for (const candidate of candidates) {
+    const rehosted = await rehostIconImage(candidate);
+    if (rehosted) return rehosted;
+  }
+  return null;
+}
+
+/** Download a favicon (full SSRF checks) and re-publish via stored files. Null on any failure. */
+async function rehostIconImage(iconUrl: string): Promise<string | null> {
+  let image;
+  try {
+    image = await fetchPublicUrl(iconUrl, {
+      maxBytes: 1024 * 1024,
+      timeoutMs: IMAGE_TIMEOUT_MS,
+      accept: "image/*",
+      requireContentTypePrefixes: ["image/"],
+    });
+  } catch {
+    return null;
+  }
+  if (!image) return null;
+  const sniffed = sniffImageType(image.body);
+  if (!sniffed) return null;
+  try {
+    const stored = storeFile(image.body, `link-preview-icon.${sniffed.ext}`, sniffed.mime);
+    return `/friday-next/files/${encodeURIComponent(stored.urlToken)}`;
+  } catch {
+    return null;
+  }
+}
+
+/** Download og:image (full SSRF checks) and re-publish via stored files. Null on any failure. */
+async function rehostCoverImage(imageUrl: string): Promise<string | null> {
+  let image;
+  try {
+    image = await fetchPublicUrl(imageUrl, {
+      maxBytes: IMAGE_MAX_BYTES,
+      timeoutMs: IMAGE_TIMEOUT_MS,
+      accept: "image/*",
+      requireContentTypePrefixes: ["image/"],
+    });
+  } catch {
+    return null; // blocked og:image just means no cover
+  }
+  if (!image) return null;
+
+  const sniffed = sniffImageType(image.body);
+  if (!sniffed) return null;
+
+  try {
+    const stored = storeFile(image.body, `link-preview-cover.${sniffed.ext}`, sniffed.mime);
+    return `/friday-next/files/${encodeURIComponent(stored.urlToken)}`;
+  } catch (err) {
+    logger.warn(`link-preview cover store failed for ${imageUrl}: ${String(err)}`);
+    return null;
+  }
+}
+
+/** Magic-byte sniff — second line of defense after the Content-Type check. */
+function sniffImageType(buffer: Buffer): { ext: string; mime: string } | null {
+  if (buffer.length < 12) return null;
+  // ICO: 00 00 01 00 (favicons are commonly .ico; iOS ImageIO decodes them).
+  if (buffer[0] === 0x00 && buffer[1] === 0x00 && buffer[2] === 0x01 && buffer[3] === 0x00) {
+    return { ext: "ico", mime: "image/x-icon" };
+  }
+  if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
+    return { ext: "png", mime: "image/png" };
+  }
+  if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+    return { ext: "jpg", mime: "image/jpeg" };
+  }
+  if (buffer.subarray(0, 4).toString("latin1") === "GIF8") {
+    return { ext: "gif", mime: "image/gif" };
+  }
+  if (
+    buffer.subarray(0, 4).toString("latin1") === "RIFF" &&
+    buffer.subarray(8, 12).toString("latin1") === "WEBP"
+  ) {
+    return { ext: "webp", mime: "image/webp" };
+  }
+  return null;
+}

package/src/link-preview/ssrf-guard.test.ts

@@ -0,0 +1,234 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("node:dns/promises", () => ({
+  default: { lookup: vi.fn() },
+}));
+
+import dns from "node:dns/promises";
+import {
+  BlockedUrlError,
+  assertPublicHttpUrl,
+  assertResolvesPublic,
+  fetchPublicUrl,
+  isPrivateAddress,
+  parseHttpUrl,
+} from "./ssrf-guard.js";
+
+const lookupMock = vi.mocked(dns.lookup);
+
+function mockPublicDns(): void {
+  lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }] as never);
+}
+
+beforeEach(() => {
+  lookupMock.mockReset();
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe("parseHttpUrl", () => {
+  it("accepts absolute http/https URLs", () => {
+    expect(parseHttpUrl("https://example.com/a?b=1")?.hostname).toBe("example.com");
+    expect(parseHttpUrl("  http://example.com  ")?.protocol).toBe("http:");
+  });
+
+  it("rejects non-http schemes and garbage", () => {
+    expect(parseHttpUrl("ftp://example.com/a")).toBeNull();
+    expect(parseHttpUrl("file:///etc/passwd")).toBeNull();
+    expect(parseHttpUrl("javascript:alert(1)")).toBeNull();
+    expect(parseHttpUrl("not a url")).toBeNull();
+    expect(parseHttpUrl("/relative/path")).toBeNull();
+  });
+});
+
+describe("assertPublicHttpUrl", () => {
+  it("rejects non-default ports", () => {
+    expect(() => assertPublicHttpUrl(new URL("http://example.com:8080/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("https://example.com:8443/"))).toThrow(BlockedUrlError);
+  });
+
+  it("allows default and explicit 80/443 ports", () => {
+    expect(() => assertPublicHttpUrl(new URL("https://example.com/"))).not.toThrow();
+    expect(() => assertPublicHttpUrl(new URL("http://example.com:80/"))).not.toThrow();
+    expect(() => assertPublicHttpUrl(new URL("https://example.com:443/"))).not.toThrow();
+  });
+
+  it("rejects localhost and internal-suffix hostnames", () => {
+    expect(() => assertPublicHttpUrl(new URL("http://localhost/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("http://gateway.local/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("http://db.internal/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("http://nas.home.arpa/"))).toThrow(BlockedUrlError);
+  });
+
+  it("rejects private IP literals including bracketed IPv6", () => {
+    expect(() => assertPublicHttpUrl(new URL("http://127.0.0.1/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("http://10.0.0.5/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("http://[::1]/"))).toThrow(BlockedUrlError);
+    expect(() => assertPublicHttpUrl(new URL("http://[fc00::1]/"))).toThrow(BlockedUrlError);
+  });
+
+  it("allows public IP literals", () => {
+    expect(() => assertPublicHttpUrl(new URL("http://93.184.216.34/"))).not.toThrow();
+  });
+});
+
+describe("isPrivateAddress", () => {
+  it("flags private/reserved IPv4 ranges", () => {
+    for (const ip of [
+      "0.0.0.0",
+      "10.1.2.3",
+      "100.64.0.1",
+      "100.127.255.255",
+      "127.0.0.1",
+      "169.254.1.1",
+      "172.16.0.1",
+      "172.31.255.255",
+      "192.0.0.1",
+      "192.168.1.1",
+      "224.0.0.1",
+      "255.255.255.255",
+    ]) {
+      expect(isPrivateAddress(ip), ip).toBe(true);
+    }
+  });
+
+  it("passes public IPv4", () => {
+    for (const ip of ["8.8.8.8", "93.184.216.34", "100.63.0.1", "100.128.0.1", "172.32.0.1", "198.20.0.1"]) {
+      expect(isPrivateAddress(ip), ip).toBe(false);
+    }
+  });
+
+  it("passes 198.18/15 (fake-IP DNS range used by clash/surge tun mode)", () => {
+    expect(isPrivateAddress("198.18.1.156")).toBe(false);
+    expect(isPrivateAddress("198.19.255.255")).toBe(false);
+  });
+
+  it("flags private/reserved IPv6 and mapped IPv4", () => {
+    for (const ip of ["::1", "::", "fc00::1", "fd12:3456::1", "fe80::1", "::ffff:10.0.0.1", "::ffff:127.0.0.1"]) {
+      expect(isPrivateAddress(ip), ip).toBe(true);
+    }
+  });
+
+  it("passes public IPv6 and mapped public IPv4", () => {
+    expect(isPrivateAddress("2606:2800:220:1:248:1893:25c8:1946")).toBe(false);
+    expect(isPrivateAddress("::ffff:93.184.216.34")).toBe(false);
+  });
+
+  it("treats non-IP strings as unsafe", () => {
+    expect(isPrivateAddress("not-an-ip")).toBe(true);
+  });
+});
+
+describe("assertResolvesPublic", () => {
+  it("passes when all resolved addresses are public", async () => {
+    lookupMock.mockResolvedValue([
+      { address: "93.184.216.34", family: 4 },
+      { address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 },
+    ] as never);
+    await expect(assertResolvesPublic(new URL("https://example.com/"))).resolves.toBeUndefined();
+  });
+
+  it("throws BlockedUrlError when any resolved address is private", async () => {
+    lookupMock.mockResolvedValue([
+      { address: "93.184.216.34", family: 4 },
+      { address: "10.0.0.5", family: 4 },
+    ] as never);
+    await expect(assertResolvesPublic(new URL("https://rebind.example.com/"))).rejects.toThrow(BlockedUrlError);
+  });
+
+  it("validates IP-literal hosts without a DNS lookup", async () => {
+    await expect(assertResolvesPublic(new URL("http://127.0.0.1/"))).rejects.toThrow(BlockedUrlError);
+    await expect(assertResolvesPublic(new URL("http://93.184.216.34/"))).resolves.toBeUndefined();
+    expect(lookupMock).not.toHaveBeenCalled();
+  });
+
+  it("throws a plain error (not BlockedUrlError) on DNS failure", async () => {
+    lookupMock.mockRejectedValue(new Error("ENOTFOUND"));
+    const err = await assertResolvesPublic(new URL("https://nope.example.com/")).catch((e) => e);
+    expect(err).toBeInstanceOf(Error);
+    expect(err).not.toBeInstanceOf(BlockedUrlError);
+  });
+});
+
+describe("fetchPublicUrl", () => {
+  const opts = { maxBytes: 1024, timeoutMs: 5000 };
+
+  it("fetches a public URL and returns body + finalUrl", async () => {
+    mockPublicDns();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => new Response("<html>hi</html>", { status: 200, headers: { "content-type": "text/html; charset=utf-8" } })),
+    );
+    const result = await fetchPublicUrl("https://example.com/page", opts);
+    expect(result?.finalUrl).toBe("https://example.com/page");
+    expect(result?.contentType).toContain("text/html");
+    expect(result?.body.toString()).toBe("<html>hi</html>");
+  });
+
+  it("follows redirects and re-validates each hop", async () => {
+    mockPublicDns();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(new Response(null, { status: 302, headers: { location: "https://other.example.com/final" } }))
+      .mockResolvedValueOnce(new Response("ok", { status: 200, headers: { "content-type": "text/html" } }));
+    vi.stubGlobal("fetch", fetchMock);
+    const result = await fetchPublicUrl("https://example.com/start", opts);
+    expect(result?.finalUrl).toBe("https://other.example.com/final");
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(lookupMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("throws BlockedUrlError when a redirect targets a private address", async () => {
+    mockPublicDns();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => new Response(null, { status: 302, headers: { location: "http://127.0.0.1/admin" } })),
+    );
+    await expect(fetchPublicUrl("https://example.com/start", opts)).rejects.toThrow(BlockedUrlError);
+  });
+
+  it("throws BlockedUrlError for a directly-blocked URL", async () => {
+    await expect(fetchPublicUrl("http://192.168.1.1/", opts)).rejects.toThrow(BlockedUrlError);
+  });
+
+  it("returns null when the body exceeds maxBytes (ignoring content-length)", async () => {
+    mockPublicDns();
+    const big = "x".repeat(2048);
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => new Response(big, { status: 200, headers: { "content-type": "text/html", "content-length": "10" } })),
+    );
+    expect(await fetchPublicUrl("https://example.com/big", opts)).toBeNull();
+  });
+
+  it("returns null when content-type does not match the required prefixes", async () => {
+    mockPublicDns();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => new Response("{}", { status: 200, headers: { "content-type": "application/json" } })),
+    );
+    expect(
+      await fetchPublicUrl("https://example.com/api", { ...opts, requireContentTypePrefixes: ["text/html", "application/xhtml+xml"] }),
+    ).toBeNull();
+  });
+
+  it("returns null on non-2xx and on DNS failure", async () => {
+    mockPublicDns();
+    vi.stubGlobal("fetch", vi.fn(async () => new Response("nope", { status: 404 })));
+    expect(await fetchPublicUrl("https://example.com/missing", opts)).toBeNull();
+
+    lookupMock.mockRejectedValue(new Error("ENOTFOUND"));
+    expect(await fetchPublicUrl("https://nope.example.com/", opts)).toBeNull();
+  });
+
+  it("gives up after too many redirects", async () => {
+    mockPublicDns();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => new Response(null, { status: 302, headers: { location: "https://example.com/loop" } })),
+    );
+    expect(await fetchPublicUrl("https://example.com/loop", opts)).toBeNull();
+  });
+});