@jant/core 0.6.10 → 0.6.11

package/src/lib/upload.ts

@@ -277,6 +277,117 @@ export function isImageMimeType(mimeType: string): boolean {
   return mimeType.startsWith("image/");
 }
 
+/** Image MIME types accepted by the remote-image sideload path. */
+const SIDELOAD_IMAGE_MIME_TYPES = new Set<string>(IMAGE_MIME_TYPES);
+
+/** Map of sideload-accepted image MIME types to their file extensions. */
+const IMAGE_MIME_EXTENSIONS: Record<string, string> = {
+  "image/jpeg": "jpg",
+  "image/png": "png",
+  "image/gif": "gif",
+  "image/webp": "webp",
+  "image/svg+xml": "svg",
+  "image/avif": "avif",
+  "image/bmp": "bmp",
+  "image/x-icon": "ico",
+};
+
+/**
+ * Whether a MIME type may be rehosted via remote-image sideload.
+ *
+ * Unlike {@link getStoredUploadPolicy} (which only allows webp/png/jpeg because
+ * the normal upload path re-encodes everything to WebP client-side), the
+ * sideload path stores the original remote bytes, so it accepts the full set of
+ * image formats Jant can display.
+ *
+ * @param contentType - The MIME type to check
+ * @returns Whether the type is an accepted sideload image
+ * @example
+ * ```ts
+ * isAllowedSideloadImageType("image/gif"); // true
+ * isAllowedSideloadImageType("text/html"); // false
+ * ```
+ */
+export function isAllowedSideloadImageType(contentType: string): boolean {
+  return SIDELOAD_IMAGE_MIME_TYPES.has(contentType);
+}
+
+/**
+ * Returns the file extension for a sideload-accepted image MIME type.
+ *
+ * @param contentType - The image MIME type
+ * @returns Extension without a dot, or null if the type isn't sideloadable
+ * @example
+ * ```ts
+ * imageExtensionForMimeType("image/jpeg"); // "jpg"
+ * ```
+ */
+export function imageExtensionForMimeType(contentType: string): string | null {
+  return IMAGE_MIME_EXTENSIONS[contentType] ?? null;
+}
+
+/**
+ * Identify an image format from its leading bytes (magic numbers), so a remote
+ * sideload can trust the actual content rather than the server's content-type
+ * header. Recognizes every {@link isAllowedSideloadImageType} format.
+ *
+ * @param bytes - The leading bytes of the file (≥ ~256 bytes recommended)
+ * @returns The detected image MIME type, or null if unrecognized
+ * @example
+ * ```ts
+ * sniffImageMimeType(pngBytes); // "image/png"
+ * ```
+ */
+export function sniffImageMimeType(bytes: Uint8Array): string | null {
+  if (
+    bytes.length >= 3 &&
+    bytes[0] === 0xff &&
+    bytes[1] === 0xd8 &&
+    bytes[2] === 0xff
+  ) {
+    return "image/jpeg";
+  }
+  if (
+    bytes.length >= 8 &&
+    bytes[0] === 0x89 &&
+    readAscii(bytes, 1, 3) === "PNG"
+  ) {
+    return "image/png";
+  }
+  if (bytes.length >= 6 && /^GIF8[79]a$/.test(readAscii(bytes, 0, 6))) {
+    return "image/gif";
+  }
+  if (
+    bytes.length >= 12 &&
+    readAscii(bytes, 0, 4) === "RIFF" &&
+    readAscii(bytes, 8, 4) === "WEBP"
+  ) {
+    return "image/webp";
+  }
+  if (bytes.length >= 12 && readAscii(bytes, 4, 4) === "ftyp") {
+    const brand = readAscii(bytes, 8, 4);
+    if (brand === "avif" || brand === "avis") return "image/avif";
+  }
+  if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) {
+    return "image/bmp";
+  }
+  if (
+    bytes.length >= 4 &&
+    bytes[0] === 0x00 &&
+    bytes[1] === 0x00 &&
+    bytes[2] === 0x01 &&
+    bytes[3] === 0x00
+  ) {
+    return "image/x-icon";
+  }
+  // SVG is text — look for an <svg> root in the leading bytes.
+  const head = new TextDecoder().decode(bytes.subarray(0, 1024)).toLowerCase();
+  if (head.includes("<svg")) {
+    return "image/svg+xml";
+  }
+  return null;
+}
+
 export interface ValidateUploadOptions {
   /** When true, only image MIME types are accepted (e.g. for avatar uploads). */
   imagesOnly?: boolean;

package/src/lib/url-fetch.ts

@@ -0,0 +1,263 @@
+/**
+ * Safe remote URL fetching for server-side image sideloading.
+ *
+ * When an author pastes an article from another site, its `<img>` tags point at
+ * remote URLs. To rehost those images into the site's own storage the server
+ * must fetch the bytes itself (a browser `fetch` of a third-party image is
+ * blocked by CORS for most hosts). Because the URL comes from pasted HTML it is
+ * attacker-influenced, so every fetch passes through an SSRF guard and a bounded
+ * reader that caps size and time.
+ *
+ * Note: DNS is not resolvable from a Cloudflare Worker, so the IP-literal checks
+ * here are defense-in-depth for self-hosted (Node) deployments and for URLs that
+ * embed a literal address. They are not a substitute for network-level egress
+ * controls.
+ */
+
+import { ValidationError } from "./errors.js";
+
+/** A browser-like UA — many CDNs serving article images block unknown bots. */
+const FETCH_USER_AGENT =
+  "Mozilla/5.0 (compatible; Jant image sideloader) AppleWebKit/537.36";
+
+export interface FetchedImage {
+  bytes: Uint8Array;
+  /** Lowercased content-type with parameters stripped, or null if absent. */
+  contentType: string | null;
+}
+
+export interface FetchImageBytesOptions {
+  /** Reject (and abort) once the body exceeds this many bytes. */
+  maxBytes: number;
+  /** Abort the whole request after this many milliseconds. */
+  timeoutMs: number;
+  /** Maximum redirect hops to follow (each re-validated). Default 3. */
+  maxRedirects?: number;
+}
+
+/**
+ * Validate that a string is a public http(s) URL safe to fetch server-side.
+ *
+ * Throws {@link ValidationError} for non-http(s) protocols, embedded
+ * credentials, localhost names, and private/loopback/link-local/ULA/CGNAT IP
+ * literals (including the `169.254.169.254` cloud-metadata address).
+ *
+ * @param raw - The candidate URL string
+ * @returns The parsed {@link URL}
+ * @example
+ * ```ts
+ * const url = assertPublicHttpUrl("https://example.com/photo.jpg");
+ * ```
+ */
+export function assertPublicHttpUrl(raw: string): URL {
+  let url: URL;
+  try {
+    url = new URL(raw);
+  } catch {
+    throw new ValidationError("That doesn't look like a valid image URL.");
+  }
+
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new ValidationError("Only http and https image URLs can be fetched.");
+  }
+  if (url.username || url.password) {
+    throw new ValidationError("Image URLs can't include credentials.");
+  }
+
+  const host = url.hostname.toLowerCase();
+  if (
+    host === "localhost" ||
+    host.endsWith(".localhost") ||
+    host === "0.0.0.0" ||
+    isPrivateIpv4(host) ||
+    isPrivateIpv6(host)
+  ) {
+    throw new ValidationError("That image URL points to a private address.");
+  }
+
+  return url;
+}
+
+/**
+ * Fetch a remote image with an SSRF-checked redirect chain, a size cap, and a
+ * timeout. Reads the body in chunks and aborts the moment it exceeds `maxBytes`
+ * so an untrusted host can't exhaust memory.
+ *
+ * @param startUrl - A URL already validated by {@link assertPublicHttpUrl}
+ * @param options - Size cap, timeout, and redirect budget
+ * @returns The raw bytes and the response content-type
+ * @example
+ * ```ts
+ * const { bytes, contentType } = await fetchImageBytes(url, {
+ *   maxBytes: 25 * 1024 * 1024,
+ *   timeoutMs: 15000,
+ * });
+ * ```
+ */
+export async function fetchImageBytes(
+  startUrl: URL,
+  options: FetchImageBytesOptions,
+): Promise<FetchedImage> {
+  const maxRedirects = options.maxRedirects ?? 3;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), options.timeoutMs);
+
+  try {
+    let url = startUrl;
+    let response: Response | null = null;
+
+    for (let hop = 0; hop <= maxRedirects; hop++) {
+      response = await fetch(url.toString(), {
+        method: "GET",
+        redirect: "manual",
+        signal: controller.signal,
+        headers: {
+          Accept: "image/*,*/*;q=0.8",
+          "User-Agent": FETCH_USER_AGENT,
+          // Many CDNs (Douban, WeChat, etc.) reject hotlinked image requests
+          // that lack a Referer. Sending the image's own origin satisfies the
+          // common "referer must be same-site" hotlink check.
+          Referer: `${url.origin}/`,
+        },
+      }).catch((error) => {
+        if (controller.signal.aborted) {
+          throw new ValidationError("Timed out fetching the image.");
+        }
+        throw new ValidationError(
+          error instanceof Error
+            ? `Couldn't fetch the image: ${error.message}`
+            : "Couldn't fetch the image.",
+        );
+      });
+
+      if (response.status >= 300 && response.status < 400) {
+        const location = response.headers.get("location");
+        if (!location) break; // No target — fall through to the (failing) checks.
+        if (hop === maxRedirects) {
+          throw new ValidationError("Too many redirects fetching the image.");
+        }
+        // Re-validate every hop so a redirect can't escape the SSRF guard.
+        url = assertPublicHttpUrl(new URL(location, url).toString());
+        continue;
+      }
+      break;
+    }
+
+    if (!response) {
+      throw new ValidationError("Couldn't fetch the image.");
+    }
+    if (!response.ok) {
+      throw new ValidationError(
+        `Couldn't fetch the image (HTTP ${response.status}).`,
+      );
+    }
+
+    const declared = Number(response.headers.get("content-length"));
+    if (Number.isFinite(declared) && declared > options.maxBytes) {
+      throw new ValidationError("That image is too large.");
+    }
+
+    const contentType = normalizeContentType(
+      response.headers.get("content-type"),
+    );
+    const bytes = await readBounded(response, options.maxBytes);
+    return { bytes, contentType };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function readBounded(
+  response: Response,
+  maxBytes: number,
+): Promise<Uint8Array> {
+  const body = response.body;
+  if (!body) {
+    const buffer = new Uint8Array(await response.arrayBuffer());
+    if (buffer.byteLength > maxBytes) {
+      throw new ValidationError("That image is too large.");
+    }
+    return buffer;
+  }
+
+  const reader = body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (!value) continue;
+    total += value.byteLength;
+    if (total > maxBytes) {
+      await reader.cancel().catch(() => {});
+      throw new ValidationError("That image is too large.");
+    }
+    chunks.push(value);
+  }
+
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return out;
+}
+
+function normalizeContentType(raw: string | null): string | null {
+  if (!raw) return null;
+  const type = raw.split(";")[0]?.trim().toLowerCase();
+  return type || null;
+}
+
+/**
+ * True when a canonicalized IPv4 dotted-quad host is in a private, loopback,
+ * link-local, CGNAT, or otherwise non-public range. The WHATWG URL parser
+ * already normalizes decimal/octal/hex IPv4 forms to dotted-quad, so checking
+ * `url.hostname` is sufficient.
+ */
+function isPrivateIpv4(host: string): boolean {
+  const match = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
+  if (!match) return false;
+  const octets = match.slice(1, 5).map(Number);
+  if (octets.some((part) => part > 255)) return true; // Malformed → unsafe.
+  const a = octets[0] ?? 0;
+  const b = octets[1] ?? 0;
+  if (a === 0) return true; // 0.0.0.0/8 "this" network
+  if (a === 10) return true; // 10/8 private
+  if (a === 127) return true; // 127/8 loopback
+  if (a === 169 && b === 254) return true; // 169.254/16 link-local (metadata)
+  if (a === 172 && b >= 16 && b <= 31) return true; // 172.16/12 private
+  if (a === 192 && b === 168) return true; // 192.168/16 private
+  if (a === 100 && b >= 64 && b <= 127) return true; // 100.64/10 CGNAT
+  if (a === 255 && b === 255) return true; // broadcast
+  return false;
+}
+
+/** True when an IPv6 host (bracketed or bare) is loopback/link-local/ULA/mapped-private. */
+function isPrivateIpv6(host: string): boolean {
+  let inner = host;
+  if (inner.startsWith("[") && inner.endsWith("]")) {
+    inner = inner.slice(1, -1);
+  }
+  if (!inner.includes(":")) return false;
+  const lower = inner.toLowerCase();
+  if (lower === "::" || lower === "::1") return true; // unspecified / loopback
+
+  // IPv4-mapped, dotted form (::ffff:1.2.3.4).
+  const dotted = lower.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
+  if (dotted?.[1]) return isPrivateIpv4(dotted[1]);
+
+  // IPv4-mapped, hex form (::ffff:a00:1) — what the URL parser normalizes to.
+  const hex = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
+  if (hex) {
+    const hi = parseInt(hex[1] ?? "0", 16);
+    const lo = parseInt(hex[2] ?? "0", 16);
+    const v4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
+    return isPrivateIpv4(v4);
+  }
+
+  if (/^fe[89ab]/.test(lower)) return true; // fe80::/10 link-local
+  if (/^f[cd]/.test(lower)) return true; // fc00::/7 unique local
+  return false;
+}

package/src/routes/api/__tests__/uploads.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { createTestApp } from "../../../__tests__/helpers/app.js";
 import type {
   PresignedPutOptions,
@@ -136,6 +136,68 @@ function createMockStorage(options?: {
   };
 }
 
+describe("POST /api/uploads/sideload", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  function pngBytes(): Uint8Array {
+    return new Uint8Array([
+      0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
+      0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06,
+      0x08, 0x06, 0x00, 0x00, 0x00,
+    ]);
+  }
+
+  it("rehosts a remote image and returns its stored URL", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(pngBytes(), {
+            headers: { "content-type": "image/png" },
+          }),
+      ),
+    );
+    const storage = createMockStorage();
+    const { app, services } = createTestApp({ authenticated: true, storage });
+    app.route("/api/uploads", uploadsApiRoutes);
+
+    const res = await app.request("/api/uploads/sideload", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/photo.png" }),
+    });
+
+    expect(res.status).toBe(200);
+    const data = (await res.json()) as {
+      id: string;
+      url: string;
+      mimeType: string;
+      width: number;
+      height: number;
+    };
+    expect(data.mimeType).toBe("image/png");
+    expect(data.width).toBe(4);
+    expect(data.height).toBe(6);
+    const media = await services.media.getById(data.id);
+    expect(media?.mimeType).toBe("image/png");
+  });
+
+  it("requires authentication", async () => {
+    const { app } = createTestApp({ authenticated: false });
+    app.route("/api/uploads", uploadsApiRoutes);
+
+    const res = await app.request("/api/uploads/sideload", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com/photo.png" }),
+    });
+
+    expect(res.status).toBe(401);
+  });
+});
+
 describe("Upload Session API Routes", () => {
   it("completes a relay image upload and stores inline media metadata", async () => {
     const storage = createMockStorage();

package/src/routes/api/uploads.ts

@@ -35,10 +35,62 @@ const CompleteUploadSchema = z.object({
     .optional(),
 });
 
+const SideloadSchema = z.object({
+  url: z.string().url(),
+  alt: z.string().max(2000).optional(),
+});
+
 export const uploadsApiRoutes = new Hono<Env>();
 
 uploadsApiRoutes.use("*", requireAuthApi());
 
+/**
+ * Rehost a remote image into the site's own storage. Used when an author pastes
+ * an article whose `<img>` tags point at external URLs — the server fetches the
+ * bytes (bypassing browser CORS) and stores them. Returns the new media's
+ * public URL so the editor can swap the node's `src`.
+ */
+uploadsApiRoutes.post("/sideload", async (c) => {
+  const storage = c.var.storage;
+  if (!storage) {
+    return c.json(
+      { error: "File storage isn't set up. Check your server config." },
+      500,
+    );
+  }
+
+  const { url, alt } = parseValidated(SideloadSchema, await c.req.json());
+
+  const media = await c.var.services.media.ingestFromUrl(
+    { url, alt },
+    {
+      storage,
+      storageDriver: c.var.appConfig.storageDriver,
+      maxFileSizeMB: c.var.appConfig.uploadMaxFileSize,
+    },
+  );
+
+  const mediaPublicUrl = getPublicUrlForProvider(
+    c.var.appConfig.storageDriver,
+    c.var.appConfig.r2PublicUrl,
+    c.var.appConfig.s3PublicUrl,
+    c.var.appConfig.localPublicUrl,
+  );
+
+  return c.json({
+    id: media.id,
+    url: getMediaUrl(
+      media.storageKey,
+      mediaPublicUrl,
+      c.var.appConfig.sitePathPrefix,
+    ),
+    width: media.width,
+    height: media.height,
+    mimeType: media.mimeType,
+    size: media.size,
+  });
+});
+
 function scheduleExpiredUploadCleanup(
   c: {
     executionCtx?: { waitUntil: (promise: Promise<unknown>) => void };

package/src/services/__tests__/media.test.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion -- Test assertions use ! for readability */
-import { describe, it, expect, beforeEach, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
 import { eq } from "drizzle-orm";
 import {
   createTestDatabase,
@@ -1546,3 +1546,170 @@ describe("MediaService", () => {
     });
   });
 });
+
+/** Minimal valid PNG header (signature + IHDR with width 4, height 6). */
+function createPngBytes(): Uint8Array {
+  return new Uint8Array([
+    0x89,
+    0x50,
+    0x4e,
+    0x47,
+    0x0d,
+    0x0a,
+    0x1a,
+    0x0a, // signature
+    0x00,
+    0x00,
+    0x00,
+    0x0d, // IHDR length (13)
+    0x49,
+    0x48,
+    0x44,
+    0x52, // "IHDR"
+    0x00,
+    0x00,
+    0x00,
+    0x04, // width = 4
+    0x00,
+    0x00,
+    0x00,
+    0x06, // height = 6
+    0x08,
+    0x06,
+    0x00,
+    0x00,
+    0x00, // bit depth, color type, ...
+  ]);
+}
+
+describe("MediaService.ingestFromUrl", () => {
+  let db: Database;
+  let mediaService: ReturnType<typeof createMediaService>;
+
+  beforeEach(() => {
+    const testDb = createTestDatabase();
+    db = testDb.db as unknown as Database;
+    mediaService = createMediaService(db, DEFAULT_TEST_SITE_ID);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  const deps = () => ({
+    storage: createMockStorage(),
+    storageDriver: "local",
+    maxFileSizeMB: 25,
+  });
+
+  it("fetches a remote image, stores it, and creates a media row", async () => {
+    const bytes = createPngBytes();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(bytes, { headers: { "content-type": "image/png" } }),
+      ),
+    );
+
+    const d = deps();
+    const media = await mediaService.ingestFromUrl(
+      { url: "https://example.com/photo.png", alt: "A photo" },
+      d,
+    );
+
+    expect(media.mimeType).toBe("image/png");
+    expect(media.mediaKind).toBe("image");
+    expect(media.width).toBe(4);
+    expect(media.height).toBe(6);
+    expect(media.alt).toBe("A photo");
+    expect(media.postId).toBeNull();
+
+    const stored = d.storage.files.get(media.storageKey);
+    expect(stored).toBeDefined();
+    expect(stored?.contentType).toBe("image/png");
+    expect(stored?.contentDisposition).toBe("inline");
+  });
+
+  it("stores SVG with attachment disposition (XSS-safe direct navigation)", async () => {
+    const svg = new TextEncoder().encode(
+      '<svg xmlns="http://www.w3.org/2000/svg"></svg>',
+    );
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(svg, {
+            headers: { "content-type": "image/svg+xml" },
+          }),
+      ),
+    );
+
+    const d = deps();
+    const media = await mediaService.ingestFromUrl(
+      { url: "https://example.com/icon.svg" },
+      d,
+    );
+
+    expect(media.mimeType).toBe("image/svg+xml");
+    const stored = d.storage.files.get(media.storageKey);
+    expect(stored?.contentDisposition).toBe("attachment");
+  });
+
+  it("rejects content that isn't a real image (content-type spoofing)", async () => {
+    const html = new TextEncoder().encode(
+      "<!doctype html><script>alert(1)</script>",
+    );
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        // Server lies: claims PNG, returns HTML.
+        async () =>
+          new Response(html, { headers: { "content-type": "image/png" } }),
+      ),
+    );
+
+    await expect(
+      mediaService.ingestFromUrl(
+        { url: "https://example.com/evil.png" },
+        deps(),
+      ),
+    ).rejects.toThrow(/supported image/i);
+  });
+
+  it("rejects an oversize image", async () => {
+    const stream = new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(new Uint8Array(2 * 1024 * 1024));
+        controller.close();
+      },
+    });
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(stream, { headers: { "content-type": "image/png" } }),
+      ),
+    );
+
+    await expect(
+      mediaService.ingestFromUrl(
+        { url: "https://example.com/huge.png" },
+        { ...deps(), maxFileSizeMB: 1 },
+      ),
+    ).rejects.toThrow(/too large/i);
+  });
+
+  it("rejects a private/SSRF URL before fetching", async () => {
+    const fetchMock = vi.fn();
+    vi.stubGlobal("fetch", fetchMock);
+
+    await expect(
+      mediaService.ingestFromUrl(
+        { url: "http://169.254.169.254/latest/meta-data" },
+        deps(),
+      ),
+    ).rejects.toThrow(/private address/i);
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+});