@bygghammar/cms-platform 0.94.0 → 0.96.0

package/src/server/media-usage-core.ts

@@ -10,6 +10,15 @@
  */
 import { variantSiblings } from "../media/srcset";
 import { cleanFileName, splitExt, uniqueFileName } from "../media/filename";
+import {
+  MEDIA_IMPORT_POLICY,
+  redactUrl,
+  safeRequest,
+  sniffMedia,
+  type MediaKind,
+  type Resolver,
+  type Transport,
+} from "./outbound-http";
 import type { MediaImportResult, MediaUsage, MediaUsageReport } from "../media/usage-types";
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
@@ -42,6 +51,8 @@ export type MediaImportHost = MediaUsageHost & {
   admin: any;
   /** User-Agent sent when fetching remote media. */
   userAgent?: string;
+  /** Test seams: a fake resolver and transport instead of the network. */
+  network?: { resolver?: Resolver; transport?: Transport };
 };
 
 const URL_RE = /https?:\/\/[^\s"'<>)]+/gi;
@@ -154,28 +165,55 @@ export function remoteUrlCandidates(rawUrl: string): string[] {
   return Array.from(candidates);
 }
 
-async function fetchRemoteMedia(rawUrl: string, userAgent: string) {
+/**
+ * Download one remote media file, under the shared outbound policy: public
+ * addresses only (looked up, pinned, every redirect hop checked), one
+ * deadline, a capped size. What it is comes from its first bytes — never from
+ * the server's content type or the address's extension.
+ */
+async function fetchRemoteMedia(
+  rawUrl: string,
+  userAgent: string,
+  network: MediaImportHost["network"],
+): Promise<{ bytes: Uint8Array; kind: MediaKind; resolvedUrl: string } | { error: string }> {
   let lastStatus: number | null = null;
   let lastError: string | null = null;
 
   const headers = {
     "User-Agent": userAgent,
-    Accept:
-      "image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,application/pdf,*/*;q=0.8",
+    Accept: "image/avif,image/webp,image/apng,image/*,video/*,audio/*,application/pdf;q=0.9",
   };
 
   for (const url of remoteUrlCandidates(rawUrl)) {
-    try {
-      const response = await fetch(url, { headers, redirect: "follow" });
-      if (response.ok) return { response, resolvedUrl: url };
-      lastStatus = response.status;
-    } catch (error) {
-      lastError = error instanceof Error ? error.message : "Network error";
+    const res = await safeRequest({
+      url,
+      method: "GET",
+      headers,
+      policy: MEDIA_IMPORT_POLICY,
+      resolver: network?.resolver,
+      transport: network?.transport,
+    });
+    if (!res.ok) {
+      // Every candidate is a spelling of the same address: a refused one is refused for all.
+      if (res.failure === "blocked-url" || res.failure === "blocked-address") {
+        return { error: `${res.message} (${redactUrl(rawUrl)})` };
+      }
+      lastError = res.message;
+      continue;
+    }
+    if (res.status < 200 || res.status >= 300) {
+      lastStatus = res.status;
+      continue;
+    }
+    const kind = sniffMedia(res.body);
+    if (!kind) {
+      return { error: `Not an image, video, audio or PDF file (${redactUrl(rawUrl)}).` };
     }
+    return { bytes: res.body, kind, resolvedUrl: url };
   }
 
   const reason = lastStatus ? `Fetch failed: ${lastStatus}` : (lastError ?? "Fetch failed");
-  return { error: `${reason} for ${rawUrl}` };
+  return { error: `${reason} (${redactUrl(rawUrl)})` };
 }
 
 /** Scan every CMS content column and report which library files are used where. */
@@ -417,19 +455,21 @@ export async function importRemoteUrlCore(
   const remote = await fetchRemoteMedia(
     data.url,
     host.userAgent ?? "Mozilla/5.0 (compatible; CmsMediaImporter/1.0)",
+    host.network,
   );
   if ("error" in remote) {
     return { path: "", url: "", skipped: true, error: remote.error ?? "Fetch failed" };
   }
 
-  const contentType = remote.response.headers.get("content-type") ?? "application/octet-stream";
-  const buffer = new Uint8Array(await remote.response.arrayBuffer());
+  // Stored as what the bytes are: their sniffed type and its extension.
+  const contentType = remote.kind.mime;
+  const buffer = remote.bytes;
   const urlObj = new URL(remote.resolvedUrl);
   const rawName =
     data.filename ||
     urlObj.pathname.split("/").filter(Boolean).map(safeDecodeURIComponent).pop() ||
     "imported-file";
-  const clean = cleanFileName(rawName, "imported-file");
+  const clean = `${splitExt(cleanFileName(rawName.replace(/[\\/]/g, "-"), "imported-file")).base}${remote.kind.ext}`;
 
   const all = await host.storage.list();
   const existing = all

package/src/server/outbound-address.ts

@@ -0,0 +1,131 @@
+/**
+ * Which network addresses a server-side request may connect to.
+ *
+ * Only globally routable unicast addresses. Everything that names this
+ * machine, its private network, a cloud metadata service, a multicast group
+ * or a reserved block is refused — in IPv4 and IPv6, and in every IPv6 form
+ * that carries an IPv4 address inside it (mapped, compatible, NAT64, 6to4).
+ *
+ * Pure and runtime-independent: it only reads address text.
+ */
+
+/** IPv4 blocks that are not public destinations: [network, prefix length]. */
+const BLOCKED_V4: ReadonlyArray<readonly [string, number]> = [
+  ["0.0.0.0", 8], // "this network"
+  ["10.0.0.0", 8], // private
+  ["100.64.0.0", 10], // carrier-grade NAT
+  ["127.0.0.0", 8], // loopback
+  ["169.254.0.0", 16], // link-local, incl. cloud metadata 169.254.169.254
+  ["172.16.0.0", 12], // private
+  ["192.0.0.0", 24], // IETF protocol assignments
+  ["192.0.2.0", 24], // documentation
+  ["192.88.99.0", 24], // 6to4 relay anycast
+  ["192.168.0.0", 16], // private
+  ["198.18.0.0", 15], // benchmarking
+  ["198.51.100.0", 24], // documentation
+  ["203.0.113.0", 24], // documentation
+  ["224.0.0.0", 4], // multicast
+  ["240.0.0.0", 4], // reserved, incl. broadcast
+];
+
+/** IPv6 blocks that are not public destinations. */
+const BLOCKED_V6: ReadonlyArray<readonly [string, number]> = [
+  ["::", 128], // unspecified
+  ["::1", 128], // loopback
+  ["::ffff:0:0", 96], // IPv4-mapped
+  ["::", 96], // IPv4-compatible (deprecated)
+  ["64:ff9b::", 96], // NAT64
+  ["64:ff9b:1::", 48], // local-use NAT64
+  ["100::", 64], // discard
+  ["2001::", 23], // IETF protocol assignments, incl. Teredo
+  ["2001:db8::", 32], // documentation
+  ["2002::", 16], // 6to4
+  ["3fff::", 20], // documentation
+  ["5f00::", 16], // segment routing
+  ["fc00::", 7], // unique local, incl. fd00:ec2::254 (cloud metadata)
+  ["fe80::", 10], // link-local
+  ["fec0::", 10], // site-local (deprecated)
+  ["ff00::", 8], // multicast
+];
+
+/** An IPv4 address as a 32-bit number, or null. Dotted decimal only (as DNS and URL produce). */
+export function parseIPv4(text: string): number | null {
+  const parts = text.split(".");
+  if (parts.length !== 4) return null;
+  let value = 0;
+  for (const part of parts) {
+    if (!/^(0|[1-9][0-9]{0,2})$/.test(part)) return null;
+    const octet = Number(part);
+    if (octet > 255) return null;
+    value = value * 256 + octet;
+  }
+  return value;
+}
+
+/** An IPv6 address as a 128-bit number, or null. Zone ids are refused. */
+export function parseIPv6(text: string): bigint | null {
+  let address = text.trim().toLowerCase();
+  if (address.startsWith("[") && address.endsWith("]")) address = address.slice(1, -1);
+  if (address === "" || address.includes("%")) return null;
+
+  // An IPv4 tail ("::ffff:1.2.3.4") becomes two hex groups.
+  const lastColon = address.lastIndexOf(":");
+  const tail = address.slice(lastColon + 1);
+  if (tail.includes(".")) {
+    const v4 = parseIPv4(tail);
+    if (v4 === null) return null;
+    address = `${address.slice(0, lastColon + 1)}${(v4 >>> 16).toString(16)}:${(v4 & 0xffff).toString(16)}`;
+  }
+
+  const halves = address.split("::");
+  if (halves.length > 2) return null;
+  const parse = (half: string) => (half === "" ? [] : half.split(":"));
+  const head = parse(halves[0] ?? "");
+  const rest = halves.length === 2 ? parse(halves[1] ?? "") : [];
+  const zeros = 8 - head.length - rest.length;
+  // "::" stands for at least one group of zeros; without it there are exactly eight groups.
+  if (halves.length === 2 ? zeros < 1 : zeros !== 0) return null;
+  const groups = [...head, ...Array<string>(halves.length === 2 ? zeros : 0).fill("0"), ...rest];
+
+  let value = 0n;
+  for (const group of groups) {
+    if (!/^[0-9a-f]{1,4}$/.test(group)) return null;
+    value = (value << 16n) | BigInt(Number.parseInt(group, 16));
+  }
+  return value;
+}
+
+function inV4Block(address: number, [network, prefix]: readonly [string, number]): boolean {
+  const base = parseIPv4(network)!;
+  if (prefix === 0) return true;
+  const shift = 32 - prefix;
+  return Math.floor(address / 2 ** shift) === Math.floor(base / 2 ** shift);
+}
+
+function inV6Block(address: bigint, [network, prefix]: readonly [string, number]): boolean {
+  const base = parseIPv6(network)!;
+  const shift = BigInt(128 - prefix);
+  return address >> shift === base >> shift;
+}
+
+export type AddressFamily = 4 | 6;
+
+/** The family of an address literal, or null when it is not one. */
+export function addressFamily(text: string): AddressFamily | null {
+  if (parseIPv4(text) !== null) return 4;
+  if (parseIPv6(text) !== null) return 6;
+  return null;
+}
+
+/** True only for a globally routable unicast address. Anything unparseable is refused. */
+export function isPublicAddress(text: string): boolean {
+  const v4 = parseIPv4(text);
+  if (v4 !== null) return !BLOCKED_V4.some((block) => inV4Block(v4, block));
+  const v6 = parseIPv6(text);
+  if (v6 !== null) {
+    if (BLOCKED_V6.some((block) => inV6Block(v6, block))) return false;
+    // Only the global unicast range 2000::/3 is public.
+    return v6 >> 125n === 1n;
+  }
+  return false;
+}