@terminus-ai/cli 0.0.1

package/bin/dev-net.mjs

@@ -0,0 +1,316 @@
+/**
+ * `net.fetch` and `net.image` under `terminus dev` — and a server op's
+ * `webFetch`, which the platform makes through the same fetch — guarded the
+ * way the platform guards them (terminus-backend src/routes/net.rs): public
+ * http(s) on the standard ports only, every resolved address of every hop
+ * checked and the connection pinned to one, at most three redirects, ten
+ * seconds in all. A page is text, XML, JSON or calendar data of at most
+ * 512 KiB — refused past that unless the call asks for its start (`partial`)
+ * — and a picture is a PNG, JPEG, GIF, WebP or AVIF by its own bytes, at most
+ * 5 MiB, served under the type the bytes say. Refusals carry the contract's
+ * codes.
+ *
+ * A developer's own site on localhost is refused here exactly as the
+ * platform refuses it. Tests hand in `fetchImpl` (the WHATWG fetch shape) to
+ * stand a local server in for a public one; the URL rules still apply.
+ */
+
+import dns from "node:dns/promises";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { isIP } from "node:net";
+
+import { LIMITS, runtimeError } from "./dev-contract.mjs";
+
+const FETCH_TIMEOUT_MS = 10_000;
+const MAX_REDIRECTS = 3;
+const USER_AGENT = "terminus-app-fetch/1.0";
+
+function ipv4Private(address) {
+  const [a, b, c] = address.split(".").map(Number);
+  return a === 0 || a >= 224 || a === 10 || a === 127
+    || (a === 169 && b === 254)
+    || (a === 172 && b >= 16 && b <= 31)
+    || (a === 192 && b === 168)
+    || (a === 192 && b === 0 && c === 0)
+    || (a === 192 && b === 0 && c === 2)
+    || (a === 198 && (b === 18 || b === 19))
+    || (a === 198 && b === 51 && c === 100)
+    || (a === 203 && b === 0 && c === 113)
+    || (a === 100 && (b & 0xc0) === 64);
+}
+
+function ipv6Groups(address) {
+  let text = address.toLowerCase();
+  const zone = text.indexOf("%");
+  if (zone !== -1) text = text.slice(0, zone);
+  let tail = [];
+  const dotted = /(\d+\.\d+\.\d+\.\d+)$/.exec(text);
+  if (dotted) {
+    const parts = dotted[1].split(".").map(Number);
+    tail = [(parts[0] << 8) | parts[1], (parts[2] << 8) | parts[3]];
+    text = text.slice(0, -dotted[1].length).replace(/:$/, "") || ":";
+    if (text === ":") text = "::";
+  }
+  const [head, rest] = text.includes("::") ? text.split("::") : [text, null];
+  const headGroups = head ? head.split(":").filter(Boolean).map((group) => parseInt(group, 16)) : [];
+  const restGroups = rest ? rest.split(":").filter(Boolean).map((group) => parseInt(group, 16)) : [];
+  const fill = 8 - headGroups.length - restGroups.length - tail.length;
+  return [...headGroups, ...(rest === null ? [] : new Array(Math.max(0, fill)).fill(0)), ...restGroups, ...tail];
+}
+
+/** `is_private_ip` (net.rs): anything a public fetch must never reach. */
+export function isPrivateAddress(address) {
+  const family = isIP(address);
+  if (family === 4) return ipv4Private(address);
+  if (family !== 6) return true;
+  const groups = ipv6Groups(address);
+  const mapped = groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff;
+  if (mapped) {
+    return ipv4Private(`${groups[6] >> 8}.${groups[6] & 0xff}.${groups[7] >> 8}.${groups[7] & 0xff}`);
+  }
+  return (groups[0] & 0xe000) !== 0x2000
+    || groups[0] === 0x2002
+    || (groups[0] === 0x2001 && (groups[1] < 0x0200 || groups[1] === 0x0db8))
+    || groups.every((group) => group === 0)
+    || (groups.slice(0, 7).every((group) => group === 0) && groups[7] === 1)
+    || (groups[0] & 0xfe00) === 0xfc00
+    || (groups[0] & 0xffc0) === 0xfe80;
+}
+
+function parseUrl(raw) {
+  try {
+    return new URL(String(raw ?? "").trim());
+  } catch {
+    throw runtimeError("bad_request", "invalid url");
+  }
+}
+
+function shapeAllowed(url) {
+  if (url.username || url.password) return false;
+  if (url.protocol !== "http:" && url.protocol !== "https:") return false;
+  if (url.port && url.port !== "80" && url.port !== "443") return false;
+  const host = url.hostname.replace(/^\[|\]$/gu, "").toLowerCase();
+  if (!host) return false;
+  if (isIP(host)) return !isPrivateAddress(host);
+  return host !== "localhost" && !host.endsWith(".localhost");
+}
+
+function requireShape(url) {
+  if (!shapeAllowed(url)) {
+    throw runtimeError("url_not_allowed", "only public http(s) URLs on standard ports can be fetched");
+  }
+}
+
+async function pinnedAddress(url) {
+  const host = url.hostname.replace(/^\[|\]$/gu, "");
+  if (isIP(host)) return null;
+  let addresses;
+  try {
+    addresses = await dns.lookup(host, { all: true, verbatim: true });
+  } catch {
+    throw runtimeError("upstream_failed", "the host could not be resolved");
+  }
+  if (!addresses.length) throw runtimeError("upstream_failed", "the host did not resolve to an address");
+  if (addresses.some((entry) => isPrivateAddress(entry.address))) {
+    throw runtimeError("url_not_allowed", "the host resolves to a private address");
+  }
+  return addresses[0];
+}
+
+function getPinned(url, pinned, signal, userAgent) {
+  return new Promise((resolve, reject) => {
+    const send = url.protocol === "https:" ? httpsRequest : httpRequest;
+    const outgoing = send(url, {
+      method: "GET",
+      headers: { "user-agent": userAgent },
+      signal,
+      ...(pinned
+        ? { lookup: (_host, options, done) => (options?.all
+          ? done(null, [{ address: pinned.address, family: pinned.family }])
+          : done(null, pinned.address, pinned.family)) }
+        : {}),
+    }, resolve);
+    outgoing.on("error", reject);
+    outgoing.end();
+  });
+}
+
+/** One answer, in one shape whichever way it was fetched. */
+function fromIncoming(url, incoming) {
+  return {
+    url: url.href,
+    status: incoming.statusCode ?? 0,
+    header: (name) => {
+      const value = incoming.headers[name];
+      return Array.isArray(value) ? value.join(", ") : (value ?? null);
+    },
+    chunks: incoming,
+    discard: () => incoming.resume(),
+  };
+}
+
+function fromResponse(url, response) {
+  return {
+    url: response.url || url.href,
+    status: response.status,
+    header: (name) => response.headers.get(name),
+    chunks: response.body ?? [],
+    discard: () => response.body?.cancel().catch(() => undefined),
+  };
+}
+
+/** Follow the public hops to the answer that is not a redirect. */
+async function openPublic(rawUrl, { fetchImpl, signal, userAgent }) {
+  let url = parseUrl(rawUrl);
+  for (let redirects = 0; ; redirects += 1) {
+    requireShape(url);
+    let answer;
+    try {
+      answer = fetchImpl
+        ? fromResponse(url, await fetchImpl(url.href, { redirect: "manual", signal, headers: { "user-agent": userAgent } }))
+        : fromIncoming(url, await getPinned(url, await pinnedAddress(url), signal, userAgent));
+    } catch (error) {
+      if (error?.apiCode) throw error;
+      throw runtimeError("upstream_failed", `fetch failed: ${error?.message ?? error}`);
+    }
+    if (![301, 302, 303, 307, 308].includes(answer.status)) return answer;
+    answer.discard();
+    if (redirects === MAX_REDIRECTS) throw runtimeError("upstream_failed", "too many redirects");
+    const location = answer.header("location");
+    if (!location) throw runtimeError("upstream_failed", "redirect response omitted Location");
+    try {
+      url = new URL(location, url);
+    } catch {
+      throw runtimeError("upstream_failed", "redirect contained an invalid URL");
+    }
+  }
+}
+
+/** A fetch within its deadline — ten seconds for all of it: the lookups,
+ * every hop and every byte. At the deadline it is dropped wherever it was
+ * (a lookup that cannot be cancelled included) and answers
+ * deadline_exceeded in the platform's words, whatever the drop broke. */
+async function withDeadline(task, timeoutMs = FETCH_TIMEOUT_MS) {
+  const controller = new AbortController();
+  const expired = runtimeError("deadline_exceeded", `the fetch did not finish within ${timeoutMs / 1000} seconds`);
+  let timer;
+  const deadline = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      controller.abort(expired);
+      reject(expired);
+    }, timeoutMs);
+  });
+  try {
+    return await Promise.race([task(controller.signal), deadline]);
+  } catch (error) {
+    throw controller.signal.aborted ? expired : error;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function readCapped(answer, cap, { partial = false, tooLarge }) {
+  const chunks = [];
+  let size = 0;
+  let truncated = false;
+  try {
+    for await (const raw of answer.chunks) {
+      const chunk = Buffer.from(raw);
+      if (size + chunk.length > cap) {
+        if (!partial) {
+          answer.discard();
+          throw runtimeError("payload_too_large", tooLarge);
+        }
+        chunks.push(chunk.subarray(0, cap - size));
+        truncated = true;
+        answer.discard();
+        break;
+      }
+      size += chunk.length;
+      chunks.push(chunk);
+    }
+  } catch (error) {
+    if (error?.apiCode) throw error;
+    throw runtimeError("upstream_failed", `fetch failed: ${error?.message ?? error}`);
+  }
+  return { bytes: Buffer.concat(chunks), truncated };
+}
+
+/** `GET /net/fetch`: a public text resource → `{url, status, content_type,
+ * body, truncated}`. `timeoutMs` is the door's deadline (tests shorten it);
+ * `userAgent` names the lane asking (server ops fetch through here too). */
+export function fetchPublicPage(rawUrl, {
+  partial = false,
+  fetchImpl = null,
+  timeoutMs = FETCH_TIMEOUT_MS,
+  userAgent = USER_AGENT,
+} = {}) {
+  return withDeadline(async (signal) => {
+    const answer = await openPublic(rawUrl, { fetchImpl, signal, userAgent });
+    const contentType = answer.header("content-type") ?? "";
+    const textShaped = !contentType
+      || contentType.startsWith("text/")
+      || contentType.includes("xml")
+      || contentType.includes("json")
+      || contentType.includes("calendar");
+    if (!textShaped) {
+      answer.discard();
+      throw runtimeError("upstream_failed", `unsupported content type '${contentType}'`);
+    }
+    const { bytes, truncated } = await readCapped(answer, LIMITS.net_fetch_bytes, {
+      partial,
+      tooLarge: "the response is larger than 512 KiB",
+    });
+    return {
+      url: answer.url,
+      status: answer.status,
+      content_type: contentType,
+      body: bytes.toString("utf8"),
+      truncated,
+    };
+  }, timeoutMs);
+}
+
+/** What a body is by its first bytes: only these are served as a picture. */
+export function sniffImageType(bytes) {
+  const starts = (...signature) => signature.every((byte, index) => bytes[index] === byte);
+  if (starts(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return "image/png";
+  if (starts(0xff, 0xd8, 0xff)) return "image/jpeg";
+  const head = bytes.subarray(0, 12).toString("latin1");
+  if (head.startsWith("GIF87a") || head.startsWith("GIF89a")) return "image/gif";
+  if (head.startsWith("RIFF") && head.slice(8, 12) === "WEBP") return "image/webp";
+  if (head.slice(4, 8) === "ftyp" && ["avif", "avis"].includes(head.slice(8, 12))) return "image/avif";
+  return null;
+}
+
+/** `GET /net/image`: a public raster image → `{url, content_type, bytes}`. */
+export function fetchPublicImage(rawUrl, { fetchImpl = null, timeoutMs = FETCH_TIMEOUT_MS } = {}) {
+  return withDeadline(async (signal) => {
+    const answer = await openPublic(rawUrl, { fetchImpl, signal, userAgent: USER_AGENT });
+    if (answer.status < 200 || answer.status > 299) {
+      answer.discard();
+      throw runtimeError("upstream_failed", `the image could not be read (HTTP ${answer.status})`);
+    }
+    const declared = (answer.header("content-type") ?? "").toLowerCase();
+    if (declared.startsWith("text/") || declared.includes("json") || declared.includes("xml")) {
+      answer.discard();
+      throw runtimeError("upstream_failed", `unsupported content type '${declared}'`);
+    }
+    const tooLarge = "the image is larger than 5 MiB";
+    const length = Number(answer.header("content-length"));
+    if (Number.isFinite(length) && length > LIMITS.net_image_bytes) {
+      answer.discard();
+      throw runtimeError("payload_too_large", tooLarge);
+    }
+    const { bytes } = await readCapped(answer, LIMITS.net_image_bytes, { tooLarge });
+    const contentType = sniffImageType(bytes);
+    if (!contentType) {
+      throw runtimeError(
+        "upstream_failed",
+        "only public raster images (PNG, JPEG, GIF, WebP, AVIF) can be fetched",
+      );
+    }
+    return { url: answer.url, content_type: contentType, bytes };
+  }, timeoutMs);
+}