@opengeni/api-router 3.0.1 → 3.0.2-canary.38894290271001

package/src/site-publishing/content-scanner.ts

@@ -0,0 +1,93 @@
+import type { Settings } from "@opengeni/config";
+
+/**
+ * Pluggable content scanning for published Sites. `none` is the default; the
+ * Google Web Risk scanner runs only with OPENGENI_SITES_WEB_RISK_API_KEY. A
+ * scan looks up the absolute URLs found in a Site's HTML when it is published
+ * and records findings for moderation. It never blocks publishing, and a
+ * disabled or failing scanner records nothing that blocks either.
+ */
+
+export type SiteContentScanFinding = { url: string; threatTypes: string[] };
+export type SiteContentScanResult = {
+  status: "none" | "clean" | "flagged" | "error";
+  findings: SiteContentScanFinding[];
+};
+
+export type SiteContentScanner = {
+  readonly name: "none" | "web_risk";
+  scanHtml(html: string): Promise<SiteContentScanResult>;
+};
+
+/** Most distinct URLs one scan looks up (bounds cost per publish). */
+export const SITE_SCAN_MAX_URLS = 20;
+
+/** Distinct absolute http(s) URLs in the HTML, first seen first, at most `limit`. */
+export function siteHtmlUrls(html: string, limit = SITE_SCAN_MAX_URLS): string[] {
+  const found = new Set<string>();
+  for (const match of html.matchAll(/https?:\/\/[^\s"'<>`()\\]{1,2000}/giu)) {
+    let url: URL;
+    try {
+      url = new URL(match[0]);
+    } catch {
+      continue;
+    }
+    found.add(url.toString());
+    if (found.size >= limit) break;
+  }
+  return [...found];
+}
+
+export const noneSiteContentScanner: SiteContentScanner = Object.freeze({
+  name: "none" as const,
+  scanHtml: async () => ({ status: "none" as const, findings: [] }),
+});
+
+const WEB_RISK_THREATS = ["MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE"];
+
+export function createWebRiskSiteContentScanner(
+  apiKey: string,
+  options: { fetch?: typeof fetch } = {},
+): SiteContentScanner {
+  const fetchImpl = options.fetch ?? fetch;
+  return {
+    name: "web_risk",
+    async scanHtml(html) {
+      const urls = siteHtmlUrls(html);
+      if (urls.length === 0) return { status: "clean", findings: [] };
+      const findings: SiteContentScanFinding[] = [];
+      try {
+        for (const url of urls) {
+          const query = new URLSearchParams();
+          for (const threat of WEB_RISK_THREATS) query.append("threatTypes", threat);
+          query.set("uri", url);
+          query.set("key", apiKey);
+          const response = await fetchImpl(
+            `https://webrisk.googleapis.com/v1/uris:search?${query.toString()}`,
+            { method: "GET" },
+          );
+          if (!response.ok) throw new Error(`Web Risk lookup failed with ${response.status}`);
+          const body = (await response.json().catch(() => ({}))) as {
+            threat?: { threatTypes?: unknown };
+          };
+          const types = Array.isArray(body.threat?.threatTypes)
+            ? body.threat.threatTypes.filter((type): type is string => typeof type === "string")
+            : [];
+          if (types.length > 0) findings.push({ url, threatTypes: types });
+        }
+      } catch {
+        return { status: "error", findings };
+      }
+      return { status: findings.length > 0 ? "flagged" : "clean", findings };
+    },
+  };
+}
+
+export function createSiteContentScanner(
+  settings: Pick<Settings, "sitesWebRiskApiKey">,
+  options: { fetch?: typeof fetch } = {},
+): SiteContentScanner {
+  return settings.sitesWebRiskApiKey
+    ? createWebRiskSiteContentScanner(settings.sitesWebRiskApiKey, options)
+    : noneSiteContentScanner;
+}

package/src/site-publishing/edge-sync.ts

@@ -0,0 +1,148 @@
+import { SiteHostMapRecord, siteHostMapKey } from "@opengeni/contracts";
+import type { Settings } from "@opengeni/config";
+
+/**
+ * Optional Cloudflare edge sync for published Sites. Postgres is the source
+ * of truth; the KV host map and the edge cache are rebuildable copies. With
+ * no Cloudflare settings every call is a no-op.
+ *
+ * Writes go through the Cloudflare REST API with plain fetch (no SDK, each
+ * call bounded by a 10 s timeout) and never block the database write: callers
+ * schedule them after commit, each is retried a few times and then logged.
+ * The caller reads what a hostname should hold in a short transaction under
+ * the hostname's lock and commits before any of these calls, so no database
+ * transaction or lock waits on Cloudflare; after writing it re-reads the
+ * hostname and writes again if it moved meanwhile. One process runs at most
+ * one sync per hostname at a time. A writer never replaces a newer
+ * generation (read, compare, write; KV has no compare-and-set, so writers in
+ * two processes racing within one round trip can still land out of order,
+ * until the entry's TTL ends it). Generations come from one database
+ * sequence shared by every hostname, so a later change to a label (a move,
+ * an unpublish tombstone, a takedown) always carries a higher generation
+ * than anything written before.
+ *
+ * Staleness bound: the API itself refuses a suspended or unpublished Site on
+ * the very next site-runtime call (it re-checks publication status and access
+ * every time). The edge copies lag behind: Cloudflare KV propagates a write
+ * to other locations within about 60 s, every KV entry expires after
+ * `sitesHostMapTtlSeconds` (default 300 s), and the gateway caches host
+ * lookups for up to 60 s and access checks for 30 s. A takedown also purges
+ * the host's edge cache when a zone id is configured. The after-commit sync
+ * runs in process, so a restart can lose a pending write; the next signed
+ * host lookup for that label re-syncs a stale record, and the TTL bounds the
+ * rest. A missed sync is therefore corrected within TTL + 60 s at worst.
+ */
+
+export type SiteEdgeSyncSettings = Pick<
+  Settings,
+  | "sitesCloudflareApiToken"
+  | "sitesCloudflareAccountId"
+  | "sitesCloudflareKvNamespaceId"
+  | "sitesCloudflareZoneId"
+  | "sitesHostMapTtlSeconds"
+>;
+
+export type SiteEdgeSync = {
+  readonly kvEnabled: boolean;
+  readonly purgeEnabled: boolean;
+  /** Write the host-map record unless the edge already holds a newer generation. */
+  putHostRecord(record: SiteHostMapRecord): Promise<"written" | "skipped_newer" | "disabled">;
+  /** Purge the edge cache for one hostname. */
+  purgeHost(hostname: string): Promise<"purged" | "disabled">;
+};
+
+export type SiteEdgeSyncOptions = {
+  fetch?: typeof fetch;
+  attempts?: number;
+  backoffMs?: number;
+};
+
+const API = "https://api.cloudflare.com/client/v4";
+const FETCH_TIMEOUT_MS = 10_000;
+
+async function withRetries<T>(attempts: number, backoffMs: number, run: () => Promise<T>) {
+  let last: unknown;
+  for (let attempt = 0; attempt < attempts; attempt += 1) {
+    try {
+      return await run();
+    } catch (error) {
+      last = error;
+      if (attempt < attempts - 1)
+        await new Promise((resolve) => setTimeout(resolve, backoffMs * 2 ** attempt));
+    }
+  }
+  throw last;
+}
+
+export function createSiteEdgeSync(
+  settings: SiteEdgeSyncSettings,
+  options: SiteEdgeSyncOptions = {},
+): SiteEdgeSync {
+  const fetchImpl = options.fetch ?? fetch;
+  const attempts = Math.max(1, options.attempts ?? 3);
+  const backoffMs = Math.max(0, options.backoffMs ?? 250);
+  const token = settings.sitesCloudflareApiToken;
+  const account = settings.sitesCloudflareAccountId;
+  const namespace = settings.sitesCloudflareKvNamespaceId;
+  const zone = settings.sitesCloudflareZoneId;
+  const kvEnabled = Boolean(token && account && namespace);
+  const purgeEnabled = Boolean(token && zone);
+  const headers = { authorization: `Bearer ${token ?? ""}` };
+  const valueUrl = (key: string) =>
+    `${API}/accounts/${encodeURIComponent(account ?? "")}/storage/kv/namespaces/${encodeURIComponent(
+      namespace ?? "",
+    )}/values/${encodeURIComponent(key)}`;
+
+  return {
+    kvEnabled,
+    purgeEnabled,
+    async putHostRecord(record) {
+      if (!kvEnabled) return "disabled";
+      const parsed = SiteHostMapRecord.parse(record);
+      const key = siteHostMapKey(parsed.hostname);
+      return await withRetries(attempts, backoffMs, async () => {
+        const current = await fetchImpl(valueUrl(key), {
+          method: "GET",
+          headers,
+          signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+        });
+        if (current.ok) {
+          const existing = SiteHostMapRecord.safeParse(await current.json().catch(() => null));
+          if (existing.success && existing.data.generation > parsed.generation)
+            return "skipped_newer" as const;
+        } else if (current.status !== 404) {
+          throw new Error(`Cloudflare KV read failed with ${current.status}`);
+        } else {
+          await current.body?.cancel().catch(() => undefined);
+        }
+        const ttl = Math.max(60, settings.sitesHostMapTtlSeconds);
+        const written = await fetchImpl(`${valueUrl(key)}?expiration_ttl=${ttl}`, {
+          method: "PUT",
+          headers: { ...headers, "content-type": "application/json" },
+          body: JSON.stringify(parsed),
+          signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+        });
+        if (!written.ok) throw new Error(`Cloudflare KV write failed with ${written.status}`);
+        await written.body?.cancel().catch(() => undefined);
+        return "written" as const;
+      });
+    },
+    async purgeHost(hostname) {
+      if (!purgeEnabled) return "disabled";
+      return await withRetries(attempts, backoffMs, async () => {
+        const response = await fetchImpl(
+          `${API}/zones/${encodeURIComponent(zone ?? "")}/purge_cache`,
+          {
+            method: "POST",
+            headers: { ...headers, "content-type": "application/json" },
+            body: JSON.stringify({ hosts: [hostname.toLowerCase()] }),
+            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+          },
+        );
+        if (!response.ok) throw new Error(`Cloudflare cache purge failed with ${response.status}`);
+        await response.body?.cancel().catch(() => undefined);
+        return "purged" as const;
+      });
+    },
+  };
+}

package/src/site-publishing/gateway-signature.ts

@@ -0,0 +1,198 @@
+import { createHash, createHmac, timingSafeEqual } from "node:crypto";
+import {
+  siteGatewayCanonicalString,
+  SITE_GATEWAY_BODY_SHA256_HEADER,
+  SITE_GATEWAY_CLIENT_IP_HEADER,
+  SITE_GATEWAY_HOST_HEADER,
+  SITE_GATEWAY_HUMAN_HEADER,
+  SITE_GATEWAY_MAX_SKEW_SECONDS,
+  SITE_GATEWAY_SESSION_HEADER,
+  SITE_GATEWAY_SIGNATURE_HEADER,
+  SITE_GATEWAY_SIGNATURE_VERSION,
+  SITE_GATEWAY_TIMESTAMP_HEADER,
+  SITE_GATEWAY_VISITOR_HEADER,
+} from "@opengeni/contracts";
+
+/**
+ * Verification of gateway-signed site-runtime requests. The gateway HMACs
+ * `siteGatewayCanonicalString` with the shared secret; the API accepts the
+ * current or the previous secret (rotation), within 30 s of its clock. The
+ * signed body hash travels in its own header, so the signature is checked
+ * from the headers alone, before the body is read; the route then hashes the
+ * body as it reads it and refuses a body that does not match. Nothing else
+ * about the request is trusted before this passes.
+ */
+
+export type SiteGatewayFields = {
+  host: string;
+  session: string;
+  visitor: string;
+  clientIp: string;
+  human: boolean;
+};
+
+export type SiteGatewayVerification =
+  | {
+      ok: true;
+      fields: SiteGatewayFields;
+      /** The signed hex SHA-256 the body read afterwards must match. */
+      bodySha256: string;
+    }
+  | {
+      ok: false;
+      reason:
+        | "not_configured"
+        | "missing_headers"
+        | "bad_timestamp"
+        | "skew"
+        | "bad_body_hash"
+        | "bad_signature"
+        | "body_mismatch"
+        | "credentials";
+    };
+
+const SHA256_HEX = /^[0-9a-f]{64}$/u;
+
+/** Request headers that carry Opengeni app credentials; site-runtime refuses them. */
+export const SITE_RUNTIME_REFUSED_HEADERS: readonly string[] = Object.freeze([
+  "authorization",
+  "proxy-authorization",
+  "cookie",
+  "x-api-key",
+  "x-opengeni-external-actor",
+]);
+
+export function sha256Hex(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+export function siteGatewaySignature(secret: string, canonical: string): string {
+  return `${SITE_GATEWAY_SIGNATURE_VERSION}=${createHmac("sha256", secret)
+    .update(canonical, "utf8")
+    .digest("base64url")}`;
+}
+
+function sameText(expected: string, presented: string): boolean {
+  const a = Buffer.from(expected, "utf8");
+  const b = Buffer.from(presented, "utf8");
+  // Equal-length compare; a length mismatch still spends a compare.
+  if (a.length !== b.length) {
+    timingSafeEqual(a, a);
+    return false;
+  }
+  return timingSafeEqual(a, b);
+}
+
+/** Bounded header value: the gateway never sends long values here. */
+function header(headers: Headers, name: string, max = 512): string | null {
+  const value = headers.get(name);
+  if (value === null) return null;
+  return value.length > max ? null : value;
+}
+
+/**
+ * Verify a signed request from its headers alone; the body is not needed.
+ * On success the caller reads the body (capped) and checks it against
+ * `bodySha256` with `sameSha256Hex`, refusing a mismatch as `body_mismatch`.
+ */
+export function verifySiteGatewayRequest(input: {
+  method: string;
+  pathWithQuery: string;
+  headers: Headers;
+  secrets: readonly (string | undefined)[];
+  nowMs?: number;
+}): SiteGatewayVerification {
+  const secrets = input.secrets.filter((secret): secret is string => Boolean(secret));
+  if (secrets.length === 0) return { ok: false, reason: "not_configured" };
+  const signature = header(input.headers, SITE_GATEWAY_SIGNATURE_HEADER, 128);
+  const timestampText = header(input.headers, SITE_GATEWAY_TIMESTAMP_HEADER, 16);
+  const host = header(input.headers, SITE_GATEWAY_HOST_HEADER, 253);
+  const bodySha256 = input.headers.get(SITE_GATEWAY_BODY_SHA256_HEADER);
+  if (!signature || !timestampText || !host || bodySha256 === null)
+    return { ok: false, reason: "missing_headers" };
+  if (!/^\d{1,12}$/u.test(timestampText)) return { ok: false, reason: "bad_timestamp" };
+  const timestamp = Number(timestampText);
+  const now = Math.floor((input.nowMs ?? Date.now()) / 1000);
+  if (Math.abs(now - timestamp) > SITE_GATEWAY_MAX_SKEW_SECONDS)
+    return { ok: false, reason: "skew" };
+  // The hash must be lowercase hex, exactly as the gateway signs it.
+  if (!SHA256_HEX.test(bodySha256)) return { ok: false, reason: "bad_body_hash" };
+  const session = header(input.headers, SITE_GATEWAY_SESSION_HEADER) ?? "";
+  const visitor = header(input.headers, SITE_GATEWAY_VISITOR_HEADER) ?? "";
+  const clientIp = header(input.headers, SITE_GATEWAY_CLIENT_IP_HEADER, 64) ?? "";
+  const human = header(input.headers, SITE_GATEWAY_HUMAN_HEADER, 8) ?? "";
+  const canonical = siteGatewayCanonicalString({
+    method: input.method,
+    host,
+    pathWithQuery: input.pathWithQuery,
+    timestamp,
+    bodySha256,
+    session,
+    visitor,
+    clientIp,
+    human,
+  });
+  // Check every configured secret, so timing does not reveal which one matched.
+  let matched = false;
+  for (const secret of secrets)
+    matched = sameText(siteGatewaySignature(secret, canonical), signature) || matched;
+  if (!matched) return { ok: false, reason: "bad_signature" };
+  // Signed requests never carry app credentials; a request that does is refused.
+  if (SITE_RUNTIME_REFUSED_HEADERS.some((name) => input.headers.has(name)))
+    return { ok: false, reason: "credentials" };
+  return {
+    ok: true,
+    fields: { host: host.toLowerCase(), session, visitor, clientIp, human: human === "1" },
+    bodySha256,
+  };
+}
+
+/** Whether two hex digests are equal, compared in constant time. */
+export function sameSha256Hex(actual: string, signed: string): boolean {
+  return sameText(actual, signed);
+}
+
+/** Sign a request as the gateway does (tests and local tools). */
+export function signSiteGatewayRequest(input: {
+  secret: string;
+  method: string;
+  pathWithQuery: string;
+  host: string;
+  body?: Uint8Array | string;
+  session?: string;
+  visitor?: string;
+  clientIp?: string;
+  human?: boolean;
+  nowMs?: number;
+}): Record<string, string> {
+  const body =
+    typeof input.body === "string"
+      ? new TextEncoder().encode(input.body)
+      : (input.body ?? new Uint8Array());
+  const timestamp = Math.floor((input.nowMs ?? Date.now()) / 1000);
+  const fields = {
+    session: input.session ?? "",
+    visitor: input.visitor ?? "",
+    clientIp: input.clientIp ?? "",
+    human: input.human ? "1" : "",
+  };
+  const bodySha256 = sha256Hex(body);
+  const canonical = siteGatewayCanonicalString({
+    method: input.method,
+    host: input.host,
+    pathWithQuery: input.pathWithQuery,
+    timestamp,
+    bodySha256,
+    ...fields,
+  });
+  return {
+    [SITE_GATEWAY_BODY_SHA256_HEADER]: bodySha256,
+    [SITE_GATEWAY_TIMESTAMP_HEADER]: String(timestamp),
+    [SITE_GATEWAY_SIGNATURE_HEADER]: siteGatewaySignature(input.secret, canonical),
+    [SITE_GATEWAY_HOST_HEADER]: input.host,
+    ...(fields.session ? { [SITE_GATEWAY_SESSION_HEADER]: fields.session } : {}),
+    ...(fields.visitor ? { [SITE_GATEWAY_VISITOR_HEADER]: fields.visitor } : {}),
+    ...(fields.clientIp ? { [SITE_GATEWAY_CLIENT_IP_HEADER]: fields.clientIp } : {}),
+    ...(fields.human ? { [SITE_GATEWAY_HUMAN_HEADER]: fields.human } : {}),
+  };
+}