@anthusai/papyrus 5.49.0 → 5.51.0

package/lib/source-frameable.ts

@@ -0,0 +1,77 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+/**
+ * Whether a source page lets another site show it in a frame. Most news sites refuse (X-Frame-Options or a CSP frame-ancestors
+ * rule), and a refused frame is just an empty box, which the browser cannot detect from outside. So the newsroom asks here,
+ * on the server, before it shows a web page's live preview.
+ */
+
+/** True when the response headers allow framing by any site. A rule naming specific origins is treated as a refusal. */
+export function frameableFromHeaders(headers: Headers): boolean {
+  const xfo = headers.get("x-frame-options")?.trim().toLowerCase();
+  if (xfo && (xfo.includes("deny") || xfo.includes("sameorigin") || xfo.startsWith("allow-from"))) return false;
+  const csp = headers.get("content-security-policy");
+  if (csp) {
+    for (const directive of csp.split(";")) {
+      const [name, ...sources] = directive.trim().split(/\s+/);
+      if (name?.toLowerCase() !== "frame-ancestors") continue;
+      if (!sources.some((source) => source === "*" || source === "https:" || source === "http:")) return false;
+    }
+  }
+  return true;
+}
+
+/** Loopback, private, link-local and other non-public addresses, which the server must never be made to fetch. */
+export function isPrivateAddress(address: string): boolean {
+  const v4 = address.startsWith("::ffff:") ? address.slice(7) : address;
+  if (isIP(v4) === 4) {
+    const [a, b] = v4.split(".").map(Number);
+    return a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254)
+      || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || a >= 224;
+  }
+  const v6 = address.toLowerCase();
+  return v6 === "::" || v6 === "::1" || v6.startsWith("fc") || v6.startsWith("fd") || v6.startsWith("fe80");
+}
+
+/** A plain public http(s) address on the default port whose host resolves only to public addresses, else null. */
+export async function publicHttpUrl(raw: string): Promise<URL | null> {
+  let url: URL;
+  try { url = new URL(raw); } catch { return null; }
+  if ((url.protocol !== "https:" && url.protocol !== "http:") || url.port || url.username || url.password) return null;
+  try {
+    const addresses = isIP(url.hostname) ? [{ address: url.hostname }] : await lookup(url.hostname, { all: true });
+    return addresses.length && addresses.every(({ address }) => !isPrivateAddress(address)) ? url : null;
+  } catch {
+    return null;
+  }
+}
+
+/** GETs a public address, following up to `maxRedirects` redirects and checking every hop with publicHttpUrl. Null on any failure. */
+export async function fetchPublic(raw: string, { accept, timeoutMs = 5000, maxRedirects = 4 }: { accept: string; timeoutMs?: number; maxRedirects?: number }): Promise<Response | null> {
+  let next = await publicHttpUrl(raw);
+  for (let hop = 0; next && hop <= maxRedirects; hop += 1) {
+    let response: Response;
+    try {
+      response = await fetch(next, { method: "GET", redirect: "manual", signal: AbortSignal.timeout(timeoutMs), headers: { accept } });
+    } catch {
+      return null;
+    }
+    const location = response.headers.get("location");
+    if (response.status >= 300 && response.status < 400 && location) {
+      void response.body?.cancel();
+      next = await publicHttpUrl(new URL(location, next).toString());
+      continue;
+    }
+    return response;
+  }
+  return null;
+}
+
+/** Asks the page itself. Anything that fails (bad address, error status, timeout, too many redirects) counts as not frameable. */
+export async function checkSourceFrameable(raw: string, options: { timeoutMs?: number; maxRedirects?: number } = {}): Promise<boolean> {
+  const response = await fetchPublic(raw, { accept: "text/html", ...options });
+  if (!response) return false;
+  void response.body?.cancel();
+  return response.ok && frameableFromHeaders(response.headers);
+}

package/lib/source-icon.ts

@@ -0,0 +1,104 @@
+import { fetchPublic } from "./source-frameable";
+
+/**
+ * A source's own icon, fetched on the server. Guessing `/favicon.ico` from the browser fails often: some sites serve a fully
+ * transparent placeholder there, and others refuse requests from another site. The page's own `<link rel="icon">` tags name
+ * the icon the site actually uses, so the newsroom reads those first and passes the image through.
+ */
+
+const MAX_HTML_BYTES = 256 * 1024;
+const MAX_ICON_BYTES = 256 * 1024;
+/** A bare 16x16 ICO is about 200 bytes; at that size it is often an empty placeholder, so the guessed path must beat it. */
+const MIN_GUESSED_ICON_BYTES = 300;
+
+export type SourceIcon = { body: ArrayBuffer; contentType: string };
+
+/** A bare host name such as `www.brookings.edu`: letters, digits, hyphens and dots only. */
+export function isHostName(host: string): boolean {
+  return host.length <= 253 && /^(?=.*[a-z])[a-z0-9-]+(\.[a-z0-9-]+)+$/i.test(host);
+}
+
+function attribute(tag: string, name: string): string | null {
+  const match = tag.match(new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i"));
+  return match ? (match[1] ?? match[2] ?? match[3] ?? "") : null;
+}
+
+/** The icons a page declares, best first: touch icons, then the largest declared size. Mask icons are single-colour stencils and are skipped. */
+export function iconCandidates(html: string, base: string): string[] {
+  const found: { href: string; score: number }[] = [];
+  for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) {
+    const rel = (attribute(tag, "rel") ?? "").toLowerCase().split(/\s+/);
+    if (!rel.some((word) => word === "icon" || word === "apple-touch-icon" || word === "apple-touch-icon-precomposed")) continue;
+    const href = attribute(tag, "href");
+    if (!href) continue;
+    let absolute: string;
+    try { absolute = new URL(href, base).toString(); } catch { continue; }
+    const size = Math.max(0, ...(attribute(tag, "sizes") ?? "").split(/\s+/).map((s) => Number.parseInt(s, 10)).filter(Number.isFinite));
+    const touch = rel.some((word) => word.startsWith("apple-touch-icon"));
+    found.push({ href: absolute, score: (touch ? 10_000 : 0) + (size || (touch ? 180 : 16)) });
+  }
+  return [...new Set(found.sort((a, b) => b.score - a.score).map(({ href }) => href))];
+}
+
+async function readLimited(response: Response, limit: number): Promise<ArrayBuffer | null> {
+  const declared = Number(response.headers.get("content-length"));
+  if (declared > limit) { void response.body?.cancel(); return null; }
+  const body = await response.arrayBuffer().catch(() => null);
+  return body && body.byteLength <= limit ? body : null;
+}
+
+/** The first `limit` bytes of a page: icons are declared in its head, and some home pages run to a megabyte. */
+async function readPrefix(response: Response, limit: number): Promise<string> {
+  const reader = response.body?.getReader();
+  if (!reader) return "";
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  try {
+    while (total < limit) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      total += value.byteLength;
+    }
+  } catch {
+    // keep what arrived
+  } finally {
+    void reader.cancel().catch(() => undefined);
+  }
+  const bytes = new Uint8Array(Math.min(total, limit));
+  let offset = 0;
+  for (const chunk of chunks) {
+    const part = chunk.subarray(0, bytes.length - offset);
+    bytes.set(part, offset);
+    offset += part.byteLength;
+    if (offset >= bytes.length) break;
+  }
+  return new TextDecoder().decode(bytes);
+}
+
+async function fetchIcon(url: string, minBytes: number): Promise<SourceIcon | null> {
+  const response = await fetchPublic(url, { accept: "image/*" });
+  if (!response) return null;
+  const contentType = (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();
+  if (!response.ok || !contentType.startsWith("image/")) { void response.body?.cancel(); return null; }
+  const body = await readLimited(response, MAX_ICON_BYTES);
+  return body && body.byteLength >= minBytes ? { body, contentType } : null;
+}
+
+/** The host's best declared icon, else its `/favicon.ico` when that is more than a placeholder, else null. */
+export async function fetchSourceIcon(host: string): Promise<SourceIcon | null> {
+  if (!isHostName(host)) return null;
+  const home = `https://${host}/`;
+  const page = await fetchPublic(home, { accept: "text/html" });
+  let candidates: string[] = [];
+  if (page?.ok && (page.headers.get("content-type") ?? "").includes("html")) {
+    candidates = iconCandidates(await readPrefix(page, MAX_HTML_BYTES), page.url || home);
+  } else {
+    void page?.body?.cancel();
+  }
+  for (const candidate of candidates.slice(0, 4)) {
+    const icon = await fetchIcon(candidate, 1);
+    if (icon) return icon;
+  }
+  return fetchIcon(new URL("/favicon.ico", home).toString(), MIN_GUESSED_ICON_BYTES);
+}

package/package.json

@@ -86,6 +86,7 @@
     "./components/editorial-findings-panel": "./components/editorial-findings-panel.tsx",
     "./components/editorial-preview-popover": "./components/editorial-preview-popover.tsx",
     "./components/item-stream-view": "./components/item-stream-view.tsx",
+    "./components/markdown-context": "./components/markdown-context.tsx",
     "./components/markus-body-editor": "./components/markus-body-editor.tsx",
     "./components/markus-ir-preview": "./components/markus-ir-preview.tsx",
     "./components/moderation-panel": "./components/moderation-panel.tsx",
@@ -114,7 +115,6 @@
     "./components/newsroom-mobile-navigation": "./components/newsroom-mobile-navigation.tsx",
     "./components/newsroom-ops-overview": "./components/newsroom-ops-overview.tsx",
     "./components/newsroom-ops-shell": "./components/newsroom-ops-shell.tsx",
-    "./components/newsroom-overview-skeleton": "./components/newsroom-overview-skeleton.tsx",
     "./components/newsroom-references-view": "./components/newsroom-references-view.tsx",
     "./components/newsroom-theme-fonts/broadsheet-green": "./components/newsroom-theme-fonts/broadsheet-green.tsx",
     "./components/newsroom-theme-fonts/newspaper-gothic": "./components/newsroom-theme-fonts/newspaper-gothic.tsx",
@@ -164,6 +164,7 @@
     "./components/story-reporting-trail-client": "./components/story-reporting-trail-client.ts",
     "./components/story-reporting-trail": "./components/story-reporting-trail.tsx",
     "./components/topic-steering-workspace": "./components/topic-steering-workspace.tsx",
+    "./components/ui/auto-grow-textarea": "./components/ui/auto-grow-textarea.tsx",
     "./components/ui/badge": "./components/ui/badge.tsx",
     "./components/ui/button-group": "./components/ui/button-group.tsx",
     "./components/ui/button": "./components/ui/button.tsx",
@@ -368,6 +369,8 @@
     "./lib/site-renderer": "./lib/site-renderer.ts",
     "./lib/site-section-suggestions": "./lib/site-section-suggestions.ts",
     "./lib/site-stack": "./lib/site-stack.ts",
+    "./lib/source-frameable": "./lib/source-frameable.ts",
+    "./lib/source-icon": "./lib/source-icon.ts",
     "./lib/staff-groups": "./lib/staff-groups.ts",
     "./lib/staging-gate": "./lib/staging-gate.ts",
     "./lib/staging-gated-path": "./lib/staging-gated-path.ts",
@@ -419,6 +422,8 @@
     "./app/api/newsroom/curation/context/route": "./app/api/newsroom/curation/context/route.ts",
     "./app/api/newsroom/curation/priming/route": "./app/api/newsroom/curation/priming/route.ts",
     "./app/api/newsroom/curation/publication/route": "./app/api/newsroom/curation/publication/route.ts",
+    "./app/api/newsroom/source-frameable/route": "./app/api/newsroom/source-frameable/route.ts",
+    "./app/api/newsroom/source-icon/route": "./app/api/newsroom/source-icon/route.ts",
     "./app/api/reader/comment/delete/route": "./app/api/reader/comment/delete/route.ts",
     "./app/api/reader/comment/edit/route": "./app/api/reader/comment/edit/route.ts",
     "./app/api/reader/comment/route": "./app/api/reader/comment/route.ts",
@@ -495,7 +500,7 @@
       "optional": true
     }
   },
-  "version": "5.49.0",
+  "version": "5.51.0",
   "dependencies": {
     "@aws-amplify/adapter-nextjs": "^1.7.3",
     "@aws-crypto/sha256-js": "5.2.0",

package/routes.manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.49.0",
+  "version": "5.51.0",
   "routes": [
     {
       "file": "app/%5F_preview/[[...path]]/route.ts",
@@ -285,6 +285,28 @@
         "runtime": "\"nodejs\""
       }
     },
+    {
+      "file": "app/api/newsroom/source-frameable/route.ts",
+      "hasDefault": false,
+      "exports": [
+        "GET"
+      ],
+      "config": {
+        "dynamic": "\"force-dynamic\"",
+        "runtime": "\"nodejs\""
+      }
+    },
+    {
+      "file": "app/api/newsroom/source-icon/route.ts",
+      "hasDefault": false,
+      "exports": [
+        "GET"
+      ],
+      "config": {
+        "dynamic": "\"force-dynamic\"",
+        "runtime": "\"nodejs\""
+      }
+    },
     {
       "file": "app/api/reader/comment/delete/route.ts",
       "hasDefault": false,

package/components/newsroom-overview-skeleton.tsx

@@ -1,25 +0,0 @@
-"use client";
-
-import * as React from "react";
-
-/** Placeholder geometry for the overview while private records or access are unresolved. */
-export function NewsroomOverviewSkeleton() {
-  return (
-    <div aria-label="Loading newsroom overview" className="space-y-8 font-sans" data-newsroom-overview-skeleton role="status">
-      <section className="newsroom-overview-feed-frame space-y-4 overflow-hidden rounded-xl border bg-card p-4" aria-label="Notification Center">
-        <div className="h-6 w-48 rounded bg-muted" />
-        <div className="h-4 w-3/4 max-w-xl rounded bg-muted" />
-        <div className="h-4 w-2/3 max-w-lg rounded bg-muted" />
-        <div className="grid gap-3 sm:grid-cols-2">
-          {Array.from({ length: 4 }, (_, index) => <div className="h-28 rounded-xl border bg-card p-4" key={index}>
-            <div className="h-4 w-2/3 rounded bg-muted" /><div className="mt-4 h-3 w-1/2 rounded bg-muted" />
-          </div>)}
-        </div>
-      </section>
-      <section aria-label="Needs attention" className="space-y-3">
-        <div className="h-4 w-32 rounded bg-muted" />
-        <div className="h-16 rounded-xl border bg-card p-4"><div className="h-4 w-1/2 rounded bg-muted" /><div className="mt-2 h-3 w-1/3 rounded bg-muted" /></div>
-      </section>
-    </div>
-  );
-}