@junejs/server 0.1.3 → 0.2.1

package/src/app.ts

@@ -18,21 +18,25 @@ import type { ComponentType } from "react";
 import { routeFromModule } from "@junejs/core/route";
 import {
   resolveAgent,
+  deployOrigin,
   resolveClientRouter,
   resolveSpeculationRules,
   type JuneConfig,
 } from "@junejs/core/config";
 import type { DocumentConfig } from "@junejs/core/document";
 import { runWithTrace, type RequestTrace } from "@junejs/core/instrumentation";
+import { blobFileStore, memoryFileStore } from "@junejs/core/agent-files";
+import type { JuneBlob } from "@junejs/core/resources";
 
 import { findMiddlewareFile, isResourceFile, listRoutes, matchRouteTree, resolveNotFound, routeFiles, type SegmentMatch } from "./router";
+import { routeProblems, scanRoutes } from "./route-scan";
 import { createPipeline, type ExtraHandler, type LayoutComponent, type MiddlewareHandler, type Pipeline, type Resolved, type ResourceHandler } from "./pipeline";
 import { discoverAgent } from "./agent-discover";
-import { createAgentRuntime, mountAgent } from "./agent-native";
-import { buildSystemPrompt } from "@junejs/core/agent-config";
+import { createAgentRuntime, mountAgent, toAgentDef, type InProcessRuntime } from "./agent-native";
 import { anthropic } from "@junejs/core/agent-models";
 import { resolveBoundary } from "./segment";
 import { memoizeResources } from "./resources";
+import { localBlob } from "./blob";
 import {
   findClientEntry,
   bundleClientSplit,
@@ -43,6 +47,15 @@ import { generateIslandRegistry } from "./island-registry";
 import { findGlobalCss, globalCssUsesTailwind, processCssCached, STYLES_URL } from "./css";
 import { buildModuleCss, registerCssModules, MODULE_STYLES_URL, type ModuleMaps } from "./css-modules";
 import { contentTypeFor, RESERVED_PREFIX, safeRelativePath } from "./static-files";
+import {
+  GENERATED_ICONS,
+  googleFontFetcher,
+  publicFileCheck,
+  renderIcons,
+  resolveIcons,
+  type GeneratedIconFile,
+  type GeneratedIcons,
+} from "./favicon";
 
 export type CreateAppOptions = {
   appDir: string;
@@ -56,6 +69,10 @@ export type JuneApp = {
   warmup(): Promise<void>;
   routePaths(): Promise<string[]>;
   earlyHints(): string[];
+  // Shut down what the app started (#317): the agent runtime built for an agent/
+  // directory — its pending announcement retries, its in-flight turns, its SQLite
+  // handle. Resolves once they finish. A no-op when no runtime was built.
+  close(): Promise<void>;
 };
 
 // Import a layout file's default export as a layout component (memoized), plus
@@ -133,6 +150,19 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
   } catch {
     /* no public/ dir → nothing to serve */
   }
+  // Default PNG icons: linked by the document, rendered once on first request
+  // (a failed render is retried next time rather than memoized).
+  const iconPlan = resolveIcons(config.site ?? {}, publicFileCheck(publicDir));
+  let iconsPromise: Promise<GeneratedIcons> | undefined;
+  const getIcons = () =>
+    (iconsPromise ??= renderIcons({
+      site: config.site ?? {},
+      fetchFont: googleFontFetcher(join(dirname(appDir), "node_modules", ".cache", "june", "fonts")),
+      warn: (msg) => console.warn(msg),
+    }).catch((err) => {
+      iconsPromise = undefined;
+      throw err;
+    }));
   const agent = resolveAgent(config.agent);
   const speculation = config.speculation;
   // app/_client.* present → the dev document loads /client.js and we serve it
@@ -144,6 +174,7 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
   const cssEntry = findGlobalCss(appDir);
   const docConfig: DocumentConfig = {
     site: config.site ?? {},
+    deployOrigin: deployOrigin(config),
     speculationRules: resolveSpeculationRules(speculation ?? undefined),
     speculationDelivery: speculation ? speculation.delivery ?? "inline" : "inline",
     viewTransitions: config.viewTransitions ?? true,
@@ -152,6 +183,7 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
     clientRouter: resolveClientRouter(config.clientRouter),
     clientScript: clientEntry ? CLIENT_SCRIPT_URL : null,
     styles: cssEntry ? STYLES_URL : null,
+    icons: iconPlan.icons,
   };
 
   // CSS Modules: glob + transform app/**/*.module.css ONCE (memoized) → the maps
@@ -202,6 +234,8 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
   // pipeline is built lazily on first fetch, memoized after.
   let pipelinePromise: Promise<Pipeline> | undefined;
   const getPipeline = (): Promise<Pipeline> => (pipelinePromise ??= buildPipeline());
+  // The agent runtime buildPipeline creates, kept so close() can shut it down (#317).
+  let agentRuntime: InProcessRuntime | undefined;
   async function buildPipeline(): Promise<Pipeline> {
     // Wire CSS Modules BEFORE any route import: the runtime interceptor must be
     // active so `import "x.module.css"` in a route resolves to the scoped map,
@@ -237,10 +271,18 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
       // instructions live on the def (single source); the runtime injects them
       // into the model per turn via withSystem — no need to bake into anthropic().
       const model = anthropic({ model: def.model });
-      const rt = await createAgentRuntime(
-        { [def.name]: { model, tools: def.tools, instructions: buildSystemPrompt(def) } },
-        { backend },
-      );
+      // toAgentDef: tools, prompt AND the per-surface policies (#149) from the one
+      // definition — channelInstructions was previously dropped on this path (#173).
+      // Agent files (docs/rfc-agent-files.md) live in the app's blob resource; an app that
+      // declares none gets the local default directory. The memory backend keeps them in
+      // memory, like everything else it holds.
+      let blobOpened: Promise<JuneBlob> | undefined;
+      const files =
+        backend === "memory"
+          ? memoryFileStore()
+          : blobFileStore(() => (blobOpened ??= resources().then(async (r) => r.blob ?? (await localBlob().open()))));
+      const rt = await createAgentRuntime({ [def.name]: toAgentDef(def, model) }, { backend, files });
+      agentRuntime = rt;
       const mounted = mountAgent(def, rt, { chatPath: agent.runtime.chat.path, channels: agent.runtime.channels });
       agentSurface = (req) => mounted.surface(req);
       await mounted.startAll();
@@ -403,9 +445,30 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
           }
         }
       }
+      // June's default PNG icons (the app has none of its own): rendered on first
+      // request with the same code `june build` writes them with.
+      if (iconPlan.generate && (request.method === "GET" || request.method === "HEAD")) {
+        const name = new URL(request.url).pathname.slice(1);
+        if ((GENERATED_ICONS as readonly string[]).includes(name)) {
+          return getIcons().then(
+            (icons) =>
+              new Response(request.method === "HEAD" ? null : icons[name as GeneratedIconFile], {
+                headers: { "content-type": contentTypeFor(name), "cache-control": "public, max-age=0, must-revalidate" },
+              }),
+            () => toPipeline(),
+          );
+        }
+      }
       return toPipeline();
     },
     async warmup() {
+      // What `june build` refuses (same-path files, a segment after an optional
+      // or catch-all), dev reports up front — it still serves what it can.
+      const problems = routeProblems(
+        [await scanRoutes(appDir), hasJuneRoutes ? await scanRoutes(juneRoutesDir) : []],
+        dirname(appDir),
+      );
+      if (problems) console.error(problems);
       const files = await routeFiles(appDir, { pageConvention: true });
       if (hasJuneRoutes) files.push(...(await routeFiles(juneRoutesDir, { pageConvention: true })));
       for (const file of files) {
@@ -416,5 +479,11 @@ export function createApp({ appDir: appDirInput, config = {} }: CreateAppOptions
     },
     routePaths,
     earlyHints: () => config.earlyHints ?? [],
+    async close() {
+      // A pipeline still being built may be about to create the runtime: let it finish
+      // (its failure is the fetch's to report, not close()'s).
+      await pipelinePromise?.catch(() => {});
+      await agentRuntime?.close();
+    },
   };
 }

package/src/assets.ts

@@ -0,0 +1,200 @@
+// assetCache — a loader for the large files an `unbundled` entry leaves out of a build (#383):
+// the PDF engine, a CJK font. Given a directory the app owns (a data volume), it serves each
+// file from there and fetches it the first time, so a single binary doesn't carry 40 MiB it
+// may never use:
+//
+//   import { assetCache } from "@junejs/server/assets";
+//   const load = assetCache({ dir: "/data/june-assets" });
+//   typstWasmRenderer({ load, fonts: [notoSansTC({ load })] });
+//
+// What it guarantees, so it can sit under a production app:
+//   • a file is returned only when its size and SHA-256 match the ref — on disk or downloaded;
+//     a stale or corrupt file is replaced, never used (the packages check again regardless);
+//   • a download stops at the ref's size, so a wrong URL can't fill the disk;
+//   • the file lands by rename from a temp file in the same directory, so a crash leaves no
+//     half-written file under the real name;
+//   • one download per file per cache, however many renders ask at once (share one `load`;
+//     two caches over the same dir each download, and the rename keeps that safe);
+//   • `offline: true` never touches the network: for hosts without egress, the files are
+//     copied in beforehand (`dir/<name>`, e.g. dir/typst.wasm), and a missing one is an error
+//     that says so.
+//
+// The request is a fixed URL: no document data leaves. `egress(refs)` says which hosts a
+// call would still contact, for a setup check.
+//
+// AssetRef is the shape @junejs/pdf-typst-wasm defines; it is repeated here, so the server
+// depends on no PDF package.
+
+import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export type AssetRef = { name: string; sha256: string; size: number; urls: string[] };
+
+export type AssetCacheOptions = {
+  // Where the files live: one per ref, at dir/<name>.
+  dir: string;
+  // Never download: serve what is in `dir`, and fail naming the file to copy in when it's
+  // missing or doesn't match.
+  offline?: boolean;
+  // Where to fetch a ref from, in order (default: ref.urls) — a mirror the app controls.
+  urls?: (ref: AssetRef) => string[];
+  // How to fetch (default: global fetch).
+  fetch?: typeof fetch;
+};
+
+export interface AssetCache {
+  // The file's bytes, from `dir` or downloaded into it; always matching the ref.
+  (ref: AssetRef): Promise<Uint8Array>;
+  // Whether `dir` already holds a file matching the ref.
+  cached(ref: AssetRef): Promise<boolean>;
+  // The hosts a call for these refs would contact: [] when all are cached, or when offline.
+  egress(refs: AssetRef[]): Promise<{ host: string; sends: "nothing" }[]>;
+}
+
+// A temp file older than this is a crashed download; younger ones may be another process's.
+const STALE_TEMP_MS = 60 * 60 * 1000;
+// Temp files carry this prefix, and no asset may: the sweep removes nothing else.
+const TEMP_PREFIX = ".june-tmp-";
+
+export function assetCache(opts: AssetCacheOptions): AssetCache {
+  const fetcher = opts.fetch ?? fetch;
+  const urlsFor = opts.urls ?? ((ref: AssetRef) => ref.urls);
+  const inflight = new Map<string, Promise<Uint8Array>>();
+  let swept: Promise<void> | undefined;
+
+  const pathOf = (ref: AssetRef) => join(opts.dir, safeName(ref.name));
+
+  // The file at dir/<name> when it matches the ref; undefined when it's missing or doesn't.
+  // Sized first, so a large wrong file is never read; a read error other than "missing" (a
+  // directory in its place, no permission) is the caller's problem, not a cache miss.
+  async function readMatching(ref: AssetRef): Promise<Uint8Array | undefined> {
+    const path = pathOf(ref);
+    let info: Awaited<ReturnType<typeof stat>>;
+    try {
+      info = await stat(path);
+    } catch (err) {
+      if ((err as { code?: string }).code === "ENOENT") return undefined;
+      throw err;
+    }
+    if (!info.isFile()) throw new Error(`${path} is in the way of ${ref.name}: it isn't a file`);
+    if (info.size !== ref.size) return undefined;
+    const bytes = new Uint8Array(await readFile(path));
+    return (await sha256(bytes)) === ref.sha256 ? bytes : undefined;
+  }
+
+  async function get(ref: AssetRef): Promise<Uint8Array> {
+    pathOf(ref); // an unsafe name fails here, before any read or fetch
+    const hit = await readMatching(ref);
+    if (hit) return hit;
+    if (opts.offline) {
+      const there = await stat(pathOf(ref)).then(() => true, () => false);
+      const state = there ? `in ${opts.dir} but doesn't match` : `not in ${opts.dir}`;
+      throw new Error(
+        `${ref.name} is ${state}, and this cache is offline: ` +
+          `copy the file with sha256 ${ref.sha256} (${ref.size} bytes) to ${pathOf(ref)} — it is at ${ref.urls.join(" or ")}`,
+      );
+    }
+    // Housekeeping, best effort: a temp it can't remove must not fail this download or any later.
+    await (swept ??= sweepTemps(opts.dir).catch(() => {}));
+    const bytes = await download(ref, urlsFor(ref), fetcher);
+    await mkdir(opts.dir, { recursive: true });
+    const temp = join(opts.dir, `${TEMP_PREFIX}${crypto.randomUUID()}-${safeName(ref.name)}`);
+    try {
+      await writeFile(temp, bytes);
+      await rename(temp, pathOf(ref)); // atomic on one filesystem: never a half-written file
+    } catch (err) {
+      await rm(temp, { force: true }).catch(() => {}); // don't leave it for the next sweep
+      throw err;
+    }
+    return bytes;
+  }
+
+  // Keyed by the ref's whole identity (name, content, size): only identical refs share a
+  // download, so each call gets bytes that match its own ref.
+  const cache = ((ref: AssetRef) => {
+    const key = `${ref.name}\0${ref.sha256}\0${ref.size}`;
+    const running = inflight.get(key);
+    if (running) return running;
+    const p = get(ref).finally(() => inflight.delete(key));
+    inflight.set(key, p);
+    return p;
+  }) as AssetCache;
+  cache.cached = async (ref) => (await readMatching(ref)) !== undefined;
+  cache.egress = async (refs) => {
+    if (opts.offline) return [];
+    const missing = (await Promise.all(refs.map(async (r) => ((await cache.cached(r)) ? [] : urlsFor(r))))).flat();
+    return [...new Set(missing.map((u) => new URL(u).host))].map((host) => ({ host, sends: "nothing" as const }));
+  };
+  return cache;
+}
+
+// Each URL in turn; the first whose bytes match wins. The error lists every attempt.
+async function download(ref: AssetRef, urls: string[], fetcher: typeof fetch): Promise<Uint8Array> {
+  const failures: string[] = [];
+  for (const url of urls) {
+    try {
+      const res = await fetcher(url);
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const declared = Number(res.headers.get("content-length") ?? NaN);
+      if (declared > ref.size) throw new Error(`${declared} bytes declared, expected ${ref.size}`);
+      const bytes = await readCapped(res, ref.size);
+      const digest = await sha256(bytes);
+      if (bytes.length !== ref.size || digest !== ref.sha256) throw new Error(`got ${bytes.length} bytes with sha256 ${digest}, expected ${ref.size} with ${ref.sha256}`);
+      return bytes;
+    } catch (err) {
+      failures.push(`${url}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  throw new Error(`${ref.name} could not be fetched (${failures.join("; ") || "no URLs"}). To run without the network, copy the file into the cache directory and pass offline: true`);
+}
+
+// The body, cut off one byte past the expected size: enough to know it's wrong, never more.
+async function readCapped(res: Response, size: number): Promise<Uint8Array> {
+  if (!res.body) return new Uint8Array(await res.arrayBuffer());
+  const out = new Uint8Array(size + 1);
+  let length = 0;
+  const reader = res.body.getReader();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (length + value.length >= out.length) {
+      out.set(value.subarray(0, out.length - length), length);
+      length = out.length;
+      await reader.cancel();
+      break;
+    }
+    out.set(value, length);
+    length += value.length;
+  }
+  return out.subarray(0, length);
+}
+
+async function sha256(bytes: Uint8Array): Promise<string> {
+  const digest = await crypto.subtle.digest("SHA-256", bytes as Uint8Array<ArrayBuffer>);
+  return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+// A ref's name becomes a file name: one path segment, nothing that climbs out of `dir`.
+function safeName(name: string): string {
+  if (!name || name === "." || name === ".." || /[\\/]/.test(name) || name.startsWith(TEMP_PREFIX)) throw new Error(`unsafe asset name: ${JSON.stringify(name)}`);
+  return name;
+}
+
+// Temp files a crashed download left behind; ones still young may belong to another process.
+async function sweepTemps(dir: string): Promise<void> {
+  let names: string[];
+  try {
+    names = await readdir(dir);
+  } catch {
+    return;
+  }
+  const now = Date.now();
+  await Promise.all(
+    names
+      .filter((n) => n.startsWith(TEMP_PREFIX))
+      .map(async (n) => {
+        const s = await stat(join(dir, n)).catch(() => undefined);
+        if (s && now - s.mtimeMs > STALE_TEMP_MS) await rm(join(dir, n), { force: true });
+      }),
+  );
+}