@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/surfaces.ts

@@ -0,0 +1,264 @@
+// An app's file tree → one merged realm.ts source, with esbuild alone.
+//
+// The merge does two jobs, and an app can need either or both: attach the
+// file-based UI surfaces, and inline the entry's relative-import closure (an app
+// can have no UI at all — just realm.ts + relative helpers — and still has to
+// arrive as one module, since `bundle()` blocks relative specifiers).
+//
+// Authoring is Next.js style: each pages/<key>.tsx / widgets/<key>.tsx exports
+// `title`, `icon`, optional `loaders` / `actions` (and `cols` / `rows` for
+// widgets) plus a default render fn; the author's realm.tsx never references
+// them. The merge generates a glue entry that imports realm.tsx's app + each
+// surface namespace, calls `app.attachSurfaces({ pages, widgets })`, and
+// re-exports the app. esbuild inlines the relatives into one module; every
+// package import — @yourrealm/sdk*, preact, zod — stays external. No TypeScript
+// compiler is involved (surface exports are read off the namespace at runtime),
+// which is what lets Home run this at install.
+//
+// Home consumes a packed artifact via `mergePackedSource` (unpack → confined tmp
+// tree → merge); `realm-sdk dev` calls `mergeSurfaces` on the live app dir. The
+// only confinement is esbuild's `--metafile` guard in `esbuildMerge`: refuse the
+// merge if esbuild read any file outside the app root.
+
+import { dirname, join } from "@std/path";
+import { REALM_ENTRIES, unpackFiles } from "./pack.ts";
+import { pkgRoot } from "./pkg-root.ts";
+
+const ID_RE = /^[A-Za-z_$][\w$]*$/;
+
+const SDK_DIR = pkgRoot(import.meta.url);
+
+interface SurfaceFile {
+  kind: "page" | "widget";
+  key: string;
+  dir: "pages" | "widgets";
+  file: string;
+}
+
+export async function globSurfaces(
+  appDir: string,
+  kind: "page" | "widget",
+): Promise<SurfaceFile[]> {
+  const dir = kind === "page" ? "pages" : "widgets";
+  const out: SurfaceFile[] = [];
+  try {
+    for await (const entry of Deno.readDir(join(appDir, dir))) {
+      if (!entry.isFile || !entry.name.endsWith(".tsx")) continue;
+      const key = entry.name.slice(0, -".tsx".length);
+      if (!ID_RE.test(key)) {
+        throw new Error(
+          `${dir}/${entry.name}: surface key "${key}" must be a valid identifier — rename the file.`,
+        );
+      }
+      out.push({ kind, key, dir, file: entry.name });
+    }
+  } catch (err) {
+    if (!(err instanceof Deno.errors.NotFound)) throw err; // no pages/ or widgets/ dir is fine
+  }
+  return out.sort((a, b) => a.key.localeCompare(b.key));
+}
+
+const surfaceNs = (f: SurfaceFile): string => `__realm_${f.kind}_${f.key}`;
+
+// `page()`/`widget()` call reading every field off the surface's namespace —
+// missing optionals read as `undefined`, which the helpers + createApp tolerate.
+function surfaceCall(f: SurfaceFile): string {
+  const ns = surfaceNs(f);
+  const factory = f.kind === "page" ? "__realmPage" : "__realmWidget";
+  const fields = [
+    `title: ${ns}.title`,
+    `icon: ${ns}.icon`,
+    `render: ${ns}.default`,
+    `stateSchema: ${ns}.stateSchema`,
+    `configSchema: ${ns}.configSchema`,
+    `loaders: ${ns}.loaders`,
+    `actions: ${ns}.actions`,
+  ];
+  if (f.kind === "widget") fields.push(`cols: ${ns}.cols`, `rows: ${ns}.rows`);
+  return `${factory}({ ${fields.join(", ")} })`;
+}
+
+// The glue entry: import the app + each surface namespace, attach the assembled
+// ui, re-export the app. esbuild inlines the relatives; the app runs first
+// (createApp during its import), then attachSurfaces in the glue body.
+function surfaceGlue(entry: string, files: SurfaceFile[]): string {
+  // Surface-less app: the glue exists only to inline the entry's
+  // relative-import closure, so keep it to a bare re-export. Importing
+  // `@yourrealm/sdk/ui` with nothing to attach would pull a UI barrel into an
+  // app that has no UI — a QuickJS load-safety risk for no gain.
+  if (files.length === 0)
+    return `import __realmApp from "./${entry}";\nexport default __realmApp;\n`;
+  const lines = [
+    `import __realmApp from "./${entry}";`,
+    `import { page as __realmPage, widget as __realmWidget } from "@yourrealm/sdk/ui";`,
+  ];
+  for (const f of files) lines.push(`import * as ${surfaceNs(f)} from "./${f.dir}/${f.file}";`);
+  lines.push("__realmApp.attachSurfaces({", "  pages: {");
+  for (const f of files) if (f.kind === "page") lines.push(`    ${f.key}: ${surfaceCall(f)},`);
+  lines.push("  },", "  widgets: {");
+  for (const f of files) if (f.kind === "widget") lines.push(`    ${f.key}: ${surfaceCall(f)},`);
+  lines.push("  },", "});", "export default __realmApp;", "");
+  return lines.join("\n");
+}
+
+// esbuild / oxfmt are devDeps in node_modules/.bin, not necessarily on PATH.
+// Walk up from the app dir to find them.
+function findBin(startDir: string, name: string): string | null {
+  let dir = startDir;
+  while (true) {
+    const candidate = join(dir, "node_modules", ".bin", name);
+    try {
+      Deno.statSync(candidate);
+      return candidate;
+    } catch {
+      /* keep walking */
+    }
+    const parent = dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+// esbuild lives in node_modules/.bin. The author's app dir has one up the tree;
+// Home's install-time tmp tree doesn't, so fall back to the SDK's own location,
+// then to PATH (Home stages the binary there).
+function findEsbuild(appDir: string): string {
+  return (
+    findBin(appDir, "esbuild") ??
+    (SDK_DIR === null ? null : findBin(SDK_DIR, "esbuild")) ??
+    "esbuild"
+  );
+}
+
+// Bundle the generated glue through esbuild via stdin, with the process cwd set
+// to the app dir so the glue's relative imports (./realm.tsx, ./pages/*) resolve
+// against the real tree. The app dir itself is never mutated — output + metafile
+// go to a scratch dir outside it.
+//
+// esbuild follows every relative/absolute import straight to disk
+// (`--packages=external` externalizes only bare specifiers), so a hostile
+// `import "../../secret"` in a packed file would otherwise inline a
+// control-plane file. The sole confinement is esbuild's own `--metafile`: refuse
+// the merge if it read ANY input outside the app root. It keys off what esbuild
+// actually resolved (static, dynamic, require, css) rather than a source scan,
+// so no import form or line layout slips past it.
+async function esbuildMerge(glue: string, appDir: string): Promise<string> {
+  const esbuild = findEsbuild(appDir);
+  const outDir = await Deno.makeTempDir({ prefix: "realm-merge-out-" });
+  const outFile = join(outDir, "merged.js");
+  const metaFile = join(outDir, "meta.json");
+  try {
+    const child = new Deno.Command(esbuild, {
+      args: [
+        "--bundle",
+        "--loader=tsx", // stdin has no filename to infer from
+        "--packages=external", // every bare import stays external; only relatives inline
+        // Normalize raw `zod` to the SDK re-export. A file shared between an
+        // app's container code and a UI surface imports `z` from raw `zod`, but
+        // Home's pruned image resolves only `@yourrealm/sdk`, which re-exports
+        // the same `z`. Alias fires before the external check, so the specifier
+        // is rewritten and left external, never resolved here.
+        "--alias:zod=@yourrealm/sdk",
+        "--format=esm",
+        "--platform=neutral",
+        "--target=esnext", // keep output close to source — no needless downleveling
+        "--jsx=preserve", // Home's bundle() does the JSX transform; keep <Page> readable
+        "--charset=utf8", // keep `…`/emoji literal instead of \u escapes
+        `--outfile=${outFile}`, // esbuild requires an output path to emit a metafile
+        `--metafile=${metaFile}`,
+      ],
+      cwd: appDir,
+      stdin: "piped",
+      stdout: "piped",
+      stderr: "piped",
+    }).spawn();
+    const writer = child.stdin.getWriter();
+    await writer.write(new TextEncoder().encode(glue));
+    await writer.close();
+    const { code, stderr } = await child.output();
+    if (code !== 0) throw new Error(new TextDecoder().decode(stderr));
+
+    // esbuild reports every input it read, keyed by a path relative to its cwd
+    // (the app dir). An in-app file never starts with `../`; anything that does
+    // (or is absolute) was read from outside the app — refuse it. `<stdin>` is
+    // the glue itself.
+    const meta = JSON.parse(await Deno.readTextFile(metaFile)) as {
+      inputs: Record<string, unknown>;
+    };
+    for (const input of Object.keys(meta.inputs)) {
+      if (input === "<stdin>") continue;
+      if (input.startsWith("/") || input.startsWith("../")) {
+        throw new Error(`merge refused: an import resolves outside the app root ("${input}")`);
+      }
+    }
+    return await Deno.readTextFile(outFile);
+  } finally {
+    await Deno.remove(outDir, { recursive: true }).catch(() => {});
+  }
+}
+
+export function findRealmEntry(appDir: string): string {
+  for (const name of REALM_ENTRIES) {
+    try {
+      Deno.statSync(join(appDir, name));
+      return name;
+    } catch {
+      /* try next */
+    }
+  }
+  throw new Error(`no ${REALM_ENTRIES.join(" or ")} in ${appDir}`);
+}
+
+/**
+ * Merge an app's tree into the one source `bundle()` ingests, using esbuild
+ * alone: attach the file-based surfaces AND inline the entry's relative-import
+ * closure. Runs at install (Home) and in dev.
+ *
+ * Every app goes through the merge, surfaces or not — a surface-less app can
+ * still be multi-file (`realm.ts` + plain relative helpers), and its relatives
+ * have to be inlined all the same or `bundle()` rejects them as blocked
+ * specifiers. Nothing is lost by merging the single-file case too: `bundle()`
+ * esbuilds the source into the QuickJS artifact immediately afterward either
+ * way.
+ */
+export async function mergeSurfaces(
+  appDir: string,
+  // The entry's filename, when the caller already knows it. Defaults to the
+  // canonical realm.tsx/realm.ts an app dir holds — but a test fixture dir can
+  // hold several apps under arbitrary names (`sample.realm.ts`), and the test
+  // harness knows exactly which one it's merging.
+  entryName?: string,
+): Promise<{ source: string; surfaceCount: number }> {
+  const entry = entryName ?? findRealmEntry(appDir);
+  const files = [
+    ...(await globSurfaces(appDir, "page")),
+    ...(await globSurfaces(appDir, "widget")),
+  ];
+  return {
+    source: await esbuildMerge(surfaceGlue(entry, files), appDir),
+    surfaceCount: files.length,
+  };
+}
+
+/**
+ * Install-time entry point: unpack a packed artifact into a confined tmp tree
+ * and merge it into the single source `bundle()` ingests. Folded into `bundle()`
+ * / `bundleUiSurfaces()`, so Home handles a packed artifact at every call site.
+ * A hostile import that reaches outside the tmp tree is refused by esbuild's
+ * `--metafile` guard in `esbuildMerge`. No TypeScript compiler — safe in Home's
+ * least-privilege runtime.
+ */
+export async function mergePackedSource(packed: string): Promise<string> {
+  const files = unpackFiles(packed);
+  const tmpDir = await Deno.makeTempDir({ prefix: "realm-unpack-" });
+  try {
+    for (const [path, content] of files) {
+      const dest = join(tmpDir, path);
+      await Deno.mkdir(dirname(dest), { recursive: true });
+      await Deno.writeTextFile(dest, content);
+    }
+    return (await mergeSurfaces(tmpDir)).source;
+  } finally {
+    await Deno.remove(tmpDir, { recursive: true }).catch(() => {});
+  }
+}

package/src/tailwind-bin.ts

@@ -0,0 +1,56 @@
+// Resolve the `tailwindcss` binary by walking up from a starting dir
+// looking for `node_modules/.bin/tailwindcss`.
+//
+// Under byonm (root `nodeModulesDir: "manual"`) pnpm writes a `.bin` into
+// each workspace package's node_modules, so walking up from any caller
+// finds the nearest one.
+
+import { dirname, join } from "@std/path";
+
+export function findTailwindBin(startDir: string): string {
+  let dir = startDir;
+  while (true) {
+    const candidate = join(dir, "node_modules", ".bin", "tailwindcss");
+    try {
+      Deno.statSync(candidate);
+      return candidate;
+    } catch {
+      // keep walking
+    }
+    const parent = dirname(dir);
+    if (parent === dir) {
+      throw new Error(
+        `tailwindcss binary not found by walking up from ${startDir}. ` +
+          `Add @tailwindcss/cli to a workspace package's devDependencies and run pnpm install.`,
+      );
+    }
+    dir = parent;
+  }
+}
+
+/**
+ * Collect every `node_modules/.bin` directory from `startDir` up to
+ * the filesystem root. Useful for `realm-sdk dev`, which spawns esbuild
+ * + tailwindcss as bare-name commands and needs PATH primed when
+ * invoked directly via `deno run` (pnpm's bin-shimming only activates
+ * inside `pnpm <script>`). pnpm hoists most devDeps to the workspace
+ * root's `.bin`, but a workspace package's local `.bin` can shadow
+ * them — return all matches in walked order so callers prepend them
+ * to PATH from nearest to farthest.
+ */
+export function findBinDirs(startDir: string): string[] {
+  const out: string[] = [];
+  let dir = startDir;
+  while (true) {
+    const candidate = join(dir, "node_modules", ".bin");
+    try {
+      const stat = Deno.statSync(candidate);
+      if (stat.isDirectory) out.push(candidate);
+    } catch {
+      // keep walking
+    }
+    const parent = dirname(dir);
+    if (parent === dir) return out;
+    dir = parent;
+  }
+}

package/src/test/boot.ts

@@ -0,0 +1,192 @@
+// `t.boot` — render compile output to docker-compose, splice a
+// Playwright sidecar, and run `docker compose up -d --wait` against a
+// unique project name. The harness returns a handle whose `page` is a
+// browser context running *inside* the same compose network, so a
+// realm.ts test can drive `http://serviceName:port` URLs the way a
+// sibling container would.
+//
+// Boot reuses `compile`'s output. A failed compile is fatal here — the
+// thrown error carries the full issues list so a test's catch site can
+// still inspect it.
+//
+// Cleanup has two layers, both automatic:
+//   1. The describe.ts wrapper tears down every stack a test booted
+//      after the body resolves or throws.
+//   2. A module-level set of active handles is drained on
+//      `beforeunload`, so a test that crashes the process still tears
+//      down before exit.
+
+import { chromium, type Page } from "playwright-core";
+import { expect } from "@playwright/test";
+import { convertToDockerCompose, type ComposeService } from "../runtime/docker/compose/index.ts";
+import type { WorkerApp, WorkerState } from "../worker-protocol/index.ts";
+import { compile as runCompile } from "./compile.ts";
+import { RealmCompileError } from "../errors.ts";
+import { bringUpStack } from "../dev/stack.ts";
+import { fixtureIngressInput } from "./ingress-fixture.ts";
+import type { CompileFixture } from "./types.ts";
+
+// Pin this in lockstep with the playwright-core / @playwright/test
+// versions in package.json AND the tag built from
+// platform/playwright/Dockerfile. The run-server protocol is version-coupled.
+const PLAYWRIGHT_IMAGE = "benjick/realm-playwright:1.58.0";
+const PLAYWRIGHT_PORT = 14000;
+
+interface BootInputs<App> {
+  bundledCode: string;
+  appName: string;
+  fixture: CompileFixture<App>;
+}
+
+export interface RealmBootHandle {
+  page: Page;
+  /**
+   * Map of service name → compose hostname. Indexable by any string;
+   * the runtime always returns `${appId}_${name}` (the renderer's compose
+   * key, which is also the in-network DNS name).
+   */
+  services: Record<string, string>;
+  expect: typeof expect;
+}
+
+/**
+ * Internal: what `boot()` actually returns to the describe.ts wrapper.
+ * The wrapper registers `cleanup` against the test body and returns
+ * the bare `handle` to the test author.
+ */
+interface BootResult {
+  handle: RealmBootHandle;
+  cleanup: () => Promise<void>;
+}
+
+export async function boot<App>(inputs: BootInputs<App>): Promise<BootResult> {
+  const { bundledCode, appName, fixture } = inputs;
+  const result = await runCompile<App>({ bundledCode, appName, fixture });
+  if (result.error) {
+    throw new RealmCompileError([...result.issues]);
+  }
+
+  const appId = fixture.appId ?? appName;
+
+  // The harness renders in-process and never serializes to the wire, and the
+  // renderer reads only the fields it knows, so the resolve model's extra
+  // Home-only fields are simply ignored — cast straight to the wire shape.
+  const workerApp: WorkerApp = {
+    appId,
+    services: result.app.services as unknown as WorkerApp["services"],
+    ...(result.app.egress ? { egress: true } : {}),
+    externalNetworks: [],
+    dependsOn: {},
+    inlineFiles: {},
+    allowedAbsoluteHostPaths: [],
+  };
+  const wireState: WorkerState = {
+    apps: [workerApp],
+    artifactHashes: [],
+    workerNetworks: [],
+    // Nothing in the harness renders ingress, but the wire has no absent
+    // state for it - so state what this box's ingress would be, read off the
+    // same fixture the eval resolved `host.management()` from.
+    ingress: fixtureIngressInput(fixture.management),
+  };
+
+  // Single-app harness: one project per app, and boot only has one.
+  const compose = convertToDockerCompose(wireState)[0]!.compose;
+
+  // Mark every realm service for `realm-sdk down`. The Playwright
+  // sidecar gets the same label below.
+  for (const svc of Object.values(compose.services)) {
+    svc.labels = { ...svc.labels, "com.realm.test": "true" };
+  }
+
+  const hostPort = await pickFreePort();
+  const sidecarName = `${appId}_realm_playwright`;
+  compose.services[sidecarName] = playwrightSidecar({ appId, hostPort });
+
+  const project = `realm-test-${appId}-${nano(8)}`;
+  const tmpDir = await Deno.makeTempDir({ prefix: `${project}-` });
+
+  let stack;
+  try {
+    stack = await bringUpStack({ compose, project, tmpDir });
+  } catch (err) {
+    await Deno.remove(tmpDir, { recursive: true }).catch(() => {});
+    throw err;
+  }
+
+  const browser = await chromium.connect(`ws://localhost:${hostPort}/`);
+  const ctx = await browser.newContext();
+  const page = await ctx.newPage();
+
+  const services = new Proxy<Record<string, string>>(
+    {},
+    {
+      get: (_t, prop) => {
+        if (typeof prop !== "string") return undefined;
+        return `${appId}_${prop}`;
+      },
+    },
+  );
+
+  const cleanup = async () => {
+    await ctx.close().catch(() => {});
+    await browser.close().catch(() => {});
+    await stack.cleanup();
+    await Deno.remove(tmpDir, { recursive: true }).catch(() => {});
+  };
+
+  return { handle: { page, services, expect }, cleanup };
+}
+
+function playwrightSidecar(opts: { appId: string; hostPort: number }): ComposeService {
+  return {
+    image: PLAYWRIGHT_IMAGE,
+    labels: { "com.realm.test": "true" },
+    ports: [{ target: PLAYWRIGHT_PORT, published: opts.hostPort, protocol: "tcp" }],
+    networks: { [`realm_${opts.appId}`]: { aliases: ["realm-playwright"] } },
+    // `--wait` only waits for healthy containers. Without a check
+    // here, compose returns as soon as the container starts, which
+    // races chromium.connect against playwright's own boot. The check
+    // is the same TCP-probe e2e uses.
+    healthcheck: {
+      test: [
+        "CMD",
+        "node",
+        "-e",
+        `require('net').createConnection(${PLAYWRIGHT_PORT}, 'localhost').on('connect', () => process.exit(0)).on('error', () => process.exit(1))`,
+      ],
+      interval: "2s",
+      timeout: "3s",
+      retries: 60,
+      start_period: "5s",
+    },
+    cap_drop: ["ALL"],
+    cap_add: [],
+    security_opt: ["no-new-privileges:true"],
+    pids_limit: 4096,
+    ulimits: { nofile: 65536, nproc: 4096 },
+    restart: "no",
+    logging: { driver: "json-file", options: { "max-size": "10m", "max-file": "3" } },
+  };
+}
+
+async function pickFreePort(): Promise<number> {
+  // Another process could grab the port between close() and `docker compose
+  // up`; the bind failure surfaces at the docker-compose layer. The window is
+  // small enough not to matter across a test suite.
+  const listener = Deno.listen({ port: 0 });
+  const port = (listener.addr as Deno.NetAddr).port;
+  listener.close();
+  return port;
+}
+
+function nano(n: number): string {
+  // Project names are docker-compose-validated; a-z 0-9 only. Cryptographic
+  // uniqueness is not needed — collision odds with 8 lowercase hex chars are
+  // negligible across a single test run.
+  const bytes = new Uint8Array(Math.ceil(n / 2));
+  crypto.getRandomValues(bytes);
+  return Array.from(bytes, (b) => b.toString(16).padStart(2, "0"))
+    .join("")
+    .slice(0, n);
+}