@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/dev/stack.ts

@@ -0,0 +1,108 @@
+// Shared compose-stack lifecycle: write a compose file, run
+// `docker compose up -d --wait`, and tear down deterministically. Both
+// `realm-sdk test --e2e` (test/boot.ts) and `realm-sdk dev` (dev/server.ts) go
+// through it. The activeStacks registry and beforeunload hook are module-level
+// so a process driving both code paths drains every stack on exit.
+
+import { join } from "@std/path";
+import { stringify as yamlStringify } from "yaml";
+import type { ComposeConfig } from "../runtime/docker/compose/types.ts";
+
+interface StackInputs {
+  /** Pre-built docker-compose object (services, networks, etc.). */
+  compose: ComposeConfig;
+  /**
+   * Compose project name. Docker-compose-validates to a-z 0-9; pick
+   * something unique enough to avoid collisions across concurrent runs.
+   */
+  project: string;
+  /** Working directory for the compose file. Caller owns its lifetime. */
+  tmpDir: string;
+}
+
+interface StackHandle {
+  project: string;
+  composeFile: string;
+  /**
+   * Take down the stack and remove its volumes. Idempotent — calling
+   * twice (e.g. once from a test's finally + again from the exit hook)
+   * is safe.
+   */
+  cleanup: () => Promise<void>;
+}
+
+interface ActiveStack {
+  project: string;
+  composeFile: string;
+}
+
+const activeStacks = new Set<ActiveStack>();
+let exitHookRegistered = false;
+
+function registerExitHook(): void {
+  if (exitHookRegistered) return;
+  exitHookRegistered = true;
+  // Deno emits `beforeunload` on a clean exit; we attach there so any
+  // stack a caller forgot still goes down before the process leaves.
+  // Ctrl-C / uncaught throws don't fire this — `realm-sdk down` is the
+  // recovery path for those.
+  globalThis.addEventListener("beforeunload", () => {
+    for (const stack of activeStacks) {
+      // Sync best-effort: we can't await inside beforeunload, but
+      // `Deno.Command.outputSync` blocks the event loop until done.
+      try {
+        new Deno.Command("docker", {
+          args: ["compose", "-p", stack.project, "-f", stack.composeFile, "down", "-v"],
+          stdout: "null",
+          stderr: "null",
+        }).outputSync();
+      } catch {
+        // Ignore — best effort.
+      }
+    }
+    activeStacks.clear();
+  });
+}
+
+async function downStack(project: string, composeFile: string): Promise<void> {
+  const cmd = new Deno.Command("docker", {
+    args: ["compose", "-p", project, "-f", composeFile, "down", "-v"],
+    stdout: "null",
+    stderr: "null",
+  });
+  await cmd.output().catch(() => {});
+}
+
+export async function bringUpStack(inputs: StackInputs): Promise<StackHandle> {
+  const { compose, project, tmpDir } = inputs;
+  const composeFile = join(tmpDir, "docker-compose.yml");
+  await Deno.writeTextFile(composeFile, yamlStringify(compose as unknown));
+
+  registerExitHook();
+
+  const upCmd = new Deno.Command("docker", {
+    args: ["compose", "-p", project, "-f", composeFile, "up", "-d", "--wait"],
+    stdout: "piped",
+    stderr: "piped",
+  });
+  const upOut = await upCmd.output();
+  if (upOut.code !== 0) {
+    const stderr = new TextDecoder().decode(upOut.stderr);
+    const stdout = new TextDecoder().decode(upOut.stdout);
+    // Best-effort cleanup before throwing — leaving the stack up after
+    // a failure makes the next run fight over names.
+    await downStack(project, composeFile);
+    throw new Error(`docker compose up failed (exit ${upOut.code})\n${stdout}\n${stderr}`);
+  }
+
+  const stack: ActiveStack = { project, composeFile };
+  activeStacks.add(stack);
+
+  const cleanup = async () => {
+    if (!activeStacks.has(stack)) return;
+    activeStacks.delete(stack);
+    await downStack(project, composeFile);
+  };
+
+  return { project, composeFile, cleanup };
+}

package/src/dev/state.ts

@@ -0,0 +1,159 @@
+// Shared compile + bring-up plumbing for `realm-sdk dev`. The HTTP server
+// (dev/server.ts) and the in-process install handler (dev/install-handler.ts)
+// both go through it, so they can't drift on how a realm.ts becomes a bundled,
+// compiled, compose-running stack: `compileAndBundle` returns a `DevState` and
+// `bringUp` runs `docker compose up -d --wait` against the resolved state.
+
+import type { Grants } from "../home.ts";
+import { bundle } from "../bundle.ts";
+import { bundleUiSurfaces, type UiSurfaceBundle } from "../bundle-ui.ts";
+import { mergeSurfaces } from "../surfaces.ts";
+import { compile } from "../test/compile.ts";
+import { convertToDockerCompose } from "../runtime/docker/compose/index.ts";
+import { extractAppName } from "../sandbox.ts";
+import { type CompileIssue, formatIssue } from "../errors.ts";
+import type { HomeApp } from "../resolve.ts";
+import type { Violation } from "../security.ts";
+import type { IngressInput, WorkerApp, WorkerState } from "../worker-protocol/index.ts";
+import { fixtureIngressInput } from "../test/ingress-fixture.ts";
+import { bringUpStack } from "./stack.ts";
+import { loadDevFixture, stableDevSeed } from "./fixture.ts";
+
+/**
+ * Compile failure thrown by `compileAndBundle`. Carries the original
+ * issue list (for the human-readable stderr summary on dev boot / file
+ * watcher reloads) and the raw violations (so the install panel can
+ * render the structured per-permission list instead of a flat blob).
+ */
+export class DevCompileError extends Error {
+  constructor(
+    message: string,
+    public readonly issues: readonly CompileIssue[],
+    public readonly violations: readonly Violation[],
+  ) {
+    super(message);
+    this.name = "DevCompileError";
+  }
+}
+
+export interface DevState {
+  appId: string;
+  appName: string;
+  source: string;
+  resolvedState: HomeApp;
+  bundles: Map<string, UiSurfaceBundle>;
+  /** Map of `${kind}:${key}` → hash. */
+  surfaceHashes: Map<string, string>;
+  /**
+   * This box's ingress, off the dev fixture's `management` block. Carried on
+   * the state because `bringUp` needs it for the wire and the fixture is read
+   * here - it is the one piece of the wire dev cannot reconstruct downstream.
+   */
+  ingress: IngressInput;
+}
+
+interface CompileAndBundleOpts {
+  cwd: string;
+  realmFile: string;
+  /**
+   * In-flight install POST from the panel. When set, `source` is the packed
+   * artifact the panel pre-filled (and the operator may have edited) — used
+   * verbatim, not re-read from disk — and `config` / `grants` replace the
+   * on-disk fixture's values for this one compile. `handleInstall` persists
+   * them to `realm-sdk.dev.ts` immediately after, so subsequent reloads pick
+   * them up from disk normally. `peers` + `seed` always come from the disk
+   * fixture so peer-world resolution and password derivation match the
+   * file-watcher path.
+   */
+  override?: {
+    source: string;
+    config: Record<string, unknown>;
+    grants: Grants;
+  };
+}
+
+export async function compileAndBundle(opts: CompileAndBundleOpts): Promise<DevState> {
+  // The install panel POSTs the packed artifact it pre-filled (what actually
+  // ships), so Preview and Install read the exact same bytes. The file-watcher
+  // path has no override and merges the app's live files off disk instead. Both
+  // feed bundle(), which unpacks + merges a packed source and passes a plain
+  // one straight through.
+  const source = opts.override ? opts.override.source : (await mergeSurfaces(opts.cwd)).source;
+  const bundledCode = await bundle(source);
+
+  const { fixture: rawFixture, source: fixtureSource } = await loadDevFixture(opts.cwd);
+  const fixture = {
+    ...rawFixture,
+    seed: rawFixture.seed ?? stableDevSeed(opts.cwd),
+    ...(opts.override
+      ? {
+          config: opts.override.config as never,
+          grants: opts.override.grants as never,
+        }
+      : {}),
+  };
+
+  const appName = await extractAppName(bundledCode);
+  const appId = fixture.appId ?? appName;
+
+  const result = await compile({ bundledCode, appName, fixture });
+  if (result.error) {
+    const lines = [
+      `compile failed${fixtureSource ? ` (using ${fixtureSource})` : " (no realm-sdk.dev.ts found — using empty fixture)"}:`,
+      ...result.issues.map((i) => `  - ${formatIssue(i)}`),
+    ];
+    throw new DevCompileError(lines.join("\n"), result.issues, result.violations);
+  }
+
+  const surfaces = await bundleUiSurfaces(source, bundledCode);
+  const bundles = new Map<string, UiSurfaceBundle>();
+  const surfaceHashes = new Map<string, string>();
+  for (const b of surfaces) {
+    bundles.set(b.hash, b);
+    surfaceHashes.set(`${b.surface.kind}:${b.surface.key}`, b.hash);
+  }
+
+  return {
+    appId,
+    appName,
+    source,
+    resolvedState: result.app,
+    bundles,
+    surfaceHashes,
+    ingress: fixtureIngressInput(fixture.management),
+  };
+}
+
+export function bringUp(state: DevState, tmpDir: string) {
+  const workerApp: WorkerApp = {
+    appId: state.appId,
+    // Dev renders in-process and never serializes to the wire, and the
+    // renderer reads only the fields it knows, so the resolve model's extra
+    // Home-only fields are simply ignored — cast straight to the wire shape.
+    services: state.resolvedState.services as unknown as WorkerApp["services"],
+    ...(state.resolvedState.egress ? { egress: true } : {}),
+    externalNetworks: [],
+    dependsOn: {},
+    inlineFiles: {},
+    allowedAbsoluteHostPaths: [],
+  };
+  const wireState: WorkerState = {
+    apps: [workerApp],
+    artifactHashes: [],
+    workerNetworks: [],
+    ingress: state.ingress,
+  };
+  // Single-app harness: the renderer returns one project per app, and
+  // dev only ever has one.
+  const compose = convertToDockerCompose(wireState)[0]!.compose;
+  // Mark every service so `realm-sdk down` (com.realm.test=true) won't
+  // sweep them, and so an operator can grep their docker ps. Also inject
+  // REALM_SDK_DEV=1 so apps can opt into dev-only behavior (synthetic
+  // fixtures, skipping connections that need a real worker, etc.).
+  for (const svc of Object.values(compose.services)) {
+    svc.labels = { ...svc.labels, "com.realm.dev": "true" };
+    svc.environment = { ...svc.environment, REALM_SDK_DEV: "1" };
+  }
+  const project = `realm-dev-${state.appId}`;
+  return bringUpStack({ compose, project, tmpDir });
+}

package/src/docker.ts

@@ -0,0 +1,96 @@
+// Docker registry v2 gate helpers — for use inside `verifyAnonymousRequest`
+// when an app fronts an OCI registry (e.g. Forge's Zot). A `docker pull`
+// answers a 401-with-Basic-challenge with `Authorization: Basic
+// base64(username:password)`; `checkDockerBasic` compares that password to the
+// app's `stableSecret(...)`, and `dockerChallenge` is the 401 that triggers it.
+//
+// These run in the QuickJS gate sandbox, which shims only
+// Request/Response/Headers/URL — there is no `atob` and no Web Crypto — so the
+// code is pure JS: base64 is decoded by hand and the compare is a plain loop.
+// Keep it that way (no `Deno`/`atob`/`TextEncoder`) or the gate bundle breaks
+// at load under QuickJS.
+
+/** Standard-base64 alphabet (RFC 4648). */
+const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+/**
+ * Decode standard base64 to a Latin-1 string, or `null` on an invalid
+ * character. Pure JS (no `atob`) so it runs in the gate sandbox. Exact for
+ * well-formed base64; looser than `atob` on a malformed *length* — a dangling
+ * final partial group is dropped rather than rejected. That only ever drops a
+ * trailing group (never alters earlier bytes), so it's safe for this
+ * fail-closed credential check: a bad credential still can't match the secret.
+ * Registry credentials are ASCII (`username:hexsecret`), so the byte-as-charcode
+ * string round-trips them exactly.
+ */
+function decodeBase64(input: string): string | null {
+  let s = input;
+  // Drop CR/LF and trailing `=` padding; the bit accumulator handles the rest.
+  s = s.replace(/[\r\n]/g, "");
+  while (s.length > 0 && s[s.length - 1] === "=") s = s.slice(0, -1);
+
+  let out = "";
+  let acc = 0;
+  let bits = 0;
+  for (let i = 0; i < s.length; i++) {
+    const idx = B64.indexOf(s[i]!);
+    if (idx < 0) return null;
+    acc = (acc << 6) | idx;
+    bits += 6;
+    if (bits >= 8) {
+      bits -= 8;
+      out += String.fromCharCode((acc >> bits) & 0xff);
+    }
+  }
+  return out;
+}
+
+/** Length-then-XOR compare; no early return on the first differing byte. */
+function constantTimeEqual(a: string, b: string): boolean {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
+
+/**
+ * `true` when the request carries an `Authorization: Basic` header whose
+ * password half equals `expectedSecret`. The username half is ignored — the
+ * single shared `stableSecret(...)` is the whole credential, so any username
+ * with the right password passes. Anything malformed (no header, wrong scheme,
+ * bad base64, no colon) is a plain `false`, so the caller falls through to
+ * {@link dockerChallenge}.
+ *
+ * ```ts
+ * verifyAnonymousRequest: (request, stableSecret) =>
+ *   checkDockerBasic(request, stableSecret("registry")) || dockerChallenge();
+ * ```
+ */
+export function checkDockerBasic(request: Request, expectedSecret: string): boolean {
+  const header = request.headers.get("authorization");
+  if (!header) return false;
+  const space = header.indexOf(" ");
+  if (space < 0) return false;
+  if (header.slice(0, space).toLowerCase() !== "basic") return false;
+
+  const decoded = decodeBase64(header.slice(space + 1).trim());
+  if (decoded == null) return false;
+  const colon = decoded.indexOf(":");
+  if (colon < 0) return false;
+  return constantTimeEqual(decoded.slice(colon + 1), expectedSecret);
+}
+
+/**
+ * The `401` that makes a `docker pull` retry with Basic credentials: an empty
+ * body plus `WWW-Authenticate: Basic`. Return it from `verifyAnonymousRequest`
+ * for an unauthenticated (or wrong-credential) registry request.
+ */
+export function dockerChallenge(realm = "Realm registry"): Response {
+  return new Response(null, {
+    status: 401,
+    headers: {
+      "WWW-Authenticate": `Basic realm="${realm}"`,
+      "Docker-Distribution-Api-Version": "registry/2.0",
+    },
+  });
+}

package/src/errors.ts

@@ -0,0 +1,61 @@
+/**
+ * Error contract for the realm.ts → state compile pipeline.
+ *
+ * Modeled on Zod: one class with a list of issues, where each issue is a
+ * discriminated union keyed on `kind`. Single-issue throw sites pass one
+ * issue; accumulating boundaries (Phase 2 `compile()`, multi-violation
+ * grant checks) pass a list. The list is always the public surface — a
+ * one-issue error is just a one-element list.
+ *
+ * No `publicMessage` on the throw site. The HTTP boundary decides what's
+ * safe to render per-kind in one place; throw sites just describe what
+ * happened.
+ */
+export type CompileIssue =
+  | { kind: "eval"; message: string; line?: number; column?: number }
+  | { kind: "validation"; path: string[]; message: string }
+  | { kind: "permission"; missing: string };
+
+export class RealmCompileError extends Error {
+  readonly issues: readonly CompileIssue[];
+
+  constructor(input: CompileIssue | CompileIssue[]) {
+    const list = Array.isArray(input) ? input : [input];
+    super(formatHeader(list));
+    this.issues = Object.freeze([...list]) as readonly CompileIssue[];
+    this.name = "RealmCompileError";
+  }
+}
+
+/**
+ * A `host.silo` require-map the Silo doesn't meet yet. Its own subclass so
+ * the eval pass structure can tell it apart from a permanent failure: a
+ * provider that evaluates later may satisfy it, so this is the one drop the
+ * fixpoint loop retries.
+ *
+ * Lives here rather than in Home because the thrower is `host.silo`'s
+ * resolver, which lives in the SDK.
+ */
+export class SiloRequiresError extends RealmCompileError {}
+
+function formatHeader(issues: readonly CompileIssue[]): string {
+  if (issues.length === 1) return formatIssue(issues[0]!);
+  return `${issues.length} compile errors`;
+}
+
+/**
+ * Render a single issue as a human-readable string. Used for the
+ * Error.message header (stack traces, logs) - NOT for HTTP responses,
+ * which redact eval issues to a generic string. The HTTP boundary
+ * lives in Home's `parseRealmError`.
+ */
+export function formatIssue(issue: CompileIssue): string {
+  switch (issue.kind) {
+    case "eval":
+      return issue.message;
+    case "validation":
+      return `${issue.path.join(".")}: ${issue.message}`;
+    case "permission":
+      return `missing grant: ${issue.missing}`;
+  }
+}