@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/runtime/log-line.ts

@@ -0,0 +1,24 @@
+import { z } from "zod";
+
+/**
+ * The wire shape of one tapped container log line: transport metadata
+ * wrapping the raw, untouched line. Produced by the logs-server and read
+ * by the logs-client — the two ends of one unix socket, so this is the
+ * single source of truth for both.
+ *
+ * Distinct from apps/logger's `LogEnvelope`, which is the *stored* record
+ * a log-collector app derives from this (ISO timestamp, app-centric, with
+ * structured fields unwrapped). Don't conflate the transport line with a
+ * persisted record.
+ */
+export const logLineSchema = z.object({
+  appId: z.string(),
+  service: z.string(),
+  containerId: z.string(),
+  // Unix milliseconds.
+  timestamp: z.number(),
+  stream: z.enum(["stdout", "stderr"]),
+  line: z.string(),
+});
+
+export type LogLine = z.infer<typeof logLineSchema>;

package/src/runtime/logs-client/index.ts

@@ -0,0 +1,175 @@
+// Consumer-side helper for the worker's logs socket. Apps with the
+// `host.logs` permission mount the socket and use this to read a stream
+// of `LogLine` records.
+//
+// Two entry points:
+// - `connectToLogsSocket` — single-shot connect; throws on disconnect.
+//   Use for tests, one-off scans, or callers that want full control.
+// - `streamLogs` — auto-reconnecting wrapper. Tracks the latest yielded
+//   timestamp and resumes from it on drop with exponential backoff.
+//
+// Cursor persistence is left to the caller via `readCursor` /
+// `writeCursor`. The consumer owns the at-least-once boundary and must
+// only persist a cursor *after* it has successfully processed the line.
+
+import { type LogLine, logLineSchema } from "../log-line.ts";
+
+export type { LogLine };
+
+export interface ConnectOpts {
+  /**
+   * Cursor in unix ms. Omit (or pass `0`) to replay everything Docker
+   * still has on disk for each managed container — bounded by the
+   * default per-container retention (`max-size: 10m`, `max-file: 3`),
+   * so a fresh consumer pays a one-time, capped backfill rather than
+   * starting blind. To start "from now", pass `Date.now()`.
+   */
+  since?: number;
+  /** Abort to close the socket and end iteration. */
+  signal?: AbortSignal;
+}
+
+export interface StreamOpts extends ConnectOpts {
+  /** First reconnect delay (default 100ms). */
+  initialBackoffMs?: number;
+  /** Max reconnect delay (default 5000ms). */
+  maxBackoffMs?: number;
+  /**
+   * Called before each reconnect attempt (after the first). `lastError`
+   * is `undefined` if the previous connection ended cleanly (server EOF
+   * with no thrown error).
+   */
+  onReconnect?(attempt: number, lastError: unknown): void;
+}
+
+const encoder = new TextEncoder();
+
+export async function* connectToLogsSocket(
+  path: string,
+  opts: ConnectOpts = {},
+): AsyncGenerator<LogLine> {
+  const conn = await Deno.connect({ transport: "unix", path });
+  const onAbort = () => {
+    try {
+      conn.close();
+    } catch {}
+  };
+  opts.signal?.addEventListener("abort", onAbort);
+  // Per-instance, streaming decoder so a multi-byte UTF-8 sequence
+  // straddling two `conn.read()` calls survives the boundary.
+  const decoder = new TextDecoder("utf-8");
+  try {
+    const line = opts.since !== undefined ? `${JSON.stringify({ since: opts.since })}\n` : "\n";
+    await conn.write(encoder.encode(line));
+
+    const buf = new Uint8Array(8192);
+    let remainder = "";
+    while (true) {
+      const n = await conn.read(buf);
+      if (n === null) {
+        // Flush any bytes still buffered inside the decoder; partial UTF-8
+        // at EOF becomes the replacement char, which is fine — we only
+        // surface complete `\n`-terminated lines below.
+        remainder += decoder.decode();
+        return;
+      }
+      remainder += decoder.decode(buf.slice(0, n), { stream: true });
+      let nl: number;
+      while ((nl = remainder.indexOf("\n")) !== -1) {
+        const raw = remainder.slice(0, nl).trim();
+        remainder = remainder.slice(nl + 1);
+        if (!raw) continue;
+        try {
+          yield logLineSchema.parse(JSON.parse(raw));
+        } catch {
+          // Skip malformed lines; the server shouldn't emit them, but a
+          // partial buffer at the boundary shouldn't kill the stream.
+        }
+      }
+    }
+  } finally {
+    opts.signal?.removeEventListener("abort", onAbort);
+    try {
+      conn.close();
+    } catch {}
+  }
+}
+
+export async function* streamLogs(path: string, opts: StreamOpts = {}): AsyncGenerator<LogLine> {
+  const initial = opts.initialBackoffMs ?? 100;
+  const max = opts.maxBackoffMs ?? 5000;
+  let cursor = opts.since;
+  let backoff = initial;
+  let attempt = 0;
+  let lastError: unknown;
+
+  while (!opts.signal?.aborted) {
+    if (attempt > 0) opts.onReconnect?.(attempt, lastError);
+    try {
+      for await (const env of connectToLogsSocket(path, {
+        since: cursor,
+        signal: opts.signal,
+      })) {
+        cursor = env.timestamp;
+        backoff = initial;
+        attempt = 0;
+        yield env;
+      }
+      lastError = undefined;
+    } catch (err) {
+      lastError = err;
+    }
+
+    if (opts.signal?.aborted) return;
+    attempt++;
+    await sleep(backoff, opts.signal);
+    backoff = Math.min(backoff * 2, max);
+  }
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve();
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+/**
+ * Read a persisted cursor from `path`. Returns `undefined` if the file
+ * doesn't exist or its contents aren't a finite number.
+ */
+export async function readCursor(path: string): Promise<number | undefined> {
+  let raw: string;
+  try {
+    raw = await Deno.readTextFile(path);
+  } catch (err) {
+    if (err instanceof Deno.errors.NotFound) return undefined;
+    throw err;
+  }
+  const n = Number(raw.trim());
+  return Number.isFinite(n) ? n : undefined;
+}
+
+/**
+ * Write `ts` to `path` via tempfile + rename. A process crash mid-write
+ * can't truncate an existing cursor: either the old value or the new
+ * value is observable, never a partial write.
+ *
+ * Not durable across power loss — there's no fsync. Combined with the
+ * at-least-once semantics of `streamLogs` (resume re-delivers the
+ * boundary line) this is the right tradeoff: a lost cursor on power
+ * loss costs at most a small replay window.
+ */
+export async function writeCursor(path: string, ts: number): Promise<void> {
+  const tmp = `${path}.tmp`;
+  await Deno.writeTextFile(tmp, String(ts));
+  await Deno.rename(tmp, path);
+}

package/src/runtime/spawn-client/index.ts

@@ -0,0 +1,93 @@
+// Consumer-side helper for the worker's `spawn.sock` broker. Apps with
+// the `host.spawn` permission mount the socket and call `buildImage` to
+// run one disposable, fully-hardened box and stream its output.
+//
+// Mirrors `logs-client`: connect, write one request line, read the log
+// lines (→ `onLog`) and the single terminal line, return.
+
+import { z } from "zod";
+import type { SpawnLogLine, SpawnRequest } from "../docker/spawn-server/protocol.ts";
+
+export { DEFAULT_KANIKO_IMAGE, kanikoArgv, TAR_NAME } from "./kaniko.ts";
+
+export interface BuildResult {
+  /** The box's exit code (-1 on a broker-side failure carried via `error`). */
+  exit: number;
+  /**
+   * Basename of the broker-owned output dir (`output-<boxId>`) when the
+   * request set `output`; otherwise absent. The caller reads its artifacts
+   * back from `${artifactsRoot}/${outputKey}` and removes the dir.
+   */
+  outputKey?: string;
+}
+
+export interface BuildOpts {
+  /** Called for each line of the box's output as it streams. */
+  onLog?(line: SpawnLogLine): void;
+  /** Abort to close the socket and stop waiting. */
+  signal?: AbortSignal;
+}
+
+const encoder = new TextEncoder();
+
+/**
+ * Spawn one box for `req` and resolve once it exits. Rejects if the
+ * broker reports an `error`, or closes without a terminal line.
+ */
+export async function buildImage(
+  socketPath: string,
+  req: SpawnRequest,
+  opts: BuildOpts = {},
+): Promise<BuildResult> {
+  const conn = await Deno.connect({ transport: "unix", path: socketPath });
+  const onAbort = () => {
+    try {
+      conn.close();
+    } catch {}
+  };
+  opts.signal?.addEventListener("abort", onAbort);
+  // Per-instance, streaming decoder so a multi-byte UTF-8 sequence
+  // straddling two reads survives the boundary.
+  const decoder = new TextDecoder("utf-8");
+  try {
+    await conn.write(encoder.encode(JSON.stringify(req) + "\n"));
+
+    const buf = new Uint8Array(8192);
+    let remainder = "";
+    let terminal: { exit: number; outputKey?: string; error?: string | null } | undefined;
+    while (true) {
+      const n = await conn.read(buf);
+      if (n === null) break;
+      remainder += decoder.decode(buf.slice(0, n), { stream: true });
+      let nl: number;
+      while ((nl = remainder.indexOf("\n")) !== -1) {
+        const raw = remainder.slice(0, nl).trim();
+        remainder = remainder.slice(nl + 1);
+        if (!raw) continue;
+        let parsed: Record<string, unknown>;
+        try {
+          parsed = z.record(z.string(), z.unknown()).parse(JSON.parse(raw));
+        } catch {
+          continue; // skip a malformed line; the boundary buffer recovers
+        }
+        if ("log" in parsed) {
+          opts.onLog?.({
+            log: String(parsed.log ?? ""),
+            stream: parsed.stream === "stderr" ? "stderr" : "stdout",
+          });
+        } else if ("exit" in parsed) {
+          terminal = parsed as { exit: number; outputKey?: string; error?: string | null };
+        }
+      }
+    }
+
+    if (!terminal) throw new Error("spawn broker closed without a terminal line");
+    if (terminal.error) throw new Error(terminal.error);
+    return { exit: terminal.exit, outputKey: terminal.outputKey };
+  } finally {
+    opts.signal?.removeEventListener("abort", onAbort);
+    try {
+      conn.close();
+    } catch {}
+  }
+}

package/src/runtime/spawn-client/kaniko.ts

@@ -0,0 +1,79 @@
+// Map a `build:` recipe to kaniko executor argv for the spawn broker.
+//
+// The broker is image-agnostic: the caller supplies the builder image +
+// argv, and the broker runs one disposable hardened box. For a build, the
+// image is kaniko (DEFAULT_KANIKO_IMAGE) and the argv is what `kanikoArgv`
+// produces:
+//
+//   buildImage(socket, { image: DEFAULT_KANIKO_IMAGE, context, argv: kanikoArgv(recipe) })
+//
+// The box NEVER pushes. It has no network path to a registry (it sits on
+// the egress-only untrusted-net DMZ, with nothing internal reachable), so
+// kaniko always runs `--no-push`. Two shapes:
+//
+// Build-only (no `output`): kaniko builds from the context the broker mounts
+// read-only at /workspace and discards the result — just a compile check.
+//
+// Output (`output`): kaniko additionally writes the built image as an OCI
+// tarball to the broker's RW output mount (`--tar-path`). The caller (in
+// prod, Forge) reads that tarball back and pushes it to the registry itself,
+// over its own trusted network — so the registry host never appears here and
+// the box never touches it.
+
+import type { BuildRecipe } from "../../service-definition.ts";
+
+// Default builder image: canonical kaniko. It runs as root and never calls
+// unshare(CLONE_NEWUSER), so it builds clean under the broker's envelope:
+// HARDENING plus the file-extraction caps (FOWNER, DAC_OVERRIDE, SETFCAP,
+// FSETID) spec.ts adds so base rootfs with non-root-owned files
+// (node:22-alpine's /home/node) unpack faithfully — the property is
+// kaniko's, not a specific distro's.
+export const DEFAULT_KANIKO_IMAGE = "gcr.io/kaniko-project/executor:latest";
+
+// Where the broker bind-mounts the staged context, read-only, in the box.
+// Must match `CONTEXT_MOUNT` in spawn-server/spec.ts — kaniko.test.ts pins
+// the two together (the client can't import spec.ts, which drags in
+// `@local/docker`).
+const CONTEXT_MOUNT = "/workspace";
+
+// Where the broker bind-mounts its output dir, read-write, in the box.
+// Must match `OUTPUT_MOUNT` in spawn-server/spec.ts — kaniko.test.ts pins
+// the two together, same as `CONTEXT_MOUNT`.
+const OUTPUT_MOUNT = "/output";
+
+/**
+ * Basename of the OCI image tarball kaniko writes under the broker's output
+ * mount. The cross-layer contract: the box writes `${OUTPUT_MOUNT}/${TAR_NAME}`,
+ * and the caller reads the same basename back off the artifacts dir
+ * (`${artifactsRoot}/${outputKey}/${TAR_NAME}`) to push it.
+ */
+export const TAR_NAME = "image.tar";
+
+/**
+ * In-box path kaniko writes the OCI image tarball to (`/output/image.tar`).
+ * Lives in the broker's RW output mount; the caller reads it back off the
+ * artifacts dir. kaniko.test.ts pins this against spec.ts's OUTPUT_MOUNT so
+ * the local copy can't drift from the broker.
+ */
+export const TAR_PATH = `${OUTPUT_MOUNT}/${TAR_NAME}`;
+
+/**
+ * kaniko executor flags for `recipe`. The context is the broker's fixed
+ * `/workspace` mount; `recipe.context` selects a sub-path within it.
+ *
+ * Always `--no-push` — the box never reaches a registry. With `output` it
+ * also emits `--tar-path=${TAR_PATH}`, so kaniko writes the built image as an
+ * OCI tarball to the broker's RW output mount for the caller to read back and
+ * push.
+ */
+export function kanikoArgv(recipe: BuildRecipe, output?: boolean): string[] {
+  const contextDir = recipe.context ? `${CONTEXT_MOUNT}/${recipe.context}` : CONTEXT_MOUNT;
+  // Build-only just compile-checks; an output run also tars the result for
+  // the caller to push. Either way the box never pushes (`--no-push`).
+  const argv = output ? ["--no-push", `--tar-path=${TAR_PATH}`] : ["--no-push"];
+  argv.push(`--context=dir://${contextDir}`, `--dockerfile=${recipe.dockerfile ?? "Dockerfile"}`);
+  for (const [key, value] of Object.entries(recipe.buildArgs ?? {})) {
+    argv.push(`--build-arg=${key}=${value}`);
+  }
+  return argv;
+}

package/src/runtime/stats-client/index.ts

@@ -0,0 +1,65 @@
+// Consumer-side helper for the worker's stats socket. Apps with the
+// `host.stats` permission mount the socket dir and use this to read a
+// point-in-time `StatsSample` of the host and every container on it.
+//
+// One entry point, because the socket is request/response: connect, ask,
+// parse, close. There's no stream to resume and no cursor to persist — the
+// contrast with logs-client is the whole design. Cadence is the caller's:
+// the worker holds no timer and caches nothing, so a sample you don't store
+// is gone.
+
+import { type StatsSample, statsSampleSchema } from "../stats-sample.ts";
+
+export type { ContainerSample, DiskSample, HostSample, StatsSample } from "../stats-sample.ts";
+export { OTHER_APP } from "../stats-sample.ts";
+
+export interface SampleOpts {
+  /** Abort to close the socket and fail the pending sample. */
+  signal?: AbortSignal;
+}
+
+const encoder = new TextEncoder();
+
+/** Take one sample from the stats socket at `path` (the `stats.sock` inside the mounted dir). */
+export async function sampleStats(path: string, opts: SampleOpts = {}): Promise<StatsSample> {
+  const conn = await Deno.connect({ transport: "unix", path });
+  const onAbort = () => {
+    try {
+      conn.close();
+    } catch {}
+  };
+  opts.signal?.addEventListener("abort", onAbort);
+  // Per-instance, streaming decoder so a multi-byte UTF-8 sequence straddling
+  // two `conn.read()` calls survives the boundary.
+  const decoder = new TextDecoder("utf-8");
+  try {
+    await conn.write(encoder.encode(JSON.stringify({ sample: true }) + "\n"));
+
+    const buf = new Uint8Array(65536);
+    let raw = "";
+    while (true) {
+      const n = await conn.read(buf);
+      if (n === null) {
+        raw += decoder.decode();
+        break;
+      }
+      raw += decoder.decode(buf.slice(0, n), { stream: true });
+      // The server writes one line and closes, but a sample large enough to
+      // straddle reads arrives in pieces — stop at the newline rather than
+      // waiting for EOF.
+      const nl = raw.indexOf("\n");
+      if (nl !== -1) {
+        raw = raw.slice(0, nl);
+        break;
+      }
+    }
+
+    if (!raw.trim()) throw new Error("stats socket closed without a sample");
+    return statsSampleSchema.parse(JSON.parse(raw));
+  } finally {
+    opts.signal?.removeEventListener("abort", onAbort);
+    try {
+      conn.close();
+    } catch {}
+  }
+}

package/src/runtime/stats-sample.ts

@@ -0,0 +1,95 @@
+import { z } from "zod";
+
+/**
+ * The wire shape of one point-in-time reading of a worker's host and every
+ * container on it. Produced by the stats-server and read by the stats-client —
+ * the two ends of one unix socket, so this is the single source of truth for
+ * both.
+ *
+ * Mirrors `log-line.ts`, with one deliberate difference: logs are a stream of
+ * events, stats are a request/response sample. The worker holds no timer, no
+ * history, and no cache — it reads the host when asked and answers.
+ *
+ * CPU, network, and block IO are CUMULATIVE COUNTERS, not rates. A consumer
+ * that wants a rate keeps the previous sample and divides by elapsed time.
+ * Counters average correctly over any interval; instantaneous gauges alias.
+ * Memory and pids are gauges because they're levels, with no counter to diff.
+ *
+ * @see https://yourrealm.eu/docs/realm-ts/host/stats/
+ */
+
+/** One real filesystem mounted on the host. Keyed by mount point. */
+export const diskSampleSchema = z.object({
+  path: z.string(),
+  total: z.number(),
+  available: z.number(),
+});
+
+export const hostSampleSchema = z.object({
+  kernelVersion: z.string(),
+  cores: z.number(),
+  /** 1, 5, 15 minute. Read against `cores`. */
+  loadAvg: z.tuple([z.number(), z.number(), z.number()]),
+  memoryTotal: z.number(),
+  memoryAvailable: z.number(),
+  swapTotal: z.number(),
+  swapUsed: z.number(),
+  /** Seconds since boot. */
+  uptime: z.number(),
+  disks: z.array(diskSampleSchema),
+});
+
+export const containerSampleSchema = z.object({
+  /** The `realm.app` label; {@link OTHER_APP} for non-Realm containers. */
+  app: z.string(),
+  service: z.string(),
+  /**
+   * Docker's own container name, always present. The only human-readable handle
+   * an {@link OTHER_APP} container has — `app` and `service` are both label-derived,
+   * so they say nothing about a container Realm didn't start.
+   */
+  name: z.string(),
+  containerId: z.string(),
+  /** running | exited | ... Counters read 0 unless this is "running". */
+  state: z.string(),
+  /** healthy | unhealthy | starting, if the container has a healthcheck. */
+  health: z.string().optional(),
+  startedAt: z.string(),
+  restartCount: z.number(),
+
+  // Cumulative counters.
+  cpuUsage: z.number(),
+  netRx: z.number(),
+  netTx: z.number(),
+  blockRead: z.number(),
+  blockWrite: z.number(),
+
+  // Gauges.
+  memoryUsage: z.number(),
+  memoryLimit: z.number(),
+  pids: z.number(),
+});
+
+export const statsSampleSchema = z.object({
+  /** ISO-8601, when the worker took the reading. */
+  ts: z.string(),
+  host: hostSampleSchema,
+  docker: z.object({
+    version: z.string(),
+    containersRunning: z.number(),
+    containersTotal: z.number(),
+  }),
+  containers: z.array(containerSampleSchema),
+});
+
+/**
+ * Reserved app name for containers on the host that Realm doesn't manage
+ * (no `realm.app` label). Underscore-prefixed so it can't collide with a real
+ * app id, which `nameSchema` constrains to hyphens.
+ */
+export const OTHER_APP = "__other";
+
+export type DiskSample = z.infer<typeof diskSampleSchema>;
+export type HostSample = z.infer<typeof hostSampleSchema>;
+export type ContainerSample = z.infer<typeof containerSampleSchema>;
+export type StatsSample = z.infer<typeof statsSampleSchema>;

package/src/safe-json.ts

@@ -0,0 +1,37 @@
+/**
+ * `JSON.parse` variant that strips prototype-pollution vectors (and any
+ * additional caller-supplied keys) during parsing. The reviver runs
+ * before the parsed object is constructed, so a hostile input like
+ * `{"__proto__": {"polluted": true}}` cannot result in
+ * `({}).polluted === true` even if downstream code does
+ * `target[key] = parsed[key]` in a loop.
+ */
+
+const DANGEROUS_KEYS: ReadonlySet<string> = new Set(["__proto__", "constructor", "prototype"]);
+
+/**
+ * Parse JSON and drop dangerous keys. `extraStrip` lets the caller
+ * remove additional keys (e.g. timestamp aliases the consumer
+ * overwrites with its own value) at parse time, so the consumer doesn't
+ * need a separate skip-list during its own iteration.
+ *
+ * Keys the caller wants to *consume* (read the value, then place it
+ * elsewhere — e.g. `msg` → `envelope.msg`) must NOT be in `extraStrip`;
+ * the consumer reads them directly off the parsed object.
+ *
+ * @deprecated Prefer `schema.parse(JSON.parse(text))`. A Zod schema drops
+ * `__proto__` in every mode (object / record / passthrough), preserves
+ * legitimately-named `constructor` / `prototype` fields that this reviver
+ * wrongly strips, and validates the shape on top. Plain `JSON.parse` does
+ * not pollute `Object.prototype` on its own, so this function only earns
+ * its keep where attacker-controlled keys from an *open dictionary* are
+ * then assigned via `obj[key] = value` — a real pollution sink. The lone
+ * such caller is apps/logger's envelope builder.
+ */
+export function hardenedJsonParse(text: string, extraStrip?: ReadonlySet<string>): unknown {
+  return JSON.parse(text, (key, value) => {
+    if (DANGEROUS_KEYS.has(key)) return undefined;
+    if (extraStrip?.has(key)) return undefined;
+    return value;
+  });
+}