@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/worker-protocol/uplink/test-support.ts

@@ -0,0 +1,188 @@
+// Test-only plumbing. A `Wire` is two cross-wired in-process byte streams
+// standing in for the plaintext side of a Noise transport, taps every frame
+// that crosses it, and can hold bytes back until the test releases them so
+// backpressure and frame priority are observable.
+
+import { type Frame, FrameParser, frameTypeName } from "./frame.ts";
+import { type ByteDuplex, type Limits, Mux, type Stream } from "./mux.ts";
+
+interface WireFrame extends Frame {
+  /** "ab" is a→b. */
+  dir: "ab" | "ba";
+  name: string;
+}
+
+interface Wire {
+  a: ByteDuplex;
+  b: ByteDuplex;
+  /** Every frame that has actually crossed, both directions, in order. */
+  seen: WireFrame[];
+  /** Lets `n` more chunks through a→b. Only meaningful when gated. */
+  release(n: number): void;
+}
+
+/** `gated` holds every a→b chunk until `release` is called. */
+export function wire(opts: { gated?: boolean } = {}): Wire {
+  const seen: WireFrame[] = [];
+  const permits = new Permits();
+  const ab = channel(seen, "ab", opts.gated === true ? permits : undefined);
+  const ba = channel(seen, "ba", undefined);
+  return {
+    a: { readable: ba.readable, writable: ab.writable },
+    b: { readable: ab.readable, writable: ba.writable },
+    seen,
+    release: (n: number) => permits.grant(n),
+  };
+}
+
+interface MuxPair {
+  a: Mux;
+  b: Mux;
+  seen: WireFrame[];
+  release(n: number): void;
+  close(): Promise<void>;
+}
+
+export function muxPair(limits: Partial<Limits> = {}, opts: { gated?: boolean } = {}): MuxPair {
+  const w = wire(opts);
+  const a = new Mux(w.a, { side: "initiator", ...limits });
+  const b = new Mux(w.b, { side: "responder", ...limits });
+  return {
+    a,
+    b,
+    seen: w.seen,
+    release: w.release,
+    close: async () => {
+      // A gated wire holds an in-flight write; abort would wait on it.
+      w.release(1_000_000);
+      a.close("test over");
+      b.close("test over");
+      await Promise.all([a.closed, b.closed]);
+    },
+  };
+}
+
+interface Accepted {
+  stream: Stream;
+  header: Uint8Array;
+}
+
+/** Turns `onOpen` into a queue the test pulls from. */
+export function acceptor(mux: Mux): () => Promise<Accepted> {
+  const queue: Accepted[] = [];
+  const waiting: Array<(accepted: Accepted) => void> = [];
+  mux.onOpen((stream, header) => {
+    const next = waiting.shift();
+    if (next === undefined) queue.push({ stream, header });
+    else next({ stream, header });
+  });
+  return () => {
+    const next = queue.shift();
+    if (next !== undefined) return Promise.resolve(next);
+    return new Promise<Accepted>((resolve) => {
+      waiting.push(resolve);
+    });
+  };
+}
+
+export function dataBytes(seen: WireFrame[], dir: "ab" | "ba"): number {
+  return seen
+    .filter((f) => f.dir === dir && f.name === "DATA")
+    .reduce((sum, f) => sum + f.payload.length, 0);
+}
+
+export function named(seen: WireFrame[], name: string, dir?: "ab" | "ba"): WireFrame[] {
+  return seen.filter((f) => f.name === name && (dir === undefined || f.dir === dir));
+}
+
+/**
+ * Reads and discards. A raw peer must do this or the tapped channel toward
+ * it stalls on backpressure and nothing is ever observed.
+ */
+export function sink(readable: ReadableStream<Uint8Array>): Promise<void> {
+  return readable.pipeTo(new WritableStream()).catch(() => {});
+}
+
+export async function collect(readable: ReadableStream<Uint8Array>): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  for await (const chunk of readable) {
+    chunks.push(chunk);
+    size += chunk.length;
+  }
+  const out = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+}
+
+/** Deterministic filler so a round trip can be compared byte for byte. */
+export function pattern(size: number, seed = 1): Uint8Array {
+  const out = new Uint8Array(size);
+  let x = seed >>> 0;
+  for (let i = 0; i < size; i++) {
+    x = (x * 1664525 + 1013904223) >>> 0;
+    out[i] = x >>> 24;
+  }
+  return out;
+}
+
+/** Lets every queued stream job run. Macrotask ticks, so nothing is left mid-pipe. */
+export async function settle(rounds = 6): Promise<void> {
+  for (let i = 0; i < rounds; i++) {
+    await new Promise<void>((resolve) => setTimeout(resolve, 0));
+  }
+}
+
+export async function waitFor(predicate: () => boolean, what = "condition"): Promise<void> {
+  for (let i = 0; i < 500; i++) {
+    if (predicate()) return;
+    await new Promise<void>((resolve) => setTimeout(resolve, 0));
+  }
+  throw new Error(`timed out waiting for ${what}`);
+}
+
+class Permits {
+  #available = 0;
+  #resolve: (() => void) | undefined;
+
+  async take(): Promise<void> {
+    while (this.#available <= 0) {
+      await new Promise<void>((resolve) => {
+        this.#resolve = resolve;
+      });
+    }
+    this.#available--;
+  }
+
+  grant(n: number): void {
+    this.#available += n;
+    const resolve = this.#resolve;
+    this.#resolve = undefined;
+    resolve?.();
+  }
+}
+
+function channel(
+  seen: WireFrame[],
+  dir: "ab" | "ba",
+  permits: Permits | undefined,
+): TransformStream<Uint8Array, Uint8Array> {
+  const parser = new FrameParser();
+  return new TransformStream<Uint8Array, Uint8Array>({
+    transform: async (chunk, controller) => {
+      if (permits !== undefined) await permits.take();
+      try {
+        for (const frame of parser.push(chunk)) {
+          seen.push({ ...frame, dir, name: frameTypeName(frame.type) });
+        }
+      } catch {
+        // Tests that inject garbage break the tap, not the transport.
+      }
+      controller.enqueue(chunk);
+    },
+  });
+}

package/src/worker-protocol/worker-mount.ts

@@ -0,0 +1,27 @@
+import type { WorkerHostPath } from "./logs-socket-path.ts";
+
+/**
+ * A mount the worker dictates: where it comes from on the host, where it
+ * lands in the container by default, and whether the app may write to it.
+ *
+ * Spread it into a service's `volumes`:
+ *
+ * ```ts
+ * volumes: [config.logs]                              // default mount point
+ * volumes: [{ ...config.logs, containerPath: "/x" }]  // somewhere else
+ * ```
+ *
+ * `containerPath` is the app's to change - it only has to agree with whatever
+ * the app's own code opens inside the container. `readOnly` is not: it says
+ * what the *permission* covers. Connecting to a broker socket needs no write
+ * access to the directory holding it, so those mounts resolve `readOnly: true`
+ * and overriding that to `false` is a grant violation rather than a
+ * preference (`readOnlyMountViolations` in `security.ts`). Without that gate
+ * every app granted `host.logs` could unlink the shared socket and bind its
+ * own listener in its place.
+ */
+export interface WorkerMount {
+  readonly hostPath: WorkerHostPath;
+  readonly containerPath: string;
+  readonly readOnly: boolean;
+}

package/src/worker-protocol/worker-state.ts

@@ -0,0 +1,190 @@
+// The Home → Worker wire contract.
+//
+// `WorkerState` is the whole payload Home ships to a worker (read from
+// `state.json`, encrypted at rest, plaintext over WireGuard) and hands to
+// runtime-specific renderers - today `convertToDockerCompose` for `apps` and
+// `buildCaddyConfig` for `ingress`, both injected by `platform/worker`.
+//
+// This is its OWN schema, deliberately ignorant of the resolve model
+// (`homeAppSchema` in `resolve.ts`), which is Home's internal working state and
+// mirrors `getDesiredState` (refs, `ui`, `trustedHeaders`, per-service
+// `router`/`serviceKind`/`output`). The wire carries only what a keyless,
+// app-isolated worker needs to run the workloads and route traffic to them.
+//
+// The wire is post-resolution, and `build-wire.ts` proves it: it walks every
+// `{__realmRef:...}` marker out (`resolveRefs`) and asserts none survive
+// (`assertNoRefMarkers`). So every field here is a plain literal - no
+// `stringOrRef` unions, and a marker that escaped resolution fails this parse.
+
+import { z } from "zod";
+import { hostKey, nameSchema } from "../identifiers.ts";
+import { ingressInputSchema } from "./ingress.ts";
+import { SPECULATIVE_SILO } from "../speculative-silo.ts";
+
+/**
+ * Per-service depends_on edge, runtime-neutral. Home computes these during
+ * stitching from the peer permissions registry; the renderer emits the
+ * runtime-specific form (compose `service_healthy`/`service_started`).
+ */
+export const dependsOnEdgeSchema = z.object({
+  providerAppId: z.string(),
+  providerServiceName: z.string(),
+  condition: z.enum(["started", "healthy"]),
+});
+export type DependsOnEdge = z.infer<typeof dependsOnEdgeSchema>;
+
+// Whole-second duration. The renderer formats `${n}s` when emitting
+// compose. Defaulted upstream in the resolve model, so on the wire it's
+// always a concrete number.
+const duration = z.number().int().nonnegative();
+const containerPort = z.number().int().min(1).max(65535);
+
+const healthcheckSchema = z.object({
+  test: z.array(z.string()),
+  interval: duration,
+  timeout: duration,
+  retries: z.number().int().nonnegative(),
+  startPeriod: duration,
+});
+
+const hostPortSchema = z.object({
+  containerPort,
+  hostPort: z.number().int().min(1).max(65535),
+  protocol: z.enum(["tcp", "udp"]).optional(),
+});
+
+// Resolved on the wire: the `pg-dump`/`valkey-save`/`mysql-dump` URLs are
+// `stringOrRef` in the resolve model; the walker has baked them to strings.
+const backupStrategySchema = z.discriminatedUnion("kind", [
+  z.object({ kind: z.literal("file-copy") }),
+  z.object({ kind: z.literal("sqlite"), file: z.string() }),
+  z.object({ kind: z.literal("pg-dump"), url: z.string(), version: z.number().int().positive() }),
+  z.object({ kind: z.literal("valkey-save"), url: z.string() }),
+  z.object({
+    kind: z.literal("mysql-dump"),
+    url: z.string(),
+    version: z.number().int().positive(),
+  }),
+]);
+
+const volumeSchema = z.object({
+  // Refused by name: the wire is where "this was fully resolved" stops being a
+  // promise and becomes a check. A speculative eval fabricates a Silo so an app
+  // can be previewed before the operator picks one (`speculative-silo.ts`), and
+  // that placeholder is otherwise an ordinary absolute path - shipping it binds
+  // an empty directory in place of the operator's library, silently. `evalApps`
+  // refuses that combination up front; this is the half that cannot be bypassed.
+  hostPath: z.string().refine((p) => p !== SPECULATIVE_SILO.hostPath, {
+    message: "the speculative Silo placeholder must never reach the wire",
+  }),
+  containerPath: z.string(),
+  // The Silo group that owns this directory. The renderer turns it into
+  // `group_add` on every service that mounts the volume, so directory
+  // preparation and group membership are driven by the same field and can
+  // never disagree.
+  gid: z.number().int().positive().optional(),
+  readOnly: z.boolean().optional(),
+  backup: z.union([backupStrategySchema, z.literal(false)]).optional(),
+});
+
+/**
+ * One service as the renderer sees it - renderer fields only. The resolve
+ * model carries extra Home-side fields (`router`, `serviceKind`, `output`,
+ * `build`); none belong on the wire. This is a plain (non-strict) object, so
+ * parsing against it drops any such field automatically - that parse is what
+ * keeps Home-only data off the wire. (`output` is the one field a producer
+ * still removes by hand, earlier, for an unrelated reason - see build-wire.)
+ *
+ * No `build`: recompute guarantees every service is image-pinned before the
+ * wire is built, so `build` has already become an `image`. The wire is always
+ * image-based.
+ */
+export const wireServiceSchema = z.object({
+  image: z.string().min(1),
+  hostPorts: z.array(hostPortSchema).optional(),
+  env: z.record(z.string(), z.union([z.string(), z.number()])).optional(),
+  // containerPath -> 64-char hex digest the renderer resolves to bytes. The two
+  // sources hash different things: artifact refs collapse to their bare content
+  // sha, while inline content is keyed by *identity* — sha256 of
+  // `appId:serviceKey:fileKey` (plaintext rides in `WorkerApp.inlineFiles`). An
+  // inline file therefore keeps the same digest and worker path across content
+  // changes, so `--watch` consumers reload without recreating the container.
+  files: z.record(z.string().startsWith("/"), z.string().regex(/^[a-f0-9]{64}$/)).optional(),
+  command: z.array(z.string()).optional(),
+  volumes: z.array(volumeSchema).optional(),
+  gpu: z.boolean().optional(),
+  iptables: z.boolean().optional(),
+  joinNetworks: z.array(z.string()).optional(),
+  healthcheck: healthcheckSchema.optional(),
+  memory: z.string().optional(),
+  cpu: z.number().positive().max(128).optional(),
+  // Defaulted to 1 upstream, so always present on the wire.
+  replicas: z.number().int().min(1).max(100),
+  labels: z.array(z.string()).optional(),
+  shmSizeMb: z.number().int().positive().optional(),
+});
+export type WireService = z.infer<typeof wireServiceSchema>;
+
+export const workerAppSchema = z.object({
+  appId: z.string(),
+  // Flattened: in the resolve model these live under `.state` (matching
+  // `getDesiredState`'s shape), but on the wire only these two of that
+  // object's fields survive, so they're hoisted onto the app directly.
+  services: z.record(nameSchema, wireServiceSchema),
+  /** `true` makes the per-app network `internal: true` (no egress). */
+  egress: z.literal(true).optional(),
+  /** Worker-owned external nets this app's services join (e.g. `management-net`). */
+  externalNetworks: z.array(z.string()),
+  /**
+   * Service-name (unprefixed) -> producer edges. The renderer namespaces
+   * `providerServiceName` with `providerAppId` to form the final compose key.
+   */
+  dependsOn: z.record(z.string(), z.array(dependsOnEdgeSchema)),
+  /** Plaintext for inline `files` entries, keyed by their identity digest. */
+  inlineFiles: z.record(z.string(), z.string()),
+  /**
+   * Absolute host paths exempted from the absolute-mount gate and
+   * bound verbatim by the renderer (peer backup volumes, the logs socket).
+   */
+  allowedAbsoluteHostPaths: z.array(z.string()),
+  /**
+   * Host path the worker binds this app's `home-proxy.sock` at. Present only
+   * when the app holds a `home.apiAccess` grant.
+   */
+  homeProxySocketPath: z.string().optional(),
+  /**
+   * Host path the worker binds this app's spawn broker socket at. Present
+   * only when the app holds a `host.spawn` grant.
+   */
+  spawnSocketPath: z.string().optional(),
+  /**
+   * This app's registry credentials, keyed by registry host. The worker
+   * renders a per-app docker `config.json` from these so private images pull.
+   */
+  registryAuth: z
+    .record(hostKey, z.object({ username: z.string(), password: z.string() }))
+    .optional(),
+});
+export type WorkerApp = z.infer<typeof workerAppSchema>;
+
+export const workerStateSchema = z.object({
+  apps: z.array(workerAppSchema),
+  /** SHAs of every artifact the worker keeps in `files/` after a reconcile. */
+  artifactHashes: z.array(z.string()),
+  /** Docker networks the worker container itself joins this reconcile. */
+  workerNetworks: z.array(z.string()),
+  /**
+   * Where traffic goes on this box, as `apps` says what runs on it. Runtime-
+   * neutral: the Docker worker renders it with `buildCaddyConfig`
+   * (`runtime/docker/ingress`) exactly as it renders `apps` with
+   * `convertToDockerCompose`.
+   *
+   * Required, not optional. Every producer can state an honest value - an
+   * unconfigured box gets the empty-identity placeholder, which is a real
+   * answer ("no domain yet") rather than a missing one - and a box with no
+   * schedulable app still needs a front door, so absence must never be a legal
+   * way to say "no ingress here".
+   */
+  ingress: ingressInputSchema,
+});
+export type WorkerState = z.infer<typeof workerStateSchema>;

package/src/xml.ts

@@ -0,0 +1,49 @@
+import type { Deferred, Ref } from "./ref.ts";
+import { serializeRefs } from "./serialize.ts";
+
+/**
+ * Element-only XML values. Attributes aren't expressible — every node is
+ * either a leaf with text content or a parent with element children.
+ * Covers config files like Sonarr/Radarr `config.xml` where each setting
+ * is a `<Key>value</Key>` line.
+ */
+type XmlValue = string | number | Ref<string> | { [key: string]: XmlValue };
+
+function escapeXmlText(s: string): string {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+// Emit a resolved (ref-free) tree. String leaves — including values that
+// were refs before resolution — go through escapeXmlText, so a resolved
+// secret containing `&`/`<`/`>` can't break the document.
+function emit(parts: string[], indent: string, key: string, value: unknown): void {
+  parts.push(`${indent}<${key}>`);
+  if (typeof value === "string") {
+    parts.push(escapeXmlText(value));
+  } else if (typeof value === "number") {
+    parts.push(String(value));
+  } else {
+    parts.push("\n");
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      emit(parts, indent + "  ", k, v);
+    }
+    parts.push(indent);
+  }
+  parts.push(`</${key}>\n`);
+}
+
+/**
+ * Serialize a tree of objects to an element-only XML string. Refs in leaf
+ * positions are resolved at deploy time (via `apply()`) before the text is
+ * XML-escaped, so a resolved secret is escaped rather than spliced raw. A
+ * ref-free tree collapses to a plain string at build time.
+ */
+export function toXml(value: Record<string, XmlValue>): string | Deferred<string> {
+  return serializeRefs(value, (resolved) => {
+    const parts: string[] = [];
+    for (const [k, v] of Object.entries(resolved as Record<string, unknown>)) {
+      emit(parts, "", k, v);
+    }
+    return parts.join("");
+  });
+}

package/src/yaml.ts

@@ -0,0 +1,15 @@
+import { stringify } from "yaml";
+import type { Deferred } from "./ref.ts";
+import { serializeRefs } from "./serialize.ts";
+
+/**
+ * Serialize a value to a YAML string. Refs anywhere in the tree are
+ * resolved at deploy time (via `apply()`) before serialization, so YAML's
+ * value-dependent quoting sees the real string. A ref-free value collapses
+ * to a plain string at build time.
+ *
+ *   files: { "/etc/app/config.yml": toYaml({ database: db.url }) }
+ */
+export function toYaml<T>(value: T): string | Deferred<string> {
+  return serializeRefs(value, (resolved) => stringify(resolved));
+}