@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/runtime/docker/compose/convert-to-docker-compose.ts

@@ -0,0 +1,278 @@
+import {
+  REALM_APP_DIR_GID,
+  isPreparedAppDir,
+  scopeBindSource,
+  type WorkerApp,
+  type WorkerState,
+} from "../../../worker-protocol/index.ts";
+import { HARDENING } from "./default-hardening.ts";
+import { buildContainerLabels } from "./labels.ts";
+import { EFFECTS, type EffectContext } from "./effects.ts";
+import type {
+  ComposeConfig,
+  ComposeDeploy,
+  ComposeHealthcheck,
+  ComposeNetworkDef,
+  ComposeService,
+  ComposeVolume,
+} from "./types.ts";
+
+/**
+ * One app's rendered compose project. The worker runs each of these as
+ * its own `docker compose --project-name realm_<appId>` up, in the order
+ * returned by `convertToDockerCompose` (providers before consumers).
+ */
+export interface AppComposeProject {
+  appId: string;
+  compose: ComposeConfig;
+}
+
+export interface ConvertOptions {
+  /**
+   * Worker-host timezone (e.g. `"Europe/Stockholm"`), typically read
+   * from `/etc/timezone` by the worker spine. Stamped as `TZ=` on
+   * every service that doesn't already define its own `TZ` env. Apps
+   * win: setting `env.TZ` in realm.ts opts out for that service.
+   */
+  tz?: string;
+  /**
+   * Platform string (`linux/amd64`, `linux/arm64`) stamped on every
+   * service. Suppresses Docker's "platform mismatch" warning on hosts
+   * that emulate (e.g. Apple Silicon running amd64 images via Rosetta).
+   * Empty/unset omits the field, leaving Docker to pick.
+   */
+  platform?: string;
+}
+
+/**
+ * Render a WorkerState — runtime-neutral, post-resolution
+ * data — into one docker-compose project per app. The worker runs each
+ * project under its own `--project-name realm_<appId>`, sequentially, in
+ * the returned order: providers before the apps that consume them.
+ *
+ * The renderer trusts the wire: refs are already walked, image digests
+ * are already pinned, cross-app `dependsOn` edges are pre-stitched. No
+ * grant checks, no peer-registry walks. Future runtimes
+ * (`convertToQuadlet`, `convertToKube`) operate on the same wire.
+ *
+ * Cross-app `depends_on` is NOT emitted into compose — it can't span
+ * projects. The topological order returned here carries that ordering
+ * instead; intra-app readiness still rides `restart: on-failure`.
+ */
+export function convertToDockerCompose(
+  state: WorkerState,
+  opts: ConvertOptions = {},
+): AppComposeProject[] {
+  return topoSortApps(state.apps).map((app) => {
+    const services: Record<string, ComposeService> = {};
+    const networks: Record<string, ComposeNetworkDef> = {};
+    renderApp(app, services, networks, opts);
+    return {
+      appId: app.appId,
+      compose: {
+        services,
+        ...(Object.keys(networks).length > 0 ? { networks } : {}),
+      },
+    };
+  });
+}
+
+/**
+ * Order apps so every provider comes before the apps that consume one of
+ * its services (Kahn's algorithm over the cross-app `dependsOn` provider
+ * edges). The worker runs the per-app ups in this order, so a consumer's
+ * provider is already up by the time its own up runs.
+ *
+ * Edges to apps not present in this wire are ignored. Apps left in a
+ * cycle (or with no edges) keep their input order and are appended last;
+ * they converge across reconciles via `restart: on-failure`.
+ */
+function topoSortApps(apps: WorkerApp[]): WorkerApp[] {
+  const present = new Set(apps.map((a) => a.appId));
+  // For each app, the set of in-wire provider appIds it depends on.
+  const providersOf = new Map<string, Set<string>>();
+  for (const app of apps) {
+    const providers = new Set<string>();
+    for (const edges of Object.values(app.dependsOn)) {
+      for (const edge of edges) {
+        if (edge.providerAppId !== app.appId && present.has(edge.providerAppId)) {
+          providers.add(edge.providerAppId);
+        }
+      }
+    }
+    providersOf.set(app.appId, providers);
+  }
+
+  const sorted: WorkerApp[] = [];
+  const placed = new Set<string>();
+  // Each pass emits, in input order, every not-yet-placed app whose
+  // providers are all placed. A pass that places nothing means the rest
+  // is a cycle.
+  let progressed = true;
+  while (progressed && placed.size < apps.length) {
+    progressed = false;
+    for (const app of apps) {
+      if (placed.has(app.appId)) continue;
+      const ready = [...providersOf.get(app.appId)!].every((p) => placed.has(p));
+      if (!ready) continue;
+      sorted.push(app);
+      placed.add(app.appId);
+      progressed = true;
+    }
+  }
+  // Cycle remainder: append in input order.
+  for (const app of apps) {
+    if (!placed.has(app.appId)) sorted.push(app);
+  }
+  return sorted;
+}
+
+function renderApp(
+  app: WorkerApp,
+  services: Record<string, ComposeService>,
+  networks: Record<string, ComposeNetworkDef>,
+  opts: ConvertOptions,
+): void {
+  const { appId, services: appServices, egress, externalNetworks, allowedAbsoluteHostPaths } = app;
+  const exemptHostPaths = new Set(allowedAbsoluteHostPaths);
+  // Home pre-computes the app's union of external networks, and an EFFECT
+  // can add more per service during render. Both are honored.
+  const appExternalNetworks = new Set<string>(externalNetworks);
+
+  for (const [name, svc] of Object.entries(appServices)) {
+    const volumes: ComposeVolume[] = [];
+    // Whether this service mounts a directory the worker prepares with
+    // REALM_APP_DIR_GID + setgid. The container has to join that group or an
+    // image running as a baked-in uid can't write its own storage; the same
+    // predicate decides membership and what the worker prepares.
+    let ownsPreparedDirs = false;
+    if (svc.volumes) {
+      for (const v of svc.volumes) {
+        const hostPath = v.hostPath;
+        const source = exemptHostPaths.has(hostPath) ? hostPath : scopeBindSource(hostPath, appId);
+        if (isPreparedAppDir(appId, v, exemptHostPaths)) ownsPreparedDirs = true;
+        // Read-only is opt-in. A mount an app declared is a mount it means to
+        // use, and the overwhelmingly common case is writing to it.
+        volumes.push({
+          type: "bind",
+          source,
+          target: v.containerPath,
+          ...(v.readOnly ? { read_only: true } : {}),
+        });
+      }
+    }
+
+    let environment: Record<string, string> | undefined;
+    if (svc.env) {
+      environment = {};
+      for (const [k, val] of Object.entries(svc.env)) {
+        environment[k] = String(val);
+      }
+    }
+    if (opts.tz && !(environment && "TZ" in environment)) {
+      environment ??= {};
+      environment.TZ = opts.tz;
+    }
+
+    let healthcheck: ComposeHealthcheck | undefined;
+    if (svc.healthcheck) {
+      const hc = svc.healthcheck;
+      // Compose-spec rejects bare integers — durations need a unit suffix.
+      healthcheck = {
+        test: hc.test,
+        interval: `${hc.interval}s`,
+        timeout: `${hc.timeout}s`,
+        retries: hc.retries,
+        start_period: `${hc.startPeriod}s`,
+      };
+    }
+
+    let deploy: ComposeDeploy | undefined;
+    const { memory, cpu, replicas } = svc;
+    if (memory != null || cpu != null || replicas > 1) {
+      deploy = {};
+      if (replicas > 1) deploy.replicas = replicas;
+      if (memory || cpu) {
+        deploy.resources = {
+          limits: {
+            ...(memory ? { memory } : {}),
+            ...(cpu ? { cpus: String(cpu) } : {}),
+          },
+        };
+      }
+    }
+
+    const labels = buildContainerLabels({
+      appId,
+      serviceName: name,
+      customLabels: svc.labels,
+    });
+
+    const composeService: ComposeService = {
+      image: svc.image,
+      ...(opts.platform ? { platform: opts.platform } : {}),
+      ...(environment ? { environment } : {}),
+      ...(svc.command ? { command: svc.command } : {}),
+      labels,
+      ...(volumes.length > 0 ? { volumes } : {}),
+      ...(ownsPreparedDirs ? { group_add: [REALM_APP_DIR_GID] } : {}),
+      ...(healthcheck ? { healthcheck } : {}),
+      ...(deploy ? { deploy } : {}),
+      cap_drop: [...HARDENING.cap_drop],
+      cap_add: [...HARDENING.cap_add],
+      security_opt: [...HARDENING.security_opt],
+      pids_limit: HARDENING.pids_limit,
+      ulimits: { ...HARDENING.ulimits },
+      ...(svc.shmSizeMb != null ? { shm_size: `${svc.shmSizeMb}m` } : {}),
+      restart: HARDENING.restart,
+      networks: { [`realm_${appId}`]: { aliases: [name] } },
+      logging: {
+        driver: HARDENING.logging.driver,
+        options: { ...HARDENING.logging.options },
+      },
+    };
+
+    const filesUnused = new Map<string, string>();
+    const ctx: EffectContext = {
+      appId,
+      serviceName: name,
+      composeService,
+      externalNetworks: appExternalNetworks,
+      // The renderer doesn't materialize inline file bytes — Home ships them
+      // over the wire — but EFFECTS keeps the `files` parameter. No effect
+      // writes to it today; one that did would surface in the golden tests.
+      files: filesUnused,
+    };
+    for (const effect of Object.values(EFFECTS)) {
+      effect(svc, ctx);
+    }
+
+    // svc.files is keyed by container path; values are sha256 hashes Home
+    // pre-computed. Inline content is keyed by IDENTITY (sha256 of
+    // `appId:serviceKey:fileKey`), so the host path stays stable across content
+    // changes and a rewrite lands in place — what lets `--watch` consumers
+    // reload without a container recreate. Artifact refs carry their own hash.
+    if (svc.files != null) {
+      for (const [containerPath, sha] of Object.entries(svc.files)) {
+        composeService.volumes ??= [];
+        composeService.volumes.push({
+          type: "bind",
+          source: `./files/${sha}`,
+          target: containerPath,
+          read_only: true,
+        });
+      }
+    }
+
+    services[`${appId}_${name}`] = composeService;
+  }
+
+  networks[`realm_${appId}`] = {
+    name: `realm_${appId}`,
+    ...(!egress ? { internal: true } : {}),
+  };
+
+  // External networks (management-net, peer `realm_<otherApp>`) get no
+  // top-level declaration: nothing in the `services` block references them.
+  // The worker connects to them post-up via `docker network connect`.
+}

package/src/runtime/docker/compose/default-hardening.ts

@@ -0,0 +1,21 @@
+// Drop every capability, add back only what an ordinary image needs to boot:
+// CHOWN to claim its own data dir, SETGID/SETUID for the gosu drop to a service
+// user, NET_BIND_SERVICE to bind below 1024.
+//
+// DAC_OVERRIDE is here because without it those same entrypoints work only on
+// their FIRST boot: uid 0 with no capabilities still faces ordinary permission
+// checks, so once an image has tightened its own data dir (postgres chmods
+// PGDATA to 0700 and refuses to start otherwise) root can no longer descend it
+// and the entrypoint dies on the traversal. It costs little — the boundary is
+// the mount gate in security.ts, not in-container DAC, and read-only mounts
+// are unaffected (MS_RDONLY is VFS-enforced). These 5 are a subset of the 14
+// capabilities Docker's own default grants.
+export const HARDENING = {
+  cap_drop: ["ALL"],
+  cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID", "NET_BIND_SERVICE"],
+  security_opt: ["no-new-privileges:true"],
+  pids_limit: 4096,
+  ulimits: { nofile: 65536, nproc: 4096 },
+  logging: { driver: "json-file", options: { "max-size": "10m", "max-file": "3" } },
+  restart: "on-failure",
+} as const;

package/src/runtime/docker/compose/effects.ts

@@ -0,0 +1,107 @@
+import type { HostPermissionKind } from "../../../permissions.ts";
+import type { WireService } from "../../../worker-protocol/index.ts";
+import type { ComposeService } from "./types.ts";
+
+// ---------------------------------------------------------------------------
+// Compose effects table
+// ---------------------------------------------------------------------------
+//
+// One entry per host permission kind. Each effect mutates the
+// `ComposeService` (and the shared externalNetworks/files maps) from the
+// resolved service's fields. The gate has already validated the service
+// against grants by the time this runs, so effects only translate.
+//
+// `Record<HostPermissionKind, EffectFn>` keeps the table in lock-step with
+// the kinds derived in permissions.ts: a new kind without an effect, or a
+// stale effect for a deleted one, is a compile error.
+
+export interface EffectContext {
+  appId: string;
+  serviceName: string;
+  composeService: ComposeService;
+  externalNetworks: Set<string>;
+  files: Map<string, string>;
+}
+
+type EffectFn = (svc: WireService, ctx: EffectContext) => void;
+
+export const EFFECTS: Record<HostPermissionKind, EffectFn> = {
+  "host.gpu": (svc, ctx) => {
+    if (svc.gpu === true) {
+      ctx.composeService.devices = [...(ctx.composeService.devices ?? []), "/dev/dri:/dev/dri"];
+    }
+  },
+
+  "host.port": (svc, ctx) => {
+    if (!svc.hostPorts || svc.hostPorts.length === 0) return;
+    const cs = ctx.composeService;
+    cs.ports ??= [];
+    for (const hp of svc.hostPorts) {
+      cs.ports.push({
+        target: hp.containerPort,
+        published: hp.hostPort,
+        ...(hp.protocol ? { protocol: hp.protocol } : {}),
+      });
+    }
+  },
+
+  // A Silo's volume is mounted by an ordinary `volumes:` entry; the one thing
+  // the renderer adds is group membership, so it lands exactly where the mount
+  // lands, per service - a sidecar that doesn't mount the shared library never
+  // joins the group. `group_add` needs no cooperation from the image (unlike
+  // `user:`), so a root image, a baked-in uid, and a PUID all work.
+  "host.silo": (svc, ctx) => {
+    // Seeded from what's already there: a service that also mounts its own
+    // storage carries REALM_APP_DIR_GID from the base render, and joining a
+    // Silo must add to that membership rather than replace it - dropping it
+    // would take away the app's *own* directory to give it a shared one.
+    const gids = new Set<number>(ctx.composeService.group_add ?? []);
+    for (const v of svc.volumes ?? []) {
+      if (v.gid !== undefined) gids.add(v.gid);
+    }
+    if (gids.size > 0) ctx.composeService.group_add = [...gids].sort((a, b) => a - b);
+  },
+
+  // joinNetworks doesn't land in compose: the worker reconciles network
+  // membership post-up with `docker network connect/disconnect`, so adding
+  // or removing one doesn't recreate the container. The data rides the wire
+  // on `services[*].joinNetworks` and the worker reads it directly.
+  "host.management": () => {},
+
+  "host.egress": () => {},
+
+  // `iptables: true` rolls up the two raw primitives the firewall needs:
+  // NET_ADMIN to mutate netfilter, and host network mode so rules apply to
+  // host traffic and Docker's DOCKER-USER chain. Authoring stays at the
+  // intent level; the grant gate has already validated the use.
+  "host.iptables": (svc, ctx) => {
+    if (svc.iptables !== true) return;
+    if (!ctx.composeService.cap_add.includes("NET_ADMIN")) {
+      ctx.composeService.cap_add.push("NET_ADMIN");
+    }
+    ctx.composeService.network_mode = "host";
+    // host network mode and the per-project default network are
+    // mutually exclusive in compose.
+    delete ctx.composeService.networks;
+  },
+
+  // host.logs has no compose effect — the socket the value resolves to is
+  // mounted by the app via a normal `volumes:` entry. The grant exempts
+  // that mount from the absolute-mount gate (see graph.ts collectAllowed).
+  "host.logs": () => {},
+
+  // host.stats has no compose effect — same shape as host.logs: the app mounts
+  // the socket dir via a normal `volumes:` entry and the grant exempts it.
+  "host.stats": () => {},
+
+  // host.spawn has no compose effect — the socket/scratch/artifacts paths
+  // are mounted via normal `volumes:` entries and the network rides
+  // `joinNetworks`. The grant exempts all three from the absolute-mount /
+  // network.join gates (see graph.ts collectAllowed).
+  "host.spawn": () => {},
+
+  // No compose effect — `host.rocketLaunch` is a probe-the-UX permission
+  // that no worker advertises, so the gate blocks any app declaring it
+  // before it could ever reach this table.
+  "host.rocketLaunch": () => {},
+};

package/src/runtime/docker/compose/index.ts

@@ -0,0 +1,17 @@
+// @yourrealm/sdk/runtime/docker/compose — docker-compose renderer.
+//
+// Transforms `WorkerState` (from @yourrealm/sdk/worker-protocol) into a
+// docker-compose.yml. No I/O, no docker socket, no network. Home uses it in
+// integration tests; the Worker uses it at reconcile time to produce the
+// file `docker compose up` consumes.
+
+export {
+  type AppComposeProject,
+  convertToDockerCompose,
+  type ConvertOptions,
+} from "./convert-to-docker-compose.ts";
+export { HARDENING } from "./default-hardening.ts";
+export { buildContainerLabels } from "./labels.ts";
+export { EFFECTS, type EffectContext } from "./effects.ts";
+export { composeSchema } from "./compose-schema.ts";
+export type { ComposeConfig, ComposeNetworkDef, ComposeService } from "./types.ts";

package/src/runtime/docker/compose/labels.ts

@@ -0,0 +1,39 @@
+import { KINDS, LABELS } from "@local/docker";
+
+interface ContainerLabelOptions {
+  appId: string;
+  serviceName: string;
+  customLabels?: string[];
+}
+
+const RESERVED_PREFIXES = ["realm.", "com.docker.", "org.opencontainers."];
+
+/**
+ * Realm container metadata labels (managed, app, service). Routing
+ * data lives in the Caddy JSON config the proxy app generates from
+ * `peers.routedApps()`, not on container labels.
+ *
+ * Custom labels declared in realm.ts pass through unchanged unless they
+ * collide with a reserved prefix, in which case they're dropped.
+ */
+export function buildContainerLabels(options: ContainerLabelOptions): Record<string, string> {
+  const labels: Record<string, string> = {};
+
+  if (options.customLabels) {
+    for (const label of options.customLabels) {
+      const eqIdx = label.indexOf("=");
+      if (eqIdx === -1) continue;
+      const key = label.slice(0, eqIdx);
+      const keyLower = key.toLowerCase();
+      if (RESERVED_PREFIXES.some((p) => keyLower.startsWith(p))) continue;
+      labels[key] = label.slice(eqIdx + 1);
+    }
+  }
+
+  labels[LABELS.MANAGED] = "true";
+  labels[LABELS.KIND] = KINDS.APP_SERVICE;
+  labels[LABELS.APP] = options.appId;
+  labels[LABELS.SERVICE] = options.serviceName;
+
+  return labels;
+}

package/src/runtime/docker/compose/types.ts

@@ -0,0 +1,78 @@
+// Compose-shape types the docker renderer emits. They live beside the
+// renderer, which owns the docker dialect end-to-end; Home uses them only as
+// TypeScript types via the @yourrealm/sdk/compile re-exports.
+
+interface ComposeVolume {
+  type: "bind" | "volume";
+  source: string;
+  target: string;
+  read_only?: boolean;
+}
+
+interface ComposePort {
+  target: number;
+  published: number;
+  protocol?: "tcp" | "udp";
+}
+
+interface ComposeHealthcheck {
+  test: string[];
+  interval?: string;
+  timeout?: string;
+  retries?: number;
+  start_period?: string;
+}
+
+interface ComposeDeploy {
+  replicas?: number;
+  resources?: {
+    limits?: {
+      memory?: string;
+      cpus?: string;
+    };
+  };
+}
+
+interface ComposeDependsOn {
+  [serviceName: string]: { condition: "service_started" | "service_healthy" };
+}
+
+export interface ComposeService {
+  image: string;
+  platform?: string;
+  environment?: Record<string, string>;
+  command?: string[];
+  entrypoint?: string[];
+  labels?: Record<string, string>;
+  volumes?: ComposeVolume[];
+  /** Supplementary groups, derived from the gid-carrying volumes mounted. */
+  group_add?: number[];
+  ports?: ComposePort[];
+  healthcheck?: ComposeHealthcheck;
+  deploy?: ComposeDeploy;
+  cap_drop: string[];
+  cap_add: string[];
+  security_opt: string[];
+  pids_limit: number;
+  ulimits: { nofile: number; nproc: number };
+  shm_size?: string;
+  devices?: string[];
+  network_mode?: string;
+  restart: string;
+  networks?: Record<string, { aliases?: string[] }>;
+  depends_on?: ComposeDependsOn;
+  logging: { driver: string; options: Record<string, string> };
+}
+
+export interface ComposeNetworkDef {
+  external?: true;
+  internal?: true;
+  name: string;
+}
+
+export interface ComposeConfig {
+  services: Record<string, ComposeService>;
+  networks?: Record<string, ComposeNetworkDef>;
+}
+
+export type { ComposeVolume, ComposePort, ComposeHealthcheck, ComposeDeploy, ComposeDependsOn };