@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/worker/start-worker.ts

@@ -0,0 +1,227 @@
+import { log } from "@local/logger";
+import type { WorkerCapability } from "../permissions.ts";
+import type { SiloOrphan, WorkerState } from "../worker-protocol/index.ts";
+import {
+  type AppApplyResult,
+  jsonHandler,
+  parseEnrollmentToken,
+  type RpcHandler,
+  type UpdateProject,
+  type UpdateStatus,
+} from "../worker-protocol/uplink/index.ts";
+import type {
+  HomeProxyRequest,
+  HomeProxyResponse,
+} from "../runtime/docker/home-proxy-server/protocol.ts";
+import { loadIdentity, readEnrollment, resolveHomePublicKey, writeEnrollment } from "./identity.ts";
+import { createReconcileQueue } from "./queue.ts";
+import { declaredPortsIn, uiProxyHandler } from "./ui-proxy.ts";
+import { startUplink, type UplinkHandle, WorkerOffline } from "./uplink.ts";
+import { startVerifyRelay } from "./verify-relay.ts";
+
+/** What a reconcile callback can reach on the live uplink. */
+export interface ReconcileContext {
+  /** Relays one `home.apiAccess` call for `appId`; rejects while offline. */
+  apiAccess: (appId: string, req: HomeProxyRequest) => Promise<HomeProxyResponse>;
+}
+
+export interface StartWorkerOptions {
+  /**
+   * Fires when a reconcile pulls a new ETag and the parsed state is ready to
+   * apply. Resolves with one result per app brought up; they ride
+   * `state.applied` beside the etag, so Home sees which apps actually came up.
+   */
+  onNewState: (state: WorkerState, ctx: ReconcileContext) => Promise<AppApplyResult[]>;
+  /**
+   * Fires on a 304 (unchanged state) with the last-known state. Mount only the
+   * cheap, idempotent healers here (broker-socket ensure) so a socket that
+   * vanished without a state change is rebuilt on the next reconcile trigger.
+   * Resolves with the results of any app it retried, empty when it only healed.
+   */
+  onUnchanged?: (state: WorkerState, ctx: ReconcileContext) => Promise<AppApplyResult[]>;
+  /**
+   * Returns the host permission kinds this worker can satisfy. Called once per
+   * announce, so probes (filesystem, devices, etc.) re-run on every connect and
+   * reach the latest host state. Home uses the list to drop apps whose required
+   * permissions aren't matched by any deployed worker. Only host kinds — peer +
+   * remote are Home-resolved at eval and never flow over this wire.
+   */
+  capabilities: () => Promise<WorkerCapability[]>;
+  /** Home → worker `siloOrphans`: `live` is every Silo id that still has a row. */
+  siloOrphans: (live: Set<string>) => Promise<SiloOrphan[]>;
+  /**
+   * Home → worker `update`: pin one of this box's compose projects onto
+   * `imageRef` for its updater sidecar. The project is on the wire because the
+   * box has one sidecar and one etc/ tree - the worker is simply the only thing
+   * here holding a session, so it is what gets told about the ingress too.
+   */
+  update: (imageRef: string, project: UpdateProject) => Promise<void>;
+  /**
+   * What this box's ingress container is running, or null where it has none.
+   * Injected rather than read here, for the reason the whole `worker` layer is
+   * Docker-free: asking a container what image it was created from is an engine
+   * question, and platform/worker owns those.
+   */
+  ingressImage: () => Promise<{ imageRef: string } | null>;
+  /**
+   * What each of this box's updater-driven compose projects is doing, for the
+   * announce. Injected for the same reason `ingressImage` is: which projects
+   * the box carries and where its `etc/` tree is mounted are platform/worker's
+   * facts, and this layer holds no more opinion about `REALM_ETC_DIR` than it
+   * already does.
+   */
+  updates: () => Promise<UpdateStatus[]>;
+  /**
+   * The one address the verify relay listens on: this worker's own on
+   * management-net, where the ingress reaches it. Injected for the same reason
+   * as `ingressImage` - the address comes off a container inspect. Undefined
+   * where the box could not resolve it, which binds every interface instead
+   * (verify-relay.ts).
+   */
+  verifyRelayHostname?: string;
+}
+
+export interface WorkerHandle {
+  /** Manually enqueue a reconcile (e.g. for tests or non-uplink triggers). */
+  trigger(): void;
+  /**
+   * Drop the session so the uplink redials, re-announcing and pulling on the
+   * way back in. For a fact the announce carries that changed under a live
+   * session - see `UplinkHandle.reconnect`. A no-op on a box with no uplink.
+   */
+  reconnect(reason: string): void;
+  /**
+   * Tell Home one project's `.update` / `.update.error` moved on disk. Rejects
+   * with `WorkerOffline` while there is no session; the next announce carries
+   * the whole set, so a caller can let that one go.
+   */
+  updateStatus(status: UpdateStatus): Promise<void>;
+  /** Stops the uplink loop and closes the session. */
+  stop(): Promise<void>;
+}
+
+const WORKER_DIR = Deno.env.get("WORKER_DIR") ?? "/opt/realm/worker";
+const ETC_DIR = Deno.env.get("REALM_ETC_DIR") ?? "/opt/realm/etc";
+const VERIFY_RELAY_PORT = 6021;
+
+/**
+ * Spine entry point. Lifecycle:
+ *   1. Bind the verify relay on :6021 — the worker's one listener, on the
+ *      management-net address the caller resolved. It answers 502 until the
+ *      uplink is up, which is the documented behaviour, so it goes first and
+ *      never waits on Home.
+ *   2. Load this box's identity and resolve the pin, waiting for Home's key on
+ *      a fresh co-located box.
+ *   3. Open the uplink and keep it open. Every RPC in either direction rides
+ *      it, and the first connect pulls state.
+ *
+ * A worker whose Home is unreachable still runs its apps: nothing below
+ * blocks on a session, and the reconcilers run off the state on disk.
+ */
+export async function startWorker(opts: StartWorkerOptions): Promise<WorkerHandle> {
+  let uplink: UplinkHandle | null = null;
+  let lastState: WorkerState | null = null;
+
+  const context: ReconcileContext = {
+    apiAccess: async (appId, req) => {
+      if (uplink === null) throw new WorkerOffline("apiAccess");
+      return await uplink.apiAccess({ ...req, appId });
+    },
+  };
+
+  const queue = createReconcileQueue({
+    workerDir: WORKER_DIR,
+    pull: async (params) => {
+      if (uplink === null) throw new WorkerOffline("state.pull");
+      return await uplink.pull(params);
+    },
+    applied: async (etag, apps) => {
+      if (uplink === null) throw new WorkerOffline("state.applied");
+      await uplink.applied(etag, apps);
+    },
+    onNewState: async (state) => {
+      lastState = state;
+      return await opts.onNewState(state, context);
+    },
+    ...(opts.onUnchanged === undefined
+      ? {}
+      : { onUnchanged: (state: WorkerState) => opts.onUnchanged!(state, context) }),
+  });
+
+  // The ingress's ForwardAuth hop on every worker, co-located with Home or not
+  // — see verify-relay.ts for why the ingress can't ask Home itself.
+  startVerifyRelay({
+    port: VERIFY_RELAY_PORT,
+    ...(opts.verifyRelayHostname === undefined ? {} : { hostname: opts.verifyRelayHostname }),
+    verify: async (params) => {
+      if (uplink === null) throw new WorkerOffline("verify");
+      return await uplink.verify(params);
+    },
+  });
+
+  const handlers: Record<string, RpcHandler> = {
+    reconcile: jsonHandler("reconcile", () => {
+      queue.trigger();
+      return { ok: true };
+    }),
+    siloOrphans: jsonHandler("siloOrphans", ({ live }) => opts.siloOrphans(new Set(live))),
+    update: jsonHandler("update", async ({ imageRef, project }) => {
+      await opts.update(imageRef, project);
+      return { ok: true };
+    }),
+    uiProxy: uiProxyHandler((appId, service) =>
+      lastState === null ? undefined : declaredPortsIn(lastState, appId, service),
+    ),
+  };
+
+  const url = Deno.env.get("UPLINK_URL");
+  if (url === undefined || url === "") {
+    // Nothing to dial. The apps on this box keep running off the state on
+    // disk; a control plane is what is missing, and only a restart with the
+    // variable set brings one back.
+    log.error("UPLINK_URL is unset, so this worker has no way to reach Home");
+    return {
+      trigger: queue.trigger,
+      reconnect: () => {},
+      updateStatus: () => Promise.reject(new WorkerOffline("update.status")),
+      stop: () => Promise.resolve(),
+    };
+  }
+
+  const token = Deno.env.get("REALM_TOKEN");
+  const parsed = token === undefined || token === "" ? null : parseEnrollmentToken(token);
+  if (token !== undefined && token !== "" && parsed === null) {
+    log.warn("REALM_TOKEN is not `{homePublicKey}.{otp}` or a bare OTP; ignoring it");
+  }
+
+  const identity = await loadIdentity(WORKER_DIR);
+  const homePublicKey = await resolveHomePublicKey({
+    workerDir: WORKER_DIR,
+    etcDir: ETC_DIR,
+    tokenKey: parsed?.homePublicKey ?? null,
+  });
+  const name = Deno.env.get("WORKER_NAME") ?? "";
+
+  uplink = startUplink({
+    url,
+    identity,
+    homePublicKey,
+    otp: parsed?.otp ?? null,
+    name: name === "" ? null : name,
+    enrolled: await readEnrollment(WORKER_DIR),
+    capabilities: opts.capabilities,
+    ingressImage: opts.ingressImage,
+    updates: opts.updates,
+    handlers,
+    appliedEtag: queue.appliedEtag,
+    onConnected: queue.trigger,
+    onEnrolled: (enrollment) => writeEnrollment(WORKER_DIR, enrollment),
+  });
+
+  return {
+    trigger: queue.trigger,
+    reconnect: (reason) => uplink!.reconnect(reason),
+    updateStatus: (status) => uplink!.updateStatus(status),
+    stop: () => uplink!.close(),
+  };
+}

package/src/worker/ui-proxy.ts

@@ -0,0 +1,335 @@
+// Worker-side surface-call forwarder. Home authenticates the call and sends
+// the request envelope down the session as `uiProxy`; we match the container
+// by labels (`realm.app` + `realm.service`) and proxy to its IP on the per-app
+// `realm_<appId>` bridge.
+//
+// Two bounds, not one. Reachability: `network-reconcile.ts` joins the worker
+// to `realm_<appId>` only for ui-having apps, and Docker's
+// DOCKER-ISOLATION-STAGE-2 chain DROPs FORWARD between user-defined bridges,
+// so an app with no `ui:` block cannot be dialed even by IP. And the port:
+// the worker's own wire state says which ports a service declares, and a call
+// naming any other port of that service is refused before the container is
+// even looked up (`todo/ideas/uplink.md` § The relays). Without it a Home that
+// has been made to send an arbitrary port reaches every port of every
+// realm-labelled container the worker can see — Caddy's admin `:2019` being
+// the concrete danger.
+
+import { LABELS, ode } from "@local/docker";
+import { uiErrorBody, type UiErrorCode } from "../ui-error.ts";
+import type { WorkerState } from "../worker-protocol/index.ts";
+import {
+  jsonResponse,
+  methods,
+  type Params,
+  type RpcHandler,
+  type RpcResponse,
+} from "../worker-protocol/uplink/index.ts";
+
+type ProxyRequest = Params<"uiProxy">;
+
+/**
+ * The ports the worker's current wire state attributes to one service.
+ *
+ * `undefined` means the wire carries no port for it, which today is the
+ * ordinary case for a ui-only service: the port a loader dials lives in the
+ * app's ui manifest, which is Home's and never crosses the wire. Those calls
+ * are let through; everything the wire CAN name is bounded to exactly it.
+ */
+export type DeclaredPorts = (appId: string, service: string) => Set<number> | undefined;
+
+// Cap how long the worker waits on the container. A hung service would
+// otherwise pile up inflight requests at three layers (iframe, Home,
+// worker); the abort makes the pile bounded.
+const UPSTREAM_TIMEOUT_MS = 30_000;
+
+// Proxy fetches must not reuse a pooled keep-alive connection: an idle
+// socket to an app container can be dropped by the container's HTTP server
+// or the Docker bridge without a RST reaching us, and Deno's global fetch
+// would then hand the next request that dead half-open connection and block
+// until UPSTREAM_TIMEOUT_MS. Created lazily because this file can be loaded
+// into a QuickJS eval where the `Deno` global is absent.
+let proxyHttpClient: Deno.HttpClient | undefined;
+function getProxyHttpClient(): Deno.HttpClient {
+  return (proxyHttpClient ??= Deno.createHttpClient({ poolMaxIdlePerHost: 0 }));
+}
+
+// Cap the response we'll buffer from the container. Mirrors the
+// inbound `MAX_BODY_BYTES` in Home's secureHeaders. A misbehaving
+// service returning gigabytes would otherwise OOM the worker (and
+// then Home when this proxy passes it through).
+const MAX_LOADER_RESPONSE_BYTES = 4 * 1024 * 1024;
+
+/**
+ * Why a fetch failed. Deno throws `TypeError: fetch failed` and puts the
+ * reason (dns, refused, reset) on `cause`; the message alone says nothing.
+ */
+export function describeFetchError(err: unknown): string {
+  const e = err as { message?: string; cause?: { message?: string } };
+  const message = e.message ?? String(err);
+  return e.cause?.message ? `${message}: ${e.cause.message}` : message;
+}
+
+/** What the `uiProxy` method answers with. The body is always JSON text. */
+export interface UiProxyResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: string;
+}
+
+/**
+ * The ports `state` declares for one service: every published `hostPorts`
+ * container port plus the routed one, if it is routed.
+ */
+export function declaredPortsIn(
+  state: WorkerState,
+  appId: string,
+  service: string,
+): Set<number> | undefined {
+  const app = state.apps.find((a) => a.appId === appId);
+  if (app === undefined) return undefined;
+  const wire = app.services[service];
+  if (wire === undefined) return undefined;
+  const ports = new Set<number>();
+  for (const published of wire.hostPorts ?? []) ports.add(published.containerPort);
+  const routed = state.ingress.routedApps.find((a) => a.appId === appId);
+  for (const entry of routed?.services ?? []) {
+    if (entry.serviceId === service) ports.add(entry.router.containerPort);
+  }
+  return ports.size === 0 ? undefined : ports;
+}
+
+/** The Home → worker `uiProxy` handler, parse and all. */
+export function uiProxyHandler(declaredPorts: DeclaredPorts): RpcHandler {
+  return async (req): Promise<RpcResponse> => {
+    let raw: unknown;
+    try {
+      raw = JSON.parse(new TextDecoder().decode(await readAll(req.body)));
+    } catch {
+      return jsonResponse(400, uiErrorBody("INTERNAL", "uiProxy: body is not JSON"));
+    }
+    const parsed = methods.uiProxy.params.safeParse(raw);
+    if (!parsed.success) {
+      const issue = parsed.error.issues[0];
+      return jsonResponse(
+        400,
+        uiErrorBody("INTERNAL", `uiProxy: ${issue === undefined ? "invalid" : issue.message}`),
+      );
+    }
+    const res = await handleUiProxy(parsed.data, declaredPorts);
+    return {
+      status: res.status,
+      headers: res.headers,
+      body: bytesBody(new TextEncoder().encode(res.body)),
+    };
+  };
+}
+
+interface ContainerEndpoint {
+  id: string;
+  ip: string;
+}
+
+// The lookup either finds a dialable endpoint or explains why not, as a
+// typed code. Listing with `all: true` and reading `State` is what
+// separates the transient boot window (`CONTAINER_STARTING`, safe to
+// retry) from a real failure (`CONTAINER_CRASHED`, surface it).
+type ContainerLookup = { endpoint: ContainerEndpoint } | { code: UiErrorCode; message: string };
+
+async function resolveContainerEndpoint(req: ProxyRequest): Promise<ContainerLookup> {
+  const filters = {
+    label: [`${LABELS.APP}=${req.appId}`, `${LABELS.SERVICE}=${req.service}`],
+  };
+  const summaries = await ode.listContainers({ all: true, filters });
+  if (summaries.length === 0) {
+    return {
+      code: "NO_CONTAINER",
+      message: `no container matches ${req.appId}/${req.service}`,
+    };
+  }
+  // Multiple containers can match (replicas) — pick the first.
+  const summary = summaries[0]!;
+  const id = summary.Id;
+  if (!id) {
+    return {
+      code: "INTERNAL",
+      message: `container for ${req.appId}/${req.service} has no id`,
+    };
+  }
+  // A container that exists but isn't running is either on its way up or
+  // down for good — `created`/`restarting` are the boot window, anything
+  // else is a real failure.
+  if (summary.State !== "running") {
+    const starting = summary.State === "created" || summary.State === "restarting";
+    return {
+      code: starting ? "CONTAINER_STARTING" : "CONTAINER_CRASHED",
+      message: `container for ${req.appId}/${req.service} is ${summary.State ?? "unknown"}`,
+    };
+  }
+  const detail = await ode.inspectContainer(id);
+  const networks = (detail.NetworkSettings?.Networks ?? {}) as Record<
+    string,
+    { IPAddress?: string | null }
+  >;
+  // Prefer the per-app network so we don't accidentally route via a
+  // shared network that might be missing the container as a member.
+  const expectedNet = `realm_${req.appId}`;
+  const preferred = networks[expectedNet];
+  if (preferred?.IPAddress) return { endpoint: { id, ip: preferred.IPAddress } };
+  for (const endpoint of Object.values(networks)) {
+    if (endpoint?.IPAddress) return { endpoint: { id, ip: endpoint.IPAddress } };
+  }
+  // Running but no IP yet — still the boot window (network attach lags
+  // container start by a beat).
+  return {
+    code: "CONTAINER_STARTING",
+    message: `container for ${req.appId}/${req.service} has no network address yet`,
+  };
+}
+
+export async function handleUiProxy(
+  parsed: ProxyRequest,
+  declaredPorts: DeclaredPorts,
+): Promise<UiProxyResponse> {
+  const allowed = declaredPorts(parsed.appId, parsed.service);
+  if (allowed !== undefined && !allowed.has(parsed.port)) {
+    // Before the container lookup on purpose: the refusal must not depend on
+    // anything the container says about itself.
+    return coded(
+      400,
+      "INTERNAL",
+      `port ${parsed.port} is not declared by ${parsed.appId}/${parsed.service}`,
+    );
+  }
+
+  const lookup = await resolveContainerEndpoint(parsed);
+  if (!("endpoint" in lookup)) return coded(502, lookup.code, lookup.message);
+  const endpoint = lookup.endpoint;
+
+  // `path` may already carry a literal query string, so the stitch fn's
+  // `query` is merged onto it via URL rather than appended.
+  const target = new URL(`http://${endpoint.ip}:${parsed.port}${parsed.path}`);
+  if (parsed.query) {
+    for (const [k, v] of Object.entries(parsed.query)) target.searchParams.set(k, String(v));
+  }
+  const headers: Record<string, string> = parsed.headers ? { ...parsed.headers } : {};
+  const sendBody = parsed.body !== undefined && parsed.method !== "GET";
+  if (sendBody) headers["content-type"] = "application/json";
+  const init: RequestInit & { client: Deno.HttpClient } = {
+    method: parsed.method,
+    headers,
+    signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
+    client: getProxyHttpClient(),
+  };
+  if (sendBody) init.body = JSON.stringify(parsed.body);
+
+  let upstream: Response;
+  try {
+    upstream = await fetch(target, init);
+  } catch (e) {
+    const err = e as Error;
+    const timedOut = err.name === "TimeoutError";
+    return coded(
+      502,
+      timedOut ? "UPSTREAM_TIMEOUT" : "UPSTREAM_ERROR",
+      timedOut
+        ? `upstream timed out after ${UPSTREAM_TIMEOUT_MS}ms`
+        : `upstream fetch failed: ${describeFetchError(err)}`,
+    );
+  }
+
+  // Pass through JSON; bail out of non-JSON responses with a wrapped
+  // error so the iframe surface gets a structured failure.
+  const contentType = upstream.headers.get("content-type") ?? "";
+  let text: string;
+  try {
+    text = await readBoundedText(upstream, MAX_LOADER_RESPONSE_BYTES);
+  } catch (e) {
+    const message =
+      e instanceof RangeError ? e.message : `upstream read failed: ${describeFetchError(e)}`;
+    return coded(502, "UPSTREAM_ERROR", message);
+  }
+  // An empty body is a success shape, not a contract break: `204 No Content`
+  // carries no content-type, so the JSON guard below would report a working
+  // PUT/DELETE as a 502. Normalize to `{}` and keep the upstream status —
+  // Home's action route still fails the call on a non-2xx.
+  if (text.trim() === "") return json(upstream.status, "{}");
+  if (!contentType.includes("application/json")) {
+    return coded(
+      502,
+      "UPSTREAM_ERROR",
+      `upstream returned non-JSON (${upstream.status}): ${text.slice(0, 256)}`,
+    );
+  }
+  // Relay the JSON bytes verbatim: a malformed body that still claims JSON
+  // passes through so the iframe's own parse surfaces the upstream error.
+  return json(upstream.status, text);
+}
+
+function json(status: number, body: string): UiProxyResponse {
+  return { status, headers: { "content-type": "application/json" }, body };
+}
+
+function coded(status: number, code: UiErrorCode, message: string): UiProxyResponse {
+  return json(status, JSON.stringify(uiErrorBody(code, message)));
+}
+
+async function readBoundedText(res: Response, maxBytes: number): Promise<string> {
+  // Trust Content-Length when the server sends one — saves us streaming
+  // a multi-GB body just to throw it away.
+  const contentLength = res.headers.get("content-length");
+  if (contentLength !== null) {
+    const declared = Number(contentLength);
+    if (Number.isFinite(declared) && declared > maxBytes) {
+      throw new RangeError(`upstream body exceeds ${maxBytes} bytes (declared ${declared})`);
+    }
+  }
+  const reader = res.body?.getReader();
+  if (!reader) return "";
+  const chunks: Uint8Array[] = [];
+  let received = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    received += value.byteLength;
+    if (received > maxBytes) {
+      await reader.cancel();
+      throw new RangeError(`upstream body exceeds ${maxBytes} bytes`);
+    }
+    chunks.push(value);
+  }
+  const merged = new Uint8Array(received);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(merged);
+}
+
+function bytesBody(bytes: Uint8Array): ReadableStream<Uint8Array> {
+  return new ReadableStream<Uint8Array>(
+    {
+      pull: (controller) => {
+        if (bytes.length > 0) controller.enqueue(bytes);
+        controller.close();
+      },
+    },
+    { highWaterMark: 0 },
+  );
+}
+
+async function readAll(body: ReadableStream<Uint8Array>): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  for await (const chunk of body) {
+    chunks.push(chunk);
+    size += chunk.length;
+  }
+  const out = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+}