@moondeck/station 0.0.0-stage → 1.0.0

package/lib/shared-files.ts

@@ -0,0 +1,156 @@
+/**
+ * ADR-062 shared files — host-side handler for the `filesOp` bridge command.
+ * Two roots per host: org files (per host+org) and personal files (per
+ * host+user), both under <workspaceRoot>/shared/. The cloud is a window
+ * only; this module is the sole writer host-side.
+ *
+ * SECURITY: the cloud relays paths from browsers, and containers mount these
+ * same dirs — every path is treated as hostile. Two gates on every op:
+ * `resolveInsideRoot` (relative-only, ".."-free, resolved-prefix check) and
+ * `assertRealInsideRoot` (realpath containment, so a symlink the container
+ * planted inside a rw mount cannot point ops outside the root).
+ */
+
+import { promises as fsp } from "node:fs";
+import { dirname, join, normalize, resolve, sep } from "node:path";
+
+import type { FilesEntry, FilesOpInput, FilesOpValue } from "../src/protocol";
+import { env } from "./env";
+
+/** Max bytes returned per read chunk (base64 expands ~4/3; WS cap is 16MB). */
+const MAX_READ_CHUNK = 4 * 1024 * 1024;
+/** Max bytes accepted per write chunk. */
+const MAX_WRITE_CHUNK = 8 * 1024 * 1024;
+/** Max entries returned by a single list. */
+const MAX_LIST_ENTRIES = 2_000;
+
+/**
+ * Filesystem-safe directory name for an org/user id. Personal org ids are
+ * `personal:<userId>` — the colon breaks docker's short-syntax bind spec
+ * (src:dst:mode), so it is mapped to "_". MUST stay in lockstep with the
+ * identical mapping in scripts/agent/task-up.sh (shared_fs_name).
+ */
+export function sharedDirName(id: string): string {
+  return id.replace(/:/g, "_");
+}
+
+export function sharedRoot(scope: "org" | "me", orgId: string, userId: string): string {
+  return scope === "org"
+    ? resolve(env.workspaceRoot, "shared", "orgs", sharedDirName(orgId))
+    : resolve(env.workspaceRoot, "shared", "users", sharedDirName(userId));
+}
+
+function resolveInsideRoot(root: string, relPath: string): string {
+  // Relative paths only — an absolute path is rejected, never reinterpreted.
+  if (/^[/\\]/.test(relPath) || /^[A-Za-z]:/.test(relPath)) {
+    throw new Error("invalid path");
+  }
+  const cleaned = normalize(relPath);
+  if (
+    cleaned.split(/[/\\]/).some((seg) => seg === "..") ||
+    cleaned.includes("\0")
+  ) {
+    throw new Error("invalid path");
+  }
+  const target = resolve(root, cleaned);
+  if (target !== root && !target.startsWith(root + sep)) {
+    throw new Error("invalid path");
+  }
+  return target;
+}
+
+/**
+ * String containment (resolveInsideRoot) can't see symlinks the container
+ * planted inside a rw mount — `root/evil → /` would pass it. Walk up from
+ * the target to its deepest EXISTING ancestor, realpath it, and require the
+ * REAL location to stay inside the real root (ADR-062 "no symlink escapes").
+ */
+async function assertRealInsideRoot(root: string, target: string): Promise<void> {
+  const realRoot = await fsp.realpath(root);
+  let probe = target;
+  for (;;) {
+    try {
+      const real = await fsp.realpath(probe);
+      if (real !== realRoot && !real.startsWith(realRoot + sep)) {
+        throw new Error("invalid path");
+      }
+      return;
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+      const parent = dirname(probe);
+      if (parent === probe) throw new Error("invalid path");
+      probe = parent;
+    }
+  }
+}
+
+export async function handleFilesOp(input: FilesOpInput): Promise<FilesOpValue> {
+  if (!input.orgId && input.scope === "org") throw new Error("missing orgId");
+  if (!input.userId && input.scope === "me") throw new Error("missing userId");
+  const root = sharedRoot(input.scope, input.orgId, input.userId);
+  await fsp.mkdir(root, { recursive: true });
+  const target = resolveInsideRoot(root, input.path ?? "");
+  await assertRealInsideRoot(root, target);
+
+  switch (input.op) {
+    case "list": {
+      const names = await fsp.readdir(target);
+      const entries: FilesEntry[] = [];
+      for (const name of names.slice(0, MAX_LIST_ENTRIES)) {
+        try {
+          const st = await fsp.lstat(join(target, name));
+          entries.push({
+            name,
+            dir: st.isDirectory(),
+            size: st.isDirectory() ? 0 : st.size,
+            mtimeMs: st.mtimeMs,
+          });
+        } catch {
+          // Raced deletion — skip.
+        }
+      }
+      entries.sort((a, b) =>
+        a.dir === b.dir ? a.name.localeCompare(b.name) : a.dir ? -1 : 1,
+      );
+      return { entries };
+    }
+    case "read": {
+      const st = await fsp.stat(target);
+      if (st.isDirectory()) throw new Error("is a directory");
+      const offset = Math.max(0, input.offset ?? 0);
+      const length = Math.min(input.length ?? MAX_READ_CHUNK, MAX_READ_CHUNK);
+      const fh = await fsp.open(target, "r");
+      try {
+        const buf = Buffer.alloc(Math.min(length, Math.max(0, st.size - offset)));
+        const { bytesRead } = await fh.read(buf, 0, buf.length, offset);
+        return {
+          dataBase64: buf.subarray(0, bytesRead).toString("base64"),
+          size: st.size,
+          eof: offset + bytesRead >= st.size,
+        };
+      } finally {
+        await fh.close();
+      }
+    }
+    case "write": {
+      if (target === root) throw new Error("invalid path");
+      const data = Buffer.from(input.dataBase64 ?? "", "base64");
+      if (data.length > MAX_WRITE_CHUNK) throw new Error("chunk too large");
+      await fsp.mkdir(resolve(target, ".."), { recursive: true });
+      await fsp.writeFile(target, data, { flag: input.append ? "a" : "w" });
+      return {};
+    }
+    case "mkdir": {
+      if (target === root) throw new Error("invalid path");
+      await fsp.mkdir(target, { recursive: true });
+      return {};
+    }
+    case "delete": {
+      if (target === root) throw new Error("invalid path");
+      await fsp.rm(target, { recursive: true, force: true });
+      return {};
+    }
+    default:
+      throw new Error(`unknown op: ${String(input.op)}`);
+  }
+}

package/lib/sidecar-assign.ts

@@ -0,0 +1,49 @@
+/**
+ * The warm-pool claim's host half (docs/machine-warm-pool.md): the cloud's
+ * `sidecarAssign` names the task this pool machine now serves. The task id
+ * is appended to the sidecar unit's EnvironmentFile and the process exits
+ * with ASSIGN_RESTART_EXIT_CODE — systemd (`Restart=on-failure`) brings the
+ * unit back up in plain sidecar mode, the tested runner-restart path.
+ * Rewriting a root-owned file and restarting is deliberately boring: no
+ * runtime re-wiring of boot-time gates, no half-pool half-task states.
+ */
+import { appendFileSync, readFileSync } from "node:fs";
+
+import { sidecarPoolMode } from "./sidecar";
+
+/** Anything but 0 and 3: 0 would not restart the unit, 3 is the lease-denied
+ *  latch (`RestartPreventExitStatus=3`). */
+export const ASSIGN_RESTART_EXIT_CODE = 64;
+const EXIT_DELAY_MS = 500;
+
+export function sidecarEnvFile(): string {
+  return process.env.UAI_SIDECAR_ENV_FILE ?? "/etc/uai/sidecar.env";
+}
+
+export function applySidecarAssign(
+  taskId: string,
+  deps: {
+    envFile?: string;
+    exit?: (code: number) => void;
+    append?: (file: string, text: string) => void;
+    read?: (file: string) => string;
+  } = {},
+): void {
+  if (!sidecarPoolMode()) {
+    throw new Error("not a pool machine (already assigned, or not a sidecar)");
+  }
+  if (!/^[a-z0-9]{1,32}$/i.test(taskId)) throw new Error("invalid task id");
+  const file = deps.envFile ?? sidecarEnvFile();
+  const read = deps.read ?? ((f: string) => readFileSync(f, "utf8"));
+  const current = read(file);
+  if (/^UAI_SIDECAR_TASK=/m.test(current)) {
+    throw new Error("this machine already carries a task assignment");
+  }
+  const append = deps.append ?? ((f: string, text: string) => appendFileSync(f, text));
+  append(file, `UAI_SIDECAR_TASK=${taskId}\n`);
+  // Ack first, then die: the delay lets the command reply flush to the
+  // bridge before the socket goes with the process.
+  const exit = deps.exit ?? ((code: number) => process.exit(code));
+  const timer = setTimeout(() => exit(ASSIGN_RESTART_EXIT_CODE), EXIT_DELAY_MS);
+  timer.unref?.();
+}

package/lib/sidecar-guard.ts

@@ -0,0 +1,58 @@
+/**
+ * A task machine's own guards, held by its sidecar (root) before the task
+ * runs anything there (ADR-133; ADR-153 §3, "Where the proxy runs"):
+ *
+ * - The runner's data directory is root's alone, so the task's user cannot
+ *   read the store's database. Its values are sealed with a root-only key,
+ *   but its key names and bindings would still be there to read.
+ * - The instance metadata service is closed to the task's user, so it can
+ *   read neither the machine's user-data, which carries the sidecar's host
+ *   token, nor the instance role's credentials.
+ *
+ * The image's prepare step sets both up too. The runner insists on them at
+ * every task-up, since an image that skipped or failed that step would
+ * otherwise let the task start without them.
+ */
+
+import { execFileSync } from "node:child_process";
+import { chmodSync, mkdirSync } from "node:fs";
+
+/** Rejects the task user's traffic to the instance metadata service, with
+ *  whichever iptables backend the kernel serves (as the proxy's chain does),
+ *  adding the rule only when it is missing. */
+export const METADATA_BLOCK_SCRIPT = [
+  "set -eu",
+  "ipt=iptables",
+  "if ! command -v iptables >/dev/null 2>&1 || ! iptables -S OUTPUT >/dev/null 2>&1; then ipt=iptables-legacy; fi",
+  'command -v "$ipt" >/dev/null 2>&1 || { echo "iptables is not installed" >&2; exit 1; }',
+  '"$ipt" -C OUTPUT -d 169.254.169.254 -m owner --uid-owner node -j REJECT 2>/dev/null \\',
+  '  || "$ipt" -I OUTPUT -d 169.254.169.254 -m owner --uid-owner node -j REJECT',
+].join("\n");
+
+export interface SidecarGuardDeps {
+  dataDir: string;
+  /** Runs a script as this (root) process; throws when it fails. */
+  runScript(script: string): void;
+}
+
+const defaultRunScript = (script: string): void => {
+  execFileSync("/bin/sh", ["-c", script], { stdio: ["ignore", "ignore", "pipe"], timeout: 15_000 });
+};
+
+/** Puts both guards in place, or says which could not be. */
+export function guardSidecarMachine(deps: Partial<SidecarGuardDeps> & Pick<SidecarGuardDeps, "dataDir">): { ok: true } | { ok: false; reason: string } {
+  const runScript = deps.runScript ?? defaultRunScript;
+  try {
+    mkdirSync(deps.dataDir, { recursive: true, mode: 0o700 });
+    chmodSync(deps.dataDir, 0o700);
+  } catch (error) {
+    return { ok: false, reason: `its data directory could not be made private (${error instanceof Error ? error.message : String(error)})` };
+  }
+  try {
+    runScript(METADATA_BLOCK_SCRIPT);
+  } catch (error) {
+    const detail = error instanceof Error && "stderr" in error ? String((error as { stderr?: unknown }).stderr ?? "").trim() : "";
+    return { ok: false, reason: `the instance metadata service could not be closed to the task${detail ? ` (${detail})` : ""}` };
+  }
+  return { ok: true };
+}

package/lib/sidecar-resume.ts

@@ -0,0 +1,178 @@
+/**
+ * ADR-133: what a sidecar re-asserts when it starts on a machine whose task
+ * is ALREADY up — a restart of the unit, not a fresh launch.
+ *
+ * Task-up mounts the shared files and launches the editor once; nothing ran
+ * them again when the runner came back, so an operator's restart (smoke 10,
+ * 2026-09-10) left /workspace/files as two dead endpoints and the Editor tab
+ * empty for the rest of the task. The unit now keeps those processes alive
+ * across a stop (KillMode=process), and this pass covers what that cannot:
+ * a crash, an older unit, a SIGKILL. Both steps are idempotent — a healthy
+ * mount and a live code-server are left alone — so running it on every
+ * lease-ready costs one stat each.
+ */
+
+import { eq } from "drizzle-orm";
+
+import * as schema from "../db/schema";
+import { getDb } from "./db";
+import { mountSharedFilesFromLease } from "./shared-files-s3";
+import { sidecarTaskId } from "./sidecar";
+import { reconstructPersistedTaskEnvironment } from "./task-environment";
+import { installMachineProjectEnvLoader } from "./machine-project-env-loader";
+import { activeSecretsProxy } from "./secrets-proxy/host";
+import { taskViewProven } from "./secrets-proxy/sidecar-view";
+import { reconcileSidecarTaskProxy } from "./secrets-proxy/task-trust";
+import { ensureCodeServer } from "./task-environment/machine-task-up";
+import type { TaskEnvironmentHandle } from "./task-environment/types";
+
+type PersistedTask = typeof schema.hostTasks.$inferSelect;
+
+export interface SidecarResumeDeps {
+  taskId?: () => string | null;
+  persistedTask?: (taskId: string) => PersistedTask | undefined;
+  reconstruct?: (task: PersistedTask) => Promise<TaskEnvironmentHandle<unknown> | null>;
+  mount?: (
+    handle: TaskEnvironmentHandle<unknown>,
+    workspacePath: string,
+    task: { sharedFiles?: string },
+  ) => Promise<string | null>;
+  /** ADR-154 Files v1: install and mount the exact recovered run's session
+   * from trusted host state.  Unlike the legacy marker path, failure is
+   * terminal for this recovery setup and is not converted to a warning. */
+  prepareFiles?: (handle: TaskEnvironmentHandle<unknown>) => Promise<void>;
+  loader?: (handle: TaskEnvironmentHandle<unknown>) => Promise<void>;
+  editor?: (handle: TaskEnvironmentHandle<unknown>, workspacePath: string) => Promise<void>;
+  proxy?: typeof reconcileSidecarTaskProxy;
+  /** Remount the shared files from the v0 lease (the default). ADR-154: a
+   *  task on cloud credentials remounts them from its own run's Files
+   *  session instead, never from the runner's process-wide v0 lease. */
+  remountFiles?: boolean;
+  /** ADR-154: aborted when the run this resume is for ends (Stop, teardown,
+   *  a newer run, shutdown): no step starts after. */
+  signal?: AbortSignal;
+}
+
+export interface SidecarResumeResult {
+  /** `no-task`: not a sidecar. `not-up`: the task has not come up on this
+   * machine yet (a fresh launch, or a start that a runner restart cut short —
+   * task-up does the work). `resumed`: the
+   * task was up before this runner started; its mounts and editor were
+   * re-asserted, with any per-step warning listed. */
+  outcome: "no-task" | "not-up" | "resumed" | "interrupted";
+  warnings: string[];
+}
+
+/** The mode task-up mounted with, recorded in the files-mounted marker; null
+ *  when the files were never mounted (mode off, or the mount failed then). */
+function mountedModeFromMarker(bytes: Uint8Array | null): "ro" | "rw" | null {
+  if (bytes === null) return null;
+  return Buffer.from(bytes).toString("utf8").trim() === "rw" ? "rw" : "ro";
+}
+
+/**
+ * ADR-153 machines, at the runner's start and before any command can reach
+ * the task: a task this machine has a row for, in whatever state (`starting`
+ * included), is held for this boot unless its hosts-view record proves it.
+ * A start driven again, or a resume, then finds the hold. Only task-up for a
+ * task with no row admits it fresh (src/index.ts).
+ */
+export function classifySidecarTaskView(
+  deps: Pick<SidecarResumeDeps, "taskId" | "persistedTask"> = {},
+): void {
+  const taskId = (deps.taskId ?? sidecarTaskId)();
+  if (!taskId) return;
+  if ((deps.persistedTask ?? defaultPersistedTask)(taskId) === undefined) return;
+  // Asked for its record: one with none is held from here on.
+  taskViewProven(taskId);
+}
+
+export async function resumeSidecarTask(
+  deps: SidecarResumeDeps = {},
+): Promise<SidecarResumeResult> {
+  const taskId = (deps.taskId ?? sidecarTaskId)();
+  if (!taskId) return { outcome: "no-task", warnings: [] };
+  const task = (deps.persistedTask ?? defaultPersistedTask)(taskId);
+  // Only a task that came all the way up: one still starting (a runner that
+  // died mid-provision) is task-up's to finish, and resuming it would race
+  // that retry's loader, proxy steps and editor.
+  if (!task?.environmentLocator || task.statusMirror !== "running") return { outcome: "not-up", warnings: [] };
+  const handle = await (deps.reconstruct ?? reconstructPersistedTaskEnvironment)(task);
+  if (!handle) return { outcome: "not-up", warnings: [] };
+  const workspacePath = handle.descriptor.workspacePath;
+  const warnings: string[] = [];
+  const ended = (): boolean => deps.signal?.aborted === true;
+  const interrupted = (): SidecarResumeResult => ({ outcome: "interrupted", warnings });
+
+  if (deps.prepareFiles) {
+    if (ended()) return interrupted();
+    await deps.prepareFiles(handle);
+  }
+
+  if (task.envMode === "loader") {
+    if (ended()) return interrupted();
+    try {
+      await (deps.loader ?? installMachineProjectEnvLoader)(handle);
+    } catch (error) {
+      // The loader remains fail-closed: no project value was baked into this
+      // machine. Keep repairing the editor/mounts, but make the degraded env
+      // channel visible to the sidecar's restart log.
+      warnings.push(
+        `project env loader: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+    // ADR-153: the proxy's CA and pins again (a reboot keeps neither the
+    // chain nor this runner's record of any of it).
+    if (ended()) return interrupted();
+    const proxyWarning = await (deps.proxy ?? reconcileSidecarTaskProxy)(
+      handle,
+      activeSecretsProxy()?.runtime ?? null,
+      { start: false, ...(deps.signal ? { isCurrent: () => !ended() } : {}) },
+    );
+    if (proxyWarning !== null) warnings.push(proxyWarning);
+  }
+
+  const marker =
+    deps.remountFiles === false
+      ? null
+      : await handle.readWorkspaceFile(`${workspacePath}/.moondeck/files-mounted`).catch(() => null);
+  const mode = mountedModeFromMarker(marker);
+  if (mode) {
+    try {
+      const warning = await (deps.mount ?? mountSharedFilesFromLease)(handle, workspacePath, {
+        sharedFiles: mode,
+      });
+      if (warning) warnings.push(warning);
+    } catch (error) {
+      warnings.push(
+        `shared files: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  if (ended()) return interrupted();
+  try {
+    await (deps.editor ?? defaultEditor)(handle, workspacePath);
+  } catch (error) {
+    warnings.push(`editor: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  return { outcome: "resumed", warnings };
+}
+
+function defaultPersistedTask(taskId: string): PersistedTask | undefined {
+  return getDb()
+    .select()
+    .from(schema.hostTasks)
+    .where(eq(schema.hostTasks.taskId, taskId))
+    .get();
+}
+
+function defaultEditor(
+  handle: TaskEnvironmentHandle<unknown>,
+  workspacePath: string,
+): Promise<void> {
+  // The theme rode task-up's preview env and is not persisted; the settings
+  // seed is skipped when the file already exists, so an existing task keeps
+  // the theme it started with.
+  return ensureCodeServer(handle, workspacePath, undefined);
+}

package/lib/sidecar.ts

@@ -0,0 +1,69 @@
+/**
+ * ADR-133 sidecar mode: this runner process lives ON the task's machine and
+ * is that one task's host. Selected by the environment the machine image's
+ * `uai-sidecar.service` writes from EC2 user-data (hostless-implementation.md
+ * §8/§9). Everything the coordinator shape needed and a single-task machine
+ * does not — the local UI, the container runtime, host-level engine accounts,
+ * the machine provider, recovery sweeps over many tasks — asks here and
+ * stays off. Owned runners never set it.
+ */
+
+export function sidecarTaskId(): string | null {
+  const value = process.env.UAI_SIDECAR_TASK?.trim();
+  return value && /^[a-z0-9]{1,32}$/i.test(value) ? value : null;
+}
+
+/**
+ * Warm-pool mode (docs/machine-warm-pool.md): a sidecar with no task yet.
+ * Same machine gates as sidecar mode, no lease (nothing to scope one to);
+ * the process waits for `sidecarAssign` to write `UAI_SIDECAR_TASK` into
+ * the unit env and restart it into plain sidecar mode. Once a task id is
+ * present, pool mode is over — even before the restart lands.
+ */
+export function sidecarPoolMode(): boolean {
+  return process.env.UAI_SIDECAR_POOL === "1" && sidecarTaskId() === null;
+}
+
+export function sidecarMode(): boolean {
+  return sidecarTaskId() !== null || sidecarPoolMode();
+}
+
+/**
+ * A sidecar is ready to run its task only once it holds a lease AND the
+ * credential store has been reconciled under it — before that there is no
+ * engine account, GitHub token, or project env on the machine, and a task-up
+ * that raced the first grant would clone without credentials and spawn
+ * agents that cannot sign in (the dispatch arrives the moment the bridge
+ * connects; the lease answer is one round trip behind it). Resolved by
+ * main.ts after the first grant's reconciles settle; never on an owned host.
+ */
+let markReady: (() => void) | null = null;
+let ready = false;
+export const sidecarReady: Promise<void> = new Promise<void>((resolve) => {
+  markReady = resolve;
+});
+export function markSidecarReady(): void {
+  if (ready) return;
+  ready = true;
+  markReady?.();
+  markReady = null;
+}
+export function isSidecarReady(): boolean {
+  return ready;
+}
+/** How long a task-up waits for the lease before giving up. */
+export const SIDECAR_READY_TIMEOUT_MS = 3 * 60_000;
+export function awaitSidecarReady(timeoutMs = SIDECAR_READY_TIMEOUT_MS): Promise<void> {
+  if (!sidecarMode() || ready) return Promise.resolve();
+  return new Promise<void>((resolve, reject) => {
+    const timer = setTimeout(
+      () => reject(new Error("the task's machine has no credentials yet (no lease granted)")),
+      timeoutMs,
+    );
+    timer.unref?.();
+    void sidecarReady.then(() => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}