@moondeck/station 0.0.1 → 1.0.0

package/lib/task-files-machine-retirement.ts

@@ -0,0 +1,116 @@
+import type { TaskMachineRetireReason } from "../src/protocol";
+import type { TaskAssignment } from "./cloud-credential-leases";
+
+export type TaskFilesMachineRetirementReason = TaskMachineRetireReason;
+
+export interface TaskFilesMachineRetirementRequest {
+  assignment: Readonly<TaskAssignment>;
+  reason: TaskFilesMachineRetirementReason;
+  error: Error;
+}
+
+/**
+ * Cutover-owned terminal path for a one-task machine whose Files boundary
+ * cannot be proved closed. The handler must synchronously quarantine local
+ * task work, then durably retry an authenticated exact-run machine-retirement
+ * request until the cloud acknowledges it or the machine dies. Exiting only
+ * the sidecar is explicitly insufficient: its systemd unit uses
+ * KillMode=process and leaves the task processes alive.
+ */
+export type TaskFilesMachineRetirementHandler = (
+  request: TaskFilesMachineRetirementRequest,
+) => undefined;
+
+let handler: TaskFilesMachineRetirementHandler | null = null;
+let retiring: TaskFilesMachineRetirementRequest | null = null;
+let damaged: Error | null = null;
+let handedOff = false;
+
+export function setTaskFilesMachineRetirementHandler(
+  next: TaskFilesMachineRetirementHandler | null,
+): void {
+  handler = next;
+}
+
+/** Files v1 must not be advertised, or activation entered, without this. */
+export function taskFilesMachineRetirementReady(): boolean {
+  return handler !== null && retiring === null && damaged === null;
+}
+
+/** An unreadable same-boot outbox record may be an irreversible retirement
+ * request. With no safe exact run to reconstruct, keep every task-process
+ * admission closed for the rest of this boot. */
+export function latchDamagedTaskFilesMachineRetirement(error: Error): void {
+  damaged ??= error;
+}
+
+/**
+ * The self provider calls this immediately around every task-process
+ * admission. Once terminal Files retirement wins, no in-flight command may
+ * repopulate the service cgroup after the retirement handler proves it empty.
+ */
+export function assertTaskFilesMachineProcessAllowed(): void {
+  if (damaged !== null) {
+    throw new Error("task-machine Files retirement state is damaged", {
+      cause: damaged,
+    });
+  }
+  if (retiring !== null) {
+    throw new Error("task-machine Files retirement is pending");
+  }
+}
+
+export function requestTaskFilesMachineRetirement(
+  request: TaskFilesMachineRetirementRequest,
+): void {
+  if (damaged !== null) {
+    throw new Error("task-machine Files retirement state is damaged", {
+      cause: damaged,
+    });
+  }
+  if (!handler) {
+    throw new Error("task-machine Files retirement is not installed", {
+      cause: request.error,
+    });
+  }
+  if (retiring === null) {
+    // First exact failure wins before any callback: a reconnect/capability
+    // pass in the callback's first await can no longer advertise this machine.
+    retiring = request;
+  } else if (
+    retiring.assignment.taskId !== request.assignment.taskId ||
+    retiring.assignment.runEpoch !== request.assignment.runEpoch
+  ) {
+    return;
+  }
+  if (handedOff) return;
+  // A synchronous handoff failure must be retryable without reopening the
+  // capability or letting a later assignment replace the first failure.
+  // Once this returns, the handler owns durable retries itself.
+  const result: unknown = handler(retiring);
+  if (result !== undefined) {
+    // A JavaScript caller can evade the type even though `undefined` (rather
+    // than `void`) rejects async handlers in TypeScript. Observe a returned
+    // promise so its later rejection is not unhandled, but never count an
+    // asynchronous callback as the required synchronous handoff.
+    if (
+      typeof result === "object" &&
+      result !== null &&
+      "then" in result &&
+      typeof (result as { then?: unknown }).then === "function"
+    ) {
+      void Promise.resolve(result).catch(() => {});
+    }
+    throw new Error("task-machine Files retirement must be handed off synchronously");
+  }
+  handedOff = true;
+}
+
+/** Tests model a fresh machine process without weakening the production
+ * one-way latch. */
+export function resetTaskFilesMachineRetirementForTests(): void {
+  handler = null;
+  retiring = null;
+  damaged = null;
+  handedOff = false;
+}

package/lib/task-files-readiness.ts

@@ -0,0 +1,127 @@
+import {
+  accessSync,
+  chmodSync,
+  constants,
+  lstatSync,
+  mkdirSync,
+  statfsSync,
+} from "node:fs";
+
+export const TASK_FILES_MOUNT_S3_PATH = "/usr/bin/mount-s3";
+export const TASK_FILES_FUSE_PATH = "/dev/fuse";
+export const TASK_FILES_HELPER_PATHS = [
+  "/usr/bin/python3",
+  "/usr/bin/env",
+  "/usr/bin/mountpoint",
+  "/usr/bin/umount",
+  "/bin/sh",
+] as const;
+export const TASK_FILES_RUNTIME_ROOT = "/run";
+export const TASK_FILES_SESSION_ROOT = "/run/uai";
+
+// linux/magic.h: credentials may be written only to a memory-backed runtime.
+export const TMPFS_MAGIC = 0x01021994;
+
+export type TaskFilesV1Readiness =
+  | { ready: true }
+  | {
+      ready: false;
+      reason:
+        | "not-task-machine"
+        | "not-root"
+        | "mount-s3-unavailable"
+        | "fuse-unavailable"
+        | "helper-unavailable"
+        | "run-not-tmpfs"
+        | "session-root-unsafe";
+    };
+
+export interface TaskFilesV1ReadinessDeps {
+  getuid(): number;
+  access(path: string, mode: number): void;
+  prepareSessionRoot(): void;
+  statfs(path: string): { type: number | bigint };
+}
+
+function prepareSessionRoot(): void {
+  try {
+    mkdirSync(TASK_FILES_SESSION_ROOT, { mode: 0o700 });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+  }
+  const info = lstatSync(TASK_FILES_SESSION_ROOT);
+  if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 0) {
+    throw new Error("the Files session root is not a root-owned directory");
+  }
+  chmodSync(TASK_FILES_SESSION_ROOT, 0o700);
+}
+
+const DEFAULT_DEPS: TaskFilesV1ReadinessDeps = {
+  getuid: () => process.getuid?.() ?? -1,
+  access: accessSync,
+  prepareSessionRoot,
+  statfs: (path) => statfsSync(path),
+};
+
+/**
+ * Local half of the Files-v1 capability check. Cloud mode is irrelevant: a
+ * machine advertises what this process can safely do before activation. Keep
+ * this synchronous so the connect capability frame cannot race a later
+ * prerequisite probe.
+ */
+export function taskFilesV1Readiness(
+  taskMachine: boolean,
+  deps: TaskFilesV1ReadinessDeps = DEFAULT_DEPS,
+): TaskFilesV1Readiness {
+  if (!taskMachine) return { ready: false, reason: "not-task-machine" };
+
+  let uid: number;
+  try {
+    uid = deps.getuid();
+  } catch {
+    return { ready: false, reason: "not-root" };
+  }
+  if (uid !== 0) return { ready: false, reason: "not-root" };
+
+  try {
+    deps.access(TASK_FILES_MOUNT_S3_PATH, constants.X_OK);
+  } catch {
+    return { ready: false, reason: "mount-s3-unavailable" };
+  }
+
+  try {
+    deps.access(TASK_FILES_FUSE_PATH, constants.R_OK | constants.W_OK);
+  } catch {
+    return { ready: false, reason: "fuse-unavailable" };
+  }
+
+  try {
+    for (const path of TASK_FILES_HELPER_PATHS) {
+      deps.access(path, constants.X_OK);
+    }
+  } catch {
+    return { ready: false, reason: "helper-unavailable" };
+  }
+
+  try {
+    if (Number(deps.statfs(TASK_FILES_RUNTIME_ROOT).type) !== TMPFS_MAGIC) {
+      return { ready: false, reason: "run-not-tmpfs" };
+    }
+  } catch {
+    return { ready: false, reason: "run-not-tmpfs" };
+  }
+
+  try {
+    deps.prepareSessionRoot();
+    // `/run` being tmpfs is insufficient: an existing `/run/uai` may be a
+    // bind mount onto persistent storage. Capability advertisement proves
+    // the exact directory in which the session will be written.
+    if (Number(deps.statfs(TASK_FILES_SESSION_ROOT).type) !== TMPFS_MAGIC) {
+      return { ready: false, reason: "session-root-unsafe" };
+    }
+  } catch {
+    return { ready: false, reason: "session-root-unsafe" };
+  }
+
+  return { ready: true };
+}

package/lib/task-gateway-port.ts

@@ -0,0 +1,22 @@
+/**
+ * The port the runner's task gateway listens on (`UAI_MCP_GATEWAY_PORT`,
+ * 5877 by default): where task forwarders relay, and where a machine's
+ * sidecar serves its own task directly on loopback. A module of its own so
+ * the secrets proxy can name it without loading the gateway.
+ */
+
+const DEFAULT_MCP_GATEWAY_PORT = 5877;
+
+function gatewayPort(value: string | undefined): number {
+  if (value === undefined || !/^\d+$/.test(value)) {
+    return DEFAULT_MCP_GATEWAY_PORT;
+  }
+  const port = Number(value);
+  return Number.isSafeInteger(port) && port >= 1 && port <= 65_535
+    ? port
+    : DEFAULT_MCP_GATEWAY_PORT;
+}
+
+export const MCP_GATEWAY_PORT = gatewayPort(
+  process.env.UAI_MCP_GATEWAY_PORT,
+);

package/lib/task-gateway-protocol.ts

@@ -0,0 +1,190 @@
+import {
+  createHmac,
+  timingSafeEqual,
+  type KeyObject,
+} from "node:crypto";
+
+/**
+ * Binary, versioned task-forwarder handshake (ADR-153).
+ *
+ * The leading NUL cannot be the first byte of either HTTP/1.1 or TLS. That
+ * lets the shared gateway reject direct TLS while keeping the existing MCP
+ * HTTP listener on the same port. Length-prefixed fields keep unauthenticated
+ * input bounded before a task identity has been proved.
+ */
+export const TASK_FORWARDER_PREAMBLE_MAGIC = Buffer.from([
+  0x00, 0x55, 0x41, 0x49, 0x46, 0x57, 0x44, 0x01,
+]);
+export const TASK_FORWARDER_CHALLENGE_MAGIC = Buffer.from("UAICHL01", "ascii");
+export const TASK_FORWARDER_RESPONSE_MAGIC = Buffer.from("UAIRES01", "ascii");
+export const TASK_FORWARDER_ACCEPT_MAGIC = Buffer.from("UAIOK001", "ascii");
+
+export const TASK_FORWARDER_SECRET_BYTES = 32;
+/** Numerical container uid reserved for the forwarder (neither root nor node). */
+export const TASK_FORWARDER_UID = 65_532;
+export const TASK_FORWARDER_NONCE_BYTES = 32;
+export const TASK_FORWARDER_MAC_BYTES = 32;
+export const TASK_FORWARDER_MAX_TASK_ID_BYTES = 128;
+export const TASK_FORWARDER_MAX_GENERATION_BYTES = 64;
+export const TASK_FORWARDER_PREAMBLE_HEADER_BYTES =
+  TASK_FORWARDER_PREAMBLE_MAGIC.length + 2;
+export const TASK_FORWARDER_MAX_PREAMBLE_BYTES =
+  TASK_FORWARDER_PREAMBLE_HEADER_BYTES +
+  TASK_FORWARDER_MAX_TASK_ID_BYTES +
+  TASK_FORWARDER_MAX_GENERATION_BYTES;
+export const TASK_FORWARDER_CHALLENGE_BYTES =
+  TASK_FORWARDER_CHALLENGE_MAGIC.length + TASK_FORWARDER_NONCE_BYTES;
+export const TASK_FORWARDER_RESPONSE_BYTES =
+  TASK_FORWARDER_RESPONSE_MAGIC.length + TASK_FORWARDER_MAC_BYTES;
+
+const FORWARDER_HMAC_DOMAIN = Buffer.from(
+  "uai-task-forwarder-v1\0",
+  "ascii",
+);
+const FIELD_SEPARATOR = Buffer.from([0]);
+const FORWARDER_GENERATION = /^[A-Za-z0-9_-]{16,64}$/;
+
+export interface TaskForwarderIdentity {
+  taskId: string;
+  generation: string;
+}
+
+export function encodeTaskForwarderPreamble(
+  identity: TaskForwarderIdentity,
+): Buffer {
+  const taskId = Buffer.from(identity.taskId, "utf8");
+  const generation = Buffer.from(identity.generation, "ascii");
+  if (
+    taskId.length < 1 ||
+    taskId.length > TASK_FORWARDER_MAX_TASK_ID_BYTES ||
+    generation.length < 1 ||
+    generation.length > TASK_FORWARDER_MAX_GENERATION_BYTES ||
+    !FORWARDER_GENERATION.test(identity.generation)
+  ) {
+    throw new Error("invalid task forwarder identity");
+  }
+  return Buffer.concat([
+    TASK_FORWARDER_PREAMBLE_MAGIC,
+    Buffer.from([taskId.length, generation.length]),
+    taskId,
+    generation,
+  ]);
+}
+
+export function taskForwarderPreambleLength(header: Buffer): number | null {
+  if (
+    header.length !== TASK_FORWARDER_PREAMBLE_HEADER_BYTES ||
+    !header.subarray(0, TASK_FORWARDER_PREAMBLE_MAGIC.length).equals(
+      TASK_FORWARDER_PREAMBLE_MAGIC,
+    )
+  ) {
+    return null;
+  }
+  const taskIdBytes = header[TASK_FORWARDER_PREAMBLE_MAGIC.length] ?? 0;
+  const generationBytes =
+    header[TASK_FORWARDER_PREAMBLE_MAGIC.length + 1] ?? 0;
+  if (
+    taskIdBytes < 1 ||
+    taskIdBytes > TASK_FORWARDER_MAX_TASK_ID_BYTES ||
+    generationBytes < 1 ||
+    generationBytes > TASK_FORWARDER_MAX_GENERATION_BYTES
+  ) {
+    return null;
+  }
+  return TASK_FORWARDER_PREAMBLE_HEADER_BYTES + taskIdBytes + generationBytes;
+}
+
+export function decodeTaskForwarderPreamble(
+  preamble: Buffer,
+): TaskForwarderIdentity | null {
+  const header = preamble.subarray(0, TASK_FORWARDER_PREAMBLE_HEADER_BYTES);
+  const expectedLength = taskForwarderPreambleLength(header);
+  if (expectedLength === null || preamble.length !== expectedLength) return null;
+
+  const taskIdBytes = header[TASK_FORWARDER_PREAMBLE_MAGIC.length] ?? 0;
+  const taskStart = TASK_FORWARDER_PREAMBLE_HEADER_BYTES;
+  const generationStart = taskStart + taskIdBytes;
+  const rawTaskId = preamble.subarray(taskStart, generationStart);
+  const rawGeneration = preamble.subarray(generationStart);
+  const taskId = rawTaskId.toString("utf8");
+  const generation = rawGeneration.toString("ascii");
+  // Both Node decoders replace or mask malformed bytes. Require exact
+  // round-trips so two wire identities can never alias the same HMAC input.
+  if (
+    !Buffer.from(taskId, "utf8").equals(rawTaskId) ||
+    !Buffer.from(generation, "ascii").equals(rawGeneration) ||
+    !FORWARDER_GENERATION.test(generation)
+  ) {
+    return null;
+  }
+  return { taskId, generation };
+}
+
+export function encodeTaskForwarderChallenge(nonce: Buffer): Buffer {
+  if (nonce.length !== TASK_FORWARDER_NONCE_BYTES) {
+    throw new Error("invalid task forwarder nonce");
+  }
+  return Buffer.concat([TASK_FORWARDER_CHALLENGE_MAGIC, nonce]);
+}
+
+export function decodeTaskForwarderChallenge(
+  challenge: Buffer,
+): Buffer | null {
+  if (
+    challenge.length !== TASK_FORWARDER_CHALLENGE_BYTES ||
+    !challenge.subarray(0, TASK_FORWARDER_CHALLENGE_MAGIC.length).equals(
+      TASK_FORWARDER_CHALLENGE_MAGIC,
+    )
+  ) {
+    return null;
+  }
+  return Buffer.from(challenge.subarray(TASK_FORWARDER_CHALLENGE_MAGIC.length));
+}
+
+export function taskForwarderMac(
+  secret: Buffer | KeyObject,
+  identity: TaskForwarderIdentity,
+  nonce: Buffer,
+): Buffer {
+  if (nonce.length !== TASK_FORWARDER_NONCE_BYTES) {
+    throw new Error("invalid task forwarder nonce");
+  }
+  return createHmac("sha256", secret)
+    .update(FORWARDER_HMAC_DOMAIN)
+    .update(identity.taskId, "utf8")
+    .update(FIELD_SEPARATOR)
+    .update(identity.generation, "ascii")
+    .update(FIELD_SEPARATOR)
+    .update(nonce)
+    .digest();
+}
+
+export function encodeTaskForwarderResponse(mac: Buffer): Buffer {
+  if (mac.length !== TASK_FORWARDER_MAC_BYTES) {
+    throw new Error("invalid task forwarder response");
+  }
+  return Buffer.concat([TASK_FORWARDER_RESPONSE_MAGIC, mac]);
+}
+
+export function decodeTaskForwarderResponse(response: Buffer): Buffer | null {
+  if (
+    response.length !== TASK_FORWARDER_RESPONSE_BYTES ||
+    !response.subarray(0, TASK_FORWARDER_RESPONSE_MAGIC.length).equals(
+      TASK_FORWARDER_RESPONSE_MAGIC,
+    )
+  ) {
+    return null;
+  }
+  return Buffer.from(response.subarray(TASK_FORWARDER_RESPONSE_MAGIC.length));
+}
+
+export function verifyTaskForwarderMac(
+  expected: Buffer,
+  provided: Buffer,
+): boolean {
+  return (
+    expected.length === TASK_FORWARDER_MAC_BYTES &&
+    provided.length === expected.length &&
+    timingSafeEqual(expected, provided)
+  );
+}

package/lib/task-gateway-recovery.ts

@@ -0,0 +1,47 @@
+import {
+  activeLoaderHostTasks,
+  hostTaskProjectEnvSpec,
+} from "./runtime-state";
+import { taskEnvResolver } from "./task-env-source";
+import { recoverTaskGatewayTask } from "./task-gateway-state";
+
+export interface TaskGatewayRecoverySummary {
+  readonly recovered: number;
+  readonly failed: number;
+}
+
+/**
+ * Reattach value resolvers before the gateway starts listening. Persisted
+ * credentials load inactive, so a malformed row or missing identity stays
+ * fail-closed and its surviving forwarder cannot authenticate.
+ */
+export function recoverActiveTaskGateways(): TaskGatewayRecoverySummary {
+  let recovered = 0;
+  let failed = 0;
+  for (const task of activeLoaderHostTasks()) {
+    try {
+      const spec = hostTaskProjectEnvSpec(task);
+      if (spec === null) {
+        throw new Error("loader task has no persisted project env spec");
+      }
+      // The generation serves the run its runtime was established for.
+      const runEpoch = task.runEpoch > 0 ? task.runEpoch : null;
+      const identity = recoverTaskGatewayTask(task.taskId, {
+        resolveEnv: taskEnvResolver(task.taskId, spec, runEpoch),
+        runEpoch,
+      });
+      if (identity === null) {
+        throw new Error("loader task has no valid persisted gateway identity");
+      }
+      recovered += 1;
+    } catch (error) {
+      failed += 1;
+      console.warn(
+        `[task-gateway] task ${task.taskId}: boot recovery stayed closed: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    }
+  }
+  return { recovered, failed };
+}