@moondeck/station 0.0.0-stage → 1.0.0

package/lib/task-env-forwarder.ts

@@ -0,0 +1,228 @@
+import type { TaskEnvironmentHandle } from "./task-environment/types";
+import { isEnvironmentTransportProvider } from "./task-environment/types";
+import { MACHINE_GATEWAY_TUNNEL_PORT } from "./machine-gateway-tunnel";
+import {
+  installMachineProjectEnvLoader,
+  installMachineTaskForwarderAsset,
+} from "./machine-project-env-loader";
+import { MCP_GATEWAY_PORT, taskGatewayHost } from "./mcp-gateway";
+import { STANDARD_DEFAULT_NODE_VERSION } from "./standard-image";
+import {
+  taskGatewayForwarderSecretInput,
+  taskGatewayRegistrationForRecovery,
+} from "./task-gateway-state";
+import { TASK_FORWARDER_SECRET_BYTES } from "./task-gateway-protocol";
+import type { TaskGatewayRegistration } from "./task-gateway-state";
+
+const STEP_OUTPUT_BYTES = 256 * 1024;
+const APPLE_FORWARDER_NODE_PATH =
+  `/run/uai/asdf-lower/installs/nodejs/${STANDARD_DEFAULT_NODE_VERSION}/bin/node` as const;
+
+type TaskForwarderNodePath =
+  | "/usr/bin/node"
+  | "/usr/local/bin/node"
+  | typeof APPLE_FORWARDER_NODE_PATH;
+
+// Shared by initial machine provisioning and every recovery topology. Keep
+// the ownership/process proof in lockstep with task-up.sh's container
+// launcher. The generation secret arrives on stdin; no task file, argv entry,
+// or environment variable ever contains it.
+const START_TASK_FORWARDER = String.raw`set -eu
+task_id=$1
+generation=$2
+gateway_host=$3
+gateway_port=$4
+node_path=$5
+case "$node_path" in
+  /usr/bin/node | /usr/local/bin/node) ;;
+  ${APPLE_FORWARDER_NODE_PATH})
+    [ ! -L "$node_path" ] && [ -f "$node_path" ] && [ -x "$node_path" ] \
+      || exit 1
+    ;;
+  *) exit 1 ;;
+esac
+case "$gateway_port" in ''|*[!0-9]*) exit 1 ;; esac
+[ "$gateway_port" -ge 1 ] && [ "$gateway_port" -le 65535 ] || exit 1
+state_dir=/run/uai/task-forwarder
+pid_file=$state_dir/pid
+ready_file=$state_dir/ready
+error_file=$state_dir/error.log
+umask 077
+[ ! -L "$state_dir" ] || exit 1
+[ ! -e "$state_dir" ] || [ -d "$state_dir" ] || exit 1
+/usr/bin/install -d -o root -g root -m 0700 "$state_dir"
+
+if [ -f "$pid_file" ] && [ ! -L "$pid_file" ]; then
+  old_pid=$(/bin/cat "$pid_file" 2>/dev/null || true)
+  case "$old_pid" in ''|*[!0-9]*) old_pid= ;; esac
+  if [ -n "$old_pid" ] && /bin/kill -0 "$old_pid" 2>/dev/null; then
+    old_uid=$(/usr/bin/awk '/^Uid:/ { print $2; exit }' "/proc/$old_pid/status" 2>/dev/null || true)
+    old_args=$(/usr/bin/tr '\000' '\n' < "/proc/$old_pid/cmdline" 2>/dev/null || true)
+    # /run survives a Docker stop/start. Its recorded pid may have been
+    # reused by uai-init or another innocent process; treat that record as
+    # stale and never signal a process unless both uid and argv prove it is
+    # this task's prior forwarder.
+    if [ "$old_uid" = 65532 ] \
+       && printf '%s\n' "$old_args" | /bin/grep -Fxq '/usr/local/share/uai/uai-task-forwarder.mjs' \
+       && printf '%s\n' "$old_args" | /bin/grep -Fxq "$task_id"; then
+      /bin/kill -TERM "$old_pid"
+      attempts=0
+      while /bin/kill -0 "$old_pid" 2>/dev/null && [ "$attempts" -lt 50 ]; do
+        /bin/sleep 0.1
+        attempts=$((attempts + 1))
+      done
+      if /bin/kill -0 "$old_pid" 2>/dev/null; then
+        /bin/kill -KILL "$old_pid"
+      fi
+    fi
+  fi
+fi
+/bin/rm -f -- "$pid_file" "$ready_file" "$error_file"
+
+exec 3<&0
+/usr/bin/env -i PATH=/usr/bin:/bin \
+  /usr/bin/setsid \
+  /usr/bin/setpriv \
+    --reuid=65532 \
+    --regid=65532 \
+    --clear-groups \
+    --bounding-set=-all \
+    --inh-caps=-all \
+    --ambient-caps=-all \
+    --no-new-privs \
+  "$node_path" \
+    /usr/local/share/uai/uai-task-forwarder.mjs \
+    --task-id "$task_id" \
+    --generation "$generation" \
+    --gateway-host "$gateway_host" \
+    --gateway-port "$gateway_port" \
+    --listen-port 5877 \
+  <&3 3<&- >"$ready_file" 2>"$error_file" &
+forwarder_pid=$!
+exec 3<&-
+printf '%s\n' "$forwarder_pid" > "$pid_file"
+
+attempts=0
+while [ "$attempts" -lt 50 ]; do
+  /bin/kill -0 "$forwarder_pid" 2>/dev/null || exit 1
+  if [ -f "$ready_file" ] \
+     && [ "$(/bin/cat "$ready_file" 2>/dev/null || true)" = READY ]; then
+    exit 0
+  fi
+  /bin/sleep 0.1
+  attempts=$((attempts + 1))
+done
+exit 1`;
+
+export interface TaskEnvForwarderLaunchOptions {
+  gatewayHost: string;
+  gatewayPort: number;
+  nodePath: TaskForwarderNodePath;
+}
+
+/** Launch or replace one authenticated task forwarder. Consumes and wipes
+ * the caller-owned secret copy on every outcome. */
+export async function launchTaskEnvForwarder(
+  environment: Pick<TaskEnvironmentHandle<unknown>, "exec">,
+  identity: TaskGatewayRegistration,
+  secret: Uint8Array,
+  options: TaskEnvForwarderLaunchOptions,
+): Promise<void> {
+  if (
+    secret.byteLength !== TASK_FORWARDER_SECRET_BYTES ||
+    !Number.isSafeInteger(options.gatewayPort) ||
+    options.gatewayPort < 1 ||
+    options.gatewayPort > 65_535
+  ) {
+    secret.fill(0);
+    throw new Error("task env forwarder launch parameters are invalid");
+  }
+  try {
+    const result = await environment.exec({
+      argv: [
+        "/bin/bash",
+        "-c",
+        START_TASK_FORWARDER,
+        "uai-task-forwarder",
+        identity.taskId,
+        identity.generation,
+        options.gatewayHost,
+        String(options.gatewayPort),
+        options.nodePath,
+      ],
+      user: "root",
+      projectEnvProfile: "maintenance",
+      stdin: secret,
+      timeoutMs: 30_000,
+      maxOutputBytes: STEP_OUTPUT_BYTES,
+    });
+    if (
+      result.exitCode !== 0 ||
+      result.signal !== null ||
+      result.stdoutTruncated ||
+      result.stderrTruncated
+    ) {
+      throw new Error(
+        `task env forwarder failed to start: ${Buffer.from(result.stderr)
+          .toString("utf8")
+          .trim()
+          .slice(-512) || `exit ${String(result.exitCode)}`}`,
+      );
+    }
+  } finally {
+    secret.fill(0);
+  }
+}
+
+/** Reassert the loader process after runtime recovery, before consumers.
+ * Uses the task only through `exec`, so a caller can fence every exec. */
+export async function ensureContainerTaskEnvForwarder(
+  environment: Pick<TaskEnvironmentHandle<unknown>, "descriptor" | "exec">,
+): Promise<void> {
+  const { descriptor } = environment;
+  if (descriptor.envMode !== "loader") return;
+  const isTransport = isEnvironmentTransportProvider(
+    descriptor.locator.provider,
+  );
+  const directMachine = descriptor.locator.provider === "machine";
+  if (isTransport && !directMachine) {
+    // A fixed-task sidecar serves its own loopback and must not bind a second
+    // listener. Other non-machine transports share that conservative shape.
+    return;
+  }
+  if (directMachine) {
+    // A machine reboot preserves disk but not /run or the uid-65532 process.
+    // Refresh the current runner's root-owned assets before exporting the
+    // generation secret, then recreate the process below.
+    await installMachineProjectEnvLoader(environment);
+    await installMachineTaskForwarderAsset(environment);
+  }
+  const identity = taskGatewayRegistrationForRecovery(descriptor.taskId);
+  if (identity === null) {
+    throw new Error("task env gateway identity is not active during recovery");
+  }
+  const launch: TaskEnvForwarderLaunchOptions = directMachine
+    ? {
+        gatewayHost: "127.0.0.1",
+        gatewayPort: MACHINE_GATEWAY_TUNNEL_PORT,
+        nodePath: "/usr/bin/node",
+      }
+    : {
+        gatewayHost: await taskGatewayHost(),
+        gatewayPort: MCP_GATEWAY_PORT,
+        nodePath:
+          descriptor.locator.provider === "apple-container"
+            ? APPLE_FORWARDER_NODE_PATH
+            : "/usr/local/bin/node",
+      };
+  // The secret copy is taken last, with nothing left to await before the
+  // launch, which wipes it on every outcome: no failure strands it.
+  const secret = taskGatewayForwarderSecretInput(
+    identity.taskId,
+    identity.generation,
+  );
+  if (secret === null) {
+    throw new Error("task env gateway secret is unavailable during recovery");
+  }
+  await launchTaskEnvForwarder(environment, identity, secret, launch);
+}

package/lib/task-env-source.ts

@@ -0,0 +1,249 @@
+import { deliveredTaskSecrets } from "./credential-consumers/task-secrets";
+/**
+ * ADR-153 §4 and §6: where a task's project env comes from, for its /env and
+ * its proxy alike, on an owned runner and on a task machine's sidecar.
+ *
+ * - The store is the runner's own records on an owned runner, and on a task
+ *   machine the sidecar's replica of the cloud's.
+ * - A store never activated gives tasks the stored values, as before
+ *   records existed.
+ * - Once activated, a task's env is the view of one snapshot of the store,
+ *   decided with the task's own schema observations: each key's value as of
+ *   its head, a placeholder the proxy swaps, or nothing. While the store's
+ *   gate is closed, or a read of it fails, the task gets no project env at
+ *   all, never the stored values.
+ *
+ * A task's read of the store is kept until anything it depends on changes:
+ * one of its projects' values or records, a binding or the store's gate
+ * (each bumps a process-local revision, and only this process writes the
+ * store). Its observations are stored once and never change. So a change
+ * reaches the next command and the next request, while a command that
+ * changes nothing reads nothing.
+ *
+ * ADR-154: once the runner begins leaving its own credentials, neither
+ * store is read. On cloud credentials a task's env is what the composer
+ * serves for exactly the run the reading generation serves (/env's gateway
+ * generation, the proxy's live one) while that run is the task's open one,
+ * read afresh every time (its gate closes the moment the run's credentials
+ * do, and no copy of it is kept here). Until the runner is active, a task
+ * gets no project env at all.
+ */
+
+import { activeCloudCredentials, onLeavingLegacy, onOwnCredentials } from "./cloud-credential-host";
+import { noteProjectEnvViewChange, projectEnvAuthorityRevision, projectEnvRevision } from "./host-env";
+import { readProjectEnvAuthoritySnapshot } from "./project-env-authority";
+import { readProjectEnvReplicaSnapshot, readProjectEnvReplicaStatus } from "./project-env-replica";
+import { taskProjectEnvResolver, type TaskEnvSource } from "./project-env-loader";
+import { captureTaskEnvSchemas, taskEnvSchemaObservation } from "./project-env-schema-capture";
+import {
+  sameTaskEnvHead,
+  taskEnvView,
+  taskEnvWithheldNote,
+  type TaskEnvHead,
+  type TaskEnvSnapshot,
+} from "./project-env-task-policy";
+import { getHostTask, hostTaskProjectEnvSpec, readTaskRunEpochState, type TaskProjectEnvSpec } from "./runtime-state";
+import { sidecarMode } from "./sidecar";
+import { taskProxiedEnv } from "./secrets-proxy/policy";
+import { taskTrustEnv } from "./secrets-proxy/readiness";
+import { currentTaskGatewayRun, type TaskGatewayEnvResolver } from "./task-gateway-state";
+
+/** The task's projects, lead first, as its launch pinned them. */
+function projectIdsOf(spec: TaskProjectEnvSpec): string[] {
+  const ordered = [...spec.projects].sort(
+    (left, right) => left.position - right.position || (left.projectId < right.projectId ? -1 : left.projectId > right.projectId ? 1 : 0),
+  );
+  return [...new Set(ordered.map((project) => project.projectId))];
+}
+
+/**
+ * One snapshot of the task's projects: `legacy` on a store that has never
+ * been activated, null while its gate is closed or it cannot be read, and
+ * otherwise the records, values and observations the task's decisions come
+ * from.
+ */
+export function readTaskEnvSnapshot(
+  taskId: string,
+  spec: TaskProjectEnvSpec,
+  runEpoch: number | null,
+): TaskEnvSnapshot | "legacy" | null {
+  const projectIds = projectIdsOf(spec);
+  if (!onOwnCredentials()) return cloudTaskEnvSnapshot(taskId, projectIds, runEpoch);
+  const key = JSON.stringify([projectEnvAuthorityRevision(), projectIds.map((projectId) => [projectId, projectEnvRevision(projectId)])]);
+  const hit = snapshots.get(taskId);
+  if (hit?.key === key) return hit.snapshot;
+  const snapshot = freshTaskEnvSnapshot(taskId, projectIds);
+  // Kept only when nothing in it can change without a revision moving: a
+  // read that failed, or one with an observation not stored yet (task-up
+  // may still be about to record its base), is read again next time.
+  if (snapshot === "legacy" || (snapshot !== null && projectIds.every((projectId) => taskEnvSchemaObservation(taskId, projectId) !== null))) {
+    snapshots.delete(taskId);
+    if (snapshots.size >= MAX_SNAPSHOTS) snapshots.delete(snapshots.keys().next().value!);
+    snapshots.set(taskId, { key, snapshot });
+  }
+  return snapshot;
+}
+
+/** Tasks' last reads, most recent last. Values stay in memory only here and
+ *  in the rendered /env documents, as before. */
+const snapshots = new Map<string, { key: string; snapshot: TaskEnvSnapshot | "legacy" }>();
+const MAX_SNAPSHOTS = 256;
+
+/** Forget a task's read, when its gateway generation goes. */
+export function forgetTaskEnvSnapshot(taskId: string): void {
+  snapshots.delete(taskId);
+}
+
+/**
+ * ADR-154: the task's env as the composer serves it for `runEpoch`, the
+ * reading generation's run, with the task's own observations; null unless
+ * the runner is active and that run is the task's admitted, open one (a
+ * generation from an older runtime gets nothing while a newer run starts),
+ * or while the composer serves nothing for it.
+ */
+function cloudTaskEnvSnapshot(
+  taskId: string,
+  projectIds: readonly string[],
+  runEpoch: number | null,
+): TaskEnvSnapshot | null {
+  const composer = activeCloudCredentials();
+  if (composer === null || runEpoch === null) return null;
+  const run = readTaskRunEpochState(taskId);
+  if (run === null || !run.assignmentOpen || run.highestRunEpoch !== runEpoch) return null;
+  const served = composer.envSnapshot({ taskId, runEpoch }, projectIds);
+  if (served === null) return null;
+  return snapshotOf(taskId, projectIds, (projectId) =>
+    (served.projects.find((project) => project.projectId === projectId)?.entries ?? []).map((entry) => ({
+      key: entry.state.key,
+      state: entry.state,
+      value: entry.materialization.kind === "installed" ? entry.value : null,
+    })),
+  );
+}
+
+function freshTaskEnvSnapshot(taskId: string, projectIds: readonly string[]): TaskEnvSnapshot | "legacy" | null {
+  return sidecarMode() ? replicaTaskEnvSnapshot(taskId, projectIds) : ownedTaskEnvSnapshot(taskId, projectIds);
+}
+
+/** An owned runner is the authority for its own records. */
+function ownedTaskEnvSnapshot(taskId: string, projectIds: readonly string[]): TaskEnvSnapshot | "legacy" | null {
+  const read = readProjectEnvAuthoritySnapshot(projectIds);
+  if (read === null) return null;
+  if (read.mode === "legacy") return "legacy";
+  return snapshotOf(taskId, projectIds, (projectId) => read.projects.find((project) => project.projectId === projectId)?.entries ?? []);
+}
+
+/**
+ * On a task machine the cloud is the authority, and the sidecar keeps a
+ * replica of its task's records (ADR-153 §4, "A machine's replica"): once it
+ * has been activated, the task's view comes from it, and a key whose value
+ * the sidecar has not installed gets none. Before that, the values the
+ * sidecar reads from the store by name, as before.
+ */
+function replicaTaskEnvSnapshot(taskId: string, projectIds: readonly string[]): TaskEnvSnapshot | "legacy" | null {
+  let activated: boolean;
+  try {
+    activated = readProjectEnvReplicaStatus().activated;
+  } catch {
+    return null;
+  }
+  if (!activated) return "legacy";
+  const read = readProjectEnvReplicaSnapshot(projectIds);
+  if (read === null) return null;
+  return snapshotOf(taskId, projectIds, (projectId) =>
+    (read.projects.find((project) => project.projectId === projectId)?.entries ?? []).map((entry) => ({
+      key: entry.state.key,
+      state: entry.state,
+      value: entry.materialization.kind === "installed" ? entry.value : null,
+    })),
+  );
+}
+
+interface StoreEntry {
+  key: string;
+  /** The record; null on a store that predates records. */
+  state: { head: TaskEnvHead; binding: unknown; policyEpoch: unknown } | null;
+  /** Null where the store holds no value for the record. */
+  value: string | null;
+}
+
+/** The task's snapshot of one read of the store, with its observations. */
+function snapshotOf(taskId: string, projectIds: readonly string[], entriesOf: (projectId: string) => readonly StoreEntry[]): TaskEnvSnapshot {
+  const projects = projectIds.map((projectId) => {
+    const entries = entriesOf(projectId);
+    return {
+      projectId,
+      // Captured here on the task's first read, from the base task-up
+      // recorded before the container started; with none recorded yet, a
+      // schema it could not read, so an unbound key is withheld.
+      observation: taskEnvSchemaObservation(taskId, projectId) ?? captureTaskEnvSchemas(taskId, [projectId], { final: false })[0],
+      records: entries.flatMap((entry) =>
+        entry.state === null
+          ? []
+          : [{ key: entry.key, head: entry.state.head, binding: entry.state.binding, policyEpoch: entry.state.policyEpoch }],
+      ),
+      entries,
+    };
+  });
+  return {
+    projects: projects.map(({ projectId, observation, records }) => ({ projectId, observation, records })),
+    valueAt: (projectId: string, key: string, head: TaskEnvHead): string | null => {
+      const entry = projects.find((project) => project.projectId === projectId)?.entries.find((candidate) => candidate.key === key);
+      return entry?.state != null && sameTaskEnvHead(entry.state.head, head) ? entry.value : null;
+    },
+  };
+}
+
+/** Where the task's /env takes its project values from right now, for a
+ *  generation serving `runEpoch`. */
+export function taskEnvSource(taskId: string, spec: TaskProjectEnvSpec, runEpoch: number | null): TaskEnvSource {
+  const snapshot = readTaskEnvSnapshot(taskId, spec, runEpoch);
+  if (snapshot === "legacy") return { kind: "stored" };
+  if (snapshot === null) return { kind: "none" };
+  return { kind: "view", view: taskEnvView(snapshot) };
+}
+
+/** The task's snapshot for its proxy, by task id alone: what its persisted
+ *  launch spec names, for the run its live gateway generation serves.
+ *  Nothing is proxied from a store never activated. */
+export function proxyTaskEnvSnapshot(taskId: string): TaskEnvSnapshot | null {
+  const task = getHostTask(taskId);
+  let spec: TaskProjectEnvSpec | null = null;
+  try {
+    spec = task === null ? null : hostTaskProjectEnvSpec(task);
+  } catch {
+    return null;
+  }
+  if (spec === null) return null;
+  const snapshot = readTaskEnvSnapshot(taskId, spec, currentTaskGatewayRun(taskId));
+  return snapshot === "legacy" ? null : snapshot;
+}
+
+/** The task's note about the keys it does not get, from the store as it is
+ *  now; null when it gets every one, or the store gives no view. */
+export function taskEnvWithheldNoteNow(taskId: string, spec: TaskProjectEnvSpec, runEpoch: number | null): string | null {
+  const source = taskEnvSource(taskId, spec, runEpoch);
+  return source.kind === "view" ? taskEnvWithheldNote(source.view.policies) : null;
+}
+
+/** The resolver a loader task's gateway generation renders /env with, for
+ *  the run that generation serves (null: one from before run epochs). */
+export function taskEnvResolver(taskId: string, spec: TaskProjectEnvSpec, runEpoch: number | null): TaskGatewayEnvResolver {
+  return taskProjectEnvResolver(
+    spec,
+    () => taskTrustEnv(taskId),
+    () => taskProxiedEnv(taskId),
+    () => taskEnvSource(taskId, spec, runEpoch),
+    () => deliveredTaskSecrets(taskId, runEpoch),
+  );
+}
+
+/**
+ * ADR-154: the moment the runner begins leaving its own credentials, every
+ * read above stops taking their values (each checks `onOwnCredentials()`),
+ * and whoever follows the env (the proxy's trust delivery, which unpins
+ * what is no longer proxied) looks again. Registered once, at boot.
+ */
+export function followLeavingLegacy(): () => void {
+  return onLeavingLegacy(() => noteProjectEnvViewChange());
+}

package/lib/task-environment/anchor.ts

@@ -0,0 +1,31 @@
+/**
+ * ADR-154 §6, for providers whose runtime is a container or a machine they
+ * exec into (Docker, Apple, a machine over ssh): an anchored request runs
+ * as root, the engine supervisor first, which drops to the task user the
+ * request would have run as before it starts the request's own argv.
+ */
+
+/** Who a request with no user runs as in a task runtime, as the engines
+ *  always have (their homes are under /home/node). */
+export const ANCHORED_TASK_USER = "node";
+
+/** `request`, run as root under its anchor; unchanged without one. */
+export function rootAnchoredRequest<
+  T extends {
+    argv: readonly [string, ...string[]];
+    user?: string;
+    anchor?: { readonly prefix: readonly [string, ...string[]] };
+  },
+>(request: T): T {
+  const { anchor } = request;
+  if (!anchor) return request;
+  if (request.user === "root") {
+    throw new Error("an anchored process runs as a task user, never as root");
+  }
+  const { anchor: _anchor, ...rest } = request;
+  return {
+    ...rest,
+    user: "root",
+    argv: [...anchor.prefix, request.user ?? ANCHORED_TASK_USER, "--", ...request.argv],
+  } as unknown as T;
+}