@vellumai/cli 0.8.10 → 0.8.11-staging.1

package/src/lib/statefulset.ts

@@ -262,6 +262,49 @@ export interface BuildServiceRunArgsOpts extends DockerRunSecrets {
   avatarDevicePath?: string;
 }
 
+interface BuilderManagedEnvKeys {
+  /** Always set by buildServiceRunArgs (spec static/secret entries, builder-computed extras, image-baked PATH). Never replay. */
+  always: ReadonlySet<string>;
+  /**
+   * Spec host-forwarded entries. `name` is the container-side env key (what
+   * docker inspect captures); `hostVar` is the host process.env variable
+   * buildServiceRunArgs reads. Exclude captured `name` from replay only when
+   * process.env[hostVar] is set.
+   */
+  hostForwarded: ReadonlyArray<{ name: string; hostVar: string }>;
+}
+
+/**
+ * Env var names that `buildServiceRunArgs` manages for a service, derived
+ * from the spec so future entries are picked up automatically.
+ */
+export function getBuilderManagedEnvKeys(
+  service: ServiceName,
+  spec = DOCKER_STATEFUL_SET_SPEC,
+): BuilderManagedEnvKeys {
+  const container = spec.containers.find((c) => c.internalName === service);
+  if (!container) throw new Error(`docker-statefulset: unknown service "${service}"`);
+
+  const always = new Set<string>(["PATH"]);
+  const hostForwarded: Array<{ name: string; hostVar: string }> = [];
+  for (const entry of container.env) {
+    if (entry.kind === "host") {
+      hostForwarded.push({ name: entry.name, hostVar: entry.hostVar ?? entry.name });
+    } else {
+      always.add(entry.name);
+    }
+  }
+
+  // Builder-computed extras added outside the spec env arrays
+  if (service === "assistant") {
+    always.add("VELLUM_ASSISTANT_NAME");
+    always.add("GATEWAY_INTERNAL_URL");
+    always.add(AVATAR_DEVICE_ENV_VAR);
+  }
+
+  return { always, hostForwarded };
+}
+
 function resolveVolume(
   spec: DockerStatefulSetSpec,
   instanceName: string,

package/src/lib/terminal-client.ts

@@ -7,6 +7,7 @@
  */
 
 import { authHeaders, getPlatformUrl } from "./platform-client.js";
+import { loopbackSafeFetch } from "./loopback-fetch.js";
 
 // ---------------------------------------------------------------------------
 // Create / Close
@@ -25,7 +26,7 @@ export async function createTerminalSession(
   if (service) {
     body.service = service;
   }
-  const response = await fetch(
+  const response = await loopbackSafeFetch(
     `${baseUrl}/v1/assistants/${assistantId}/terminal/sessions/`,
     {
       method: "POST",
@@ -49,7 +50,7 @@ export async function closeTerminalSession(
   platformUrl?: string,
 ): Promise<void> {
   const baseUrl = platformUrl || getPlatformUrl();
-  const response = await fetch(
+  const response = await loopbackSafeFetch(
     `${baseUrl}/v1/assistants/${assistantId}/terminal/sessions/${sessionId}/`,
     {
       method: "DELETE",
@@ -76,7 +77,7 @@ export async function sendTerminalInput(
   platformUrl?: string,
 ): Promise<void> {
   const baseUrl = platformUrl || getPlatformUrl();
-  const response = await fetch(
+  const response = await loopbackSafeFetch(
     `${baseUrl}/v1/assistants/${assistantId}/terminal/sessions/${sessionId}/input/`,
     {
       method: "POST",
@@ -100,7 +101,7 @@ export async function resizeTerminalSession(
   platformUrl?: string,
 ): Promise<void> {
   const baseUrl = platformUrl || getPlatformUrl();
-  const response = await fetch(
+  const response = await loopbackSafeFetch(
     `${baseUrl}/v1/assistants/${assistantId}/terminal/sessions/${sessionId}/resize/`,
     {
       method: "POST",
@@ -137,7 +138,7 @@ export async function* subscribeTerminalEvents(
   signal?: AbortSignal,
 ): AsyncGenerator<TerminalOutputEvent> {
   const baseUrl = platformUrl || getPlatformUrl();
-  const response = await fetch(
+  const response = await loopbackSafeFetch(
     `${baseUrl}/v1/assistants/${assistantId}/terminal/sessions/${sessionId}/events/`,
     {
       headers: await authHeaders(token, platformUrl),

package/src/lib/upgrade-lifecycle.ts

@@ -7,6 +7,7 @@ import type { AssistantEntry } from "./assistant-config.js";
 import { saveAssistantEntry } from "./assistant-config.js";
 import { createBackup, pruneOldBackups, restoreBackup } from "./backup-ops.js";
 import { emitCliError } from "./cli-error.js";
+import { getOrCreateHostDeviceId } from "./device-id.js";
 import {
   captureImageRefs,
   DOCKER_READY_TIMEOUT_MS,
@@ -19,8 +20,14 @@ import { getStateDir } from "./environments/paths.js";
 import { getCurrentEnvironment } from "./environments/resolve.js";
 import { loadGuardianToken } from "./guardian-token.js";
 import { resolveImageRefs } from "./platform-releases.js";
+import {
+  getBuilderManagedEnvKeys,
+  type DockerStatefulSetSpec,
+  type ServiceName,
+} from "./statefulset.js";
 import { exec, execOutput } from "./step-runner.js";
 import { compareVersions } from "./version-compat.js";
+import { loopbackSafeFetch } from "./loopback-fetch.js";
 
 // ---------------------------------------------------------------------------
 // Failure log capture
@@ -141,20 +148,6 @@ export function buildUpgradeCommitMessage(options: {
   return lines.join("\n");
 }
 
-/**
- * Environment variable keys that are set by CLI run arguments and should
- * not be replayed from a captured container environment during upgrades
- * or rollbacks. Shared between upgrade.ts and rollback.ts.
- */
-export const CONTAINER_ENV_EXCLUDE_KEYS: ReadonlySet<string> = new Set([
-  "CES_SERVICE_TOKEN",
-  "GUARDIAN_BOOTSTRAP_SECRET",
-  "VELLUM_ASSISTANT_NAME",
-  "RUNTIME_HTTP_HOST",
-  "PATH",
-  "ACTOR_TOKEN_SIGNING_KEY",
-]);
-
 /**
  * Capture environment variables from a running Docker container so they
  * can be replayed onto the replacement container after upgrade.
@@ -183,6 +176,96 @@ export async function captureContainerEnv(
   return captured;
 }
 
+/**
+ * Filter a captured container env down to the entries safe to replay onto a
+ * replacement container.
+ *
+ * Drops every key `buildServiceRunArgs` sets itself (spec static/secret
+ * entries, builder-computed extras, PATH). Spec-managed secrets re-enter via
+ * the dedicated `DockerRunSecrets` path, so adding a secret to the spec
+ * automatically excludes it here. Spec host-forwarded keys are dropped only
+ * when the host variable is currently set, so fresh host values win over
+ * stale captured ones.
+ *
+ * Security contract: the returned env is memory-only — never persist it to
+ * disk, and log counts only, never values.
+ */
+export function buildReplayEnv(
+  capturedEnv: Record<string, string>,
+  service: ServiceName,
+  spec?: DockerStatefulSetSpec,
+): Record<string, string> {
+  const { always, hostForwarded } = getBuilderManagedEnvKeys(service, spec);
+  const hostManaged = new Set(
+    hostForwarded.filter((h) => process.env[h.hostVar]).map((h) => h.name),
+  );
+  return Object.fromEntries(
+    Object.entries(capturedEnv).filter(
+      ([key]) => !always.has(key) && !hostManaged.has(key),
+    ),
+  );
+}
+
+/** Secrets and replay env derived from the outgoing containers. */
+interface ReplayState {
+  bootstrapSecret: string | undefined;
+  cesServiceToken: string;
+  signingKey: string;
+  extraAssistantEnv: Record<string, string>;
+  extraGatewayEnv: Record<string, string>;
+}
+
+/**
+ * Derive the secrets and replay env for replacement containers from
+ * already-captured assistant/gateway envs. GUARDIAN_BOOTSTRAP_SECRET is only
+ * set on the gateway; CES_SERVICE_TOKEN and ACTOR_TOKEN_SIGNING_KEY fall back
+ * to fresh values for instances that predate them. VELLUM_DEVICE_ID is
+ * backfilled on the gateway from the host, and the assistant inherits the
+ * gateway's value (captured values win) so the pair always matches.
+ */
+export function buildReplayState(
+  capturedEnv: Record<string, string>,
+  gatewayEnv: Record<string, string>,
+): ReplayState {
+  const extraGatewayEnv = buildReplayEnv(gatewayEnv, "gateway");
+  extraGatewayEnv.VELLUM_DEVICE_ID ??= getOrCreateHostDeviceId();
+
+  const extraAssistantEnv = buildReplayEnv(capturedEnv, "assistant");
+  extraAssistantEnv.VELLUM_DEVICE_ID ??= extraGatewayEnv.VELLUM_DEVICE_ID;
+
+  return {
+    bootstrapSecret: gatewayEnv["GUARDIAN_BOOTSTRAP_SECRET"],
+    cesServiceToken:
+      capturedEnv["CES_SERVICE_TOKEN"] || randomBytes(32).toString("hex"),
+    signingKey:
+      capturedEnv["ACTOR_TOKEN_SIGNING_KEY"] || randomBytes(32).toString("hex"),
+    extraAssistantEnv,
+    extraGatewayEnv,
+  };
+}
+
+/**
+ * Capture the assistant and gateway container envs and derive the replay
+ * state for the replacement containers. Logs only the assistant env-var
+ * count (security contract on `buildReplayEnv`).
+ */
+export async function captureReplayState(
+  res: Pick<
+    ReturnType<typeof dockerResourceNames>,
+    "assistantContainer" | "gatewayContainer"
+  >,
+): Promise<ReplayState> {
+  console.log("💾 Capturing existing container environment...");
+  const [capturedEnv, gatewayEnv] = await Promise.all([
+    captureContainerEnv(res.assistantContainer),
+    captureContainerEnv(res.gatewayContainer),
+  ]);
+  console.log(
+    `   Captured ${Object.keys(capturedEnv).length} env var(s) from ${res.assistantContainer}\n`,
+  );
+  return buildReplayState(capturedEnv, gatewayEnv);
+}
+
 /**
  * Best-effort fetch of the running service group version from the gateway
  * `/healthz` endpoint.  Returns `undefined` when the endpoint is
@@ -192,7 +275,7 @@ export async function fetchCurrentVersion(
   runtimeUrl: string,
 ): Promise<string | undefined> {
   try {
-    const resp = await fetch(`${runtimeUrl}/healthz`, {
+    const resp = await loopbackSafeFetch(`${runtimeUrl}/healthz`, {
       signal: AbortSignal.timeout(5000),
     });
     if (resp.ok) {
@@ -217,7 +300,7 @@ export async function fetchAssistantIngressUrl(
 ): Promise<string | undefined> {
   if (!bearerToken) return undefined;
   try {
-    const resp = await fetch(`${runtimeUrl}/integrations/ingress/config`, {
+    const resp = await loopbackSafeFetch(`${runtimeUrl}/integrations/ingress/config`, {
       headers: { Authorization: `Bearer ${bearerToken}` },
       signal: AbortSignal.timeout(5000),
     });
@@ -259,7 +342,7 @@ export async function fetchPreviousVersion(
   try {
     const { getPlatformUrl } = await import("./platform-client.js");
     const platformUrl = getPlatformUrl();
-    const resp = await fetch(`${platformUrl}/v1/releases/?stable=true`, {
+    const resp = await loopbackSafeFetch(`${platformUrl}/v1/releases/?stable=true`, {
       signal: AbortSignal.timeout(10_000),
     });
     if (!resp.ok) return undefined;
@@ -291,7 +374,7 @@ export async function waitForReady(runtimeUrl: string): Promise<boolean> {
 
   while (Date.now() - start < DOCKER_READY_TIMEOUT_MS) {
     try {
-      const resp = await fetch(readyUrl, {
+      const resp = await loopbackSafeFetch(readyUrl, {
         signal: AbortSignal.timeout(5000),
       });
       if (resp.ok) {
@@ -337,7 +420,7 @@ export async function broadcastUpgradeEvent(
     if (token?.accessToken) {
       headers["Authorization"] = `Bearer ${token.accessToken}`;
     }
-    await fetch(`${gatewayUrl}/v1/admin/upgrade-broadcast`, {
+    await loopbackSafeFetch(`${gatewayUrl}/v1/admin/upgrade-broadcast`, {
       method: "POST",
       headers,
       body: JSON.stringify(event),
@@ -366,7 +449,7 @@ export async function commitWorkspaceViaGateway(
     if (token?.accessToken) {
       headers["Authorization"] = `Bearer ${token.accessToken}`;
     }
-    await fetch(`${gatewayUrl}/v1/admin/workspace-commit`, {
+    await loopbackSafeFetch(`${gatewayUrl}/v1/admin/workspace-commit`, {
       method: "POST",
       headers,
       body: JSON.stringify({ message }),
@@ -409,7 +492,7 @@ export async function rollbackMigrations(
       body.targetWorkspaceMigrationId = targetWorkspaceMigrationId;
     if (rollbackToRegistryCeiling) body.rollbackToRegistryCeiling = true;
 
-    const resp = await fetch(`${gatewayUrl}/v1/admin/rollback-migrations`, {
+    const resp = await loopbackSafeFetch(`${gatewayUrl}/v1/admin/rollback-migrations`, {
       method: "POST",
       headers,
       body: JSON.stringify(body),
@@ -490,7 +573,7 @@ export async function performDockerRollback(
     lastWorkspaceMigrationId?: string;
   } = {};
   try {
-    const healthResp = await fetch(
+    const healthResp = await loopbackSafeFetch(
       `${entry.runtimeUrl}/healthz?include=migrations`,
       { signal: AbortSignal.timeout(5000) },
     );
@@ -581,37 +664,13 @@ export async function performDockerRollback(
     console.warn("⚠️  Pre-rollback backup failed (continuing with rollback)\n");
   }
 
-  // Capture container env, extract secrets
-  console.log("💾 Capturing existing container environment...");
-  const capturedEnv = await captureContainerEnv(res.assistantContainer);
-  console.log(
-    `   Captured ${Object.keys(capturedEnv).length} env var(s) from ${res.assistantContainer}\n`,
-  );
-
-  // Capture GUARDIAN_BOOTSTRAP_SECRET from the gateway container (it is only
-  // set on gateway, not assistant) so it persists across container restarts.
-  const gatewayEnv = await captureContainerEnv(res.gatewayContainer);
-  const bootstrapSecret = gatewayEnv["GUARDIAN_BOOTSTRAP_SECRET"];
-
-  const cesServiceToken =
-    capturedEnv["CES_SERVICE_TOKEN"] || randomBytes(32).toString("hex");
-
-  const signingKey =
-    capturedEnv["ACTOR_TOKEN_SIGNING_KEY"] || randomBytes(32).toString("hex");
-
-  // Build extra env vars, excluding keys managed by buildServiceRunArgs
-  const envKeysSetByRunArgs = new Set(CONTAINER_ENV_EXCLUDE_KEYS);
-  for (const envVar of ["ANTHROPIC_API_KEY", "VELLUM_PLATFORM_URL"]) {
-    if (process.env[envVar]) {
-      envKeysSetByRunArgs.add(envVar);
-    }
-  }
-  const extraAssistantEnv: Record<string, string> = {};
-  for (const [key, value] of Object.entries(capturedEnv)) {
-    if (!envKeysSetByRunArgs.has(key)) {
-      extraAssistantEnv[key] = value;
-    }
-  }
+  const {
+    bootstrapSecret,
+    cesServiceToken,
+    signingKey,
+    extraAssistantEnv,
+    extraGatewayEnv,
+  } = await captureReplayState(res);
 
   // Parse gateway port from entry's runtimeUrl
   let gatewayPort = GATEWAY_INTERNAL_PORT;
@@ -684,6 +743,7 @@ export async function performDockerRollback(
       bootstrapSecret,
       cesServiceToken,
       extraAssistantEnv,
+      extraGatewayEnv,
       gatewayPort,
       imageTags: targetImageTags,
       instanceName,
@@ -801,6 +861,7 @@ export async function performDockerRollback(
             bootstrapSecret,
             cesServiceToken,
             extraAssistantEnv,
+            extraGatewayEnv,
             gatewayPort,
             imageTags: currentImageRefs,
             instanceName,